Subversion Repositories HelenOS

Rev

Rev 3913 | Rev 4137 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (c) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. /** @addtogroup mips32mm
  30.  * @{
  31.  */
  32. /** @file
  33.  */
  34.  
  35. #include <arch/mm/tlb.h>
  36. #include <mm/asid.h>
  37. #include <mm/tlb.h>
  38. #include <mm/page.h>
  39. #include <mm/as.h>
  40. #include <arch/cp0.h>
  41. #include <panic.h>
  42. #include <arch.h>
  43. #include <synch/mutex.h>
  44. #include <print.h>
  45. #include <debug.h>
  46. #include <align.h>
  47. #include <interrupt.h>
  48.  
  49. #ifdef CONFIG_SYMTAB
  50. #include <symtab.h>
  51. #endif
  52.  
  53. static void tlb_refill_fail(istate_t *);
  54. static void tlb_invalid_fail(istate_t *);
  55. static void tlb_modified_fail(istate_t *);
  56.  
  57. static pte_t *find_mapping_and_check(uintptr_t, int, istate_t *, int *);
  58.  
  59. /** Initialize TLB.
  60.  *
  61.  * Invalidate all entries and mark wired entries.
  62.  */
  63. void tlb_arch_init(void)
  64. {
  65.     int i;
  66.  
  67.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  68.     cp0_entry_hi_write(0);
  69.     cp0_entry_lo0_write(0);
  70.     cp0_entry_lo1_write(0);
  71.  
  72.     /* Clear and initialize TLB. */
  73.    
  74.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  75.         cp0_index_write(i);
  76.         tlbwi();
  77.     }
  78.        
  79.     /*
  80.      * The kernel is going to make use of some wired
  81.      * entries (e.g. mapping kernel stacks in kseg3).
  82.      */
  83.     cp0_wired_write(TLB_WIRED);
  84. }
  85.  
  86. /** Process TLB Refill Exception.
  87.  *
  88.  * @param istate    Interrupted register context.
  89.  */
  90. void tlb_refill(istate_t *istate)
  91. {
  92.     entry_lo_t lo;
  93.     entry_hi_t hi;
  94.     asid_t asid;
  95.     uintptr_t badvaddr;
  96.     pte_t *pte;
  97.     int pfrc;
  98.    
  99.     badvaddr = cp0_badvaddr_read();
  100.    
  101.     mutex_lock(&AS->lock);
  102.     asid = AS->asid;
  103.     mutex_unlock(&AS->lock);
  104.    
  105.     page_table_lock(AS, true);
  106.    
  107.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  108.     if (!pte) {
  109.         switch (pfrc) {
  110.         case AS_PF_FAULT:
  111.             goto fail;
  112.             break;
  113.         case AS_PF_DEFER:
  114.             /*
  115.              * The page fault came during copy_from_uspace()
  116.              * or copy_to_uspace().
  117.              */
  118.             page_table_unlock(AS, true);
  119.             return;
  120.         default:
  121.             panic("Unexpected pfrc (%d).", pfrc);
  122.         }
  123.     }
  124.  
  125.     /*
  126.      * Record access to PTE.
  127.      */
  128.     pte->a = 1;
  129.  
  130.     tlb_prepare_entry_hi(&hi, asid, badvaddr);
  131.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
  132.         pte->pfn);
  133.  
  134.     /*
  135.      * New entry is to be inserted into TLB
  136.      */
  137.     cp0_entry_hi_write(hi.value);
  138.     if ((badvaddr / PAGE_SIZE) % 2 == 0) {
  139.         cp0_entry_lo0_write(lo.value);
  140.         cp0_entry_lo1_write(0);
  141.     }
  142.     else {
  143.         cp0_entry_lo0_write(0);
  144.         cp0_entry_lo1_write(lo.value);
  145.     }
  146.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  147.     tlbwr();
  148.  
  149.     page_table_unlock(AS, true);
  150.     return;
  151.    
  152. fail:
  153.     page_table_unlock(AS, true);
  154.     tlb_refill_fail(istate);
  155. }
  156.  
  157. /** Process TLB Invalid Exception.
  158.  *
  159.  * @param istate    Interrupted register context.
  160.  */
  161. void tlb_invalid(istate_t *istate)
  162. {
  163.     tlb_index_t index;
  164.     uintptr_t badvaddr;
  165.     entry_lo_t lo;
  166.     entry_hi_t hi;
  167.     pte_t *pte;
  168.     int pfrc;
  169.  
  170.     badvaddr = cp0_badvaddr_read();
  171.  
  172.     /*
  173.      * Locate the faulting entry in TLB.
  174.      */
  175.     hi.value = cp0_entry_hi_read();
  176.     tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
  177.     cp0_entry_hi_write(hi.value);
  178.     tlbp();
  179.     index.value = cp0_index_read();
  180.  
  181.     page_table_lock(AS, true); 
  182.    
  183.     /*
  184.      * Fail if the entry is not in TLB.
  185.      */
  186.     if (index.p) {
  187.         printf("TLB entry not found.\n");
  188.         goto fail;
  189.     }
  190.  
  191.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  192.     if (!pte) {
  193.         switch (pfrc) {
  194.         case AS_PF_FAULT:
  195.             goto fail;
  196.             break;
  197.         case AS_PF_DEFER:
  198.             /*
  199.              * The page fault came during copy_from_uspace()
  200.              * or copy_to_uspace().
  201.              */
  202.             page_table_unlock(AS, true);             
  203.             return;
  204.         default:
  205.             panic("Unexpected pfrc (%d).", pfrc);
  206.         }
  207.     }
  208.  
  209.     /*
  210.      * Read the faulting TLB entry.
  211.      */
  212.     tlbr();
  213.  
  214.     /*
  215.      * Record access to PTE.
  216.      */
  217.     pte->a = 1;
  218.  
  219.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
  220.         pte->pfn);
  221.  
  222.     /*
  223.      * The entry is to be updated in TLB.
  224.      */
  225.     if ((badvaddr / PAGE_SIZE) % 2 == 0)
  226.         cp0_entry_lo0_write(lo.value);
  227.     else
  228.         cp0_entry_lo1_write(lo.value);
  229.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  230.     tlbwi();
  231.  
  232.     page_table_unlock(AS, true);
  233.     return;
  234.    
  235. fail:
  236.     page_table_unlock(AS, true);
  237.     tlb_invalid_fail(istate);
  238. }
  239.  
  240. /** Process TLB Modified Exception.
  241.  *
  242.  * @param istate    Interrupted register context.
  243.  */
  244. void tlb_modified(istate_t *istate)
  245. {
  246.     tlb_index_t index;
  247.     uintptr_t badvaddr;
  248.     entry_lo_t lo;
  249.     entry_hi_t hi;
  250.     pte_t *pte;
  251.     int pfrc;
  252.  
  253.     badvaddr = cp0_badvaddr_read();
  254.  
  255.     /*
  256.      * Locate the faulting entry in TLB.
  257.      */
  258.     hi.value = cp0_entry_hi_read();
  259.     tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
  260.     cp0_entry_hi_write(hi.value);
  261.     tlbp();
  262.     index.value = cp0_index_read();
  263.  
  264.     page_table_lock(AS, true); 
  265.    
  266.     /*
  267.      * Fail if the entry is not in TLB.
  268.      */
  269.     if (index.p) {
  270.         printf("TLB entry not found.\n");
  271.         goto fail;
  272.     }
  273.  
  274.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
  275.     if (!pte) {
  276.         switch (pfrc) {
  277.         case AS_PF_FAULT:
  278.             goto fail;
  279.             break;
  280.         case AS_PF_DEFER:
  281.             /*
  282.              * The page fault came during copy_from_uspace()
  283.              * or copy_to_uspace().
  284.              */
  285.             page_table_unlock(AS, true);             
  286.             return;
  287.         default:
  288.             panic("Unexpected pfrc (%d).", pfrc);
  289.         }
  290.     }
  291.  
  292.     /*
  293.      * Read the faulting TLB entry.
  294.      */
  295.     tlbr();
  296.  
  297.     /*
  298.      * Record access and write to PTE.
  299.      */
  300.     pte->a = 1;
  301.     pte->d = 1;
  302.  
  303.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable,
  304.         pte->pfn);
  305.  
  306.     /*
  307.      * The entry is to be updated in TLB.
  308.      */
  309.     if ((badvaddr / PAGE_SIZE) % 2 == 0)
  310.         cp0_entry_lo0_write(lo.value);
  311.     else
  312.         cp0_entry_lo1_write(lo.value);
  313.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  314.     tlbwi();
  315.  
  316.     page_table_unlock(AS, true);
  317.     return;
  318.    
  319. fail:
  320.     page_table_unlock(AS, true);
  321.     tlb_modified_fail(istate);
  322. }
  323.  
  324. void tlb_refill_fail(istate_t *istate)
  325. {
  326.     char *symbol = "";
  327.     char *sym2 = "";
  328.  
  329. #ifdef CONFIG_SYMTAB
  330.     char *s = get_symtab_entry(istate->epc);
  331.     if (s)
  332.         symbol = s;
  333.     s = get_symtab_entry(istate->ra);
  334.     if (s)
  335.         sym2 = s;
  336. #endif
  337.  
  338.     fault_if_from_uspace(istate, "TLB Refill Exception on %p.",
  339.         cp0_badvaddr_read());
  340.     panic("%x: TLB Refill Exception at %x(%s<-%s).", cp0_badvaddr_read(),
  341.         istate->epc, symbol, sym2);
  342. }
  343.  
  344.  
  345. void tlb_invalid_fail(istate_t *istate)
  346. {
  347.     char *symbol = "";
  348.  
  349. #ifdef CONFIG_SYMTAB
  350.     char *s = get_symtab_entry(istate->epc);
  351.     if (s)
  352.         symbol = s;
  353. #endif
  354.  
  355.     fault_if_from_uspace(istate, "TLB Invalid Exception on %p.",
  356.         cp0_badvaddr_read());
  357.     panic("%x: TLB Invalid Exception at %x(%s).", cp0_badvaddr_read(),
  358.         istate->epc, symbol);
  359. }
  360.  
  361. void tlb_modified_fail(istate_t *istate)
  362. {
  363.     char *symbol = "";
  364.  
  365. #ifdef CONFIG_SYMTAB
  366.     char *s = get_symtab_entry(istate->epc);
  367.     if (s)
  368.         symbol = s;
  369. #endif
  370.  
  371.     fault_if_from_uspace(istate, "TLB Modified Exception on %p.",
  372.         cp0_badvaddr_read());
  373.     panic("%x: TLB Modified Exception at %x(%s).", cp0_badvaddr_read(),
  374.         istate->epc, symbol);
  375. }
  376.  
  377. /** Try to find PTE for faulting address.
  378.  *
  379.  * The AS->lock must be held on entry to this function.
  380.  *
  381.  * @param badvaddr  Faulting virtual address.
  382.  * @param access    Access mode that caused the fault.
  383.  * @param istate    Pointer to interrupted state.
  384.  * @param pfrc      Pointer to variable where as_page_fault() return code
  385.  *          will be stored.
  386.  *
  387.  * @return      PTE on success, NULL otherwise.
  388.  */
  389. pte_t *
  390. find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate,
  391.     int *pfrc)
  392. {
  393.     entry_hi_t hi;
  394.     pte_t *pte;
  395.  
  396.     hi.value = cp0_entry_hi_read();
  397.  
  398.     /*
  399.      * Handler cannot succeed if the ASIDs don't match.
  400.      */
  401.     if (hi.asid != AS->asid) {
  402.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  403.         return NULL;
  404.     }
  405.  
  406.     /*
  407.      * Check if the mapping exists in page tables.
  408.      */
  409.     pte = page_mapping_find(AS, badvaddr);
  410.     if (pte && pte->p && (pte->w || access != PF_ACCESS_WRITE)) {
  411.         /*
  412.          * Mapping found in page tables.
  413.          * Immediately succeed.
  414.          */
  415.         return pte;
  416.     } else {
  417.         int rc;
  418.        
  419.         /*
  420.          * Mapping not found in page tables.
  421.          * Resort to higher-level page fault handler.
  422.          */
  423.         page_table_unlock(AS, true);
  424.         switch (rc = as_page_fault(badvaddr, access, istate)) {
  425.         case AS_PF_OK:
  426.             /*
  427.              * The higher-level page fault handler succeeded,
  428.              * The mapping ought to be in place.
  429.              */
  430.             page_table_lock(AS, true);
  431.             pte = page_mapping_find(AS, badvaddr);
  432.             ASSERT(pte && pte->p);
  433.             ASSERT(pte->w || access != PF_ACCESS_WRITE);
  434.             return pte;
  435.             break;
  436.         case AS_PF_DEFER:
  437.             page_table_lock(AS, true);
  438.             *pfrc = AS_PF_DEFER;
  439.             return NULL;
  440.             break;
  441.         case AS_PF_FAULT:
  442.             page_table_lock(AS, true);
  443.             *pfrc = AS_PF_FAULT;
  444.             return NULL;
  445.             break;
  446.         default:
  447.             panic("Unexpected rc (%d).", rc);
  448.         }
  449.        
  450.     }
  451. }
  452.  
  453. void
  454. tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable,
  455.     uintptr_t pfn)
  456. {
  457.     lo->value = 0;
  458.     lo->g = g;
  459.     lo->v = v;
  460.     lo->d = d;
  461.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  462.     lo->pfn = pfn;
  463. }
  464.  
  465. void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
  466. {
  467.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  468.     hi->asid = asid;
  469. }
  470.  
  471. /** Print contents of TLB. */
  472. void tlb_print(void)
  473. {
  474.     page_mask_t mask;
  475.     entry_lo_t lo0, lo1;
  476.     entry_hi_t hi, hi_save;
  477.     unsigned int i;
  478.  
  479.     hi_save.value = cp0_entry_hi_read();
  480.    
  481.     printf("#  ASID VPN2   MASK G V D C PFN\n");
  482.     printf("-- ---- ------ ---- - - - - ------\n");
  483.    
  484.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  485.         cp0_index_write(i);
  486.         tlbr();
  487.        
  488.         mask.value = cp0_pagemask_read();
  489.         hi.value = cp0_entry_hi_read();
  490.         lo0.value = cp0_entry_lo0_read();
  491.         lo1.value = cp0_entry_lo1_read();
  492.        
  493.         printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
  494.             i, hi.asid, hi.vpn2, mask.mask,
  495.             lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
  496.         printf("                    %1u %1u %1u %1u %#6x\n",
  497.             lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  498.     }
  499.    
  500.     cp0_entry_hi_write(hi_save.value);
  501. }
  502.  
  503. /** Invalidate all not wired TLB entries. */
  504. void tlb_invalidate_all(void)
  505. {
  506.     ipl_t ipl;
  507.     entry_lo_t lo0, lo1;
  508.     entry_hi_t hi_save;
  509.     int i;
  510.  
  511.     hi_save.value = cp0_entry_hi_read();
  512.     ipl = interrupts_disable();
  513.  
  514.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  515.         cp0_index_write(i);
  516.         tlbr();
  517.  
  518.         lo0.value = cp0_entry_lo0_read();
  519.         lo1.value = cp0_entry_lo1_read();
  520.  
  521.         lo0.v = 0;
  522.         lo1.v = 0;
  523.  
  524.         cp0_entry_lo0_write(lo0.value);
  525.         cp0_entry_lo1_write(lo1.value);
  526.                
  527.         tlbwi();
  528.     }
  529.    
  530.     interrupts_restore(ipl);
  531.     cp0_entry_hi_write(hi_save.value);
  532. }
  533.  
  534. /** Invalidate all TLB entries belonging to specified address space.
  535.  *
  536.  * @param asid Address space identifier.
  537.  */
  538. void tlb_invalidate_asid(asid_t asid)
  539. {
  540.     ipl_t ipl;
  541.     entry_lo_t lo0, lo1;
  542.     entry_hi_t hi, hi_save;
  543.     int i;
  544.  
  545.     ASSERT(asid != ASID_INVALID);
  546.  
  547.     hi_save.value = cp0_entry_hi_read();
  548.     ipl = interrupts_disable();
  549.    
  550.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  551.         cp0_index_write(i);
  552.         tlbr();
  553.        
  554.         hi.value = cp0_entry_hi_read();
  555.        
  556.         if (hi.asid == asid) {
  557.             lo0.value = cp0_entry_lo0_read();
  558.             lo1.value = cp0_entry_lo1_read();
  559.  
  560.             lo0.v = 0;
  561.             lo1.v = 0;
  562.  
  563.             cp0_entry_lo0_write(lo0.value);
  564.             cp0_entry_lo1_write(lo1.value);
  565.  
  566.             tlbwi();
  567.         }
  568.     }
  569.    
  570.     interrupts_restore(ipl);
  571.     cp0_entry_hi_write(hi_save.value);
  572. }
  573.  
  574. /** Invalidate TLB entries for specified page range belonging to specified
  575.  * address space.
  576.  *
  577.  * @param asid      Address space identifier.
  578.  * @param page      First page whose TLB entry is to be invalidated.
  579.  * @param cnt       Number of entries to invalidate.
  580.  */
  581. void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
  582. {
  583.     unsigned int i;
  584.     ipl_t ipl;
  585.     entry_lo_t lo0, lo1;
  586.     entry_hi_t hi, hi_save;
  587.     tlb_index_t index;
  588.  
  589.     ASSERT(asid != ASID_INVALID);
  590.  
  591.     hi_save.value = cp0_entry_hi_read();
  592.     ipl = interrupts_disable();
  593.  
  594.     for (i = 0; i < cnt + 1; i += 2) {
  595.         hi.value = 0;
  596.         tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  597.         cp0_entry_hi_write(hi.value);
  598.  
  599.         tlbp();
  600.         index.value = cp0_index_read();
  601.  
  602.         if (!index.p) {
  603.             /*
  604.              * Entry was found, index register contains valid
  605.              * index.
  606.              */
  607.             tlbr();
  608.  
  609.             lo0.value = cp0_entry_lo0_read();
  610.             lo1.value = cp0_entry_lo1_read();
  611.  
  612.             lo0.v = 0;
  613.             lo1.v = 0;
  614.  
  615.             cp0_entry_lo0_write(lo0.value);
  616.             cp0_entry_lo1_write(lo1.value);
  617.  
  618.             tlbwi();
  619.         }
  620.     }
  621.    
  622.     interrupts_restore(ipl);
  623.     cp0_entry_hi_write(hi_save.value);
  624. }
  625.  
  626. /** @}
  627.  */
  628.