Subversion Repositories HelenOS

Rev

Rev 3228 | Rev 3393 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (c) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. /** @addtogroup mips32mm   
  30.  * @{
  31.  */
  32. /** @file
  33.  */
  34.  
  35. #include <arch/mm/tlb.h>
  36. #include <mm/asid.h>
  37. #include <mm/tlb.h>
  38. #include <mm/page.h>
  39. #include <mm/as.h>
  40. #include <arch/cp0.h>
  41. #include <panic.h>
  42. #include <arch.h>
  43. #include <symtab.h>
  44. #include <synch/spinlock.h>
  45. #include <print.h>
  46. #include <debug.h>
  47. #include <align.h>
  48. #include <interrupt.h>
  49.  
  50. static void tlb_refill_fail(istate_t *);
  51. static void tlb_invalid_fail(istate_t *);
  52. static void tlb_modified_fail(istate_t *);
  53.  
  54. static pte_t *find_mapping_and_check(uintptr_t, int, istate_t *, int *);
  55.  
  56. /** Initialize TLB.
  57.  *
  58.  * Invalidate all entries and mark wired entries.
  59.  */
  60. void tlb_arch_init(void)
  61. {
  62.     int i;
  63.  
  64.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  65.     cp0_entry_hi_write(0);
  66.     cp0_entry_lo0_write(0);
  67.     cp0_entry_lo1_write(0);
  68.  
  69.     /* Clear and initialize TLB. */
  70.    
  71.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  72.         cp0_index_write(i);
  73.         tlbwi();
  74.     }
  75.        
  76.     /*
  77.      * The kernel is going to make use of some wired
  78.      * entries (e.g. mapping kernel stacks in kseg3).
  79.      */
  80.     cp0_wired_write(TLB_WIRED);
  81. }
  82.  
  83. /** Process TLB Refill Exception.
  84.  *
  85.  * @param istate    Interrupted register context.
  86.  */
  87. void tlb_refill(istate_t *istate)
  88. {
  89.     entry_lo_t lo;
  90.     entry_hi_t hi;
  91.     asid_t asid;
  92.     uintptr_t badvaddr;
  93.     pte_t *pte;
  94.     int pfrc;
  95.  
  96.     badvaddr = cp0_badvaddr_read();
  97.  
  98.     spinlock_lock(&AS->lock);
  99.     asid = AS->asid;
  100.     spinlock_unlock(&AS->lock);
  101.  
  102.     page_table_lock(AS, true);
  103.  
  104.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  105.     if (!pte) {
  106.         switch (pfrc) {
  107.         case AS_PF_FAULT:
  108.             goto fail;
  109.             break;
  110.         case AS_PF_DEFER:
  111.             /*
  112.              * The page fault came during copy_from_uspace()
  113.              * or copy_to_uspace().
  114.              */
  115.             page_table_unlock(AS, true);
  116.             return;
  117.         default:
  118.             panic("unexpected pfrc (%d)\n", pfrc);
  119.         }
  120.     }
  121.  
  122.     /*
  123.      * Record access to PTE.
  124.      */
  125.     pte->a = 1;
  126.  
  127.     tlb_prepare_entry_hi(&hi, asid, badvaddr);
  128.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
  129.         pte->pfn);
  130.  
  131.     /*
  132.      * New entry is to be inserted into TLB
  133.      */
  134.     cp0_entry_hi_write(hi.value);
  135.     if ((badvaddr / PAGE_SIZE) % 2 == 0) {
  136.         cp0_entry_lo0_write(lo.value);
  137.         cp0_entry_lo1_write(0);
  138.     }
  139.     else {
  140.         cp0_entry_lo0_write(0);
  141.         cp0_entry_lo1_write(lo.value);
  142.     }
  143.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  144.     tlbwr();
  145.  
  146.     page_table_unlock(AS, true);
  147.     return;
  148.    
  149. fail:
  150.     page_table_unlock(AS, true);
  151.     tlb_refill_fail(istate);
  152. }
  153.  
  154. /** Process TLB Invalid Exception.
  155.  *
  156.  * @param istate    Interrupted register context.
  157.  */
  158. void tlb_invalid(istate_t *istate)
  159. {
  160.     tlb_index_t index;
  161.     uintptr_t badvaddr;
  162.     entry_lo_t lo;
  163.     entry_hi_t hi;
  164.     pte_t *pte;
  165.     int pfrc;
  166.  
  167.     badvaddr = cp0_badvaddr_read();
  168.  
  169.     /*
  170.      * Locate the faulting entry in TLB.
  171.      */
  172.     hi.value = cp0_entry_hi_read();
  173.     tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
  174.     cp0_entry_hi_write(hi.value);
  175.     tlbp();
  176.     index.value = cp0_index_read();
  177.  
  178.     page_table_lock(AS, true); 
  179.    
  180.     /*
  181.      * Fail if the entry is not in TLB.
  182.      */
  183.     if (index.p) {
  184.         printf("TLB entry not found.\n");
  185.         goto fail;
  186.     }
  187.  
  188.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  189.     if (!pte) {
  190.         switch (pfrc) {
  191.         case AS_PF_FAULT:
  192.             goto fail;
  193.             break;
  194.         case AS_PF_DEFER:
  195.             /*
  196.              * The page fault came during copy_from_uspace()
  197.              * or copy_to_uspace().
  198.              */
  199.             page_table_unlock(AS, true);             
  200.             return;
  201.         default:
  202.             panic("unexpected pfrc (%d)\n", pfrc);
  203.         }
  204.     }
  205.  
  206.     /*
  207.      * Read the faulting TLB entry.
  208.      */
  209.     tlbr();
  210.  
  211.     /*
  212.      * Record access to PTE.
  213.      */
  214.     pte->a = 1;
  215.  
  216.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
  217.         pte->pfn);
  218.  
  219.     /*
  220.      * The entry is to be updated in TLB.
  221.      */
  222.     if ((badvaddr / PAGE_SIZE) % 2 == 0)
  223.         cp0_entry_lo0_write(lo.value);
  224.     else
  225.         cp0_entry_lo1_write(lo.value);
  226.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  227.     tlbwi();
  228.  
  229.     page_table_unlock(AS, true);
  230.     return;
  231.    
  232. fail:
  233.     page_table_unlock(AS, true);
  234.     tlb_invalid_fail(istate);
  235. }
  236.  
  237. /** Process TLB Modified Exception.
  238.  *
  239.  * @param istate    Interrupted register context.
  240.  */
  241. void tlb_modified(istate_t *istate)
  242. {
  243.     tlb_index_t index;
  244.     uintptr_t badvaddr;
  245.     entry_lo_t lo;
  246.     entry_hi_t hi;
  247.     pte_t *pte;
  248.     int pfrc;
  249.  
  250.     badvaddr = cp0_badvaddr_read();
  251.  
  252.     /*
  253.      * Locate the faulting entry in TLB.
  254.      */
  255.     hi.value = cp0_entry_hi_read();
  256.     tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
  257.     cp0_entry_hi_write(hi.value);
  258.     tlbp();
  259.     index.value = cp0_index_read();
  260.  
  261.     page_table_lock(AS, true); 
  262.    
  263.     /*
  264.      * Fail if the entry is not in TLB.
  265.      */
  266.     if (index.p) {
  267.         printf("TLB entry not found.\n");
  268.         goto fail;
  269.     }
  270.  
  271.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
  272.     if (!pte) {
  273.         switch (pfrc) {
  274.         case AS_PF_FAULT:
  275.             goto fail;
  276.             break;
  277.         case AS_PF_DEFER:
  278.             /*
  279.              * The page fault came during copy_from_uspace()
  280.              * or copy_to_uspace().
  281.              */
  282.             page_table_unlock(AS, true);             
  283.             return;
  284.         default:
  285.             panic("unexpected pfrc (%d)\n", pfrc);
  286.         }
  287.     }
  288.  
  289.     /*
  290.      * Fail if the page is not writable.
  291.      */
  292.     if (!pte->w)
  293.         goto fail;
  294.  
  295.     /*
  296.      * Read the faulting TLB entry.
  297.      */
  298.     tlbr();
  299.  
  300.     /*
  301.      * Record access and write to PTE.
  302.      */
  303.     pte->a = 1;
  304.     pte->d = 1;
  305.  
  306.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable,
  307.         pte->pfn);
  308.  
  309.     /*
  310.      * The entry is to be updated in TLB.
  311.      */
  312.     if ((badvaddr / PAGE_SIZE) % 2 == 0)
  313.         cp0_entry_lo0_write(lo.value);
  314.     else
  315.         cp0_entry_lo1_write(lo.value);
  316.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  317.     tlbwi();
  318.  
  319.     page_table_unlock(AS, true);
  320.     return;
  321.    
  322. fail:
  323.     page_table_unlock(AS, true);
  324.     tlb_modified_fail(istate);
  325. }
  326.  
  327. void tlb_refill_fail(istate_t *istate)
  328. {
  329.     char *symbol = "";
  330.     char *sym2 = "";
  331.  
  332.     char *s = get_symtab_entry(istate->epc);
  333.     if (s)
  334.         symbol = s;
  335.     s = get_symtab_entry(istate->ra);
  336.     if (s)
  337.         sym2 = s;
  338.  
  339.     fault_if_from_uspace(istate, "TLB Refill Exception on %p",
  340.         cp0_badvaddr_read());
  341.     panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(),
  342.         istate->epc, symbol, sym2);
  343. }
  344.  
  345.  
  346. void tlb_invalid_fail(istate_t *istate)
  347. {
  348.     char *symbol = "";
  349.  
  350.     char *s = get_symtab_entry(istate->epc);
  351.     if (s)
  352.         symbol = s;
  353.     fault_if_from_uspace(istate, "TLB Invalid Exception on %p",
  354.         cp0_badvaddr_read());
  355.     panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(),
  356.         istate->epc, symbol);
  357. }
  358.  
  359. void tlb_modified_fail(istate_t *istate)
  360. {
  361.     char *symbol = "";
  362.  
  363.     char *s = get_symtab_entry(istate->epc);
  364.     if (s)
  365.         symbol = s;
  366.     fault_if_from_uspace(istate, "TLB Modified Exception on %p",
  367.         cp0_badvaddr_read());
  368.     panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(),
  369.         istate->epc, symbol);
  370. }
  371.  
  372. /** Try to find PTE for faulting address.
  373.  *
  374.  * The AS->lock must be held on entry to this function.
  375.  *
  376.  * @param badvaddr  Faulting virtual address.
  377.  * @param access    Access mode that caused the fault.
  378.  * @param istate    Pointer to interrupted state.
  379.  * @param pfrc      Pointer to variable where as_page_fault() return code
  380.  *          will be stored.
  381.  *
  382.  * @return      PTE on success, NULL otherwise.
  383.  */
  384. pte_t *
  385. find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate,
  386.     int *pfrc)
  387. {
  388.     entry_hi_t hi;
  389.     pte_t *pte;
  390.  
  391.     hi.value = cp0_entry_hi_read();
  392.  
  393.     /*
  394.      * Handler cannot succeed if the ASIDs don't match.
  395.      */
  396.     if (hi.asid != AS->asid) {
  397.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  398.         return NULL;
  399.     }
  400.  
  401.     /*
  402.      * Check if the mapping exists in page tables.
  403.      */
  404.     pte = page_mapping_find(AS, badvaddr);
  405.     if (pte && pte->p) {
  406.         /*
  407.          * Mapping found in page tables.
  408.          * Immediately succeed.
  409.          */
  410.         return pte;
  411.     } else {
  412.         int rc;
  413.        
  414.         /*
  415.          * Mapping not found in page tables.
  416.          * Resort to higher-level page fault handler.
  417.          */
  418.         page_table_unlock(AS, true);
  419.         switch (rc = as_page_fault(badvaddr, access, istate)) {
  420.         case AS_PF_OK:
  421.             /*
  422.              * The higher-level page fault handler succeeded,
  423.              * The mapping ought to be in place.
  424.              */
  425.             page_table_lock(AS, true);
  426.             pte = page_mapping_find(AS, badvaddr);
  427.             ASSERT(pte && pte->p);
  428.             return pte;
  429.             break;
  430.         case AS_PF_DEFER:
  431.             page_table_lock(AS, true);
  432.             *pfrc = AS_PF_DEFER;
  433.             return NULL;
  434.             break;
  435.         case AS_PF_FAULT:
  436.             page_table_lock(AS, true);
  437.             printf("Page fault.\n");
  438.             *pfrc = AS_PF_FAULT;
  439.             return NULL;
  440.             break;
  441.         default:
  442.             panic("unexpected rc (%d)\n", rc);
  443.         }
  444.        
  445.     }
  446. }
  447.  
  448. void
  449. tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable,
  450.     uintptr_t pfn)
  451. {
  452.     lo->value = 0;
  453.     lo->g = g;
  454.     lo->v = v;
  455.     lo->d = d;
  456.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  457.     lo->pfn = pfn;
  458. }
  459.  
  460. void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
  461. {
  462.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  463.     hi->asid = asid;
  464. }
  465.  
  466. /** Print contents of TLB. */
  467. void tlb_print(void)
  468. {
  469.     page_mask_t mask;
  470.     entry_lo_t lo0, lo1;
  471.     entry_hi_t hi, hi_save;
  472.     unsigned int i;
  473.  
  474.     hi_save.value = cp0_entry_hi_read();
  475.    
  476.     printf("#  ASID VPN2   MASK G V D C PFN\n");
  477.     printf("-- ---- ------ ---- - - - - ------\n");
  478.    
  479.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  480.         cp0_index_write(i);
  481.         tlbr();
  482.        
  483.         mask.value = cp0_pagemask_read();
  484.         hi.value = cp0_entry_hi_read();
  485.         lo0.value = cp0_entry_lo0_read();
  486.         lo1.value = cp0_entry_lo1_read();
  487.        
  488.         printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
  489.             i, hi.asid, hi.vpn2, mask.mask,
  490.             lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
  491.         printf("                    %1u %1u %1u %1u %#6x\n",
  492.             lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  493.     }
  494.    
  495.     cp0_entry_hi_write(hi_save.value);
  496. }
  497.  
  498. /** Invalidate all not wired TLB entries. */
  499. void tlb_invalidate_all(void)
  500. {
  501.     ipl_t ipl;
  502.     entry_lo_t lo0, lo1;
  503.     entry_hi_t hi_save;
  504.     int i;
  505.  
  506.     hi_save.value = cp0_entry_hi_read();
  507.     ipl = interrupts_disable();
  508.  
  509.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  510.         cp0_index_write(i);
  511.         tlbr();
  512.  
  513.         lo0.value = cp0_entry_lo0_read();
  514.         lo1.value = cp0_entry_lo1_read();
  515.  
  516.         lo0.v = 0;
  517.         lo1.v = 0;
  518.  
  519.         cp0_entry_lo0_write(lo0.value);
  520.         cp0_entry_lo1_write(lo1.value);
  521.                
  522.         tlbwi();
  523.     }
  524.    
  525.     interrupts_restore(ipl);
  526.     cp0_entry_hi_write(hi_save.value);
  527. }
  528.  
  529. /** Invalidate all TLB entries belonging to specified address space.
  530.  *
  531.  * @param asid Address space identifier.
  532.  */
  533. void tlb_invalidate_asid(asid_t asid)
  534. {
  535.     ipl_t ipl;
  536.     entry_lo_t lo0, lo1;
  537.     entry_hi_t hi, hi_save;
  538.     int i;
  539.  
  540.     ASSERT(asid != ASID_INVALID);
  541.  
  542.     hi_save.value = cp0_entry_hi_read();
  543.     ipl = interrupts_disable();
  544.    
  545.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  546.         cp0_index_write(i);
  547.         tlbr();
  548.        
  549.         hi.value = cp0_entry_hi_read();
  550.        
  551.         if (hi.asid == asid) {
  552.             lo0.value = cp0_entry_lo0_read();
  553.             lo1.value = cp0_entry_lo1_read();
  554.  
  555.             lo0.v = 0;
  556.             lo1.v = 0;
  557.  
  558.             cp0_entry_lo0_write(lo0.value);
  559.             cp0_entry_lo1_write(lo1.value);
  560.  
  561.             tlbwi();
  562.         }
  563.     }
  564.    
  565.     interrupts_restore(ipl);
  566.     cp0_entry_hi_write(hi_save.value);
  567. }
  568.  
  569. /** Invalidate TLB entries for specified page range belonging to specified
  570.  * address space.
  571.  *
  572.  * @param asid      Address space identifier.
  573.  * @param page      First page whose TLB entry is to be invalidated.
  574.  * @param cnt       Number of entries to invalidate.
  575.  */
  576. void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
  577. {
  578.     unsigned int i;
  579.     ipl_t ipl;
  580.     entry_lo_t lo0, lo1;
  581.     entry_hi_t hi, hi_save;
  582.     tlb_index_t index;
  583.  
  584.     ASSERT(asid != ASID_INVALID);
  585.  
  586.     hi_save.value = cp0_entry_hi_read();
  587.     ipl = interrupts_disable();
  588.  
  589.     for (i = 0; i < cnt + 1; i += 2) {
  590.         hi.value = 0;
  591.         tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  592.         cp0_entry_hi_write(hi.value);
  593.  
  594.         tlbp();
  595.         index.value = cp0_index_read();
  596.  
  597.         if (!index.p) {
  598.             /*
  599.              * Entry was found, index register contains valid
  600.              * index.
  601.              */
  602.             tlbr();
  603.  
  604.             lo0.value = cp0_entry_lo0_read();
  605.             lo1.value = cp0_entry_lo1_read();
  606.  
  607.             lo0.v = 0;
  608.             lo1.v = 0;
  609.  
  610.             cp0_entry_lo0_write(lo0.value);
  611.             cp0_entry_lo1_write(lo1.value);
  612.  
  613.             tlbwi();
  614.         }
  615.     }
  616.    
  617.     interrupts_restore(ipl);
  618.     cp0_entry_hi_write(hi_save.value);
  619. }
  620.  
  621. /** @}
  622.  */
  623.