Subversion Repositories HelenOS

Rev

Rev 3228 | Rev 3397 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (c) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. /** @addtogroup mips32mm   
  30.  * @{
  31.  */
  32. /** @file
  33.  */
  34.  
  35. #include <arch/mm/tlb.h>
  36. #include <mm/asid.h>
  37. #include <mm/tlb.h>
  38. #include <mm/page.h>
  39. #include <mm/as.h>
  40. #include <arch/cp0.h>
  41. #include <panic.h>
  42. #include <arch.h>
  43. #include <symtab.h>
  44. #include <synch/spinlock.h>
  45. #include <print.h>
  46. #include <debug.h>
  47. #include <align.h>
  48. #include <interrupt.h>
  49.  
  50. static void tlb_refill_fail(istate_t *istate);
  51. static void tlb_invalid_fail(istate_t *istate);
  52. static void tlb_modified_fail(istate_t *istate);
  53.  
  54. static pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc);
  55.  
  56. /** Initialize TLB
  57.  *
  58.  * Initialize TLB.
  59.  * Invalidate all entries and mark wired entries.
  60.  */
  61. void tlb_arch_init(void)
  62. {
  63.     int i;
  64.  
  65.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  66.     cp0_entry_hi_write(0);
  67.     cp0_entry_lo0_write(0);
  68.     cp0_entry_lo1_write(0);
  69.  
  70.     /* Clear and initialize TLB. */
  71.    
  72.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  73.         cp0_index_write(i);
  74.         tlbwi();
  75.     }
  76.        
  77.     /*
  78.      * The kernel is going to make use of some wired
  79.      * entries (e.g. mapping kernel stacks in kseg3).
  80.      */
  81.     cp0_wired_write(TLB_WIRED);
  82. }
  83.  
  84. /** Process TLB Refill Exception
  85.  *
  86.  * Process TLB Refill Exception.
  87.  *
  88.  * @param istate Interrupted register context.
  89.  */
  90. void tlb_refill(istate_t *istate)
  91. {
  92.     entry_lo_t lo;
  93.     entry_hi_t hi;
  94.     asid_t asid;
  95.     uintptr_t badvaddr;
  96.     pte_t *pte;
  97.     int pfrc;
  98.  
  99.     badvaddr = cp0_badvaddr_read();
  100.  
  101.     spinlock_lock(&AS->lock);
  102.     asid = AS->asid;
  103.     spinlock_unlock(&AS->lock);
  104.  
  105.     page_table_lock(AS, true);
  106.  
  107.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  108.     if (!pte) {
  109.         switch (pfrc) {
  110.         case AS_PF_FAULT:
  111.             goto fail;
  112.             break;
  113.         case AS_PF_DEFER:
  114.             /*
  115.              * The page fault came during copy_from_uspace()
  116.              * or copy_to_uspace().
  117.              */
  118.             page_table_unlock(AS, true);
  119.             return;
  120.         default:
  121.             panic("unexpected pfrc (%d)\n", pfrc);
  122.         }
  123.     }
  124.  
  125.     /*
  126.      * Record access to PTE.
  127.      */
  128.     pte->a = 1;
  129.  
  130.     tlb_prepare_entry_hi(&hi, asid, badvaddr);
  131.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  132.  
  133.     /*
  134.      * New entry is to be inserted into TLB
  135.      */
  136.     cp0_entry_hi_write(hi.value);
  137.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  138.         cp0_entry_lo0_write(lo.value);
  139.         cp0_entry_lo1_write(0);
  140.     }
  141.     else {
  142.         cp0_entry_lo0_write(0);
  143.         cp0_entry_lo1_write(lo.value);
  144.     }
  145.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  146.     tlbwr();
  147.  
  148.     page_table_unlock(AS, true);
  149.     return;
  150.    
  151. fail:
  152.     page_table_unlock(AS, true);
  153.     tlb_refill_fail(istate);
  154. }
  155.  
  156. /** Process TLB Invalid Exception
  157.  *
  158.  * Process TLB Invalid Exception.
  159.  *
  160.  * @param istate Interrupted register context.
  161.  */
  162. void tlb_invalid(istate_t *istate)
  163. {
  164.     tlb_index_t index;
  165.     uintptr_t badvaddr;
  166.     entry_lo_t lo;
  167.     entry_hi_t hi;
  168.     pte_t *pte;
  169.     int pfrc;
  170.  
  171.     badvaddr = cp0_badvaddr_read();
  172.  
  173.     /*
  174.      * Locate the faulting entry in TLB.
  175.      */
  176.     hi.value = cp0_entry_hi_read();
  177.     tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
  178.     cp0_entry_hi_write(hi.value);
  179.     tlbp();
  180.     index.value = cp0_index_read();
  181.  
  182.     page_table_lock(AS, true); 
  183.    
  184.     /*
  185.      * Fail if the entry is not in TLB.
  186.      */
  187.     if (index.p) {
  188.         printf("TLB entry not found.\n");
  189.         goto fail;
  190.     }
  191.  
  192.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  193.     if (!pte) {
  194.         switch (pfrc) {
  195.         case AS_PF_FAULT:
  196.             goto fail;
  197.             break;
  198.         case AS_PF_DEFER:
  199.             /*
  200.              * The page fault came during copy_from_uspace()
  201.              * or copy_to_uspace().
  202.              */
  203.             page_table_unlock(AS, true);             
  204.             return;
  205.         default:
  206.             panic("unexpected pfrc (%d)\n", pfrc);
  207.         }
  208.     }
  209.  
  210.     /*
  211.      * Read the faulting TLB entry.
  212.      */
  213.     tlbr();
  214.  
  215.     /*
  216.      * Record access to PTE.
  217.      */
  218.     pte->a = 1;
  219.  
  220.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  221.  
  222.     /*
  223.      * The entry is to be updated in TLB.
  224.      */
  225.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  226.         cp0_entry_lo0_write(lo.value);
  227.     else
  228.         cp0_entry_lo1_write(lo.value);
  229.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  230.     tlbwi();
  231.  
  232.     page_table_unlock(AS, true);
  233.     return;
  234.    
  235. fail:
  236.     page_table_unlock(AS, true);
  237.     tlb_invalid_fail(istate);
  238. }
  239.  
  240. /** Process TLB Modified Exception
  241.  *
  242.  * Process TLB Modified Exception.
  243.  *
  244.  * @param istate Interrupted register context.
  245.  */
  246. void tlb_modified(istate_t *istate)
  247. {
  248.     tlb_index_t index;
  249.     uintptr_t badvaddr;
  250.     entry_lo_t lo;
  251.     entry_hi_t hi;
  252.     pte_t *pte;
  253.     int pfrc;
  254.  
  255.     badvaddr = cp0_badvaddr_read();
  256.  
  257.     /*
  258.      * Locate the faulting entry in TLB.
  259.      */
  260.     hi.value = cp0_entry_hi_read();
  261.     tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
  262.     cp0_entry_hi_write(hi.value);
  263.     tlbp();
  264.     index.value = cp0_index_read();
  265.  
  266.     page_table_lock(AS, true); 
  267.    
  268.     /*
  269.      * Fail if the entry is not in TLB.
  270.      */
  271.     if (index.p) {
  272.         printf("TLB entry not found.\n");
  273.         goto fail;
  274.     }
  275.  
  276.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
  277.     if (!pte) {
  278.         switch (pfrc) {
  279.         case AS_PF_FAULT:
  280.             goto fail;
  281.             break;
  282.         case AS_PF_DEFER:
  283.             /*
  284.              * The page fault came during copy_from_uspace()
  285.              * or copy_to_uspace().
  286.              */
  287.             page_table_unlock(AS, true);             
  288.             return;
  289.         default:
  290.             panic("unexpected pfrc (%d)\n", pfrc);
  291.         }
  292.     }
  293.  
  294.     /*
  295.      * Fail if the page is not writable.
  296.      */
  297.     if (!pte->w)
  298.         goto fail;
  299.  
  300.     /*
  301.      * Read the faulting TLB entry.
  302.      */
  303.     tlbr();
  304.  
  305.     /*
  306.      * Record access and write to PTE.
  307.      */
  308.     pte->a = 1;
  309.     pte->d = 1;
  310.  
  311.     tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
  312.  
  313.     /*
  314.      * The entry is to be updated in TLB.
  315.      */
  316.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  317.         cp0_entry_lo0_write(lo.value);
  318.     else
  319.         cp0_entry_lo1_write(lo.value);
  320.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  321.     tlbwi();
  322.  
  323.     page_table_unlock(AS, true);
  324.     return;
  325.    
  326. fail:
  327.     page_table_unlock(AS, true);
  328.     tlb_modified_fail(istate);
  329. }
  330.  
  331. void tlb_refill_fail(istate_t *istate)
  332. {
  333.     char *symbol = "";
  334.     char *sym2 = "";
  335.  
  336.     char *s = get_symtab_entry(istate->epc);
  337.     if (s)
  338.         symbol = s;
  339.     s = get_symtab_entry(istate->ra);
  340.     if (s)
  341.         sym2 = s;
  342.  
  343.     fault_if_from_uspace(istate, "TLB Refill Exception on %p", cp0_badvaddr_read());
  344.     panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
  345. }
  346.  
  347.  
  348. void tlb_invalid_fail(istate_t *istate)
  349. {
  350.     char *symbol = "";
  351.  
  352.     char *s = get_symtab_entry(istate->epc);
  353.     if (s)
  354.         symbol = s;
  355.     fault_if_from_uspace(istate, "TLB Invalid Exception on %p", cp0_badvaddr_read());
  356.     panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  357. }
  358.  
  359. void tlb_modified_fail(istate_t *istate)
  360. {
  361.     char *symbol = "";
  362.  
  363.     char *s = get_symtab_entry(istate->epc);
  364.     if (s)
  365.         symbol = s;
  366.     fault_if_from_uspace(istate, "TLB Modified Exception on %p", cp0_badvaddr_read());
  367.     panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  368. }
  369.  
  370. /** Try to find PTE for faulting address
  371.  *
  372.  * Try to find PTE for faulting address.
  373.  * The AS->lock must be held on entry to this function.
  374.  *
  375.  * @param badvaddr Faulting virtual address.
  376.  * @param access Access mode that caused the fault.
  377.  * @param istate Pointer to interrupted state.
  378.  * @param pfrc Pointer to variable where as_page_fault() return code will be stored.
  379.  *
  380.  * @return PTE on success, NULL otherwise.
  381.  */
  382. pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc)
  383. {
  384.     entry_hi_t hi;
  385.     pte_t *pte;
  386.  
  387.     hi.value = cp0_entry_hi_read();
  388.  
  389.     /*
  390.      * Handler cannot succeed if the ASIDs don't match.
  391.      */
  392.     if (hi.asid != AS->asid) {
  393.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  394.         return NULL;
  395.     }
  396.  
  397.     /*
  398.      * Check if the mapping exists in page tables.
  399.      */
  400.     pte = page_mapping_find(AS, badvaddr);
  401.     if (pte && pte->p) {
  402.         /*
  403.          * Mapping found in page tables.
  404.          * Immediately succeed.
  405.          */
  406.         return pte;
  407.     } else {
  408.         int rc;
  409.        
  410.         /*
  411.          * Mapping not found in page tables.
  412.          * Resort to higher-level page fault handler.
  413.          */
  414.         page_table_unlock(AS, true);
  415.         switch (rc = as_page_fault(badvaddr, access, istate)) {
  416.         case AS_PF_OK:
  417.             /*
  418.              * The higher-level page fault handler succeeded,
  419.              * The mapping ought to be in place.
  420.              */
  421.             page_table_lock(AS, true);
  422.             pte = page_mapping_find(AS, badvaddr);
  423.             ASSERT(pte && pte->p);
  424.             return pte;
  425.             break;
  426.         case AS_PF_DEFER:
  427.             page_table_lock(AS, true);
  428.             *pfrc = AS_PF_DEFER;
  429.             return NULL;
  430.             break;
  431.         case AS_PF_FAULT:
  432.             page_table_lock(AS, true);
  433.             printf("Page fault.\n");
  434.             *pfrc = AS_PF_FAULT;
  435.             return NULL;
  436.             break;
  437.         default:
  438.             panic("unexpected rc (%d)\n", rc);
  439.         }
  440.        
  441.     }
  442. }
  443.  
  444. void tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn)
  445. {
  446.     lo->value = 0;
  447.     lo->g = g;
  448.     lo->v = v;
  449.     lo->d = d;
  450.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  451.     lo->pfn = pfn;
  452. }
  453.  
  454. void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
  455. {
  456.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  457.     hi->asid = asid;
  458. }
  459.  
  460. /** Print contents of TLB. */
  461. void tlb_print(void)
  462. {
  463.     page_mask_t mask;
  464.     entry_lo_t lo0, lo1;
  465.     entry_hi_t hi, hi_save;
  466.     unsigned int i;
  467.  
  468.     hi_save.value = cp0_entry_hi_read();
  469.    
  470.     printf("#  ASID VPN2   MASK G V D C PFN\n");
  471.     printf("-- ---- ------ ---- - - - - ------\n");
  472.    
  473.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  474.         cp0_index_write(i);
  475.         tlbr();
  476.        
  477.         mask.value = cp0_pagemask_read();
  478.         hi.value = cp0_entry_hi_read();
  479.         lo0.value = cp0_entry_lo0_read();
  480.         lo1.value = cp0_entry_lo1_read();
  481.        
  482.         printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
  483.             i, hi.asid, hi.vpn2, mask.mask,
  484.             lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
  485.         printf("                    %1u %1u %1u %1u %#6x\n",
  486.             lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  487.     }
  488.    
  489.     cp0_entry_hi_write(hi_save.value);
  490. }
  491.  
  492. /** Invalidate all not wired TLB entries. */
  493. void tlb_invalidate_all(void)
  494. {
  495.     ipl_t ipl;
  496.     entry_lo_t lo0, lo1;
  497.     entry_hi_t hi_save;
  498.     int i;
  499.  
  500.     hi_save.value = cp0_entry_hi_read();
  501.     ipl = interrupts_disable();
  502.  
  503.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  504.         cp0_index_write(i);
  505.         tlbr();
  506.  
  507.         lo0.value = cp0_entry_lo0_read();
  508.         lo1.value = cp0_entry_lo1_read();
  509.  
  510.         lo0.v = 0;
  511.         lo1.v = 0;
  512.  
  513.         cp0_entry_lo0_write(lo0.value);
  514.         cp0_entry_lo1_write(lo1.value);
  515.                
  516.         tlbwi();
  517.     }
  518.    
  519.     interrupts_restore(ipl);
  520.     cp0_entry_hi_write(hi_save.value);
  521. }
  522.  
  523. /** Invalidate all TLB entries belonging to specified address space.
  524.  *
  525.  * @param asid Address space identifier.
  526.  */
  527. void tlb_invalidate_asid(asid_t asid)
  528. {
  529.     ipl_t ipl;
  530.     entry_lo_t lo0, lo1;
  531.     entry_hi_t hi, hi_save;
  532.     int i;
  533.  
  534.     ASSERT(asid != ASID_INVALID);
  535.  
  536.     hi_save.value = cp0_entry_hi_read();
  537.     ipl = interrupts_disable();
  538.    
  539.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  540.         cp0_index_write(i);
  541.         tlbr();
  542.        
  543.         hi.value = cp0_entry_hi_read();
  544.        
  545.         if (hi.asid == asid) {
  546.             lo0.value = cp0_entry_lo0_read();
  547.             lo1.value = cp0_entry_lo1_read();
  548.  
  549.             lo0.v = 0;
  550.             lo1.v = 0;
  551.  
  552.             cp0_entry_lo0_write(lo0.value);
  553.             cp0_entry_lo1_write(lo1.value);
  554.  
  555.             tlbwi();
  556.         }
  557.     }
  558.    
  559.     interrupts_restore(ipl);
  560.     cp0_entry_hi_write(hi_save.value);
  561. }
  562.  
  563. /** Invalidate TLB entries for specified page range belonging to specified address space.
  564.  *
  565.  * @param asid Address space identifier.
  566.  * @param page First page whose TLB entry is to be invalidated.
  567.  * @param cnt Number of entries to invalidate.
  568.  */
  569. void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
  570. {
  571.     unsigned int i;
  572.     ipl_t ipl;
  573.     entry_lo_t lo0, lo1;
  574.     entry_hi_t hi, hi_save;
  575.     tlb_index_t index;
  576.  
  577.     ASSERT(asid != ASID_INVALID);
  578.  
  579.     hi_save.value = cp0_entry_hi_read();
  580.     ipl = interrupts_disable();
  581.  
  582.     for (i = 0; i < cnt + 1; i += 2) {
  583.         hi.value = 0;
  584.         tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  585.         cp0_entry_hi_write(hi.value);
  586.  
  587.         tlbp();
  588.         index.value = cp0_index_read();
  589.  
  590.         if (!index.p) {
  591.             /* Entry was found, index register contains valid index. */
  592.             tlbr();
  593.  
  594.             lo0.value = cp0_entry_lo0_read();
  595.             lo1.value = cp0_entry_lo1_read();
  596.  
  597.             lo0.v = 0;
  598.             lo1.v = 0;
  599.  
  600.             cp0_entry_lo0_write(lo0.value);
  601.             cp0_entry_lo1_write(lo1.value);
  602.  
  603.             tlbwi();
  604.         }
  605.     }
  606.    
  607.     interrupts_restore(ipl);
  608.     cp0_entry_hi_write(hi_save.value);
  609. }
  610.  
  611. /** @}
  612.  */
  613.