Subversion Repositories HelenOS

Rev

Rev 2720 | Rev 2927 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (c) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. /** @addtogroup mips32mm   
  30.  * @{
  31.  */
  32. /** @file
  33.  */
  34.  
  35. #include <arch/mm/tlb.h>
  36. #include <mm/asid.h>
  37. #include <mm/tlb.h>
  38. #include <mm/page.h>
  39. #include <mm/as.h>
  40. #include <arch/cp0.h>
  41. #include <panic.h>
  42. #include <arch.h>
  43. #include <symtab.h>
  44. #include <synch/spinlock.h>
  45. #include <print.h>
  46. #include <debug.h>
  47. #include <align.h>
  48. #include <interrupt.h>
  49.  
  50. static void tlb_refill_fail(istate_t *istate);
  51. static void tlb_invalid_fail(istate_t *istate);
  52. static void tlb_modified_fail(istate_t *istate);
  53.  
  54. static pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc);
  55.  
  56. static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn);
  57. static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr);
  58.  
  59. /** Initialize TLB
  60.  *
  61.  * Initialize TLB.
  62.  * Invalidate all entries and mark wired entries.
  63.  */
  64. void tlb_arch_init(void)
  65. {
  66.     int i;
  67.  
  68.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  69.     cp0_entry_hi_write(0);
  70.     cp0_entry_lo0_write(0);
  71.     cp0_entry_lo1_write(0);
  72.  
  73.     /* Clear and initialize TLB. */
  74.    
  75.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  76.         cp0_index_write(i);
  77.         tlbwi();
  78.     }
  79.  
  80.        
  81.     /*
  82.      * The kernel is going to make use of some wired
  83.      * entries (e.g. mapping kernel stacks in kseg3).
  84.      */
  85.     cp0_wired_write(TLB_WIRED);
  86. }
  87.  
  88. /** Process TLB Refill Exception
  89.  *
  90.  * Process TLB Refill Exception.
  91.  *
  92.  * @param istate Interrupted register context.
  93.  */
  94. void tlb_refill(istate_t *istate)
  95. {
  96.     entry_lo_t lo;
  97.     entry_hi_t hi;
  98.     asid_t asid;
  99.     uintptr_t badvaddr;
  100.     pte_t *pte;
  101.     int pfrc;
  102.  
  103.     badvaddr = cp0_badvaddr_read();
  104.  
  105.     spinlock_lock(&AS->lock);
  106.     asid = AS->asid;
  107.     spinlock_unlock(&AS->lock);
  108.  
  109.     page_table_lock(AS, true);
  110.  
  111.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  112.     if (!pte) {
  113.         switch (pfrc) {
  114.         case AS_PF_FAULT:
  115.             goto fail;
  116.             break;
  117.         case AS_PF_DEFER:
  118.             /*
  119.              * The page fault came during copy_from_uspace()
  120.              * or copy_to_uspace().
  121.              */
  122.             page_table_unlock(AS, true);
  123.             return;
  124.         default:
  125.             panic("unexpected pfrc (%d)\n", pfrc);
  126.         }
  127.     }
  128.  
  129.     /*
  130.      * Record access to PTE.
  131.      */
  132.     pte->a = 1;
  133.  
  134.     prepare_entry_hi(&hi, asid, badvaddr);
  135.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  136.  
  137.     /*
  138.      * New entry is to be inserted into TLB
  139.      */
  140.     cp0_entry_hi_write(hi.value);
  141.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  142.         cp0_entry_lo0_write(lo.value);
  143.         cp0_entry_lo1_write(0);
  144.     }
  145.     else {
  146.         cp0_entry_lo0_write(0);
  147.         cp0_entry_lo1_write(lo.value);
  148.     }
  149.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  150.     tlbwr();
  151.  
  152.     page_table_unlock(AS, true);
  153.     return;
  154.    
  155. fail:
  156.     page_table_unlock(AS, true);
  157.     tlb_refill_fail(istate);
  158. }
  159.  
  160. /** Process TLB Invalid Exception
  161.  *
  162.  * Process TLB Invalid Exception.
  163.  *
  164.  * @param istate Interrupted register context.
  165.  */
  166. void tlb_invalid(istate_t *istate)
  167. {
  168.     tlb_index_t index;
  169.     uintptr_t badvaddr;
  170.     entry_lo_t lo;
  171.     entry_hi_t hi;
  172.     pte_t *pte;
  173.     int pfrc;
  174.  
  175.     badvaddr = cp0_badvaddr_read();
  176.  
  177.     /*
  178.      * Locate the faulting entry in TLB.
  179.      */
  180.     hi.value = cp0_entry_hi_read();
  181.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  182.     cp0_entry_hi_write(hi.value);
  183.     tlbp();
  184.     index.value = cp0_index_read();
  185.  
  186.     page_table_lock(AS, true); 
  187.    
  188.     /*
  189.      * Fail if the entry is not in TLB.
  190.      */
  191.     if (index.p) {
  192.         printf("TLB entry not found.\n");
  193.         goto fail;
  194.     }
  195.  
  196.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  197.     if (!pte) {
  198.         switch (pfrc) {
  199.         case AS_PF_FAULT:
  200.             goto fail;
  201.             break;
  202.         case AS_PF_DEFER:
  203.             /*
  204.              * The page fault came during copy_from_uspace()
  205.              * or copy_to_uspace().
  206.              */
  207.             page_table_unlock(AS, true);             
  208.             return;
  209.         default:
  210.             panic("unexpected pfrc (%d)\n", pfrc);
  211.         }
  212.     }
  213.  
  214.     /*
  215.      * Read the faulting TLB entry.
  216.      */
  217.     tlbr();
  218.  
  219.     /*
  220.      * Record access to PTE.
  221.      */
  222.     pte->a = 1;
  223.  
  224.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  225.  
  226.     /*
  227.      * The entry is to be updated in TLB.
  228.      */
  229.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  230.         cp0_entry_lo0_write(lo.value);
  231.     else
  232.         cp0_entry_lo1_write(lo.value);
  233.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  234.     tlbwi();
  235.  
  236.     page_table_unlock(AS, true);
  237.     return;
  238.    
  239. fail:
  240.     page_table_unlock(AS, true);
  241.     tlb_invalid_fail(istate);
  242. }
  243.  
  244. /** Process TLB Modified Exception
  245.  *
  246.  * Process TLB Modified Exception.
  247.  *
  248.  * @param istate Interrupted register context.
  249.  */
  250. void tlb_modified(istate_t *istate)
  251. {
  252.     tlb_index_t index;
  253.     uintptr_t badvaddr;
  254.     entry_lo_t lo;
  255.     entry_hi_t hi;
  256.     pte_t *pte;
  257.     int pfrc;
  258.  
  259.     badvaddr = cp0_badvaddr_read();
  260.  
  261.     /*
  262.      * Locate the faulting entry in TLB.
  263.      */
  264.     hi.value = cp0_entry_hi_read();
  265.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  266.     cp0_entry_hi_write(hi.value);
  267.     tlbp();
  268.     index.value = cp0_index_read();
  269.  
  270.     page_table_lock(AS, true); 
  271.    
  272.     /*
  273.      * Fail if the entry is not in TLB.
  274.      */
  275.     if (index.p) {
  276.         printf("TLB entry not found.\n");
  277.         goto fail;
  278.     }
  279.  
  280.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
  281.     if (!pte) {
  282.         switch (pfrc) {
  283.         case AS_PF_FAULT:
  284.             goto fail;
  285.             break;
  286.         case AS_PF_DEFER:
  287.             /*
  288.              * The page fault came during copy_from_uspace()
  289.              * or copy_to_uspace().
  290.              */
  291.             page_table_unlock(AS, true);             
  292.             return;
  293.         default:
  294.             panic("unexpected pfrc (%d)\n", pfrc);
  295.         }
  296.     }
  297.  
  298.     /*
  299.      * Fail if the page is not writable.
  300.      */
  301.     if (!pte->w)
  302.         goto fail;
  303.  
  304.     /*
  305.      * Read the faulting TLB entry.
  306.      */
  307.     tlbr();
  308.  
  309.     /*
  310.      * Record access and write to PTE.
  311.      */
  312.     pte->a = 1;
  313.     pte->d = 1;
  314.  
  315.     prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
  316.  
  317.     /*
  318.      * The entry is to be updated in TLB.
  319.      */
  320.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  321.         cp0_entry_lo0_write(lo.value);
  322.     else
  323.         cp0_entry_lo1_write(lo.value);
  324.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  325.     tlbwi();
  326.  
  327.     page_table_unlock(AS, true);
  328.     return;
  329.    
  330. fail:
  331.     page_table_unlock(AS, true);
  332.     tlb_modified_fail(istate);
  333. }
  334.  
  335. void tlb_refill_fail(istate_t *istate)
  336. {
  337.     char *symbol = "";
  338.     char *sym2 = "";
  339.  
  340.     char *s = get_symtab_entry(istate->epc);
  341.     if (s)
  342.         symbol = s;
  343.     s = get_symtab_entry(istate->ra);
  344.     if (s)
  345.         sym2 = s;
  346.  
  347.     fault_if_from_uspace(istate, "TLB Refill Exception on %p", cp0_badvaddr_read());
  348.     panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
  349. }
  350.  
  351.  
  352. void tlb_invalid_fail(istate_t *istate)
  353. {
  354.     char *symbol = "";
  355.  
  356.     char *s = get_symtab_entry(istate->epc);
  357.     if (s)
  358.         symbol = s;
  359.     fault_if_from_uspace(istate, "TLB Invalid Exception on %p", cp0_badvaddr_read());
  360.     panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  361. }
  362.  
  363. void tlb_modified_fail(istate_t *istate)
  364. {
  365.     char *symbol = "";
  366.  
  367.     char *s = get_symtab_entry(istate->epc);
  368.     if (s)
  369.         symbol = s;
  370.     fault_if_from_uspace(istate, "TLB Modified Exception on %p", cp0_badvaddr_read());
  371.     panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  372. }
  373.  
  374. /** Try to find PTE for faulting address
  375.  *
  376.  * Try to find PTE for faulting address.
  377.  * The AS->lock must be held on entry to this function.
  378.  *
  379.  * @param badvaddr Faulting virtual address.
  380.  * @param access Access mode that caused the fault.
  381.  * @param istate Pointer to interrupted state.
  382.  * @param pfrc Pointer to variable where as_page_fault() return code will be stored.
  383.  *
  384.  * @return PTE on success, NULL otherwise.
  385.  */
  386. pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc)
  387. {
  388.     entry_hi_t hi;
  389.     pte_t *pte;
  390.  
  391.     hi.value = cp0_entry_hi_read();
  392.  
  393.     /*
  394.      * Handler cannot succeed if the ASIDs don't match.
  395.      */
  396.     if (hi.asid != AS->asid) {
  397.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  398.         return NULL;
  399.     }
  400.  
  401.     /*
  402.      * Check if the mapping exists in page tables.
  403.      */
  404.     pte = page_mapping_find(AS, badvaddr);
  405.     if (pte && pte->p) {
  406.         /*
  407.          * Mapping found in page tables.
  408.          * Immediately succeed.
  409.          */
  410.         return pte;
  411.     } else {
  412.         int rc;
  413.        
  414.         /*
  415.          * Mapping not found in page tables.
  416.          * Resort to higher-level page fault handler.
  417.          */
  418.         page_table_unlock(AS, true);
  419.         switch (rc = as_page_fault(badvaddr, access, istate)) {
  420.         case AS_PF_OK:
  421.             /*
  422.              * The higher-level page fault handler succeeded,
  423.              * The mapping ought to be in place.
  424.              */
  425.             page_table_lock(AS, true);
  426.             pte = page_mapping_find(AS, badvaddr);
  427.             ASSERT(pte && pte->p);
  428.             return pte;
  429.             break;
  430.         case AS_PF_DEFER:
  431.             page_table_lock(AS, true);
  432.             *pfrc = AS_PF_DEFER;
  433.             return NULL;
  434.             break;
  435.         case AS_PF_FAULT:
  436.             page_table_lock(AS, true);
  437.             printf("Page fault.\n");
  438.             *pfrc = AS_PF_FAULT;
  439.             return NULL;
  440.             break;
  441.         default:
  442.             panic("unexpected rc (%d)\n", rc);
  443.         }
  444.        
  445.     }
  446. }
  447.  
  448. void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn)
  449. {
  450.     lo->value = 0;
  451.     lo->g = g;
  452.     lo->v = v;
  453.     lo->d = d;
  454.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  455.     lo->pfn = pfn;
  456. }
  457.  
  458. void prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
  459. {
  460.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  461.     hi->asid = asid;
  462. }
  463.  
  464. /** Print contents of TLB. */
  465. void tlb_print(void)
  466. {
  467.     page_mask_t mask;
  468.     entry_lo_t lo0, lo1;
  469.     entry_hi_t hi, hi_save;
  470.     unsigned int i;
  471.  
  472.     hi_save.value = cp0_entry_hi_read();
  473.    
  474.     printf("#  ASID VPN2   MASK G V D C PFN\n");
  475.     printf("-- ---- ------ ---- - - - - ------\n");
  476.    
  477.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  478.         cp0_index_write(i);
  479.         tlbr();
  480.        
  481.         mask.value = cp0_pagemask_read();
  482.         hi.value = cp0_entry_hi_read();
  483.         lo0.value = cp0_entry_lo0_read();
  484.         lo1.value = cp0_entry_lo1_read();
  485.        
  486.         printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
  487.             i, hi.asid, hi.vpn2, mask.mask,
  488.             lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
  489.         printf("                    %1u %1u %1u %1u %#6x\n",
  490.             lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  491.     }
  492.    
  493.     cp0_entry_hi_write(hi_save.value);
  494. }
  495.  
  496. /** Invalidate all not wired TLB entries. */
  497. void tlb_invalidate_all(void)
  498. {
  499.     ipl_t ipl;
  500.     entry_lo_t lo0, lo1;
  501.     entry_hi_t hi_save;
  502.     int i;
  503.  
  504.     hi_save.value = cp0_entry_hi_read();
  505.     ipl = interrupts_disable();
  506.  
  507.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  508.         cp0_index_write(i);
  509.         tlbr();
  510.  
  511.         lo0.value = cp0_entry_lo0_read();
  512.         lo1.value = cp0_entry_lo1_read();
  513.  
  514.         lo0.v = 0;
  515.         lo1.v = 0;
  516.  
  517.         cp0_entry_lo0_write(lo0.value);
  518.         cp0_entry_lo1_write(lo1.value);
  519.                
  520.         tlbwi();
  521.     }
  522.    
  523.     interrupts_restore(ipl);
  524.     cp0_entry_hi_write(hi_save.value);
  525. }
  526.  
  527. /** Invalidate all TLB entries belonging to specified address space.
  528.  *
  529.  * @param asid Address space identifier.
  530.  */
  531. void tlb_invalidate_asid(asid_t asid)
  532. {
  533.     ipl_t ipl;
  534.     entry_lo_t lo0, lo1;
  535.     entry_hi_t hi, hi_save;
  536.     int i;
  537.  
  538.     ASSERT(asid != ASID_INVALID);
  539.  
  540.     hi_save.value = cp0_entry_hi_read();
  541.     ipl = interrupts_disable();
  542.    
  543.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  544.         cp0_index_write(i);
  545.         tlbr();
  546.        
  547.         hi.value = cp0_entry_hi_read();
  548.        
  549.         if (hi.asid == asid) {
  550.             lo0.value = cp0_entry_lo0_read();
  551.             lo1.value = cp0_entry_lo1_read();
  552.  
  553.             lo0.v = 0;
  554.             lo1.v = 0;
  555.  
  556.             cp0_entry_lo0_write(lo0.value);
  557.             cp0_entry_lo1_write(lo1.value);
  558.  
  559.             tlbwi();
  560.         }
  561.     }
  562.    
  563.     interrupts_restore(ipl);
  564.     cp0_entry_hi_write(hi_save.value);
  565. }
  566.  
  567. /** Invalidate TLB entries for specified page range belonging to specified address space.
  568.  *
  569.  * @param asid Address space identifier.
  570.  * @param page First page whose TLB entry is to be invalidated.
  571.  * @param cnt Number of entries to invalidate.
  572.  */
  573. void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
  574. {
  575.     unsigned int i;
  576.     ipl_t ipl;
  577.     entry_lo_t lo0, lo1;
  578.     entry_hi_t hi, hi_save;
  579.     tlb_index_t index;
  580.  
  581.     ASSERT(asid != ASID_INVALID);
  582.  
  583.     hi_save.value = cp0_entry_hi_read();
  584.     ipl = interrupts_disable();
  585.  
  586.     for (i = 0; i < cnt + 1; i += 2) {
  587.         hi.value = 0;
  588.         prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  589.         cp0_entry_hi_write(hi.value);
  590.  
  591.         tlbp();
  592.         index.value = cp0_index_read();
  593.  
  594.         if (!index.p) {
  595.             /* Entry was found, index register contains valid index. */
  596.             tlbr();
  597.  
  598.             lo0.value = cp0_entry_lo0_read();
  599.             lo1.value = cp0_entry_lo1_read();
  600.  
  601.             lo0.v = 0;
  602.             lo1.v = 0;
  603.  
  604.             cp0_entry_lo0_write(lo0.value);
  605.             cp0_entry_lo1_write(lo1.value);
  606.  
  607.             tlbwi();
  608.         }
  609.     }
  610.    
  611.     interrupts_restore(ipl);
  612.     cp0_entry_hi_write(hi_save.value);
  613. }
  614.  
  615. /** @}
  616.  */
  617.