Subversion Repositories HelenOS

Rev

Rev 1965 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (C) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. #include <arch/mm/tlb.h>
  30. #include <mm/asid.h>
  31. #include <mm/tlb.h>
  32. #include <mm/page.h>
  33. #include <mm/as.h>
  34. #include <arch/cp0.h>
  35. #include <panic.h>
  36. #include <arch.h>
  37. #include <symtab.h>
  38. #include <synch/spinlock.h>
  39. #include <print.h>
  40. #include <debug.h>
  41. #include <align.h>
  42. #include <interrupt.h>
  43.  
  44. static void tlb_refill_fail(istate_t *istate);
  45. static void tlb_invalid_fail(istate_t *istate);
  46. static void tlb_modified_fail(istate_t *istate);
  47.  
  48. static pte_t *find_mapping_and_check(__address badvaddr, int access, istate_t *istate, int *pfrc);
  49.  
  50. static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
  51. static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
  52.  
  53. /** Initialize TLB
  54.  *
  55.  * Initialize TLB.
  56.  * Invalidate all entries and mark wired entries.
  57.  */
  58. void tlb_arch_init(void)
  59. {
  60.     int i;
  61.  
  62.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  63.     cp0_entry_hi_write(0);
  64.     cp0_entry_lo0_write(0);
  65.     cp0_entry_lo1_write(0);
  66.  
  67.     /* Clear and initialize TLB. */
  68.    
  69.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  70.         cp0_index_write(i);
  71.         tlbwi();
  72.     }
  73.  
  74.        
  75.     /*
  76.      * The kernel is going to make use of some wired
  77.      * entries (e.g. mapping kernel stacks in kseg3).
  78.      */
  79.     cp0_wired_write(TLB_WIRED);
  80. }
  81.  
  82. /** Process TLB Refill Exception
  83.  *
  84.  * Process TLB Refill Exception.
  85.  *
  86.  * @param istate Interrupted register context.
  87.  */
  88. void tlb_refill(istate_t *istate)
  89. {
  90.     entry_lo_t lo;
  91.     entry_hi_t hi;
  92.     asid_t asid;
  93.     __address badvaddr;
  94.     pte_t *pte;
  95.     int pfrc;
  96.  
  97.     badvaddr = cp0_badvaddr_read();
  98.  
  99.     spinlock_lock(&AS->lock);
  100.     asid = AS->asid;
  101.     spinlock_unlock(&AS->lock);
  102.  
  103.     page_table_lock(AS, true);
  104.  
  105.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  106.     if (!pte) {
  107.         switch (pfrc) {
  108.         case AS_PF_FAULT:
  109.             goto fail;
  110.             break;
  111.         case AS_PF_DEFER:
  112.             /*
  113.              * The page fault came during copy_from_uspace()
  114.              * or copy_to_uspace().
  115.              */
  116.             page_table_unlock(AS, true);
  117.             return;
  118.         default:
  119.             panic("unexpected pfrc (%d)\n", pfrc);
  120.         }
  121.     }
  122.  
  123.     /*
  124.      * Record access to PTE.
  125.      */
  126.     pte->a = 1;
  127.  
  128.     prepare_entry_hi(&hi, asid, badvaddr);
  129.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  130.  
  131.     /*
  132.      * New entry is to be inserted into TLB
  133.      */
  134.     cp0_entry_hi_write(hi.value);
  135.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  136.         cp0_entry_lo0_write(lo.value);
  137.         cp0_entry_lo1_write(0);
  138.     }
  139.     else {
  140.         cp0_entry_lo0_write(0);
  141.         cp0_entry_lo1_write(lo.value);
  142.     }
  143.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  144.     tlbwr();
  145.  
  146.     page_table_unlock(AS, true);
  147.     return;
  148.    
  149. fail:
  150.     page_table_unlock(AS, true);
  151.     tlb_refill_fail(istate);
  152. }
  153.  
  154. /** Process TLB Invalid Exception
  155.  *
  156.  * Process TLB Invalid Exception.
  157.  *
  158.  * @param istate Interrupted register context.
  159.  */
  160. void tlb_invalid(istate_t *istate)
  161. {
  162.     tlb_index_t index;
  163.     __address badvaddr;
  164.     entry_lo_t lo;
  165.     entry_hi_t hi;
  166.     pte_t *pte;
  167.     int pfrc;
  168.  
  169.     badvaddr = cp0_badvaddr_read();
  170.  
  171.     /*
  172.      * Locate the faulting entry in TLB.
  173.      */
  174.     hi.value = cp0_entry_hi_read();
  175.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  176.     cp0_entry_hi_write(hi.value);
  177.     tlbp();
  178.     index.value = cp0_index_read();
  179.  
  180.     page_table_lock(AS, true); 
  181.    
  182.     /*
  183.      * Fail if the entry is not in TLB.
  184.      */
  185.     if (index.p) {
  186.         printf("TLB entry not found.\n");
  187.         goto fail;
  188.     }
  189.  
  190.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
  191.     if (!pte) {
  192.         switch (pfrc) {
  193.         case AS_PF_FAULT:
  194.             goto fail;
  195.             break;
  196.         case AS_PF_DEFER:
  197.             /*
  198.              * The page fault came during copy_from_uspace()
  199.              * or copy_to_uspace().
  200.              */
  201.             page_table_unlock(AS, true);             
  202.             return;
  203.         default:
  204.             panic("unexpected pfrc (%d)\n", pfrc);
  205.         }
  206.     }
  207.  
  208.     /*
  209.      * Read the faulting TLB entry.
  210.      */
  211.     tlbr();
  212.  
  213.     /*
  214.      * Record access to PTE.
  215.      */
  216.     pte->a = 1;
  217.  
  218.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  219.  
  220.     /*
  221.      * The entry is to be updated in TLB.
  222.      */
  223.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  224.         cp0_entry_lo0_write(lo.value);
  225.     else
  226.         cp0_entry_lo1_write(lo.value);
  227.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  228.     tlbwi();
  229.  
  230.     page_table_unlock(AS, true);
  231.     return;
  232.    
  233. fail:
  234.     page_table_unlock(AS, true);
  235.     tlb_invalid_fail(istate);
  236. }
  237.  
  238. /** Process TLB Modified Exception
  239.  *
  240.  * Process TLB Modified Exception.
  241.  *
  242.  * @param istate Interrupted register context.
  243.  */
  244. void tlb_modified(istate_t *istate)
  245. {
  246.     tlb_index_t index;
  247.     __address badvaddr;
  248.     entry_lo_t lo;
  249.     entry_hi_t hi;
  250.     pte_t *pte;
  251.     int pfrc;
  252.  
  253.     badvaddr = cp0_badvaddr_read();
  254.  
  255.     /*
  256.      * Locate the faulting entry in TLB.
  257.      */
  258.     hi.value = cp0_entry_hi_read();
  259.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  260.     cp0_entry_hi_write(hi.value);
  261.     tlbp();
  262.     index.value = cp0_index_read();
  263.  
  264.     page_table_lock(AS, true); 
  265.    
  266.     /*
  267.      * Fail if the entry is not in TLB.
  268.      */
  269.     if (index.p) {
  270.         printf("TLB entry not found.\n");
  271.         goto fail;
  272.     }
  273.  
  274.     pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
  275.     if (!pte) {
  276.         switch (pfrc) {
  277.         case AS_PF_FAULT:
  278.             goto fail;
  279.             break;
  280.         case AS_PF_DEFER:
  281.             /*
  282.              * The page fault came during copy_from_uspace()
  283.              * or copy_to_uspace().
  284.              */
  285.             page_table_unlock(AS, true);             
  286.             return;
  287.         default:
  288.             panic("unexpected pfrc (%d)\n", pfrc);
  289.         }
  290.     }
  291.  
  292.     /*
  293.      * Fail if the page is not writable.
  294.      */
  295.     if (!pte->w)
  296.         goto fail;
  297.  
  298.     /*
  299.      * Read the faulting TLB entry.
  300.      */
  301.     tlbr();
  302.  
  303.     /*
  304.      * Record access and write to PTE.
  305.      */
  306.     pte->a = 1;
  307.     pte->d = 1;
  308.  
  309.     prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
  310.  
  311.     /*
  312.      * The entry is to be updated in TLB.
  313.      */
  314.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  315.         cp0_entry_lo0_write(lo.value);
  316.     else
  317.         cp0_entry_lo1_write(lo.value);
  318.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  319.     tlbwi();
  320.  
  321.     page_table_unlock(AS, true);
  322.     return;
  323.    
  324. fail:
  325.     page_table_unlock(AS, true);
  326.     tlb_modified_fail(istate);
  327. }
  328.  
  329. void tlb_refill_fail(istate_t *istate)
  330. {
  331.     char *symbol = "";
  332.     char *sym2 = "";
  333.  
  334.     char *s = get_symtab_entry(istate->epc);
  335.     if (s)
  336.         symbol = s;
  337.     s = get_symtab_entry(istate->ra);
  338.     if (s)
  339.         sym2 = s;
  340.  
  341.     fault_if_from_uspace(istate, "TLB Refill Exception on %P", cp0_badvaddr_read());
  342.     panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
  343. }
  344.  
  345.  
  346. void tlb_invalid_fail(istate_t *istate)
  347. {
  348.     char *symbol = "";
  349.  
  350.     char *s = get_symtab_entry(istate->epc);
  351.     if (s)
  352.         symbol = s;
  353.     fault_if_from_uspace(istate, "TLB Invalid Exception on %P", cp0_badvaddr_read());
  354.     panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  355. }
  356.  
  357. void tlb_modified_fail(istate_t *istate)
  358. {
  359.     char *symbol = "";
  360.  
  361.     char *s = get_symtab_entry(istate->epc);
  362.     if (s)
  363.         symbol = s;
  364.     fault_if_from_uspace(istate, "TLB Modified Exception on %P", cp0_badvaddr_read());
  365.     panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  366. }
  367.  
  368. /** Try to find PTE for faulting address
  369.  *
  370.  * Try to find PTE for faulting address.
  371.  * The AS->lock must be held on entry to this function.
  372.  *
  373.  * @param badvaddr Faulting virtual address.
  374.  * @param access Access mode that caused the fault.
  375.  * @param istate Pointer to interrupted state.
  376.  * @param pfrc Pointer to variable where as_page_fault() return code will be stored.
  377.  *
  378.  * @return PTE on success, NULL otherwise.
  379.  */
  380. pte_t *find_mapping_and_check(__address badvaddr, int access, istate_t *istate, int *pfrc)
  381. {
  382.     entry_hi_t hi;
  383.     pte_t *pte;
  384.  
  385.     hi.value = cp0_entry_hi_read();
  386.  
  387.     /*
  388.      * Handler cannot succeed if the ASIDs don't match.
  389.      */
  390.     if (hi.asid != AS->asid) {
  391.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  392.         return NULL;
  393.     }
  394.  
  395.     /*
  396.      * Check if the mapping exists in page tables.
  397.      */
  398.     pte = page_mapping_find(AS, badvaddr);
  399.     if (pte && pte->p) {
  400.         /*
  401.          * Mapping found in page tables.
  402.          * Immediately succeed.
  403.          */
  404.         return pte;
  405.     } else {
  406.         int rc;
  407.        
  408.         /*
  409.          * Mapping not found in page tables.
  410.          * Resort to higher-level page fault handler.
  411.          */
  412.         page_table_unlock(AS, true);
  413.         switch (rc = as_page_fault(badvaddr, access, istate)) {
  414.         case AS_PF_OK:
  415.             /*
  416.              * The higher-level page fault handler succeeded,
  417.              * The mapping ought to be in place.
  418.              */
  419.             page_table_lock(AS, true);
  420.             pte = page_mapping_find(AS, badvaddr);
  421.             ASSERT(pte && pte->p);
  422.             return pte;
  423.             break;
  424.         case AS_PF_DEFER:
  425.             page_table_lock(AS, true);
  426.             *pfrc = AS_PF_DEFER;
  427.             return NULL;
  428.             break;
  429.         case AS_PF_FAULT:
  430.             page_table_lock(AS, true);
  431.             printf("Page fault.\n");
  432.             *pfrc = AS_PF_FAULT;
  433.             return NULL;
  434.             break;
  435.         default:
  436.             panic("unexpected rc (%d)\n", rc);
  437.         }
  438.        
  439.     }
  440. }
  441.  
  442. void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
  443. {
  444.     lo->value = 0;
  445.     lo->g = g;
  446.     lo->v = v;
  447.     lo->d = d;
  448.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  449.     lo->pfn = pfn;
  450. }
  451.  
  452. void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
  453. {
  454.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  455.     hi->asid = asid;
  456. }
  457.  
  458. /** Print contents of TLB. */
  459. void tlb_print(void)
  460. {
  461.     page_mask_t mask;
  462.     entry_lo_t lo0, lo1;
  463.     entry_hi_t hi, hi_save;
  464.     int i;
  465.  
  466.     hi_save.value = cp0_entry_hi_read();
  467.  
  468.     printf("TLB:\n");
  469.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  470.         cp0_index_write(i);
  471.         tlbr();
  472.        
  473.         mask.value = cp0_pagemask_read();
  474.         hi.value = cp0_entry_hi_read();
  475.         lo0.value = cp0_entry_lo0_read();
  476.         lo1.value = cp0_entry_lo1_read();
  477.        
  478.         printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%hhd, pfn[0]=%d\n"
  479.                "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%hhd, pfn[1]=%d\n",
  480.                i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
  481.                lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  482.     }
  483.    
  484.     cp0_entry_hi_write(hi_save.value);
  485. }
  486.  
  487. /** Invalidate all not wired TLB entries. */
  488. void tlb_invalidate_all(void)
  489. {
  490.     ipl_t ipl;
  491.     entry_lo_t lo0, lo1;
  492.     entry_hi_t hi_save;
  493.     int i;
  494.  
  495.     hi_save.value = cp0_entry_hi_read();
  496.     ipl = interrupts_disable();
  497.  
  498.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  499.         cp0_index_write(i);
  500.         tlbr();
  501.  
  502.         lo0.value = cp0_entry_lo0_read();
  503.         lo1.value = cp0_entry_lo1_read();
  504.  
  505.         lo0.v = 0;
  506.         lo1.v = 0;
  507.  
  508.         cp0_entry_lo0_write(lo0.value);
  509.         cp0_entry_lo1_write(lo1.value);
  510.                
  511.         tlbwi();
  512.     }
  513.    
  514.     interrupts_restore(ipl);
  515.     cp0_entry_hi_write(hi_save.value);
  516. }
  517.  
  518. /** Invalidate all TLB entries belonging to specified address space.
  519.  *
  520.  * @param asid Address space identifier.
  521.  */
  522. void tlb_invalidate_asid(asid_t asid)
  523. {
  524.     ipl_t ipl;
  525.     entry_lo_t lo0, lo1;
  526.     entry_hi_t hi, hi_save;
  527.     int i;
  528.  
  529.     ASSERT(asid != ASID_INVALID);
  530.  
  531.     hi_save.value = cp0_entry_hi_read();
  532.     ipl = interrupts_disable();
  533.    
  534.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  535.         cp0_index_write(i);
  536.         tlbr();
  537.        
  538.         hi.value = cp0_entry_hi_read();
  539.        
  540.         if (hi.asid == asid) {
  541.             lo0.value = cp0_entry_lo0_read();
  542.             lo1.value = cp0_entry_lo1_read();
  543.  
  544.             lo0.v = 0;
  545.             lo1.v = 0;
  546.  
  547.             cp0_entry_lo0_write(lo0.value);
  548.             cp0_entry_lo1_write(lo1.value);
  549.  
  550.             tlbwi();
  551.         }
  552.     }
  553.    
  554.     interrupts_restore(ipl);
  555.     cp0_entry_hi_write(hi_save.value);
  556. }
  557.  
  558. /** Invalidate TLB entries for specified page range belonging to specified address space.
  559.  *
  560.  * @param asid Address space identifier.
  561.  * @param page First page whose TLB entry is to be invalidated.
  562.  * @param cnt Number of entries to invalidate.
  563.  */
  564. void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
  565. {
  566.     int i;
  567.     ipl_t ipl;
  568.     entry_lo_t lo0, lo1;
  569.     entry_hi_t hi, hi_save;
  570.     tlb_index_t index;
  571.  
  572.     ASSERT(asid != ASID_INVALID);
  573.  
  574.     hi_save.value = cp0_entry_hi_read();
  575.     ipl = interrupts_disable();
  576.  
  577.     for (i = 0; i < cnt+1; i+=2) {
  578.         hi.value = 0;
  579.         prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  580.         cp0_entry_hi_write(hi.value);
  581.  
  582.         tlbp();
  583.         index.value = cp0_index_read();
  584.  
  585.         if (!index.p) {
  586.             /* Entry was found, index register contains valid index. */
  587.             tlbr();
  588.  
  589.             lo0.value = cp0_entry_lo0_read();
  590.             lo1.value = cp0_entry_lo1_read();
  591.  
  592.             lo0.v = 0;
  593.             lo1.v = 0;
  594.  
  595.             cp0_entry_lo0_write(lo0.value);
  596.             cp0_entry_lo1_write(lo1.value);
  597.  
  598.             tlbwi();
  599.         }
  600.     }
  601.    
  602.     interrupts_restore(ipl);
  603.     cp0_entry_hi_write(hi_save.value);
  604. }
  605.