Subversion Repositories HelenOS-historic

Rev

Rev 983 | Rev 1288 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (C) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. #include <arch/mm/tlb.h>
  30. #include <mm/asid.h>
  31. #include <mm/tlb.h>
  32. #include <mm/page.h>
  33. #include <mm/as.h>
  34. #include <arch/cp0.h>
  35. #include <panic.h>
  36. #include <arch.h>
  37. #include <symtab.h>
  38. #include <synch/spinlock.h>
  39. #include <print.h>
  40. #include <debug.h>
  41. #include <align.h>
  42.  
  43. static void tlb_refill_fail(istate_t *istate);
  44. static void tlb_invalid_fail(istate_t *istate);
  45. static void tlb_modified_fail(istate_t *istate);
  46.  
  47. static pte_t *find_mapping_and_check(__address badvaddr);
  48.  
  49. static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
  50. static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
  51.  
  52. /** Initialize TLB
  53.  *
  54.  * Initialize TLB.
  55.  * Invalidate all entries and mark wired entries.
  56.  */
  57. void tlb_arch_init(void)
  58. {
  59.     int i;
  60.  
  61.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  62.     cp0_entry_hi_write(0);
  63.     cp0_entry_lo0_write(0);
  64.     cp0_entry_lo1_write(0);
  65.  
  66.     /* Clear and initialize TLB. */
  67.    
  68.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  69.         cp0_index_write(i);
  70.         tlbwi();
  71.     }
  72.  
  73.        
  74.     /*
  75.      * The kernel is going to make use of some wired
  76.      * entries (e.g. mapping kernel stacks in kseg3).
  77.      */
  78.     cp0_wired_write(TLB_WIRED);
  79. }
  80.  
  81. /** Process TLB Refill Exception
  82.  *
  83.  * Process TLB Refill Exception.
  84.  *
  85.  * @param istate Interrupted register context.
  86.  */
  87. void tlb_refill(istate_t *istate)
  88. {
  89.     entry_lo_t lo;
  90.     entry_hi_t hi;
  91.     asid_t asid;
  92.     __address badvaddr;
  93.     pte_t *pte;
  94.  
  95.     badvaddr = cp0_badvaddr_read();
  96.  
  97.     spinlock_lock(&AS->lock);
  98.     asid = AS->asid;
  99.     spinlock_unlock(&AS->lock);
  100.  
  101.     page_table_lock(AS, true);
  102.  
  103.     pte = find_mapping_and_check(badvaddr);
  104.     if (!pte)
  105.         goto fail;
  106.  
  107.     /*
  108.      * Record access to PTE.
  109.      */
  110.     pte->a = 1;
  111.  
  112.     prepare_entry_hi(&hi, asid, badvaddr);
  113.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  114.  
  115.     /*
  116.      * New entry is to be inserted into TLB
  117.      */
  118.     cp0_entry_hi_write(hi.value);
  119.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  120.         cp0_entry_lo0_write(lo.value);
  121.         cp0_entry_lo1_write(0);
  122.     }
  123.     else {
  124.         cp0_entry_lo0_write(0);
  125.         cp0_entry_lo1_write(lo.value);
  126.     }
  127.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  128.     tlbwr();
  129.  
  130.     page_table_unlock(AS, true);
  131.     return;
  132.    
  133. fail:
  134.     page_table_unlock(AS, true);
  135.     tlb_refill_fail(istate);
  136. }
  137.  
  138. /** Process TLB Invalid Exception
  139.  *
  140.  * Process TLB Invalid Exception.
  141.  *
  142.  * @param istate Interrupted register context.
  143.  */
  144. void tlb_invalid(istate_t *istate)
  145. {
  146.     tlb_index_t index;
  147.     __address badvaddr;
  148.     entry_lo_t lo;
  149.     entry_hi_t hi;
  150.     pte_t *pte;
  151.  
  152.     badvaddr = cp0_badvaddr_read();
  153.  
  154.     /*
  155.      * Locate the faulting entry in TLB.
  156.      */
  157.     hi.value = cp0_entry_hi_read();
  158.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  159.     cp0_entry_hi_write(hi.value);
  160.     tlbp();
  161.     index.value = cp0_index_read();
  162.  
  163.     page_table_lock(AS, true); 
  164.    
  165.     /*
  166.      * Fail if the entry is not in TLB.
  167.      */
  168.     if (index.p) {
  169.         printf("TLB entry not found.\n");
  170.         goto fail;
  171.     }
  172.  
  173.     pte = find_mapping_and_check(badvaddr);
  174.     if (!pte)
  175.         goto fail;
  176.  
  177.     /*
  178.      * Read the faulting TLB entry.
  179.      */
  180.     tlbr();
  181.  
  182.     /*
  183.      * Record access to PTE.
  184.      */
  185.     pte->a = 1;
  186.  
  187.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  188.  
  189.     /*
  190.      * The entry is to be updated in TLB.
  191.      */
  192.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  193.         cp0_entry_lo0_write(lo.value);
  194.     else
  195.         cp0_entry_lo1_write(lo.value);
  196.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  197.     tlbwi();
  198.  
  199.     page_table_unlock(AS, true);
  200.     return;
  201.    
  202. fail:
  203.     page_table_unlock(AS, true);
  204.     tlb_invalid_fail(istate);
  205. }
  206.  
  207. /** Process TLB Modified Exception
  208.  *
  209.  * Process TLB Modified Exception.
  210.  *
  211.  * @param istate Interrupted register context.
  212.  */
  213. void tlb_modified(istate_t *istate)
  214. {
  215.     tlb_index_t index;
  216.     __address badvaddr;
  217.     entry_lo_t lo;
  218.     entry_hi_t hi;
  219.     pte_t *pte;
  220.  
  221.     badvaddr = cp0_badvaddr_read();
  222.  
  223.     /*
  224.      * Locate the faulting entry in TLB.
  225.      */
  226.     hi.value = cp0_entry_hi_read();
  227.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  228.     cp0_entry_hi_write(hi.value);
  229.     tlbp();
  230.     index.value = cp0_index_read();
  231.  
  232.     page_table_lock(AS, true); 
  233.    
  234.     /*
  235.      * Fail if the entry is not in TLB.
  236.      */
  237.     if (index.p) {
  238.         printf("TLB entry not found.\n");
  239.         goto fail;
  240.     }
  241.  
  242.     pte = find_mapping_and_check(badvaddr);
  243.     if (!pte)
  244.         goto fail;
  245.  
  246.     /*
  247.      * Fail if the page is not writable.
  248.      */
  249.     if (!pte->w)
  250.         goto fail;
  251.  
  252.     /*
  253.      * Read the faulting TLB entry.
  254.      */
  255.     tlbr();
  256.  
  257.     /*
  258.      * Record access and write to PTE.
  259.      */
  260.     pte->a = 1;
  261.     pte->d = 1;
  262.  
  263.     prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
  264.  
  265.     /*
  266.      * The entry is to be updated in TLB.
  267.      */
  268.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  269.         cp0_entry_lo0_write(lo.value);
  270.     else
  271.         cp0_entry_lo1_write(lo.value);
  272.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  273.     tlbwi();
  274.  
  275.     page_table_unlock(AS, true);
  276.     return;
  277.    
  278. fail:
  279.     page_table_unlock(AS, true);
  280.     tlb_modified_fail(istate);
  281. }
  282.  
  283. void tlb_refill_fail(istate_t *istate)
  284. {
  285.     char *symbol = "";
  286.     char *sym2 = "";
  287.  
  288.     char *s = get_symtab_entry(istate->epc);
  289.     if (s)
  290.         symbol = s;
  291.     s = get_symtab_entry(istate->ra);
  292.     if (s)
  293.         sym2 = s;
  294.     panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
  295. }
  296.  
  297.  
  298. void tlb_invalid_fail(istate_t *istate)
  299. {
  300.     char *symbol = "";
  301.  
  302.     char *s = get_symtab_entry(istate->epc);
  303.     if (s)
  304.         symbol = s;
  305.     panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  306. }
  307.  
  308. void tlb_modified_fail(istate_t *istate)
  309. {
  310.     char *symbol = "";
  311.  
  312.     char *s = get_symtab_entry(istate->epc);
  313.     if (s)
  314.         symbol = s;
  315.     panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  316. }
  317.  
  318. /** Try to find PTE for faulting address
  319.  *
  320.  * Try to find PTE for faulting address.
  321.  * The AS->lock must be held on entry to this function.
  322.  *
  323.  * @param badvaddr Faulting virtual address.
  324.  *
  325.  * @return PTE on success, NULL otherwise.
  326.  */
  327. pte_t *find_mapping_and_check(__address badvaddr)
  328. {
  329.     entry_hi_t hi;
  330.     pte_t *pte;
  331.  
  332.     hi.value = cp0_entry_hi_read();
  333.  
  334.     /*
  335.      * Handler cannot succeed if the ASIDs don't match.
  336.      */
  337.     if (hi.asid != AS->asid) {
  338.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  339.         return NULL;
  340.     }
  341.  
  342.     /*
  343.      * Check if the mapping exists in page tables.
  344.      */
  345.     pte = page_mapping_find(AS, badvaddr);
  346.     if (pte && pte->p) {
  347.         /*
  348.          * Mapping found in page tables.
  349.          * Immediately succeed.
  350.          */
  351.         return pte;
  352.     } else {
  353.         /*
  354.          * Mapping not found in page tables.
  355.          * Resort to higher-level page fault handler.
  356.          */
  357.         page_table_unlock(AS, true);
  358.         if (as_page_fault(badvaddr)) {
  359.             /*
  360.              * The higher-level page fault handler succeeded,
  361.              * The mapping ought to be in place.
  362.              */
  363.             page_table_lock(AS, true);
  364.             pte = page_mapping_find(AS, badvaddr);
  365.             ASSERT(pte && pte->p);
  366.             return pte;
  367.         } else {
  368.             page_table_lock(AS, true);
  369.             printf("Page fault.\n");
  370.             return NULL;
  371.         }
  372.        
  373.     }
  374. }
  375.  
  376. void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
  377. {
  378.     lo->value = 0;
  379.     lo->g = g;
  380.     lo->v = v;
  381.     lo->d = d;
  382.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  383.     lo->pfn = pfn;
  384. }
  385.  
  386. void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
  387. {
  388.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  389.     hi->asid = asid;
  390. }
  391.  
  392. /** Print contents of TLB. */
  393. void tlb_print(void)
  394. {
  395.     page_mask_t mask;
  396.     entry_lo_t lo0, lo1;
  397.     entry_hi_t hi, hi_save;
  398.     int i;
  399.  
  400.     hi_save.value = cp0_entry_hi_read();
  401.  
  402.     printf("TLB:\n");
  403.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  404.         cp0_index_write(i);
  405.         tlbr();
  406.        
  407.         mask.value = cp0_pagemask_read();
  408.         hi.value = cp0_entry_hi_read();
  409.         lo0.value = cp0_entry_lo0_read();
  410.         lo1.value = cp0_entry_lo1_read();
  411.        
  412.         printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
  413.                "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
  414.                i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
  415.                lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  416.     }
  417.    
  418.     cp0_entry_hi_write(hi_save.value);
  419. }
  420.  
  421. /** Invalidate all not wired TLB entries. */
  422. void tlb_invalidate_all(void)
  423. {
  424.     ipl_t ipl;
  425.     entry_lo_t lo0, lo1;
  426.     entry_hi_t hi_save;
  427.     int i;
  428.  
  429.     hi_save.value = cp0_entry_hi_read();
  430.     ipl = interrupts_disable();
  431.  
  432.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  433.         cp0_index_write(i);
  434.         tlbr();
  435.  
  436.         lo0.value = cp0_entry_lo0_read();
  437.         lo1.value = cp0_entry_lo1_read();
  438.  
  439.         lo0.v = 0;
  440.         lo1.v = 0;
  441.  
  442.         cp0_entry_lo0_write(lo0.value);
  443.         cp0_entry_lo1_write(lo1.value);
  444.                
  445.         tlbwi();
  446.     }
  447.    
  448.     interrupts_restore(ipl);
  449.     cp0_entry_hi_write(hi_save.value);
  450. }
  451.  
  452. /** Invalidate all TLB entries belonging to specified address space.
  453.  *
  454.  * @param asid Address space identifier.
  455.  */
  456. void tlb_invalidate_asid(asid_t asid)
  457. {
  458.     ipl_t ipl;
  459.     entry_lo_t lo0, lo1;
  460.     entry_hi_t hi, hi_save;
  461.     int i;
  462.  
  463.     ASSERT(asid != ASID_INVALID);
  464.  
  465.     hi_save.value = cp0_entry_hi_read();
  466.     ipl = interrupts_disable();
  467.    
  468.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  469.         cp0_index_write(i);
  470.         tlbr();
  471.        
  472.         hi.value = cp0_entry_hi_read();
  473.        
  474.         if (hi.asid == asid) {
  475.             lo0.value = cp0_entry_lo0_read();
  476.             lo1.value = cp0_entry_lo1_read();
  477.  
  478.             lo0.v = 0;
  479.             lo1.v = 0;
  480.  
  481.             cp0_entry_lo0_write(lo0.value);
  482.             cp0_entry_lo1_write(lo1.value);
  483.  
  484.             tlbwi();
  485.         }
  486.     }
  487.    
  488.     interrupts_restore(ipl);
  489.     cp0_entry_hi_write(hi_save.value);
  490. }
  491.  
  492. /** Invalidate TLB entries for specified page range belonging to specified address space.
  493.  *
  494.  * @param asid Address space identifier.
  495.  * @param page First page whose TLB entry is to be invalidated.
  496.  * @param cnt Number of entries to invalidate.
  497.  */
  498. void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
  499. {
  500.     int i;
  501.     ipl_t ipl;
  502.     entry_lo_t lo0, lo1;
  503.     entry_hi_t hi, hi_save;
  504.     tlb_index_t index;
  505.  
  506.     ASSERT(asid != ASID_INVALID);
  507.  
  508.     hi_save.value = cp0_entry_hi_read();
  509.     ipl = interrupts_disable();
  510.  
  511.     for (i = 0; i < cnt+1; i+=2) {
  512.         hi.value = 0;
  513.         prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  514.         cp0_entry_hi_write(hi.value);
  515.  
  516.         tlbp();
  517.         index.value = cp0_index_read();
  518.  
  519.         if (!index.p) {
  520.             /* Entry was found, index register contains valid index. */
  521.             tlbr();
  522.  
  523.             lo0.value = cp0_entry_lo0_read();
  524.             lo1.value = cp0_entry_lo1_read();
  525.  
  526.             lo0.v = 0;
  527.             lo1.v = 0;
  528.  
  529.             cp0_entry_lo0_write(lo0.value);
  530.             cp0_entry_lo1_write(lo1.value);
  531.  
  532.             tlbwi();
  533.         }
  534.     }
  535.    
  536.     interrupts_restore(ipl);
  537.     cp0_entry_hi_write(hi_save.value);
  538. }
  539.