Subversion Repositories HelenOS-historic

Rev

Rev 958 | Rev 1196 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (C) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. #include <arch/mm/tlb.h>
  30. #include <mm/asid.h>
  31. #include <mm/tlb.h>
  32. #include <mm/page.h>
  33. #include <mm/as.h>
  34. #include <arch/cp0.h>
  35. #include <panic.h>
  36. #include <arch.h>
  37. #include <symtab.h>
  38. #include <synch/spinlock.h>
  39. #include <print.h>
  40. #include <debug.h>
  41. #include <align.h>
  42.  
  43. static void tlb_refill_fail(istate_t *istate);
  44. static void tlb_invalid_fail(istate_t *istate);
  45. static void tlb_modified_fail(istate_t *istate);
  46.  
  47. static pte_t *find_mapping_and_check(__address badvaddr);
  48.  
  49. static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
  50. static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
  51.  
  52. /** Initialize TLB
  53.  *
  54.  * Initialize TLB.
  55.  * Invalidate all entries and mark wired entries.
  56.  */
  57. void tlb_arch_init(void)
  58. {
  59.     int i;
  60.  
  61.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  62.     cp0_entry_hi_write(0);
  63.     cp0_entry_lo0_write(0);
  64.     cp0_entry_lo1_write(0);
  65.  
  66.     /* Clear and initialize TLB. */
  67.    
  68.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  69.         cp0_index_write(i);
  70.         tlbwi();
  71.     }
  72.  
  73.        
  74.     /*
  75.      * The kernel is going to make use of some wired
  76.      * entries (e.g. mapping kernel stacks in kseg3).
  77.      */
  78.     cp0_wired_write(TLB_WIRED);
  79. }
  80.  
  81. /** Process TLB Refill Exception
  82.  *
  83.  * Process TLB Refill Exception.
  84.  *
  85.  * @param istate Interrupted register context.
  86.  */
  87. void tlb_refill(istate_t *istate)
  88. {
  89.     entry_lo_t lo;
  90.     entry_hi_t hi; 
  91.     __address badvaddr;
  92.     pte_t *pte;
  93.  
  94.     badvaddr = cp0_badvaddr_read();
  95.  
  96.     spinlock_lock(&AS->lock);      
  97.  
  98.     pte = find_mapping_and_check(badvaddr);
  99.     if (!pte)
  100.         goto fail;
  101.  
  102.     /*
  103.      * Record access to PTE.
  104.      */
  105.     pte->a = 1;
  106.  
  107.     prepare_entry_hi(&hi, AS->asid, badvaddr);
  108.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  109.  
  110.     /*
  111.      * New entry is to be inserted into TLB
  112.      */
  113.     cp0_entry_hi_write(hi.value);
  114.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  115.         cp0_entry_lo0_write(lo.value);
  116.         cp0_entry_lo1_write(0);
  117.     }
  118.     else {
  119.         cp0_entry_lo0_write(0);
  120.         cp0_entry_lo1_write(lo.value);
  121.     }
  122.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  123.     tlbwr();
  124.  
  125.     spinlock_unlock(&AS->lock);
  126.     return;
  127.    
  128. fail:
  129.     spinlock_unlock(&AS->lock);
  130.     tlb_refill_fail(istate);
  131. }
  132.  
  133. /** Process TLB Invalid Exception
  134.  *
  135.  * Process TLB Invalid Exception.
  136.  *
  137.  * @param istate Interrupted register context.
  138.  */
  139. void tlb_invalid(istate_t *istate)
  140. {
  141.     tlb_index_t index;
  142.     __address badvaddr;
  143.     entry_lo_t lo;
  144.     entry_hi_t hi;
  145.     pte_t *pte;
  146.  
  147.     badvaddr = cp0_badvaddr_read();
  148.  
  149.     /*
  150.      * Locate the faulting entry in TLB.
  151.      */
  152.     hi.value = cp0_entry_hi_read();
  153.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  154.     cp0_entry_hi_write(hi.value);
  155.     tlbp();
  156.     index.value = cp0_index_read();
  157.    
  158.     spinlock_lock(&AS->lock);  
  159.    
  160.     /*
  161.      * Fail if the entry is not in TLB.
  162.      */
  163.     if (index.p) {
  164.         printf("TLB entry not found.\n");
  165.         goto fail;
  166.     }
  167.  
  168.     pte = find_mapping_and_check(badvaddr);
  169.     if (!pte)
  170.         goto fail;
  171.  
  172.     /*
  173.      * Read the faulting TLB entry.
  174.      */
  175.     tlbr();
  176.  
  177.     /*
  178.      * Record access to PTE.
  179.      */
  180.     pte->a = 1;
  181.  
  182.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  183.  
  184.     /*
  185.      * The entry is to be updated in TLB.
  186.      */
  187.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  188.         cp0_entry_lo0_write(lo.value);
  189.     else
  190.         cp0_entry_lo1_write(lo.value);
  191.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  192.     tlbwi();
  193.  
  194.     spinlock_unlock(&AS->lock);
  195.     return;
  196.    
  197. fail:
  198.     spinlock_unlock(&AS->lock);
  199.     tlb_invalid_fail(istate);
  200. }
  201.  
  202. /** Process TLB Modified Exception
  203.  *
  204.  * Process TLB Modified Exception.
  205.  *
  206.  * @param istate Interrupted register context.
  207.  */
  208. void tlb_modified(istate_t *istate)
  209. {
  210.     tlb_index_t index;
  211.     __address badvaddr;
  212.     entry_lo_t lo;
  213.     entry_hi_t hi;
  214.     pte_t *pte;
  215.  
  216.     badvaddr = cp0_badvaddr_read();
  217.  
  218.     /*
  219.      * Locate the faulting entry in TLB.
  220.      */
  221.     hi.value = cp0_entry_hi_read();
  222.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  223.     cp0_entry_hi_write(hi.value);
  224.     tlbp();
  225.     index.value = cp0_index_read();
  226.    
  227.     spinlock_lock(&AS->lock);  
  228.    
  229.     /*
  230.      * Fail if the entry is not in TLB.
  231.      */
  232.     if (index.p) {
  233.         printf("TLB entry not found.\n");
  234.         goto fail;
  235.     }
  236.  
  237.     pte = find_mapping_and_check(badvaddr);
  238.     if (!pte)
  239.         goto fail;
  240.  
  241.     /*
  242.      * Fail if the page is not writable.
  243.      */
  244.     if (!pte->w)
  245.         goto fail;
  246.  
  247.     /*
  248.      * Read the faulting TLB entry.
  249.      */
  250.     tlbr();
  251.  
  252.     /*
  253.      * Record access and write to PTE.
  254.      */
  255.     pte->a = 1;
  256.     pte->d = 1;
  257.  
  258.     prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
  259.  
  260.     /*
  261.      * The entry is to be updated in TLB.
  262.      */
  263.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  264.         cp0_entry_lo0_write(lo.value);
  265.     else
  266.         cp0_entry_lo1_write(lo.value);
  267.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  268.     tlbwi();
  269.  
  270.     spinlock_unlock(&AS->lock);
  271.     return;
  272.    
  273. fail:
  274.     spinlock_unlock(&AS->lock);
  275.     tlb_modified_fail(istate);
  276. }
  277.  
  278. void tlb_refill_fail(istate_t *istate)
  279. {
  280.     char *symbol = "";
  281.     char *sym2 = "";
  282.  
  283.     char *s = get_symtab_entry(istate->epc);
  284.     if (s)
  285.         symbol = s;
  286.     s = get_symtab_entry(istate->ra);
  287.     if (s)
  288.         sym2 = s;
  289.     panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
  290. }
  291.  
  292.  
  293. void tlb_invalid_fail(istate_t *istate)
  294. {
  295.     char *symbol = "";
  296.  
  297.     char *s = get_symtab_entry(istate->epc);
  298.     if (s)
  299.         symbol = s;
  300.     panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  301. }
  302.  
  303. void tlb_modified_fail(istate_t *istate)
  304. {
  305.     char *symbol = "";
  306.  
  307.     char *s = get_symtab_entry(istate->epc);
  308.     if (s)
  309.         symbol = s;
  310.     panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
  311. }
  312.  
  313. /** Try to find PTE for faulting address
  314.  *
  315.  * Try to find PTE for faulting address.
  316.  * The AS->lock must be held on entry to this function.
  317.  *
  318.  * @param badvaddr Faulting virtual address.
  319.  *
  320.  * @return PTE on success, NULL otherwise.
  321.  */
  322. pte_t *find_mapping_and_check(__address badvaddr)
  323. {
  324.     entry_hi_t hi;
  325.     pte_t *pte;
  326.  
  327.     hi.value = cp0_entry_hi_read();
  328.  
  329.     /*
  330.      * Handler cannot succeed if the ASIDs don't match.
  331.      */
  332.     if (hi.asid != AS->asid) {
  333.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  334.         return NULL;
  335.     }
  336.  
  337.     /*
  338.      * Check if the mapping exists in page tables.
  339.      */
  340.     pte = page_mapping_find(AS, badvaddr);
  341.     if (pte && pte->p) {
  342.         /*
  343.          * Mapping found in page tables.
  344.          * Immediately succeed.
  345.          */
  346.         return pte;
  347.     } else {
  348.         /*
  349.          * Mapping not found in page tables.
  350.          * Resort to higher-level page fault handler.
  351.          */
  352.         if (as_page_fault(badvaddr)) {
  353.             /*
  354.              * The higher-level page fault handler succeeded,
  355.              * The mapping ought to be in place.
  356.              */
  357.             pte = page_mapping_find(AS, badvaddr);
  358.             ASSERT(pte && pte->p);
  359.             return pte;
  360.         }
  361.     }
  362.  
  363.     /*
  364.      * Handler cannot succeed if badvaddr has no mapping.
  365.      */
  366.     if (!pte) {
  367.         printf("No such mapping.\n");
  368.         return NULL;
  369.     }
  370.  
  371.     /*
  372.      * Handler cannot succeed if the mapping is marked as invalid.
  373.      */
  374.     if (!pte->p) {
  375.         printf("Invalid mapping.\n");
  376.         return NULL;
  377.     }
  378.  
  379.     return pte;
  380. }
  381.  
  382. void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
  383. {
  384.     lo->value = 0;
  385.     lo->g = g;
  386.     lo->v = v;
  387.     lo->d = d;
  388.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  389.     lo->pfn = pfn;
  390. }
  391.  
  392. void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
  393. {
  394.     hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
  395.     hi->asid = asid;
  396. }
  397.  
  398. /** Print contents of TLB. */
  399. void tlb_print(void)
  400. {
  401.     page_mask_t mask;
  402.     entry_lo_t lo0, lo1;
  403.     entry_hi_t hi, hi_save;
  404.     int i;
  405.  
  406.     hi_save.value = cp0_entry_hi_read();
  407.  
  408.     printf("TLB:\n");
  409.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  410.         cp0_index_write(i);
  411.         tlbr();
  412.        
  413.         mask.value = cp0_pagemask_read();
  414.         hi.value = cp0_entry_hi_read();
  415.         lo0.value = cp0_entry_lo0_read();
  416.         lo1.value = cp0_entry_lo1_read();
  417.        
  418.         printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
  419.                "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
  420.                i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
  421.                lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  422.     }
  423.    
  424.     cp0_entry_hi_write(hi_save.value);
  425. }
  426.  
  427. /** Invalidate all not wired TLB entries. */
  428. void tlb_invalidate_all(void)
  429. {
  430.     ipl_t ipl;
  431.     entry_lo_t lo0, lo1;
  432.     entry_hi_t hi_save;
  433.     int i;
  434.  
  435.     hi_save.value = cp0_entry_hi_read();
  436.     ipl = interrupts_disable();
  437.  
  438.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  439.         cp0_index_write(i);
  440.         tlbr();
  441.  
  442.         lo0.value = cp0_entry_lo0_read();
  443.         lo1.value = cp0_entry_lo1_read();
  444.  
  445.         lo0.v = 0;
  446.         lo1.v = 0;
  447.  
  448.         cp0_entry_lo0_write(lo0.value);
  449.         cp0_entry_lo1_write(lo1.value);
  450.                
  451.         tlbwi();
  452.     }
  453.    
  454.     interrupts_restore(ipl);
  455.     cp0_entry_hi_write(hi_save.value);
  456. }
  457.  
  458. /** Invalidate all TLB entries belonging to specified address space.
  459.  *
  460.  * @param asid Address space identifier.
  461.  */
  462. void tlb_invalidate_asid(asid_t asid)
  463. {
  464.     ipl_t ipl;
  465.     entry_lo_t lo0, lo1;
  466.     entry_hi_t hi, hi_save;
  467.     int i;
  468.  
  469.     ASSERT(asid != ASID_INVALID);
  470.  
  471.     hi_save.value = cp0_entry_hi_read();
  472.     ipl = interrupts_disable();
  473.    
  474.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  475.         cp0_index_write(i);
  476.         tlbr();
  477.        
  478.         hi.value = cp0_entry_hi_read();
  479.        
  480.         if (hi.asid == asid) {
  481.             lo0.value = cp0_entry_lo0_read();
  482.             lo1.value = cp0_entry_lo1_read();
  483.  
  484.             lo0.v = 0;
  485.             lo1.v = 0;
  486.  
  487.             cp0_entry_lo0_write(lo0.value);
  488.             cp0_entry_lo1_write(lo1.value);
  489.  
  490.             tlbwi();
  491.         }
  492.     }
  493.    
  494.     interrupts_restore(ipl);
  495.     cp0_entry_hi_write(hi_save.value);
  496. }
  497.  
  498. /** Invalidate TLB entries for specified page range belonging to specified address space.
  499.  *
  500.  * @param asid Address space identifier.
  501.  * @param page First page whose TLB entry is to be invalidated.
  502.  * @param cnt Number of entries to invalidate.
  503.  */
  504. void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
  505. {
  506.     int i;
  507.     ipl_t ipl;
  508.     entry_lo_t lo0, lo1;
  509.     entry_hi_t hi, hi_save;
  510.     tlb_index_t index;
  511.  
  512.     ASSERT(asid != ASID_INVALID);
  513.  
  514.     hi_save.value = cp0_entry_hi_read();
  515.     ipl = interrupts_disable();
  516.  
  517.     for (i = 0; i < cnt+1; i+=2) {
  518.         hi.value = 0;
  519.         prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  520.         cp0_entry_hi_write(hi.value);
  521.  
  522.         tlbp();
  523.         index.value = cp0_index_read();
  524.  
  525.         if (!index.p) {
  526.             /* Entry was found, index register contains valid index. */
  527.             tlbr();
  528.  
  529.             lo0.value = cp0_entry_lo0_read();
  530.             lo1.value = cp0_entry_lo1_read();
  531.  
  532.             lo0.v = 0;
  533.             lo1.v = 0;
  534.  
  535.             cp0_entry_lo0_write(lo0.value);
  536.             cp0_entry_lo1_write(lo1.value);
  537.  
  538.             tlbwi();
  539.         }
  540.     }
  541.    
  542.     interrupts_restore(ipl);
  543.     cp0_entry_hi_write(hi_save.value);
  544. }
  545.