Subversion Repositories HelenOS-historic

Rev

Rev 830 | Rev 958 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (C) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. #include <arch/mm/tlb.h>
  30. #include <mm/asid.h>
  31. #include <mm/tlb.h>
  32. #include <mm/page.h>
  33. #include <mm/as.h>
  34. #include <arch/cp0.h>
  35. #include <panic.h>
  36. #include <arch.h>
  37. #include <symtab.h>
  38. #include <synch/spinlock.h>
  39. #include <print.h>
  40. #include <debug.h>
  41.  
  42. static void tlb_refill_fail(struct exception_regdump *pstate);
  43. static void tlb_invalid_fail(struct exception_regdump *pstate);
  44. static void tlb_modified_fail(struct exception_regdump *pstate);
  45.  
  46. static pte_t *find_mapping_and_check(__address badvaddr);
  47.  
  48. static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
  49. static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
  50.  
  51. /** Initialize TLB
  52.  *
  53.  * Initialize TLB.
  54.  * Invalidate all entries and mark wired entries.
  55.  */
  56. void tlb_arch_init(void)
  57. {
  58.     int i;
  59.  
  60.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  61.     cp0_entry_hi_write(0);
  62.     cp0_entry_lo0_write(0);
  63.     cp0_entry_lo1_write(0);
  64.  
  65.     /* Clear and initialize TLB. */
  66.    
  67.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  68.         cp0_index_write(i);
  69.         tlbwi();
  70.     }
  71.  
  72.        
  73.     /*
  74.      * The kernel is going to make use of some wired
  75.      * entries (e.g. mapping kernel stacks in kseg3).
  76.      */
  77.     cp0_wired_write(TLB_WIRED);
  78. }
  79.  
  80. /** Process TLB Refill Exception
  81.  *
  82.  * Process TLB Refill Exception.
  83.  *
  84.  * @param pstate Interrupted register context.
  85.  */
  86. void tlb_refill(struct exception_regdump *pstate)
  87. {
  88.     entry_lo_t lo;
  89.     entry_hi_t hi; 
  90.     __address badvaddr;
  91.     pte_t *pte;
  92.  
  93.     badvaddr = cp0_badvaddr_read();
  94.  
  95.     spinlock_lock(&AS->lock);      
  96.  
  97.     pte = find_mapping_and_check(badvaddr);
  98.     if (!pte)
  99.         goto fail;
  100.  
  101.     /*
  102.      * Record access to PTE.
  103.      */
  104.     pte->a = 1;
  105.  
  106.     prepare_entry_hi(&hi, AS->asid, badvaddr);
  107.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  108.  
  109.     /*
  110.      * New entry is to be inserted into TLB
  111.      */
  112.     cp0_entry_hi_write(hi.value);
  113.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  114.         cp0_entry_lo0_write(lo.value);
  115.         cp0_entry_lo1_write(0);
  116.     }
  117.     else {
  118.         cp0_entry_lo0_write(0);
  119.         cp0_entry_lo1_write(lo.value);
  120.     }
  121.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  122.     tlbwr();
  123.  
  124.     spinlock_unlock(&AS->lock);
  125.     return;
  126.    
  127. fail:
  128.     spinlock_unlock(&AS->lock);
  129.     tlb_refill_fail(pstate);
  130. }
  131.  
  132. /** Process TLB Invalid Exception
  133.  *
  134.  * Process TLB Invalid Exception.
  135.  *
  136.  * @param pstate Interrupted register context.
  137.  */
  138. void tlb_invalid(struct exception_regdump *pstate)
  139. {
  140.     tlb_index_t index;
  141.     __address badvaddr;
  142.     entry_lo_t lo;
  143.     entry_hi_t hi;
  144.     pte_t *pte;
  145.  
  146.     badvaddr = cp0_badvaddr_read();
  147.  
  148.     /*
  149.      * Locate the faulting entry in TLB.
  150.      */
  151.     hi.value = cp0_entry_hi_read();
  152.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  153.     cp0_entry_hi_write(hi.value);
  154.     tlbp();
  155.     index.value = cp0_index_read();
  156.    
  157.     spinlock_lock(&AS->lock);  
  158.    
  159.     /*
  160.      * Fail if the entry is not in TLB.
  161.      */
  162.     if (index.p) {
  163.         printf("TLB entry not found.\n");
  164.         goto fail;
  165.     }
  166.  
  167.     pte = find_mapping_and_check(badvaddr);
  168.     if (!pte)
  169.         goto fail;
  170.  
  171.     /*
  172.      * Read the faulting TLB entry.
  173.      */
  174.     tlbr();
  175.  
  176.     /*
  177.      * Record access to PTE.
  178.      */
  179.     pte->a = 1;
  180.  
  181.     prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
  182.  
  183.     /*
  184.      * The entry is to be updated in TLB.
  185.      */
  186.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  187.         cp0_entry_lo0_write(lo.value);
  188.     else
  189.         cp0_entry_lo1_write(lo.value);
  190.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  191.     tlbwi();
  192.  
  193.     spinlock_unlock(&AS->lock);
  194.     return;
  195.    
  196. fail:
  197.     spinlock_unlock(&AS->lock);
  198.     tlb_invalid_fail(pstate);
  199. }
  200.  
  201. /** Process TLB Modified Exception
  202.  *
  203.  * Process TLB Modified Exception.
  204.  *
  205.  * @param pstate Interrupted register context.
  206.  */
  207. void tlb_modified(struct exception_regdump *pstate)
  208. {
  209.     tlb_index_t index;
  210.     __address badvaddr;
  211.     entry_lo_t lo;
  212.     entry_hi_t hi;
  213.     pte_t *pte;
  214.  
  215.     badvaddr = cp0_badvaddr_read();
  216.  
  217.     /*
  218.      * Locate the faulting entry in TLB.
  219.      */
  220.     hi.value = cp0_entry_hi_read();
  221.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  222.     cp0_entry_hi_write(hi.value);
  223.     tlbp();
  224.     index.value = cp0_index_read();
  225.    
  226.     spinlock_lock(&AS->lock);  
  227.    
  228.     /*
  229.      * Fail if the entry is not in TLB.
  230.      */
  231.     if (index.p) {
  232.         printf("TLB entry not found.\n");
  233.         goto fail;
  234.     }
  235.  
  236.     pte = find_mapping_and_check(badvaddr);
  237.     if (!pte)
  238.         goto fail;
  239.  
  240.     /*
  241.      * Fail if the page is not writable.
  242.      */
  243.     if (!pte->w)
  244.         goto fail;
  245.  
  246.     /*
  247.      * Read the faulting TLB entry.
  248.      */
  249.     tlbr();
  250.  
  251.     /*
  252.      * Record access and write to PTE.
  253.      */
  254.     pte->a = 1;
  255.     pte->d = 1;
  256.  
  257.     prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
  258.  
  259.     /*
  260.      * The entry is to be updated in TLB.
  261.      */
  262.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  263.         cp0_entry_lo0_write(lo.value);
  264.     else
  265.         cp0_entry_lo1_write(lo.value);
  266.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  267.     tlbwi();
  268.  
  269.     spinlock_unlock(&AS->lock);
  270.     return;
  271.    
  272. fail:
  273.     spinlock_unlock(&AS->lock);
  274.     tlb_modified_fail(pstate);
  275. }
  276.  
  277. void tlb_refill_fail(struct exception_regdump *pstate)
  278. {
  279.     char *symbol = "";
  280.     char *sym2 = "";
  281.  
  282.     char *s = get_symtab_entry(pstate->epc);
  283.     if (s)
  284.         symbol = s;
  285.     s = get_symtab_entry(pstate->ra);
  286.     if (s)
  287.         sym2 = s;
  288.     panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), pstate->epc, symbol, sym2);
  289. }
  290.  
  291.  
  292. void tlb_invalid_fail(struct exception_regdump *pstate)
  293. {
  294.     char *symbol = "";
  295.  
  296.     char *s = get_symtab_entry(pstate->epc);
  297.     if (s)
  298.         symbol = s;
  299.     panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
  300. }
  301.  
  302. void tlb_modified_fail(struct exception_regdump *pstate)
  303. {
  304.     char *symbol = "";
  305.  
  306.     char *s = get_symtab_entry(pstate->epc);
  307.     if (s)
  308.         symbol = s;
  309.     panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
  310. }
  311.  
  312. /** Try to find PTE for faulting address
  313.  *
  314.  * Try to find PTE for faulting address.
  315.  * The AS->lock must be held on entry to this function.
  316.  *
  317.  * @param badvaddr Faulting virtual address.
  318.  *
  319.  * @return PTE on success, NULL otherwise.
  320.  */
  321. pte_t *find_mapping_and_check(__address badvaddr)
  322. {
  323.     entry_hi_t hi;
  324.     pte_t *pte;
  325.  
  326.     hi.value = cp0_entry_hi_read();
  327.  
  328.     /*
  329.      * Handler cannot succeed if the ASIDs don't match.
  330.      */
  331.     if (hi.asid != AS->asid) {
  332.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  333.         return NULL;
  334.     }
  335.  
  336.     /*
  337.      * Check if the mapping exists in page tables.
  338.      */
  339.     pte = page_mapping_find(AS, badvaddr);
  340.     if (pte && pte->p) {
  341.         /*
  342.          * Mapping found in page tables.
  343.          * Immediately succeed.
  344.          */
  345.         return pte;
  346.     } else {
  347.         /*
  348.          * Mapping not found in page tables.
  349.          * Resort to higher-level page fault handler.
  350.          */
  351.         if (as_page_fault(badvaddr)) {
  352.             /*
  353.              * The higher-level page fault handler succeeded,
  354.              * The mapping ought to be in place.
  355.              */
  356.             pte = page_mapping_find(AS, badvaddr);
  357.             ASSERT(pte && pte->p);
  358.             return pte;
  359.         }
  360.     }
  361.  
  362.     /*
  363.      * Handler cannot succeed if badvaddr has no mapping.
  364.      */
  365.     if (!pte) {
  366.         printf("No such mapping.\n");
  367.         return NULL;
  368.     }
  369.  
  370.     /*
  371.      * Handler cannot succeed if the mapping is marked as invalid.
  372.      */
  373.     if (!pte->p) {
  374.         printf("Invalid mapping.\n");
  375.         return NULL;
  376.     }
  377.  
  378.     return pte;
  379. }
  380.  
  381. void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
  382. {
  383.     lo->value = 0;
  384.     lo->g = g;
  385.     lo->v = v;
  386.     lo->d = d;
  387.     lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
  388.     lo->pfn = pfn;
  389. }
  390.  
  391. void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
  392. {
  393.     hi->value = (((addr/PAGE_SIZE)/2)*PAGE_SIZE*2);
  394.     hi->asid = asid;
  395. }
  396.  
  397. /** Print contents of TLB. */
  398. void tlb_print(void)
  399. {
  400.     page_mask_t mask;
  401.     entry_lo_t lo0, lo1;
  402.     entry_hi_t hi, hi_save;
  403.     int i;
  404.  
  405.     hi_save.value = cp0_entry_hi_read();
  406.  
  407.     printf("TLB:\n");
  408.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  409.         cp0_index_write(i);
  410.         tlbr();
  411.        
  412.         mask.value = cp0_pagemask_read();
  413.         hi.value = cp0_entry_hi_read();
  414.         lo0.value = cp0_entry_lo0_read();
  415.         lo1.value = cp0_entry_lo1_read();
  416.        
  417.         printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
  418.                "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
  419.                i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
  420.                lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  421.     }
  422.    
  423.     cp0_entry_hi_write(hi_save.value);
  424. }
  425.  
  426. /** Invalidate all not wired TLB entries. */
  427. void tlb_invalidate_all(void)
  428. {
  429.     ipl_t ipl;
  430.     entry_lo_t lo0, lo1;
  431.     entry_hi_t hi_save;
  432.     int i;
  433.  
  434.     hi_save.value = cp0_entry_hi_read();
  435.     ipl = interrupts_disable();
  436.  
  437.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  438.         cp0_index_write(i);
  439.         tlbr();
  440.  
  441.         lo0.value = cp0_entry_lo0_read();
  442.         lo1.value = cp0_entry_lo1_read();
  443.  
  444.         lo0.v = 0;
  445.         lo1.v = 0;
  446.  
  447.         cp0_entry_lo0_write(lo0.value);
  448.         cp0_entry_lo1_write(lo1.value);
  449.                
  450.         tlbwi();
  451.     }
  452.    
  453.     interrupts_restore(ipl);
  454.     cp0_entry_hi_write(hi_save.value);
  455. }
  456.  
  457. /** Invalidate all TLB entries belonging to specified address space.
  458.  *
  459.  * @param asid Address space identifier.
  460.  */
  461. void tlb_invalidate_asid(asid_t asid)
  462. {
  463.     ipl_t ipl;
  464.     entry_lo_t lo0, lo1;
  465.     entry_hi_t hi, hi_save;
  466.     int i;
  467.  
  468.     ASSERT(asid != ASID_INVALID);
  469.  
  470.     hi_save.value = cp0_entry_hi_read();
  471.     ipl = interrupts_disable();
  472.    
  473.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  474.         cp0_index_write(i);
  475.         tlbr();
  476.        
  477.         hi.value = cp0_entry_hi_read();
  478.        
  479.         if (hi.asid == asid) {
  480.             lo0.value = cp0_entry_lo0_read();
  481.             lo1.value = cp0_entry_lo1_read();
  482.  
  483.             lo0.v = 0;
  484.             lo1.v = 0;
  485.  
  486.             cp0_entry_lo0_write(lo0.value);
  487.             cp0_entry_lo1_write(lo1.value);
  488.  
  489.             tlbwi();
  490.         }
  491.     }
  492.    
  493.     interrupts_restore(ipl);
  494.     cp0_entry_hi_write(hi_save.value);
  495. }
  496.  
  497. /** Invalidate TLB entries for specified page range belonging to specified address space.
  498.  *
  499.  * @param asid Address space identifier.
  500.  * @param page First page whose TLB entry is to be invalidated.
  501.  * @param cnt Number of entries to invalidate.
  502.  */
  503. void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
  504. {
  505.     int i;
  506.     ipl_t ipl;
  507.     entry_lo_t lo0, lo1;
  508.     entry_hi_t hi, hi_save;
  509.     tlb_index_t index;
  510.  
  511.     ASSERT(asid != ASID_INVALID);
  512.  
  513.     hi_save.value = cp0_entry_hi_read();
  514.     ipl = interrupts_disable();
  515.  
  516.     for (i = 0; i < cnt; i++) {
  517.         hi.value = 0;
  518.         prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  519.         cp0_entry_hi_write(hi.value);
  520.  
  521.         tlbp();
  522.         index.value = cp0_index_read();
  523.  
  524.         if (!index.p) {
  525.             /* Entry was found, index register contains valid index. */
  526.             tlbr();
  527.  
  528.             lo0.value = cp0_entry_lo0_read();
  529.             lo1.value = cp0_entry_lo1_read();
  530.  
  531.             lo0.v = 0;
  532.             lo1.v = 0;
  533.  
  534.             cp0_entry_lo0_write(lo0.value);
  535.             cp0_entry_lo1_write(lo1.value);
  536.  
  537.             tlbwi();
  538.         }
  539.     }
  540.    
  541.     interrupts_restore(ipl);
  542.     cp0_entry_hi_write(hi_save.value);
  543. }
  544.