Subversion Repositories HelenOS-historic

Rev

Rev 727 | Rev 755 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (C) 2003-2004 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. #include <arch/mm/tlb.h>
  30. #include <mm/asid.h>
  31. #include <genarch/mm/asid_fifo.h>
  32. #include <mm/tlb.h>
  33. #include <mm/page.h>
  34. #include <mm/as.h>
  35. #include <arch/cp0.h>
  36. #include <panic.h>
  37. #include <arch.h>
  38. #include <symtab.h>
  39. #include <synch/spinlock.h>
  40. #include <print.h>
  41. #include <debug.h>
  42.  
  43. static void tlb_refill_fail(struct exception_regdump *pstate);
  44. static void tlb_invalid_fail(struct exception_regdump *pstate);
  45. static void tlb_modified_fail(struct exception_regdump *pstate);
  46.  
  47. static pte_t *find_mapping_and_check(__address badvaddr);
  48.  
  49. static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn);
  50. static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
  51.  
  52. /** Initialize TLB
  53.  *
  54.  * Initialize TLB.
  55.  * Invalidate all entries and mark wired entries.
  56.  */
  57. void tlb_arch_init(void)
  58. {
  59.     int i;
  60.  
  61.     asid_fifo_init();
  62.  
  63.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  64.     cp0_entry_hi_write(0);
  65.     cp0_entry_lo0_write(0);
  66.     cp0_entry_lo1_write(0);
  67.  
  68.     /* Clear and initialize TLB. */
  69.    
  70.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  71.         cp0_index_write(i);
  72.         tlbwi();
  73.     }
  74.  
  75.        
  76.     /*
  77.      * The kernel is going to make use of some wired
  78.      * entries (e.g. mapping kernel stacks in kseg3).
  79.      */
  80.     cp0_wired_write(TLB_WIRED);
  81. }
  82.  
  83. /** Process TLB Refill Exception
  84.  *
  85.  * Process TLB Refill Exception.
  86.  *
  87.  * @param pstate Interrupted register context.
  88.  */
  89. void tlb_refill(struct exception_regdump *pstate)
  90. {
  91.     entry_lo_t lo;
  92.     entry_hi_t hi; 
  93.     __address badvaddr;
  94.     pte_t *pte;
  95.  
  96.     badvaddr = cp0_badvaddr_read();
  97.  
  98.     spinlock_lock(&AS->lock);      
  99.  
  100.     pte = find_mapping_and_check(badvaddr);
  101.     if (!pte)
  102.         goto fail;
  103.  
  104.     /*
  105.      * Record access to PTE.
  106.      */
  107.     pte->a = 1;
  108.  
  109.     prepare_entry_hi(&hi, AS->asid, badvaddr);
  110.     prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
  111.  
  112.     /*
  113.      * New entry is to be inserted into TLB
  114.      */
  115.     cp0_entry_hi_write(hi.value);
  116.     if ((badvaddr/PAGE_SIZE) % 2 == 0) {
  117.         cp0_entry_lo0_write(lo.value);
  118.         cp0_entry_lo1_write(0);
  119.     }
  120.     else {
  121.         cp0_entry_lo0_write(0);
  122.         cp0_entry_lo1_write(lo.value);
  123.     }
  124.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  125.     tlbwr();
  126.  
  127.     spinlock_unlock(&AS->lock);
  128.     return;
  129.    
  130. fail:
  131.     spinlock_unlock(&AS->lock);
  132.     tlb_refill_fail(pstate);
  133. }
  134.  
  135. /** Process TLB Invalid Exception
  136.  *
  137.  * Process TLB Invalid Exception.
  138.  *
  139.  * @param pstate Interrupted register context.
  140.  */
  141. void tlb_invalid(struct exception_regdump *pstate)
  142. {
  143.     tlb_index_t index;
  144.     __address badvaddr;
  145.     entry_lo_t lo;
  146.     entry_hi_t hi;
  147.     pte_t *pte;
  148.  
  149.     badvaddr = cp0_badvaddr_read();
  150.  
  151.     /*
  152.      * Locate the faulting entry in TLB.
  153.      */
  154.     hi.value = cp0_entry_hi_read();
  155.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  156.     cp0_entry_hi_write(hi.value);
  157.     tlbp();
  158.     index.value = cp0_index_read();
  159.    
  160.     spinlock_lock(&AS->lock);  
  161.    
  162.     /*
  163.      * Fail if the entry is not in TLB.
  164.      */
  165.     if (index.p) {
  166.         printf("TLB entry not found.\n");
  167.         goto fail;
  168.     }
  169.  
  170.     pte = find_mapping_and_check(badvaddr);
  171.     if (!pte)
  172.         goto fail;
  173.  
  174.     /*
  175.      * Read the faulting TLB entry.
  176.      */
  177.     tlbr();
  178.  
  179.     /*
  180.      * Record access to PTE.
  181.      */
  182.     pte->a = 1;
  183.  
  184.     prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
  185.  
  186.     /*
  187.      * The entry is to be updated in TLB.
  188.      */
  189.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  190.         cp0_entry_lo0_write(lo.value);
  191.     else
  192.         cp0_entry_lo1_write(lo.value);
  193.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  194.     tlbwi();
  195.  
  196.     spinlock_unlock(&AS->lock);
  197.     return;
  198.    
  199. fail:
  200.     spinlock_unlock(&AS->lock);
  201.     tlb_invalid_fail(pstate);
  202. }
  203.  
  204. /** Process TLB Modified Exception
  205.  *
  206.  * Process TLB Modified Exception.
  207.  *
  208.  * @param pstate Interrupted register context.
  209.  */
  210. void tlb_modified(struct exception_regdump *pstate)
  211. {
  212.     tlb_index_t index;
  213.     __address badvaddr;
  214.     entry_lo_t lo;
  215.     entry_hi_t hi;
  216.     pte_t *pte;
  217.  
  218.     badvaddr = cp0_badvaddr_read();
  219.  
  220.     /*
  221.      * Locate the faulting entry in TLB.
  222.      */
  223.     hi.value = cp0_entry_hi_read();
  224.     prepare_entry_hi(&hi, hi.asid, badvaddr);
  225.     cp0_entry_hi_write(hi.value);
  226.     tlbp();
  227.     index.value = cp0_index_read();
  228.    
  229.     spinlock_lock(&AS->lock);  
  230.    
  231.     /*
  232.      * Fail if the entry is not in TLB.
  233.      */
  234.     if (index.p) {
  235.         printf("TLB entry not found.\n");
  236.         goto fail;
  237.     }
  238.  
  239.     pte = find_mapping_and_check(badvaddr);
  240.     if (!pte)
  241.         goto fail;
  242.  
  243.     /*
  244.      * Fail if the page is not writable.
  245.      */
  246.     if (!pte->w)
  247.         goto fail;
  248.  
  249.     /*
  250.      * Read the faulting TLB entry.
  251.      */
  252.     tlbr();
  253.  
  254.     /*
  255.      * Record access and write to PTE.
  256.      */
  257.     pte->a = 1;
  258.     pte->lo.d = 1;
  259.  
  260.     prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->w, pte->lo.c, pte->lo.pfn);
  261.  
  262.     /*
  263.      * The entry is to be updated in TLB.
  264.      */
  265.     if ((badvaddr/PAGE_SIZE) % 2 == 0)
  266.         cp0_entry_lo0_write(lo.value);
  267.     else
  268.         cp0_entry_lo1_write(lo.value);
  269.     cp0_pagemask_write(TLB_PAGE_MASK_16K);
  270.     tlbwi();
  271.  
  272.     spinlock_unlock(&AS->lock);
  273.     return;
  274.    
  275. fail:
  276.     spinlock_unlock(&AS->lock);
  277.     tlb_modified_fail(pstate);
  278. }
  279.  
  280. void tlb_refill_fail(struct exception_regdump *pstate)
  281. {
  282.     char *symbol = "";
  283.     char *sym2 = "";
  284.  
  285.     char *s = get_symtab_entry(pstate->epc);
  286.     if (s)
  287.         symbol = s;
  288.     s = get_symtab_entry(pstate->ra);
  289.     if (s)
  290.         sym2 = s;
  291.     panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), pstate->epc, symbol, sym2);
  292. }
  293.  
  294.  
  295. void tlb_invalid_fail(struct exception_regdump *pstate)
  296. {
  297.     char *symbol = "";
  298.  
  299.     char *s = get_symtab_entry(pstate->epc);
  300.     if (s)
  301.         symbol = s;
  302.     panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
  303. }
  304.  
  305. void tlb_modified_fail(struct exception_regdump *pstate)
  306. {
  307.     char *symbol = "";
  308.  
  309.     char *s = get_symtab_entry(pstate->epc);
  310.     if (s)
  311.         symbol = s;
  312.     panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
  313. }
  314.  
  315. /** Try to find PTE for faulting address
  316.  *
  317.  * Try to find PTE for faulting address.
  318.  * The AS->lock must be held on entry to this function.
  319.  *
  320.  * @param badvaddr Faulting virtual address.
  321.  *
  322.  * @return PTE on success, NULL otherwise.
  323.  */
  324. pte_t *find_mapping_and_check(__address badvaddr)
  325. {
  326.     entry_hi_t hi;
  327.     pte_t *pte;
  328.  
  329.     hi.value = cp0_entry_hi_read();
  330.  
  331.     /*
  332.      * Handler cannot succeed if the ASIDs don't match.
  333.      */
  334.     if (hi.asid != AS->asid) {
  335.         printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
  336.         return NULL;
  337.     }
  338.  
  339.     /*
  340.      * Check if the mapping exists in page tables.
  341.      */
  342.     pte = page_mapping_find(badvaddr, AS->asid, 0);
  343.     if (pte && pte->lo.v) {
  344.         /*
  345.          * Mapping found in page tables.
  346.          * Immediately succeed.
  347.          */
  348.         return pte;
  349.     } else {
  350.         /*
  351.          * Mapping not found in page tables.
  352.          * Resort to higher-level page fault handler.
  353.          */
  354.         if (as_page_fault(badvaddr)) {
  355.             /*
  356.              * The higher-level page fault handler succeeded,
  357.              * The mapping ought to be in place.
  358.              */
  359.             pte = page_mapping_find(badvaddr, AS->asid, 0);
  360.             ASSERT(pte && pte->lo.v);
  361.             return pte;
  362.         }
  363.     }
  364.  
  365.     /*
  366.      * Handler cannot succeed if badvaddr has no mapping.
  367.      */
  368.     if (!pte) {
  369.         printf("No such mapping.\n");
  370.         return NULL;
  371.     }
  372.  
  373.     /*
  374.      * Handler cannot succeed if the mapping is marked as invalid.
  375.      */
  376.     if (!pte->lo.v) {
  377.         printf("Invalid mapping.\n");
  378.         return NULL;
  379.     }
  380.  
  381.     return pte;
  382. }
  383.  
  384. void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn)
  385. {
  386.     lo->value = 0;
  387.     lo->g = g;
  388.     lo->v = v;
  389.     lo->d = d;
  390.     lo->c = c;
  391.     lo->pfn = pfn;
  392. }
  393.  
  394. void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
  395. {
  396.     hi->value = (((addr/PAGE_SIZE)/2)*PAGE_SIZE*2);
  397.     hi->asid = asid;
  398. }
  399.  
  400. /** Print contents of TLB. */
  401. void tlb_print(void)
  402. {
  403.     page_mask_t mask;
  404.     entry_lo_t lo0, lo1;
  405.     entry_hi_t hi, hi_save;
  406.     int i;
  407.  
  408.     hi_save.value = cp0_entry_hi_read();
  409.  
  410.     printf("TLB:\n");
  411.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  412.         cp0_index_write(i);
  413.         tlbr();
  414.        
  415.         mask.value = cp0_pagemask_read();
  416.         hi.value = cp0_entry_hi_read();
  417.         lo0.value = cp0_entry_lo0_read();
  418.         lo1.value = cp0_entry_lo1_read();
  419.        
  420.         printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
  421.                "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
  422.                i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
  423.                lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
  424.     }
  425.    
  426.     cp0_entry_hi_write(hi_save.value);
  427. }
  428.  
  429. /** Invalidate all not wired TLB entries. */
  430. void tlb_invalidate_all(void)
  431. {
  432.     ipl_t ipl;
  433.     entry_lo_t lo0, lo1;
  434.     entry_hi_t hi_save;
  435.     int i;
  436.  
  437.     hi_save.value = cp0_entry_hi_read();
  438.     ipl = interrupts_disable();
  439.  
  440.     for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
  441.         cp0_index_write(i);
  442.         tlbr();
  443.  
  444.         lo0.value = cp0_entry_lo0_read();
  445.         lo1.value = cp0_entry_lo1_read();
  446.  
  447.         lo0.v = 0;
  448.         lo1.v = 0;
  449.  
  450.         cp0_entry_lo0_write(lo0.value);
  451.         cp0_entry_lo1_write(lo1.value);
  452.                
  453.         tlbwi();
  454.     }
  455.    
  456.     interrupts_restore(ipl);
  457.     cp0_entry_hi_write(hi_save.value);
  458. }
  459.  
  460. /** Invalidate all TLB entries belonging to specified address space.
  461.  *
  462.  * @param asid Address space identifier.
  463.  */
  464. void tlb_invalidate_asid(asid_t asid)
  465. {
  466.     ipl_t ipl;
  467.     entry_lo_t lo0, lo1;
  468.     entry_hi_t hi, hi_save;
  469.     int i;
  470.  
  471.     ASSERT(asid != ASID_INVALID);
  472.  
  473.     hi_save.value = cp0_entry_hi_read();
  474.     ipl = interrupts_disable();
  475.    
  476.     for (i = 0; i < TLB_ENTRY_COUNT; i++) {
  477.         cp0_index_write(i);
  478.         tlbr();
  479.        
  480.         hi.value = cp0_entry_hi_read();
  481.        
  482.         if (hi.asid == asid) {
  483.             lo0.value = cp0_entry_lo0_read();
  484.             lo1.value = cp0_entry_lo1_read();
  485.  
  486.             lo0.v = 0;
  487.             lo1.v = 0;
  488.  
  489.             cp0_entry_lo0_write(lo0.value);
  490.             cp0_entry_lo1_write(lo1.value);
  491.  
  492.             tlbwi();
  493.         }
  494.     }
  495.    
  496.     interrupts_restore(ipl);
  497.     cp0_entry_hi_write(hi_save.value);
  498. }
  499.  
  500. /** Invalidate TLB entries for specified page range belonging to specified address space.
  501.  *
  502.  * @param asid Address space identifier.
  503.  * @param page First page whose TLB entry is to be invalidated.
  504.  * @param cnt Number of entries to invalidate.
  505.  */
  506. void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
  507. {
  508.     int i;
  509.     ipl_t ipl;
  510.     entry_lo_t lo0, lo1;
  511.     entry_hi_t hi, hi_save;
  512.     tlb_index_t index;
  513.  
  514.     ASSERT(asid != ASID_INVALID);
  515.  
  516.     hi_save.value = cp0_entry_hi_read();
  517.     ipl = interrupts_disable();
  518.  
  519.     for (i = 0; i < cnt; i++) {
  520.         hi.value = 0;
  521.         prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
  522.         cp0_entry_hi_write(hi.value);
  523.  
  524.         tlbp();
  525.         index.value = cp0_index_read();
  526.  
  527.         if (!index.p) {
  528.             /* Entry was found, index register contains valid index. */
  529.             tlbr();
  530.  
  531.             lo0.value = cp0_entry_lo0_read();
  532.             lo1.value = cp0_entry_lo1_read();
  533.  
  534.             lo0.v = 0;
  535.             lo1.v = 0;
  536.  
  537.             cp0_entry_lo0_write(lo0.value);
  538.             cp0_entry_lo1_write(lo1.value);
  539.  
  540.             tlbwi();
  541.         }
  542.     }
  543.    
  544.     interrupts_restore(ipl);
  545.     cp0_entry_hi_write(hi_save.value);
  546. }
  547.