Subversion Repositories HelenOS

Rev

Rev 3838 | Rev 3857 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (c) 2006 Martin Decky
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. /** @addtogroup ppc32mm
  30.  * @{
  31.  */
  32. /** @file
  33.  */
  34.  
  35. #include <mm/tlb.h>
  36. #include <arch/mm/tlb.h>
  37. #include <arch/interrupt.h>
  38. #include <interrupt.h>
  39. #include <mm/as.h>
  40. #include <arch.h>
  41. #include <print.h>
  42. #include <symtab.h>
  43. #include <macros.h>
  44.  
  45.  
  46. static unsigned int seed = 10;
  47. static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
  48.  
  49.  
  50. /** Try to find PTE for faulting address
  51.  *
  52.  * Try to find PTE for faulting address.
  53.  * The as->lock must be held on entry to this function
  54.  * if lock is true.
  55.  *
  56.  * @param as        Address space.
  57.  * @param lock      Lock/unlock the address space.
  58.  * @param badvaddr  Faulting virtual address.
  59.  * @param access    Access mode that caused the fault.
  60.  * @param istate    Pointer to interrupted state.
  61.  * @param pfrc      Pointer to variable where as_page_fault() return code
  62.  *          will be stored.
  63.  * @return      PTE on success, NULL otherwise.
  64.  *
  65.  */
  66. static pte_t *
  67. find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
  68.     istate_t *istate, int *pfrc)
  69. {
  70.     /*
  71.      * Check if the mapping exists in page tables.
  72.      */
  73.     pte_t *pte = page_mapping_find(as, badvaddr);
  74.     if ((pte) && (pte->present)) {
  75.         /*
  76.          * Mapping found in page tables.
  77.          * Immediately succeed.
  78.          */
  79.         return pte;
  80.     } else {
  81.         int rc;
  82.    
  83.         /*
  84.          * Mapping not found in page tables.
  85.          * Resort to higher-level page fault handler.
  86.          */
  87.         page_table_unlock(as, lock);
  88.         switch (rc = as_page_fault(badvaddr, access, istate)) {
  89.         case AS_PF_OK:
  90.             /*
  91.              * The higher-level page fault handler succeeded,
  92.              * The mapping ought to be in place.
  93.              */
  94.             page_table_lock(as, lock);
  95.             pte = page_mapping_find(as, badvaddr);
  96.             ASSERT((pte) && (pte->present));
  97.             *pfrc = 0;
  98.             return pte;
  99.         case AS_PF_DEFER:
  100.             page_table_lock(as, lock);
  101.             *pfrc = rc;
  102.             return NULL;
  103.         case AS_PF_FAULT:
  104.             page_table_lock(as, lock);
  105.             *pfrc = rc;
  106.             return NULL;
  107.         default:
  108.             panic("Unexpected rc (%d).", rc);
  109.         }  
  110.     }
  111. }
  112.  
  113.  
  114. static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
  115. {
  116.     char *symbol = "";
  117.     char *sym2 = "";
  118.  
  119.     char *str = get_symtab_entry(istate->pc);
  120.     if (str)
  121.         symbol = str;
  122.     str = get_symtab_entry(istate->lr);
  123.     if (str)
  124.         sym2 = str;
  125.  
  126.     fault_if_from_uspace(istate,
  127.         "PHT Refill Exception on %p.", badvaddr);
  128.     panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
  129.         istate->pc, symbol, sym2);
  130. }
  131.  
  132.  
  133. static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
  134. {
  135.     uint32_t page = (vaddr >> 12) & 0xffff;
  136.     uint32_t api = (vaddr >> 22) & 0x3f;
  137.    
  138.     uint32_t vsid;
  139.     asm volatile (
  140.         "mfsrin %0, %1\n"
  141.         : "=r" (vsid)
  142.         : "r" (vaddr)
  143.     );
  144.    
  145.     uint32_t sdr1;
  146.     asm volatile (
  147.         "mfsdr1 %0\n"
  148.         : "=r" (sdr1)
  149.     );
  150.     phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
  151.    
  152.     /* Primary hash (xor) */
  153.     uint32_t h = 0;
  154.     uint32_t hash = vsid ^ page;
  155.     uint32_t base = (hash & 0x3ff) << 3;
  156.     uint32_t i;
  157.     bool found = false;
  158.    
  159.     /* Find colliding PTE in PTEG */
  160.     for (i = 0; i < 8; i++) {
  161.         if ((phte[base + i].v)
  162.             && (phte[base + i].vsid == vsid)
  163.             && (phte[base + i].api == api)
  164.             && (phte[base + i].h == 0)) {
  165.             found = true;
  166.             break;
  167.         }
  168.     }
  169.    
  170.     if (!found) {
  171.         /* Find unused PTE in PTEG */
  172.         for (i = 0; i < 8; i++) {
  173.             if (!phte[base + i].v) {
  174.                 found = true;
  175.                 break;
  176.             }
  177.         }
  178.     }
  179.    
  180.     if (!found) {
  181.         /* Secondary hash (not) */
  182.         uint32_t base2 = (~hash & 0x3ff) << 3;
  183.        
  184.         /* Find colliding PTE in PTEG */
  185.         for (i = 0; i < 8; i++) {
  186.             if ((phte[base2 + i].v)
  187.                 && (phte[base2 + i].vsid == vsid)
  188.                 && (phte[base2 + i].api == api)
  189.                 && (phte[base2 + i].h == 1)) {
  190.                 found = true;
  191.                 base = base2;
  192.                 h = 1;
  193.                 break;
  194.             }
  195.         }
  196.        
  197.         if (!found) {
  198.             /* Find unused PTE in PTEG */
  199.             for (i = 0; i < 8; i++) {
  200.                 if (!phte[base2 + i].v) {
  201.                     found = true;
  202.                     base = base2;
  203.                     h = 1;
  204.                     break;
  205.                 }
  206.             }
  207.         }
  208.        
  209.         if (!found)
  210.             i = RANDI(seed) % 8;
  211.     }
  212.    
  213.     phte[base + i].v = 1;
  214.     phte[base + i].vsid = vsid;
  215.     phte[base + i].h = h;
  216.     phte[base + i].api = api;
  217.     phte[base + i].rpn = pte->pfn;
  218.     phte[base + i].r = 0;
  219.     phte[base + i].c = 0;
  220.     phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
  221.     phte[base + i].pp = 2; // FIXME
  222. }
  223.  
  224.  
  225. /** Process Instruction/Data Storage Interrupt
  226.  *
  227.  * @param n     Interrupt vector number.
  228.  * @param istate    Interrupted register context.
  229.  *
  230.  */
  231. void pht_refill(int n, istate_t *istate)
  232. {
  233.     uintptr_t badvaddr;
  234.     pte_t *pte;
  235.     int pfrc;
  236.     as_t *as;
  237.     bool lock;
  238.    
  239.     if (AS == NULL) {
  240.         as = AS_KERNEL;
  241.         lock = false;
  242.     } else {
  243.         as = AS;
  244.         lock = true;
  245.     }
  246.    
  247.     if (n == VECTOR_DATA_STORAGE)
  248.         badvaddr = istate->dar;
  249.     else
  250.         badvaddr = istate->pc;
  251.        
  252.     page_table_lock(as, lock);
  253.    
  254.     pte = find_mapping_and_check(as, lock, badvaddr,
  255.         PF_ACCESS_READ /* FIXME */, istate, &pfrc);
  256.     if (!pte) {
  257.         switch (pfrc) {
  258.         case AS_PF_FAULT:
  259.             goto fail;
  260.             break;
  261.         case AS_PF_DEFER:
  262.             /*
  263.              * The page fault came during copy_from_uspace()
  264.              * or copy_to_uspace().
  265.              */
  266.             page_table_unlock(as, lock);
  267.             return;
  268.         default:
  269.             panic("Unexpected pfrc (%d).", pfrc);
  270.         }
  271.     }
  272.    
  273.     pte->accessed = 1; /* Record access to PTE */
  274.     pht_insert(badvaddr, pte);
  275.    
  276.     page_table_unlock(as, lock);
  277.     return;
  278.    
  279. fail:
  280.     page_table_unlock(as, lock);
  281.     pht_refill_fail(badvaddr, istate);
  282. }
  283.  
  284.  
  285. /** Process Instruction/Data Storage Interrupt in Real Mode
  286.  *
  287.  * @param n     Interrupt vector number.
  288.  * @param istate    Interrupted register context.
  289.  *
  290.  */
  291. bool pht_refill_real(int n, istate_t *istate)
  292. {
  293.     uintptr_t badvaddr;
  294.    
  295.     if (n == VECTOR_DATA_STORAGE)
  296.         badvaddr = istate->dar;
  297.     else
  298.         badvaddr = istate->pc;
  299.    
  300.     uint32_t physmem;
  301.     asm volatile (
  302.         "mfsprg3 %0\n"
  303.         : "=r" (physmem)
  304.     );
  305.    
  306.     if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
  307.         return false;
  308.    
  309.     uint32_t page = (badvaddr >> 12) & 0xffff;
  310.     uint32_t api = (badvaddr >> 22) & 0x3f;
  311.    
  312.     uint32_t vsid;
  313.     asm volatile (
  314.         "mfsrin %0, %1\n"
  315.         : "=r" (vsid)
  316.         : "r" (badvaddr)
  317.     );
  318.    
  319.     uint32_t sdr1;
  320.     asm volatile (
  321.         "mfsdr1 %0\n"
  322.         : "=r" (sdr1)
  323.     );
  324.     phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
  325.    
  326.     /* Primary hash (xor) */
  327.     uint32_t h = 0;
  328.     uint32_t hash = vsid ^ page;
  329.     uint32_t base = (hash & 0x3ff) << 3;
  330.     uint32_t i;
  331.     bool found = false;
  332.    
  333.     /* Find colliding PTE in PTEG */
  334.     for (i = 0; i < 8; i++) {
  335.         if ((phte_real[base + i].v)
  336.             && (phte_real[base + i].vsid == vsid)
  337.             && (phte_real[base + i].api == api)
  338.             && (phte_real[base + i].h == 0)) {
  339.             found = true;
  340.             break;
  341.         }
  342.     }
  343.    
  344.     if (!found) {
  345.         /* Find unused PTE in PTEG */
  346.         for (i = 0; i < 8; i++) {
  347.             if (!phte_real[base + i].v) {
  348.                 found = true;
  349.                 break;
  350.             }
  351.         }
  352.     }
  353.    
  354.     if (!found) {
  355.         /* Secondary hash (not) */
  356.         uint32_t base2 = (~hash & 0x3ff) << 3;
  357.        
  358.         /* Find colliding PTE in PTEG */
  359.         for (i = 0; i < 8; i++) {
  360.             if ((phte_real[base2 + i].v)
  361.                 && (phte_real[base2 + i].vsid == vsid)
  362.                 && (phte_real[base2 + i].api == api)
  363.                 && (phte_real[base2 + i].h == 1)) {
  364.                 found = true;
  365.                 base = base2;
  366.                 h = 1;
  367.                 break;
  368.             }
  369.         }
  370.        
  371.         if (!found) {
  372.             /* Find unused PTE in PTEG */
  373.             for (i = 0; i < 8; i++) {
  374.                 if (!phte_real[base2 + i].v) {
  375.                     found = true;
  376.                     base = base2;
  377.                     h = 1;
  378.                     break;
  379.                 }
  380.             }
  381.         }
  382.        
  383.         if (!found) {
  384.             /* Use secondary hash to avoid collisions
  385.                with usual PHT refill handler. */
  386.             i = RANDI(seed_real) % 8;
  387.             base = base2;
  388.             h = 1;
  389.         }
  390.     }
  391.    
  392.     phte_real[base + i].v = 1;
  393.     phte_real[base + i].vsid = vsid;
  394.     phte_real[base + i].h = h;
  395.     phte_real[base + i].api = api;
  396.     phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
  397.     phte_real[base + i].r = 0;
  398.     phte_real[base + i].c = 0;
  399.     phte_real[base + i].wimg = 0;
  400.     phte_real[base + i].pp = 2; // FIXME
  401.    
  402.     return true;
  403. }
  404.  
  405.  
  406. void tlb_arch_init(void)
  407. {
  408.     tlb_invalidate_all();
  409. }
  410.  
  411.  
  412. void tlb_invalidate_all(void)
  413. {
  414.     asm volatile (
  415.         "tlbsync\n"
  416.     );
  417. }
  418.  
  419.  
  420. void tlb_invalidate_asid(asid_t asid)
  421. {
  422.     uint32_t sdr1;
  423.     asm volatile (
  424.         "mfsdr1 %0\n"
  425.         : "=r" (sdr1)
  426.     );
  427.     phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
  428.    
  429.     uint32_t i;
  430.     for (i = 0; i < 8192; i++) {
  431.         if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
  432.             (phte[i].vsid < ((asid << 4) + 16)))
  433.             phte[i].v = 0;
  434.     }
  435.     tlb_invalidate_all();
  436. }
  437.  
  438.  
  439. void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
  440. {
  441.     // TODO
  442.     tlb_invalidate_all();
  443. }
  444.  
  445.  
  446. #define PRINT_BAT(name, ureg, lreg) \
  447.     asm volatile ( \
  448.         "mfspr %0," #ureg "\n" \
  449.         "mfspr %1," #lreg "\n" \
  450.         : "=r" (upper), "=r" (lower) \
  451.     ); \
  452.     mask = (upper & 0x1ffc) >> 2; \
  453.     if (upper & 3) { \
  454.         uint32_t tmp = mask; \
  455.         length = 128; \
  456.         while (tmp) { \
  457.             if ((tmp & 1) == 0) { \
  458.                 printf("ibat[0]: error in mask\n"); \
  459.                 break; \
  460.             } \
  461.             length <<= 1; \
  462.             tmp >>= 1; \
  463.         } \
  464.     } else \
  465.         length = 0; \
  466.     printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
  467.         sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
  468.         lower & 0xffff0000, length, mask, \
  469.         ((upper >> 1) & 1) ? " supervisor" : "", \
  470.         (upper & 1) ? " user" : "");
  471.  
  472.  
  473. void tlb_print(void)
  474. {
  475.     uint32_t sr;
  476.    
  477.     for (sr = 0; sr < 16; sr++) {
  478.         uint32_t vsid;
  479.         asm volatile (
  480.             "mfsrin %0, %1\n"
  481.             : "=r" (vsid)
  482.             : "r" (sr << 28)
  483.         );
  484.         printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
  485.             sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
  486.             ((vsid >> 30) & 1) ? " supervisor" : "",
  487.             ((vsid >> 29) & 1) ? " user" : "");
  488.     }
  489.    
  490.     uint32_t upper;
  491.     uint32_t lower;
  492.     uint32_t mask;
  493.     uint32_t length;
  494.    
  495.     PRINT_BAT("ibat[0]", 528, 529);
  496.     PRINT_BAT("ibat[1]", 530, 531);
  497.     PRINT_BAT("ibat[2]", 532, 533);
  498.     PRINT_BAT("ibat[3]", 534, 535);
  499.    
  500.     PRINT_BAT("dbat[0]", 536, 537);
  501.     PRINT_BAT("dbat[1]", 538, 539);
  502.     PRINT_BAT("dbat[2]", 540, 541);
  503.     PRINT_BAT("dbat[3]", 542, 543);
  504. }
  505.  
  506. /** @}
  507.  */
  508.