WebSVN – HelenOS – Path Comparison – / – /branches/dd/kernel/arch/ppc32/src/mm/tlb.c Rev 4054 and /branches/dd/kernel/arch/ppc32/src/mm/tlb.c Rev 4055

Ignore whitespace Rev 4054 → Rev 4055

 /branches/dd/kernel/arch/ppc32/src/mm/tlb.c
 ,7 → 26,7
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-/** @addtogroup ppc32mm
+/** @addtogroup ppc32mm
  * @{
  */
 /** @file
 ,12 → 35,23
 #include <mm/tlb.h>
 #include <arch/mm/tlb.h>
 #include <arch/interrupt.h>
+#include <interrupt.h>
 #include <mm/as.h>
 #include <arch.h>
 #include <print.h>
 #include <symtab.h>
+#include <macros.h>
+static unsigned int seed = 10;
+static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
+#define TLB_FLUSH \
+    "tlbie %0\n" \
+    "addi %0, %0, 0x1000\n"
 /** Try to find PTE for faulting address
  *
  * Try to find PTE for faulting address.
 ,22 → 58,25
  * The as->lock must be held on entry to this function
  * if lock is true.
  *
- * @param as       Address space.
- * @param lock     Lock/unlock the address space.
- * @param badvaddr Faulting virtual address.
- * @param access   Access mode that caused the fault.
- * @param istate   Pointer to interrupted state.
- * @param pfrc     Pointer to variable where as_page_fault() return code will be stored.
- * @return         PTE on success, NULL otherwise.
+ * @param as        Address space.
+ * @param lock      Lock/unlock the address space.
+ * @param badvaddr  Faulting virtual address.
+ * @param access    Access mode that caused the fault.
+ * @param istate    Pointer to interrupted state.
+ * @param pfrc      Pointer to variable where as_page_fault() return code
+ *          will be stored.
+ * @return      PTE on success, NULL otherwise.
  *
  */
-static pte_t *find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access, istate_t *istate, int *pfrc)
+static pte_t *
+find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
+    istate_t *istate, int *pfrc)
 {
     /*
      * Check if the mapping exists in page tables.
      */
     pte_t *pte = page_mapping_find(as, badvaddr);
-    if ((pte) && (pte->p)) {
+    if ((pte) && (pte->present)) {
         /*
          * Mapping found in page tables.
          * Immediately succeed.
 ,27 → 91,26
          */
         page_table_unlock(as, lock);
         switch (rc = as_page_fault(badvaddr, access, istate)) {
-            case AS_PF_OK:
-                /*
-                 * The higher-level page fault handler succeeded,
-                 * The mapping ought to be in place.
-                 */
-                page_table_lock(as, lock);
-                pte = page_mapping_find(as, badvaddr);
-                ASSERT((pte) && (pte->p));
-                *pfrc = 0;
-                return pte;
-            case AS_PF_DEFER:
-                page_table_lock(as, lock);
-                *pfrc = rc;
-                return NULL;
-            case AS_PF_FAULT:
-                page_table_lock(as, lock);
-                printf("Page fault.\n");
-                *pfrc = rc;
-                return NULL;
-            default:
-                panic("unexpected rc (%d)\n", rc);
+        case AS_PF_OK:
+            /*
+             * The higher-level page fault handler succeeded,
+             * The mapping ought to be in place.
+             */
+            page_table_lock(as, lock);
+            pte = page_mapping_find(as, badvaddr);
+            ASSERT((pte) && (pte->present));
+            *pfrc = 0;
+            return pte;
+        case AS_PF_DEFER:
+            page_table_lock(as, lock);
+            *pfrc = rc;
+            return NULL;
+        case AS_PF_FAULT:
+            page_table_lock(as, lock);
+            *pfrc = rc;
+            return NULL;
+        default:
+            panic("Unexpected rc (%d).", rc);
         }
     }
 }
 ,17 → 121,21
     char *symbol = "";
     char *sym2 = "";
-    char *s = get_symtab_entry(istate->pc);
-    if (s)
-        symbol = s;
-    s = get_symtab_entry(istate->lr);
-    if (s)
-        sym2 = s;
-    panic("%p: PHT Refill Exception at %p (%s<-%s)\n", badvaddr, istate->pc, symbol, sym2);
+    char *str = get_symtab_entry(istate->pc);
+    if (str)
+        symbol = str;
+    str = get_symtab_entry(istate->lr);
+    if (str)
+        sym2 = str;
+    fault_if_from_uspace(istate,
+        "PHT Refill Exception on %p.", badvaddr);
+    panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
+        istate->pc, symbol, sym2);
 }
-static void pht_insert(const uintptr_t vaddr, const pfn_t pfn)
+static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
 {
     uint32_t page = (vaddr >> 12) & 0xffff;
     uint32_t api = (vaddr >> 22) & 0x3f;
 ,10 → 161,12
     uint32_t i;
     bool found = false;
-    /* Find unused or colliding
-       PTE in PTEG */
+    /* Find colliding PTE in PTEG */
     for (i = 0; i < 8; i++) {
-        if ((!phte[base + i].v) || ((phte[base + i].vsid == vsid) && (phte[base + i].api == api))) {
+        if ((phte[base + i].v)
+            && (phte[base + i].vsid == vsid)
+            && (phte[base + i].api == api)
+            && (phte[base + i].h == 0)) {
             found = true;
             break;
         }
 ,13 → 173,25
     }
     if (!found) {
+        /* Find unused PTE in PTEG */
+        for (i = 0; i < 8; i++) {
+            if (!phte[base + i].v) {
+                found = true;
+                break;
+            }
+        }
+    }
+    if (!found) {
         /* Secondary hash (not) */
         uint32_t base2 = (~hash & 0x3ff) << 3;
-        /* Find unused or colliding
-           PTE in PTEG */
+        /* Find colliding PTE in PTEG */
         for (i = 0; i < 8; i++) {
-            if ((!phte[base2 + i].v) || ((phte[base2 + i].vsid == vsid) && (phte[base2 + i].api == api))) {
+            if ((phte[base2 + i].v)
+                && (phte[base2 + i].vsid == vsid)
+                && (phte[base2 + i].api == api)
+                && (phte[base2 + i].h == 1)) {
                 found = true;
                 base = base2;
                 h = 1;
 ,9 → 200,19
         }
         if (!found) {
-            // TODO: A/C precedence groups
-            i = page % 8;
+            /* Find unused PTE in PTEG */
+            for (i = 0; i < 8; i++) {
+                if (!phte[base2 + i].v) {
+                    found = true;
+                    base = base2;
+                    h = 1;
+                    break;
+                }
+            }
         }
+        if (!found)
+            i = RANDI(seed) % 8;
     }
     phte[base + i].v = 1;
 ,23 → 219,106
     phte[base + i].vsid = vsid;
     phte[base + i].h = h;
     phte[base + i].api = api;
-    phte[base + i].rpn = pfn;
+    phte[base + i].rpn = pte->pfn;
     phte[base + i].r = 0;
     phte[base + i].c = 0;
+    phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
     phte[base + i].pp = 2; // FIXME
 }
-static void pht_real_insert(const uintptr_t vaddr, const pfn_t pfn)
+/** Process Instruction/Data Storage Exception
+ *
+ * @param n      Exception vector number.
+ * @param istate Interrupted register context.
+ *
+ */
+void pht_refill(int n, istate_t *istate)
 {
-    uint32_t page = (vaddr >> 12) & 0xffff;
-    uint32_t api = (vaddr >> 22) & 0x3f;
+    uintptr_t badvaddr;
+    pte_t *pte;
+    int pfrc;
+    as_t *as;
+    bool lock;
+    if (AS == NULL) {
+        as = AS_KERNEL;
+        lock = false;
+    } else {
+        as = AS;
+        lock = true;
+    }
+    if (n == VECTOR_DATA_STORAGE)
+        badvaddr = istate->dar;
+    else
+        badvaddr = istate->pc;
+    page_table_lock(as, lock);
+    pte = find_mapping_and_check(as, lock, badvaddr,
+        PF_ACCESS_READ /* FIXME */, istate, &pfrc);
+    if (!pte) {
+        switch (pfrc) {
+        case AS_PF_FAULT:
+            goto fail;
+            break;
+        case AS_PF_DEFER:
+            /*
+             * The page fault came during copy_from_uspace()
+             * or copy_to_uspace().
+             */
+            page_table_unlock(as, lock);
+            return;
+        default:
+            panic("Unexpected pfrc (%d).", pfrc);
+        }
+    }
+    pte->accessed = 1; /* Record access to PTE */
+    pht_insert(badvaddr, pte);
+    page_table_unlock(as, lock);
+    return;
+fail:
+    page_table_unlock(as, lock);
+    pht_refill_fail(badvaddr, istate);
+}
+/** Process Instruction/Data Storage Exception in Real Mode
+ *
+ * @param n      Exception vector number.
+ * @param istate Interrupted register context.
+ *
+ */
+bool pht_refill_real(int n, istate_t *istate)
+{
+    uintptr_t badvaddr;
+    if (n == VECTOR_DATA_STORAGE)
+        badvaddr = istate->dar;
+    else
+        badvaddr = istate->pc;
+    uint32_t physmem;
+    asm volatile (
+        "mfsprg3 %0\n"
+        : "=r" (physmem)
+    );
+    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
+        return false;
+    uint32_t page = (badvaddr >> 12) & 0xffff;
+    uint32_t api = (badvaddr >> 22) & 0x3f;
     uint32_t vsid;
     asm volatile (
         "mfsrin %0, %1\n"
         : "=r" (vsid)
-        : "r" (vaddr)
+        : "r" (badvaddr)
     );
     uint32_t sdr1;
 ,7 → 326,7
         "mfsdr1 %0\n"
         : "=r" (sdr1)
     );
-    phte_t *phte_physical = (phte_t *) (sdr1 & 0xffff0000);
+    phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
     /* Primary hash (xor) */
     uint32_t h = 0;
 ,10 → 335,12
     uint32_t i;
     bool found = false;
-    /* Find unused or colliding
-       PTE in PTEG */
+    /* Find colliding PTE in PTEG */
     for (i = 0; i < 8; i++) {
-        if ((!phte_physical[base + i].v) || ((phte_physical[base + i].vsid == vsid) && (phte_physical[base + i].api == api))) {
+        if ((phte_real[base + i].v)
+            && (phte_real[base + i].vsid == vsid)
+            && (phte_real[base + i].api == api)
+            && (phte_real[base + i].h == 0)) {
             found = true;
             break;
         }
 ,13 → 347,25
     }
     if (!found) {
+        /* Find unused PTE in PTEG */
+        for (i = 0; i < 8; i++) {
+            if (!phte_real[base + i].v) {
+                found = true;
+                break;
+            }
+        }
+    }
+    if (!found) {
         /* Secondary hash (not) */
         uint32_t base2 = (~hash & 0x3ff) << 3;
-        /* Find unused or colliding
-           PTE in PTEG */
+        /* Find colliding PTE in PTEG */
         for (i = 0; i < 8; i++) {
-            if ((!phte_physical[base2 + i].v) || ((phte_physical[base2 + i].vsid == vsid) && (phte_physical[base2 + i].api == api))) {
+            if ((phte_real[base2 + i].v)
+                && (phte_real[base2 + i].vsid == vsid)
+                && (phte_real[base2 + i].api == api)
+                && (phte_real[base2 + i].h == 1)) {
                 found = true;
                 base = base2;
                 h = 1;
 ,102 → 374,48
         }
         if (!found) {
-            // TODO: A/C precedence groups
-            i = page % 8;
+            /* Find unused PTE in PTEG */
+            for (i = 0; i < 8; i++) {
+                if (!phte_real[base2 + i].v) {
+                    found = true;
+                    base = base2;
+                    h = 1;
+                    break;
+                }
+            }
         }
-    }
-    phte_physical[base + i].v = 1;
-    phte_physical[base + i].vsid = vsid;
-    phte_physical[base + i].h = h;
-    phte_physical[base + i].api = api;
-    phte_physical[base + i].rpn = pfn;
-    phte_physical[base + i].r = 0;
-    phte_physical[base + i].c = 0;
-    phte_physical[base + i].pp = 2; // FIXME
-}
-/** Process Instruction/Data Storage Interrupt
- *
- * @param n Interrupt vector number.
- * @param istate Interrupted register context.
- *
- */
-void pht_refill(int n, istate_t *istate)
-{
-    uintptr_t badvaddr;
-    pte_t *pte;
-    int pfrc;
-    as_t *as;
-    bool lock;
-    if (AS == NULL) {
-        as = AS_KERNEL;
-        lock = false;
-    } else {
-        as = AS;
-        lock = true;
-    }
-    if (n == VECTOR_DATA_STORAGE) {
-        asm volatile (
-            "mfdar %0\n"
-            : "=r" (badvaddr)
-        );
-    } else
-        badvaddr = istate->pc;
-    page_table_lock(as, lock);
-    pte = find_mapping_and_check(as, lock, badvaddr, PF_ACCESS_READ /* FIXME */, istate, &pfrc);
-    if (!pte) {
-        switch (pfrc) {
-            case AS_PF_FAULT:
-                goto fail;
-                break;
-            case AS_PF_DEFER:
-                /*
-                 * The page fault came during copy_from_uspace()
-                 * or copy_to_uspace().
-                 */
-                page_table_unlock(as, lock);
-                return;
-            default:
-                panic("Unexpected pfrc (%d)\n", pfrc);
+        if (!found) {
+            /* Use secondary hash to avoid collisions
+               with usual PHT refill handler. */
+            i = RANDI(seed_real) % 8;
+            base = base2;
+            h = 1;
         }
     }
-    pte->a = 1; /* Record access to PTE */
-    pht_insert(badvaddr, pte->pfn);
+    phte_real[base + i].v = 1;
+    phte_real[base + i].vsid = vsid;
+    phte_real[base + i].h = h;
+    phte_real[base + i].api = api;
+    phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
+    phte_real[base + i].r = 0;
+    phte_real[base + i].c = 0;
+    phte_real[base + i].wimg = 0;
+    phte_real[base + i].pp = 2; // FIXME
-    page_table_unlock(as, lock);
-    return;
-fail:
-    page_table_unlock(as, lock);
-    pht_refill_fail(badvaddr, istate);
+    return true;
 }
-/** Process Instruction/Data Storage Interrupt in Real Mode
+/** Process ITLB/DTLB Miss Exception in Real Mode
  *
- * @param n Interrupt vector number.
- * @param istate Interrupted register context.
  *
  */
-bool pht_real_refill(int n, istate_t *istate)
+void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
 {
-    uintptr_t badvaddr;
+    uint32_t badvaddr = tlbmiss & 0xfffffffc;
-    if (n == VECTOR_DATA_STORAGE) {
-        asm volatile (
-            "mfdar %0\n"
-            : "=r" (badvaddr)
-        );
-    } else
-        badvaddr = istate->pc;
     uint32_t physmem;
     asm volatile (
         "mfsprg3 %0\n"
 ,12 → 422,23
         : "=r" (physmem)
     );
-    if ((badvaddr >= PA2KA(0)) && (badvaddr < PA2KA(physmem))) {
-        pht_real_insert(badvaddr, KA2PA(badvaddr) >> 12);
-        return true;
-    }
+    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
+        return; // FIXME
-    return false;
+    ptelo.rpn = KA2PA(badvaddr) >> 12;
+    ptelo.wimg = 0;
+    ptelo.pp = 2; // FIXME
+    uint32_t index = 0;
+    asm volatile (
+        "mtspr 981, %0\n"
+        "mtspr 982, %1\n"
+        "tlbld %2\n"
+        "tlbli %2\n"
+        : "=r" (index)
+        : "r" (ptehi),
+          "r" (ptelo)
+    );
 }
 ,9 → 450,87
 void tlb_invalidate_all(void)
 {
+    uint32_t index;
     asm volatile (
-        "tlbia\n"
+        "li %0, 0\n"
+        "sync\n"
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        TLB_FLUSH
+        "eieio\n"
         "tlbsync\n"
+        "sync\n"
+        : "=r" (index)
     );
 }
 ,7 → 546,8
     uint32_t i;
     for (i = 0; i < 8192; i++) {
-        if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) && (phte[i].vsid < ((asid << 4) + 16)))
+        if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
+            (phte[i].vsid < ((asid << 4) + 16)))
             phte[i].v = 0;
     }
     tlb_invalidate_all();
 ,7 → 581,11
         } \
     } else \
         length = 0; \
-    printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, lower & 0xffff0000, length, mask, ((upper >> 1) & 1) ? " supervisor" : "", (upper & 1) ? " user" : "");
+    printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
+        sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
+        lower & 0xffff0000, length, mask, \
+        ((upper >> 1) & 1) ? " supervisor" : "", \
+        (upper & 1) ? " user" : "");
 void tlb_print(void)
 ,7 → 599,10
             : "=r" (vsid)
             : "r" (sr << 28)
         );
-        printf("vsid[%d]: VSID=%.*p (ASID=%d)%s%s\n", sr, sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4, ((vsid >> 30) & 1) ? " supervisor" : "", ((vsid >> 29) & 1) ? " user" : "");
+        printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
+            sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
+            ((vsid >> 30) & 1) ? " supervisor" : "",
+            ((vsid >> 29) & 1) ? " user" : "");
     }
     uint32_t upper;

Subversion Repositories HelenOS

Compare Revisions

Ignore whitespace Rev 4054 → Rev 4055