WebSVN – HelenOS – Diff – /trunk/kernel/arch/mips32/src/mm/tlb.c


/*
 * Copyright (c) 2003-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
/** @addtogroup mips32mm   
 * @{
 */
/** @file
 */
 
#include <arch/mm/tlb.h>
#include <mm/asid.h>
#include <mm/tlb.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/cp0.h>
#include <panic.h>
#include <arch.h>
#include <symtab.h>
#include <synch/spinlock.h>
#include <print.h>
#include <debug.h>
#include <align.h>
#include <interrupt.h>
 
static void tlb_refill_fail(istate_t *);
static void tlb_invalid_fail(istate_t *);
static void tlb_modified_fail(istate_t *);
 
static pte_t *find_mapping_and_check(uintptr_t, int, istate_t *, int *);
 
/** Initialize TLB.
 *
 
 * Invalidate all entries and mark wired entries.
 */
void tlb_arch_init(void)
{
    int i;
 
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    cp0_entry_hi_write(0);
    cp0_entry_lo0_write(0);
    cp0_entry_lo1_write(0);
 
    /* Clear and initialize TLB. */
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbwi();
    }
       
    /*
     * The kernel is going to make use of some wired
     * entries (e.g. mapping kernel stacks in kseg3).
     */
    cp0_wired_write(TLB_WIRED);
}
 
/** Process TLB Refill Exception.
 *
 
 
 * @param istate    Interrupted register context.
 */
void tlb_refill(istate_t *istate)
{
    entry_lo_t lo;
    entry_hi_t hi;
    asid_t asid;
    uintptr_t badvaddr;
    pte_t *pte;
    int pfrc;
 
    badvaddr = cp0_badvaddr_read();
 
    spinlock_lock(&AS->lock);
    asid = AS->asid;
    spinlock_unlock(&AS->lock);
 
    page_table_lock(AS, true);
 
    pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
    if (!pte) {
        switch (pfrc) {
        case AS_PF_FAULT:
            goto fail;
            break;
        case AS_PF_DEFER:
            /*
             * The page fault came during copy_from_uspace()
             * or copy_to_uspace().
             */
            page_table_unlock(AS, true);
            return;
        default:
            panic("unexpected pfrc (%d)\n", pfrc);
        }
    }
 
    /*
     * Record access to PTE.
     */
    pte->a = 1;
 
    tlb_prepare_entry_hi(&hi, asid, badvaddr);
    tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
        pte->pfn);
 
    /*
     * New entry is to be inserted into TLB
     */
    cp0_entry_hi_write(hi.value);
    if ((badvaddr / PAGE_SIZE) % 2 == 0) {
        cp0_entry_lo0_write(lo.value);
        cp0_entry_lo1_write(0);
    }
    else {
        cp0_entry_lo0_write(0);
        cp0_entry_lo1_write(lo.value);
    }
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwr();
 
    page_table_unlock(AS, true);
    return;
   
fail:
    page_table_unlock(AS, true);
    tlb_refill_fail(istate);
}
 
/** Process TLB Invalid Exception.
 
 
 *
 * @param istate    Interrupted register context.
 */
void tlb_invalid(istate_t *istate)
{
    tlb_index_t index;
    uintptr_t badvaddr;
    entry_lo_t lo;
    entry_hi_t hi;
    pte_t *pte;
    int pfrc;
 
    badvaddr = cp0_badvaddr_read();
 
    /*
     * Locate the faulting entry in TLB.
     */
    hi.value = cp0_entry_hi_read();
    tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
    cp0_entry_hi_write(hi.value);
    tlbp();
    index.value = cp0_index_read();
 
    page_table_lock(AS, true); 
   
    /*
     * Fail if the entry is not in TLB.
     */
    if (index.p) {
        printf("TLB entry not found.\n");
        goto fail;
    }
 
    pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
    if (!pte) {
        switch (pfrc) {
        case AS_PF_FAULT:
            goto fail;
            break;
        case AS_PF_DEFER:
            /*
             * The page fault came during copy_from_uspace()
             * or copy_to_uspace().
             */
            page_table_unlock(AS, true);             
            return;
        default:
            panic("unexpected pfrc (%d)\n", pfrc);
        }
    }
 
    /*
     * Read the faulting TLB entry.
     */
    tlbr();
 
    /*
     * Record access to PTE.
     */
    pte->a = 1;
 
    tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
        pte->pfn);
 
    /*
     * The entry is to be updated in TLB.
     */
    if ((badvaddr / PAGE_SIZE) % 2 == 0)
        cp0_entry_lo0_write(lo.value);
    else
        cp0_entry_lo1_write(lo.value);
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwi();
 
    page_table_unlock(AS, true);
    return;
   
fail:
    page_table_unlock(AS, true);
    tlb_invalid_fail(istate);
}
 
/** Process TLB Modified Exception.
 
 
 *
 * @param istate    Interrupted register context.
 */
void tlb_modified(istate_t *istate)
{
    tlb_index_t index;
    uintptr_t badvaddr;
    entry_lo_t lo;
    entry_hi_t hi;
    pte_t *pte;
    int pfrc;
 
    badvaddr = cp0_badvaddr_read();
 
    /*
     * Locate the faulting entry in TLB.
     */
    hi.value = cp0_entry_hi_read();
    tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
    cp0_entry_hi_write(hi.value);
    tlbp();
    index.value = cp0_index_read();
 
    page_table_lock(AS, true); 
   
    /*
     * Fail if the entry is not in TLB.
     */
    if (index.p) {
        printf("TLB entry not found.\n");
        goto fail;
    }
 
    pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
    if (!pte) {
        switch (pfrc) {
        case AS_PF_FAULT:
            goto fail;
            break;
        case AS_PF_DEFER:
            /*
             * The page fault came during copy_from_uspace()
             * or copy_to_uspace().
             */
            page_table_unlock(AS, true);             
            return;
        default:
            panic("unexpected pfrc (%d)\n", pfrc);
        }
    }
 
    /*
     * Fail if the page is not writable.
     */
    if (!pte->w)
        goto fail;
 
    /*
     * Read the faulting TLB entry.
     */
    tlbr();
 
    /*
     * Record access and write to PTE.
     */
    pte->a = 1;
    pte->d = 1;
 
    tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable,
        pte->pfn);
 
    /*
     * The entry is to be updated in TLB.
     */
    if ((badvaddr / PAGE_SIZE) % 2 == 0)
        cp0_entry_lo0_write(lo.value);
    else
        cp0_entry_lo1_write(lo.value);
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwi();
 
    page_table_unlock(AS, true);
    return;
   
fail:
    page_table_unlock(AS, true);
    tlb_modified_fail(istate);
}
 
void tlb_refill_fail(istate_t *istate)
{
    char *symbol = "";
    char *sym2 = "";
 
    char *s = get_symtab_entry(istate->epc);
    if (s)
        symbol = s;
    s = get_symtab_entry(istate->ra);
    if (s)
        sym2 = s;
 
    fault_if_from_uspace(istate, "TLB Refill Exception on %p",
        cp0_badvaddr_read());
    panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(),
        istate->epc, symbol, sym2);
}
 
 
void tlb_invalid_fail(istate_t *istate)
{
    char *symbol = "";
 
    char *s = get_symtab_entry(istate->epc);
    if (s)
        symbol = s;
    fault_if_from_uspace(istate, "TLB Invalid Exception on %p",
        cp0_badvaddr_read());
    panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(),
        istate->epc, symbol);
}
 
void tlb_modified_fail(istate_t *istate)
{
    char *symbol = "";
 
    char *s = get_symtab_entry(istate->epc);
    if (s)
        symbol = s;
    fault_if_from_uspace(istate, "TLB Modified Exception on %p",
        cp0_badvaddr_read());
    panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(),
        istate->epc, symbol);
}
 
/** Try to find PTE for faulting address.
 *
 
 * The AS->lock must be held on entry to this function.
 *
 * @param badvaddr  Faulting virtual address.
 * @param access    Access mode that caused the fault.
 * @param istate    Pointer to interrupted state.
 * @param pfrc      Pointer to variable where as_page_fault() return code
 *          will be stored.
 *
 * @return      PTE on success, NULL otherwise.
 */
pte_t *
find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate,
    int *pfrc)
{
    entry_hi_t hi;
    pte_t *pte;
 
    hi.value = cp0_entry_hi_read();
 
    /*
     * Handler cannot succeed if the ASIDs don't match.
     */
    if (hi.asid != AS->asid) {
        printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
        return NULL;
    }
 
    /*
     * Check if the mapping exists in page tables.
     */
    pte = page_mapping_find(AS, badvaddr);
    if (pte && pte->p) {
        /*
         * Mapping found in page tables.
         * Immediately succeed.
         */
        return pte;
    } else {
        int rc;
       
        /*
         * Mapping not found in page tables.
         * Resort to higher-level page fault handler.
         */
        page_table_unlock(AS, true);
        switch (rc = as_page_fault(badvaddr, access, istate)) {
        case AS_PF_OK:
            /*
             * The higher-level page fault handler succeeded,
             * The mapping ought to be in place.
             */
            page_table_lock(AS, true);
            pte = page_mapping_find(AS, badvaddr);
            ASSERT(pte && pte->p);
            return pte;
            break;
        case AS_PF_DEFER:
            page_table_lock(AS, true);
            *pfrc = AS_PF_DEFER;
            return NULL;
            break;
        case AS_PF_FAULT:
            page_table_lock(AS, true);
            printf("Page fault.\n");
            *pfrc = AS_PF_FAULT;
            return NULL;
            break;
        default:
            panic("unexpected rc (%d)\n", rc);
        }
       
    }
}
 
void
tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable,
    uintptr_t pfn)
{
    lo->value = 0;
    lo->g = g;
    lo->v = v;
    lo->d = d;
    lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
    lo->pfn = pfn;
}
 
void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
{
    hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
    hi->asid = asid;
}
 
/** Print contents of TLB. */
void tlb_print(void)
{
    page_mask_t mask;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    unsigned int i;
 
    hi_save.value = cp0_entry_hi_read();
   
    printf("#  ASID VPN2   MASK G V D C PFN\n");
    printf("-- ---- ------ ---- - - - - ------\n");
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
       
        mask.value = cp0_pagemask_read();
        hi.value = cp0_entry_hi_read();
        lo0.value = cp0_entry_lo0_read();
        lo1.value = cp0_entry_lo1_read();
       
        printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
            i, hi.asid, hi.vpn2, mask.mask,
            lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
        printf("                    %1u %1u %1u %1u %#6x\n",
            lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
    }
   
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate all not wired TLB entries. */
void tlb_invalidate_all(void)
{
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi_save;
    int i;
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
 
    for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
 
        lo0.value = cp0_entry_lo0_read();
        lo1.value = cp0_entry_lo1_read();
 
        lo0.v = 0;
        lo1.v = 0;
 
        cp0_entry_lo0_write(lo0.value);
        cp0_entry_lo1_write(lo1.value);
               
        tlbwi();
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate all TLB entries belonging to specified address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    int i;
 
    ASSERT(asid != ASID_INVALID);
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
       
        hi.value = cp0_entry_hi_read();
       
        if (hi.asid == asid) {
            lo0.value = cp0_entry_lo0_read();
            lo1.value = cp0_entry_lo1_read();
 
            lo0.v = 0;
            lo1.v = 0;
 
            cp0_entry_lo0_write(lo0.value);
            cp0_entry_lo1_write(lo1.value);
 
            tlbwi();
        }
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate TLB entries for specified page range belonging to specified
 * address space.
 *
 * @param asid      Address space identifier.
 * @param page      First page whose TLB entry is to be invalidated.
 * @param cnt       Number of entries to invalidate.
 */
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
{
    unsigned int i;
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    tlb_index_t index;
 
    ASSERT(asid != ASID_INVALID);
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
 
    for (i = 0; i < cnt + 1; i += 2) {
        hi.value = 0;
        tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
        cp0_entry_hi_write(hi.value);
 
        tlbp();
        index.value = cp0_index_read();
 
        if (!index.p) {
            /*
             * Entry was found, index register contains valid
             * index.
             */
            tlbr();
 
            lo0.value = cp0_entry_lo0_read();
            lo1.value = cp0_entry_lo1_read();
 
            lo0.v = 0;
            lo1.v = 0;
 
            cp0_entry_lo0_write(lo0.value);
            cp0_entry_lo1_write(lo1.value);
 
            tlbwi();
        }
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** @}
 */
 

Rev 3228	Rev 3392
1	/*	1	/*
2	* Copyright (c) 2003-2004 Jakub Jermar	2	* Copyright (c) 2003-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	/** @addtogroup mips32mm	29	/** @addtogroup mips32mm
30	* @{	30	* @{
31	*/	31	*/
32	/** @file	32	/** @file
33	*/	33	*/
34		34
35	#include <arch/mm/tlb.h>	35	#include <arch/mm/tlb.h>
36	#include <mm/asid.h>	36	#include <mm/asid.h>
37	#include <mm/tlb.h>	37	#include <mm/tlb.h>
38	#include <mm/page.h>	38	#include <mm/page.h>
39	#include <mm/as.h>	39	#include <mm/as.h>
40	#include <arch/cp0.h>	40	#include <arch/cp0.h>
41	#include <panic.h>	41	#include <panic.h>
42	#include <arch.h>	42	#include <arch.h>
43	#include <symtab.h>	43	#include <symtab.h>
44	#include <synch/spinlock.h>	44	#include <synch/spinlock.h>
45	#include <print.h>	45	#include <print.h>
46	#include <debug.h>	46	#include <debug.h>
47	#include <align.h>	47	#include <align.h>
48	#include <interrupt.h>	48	#include <interrupt.h>
49		49
50	static void tlb_refill_fail(istate_t *istate);	50	static void tlb_refill_fail(istate_t *);
51	static void tlb_invalid_fail(istate_t *istate);	51	static void tlb_invalid_fail(istate_t *);
52	static void tlb_modified_fail(istate_t *istate);	52	static void tlb_modified_fail(istate_t *);
53		53
54	static pte_t find_mapping_and_check(uintptr_t badvaddr, int access, istate_t istate, int *pfrc);	54	static pte_t find_mapping_and_check(uintptr_t, int, istate_t , int *);
55		55
56	/** Initialize TLB	56	/** Initialize TLB.
57	*	57	*
58	* Initialize TLB.	-
59	* Invalidate all entries and mark wired entries.	58	* Invalidate all entries and mark wired entries.
60	*/	59	*/
61	void tlb_arch_init(void)	60	void tlb_arch_init(void)
62	{	61	{
63	int i;	62	int i;
64		63
65	cp0_pagemask_write(TLB_PAGE_MASK_16K);	64	cp0_pagemask_write(TLB_PAGE_MASK_16K);
66	cp0_entry_hi_write(0);	65	cp0_entry_hi_write(0);
67	cp0_entry_lo0_write(0);	66	cp0_entry_lo0_write(0);
68	cp0_entry_lo1_write(0);	67	cp0_entry_lo1_write(0);
69		68
70	/* Clear and initialize TLB. */	69	/* Clear and initialize TLB. */
71		70
72	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	71	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
73	cp0_index_write(i);	72	cp0_index_write(i);
74	tlbwi();	73	tlbwi();
75	}	74	}
76		75
77	/*	76	/*
78	* The kernel is going to make use of some wired	77	* The kernel is going to make use of some wired
79	* entries (e.g. mapping kernel stacks in kseg3).	78	* entries (e.g. mapping kernel stacks in kseg3).
80	*/	79	*/
81	cp0_wired_write(TLB_WIRED);	80	cp0_wired_write(TLB_WIRED);
82	}	81	}
83		82
84	/** Process TLB Refill Exception	83	/** Process TLB Refill Exception.
85	*	84	*
86	* Process TLB Refill Exception.	-
87	*	-
88	* @param istate Interrupted register context.	85	* @param istate Interrupted register context.
89	*/	86	*/
90	void tlb_refill(istate_t *istate)	87	void tlb_refill(istate_t *istate)
91	{	88	{
92	entry_lo_t lo;	89	entry_lo_t lo;
93	entry_hi_t hi;	90	entry_hi_t hi;
94	asid_t asid;	91	asid_t asid;
95	uintptr_t badvaddr;	92	uintptr_t badvaddr;
96	pte_t *pte;	93	pte_t *pte;
97	int pfrc;	94	int pfrc;
98		95
99	badvaddr = cp0_badvaddr_read();	96	badvaddr = cp0_badvaddr_read();
100		97
101	spinlock_lock(&AS->lock);	98	spinlock_lock(&AS->lock);
102	asid = AS->asid;	99	asid = AS->asid;
103	spinlock_unlock(&AS->lock);	100	spinlock_unlock(&AS->lock);
104		101
105	page_table_lock(AS, true);	102	page_table_lock(AS, true);
106		103
107	pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);	104	pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
108	if (!pte) {	105	if (!pte) {
109	switch (pfrc) {	106	switch (pfrc) {
110	case AS_PF_FAULT:	107	case AS_PF_FAULT:
111	goto fail;	108	goto fail;
112	break;	109	break;
113	case AS_PF_DEFER:	110	case AS_PF_DEFER:
114	/*	111	/*
115	* The page fault came during copy_from_uspace()	112	* The page fault came during copy_from_uspace()
116	* or copy_to_uspace().	113	* or copy_to_uspace().
117	*/	114	*/
118	page_table_unlock(AS, true);	115	page_table_unlock(AS, true);
119	return;	116	return;
120	default:	117	default:
121	panic("unexpected pfrc (%d)\n", pfrc);	118	panic("unexpected pfrc (%d)\n", pfrc);
122	}	119	}
123	}	120	}
124		121
125	/*	122	/*
126	* Record access to PTE.	123	* Record access to PTE.
127	*/	124	*/
128	pte->a = 1;	125	pte->a = 1;
129		126
130	tlb_prepare_entry_hi(&hi, asid, badvaddr);	127	tlb_prepare_entry_hi(&hi, asid, badvaddr);
131	tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);	128	tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
-		129	pte->pfn);
132		130
133	/*	131	/*
134	* New entry is to be inserted into TLB	132	* New entry is to be inserted into TLB
135	*/	133	*/
136	cp0_entry_hi_write(hi.value);	134	cp0_entry_hi_write(hi.value);
137	if ((badvaddr/PAGE_SIZE) % 2 == 0) {	135	if ((badvaddr / PAGE_SIZE) % 2 == 0) {
138	cp0_entry_lo0_write(lo.value);	136	cp0_entry_lo0_write(lo.value);
139	cp0_entry_lo1_write(0);	137	cp0_entry_lo1_write(0);
140	}	138	}
141	else {	139	else {
142	cp0_entry_lo0_write(0);	140	cp0_entry_lo0_write(0);
143	cp0_entry_lo1_write(lo.value);	141	cp0_entry_lo1_write(lo.value);
144	}	142	}
145	cp0_pagemask_write(TLB_PAGE_MASK_16K);	143	cp0_pagemask_write(TLB_PAGE_MASK_16K);
146	tlbwr();	144	tlbwr();
147		145
148	page_table_unlock(AS, true);	146	page_table_unlock(AS, true);
149	return;	147	return;
150		148
151	fail:	149	fail:
152	page_table_unlock(AS, true);	150	page_table_unlock(AS, true);
153	tlb_refill_fail(istate);	151	tlb_refill_fail(istate);
154	}	152	}
155		153
156	/** Process TLB Invalid Exception	154	/** Process TLB Invalid Exception.
157	*	-
158	* Process TLB Invalid Exception.	-
159	*	155	*
160	* @param istate Interrupted register context.	156	* @param istate Interrupted register context.
161	*/	157	*/
162	void tlb_invalid(istate_t *istate)	158	void tlb_invalid(istate_t *istate)
163	{	159	{
164	tlb_index_t index;	160	tlb_index_t index;
165	uintptr_t badvaddr;	161	uintptr_t badvaddr;
166	entry_lo_t lo;	162	entry_lo_t lo;
167	entry_hi_t hi;	163	entry_hi_t hi;
168	pte_t *pte;	164	pte_t *pte;
169	int pfrc;	165	int pfrc;
170		166
171	badvaddr = cp0_badvaddr_read();	167	badvaddr = cp0_badvaddr_read();
172		168
173	/*	169	/*
174	* Locate the faulting entry in TLB.	170	* Locate the faulting entry in TLB.
175	*/	171	*/
176	hi.value = cp0_entry_hi_read();	172	hi.value = cp0_entry_hi_read();
177	tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);	173	tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
178	cp0_entry_hi_write(hi.value);	174	cp0_entry_hi_write(hi.value);
179	tlbp();	175	tlbp();
180	index.value = cp0_index_read();	176	index.value = cp0_index_read();
181		177
182	page_table_lock(AS, true);	178	page_table_lock(AS, true);
183		179
184	/*	180	/*
185	* Fail if the entry is not in TLB.	181	* Fail if the entry is not in TLB.
186	*/	182	*/
187	if (index.p) {	183	if (index.p) {
188	printf("TLB entry not found.\n");	184	printf("TLB entry not found.\n");
189	goto fail;	185	goto fail;
190	}	186	}
191		187
192	pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);	188	pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
193	if (!pte) {	189	if (!pte) {
194	switch (pfrc) {	190	switch (pfrc) {
195	case AS_PF_FAULT:	191	case AS_PF_FAULT:
196	goto fail;	192	goto fail;
197	break;	193	break;
198	case AS_PF_DEFER:	194	case AS_PF_DEFER:
199	/*	195	/*
200	* The page fault came during copy_from_uspace()	196	* The page fault came during copy_from_uspace()
201	* or copy_to_uspace().	197	* or copy_to_uspace().
202	*/	198	*/
203	page_table_unlock(AS, true);	199	page_table_unlock(AS, true);
204	return;	200	return;
205	default:	201	default:
206	panic("unexpected pfrc (%d)\n", pfrc);	202	panic("unexpected pfrc (%d)\n", pfrc);
207	}	203	}
208	}	204	}
209		205
210	/*	206	/*
211	* Read the faulting TLB entry.	207	* Read the faulting TLB entry.
212	*/	208	*/
213	tlbr();	209	tlbr();
214		210
215	/*	211	/*
216	* Record access to PTE.	212	* Record access to PTE.
217	*/	213	*/
218	pte->a = 1;	214	pte->a = 1;
219		215
220	tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);	216	tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
-		217	pte->pfn);
221		218
222	/*	219	/*
223	* The entry is to be updated in TLB.	220	* The entry is to be updated in TLB.
224	*/	221	*/
225	if ((badvaddr/PAGE_SIZE) % 2 == 0)	222	if ((badvaddr / PAGE_SIZE) % 2 == 0)
226	cp0_entry_lo0_write(lo.value);	223	cp0_entry_lo0_write(lo.value);
227	else	224	else
228	cp0_entry_lo1_write(lo.value);	225	cp0_entry_lo1_write(lo.value);
229	cp0_pagemask_write(TLB_PAGE_MASK_16K);	226	cp0_pagemask_write(TLB_PAGE_MASK_16K);
230	tlbwi();	227	tlbwi();
231		228
232	page_table_unlock(AS, true);	229	page_table_unlock(AS, true);
233	return;	230	return;
234		231
235	fail:	232	fail:
236	page_table_unlock(AS, true);	233	page_table_unlock(AS, true);
237	tlb_invalid_fail(istate);	234	tlb_invalid_fail(istate);
238	}	235	}
239		236
240	/** Process TLB Modified Exception	237	/** Process TLB Modified Exception.
241	*	-
242	* Process TLB Modified Exception.	-
243	*	238	*
244	* @param istate Interrupted register context.	239	* @param istate Interrupted register context.
245	*/	240	*/
246	void tlb_modified(istate_t *istate)	241	void tlb_modified(istate_t *istate)
247	{	242	{
248	tlb_index_t index;	243	tlb_index_t index;
249	uintptr_t badvaddr;	244	uintptr_t badvaddr;
250	entry_lo_t lo;	245	entry_lo_t lo;
251	entry_hi_t hi;	246	entry_hi_t hi;
252	pte_t *pte;	247	pte_t *pte;
253	int pfrc;	248	int pfrc;
254		249
255	badvaddr = cp0_badvaddr_read();	250	badvaddr = cp0_badvaddr_read();
256		251
257	/*	252	/*
258	* Locate the faulting entry in TLB.	253	* Locate the faulting entry in TLB.
259	*/	254	*/
260	hi.value = cp0_entry_hi_read();	255	hi.value = cp0_entry_hi_read();
261	tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);	256	tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
262	cp0_entry_hi_write(hi.value);	257	cp0_entry_hi_write(hi.value);
263	tlbp();	258	tlbp();
264	index.value = cp0_index_read();	259	index.value = cp0_index_read();
265		260
266	page_table_lock(AS, true);	261	page_table_lock(AS, true);
267		262
268	/*	263	/*
269	* Fail if the entry is not in TLB.	264	* Fail if the entry is not in TLB.
270	*/	265	*/
271	if (index.p) {	266	if (index.p) {
272	printf("TLB entry not found.\n");	267	printf("TLB entry not found.\n");
273	goto fail;	268	goto fail;
274	}	269	}
275		270
276	pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);	271	pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
277	if (!pte) {	272	if (!pte) {
278	switch (pfrc) {	273	switch (pfrc) {
279	case AS_PF_FAULT:	274	case AS_PF_FAULT:
280	goto fail;	275	goto fail;
281	break;	276	break;
282	case AS_PF_DEFER:	277	case AS_PF_DEFER:
283	/*	278	/*
284	* The page fault came during copy_from_uspace()	279	* The page fault came during copy_from_uspace()
285	* or copy_to_uspace().	280	* or copy_to_uspace().
286	*/	281	*/
287	page_table_unlock(AS, true);	282	page_table_unlock(AS, true);
288	return;	283	return;
289	default:	284	default:
290	panic("unexpected pfrc (%d)\n", pfrc);	285	panic("unexpected pfrc (%d)\n", pfrc);
291	}	286	}
292	}	287	}
293		288
294	/*	289	/*
295	* Fail if the page is not writable.	290	* Fail if the page is not writable.
296	*/	291	*/
297	if (!pte->w)	292	if (!pte->w)
298	goto fail;	293	goto fail;
299		294
300	/*	295	/*
301	* Read the faulting TLB entry.	296	* Read the faulting TLB entry.
302	*/	297	*/
303	tlbr();	298	tlbr();
304		299
305	/*	300	/*
306	* Record access and write to PTE.	301	* Record access and write to PTE.
307	*/	302	*/
308	pte->a = 1;	303	pte->a = 1;
309	pte->d = 1;	304	pte->d = 1;
310		305
311	tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);	306	tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable,
-		307	pte->pfn);
312		308
313	/*	309	/*
314	* The entry is to be updated in TLB.	310	* The entry is to be updated in TLB.
315	*/	311	*/
316	if ((badvaddr/PAGE_SIZE) % 2 == 0)	312	if ((badvaddr / PAGE_SIZE) % 2 == 0)
317	cp0_entry_lo0_write(lo.value);	313	cp0_entry_lo0_write(lo.value);
318	else	314	else
319	cp0_entry_lo1_write(lo.value);	315	cp0_entry_lo1_write(lo.value);
320	cp0_pagemask_write(TLB_PAGE_MASK_16K);	316	cp0_pagemask_write(TLB_PAGE_MASK_16K);
321	tlbwi();	317	tlbwi();
322		318
323	page_table_unlock(AS, true);	319	page_table_unlock(AS, true);
324	return;	320	return;
325		321
326	fail:	322	fail:
327	page_table_unlock(AS, true);	323	page_table_unlock(AS, true);
328	tlb_modified_fail(istate);	324	tlb_modified_fail(istate);
329	}	325	}
330		326
331	void tlb_refill_fail(istate_t *istate)	327	void tlb_refill_fail(istate_t *istate)
332	{	328	{
333	char *symbol = "";	329	char *symbol = "";
334	char *sym2 = "";	330	char *sym2 = "";
335		331
336	char *s = get_symtab_entry(istate->epc);	332	char *s = get_symtab_entry(istate->epc);
337	if (s)	333	if (s)
338	symbol = s;	334	symbol = s;
339	s = get_symtab_entry(istate->ra);	335	s = get_symtab_entry(istate->ra);
340	if (s)	336	if (s)
341	sym2 = s;	337	sym2 = s;
342		338
343	fault_if_from_uspace(istate, "TLB Refill Exception on %p", cp0_badvaddr_read());	339	fault_if_from_uspace(istate, "TLB Refill Exception on %p",
-		340	cp0_badvaddr_read());
344	panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);	341	panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(),
-		342	istate->epc, symbol, sym2);
345	}	343	}
346		344
347		345
348	void tlb_invalid_fail(istate_t *istate)	346	void tlb_invalid_fail(istate_t *istate)
349	{	347	{
350	char *symbol = "";	348	char *symbol = "";
351		349
352	char *s = get_symtab_entry(istate->epc);	350	char *s = get_symtab_entry(istate->epc);
353	if (s)	351	if (s)
354	symbol = s;	352	symbol = s;
355	fault_if_from_uspace(istate, "TLB Invalid Exception on %p", cp0_badvaddr_read());	353	fault_if_from_uspace(istate, "TLB Invalid Exception on %p",
-		354	cp0_badvaddr_read());
356	panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);	355	panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(),
-		356	istate->epc, symbol);
357	}	357	}
358		358
359	void tlb_modified_fail(istate_t *istate)	359	void tlb_modified_fail(istate_t *istate)
360	{	360	{
361	char *symbol = "";	361	char *symbol = "";
362		362
363	char *s = get_symtab_entry(istate->epc);	363	char *s = get_symtab_entry(istate->epc);
364	if (s)	364	if (s)
365	symbol = s;	365	symbol = s;
366	fault_if_from_uspace(istate, "TLB Modified Exception on %p", cp0_badvaddr_read());	366	fault_if_from_uspace(istate, "TLB Modified Exception on %p",
-		367	cp0_badvaddr_read());
367	panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);	368	panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(),
-		369	istate->epc, symbol);
368	}	370	}
369		371
370	/** Try to find PTE for faulting address	372	/** Try to find PTE for faulting address.
371	*	373	*
372	* Try to find PTE for faulting address.	-
373	* The AS->lock must be held on entry to this function.	374	* The AS->lock must be held on entry to this function.
374	*	375	*
375	* @param badvaddr Faulting virtual address.	376	* @param badvaddr Faulting virtual address.
376	* @param access Access mode that caused the fault.	377	* @param access Access mode that caused the fault.
377	* @param istate Pointer to interrupted state.	378	* @param istate Pointer to interrupted state.
378	* @param pfrc Pointer to variable where as_page_fault() return code will be stored.	379	* @param pfrc Pointer to variable where as_page_fault() return code
-		380	* will be stored.
379	*	381	*
380	* @return PTE on success, NULL otherwise.	382	* @return PTE on success, NULL otherwise.
381	*/	383	*/
-		384	pte_t *
382	pte_t find_mapping_and_check(uintptr_t badvaddr, int access, istate_t istate, int *pfrc)	385	find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate,
-		386	int *pfrc)
383	{	387	{
384	entry_hi_t hi;	388	entry_hi_t hi;
385	pte_t *pte;	389	pte_t *pte;
386		390
387	hi.value = cp0_entry_hi_read();	391	hi.value = cp0_entry_hi_read();
388		392
389	/*	393	/*
390	* Handler cannot succeed if the ASIDs don't match.	394	* Handler cannot succeed if the ASIDs don't match.
391	*/	395	*/
392	if (hi.asid != AS->asid) {	396	if (hi.asid != AS->asid) {
393	printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);	397	printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
394	return NULL;	398	return NULL;
395	}	399	}
396		400
397	/*	401	/*
398	* Check if the mapping exists in page tables.	402	* Check if the mapping exists in page tables.
399	*/	403	*/
400	pte = page_mapping_find(AS, badvaddr);	404	pte = page_mapping_find(AS, badvaddr);
401	if (pte && pte->p) {	405	if (pte && pte->p) {
402	/*	406	/*
403	* Mapping found in page tables.	407	* Mapping found in page tables.
404	* Immediately succeed.	408	* Immediately succeed.
405	*/	409	*/
406	return pte;	410	return pte;
407	} else {	411	} else {
408	int rc;	412	int rc;
409		413
410	/*	414	/*
411	* Mapping not found in page tables.	415	* Mapping not found in page tables.
412	* Resort to higher-level page fault handler.	416	* Resort to higher-level page fault handler.
413	*/	417	*/
414	page_table_unlock(AS, true);	418	page_table_unlock(AS, true);
415	switch (rc = as_page_fault(badvaddr, access, istate)) {	419	switch (rc = as_page_fault(badvaddr, access, istate)) {
416	case AS_PF_OK:	420	case AS_PF_OK:
417	/*	421	/*
418	* The higher-level page fault handler succeeded,	422	* The higher-level page fault handler succeeded,
419	* The mapping ought to be in place.	423	* The mapping ought to be in place.
420	*/	424	*/
421	page_table_lock(AS, true);	425	page_table_lock(AS, true);
422	pte = page_mapping_find(AS, badvaddr);	426	pte = page_mapping_find(AS, badvaddr);
423	ASSERT(pte && pte->p);	427	ASSERT(pte && pte->p);
424	return pte;	428	return pte;
425	break;	429	break;
426	case AS_PF_DEFER:	430	case AS_PF_DEFER:
427	page_table_lock(AS, true);	431	page_table_lock(AS, true);
428	*pfrc = AS_PF_DEFER;	432	*pfrc = AS_PF_DEFER;
429	return NULL;	433	return NULL;
430	break;	434	break;
431	case AS_PF_FAULT:	435	case AS_PF_FAULT:
432	page_table_lock(AS, true);	436	page_table_lock(AS, true);
433	printf("Page fault.\n");	437	printf("Page fault.\n");
434	*pfrc = AS_PF_FAULT;	438	*pfrc = AS_PF_FAULT;
435	return NULL;	439	return NULL;
436	break;	440	break;
437	default:	441	default:
438	panic("unexpected rc (%d)\n", rc);	442	panic("unexpected rc (%d)\n", rc);
439	}	443	}
440		444
441	}	445	}
442	}	446	}
443		447
-		448	void
444	void tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn)	449	tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable,
-		450	uintptr_t pfn)
445	{	451	{
446	lo->value = 0;	452	lo->value = 0;
447	lo->g = g;	453	lo->g = g;
448	lo->v = v;	454	lo->v = v;
449	lo->d = d;	455	lo->d = d;
450	lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;	456	lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
451	lo->pfn = pfn;	457	lo->pfn = pfn;
452	}	458	}
453		459
454	void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)	460	void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
455	{	461	{
456	hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);	462	hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
457	hi->asid = asid;	463	hi->asid = asid;
458	}	464	}
459		465
460	/** Print contents of TLB. */	466	/** Print contents of TLB. */
461	void tlb_print(void)	467	void tlb_print(void)
462	{	468	{
463	page_mask_t mask;	469	page_mask_t mask;
464	entry_lo_t lo0, lo1;	470	entry_lo_t lo0, lo1;
465	entry_hi_t hi, hi_save;	471	entry_hi_t hi, hi_save;
466	unsigned int i;	472	unsigned int i;
467		473
468	hi_save.value = cp0_entry_hi_read();	474	hi_save.value = cp0_entry_hi_read();
469		475
470	printf("# ASID VPN2 MASK G V D C PFN\n");	476	printf("# ASID VPN2 MASK G V D C PFN\n");
471	printf("-- ---- ------ ---- - - - - ------\n");	477	printf("-- ---- ------ ---- - - - - ------\n");
472		478
473	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	479	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
474	cp0_index_write(i);	480	cp0_index_write(i);
475	tlbr();	481	tlbr();
476		482
477	mask.value = cp0_pagemask_read();	483	mask.value = cp0_pagemask_read();
478	hi.value = cp0_entry_hi_read();	484	hi.value = cp0_entry_hi_read();
479	lo0.value = cp0_entry_lo0_read();	485	lo0.value = cp0_entry_lo0_read();
480	lo1.value = cp0_entry_lo1_read();	486	lo1.value = cp0_entry_lo1_read();
481		487
482	printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",	488	printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
483	i, hi.asid, hi.vpn2, mask.mask,	489	i, hi.asid, hi.vpn2, mask.mask,
484	lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);	490	lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
485	printf(" %1u %1u %1u %1u %#6x\n",	491	printf(" %1u %1u %1u %1u %#6x\n",
486	lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);	492	lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);

Subversion Repositories HelenOS

(root)/trunk/kernel/arch/mips32/src/mm/tlb.c – Rev 3228 → 3392