WebSVN – HelenOS-historic – Diff – /kernel/trunk/arch/mips32/src/mm/tlb.c


/*
 * Copyright (C) 2003-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <arch/mm/tlb.h>
#include <mm/asid.h>
#include <mm/tlb.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/cp0.h>
#include <panic.h>
#include <arch.h>
#include <symtab.h>
#include <synch/spinlock.h>
#include <print.h>
#include <debug.h>
#include <align.h>
 
static void tlb_refill_fail(istate_t *istate);
static void tlb_invalid_fail(istate_t *istate);
static void tlb_modified_fail(istate_t *istate);
 
static pte_t *find_mapping_and_check(__address badvaddr);
 
static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
 
/** Initialize TLB
 *
 * Initialize TLB.
 * Invalidate all entries and mark wired entries.
 */
void tlb_arch_init(void)
{
    int i;
 
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    cp0_entry_hi_write(0);
    cp0_entry_lo0_write(0);
    cp0_entry_lo1_write(0);
 
    /* Clear and initialize TLB. */
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbwi();
    }
 
       
    /*
     * The kernel is going to make use of some wired
     * entries (e.g. mapping kernel stacks in kseg3).
     */
    cp0_wired_write(TLB_WIRED);
}
 
/** Process TLB Refill Exception
 *
 * Process TLB Refill Exception.
 *
 * @param istate Interrupted register context.
 */
void tlb_refill(istate_t *istate)
{
    entry_lo_t lo;
    entry_hi_t hi; 
    __address badvaddr;
    pte_t *pte;
 
    badvaddr = cp0_badvaddr_read();
 
    spinlock_lock(&AS->lock);      
 
    pte = find_mapping_and_check(badvaddr);
    if (!pte)
        goto fail;
 
    /*
     * Record access to PTE.
     */
    pte->a = 1;
 
    prepare_entry_hi(&hi, AS->asid, badvaddr);
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
 
    /*
     * New entry is to be inserted into TLB
     */
    cp0_entry_hi_write(hi.value);
    if ((badvaddr/PAGE_SIZE) % 2 == 0) {
        cp0_entry_lo0_write(lo.value);
        cp0_entry_lo1_write(0);
    }
    else {
        cp0_entry_lo0_write(0);
        cp0_entry_lo1_write(lo.value);
    }
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwr();
 
    spinlock_unlock(&AS->lock);
    return;
   
fail:
    spinlock_unlock(&AS->lock);
    tlb_refill_fail(istate);
}
 
/** Process TLB Invalid Exception
 *
 * Process TLB Invalid Exception.
 *
 * @param istate Interrupted register context.
 */
void tlb_invalid(istate_t *istate)
{
    tlb_index_t index;
    __address badvaddr;
    entry_lo_t lo;
    entry_hi_t hi;
    pte_t *pte;
 
    badvaddr = cp0_badvaddr_read();
 
    /*
     * Locate the faulting entry in TLB.
     */
    hi.value = cp0_entry_hi_read();
    prepare_entry_hi(&hi, hi.asid, badvaddr);
    cp0_entry_hi_write(hi.value);
    tlbp();
    index.value = cp0_index_read();
   
    spinlock_lock(&AS->lock);  
   
    /*
     * Fail if the entry is not in TLB.
     */
    if (index.p) {
        printf("TLB entry not found.\n");
        goto fail;
    }
 
    pte = find_mapping_and_check(badvaddr);
    if (!pte)
        goto fail;
 
    /*
     * Read the faulting TLB entry.
     */
    tlbr();
 
    /*
     * Record access to PTE.
     */
    pte->a = 1;
 
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
 
    /*
     * The entry is to be updated in TLB.
     */
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
        cp0_entry_lo0_write(lo.value);
    else
        cp0_entry_lo1_write(lo.value);
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwi();
 
    spinlock_unlock(&AS->lock);
    return;
   
fail:
    spinlock_unlock(&AS->lock);
    tlb_invalid_fail(istate);
}
 
/** Process TLB Modified Exception
 *
 * Process TLB Modified Exception.
 *
 * @param istate Interrupted register context.
 */
void tlb_modified(istate_t *istate)
{
    tlb_index_t index;
    __address badvaddr;
    entry_lo_t lo;
    entry_hi_t hi;
    pte_t *pte;
 
    badvaddr = cp0_badvaddr_read();
 
    /*
     * Locate the faulting entry in TLB.
     */
    hi.value = cp0_entry_hi_read();
    prepare_entry_hi(&hi, hi.asid, badvaddr);
    cp0_entry_hi_write(hi.value);
    tlbp();
    index.value = cp0_index_read();
   
    spinlock_lock(&AS->lock);  
   
    /*
     * Fail if the entry is not in TLB.
     */
    if (index.p) {
        printf("TLB entry not found.\n");
        goto fail;
    }
 
    pte = find_mapping_and_check(badvaddr);
    if (!pte)
        goto fail;
 
    /*
     * Fail if the page is not writable.
     */
    if (!pte->w)
        goto fail;
 
    /*
     * Read the faulting TLB entry.
     */
    tlbr();
 
    /*
     * Record access and write to PTE.
     */
    pte->a = 1;
    pte->d = 1;
 
    prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
 
    /*
     * The entry is to be updated in TLB.
     */
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
        cp0_entry_lo0_write(lo.value);
    else
        cp0_entry_lo1_write(lo.value);
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwi();
 
    spinlock_unlock(&AS->lock);
    return;
   
fail:
    spinlock_unlock(&AS->lock);
    tlb_modified_fail(istate);
}
 
void tlb_refill_fail(istate_t *istate)
{
    char *symbol = "";
    char *sym2 = "";
 
    char *s = get_symtab_entry(istate->epc);
    if (s)
        symbol = s;
    s = get_symtab_entry(istate->ra);
    if (s)
        sym2 = s;
    panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
}
 
 
void tlb_invalid_fail(istate_t *istate)
{
    char *symbol = "";
 
    char *s = get_symtab_entry(istate->epc);
    if (s)
        symbol = s;
    panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
}
 
void tlb_modified_fail(istate_t *istate)
{
    char *symbol = "";
 
    char *s = get_symtab_entry(istate->epc);
    if (s)
        symbol = s;
    panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
}
 
/** Try to find PTE for faulting address
 *
 * Try to find PTE for faulting address.
 * The AS->lock must be held on entry to this function.
 *
 * @param badvaddr Faulting virtual address.
 *
 * @return PTE on success, NULL otherwise.
 */
pte_t *find_mapping_and_check(__address badvaddr)
{
    entry_hi_t hi;
    pte_t *pte;
 
    hi.value = cp0_entry_hi_read();
 
    /*
     * Handler cannot succeed if the ASIDs don't match.
     */
    if (hi.asid != AS->asid) {
        printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
        return NULL;
    }
 
    /*
     * Check if the mapping exists in page tables.
     */
    pte = page_mapping_find(AS, badvaddr);
    if (pte && pte->p) {
        /*
         * Mapping found in page tables.
         * Immediately succeed.
         */
        return pte;
    } else {
        /*
         * Mapping not found in page tables.
         * Resort to higher-level page fault handler.
         */
        if (as_page_fault(badvaddr)) {
            /*
             * The higher-level page fault handler succeeded,
             * The mapping ought to be in place.
             */
            pte = page_mapping_find(AS, badvaddr);
            ASSERT(pte && pte->p);
            return pte;
        }
    }
 
    /*
     * Handler cannot succeed if badvaddr has no mapping.
     */
    if (!pte) {
        printf("No such mapping.\n");
        return NULL;
    }
 
    /*
     * Handler cannot succeed if the mapping is marked as invalid.
     */
    if (!pte->p) {
        printf("Invalid mapping.\n");
        return NULL;
    }
 
    return pte;
}
 
void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
{
    lo->value = 0;
    lo->g = g;
    lo->v = v;
    lo->d = d;
    lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
    lo->pfn = pfn;
}
 
void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
{
    hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
    hi->asid = asid;
}
 
/** Print contents of TLB. */
void tlb_print(void)
{
    page_mask_t mask;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    int i;
 
    hi_save.value = cp0_entry_hi_read();
 
    printf("TLB:\n");
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
       
        mask.value = cp0_pagemask_read();
        hi.value = cp0_entry_hi_read();
        lo0.value = cp0_entry_lo0_read();
        lo1.value = cp0_entry_lo1_read();
       
        printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
               "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
               i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
               lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
    }
   
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate all not wired TLB entries. */
void tlb_invalidate_all(void)
{
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi_save;
    int i;
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
 
    for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
 
        lo0.value = cp0_entry_lo0_read();
        lo1.value = cp0_entry_lo1_read();
 
        lo0.v = 0;
        lo1.v = 0;
 
        cp0_entry_lo0_write(lo0.value);
        cp0_entry_lo1_write(lo1.value);
               
        tlbwi();
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate all TLB entries belonging to specified address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    int i;
 
    ASSERT(asid != ASID_INVALID);
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
       
        hi.value = cp0_entry_hi_read();
       
        if (hi.asid == asid) {
            lo0.value = cp0_entry_lo0_read();
            lo1.value = cp0_entry_lo1_read();
 
            lo0.v = 0;
            lo1.v = 0;
 
            cp0_entry_lo0_write(lo0.value);
            cp0_entry_lo1_write(lo1.value);
 
            tlbwi();
        }
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate TLB entries for specified page range belonging to specified address space.
 *
 * @param asid Address space identifier.
 * @param page First page whose TLB entry is to be invalidated.
 * @param cnt Number of entries to invalidate.
 */
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
{
    int i;
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    tlb_index_t index;
 
    ASSERT(asid != ASID_INVALID);
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
 
    for (i = 0; i < cnt+1; i+=2) {
        hi.value = 0;
        prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
        cp0_entry_hi_write(hi.value);
 
        tlbp();
        index.value = cp0_index_read();
 
        if (!index.p) {
            /* Entry was found, index register contains valid index. */
            tlbr();
 
            lo0.value = cp0_entry_lo0_read();
            lo1.value = cp0_entry_lo1_read();
 
            lo0.v = 0;
            lo1.v = 0;
 
            cp0_entry_lo0_write(lo0.value);
            cp0_entry_lo1_write(lo1.value);
 
            tlbwi();
        }
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 

Rev 958	Rev 983
1	/*	1	/*
2	* Copyright (C) 2003-2004 Jakub Jermar	2	* Copyright (C) 2003-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <arch/mm/tlb.h>	29	#include <arch/mm/tlb.h>
30	#include <mm/asid.h>	30	#include <mm/asid.h>
31	#include <mm/tlb.h>	31	#include <mm/tlb.h>
32	#include <mm/page.h>	32	#include <mm/page.h>
33	#include <mm/as.h>	33	#include <mm/as.h>
34	#include <arch/cp0.h>	34	#include <arch/cp0.h>
35	#include <panic.h>	35	#include <panic.h>
36	#include <arch.h>	36	#include <arch.h>
37	#include <symtab.h>	37	#include <symtab.h>
38	#include <synch/spinlock.h>	38	#include <synch/spinlock.h>
39	#include <print.h>	39	#include <print.h>
40	#include <debug.h>	40	#include <debug.h>
-		41	#include <align.h>
41		42
42	static void tlb_refill_fail(istate_t *istate);	43	static void tlb_refill_fail(istate_t *istate);
43	static void tlb_invalid_fail(istate_t *istate);	44	static void tlb_invalid_fail(istate_t *istate);
44	static void tlb_modified_fail(istate_t *istate);	45	static void tlb_modified_fail(istate_t *istate);
45		46
46	static pte_t *find_mapping_and_check(__address badvaddr);	47	static pte_t *find_mapping_and_check(__address badvaddr);
47		48
48	static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);	49	static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
49	static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);	50	static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
50		51
51	/** Initialize TLB	52	/** Initialize TLB
52	*	53	*
53	* Initialize TLB.	54	* Initialize TLB.
54	* Invalidate all entries and mark wired entries.	55	* Invalidate all entries and mark wired entries.
55	*/	56	*/
56	void tlb_arch_init(void)	57	void tlb_arch_init(void)
57	{	58	{
58	int i;	59	int i;
59		60
60	cp0_pagemask_write(TLB_PAGE_MASK_16K);	61	cp0_pagemask_write(TLB_PAGE_MASK_16K);
61	cp0_entry_hi_write(0);	62	cp0_entry_hi_write(0);
62	cp0_entry_lo0_write(0);	63	cp0_entry_lo0_write(0);
63	cp0_entry_lo1_write(0);	64	cp0_entry_lo1_write(0);
64		65
65	/* Clear and initialize TLB. */	66	/* Clear and initialize TLB. */
66		67
67	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	68	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
68	cp0_index_write(i);	69	cp0_index_write(i);
69	tlbwi();	70	tlbwi();
70	}	71	}
71		72
72		73
73	/*	74	/*
74	* The kernel is going to make use of some wired	75	* The kernel is going to make use of some wired
75	* entries (e.g. mapping kernel stacks in kseg3).	76	* entries (e.g. mapping kernel stacks in kseg3).
76	*/	77	*/
77	cp0_wired_write(TLB_WIRED);	78	cp0_wired_write(TLB_WIRED);
78	}	79	}
79		80
80	/** Process TLB Refill Exception	81	/** Process TLB Refill Exception
81	*	82	*
82	* Process TLB Refill Exception.	83	* Process TLB Refill Exception.
83	*	84	*
84	* @param istate Interrupted register context.	85	* @param istate Interrupted register context.
85	*/	86	*/
86	void tlb_refill(istate_t *istate)	87	void tlb_refill(istate_t *istate)
87	{	88	{
88	entry_lo_t lo;	89	entry_lo_t lo;
89	entry_hi_t hi;	90	entry_hi_t hi;
90	__address badvaddr;	91	__address badvaddr;
91	pte_t *pte;	92	pte_t *pte;
92		93
93	badvaddr = cp0_badvaddr_read();	94	badvaddr = cp0_badvaddr_read();
94		95
95	spinlock_lock(&AS->lock);	96	spinlock_lock(&AS->lock);
96		97
97	pte = find_mapping_and_check(badvaddr);	98	pte = find_mapping_and_check(badvaddr);
98	if (!pte)	99	if (!pte)
99	goto fail;	100	goto fail;
100		101
101	/*	102	/*
102	* Record access to PTE.	103	* Record access to PTE.
103	*/	104	*/
104	pte->a = 1;	105	pte->a = 1;
105		106
106	prepare_entry_hi(&hi, AS->asid, badvaddr);	107	prepare_entry_hi(&hi, AS->asid, badvaddr);
107	prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);	108	prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
108		109
109	/*	110	/*
110	* New entry is to be inserted into TLB	111	* New entry is to be inserted into TLB
111	*/	112	*/
112	cp0_entry_hi_write(hi.value);	113	cp0_entry_hi_write(hi.value);
113	if ((badvaddr/PAGE_SIZE) % 2 == 0) {	114	if ((badvaddr/PAGE_SIZE) % 2 == 0) {
114	cp0_entry_lo0_write(lo.value);	115	cp0_entry_lo0_write(lo.value);
115	cp0_entry_lo1_write(0);	116	cp0_entry_lo1_write(0);
116	}	117	}
117	else {	118	else {
118	cp0_entry_lo0_write(0);	119	cp0_entry_lo0_write(0);
119	cp0_entry_lo1_write(lo.value);	120	cp0_entry_lo1_write(lo.value);
120	}	121	}
121	cp0_pagemask_write(TLB_PAGE_MASK_16K);	122	cp0_pagemask_write(TLB_PAGE_MASK_16K);
122	tlbwr();	123	tlbwr();
123		124
124	spinlock_unlock(&AS->lock);	125	spinlock_unlock(&AS->lock);
125	return;	126	return;
126		127
127	fail:	128	fail:
128	spinlock_unlock(&AS->lock);	129	spinlock_unlock(&AS->lock);
129	tlb_refill_fail(istate);	130	tlb_refill_fail(istate);
130	}	131	}
131		132
132	/** Process TLB Invalid Exception	133	/** Process TLB Invalid Exception
133	*	134	*
134	* Process TLB Invalid Exception.	135	* Process TLB Invalid Exception.
135	*	136	*
136	* @param istate Interrupted register context.	137	* @param istate Interrupted register context.
137	*/	138	*/
138	void tlb_invalid(istate_t *istate)	139	void tlb_invalid(istate_t *istate)
139	{	140	{
140	tlb_index_t index;	141	tlb_index_t index;
141	__address badvaddr;	142	__address badvaddr;
142	entry_lo_t lo;	143	entry_lo_t lo;
143	entry_hi_t hi;	144	entry_hi_t hi;
144	pte_t *pte;	145	pte_t *pte;
145		146
146	badvaddr = cp0_badvaddr_read();	147	badvaddr = cp0_badvaddr_read();
147		148
148	/*	149	/*
149	* Locate the faulting entry in TLB.	150	* Locate the faulting entry in TLB.
150	*/	151	*/
151	hi.value = cp0_entry_hi_read();	152	hi.value = cp0_entry_hi_read();
152	prepare_entry_hi(&hi, hi.asid, badvaddr);	153	prepare_entry_hi(&hi, hi.asid, badvaddr);
153	cp0_entry_hi_write(hi.value);	154	cp0_entry_hi_write(hi.value);
154	tlbp();	155	tlbp();
155	index.value = cp0_index_read();	156	index.value = cp0_index_read();
156		157
157	spinlock_lock(&AS->lock);	158	spinlock_lock(&AS->lock);
158		159
159	/*	160	/*
160	* Fail if the entry is not in TLB.	161	* Fail if the entry is not in TLB.
161	*/	162	*/
162	if (index.p) {	163	if (index.p) {
163	printf("TLB entry not found.\n");	164	printf("TLB entry not found.\n");
164	goto fail;	165	goto fail;
165	}	166	}
166		167
167	pte = find_mapping_and_check(badvaddr);	168	pte = find_mapping_and_check(badvaddr);
168	if (!pte)	169	if (!pte)
169	goto fail;	170	goto fail;
170		171
171	/*	172	/*
172	* Read the faulting TLB entry.	173	* Read the faulting TLB entry.
173	*/	174	*/
174	tlbr();	175	tlbr();
175		176
176	/*	177	/*
177	* Record access to PTE.	178	* Record access to PTE.
178	*/	179	*/
179	pte->a = 1;	180	pte->a = 1;
180		181
181	prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);	182	prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
182		183
183	/*	184	/*
184	* The entry is to be updated in TLB.	185	* The entry is to be updated in TLB.
185	*/	186	*/
186	if ((badvaddr/PAGE_SIZE) % 2 == 0)	187	if ((badvaddr/PAGE_SIZE) % 2 == 0)
187	cp0_entry_lo0_write(lo.value);	188	cp0_entry_lo0_write(lo.value);
188	else	189	else
189	cp0_entry_lo1_write(lo.value);	190	cp0_entry_lo1_write(lo.value);
190	cp0_pagemask_write(TLB_PAGE_MASK_16K);	191	cp0_pagemask_write(TLB_PAGE_MASK_16K);
191	tlbwi();	192	tlbwi();
192		193
193	spinlock_unlock(&AS->lock);	194	spinlock_unlock(&AS->lock);
194	return;	195	return;
195		196
196	fail:	197	fail:
197	spinlock_unlock(&AS->lock);	198	spinlock_unlock(&AS->lock);
198	tlb_invalid_fail(istate);	199	tlb_invalid_fail(istate);
199	}	200	}
200		201
201	/** Process TLB Modified Exception	202	/** Process TLB Modified Exception
202	*	203	*
203	* Process TLB Modified Exception.	204	* Process TLB Modified Exception.
204	*	205	*
205	* @param istate Interrupted register context.	206	* @param istate Interrupted register context.
206	*/	207	*/
207	void tlb_modified(istate_t *istate)	208	void tlb_modified(istate_t *istate)
208	{	209	{
209	tlb_index_t index;	210	tlb_index_t index;
210	__address badvaddr;	211	__address badvaddr;
211	entry_lo_t lo;	212	entry_lo_t lo;
212	entry_hi_t hi;	213	entry_hi_t hi;
213	pte_t *pte;	214	pte_t *pte;
214		215
215	badvaddr = cp0_badvaddr_read();	216	badvaddr = cp0_badvaddr_read();
216		217
217	/*	218	/*
218	* Locate the faulting entry in TLB.	219	* Locate the faulting entry in TLB.
219	*/	220	*/
220	hi.value = cp0_entry_hi_read();	221	hi.value = cp0_entry_hi_read();
221	prepare_entry_hi(&hi, hi.asid, badvaddr);	222	prepare_entry_hi(&hi, hi.asid, badvaddr);
222	cp0_entry_hi_write(hi.value);	223	cp0_entry_hi_write(hi.value);
223	tlbp();	224	tlbp();
224	index.value = cp0_index_read();	225	index.value = cp0_index_read();
225		226
226	spinlock_lock(&AS->lock);	227	spinlock_lock(&AS->lock);
227		228
228	/*	229	/*
229	* Fail if the entry is not in TLB.	230	* Fail if the entry is not in TLB.
230	*/	231	*/
231	if (index.p) {	232	if (index.p) {
232	printf("TLB entry not found.\n");	233	printf("TLB entry not found.\n");
233	goto fail;	234	goto fail;
234	}	235	}
235		236
236	pte = find_mapping_and_check(badvaddr);	237	pte = find_mapping_and_check(badvaddr);
237	if (!pte)	238	if (!pte)
238	goto fail;	239	goto fail;
239		240
240	/*	241	/*
241	* Fail if the page is not writable.	242	* Fail if the page is not writable.
242	*/	243	*/
243	if (!pte->w)	244	if (!pte->w)
244	goto fail;	245	goto fail;
245		246
246	/*	247	/*
247	* Read the faulting TLB entry.	248	* Read the faulting TLB entry.
248	*/	249	*/
249	tlbr();	250	tlbr();
250		251
251	/*	252	/*
252	* Record access and write to PTE.	253	* Record access and write to PTE.
253	*/	254	*/
254	pte->a = 1;	255	pte->a = 1;
255	pte->d = 1;	256	pte->d = 1;
256		257
257	prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);	258	prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
258		259
259	/*	260	/*
260	* The entry is to be updated in TLB.	261	* The entry is to be updated in TLB.
261	*/	262	*/
262	if ((badvaddr/PAGE_SIZE) % 2 == 0)	263	if ((badvaddr/PAGE_SIZE) % 2 == 0)
263	cp0_entry_lo0_write(lo.value);	264	cp0_entry_lo0_write(lo.value);
264	else	265	else
265	cp0_entry_lo1_write(lo.value);	266	cp0_entry_lo1_write(lo.value);
266	cp0_pagemask_write(TLB_PAGE_MASK_16K);	267	cp0_pagemask_write(TLB_PAGE_MASK_16K);
267	tlbwi();	268	tlbwi();
268		269
269	spinlock_unlock(&AS->lock);	270	spinlock_unlock(&AS->lock);
270	return;	271	return;
271		272
272	fail:	273	fail:
273	spinlock_unlock(&AS->lock);	274	spinlock_unlock(&AS->lock);
274	tlb_modified_fail(istate);	275	tlb_modified_fail(istate);
275	}	276	}
276		277
277	void tlb_refill_fail(istate_t *istate)	278	void tlb_refill_fail(istate_t *istate)
278	{	279	{
279	char *symbol = "";	280	char *symbol = "";
280	char *sym2 = "";	281	char *sym2 = "";
281		282
282	char *s = get_symtab_entry(istate->epc);	283	char *s = get_symtab_entry(istate->epc);
283	if (s)	284	if (s)
284	symbol = s;	285	symbol = s;
285	s = get_symtab_entry(istate->ra);	286	s = get_symtab_entry(istate->ra);
286	if (s)	287	if (s)
287	sym2 = s;	288	sym2 = s;
288	panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);	289	panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
289	}	290	}
290		291
291		292
292	void tlb_invalid_fail(istate_t *istate)	293	void tlb_invalid_fail(istate_t *istate)
293	{	294	{
294	char *symbol = "";	295	char *symbol = "";
295		296
296	char *s = get_symtab_entry(istate->epc);	297	char *s = get_symtab_entry(istate->epc);
297	if (s)	298	if (s)
298	symbol = s;	299	symbol = s;
299	panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);	300	panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
300	}	301	}
301		302
302	void tlb_modified_fail(istate_t *istate)	303	void tlb_modified_fail(istate_t *istate)
303	{	304	{
304	char *symbol = "";	305	char *symbol = "";
305		306
306	char *s = get_symtab_entry(istate->epc);	307	char *s = get_symtab_entry(istate->epc);
307	if (s)	308	if (s)
308	symbol = s;	309	symbol = s;
309	panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);	310	panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
310	}	311	}
311		312
312	/** Try to find PTE for faulting address	313	/** Try to find PTE for faulting address
313	*	314	*
314	* Try to find PTE for faulting address.	315	* Try to find PTE for faulting address.
315	* The AS->lock must be held on entry to this function.	316	* The AS->lock must be held on entry to this function.
316	*	317	*
317	* @param badvaddr Faulting virtual address.	318	* @param badvaddr Faulting virtual address.
318	*	319	*
319	* @return PTE on success, NULL otherwise.	320	* @return PTE on success, NULL otherwise.
320	*/	321	*/
321	pte_t *find_mapping_and_check(__address badvaddr)	322	pte_t *find_mapping_and_check(__address badvaddr)
322	{	323	{
323	entry_hi_t hi;	324	entry_hi_t hi;
324	pte_t *pte;	325	pte_t *pte;
325		326
326	hi.value = cp0_entry_hi_read();	327	hi.value = cp0_entry_hi_read();
327		328
328	/*	329	/*
329	* Handler cannot succeed if the ASIDs don't match.	330	* Handler cannot succeed if the ASIDs don't match.
330	*/	331	*/
331	if (hi.asid != AS->asid) {	332	if (hi.asid != AS->asid) {
332	printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);	333	printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
333	return NULL;	334	return NULL;
334	}	335	}
335		336
336	/*	337	/*
337	* Check if the mapping exists in page tables.	338	* Check if the mapping exists in page tables.
338	*/	339	*/
339	pte = page_mapping_find(AS, badvaddr);	340	pte = page_mapping_find(AS, badvaddr);
340	if (pte && pte->p) {	341	if (pte && pte->p) {
341	/*	342	/*
342	* Mapping found in page tables.	343	* Mapping found in page tables.
343	* Immediately succeed.	344	* Immediately succeed.
344	*/	345	*/
345	return pte;	346	return pte;
346	} else {	347	} else {
347	/*	348	/*
348	* Mapping not found in page tables.	349	* Mapping not found in page tables.
349	* Resort to higher-level page fault handler.	350	* Resort to higher-level page fault handler.
350	*/	351	*/
351	if (as_page_fault(badvaddr)) {	352	if (as_page_fault(badvaddr)) {
352	/*	353	/*
353	* The higher-level page fault handler succeeded,	354	* The higher-level page fault handler succeeded,
354	* The mapping ought to be in place.	355	* The mapping ought to be in place.
355	*/	356	*/
356	pte = page_mapping_find(AS, badvaddr);	357	pte = page_mapping_find(AS, badvaddr);
357	ASSERT(pte && pte->p);	358	ASSERT(pte && pte->p);
358	return pte;	359	return pte;
359	}	360	}
360	}	361	}
361		362
362	/*	363	/*
363	* Handler cannot succeed if badvaddr has no mapping.	364	* Handler cannot succeed if badvaddr has no mapping.
364	*/	365	*/
365	if (!pte) {	366	if (!pte) {
366	printf("No such mapping.\n");	367	printf("No such mapping.\n");
367	return NULL;	368	return NULL;
368	}	369	}
369		370
370	/*	371	/*
371	* Handler cannot succeed if the mapping is marked as invalid.	372	* Handler cannot succeed if the mapping is marked as invalid.
372	*/	373	*/
373	if (!pte->p) {	374	if (!pte->p) {
374	printf("Invalid mapping.\n");	375	printf("Invalid mapping.\n");
375	return NULL;	376	return NULL;
376	}	377	}
377		378
378	return pte;	379	return pte;
379	}	380	}
380		381
381	void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)	382	void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
382	{	383	{
383	lo->value = 0;	384	lo->value = 0;
384	lo->g = g;	385	lo->g = g;
385	lo->v = v;	386	lo->v = v;
386	lo->d = d;	387	lo->d = d;
387	lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;	388	lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
388	lo->pfn = pfn;	389	lo->pfn = pfn;
389	}	390	}
390		391
391	void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)	392	void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
392	{	393	{
393	hi->value = (((addr/PAGE_SIZE)/2)PAGE_SIZE2);	394	hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
394	hi->asid = asid;	395	hi->asid = asid;
395	}	396	}
396		397
397	/** Print contents of TLB. */	398	/** Print contents of TLB. */
398	void tlb_print(void)	399	void tlb_print(void)
399	{	400	{
400	page_mask_t mask;	401	page_mask_t mask;
401	entry_lo_t lo0, lo1;	402	entry_lo_t lo0, lo1;
402	entry_hi_t hi, hi_save;	403	entry_hi_t hi, hi_save;
403	int i;	404	int i;
404		405
405	hi_save.value = cp0_entry_hi_read();	406	hi_save.value = cp0_entry_hi_read();
406		407
407	printf("TLB:\n");	408	printf("TLB:\n");
408	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	409	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
409	cp0_index_write(i);	410	cp0_index_write(i);
410	tlbr();	411	tlbr();
411		412
412	mask.value = cp0_pagemask_read();	413	mask.value = cp0_pagemask_read();
413	hi.value = cp0_entry_hi_read();	414	hi.value = cp0_entry_hi_read();
414	lo0.value = cp0_entry_lo0_read();	415	lo0.value = cp0_entry_lo0_read();
415	lo1.value = cp0_entry_lo1_read();	416	lo1.value = cp0_entry_lo1_read();
416		417
417	printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"	418	printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
418	"\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",	419	"\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
419	i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,	420	i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
420	lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);	421	lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
421	}	422	}
422		423
423	cp0_entry_hi_write(hi_save.value);	424	cp0_entry_hi_write(hi_save.value);
424	}	425	}
425		426
426	/** Invalidate all not wired TLB entries. */	427	/** Invalidate all not wired TLB entries. */
427	void tlb_invalidate_all(void)	428	void tlb_invalidate_all(void)
428	{	429	{
429	ipl_t ipl;	430	ipl_t ipl;
430	entry_lo_t lo0, lo1;	431	entry_lo_t lo0, lo1;
431	entry_hi_t hi_save;	432	entry_hi_t hi_save;
432	int i;	433	int i;
433		434
434	hi_save.value = cp0_entry_hi_read();	435	hi_save.value = cp0_entry_hi_read();
435	ipl = interrupts_disable();	436	ipl = interrupts_disable();
436		437
437	for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {	438	for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
438	cp0_index_write(i);	439	cp0_index_write(i);
439	tlbr();	440	tlbr();
440		441
441	lo0.value = cp0_entry_lo0_read();	442	lo0.value = cp0_entry_lo0_read();
442	lo1.value = cp0_entry_lo1_read();	443	lo1.value = cp0_entry_lo1_read();
443		444
444	lo0.v = 0;	445	lo0.v = 0;
445	lo1.v = 0;	446	lo1.v = 0;
446		447
447	cp0_entry_lo0_write(lo0.value);	448	cp0_entry_lo0_write(lo0.value);
448	cp0_entry_lo1_write(lo1.value);	449	cp0_entry_lo1_write(lo1.value);
449		450
450	tlbwi();	451	tlbwi();
451	}	452	}
452		453
453	interrupts_restore(ipl);	454	interrupts_restore(ipl);
454	cp0_entry_hi_write(hi_save.value);	455	cp0_entry_hi_write(hi_save.value);
455	}	456	}
456		457
457	/** Invalidate all TLB entries belonging to specified address space.	458	/** Invalidate all TLB entries belonging to specified address space.
458	*	459	*
459	* @param asid Address space identifier.	460	* @param asid Address space identifier.
460	*/	461	*/
461	void tlb_invalidate_asid(asid_t asid)	462	void tlb_invalidate_asid(asid_t asid)
462	{	463	{
463	ipl_t ipl;	464	ipl_t ipl;
464	entry_lo_t lo0, lo1;	465	entry_lo_t lo0, lo1;
465	entry_hi_t hi, hi_save;	466	entry_hi_t hi, hi_save;
466	int i;	467	int i;
467		468
468	ASSERT(asid != ASID_INVALID);	469	ASSERT(asid != ASID_INVALID);
469		470
470	hi_save.value = cp0_entry_hi_read();	471	hi_save.value = cp0_entry_hi_read();
471	ipl = interrupts_disable();	472	ipl = interrupts_disable();
472		473
473	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	474	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
474	cp0_index_write(i);	475	cp0_index_write(i);
475	tlbr();	476	tlbr();
476		477
477	hi.value = cp0_entry_hi_read();	478	hi.value = cp0_entry_hi_read();
478		479
479	if (hi.asid == asid) {	480	if (hi.asid == asid) {
480	lo0.value = cp0_entry_lo0_read();	481	lo0.value = cp0_entry_lo0_read();
481	lo1.value = cp0_entry_lo1_read();	482	lo1.value = cp0_entry_lo1_read();
482		483
483	lo0.v = 0;	484	lo0.v = 0;
484	lo1.v = 0;	485	lo1.v = 0;
485		486
486	cp0_entry_lo0_write(lo0.value);	487	cp0_entry_lo0_write(lo0.value);
487	cp0_entry_lo1_write(lo1.value);	488	cp0_entry_lo1_write(lo1.value);
488		489
489	tlbwi();	490	tlbwi();
490	}	491	}
491	}	492	}
492		493
493	interrupts_restore(ipl);	494	interrupts_restore(ipl);
494	cp0_entry_hi_write(hi_save.value);	495	cp0_entry_hi_write(hi_save.value);
495	}	496	}
496		497
497	/** Invalidate TLB entries for specified page range belonging to specified address space.	498	/** Invalidate TLB entries for specified page range belonging to specified address space.
498	*	499	*
499	* @param asid Address space identifier.	500	* @param asid Address space identifier.

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/arch/mips32/src/mm/tlb.c – Rev 958 → 983