WebSVN – HelenOS-historic – Diff – /kernel/trunk/arch/mips32/src/mm/tlb.c


/*
 * Copyright (C) 2003-2004 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <arch/mm/tlb.h>
#include <mm/asid.h>
#include <genarch/mm/asid_fifo.h>
#include <mm/tlb.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/cp0.h>
#include <panic.h>
#include <arch.h>
#include <symtab.h>
#include <synch/spinlock.h>
#include <print.h>
#include <debug.h>
 
static void tlb_refill_fail(struct exception_regdump *pstate);
static void tlb_invalid_fail(struct exception_regdump *pstate);
static void tlb_modified_fail(struct exception_regdump *pstate);
 
static pte_t *find_mapping_and_check(__address badvaddr);
 
static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn);
static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
 
/** Initialize TLB
 *
 * Initialize TLB.
 * Invalidate all entries and mark wired entries.
 */
void tlb_arch_init(void)
{
    int i;
 
    asid_fifo_init();
 
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    cp0_entry_hi_write(0);
    cp0_entry_lo0_write(0);
    cp0_entry_lo1_write(0);
 
    /* Clear and initialize TLB. */
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbwi();
    }
 
       
    /*
     * The kernel is going to make use of some wired
     * entries (e.g. mapping kernel stacks in kseg3).
     */
    cp0_wired_write(TLB_WIRED);
}
 
/** Process TLB Refill Exception
 *
 * Process TLB Refill Exception.
 *
 * @param pstate Interrupted register context.
 */
void tlb_refill(struct exception_regdump *pstate)
{
    entry_lo_t lo;
    entry_hi_t hi; 
    __address badvaddr;
    pte_t *pte;
 
    badvaddr = cp0_badvaddr_read();
 
    spinlock_lock(&AS->lock);      
 
    pte = find_mapping_and_check(badvaddr);
    if (!pte)
        goto fail;
 
    /*
     * Record access to PTE.
     */
    pte->a = 1;
 
    prepare_entry_hi(&hi, AS->asid, badvaddr);
    prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
 
    /*
     * New entry is to be inserted into TLB
     */
    cp0_entry_hi_write(hi.value);
    if ((badvaddr/PAGE_SIZE) % 2 == 0) {
        cp0_entry_lo0_write(lo.value);
        cp0_entry_lo1_write(0);
    }
    else {
        cp0_entry_lo0_write(0);
        cp0_entry_lo1_write(lo.value);
    }
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwr();
 
    spinlock_unlock(&AS->lock);
    return;
   
fail:
    spinlock_unlock(&AS->lock);
    tlb_refill_fail(pstate);
}
 
/** Process TLB Invalid Exception
 *
 * Process TLB Invalid Exception.
 *
 * @param pstate Interrupted register context.
 */
void tlb_invalid(struct exception_regdump *pstate)
{
    tlb_index_t index;
    __address badvaddr;
    entry_lo_t lo;
    entry_hi_t hi;
    pte_t *pte;
 
    badvaddr = cp0_badvaddr_read();
 
    /*
     * Locate the faulting entry in TLB.
     */
    hi.value = cp0_entry_hi_read();
    prepare_entry_hi(&hi, hi.asid, badvaddr);
    cp0_entry_hi_write(hi.value);
    tlbp();
    index.value = cp0_index_read();
   
    spinlock_lock(&AS->lock);  
   
    /*
     * Fail if the entry is not in TLB.
     */
    if (index.p) {
        printf("TLB entry not found.\n");
        goto fail;
    }
 
    pte = find_mapping_and_check(badvaddr);
    if (!pte)
        goto fail;
 
    /*
     * Read the faulting TLB entry.
     */
    tlbr();
 
    /*
     * Record access to PTE.
     */
    pte->a = 1;
 
    prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
 
    /*
     * The entry is to be updated in TLB.
     */
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
        cp0_entry_lo0_write(lo.value);
    else
        cp0_entry_lo1_write(lo.value);
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwi();
 
    spinlock_unlock(&AS->lock);
    return;
   
fail:
    spinlock_unlock(&AS->lock);
    tlb_invalid_fail(pstate);
}
 
/** Process TLB Modified Exception
 *
 * Process TLB Modified Exception.
 *
 * @param pstate Interrupted register context.
 */
void tlb_modified(struct exception_regdump *pstate)
{
    tlb_index_t index;
    __address badvaddr;
    entry_lo_t lo;
    entry_hi_t hi;
    pte_t *pte;
 
    badvaddr = cp0_badvaddr_read();
 
    /*
     * Locate the faulting entry in TLB.
     */
    hi.value = cp0_entry_hi_read();
    prepare_entry_hi(&hi, hi.asid, badvaddr);
    cp0_entry_hi_write(hi.value);
    tlbp();
    index.value = cp0_index_read();
   
    spinlock_lock(&AS->lock);  
   
    /*
     * Fail if the entry is not in TLB.
     */
    if (index.p) {
        printf("TLB entry not found.\n");
        goto fail;
    }
 
    pte = find_mapping_and_check(badvaddr);
    if (!pte)
        goto fail;
 
    /*
     * Fail if the page is not writable.
     */
    if (!pte->w)
        goto fail;
 
    /*
     * Read the faulting TLB entry.
     */
    tlbr();
 
    /*
     * Record access and write to PTE.
     */
    pte->a = 1;
    pte->lo.d = 1;
 
    prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->w, pte->lo.c, pte->lo.pfn);
 
    /*
     * The entry is to be updated in TLB.
     */
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
        cp0_entry_lo0_write(lo.value);
    else
        cp0_entry_lo1_write(lo.value);
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
    tlbwi();
 
    spinlock_unlock(&AS->lock);
    return;
   
fail:
    spinlock_unlock(&AS->lock);
    tlb_modified_fail(pstate);
}
 
void tlb_refill_fail(struct exception_regdump *pstate)
{
    char *symbol = "";
    char *sym2 = "";
 
    char *s = get_symtab_entry(pstate->epc);
    if (s)
        symbol = s;
    s = get_symtab_entry(pstate->ra);
    if (s)
        sym2 = s;
    panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), pstate->epc, symbol, sym2);
}
 
 
void tlb_invalid_fail(struct exception_regdump *pstate)
{
    char *symbol = "";
 
    char *s = get_symtab_entry(pstate->epc);
    if (s)
        symbol = s;
    panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
}
 
void tlb_modified_fail(struct exception_regdump *pstate)
{
    char *symbol = "";
 
    char *s = get_symtab_entry(pstate->epc);
    if (s)
        symbol = s;
    panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
}
 
/** Try to find PTE for faulting address
 *
 * Try to find PTE for faulting address.
 * The AS->lock must be held on entry to this function.
 *
 * @param badvaddr Faulting virtual address.
 *
 * @return PTE on success, NULL otherwise.
 */
pte_t *find_mapping_and_check(__address badvaddr)
{
    entry_hi_t hi;
    pte_t *pte;
 
    hi.value = cp0_entry_hi_read();
 
    /*
     * Handler cannot succeed if the ASIDs don't match.
     */
    if (hi.asid != AS->asid) {
        printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
        return NULL;
    }
 
    /*
     * Check if the mapping exists in page tables.
     */
    pte = page_mapping_find(AS, badvaddr, 0);
    if (pte && pte->lo.v) {
        /*
         * Mapping found in page tables.
         * Immediately succeed.
         */
        return pte;
    } else {
        /*
         * Mapping not found in page tables.
         * Resort to higher-level page fault handler.
         */
        if (as_page_fault(badvaddr)) {
            /*
             * The higher-level page fault handler succeeded,
             * The mapping ought to be in place.
             */
            pte = page_mapping_find(AS, badvaddr, 0);
            ASSERT(pte && pte->lo.v);
            return pte;
        }
    }
 
    /*
     * Handler cannot succeed if badvaddr has no mapping.
     */
    if (!pte) {
        printf("No such mapping.\n");
        return NULL;
    }
 
    /*
     * Handler cannot succeed if the mapping is marked as invalid.
     */
    if (!pte->lo.v) {
        printf("Invalid mapping.\n");
        return NULL;
    }
 
    return pte;
}
 
void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn)
{
    lo->value = 0;
    lo->g = g;
    lo->v = v;
    lo->d = d;
    lo->c = c;
    lo->pfn = pfn;
}
 
void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
{
    hi->value = (((addr/PAGE_SIZE)/2)*PAGE_SIZE*2);
    hi->asid = asid;
}
 
/** Print contents of TLB. */
void tlb_print(void)
{
    page_mask_t mask;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    int i;
 
    hi_save.value = cp0_entry_hi_read();
 
    printf("TLB:\n");
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
       
        mask.value = cp0_pagemask_read();
        hi.value = cp0_entry_hi_read();
        lo0.value = cp0_entry_lo0_read();
        lo1.value = cp0_entry_lo1_read();
       
        printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
               "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
               i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
               lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
    }
   
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate all not wired TLB entries. */
void tlb_invalidate_all(void)
{
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi_save;
    int i;
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
 
    for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
 
        lo0.value = cp0_entry_lo0_read();
        lo1.value = cp0_entry_lo1_read();
 
        lo0.v = 0;
        lo1.v = 0;
 
        cp0_entry_lo0_write(lo0.value);
        cp0_entry_lo1_write(lo1.value);
               
        tlbwi();
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate all TLB entries belonging to specified address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    int i;
 
    ASSERT(asid != ASID_INVALID);
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
   
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
        cp0_index_write(i);
        tlbr();
       
        hi.value = cp0_entry_hi_read();
       
        if (hi.asid == asid) {
            lo0.value = cp0_entry_lo0_read();
            lo1.value = cp0_entry_lo1_read();
 
            lo0.v = 0;
            lo1.v = 0;
 
            cp0_entry_lo0_write(lo0.value);
            cp0_entry_lo1_write(lo1.value);
 
            tlbwi();
        }
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 
/** Invalidate TLB entries for specified page range belonging to specified address space.
 *
 * @param asid Address space identifier.
 * @param page First page whose TLB entry is to be invalidated.
 * @param cnt Number of entries to invalidate.
 */
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
{
    int i;
    ipl_t ipl;
    entry_lo_t lo0, lo1;
    entry_hi_t hi, hi_save;
    tlb_index_t index;
 
    ASSERT(asid != ASID_INVALID);
 
    hi_save.value = cp0_entry_hi_read();
    ipl = interrupts_disable();
 
    for (i = 0; i < cnt; i++) {
        hi.value = 0;
        prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
        cp0_entry_hi_write(hi.value);
 
        tlbp();
        index.value = cp0_index_read();
 
        if (!index.p) {
            /* Entry was found, index register contains valid index. */
            tlbr();
 
            lo0.value = cp0_entry_lo0_read();
            lo1.value = cp0_entry_lo1_read();
 
            lo0.v = 0;
            lo1.v = 0;
 
            cp0_entry_lo0_write(lo0.value);
            cp0_entry_lo1_write(lo1.value);
 
            tlbwi();
        }
    }
   
    interrupts_restore(ipl);
    cp0_entry_hi_write(hi_save.value);
}
 

Rev 730	Rev 755
1	/*	1	/*
2	* Copyright (C) 2003-2004 Jakub Jermar	2	* Copyright (C) 2003-2004 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <arch/mm/tlb.h>	29	#include <arch/mm/tlb.h>
30	#include <mm/asid.h>	30	#include <mm/asid.h>
31	#include <genarch/mm/asid_fifo.h>	31	#include <genarch/mm/asid_fifo.h>
32	#include <mm/tlb.h>	32	#include <mm/tlb.h>
33	#include <mm/page.h>	33	#include <mm/page.h>
34	#include <mm/as.h>	34	#include <mm/as.h>
35	#include <arch/cp0.h>	35	#include <arch/cp0.h>
36	#include <panic.h>	36	#include <panic.h>
37	#include <arch.h>	37	#include <arch.h>
38	#include <symtab.h>	38	#include <symtab.h>
39	#include <synch/spinlock.h>	39	#include <synch/spinlock.h>
40	#include <print.h>	40	#include <print.h>
41	#include <debug.h>	41	#include <debug.h>
42		42
43	static void tlb_refill_fail(struct exception_regdump *pstate);	43	static void tlb_refill_fail(struct exception_regdump *pstate);
44	static void tlb_invalid_fail(struct exception_regdump *pstate);	44	static void tlb_invalid_fail(struct exception_regdump *pstate);
45	static void tlb_modified_fail(struct exception_regdump *pstate);	45	static void tlb_modified_fail(struct exception_regdump *pstate);
46		46
47	static pte_t *find_mapping_and_check(__address badvaddr);	47	static pte_t *find_mapping_and_check(__address badvaddr);
48		48
49	static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn);	49	static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn);
50	static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);	50	static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
51		51
52	/** Initialize TLB	52	/** Initialize TLB
53	*	53	*
54	* Initialize TLB.	54	* Initialize TLB.
55	* Invalidate all entries and mark wired entries.	55	* Invalidate all entries and mark wired entries.
56	*/	56	*/
57	void tlb_arch_init(void)	57	void tlb_arch_init(void)
58	{	58	{
59	int i;	59	int i;
60		60
61	asid_fifo_init();	61	asid_fifo_init();
62		62
63	cp0_pagemask_write(TLB_PAGE_MASK_16K);	63	cp0_pagemask_write(TLB_PAGE_MASK_16K);
64	cp0_entry_hi_write(0);	64	cp0_entry_hi_write(0);
65	cp0_entry_lo0_write(0);	65	cp0_entry_lo0_write(0);
66	cp0_entry_lo1_write(0);	66	cp0_entry_lo1_write(0);
67		67
68	/* Clear and initialize TLB. */	68	/* Clear and initialize TLB. */
69		69
70	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	70	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
71	cp0_index_write(i);	71	cp0_index_write(i);
72	tlbwi();	72	tlbwi();
73	}	73	}
74		74
75		75
76	/*	76	/*
77	* The kernel is going to make use of some wired	77	* The kernel is going to make use of some wired
78	* entries (e.g. mapping kernel stacks in kseg3).	78	* entries (e.g. mapping kernel stacks in kseg3).
79	*/	79	*/
80	cp0_wired_write(TLB_WIRED);	80	cp0_wired_write(TLB_WIRED);
81	}	81	}
82		82
83	/** Process TLB Refill Exception	83	/** Process TLB Refill Exception
84	*	84	*
85	* Process TLB Refill Exception.	85	* Process TLB Refill Exception.
86	*	86	*
87	* @param pstate Interrupted register context.	87	* @param pstate Interrupted register context.
88	*/	88	*/
89	void tlb_refill(struct exception_regdump *pstate)	89	void tlb_refill(struct exception_regdump *pstate)
90	{	90	{
91	entry_lo_t lo;	91	entry_lo_t lo;
92	entry_hi_t hi;	92	entry_hi_t hi;
93	__address badvaddr;	93	__address badvaddr;
94	pte_t *pte;	94	pte_t *pte;
95		95
96	badvaddr = cp0_badvaddr_read();	96	badvaddr = cp0_badvaddr_read();
97		97
98	spinlock_lock(&AS->lock);	98	spinlock_lock(&AS->lock);
99		99
100	pte = find_mapping_and_check(badvaddr);	100	pte = find_mapping_and_check(badvaddr);
101	if (!pte)	101	if (!pte)
102	goto fail;	102	goto fail;
103		103
104	/*	104	/*
105	* Record access to PTE.	105	* Record access to PTE.
106	*/	106	*/
107	pte->a = 1;	107	pte->a = 1;
108		108
109	prepare_entry_hi(&hi, AS->asid, badvaddr);	109	prepare_entry_hi(&hi, AS->asid, badvaddr);
110	prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);	110	prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
111		111
112	/*	112	/*
113	* New entry is to be inserted into TLB	113	* New entry is to be inserted into TLB
114	*/	114	*/
115	cp0_entry_hi_write(hi.value);	115	cp0_entry_hi_write(hi.value);
116	if ((badvaddr/PAGE_SIZE) % 2 == 0) {	116	if ((badvaddr/PAGE_SIZE) % 2 == 0) {
117	cp0_entry_lo0_write(lo.value);	117	cp0_entry_lo0_write(lo.value);
118	cp0_entry_lo1_write(0);	118	cp0_entry_lo1_write(0);
119	}	119	}
120	else {	120	else {
121	cp0_entry_lo0_write(0);	121	cp0_entry_lo0_write(0);
122	cp0_entry_lo1_write(lo.value);	122	cp0_entry_lo1_write(lo.value);
123	}	123	}
124	cp0_pagemask_write(TLB_PAGE_MASK_16K);	124	cp0_pagemask_write(TLB_PAGE_MASK_16K);
125	tlbwr();	125	tlbwr();
126		126
127	spinlock_unlock(&AS->lock);	127	spinlock_unlock(&AS->lock);
128	return;	128	return;
129		129
130	fail:	130	fail:
131	spinlock_unlock(&AS->lock);	131	spinlock_unlock(&AS->lock);
132	tlb_refill_fail(pstate);	132	tlb_refill_fail(pstate);
133	}	133	}
134		134
135	/** Process TLB Invalid Exception	135	/** Process TLB Invalid Exception
136	*	136	*
137	* Process TLB Invalid Exception.	137	* Process TLB Invalid Exception.
138	*	138	*
139	* @param pstate Interrupted register context.	139	* @param pstate Interrupted register context.
140	*/	140	*/
141	void tlb_invalid(struct exception_regdump *pstate)	141	void tlb_invalid(struct exception_regdump *pstate)
142	{	142	{
143	tlb_index_t index;	143	tlb_index_t index;
144	__address badvaddr;	144	__address badvaddr;
145	entry_lo_t lo;	145	entry_lo_t lo;
146	entry_hi_t hi;	146	entry_hi_t hi;
147	pte_t *pte;	147	pte_t *pte;
148		148
149	badvaddr = cp0_badvaddr_read();	149	badvaddr = cp0_badvaddr_read();
150		150
151	/*	151	/*
152	* Locate the faulting entry in TLB.	152	* Locate the faulting entry in TLB.
153	*/	153	*/
154	hi.value = cp0_entry_hi_read();	154	hi.value = cp0_entry_hi_read();
155	prepare_entry_hi(&hi, hi.asid, badvaddr);	155	prepare_entry_hi(&hi, hi.asid, badvaddr);
156	cp0_entry_hi_write(hi.value);	156	cp0_entry_hi_write(hi.value);
157	tlbp();	157	tlbp();
158	index.value = cp0_index_read();	158	index.value = cp0_index_read();
159		159
160	spinlock_lock(&AS->lock);	160	spinlock_lock(&AS->lock);
161		161
162	/*	162	/*
163	* Fail if the entry is not in TLB.	163	* Fail if the entry is not in TLB.
164	*/	164	*/
165	if (index.p) {	165	if (index.p) {
166	printf("TLB entry not found.\n");	166	printf("TLB entry not found.\n");
167	goto fail;	167	goto fail;
168	}	168	}
169		169
170	pte = find_mapping_and_check(badvaddr);	170	pte = find_mapping_and_check(badvaddr);
171	if (!pte)	171	if (!pte)
172	goto fail;	172	goto fail;
173		173
174	/*	174	/*
175	* Read the faulting TLB entry.	175	* Read the faulting TLB entry.
176	*/	176	*/
177	tlbr();	177	tlbr();
178		178
179	/*	179	/*
180	* Record access to PTE.	180	* Record access to PTE.
181	*/	181	*/
182	pte->a = 1;	182	pte->a = 1;
183		183
184	prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);	184	prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
185		185
186	/*	186	/*
187	* The entry is to be updated in TLB.	187	* The entry is to be updated in TLB.
188	*/	188	*/
189	if ((badvaddr/PAGE_SIZE) % 2 == 0)	189	if ((badvaddr/PAGE_SIZE) % 2 == 0)
190	cp0_entry_lo0_write(lo.value);	190	cp0_entry_lo0_write(lo.value);
191	else	191	else
192	cp0_entry_lo1_write(lo.value);	192	cp0_entry_lo1_write(lo.value);
193	cp0_pagemask_write(TLB_PAGE_MASK_16K);	193	cp0_pagemask_write(TLB_PAGE_MASK_16K);
194	tlbwi();	194	tlbwi();
195		195
196	spinlock_unlock(&AS->lock);	196	spinlock_unlock(&AS->lock);
197	return;	197	return;
198		198
199	fail:	199	fail:
200	spinlock_unlock(&AS->lock);	200	spinlock_unlock(&AS->lock);
201	tlb_invalid_fail(pstate);	201	tlb_invalid_fail(pstate);
202	}	202	}
203		203
204	/** Process TLB Modified Exception	204	/** Process TLB Modified Exception
205	*	205	*
206	* Process TLB Modified Exception.	206	* Process TLB Modified Exception.
207	*	207	*
208	* @param pstate Interrupted register context.	208	* @param pstate Interrupted register context.
209	*/	209	*/
210	void tlb_modified(struct exception_regdump *pstate)	210	void tlb_modified(struct exception_regdump *pstate)
211	{	211	{
212	tlb_index_t index;	212	tlb_index_t index;
213	__address badvaddr;	213	__address badvaddr;
214	entry_lo_t lo;	214	entry_lo_t lo;
215	entry_hi_t hi;	215	entry_hi_t hi;
216	pte_t *pte;	216	pte_t *pte;
217		217
218	badvaddr = cp0_badvaddr_read();	218	badvaddr = cp0_badvaddr_read();
219		219
220	/*	220	/*
221	* Locate the faulting entry in TLB.	221	* Locate the faulting entry in TLB.
222	*/	222	*/
223	hi.value = cp0_entry_hi_read();	223	hi.value = cp0_entry_hi_read();
224	prepare_entry_hi(&hi, hi.asid, badvaddr);	224	prepare_entry_hi(&hi, hi.asid, badvaddr);
225	cp0_entry_hi_write(hi.value);	225	cp0_entry_hi_write(hi.value);
226	tlbp();	226	tlbp();
227	index.value = cp0_index_read();	227	index.value = cp0_index_read();
228		228
229	spinlock_lock(&AS->lock);	229	spinlock_lock(&AS->lock);
230		230
231	/*	231	/*
232	* Fail if the entry is not in TLB.	232	* Fail if the entry is not in TLB.
233	*/	233	*/
234	if (index.p) {	234	if (index.p) {
235	printf("TLB entry not found.\n");	235	printf("TLB entry not found.\n");
236	goto fail;	236	goto fail;
237	}	237	}
238		238
239	pte = find_mapping_and_check(badvaddr);	239	pte = find_mapping_and_check(badvaddr);
240	if (!pte)	240	if (!pte)
241	goto fail;	241	goto fail;
242		242
243	/*	243	/*
244	* Fail if the page is not writable.	244	* Fail if the page is not writable.
245	*/	245	*/
246	if (!pte->w)	246	if (!pte->w)
247	goto fail;	247	goto fail;
248		248
249	/*	249	/*
250	* Read the faulting TLB entry.	250	* Read the faulting TLB entry.
251	*/	251	*/
252	tlbr();	252	tlbr();
253		253
254	/*	254	/*
255	* Record access and write to PTE.	255	* Record access and write to PTE.
256	*/	256	*/
257	pte->a = 1;	257	pte->a = 1;
258	pte->lo.d = 1;	258	pte->lo.d = 1;
259		259
260	prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->w, pte->lo.c, pte->lo.pfn);	260	prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->w, pte->lo.c, pte->lo.pfn);
261		261
262	/*	262	/*
263	* The entry is to be updated in TLB.	263	* The entry is to be updated in TLB.
264	*/	264	*/
265	if ((badvaddr/PAGE_SIZE) % 2 == 0)	265	if ((badvaddr/PAGE_SIZE) % 2 == 0)
266	cp0_entry_lo0_write(lo.value);	266	cp0_entry_lo0_write(lo.value);
267	else	267	else
268	cp0_entry_lo1_write(lo.value);	268	cp0_entry_lo1_write(lo.value);
269	cp0_pagemask_write(TLB_PAGE_MASK_16K);	269	cp0_pagemask_write(TLB_PAGE_MASK_16K);
270	tlbwi();	270	tlbwi();
271		271
272	spinlock_unlock(&AS->lock);	272	spinlock_unlock(&AS->lock);
273	return;	273	return;
274		274
275	fail:	275	fail:
276	spinlock_unlock(&AS->lock);	276	spinlock_unlock(&AS->lock);
277	tlb_modified_fail(pstate);	277	tlb_modified_fail(pstate);
278	}	278	}
279		279
280	void tlb_refill_fail(struct exception_regdump *pstate)	280	void tlb_refill_fail(struct exception_regdump *pstate)
281	{	281	{
282	char *symbol = "";	282	char *symbol = "";
283	char *sym2 = "";	283	char *sym2 = "";
284		284
285	char *s = get_symtab_entry(pstate->epc);	285	char *s = get_symtab_entry(pstate->epc);
286	if (s)	286	if (s)
287	symbol = s;	287	symbol = s;
288	s = get_symtab_entry(pstate->ra);	288	s = get_symtab_entry(pstate->ra);
289	if (s)	289	if (s)
290	sym2 = s;	290	sym2 = s;
291	panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), pstate->epc, symbol, sym2);	291	panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), pstate->epc, symbol, sym2);
292	}	292	}
293		293
294		294
295	void tlb_invalid_fail(struct exception_regdump *pstate)	295	void tlb_invalid_fail(struct exception_regdump *pstate)
296	{	296	{
297	char *symbol = "";	297	char *symbol = "";
298		298
299	char *s = get_symtab_entry(pstate->epc);	299	char *s = get_symtab_entry(pstate->epc);
300	if (s)	300	if (s)
301	symbol = s;	301	symbol = s;
302	panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);	302	panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
303	}	303	}
304		304
305	void tlb_modified_fail(struct exception_regdump *pstate)	305	void tlb_modified_fail(struct exception_regdump *pstate)
306	{	306	{
307	char *symbol = "";	307	char *symbol = "";
308		308
309	char *s = get_symtab_entry(pstate->epc);	309	char *s = get_symtab_entry(pstate->epc);
310	if (s)	310	if (s)
311	symbol = s;	311	symbol = s;
312	panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);	312	panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
313	}	313	}
314		314
315	/** Try to find PTE for faulting address	315	/** Try to find PTE for faulting address
316	*	316	*
317	* Try to find PTE for faulting address.	317	* Try to find PTE for faulting address.
318	* The AS->lock must be held on entry to this function.	318	* The AS->lock must be held on entry to this function.
319	*	319	*
320	* @param badvaddr Faulting virtual address.	320	* @param badvaddr Faulting virtual address.
321	*	321	*
322	* @return PTE on success, NULL otherwise.	322	* @return PTE on success, NULL otherwise.
323	*/	323	*/
324	pte_t *find_mapping_and_check(__address badvaddr)	324	pte_t *find_mapping_and_check(__address badvaddr)
325	{	325	{
326	entry_hi_t hi;	326	entry_hi_t hi;
327	pte_t *pte;	327	pte_t *pte;
328		328
329	hi.value = cp0_entry_hi_read();	329	hi.value = cp0_entry_hi_read();
330		330
331	/*	331	/*
332	* Handler cannot succeed if the ASIDs don't match.	332	* Handler cannot succeed if the ASIDs don't match.
333	*/	333	*/
334	if (hi.asid != AS->asid) {	334	if (hi.asid != AS->asid) {
335	printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);	335	printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
336	return NULL;	336	return NULL;
337	}	337	}
338		338
339	/*	339	/*
340	* Check if the mapping exists in page tables.	340	* Check if the mapping exists in page tables.
341	*/	341	*/
342	pte = page_mapping_find(badvaddr, AS->asid, 0);	342	pte = page_mapping_find(AS, badvaddr, 0);
343	if (pte && pte->lo.v) {	343	if (pte && pte->lo.v) {
344	/*	344	/*
345	* Mapping found in page tables.	345	* Mapping found in page tables.
346	* Immediately succeed.	346	* Immediately succeed.
347	*/	347	*/
348	return pte;	348	return pte;
349	} else {	349	} else {
350	/*	350	/*
351	* Mapping not found in page tables.	351	* Mapping not found in page tables.
352	* Resort to higher-level page fault handler.	352	* Resort to higher-level page fault handler.
353	*/	353	*/
354	if (as_page_fault(badvaddr)) {	354	if (as_page_fault(badvaddr)) {
355	/*	355	/*
356	* The higher-level page fault handler succeeded,	356	* The higher-level page fault handler succeeded,
357	* The mapping ought to be in place.	357	* The mapping ought to be in place.
358	*/	358	*/
359	pte = page_mapping_find(badvaddr, AS->asid, 0);	359	pte = page_mapping_find(AS, badvaddr, 0);
360	ASSERT(pte && pte->lo.v);	360	ASSERT(pte && pte->lo.v);
361	return pte;	361	return pte;
362	}	362	}
363	}	363	}
364		364
365	/*	365	/*
366	* Handler cannot succeed if badvaddr has no mapping.	366	* Handler cannot succeed if badvaddr has no mapping.
367	*/	367	*/
368	if (!pte) {	368	if (!pte) {
369	printf("No such mapping.\n");	369	printf("No such mapping.\n");
370	return NULL;	370	return NULL;
371	}	371	}
372		372
373	/*	373	/*
374	* Handler cannot succeed if the mapping is marked as invalid.	374	* Handler cannot succeed if the mapping is marked as invalid.
375	*/	375	*/
376	if (!pte->lo.v) {	376	if (!pte->lo.v) {
377	printf("Invalid mapping.\n");	377	printf("Invalid mapping.\n");
378	return NULL;	378	return NULL;
379	}	379	}
380		380
381	return pte;	381	return pte;
382	}	382	}
383		383
384	void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn)	384	void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn)
385	{	385	{
386	lo->value = 0;	386	lo->value = 0;
387	lo->g = g;	387	lo->g = g;
388	lo->v = v;	388	lo->v = v;
389	lo->d = d;	389	lo->d = d;
390	lo->c = c;	390	lo->c = c;
391	lo->pfn = pfn;	391	lo->pfn = pfn;
392	}	392	}
393		393
394	void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)	394	void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
395	{	395	{
396	hi->value = (((addr/PAGE_SIZE)/2)PAGE_SIZE2);	396	hi->value = (((addr/PAGE_SIZE)/2)PAGE_SIZE2);
397	hi->asid = asid;	397	hi->asid = asid;
398	}	398	}
399		399
400	/** Print contents of TLB. */	400	/** Print contents of TLB. */
401	void tlb_print(void)	401	void tlb_print(void)
402	{	402	{
403	page_mask_t mask;	403	page_mask_t mask;
404	entry_lo_t lo0, lo1;	404	entry_lo_t lo0, lo1;
405	entry_hi_t hi, hi_save;	405	entry_hi_t hi, hi_save;
406	int i;	406	int i;
407		407
408	hi_save.value = cp0_entry_hi_read();	408	hi_save.value = cp0_entry_hi_read();
409		409
410	printf("TLB:\n");	410	printf("TLB:\n");
411	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	411	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
412	cp0_index_write(i);	412	cp0_index_write(i);
413	tlbr();	413	tlbr();
414		414
415	mask.value = cp0_pagemask_read();	415	mask.value = cp0_pagemask_read();
416	hi.value = cp0_entry_hi_read();	416	hi.value = cp0_entry_hi_read();
417	lo0.value = cp0_entry_lo0_read();	417	lo0.value = cp0_entry_lo0_read();
418	lo1.value = cp0_entry_lo1_read();	418	lo1.value = cp0_entry_lo1_read();
419		419
420	printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"	420	printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
421	"\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",	421	"\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
422	i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,	422	i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
423	lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);	423	lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
424	}	424	}
425		425
426	cp0_entry_hi_write(hi_save.value);	426	cp0_entry_hi_write(hi_save.value);
427	}	427	}
428		428
429	/** Invalidate all not wired TLB entries. */	429	/** Invalidate all not wired TLB entries. */
430	void tlb_invalidate_all(void)	430	void tlb_invalidate_all(void)
431	{	431	{
432	ipl_t ipl;	432	ipl_t ipl;
433	entry_lo_t lo0, lo1;	433	entry_lo_t lo0, lo1;
434	entry_hi_t hi_save;	434	entry_hi_t hi_save;
435	int i;	435	int i;
436		436
437	hi_save.value = cp0_entry_hi_read();	437	hi_save.value = cp0_entry_hi_read();
438	ipl = interrupts_disable();	438	ipl = interrupts_disable();
439		439
440	for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {	440	for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
441	cp0_index_write(i);	441	cp0_index_write(i);
442	tlbr();	442	tlbr();
443		443
444	lo0.value = cp0_entry_lo0_read();	444	lo0.value = cp0_entry_lo0_read();
445	lo1.value = cp0_entry_lo1_read();	445	lo1.value = cp0_entry_lo1_read();
446		446
447	lo0.v = 0;	447	lo0.v = 0;
448	lo1.v = 0;	448	lo1.v = 0;
449		449
450	cp0_entry_lo0_write(lo0.value);	450	cp0_entry_lo0_write(lo0.value);
451	cp0_entry_lo1_write(lo1.value);	451	cp0_entry_lo1_write(lo1.value);
452		452
453	tlbwi();	453	tlbwi();
454	}	454	}
455		455
456	interrupts_restore(ipl);	456	interrupts_restore(ipl);
457	cp0_entry_hi_write(hi_save.value);	457	cp0_entry_hi_write(hi_save.value);
458	}	458	}
459		459
460	/** Invalidate all TLB entries belonging to specified address space.	460	/** Invalidate all TLB entries belonging to specified address space.
461	*	461	*
462	* @param asid Address space identifier.	462	* @param asid Address space identifier.
463	*/	463	*/
464	void tlb_invalidate_asid(asid_t asid)	464	void tlb_invalidate_asid(asid_t asid)
465	{	465	{
466	ipl_t ipl;	466	ipl_t ipl;
467	entry_lo_t lo0, lo1;	467	entry_lo_t lo0, lo1;
468	entry_hi_t hi, hi_save;	468	entry_hi_t hi, hi_save;
469	int i;	469	int i;
470		470
471	ASSERT(asid != ASID_INVALID);	471	ASSERT(asid != ASID_INVALID);
472		472
473	hi_save.value = cp0_entry_hi_read();	473	hi_save.value = cp0_entry_hi_read();
474	ipl = interrupts_disable();	474	ipl = interrupts_disable();
475		475
476	for (i = 0; i < TLB_ENTRY_COUNT; i++) {	476	for (i = 0; i < TLB_ENTRY_COUNT; i++) {
477	cp0_index_write(i);	477	cp0_index_write(i);
478	tlbr();	478	tlbr();
479		479
480	hi.value = cp0_entry_hi_read();	480	hi.value = cp0_entry_hi_read();
481		481
482	if (hi.asid == asid) {	482	if (hi.asid == asid) {
483	lo0.value = cp0_entry_lo0_read();	483	lo0.value = cp0_entry_lo0_read();
484	lo1.value = cp0_entry_lo1_read();	484	lo1.value = cp0_entry_lo1_read();
485		485
486	lo0.v = 0;	486	lo0.v = 0;
487	lo1.v = 0;	487	lo1.v = 0;
488		488
489	cp0_entry_lo0_write(lo0.value);	489	cp0_entry_lo0_write(lo0.value);
490	cp0_entry_lo1_write(lo1.value);	490	cp0_entry_lo1_write(lo1.value);
491		491
492	tlbwi();	492	tlbwi();
493	}	493	}
494	}	494	}
495		495
496	interrupts_restore(ipl);	496	interrupts_restore(ipl);
497	cp0_entry_hi_write(hi_save.value);	497	cp0_entry_hi_write(hi_save.value);
498	}	498	}
499		499
500	/** Invalidate TLB entries for specified page range belonging to specified address space.	500	/** Invalidate TLB entries for specified page range belonging to specified address space.

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/arch/mips32/src/mm/tlb.c – Rev 730 → 755