WebSVN – HelenOS-historic – Diff – /kernel/trunk/arch/ia64/src/mm/tlb.c


/*
 * Copyright (C) 2006 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
/*
 * TLB management.
 */
 
#include <mm/tlb.h>
#include <mm/asid.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/mm/tlb.h>
#include <arch/mm/page.h>
#include <arch/barrier.h>
#include <arch/interrupt.h>
#include <arch/pal/pal.h>
#include <arch/asm.h>
#include <typedefs.h>
#include <panic.h>
#include <print.h>
#include <arch.h>
 
/** Invalidate all TLB entries. */
void tlb_invalidate_all(void)
{
        ipl_t ipl;
        __address adr;
        __u32 count1, count2, stride1, stride2;
       
        int i,j;
       
        adr = PAL_PTCE_INFO_BASE();
        count1 = PAL_PTCE_INFO_COUNT1();
        count2 = PAL_PTCE_INFO_COUNT2();
        stride1 = PAL_PTCE_INFO_STRIDE1();
        stride2 = PAL_PTCE_INFO_STRIDE2();
       
        ipl = interrupts_disable();
 
        for(i = 0; i < count1; i++) {
            for(j = 0; j < count2; j++) {
                __asm__ volatile (
                    "ptc.e %0 ;;"
                    :
                    : "r" (adr)
                );
                adr += stride2;
            }
            adr += stride1;
        }
 
        interrupts_restore(ipl);
 
        srlz_d();
        srlz_i();
}
 
/** Invalidate entries belonging to an address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    tlb_invalidate_all();
}
 
 
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
{
    region_register rr;
    bool restore_rr = false;
    int b = 0;
    int c = cnt;
 
    __address va;
    va = page;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    while(c >>= 1)
        b++;
    b >>= 1;
    __u64 ps;
   
    switch (b) {
        case 0: /*cnt 1-3*/
            ps = PAGE_WIDTH;
            break;
        case 1: /*cnt 4-15*/
            /*cnt=((cnt-1)/4)+1;*/
            ps = PAGE_WIDTH+2;
            va &= ~((1<<ps)-1);
            break;
        case 2: /*cnt 16-63*/
            /*cnt=((cnt-1)/16)+1;*/
            ps = PAGE_WIDTH+4;
            va &= ~((1<<ps)-1);
            break;
        case 3: /*cnt 64-255*/
            /*cnt=((cnt-1)/64)+1;*/
            ps = PAGE_WIDTH+6;
            va &= ~((1<<ps)-1);
            break;
        case 4: /*cnt 256-1023*/
            /*cnt=((cnt-1)/256)+1;*/
            ps = PAGE_WIDTH+8;
            va &= ~((1<<ps)-1);
            break;
        case 5: /*cnt 1024-4095*/
            /*cnt=((cnt-1)/1024)+1;*/
            ps = PAGE_WIDTH+10;
            va &= ~((1<<ps)-1);
            break;
        case 6: /*cnt 4096-16383*/
            /*cnt=((cnt-1)/4096)+1;*/
            ps = PAGE_WIDTH+12;
            va &= ~((1<<ps)-1);
            break;
        case 7: /*cnt 16384-65535*/
        case 8: /*cnt 65536-(256K-1)*/
            /*cnt=((cnt-1)/16384)+1;*/
            ps = PAGE_WIDTH+14;
            va &= ~((1<<ps)-1);
            break;
        default:
            /*cnt=((cnt-1)/(16384*16))+1;*/
            ps=PAGE_WIDTH+18;
            va&=~((1<<ps)-1);
            break;
    }
    /*cnt+=(page!=va);*/
    for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
        __asm__ volatile (
            "ptc.l %0,%1;;"
            :
            : "r" (va), "r" (ps<<2)
        );
    }
    srlz_d();
    srlz_i();
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
 
/** Insert data into data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, true);
}
 
/** Insert data into instruction translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, false);
}
 
/** Insert data into instruction or data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
 */
void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    __asm__ volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"       /* va */
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
        "(p6) itc.i %3;;\n"
        "(p7) itc.d %3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into instruction translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, false, tr);
}
 
/** Insert data into data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, true, tr);
}
 
/** Insert data into instruction or data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation register, use instruction translation register otherwise.
 * @param tr Translation register.
 */
void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
 
    __asm__ volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"           /* va */         
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7=%5,r0;;\n"    /* decide between itr and dtr */
        "(p6) itr.i itr[%4]=%3;;\n"
        "(p7) itr.d dtr[%4]=%3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into DTLB.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtr If true, insert into data translation register, use data translation cache otherwise.
 * @param tr Translation register if dtr is true, ignored otherwise.
 */
void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
{
    tlb_entry_t entry;
   
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = true;         /* present */
    entry.ma = MA_WRITEBACK;
    entry.a = true;         /* already accessed */
    entry.d = true;         /* already dirty */
    entry.pl = PL_KERNEL;
    entry.ar = AR_READ | AR_WRITE;
    entry.ppn = frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    if (dtr)
        dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
    else
        dtc_mapping_insert(page, ASID_KERNEL, entry);
}
 
/** Copy content of PTE into data translation cache.
 *
 * @param t PTE.
 */
void dtc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.d = t->d;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->w ? AR_WRITE : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    dtc_mapping_insert(t->page, t->as->asid, entry);
}
 
/** Copy content of PTE into instruction translation cache.
 *
 * @param t PTE.
 */
void itc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    ASSERT(t->x);
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    itc_mapping_insert(t->page, t->as->asid, entry);
}
 
/** Instruction TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        itc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (!as_page_fault(va)) {
            panic("%s: va=%P, rid=%d, iip=%P\n", __FUNCTION__, istate->cr_ifa, rr.map.rid, istate->cr_iip);
        }
    }
}
 
/** Data TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
    if (RID2ASID(rid) == ASID_KERNEL) {
        if (VA2VRN(va) == VRN_KERNEL) {
            /*
             * Provide KA2PA(identity) mapping for faulting piece of
             * kernel address space.
             */
            dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
            return;
        }
    }
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (!as_page_fault(va)) {
            panic("%s: va=%P, rid=%d, iip=%P\n", __FUNCTION__, va, rid, istate->cr_iip);
        }
    }
}
 
/** Data nested TLB fault handler.
 *
 * This fault should not occur.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_nested_tlb_fault(__u64 vector, istate_t *istate)
{
    panic("%s\n", __FUNCTION__);
}
 
/** Data Dirty bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_dirty_bit_fault(__u64 vector, istate_t *istate)
{
    pte_t *t;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, istate->cr_ifa);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Dirty bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->d = true;
        dtc_pte_copy(t);
    }
    page_table_unlock(AS, true);
}
 
/** Instruction access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void instruction_access_bit_fault(__u64 vector, istate_t *istate)
{
    pte_t *t;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, istate->cr_ifa);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into ITC.
         */
        t->a = true;
        itc_pte_copy(t);
    }
    page_table_unlock(AS, true);
}
 
/** Data access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_access_bit_fault(__u64 vector, istate_t *istate)
{
    pte_t *t;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, istate->cr_ifa);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->a = true;
        dtc_pte_copy(t);
    }
    page_table_unlock(AS, true);
}
 
/** Page not present fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void page_not_present(__u64 vector, istate_t *istate)
{
    region_register rr;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t);
   
    if (t->p) {
        /*
         * If the Present bit is set in page hash table, just copy it
         * and update ITC/DTC.
         */
        if (t->x)
            itc_pte_copy(t);
        else
            dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        page_table_unlock(AS, true);
        if (!as_page_fault(va)) {
            panic("%s: va=%P, rid=%d\n", __FUNCTION__, va, rr.map.rid);
        }
    }
}
 

Rev 1080	Rev 1104
1	/*	1	/*
2	* Copyright (C) 2006 Jakub Jermar	2	* Copyright (C) 2006 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	/*	29	/*
30	* TLB management.	30	* TLB management.
31	*/	31	*/
32		32
33	#include <mm/tlb.h>	33	#include <mm/tlb.h>
34	#include <mm/asid.h>	34	#include <mm/asid.h>
35	#include <mm/page.h>	35	#include <mm/page.h>
36	#include <mm/as.h>	36	#include <mm/as.h>
37	#include <arch/mm/tlb.h>	37	#include <arch/mm/tlb.h>
38	#include <arch/mm/page.h>	38	#include <arch/mm/page.h>
39	#include <arch/barrier.h>	39	#include <arch/barrier.h>
40	#include <arch/interrupt.h>	40	#include <arch/interrupt.h>
41	#include <arch/pal/pal.h>	41	#include <arch/pal/pal.h>
42	#include <arch/asm.h>	42	#include <arch/asm.h>
43	#include <typedefs.h>	43	#include <typedefs.h>
44	#include <panic.h>	44	#include <panic.h>
45	#include <print.h>	45	#include <print.h>
46	#include <arch.h>	46	#include <arch.h>
47		47
48	/** Invalidate all TLB entries. */	48	/** Invalidate all TLB entries. */
49	void tlb_invalidate_all(void)	49	void tlb_invalidate_all(void)
50	{	50	{
51	ipl_t ipl;	51	ipl_t ipl;
52	__address adr;	52	__address adr;
53	__u32 count1, count2, stride1, stride2;	53	__u32 count1, count2, stride1, stride2;
54		54
55	int i,j;	55	int i,j;
56		56
57	adr = PAL_PTCE_INFO_BASE();	57	adr = PAL_PTCE_INFO_BASE();
58	count1 = PAL_PTCE_INFO_COUNT1();	58	count1 = PAL_PTCE_INFO_COUNT1();
59	count2 = PAL_PTCE_INFO_COUNT2();	59	count2 = PAL_PTCE_INFO_COUNT2();
60	stride1 = PAL_PTCE_INFO_STRIDE1();	60	stride1 = PAL_PTCE_INFO_STRIDE1();
61	stride2 = PAL_PTCE_INFO_STRIDE2();	61	stride2 = PAL_PTCE_INFO_STRIDE2();
62		62
63	ipl = interrupts_disable();	63	ipl = interrupts_disable();
64		64
65	for(i = 0; i < count1; i++) {	65	for(i = 0; i < count1; i++) {
66	for(j = 0; j < count2; j++) {	66	for(j = 0; j < count2; j++) {
67	__asm__ volatile (	67	__asm__ volatile (
68	"ptc.e %0 ;;"	68	"ptc.e %0 ;;"
69	:	69	:
70	: "r" (adr)	70	: "r" (adr)
71	);	71	);
72	adr += stride2;	72	adr += stride2;
73	}	73	}
74	adr += stride1;	74	adr += stride1;
75	}	75	}
76		76
77	interrupts_restore(ipl);	77	interrupts_restore(ipl);
78		78
79	srlz_d();	79	srlz_d();
80	srlz_i();	80	srlz_i();
81	}	81	}
82		82
83	/** Invalidate entries belonging to an address space.	83	/** Invalidate entries belonging to an address space.
84	*	84	*
85	* @param asid Address space identifier.	85	* @param asid Address space identifier.
86	*/	86	*/
87	void tlb_invalidate_asid(asid_t asid)	87	void tlb_invalidate_asid(asid_t asid)
88	{	88	{
89	tlb_invalidate_all();	89	tlb_invalidate_all();
90	}	90	}
91		91
92		92
93	void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)	93	void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
94	{	94	{
95	region_register rr;	95	region_register rr;
96	bool restore_rr = false;	96	bool restore_rr = false;
97	int b = 0;	97	int b = 0;
98	int c = cnt;	98	int c = cnt;
99		99
100	__address va;	100	__address va;
101	va = page;	101	va = page;
102		102
103	rr.word = rr_read(VA2VRN(va));	103	rr.word = rr_read(VA2VRN(va));
104	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	104	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
105	/*	105	/*
106	* The selected region register does not contain required RID.	106	* The selected region register does not contain required RID.
107	* Save the old content of the register and replace the RID.	107	* Save the old content of the register and replace the RID.
108	*/	108	*/
109	region_register rr0;	109	region_register rr0;
110		110
111	rr0 = rr;	111	rr0 = rr;
112	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	112	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
113	rr_write(VA2VRN(va), rr0.word);	113	rr_write(VA2VRN(va), rr0.word);
114	srlz_d();	114	srlz_d();
115	srlz_i();	115	srlz_i();
116	}	116	}
117		117
118	while(c >>= 1)	118	while(c >>= 1)
119	b++;	119	b++;
120	b >>= 1;	120	b >>= 1;
121	__u64 ps;	121	__u64 ps;
122		122
123	switch (b) {	123	switch (b) {
124	case 0: /cnt 1-3/	124	case 0: /cnt 1-3/
125	ps = PAGE_WIDTH;	125	ps = PAGE_WIDTH;
126	break;	126	break;
127	case 1: /cnt 4-15/	127	case 1: /cnt 4-15/
128	/cnt=((cnt-1)/4)+1;/	128	/cnt=((cnt-1)/4)+1;/
129	ps = PAGE_WIDTH+2;	129	ps = PAGE_WIDTH+2;
130	va &= ~((1<<ps)-1);	130	va &= ~((1<<ps)-1);
131	break;	131	break;
132	case 2: /cnt 16-63/	132	case 2: /cnt 16-63/
133	/cnt=((cnt-1)/16)+1;/	133	/cnt=((cnt-1)/16)+1;/
134	ps = PAGE_WIDTH+4;	134	ps = PAGE_WIDTH+4;
135	va &= ~((1<<ps)-1);	135	va &= ~((1<<ps)-1);
136	break;	136	break;
137	case 3: /cnt 64-255/	137	case 3: /cnt 64-255/
138	/cnt=((cnt-1)/64)+1;/	138	/cnt=((cnt-1)/64)+1;/
139	ps = PAGE_WIDTH+6;	139	ps = PAGE_WIDTH+6;
140	va &= ~((1<<ps)-1);	140	va &= ~((1<<ps)-1);
141	break;	141	break;
142	case 4: /cnt 256-1023/	142	case 4: /cnt 256-1023/
143	/cnt=((cnt-1)/256)+1;/	143	/cnt=((cnt-1)/256)+1;/
144	ps = PAGE_WIDTH+8;	144	ps = PAGE_WIDTH+8;
145	va &= ~((1<<ps)-1);	145	va &= ~((1<<ps)-1);
146	break;	146	break;
147	case 5: /cnt 1024-4095/	147	case 5: /cnt 1024-4095/
148	/cnt=((cnt-1)/1024)+1;/	148	/cnt=((cnt-1)/1024)+1;/
149	ps = PAGE_WIDTH+10;	149	ps = PAGE_WIDTH+10;
150	va &= ~((1<<ps)-1);	150	va &= ~((1<<ps)-1);
151	break;	151	break;
152	case 6: /cnt 4096-16383/	152	case 6: /cnt 4096-16383/
153	/cnt=((cnt-1)/4096)+1;/	153	/cnt=((cnt-1)/4096)+1;/
154	ps = PAGE_WIDTH+12;	154	ps = PAGE_WIDTH+12;
155	va &= ~((1<<ps)-1);	155	va &= ~((1<<ps)-1);
156	break;	156	break;
157	case 7: /cnt 16384-65535/	157	case 7: /cnt 16384-65535/
158	case 8: /cnt 65536-(256K-1)/	158	case 8: /cnt 65536-(256K-1)/
159	/cnt=((cnt-1)/16384)+1;/	159	/cnt=((cnt-1)/16384)+1;/
160	ps = PAGE_WIDTH+14;	160	ps = PAGE_WIDTH+14;
161	va &= ~((1<<ps)-1);	161	va &= ~((1<<ps)-1);
162	break;	162	break;
163	default:	163	default:
164	/cnt=((cnt-1)/(1638416))+1;*/	164	/cnt=((cnt-1)/(1638416))+1;*/
165	ps=PAGE_WIDTH+18;	165	ps=PAGE_WIDTH+18;
166	va&=~((1<<ps)-1);	166	va&=~((1<<ps)-1);
167	break;	167	break;
168	}	168	}
169	/cnt+=(page!=va);/	169	/cnt+=(page!=va);/
170	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {	170	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
171	__asm__ volatile (	171	__asm__ volatile (
172	"ptc.l %0,%1;;"	172	"ptc.l %0,%1;;"
173	:	173	:
174	: "r" (va), "r" (ps<<2)	174	: "r" (va), "r" (ps<<2)
175	);	175	);
176	}	176	}
177	srlz_d();	177	srlz_d();
178	srlz_i();	178	srlz_i();
179		179
180	if (restore_rr) {	180	if (restore_rr) {
181	rr_write(VA2VRN(va), rr.word);	181	rr_write(VA2VRN(va), rr.word);
182	srlz_d();	182	srlz_d();
183	srlz_i();	183	srlz_i();
184	}	184	}
185	}	185	}
186		186
187		187
188	/** Insert data into data translation cache.	188	/** Insert data into data translation cache.
189	*	189	*
190	* @param va Virtual page address.	190	* @param va Virtual page address.
191	* @param asid Address space identifier.	191	* @param asid Address space identifier.
192	* @param entry The rest of TLB entry as required by TLB insertion format.	192	* @param entry The rest of TLB entry as required by TLB insertion format.
193	*/	193	*/
194	void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)	194	void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
195	{	195	{
196	tc_mapping_insert(va, asid, entry, true);	196	tc_mapping_insert(va, asid, entry, true);
197	}	197	}
198		198
199	/** Insert data into instruction translation cache.	199	/** Insert data into instruction translation cache.
200	*	200	*
201	* @param va Virtual page address.	201	* @param va Virtual page address.
202	* @param asid Address space identifier.	202	* @param asid Address space identifier.
203	* @param entry The rest of TLB entry as required by TLB insertion format.	203	* @param entry The rest of TLB entry as required by TLB insertion format.
204	*/	204	*/
205	void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)	205	void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
206	{	206	{
207	tc_mapping_insert(va, asid, entry, false);	207	tc_mapping_insert(va, asid, entry, false);
208	}	208	}
209		209
210	/** Insert data into instruction or data translation cache.	210	/** Insert data into instruction or data translation cache.
211	*	211	*
212	* @param va Virtual page address.	212	* @param va Virtual page address.
213	* @param asid Address space identifier.	213	* @param asid Address space identifier.
214	* @param entry The rest of TLB entry as required by TLB insertion format.	214	* @param entry The rest of TLB entry as required by TLB insertion format.
215	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.	215	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
216	*/	216	*/
217	void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)	217	void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
218	{	218	{
219	region_register rr;	219	region_register rr;
220	bool restore_rr = false;	220	bool restore_rr = false;
221		221
222	rr.word = rr_read(VA2VRN(va));	222	rr.word = rr_read(VA2VRN(va));
223	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	223	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
224	/*	224	/*
225	* The selected region register does not contain required RID.	225	* The selected region register does not contain required RID.
226	* Save the old content of the register and replace the RID.	226	* Save the old content of the register and replace the RID.
227	*/	227	*/
228	region_register rr0;	228	region_register rr0;
229		229
230	rr0 = rr;	230	rr0 = rr;
231	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	231	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
232	rr_write(VA2VRN(va), rr0.word);	232	rr_write(VA2VRN(va), rr0.word);
233	srlz_d();	233	srlz_d();
234	srlz_i();	234	srlz_i();
235	}	235	}
236		236
237	__asm__ volatile (	237	__asm__ volatile (
238	"mov r8=psr;;\n"	238	"mov r8=psr;;\n"
239	"rsm %0;;\n" /* PSR_IC_MASK */	239	"rsm %0;;\n" /* PSR_IC_MASK */
240	"srlz.d;;\n"	240	"srlz.d;;\n"
241	"srlz.i;;\n"	241	"srlz.i;;\n"
242	"mov cr.ifa=%1\n" /* va */	242	"mov cr.ifa=%1\n" /* va */
243	"mov cr.itir=%2;;\n" /* entry.word[1] */	243	"mov cr.itir=%2;;\n" /* entry.word[1] */
244	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */	244	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */
245	"(p6) itc.i %3;;\n"	245	"(p6) itc.i %3;;\n"
246	"(p7) itc.d %3;;\n"	246	"(p7) itc.d %3;;\n"
247	"mov psr.l=r8;;\n"	247	"mov psr.l=r8;;\n"
248	"srlz.d;;\n"	248	"srlz.d;;\n"
249	:	249	:
250	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)	250	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
251	: "p6", "p7", "r8"	251	: "p6", "p7", "r8"
252	);	252	);
253		253
254	if (restore_rr) {	254	if (restore_rr) {
255	rr_write(VA2VRN(va), rr.word);	255	rr_write(VA2VRN(va), rr.word);
256	srlz_d();	256	srlz_d();
257	srlz_i();	257	srlz_i();
258	}	258	}
259	}	259	}
260		260
261	/** Insert data into instruction translation register.	261	/** Insert data into instruction translation register.
262	*	262	*
263	* @param va Virtual page address.	263	* @param va Virtual page address.
264	* @param asid Address space identifier.	264	* @param asid Address space identifier.
265	* @param entry The rest of TLB entry as required by TLB insertion format.	265	* @param entry The rest of TLB entry as required by TLB insertion format.
266	* @param tr Translation register.	266	* @param tr Translation register.
267	*/	267	*/
268	void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)	268	void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
269	{	269	{
270	tr_mapping_insert(va, asid, entry, false, tr);	270	tr_mapping_insert(va, asid, entry, false, tr);
271	}	271	}
272		272
273	/** Insert data into data translation register.	273	/** Insert data into data translation register.
274	*	274	*
275	* @param va Virtual page address.	275	* @param va Virtual page address.
276	* @param asid Address space identifier.	276	* @param asid Address space identifier.
277	* @param entry The rest of TLB entry as required by TLB insertion format.	277	* @param entry The rest of TLB entry as required by TLB insertion format.
278	* @param tr Translation register.	278	* @param tr Translation register.
279	*/	279	*/
280	void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)	280	void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
281	{	281	{
282	tr_mapping_insert(va, asid, entry, true, tr);	282	tr_mapping_insert(va, asid, entry, true, tr);
283	}	283	}
284		284
285	/** Insert data into instruction or data translation register.	285	/** Insert data into instruction or data translation register.
286	*	286	*
287	* @param va Virtual page address.	287	* @param va Virtual page address.
288	* @param asid Address space identifier.	288	* @param asid Address space identifier.
289	* @param entry The rest of TLB entry as required by TLB insertion format.	289	* @param entry The rest of TLB entry as required by TLB insertion format.
290	* @param dtc If true, insert into data translation register, use instruction translation register otherwise.	290	* @param dtc If true, insert into data translation register, use instruction translation register otherwise.
291	* @param tr Translation register.	291	* @param tr Translation register.
292	*/	292	*/
293	void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)	293	void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
294	{	294	{
295	region_register rr;	295	region_register rr;
296	bool restore_rr = false;	296	bool restore_rr = false;
297		297
298	rr.word = rr_read(VA2VRN(va));	298	rr.word = rr_read(VA2VRN(va));
299	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	299	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
300	/*	300	/*
301	* The selected region register does not contain required RID.	301	* The selected region register does not contain required RID.
302	* Save the old content of the register and replace the RID.	302	* Save the old content of the register and replace the RID.
303	*/	303	*/
304	region_register rr0;	304	region_register rr0;
305		305
306	rr0 = rr;	306	rr0 = rr;
307	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	307	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
308	rr_write(VA2VRN(va), rr0.word);	308	rr_write(VA2VRN(va), rr0.word);
309	srlz_d();	309	srlz_d();
310	srlz_i();	310	srlz_i();
311	}	311	}
312		312
313	__asm__ volatile (	313	__asm__ volatile (
314	"mov r8=psr;;\n"	314	"mov r8=psr;;\n"
315	"rsm %0;;\n" /* PSR_IC_MASK */	315	"rsm %0;;\n" /* PSR_IC_MASK */
316	"srlz.d;;\n"	316	"srlz.d;;\n"
317	"srlz.i;;\n"	317	"srlz.i;;\n"
318	"mov cr.ifa=%1\n" /* va */	318	"mov cr.ifa=%1\n" /* va */
319	"mov cr.itir=%2;;\n" /* entry.word[1] */	319	"mov cr.itir=%2;;\n" /* entry.word[1] */
320	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */	320	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */
321	"(p6) itr.i itr[%4]=%3;;\n"	321	"(p6) itr.i itr[%4]=%3;;\n"
322	"(p7) itr.d dtr[%4]=%3;;\n"	322	"(p7) itr.d dtr[%4]=%3;;\n"
323	"mov psr.l=r8;;\n"	323	"mov psr.l=r8;;\n"
324	"srlz.d;;\n"	324	"srlz.d;;\n"
325	:	325	:
326	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)	326	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
327	: "p6", "p7", "r8"	327	: "p6", "p7", "r8"
328	);	328	);
329		329
330	if (restore_rr) {	330	if (restore_rr) {
331	rr_write(VA2VRN(va), rr.word);	331	rr_write(VA2VRN(va), rr.word);
332	srlz_d();	332	srlz_d();
333	srlz_i();	333	srlz_i();
334	}	334	}
335	}	335	}
336		336
337	/** Insert data into DTLB.	337	/** Insert data into DTLB.
338	*	338	*
339	* @param va Virtual page address.	339	* @param va Virtual page address.
340	* @param asid Address space identifier.	340	* @param asid Address space identifier.
341	* @param entry The rest of TLB entry as required by TLB insertion format.	341	* @param entry The rest of TLB entry as required by TLB insertion format.
342	* @param dtr If true, insert into data translation register, use data translation cache otherwise.	342	* @param dtr If true, insert into data translation register, use data translation cache otherwise.
343	* @param tr Translation register if dtr is true, ignored otherwise.	343	* @param tr Translation register if dtr is true, ignored otherwise.
344	*/	344	*/
345	void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)	345	void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
346	{	346	{
347	tlb_entry_t entry;	347	tlb_entry_t entry;
348		348
349	entry.word[0] = 0;	349	entry.word[0] = 0;
350	entry.word[1] = 0;	350	entry.word[1] = 0;
351		351
352	entry.p = true; /* present */	352	entry.p = true; /* present */
353	entry.ma = MA_WRITEBACK;	353	entry.ma = MA_WRITEBACK;
354	entry.a = true; /* already accessed */	354	entry.a = true; /* already accessed */
355	entry.d = true; /* already dirty */	355	entry.d = true; /* already dirty */
356	entry.pl = PL_KERNEL;	356	entry.pl = PL_KERNEL;
357	entry.ar = AR_READ \| AR_WRITE;	357	entry.ar = AR_READ \| AR_WRITE;
358	entry.ppn = frame >> PPN_SHIFT;	358	entry.ppn = frame >> PPN_SHIFT;
359	entry.ps = PAGE_WIDTH;	359	entry.ps = PAGE_WIDTH;
360		360
361	if (dtr)	361	if (dtr)
362	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);	362	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
363	else	363	else
364	dtc_mapping_insert(page, ASID_KERNEL, entry);	364	dtc_mapping_insert(page, ASID_KERNEL, entry);
365	}	365	}
366		366
367	/** Copy content of PTE into data translation cache.	367	/** Copy content of PTE into data translation cache.
368	*	368	*
369	* @param t PTE.	369	* @param t PTE.
370	*/	370	*/
371	void dtc_pte_copy(pte_t *t)	371	void dtc_pte_copy(pte_t *t)
372	{	372	{
373	tlb_entry_t entry;	373	tlb_entry_t entry;
374		374
375	entry.word[0] = 0;	375	entry.word[0] = 0;
376	entry.word[1] = 0;	376	entry.word[1] = 0;
377		377
378	entry.p = t->p;	378	entry.p = t->p;
379	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	379	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
380	entry.a = t->a;	380	entry.a = t->a;
381	entry.d = t->d;	381	entry.d = t->d;
382	entry.pl = t->k ? PL_KERNEL : PL_USER;	382	entry.pl = t->k ? PL_KERNEL : PL_USER;
383	entry.ar = t->w ? AR_WRITE : AR_READ;	383	entry.ar = t->w ? AR_WRITE : AR_READ;
384	entry.ppn = t->frame >> PPN_SHIFT;	384	entry.ppn = t->frame >> PPN_SHIFT;
385	entry.ps = PAGE_WIDTH;	385	entry.ps = PAGE_WIDTH;
386		386
387	dtc_mapping_insert(t->page, t->as->asid, entry);	387	dtc_mapping_insert(t->page, t->as->asid, entry);
388	}	388	}
389		389
390	/** Copy content of PTE into instruction translation cache.	390	/** Copy content of PTE into instruction translation cache.
391	*	391	*
392	* @param t PTE.	392	* @param t PTE.
393	*/	393	*/
394	void itc_pte_copy(pte_t *t)	394	void itc_pte_copy(pte_t *t)
395	{	395	{
396	tlb_entry_t entry;	396	tlb_entry_t entry;
397		397
398	entry.word[0] = 0;	398	entry.word[0] = 0;
399	entry.word[1] = 0;	399	entry.word[1] = 0;
400		400
401	ASSERT(t->x);	401	ASSERT(t->x);
402		402
403	entry.p = t->p;	403	entry.p = t->p;
404	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	404	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
405	entry.a = t->a;	405	entry.a = t->a;
406	entry.pl = t->k ? PL_KERNEL : PL_USER;	406	entry.pl = t->k ? PL_KERNEL : PL_USER;
407	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;	407	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;
408	entry.ppn = t->frame >> PPN_SHIFT;	408	entry.ppn = t->frame >> PPN_SHIFT;
409	entry.ps = PAGE_WIDTH;	409	entry.ps = PAGE_WIDTH;
410		410
411	itc_mapping_insert(t->page, t->as->asid, entry);	411	itc_mapping_insert(t->page, t->as->asid, entry);
412	}	412	}
413		413
414	/** Instruction TLB fault handler for faults with VHPT turned off.	414	/** Instruction TLB fault handler for faults with VHPT turned off.
415	*	415	*
416	* @param vector Interruption vector.	416	* @param vector Interruption vector.
417	* @param istate Structure with saved interruption state.	417	* @param istate Structure with saved interruption state.
418	*/	418	*/
419	void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)	419	void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
420	{	420	{
421	region_register rr;	421	region_register rr;
422	__address va;	422	__address va;
423	pte_t *t;	423	pte_t *t;
424		424
425	va = istate->cr_ifa; /* faulting address */	425	va = istate->cr_ifa; /* faulting address */
426	page_table_lock(AS, true);	426	page_table_lock(AS, true);
427	t = page_mapping_find(AS, va);	427	t = page_mapping_find(AS, va);
428	if (t) {	428	if (t) {
429	/*	429	/*
430	* The mapping was found in software page hash table.	430	* The mapping was found in software page hash table.
431	* Insert it into data translation cache.	431	* Insert it into data translation cache.
432	*/	432	*/
433	itc_pte_copy(t);	433	itc_pte_copy(t);
434	page_table_unlock(AS, true);	434	page_table_unlock(AS, true);
435	} else {	435	} else {
436	/*	436	/*
437	* Forward the page fault to address space page fault handler.	437	* Forward the page fault to address space page fault handler.
438	*/	438	*/
439	page_table_unlock(AS, true);	439	page_table_unlock(AS, true);
440	if (!as_page_fault(va)) {	440	if (!as_page_fault(va)) {
441	panic("%s: va=%P, rid=%d\n", __FUNCTION__, istate->cr_ifa, rr.map.rid);	441	panic("%s: va=%P, rid=%d, iip=%P\n", __FUNCTION__, istate->cr_ifa, rr.map.rid, istate->cr_iip);
442	}	442	}
443	}	443	}
444	}	444	}
445		445
446	/** Data TLB fault handler for faults with VHPT turned off.	446	/** Data TLB fault handler for faults with VHPT turned off.
447	*	447	*
448	* @param vector Interruption vector.	448	* @param vector Interruption vector.
449	* @param istate Structure with saved interruption state.	449	* @param istate Structure with saved interruption state.
450	*/	450	*/
451	void alternate_data_tlb_fault(__u64 vector, istate_t *istate)	451	void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
452	{	452	{
453	region_register rr;	453	region_register rr;
454	rid_t rid;	454	rid_t rid;
455	__address va;	455	__address va;
456	pte_t *t;	456	pte_t *t;
457		457
458	va = istate->cr_ifa; /* faulting address */	458	va = istate->cr_ifa; /* faulting address */
459	rr.word = rr_read(VA2VRN(va));	459	rr.word = rr_read(VA2VRN(va));
460	rid = rr.map.rid;	460	rid = rr.map.rid;
461	if (RID2ASID(rid) == ASID_KERNEL) {	461	if (RID2ASID(rid) == ASID_KERNEL) {
462	if (VA2VRN(va) == VRN_KERNEL) {	462	if (VA2VRN(va) == VRN_KERNEL) {
463	/*	463	/*
464	* Provide KA2PA(identity) mapping for faulting piece of	464	* Provide KA2PA(identity) mapping for faulting piece of
465	* kernel address space.	465	* kernel address space.
466	*/	466	*/
467	dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);	467	dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
468	return;	468	return;
469	}	469	}
470	}	470	}
471		471
472	page_table_lock(AS, true);	472	page_table_lock(AS, true);
473	t = page_mapping_find(AS, va);	473	t = page_mapping_find(AS, va);
474	if (t) {	474	if (t) {
475	/*	475	/*
476	* The mapping was found in software page hash table.	476	* The mapping was found in software page hash table.
477	* Insert it into data translation cache.	477	* Insert it into data translation cache.
478	*/	478	*/
479	dtc_pte_copy(t);	479	dtc_pte_copy(t);
480	page_table_unlock(AS, true);	480	page_table_unlock(AS, true);
481	} else {	481	} else {
482	/*	482	/*
483	* Forward the page fault to address space page fault handler.	483	* Forward the page fault to address space page fault handler.
484	*/	484	*/
485	page_table_unlock(AS, true);	485	page_table_unlock(AS, true);
486	if (!as_page_fault(va)) {	486	if (!as_page_fault(va)) {
487	panic("%s: va=%P, rid=%d, iip=%P\n", __FUNCTION__, va, rid, istate->cr_iip);	487	panic("%s: va=%P, rid=%d, iip=%P\n", __FUNCTION__, va, rid, istate->cr_iip);
488	}	488	}
489	}	489	}
490	}	490	}
491		491
492	/** Data nested TLB fault handler.	492	/** Data nested TLB fault handler.
493	*	493	*
494	* This fault should not occur.	494	* This fault should not occur.
495	*	495	*
496	* @param vector Interruption vector.	496	* @param vector Interruption vector.
497	* @param istate Structure with saved interruption state.	497	* @param istate Structure with saved interruption state.
498	*/	498	*/
499	void data_nested_tlb_fault(__u64 vector, istate_t *istate)	499	void data_nested_tlb_fault(__u64 vector, istate_t *istate)
500	{	500	{

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/arch/ia64/src/mm/tlb.c – Rev 1080 → 1104