WebSVN – HelenOS – Diff – /kernel/trunk/arch/ia64/src/mm/tlb.c


/*
 * Copyright (C) 2006 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
/*
 * TLB management.
 */
 
#include <mm/tlb.h>
#include <mm/asid.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/mm/tlb.h>
#include <arch/mm/page.h>
#include <arch/mm/vhpt.h>
#include <arch/barrier.h>
#include <arch/interrupt.h>
#include <arch/pal/pal.h>
#include <arch/asm.h>
#include <typedefs.h>
#include <panic.h>
#include <print.h>
#include <arch.h>
 
/** Invalidate all TLB entries. */
void tlb_invalidate_all(void)
{
        ipl_t ipl;
        __address adr;
        __u32 count1, count2, stride1, stride2;
       
        int i,j;
       
        adr = PAL_PTCE_INFO_BASE();
        count1 = PAL_PTCE_INFO_COUNT1();
        count2 = PAL_PTCE_INFO_COUNT2();
        stride1 = PAL_PTCE_INFO_STRIDE1();
        stride2 = PAL_PTCE_INFO_STRIDE2();
       
        ipl = interrupts_disable();
 
        for(i = 0; i < count1; i++) {
            for(j = 0; j < count2; j++) {
                __asm__ volatile (
                    "ptc.e %0 ;;"
                    :
                    : "r" (adr)
                );
                adr += stride2;
            }
            adr += stride1;
        }
 
        interrupts_restore(ipl);
 
        srlz_d();
        srlz_i();
#ifdef CONFIG_VHPT
        vhpt_invalidate_all();
#endif  
}
 
/** Invalidate entries belonging to an address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    tlb_invalidate_all();
}
 
 
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
{
    region_register rr;
    bool restore_rr = false;
    int b = 0;
    int c = cnt;
 
    __address va;
    va = page;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    while(c >>= 1)
        b++;
    b >>= 1;
    __u64 ps;
   
    switch (b) {
        case 0: /*cnt 1-3*/
            ps = PAGE_WIDTH;
            break;
        case 1: /*cnt 4-15*/
            /*cnt=((cnt-1)/4)+1;*/
            ps = PAGE_WIDTH+2;
            va &= ~((1<<ps)-1);
            break;
        case 2: /*cnt 16-63*/
            /*cnt=((cnt-1)/16)+1;*/
            ps = PAGE_WIDTH+4;
            va &= ~((1<<ps)-1);
            break;
        case 3: /*cnt 64-255*/
            /*cnt=((cnt-1)/64)+1;*/
            ps = PAGE_WIDTH+6;
            va &= ~((1<<ps)-1);
            break;
        case 4: /*cnt 256-1023*/
            /*cnt=((cnt-1)/256)+1;*/
            ps = PAGE_WIDTH+8;
            va &= ~((1<<ps)-1);
            break;
        case 5: /*cnt 1024-4095*/
            /*cnt=((cnt-1)/1024)+1;*/
            ps = PAGE_WIDTH+10;
            va &= ~((1<<ps)-1);
            break;
        case 6: /*cnt 4096-16383*/
            /*cnt=((cnt-1)/4096)+1;*/
            ps = PAGE_WIDTH+12;
            va &= ~((1<<ps)-1);
            break;
        case 7: /*cnt 16384-65535*/
        case 8: /*cnt 65536-(256K-1)*/
            /*cnt=((cnt-1)/16384)+1;*/
            ps = PAGE_WIDTH+14;
            va &= ~((1<<ps)-1);
            break;
        default:
            /*cnt=((cnt-1)/(16384*16))+1;*/
            ps=PAGE_WIDTH+18;
            va&=~((1<<ps)-1);
            break;
    }
    /*cnt+=(page!=va);*/
    for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
        __asm__ volatile (
            "ptc.l %0,%1;;"
            :
            : "r" (va), "r" (ps<<2)
        );
    }
    srlz_d();
    srlz_i();
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
 
/** Insert data into data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, true);
}
 
/** Insert data into instruction translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, false);
}
 
/** Insert data into instruction or data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
 */
void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    __asm__ volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"       /* va */
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
        "(p6) itc.i %3;;\n"
        "(p7) itc.d %3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into instruction translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, false, tr);
}
 
/** Insert data into data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, true, tr);
}
 
/** Insert data into instruction or data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation register, use instruction translation register otherwise.
 * @param tr Translation register.
 */
void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
 
    __asm__ volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"           /* va */         
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7=%5,r0;;\n"    /* decide between itr and dtr */
        "(p6) itr.i itr[%4]=%3;;\n"
        "(p7) itr.d dtr[%4]=%3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into DTLB.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtr If true, insert into data translation register, use data translation cache otherwise.
 * @param tr Translation register if dtr is true, ignored otherwise.
 */
void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
{
    tlb_entry_t entry;
   
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = true;         /* present */
    entry.ma = MA_WRITEBACK;
    entry.a = true;         /* already accessed */
    entry.d = true;         /* already dirty */
    entry.pl = PL_KERNEL;
    entry.ar = AR_READ | AR_WRITE;
    entry.ppn = frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    if (dtr)
        dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
    else
        dtc_mapping_insert(page, ASID_KERNEL, entry);
}
 
/** Copy content of PTE into data translation cache.
 *
 * @param t PTE.
 */
void dtc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.d = t->d;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->w ? AR_WRITE : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    dtc_mapping_insert(t->page, t->as->asid, entry);
#ifdef CONFIG_VHPT
    vhpt_mapping_insert(t->page, t->as->asid, entry);
#endif  
}
 
/** Copy content of PTE into instruction translation cache.
 *
 * @param t PTE.
 */
void itc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    ASSERT(t->x);
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    itc_mapping_insert(t->page, t->as->asid, entry);
#ifdef CONFIG_VHPT
    vhpt_mapping_insert(t->page, t->as->asid, entry);
#endif  
}
 
/** Instruction TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        itc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (as_page_fault(va, istate) == AS_PF_FAULT) {
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, istate->cr_ifa, rr.map.rid, istate->cr_iip);
        }
    }
}
 
/** Data TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
    if (RID2ASID(rid) == ASID_KERNEL) {
        if (VA2VRN(va) == VRN_KERNEL) {
            /*
             * Provide KA2PA(identity) mapping for faulting piece of
             * kernel address space.
             */
            dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
            return;
        }
    }
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (as_page_fault(va, istate) == AS_PF_FAULT) {
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
        }
    }
}
 
/** Data nested TLB fault handler.
 *
 * This fault should not occur.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_nested_tlb_fault(__u64 vector, istate_t *istate)
{
    panic("%s\n", __FUNCTION__);
}
 
/** Data Dirty bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_dirty_bit_fault(__u64 vector, istate_t *istate)
{
    pte_t *t;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, istate->cr_ifa);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Dirty bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->d = true;
        dtc_pte_copy(t);
    }
    page_table_unlock(AS, true);
}
 
/** Instruction access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void instruction_access_bit_fault(__u64 vector, istate_t *istate)
{
    pte_t *t;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, istate->cr_ifa);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into ITC.
         */
        t->a = true;
        itc_pte_copy(t);
    }
    page_table_unlock(AS, true);
}
 
/** Data access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_access_bit_fault(__u64 vector, istate_t *istate)
{
    pte_t *t;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, istate->cr_ifa);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->a = true;
        dtc_pte_copy(t);
    }
    page_table_unlock(AS, true);
}
 
/** Page not present fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void page_not_present(__u64 vector, istate_t *istate)
{
    region_register rr;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t);
   
    if (t->p) {
        /*
         * If the Present bit is set in page hash table, just copy it
         * and update ITC/DTC.
         */
        if (t->x)
            itc_pte_copy(t);
        else
            dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        page_table_unlock(AS, true);
        if (as_page_fault(va, istate) == AS_PF_FAULT) {
            panic("%s: va=%p, rid=%d\n", __FUNCTION__, va, rr.map.rid);
        }
    }
}
 

Rev 1221	Rev 1288
1	/*	1	/*
2	* Copyright (C) 2006 Jakub Jermar	2	* Copyright (C) 2006 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	/*	29	/*
30	* TLB management.	30	* TLB management.
31	*/	31	*/
32		32
33	#include <mm/tlb.h>	33	#include <mm/tlb.h>
34	#include <mm/asid.h>	34	#include <mm/asid.h>
35	#include <mm/page.h>	35	#include <mm/page.h>
36	#include <mm/as.h>	36	#include <mm/as.h>
37	#include <arch/mm/tlb.h>	37	#include <arch/mm/tlb.h>
38	#include <arch/mm/page.h>	38	#include <arch/mm/page.h>
39	#include <arch/mm/vhpt.h>	39	#include <arch/mm/vhpt.h>
40	#include <arch/barrier.h>	40	#include <arch/barrier.h>
41	#include <arch/interrupt.h>	41	#include <arch/interrupt.h>
42	#include <arch/pal/pal.h>	42	#include <arch/pal/pal.h>
43	#include <arch/asm.h>	43	#include <arch/asm.h>
44	#include <typedefs.h>	44	#include <typedefs.h>
45	#include <panic.h>	45	#include <panic.h>
46	#include <print.h>	46	#include <print.h>
47	#include <arch.h>	47	#include <arch.h>
48		48
49	/** Invalidate all TLB entries. */	49	/** Invalidate all TLB entries. */
50	void tlb_invalidate_all(void)	50	void tlb_invalidate_all(void)
51	{	51	{
52	ipl_t ipl;	52	ipl_t ipl;
53	__address adr;	53	__address adr;
54	__u32 count1, count2, stride1, stride2;	54	__u32 count1, count2, stride1, stride2;
55		55
56	int i,j;	56	int i,j;
57		57
58	adr = PAL_PTCE_INFO_BASE();	58	adr = PAL_PTCE_INFO_BASE();
59	count1 = PAL_PTCE_INFO_COUNT1();	59	count1 = PAL_PTCE_INFO_COUNT1();
60	count2 = PAL_PTCE_INFO_COUNT2();	60	count2 = PAL_PTCE_INFO_COUNT2();
61	stride1 = PAL_PTCE_INFO_STRIDE1();	61	stride1 = PAL_PTCE_INFO_STRIDE1();
62	stride2 = PAL_PTCE_INFO_STRIDE2();	62	stride2 = PAL_PTCE_INFO_STRIDE2();
63		63
64	ipl = interrupts_disable();	64	ipl = interrupts_disable();
65		65
66	for(i = 0; i < count1; i++) {	66	for(i = 0; i < count1; i++) {
67	for(j = 0; j < count2; j++) {	67	for(j = 0; j < count2; j++) {
68	__asm__ volatile (	68	__asm__ volatile (
69	"ptc.e %0 ;;"	69	"ptc.e %0 ;;"
70	:	70	:
71	: "r" (adr)	71	: "r" (adr)
72	);	72	);
73	adr += stride2;	73	adr += stride2;
74	}	74	}
75	adr += stride1;	75	adr += stride1;
76	}	76	}
77		77
78	interrupts_restore(ipl);	78	interrupts_restore(ipl);
79		79
80	srlz_d();	80	srlz_d();
81	srlz_i();	81	srlz_i();
82	#ifdef CONFIG_VHPT	82	#ifdef CONFIG_VHPT
83	vhpt_invalidate_all();	83	vhpt_invalidate_all();
84	#endif	84	#endif
85	}	85	}
86		86
87	/** Invalidate entries belonging to an address space.	87	/** Invalidate entries belonging to an address space.
88	*	88	*
89	* @param asid Address space identifier.	89	* @param asid Address space identifier.
90	*/	90	*/
91	void tlb_invalidate_asid(asid_t asid)	91	void tlb_invalidate_asid(asid_t asid)
92	{	92	{
93	tlb_invalidate_all();	93	tlb_invalidate_all();
94	}	94	}
95		95
96		96
97	void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)	97	void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
98	{	98	{
99	region_register rr;	99	region_register rr;
100	bool restore_rr = false;	100	bool restore_rr = false;
101	int b = 0;	101	int b = 0;
102	int c = cnt;	102	int c = cnt;
103		103
104	__address va;	104	__address va;
105	va = page;	105	va = page;
106		106
107	rr.word = rr_read(VA2VRN(va));	107	rr.word = rr_read(VA2VRN(va));
108	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	108	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
109	/*	109	/*
110	* The selected region register does not contain required RID.	110	* The selected region register does not contain required RID.
111	* Save the old content of the register and replace the RID.	111	* Save the old content of the register and replace the RID.
112	*/	112	*/
113	region_register rr0;	113	region_register rr0;
114		114
115	rr0 = rr;	115	rr0 = rr;
116	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	116	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
117	rr_write(VA2VRN(va), rr0.word);	117	rr_write(VA2VRN(va), rr0.word);
118	srlz_d();	118	srlz_d();
119	srlz_i();	119	srlz_i();
120	}	120	}
121		121
122	while(c >>= 1)	122	while(c >>= 1)
123	b++;	123	b++;
124	b >>= 1;	124	b >>= 1;
125	__u64 ps;	125	__u64 ps;
126		126
127	switch (b) {	127	switch (b) {
128	case 0: /cnt 1-3/	128	case 0: /cnt 1-3/
129	ps = PAGE_WIDTH;	129	ps = PAGE_WIDTH;
130	break;	130	break;
131	case 1: /cnt 4-15/	131	case 1: /cnt 4-15/
132	/cnt=((cnt-1)/4)+1;/	132	/cnt=((cnt-1)/4)+1;/
133	ps = PAGE_WIDTH+2;	133	ps = PAGE_WIDTH+2;
134	va &= ~((1<<ps)-1);	134	va &= ~((1<<ps)-1);
135	break;	135	break;
136	case 2: /cnt 16-63/	136	case 2: /cnt 16-63/
137	/cnt=((cnt-1)/16)+1;/	137	/cnt=((cnt-1)/16)+1;/
138	ps = PAGE_WIDTH+4;	138	ps = PAGE_WIDTH+4;
139	va &= ~((1<<ps)-1);	139	va &= ~((1<<ps)-1);
140	break;	140	break;
141	case 3: /cnt 64-255/	141	case 3: /cnt 64-255/
142	/cnt=((cnt-1)/64)+1;/	142	/cnt=((cnt-1)/64)+1;/
143	ps = PAGE_WIDTH+6;	143	ps = PAGE_WIDTH+6;
144	va &= ~((1<<ps)-1);	144	va &= ~((1<<ps)-1);
145	break;	145	break;
146	case 4: /cnt 256-1023/	146	case 4: /cnt 256-1023/
147	/cnt=((cnt-1)/256)+1;/	147	/cnt=((cnt-1)/256)+1;/
148	ps = PAGE_WIDTH+8;	148	ps = PAGE_WIDTH+8;
149	va &= ~((1<<ps)-1);	149	va &= ~((1<<ps)-1);
150	break;	150	break;
151	case 5: /cnt 1024-4095/	151	case 5: /cnt 1024-4095/
152	/cnt=((cnt-1)/1024)+1;/	152	/cnt=((cnt-1)/1024)+1;/
153	ps = PAGE_WIDTH+10;	153	ps = PAGE_WIDTH+10;
154	va &= ~((1<<ps)-1);	154	va &= ~((1<<ps)-1);
155	break;	155	break;
156	case 6: /cnt 4096-16383/	156	case 6: /cnt 4096-16383/
157	/cnt=((cnt-1)/4096)+1;/	157	/cnt=((cnt-1)/4096)+1;/
158	ps = PAGE_WIDTH+12;	158	ps = PAGE_WIDTH+12;
159	va &= ~((1<<ps)-1);	159	va &= ~((1<<ps)-1);
160	break;	160	break;
161	case 7: /cnt 16384-65535/	161	case 7: /cnt 16384-65535/
162	case 8: /cnt 65536-(256K-1)/	162	case 8: /cnt 65536-(256K-1)/
163	/cnt=((cnt-1)/16384)+1;/	163	/cnt=((cnt-1)/16384)+1;/
164	ps = PAGE_WIDTH+14;	164	ps = PAGE_WIDTH+14;
165	va &= ~((1<<ps)-1);	165	va &= ~((1<<ps)-1);
166	break;	166	break;
167	default:	167	default:
168	/cnt=((cnt-1)/(1638416))+1;*/	168	/cnt=((cnt-1)/(1638416))+1;*/
169	ps=PAGE_WIDTH+18;	169	ps=PAGE_WIDTH+18;
170	va&=~((1<<ps)-1);	170	va&=~((1<<ps)-1);
171	break;	171	break;
172	}	172	}
173	/cnt+=(page!=va);/	173	/cnt+=(page!=va);/
174	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {	174	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
175	__asm__ volatile (	175	__asm__ volatile (
176	"ptc.l %0,%1;;"	176	"ptc.l %0,%1;;"
177	:	177	:
178	: "r" (va), "r" (ps<<2)	178	: "r" (va), "r" (ps<<2)
179	);	179	);
180	}	180	}
181	srlz_d();	181	srlz_d();
182	srlz_i();	182	srlz_i();
183		183
184	if (restore_rr) {	184	if (restore_rr) {
185	rr_write(VA2VRN(va), rr.word);	185	rr_write(VA2VRN(va), rr.word);
186	srlz_d();	186	srlz_d();
187	srlz_i();	187	srlz_i();
188	}	188	}
189	}	189	}
190		190
191		191
192	/** Insert data into data translation cache.	192	/** Insert data into data translation cache.
193	*	193	*
194	* @param va Virtual page address.	194	* @param va Virtual page address.
195	* @param asid Address space identifier.	195	* @param asid Address space identifier.
196	* @param entry The rest of TLB entry as required by TLB insertion format.	196	* @param entry The rest of TLB entry as required by TLB insertion format.
197	*/	197	*/
198	void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)	198	void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
199	{	199	{
200	tc_mapping_insert(va, asid, entry, true);	200	tc_mapping_insert(va, asid, entry, true);
201	}	201	}
202		202
203	/** Insert data into instruction translation cache.	203	/** Insert data into instruction translation cache.
204	*	204	*
205	* @param va Virtual page address.	205	* @param va Virtual page address.
206	* @param asid Address space identifier.	206	* @param asid Address space identifier.
207	* @param entry The rest of TLB entry as required by TLB insertion format.	207	* @param entry The rest of TLB entry as required by TLB insertion format.
208	*/	208	*/
209	void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)	209	void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
210	{	210	{
211	tc_mapping_insert(va, asid, entry, false);	211	tc_mapping_insert(va, asid, entry, false);
212	}	212	}
213		213
214	/** Insert data into instruction or data translation cache.	214	/** Insert data into instruction or data translation cache.
215	*	215	*
216	* @param va Virtual page address.	216	* @param va Virtual page address.
217	* @param asid Address space identifier.	217	* @param asid Address space identifier.
218	* @param entry The rest of TLB entry as required by TLB insertion format.	218	* @param entry The rest of TLB entry as required by TLB insertion format.
219	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.	219	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
220	*/	220	*/
221	void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)	221	void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
222	{	222	{
223	region_register rr;	223	region_register rr;
224	bool restore_rr = false;	224	bool restore_rr = false;
225		225
226	rr.word = rr_read(VA2VRN(va));	226	rr.word = rr_read(VA2VRN(va));
227	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	227	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
228	/*	228	/*
229	* The selected region register does not contain required RID.	229	* The selected region register does not contain required RID.
230	* Save the old content of the register and replace the RID.	230	* Save the old content of the register and replace the RID.
231	*/	231	*/
232	region_register rr0;	232	region_register rr0;
233		233
234	rr0 = rr;	234	rr0 = rr;
235	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	235	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
236	rr_write(VA2VRN(va), rr0.word);	236	rr_write(VA2VRN(va), rr0.word);
237	srlz_d();	237	srlz_d();
238	srlz_i();	238	srlz_i();
239	}	239	}
240		240
241	__asm__ volatile (	241	__asm__ volatile (
242	"mov r8=psr;;\n"	242	"mov r8=psr;;\n"
243	"rsm %0;;\n" /* PSR_IC_MASK */	243	"rsm %0;;\n" /* PSR_IC_MASK */
244	"srlz.d;;\n"	244	"srlz.d;;\n"
245	"srlz.i;;\n"	245	"srlz.i;;\n"
246	"mov cr.ifa=%1\n" /* va */	246	"mov cr.ifa=%1\n" /* va */
247	"mov cr.itir=%2;;\n" /* entry.word[1] */	247	"mov cr.itir=%2;;\n" /* entry.word[1] */
248	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */	248	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */
249	"(p6) itc.i %3;;\n"	249	"(p6) itc.i %3;;\n"
250	"(p7) itc.d %3;;\n"	250	"(p7) itc.d %3;;\n"
251	"mov psr.l=r8;;\n"	251	"mov psr.l=r8;;\n"
252	"srlz.d;;\n"	252	"srlz.d;;\n"
253	:	253	:
254	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)	254	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
255	: "p6", "p7", "r8"	255	: "p6", "p7", "r8"
256	);	256	);
257		257
258	if (restore_rr) {	258	if (restore_rr) {
259	rr_write(VA2VRN(va), rr.word);	259	rr_write(VA2VRN(va), rr.word);
260	srlz_d();	260	srlz_d();
261	srlz_i();	261	srlz_i();
262	}	262	}
263	}	263	}
264		264
265	/** Insert data into instruction translation register.	265	/** Insert data into instruction translation register.
266	*	266	*
267	* @param va Virtual page address.	267	* @param va Virtual page address.
268	* @param asid Address space identifier.	268	* @param asid Address space identifier.
269	* @param entry The rest of TLB entry as required by TLB insertion format.	269	* @param entry The rest of TLB entry as required by TLB insertion format.
270	* @param tr Translation register.	270	* @param tr Translation register.
271	*/	271	*/
272	void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)	272	void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
273	{	273	{
274	tr_mapping_insert(va, asid, entry, false, tr);	274	tr_mapping_insert(va, asid, entry, false, tr);
275	}	275	}
276		276
277	/** Insert data into data translation register.	277	/** Insert data into data translation register.
278	*	278	*
279	* @param va Virtual page address.	279	* @param va Virtual page address.
280	* @param asid Address space identifier.	280	* @param asid Address space identifier.
281	* @param entry The rest of TLB entry as required by TLB insertion format.	281	* @param entry The rest of TLB entry as required by TLB insertion format.
282	* @param tr Translation register.	282	* @param tr Translation register.
283	*/	283	*/
284	void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)	284	void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
285	{	285	{
286	tr_mapping_insert(va, asid, entry, true, tr);	286	tr_mapping_insert(va, asid, entry, true, tr);
287	}	287	}
288		288
289	/** Insert data into instruction or data translation register.	289	/** Insert data into instruction or data translation register.
290	*	290	*
291	* @param va Virtual page address.	291	* @param va Virtual page address.
292	* @param asid Address space identifier.	292	* @param asid Address space identifier.
293	* @param entry The rest of TLB entry as required by TLB insertion format.	293	* @param entry The rest of TLB entry as required by TLB insertion format.
294	* @param dtc If true, insert into data translation register, use instruction translation register otherwise.	294	* @param dtc If true, insert into data translation register, use instruction translation register otherwise.
295	* @param tr Translation register.	295	* @param tr Translation register.
296	*/	296	*/
297	void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)	297	void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
298	{	298	{
299	region_register rr;	299	region_register rr;
300	bool restore_rr = false;	300	bool restore_rr = false;
301		301
302	rr.word = rr_read(VA2VRN(va));	302	rr.word = rr_read(VA2VRN(va));
303	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	303	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
304	/*	304	/*
305	* The selected region register does not contain required RID.	305	* The selected region register does not contain required RID.
306	* Save the old content of the register and replace the RID.	306	* Save the old content of the register and replace the RID.
307	*/	307	*/
308	region_register rr0;	308	region_register rr0;
309		309
310	rr0 = rr;	310	rr0 = rr;
311	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	311	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
312	rr_write(VA2VRN(va), rr0.word);	312	rr_write(VA2VRN(va), rr0.word);
313	srlz_d();	313	srlz_d();
314	srlz_i();	314	srlz_i();
315	}	315	}
316		316
317	__asm__ volatile (	317	__asm__ volatile (
318	"mov r8=psr;;\n"	318	"mov r8=psr;;\n"
319	"rsm %0;;\n" /* PSR_IC_MASK */	319	"rsm %0;;\n" /* PSR_IC_MASK */
320	"srlz.d;;\n"	320	"srlz.d;;\n"
321	"srlz.i;;\n"	321	"srlz.i;;\n"
322	"mov cr.ifa=%1\n" /* va */	322	"mov cr.ifa=%1\n" /* va */
323	"mov cr.itir=%2;;\n" /* entry.word[1] */	323	"mov cr.itir=%2;;\n" /* entry.word[1] */
324	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */	324	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */
325	"(p6) itr.i itr[%4]=%3;;\n"	325	"(p6) itr.i itr[%4]=%3;;\n"
326	"(p7) itr.d dtr[%4]=%3;;\n"	326	"(p7) itr.d dtr[%4]=%3;;\n"
327	"mov psr.l=r8;;\n"	327	"mov psr.l=r8;;\n"
328	"srlz.d;;\n"	328	"srlz.d;;\n"
329	:	329	:
330	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)	330	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
331	: "p6", "p7", "r8"	331	: "p6", "p7", "r8"
332	);	332	);
333		333
334	if (restore_rr) {	334	if (restore_rr) {
335	rr_write(VA2VRN(va), rr.word);	335	rr_write(VA2VRN(va), rr.word);
336	srlz_d();	336	srlz_d();
337	srlz_i();	337	srlz_i();
338	}	338	}
339	}	339	}
340		340
341	/** Insert data into DTLB.	341	/** Insert data into DTLB.
342	*	342	*
343	* @param va Virtual page address.	343	* @param va Virtual page address.
344	* @param asid Address space identifier.	344	* @param asid Address space identifier.
345	* @param entry The rest of TLB entry as required by TLB insertion format.	345	* @param entry The rest of TLB entry as required by TLB insertion format.
346	* @param dtr If true, insert into data translation register, use data translation cache otherwise.	346	* @param dtr If true, insert into data translation register, use data translation cache otherwise.
347	* @param tr Translation register if dtr is true, ignored otherwise.	347	* @param tr Translation register if dtr is true, ignored otherwise.
348	*/	348	*/
349	void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)	349	void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
350	{	350	{
351	tlb_entry_t entry;	351	tlb_entry_t entry;
352		352
353	entry.word[0] = 0;	353	entry.word[0] = 0;
354	entry.word[1] = 0;	354	entry.word[1] = 0;
355		355
356	entry.p = true; /* present */	356	entry.p = true; /* present */
357	entry.ma = MA_WRITEBACK;	357	entry.ma = MA_WRITEBACK;
358	entry.a = true; /* already accessed */	358	entry.a = true; /* already accessed */
359	entry.d = true; /* already dirty */	359	entry.d = true; /* already dirty */
360	entry.pl = PL_KERNEL;	360	entry.pl = PL_KERNEL;
361	entry.ar = AR_READ \| AR_WRITE;	361	entry.ar = AR_READ \| AR_WRITE;
362	entry.ppn = frame >> PPN_SHIFT;	362	entry.ppn = frame >> PPN_SHIFT;
363	entry.ps = PAGE_WIDTH;	363	entry.ps = PAGE_WIDTH;
364		364
365	if (dtr)	365	if (dtr)
366	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);	366	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
367	else	367	else
368	dtc_mapping_insert(page, ASID_KERNEL, entry);	368	dtc_mapping_insert(page, ASID_KERNEL, entry);
369	}	369	}
370		370
371	/** Copy content of PTE into data translation cache.	371	/** Copy content of PTE into data translation cache.
372	*	372	*
373	* @param t PTE.	373	* @param t PTE.
374	*/	374	*/
375	void dtc_pte_copy(pte_t *t)	375	void dtc_pte_copy(pte_t *t)
376	{	376	{
377	tlb_entry_t entry;	377	tlb_entry_t entry;
378		378
379	entry.word[0] = 0;	379	entry.word[0] = 0;
380	entry.word[1] = 0;	380	entry.word[1] = 0;
381		381
382	entry.p = t->p;	382	entry.p = t->p;
383	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	383	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
384	entry.a = t->a;	384	entry.a = t->a;
385	entry.d = t->d;	385	entry.d = t->d;
386	entry.pl = t->k ? PL_KERNEL : PL_USER;	386	entry.pl = t->k ? PL_KERNEL : PL_USER;
387	entry.ar = t->w ? AR_WRITE : AR_READ;	387	entry.ar = t->w ? AR_WRITE : AR_READ;
388	entry.ppn = t->frame >> PPN_SHIFT;	388	entry.ppn = t->frame >> PPN_SHIFT;
389	entry.ps = PAGE_WIDTH;	389	entry.ps = PAGE_WIDTH;
390		390
391	dtc_mapping_insert(t->page, t->as->asid, entry);	391	dtc_mapping_insert(t->page, t->as->asid, entry);
392	#ifdef CONFIG_VHPT	392	#ifdef CONFIG_VHPT
393	vhpt_mapping_insert(t->page, t->as->asid, entry);	393	vhpt_mapping_insert(t->page, t->as->asid, entry);
394	#endif	394	#endif
395	}	395	}
396		396
397	/** Copy content of PTE into instruction translation cache.	397	/** Copy content of PTE into instruction translation cache.
398	*	398	*
399	* @param t PTE.	399	* @param t PTE.
400	*/	400	*/
401	void itc_pte_copy(pte_t *t)	401	void itc_pte_copy(pte_t *t)
402	{	402	{
403	tlb_entry_t entry;	403	tlb_entry_t entry;
404		404
405	entry.word[0] = 0;	405	entry.word[0] = 0;
406	entry.word[1] = 0;	406	entry.word[1] = 0;
407		407
408	ASSERT(t->x);	408	ASSERT(t->x);
409		409
410	entry.p = t->p;	410	entry.p = t->p;
411	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	411	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
412	entry.a = t->a;	412	entry.a = t->a;
413	entry.pl = t->k ? PL_KERNEL : PL_USER;	413	entry.pl = t->k ? PL_KERNEL : PL_USER;
414	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;	414	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;
415	entry.ppn = t->frame >> PPN_SHIFT;	415	entry.ppn = t->frame >> PPN_SHIFT;
416	entry.ps = PAGE_WIDTH;	416	entry.ps = PAGE_WIDTH;
417		417
418	itc_mapping_insert(t->page, t->as->asid, entry);	418	itc_mapping_insert(t->page, t->as->asid, entry);
419	#ifdef CONFIG_VHPT	419	#ifdef CONFIG_VHPT
420	vhpt_mapping_insert(t->page, t->as->asid, entry);	420	vhpt_mapping_insert(t->page, t->as->asid, entry);
421	#endif	421	#endif
422	}	422	}
423		423
424	/** Instruction TLB fault handler for faults with VHPT turned off.	424	/** Instruction TLB fault handler for faults with VHPT turned off.
425	*	425	*
426	* @param vector Interruption vector.	426	* @param vector Interruption vector.
427	* @param istate Structure with saved interruption state.	427	* @param istate Structure with saved interruption state.
428	*/	428	*/
429	void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)	429	void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
430	{	430	{
431	region_register rr;	431	region_register rr;
432	__address va;	432	__address va;
433	pte_t *t;	433	pte_t *t;
434		434
435	va = istate->cr_ifa; /* faulting address */	435	va = istate->cr_ifa; /* faulting address */
436	page_table_lock(AS, true);	436	page_table_lock(AS, true);
437	t = page_mapping_find(AS, va);	437	t = page_mapping_find(AS, va);
438	if (t) {	438	if (t) {
439	/*	439	/*
440	* The mapping was found in software page hash table.	440	* The mapping was found in software page hash table.
441	* Insert it into data translation cache.	441	* Insert it into data translation cache.
442	*/	442	*/
443	itc_pte_copy(t);	443	itc_pte_copy(t);
444	page_table_unlock(AS, true);	444	page_table_unlock(AS, true);
445	} else {	445	} else {
446	/*	446	/*
447	* Forward the page fault to address space page fault handler.	447	* Forward the page fault to address space page fault handler.
448	*/	448	*/
449	page_table_unlock(AS, true);	449	page_table_unlock(AS, true);
450	if (!as_page_fault(va)) {	450	if (as_page_fault(va, istate) == AS_PF_FAULT) {
451	panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, istate->cr_ifa, rr.map.rid, istate->cr_iip);	451	panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, istate->cr_ifa, rr.map.rid, istate->cr_iip);
452	}	452	}
453	}	453	}
454	}	454	}
455		455
456	/** Data TLB fault handler for faults with VHPT turned off.	456	/** Data TLB fault handler for faults with VHPT turned off.
457	*	457	*
458	* @param vector Interruption vector.	458	* @param vector Interruption vector.
459	* @param istate Structure with saved interruption state.	459	* @param istate Structure with saved interruption state.
460	*/	460	*/
461	void alternate_data_tlb_fault(__u64 vector, istate_t *istate)	461	void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
462	{	462	{
463	region_register rr;	463	region_register rr;
464	rid_t rid;	464	rid_t rid;
465	__address va;	465	__address va;
466	pte_t *t;	466	pte_t *t;
467		467
468	va = istate->cr_ifa; /* faulting address */	468	va = istate->cr_ifa; /* faulting address */
469	rr.word = rr_read(VA2VRN(va));	469	rr.word = rr_read(VA2VRN(va));
470	rid = rr.map.rid;	470	rid = rr.map.rid;
471	if (RID2ASID(rid) == ASID_KERNEL) {	471	if (RID2ASID(rid) == ASID_KERNEL) {
472	if (VA2VRN(va) == VRN_KERNEL) {	472	if (VA2VRN(va) == VRN_KERNEL) {
473	/*	473	/*
474	* Provide KA2PA(identity) mapping for faulting piece of	474	* Provide KA2PA(identity) mapping for faulting piece of
475	* kernel address space.	475	* kernel address space.
476	*/	476	*/
477	dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);	477	dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
478	return;	478	return;
479	}	479	}
480	}	480	}
481		481
482	page_table_lock(AS, true);	482	page_table_lock(AS, true);
483	t = page_mapping_find(AS, va);	483	t = page_mapping_find(AS, va);
484	if (t) {	484	if (t) {
485	/*	485	/*
486	* The mapping was found in software page hash table.	486	* The mapping was found in software page hash table.
487	* Insert it into data translation cache.	487	* Insert it into data translation cache.
488	*/	488	*/
489	dtc_pte_copy(t);	489	dtc_pte_copy(t);
490	page_table_unlock(AS, true);	490	page_table_unlock(AS, true);
491	} else {	491	} else {
492	/*	492	/*
493	* Forward the page fault to address space page fault handler.	493	* Forward the page fault to address space page fault handler.
494	*/	494	*/
495	page_table_unlock(AS, true);	495	page_table_unlock(AS, true);
496	if (!as_page_fault(va)) {	496	if (as_page_fault(va, istate) == AS_PF_FAULT) {
497	panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);	497	panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
498	}	498	}
499	}	499	}
500	}	500	}

Subversion Repositories HelenOS

(root)/kernel/trunk/arch/ia64/src/mm/tlb.c – Rev 1221 → 1288