WebSVN – HelenOS – Diff – /tags/0.2.0/kernel/arch/ia64/src/mm/tlb.c


/*
 * Copyright (C) 2006 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
/*
 * TLB management.
 */
 
#include <mm/tlb.h>
#include <mm/asid.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/mm/tlb.h>
#include <arch/mm/page.h>
#include <arch/mm/vhpt.h>
#include <arch/barrier.h>
#include <arch/interrupt.h>
#include <arch/pal/pal.h>
#include <arch/asm.h>
#include <typedefs.h>
#include <panic.h>
#include <print.h>
#include <arch.h>
#include <interrupt.h>
 
/** Invalidate all TLB entries. */
void tlb_invalidate_all(void)
{
        ipl_t ipl;
        __address adr;
        __u32 count1, count2, stride1, stride2;
       
        int i,j;
       
        adr = PAL_PTCE_INFO_BASE();
        count1 = PAL_PTCE_INFO_COUNT1();
        count2 = PAL_PTCE_INFO_COUNT2();
        stride1 = PAL_PTCE_INFO_STRIDE1();
        stride2 = PAL_PTCE_INFO_STRIDE2();
       
        ipl = interrupts_disable();
 
        for(i = 0; i < count1; i++) {
            for(j = 0; j < count2; j++) {
                __asm__ volatile (
                    "ptc.e %0 ;;"
                    :
                    : "r" (adr)
                );
                adr += stride2;
            }
            adr += stride1;
        }
 
        interrupts_restore(ipl);
 
        srlz_d();
        srlz_i();
#ifdef CONFIG_VHPT
        vhpt_invalidate_all();
#endif  
}
 
/** Invalidate entries belonging to an address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    tlb_invalidate_all();
}
 
 
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
{
    region_register rr;
    bool restore_rr = false;
    int b = 0;
    int c = cnt;
 
    __address va;
    va = page;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    while(c >>= 1)
        b++;
    b >>= 1;
    __u64 ps;
   
    switch (b) {
        case 0: /*cnt 1-3*/
            ps = PAGE_WIDTH;
            break;
        case 1: /*cnt 4-15*/
            /*cnt=((cnt-1)/4)+1;*/
            ps = PAGE_WIDTH+2;
            va &= ~((1<<ps)-1);
            break;
        case 2: /*cnt 16-63*/
            /*cnt=((cnt-1)/16)+1;*/
            ps = PAGE_WIDTH+4;
            va &= ~((1<<ps)-1);
            break;
        case 3: /*cnt 64-255*/
            /*cnt=((cnt-1)/64)+1;*/
            ps = PAGE_WIDTH+6;
            va &= ~((1<<ps)-1);
            break;
        case 4: /*cnt 256-1023*/
            /*cnt=((cnt-1)/256)+1;*/
            ps = PAGE_WIDTH+8;
            va &= ~((1<<ps)-1);
            break;
        case 5: /*cnt 1024-4095*/
            /*cnt=((cnt-1)/1024)+1;*/
            ps = PAGE_WIDTH+10;
            va &= ~((1<<ps)-1);
            break;
        case 6: /*cnt 4096-16383*/
            /*cnt=((cnt-1)/4096)+1;*/
            ps = PAGE_WIDTH+12;
            va &= ~((1<<ps)-1);
            break;
        case 7: /*cnt 16384-65535*/
        case 8: /*cnt 65536-(256K-1)*/
            /*cnt=((cnt-1)/16384)+1;*/
            ps = PAGE_WIDTH+14;
            va &= ~((1<<ps)-1);
            break;
        default:
            /*cnt=((cnt-1)/(16384*16))+1;*/
            ps=PAGE_WIDTH+18;
            va&=~((1<<ps)-1);
            break;
    }
    /*cnt+=(page!=va);*/
    for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
        __asm__ volatile (
            "ptc.l %0,%1;;"
            :
            : "r" (va), "r" (ps<<2)
        );
    }
    srlz_d();
    srlz_i();
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
 
/** Insert data into data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, true);
}
 
/** Insert data into instruction translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, false);
}
 
/** Insert data into instruction or data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
 */
void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    __asm__ volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"       /* va */
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
        "(p6) itc.i %3;;\n"
        "(p7) itc.d %3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into instruction translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, false, tr);
}
 
/** Insert data into data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, true, tr);
}
 
/** Insert data into instruction or data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation register, use instruction translation register otherwise.
 * @param tr Translation register.
 */
void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
 
    __asm__ volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"           /* va */         
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7=%5,r0;;\n"    /* decide between itr and dtr */
        "(p6) itr.i itr[%4]=%3;;\n"
        "(p7) itr.d dtr[%4]=%3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into DTLB.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtr If true, insert into data translation register, use data translation cache otherwise.
 * @param tr Translation register if dtr is true, ignored otherwise.
 */
void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
{
    tlb_entry_t entry;
   
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = true;         /* present */
    entry.ma = MA_WRITEBACK;
    entry.a = true;         /* already accessed */
    entry.d = true;         /* already dirty */
    entry.pl = PL_KERNEL;
    entry.ar = AR_READ | AR_WRITE;
    entry.ppn = frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    if (dtr)
        dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
    else
        dtc_mapping_insert(page, ASID_KERNEL, entry);
}
 
/** Copy content of PTE into data translation cache.
 *
 * @param t PTE.
 */
void dtc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.d = t->d;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->w ? AR_WRITE : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    dtc_mapping_insert(t->page, t->as->asid, entry);
#ifdef CONFIG_VHPT
    vhpt_mapping_insert(t->page, t->as->asid, entry);
#endif  
}
 
/** Copy content of PTE into instruction translation cache.
 *
 * @param t PTE.
 */
void itc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    ASSERT(t->x);
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    itc_mapping_insert(t->page, t->as->asid, entry);
#ifdef CONFIG_VHPT
    vhpt_mapping_insert(t->page, t->as->asid, entry);
#endif  
}
 
/** Instruction TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        itc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %P",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
        }
    }
}
 
/** Data TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
    if (RID2ASID(rid) == ASID_KERNEL) {
        if (VA2VRN(va) == VRN_KERNEL) {
            /*
             * Provide KA2PA(identity) mapping for faulting piece of
             * kernel address space.
             */
            dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
            return;
        }
    }
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %P",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
        }
    }
}
 
/** Data nested TLB fault handler.
 *
 * This fault should not occur.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_nested_tlb_fault(__u64 vector, istate_t *istate)
{
    panic("%s\n", __FUNCTION__);
}
 
/** Data Dirty bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_dirty_bit_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t && t->p);
    if (t && t->p && t->w) {
        /*
         * Update the Dirty bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->d = true;
        dtc_pte_copy(t);
    } else {
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %P",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
            t->d = true;
            dtc_pte_copy(t);
        }
    }
    page_table_unlock(AS, true);
}
 
/** Instruction access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void instruction_access_bit_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;  
 
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t && t->p);
    if (t && t->p && t->x) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into ITC.
         */
        t->a = true;
        itc_pte_copy(t);
    } else {
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %P",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
            t->a = true;
            itc_pte_copy(t);
        }
    }
    page_table_unlock(AS, true);
}
 
/** Data access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_access_bit_fault(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
 
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->a = true;
        dtc_pte_copy(t);
    } else {
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %P",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
            t->a = true;
            itc_pte_copy(t);
        }
    }
    page_table_unlock(AS, true);
}
 
/** Page not present fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void page_not_present(__u64 vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    __address va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t);
   
    if (t->p) {
        /*
         * If the Present bit is set in page hash table, just copy it
         * and update ITC/DTC.
         */
        if (t->x)
            itc_pte_copy(t);
        else
            dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        page_table_unlock(AS, true);
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %P",va);
            panic("%s: va=%p, rid=%d\n", __FUNCTION__, va, rid);
        }
    }
}
 

Rev 1964	Rev 1968
1	/*	1	/*
2	* Copyright (C) 2006 Jakub Jermar	2	* Copyright (C) 2006 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	/*	29	/*
30	* TLB management.	30	* TLB management.
31	*/	31	*/
32		32
33	#include <mm/tlb.h>	33	#include <mm/tlb.h>
34	#include <mm/asid.h>	34	#include <mm/asid.h>
35	#include <mm/page.h>	35	#include <mm/page.h>
36	#include <mm/as.h>	36	#include <mm/as.h>
37	#include <arch/mm/tlb.h>	37	#include <arch/mm/tlb.h>
38	#include <arch/mm/page.h>	38	#include <arch/mm/page.h>
39	#include <arch/mm/vhpt.h>	39	#include <arch/mm/vhpt.h>
40	#include <arch/barrier.h>	40	#include <arch/barrier.h>
41	#include <arch/interrupt.h>	41	#include <arch/interrupt.h>
42	#include <arch/pal/pal.h>	42	#include <arch/pal/pal.h>
43	#include <arch/asm.h>	43	#include <arch/asm.h>
44	#include <typedefs.h>	44	#include <typedefs.h>
45	#include <panic.h>	45	#include <panic.h>
46	#include <print.h>	46	#include <print.h>
47	#include <arch.h>	47	#include <arch.h>
48	#include <interrupt.h>	48	#include <interrupt.h>
49		49
50	/** Invalidate all TLB entries. */	50	/** Invalidate all TLB entries. */
51	void tlb_invalidate_all(void)	51	void tlb_invalidate_all(void)
52	{	52	{
53	ipl_t ipl;	53	ipl_t ipl;
54	__address adr;	54	__address adr;
55	__u32 count1, count2, stride1, stride2;	55	__u32 count1, count2, stride1, stride2;
56		56
57	int i,j;	57	int i,j;
58		58
59	adr = PAL_PTCE_INFO_BASE();	59	adr = PAL_PTCE_INFO_BASE();
60	count1 = PAL_PTCE_INFO_COUNT1();	60	count1 = PAL_PTCE_INFO_COUNT1();
61	count2 = PAL_PTCE_INFO_COUNT2();	61	count2 = PAL_PTCE_INFO_COUNT2();
62	stride1 = PAL_PTCE_INFO_STRIDE1();	62	stride1 = PAL_PTCE_INFO_STRIDE1();
63	stride2 = PAL_PTCE_INFO_STRIDE2();	63	stride2 = PAL_PTCE_INFO_STRIDE2();
64		64
65	ipl = interrupts_disable();	65	ipl = interrupts_disable();
66		66
67	for(i = 0; i < count1; i++) {	67	for(i = 0; i < count1; i++) {
68	for(j = 0; j < count2; j++) {	68	for(j = 0; j < count2; j++) {
69	__asm__ volatile (	69	__asm__ volatile (
70	"ptc.e %0 ;;"	70	"ptc.e %0 ;;"
71	:	71	:
72	: "r" (adr)	72	: "r" (adr)
73	);	73	);
74	adr += stride2;	74	adr += stride2;
75	}	75	}
76	adr += stride1;	76	adr += stride1;
77	}	77	}
78		78
79	interrupts_restore(ipl);	79	interrupts_restore(ipl);
80		80
81	srlz_d();	81	srlz_d();
82	srlz_i();	82	srlz_i();
83	#ifdef CONFIG_VHPT	83	#ifdef CONFIG_VHPT
84	vhpt_invalidate_all();	84	vhpt_invalidate_all();
85	#endif	85	#endif
86	}	86	}
87		87
88	/** Invalidate entries belonging to an address space.	88	/** Invalidate entries belonging to an address space.
89	*	89	*
90	* @param asid Address space identifier.	90	* @param asid Address space identifier.
91	*/	91	*/
92	void tlb_invalidate_asid(asid_t asid)	92	void tlb_invalidate_asid(asid_t asid)
93	{	93	{
94	tlb_invalidate_all();	94	tlb_invalidate_all();
95	}	95	}
96		96
97		97
98	void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)	98	void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
99	{	99	{
100	region_register rr;	100	region_register rr;
101	bool restore_rr = false;	101	bool restore_rr = false;
102	int b = 0;	102	int b = 0;
103	int c = cnt;	103	int c = cnt;
104		104
105	__address va;	105	__address va;
106	va = page;	106	va = page;
107		107
108	rr.word = rr_read(VA2VRN(va));	108	rr.word = rr_read(VA2VRN(va));
109	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	109	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
110	/*	110	/*
111	* The selected region register does not contain required RID.	111	* The selected region register does not contain required RID.
112	* Save the old content of the register and replace the RID.	112	* Save the old content of the register and replace the RID.
113	*/	113	*/
114	region_register rr0;	114	region_register rr0;
115		115
116	rr0 = rr;	116	rr0 = rr;
117	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	117	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
118	rr_write(VA2VRN(va), rr0.word);	118	rr_write(VA2VRN(va), rr0.word);
119	srlz_d();	119	srlz_d();
120	srlz_i();	120	srlz_i();
121	}	121	}
122		122
123	while(c >>= 1)	123	while(c >>= 1)
124	b++;	124	b++;
125	b >>= 1;	125	b >>= 1;
126	__u64 ps;	126	__u64 ps;
127		127
128	switch (b) {	128	switch (b) {
129	case 0: /cnt 1-3/	129	case 0: /cnt 1-3/
130	ps = PAGE_WIDTH;	130	ps = PAGE_WIDTH;
131	break;	131	break;
132	case 1: /cnt 4-15/	132	case 1: /cnt 4-15/
133	/cnt=((cnt-1)/4)+1;/	133	/cnt=((cnt-1)/4)+1;/
134	ps = PAGE_WIDTH+2;	134	ps = PAGE_WIDTH+2;
135	va &= ~((1<<ps)-1);	135	va &= ~((1<<ps)-1);
136	break;	136	break;
137	case 2: /cnt 16-63/	137	case 2: /cnt 16-63/
138	/cnt=((cnt-1)/16)+1;/	138	/cnt=((cnt-1)/16)+1;/
139	ps = PAGE_WIDTH+4;	139	ps = PAGE_WIDTH+4;
140	va &= ~((1<<ps)-1);	140	va &= ~((1<<ps)-1);
141	break;	141	break;
142	case 3: /cnt 64-255/	142	case 3: /cnt 64-255/
143	/cnt=((cnt-1)/64)+1;/	143	/cnt=((cnt-1)/64)+1;/
144	ps = PAGE_WIDTH+6;	144	ps = PAGE_WIDTH+6;
145	va &= ~((1<<ps)-1);	145	va &= ~((1<<ps)-1);
146	break;	146	break;
147	case 4: /cnt 256-1023/	147	case 4: /cnt 256-1023/
148	/cnt=((cnt-1)/256)+1;/	148	/cnt=((cnt-1)/256)+1;/
149	ps = PAGE_WIDTH+8;	149	ps = PAGE_WIDTH+8;
150	va &= ~((1<<ps)-1);	150	va &= ~((1<<ps)-1);
151	break;	151	break;
152	case 5: /cnt 1024-4095/	152	case 5: /cnt 1024-4095/
153	/cnt=((cnt-1)/1024)+1;/	153	/cnt=((cnt-1)/1024)+1;/
154	ps = PAGE_WIDTH+10;	154	ps = PAGE_WIDTH+10;
155	va &= ~((1<<ps)-1);	155	va &= ~((1<<ps)-1);
156	break;	156	break;
157	case 6: /cnt 4096-16383/	157	case 6: /cnt 4096-16383/
158	/cnt=((cnt-1)/4096)+1;/	158	/cnt=((cnt-1)/4096)+1;/
159	ps = PAGE_WIDTH+12;	159	ps = PAGE_WIDTH+12;
160	va &= ~((1<<ps)-1);	160	va &= ~((1<<ps)-1);
161	break;	161	break;
162	case 7: /cnt 16384-65535/	162	case 7: /cnt 16384-65535/
163	case 8: /cnt 65536-(256K-1)/	163	case 8: /cnt 65536-(256K-1)/
164	/cnt=((cnt-1)/16384)+1;/	164	/cnt=((cnt-1)/16384)+1;/
165	ps = PAGE_WIDTH+14;	165	ps = PAGE_WIDTH+14;
166	va &= ~((1<<ps)-1);	166	va &= ~((1<<ps)-1);
167	break;	167	break;
168	default:	168	default:
169	/cnt=((cnt-1)/(1638416))+1;*/	169	/cnt=((cnt-1)/(1638416))+1;*/
170	ps=PAGE_WIDTH+18;	170	ps=PAGE_WIDTH+18;
171	va&=~((1<<ps)-1);	171	va&=~((1<<ps)-1);
172	break;	172	break;
173	}	173	}
174	/cnt+=(page!=va);/	174	/cnt+=(page!=va);/
175	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {	175	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
176	__asm__ volatile (	176	__asm__ volatile (
177	"ptc.l %0,%1;;"	177	"ptc.l %0,%1;;"
178	:	178	:
179	: "r" (va), "r" (ps<<2)	179	: "r" (va), "r" (ps<<2)
180	);	180	);
181	}	181	}
182	srlz_d();	182	srlz_d();
183	srlz_i();	183	srlz_i();
184		184
185	if (restore_rr) {	185	if (restore_rr) {
186	rr_write(VA2VRN(va), rr.word);	186	rr_write(VA2VRN(va), rr.word);
187	srlz_d();	187	srlz_d();
188	srlz_i();	188	srlz_i();
189	}	189	}
190	}	190	}
191		191
192		192
193	/** Insert data into data translation cache.	193	/** Insert data into data translation cache.
194	*	194	*
195	* @param va Virtual page address.	195	* @param va Virtual page address.
196	* @param asid Address space identifier.	196	* @param asid Address space identifier.
197	* @param entry The rest of TLB entry as required by TLB insertion format.	197	* @param entry The rest of TLB entry as required by TLB insertion format.
198	*/	198	*/
199	void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)	199	void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
200	{	200	{
201	tc_mapping_insert(va, asid, entry, true);	201	tc_mapping_insert(va, asid, entry, true);
202	}	202	}
203		203
204	/** Insert data into instruction translation cache.	204	/** Insert data into instruction translation cache.
205	*	205	*
206	* @param va Virtual page address.	206	* @param va Virtual page address.
207	* @param asid Address space identifier.	207	* @param asid Address space identifier.
208	* @param entry The rest of TLB entry as required by TLB insertion format.	208	* @param entry The rest of TLB entry as required by TLB insertion format.
209	*/	209	*/
210	void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)	210	void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
211	{	211	{
212	tc_mapping_insert(va, asid, entry, false);	212	tc_mapping_insert(va, asid, entry, false);
213	}	213	}
214		214
215	/** Insert data into instruction or data translation cache.	215	/** Insert data into instruction or data translation cache.
216	*	216	*
217	* @param va Virtual page address.	217	* @param va Virtual page address.
218	* @param asid Address space identifier.	218	* @param asid Address space identifier.
219	* @param entry The rest of TLB entry as required by TLB insertion format.	219	* @param entry The rest of TLB entry as required by TLB insertion format.
220	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.	220	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
221	*/	221	*/
222	void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)	222	void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
223	{	223	{
224	region_register rr;	224	region_register rr;
225	bool restore_rr = false;	225	bool restore_rr = false;
226		226
227	rr.word = rr_read(VA2VRN(va));	227	rr.word = rr_read(VA2VRN(va));
228	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	228	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
229	/*	229	/*
230	* The selected region register does not contain required RID.	230	* The selected region register does not contain required RID.
231	* Save the old content of the register and replace the RID.	231	* Save the old content of the register and replace the RID.
232	*/	232	*/
233	region_register rr0;	233	region_register rr0;
234		234
235	rr0 = rr;	235	rr0 = rr;
236	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	236	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
237	rr_write(VA2VRN(va), rr0.word);	237	rr_write(VA2VRN(va), rr0.word);
238	srlz_d();	238	srlz_d();
239	srlz_i();	239	srlz_i();
240	}	240	}
241		241
242	__asm__ volatile (	242	__asm__ volatile (
243	"mov r8=psr;;\n"	243	"mov r8=psr;;\n"
244	"rsm %0;;\n" /* PSR_IC_MASK */	244	"rsm %0;;\n" /* PSR_IC_MASK */
245	"srlz.d;;\n"	245	"srlz.d;;\n"
246	"srlz.i;;\n"	246	"srlz.i;;\n"
247	"mov cr.ifa=%1\n" /* va */	247	"mov cr.ifa=%1\n" /* va */
248	"mov cr.itir=%2;;\n" /* entry.word[1] */	248	"mov cr.itir=%2;;\n" /* entry.word[1] */
249	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */	249	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */
250	"(p6) itc.i %3;;\n"	250	"(p6) itc.i %3;;\n"
251	"(p7) itc.d %3;;\n"	251	"(p7) itc.d %3;;\n"
252	"mov psr.l=r8;;\n"	252	"mov psr.l=r8;;\n"
253	"srlz.d;;\n"	253	"srlz.d;;\n"
254	:	254	:
255	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)	255	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
256	: "p6", "p7", "r8"	256	: "p6", "p7", "r8"
257	);	257	);
258		258
259	if (restore_rr) {	259	if (restore_rr) {
260	rr_write(VA2VRN(va), rr.word);	260	rr_write(VA2VRN(va), rr.word);
261	srlz_d();	261	srlz_d();
262	srlz_i();	262	srlz_i();
263	}	263	}
264	}	264	}
265		265
266	/** Insert data into instruction translation register.	266	/** Insert data into instruction translation register.
267	*	267	*
268	* @param va Virtual page address.	268	* @param va Virtual page address.
269	* @param asid Address space identifier.	269	* @param asid Address space identifier.
270	* @param entry The rest of TLB entry as required by TLB insertion format.	270	* @param entry The rest of TLB entry as required by TLB insertion format.
271	* @param tr Translation register.	271	* @param tr Translation register.
272	*/	272	*/
273	void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)	273	void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
274	{	274	{
275	tr_mapping_insert(va, asid, entry, false, tr);	275	tr_mapping_insert(va, asid, entry, false, tr);
276	}	276	}
277		277
278	/** Insert data into data translation register.	278	/** Insert data into data translation register.
279	*	279	*
280	* @param va Virtual page address.	280	* @param va Virtual page address.
281	* @param asid Address space identifier.	281	* @param asid Address space identifier.
282	* @param entry The rest of TLB entry as required by TLB insertion format.	282	* @param entry The rest of TLB entry as required by TLB insertion format.
283	* @param tr Translation register.	283	* @param tr Translation register.
284	*/	284	*/
285	void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)	285	void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
286	{	286	{
287	tr_mapping_insert(va, asid, entry, true, tr);	287	tr_mapping_insert(va, asid, entry, true, tr);
288	}	288	}
289		289
290	/** Insert data into instruction or data translation register.	290	/** Insert data into instruction or data translation register.
291	*	291	*
292	* @param va Virtual page address.	292	* @param va Virtual page address.
293	* @param asid Address space identifier.	293	* @param asid Address space identifier.
294	* @param entry The rest of TLB entry as required by TLB insertion format.	294	* @param entry The rest of TLB entry as required by TLB insertion format.
295	* @param dtc If true, insert into data translation register, use instruction translation register otherwise.	295	* @param dtc If true, insert into data translation register, use instruction translation register otherwise.
296	* @param tr Translation register.	296	* @param tr Translation register.
297	*/	297	*/
298	void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)	298	void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
299	{	299	{
300	region_register rr;	300	region_register rr;
301	bool restore_rr = false;	301	bool restore_rr = false;
302		302
303	rr.word = rr_read(VA2VRN(va));	303	rr.word = rr_read(VA2VRN(va));
304	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	304	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
305	/*	305	/*
306	* The selected region register does not contain required RID.	306	* The selected region register does not contain required RID.
307	* Save the old content of the register and replace the RID.	307	* Save the old content of the register and replace the RID.
308	*/	308	*/
309	region_register rr0;	309	region_register rr0;
310		310
311	rr0 = rr;	311	rr0 = rr;
312	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	312	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
313	rr_write(VA2VRN(va), rr0.word);	313	rr_write(VA2VRN(va), rr0.word);
314	srlz_d();	314	srlz_d();
315	srlz_i();	315	srlz_i();
316	}	316	}
317		317
318	__asm__ volatile (	318	__asm__ volatile (
319	"mov r8=psr;;\n"	319	"mov r8=psr;;\n"
320	"rsm %0;;\n" /* PSR_IC_MASK */	320	"rsm %0;;\n" /* PSR_IC_MASK */
321	"srlz.d;;\n"	321	"srlz.d;;\n"
322	"srlz.i;;\n"	322	"srlz.i;;\n"
323	"mov cr.ifa=%1\n" /* va */	323	"mov cr.ifa=%1\n" /* va */
324	"mov cr.itir=%2;;\n" /* entry.word[1] */	324	"mov cr.itir=%2;;\n" /* entry.word[1] */
325	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */	325	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */
326	"(p6) itr.i itr[%4]=%3;;\n"	326	"(p6) itr.i itr[%4]=%3;;\n"
327	"(p7) itr.d dtr[%4]=%3;;\n"	327	"(p7) itr.d dtr[%4]=%3;;\n"
328	"mov psr.l=r8;;\n"	328	"mov psr.l=r8;;\n"
329	"srlz.d;;\n"	329	"srlz.d;;\n"
330	:	330	:
331	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)	331	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
332	: "p6", "p7", "r8"	332	: "p6", "p7", "r8"
333	);	333	);
334		334
335	if (restore_rr) {	335	if (restore_rr) {
336	rr_write(VA2VRN(va), rr.word);	336	rr_write(VA2VRN(va), rr.word);
337	srlz_d();	337	srlz_d();
338	srlz_i();	338	srlz_i();
339	}	339	}
340	}	340	}
341		341
342	/** Insert data into DTLB.	342	/** Insert data into DTLB.
343	*	343	*
344	* @param va Virtual page address.	344	* @param va Virtual page address.
345	* @param asid Address space identifier.	345	* @param asid Address space identifier.
346	* @param entry The rest of TLB entry as required by TLB insertion format.	346	* @param entry The rest of TLB entry as required by TLB insertion format.
347	* @param dtr If true, insert into data translation register, use data translation cache otherwise.	347	* @param dtr If true, insert into data translation register, use data translation cache otherwise.
348	* @param tr Translation register if dtr is true, ignored otherwise.	348	* @param tr Translation register if dtr is true, ignored otherwise.
349	*/	349	*/
350	void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)	350	void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
351	{	351	{
352	tlb_entry_t entry;	352	tlb_entry_t entry;
353		353
354	entry.word[0] = 0;	354	entry.word[0] = 0;
355	entry.word[1] = 0;	355	entry.word[1] = 0;
356		356
357	entry.p = true; /* present */	357	entry.p = true; /* present */
358	entry.ma = MA_WRITEBACK;	358	entry.ma = MA_WRITEBACK;
359	entry.a = true; /* already accessed */	359	entry.a = true; /* already accessed */
360	entry.d = true; /* already dirty */	360	entry.d = true; /* already dirty */
361	entry.pl = PL_KERNEL;	361	entry.pl = PL_KERNEL;
362	entry.ar = AR_READ \| AR_WRITE;	362	entry.ar = AR_READ \| AR_WRITE;
363	entry.ppn = frame >> PPN_SHIFT;	363	entry.ppn = frame >> PPN_SHIFT;
364	entry.ps = PAGE_WIDTH;	364	entry.ps = PAGE_WIDTH;
365		365
366	if (dtr)	366	if (dtr)
367	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);	367	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
368	else	368	else
369	dtc_mapping_insert(page, ASID_KERNEL, entry);	369	dtc_mapping_insert(page, ASID_KERNEL, entry);
370	}	370	}
371		371
372	/** Copy content of PTE into data translation cache.	372	/** Copy content of PTE into data translation cache.
373	*	373	*
374	* @param t PTE.	374	* @param t PTE.
375	*/	375	*/
376	void dtc_pte_copy(pte_t *t)	376	void dtc_pte_copy(pte_t *t)
377	{	377	{
378	tlb_entry_t entry;	378	tlb_entry_t entry;
379		379
380	entry.word[0] = 0;	380	entry.word[0] = 0;
381	entry.word[1] = 0;	381	entry.word[1] = 0;
382		382
383	entry.p = t->p;	383	entry.p = t->p;
384	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	384	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
385	entry.a = t->a;	385	entry.a = t->a;
386	entry.d = t->d;	386	entry.d = t->d;
387	entry.pl = t->k ? PL_KERNEL : PL_USER;	387	entry.pl = t->k ? PL_KERNEL : PL_USER;
388	entry.ar = t->w ? AR_WRITE : AR_READ;	388	entry.ar = t->w ? AR_WRITE : AR_READ;
389	entry.ppn = t->frame >> PPN_SHIFT;	389	entry.ppn = t->frame >> PPN_SHIFT;
390	entry.ps = PAGE_WIDTH;	390	entry.ps = PAGE_WIDTH;
391		391
392	dtc_mapping_insert(t->page, t->as->asid, entry);	392	dtc_mapping_insert(t->page, t->as->asid, entry);
393	#ifdef CONFIG_VHPT	393	#ifdef CONFIG_VHPT
394	vhpt_mapping_insert(t->page, t->as->asid, entry);	394	vhpt_mapping_insert(t->page, t->as->asid, entry);
395	#endif	395	#endif
396	}	396	}
397		397
398	/** Copy content of PTE into instruction translation cache.	398	/** Copy content of PTE into instruction translation cache.
399	*	399	*
400	* @param t PTE.	400	* @param t PTE.
401	*/	401	*/
402	void itc_pte_copy(pte_t *t)	402	void itc_pte_copy(pte_t *t)
403	{	403	{
404	tlb_entry_t entry;	404	tlb_entry_t entry;
405		405
406	entry.word[0] = 0;	406	entry.word[0] = 0;
407	entry.word[1] = 0;	407	entry.word[1] = 0;
408		408
409	ASSERT(t->x);	409	ASSERT(t->x);
410		410
411	entry.p = t->p;	411	entry.p = t->p;
412	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	412	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
413	entry.a = t->a;	413	entry.a = t->a;
414	entry.pl = t->k ? PL_KERNEL : PL_USER;	414	entry.pl = t->k ? PL_KERNEL : PL_USER;
415	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;	415	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;
416	entry.ppn = t->frame >> PPN_SHIFT;	416	entry.ppn = t->frame >> PPN_SHIFT;
417	entry.ps = PAGE_WIDTH;	417	entry.ps = PAGE_WIDTH;
418		418
419	itc_mapping_insert(t->page, t->as->asid, entry);	419	itc_mapping_insert(t->page, t->as->asid, entry);
420	#ifdef CONFIG_VHPT	420	#ifdef CONFIG_VHPT
421	vhpt_mapping_insert(t->page, t->as->asid, entry);	421	vhpt_mapping_insert(t->page, t->as->asid, entry);
422	#endif	422	#endif
423	}	423	}
424		424
425	/** Instruction TLB fault handler for faults with VHPT turned off.	425	/** Instruction TLB fault handler for faults with VHPT turned off.
426	*	426	*
427	* @param vector Interruption vector.	427	* @param vector Interruption vector.
428	* @param istate Structure with saved interruption state.	428	* @param istate Structure with saved interruption state.
429	*/	429	*/
430	void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)	430	void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
431	{	431	{
432	region_register rr;	432	region_register rr;
433	rid_t rid;	433	rid_t rid;
434	__address va;	434	__address va;
435	pte_t *t;	435	pte_t *t;
436		436
437	va = istate->cr_ifa; /* faulting address */	437	va = istate->cr_ifa; /* faulting address */
438	rr.word = rr_read(VA2VRN(va));	438	rr.word = rr_read(VA2VRN(va));
439	rid = rr.map.rid;	439	rid = rr.map.rid;
440		440
441	page_table_lock(AS, true);	441	page_table_lock(AS, true);
442	t = page_mapping_find(AS, va);	442	t = page_mapping_find(AS, va);
443	if (t) {	443	if (t) {
444	/*	444	/*
445	* The mapping was found in software page hash table.	445	* The mapping was found in software page hash table.
446	* Insert it into data translation cache.	446	* Insert it into data translation cache.
447	*/	447	*/
448	itc_pte_copy(t);	448	itc_pte_copy(t);
449	page_table_unlock(AS, true);	449	page_table_unlock(AS, true);
450	} else {	450	} else {
451	/*	451	/*
452	* Forward the page fault to address space page fault handler.	452	* Forward the page fault to address space page fault handler.
453	*/	453	*/
454	page_table_unlock(AS, true);	454	page_table_unlock(AS, true);
455	if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {	455	if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
456	fault_if_from_uspace(istate,"Page fault at %P",va);	456	fault_if_from_uspace(istate,"Page fault at %P",va);
457	panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);	457	panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
458	}	458	}
459	}	459	}
460	}	460	}
461		461
462	/** Data TLB fault handler for faults with VHPT turned off.	462	/** Data TLB fault handler for faults with VHPT turned off.
463	*	463	*
464	* @param vector Interruption vector.	464	* @param vector Interruption vector.
465	* @param istate Structure with saved interruption state.	465	* @param istate Structure with saved interruption state.
466	*/	466	*/
467	void alternate_data_tlb_fault(__u64 vector, istate_t *istate)	467	void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
468	{	468	{
469	region_register rr;	469	region_register rr;
470	rid_t rid;	470	rid_t rid;
471	__address va;	471	__address va;
472	pte_t *t;	472	pte_t *t;
473		473
474	va = istate->cr_ifa; /* faulting address */	474	va = istate->cr_ifa; /* faulting address */
475	rr.word = rr_read(VA2VRN(va));	475	rr.word = rr_read(VA2VRN(va));
476	rid = rr.map.rid;	476	rid = rr.map.rid;
477	if (RID2ASID(rid) == ASID_KERNEL) {	477	if (RID2ASID(rid) == ASID_KERNEL) {
478	if (VA2VRN(va) == VRN_KERNEL) {	478	if (VA2VRN(va) == VRN_KERNEL) {
479	/*	479	/*
480	* Provide KA2PA(identity) mapping for faulting piece of	480	* Provide KA2PA(identity) mapping for faulting piece of
481	* kernel address space.	481	* kernel address space.
482	*/	482	*/
483	dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);	483	dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
484	return;	484	return;
485	}	485	}
486	}	486	}
487		487
488	page_table_lock(AS, true);	488	page_table_lock(AS, true);
489	t = page_mapping_find(AS, va);	489	t = page_mapping_find(AS, va);
490	if (t) {	490	if (t) {
491	/*	491	/*
492	* The mapping was found in software page hash table.	492	* The mapping was found in software page hash table.
493	* Insert it into data translation cache.	493	* Insert it into data translation cache.
494	*/	494	*/
495	dtc_pte_copy(t);	495	dtc_pte_copy(t);
496	page_table_unlock(AS, true);	496	page_table_unlock(AS, true);
497	} else {	497	} else {
498	/*	498	/*
499	* Forward the page fault to address space page fault handler.	499	* Forward the page fault to address space page fault handler.
500	*/	500	*/

Subversion Repositories HelenOS

(root)/tags/0.2.0/kernel/arch/ia64/src/mm/tlb.c – Rev 1964 → 1968