WebSVN – HelenOS – Diff – /trunk/kernel/arch/ia64/src/mm/tlb.c


/*
 * Copyright (c) 2006 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
/** @addtogroup ia64mm 
 * @{
 */
/** @file
 */
 
/*
 * TLB management.
 */
 
#include <mm/tlb.h>
#include <mm/asid.h>
#include <mm/page.h>
#include <mm/as.h>
#include <arch/mm/tlb.h>
#include <arch/mm/page.h>
#include <arch/mm/vhpt.h>
#include <arch/barrier.h>
#include <arch/interrupt.h>
#include <arch/pal/pal.h>
#include <arch/asm.h>
#include <panic.h>
#include <print.h>
#include <arch.h>
#include <interrupt.h>
 
/** Invalidate all TLB entries. */
void tlb_invalidate_all(void)
{
    ipl_t ipl;
    uintptr_t adr;
    uint32_t count1, count2, stride1, stride2;
       
    unsigned int i, j;
       
    adr = PAL_PTCE_INFO_BASE();
    count1 = PAL_PTCE_INFO_COUNT1();
    count2 = PAL_PTCE_INFO_COUNT2();
    stride1 = PAL_PTCE_INFO_STRIDE1();
    stride2 = PAL_PTCE_INFO_STRIDE2();
       
    ipl = interrupts_disable();
 
    for (i = 0; i < count1; i++) {
        for (j = 0; j < count2; j++) {
            asm volatile (
                "ptc.e %0 ;;"
                :
                : "r" (adr)
            );
            adr += stride2;
        }
        adr += stride1;
    }
 
    interrupts_restore(ipl);
 
    srlz_d();
    srlz_i();
#ifdef CONFIG_VHPT
    vhpt_invalidate_all();
#endif  
}
 
/** Invalidate entries belonging to an address space.
 *
 * @param asid Address space identifier.
 */
void tlb_invalidate_asid(asid_t asid)
{
    tlb_invalidate_all();
}
 
 
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
{
    region_register rr;
    bool restore_rr = false;
    int b = 0;
    int c = cnt;
 
    uintptr_t va;
    va = page;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    while(c >>= 1)
        b++;
    b >>= 1;
    uint64_t ps;
   
    switch (b) {
    case 0: /*cnt 1-3*/
        ps = PAGE_WIDTH;
        break;
    case 1: /*cnt 4-15*/
 
        ps = PAGE_WIDTH+2;
        va &= ~((1<<ps)-1);
        break;
    case 2: /*cnt 16-63*/
 
        ps = PAGE_WIDTH+4;
        va &= ~((1<<ps)-1);
        break;
    case 3: /*cnt 64-255*/
 
        ps = PAGE_WIDTH+6;
        va &= ~((1<<ps)-1);
        break;
    case 4: /*cnt 256-1023*/
 
        ps = PAGE_WIDTH+8;
        va &= ~((1<<ps)-1);
        break;
    case 5: /*cnt 1024-4095*/
 
        ps = PAGE_WIDTH+10;
        va &= ~((1<<ps)-1);
        break;
    case 6: /*cnt 4096-16383*/
 
        ps = PAGE_WIDTH+12;
        va &= ~((1<<ps)-1);
        break;
    case 7: /*cnt 16384-65535*/
    case 8: /*cnt 65536-(256K-1)*/
 
        ps = PAGE_WIDTH+14;
        va &= ~((1<<ps)-1);
        break;
    default:
 
        ps=PAGE_WIDTH+18;
        va&=~((1<<ps)-1);
        break;
    }
 
    for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
        asm volatile (
            "ptc.l %0,%1;;"
            :
            : "r" (va), "r" (ps<<2)
        );
    }
    srlz_d();
    srlz_i();
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void dtc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, true);
}
 
/** Insert data into instruction translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 */
void itc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)
{
    tc_mapping_insert(va, asid, entry, false);
}
 
/** Insert data into instruction or data translation cache.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
 */
void tc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtc)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
   
    asm volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"       /* va */
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
        "(p6) itc.i %3;;\n"
        "(p7) itc.d %3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into instruction translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void itr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, false, tr);
}
 
/** Insert data into data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param tr Translation register.
 */
void dtr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr)
{
    tr_mapping_insert(va, asid, entry, true, tr);
}
 
/** Insert data into instruction or data translation register.
 *
 * @param va Virtual page address.
 * @param asid Address space identifier.
 * @param entry The rest of TLB entry as required by TLB insertion format.
 * @param dtr If true, insert into data translation register, use instruction translation register otherwise.
 * @param tr Translation register.
 */
void tr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
{
    region_register rr;
    bool restore_rr = false;
 
    rr.word = rr_read(VA2VRN(va));
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
        /*
         * The selected region register does not contain required RID.
         * Save the old content of the register and replace the RID.
         */
        region_register rr0;
 
        rr0 = rr;
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
        rr_write(VA2VRN(va), rr0.word);
        srlz_d();
        srlz_i();
    }
 
    asm volatile (
        "mov r8=psr;;\n"
        "rsm %0;;\n"            /* PSR_IC_MASK */
        "srlz.d;;\n"
        "srlz.i;;\n"
        "mov cr.ifa=%1\n"           /* va */         
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
        "cmp.eq p6,p7=%5,r0;;\n"    /* decide between itr and dtr */
        "(p6) itr.i itr[%4]=%3;;\n"
        "(p7) itr.d dtr[%4]=%3;;\n"
        "mov psr.l=r8;;\n"
        "srlz.d;;\n"
        :
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
        : "p6", "p7", "r8"
    );
   
    if (restore_rr) {
        rr_write(VA2VRN(va), rr.word);
        srlz_d();
        srlz_i();
    }
}
 
/** Insert data into DTLB.
 *
 * @param page Virtual page address including VRN bits.
 * @param frame Physical frame address.
 * @param dtr If true, insert into data translation register, use data translation cache otherwise.
 * @param tr Translation register if dtr is true, ignored otherwise.
 */
void dtlb_kernel_mapping_insert(uintptr_t page, uintptr_t frame, bool dtr, index_t tr)
{
    tlb_entry_t entry;
   
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = true;         /* present */
    entry.ma = MA_WRITEBACK;
    entry.a = true;         /* already accessed */
    entry.d = true;         /* already dirty */
    entry.pl = PL_KERNEL;
    entry.ar = AR_READ | AR_WRITE;
    entry.ppn = frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    if (dtr)
        dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
    else
        dtc_mapping_insert(page, ASID_KERNEL, entry);
}
 
/** Purge kernel entries from DTR.
 *
 * Purge DTR entries used by the kernel.
 *
 * @param page Virtual page address including VRN bits.
 * @param width Width of the purge in bits.
 */
void dtr_purge(uintptr_t page, count_t width)
{
    asm volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));
}
 
 
/** Copy content of PTE into data translation cache.
 *
 * @param t PTE.
 */
void dtc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.d = t->d;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->w ? AR_WRITE : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    dtc_mapping_insert(t->page, t->as->asid, entry);
#ifdef CONFIG_VHPT
    vhpt_mapping_insert(t->page, t->as->asid, entry);
#endif  
}
 
/** Copy content of PTE into instruction translation cache.
 *
 * @param t PTE.
 */
void itc_pte_copy(pte_t *t)
{
    tlb_entry_t entry;
 
    entry.word[0] = 0;
    entry.word[1] = 0;
   
    ASSERT(t->x);
   
    entry.p = t->p;
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
    entry.a = t->a;
    entry.pl = t->k ? PL_KERNEL : PL_USER;
    entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
    entry.ppn = t->frame >> PPN_SHIFT;
    entry.ps = PAGE_WIDTH;
   
    itc_mapping_insert(t->page, t->as->asid, entry);
#ifdef CONFIG_VHPT
    vhpt_mapping_insert(t->page, t->as->asid, entry);
#endif  
}
 
/** Instruction TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_instruction_tlb_fault(uint64_t vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    uintptr_t va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in software page hash table.
         * Insert it into data translation cache.
         */
        itc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        /*
         * Forward the page fault to address space page fault handler.
         */
        page_table_unlock(AS, true);
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %p",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);
        }
    }
}
 
 
 
static int is_io_page_accessible(int page)
{
    if(TASK->arch.iomap) return bitmap_get(TASK->arch.iomap,page);
    else return 0;
}
 
#define IO_FRAME_BASE 0xFFFFC000000
 
/** There is special handling of memmaped lagacy io, because
 * of 4KB sized access
 * only for userspace
 *
 * @param va virtual address of page fault
 * @param istate Structure with saved interruption state.
 *
 *
 * @return 1 on success, 0 on fail
 */
static int try_memmap_io_insertion(uintptr_t va, istate_t *istate)
{
    if((va >= IO_OFFSET ) && (va < IO_OFFSET + (1<<IO_PAGE_WIDTH)))
        if(TASK){
           
            uint64_t io_page=(va &  ((1<<IO_PAGE_WIDTH)-1)) >> (USPACE_IO_PAGE_WIDTH);
            if(is_io_page_accessible(io_page)){
 
 
                uint64_t page,frame;
 
                page = IO_OFFSET + (1 << USPACE_IO_PAGE_WIDTH) * io_page;
                frame = IO_FRAME_BASE + (1 << USPACE_IO_PAGE_WIDTH) * io_page;
 
 
                tlb_entry_t entry;
   
                entry.word[0] = 0;
                entry.word[1] = 0;
   
                entry.p = true;         /* present */
                entry.ma = MA_UNCACHEABLE;     
                entry.a = true;         /* already accessed */
                entry.d = true;         /* already dirty */
                entry.pl = PL_USER;
                entry.ar = AR_READ | AR_WRITE;
                entry.ppn = frame >> PPN_SHIFT;
                entry.ps = USPACE_IO_PAGE_WIDTH;
   
                dtc_mapping_insert(page, TASK->as->asid, entry);
                return 1;
            }else {
                fault_if_from_uspace(istate,"IO access fault at %p",va);
                return 0;
            }      
        } else
            return 0;
    else
        return 0;
       
    return 0;
 
}
 
 
 
 
/** Data TLB fault handler for faults with VHPT turned off.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void alternate_data_tlb_fault(uint64_t vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    uintptr_t va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
    if (RID2ASID(rid) == ASID_KERNEL) {
        if (VA2VRN(va) == VRN_KERNEL) {
            /*
             * Provide KA2PA(identity) mapping for faulting piece of
             * kernel address space.
             */
            dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
            return;
        }
    }
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    if (t) {
        /*
         * The mapping was found in the software page hash table.
         * Insert it into data translation cache.
         */
        dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        page_table_unlock(AS, true);
        if (try_memmap_io_insertion(va,istate)) return;
        /*
         * Forward the page fault to the address space page fault handler.
         */
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %p",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);
        }
    }
}
 
/** Data nested TLB fault handler.
 *
 * This fault should not occur.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_nested_tlb_fault(uint64_t vector, istate_t *istate)
{
    panic("%s\n", __func__);
}
 
/** Data Dirty bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_dirty_bit_fault(uint64_t vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    uintptr_t va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t && t->p);
    if (t && t->p && t->w) {
        /*
         * Update the Dirty bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->d = true;
        dtc_pte_copy(t);
    } else {
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %p",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);
            t->d = true;
            dtc_pte_copy(t);
        }
    }
    page_table_unlock(AS, true);
}
 
/** Instruction access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void instruction_access_bit_fault(uint64_t vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    uintptr_t va;
    pte_t *t;  
 
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t && t->p);
    if (t && t->p && t->x) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into ITC.
         */
        t->a = true;
        itc_pte_copy(t);
    } else {
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %p",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);
            t->a = true;
            itc_pte_copy(t);
        }
    }
    page_table_unlock(AS, true);
}
 
/** Data access bit fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void data_access_bit_fault(uint64_t vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    uintptr_t va;
    pte_t *t;
 
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t && t->p);
    if (t && t->p) {
        /*
         * Update the Accessed bit in page tables and reinsert
         * the mapping into DTC.
         */
        t->a = true;
        dtc_pte_copy(t);
    } else {
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %p",va);
            panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);
            t->a = true;
            itc_pte_copy(t);
        }
    }
    page_table_unlock(AS, true);
}
 
/** Page not present fault handler.
 *
 * @param vector Interruption vector.
 * @param istate Structure with saved interruption state.
 */
void page_not_present(uint64_t vector, istate_t *istate)
{
    region_register rr;
    rid_t rid;
    uintptr_t va;
    pte_t *t;
   
    va = istate->cr_ifa;    /* faulting address */
    rr.word = rr_read(VA2VRN(va));
    rid = rr.map.rid;
 
    page_table_lock(AS, true);
    t = page_mapping_find(AS, va);
    ASSERT(t);
   
    if (t->p) {
        /*
         * If the Present bit is set in page hash table, just copy it
         * and update ITC/DTC.
         */
        if (t->x)
            itc_pte_copy(t);
        else
            dtc_pte_copy(t);
        page_table_unlock(AS, true);
    } else {
        page_table_unlock(AS, true);
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
            fault_if_from_uspace(istate,"Page fault at %p",va);
            panic("%s: va=%p, rid=%d\n", __func__, va, rid);
        }
    }
}
 
/** @}
 */
 

Rev 3635	Rev 3763
1	/*	1	/*
2	* Copyright (c) 2006 Jakub Jermar	2	* Copyright (c) 2006 Jakub Jermar
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	/** @addtogroup ia64mm	29	/** @addtogroup ia64mm
30	* @{	30	* @{
31	*/	31	*/
32	/** @file	32	/** @file
33	*/	33	*/
34		34
35	/*	35	/*
36	* TLB management.	36	* TLB management.
37	*/	37	*/
38		38
39	#include <mm/tlb.h>	39	#include <mm/tlb.h>
40	#include <mm/asid.h>	40	#include <mm/asid.h>
41	#include <mm/page.h>	41	#include <mm/page.h>
42	#include <mm/as.h>	42	#include <mm/as.h>
43	#include <arch/mm/tlb.h>	43	#include <arch/mm/tlb.h>
44	#include <arch/mm/page.h>	44	#include <arch/mm/page.h>
45	#include <arch/mm/vhpt.h>	45	#include <arch/mm/vhpt.h>
46	#include <arch/barrier.h>	46	#include <arch/barrier.h>
47	#include <arch/interrupt.h>	47	#include <arch/interrupt.h>
48	#include <arch/pal/pal.h>	48	#include <arch/pal/pal.h>
49	#include <arch/asm.h>	49	#include <arch/asm.h>
50	#include <panic.h>	50	#include <panic.h>
51	#include <print.h>	51	#include <print.h>
52	#include <arch.h>	52	#include <arch.h>
53	#include <interrupt.h>	53	#include <interrupt.h>
54		54
55	/** Invalidate all TLB entries. */	55	/** Invalidate all TLB entries. */
56	void tlb_invalidate_all(void)	56	void tlb_invalidate_all(void)
57	{	57	{
58	ipl_t ipl;	58	ipl_t ipl;
59	uintptr_t adr;	59	uintptr_t adr;
60	uint32_t count1, count2, stride1, stride2;	60	uint32_t count1, count2, stride1, stride2;
61		61
62	unsigned int i, j;	62	unsigned int i, j;
63		63
64	adr = PAL_PTCE_INFO_BASE();	64	adr = PAL_PTCE_INFO_BASE();
65	count1 = PAL_PTCE_INFO_COUNT1();	65	count1 = PAL_PTCE_INFO_COUNT1();
66	count2 = PAL_PTCE_INFO_COUNT2();	66	count2 = PAL_PTCE_INFO_COUNT2();
67	stride1 = PAL_PTCE_INFO_STRIDE1();	67	stride1 = PAL_PTCE_INFO_STRIDE1();
68	stride2 = PAL_PTCE_INFO_STRIDE2();	68	stride2 = PAL_PTCE_INFO_STRIDE2();
69		69
70	ipl = interrupts_disable();	70	ipl = interrupts_disable();
71		71
72	for (i = 0; i < count1; i++) {	72	for (i = 0; i < count1; i++) {
73	for (j = 0; j < count2; j++) {	73	for (j = 0; j < count2; j++) {
74	asm volatile (	74	asm volatile (
75	"ptc.e %0 ;;"	75	"ptc.e %0 ;;"
76	:	76	:
77	: "r" (adr)	77	: "r" (adr)
78	);	78	);
79	adr += stride2;	79	adr += stride2;
80	}	80	}
81	adr += stride1;	81	adr += stride1;
82	}	82	}
83		83
84	interrupts_restore(ipl);	84	interrupts_restore(ipl);
85		85
86	srlz_d();	86	srlz_d();
87	srlz_i();	87	srlz_i();
88	#ifdef CONFIG_VHPT	88	#ifdef CONFIG_VHPT
89	vhpt_invalidate_all();	89	vhpt_invalidate_all();
90	#endif	90	#endif
91	}	91	}
92		92
93	/** Invalidate entries belonging to an address space.	93	/** Invalidate entries belonging to an address space.
94	*	94	*
95	* @param asid Address space identifier.	95	* @param asid Address space identifier.
96	*/	96	*/
97	void tlb_invalidate_asid(asid_t asid)	97	void tlb_invalidate_asid(asid_t asid)
98	{	98	{
99	tlb_invalidate_all();	99	tlb_invalidate_all();
100	}	100	}
101		101
102		102
103	void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)	103	void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
104	{	104	{
105	region_register rr;	105	region_register rr;
106	bool restore_rr = false;	106	bool restore_rr = false;
107	int b = 0;	107	int b = 0;
108	int c = cnt;	108	int c = cnt;
109		109
110	uintptr_t va;	110	uintptr_t va;
111	va = page;	111	va = page;
112		112
113	rr.word = rr_read(VA2VRN(va));	113	rr.word = rr_read(VA2VRN(va));
114	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	114	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
115	/*	115	/*
116	* The selected region register does not contain required RID.	116	* The selected region register does not contain required RID.
117	* Save the old content of the register and replace the RID.	117	* Save the old content of the register and replace the RID.
118	*/	118	*/
119	region_register rr0;	119	region_register rr0;
120		120
121	rr0 = rr;	121	rr0 = rr;
122	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	122	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
123	rr_write(VA2VRN(va), rr0.word);	123	rr_write(VA2VRN(va), rr0.word);
124	srlz_d();	124	srlz_d();
125	srlz_i();	125	srlz_i();
126	}	126	}
127		127
128	while(c >>= 1)	128	while(c >>= 1)
129	b++;	129	b++;
130	b >>= 1;	130	b >>= 1;
131	uint64_t ps;	131	uint64_t ps;
132		132
133	switch (b) {	133	switch (b) {
134	case 0: /cnt 1-3/	134	case 0: /cnt 1-3/
135	ps = PAGE_WIDTH;	135	ps = PAGE_WIDTH;
136	break;	136	break;
137	case 1: /cnt 4-15/	137	case 1: /cnt 4-15/
138	/cnt=((cnt-1)/4)+1;/	-
139	ps = PAGE_WIDTH+2;	138	ps = PAGE_WIDTH+2;
140	va &= ~((1<<ps)-1);	139	va &= ~((1<<ps)-1);
141	break;	140	break;
142	case 2: /cnt 16-63/	141	case 2: /cnt 16-63/
143	/cnt=((cnt-1)/16)+1;/	-
144	ps = PAGE_WIDTH+4;	142	ps = PAGE_WIDTH+4;
145	va &= ~((1<<ps)-1);	143	va &= ~((1<<ps)-1);
146	break;	144	break;
147	case 3: /cnt 64-255/	145	case 3: /cnt 64-255/
148	/cnt=((cnt-1)/64)+1;/	-
149	ps = PAGE_WIDTH+6;	146	ps = PAGE_WIDTH+6;
150	va &= ~((1<<ps)-1);	147	va &= ~((1<<ps)-1);
151	break;	148	break;
152	case 4: /cnt 256-1023/	149	case 4: /cnt 256-1023/
153	/cnt=((cnt-1)/256)+1;/	-
154	ps = PAGE_WIDTH+8;	150	ps = PAGE_WIDTH+8;
155	va &= ~((1<<ps)-1);	151	va &= ~((1<<ps)-1);
156	break;	152	break;
157	case 5: /cnt 1024-4095/	153	case 5: /cnt 1024-4095/
158	/cnt=((cnt-1)/1024)+1;/	-
159	ps = PAGE_WIDTH+10;	154	ps = PAGE_WIDTH+10;
160	va &= ~((1<<ps)-1);	155	va &= ~((1<<ps)-1);
161	break;	156	break;
162	case 6: /cnt 4096-16383/	157	case 6: /cnt 4096-16383/
163	/cnt=((cnt-1)/4096)+1;/	-
164	ps = PAGE_WIDTH+12;	158	ps = PAGE_WIDTH+12;
165	va &= ~((1<<ps)-1);	159	va &= ~((1<<ps)-1);
166	break;	160	break;
167	case 7: /cnt 16384-65535/	161	case 7: /cnt 16384-65535/
168	case 8: /cnt 65536-(256K-1)/	162	case 8: /cnt 65536-(256K-1)/
169	/cnt=((cnt-1)/16384)+1;/	-
170	ps = PAGE_WIDTH+14;	163	ps = PAGE_WIDTH+14;
171	va &= ~((1<<ps)-1);	164	va &= ~((1<<ps)-1);
172	break;	165	break;
173	default:	166	default:
174	/cnt=((cnt-1)/(1638416))+1;*/	-
175	ps=PAGE_WIDTH+18;	167	ps=PAGE_WIDTH+18;
176	va&=~((1<<ps)-1);	168	va&=~((1<<ps)-1);
177	break;	169	break;
178	}	170	}
179	/cnt+=(page!=va);/	-
180	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {	171	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
181	asm volatile (	172	asm volatile (
182	"ptc.l %0,%1;;"	173	"ptc.l %0,%1;;"
183	:	174	:
184	: "r" (va), "r" (ps<<2)	175	: "r" (va), "r" (ps<<2)
185	);	176	);
186	}	177	}
187	srlz_d();	178	srlz_d();
188	srlz_i();	179	srlz_i();
189		180
190	if (restore_rr) {	181	if (restore_rr) {
191	rr_write(VA2VRN(va), rr.word);	182	rr_write(VA2VRN(va), rr.word);
192	srlz_d();	183	srlz_d();
193	srlz_i();	184	srlz_i();
194	}	185	}
195	}	186	}
196		187
197	/** Insert data into data translation cache.	188	/** Insert data into data translation cache.
198	*	189	*
199	* @param va Virtual page address.	190	* @param va Virtual page address.
200	* @param asid Address space identifier.	191	* @param asid Address space identifier.
201	* @param entry The rest of TLB entry as required by TLB insertion format.	192	* @param entry The rest of TLB entry as required by TLB insertion format.
202	*/	193	*/
203	void dtc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)	194	void dtc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)
204	{	195	{
205	tc_mapping_insert(va, asid, entry, true);	196	tc_mapping_insert(va, asid, entry, true);
206	}	197	}
207		198
208	/** Insert data into instruction translation cache.	199	/** Insert data into instruction translation cache.
209	*	200	*
210	* @param va Virtual page address.	201	* @param va Virtual page address.
211	* @param asid Address space identifier.	202	* @param asid Address space identifier.
212	* @param entry The rest of TLB entry as required by TLB insertion format.	203	* @param entry The rest of TLB entry as required by TLB insertion format.
213	*/	204	*/
214	void itc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)	205	void itc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)
215	{	206	{
216	tc_mapping_insert(va, asid, entry, false);	207	tc_mapping_insert(va, asid, entry, false);
217	}	208	}
218		209
219	/** Insert data into instruction or data translation cache.	210	/** Insert data into instruction or data translation cache.
220	*	211	*
221	* @param va Virtual page address.	212	* @param va Virtual page address.
222	* @param asid Address space identifier.	213	* @param asid Address space identifier.
223	* @param entry The rest of TLB entry as required by TLB insertion format.	214	* @param entry The rest of TLB entry as required by TLB insertion format.
224	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.	215	* @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
225	*/	216	*/
226	void tc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtc)	217	void tc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtc)
227	{	218	{
228	region_register rr;	219	region_register rr;
229	bool restore_rr = false;	220	bool restore_rr = false;
230		221
231	rr.word = rr_read(VA2VRN(va));	222	rr.word = rr_read(VA2VRN(va));
232	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	223	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
233	/*	224	/*
234	* The selected region register does not contain required RID.	225	* The selected region register does not contain required RID.
235	* Save the old content of the register and replace the RID.	226	* Save the old content of the register and replace the RID.
236	*/	227	*/
237	region_register rr0;	228	region_register rr0;
238		229
239	rr0 = rr;	230	rr0 = rr;
240	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	231	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
241	rr_write(VA2VRN(va), rr0.word);	232	rr_write(VA2VRN(va), rr0.word);
242	srlz_d();	233	srlz_d();
243	srlz_i();	234	srlz_i();
244	}	235	}
245		236
246	asm volatile (	237	asm volatile (
247	"mov r8=psr;;\n"	238	"mov r8=psr;;\n"
248	"rsm %0;;\n" /* PSR_IC_MASK */	239	"rsm %0;;\n" /* PSR_IC_MASK */
249	"srlz.d;;\n"	240	"srlz.d;;\n"
250	"srlz.i;;\n"	241	"srlz.i;;\n"
251	"mov cr.ifa=%1\n" /* va */	242	"mov cr.ifa=%1\n" /* va */
252	"mov cr.itir=%2;;\n" /* entry.word[1] */	243	"mov cr.itir=%2;;\n" /* entry.word[1] */
253	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */	244	"cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */
254	"(p6) itc.i %3;;\n"	245	"(p6) itc.i %3;;\n"
255	"(p7) itc.d %3;;\n"	246	"(p7) itc.d %3;;\n"
256	"mov psr.l=r8;;\n"	247	"mov psr.l=r8;;\n"
257	"srlz.d;;\n"	248	"srlz.d;;\n"
258	:	249	:
259	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)	250	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
260	: "p6", "p7", "r8"	251	: "p6", "p7", "r8"
261	);	252	);
262		253
263	if (restore_rr) {	254	if (restore_rr) {
264	rr_write(VA2VRN(va), rr.word);	255	rr_write(VA2VRN(va), rr.word);
265	srlz_d();	256	srlz_d();
266	srlz_i();	257	srlz_i();
267	}	258	}
268	}	259	}
269		260
270	/** Insert data into instruction translation register.	261	/** Insert data into instruction translation register.
271	*	262	*
272	* @param va Virtual page address.	263	* @param va Virtual page address.
273	* @param asid Address space identifier.	264	* @param asid Address space identifier.
274	* @param entry The rest of TLB entry as required by TLB insertion format.	265	* @param entry The rest of TLB entry as required by TLB insertion format.
275	* @param tr Translation register.	266	* @param tr Translation register.
276	*/	267	*/
277	void itr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr)	268	void itr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr)
278	{	269	{
279	tr_mapping_insert(va, asid, entry, false, tr);	270	tr_mapping_insert(va, asid, entry, false, tr);
280	}	271	}
281		272
282	/** Insert data into data translation register.	273	/** Insert data into data translation register.
283	*	274	*
284	* @param va Virtual page address.	275	* @param va Virtual page address.
285	* @param asid Address space identifier.	276	* @param asid Address space identifier.
286	* @param entry The rest of TLB entry as required by TLB insertion format.	277	* @param entry The rest of TLB entry as required by TLB insertion format.
287	* @param tr Translation register.	278	* @param tr Translation register.
288	*/	279	*/
289	void dtr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr)	280	void dtr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr)
290	{	281	{
291	tr_mapping_insert(va, asid, entry, true, tr);	282	tr_mapping_insert(va, asid, entry, true, tr);
292	}	283	}
293		284
294	/** Insert data into instruction or data translation register.	285	/** Insert data into instruction or data translation register.
295	*	286	*
296	* @param va Virtual page address.	287	* @param va Virtual page address.
297	* @param asid Address space identifier.	288	* @param asid Address space identifier.
298	* @param entry The rest of TLB entry as required by TLB insertion format.	289	* @param entry The rest of TLB entry as required by TLB insertion format.
299	* @param dtr If true, insert into data translation register, use instruction translation register otherwise.	290	* @param dtr If true, insert into data translation register, use instruction translation register otherwise.
300	* @param tr Translation register.	291	* @param tr Translation register.
301	*/	292	*/
302	void tr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)	293	void tr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
303	{	294	{
304	region_register rr;	295	region_register rr;
305	bool restore_rr = false;	296	bool restore_rr = false;
306		297
307	rr.word = rr_read(VA2VRN(va));	298	rr.word = rr_read(VA2VRN(va));
308	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {	299	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
309	/*	300	/*
310	* The selected region register does not contain required RID.	301	* The selected region register does not contain required RID.
311	* Save the old content of the register and replace the RID.	302	* Save the old content of the register and replace the RID.
312	*/	303	*/
313	region_register rr0;	304	region_register rr0;
314		305
315	rr0 = rr;	306	rr0 = rr;
316	rr0.map.rid = ASID2RID(asid, VA2VRN(va));	307	rr0.map.rid = ASID2RID(asid, VA2VRN(va));
317	rr_write(VA2VRN(va), rr0.word);	308	rr_write(VA2VRN(va), rr0.word);
318	srlz_d();	309	srlz_d();
319	srlz_i();	310	srlz_i();
320	}	311	}
321		312
322	asm volatile (	313	asm volatile (
323	"mov r8=psr;;\n"	314	"mov r8=psr;;\n"
324	"rsm %0;;\n" /* PSR_IC_MASK */	315	"rsm %0;;\n" /* PSR_IC_MASK */
325	"srlz.d;;\n"	316	"srlz.d;;\n"
326	"srlz.i;;\n"	317	"srlz.i;;\n"
327	"mov cr.ifa=%1\n" /* va */	318	"mov cr.ifa=%1\n" /* va */
328	"mov cr.itir=%2;;\n" /* entry.word[1] */	319	"mov cr.itir=%2;;\n" /* entry.word[1] */
329	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */	320	"cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */
330	"(p6) itr.i itr[%4]=%3;;\n"	321	"(p6) itr.i itr[%4]=%3;;\n"
331	"(p7) itr.d dtr[%4]=%3;;\n"	322	"(p7) itr.d dtr[%4]=%3;;\n"
332	"mov psr.l=r8;;\n"	323	"mov psr.l=r8;;\n"
333	"srlz.d;;\n"	324	"srlz.d;;\n"
334	:	325	:
335	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)	326	: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
336	: "p6", "p7", "r8"	327	: "p6", "p7", "r8"
337	);	328	);
338		329
339	if (restore_rr) {	330	if (restore_rr) {
340	rr_write(VA2VRN(va), rr.word);	331	rr_write(VA2VRN(va), rr.word);
341	srlz_d();	332	srlz_d();
342	srlz_i();	333	srlz_i();
343	}	334	}
344	}	335	}
345		336
346	/** Insert data into DTLB.	337	/** Insert data into DTLB.
347	*	338	*
348	* @param page Virtual page address including VRN bits.	339	* @param page Virtual page address including VRN bits.
349	* @param frame Physical frame address.	340	* @param frame Physical frame address.
350	* @param dtr If true, insert into data translation register, use data translation cache otherwise.	341	* @param dtr If true, insert into data translation register, use data translation cache otherwise.
351	* @param tr Translation register if dtr is true, ignored otherwise.	342	* @param tr Translation register if dtr is true, ignored otherwise.
352	*/	343	*/
353	void dtlb_kernel_mapping_insert(uintptr_t page, uintptr_t frame, bool dtr, index_t tr)	344	void dtlb_kernel_mapping_insert(uintptr_t page, uintptr_t frame, bool dtr, index_t tr)
354	{	345	{
355	tlb_entry_t entry;	346	tlb_entry_t entry;
356		347
357	entry.word[0] = 0;	348	entry.word[0] = 0;
358	entry.word[1] = 0;	349	entry.word[1] = 0;
359		350
360	entry.p = true; /* present */	351	entry.p = true; /* present */
361	entry.ma = MA_WRITEBACK;	352	entry.ma = MA_WRITEBACK;
362	entry.a = true; /* already accessed */	353	entry.a = true; /* already accessed */
363	entry.d = true; /* already dirty */	354	entry.d = true; /* already dirty */
364	entry.pl = PL_KERNEL;	355	entry.pl = PL_KERNEL;
365	entry.ar = AR_READ \| AR_WRITE;	356	entry.ar = AR_READ \| AR_WRITE;
366	entry.ppn = frame >> PPN_SHIFT;	357	entry.ppn = frame >> PPN_SHIFT;
367	entry.ps = PAGE_WIDTH;	358	entry.ps = PAGE_WIDTH;
368		359
369	if (dtr)	360	if (dtr)
370	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);	361	dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
371	else	362	else
372	dtc_mapping_insert(page, ASID_KERNEL, entry);	363	dtc_mapping_insert(page, ASID_KERNEL, entry);
373	}	364	}
374		365
375	/** Purge kernel entries from DTR.	366	/** Purge kernel entries from DTR.
376	*	367	*
377	* Purge DTR entries used by the kernel.	368	* Purge DTR entries used by the kernel.
378	*	369	*
379	* @param page Virtual page address including VRN bits.	370	* @param page Virtual page address including VRN bits.
380	* @param width Width of the purge in bits.	371	* @param width Width of the purge in bits.
381	*/	372	*/
382	void dtr_purge(uintptr_t page, count_t width)	373	void dtr_purge(uintptr_t page, count_t width)
383	{	374	{
384	asm volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));	375	asm volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));
385	}	376	}
386		377
387		378
388	/** Copy content of PTE into data translation cache.	379	/** Copy content of PTE into data translation cache.
389	*	380	*
390	* @param t PTE.	381	* @param t PTE.
391	*/	382	*/
392	void dtc_pte_copy(pte_t *t)	383	void dtc_pte_copy(pte_t *t)
393	{	384	{
394	tlb_entry_t entry;	385	tlb_entry_t entry;
395		386
396	entry.word[0] = 0;	387	entry.word[0] = 0;
397	entry.word[1] = 0;	388	entry.word[1] = 0;
398		389
399	entry.p = t->p;	390	entry.p = t->p;
400	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	391	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
401	entry.a = t->a;	392	entry.a = t->a;
402	entry.d = t->d;	393	entry.d = t->d;
403	entry.pl = t->k ? PL_KERNEL : PL_USER;	394	entry.pl = t->k ? PL_KERNEL : PL_USER;
404	entry.ar = t->w ? AR_WRITE : AR_READ;	395	entry.ar = t->w ? AR_WRITE : AR_READ;
405	entry.ppn = t->frame >> PPN_SHIFT;	396	entry.ppn = t->frame >> PPN_SHIFT;
406	entry.ps = PAGE_WIDTH;	397	entry.ps = PAGE_WIDTH;
407		398
408	dtc_mapping_insert(t->page, t->as->asid, entry);	399	dtc_mapping_insert(t->page, t->as->asid, entry);
409	#ifdef CONFIG_VHPT	400	#ifdef CONFIG_VHPT
410	vhpt_mapping_insert(t->page, t->as->asid, entry);	401	vhpt_mapping_insert(t->page, t->as->asid, entry);
411	#endif	402	#endif
412	}	403	}
413		404
414	/** Copy content of PTE into instruction translation cache.	405	/** Copy content of PTE into instruction translation cache.
415	*	406	*
416	* @param t PTE.	407	* @param t PTE.
417	*/	408	*/
418	void itc_pte_copy(pte_t *t)	409	void itc_pte_copy(pte_t *t)
419	{	410	{
420	tlb_entry_t entry;	411	tlb_entry_t entry;
421		412
422	entry.word[0] = 0;	413	entry.word[0] = 0;
423	entry.word[1] = 0;	414	entry.word[1] = 0;
424		415
425	ASSERT(t->x);	416	ASSERT(t->x);
426		417
427	entry.p = t->p;	418	entry.p = t->p;
428	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;	419	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
429	entry.a = t->a;	420	entry.a = t->a;
430	entry.pl = t->k ? PL_KERNEL : PL_USER;	421	entry.pl = t->k ? PL_KERNEL : PL_USER;
431	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;	422	entry.ar = t->x ? (AR_EXECUTE \| AR_READ) : AR_READ;
432	entry.ppn = t->frame >> PPN_SHIFT;	423	entry.ppn = t->frame >> PPN_SHIFT;
433	entry.ps = PAGE_WIDTH;	424	entry.ps = PAGE_WIDTH;
434		425
435	itc_mapping_insert(t->page, t->as->asid, entry);	426	itc_mapping_insert(t->page, t->as->asid, entry);
436	#ifdef CONFIG_VHPT	427	#ifdef CONFIG_VHPT
437	vhpt_mapping_insert(t->page, t->as->asid, entry);	428	vhpt_mapping_insert(t->page, t->as->asid, entry);
438	#endif	429	#endif
439	}	430	}
440		431
441	/** Instruction TLB fault handler for faults with VHPT turned off.	432	/** Instruction TLB fault handler for faults with VHPT turned off.
442	*	433	*
443	* @param vector Interruption vector.	434	* @param vector Interruption vector.
444	* @param istate Structure with saved interruption state.	435	* @param istate Structure with saved interruption state.
445	*/	436	*/
446	void alternate_instruction_tlb_fault(uint64_t vector, istate_t *istate)	437	void alternate_instruction_tlb_fault(uint64_t vector, istate_t *istate)
447	{	438	{
448	region_register rr;	439	region_register rr;
449	rid_t rid;	440	rid_t rid;
450	uintptr_t va;	441	uintptr_t va;
451	pte_t *t;	442	pte_t *t;
452		443
453	va = istate->cr_ifa; /* faulting address */	444	va = istate->cr_ifa; /* faulting address */
454	rr.word = rr_read(VA2VRN(va));	445	rr.word = rr_read(VA2VRN(va));
455	rid = rr.map.rid;	446	rid = rr.map.rid;
456		447
457	page_table_lock(AS, true);	448	page_table_lock(AS, true);
458	t = page_mapping_find(AS, va);	449	t = page_mapping_find(AS, va);
459	if (t) {	450	if (t) {
460	/*	451	/*
461	* The mapping was found in software page hash table.	452	* The mapping was found in software page hash table.
462	* Insert it into data translation cache.	453	* Insert it into data translation cache.
463	*/	454	*/
464	itc_pte_copy(t);	455	itc_pte_copy(t);
465	page_table_unlock(AS, true);	456	page_table_unlock(AS, true);
466	} else {	457	} else {
467	/*	458	/*
468	* Forward the page fault to address space page fault handler.	459	* Forward the page fault to address space page fault handler.
469	*/	460	*/
470	page_table_unlock(AS, true);	461	page_table_unlock(AS, true);
471	if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {	462	if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
472	fault_if_from_uspace(istate,"Page fault at %p",va);	463	fault_if_from_uspace(istate,"Page fault at %p",va);
473	panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);	464	panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip);
474	}	465	}
475	}	466	}
476	}	467	}
477		468
478		469
479		470
480	static int is_io_page_accessible(int page)	471	static int is_io_page_accessible(int page)
481	{	472	{
482	if(TASK->arch.iomap) return bitmap_get(TASK->arch.iomap,page);	473	if(TASK->arch.iomap) return bitmap_get(TASK->arch.iomap,page);
483	else return 0;	474	else return 0;
484	}	475	}
485		476
486	#define IO_FRAME_BASE 0xFFFFC000000	477	#define IO_FRAME_BASE 0xFFFFC000000
487		478
488	/** There is special handling of memmaped lagacy io, because	479	/** There is special handling of memmaped lagacy io, because
489	* of 4KB sized access	480	* of 4KB sized access
490	* only for userspace	481	* only for userspace
491	*	482	*
492	* @param va virtual address of page fault	483	* @param va virtual address of page fault
493	* @param istate Structure with saved interruption state.	484	* @param istate Structure with saved interruption state.
494	*	485	*
495	*	486	*
496	* @return 1 on success, 0 on fail	487	* @return 1 on success, 0 on fail
497	*/	488	*/
498	static int try_memmap_io_insertion(uintptr_t va, istate_t *istate)	489	static int try_memmap_io_insertion(uintptr_t va, istate_t *istate)
499	{	490	{
500	if((va >= IO_OFFSET ) && (va < IO_OFFSET + (1<<IO_PAGE_WIDTH)))	491	if((va >= IO_OFFSET ) && (va < IO_OFFSET + (1<<IO_PAGE_WIDTH)))

Subversion Repositories HelenOS

(root)/trunk/kernel/arch/ia64/src/mm/tlb.c – Rev 3635 → 3763