Subversion Repositories HelenOS

/*

 * Copyright (C) 2006 Jakub Jermar

 * All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions

 * are met:

 *

 * - Redistributions of source code must retain the above copyright

 *   notice, this list of conditions and the following disclaimer.

 * - Redistributions in binary form must reproduce the above copyright

 *   notice, this list of conditions and the following disclaimer in the

 *   documentation and/or other materials provided with the distribution.

 * - The name of the author may not be used to endorse or promote products

 *   derived from this software without specific prior written permission.

 *

 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR

 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.

 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,

 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

/*

 * TLB management.

 */

#include <mm/tlb.h>

#include <mm/asid.h>

#include <mm/page.h>

#include <mm/as.h>

#include <arch/mm/tlb.h>

#include <arch/mm/page.h>

#include <arch/mm/vhpt.h>

#include <arch/barrier.h>

#include <arch/interrupt.h>

#include <arch/pal/pal.h>

#include <arch/asm.h>

#include <typedefs.h>

#include <panic.h>

#include <print.h>

#include <arch.h>

#include <interrupt.h>

/** Invalidate all TLB entries. */

void tlb_invalidate_all(void)

{

		ipl_t ipl;

		__address adr;

		__u32 count1, count2, stride1, stride2;

		int i,j;

		adr = PAL_PTCE_INFO_BASE();

		count1 = PAL_PTCE_INFO_COUNT1();

		count2 = PAL_PTCE_INFO_COUNT2();

		stride1 = PAL_PTCE_INFO_STRIDE1();

		stride2 = PAL_PTCE_INFO_STRIDE2();

		ipl = interrupts_disable();

		for(i = 0; i < count1; i++) {

			for(j = 0; j < count2; j++) {

				__asm__ volatile (

					"ptc.e %0 ;;"

					:

					: "r" (adr)

				);

				adr += stride2;

			}

			adr += stride1;

		}

		interrupts_restore(ipl);

		srlz_d();

		srlz_i();

#ifdef CONFIG_VHPT

		vhpt_invalidate_all();

#endif	

}

/** Invalidate entries belonging to an address space.

 *

 * @param asid Address space identifier.

 */

void tlb_invalidate_asid(asid_t asid)

{

	tlb_invalidate_all();

}

void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)

{

	region_register rr;

	bool restore_rr = false;

	int b = 0;

	int c = cnt;

	__address va;

	va = page;

	rr.word = rr_read(VA2VRN(va));

	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {

		/*

		 * The selected region register does not contain required RID.

		 * Save the old content of the register and replace the RID.

		 */

		region_register rr0;

		rr0 = rr;

		rr0.map.rid = ASID2RID(asid, VA2VRN(va));

		rr_write(VA2VRN(va), rr0.word);

		srlz_d();

		srlz_i();

	}

	while(c >>= 1)

		b++;

	b >>= 1;

	__u64 ps;

	switch (b) {

		case 0: /*cnt 1-3*/

			ps = PAGE_WIDTH;

			break;

		case 1: /*cnt 4-15*/

			/*cnt=((cnt-1)/4)+1;*/

			ps = PAGE_WIDTH+2;

			va &= ~((1<<ps)-1);

			break;

		case 2: /*cnt 16-63*/

			/*cnt=((cnt-1)/16)+1;*/

			ps = PAGE_WIDTH+4;

			va &= ~((1<<ps)-1);

			break;

		case 3: /*cnt 64-255*/

			/*cnt=((cnt-1)/64)+1;*/

			ps = PAGE_WIDTH+6;

			va &= ~((1<<ps)-1);

			break;

		case 4: /*cnt 256-1023*/

			/*cnt=((cnt-1)/256)+1;*/

			ps = PAGE_WIDTH+8;

			va &= ~((1<<ps)-1);

			break;

		case 5: /*cnt 1024-4095*/

			/*cnt=((cnt-1)/1024)+1;*/

			ps = PAGE_WIDTH+10;

			va &= ~((1<<ps)-1);

			break;

		case 6: /*cnt 4096-16383*/

			/*cnt=((cnt-1)/4096)+1;*/

			ps = PAGE_WIDTH+12;

			va &= ~((1<<ps)-1);

			break;

		case 7: /*cnt 16384-65535*/

		case 8: /*cnt 65536-(256K-1)*/

			/*cnt=((cnt-1)/16384)+1;*/

			ps = PAGE_WIDTH+14;

			va &= ~((1<<ps)-1);

			break;

		default:

			/*cnt=((cnt-1)/(16384*16))+1;*/

			ps=PAGE_WIDTH+18;

			va&=~((1<<ps)-1);

			break;

	}

	/*cnt+=(page!=va);*/

	for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps))	{

		__asm__ volatile (

			"ptc.l %0,%1;;"

			:

			: "r" (va), "r" (ps<<2)

		);

	}

	srlz_d();

	srlz_i();

	if (restore_rr) {

		rr_write(VA2VRN(va), rr.word);

		srlz_d();

		srlz_i();

	}

}

/** Insert data into data translation cache.

 *

 * @param va Virtual page address.

 * @param asid Address space identifier.

 * @param entry The rest of TLB entry as required by TLB insertion format.

 */

void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)

{

	tc_mapping_insert(va, asid, entry, true);

}

/** Insert data into instruction translation cache.

 *

 * @param va Virtual page address.

 * @param asid Address space identifier.

 * @param entry The rest of TLB entry as required by TLB insertion format.

 */

void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)

{

	tc_mapping_insert(va, asid, entry, false);

}

/** Insert data into instruction or data translation cache.

 *

 * @param va Virtual page address.

 * @param asid Address space identifier.

 * @param entry The rest of TLB entry as required by TLB insertion format.

 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.

 */

void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)

{

	region_register rr;

	bool restore_rr = false;

	rr.word = rr_read(VA2VRN(va));

	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {

		/*

		 * The selected region register does not contain required RID.

		 * Save the old content of the register and replace the RID.

		 */

		region_register rr0;

		rr0 = rr;

		rr0.map.rid = ASID2RID(asid, VA2VRN(va));

		rr_write(VA2VRN(va), rr0.word);

		srlz_d();

		srlz_i();

	}

	__asm__ volatile (

		"mov r8=psr;;\n"

		"rsm %0;;\n"   			/* PSR_IC_MASK */

		"srlz.d;;\n"

		"srlz.i;;\n"

		"mov cr.ifa=%1\n"		/* va */

		"mov cr.itir=%2;;\n"		/* entry.word[1] */

		"cmp.eq p6,p7 = %4,r0;;\n"	/* decide between itc and dtc */ 

		"(p6) itc.i %3;;\n"

		"(p7) itc.d %3;;\n"

		"mov psr.l=r8;;\n"

		"srlz.d;;\n"

		:

		: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)

		: "p6", "p7", "r8"

	);

	if (restore_rr) {

		rr_write(VA2VRN(va), rr.word);

		srlz_d();

		srlz_i();

	}

}

/** Insert data into instruction translation register.

 *

 * @param va Virtual page address.

 * @param asid Address space identifier.

 * @param entry The rest of TLB entry as required by TLB insertion format.

 * @param tr Translation register.

 */

void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)

{

	tr_mapping_insert(va, asid, entry, false, tr);

}

/** Insert data into data translation register.

 *

 * @param va Virtual page address.

 * @param asid Address space identifier.

 * @param entry The rest of TLB entry as required by TLB insertion format.

 * @param tr Translation register.

 */

void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)

{

	tr_mapping_insert(va, asid, entry, true, tr);

}

/** Insert data into instruction or data translation register.

 *

 * @param va Virtual page address.

 * @param asid Address space identifier.

 * @param entry The rest of TLB entry as required by TLB insertion format.

 * @param dtc If true, insert into data translation register, use instruction translation register otherwise.

 * @param tr Translation register.

 */

void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)

{

	region_register rr;

	bool restore_rr = false;

	rr.word = rr_read(VA2VRN(va));

	if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {

		/*

		 * The selected region register does not contain required RID.

		 * Save the old content of the register and replace the RID.

		 */

		region_register rr0;

		rr0 = rr;

		rr0.map.rid = ASID2RID(asid, VA2VRN(va));

		rr_write(VA2VRN(va), rr0.word);

		srlz_d();

		srlz_i();

	}

	__asm__ volatile (

		"mov r8=psr;;\n"

		"rsm %0;;\n"			/* PSR_IC_MASK */

		"srlz.d;;\n"

		"srlz.i;;\n"

		"mov cr.ifa=%1\n"        	/* va */		 

		"mov cr.itir=%2;;\n"		/* entry.word[1] */ 

		"cmp.eq p6,p7=%5,r0;;\n"	/* decide between itr and dtr */

		"(p6) itr.i itr[%4]=%3;;\n"

		"(p7) itr.d dtr[%4]=%3;;\n"

		"mov psr.l=r8;;\n"

		"srlz.d;;\n"

		:

		: "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)

		: "p6", "p7", "r8"

	);

	if (restore_rr) {

		rr_write(VA2VRN(va), rr.word);

		srlz_d();

		srlz_i();

	}

}

/** Insert data into DTLB.

 *

 * @param page Virtual page address including VRN bits.

 * @param frame Physical frame address.

 * @param dtr If true, insert into data translation register, use data translation cache otherwise.

 * @param tr Translation register if dtr is true, ignored otherwise.

 */

void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)

{

	tlb_entry_t entry;

	entry.word[0] = 0;

	entry.word[1] = 0;

	entry.p = true;			/* present */

	entry.ma = MA_WRITEBACK;

	entry.a = true;			/* already accessed */

	entry.d = true;			/* already dirty */

	entry.pl = PL_KERNEL;

	entry.ar = AR_READ | AR_WRITE;

	entry.ppn = frame >> PPN_SHIFT;

	entry.ps = PAGE_WIDTH;

	if (dtr)

		dtr_mapping_insert(page, ASID_KERNEL, entry, tr);

	else

		dtc_mapping_insert(page, ASID_KERNEL, entry);

}

/** Purge kernel entries from DTR.

 *

 * Purge DTR entries used by the kernel.

 *

 * @param page Virtual page address including VRN bits.

 * @param width Width of the purge in bits.

 */

void dtr_purge(__address page, count_t width)

{

	__asm__ volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));

}

/** Copy content of PTE into data translation cache.

 *

 * @param t PTE.

 */

void dtc_pte_copy(pte_t *t)

{

	tlb_entry_t entry;

	entry.word[0] = 0;

	entry.word[1] = 0;

	entry.p = t->p;

	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;

	entry.a = t->a;

	entry.d = t->d;

	entry.pl = t->k ? PL_KERNEL : PL_USER;

	entry.ar = t->w ? AR_WRITE : AR_READ;

	entry.ppn = t->frame >> PPN_SHIFT;

	entry.ps = PAGE_WIDTH;

	dtc_mapping_insert(t->page, t->as->asid, entry);

#ifdef CONFIG_VHPT

	vhpt_mapping_insert(t->page, t->as->asid, entry);

#endif	

}

/** Copy content of PTE into instruction translation cache.

 *

 * @param t PTE.

 */

void itc_pte_copy(pte_t *t)

{

	tlb_entry_t entry;

	entry.word[0] = 0;

	entry.word[1] = 0;

	ASSERT(t->x);

	entry.p = t->p;

	entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;

	entry.a = t->a;

	entry.pl = t->k ? PL_KERNEL : PL_USER;

	entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;

	entry.ppn = t->frame >> PPN_SHIFT;

	entry.ps = PAGE_WIDTH;

	itc_mapping_insert(t->page, t->as->asid, entry);

#ifdef CONFIG_VHPT

	vhpt_mapping_insert(t->page, t->as->asid, entry);

#endif	

}

/** Instruction TLB fault handler for faults with VHPT turned off.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)

{

	region_register rr;

	rid_t rid;

	__address va;

	pte_t *t;

	va = istate->cr_ifa;	/* faulting address */

	rr.word = rr_read(VA2VRN(va));

	rid = rr.map.rid;

	page_table_lock(AS, true);

	t = page_mapping_find(AS, va);

	if (t) {

		/*

		 * The mapping was found in software page hash table.

		 * Insert it into data translation cache.

		 */

		itc_pte_copy(t);

		page_table_unlock(AS, true);

	} else {

		/*

		 * Forward the page fault to address space page fault handler.

		 */

		page_table_unlock(AS, true);

		if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {

			fault_if_from_uspace(istate,"Page fault at %P",va);

			panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);

		}

	}

}

/** Data TLB fault handler for faults with VHPT turned off.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void alternate_data_tlb_fault(__u64 vector, istate_t *istate)

{

	region_register rr;

	rid_t rid;

	__address va;

	pte_t *t;

	va = istate->cr_ifa;	/* faulting address */

	rr.word = rr_read(VA2VRN(va));

	rid = rr.map.rid;

	if (RID2ASID(rid) == ASID_KERNEL) {

		if (VA2VRN(va) == VRN_KERNEL) {

			/*

			 * Provide KA2PA(identity) mapping for faulting piece of

			 * kernel address space.

			 */

			dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);

			return;

		}

	}

	page_table_lock(AS, true);

	t = page_mapping_find(AS, va);

	if (t) {

		/*

		 * The mapping was found in software page hash table.

		 * Insert it into data translation cache.

		 */

		dtc_pte_copy(t);

		page_table_unlock(AS, true);

	} else {

		/*

		 * Forward the page fault to address space page fault handler.

		 */

		page_table_unlock(AS, true);

		if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {

			fault_if_from_uspace(istate,"Page fault at %P",va);

			panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);

		}

	}

}

/** Data nested TLB fault handler.

 *

 * This fault should not occur.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void data_nested_tlb_fault(__u64 vector, istate_t *istate)

{

	panic("%s\n", __FUNCTION__);

}

/** Data Dirty bit fault handler.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void data_dirty_bit_fault(__u64 vector, istate_t *istate)

{

	region_register rr;

	rid_t rid;

	__address va;

	pte_t *t;

	va = istate->cr_ifa;	/* faulting address */

	rr.word = rr_read(VA2VRN(va));

	rid = rr.map.rid;

	page_table_lock(AS, true);

	t = page_mapping_find(AS, va);

	ASSERT(t && t->p);

	if (t && t->p && t->w) {

		/*

		 * Update the Dirty bit in page tables and reinsert

		 * the mapping into DTC.

		 */

		t->d = true;

		dtc_pte_copy(t);

	} else {

		if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {

			fault_if_from_uspace(istate,"Page fault at %P",va);

			panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);

			t->d = true;

			dtc_pte_copy(t);

		}

	}

	page_table_unlock(AS, true);

}

/** Instruction access bit fault handler.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void instruction_access_bit_fault(__u64 vector, istate_t *istate)

{

	region_register rr;

	rid_t rid;

	__address va;

	pte_t *t;	

	va = istate->cr_ifa;	/* faulting address */

	rr.word = rr_read(VA2VRN(va));

	rid = rr.map.rid;

	page_table_lock(AS, true);

	t = page_mapping_find(AS, va);

	ASSERT(t && t->p);

	if (t && t->p && t->x) {

		/*

		 * Update the Accessed bit in page tables and reinsert

		 * the mapping into ITC.

		 */

		t->a = true;

		itc_pte_copy(t);

	} else {

		if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {

			fault_if_from_uspace(istate,"Page fault at %P",va);

			panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);

			t->a = true;

			itc_pte_copy(t);

		}

	}

	page_table_unlock(AS, true);

}

/** Data access bit fault handler.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void data_access_bit_fault(__u64 vector, istate_t *istate)

{

	region_register rr;

	rid_t rid;

	__address va;

	pte_t *t;

	va = istate->cr_ifa;	/* faulting address */

	rr.word = rr_read(VA2VRN(va));

	rid = rr.map.rid;

	page_table_lock(AS, true);

	t = page_mapping_find(AS, va);

	ASSERT(t && t->p);

	if (t && t->p) {

		/*

		 * Update the Accessed bit in page tables and reinsert

		 * the mapping into DTC.

		 */

		t->a = true;

		dtc_pte_copy(t);

	} else {

		if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {

			fault_if_from_uspace(istate,"Page fault at %P",va);

			panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);

			t->a = true;

			itc_pte_copy(t);

		}

	}

	page_table_unlock(AS, true);

}

/** Page not present fault handler.

 *

 * @param vector Interruption vector.

 * @param istate Structure with saved interruption state.

 */

void page_not_present(__u64 vector, istate_t *istate)

{

	region_register rr;

	rid_t rid;

	__address va;

	pte_t *t;

	va = istate->cr_ifa;	/* faulting address */

	rr.word = rr_read(VA2VRN(va));

	rid = rr.map.rid;

	page_table_lock(AS, true);

	t = page_mapping_find(AS, va);

	ASSERT(t);

	if (t->p) {

		/*

		 * If the Present bit is set in page hash table, just copy it

		 * and update ITC/DTC.

		 */

		if (t->x)

			itc_pte_copy(t);

		else

			dtc_pte_copy(t);

		page_table_unlock(AS, true);

	} else {

		page_table_unlock(AS, true);

		if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {

			fault_if_from_uspace(istate,"Page fault at %P",va);

			panic("%s: va=%p, rid=%d\n", __FUNCTION__, va, rid);

		}

	}

}