WebSVN – HelenOS – Blame – /branches/network/kernel/arch/ppc32/src/mm/tlb.c

Rev	Author	Line No.	Line
1215	decky	1	/*
2071	jermar	2	* Copyright (c) 2006 Martin Decky
1215	decky	3	* All rights reserved.
		4	*
		5	* Redistribution and use in source and binary forms, with or without
		6	* modification, are permitted provided that the following conditions
		7	* are met:
		8	*
		9	* - Redistributions of source code must retain the above copyright
		10	* notice, this list of conditions and the following disclaimer.
		11	* - Redistributions in binary form must reproduce the above copyright
		12	* notice, this list of conditions and the following disclaimer in the
		13	* documentation and/or other materials provided with the distribution.
		14	* - The name of the author may not be used to endorse or promote products
		15	* derived from this software without specific prior written permission.
		16	*
		17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
		18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
		19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
		20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
		21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
		22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
		23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
		24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
		25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
		26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
		27	*/
		28
4153	mejdrech	29	/** @addtogroup ppc32mm
1702	cejka	30	* @{
		31	*/
		32	/** @file
		33	*/
		34
1215	decky	35	#include <mm/tlb.h>
1730	decky	36	#include <arch/mm/tlb.h>
		37	#include <arch/interrupt.h>
4153	mejdrech	38	#include <interrupt.h>
1730	decky	39	#include <mm/as.h>
		40	#include <arch.h>
		41	#include <print.h>
4153	mejdrech	42	#include <macros.h>
1730	decky	43	#include <symtab.h>
1215	decky	44
4153	mejdrech	45	static unsigned int seed = 10;
		46	static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
1215	decky	47
4153	mejdrech	48
		49	#define TLB_FLUSH \
		50	"tlbie %0\n" \
		51	"addi %0, %0, 0x1000\n"
		52
		53
1730	decky	54	/** Try to find PTE for faulting address
1215	decky	55	*
1730	decky	56	* Try to find PTE for faulting address.
		57	* The as->lock must be held on entry to this function
		58	* if lock is true.
1215	decky	59	*
3193	jermar	60	* @param as Address space.
		61	* @param lock Lock/unlock the address space.
		62	* @param badvaddr Faulting virtual address.
		63	* @param access Access mode that caused the fault.
		64	* @param istate Pointer to interrupted state.
		65	* @param pfrc Pointer to variable where as_page_fault() return code
		66	* will be stored.
		67	* @return PTE on success, NULL otherwise.
1730	decky	68	*
1215	decky	69	*/
3193	jermar	70	static pte_t *
		71	find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
		72	istate_t istate, int pfrc)
1730	decky	73	{
		74	/*
		75	* Check if the mapping exists in page tables.
		76	*/
		77	pte_t *pte = page_mapping_find(as, badvaddr);
4153	mejdrech	78	if ((pte) && (pte->present)) {
1730	decky	79	/*
		80	* Mapping found in page tables.
		81	* Immediately succeed.
		82	*/
		83	return pte;
		84	} else {
		85	int rc;
		86
		87	/*
		88	* Mapping not found in page tables.
		89	* Resort to higher-level page fault handler.
		90	*/
		91	page_table_unlock(as, lock);
		92	switch (rc = as_page_fault(badvaddr, access, istate)) {
3193	jermar	93	case AS_PF_OK:
		94	/*
		95	* The higher-level page fault handler succeeded,
		96	* The mapping ought to be in place.
		97	*/
		98	page_table_lock(as, lock);
		99	pte = page_mapping_find(as, badvaddr);
4153	mejdrech	100	ASSERT((pte) && (pte->present));
3193	jermar	101	*pfrc = 0;
		102	return pte;
		103	case AS_PF_DEFER:
		104	page_table_lock(as, lock);
		105	*pfrc = rc;
		106	return NULL;
		107	case AS_PF_FAULT:
		108	page_table_lock(as, lock);
		109	*pfrc = rc;
		110	return NULL;
		111	default:
4153	mejdrech	112	panic("Unexpected rc (%d).", rc);
1730	decky	113	}
		114	}
		115	}
		116
		117
1780	jermar	118	static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
1730	decky	119	{
4153	mejdrech	120	char *symbol;
		121	char *sym2;
1730	decky	122
4153	mejdrech	123	symbol = symtab_fmt_name_lookup(istate->pc);
		124	sym2 = symtab_fmt_name_lookup(istate->lr);
		125
		126	fault_if_from_uspace(istate,
		127	"PHT Refill Exception on %p.", badvaddr);
		128	panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
3193	jermar	129	istate->pc, symbol, sym2);
1730	decky	130	}
		131
		132
4153	mejdrech	133	static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
1730	decky	134	{
1780	jermar	135	uint32_t page = (vaddr >> 12) & 0xffff;
		136	uint32_t api = (vaddr >> 22) & 0x3f;
1730	decky	137
1780	jermar	138	uint32_t vsid;
1730	decky	139	asm volatile (
		140	"mfsrin %0, %1\n"
		141	: "=r" (vsid)
		142	: "r" (vaddr)
		143	);
		144
1780	jermar	145	uint32_t sdr1;
1730	decky	146	asm volatile (
		147	"mfsdr1 %0\n"
		148	: "=r" (sdr1)
		149	);
		150	phte_t phte = (phte_t ) PA2KA(sdr1 & 0xffff0000);
		151
		152	/* Primary hash (xor) */
1780	jermar	153	uint32_t h = 0;
		154	uint32_t hash = vsid ^ page;
		155	uint32_t base = (hash & 0x3ff) << 3;
		156	uint32_t i;
1730	decky	157	bool found = false;
		158
4153	mejdrech	159	/* Find colliding PTE in PTEG */
1730	decky	160	for (i = 0; i < 8; i++) {
4153	mejdrech	161	if ((phte[base + i].v)
		162	&& (phte[base + i].vsid == vsid)
		163	&& (phte[base + i].api == api)
		164	&& (phte[base + i].h == 0)) {
1730	decky	165	found = true;
		166	break;
		167	}
		168	}
		169
		170	if (!found) {
4153	mejdrech	171	/* Find unused PTE in PTEG */
1730	decky	172	for (i = 0; i < 8; i++) {
4153	mejdrech	173	if (!phte[base + i].v) {
1730	decky	174	found = true;
		175	break;
		176	}
		177	}
		178	}
		179
		180	if (!found) {
		181	/* Secondary hash (not) */
1780	jermar	182	uint32_t base2 = (~hash & 0x3ff) << 3;
1730	decky	183
4153	mejdrech	184	/* Find colliding PTE in PTEG */
1730	decky	185	for (i = 0; i < 8; i++) {
4153	mejdrech	186	if ((phte[base2 + i].v)
		187	&& (phte[base2 + i].vsid == vsid)
		188	&& (phte[base2 + i].api == api)
		189	&& (phte[base2 + i].h == 1)) {
1730	decky	190	found = true;
		191	base = base2;
		192	h = 1;
		193	break;
		194	}
		195	}
		196
		197	if (!found) {
4153	mejdrech	198	/* Find unused PTE in PTEG */
		199	for (i = 0; i < 8; i++) {
		200	if (!phte[base2 + i].v) {
		201	found = true;
		202	base = base2;
		203	h = 1;
		204	break;
		205	}
		206	}
1730	decky	207	}
4153	mejdrech	208
		209	if (!found)
		210	i = RANDI(seed) % 8;
1730	decky	211	}
		212
4153	mejdrech	213	phte[base + i].v = 1;
		214	phte[base + i].vsid = vsid;
		215	phte[base + i].h = h;
		216	phte[base + i].api = api;
		217	phte[base + i].rpn = pte->pfn;
		218	phte[base + i].r = 0;
		219	phte[base + i].c = 0;
		220	phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
		221	phte[base + i].pp = 2; // FIXME
1730	decky	222	}
		223
		224
4153	mejdrech	225	/** Process Instruction/Data Storage Exception
1730	decky	226	*
4153	mejdrech	227	* @param n Exception vector number.
		228	* @param istate Interrupted register context.
1730	decky	229	*
		230	*/
		231	void pht_refill(int n, istate_t *istate)
		232	{
1780	jermar	233	uintptr_t badvaddr;
1730	decky	234	pte_t *pte;
		235	int pfrc;
		236	as_t *as;
		237	bool lock;
		238
		239	if (AS == NULL) {
		240	as = AS_KERNEL;
		241	lock = false;
		242	} else {
		243	as = AS;
		244	lock = true;
		245	}
		246
4153	mejdrech	247	if (n == VECTOR_DATA_STORAGE)
		248	badvaddr = istate->dar;
		249	else
1730	decky	250	badvaddr = istate->pc;
		251
		252	page_table_lock(as, lock);
		253
3193	jermar	254	pte = find_mapping_and_check(as, lock, badvaddr,
		255	PF_ACCESS_READ /* FIXME */, istate, &pfrc);
1730	decky	256	if (!pte) {
		257	switch (pfrc) {
3193	jermar	258	case AS_PF_FAULT:
		259	goto fail;
		260	break;
		261	case AS_PF_DEFER:
		262	/*
		263	* The page fault came during copy_from_uspace()
		264	* or copy_to_uspace().
		265	*/
		266	page_table_unlock(as, lock);
		267	return;
		268	default:
4153	mejdrech	269	panic("Unexpected pfrc (%d).", pfrc);
1730	decky	270	}
		271	}
		272
4153	mejdrech	273	pte->accessed = 1; /* Record access to PTE */
		274	pht_insert(badvaddr, pte);
1730	decky	275
		276	page_table_unlock(as, lock);
		277	return;
		278
		279	fail:
		280	page_table_unlock(as, lock);
		281	pht_refill_fail(badvaddr, istate);
		282	}
		283
		284
4153	mejdrech	285	/** Process Instruction/Data Storage Exception in Real Mode
1730	decky	286	*
4153	mejdrech	287	* @param n Exception vector number.
		288	* @param istate Interrupted register context.
1730	decky	289	*
		290	*/
4153	mejdrech	291	bool pht_refill_real(int n, istate_t *istate)
1730	decky	292	{
1780	jermar	293	uintptr_t badvaddr;
1730	decky	294
4153	mejdrech	295	if (n == VECTOR_DATA_STORAGE)
		296	badvaddr = istate->dar;
		297	else
1730	decky	298	badvaddr = istate->pc;
		299
1780	jermar	300	uint32_t physmem;
1730	decky	301	asm volatile (
		302	"mfsprg3 %0\n"
		303	: "=r" (physmem)
		304	);
		305
4153	mejdrech	306	if ((badvaddr < PA2KA(0)) \|\| (badvaddr >= PA2KA(physmem)))
		307	return false;
		308
		309	uint32_t page = (badvaddr >> 12) & 0xffff;
		310	uint32_t api = (badvaddr >> 22) & 0x3f;
		311
		312	uint32_t vsid;
		313	asm volatile (
		314	"mfsrin %0, %1\n"
		315	: "=r" (vsid)
		316	: "r" (badvaddr)
		317	);
		318
		319	uint32_t sdr1;
		320	asm volatile (
		321	"mfsdr1 %0\n"
		322	: "=r" (sdr1)
		323	);
		324	phte_t phte_real = (phte_t ) (sdr1 & 0xffff0000);
		325
		326	/* Primary hash (xor) */
		327	uint32_t h = 0;
		328	uint32_t hash = vsid ^ page;
		329	uint32_t base = (hash & 0x3ff) << 3;
		330	uint32_t i;
		331	bool found = false;
		332
		333	/* Find colliding PTE in PTEG */
		334	for (i = 0; i < 8; i++) {
		335	if ((phte_real[base + i].v)
		336	&& (phte_real[base + i].vsid == vsid)
		337	&& (phte_real[base + i].api == api)
		338	&& (phte_real[base + i].h == 0)) {
		339	found = true;
		340	break;
		341	}
1730	decky	342	}
		343
4153	mejdrech	344	if (!found) {
		345	/* Find unused PTE in PTEG */
		346	for (i = 0; i < 8; i++) {
		347	if (!phte_real[base + i].v) {
		348	found = true;
		349	break;
		350	}
		351	}
		352	}
		353
		354	if (!found) {
		355	/* Secondary hash (not) */
		356	uint32_t base2 = (~hash & 0x3ff) << 3;
		357
		358	/* Find colliding PTE in PTEG */
		359	for (i = 0; i < 8; i++) {
		360	if ((phte_real[base2 + i].v)
		361	&& (phte_real[base2 + i].vsid == vsid)
		362	&& (phte_real[base2 + i].api == api)
		363	&& (phte_real[base2 + i].h == 1)) {
		364	found = true;
		365	base = base2;
		366	h = 1;
		367	break;
		368	}
		369	}
		370
		371	if (!found) {
		372	/* Find unused PTE in PTEG */
		373	for (i = 0; i < 8; i++) {
		374	if (!phte_real[base2 + i].v) {
		375	found = true;
		376	base = base2;
		377	h = 1;
		378	break;
		379	}
		380	}
		381	}
		382
		383	if (!found) {
		384	/* Use secondary hash to avoid collisions
		385	with usual PHT refill handler. */
		386	i = RANDI(seed_real) % 8;
		387	base = base2;
		388	h = 1;
		389	}
		390	}
		391
		392	phte_real[base + i].v = 1;
		393	phte_real[base + i].vsid = vsid;
		394	phte_real[base + i].h = h;
		395	phte_real[base + i].api = api;
		396	phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
		397	phte_real[base + i].r = 0;
		398	phte_real[base + i].c = 0;
		399	phte_real[base + i].wimg = 0;
		400	phte_real[base + i].pp = 2; // FIXME
		401
		402	return true;
1730	decky	403	}
		404
		405
4153	mejdrech	406	/** Process ITLB/DTLB Miss Exception in Real Mode
		407	*
		408	*
		409	*/
		410	void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
		411	{
		412	uint32_t badvaddr = tlbmiss & 0xfffffffc;
		413
		414	uint32_t physmem;
		415	asm volatile (
		416	"mfsprg3 %0\n"
		417	: "=r" (physmem)
		418	);
		419
		420	if ((badvaddr < PA2KA(0)) \|\| (badvaddr >= PA2KA(physmem)))
		421	return; // FIXME
		422
		423	ptelo.rpn = KA2PA(badvaddr) >> 12;
		424	ptelo.wimg = 0;
		425	ptelo.pp = 2; // FIXME
		426
		427	uint32_t index = 0;
		428	asm volatile (
		429	"mtspr 981, %0\n"
		430	"mtspr 982, %1\n"
		431	"tlbld %2\n"
		432	"tlbli %2\n"
		433	: "=r" (index)
		434	: "r" (ptehi),
		435	"r" (ptelo)
		436	);
		437	}
		438
		439
1215	decky	440	void tlb_arch_init(void)
		441	{
1384	decky	442	tlb_invalidate_all();
		443	}
		444
		445
		446	void tlb_invalidate_all(void)
		447	{
4153	mejdrech	448	uint32_t index;
1269	decky	449	asm volatile (
4153	mejdrech	450	"li %0, 0\n"
		451	"sync\n"
		452
		453	TLB_FLUSH
		454	TLB_FLUSH
		455	TLB_FLUSH
		456	TLB_FLUSH
		457	TLB_FLUSH
		458	TLB_FLUSH
		459	TLB_FLUSH
		460	TLB_FLUSH
		461
		462	TLB_FLUSH
		463	TLB_FLUSH
		464	TLB_FLUSH
		465	TLB_FLUSH
		466	TLB_FLUSH
		467	TLB_FLUSH
		468	TLB_FLUSH
		469	TLB_FLUSH
		470
		471	TLB_FLUSH
		472	TLB_FLUSH
		473	TLB_FLUSH
		474	TLB_FLUSH
		475	TLB_FLUSH
		476	TLB_FLUSH
		477	TLB_FLUSH
		478	TLB_FLUSH
		479
		480	TLB_FLUSH
		481	TLB_FLUSH
		482	TLB_FLUSH
		483	TLB_FLUSH
		484	TLB_FLUSH
		485	TLB_FLUSH
		486	TLB_FLUSH
		487	TLB_FLUSH
		488
		489	TLB_FLUSH
		490	TLB_FLUSH
		491	TLB_FLUSH
		492	TLB_FLUSH
		493	TLB_FLUSH
		494	TLB_FLUSH
		495	TLB_FLUSH
		496	TLB_FLUSH
		497
		498	TLB_FLUSH
		499	TLB_FLUSH
		500	TLB_FLUSH
		501	TLB_FLUSH
		502	TLB_FLUSH
		503	TLB_FLUSH
		504	TLB_FLUSH
		505	TLB_FLUSH
		506
		507	TLB_FLUSH
		508	TLB_FLUSH
		509	TLB_FLUSH
		510	TLB_FLUSH
		511	TLB_FLUSH
		512	TLB_FLUSH
		513	TLB_FLUSH
		514	TLB_FLUSH
		515
		516	TLB_FLUSH
		517	TLB_FLUSH
		518	TLB_FLUSH
		519	TLB_FLUSH
		520	TLB_FLUSH
		521	TLB_FLUSH
		522	TLB_FLUSH
		523	TLB_FLUSH
		524
		525	"eieio\n"
1384	decky	526	"tlbsync\n"
4153	mejdrech	527	"sync\n"
		528	: "=r" (index)
1269	decky	529	);
1215	decky	530	}
		531
		532
1384	decky	533	void tlb_invalidate_asid(asid_t asid)
1328	decky	534	{
1780	jermar	535	uint32_t sdr1;
1758	decky	536	asm volatile (
		537	"mfsdr1 %0\n"
		538	: "=r" (sdr1)
		539	);
		540	phte_t phte = (phte_t ) PA2KA(sdr1 & 0xffff0000);
		541
1780	jermar	542	uint32_t i;
1758	decky	543	for (i = 0; i < 8192; i++) {
3193	jermar	544	if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
		545	(phte[i].vsid < ((asid << 4) + 16)))
1758	decky	546	phte[i].v = 0;
		547	}
1384	decky	548	tlb_invalidate_all();
1328	decky	549	}
		550
1730	decky	551
4581	mejdrech	552	void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
1384	decky	553	{
1730	decky	554	// TODO
1384	decky	555	tlb_invalidate_all();
		556	}
1328	decky	557
1384	decky	558
1736	decky	559	#define PRINT_BAT(name, ureg, lreg) \
		560	asm volatile ( \
		561	"mfspr %0," #ureg "\n" \
		562	"mfspr %1," #lreg "\n" \
		563	: "=r" (upper), "=r" (lower) \
		564	); \
		565	mask = (upper & 0x1ffc) >> 2; \
		566	if (upper & 3) { \
1780	jermar	567	uint32_t tmp = mask; \
1736	decky	568	length = 128; \
		569	while (tmp) { \
		570	if ((tmp & 1) == 0) { \
		571	printf("ibat[0]: error in mask\n"); \
		572	break; \
		573	} \
		574	length <<= 1; \
		575	tmp >>= 1; \
		576	} \
		577	} else \
		578	length = 0; \
3193	jermar	579	printf(name ": page=%.p frame=%.p length=%d KB (mask=%#x)%s%s\n", \
		580	sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
		581	lower & 0xffff0000, length, mask, \
		582	((upper >> 1) & 1) ? " supervisor" : "", \
		583	(upper & 1) ? " user" : "");
1736	decky	584
		585
1215	decky	586	void tlb_print(void)
		587	{
1780	jermar	588	uint32_t sr;
1733	decky	589
		590	for (sr = 0; sr < 16; sr++) {
1780	jermar	591	uint32_t vsid;
1733	decky	592	asm volatile (
		593	"mfsrin %0, %1\n"
		594	: "=r" (vsid)
		595	: "r" (sr << 28)
		596	);
4153	mejdrech	597	printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
3193	jermar	598	sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
		599	((vsid >> 30) & 1) ? " supervisor" : "",
		600	((vsid >> 29) & 1) ? " user" : "");
1733	decky	601	}
1736	decky	602
1780	jermar	603	uint32_t upper;
		604	uint32_t lower;
		605	uint32_t mask;
		606	uint32_t length;
1736	decky	607
		608	PRINT_BAT("ibat[0]", 528, 529);
		609	PRINT_BAT("ibat[1]", 530, 531);
		610	PRINT_BAT("ibat[2]", 532, 533);
		611	PRINT_BAT("ibat[3]", 534, 535);
		612
		613	PRINT_BAT("dbat[0]", 536, 537);
		614	PRINT_BAT("dbat[1]", 538, 539);
		615	PRINT_BAT("dbat[2]", 540, 541);
		616	PRINT_BAT("dbat[3]", 542, 543);
1215	decky	617	}
1702	cejka	618
1730	decky	619	/** @}
1702	cejka	620	*/

Subversion Repositories HelenOS

(root)/branches/network/kernel/arch/ppc32/src/mm/tlb.c – Rev 4704