WebSVN – HelenOS – Blame – /trunk/kernel/arch/ppc32/src/mm/tlb.c

Rev	Author	Line No.	Line
1215	decky	1	/*
2071	jermar	2	* Copyright (c) 2006 Martin Decky
1215	decky	3	* All rights reserved.
		4	*
		5	* Redistribution and use in source and binary forms, with or without
		6	* modification, are permitted provided that the following conditions
		7	* are met:
		8	*
		9	* - Redistributions of source code must retain the above copyright
		10	* notice, this list of conditions and the following disclaimer.
		11	* - Redistributions in binary form must reproduce the above copyright
		12	* notice, this list of conditions and the following disclaimer in the
		13	* documentation and/or other materials provided with the distribution.
		14	* - The name of the author may not be used to endorse or promote products
		15	* derived from this software without specific prior written permission.
		16	*
		17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
		18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
		19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
		20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
		21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
		22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
		23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
		24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
		25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
		26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
		27	*/
		28
3856	decky	29	/** @addtogroup ppc32mm
1702	cejka	30	* @{
		31	*/
		32	/** @file
		33	*/
		34
1215	decky	35	#include <mm/tlb.h>
1730	decky	36	#include <arch/mm/tlb.h>
		37	#include <arch/interrupt.h>
3594	svoboda	38	#include <interrupt.h>
1730	decky	39	#include <mm/as.h>
		40	#include <arch.h>
		41	#include <print.h>
3838	decky	42	#include <macros.h>
1215	decky	43
4132	svoboda	44	#ifdef CONFIG_SYMTAB
		45	#include <symtab.h>
		46	#endif
1215	decky	47
3837	decky	48	static unsigned int seed = 10;
		49	static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
		50
		51
3856	decky	52	#define TLB_FLUSH \
		53	"tlbie %0\n" \
		54	"addi %0, %0, 0x1000\n"
		55
		56
1730	decky	57	/** Try to find PTE for faulting address
1215	decky	58	*
1730	decky	59	* Try to find PTE for faulting address.
		60	* The as->lock must be held on entry to this function
		61	* if lock is true.
1215	decky	62	*
3193	jermar	63	* @param as Address space.
		64	* @param lock Lock/unlock the address space.
		65	* @param badvaddr Faulting virtual address.
		66	* @param access Access mode that caused the fault.
		67	* @param istate Pointer to interrupted state.
		68	* @param pfrc Pointer to variable where as_page_fault() return code
		69	* will be stored.
		70	* @return PTE on success, NULL otherwise.
1730	decky	71	*
1215	decky	72	*/
3193	jermar	73	static pte_t *
		74	find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
		75	istate_t istate, int pfrc)
1730	decky	76	{
		77	/*
		78	* Check if the mapping exists in page tables.
		79	*/
		80	pte_t *pte = page_mapping_find(as, badvaddr);
3830	decky	81	if ((pte) && (pte->present)) {
1730	decky	82	/*
		83	* Mapping found in page tables.
		84	* Immediately succeed.
		85	*/
		86	return pte;
		87	} else {
		88	int rc;
		89
		90	/*
		91	* Mapping not found in page tables.
		92	* Resort to higher-level page fault handler.
		93	*/
		94	page_table_unlock(as, lock);
		95	switch (rc = as_page_fault(badvaddr, access, istate)) {
3193	jermar	96	case AS_PF_OK:
		97	/*
		98	* The higher-level page fault handler succeeded,
		99	* The mapping ought to be in place.
		100	*/
		101	page_table_lock(as, lock);
		102	pte = page_mapping_find(as, badvaddr);
3830	decky	103	ASSERT((pte) && (pte->present));
3193	jermar	104	*pfrc = 0;
		105	return pte;
		106	case AS_PF_DEFER:
		107	page_table_lock(as, lock);
		108	*pfrc = rc;
		109	return NULL;
		110	case AS_PF_FAULT:
		111	page_table_lock(as, lock);
		112	*pfrc = rc;
		113	return NULL;
		114	default:
3790	svoboda	115	panic("Unexpected rc (%d).", rc);
1730	decky	116	}
		117	}
		118	}
		119
		120
1780	jermar	121	static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
1730	decky	122	{
		123	char *symbol = "";
		124	char *sym2 = "";
		125
4132	svoboda	126	#ifdef CONFIG_SYMTAB
3833	decky	127	char *str = get_symtab_entry(istate->pc);
		128	if (str)
		129	symbol = str;
		130	str = get_symtab_entry(istate->lr);
		131	if (str)
		132	sym2 = str;
4132	svoboda	133	#endif
3594	svoboda	134
		135	fault_if_from_uspace(istate,
3833	decky	136	"PHT Refill Exception on %p.", badvaddr);
3790	svoboda	137	panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
3193	jermar	138	istate->pc, symbol, sym2);
1730	decky	139	}
		140
		141
3830	decky	142	static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
1730	decky	143	{
1780	jermar	144	uint32_t page = (vaddr >> 12) & 0xffff;
		145	uint32_t api = (vaddr >> 22) & 0x3f;
1730	decky	146
1780	jermar	147	uint32_t vsid;
1730	decky	148	asm volatile (
		149	"mfsrin %0, %1\n"
		150	: "=r" (vsid)
		151	: "r" (vaddr)
		152	);
		153
1780	jermar	154	uint32_t sdr1;
1730	decky	155	asm volatile (
		156	"mfsdr1 %0\n"
		157	: "=r" (sdr1)
		158	);
		159	phte_t phte = (phte_t ) PA2KA(sdr1 & 0xffff0000);
		160
		161	/* Primary hash (xor) */
1780	jermar	162	uint32_t h = 0;
		163	uint32_t hash = vsid ^ page;
		164	uint32_t base = (hash & 0x3ff) << 3;
		165	uint32_t i;
1730	decky	166	bool found = false;
		167
3839	decky	168	/* Find colliding PTE in PTEG */
1730	decky	169	for (i = 0; i < 8; i++) {
3839	decky	170	if ((phte[base + i].v)
		171	&& (phte[base + i].vsid == vsid)
3836	decky	172	&& (phte[base + i].api == api)
3839	decky	173	&& (phte[base + i].h == 0)) {
1730	decky	174	found = true;
		175	break;
		176	}
		177	}
		178
		179	if (!found) {
3839	decky	180	/* Find unused PTE in PTEG */
		181	for (i = 0; i < 8; i++) {
		182	if (!phte[base + i].v) {
		183	found = true;
		184	break;
		185	}
		186	}
		187	}
		188
		189	if (!found) {
1730	decky	190	/* Secondary hash (not) */
1780	jermar	191	uint32_t base2 = (~hash & 0x3ff) << 3;
1730	decky	192
3839	decky	193	/* Find colliding PTE in PTEG */
1730	decky	194	for (i = 0; i < 8; i++) {
3839	decky	195	if ((phte[base2 + i].v)
		196	&& (phte[base2 + i].vsid == vsid)
3836	decky	197	&& (phte[base2 + i].api == api)
3839	decky	198	&& (phte[base2 + i].h == 1)) {
1730	decky	199	found = true;
		200	base = base2;
		201	h = 1;
		202	break;
		203	}
		204	}
		205
3839	decky	206	if (!found) {
		207	/* Find unused PTE in PTEG */
		208	for (i = 0; i < 8; i++) {
		209	if (!phte[base2 + i].v) {
		210	found = true;
		211	base = base2;
		212	h = 1;
		213	break;
		214	}
		215	}
		216	}
		217
3836	decky	218	if (!found)
3837	decky	219	i = RANDI(seed) % 8;
1730	decky	220	}
		221
		222	phte[base + i].v = 1;
		223	phte[base + i].vsid = vsid;
		224	phte[base + i].h = h;
		225	phte[base + i].api = api;
3830	decky	226	phte[base + i].rpn = pte->pfn;
1730	decky	227	phte[base + i].r = 0;
		228	phte[base + i].c = 0;
3830	decky	229	phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
1730	decky	230	phte[base + i].pp = 2; // FIXME
		231	}
		232
		233
3857	decky	234	/** Process Instruction/Data Storage Exception
1730	decky	235	*
3857	decky	236	* @param n Exception vector number.
		237	* @param istate Interrupted register context.
1730	decky	238	*
		239	*/
		240	void pht_refill(int n, istate_t *istate)
		241	{
1780	jermar	242	uintptr_t badvaddr;
1730	decky	243	pte_t *pte;
		244	int pfrc;
		245	as_t *as;
		246	bool lock;
		247
		248	if (AS == NULL) {
		249	as = AS_KERNEL;
		250	lock = false;
		251	} else {
		252	as = AS;
		253	lock = true;
		254	}
		255
3833	decky	256	if (n == VECTOR_DATA_STORAGE)
		257	badvaddr = istate->dar;
		258	else
1730	decky	259	badvaddr = istate->pc;
		260
		261	page_table_lock(as, lock);
		262
3193	jermar	263	pte = find_mapping_and_check(as, lock, badvaddr,
		264	PF_ACCESS_READ /* FIXME */, istate, &pfrc);
1730	decky	265	if (!pte) {
		266	switch (pfrc) {
3193	jermar	267	case AS_PF_FAULT:
		268	goto fail;
		269	break;
		270	case AS_PF_DEFER:
		271	/*
		272	* The page fault came during copy_from_uspace()
		273	* or copy_to_uspace().
		274	*/
		275	page_table_unlock(as, lock);
		276	return;
		277	default:
3790	svoboda	278	panic("Unexpected pfrc (%d).", pfrc);
1730	decky	279	}
		280	}
		281
3830	decky	282	pte->accessed = 1; /* Record access to PTE */
		283	pht_insert(badvaddr, pte);
1730	decky	284
		285	page_table_unlock(as, lock);
		286	return;
		287
		288	fail:
		289	page_table_unlock(as, lock);
		290	pht_refill_fail(badvaddr, istate);
		291	}
		292
		293
3857	decky	294	/** Process Instruction/Data Storage Exception in Real Mode
1730	decky	295	*
3857	decky	296	* @param n Exception vector number.
		297	* @param istate Interrupted register context.
1730	decky	298	*
		299	*/
3837	decky	300	bool pht_refill_real(int n, istate_t *istate)
1730	decky	301	{
1780	jermar	302	uintptr_t badvaddr;
1730	decky	303
3833	decky	304	if (n == VECTOR_DATA_STORAGE)
		305	badvaddr = istate->dar;
		306	else
1730	decky	307	badvaddr = istate->pc;
		308
1780	jermar	309	uint32_t physmem;
1730	decky	310	asm volatile (
		311	"mfsprg3 %0\n"
		312	: "=r" (physmem)
		313	);
		314
3837	decky	315	if ((badvaddr < PA2KA(0)) \|\| (badvaddr >= PA2KA(physmem)))
		316	return false;
		317
		318	uint32_t page = (badvaddr >> 12) & 0xffff;
		319	uint32_t api = (badvaddr >> 22) & 0x3f;
		320
		321	uint32_t vsid;
		322	asm volatile (
		323	"mfsrin %0, %1\n"
		324	: "=r" (vsid)
		325	: "r" (badvaddr)
		326	);
		327
		328	uint32_t sdr1;
		329	asm volatile (
		330	"mfsdr1 %0\n"
		331	: "=r" (sdr1)
		332	);
		333	phte_t phte_real = (phte_t ) (sdr1 & 0xffff0000);
		334
		335	/* Primary hash (xor) */
		336	uint32_t h = 0;
		337	uint32_t hash = vsid ^ page;
		338	uint32_t base = (hash & 0x3ff) << 3;
		339	uint32_t i;
		340	bool found = false;
		341
3839	decky	342	/* Find colliding PTE in PTEG */
3837	decky	343	for (i = 0; i < 8; i++) {
3839	decky	344	if ((phte_real[base + i].v)
		345	&& (phte_real[base + i].vsid == vsid)
3837	decky	346	&& (phte_real[base + i].api == api)
3839	decky	347	&& (phte_real[base + i].h == 0)) {
3837	decky	348	found = true;
		349	break;
		350	}
1730	decky	351	}
		352
3837	decky	353	if (!found) {
3839	decky	354	/* Find unused PTE in PTEG */
		355	for (i = 0; i < 8; i++) {
		356	if (!phte_real[base + i].v) {
		357	found = true;
		358	break;
		359	}
		360	}
		361	}
		362
		363	if (!found) {
3837	decky	364	/* Secondary hash (not) */
		365	uint32_t base2 = (~hash & 0x3ff) << 3;
		366
3839	decky	367	/* Find colliding PTE in PTEG */
3837	decky	368	for (i = 0; i < 8; i++) {
3839	decky	369	if ((phte_real[base2 + i].v)
		370	&& (phte_real[base2 + i].vsid == vsid)
3837	decky	371	&& (phte_real[base2 + i].api == api)
3839	decky	372	&& (phte_real[base2 + i].h == 1)) {
3837	decky	373	found = true;
		374	base = base2;
		375	h = 1;
		376	break;
		377	}
		378	}
		379
		380	if (!found) {
3839	decky	381	/* Find unused PTE in PTEG */
		382	for (i = 0; i < 8; i++) {
		383	if (!phte_real[base2 + i].v) {
		384	found = true;
		385	base = base2;
		386	h = 1;
		387	break;
		388	}
		389	}
		390	}
		391
		392	if (!found) {
3837	decky	393	/* Use secondary hash to avoid collisions
		394	with usual PHT refill handler. */
		395	i = RANDI(seed_real) % 8;
		396	base = base2;
		397	h = 1;
		398	}
		399	}
		400
		401	phte_real[base + i].v = 1;
		402	phte_real[base + i].vsid = vsid;
		403	phte_real[base + i].h = h;
		404	phte_real[base + i].api = api;
		405	phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
		406	phte_real[base + i].r = 0;
		407	phte_real[base + i].c = 0;
		408	phte_real[base + i].wimg = 0;
		409	phte_real[base + i].pp = 2; // FIXME
		410
		411	return true;
1730	decky	412	}
		413
		414
3857	decky	415	/** Process ITLB/DTLB Miss Exception in Real Mode
		416	*
		417	*
		418	*/
		419	void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
		420	{
		421	uint32_t badvaddr = tlbmiss & 0xfffffffc;
		422
		423	uint32_t physmem;
		424	asm volatile (
		425	"mfsprg3 %0\n"
		426	: "=r" (physmem)
		427	);
		428
		429	if ((badvaddr < PA2KA(0)) \|\| (badvaddr >= PA2KA(physmem)))
		430	return; // FIXME
		431
		432	ptelo.rpn = KA2PA(badvaddr) >> 12;
		433	ptelo.wimg = 0;
		434	ptelo.pp = 2; // FIXME
		435
		436	uint32_t index = 0;
		437	asm volatile (
		438	"mtspr 981, %0\n"
		439	"mtspr 982, %1\n"
		440	"tlbld %2\n"
		441	"tlbli %2\n"
		442	: "=r" (index)
		443	: "r" (ptehi),
		444	"r" (ptelo)
		445	);
		446	}
		447
		448
1215	decky	449	void tlb_arch_init(void)
		450	{
1384	decky	451	tlb_invalidate_all();
		452	}
		453
		454
		455	void tlb_invalidate_all(void)
		456	{
3856	decky	457	uint32_t index;
1269	decky	458	asm volatile (
3856	decky	459	"li %0, 0\n"
		460	"sync\n"
		461
		462	TLB_FLUSH
		463	TLB_FLUSH
		464	TLB_FLUSH
		465	TLB_FLUSH
		466	TLB_FLUSH
		467	TLB_FLUSH
		468	TLB_FLUSH
		469	TLB_FLUSH
		470
		471	TLB_FLUSH
		472	TLB_FLUSH
		473	TLB_FLUSH
		474	TLB_FLUSH
		475	TLB_FLUSH
		476	TLB_FLUSH
		477	TLB_FLUSH
		478	TLB_FLUSH
		479
		480	TLB_FLUSH
		481	TLB_FLUSH
		482	TLB_FLUSH
		483	TLB_FLUSH
		484	TLB_FLUSH
		485	TLB_FLUSH
		486	TLB_FLUSH
		487	TLB_FLUSH
		488
		489	TLB_FLUSH
		490	TLB_FLUSH
		491	TLB_FLUSH
		492	TLB_FLUSH
		493	TLB_FLUSH
		494	TLB_FLUSH
		495	TLB_FLUSH
		496	TLB_FLUSH
		497
		498	TLB_FLUSH
		499	TLB_FLUSH
		500	TLB_FLUSH
		501	TLB_FLUSH
		502	TLB_FLUSH
		503	TLB_FLUSH
		504	TLB_FLUSH
		505	TLB_FLUSH
		506
		507	TLB_FLUSH
		508	TLB_FLUSH
		509	TLB_FLUSH
		510	TLB_FLUSH
		511	TLB_FLUSH
		512	TLB_FLUSH
		513	TLB_FLUSH
		514	TLB_FLUSH
		515
		516	TLB_FLUSH
		517	TLB_FLUSH
		518	TLB_FLUSH
		519	TLB_FLUSH
		520	TLB_FLUSH
		521	TLB_FLUSH
		522	TLB_FLUSH
		523	TLB_FLUSH
		524
		525	TLB_FLUSH
		526	TLB_FLUSH
		527	TLB_FLUSH
		528	TLB_FLUSH
		529	TLB_FLUSH
		530	TLB_FLUSH
		531	TLB_FLUSH
		532	TLB_FLUSH
		533
		534	"eieio\n"
1384	decky	535	"tlbsync\n"
3856	decky	536	"sync\n"
		537	: "=r" (index)
1269	decky	538	);
1215	decky	539	}
		540
		541
1384	decky	542	void tlb_invalidate_asid(asid_t asid)
1328	decky	543	{
1780	jermar	544	uint32_t sdr1;
1758	decky	545	asm volatile (
		546	"mfsdr1 %0\n"
		547	: "=r" (sdr1)
		548	);
		549	phte_t phte = (phte_t ) PA2KA(sdr1 & 0xffff0000);
		550
1780	jermar	551	uint32_t i;
1758	decky	552	for (i = 0; i < 8192; i++) {
3193	jermar	553	if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
		554	(phte[i].vsid < ((asid << 4) + 16)))
1758	decky	555	phte[i].v = 0;
		556	}
1384	decky	557	tlb_invalidate_all();
1328	decky	558	}
		559
1730	decky	560
1780	jermar	561	void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
1384	decky	562	{
1730	decky	563	// TODO
1384	decky	564	tlb_invalidate_all();
		565	}
1328	decky	566
1384	decky	567
1736	decky	568	#define PRINT_BAT(name, ureg, lreg) \
		569	asm volatile ( \
		570	"mfspr %0," #ureg "\n" \
		571	"mfspr %1," #lreg "\n" \
		572	: "=r" (upper), "=r" (lower) \
		573	); \
		574	mask = (upper & 0x1ffc) >> 2; \
		575	if (upper & 3) { \
1780	jermar	576	uint32_t tmp = mask; \
1736	decky	577	length = 128; \
		578	while (tmp) { \
		579	if ((tmp & 1) == 0) { \
		580	printf("ibat[0]: error in mask\n"); \
		581	break; \
		582	} \
		583	length <<= 1; \
		584	tmp >>= 1; \
		585	} \
		586	} else \
		587	length = 0; \
3193	jermar	588	printf(name ": page=%.p frame=%.p length=%d KB (mask=%#x)%s%s\n", \
		589	sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
		590	lower & 0xffff0000, length, mask, \
		591	((upper >> 1) & 1) ? " supervisor" : "", \
		592	(upper & 1) ? " user" : "");
1736	decky	593
		594
1215	decky	595	void tlb_print(void)
		596	{
1780	jermar	597	uint32_t sr;
1733	decky	598
		599	for (sr = 0; sr < 16; sr++) {
1780	jermar	600	uint32_t vsid;
1733	decky	601	asm volatile (
		602	"mfsrin %0, %1\n"
		603	: "=r" (vsid)
		604	: "r" (sr << 28)
		605	);
3837	decky	606	printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
3193	jermar	607	sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
		608	((vsid >> 30) & 1) ? " supervisor" : "",
		609	((vsid >> 29) & 1) ? " user" : "");
1733	decky	610	}
1736	decky	611
1780	jermar	612	uint32_t upper;
		613	uint32_t lower;
		614	uint32_t mask;
		615	uint32_t length;
1736	decky	616
		617	PRINT_BAT("ibat[0]", 528, 529);
		618	PRINT_BAT("ibat[1]", 530, 531);
		619	PRINT_BAT("ibat[2]", 532, 533);
		620	PRINT_BAT("ibat[3]", 534, 535);
		621
		622	PRINT_BAT("dbat[0]", 536, 537);
		623	PRINT_BAT("dbat[1]", 538, 539);
		624	PRINT_BAT("dbat[2]", 540, 541);
		625	PRINT_BAT("dbat[3]", 542, 543);
1215	decky	626	}
1702	cejka	627
1730	decky	628	/** @}
1702	cejka	629	*/

Subversion Repositories HelenOS

(root)/trunk/kernel/arch/ppc32/src/mm/tlb.c – Rev 4136