Rev 2787 | Rev 4377 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
740 | jermar | 1 | /* |
2071 | jermar | 2 | * Copyright (c) 2006 Jakub Jermar |
740 | jermar | 3 | * All rights reserved. |
4 | * |
||
5 | * Redistribution and use in source and binary forms, with or without |
||
6 | * modification, are permitted provided that the following conditions |
||
7 | * are met: |
||
8 | * |
||
9 | * - Redistributions of source code must retain the above copyright |
||
10 | * notice, this list of conditions and the following disclaimer. |
||
11 | * - Redistributions in binary form must reproduce the above copyright |
||
12 | * notice, this list of conditions and the following disclaimer in the |
||
13 | * documentation and/or other materials provided with the distribution. |
||
14 | * - The name of the author may not be used to endorse or promote products |
||
15 | * derived from this software without specific prior written permission. |
||
16 | * |
||
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
||
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
||
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
||
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
||
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||
27 | */ |
||
28 | |||
1850 | jermar | 29 | /** @addtogroup ia64mm |
1702 | cejka | 30 | * @{ |
31 | */ |
||
32 | /** @file |
||
33 | */ |
||
34 | |||
740 | jermar | 35 | /* |
36 | * TLB management. |
||
37 | */ |
||
38 | |||
39 | #include <mm/tlb.h> |
||
901 | jermar | 40 | #include <mm/asid.h> |
902 | jermar | 41 | #include <mm/page.h> |
42 | #include <mm/as.h> |
||
818 | vana | 43 | #include <arch/mm/tlb.h> |
901 | jermar | 44 | #include <arch/mm/page.h> |
1210 | vana | 45 | #include <arch/mm/vhpt.h> |
819 | vana | 46 | #include <arch/barrier.h> |
900 | jermar | 47 | #include <arch/interrupt.h> |
928 | vana | 48 | #include <arch/pal/pal.h> |
49 | #include <arch/asm.h> |
||
900 | jermar | 50 | #include <panic.h> |
993 | jermar | 51 | #include <print.h> |
902 | jermar | 52 | #include <arch.h> |
1621 | vana | 53 | #include <interrupt.h> |
740 | jermar | 54 | |
756 | jermar | 55 | /** Invalidate all TLB entries. */ |
740 | jermar | 56 | void tlb_invalidate_all(void) |
57 | { |
||
1850 | jermar | 58 | ipl_t ipl; |
59 | uintptr_t adr; |
||
60 | uint32_t count1, count2, stride1, stride2; |
||
928 | vana | 61 | |
2745 | decky | 62 | unsigned int i, j; |
928 | vana | 63 | |
1850 | jermar | 64 | adr = PAL_PTCE_INFO_BASE(); |
65 | count1 = PAL_PTCE_INFO_COUNT1(); |
||
66 | count2 = PAL_PTCE_INFO_COUNT2(); |
||
67 | stride1 = PAL_PTCE_INFO_STRIDE1(); |
||
68 | stride2 = PAL_PTCE_INFO_STRIDE2(); |
||
928 | vana | 69 | |
1850 | jermar | 70 | ipl = interrupts_disable(); |
928 | vana | 71 | |
2745 | decky | 72 | for (i = 0; i < count1; i++) { |
73 | for (j = 0; j < count2; j++) { |
||
2082 | decky | 74 | asm volatile ( |
1850 | jermar | 75 | "ptc.e %0 ;;" |
76 | : |
||
77 | : "r" (adr) |
||
78 | ); |
||
79 | adr += stride2; |
||
928 | vana | 80 | } |
1850 | jermar | 81 | adr += stride1; |
82 | } |
||
928 | vana | 83 | |
1850 | jermar | 84 | interrupts_restore(ipl); |
928 | vana | 85 | |
1850 | jermar | 86 | srlz_d(); |
87 | srlz_i(); |
||
1210 | vana | 88 | #ifdef CONFIG_VHPT |
1850 | jermar | 89 | vhpt_invalidate_all(); |
1210 | vana | 90 | #endif |
740 | jermar | 91 | } |
92 | |||
93 | /** Invalidate entries belonging to an address space. |
||
94 | * |
||
95 | * @param asid Address space identifier. |
||
96 | */ |
||
97 | void tlb_invalidate_asid(asid_t asid) |
||
98 | { |
||
935 | vana | 99 | tlb_invalidate_all(); |
740 | jermar | 100 | } |
818 | vana | 101 | |
935 | vana | 102 | |
1780 | jermar | 103 | void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt) |
935 | vana | 104 | { |
944 | vana | 105 | region_register rr; |
106 | bool restore_rr = false; |
||
993 | jermar | 107 | int b = 0; |
108 | int c = cnt; |
||
944 | vana | 109 | |
1780 | jermar | 110 | uintptr_t va; |
993 | jermar | 111 | va = page; |
947 | vana | 112 | |
944 | vana | 113 | rr.word = rr_read(VA2VRN(va)); |
114 | if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) { |
||
115 | /* |
||
116 | * The selected region register does not contain required RID. |
||
117 | * Save the old content of the register and replace the RID. |
||
118 | */ |
||
119 | region_register rr0; |
||
120 | |||
121 | rr0 = rr; |
||
122 | rr0.map.rid = ASID2RID(asid, VA2VRN(va)); |
||
123 | rr_write(VA2VRN(va), rr0.word); |
||
124 | srlz_d(); |
||
125 | srlz_i(); |
||
126 | } |
||
127 | |||
993 | jermar | 128 | while(c >>= 1) |
129 | b++; |
||
130 | b >>= 1; |
||
1780 | jermar | 131 | uint64_t ps; |
944 | vana | 132 | |
993 | jermar | 133 | switch (b) { |
1850 | jermar | 134 | case 0: /*cnt 1-3*/ |
135 | ps = PAGE_WIDTH; |
||
136 | break; |
||
137 | case 1: /*cnt 4-15*/ |
||
138 | /*cnt=((cnt-1)/4)+1;*/ |
||
139 | ps = PAGE_WIDTH+2; |
||
140 | va &= ~((1<<ps)-1); |
||
141 | break; |
||
142 | case 2: /*cnt 16-63*/ |
||
143 | /*cnt=((cnt-1)/16)+1;*/ |
||
144 | ps = PAGE_WIDTH+4; |
||
145 | va &= ~((1<<ps)-1); |
||
146 | break; |
||
147 | case 3: /*cnt 64-255*/ |
||
148 | /*cnt=((cnt-1)/64)+1;*/ |
||
149 | ps = PAGE_WIDTH+6; |
||
150 | va &= ~((1<<ps)-1); |
||
151 | break; |
||
152 | case 4: /*cnt 256-1023*/ |
||
153 | /*cnt=((cnt-1)/256)+1;*/ |
||
154 | ps = PAGE_WIDTH+8; |
||
155 | va &= ~((1<<ps)-1); |
||
156 | break; |
||
157 | case 5: /*cnt 1024-4095*/ |
||
158 | /*cnt=((cnt-1)/1024)+1;*/ |
||
159 | ps = PAGE_WIDTH+10; |
||
160 | va &= ~((1<<ps)-1); |
||
161 | break; |
||
162 | case 6: /*cnt 4096-16383*/ |
||
163 | /*cnt=((cnt-1)/4096)+1;*/ |
||
164 | ps = PAGE_WIDTH+12; |
||
165 | va &= ~((1<<ps)-1); |
||
166 | break; |
||
167 | case 7: /*cnt 16384-65535*/ |
||
168 | case 8: /*cnt 65536-(256K-1)*/ |
||
169 | /*cnt=((cnt-1)/16384)+1;*/ |
||
170 | ps = PAGE_WIDTH+14; |
||
171 | va &= ~((1<<ps)-1); |
||
172 | break; |
||
173 | default: |
||
174 | /*cnt=((cnt-1)/(16384*16))+1;*/ |
||
175 | ps=PAGE_WIDTH+18; |
||
176 | va&=~((1<<ps)-1); |
||
177 | break; |
||
944 | vana | 178 | } |
947 | vana | 179 | /*cnt+=(page!=va);*/ |
993 | jermar | 180 | for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) { |
2082 | decky | 181 | asm volatile ( |
947 | vana | 182 | "ptc.l %0,%1;;" |
183 | : |
||
993 | jermar | 184 | : "r" (va), "r" (ps<<2) |
947 | vana | 185 | ); |
944 | vana | 186 | } |
187 | srlz_d(); |
||
188 | srlz_i(); |
||
189 | |||
190 | if (restore_rr) { |
||
191 | rr_write(VA2VRN(va), rr.word); |
||
192 | srlz_d(); |
||
193 | srlz_i(); |
||
194 | } |
||
935 | vana | 195 | } |
196 | |||
899 | jermar | 197 | /** Insert data into data translation cache. |
198 | * |
||
199 | * @param va Virtual page address. |
||
200 | * @param asid Address space identifier. |
||
201 | * @param entry The rest of TLB entry as required by TLB insertion format. |
||
202 | */ |
||
1780 | jermar | 203 | void dtc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry) |
919 | jermar | 204 | { |
899 | jermar | 205 | tc_mapping_insert(va, asid, entry, true); |
206 | } |
||
818 | vana | 207 | |
899 | jermar | 208 | /** Insert data into instruction translation cache. |
209 | * |
||
210 | * @param va Virtual page address. |
||
211 | * @param asid Address space identifier. |
||
212 | * @param entry The rest of TLB entry as required by TLB insertion format. |
||
213 | */ |
||
1780 | jermar | 214 | void itc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry) |
919 | jermar | 215 | { |
899 | jermar | 216 | tc_mapping_insert(va, asid, entry, false); |
217 | } |
||
818 | vana | 218 | |
899 | jermar | 219 | /** Insert data into instruction or data translation cache. |
220 | * |
||
221 | * @param va Virtual page address. |
||
222 | * @param asid Address space identifier. |
||
223 | * @param entry The rest of TLB entry as required by TLB insertion format. |
||
224 | * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise. |
||
225 | */ |
||
1780 | jermar | 226 | void tc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtc) |
818 | vana | 227 | { |
228 | region_register rr; |
||
899 | jermar | 229 | bool restore_rr = false; |
818 | vana | 230 | |
901 | jermar | 231 | rr.word = rr_read(VA2VRN(va)); |
232 | if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) { |
||
899 | jermar | 233 | /* |
234 | * The selected region register does not contain required RID. |
||
235 | * Save the old content of the register and replace the RID. |
||
236 | */ |
||
237 | region_register rr0; |
||
818 | vana | 238 | |
899 | jermar | 239 | rr0 = rr; |
901 | jermar | 240 | rr0.map.rid = ASID2RID(asid, VA2VRN(va)); |
241 | rr_write(VA2VRN(va), rr0.word); |
||
899 | jermar | 242 | srlz_d(); |
243 | srlz_i(); |
||
818 | vana | 244 | } |
899 | jermar | 245 | |
2082 | decky | 246 | asm volatile ( |
899 | jermar | 247 | "mov r8=psr;;\n" |
900 | jermar | 248 | "rsm %0;;\n" /* PSR_IC_MASK */ |
899 | jermar | 249 | "srlz.d;;\n" |
250 | "srlz.i;;\n" |
||
251 | "mov cr.ifa=%1\n" /* va */ |
||
252 | "mov cr.itir=%2;;\n" /* entry.word[1] */ |
||
253 | "cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */ |
||
254 | "(p6) itc.i %3;;\n" |
||
255 | "(p7) itc.d %3;;\n" |
||
256 | "mov psr.l=r8;;\n" |
||
257 | "srlz.d;;\n" |
||
258 | : |
||
900 | jermar | 259 | : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc) |
260 | : "p6", "p7", "r8" |
||
899 | jermar | 261 | ); |
262 | |||
263 | if (restore_rr) { |
||
901 | jermar | 264 | rr_write(VA2VRN(va), rr.word); |
819 | vana | 265 | srlz_d(); |
899 | jermar | 266 | srlz_i(); |
818 | vana | 267 | } |
899 | jermar | 268 | } |
818 | vana | 269 | |
899 | jermar | 270 | /** Insert data into instruction translation register. |
271 | * |
||
272 | * @param va Virtual page address. |
||
273 | * @param asid Address space identifier. |
||
274 | * @param entry The rest of TLB entry as required by TLB insertion format. |
||
275 | * @param tr Translation register. |
||
276 | */ |
||
1780 | jermar | 277 | void itr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr) |
899 | jermar | 278 | { |
279 | tr_mapping_insert(va, asid, entry, false, tr); |
||
280 | } |
||
818 | vana | 281 | |
899 | jermar | 282 | /** Insert data into data translation register. |
283 | * |
||
284 | * @param va Virtual page address. |
||
285 | * @param asid Address space identifier. |
||
286 | * @param entry The rest of TLB entry as required by TLB insertion format. |
||
287 | * @param tr Translation register. |
||
288 | */ |
||
1780 | jermar | 289 | void dtr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, index_t tr) |
899 | jermar | 290 | { |
291 | tr_mapping_insert(va, asid, entry, true, tr); |
||
818 | vana | 292 | } |
293 | |||
899 | jermar | 294 | /** Insert data into instruction or data translation register. |
295 | * |
||
296 | * @param va Virtual page address. |
||
297 | * @param asid Address space identifier. |
||
298 | * @param entry The rest of TLB entry as required by TLB insertion format. |
||
1708 | jermar | 299 | * @param dtr If true, insert into data translation register, use instruction translation register otherwise. |
899 | jermar | 300 | * @param tr Translation register. |
301 | */ |
||
1780 | jermar | 302 | void tr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr) |
818 | vana | 303 | { |
304 | region_register rr; |
||
899 | jermar | 305 | bool restore_rr = false; |
818 | vana | 306 | |
901 | jermar | 307 | rr.word = rr_read(VA2VRN(va)); |
308 | if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) { |
||
899 | jermar | 309 | /* |
310 | * The selected region register does not contain required RID. |
||
311 | * Save the old content of the register and replace the RID. |
||
312 | */ |
||
313 | region_register rr0; |
||
818 | vana | 314 | |
899 | jermar | 315 | rr0 = rr; |
901 | jermar | 316 | rr0.map.rid = ASID2RID(asid, VA2VRN(va)); |
317 | rr_write(VA2VRN(va), rr0.word); |
||
899 | jermar | 318 | srlz_d(); |
319 | srlz_i(); |
||
320 | } |
||
818 | vana | 321 | |
2082 | decky | 322 | asm volatile ( |
899 | jermar | 323 | "mov r8=psr;;\n" |
900 | jermar | 324 | "rsm %0;;\n" /* PSR_IC_MASK */ |
899 | jermar | 325 | "srlz.d;;\n" |
326 | "srlz.i;;\n" |
||
327 | "mov cr.ifa=%1\n" /* va */ |
||
328 | "mov cr.itir=%2;;\n" /* entry.word[1] */ |
||
329 | "cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */ |
||
330 | "(p6) itr.i itr[%4]=%3;;\n" |
||
331 | "(p7) itr.d dtr[%4]=%3;;\n" |
||
332 | "mov psr.l=r8;;\n" |
||
333 | "srlz.d;;\n" |
||
334 | : |
||
900 | jermar | 335 | : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr) |
336 | : "p6", "p7", "r8" |
||
899 | jermar | 337 | ); |
338 | |||
339 | if (restore_rr) { |
||
901 | jermar | 340 | rr_write(VA2VRN(va), rr.word); |
819 | vana | 341 | srlz_d(); |
899 | jermar | 342 | srlz_i(); |
818 | vana | 343 | } |
899 | jermar | 344 | } |
818 | vana | 345 | |
901 | jermar | 346 | /** Insert data into DTLB. |
347 | * |
||
1675 | jermar | 348 | * @param page Virtual page address including VRN bits. |
349 | * @param frame Physical frame address. |
||
901 | jermar | 350 | * @param dtr If true, insert into data translation register, use data translation cache otherwise. |
351 | * @param tr Translation register if dtr is true, ignored otherwise. |
||
352 | */ |
||
1780 | jermar | 353 | void dtlb_kernel_mapping_insert(uintptr_t page, uintptr_t frame, bool dtr, index_t tr) |
901 | jermar | 354 | { |
355 | tlb_entry_t entry; |
||
356 | |||
357 | entry.word[0] = 0; |
||
358 | entry.word[1] = 0; |
||
359 | |||
360 | entry.p = true; /* present */ |
||
361 | entry.ma = MA_WRITEBACK; |
||
362 | entry.a = true; /* already accessed */ |
||
363 | entry.d = true; /* already dirty */ |
||
364 | entry.pl = PL_KERNEL; |
||
365 | entry.ar = AR_READ | AR_WRITE; |
||
366 | entry.ppn = frame >> PPN_SHIFT; |
||
367 | entry.ps = PAGE_WIDTH; |
||
368 | |||
369 | if (dtr) |
||
370 | dtr_mapping_insert(page, ASID_KERNEL, entry, tr); |
||
371 | else |
||
372 | dtc_mapping_insert(page, ASID_KERNEL, entry); |
||
373 | } |
||
374 | |||
1675 | jermar | 375 | /** Purge kernel entries from DTR. |
376 | * |
||
377 | * Purge DTR entries used by the kernel. |
||
378 | * |
||
379 | * @param page Virtual page address including VRN bits. |
||
380 | * @param width Width of the purge in bits. |
||
381 | */ |
||
1780 | jermar | 382 | void dtr_purge(uintptr_t page, count_t width) |
1675 | jermar | 383 | { |
2082 | decky | 384 | asm volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2)); |
1675 | jermar | 385 | } |
386 | |||
387 | |||
902 | jermar | 388 | /** Copy content of PTE into data translation cache. |
389 | * |
||
390 | * @param t PTE. |
||
391 | */ |
||
392 | void dtc_pte_copy(pte_t *t) |
||
393 | { |
||
394 | tlb_entry_t entry; |
||
395 | |||
396 | entry.word[0] = 0; |
||
397 | entry.word[1] = 0; |
||
398 | |||
399 | entry.p = t->p; |
||
400 | entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE; |
||
401 | entry.a = t->a; |
||
402 | entry.d = t->d; |
||
403 | entry.pl = t->k ? PL_KERNEL : PL_USER; |
||
404 | entry.ar = t->w ? AR_WRITE : AR_READ; |
||
405 | entry.ppn = t->frame >> PPN_SHIFT; |
||
406 | entry.ps = PAGE_WIDTH; |
||
407 | |||
408 | dtc_mapping_insert(t->page, t->as->asid, entry); |
||
1210 | vana | 409 | #ifdef CONFIG_VHPT |
410 | vhpt_mapping_insert(t->page, t->as->asid, entry); |
||
411 | #endif |
||
902 | jermar | 412 | } |
413 | |||
414 | /** Copy content of PTE into instruction translation cache. |
||
415 | * |
||
416 | * @param t PTE. |
||
417 | */ |
||
418 | void itc_pte_copy(pte_t *t) |
||
419 | { |
||
420 | tlb_entry_t entry; |
||
421 | |||
422 | entry.word[0] = 0; |
||
423 | entry.word[1] = 0; |
||
424 | |||
425 | ASSERT(t->x); |
||
426 | |||
427 | entry.p = t->p; |
||
428 | entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE; |
||
429 | entry.a = t->a; |
||
430 | entry.pl = t->k ? PL_KERNEL : PL_USER; |
||
431 | entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ; |
||
432 | entry.ppn = t->frame >> PPN_SHIFT; |
||
433 | entry.ps = PAGE_WIDTH; |
||
434 | |||
435 | itc_mapping_insert(t->page, t->as->asid, entry); |
||
1210 | vana | 436 | #ifdef CONFIG_VHPT |
437 | vhpt_mapping_insert(t->page, t->as->asid, entry); |
||
438 | #endif |
||
902 | jermar | 439 | } |
440 | |||
441 | /** Instruction TLB fault handler for faults with VHPT turned off. |
||
442 | * |
||
443 | * @param vector Interruption vector. |
||
958 | jermar | 444 | * @param istate Structure with saved interruption state. |
902 | jermar | 445 | */ |
1780 | jermar | 446 | void alternate_instruction_tlb_fault(uint64_t vector, istate_t *istate) |
899 | jermar | 447 | { |
902 | jermar | 448 | region_register rr; |
1411 | jermar | 449 | rid_t rid; |
1780 | jermar | 450 | uintptr_t va; |
902 | jermar | 451 | pte_t *t; |
452 | |||
958 | jermar | 453 | va = istate->cr_ifa; /* faulting address */ |
1411 | jermar | 454 | rr.word = rr_read(VA2VRN(va)); |
455 | rid = rr.map.rid; |
||
456 | |||
1044 | jermar | 457 | page_table_lock(AS, true); |
902 | jermar | 458 | t = page_mapping_find(AS, va); |
459 | if (t) { |
||
460 | /* |
||
461 | * The mapping was found in software page hash table. |
||
462 | * Insert it into data translation cache. |
||
463 | */ |
||
464 | itc_pte_copy(t); |
||
1044 | jermar | 465 | page_table_unlock(AS, true); |
902 | jermar | 466 | } else { |
467 | /* |
||
468 | * Forward the page fault to address space page fault handler. |
||
469 | */ |
||
1044 | jermar | 470 | page_table_unlock(AS, true); |
1411 | jermar | 471 | if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) { |
1735 | decky | 472 | fault_if_from_uspace(istate,"Page fault at %p",va); |
2462 | jermar | 473 | panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip); |
902 | jermar | 474 | } |
475 | } |
||
899 | jermar | 476 | } |
818 | vana | 477 | |
3675 | svoboda | 478 | |
479 | |||
480 | static int is_io_page_accessible(int page) |
||
481 | { |
||
482 | if(TASK->arch.iomap) return bitmap_get(TASK->arch.iomap,page); |
||
483 | else return 0; |
||
484 | } |
||
485 | |||
486 | #define IO_FRAME_BASE 0xFFFFC000000 |
||
487 | |||
488 | /** There is special handling of memmaped lagacy io, because |
||
489 | * of 4KB sized access |
||
490 | * only for userspace |
||
491 | * |
||
492 | * @param va virtual address of page fault |
||
493 | * @param istate Structure with saved interruption state. |
||
494 | * |
||
495 | * |
||
496 | * @return 1 on success, 0 on fail |
||
497 | */ |
||
498 | static int try_memmap_io_insertion(uintptr_t va, istate_t *istate) |
||
499 | { |
||
500 | if((va >= IO_OFFSET ) && (va < IO_OFFSET + (1<<IO_PAGE_WIDTH))) |
||
501 | if(TASK){ |
||
502 | |||
503 | uint64_t io_page=(va & ((1<<IO_PAGE_WIDTH)-1)) >> (USPACE_IO_PAGE_WIDTH); |
||
504 | if(is_io_page_accessible(io_page)){ |
||
505 | //printf("Insert %llX\n",va); |
||
506 | |||
507 | uint64_t page,frame; |
||
508 | |||
509 | page = IO_OFFSET + (1 << USPACE_IO_PAGE_WIDTH) * io_page; |
||
510 | frame = IO_FRAME_BASE + (1 << USPACE_IO_PAGE_WIDTH) * io_page; |
||
511 | |||
512 | |||
513 | tlb_entry_t entry; |
||
514 | |||
515 | entry.word[0] = 0; |
||
516 | entry.word[1] = 0; |
||
517 | |||
518 | entry.p = true; /* present */ |
||
519 | entry.ma = MA_UNCACHEABLE; |
||
520 | entry.a = true; /* already accessed */ |
||
521 | entry.d = true; /* already dirty */ |
||
522 | entry.pl = PL_USER; |
||
523 | entry.ar = AR_READ | AR_WRITE; |
||
524 | entry.ppn = frame >> PPN_SHIFT; //MUSIM spocitat frame |
||
525 | entry.ps = USPACE_IO_PAGE_WIDTH; |
||
526 | |||
527 | dtc_mapping_insert(page, TASK->as->asid, entry); //Musim zjistit ASID |
||
528 | return 1; |
||
529 | }else { |
||
530 | fault_if_from_uspace(istate,"IO access fault at %p",va); |
||
531 | return 0; |
||
532 | } |
||
533 | } else |
||
534 | return 0; |
||
535 | else |
||
536 | return 0; |
||
537 | |||
538 | return 0; |
||
539 | |||
540 | } |
||
541 | |||
542 | |||
543 | |||
544 | |||
902 | jermar | 545 | /** Data TLB fault handler for faults with VHPT turned off. |
901 | jermar | 546 | * |
547 | * @param vector Interruption vector. |
||
958 | jermar | 548 | * @param istate Structure with saved interruption state. |
901 | jermar | 549 | */ |
1780 | jermar | 550 | void alternate_data_tlb_fault(uint64_t vector, istate_t *istate) |
899 | jermar | 551 | { |
901 | jermar | 552 | region_register rr; |
553 | rid_t rid; |
||
1780 | jermar | 554 | uintptr_t va; |
902 | jermar | 555 | pte_t *t; |
901 | jermar | 556 | |
958 | jermar | 557 | va = istate->cr_ifa; /* faulting address */ |
901 | jermar | 558 | rr.word = rr_read(VA2VRN(va)); |
559 | rid = rr.map.rid; |
||
560 | if (RID2ASID(rid) == ASID_KERNEL) { |
||
561 | if (VA2VRN(va) == VRN_KERNEL) { |
||
562 | /* |
||
563 | * Provide KA2PA(identity) mapping for faulting piece of |
||
564 | * kernel address space. |
||
565 | */ |
||
902 | jermar | 566 | dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0); |
901 | jermar | 567 | return; |
568 | } |
||
569 | } |
||
919 | jermar | 570 | |
1044 | jermar | 571 | page_table_lock(AS, true); |
902 | jermar | 572 | t = page_mapping_find(AS, va); |
573 | if (t) { |
||
574 | /* |
||
1851 | jermar | 575 | * The mapping was found in the software page hash table. |
902 | jermar | 576 | * Insert it into data translation cache. |
577 | */ |
||
578 | dtc_pte_copy(t); |
||
1044 | jermar | 579 | page_table_unlock(AS, true); |
902 | jermar | 580 | } else { |
3675 | svoboda | 581 | page_table_unlock(AS, true); |
582 | if (try_memmap_io_insertion(va,istate)) return; |
||
902 | jermar | 583 | /* |
1851 | jermar | 584 | * Forward the page fault to the address space page fault handler. |
902 | jermar | 585 | */ |
1411 | jermar | 586 | if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) { |
1735 | decky | 587 | fault_if_from_uspace(istate,"Page fault at %p",va); |
2462 | jermar | 588 | panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip); |
902 | jermar | 589 | } |
590 | } |
||
818 | vana | 591 | } |
592 | |||
902 | jermar | 593 | /** Data nested TLB fault handler. |
594 | * |
||
595 | * This fault should not occur. |
||
596 | * |
||
597 | * @param vector Interruption vector. |
||
958 | jermar | 598 | * @param istate Structure with saved interruption state. |
902 | jermar | 599 | */ |
1780 | jermar | 600 | void data_nested_tlb_fault(uint64_t vector, istate_t *istate) |
899 | jermar | 601 | { |
2462 | jermar | 602 | panic("%s\n", __func__); |
899 | jermar | 603 | } |
818 | vana | 604 | |
902 | jermar | 605 | /** Data Dirty bit fault handler. |
606 | * |
||
607 | * @param vector Interruption vector. |
||
958 | jermar | 608 | * @param istate Structure with saved interruption state. |
902 | jermar | 609 | */ |
1780 | jermar | 610 | void data_dirty_bit_fault(uint64_t vector, istate_t *istate) |
819 | vana | 611 | { |
1411 | jermar | 612 | region_register rr; |
613 | rid_t rid; |
||
1780 | jermar | 614 | uintptr_t va; |
902 | jermar | 615 | pte_t *t; |
1411 | jermar | 616 | |
617 | va = istate->cr_ifa; /* faulting address */ |
||
618 | rr.word = rr_read(VA2VRN(va)); |
||
619 | rid = rr.map.rid; |
||
902 | jermar | 620 | |
1044 | jermar | 621 | page_table_lock(AS, true); |
1411 | jermar | 622 | t = page_mapping_find(AS, va); |
902 | jermar | 623 | ASSERT(t && t->p); |
1411 | jermar | 624 | if (t && t->p && t->w) { |
902 | jermar | 625 | /* |
626 | * Update the Dirty bit in page tables and reinsert |
||
627 | * the mapping into DTC. |
||
628 | */ |
||
629 | t->d = true; |
||
630 | dtc_pte_copy(t); |
||
1411 | jermar | 631 | } else { |
632 | if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) { |
||
1735 | decky | 633 | fault_if_from_uspace(istate,"Page fault at %p",va); |
2462 | jermar | 634 | panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip); |
1411 | jermar | 635 | t->d = true; |
636 | dtc_pte_copy(t); |
||
637 | } |
||
902 | jermar | 638 | } |
1044 | jermar | 639 | page_table_unlock(AS, true); |
899 | jermar | 640 | } |
819 | vana | 641 | |
902 | jermar | 642 | /** Instruction access bit fault handler. |
643 | * |
||
644 | * @param vector Interruption vector. |
||
958 | jermar | 645 | * @param istate Structure with saved interruption state. |
902 | jermar | 646 | */ |
1780 | jermar | 647 | void instruction_access_bit_fault(uint64_t vector, istate_t *istate) |
899 | jermar | 648 | { |
1411 | jermar | 649 | region_register rr; |
650 | rid_t rid; |
||
1780 | jermar | 651 | uintptr_t va; |
1411 | jermar | 652 | pte_t *t; |
902 | jermar | 653 | |
1411 | jermar | 654 | va = istate->cr_ifa; /* faulting address */ |
655 | rr.word = rr_read(VA2VRN(va)); |
||
656 | rid = rr.map.rid; |
||
657 | |||
1044 | jermar | 658 | page_table_lock(AS, true); |
1411 | jermar | 659 | t = page_mapping_find(AS, va); |
902 | jermar | 660 | ASSERT(t && t->p); |
1411 | jermar | 661 | if (t && t->p && t->x) { |
902 | jermar | 662 | /* |
663 | * Update the Accessed bit in page tables and reinsert |
||
664 | * the mapping into ITC. |
||
665 | */ |
||
666 | t->a = true; |
||
667 | itc_pte_copy(t); |
||
1411 | jermar | 668 | } else { |
669 | if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) { |
||
1735 | decky | 670 | fault_if_from_uspace(istate,"Page fault at %p",va); |
2462 | jermar | 671 | panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip); |
1411 | jermar | 672 | t->a = true; |
673 | itc_pte_copy(t); |
||
674 | } |
||
902 | jermar | 675 | } |
1044 | jermar | 676 | page_table_unlock(AS, true); |
899 | jermar | 677 | } |
819 | vana | 678 | |
902 | jermar | 679 | /** Data access bit fault handler. |
680 | * |
||
681 | * @param vector Interruption vector. |
||
958 | jermar | 682 | * @param istate Structure with saved interruption state. |
902 | jermar | 683 | */ |
1780 | jermar | 684 | void data_access_bit_fault(uint64_t vector, istate_t *istate) |
899 | jermar | 685 | { |
1411 | jermar | 686 | region_register rr; |
687 | rid_t rid; |
||
1780 | jermar | 688 | uintptr_t va; |
902 | jermar | 689 | pte_t *t; |
690 | |||
1411 | jermar | 691 | va = istate->cr_ifa; /* faulting address */ |
692 | rr.word = rr_read(VA2VRN(va)); |
||
693 | rid = rr.map.rid; |
||
694 | |||
1044 | jermar | 695 | page_table_lock(AS, true); |
1411 | jermar | 696 | t = page_mapping_find(AS, va); |
902 | jermar | 697 | ASSERT(t && t->p); |
698 | if (t && t->p) { |
||
699 | /* |
||
700 | * Update the Accessed bit in page tables and reinsert |
||
701 | * the mapping into DTC. |
||
702 | */ |
||
703 | t->a = true; |
||
704 | dtc_pte_copy(t); |
||
1411 | jermar | 705 | } else { |
706 | if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) { |
||
1735 | decky | 707 | fault_if_from_uspace(istate,"Page fault at %p",va); |
2462 | jermar | 708 | panic("%s: va=%p, rid=%d, iip=%p\n", __func__, va, rid, istate->cr_iip); |
1411 | jermar | 709 | t->a = true; |
710 | itc_pte_copy(t); |
||
711 | } |
||
902 | jermar | 712 | } |
1044 | jermar | 713 | page_table_unlock(AS, true); |
819 | vana | 714 | } |
715 | |||
902 | jermar | 716 | /** Page not present fault handler. |
717 | * |
||
718 | * @param vector Interruption vector. |
||
958 | jermar | 719 | * @param istate Structure with saved interruption state. |
902 | jermar | 720 | */ |
1780 | jermar | 721 | void page_not_present(uint64_t vector, istate_t *istate) |
819 | vana | 722 | { |
902 | jermar | 723 | region_register rr; |
1411 | jermar | 724 | rid_t rid; |
1780 | jermar | 725 | uintptr_t va; |
902 | jermar | 726 | pte_t *t; |
727 | |||
958 | jermar | 728 | va = istate->cr_ifa; /* faulting address */ |
1411 | jermar | 729 | rr.word = rr_read(VA2VRN(va)); |
730 | rid = rr.map.rid; |
||
731 | |||
1044 | jermar | 732 | page_table_lock(AS, true); |
902 | jermar | 733 | t = page_mapping_find(AS, va); |
734 | ASSERT(t); |
||
735 | |||
736 | if (t->p) { |
||
737 | /* |
||
738 | * If the Present bit is set in page hash table, just copy it |
||
739 | * and update ITC/DTC. |
||
740 | */ |
||
741 | if (t->x) |
||
742 | itc_pte_copy(t); |
||
743 | else |
||
744 | dtc_pte_copy(t); |
||
1044 | jermar | 745 | page_table_unlock(AS, true); |
902 | jermar | 746 | } else { |
1044 | jermar | 747 | page_table_unlock(AS, true); |
1411 | jermar | 748 | if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) { |
1735 | decky | 749 | fault_if_from_uspace(istate,"Page fault at %p",va); |
2462 | jermar | 750 | panic("%s: va=%p, rid=%d\n", __func__, va, rid); |
902 | jermar | 751 | } |
752 | } |
||
819 | vana | 753 | } |
1702 | cejka | 754 | |
1850 | jermar | 755 | /** @} |
1702 | cejka | 756 | */ |