Subversion Repositories HelenOS

Rev

Rev 4625 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
740 jermar 1
/*
2071 jermar 2
 * Copyright (c) 2006 Jakub Jermar
740 jermar 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
1850 jermar 29
/** @addtogroup ia64mm 
1702 cejka 30
 * @{
31
 */
32
/** @file
33
 */
34
 
740 jermar 35
/*
36
 * TLB management.
37
 */
38
 
39
#include <mm/tlb.h>
901 jermar 40
#include <mm/asid.h>
902 jermar 41
#include <mm/page.h>
42
#include <mm/as.h>
818 vana 43
#include <arch/mm/tlb.h>
901 jermar 44
#include <arch/mm/page.h>
1210 vana 45
#include <arch/mm/vhpt.h>
819 vana 46
#include <arch/barrier.h>
900 jermar 47
#include <arch/interrupt.h>
928 vana 48
#include <arch/pal/pal.h>
49
#include <arch/asm.h>
900 jermar 50
#include <panic.h>
993 jermar 51
#include <print.h>
902 jermar 52
#include <arch.h>
1621 vana 53
#include <interrupt.h>
740 jermar 54
 
756 jermar 55
/** Invalidate all TLB entries. */
740 jermar 56
void tlb_invalidate_all(void)
57
{
1850 jermar 58
    ipl_t ipl;
59
    uintptr_t adr;
60
    uint32_t count1, count2, stride1, stride2;
928 vana 61
 
2745 decky 62
    unsigned int i, j;
928 vana 63
 
1850 jermar 64
    adr = PAL_PTCE_INFO_BASE();
65
    count1 = PAL_PTCE_INFO_COUNT1();
66
    count2 = PAL_PTCE_INFO_COUNT2();
67
    stride1 = PAL_PTCE_INFO_STRIDE1();
68
    stride2 = PAL_PTCE_INFO_STRIDE2();
928 vana 69
 
1850 jermar 70
    ipl = interrupts_disable();
928 vana 71
 
2745 decky 72
    for (i = 0; i < count1; i++) {
73
        for (j = 0; j < count2; j++) {
2082 decky 74
            asm volatile (
1850 jermar 75
                "ptc.e %0 ;;"
76
                :
77
                : "r" (adr)
78
            );
79
            adr += stride2;
928 vana 80
        }
1850 jermar 81
        adr += stride1;
82
    }
928 vana 83
 
1850 jermar 84
    interrupts_restore(ipl);
928 vana 85
 
1850 jermar 86
    srlz_d();
87
    srlz_i();
1210 vana 88
#ifdef CONFIG_VHPT
1850 jermar 89
    vhpt_invalidate_all();
1210 vana 90
#endif  
740 jermar 91
}
92
 
93
/** Invalidate entries belonging to an address space.
94
 *
3766 jermar 95
 * @param asid      Address space identifier.
740 jermar 96
 */
97
void tlb_invalidate_asid(asid_t asid)
98
{
935 vana 99
    tlb_invalidate_all();
740 jermar 100
}
818 vana 101
 
935 vana 102
 
4490 decky 103
void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
935 vana 104
{
944 vana 105
    region_register rr;
106
    bool restore_rr = false;
993 jermar 107
    int b = 0;
108
    int c = cnt;
944 vana 109
 
1780 jermar 110
    uintptr_t va;
993 jermar 111
    va = page;
947 vana 112
 
944 vana 113
    rr.word = rr_read(VA2VRN(va));
114
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
115
        /*
116
         * The selected region register does not contain required RID.
117
         * Save the old content of the register and replace the RID.
118
         */
119
        region_register rr0;
120
 
121
        rr0 = rr;
122
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
123
        rr_write(VA2VRN(va), rr0.word);
124
        srlz_d();
125
        srlz_i();
126
    }
127
 
993 jermar 128
    while(c >>= 1)
129
        b++;
130
    b >>= 1;
1780 jermar 131
    uint64_t ps;
944 vana 132
 
993 jermar 133
    switch (b) {
3766 jermar 134
    case 0: /* cnt 1 - 3 */
1850 jermar 135
        ps = PAGE_WIDTH;
136
        break;
3766 jermar 137
    case 1: /* cnt 4 - 15 */
138
        ps = PAGE_WIDTH + 2;
139
        va &= ~((1 << ps) - 1);
1850 jermar 140
        break;
3766 jermar 141
    case 2: /* cnt 16 - 63 */
142
        ps = PAGE_WIDTH + 4;
143
        va &= ~((1 << ps) - 1);
1850 jermar 144
        break;
3766 jermar 145
    case 3: /* cnt 64 - 255 */
146
        ps = PAGE_WIDTH + 6;
147
        va &= ~((1 << ps) - 1);
1850 jermar 148
        break;
3766 jermar 149
    case 4: /* cnt 256 - 1023 */
150
        ps = PAGE_WIDTH + 8;
151
        va &= ~((1 << ps) - 1);
1850 jermar 152
        break;
3766 jermar 153
    case 5: /* cnt 1024 - 4095 */
154
        ps = PAGE_WIDTH + 10;
155
        va &= ~((1 << ps) - 1);
1850 jermar 156
        break;
3766 jermar 157
    case 6: /* cnt 4096 - 16383 */
158
        ps = PAGE_WIDTH + 12;
159
        va &= ~((1 << ps) - 1);
1850 jermar 160
        break;
3766 jermar 161
    case 7: /* cnt 16384 - 65535 */
162
    case 8: /* cnt 65536 - (256K - 1) */
163
        ps = PAGE_WIDTH + 14;
164
        va &= ~((1 << ps) - 1);
1850 jermar 165
        break;
166
    default:
3766 jermar 167
        ps = PAGE_WIDTH + 18;
168
        va &= ~((1 << ps) - 1);
1850 jermar 169
        break;
944 vana 170
    }
3766 jermar 171
    for(; va < (page + cnt * PAGE_SIZE); va += (1 << ps))
172
        asm volatile ("ptc.l %0, %1;;" :: "r" (va), "r" (ps << 2));
944 vana 173
    srlz_d();
174
    srlz_i();
175
 
176
    if (restore_rr) {
177
        rr_write(VA2VRN(va), rr.word);
178
        srlz_d();
179
        srlz_i();
180
    }
935 vana 181
}
182
 
899 jermar 183
/** Insert data into data translation cache.
184
 *
3766 jermar 185
 * @param va        Virtual page address.
186
 * @param asid      Address space identifier.
187
 * @param entry     The rest of TLB entry as required by TLB insertion
188
 *          format.
899 jermar 189
 */
1780 jermar 190
void dtc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)
919 jermar 191
{
899 jermar 192
    tc_mapping_insert(va, asid, entry, true);
193
}
818 vana 194
 
899 jermar 195
/** Insert data into instruction translation cache.
196
 *
3766 jermar 197
 * @param va        Virtual page address.
198
 * @param asid      Address space identifier.
199
 * @param entry     The rest of TLB entry as required by TLB insertion
200
 *          format.
899 jermar 201
 */
1780 jermar 202
void itc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry)
919 jermar 203
{
899 jermar 204
    tc_mapping_insert(va, asid, entry, false);
205
}
818 vana 206
 
899 jermar 207
/** Insert data into instruction or data translation cache.
208
 *
3766 jermar 209
 * @param va        Virtual page address.
210
 * @param asid      Address space identifier.
211
 * @param entry     The rest of TLB entry as required by TLB insertion
212
 *          format.
213
 * @param dtc       If true, insert into data translation cache, use
214
 *          instruction translation cache otherwise.
899 jermar 215
 */
1780 jermar 216
void tc_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtc)
818 vana 217
{
218
    region_register rr;
899 jermar 219
    bool restore_rr = false;
818 vana 220
 
901 jermar 221
    rr.word = rr_read(VA2VRN(va));
222
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
899 jermar 223
        /*
224
         * The selected region register does not contain required RID.
225
         * Save the old content of the register and replace the RID.
226
         */
227
        region_register rr0;
818 vana 228
 
899 jermar 229
        rr0 = rr;
901 jermar 230
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
231
        rr_write(VA2VRN(va), rr0.word);
899 jermar 232
        srlz_d();
233
        srlz_i();
818 vana 234
    }
899 jermar 235
 
2082 decky 236
    asm volatile (
3766 jermar 237
        "mov r8 = psr;;\n"
900 jermar 238
        "rsm %0;;\n"            /* PSR_IC_MASK */
899 jermar 239
        "srlz.d;;\n"
240
        "srlz.i;;\n"
3766 jermar 241
        "mov cr.ifa = %1\n"     /* va */
242
        "mov cr.itir = %2;;\n"      /* entry.word[1] */
243
        "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
899 jermar 244
        "(p6) itc.i %3;;\n"
245
        "(p7) itc.d %3;;\n"
3766 jermar 246
        "mov psr.l = r8;;\n"
899 jermar 247
        "srlz.d;;\n"
248
        :
3766 jermar 249
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]),
250
            "r" (entry.word[0]), "r" (dtc)
900 jermar 251
        : "p6", "p7", "r8"
899 jermar 252
    );
253
 
254
    if (restore_rr) {
901 jermar 255
        rr_write(VA2VRN(va), rr.word);
819 vana 256
        srlz_d();
899 jermar 257
        srlz_i();
818 vana 258
    }
899 jermar 259
}
818 vana 260
 
899 jermar 261
/** Insert data into instruction translation register.
262
 *
3766 jermar 263
 * @param va        Virtual page address.
264
 * @param asid      Address space identifier.
265
 * @param entry     The rest of TLB entry as required by TLB insertion
266
 *          format.
267
 * @param tr        Translation register.
899 jermar 268
 */
3766 jermar 269
void
4490 decky 270
itr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, size_t tr)
899 jermar 271
{
272
    tr_mapping_insert(va, asid, entry, false, tr);
273
}
818 vana 274
 
899 jermar 275
/** Insert data into data translation register.
276
 *
3766 jermar 277
 * @param va        Virtual page address.
278
 * @param asid      Address space identifier.
279
 * @param entry     The rest of TLB entry as required by TLB insertion
280
 *          format.
281
 * @param tr        Translation register.
899 jermar 282
 */
3766 jermar 283
void
4490 decky 284
dtr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, size_t tr)
899 jermar 285
{
286
    tr_mapping_insert(va, asid, entry, true, tr);
818 vana 287
}
288
 
899 jermar 289
/** Insert data into instruction or data translation register.
290
 *
3766 jermar 291
 * @param va        Virtual page address.
292
 * @param asid      Address space identifier.
293
 * @param entry     The rest of TLB entry as required by TLB insertion
294
 *          format.
295
 * @param dtr       If true, insert into data translation register, use
296
 *          instruction translation register otherwise.
297
 * @param tr        Translation register.
899 jermar 298
 */
3766 jermar 299
void
300
tr_mapping_insert(uintptr_t va, asid_t asid, tlb_entry_t entry, bool dtr,
4490 decky 301
    size_t tr)
818 vana 302
{
303
    region_register rr;
899 jermar 304
    bool restore_rr = false;
818 vana 305
 
901 jermar 306
    rr.word = rr_read(VA2VRN(va));
307
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
899 jermar 308
        /*
309
         * The selected region register does not contain required RID.
310
         * Save the old content of the register and replace the RID.
311
         */
312
        region_register rr0;
818 vana 313
 
899 jermar 314
        rr0 = rr;
901 jermar 315
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
316
        rr_write(VA2VRN(va), rr0.word);
899 jermar 317
        srlz_d();
318
        srlz_i();
319
    }
818 vana 320
 
2082 decky 321
    asm volatile (
3766 jermar 322
        "mov r8 = psr;;\n"
900 jermar 323
        "rsm %0;;\n"            /* PSR_IC_MASK */
899 jermar 324
        "srlz.d;;\n"
325
        "srlz.i;;\n"
3766 jermar 326
        "mov cr.ifa = %1\n"         /* va */         
327
        "mov cr.itir = %2;;\n"      /* entry.word[1] */
328
        "cmp.eq p6,p7 = %5,r0;;\n"  /* decide between itr and dtr */
329
        "(p6) itr.i itr[%4] = %3;;\n"
330
        "(p7) itr.d dtr[%4] = %3;;\n"
331
        "mov psr.l = r8;;\n"
899 jermar 332
        "srlz.d;;\n"
333
        :
3766 jermar 334
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]),
335
            "r" (entry.word[0]), "r" (tr), "r" (dtr)
900 jermar 336
        : "p6", "p7", "r8"
899 jermar 337
    );
338
 
339
    if (restore_rr) {
901 jermar 340
        rr_write(VA2VRN(va), rr.word);
819 vana 341
        srlz_d();
899 jermar 342
        srlz_i();
818 vana 343
    }
899 jermar 344
}
818 vana 345
 
901 jermar 346
/** Insert data into DTLB.
347
 *
3766 jermar 348
 * @param page      Virtual page address including VRN bits.
349
 * @param frame     Physical frame address.
350
 * @param dtr       If true, insert into data translation register, use data
351
 *          translation cache otherwise.
352
 * @param tr        Translation register if dtr is true, ignored otherwise.
901 jermar 353
 */
3766 jermar 354
void
355
dtlb_kernel_mapping_insert(uintptr_t page, uintptr_t frame, bool dtr,
4490 decky 356
    size_t tr)
901 jermar 357
{
358
    tlb_entry_t entry;
359
 
360
    entry.word[0] = 0;
361
    entry.word[1] = 0;
362
 
363
    entry.p = true;         /* present */
364
    entry.ma = MA_WRITEBACK;
365
    entry.a = true;         /* already accessed */
366
    entry.d = true;         /* already dirty */
367
    entry.pl = PL_KERNEL;
368
    entry.ar = AR_READ | AR_WRITE;
369
    entry.ppn = frame >> PPN_SHIFT;
370
    entry.ps = PAGE_WIDTH;
371
 
372
    if (dtr)
373
        dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
374
    else
375
        dtc_mapping_insert(page, ASID_KERNEL, entry);
376
}
377
 
1675 jermar 378
/** Purge kernel entries from DTR.
379
 *
380
 * Purge DTR entries used by the kernel.
381
 *
3766 jermar 382
 * @param page      Virtual page address including VRN bits.
383
 * @param width     Width of the purge in bits.
1675 jermar 384
 */
4490 decky 385
void dtr_purge(uintptr_t page, size_t width)
1675 jermar 386
{
3766 jermar 387
    asm volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width << 2));
1675 jermar 388
}
389
 
390
 
902 jermar 391
/** Copy content of PTE into data translation cache.
392
 *
3766 jermar 393
 * @param t     PTE.
902 jermar 394
 */
395
void dtc_pte_copy(pte_t *t)
396
{
397
    tlb_entry_t entry;
398
 
399
    entry.word[0] = 0;
400
    entry.word[1] = 0;
401
 
402
    entry.p = t->p;
403
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
404
    entry.a = t->a;
405
    entry.d = t->d;
406
    entry.pl = t->k ? PL_KERNEL : PL_USER;
407
    entry.ar = t->w ? AR_WRITE : AR_READ;
408
    entry.ppn = t->frame >> PPN_SHIFT;
409
    entry.ps = PAGE_WIDTH;
410
 
411
    dtc_mapping_insert(t->page, t->as->asid, entry);
1210 vana 412
#ifdef CONFIG_VHPT
413
    vhpt_mapping_insert(t->page, t->as->asid, entry);
414
#endif  
902 jermar 415
}
416
 
417
/** Copy content of PTE into instruction translation cache.
418
 *
3766 jermar 419
 * @param t     PTE.
902 jermar 420
 */
421
void itc_pte_copy(pte_t *t)
422
{
423
    tlb_entry_t entry;
424
 
425
    entry.word[0] = 0;
426
    entry.word[1] = 0;
427
 
428
    ASSERT(t->x);
429
 
430
    entry.p = t->p;
431
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
432
    entry.a = t->a;
433
    entry.pl = t->k ? PL_KERNEL : PL_USER;
434
    entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
435
    entry.ppn = t->frame >> PPN_SHIFT;
436
    entry.ps = PAGE_WIDTH;
437
 
438
    itc_mapping_insert(t->page, t->as->asid, entry);
1210 vana 439
#ifdef CONFIG_VHPT
440
    vhpt_mapping_insert(t->page, t->as->asid, entry);
441
#endif  
902 jermar 442
}
443
 
444
/** Instruction TLB fault handler for faults with VHPT turned off.
445
 *
3766 jermar 446
 * @param vector        Interruption vector.
447
 * @param istate        Structure with saved interruption state.
902 jermar 448
 */
1780 jermar 449
void alternate_instruction_tlb_fault(uint64_t vector, istate_t *istate)
899 jermar 450
{
902 jermar 451
    region_register rr;
1411 jermar 452
    rid_t rid;
1780 jermar 453
    uintptr_t va;
902 jermar 454
    pte_t *t;
455
 
958 jermar 456
    va = istate->cr_ifa;    /* faulting address */
1411 jermar 457
    rr.word = rr_read(VA2VRN(va));
458
    rid = rr.map.rid;
459
 
1044 jermar 460
    page_table_lock(AS, true);
902 jermar 461
    t = page_mapping_find(AS, va);
462
    if (t) {
463
        /*
464
         * The mapping was found in software page hash table.
465
         * Insert it into data translation cache.
466
         */
467
        itc_pte_copy(t);
1044 jermar 468
        page_table_unlock(AS, true);
902 jermar 469
    } else {
470
        /*
471
         * Forward the page fault to address space page fault handler.
472
         */
1044 jermar 473
        page_table_unlock(AS, true);
1411 jermar 474
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
3790 svoboda 475
            fault_if_from_uspace(istate,"Page fault at %p.",va);
476
            panic("%s: va=%p, rid=%d, iip=%p.", __func__, va, rid,
3766 jermar 477
                istate->cr_iip);
902 jermar 478
        }
479
    }
899 jermar 480
}
818 vana 481
 
3635 vana 482
static int is_io_page_accessible(int page)
483
{
3766 jermar 484
    if (TASK->arch.iomap)
3777 jermar 485
        return bitmap_get(TASK->arch.iomap, page);
3766 jermar 486
    else
487
        return 0;
3635 vana 488
}
489
 
490
#define IO_FRAME_BASE 0xFFFFC000000
491
 
3766 jermar 492
/**
493
 * There is special handling of memory mapped legacy io, because of 4KB sized
494
 * access for userspace.
3635 vana 495
 *
3766 jermar 496
 * @param va        Virtual address of page fault.
497
 * @param istate    Structure with saved interruption state.
3635 vana 498
 *
3766 jermar 499
 * @return      One on success, zero on failure.
3635 vana 500
 */
501
static int try_memmap_io_insertion(uintptr_t va, istate_t *istate)
502
{
3766 jermar 503
    if ((va >= IO_OFFSET ) && (va < IO_OFFSET + (1 << IO_PAGE_WIDTH))) {
504
        if (TASK) {
505
            uint64_t io_page = (va & ((1 << IO_PAGE_WIDTH) - 1)) >>
506
                USPACE_IO_PAGE_WIDTH;
3635 vana 507
 
3766 jermar 508
            if (is_io_page_accessible(io_page)) {
509
                uint64_t page, frame;
3635 vana 510
 
3766 jermar 511
                page = IO_OFFSET +
512
                    (1 << USPACE_IO_PAGE_WIDTH) * io_page;
513
                frame = IO_FRAME_BASE +
514
                    (1 << USPACE_IO_PAGE_WIDTH) * io_page;
3635 vana 515
 
516
                tlb_entry_t entry;
517
 
518
                entry.word[0] = 0;
519
                entry.word[1] = 0;
520
 
3766 jermar 521
                entry.p = true;     /* present */
3635 vana 522
                entry.ma = MA_UNCACHEABLE;     
3766 jermar 523
                entry.a = true;     /* already accessed */
524
                entry.d = true;     /* already dirty */
3635 vana 525
                entry.pl = PL_USER;
526
                entry.ar = AR_READ | AR_WRITE;
3763 jermar 527
                entry.ppn = frame >> PPN_SHIFT;
3635 vana 528
                entry.ps = USPACE_IO_PAGE_WIDTH;
529
 
3763 jermar 530
                dtc_mapping_insert(page, TASK->as->asid, entry);
3635 vana 531
                return 1;
3766 jermar 532
            } else {
533
                fault_if_from_uspace(istate,
3790 svoboda 534
                    "IO access fault at %p.", va);
3766 jermar 535
            }
536
        }
537
    }
3635 vana 538
 
539
    return 0;
540
}
541
 
902 jermar 542
/** Data TLB fault handler for faults with VHPT turned off.
901 jermar 543
 *
3766 jermar 544
 * @param vector    Interruption vector.
545
 * @param istate    Structure with saved interruption state.
901 jermar 546
 */
1780 jermar 547
void alternate_data_tlb_fault(uint64_t vector, istate_t *istate)
899 jermar 548
{
901 jermar 549
    region_register rr;
550
    rid_t rid;
1780 jermar 551
    uintptr_t va;
902 jermar 552
    pte_t *t;
901 jermar 553
 
958 jermar 554
    va = istate->cr_ifa;    /* faulting address */
901 jermar 555
    rr.word = rr_read(VA2VRN(va));
556
    rid = rr.map.rid;
557
    if (RID2ASID(rid) == ASID_KERNEL) {
558
        if (VA2VRN(va) == VRN_KERNEL) {
559
            /*
560
             * Provide KA2PA(identity) mapping for faulting piece of
561
             * kernel address space.
562
             */
902 jermar 563
            dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
901 jermar 564
            return;
565
        }
566
    }
919 jermar 567
 
1044 jermar 568
    page_table_lock(AS, true);
902 jermar 569
    t = page_mapping_find(AS, va);
570
    if (t) {
571
        /*
1851 jermar 572
         * The mapping was found in the software page hash table.
902 jermar 573
         * Insert it into data translation cache.
574
         */
575
        dtc_pte_copy(t);
1044 jermar 576
        page_table_unlock(AS, true);
902 jermar 577
    } else {
3635 vana 578
        page_table_unlock(AS, true);
3766 jermar 579
        if (try_memmap_io_insertion(va, istate))
580
            return;
902 jermar 581
        /*
3766 jermar 582
         * Forward the page fault to the address space page fault
583
         * handler.
902 jermar 584
         */
1411 jermar 585
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
3790 svoboda 586
            fault_if_from_uspace(istate,"Page fault at %p.",va);
587
            panic("%s: va=%p, rid=%d, iip=%p.", __func__, va, rid,
3766 jermar 588
                istate->cr_iip);
902 jermar 589
        }
590
    }
818 vana 591
}
592
 
902 jermar 593
/** Data nested TLB fault handler.
594
 *
595
 * This fault should not occur.
596
 *
3766 jermar 597
 * @param vector    Interruption vector.
598
 * @param istate    Structure with saved interruption state.
902 jermar 599
 */
1780 jermar 600
void data_nested_tlb_fault(uint64_t vector, istate_t *istate)
899 jermar 601
{
3790 svoboda 602
    panic("%s.", __func__);
899 jermar 603
}
818 vana 604
 
902 jermar 605
/** Data Dirty bit fault handler.
606
 *
3766 jermar 607
 * @param vector    Interruption vector.
608
 * @param istate    Structure with saved interruption state.
902 jermar 609
 */
1780 jermar 610
void data_dirty_bit_fault(uint64_t vector, istate_t *istate)
819 vana 611
{
1411 jermar 612
    region_register rr;
613
    rid_t rid;
1780 jermar 614
    uintptr_t va;
902 jermar 615
    pte_t *t;
1411 jermar 616
 
617
    va = istate->cr_ifa;    /* faulting address */
618
    rr.word = rr_read(VA2VRN(va));
619
    rid = rr.map.rid;
902 jermar 620
 
1044 jermar 621
    page_table_lock(AS, true);
1411 jermar 622
    t = page_mapping_find(AS, va);
902 jermar 623
    ASSERT(t && t->p);
1411 jermar 624
    if (t && t->p && t->w) {
902 jermar 625
        /*
626
         * Update the Dirty bit in page tables and reinsert
627
         * the mapping into DTC.
628
         */
629
        t->d = true;
630
        dtc_pte_copy(t);
1411 jermar 631
    } else {
632
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
3790 svoboda 633
            fault_if_from_uspace(istate,"Page fault at %p.",va);
634
            panic("%s: va=%p, rid=%d, iip=%p.", __func__, va, rid,
3766 jermar 635
                istate->cr_iip);
1411 jermar 636
        }
902 jermar 637
    }
1044 jermar 638
    page_table_unlock(AS, true);
899 jermar 639
}
819 vana 640
 
902 jermar 641
/** Instruction access bit fault handler.
642
 *
3766 jermar 643
 * @param vector    Interruption vector.
644
 * @param istate    Structure with saved interruption state.
902 jermar 645
 */
1780 jermar 646
void instruction_access_bit_fault(uint64_t vector, istate_t *istate)
899 jermar 647
{
1411 jermar 648
    region_register rr;
649
    rid_t rid;
1780 jermar 650
    uintptr_t va;
1411 jermar 651
    pte_t *t;  
902 jermar 652
 
1411 jermar 653
    va = istate->cr_ifa;    /* faulting address */
654
    rr.word = rr_read(VA2VRN(va));
655
    rid = rr.map.rid;
656
 
1044 jermar 657
    page_table_lock(AS, true);
1411 jermar 658
    t = page_mapping_find(AS, va);
902 jermar 659
    ASSERT(t && t->p);
1411 jermar 660
    if (t && t->p && t->x) {
902 jermar 661
        /*
662
         * Update the Accessed bit in page tables and reinsert
663
         * the mapping into ITC.
664
         */
665
        t->a = true;
666
        itc_pte_copy(t);
1411 jermar 667
    } else {
668
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
3790 svoboda 669
            fault_if_from_uspace(istate, "Page fault at %p.", va);
670
            panic("%s: va=%p, rid=%d, iip=%p.", __func__, va, rid,
3766 jermar 671
                istate->cr_iip);
1411 jermar 672
        }
902 jermar 673
    }
1044 jermar 674
    page_table_unlock(AS, true);
899 jermar 675
}
819 vana 676
 
902 jermar 677
/** Data access bit fault handler.
678
 *
679
 * @param vector Interruption vector.
958 jermar 680
 * @param istate Structure with saved interruption state.
902 jermar 681
 */
1780 jermar 682
void data_access_bit_fault(uint64_t vector, istate_t *istate)
899 jermar 683
{
1411 jermar 684
    region_register rr;
685
    rid_t rid;
1780 jermar 686
    uintptr_t va;
902 jermar 687
    pte_t *t;
688
 
1411 jermar 689
    va = istate->cr_ifa;    /* faulting address */
690
    rr.word = rr_read(VA2VRN(va));
691
    rid = rr.map.rid;
692
 
1044 jermar 693
    page_table_lock(AS, true);
1411 jermar 694
    t = page_mapping_find(AS, va);
902 jermar 695
    ASSERT(t && t->p);
696
    if (t && t->p) {
697
        /*
698
         * Update the Accessed bit in page tables and reinsert
699
         * the mapping into DTC.
700
         */
701
        t->a = true;
702
        dtc_pte_copy(t);
1411 jermar 703
    } else {
704
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
3790 svoboda 705
            fault_if_from_uspace(istate, "Page fault at %p.", va);
706
            panic("%s: va=%p, rid=%d, iip=%p.", __func__, va, rid,
3766 jermar 707
                istate->cr_iip);
1411 jermar 708
        }
902 jermar 709
    }
1044 jermar 710
    page_table_unlock(AS, true);
819 vana 711
}
712
 
4625 jermar 713
/** Data access rights fault handler.
714
 *
715
 * @param vector Interruption vector.
716
 * @param istate Structure with saved interruption state.
717
 */
718
void data_access_rights_fault(uint64_t vector, istate_t *istate)
719
{
720
    region_register rr;
721
    rid_t rid;
722
    uintptr_t va;
723
    pte_t *t;
724
 
725
    va = istate->cr_ifa;    /* faulting address */
726
    rr.word = rr_read(VA2VRN(va));
727
    rid = rr.map.rid;
728
 
729
    /*
730
     * Assume a write to a read-only page.
731
     */
732
    page_table_lock(AS, true);
733
    t = page_mapping_find(AS, va);
734
    ASSERT(t && t->p);
735
    ASSERT(!t->w);
736
    if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
737
        fault_if_from_uspace(istate, "Page fault at %p.", va);
738
        panic("%s: va=%p, rid=%d, iip=%p.", __func__, va, rid,
739
            istate->cr_iip);
740
    }
741
    page_table_unlock(AS, true);
742
}
743
 
902 jermar 744
/** Page not present fault handler.
745
 *
746
 * @param vector Interruption vector.
958 jermar 747
 * @param istate Structure with saved interruption state.
902 jermar 748
 */
1780 jermar 749
void page_not_present(uint64_t vector, istate_t *istate)
819 vana 750
{
902 jermar 751
    region_register rr;
1411 jermar 752
    rid_t rid;
1780 jermar 753
    uintptr_t va;
902 jermar 754
    pte_t *t;
755
 
958 jermar 756
    va = istate->cr_ifa;    /* faulting address */
1411 jermar 757
    rr.word = rr_read(VA2VRN(va));
758
    rid = rr.map.rid;
759
 
1044 jermar 760
    page_table_lock(AS, true);
902 jermar 761
    t = page_mapping_find(AS, va);
762
    ASSERT(t);
763
 
764
    if (t->p) {
765
        /*
766
         * If the Present bit is set in page hash table, just copy it
767
         * and update ITC/DTC.
768
         */
769
        if (t->x)
770
            itc_pte_copy(t);
771
        else
772
            dtc_pte_copy(t);
1044 jermar 773
        page_table_unlock(AS, true);
902 jermar 774
    } else {
1044 jermar 775
        page_table_unlock(AS, true);
1411 jermar 776
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
3790 svoboda 777
            fault_if_from_uspace(istate, "Page fault at %p.", va);
778
            panic("%s: va=%p, rid=%d.", __func__, va, rid);
902 jermar 779
        }
780
    }
819 vana 781
}
1702 cejka 782
 
3940 decky 783
void tlb_arch_init(void)
784
{
785
}
786
 
787
void tlb_print(void)
788
{
789
}
790
 
1850 jermar 791
/** @}
1702 cejka 792
 */