Subversion Repositories HelenOS-historic

Rev

Rev 1411 | Rev 1735 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1 jermar 1
/*
319 jermar 2
 * Copyright (C) 2003-2004 Jakub Jermar
1 jermar 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
29
#include <arch/mm/tlb.h>
727 jermar 30
#include <mm/asid.h>
1 jermar 31
#include <mm/tlb.h>
391 jermar 32
#include <mm/page.h>
703 jermar 33
#include <mm/as.h>
1 jermar 34
#include <arch/cp0.h>
35
#include <panic.h>
36
#include <arch.h>
268 palkovsky 37
#include <symtab.h>
391 jermar 38
#include <synch/spinlock.h>
39
#include <print.h>
396 jermar 40
#include <debug.h>
983 palkovsky 41
#include <align.h>
1595 palkovsky 42
#include <interrupt.h>
268 palkovsky 43
 
958 jermar 44
static void tlb_refill_fail(istate_t *istate);
45
static void tlb_invalid_fail(istate_t *istate);
46
static void tlb_modified_fail(istate_t *istate);
391 jermar 47
 
1411 jermar 48
static pte_t *find_mapping_and_check(__address badvaddr, int access, istate_t *istate, int *pfrc);
399 jermar 49
 
831 jermar 50
static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
399 jermar 51
static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
394 jermar 52
 
391 jermar 53
/** Initialize TLB
54
 *
55
 * Initialize TLB.
56
 * Invalidate all entries and mark wired entries.
57
 */
569 jermar 58
void tlb_arch_init(void)
389 jermar 59
{
599 jermar 60
    int i;
61
 
389 jermar 62
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
599 jermar 63
    cp0_entry_hi_write(0);
64
    cp0_entry_lo0_write(0);
65
    cp0_entry_lo1_write(0);
389 jermar 66
 
599 jermar 67
    /* Clear and initialize TLB. */
68
 
69
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
70
        cp0_index_write(i);
71
        tlbwi();
72
    }
612 jermar 73
 
598 jermar 74
 
389 jermar 75
    /*
76
     * The kernel is going to make use of some wired
391 jermar 77
     * entries (e.g. mapping kernel stacks in kseg3).
389 jermar 78
     */
79
    cp0_wired_write(TLB_WIRED);
80
}
81
 
391 jermar 82
/** Process TLB Refill Exception
83
 *
84
 * Process TLB Refill Exception.
85
 *
958 jermar 86
 * @param istate Interrupted register context.
391 jermar 87
 */
958 jermar 88
void tlb_refill(istate_t *istate)
1 jermar 89
{
396 jermar 90
    entry_lo_t lo;
1044 jermar 91
    entry_hi_t hi;
92
    asid_t asid;
391 jermar 93
    __address badvaddr;
94
    pte_t *pte;
1288 jermar 95
    int pfrc;
397 jermar 96
 
391 jermar 97
    badvaddr = cp0_badvaddr_read();
397 jermar 98
 
1044 jermar 99
    spinlock_lock(&AS->lock);
100
    asid = AS->asid;
101
    spinlock_unlock(&AS->lock);
399 jermar 102
 
1044 jermar 103
    page_table_lock(AS, true);
104
 
1411 jermar 105
    pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
1288 jermar 106
    if (!pte) {
107
        switch (pfrc) {
108
        case AS_PF_FAULT:
109
            goto fail;
110
            break;
111
        case AS_PF_DEFER:
112
            /*
113
             * The page fault came during copy_from_uspace()
114
             * or copy_to_uspace().
115
             */
116
            page_table_unlock(AS, true);
117
            return;
118
        default:
119
            panic("unexpected pfrc (%d)\n", pfrc);
120
        }
121
    }
391 jermar 122
 
123
    /*
394 jermar 124
     * Record access to PTE.
391 jermar 125
     */
394 jermar 126
    pte->a = 1;
391 jermar 127
 
1044 jermar 128
    prepare_entry_hi(&hi, asid, badvaddr);
831 jermar 129
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
394 jermar 130
 
391 jermar 131
    /*
132
     * New entry is to be inserted into TLB
133
     */
399 jermar 134
    cp0_entry_hi_write(hi.value);
391 jermar 135
    if ((badvaddr/PAGE_SIZE) % 2 == 0) {
396 jermar 136
        cp0_entry_lo0_write(lo.value);
391 jermar 137
        cp0_entry_lo1_write(0);
138
    }
139
    else {
140
        cp0_entry_lo0_write(0);
396 jermar 141
        cp0_entry_lo1_write(lo.value);
391 jermar 142
    }
612 jermar 143
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
391 jermar 144
    tlbwr();
145
 
1044 jermar 146
    page_table_unlock(AS, true);
391 jermar 147
    return;
148
 
149
fail:
1044 jermar 150
    page_table_unlock(AS, true);
958 jermar 151
    tlb_refill_fail(istate);
391 jermar 152
}
153
 
394 jermar 154
/** Process TLB Invalid Exception
155
 *
156
 * Process TLB Invalid Exception.
157
 *
958 jermar 158
 * @param istate Interrupted register context.
394 jermar 159
 */
958 jermar 160
void tlb_invalid(istate_t *istate)
391 jermar 161
{
396 jermar 162
    tlb_index_t index;
394 jermar 163
    __address badvaddr;
396 jermar 164
    entry_lo_t lo;
399 jermar 165
    entry_hi_t hi;
394 jermar 166
    pte_t *pte;
1288 jermar 167
    int pfrc;
394 jermar 168
 
169
    badvaddr = cp0_badvaddr_read();
170
 
171
    /*
172
     * Locate the faulting entry in TLB.
173
     */
399 jermar 174
    hi.value = cp0_entry_hi_read();
175
    prepare_entry_hi(&hi, hi.asid, badvaddr);
176
    cp0_entry_hi_write(hi.value);
394 jermar 177
    tlbp();
396 jermar 178
    index.value = cp0_index_read();
1044 jermar 179
 
180
    page_table_lock(AS, true); 
394 jermar 181
 
182
    /*
183
     * Fail if the entry is not in TLB.
184
     */
396 jermar 185
    if (index.p) {
186
        printf("TLB entry not found.\n");
394 jermar 187
        goto fail;
396 jermar 188
    }
394 jermar 189
 
1411 jermar 190
    pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
1288 jermar 191
    if (!pte) {
192
        switch (pfrc) {
193
        case AS_PF_FAULT:
194
            goto fail;
195
            break;
196
        case AS_PF_DEFER:
197
            /*
198
             * The page fault came during copy_from_uspace()
199
             * or copy_to_uspace().
200
             */
201
            page_table_unlock(AS, true);             
202
            return;
203
        default:
204
            panic("unexpected pfrc (%d)\n", pfrc);
205
        }
206
    }
394 jermar 207
 
208
    /*
209
     * Read the faulting TLB entry.
210
     */
211
    tlbr();
212
 
213
    /*
214
     * Record access to PTE.
215
     */
216
    pte->a = 1;
217
 
831 jermar 218
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
394 jermar 219
 
220
    /*
221
     * The entry is to be updated in TLB.
222
     */
223
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
396 jermar 224
        cp0_entry_lo0_write(lo.value);
394 jermar 225
    else
396 jermar 226
        cp0_entry_lo1_write(lo.value);
612 jermar 227
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
394 jermar 228
    tlbwi();
229
 
1044 jermar 230
    page_table_unlock(AS, true);
394 jermar 231
    return;
232
 
233
fail:
1044 jermar 234
    page_table_unlock(AS, true);
958 jermar 235
    tlb_invalid_fail(istate);
391 jermar 236
}
237
 
394 jermar 238
/** Process TLB Modified Exception
239
 *
240
 * Process TLB Modified Exception.
241
 *
958 jermar 242
 * @param istate Interrupted register context.
394 jermar 243
 */
958 jermar 244
void tlb_modified(istate_t *istate)
391 jermar 245
{
396 jermar 246
    tlb_index_t index;
394 jermar 247
    __address badvaddr;
396 jermar 248
    entry_lo_t lo;
399 jermar 249
    entry_hi_t hi;
394 jermar 250
    pte_t *pte;
1288 jermar 251
    int pfrc;
394 jermar 252
 
253
    badvaddr = cp0_badvaddr_read();
254
 
255
    /*
256
     * Locate the faulting entry in TLB.
257
     */
399 jermar 258
    hi.value = cp0_entry_hi_read();
259
    prepare_entry_hi(&hi, hi.asid, badvaddr);
260
    cp0_entry_hi_write(hi.value);
394 jermar 261
    tlbp();
396 jermar 262
    index.value = cp0_index_read();
1044 jermar 263
 
264
    page_table_lock(AS, true); 
394 jermar 265
 
266
    /*
267
     * Fail if the entry is not in TLB.
268
     */
396 jermar 269
    if (index.p) {
270
        printf("TLB entry not found.\n");
394 jermar 271
        goto fail;
396 jermar 272
    }
394 jermar 273
 
1411 jermar 274
    pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
1288 jermar 275
    if (!pte) {
276
        switch (pfrc) {
277
        case AS_PF_FAULT:
278
            goto fail;
279
            break;
280
        case AS_PF_DEFER:
281
            /*
282
             * The page fault came during copy_from_uspace()
283
             * or copy_to_uspace().
284
             */
285
            page_table_unlock(AS, true);             
286
            return;
287
        default:
288
            panic("unexpected pfrc (%d)\n", pfrc);
289
        }
290
    }
394 jermar 291
 
292
    /*
293
     * Fail if the page is not writable.
294
     */
295
    if (!pte->w)
296
        goto fail;
297
 
298
    /*
299
     * Read the faulting TLB entry.
300
     */
301
    tlbr();
302
 
303
    /*
304
     * Record access and write to PTE.
305
     */
306
    pte->a = 1;
831 jermar 307
    pte->d = 1;
394 jermar 308
 
831 jermar 309
    prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
394 jermar 310
 
311
    /*
312
     * The entry is to be updated in TLB.
313
     */
314
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
396 jermar 315
        cp0_entry_lo0_write(lo.value);
394 jermar 316
    else
396 jermar 317
        cp0_entry_lo1_write(lo.value);
612 jermar 318
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
394 jermar 319
    tlbwi();
320
 
1044 jermar 321
    page_table_unlock(AS, true);
394 jermar 322
    return;
323
 
324
fail:
1044 jermar 325
    page_table_unlock(AS, true);
958 jermar 326
    tlb_modified_fail(istate);
391 jermar 327
}
328
 
958 jermar 329
void tlb_refill_fail(istate_t *istate)
391 jermar 330
{
324 palkovsky 331
    char *symbol = "";
332
    char *sym2 = "";
333
 
958 jermar 334
    char *s = get_symtab_entry(istate->epc);
332 palkovsky 335
    if (s)
336
        symbol = s;
958 jermar 337
    s = get_symtab_entry(istate->ra);
332 palkovsky 338
    if (s)
339
        sym2 = s;
1595 palkovsky 340
 
341
    fault_if_from_uspace(istate, "TLB Refill Exception on %P", cp0_badvaddr_read());
958 jermar 342
    panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
1 jermar 343
}
344
 
391 jermar 345
 
958 jermar 346
void tlb_invalid_fail(istate_t *istate)
1 jermar 347
{
268 palkovsky 348
    char *symbol = "";
349
 
958 jermar 350
    char *s = get_symtab_entry(istate->epc);
332 palkovsky 351
    if (s)
352
        symbol = s;
1595 palkovsky 353
    fault_if_from_uspace(istate, "TLB Invalid Exception on %P", cp0_badvaddr_read());
958 jermar 354
    panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
1 jermar 355
}
356
 
958 jermar 357
void tlb_modified_fail(istate_t *istate)
389 jermar 358
{
359
    char *symbol = "";
360
 
958 jermar 361
    char *s = get_symtab_entry(istate->epc);
389 jermar 362
    if (s)
363
        symbol = s;
1595 palkovsky 364
    fault_if_from_uspace(istate, "TLB Modified Exception on %P", cp0_badvaddr_read());
958 jermar 365
    panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
389 jermar 366
}
367
 
394 jermar 368
/** Try to find PTE for faulting address
369
 *
370
 * Try to find PTE for faulting address.
703 jermar 371
 * The AS->lock must be held on entry to this function.
394 jermar 372
 *
373
 * @param badvaddr Faulting virtual address.
1411 jermar 374
 * @param access Access mode that caused the fault.
1288 jermar 375
 * @param istate Pointer to interrupted state.
376
 * @param pfrc Pointer to variable where as_page_fault() return code will be stored.
394 jermar 377
 *
378
 * @return PTE on success, NULL otherwise.
379
 */
1411 jermar 380
pte_t *find_mapping_and_check(__address badvaddr, int access, istate_t *istate, int *pfrc)
394 jermar 381
{
396 jermar 382
    entry_hi_t hi;
394 jermar 383
    pte_t *pte;
384
 
396 jermar 385
    hi.value = cp0_entry_hi_read();
394 jermar 386
 
387
    /*
388
     * Handler cannot succeed if the ASIDs don't match.
389
     */
703 jermar 390
    if (hi.asid != AS->asid) {
391
        printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
394 jermar 392
        return NULL;
396 jermar 393
    }
703 jermar 394
 
394 jermar 395
    /*
703 jermar 396
     * Check if the mapping exists in page tables.
397
     */
756 jermar 398
    pte = page_mapping_find(AS, badvaddr);
831 jermar 399
    if (pte && pte->p) {
703 jermar 400
        /*
401
         * Mapping found in page tables.
402
         * Immediately succeed.
403
         */
404
        return pte;
405
    } else {
1288 jermar 406
        int rc;
407
 
703 jermar 408
        /*
409
         * Mapping not found in page tables.
410
         * Resort to higher-level page fault handler.
411
         */
1044 jermar 412
        page_table_unlock(AS, true);
1411 jermar 413
        switch (rc = as_page_fault(badvaddr, access, istate)) {
1288 jermar 414
        case AS_PF_OK:
703 jermar 415
            /*
416
             * The higher-level page fault handler succeeded,
417
             * The mapping ought to be in place.
418
             */
1044 jermar 419
            page_table_lock(AS, true);
756 jermar 420
            pte = page_mapping_find(AS, badvaddr);
831 jermar 421
            ASSERT(pte && pte->p);
703 jermar 422
            return pte;
1288 jermar 423
            break;
424
        case AS_PF_DEFER:
1044 jermar 425
            page_table_lock(AS, true);
1288 jermar 426
            *pfrc = AS_PF_DEFER;
427
            return NULL;
428
            break;
429
        case AS_PF_FAULT:
430
            page_table_lock(AS, true);
1044 jermar 431
            printf("Page fault.\n");
1288 jermar 432
            *pfrc = AS_PF_FAULT;
1044 jermar 433
            return NULL;
1288 jermar 434
            break;
435
        default:
436
            panic("unexpected rc (%d)\n", rc);
703 jermar 437
        }
1044 jermar 438
 
703 jermar 439
    }
394 jermar 440
}
441
 
831 jermar 442
void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
394 jermar 443
{
399 jermar 444
    lo->value = 0;
394 jermar 445
    lo->g = g;
446
    lo->v = v;
447
    lo->d = d;
831 jermar 448
    lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
394 jermar 449
    lo->pfn = pfn;
450
}
399 jermar 451
 
452
void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
453
{
983 palkovsky 454
    hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
399 jermar 455
    hi->asid = asid;
456
}
569 jermar 457
 
594 jermar 458
/** Print contents of TLB. */
569 jermar 459
void tlb_print(void)
460
{
612 jermar 461
    page_mask_t mask;
594 jermar 462
    entry_lo_t lo0, lo1;
704 jermar 463
    entry_hi_t hi, hi_save;
594 jermar 464
    int i;
465
 
704 jermar 466
    hi_save.value = cp0_entry_hi_read();
467
 
594 jermar 468
    printf("TLB:\n");
469
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
470
        cp0_index_write(i);
471
        tlbr();
472
 
612 jermar 473
        mask.value = cp0_pagemask_read();
594 jermar 474
        hi.value = cp0_entry_hi_read();
475
        lo0.value = cp0_entry_lo0_read();
476
        lo1.value = cp0_entry_lo1_read();
477
 
1196 cejka 478
        printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%hhd, pfn[0]=%d\n"
479
               "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%hhd, pfn[1]=%d\n",
612 jermar 480
               i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
594 jermar 481
               lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
482
    }
704 jermar 483
 
484
    cp0_entry_hi_write(hi_save.value);
569 jermar 485
}
598 jermar 486
 
618 jermar 487
/** Invalidate all not wired TLB entries. */
598 jermar 488
void tlb_invalidate_all(void)
489
{
599 jermar 490
    ipl_t ipl;
491
    entry_lo_t lo0, lo1;
704 jermar 492
    entry_hi_t hi_save;
598 jermar 493
    int i;
494
 
704 jermar 495
    hi_save.value = cp0_entry_hi_read();
599 jermar 496
    ipl = interrupts_disable();
598 jermar 497
 
618 jermar 498
    for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
598 jermar 499
        cp0_index_write(i);
599 jermar 500
        tlbr();
501
 
502
        lo0.value = cp0_entry_lo0_read();
503
        lo1.value = cp0_entry_lo1_read();
504
 
505
        lo0.v = 0;
506
        lo1.v = 0;
507
 
508
        cp0_entry_lo0_write(lo0.value);
509
        cp0_entry_lo1_write(lo1.value);
510
 
598 jermar 511
        tlbwi();
512
    }
599 jermar 513
 
514
    interrupts_restore(ipl);
704 jermar 515
    cp0_entry_hi_write(hi_save.value);
598 jermar 516
}
517
 
518
/** Invalidate all TLB entries belonging to specified address space.
519
 *
520
 * @param asid Address space identifier.
521
 */
522
void tlb_invalidate_asid(asid_t asid)
523
{
599 jermar 524
    ipl_t ipl;
525
    entry_lo_t lo0, lo1;
704 jermar 526
    entry_hi_t hi, hi_save;
598 jermar 527
    int i;
528
 
599 jermar 529
    ASSERT(asid != ASID_INVALID);
530
 
704 jermar 531
    hi_save.value = cp0_entry_hi_read();
599 jermar 532
    ipl = interrupts_disable();
533
 
598 jermar 534
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
535
        cp0_index_write(i);
536
        tlbr();
537
 
599 jermar 538
        hi.value = cp0_entry_hi_read();
539
 
598 jermar 540
        if (hi.asid == asid) {
599 jermar 541
            lo0.value = cp0_entry_lo0_read();
542
            lo1.value = cp0_entry_lo1_read();
543
 
544
            lo0.v = 0;
545
            lo1.v = 0;
546
 
547
            cp0_entry_lo0_write(lo0.value);
548
            cp0_entry_lo1_write(lo1.value);
549
 
598 jermar 550
            tlbwi();
551
        }
552
    }
599 jermar 553
 
554
    interrupts_restore(ipl);
704 jermar 555
    cp0_entry_hi_write(hi_save.value);
598 jermar 556
}
557
 
727 jermar 558
/** Invalidate TLB entries for specified page range belonging to specified address space.
598 jermar 559
 *
560
 * @param asid Address space identifier.
727 jermar 561
 * @param page First page whose TLB entry is to be invalidated.
562
 * @param cnt Number of entries to invalidate.
598 jermar 563
 */
727 jermar 564
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
598 jermar 565
{
727 jermar 566
    int i;
599 jermar 567
    ipl_t ipl;
568
    entry_lo_t lo0, lo1;
704 jermar 569
    entry_hi_t hi, hi_save;
598 jermar 570
    tlb_index_t index;
571
 
599 jermar 572
    ASSERT(asid != ASID_INVALID);
573
 
704 jermar 574
    hi_save.value = cp0_entry_hi_read();
599 jermar 575
    ipl = interrupts_disable();
576
 
983 palkovsky 577
    for (i = 0; i < cnt+1; i+=2) {
727 jermar 578
        hi.value = 0;
579
        prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
580
        cp0_entry_hi_write(hi.value);
599 jermar 581
 
727 jermar 582
        tlbp();
583
        index.value = cp0_index_read();
598 jermar 584
 
727 jermar 585
        if (!index.p) {
586
            /* Entry was found, index register contains valid index. */
587
            tlbr();
599 jermar 588
 
727 jermar 589
            lo0.value = cp0_entry_lo0_read();
590
            lo1.value = cp0_entry_lo1_read();
599 jermar 591
 
727 jermar 592
            lo0.v = 0;
593
            lo1.v = 0;
599 jermar 594
 
727 jermar 595
            cp0_entry_lo0_write(lo0.value);
596
            cp0_entry_lo1_write(lo1.value);
599 jermar 597
 
727 jermar 598
            tlbwi();
599
        }
598 jermar 600
    }
599 jermar 601
 
602
    interrupts_restore(ipl);
704 jermar 603
    cp0_entry_hi_write(hi_save.value);
598 jermar 604
}