Subversion Repositories HelenOS-historic

Rev

Rev 756 | Rev 765 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1 jermar 1
/*
319 jermar 2
 * Copyright (C) 2003-2004 Jakub Jermar
1 jermar 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
29
#include <arch/mm/tlb.h>
727 jermar 30
#include <mm/asid.h>
730 jermar 31
#include <genarch/mm/asid_fifo.h>
1 jermar 32
#include <mm/tlb.h>
391 jermar 33
#include <mm/page.h>
703 jermar 34
#include <mm/as.h>
1 jermar 35
#include <arch/cp0.h>
36
#include <panic.h>
37
#include <arch.h>
268 palkovsky 38
#include <symtab.h>
391 jermar 39
#include <synch/spinlock.h>
40
#include <print.h>
396 jermar 41
#include <debug.h>
268 palkovsky 42
 
391 jermar 43
static void tlb_refill_fail(struct exception_regdump *pstate);
44
static void tlb_invalid_fail(struct exception_regdump *pstate);
45
static void tlb_modified_fail(struct exception_regdump *pstate);
46
 
394 jermar 47
static pte_t *find_mapping_and_check(__address badvaddr);
399 jermar 48
 
396 jermar 49
static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn);
399 jermar 50
static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
394 jermar 51
 
391 jermar 52
/** Initialize TLB
53
 *
54
 * Initialize TLB.
55
 * Invalidate all entries and mark wired entries.
56
 */
569 jermar 57
void tlb_arch_init(void)
389 jermar 58
{
599 jermar 59
    int i;
60
 
730 jermar 61
    asid_fifo_init();
62
 
389 jermar 63
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
599 jermar 64
    cp0_entry_hi_write(0);
65
    cp0_entry_lo0_write(0);
66
    cp0_entry_lo1_write(0);
389 jermar 67
 
599 jermar 68
    /* Clear and initialize TLB. */
69
 
70
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
71
        cp0_index_write(i);
72
        tlbwi();
73
    }
612 jermar 74
 
598 jermar 75
 
389 jermar 76
    /*
77
     * The kernel is going to make use of some wired
391 jermar 78
     * entries (e.g. mapping kernel stacks in kseg3).
389 jermar 79
     */
80
    cp0_wired_write(TLB_WIRED);
81
}
82
 
391 jermar 83
/** Process TLB Refill Exception
84
 *
85
 * Process TLB Refill Exception.
86
 *
87
 * @param pstate Interrupted register context.
88
 */
317 palkovsky 89
void tlb_refill(struct exception_regdump *pstate)
1 jermar 90
{
396 jermar 91
    entry_lo_t lo;
399 jermar 92
    entry_hi_t hi; 
391 jermar 93
    __address badvaddr;
94
    pte_t *pte;
397 jermar 95
 
391 jermar 96
    badvaddr = cp0_badvaddr_read();
397 jermar 97
 
703 jermar 98
    spinlock_lock(&AS->lock);      
399 jermar 99
 
394 jermar 100
    pte = find_mapping_and_check(badvaddr);
101
    if (!pte)
391 jermar 102
        goto fail;
103
 
104
    /*
394 jermar 105
     * Record access to PTE.
391 jermar 106
     */
394 jermar 107
    pte->a = 1;
391 jermar 108
 
703 jermar 109
    prepare_entry_hi(&hi, AS->asid, badvaddr);
403 jermar 110
    prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
394 jermar 111
 
391 jermar 112
    /*
113
     * New entry is to be inserted into TLB
114
     */
399 jermar 115
    cp0_entry_hi_write(hi.value);
391 jermar 116
    if ((badvaddr/PAGE_SIZE) % 2 == 0) {
396 jermar 117
        cp0_entry_lo0_write(lo.value);
391 jermar 118
        cp0_entry_lo1_write(0);
119
    }
120
    else {
121
        cp0_entry_lo0_write(0);
396 jermar 122
        cp0_entry_lo1_write(lo.value);
391 jermar 123
    }
612 jermar 124
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
391 jermar 125
    tlbwr();
126
 
703 jermar 127
    spinlock_unlock(&AS->lock);
391 jermar 128
    return;
129
 
130
fail:
703 jermar 131
    spinlock_unlock(&AS->lock);
391 jermar 132
    tlb_refill_fail(pstate);
133
}
134
 
394 jermar 135
/** Process TLB Invalid Exception
136
 *
137
 * Process TLB Invalid Exception.
138
 *
139
 * @param pstate Interrupted register context.
140
 */
391 jermar 141
void tlb_invalid(struct exception_regdump *pstate)
142
{
396 jermar 143
    tlb_index_t index;
394 jermar 144
    __address badvaddr;
396 jermar 145
    entry_lo_t lo;
399 jermar 146
    entry_hi_t hi;
394 jermar 147
    pte_t *pte;
148
 
149
    badvaddr = cp0_badvaddr_read();
150
 
151
    /*
152
     * Locate the faulting entry in TLB.
153
     */
399 jermar 154
    hi.value = cp0_entry_hi_read();
155
    prepare_entry_hi(&hi, hi.asid, badvaddr);
156
    cp0_entry_hi_write(hi.value);
394 jermar 157
    tlbp();
396 jermar 158
    index.value = cp0_index_read();
394 jermar 159
 
703 jermar 160
    spinlock_lock(&AS->lock);  
394 jermar 161
 
162
    /*
163
     * Fail if the entry is not in TLB.
164
     */
396 jermar 165
    if (index.p) {
166
        printf("TLB entry not found.\n");
394 jermar 167
        goto fail;
396 jermar 168
    }
394 jermar 169
 
170
    pte = find_mapping_and_check(badvaddr);
171
    if (!pte)
172
        goto fail;
173
 
174
    /*
175
     * Read the faulting TLB entry.
176
     */
177
    tlbr();
178
 
179
    /*
180
     * Record access to PTE.
181
     */
182
    pte->a = 1;
183
 
403 jermar 184
    prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->lo.d, pte->lo.c, pte->lo.pfn);
394 jermar 185
 
186
    /*
187
     * The entry is to be updated in TLB.
188
     */
189
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
396 jermar 190
        cp0_entry_lo0_write(lo.value);
394 jermar 191
    else
396 jermar 192
        cp0_entry_lo1_write(lo.value);
612 jermar 193
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
394 jermar 194
    tlbwi();
195
 
703 jermar 196
    spinlock_unlock(&AS->lock);
394 jermar 197
    return;
198
 
199
fail:
703 jermar 200
    spinlock_unlock(&AS->lock);
391 jermar 201
    tlb_invalid_fail(pstate);
202
}
203
 
394 jermar 204
/** Process TLB Modified Exception
205
 *
206
 * Process TLB Modified Exception.
207
 *
208
 * @param pstate Interrupted register context.
209
 */
391 jermar 210
void tlb_modified(struct exception_regdump *pstate)
211
{
396 jermar 212
    tlb_index_t index;
394 jermar 213
    __address badvaddr;
396 jermar 214
    entry_lo_t lo;
399 jermar 215
    entry_hi_t hi;
394 jermar 216
    pte_t *pte;
217
 
218
    badvaddr = cp0_badvaddr_read();
219
 
220
    /*
221
     * Locate the faulting entry in TLB.
222
     */
399 jermar 223
    hi.value = cp0_entry_hi_read();
224
    prepare_entry_hi(&hi, hi.asid, badvaddr);
225
    cp0_entry_hi_write(hi.value);
394 jermar 226
    tlbp();
396 jermar 227
    index.value = cp0_index_read();
394 jermar 228
 
703 jermar 229
    spinlock_lock(&AS->lock);  
394 jermar 230
 
231
    /*
232
     * Fail if the entry is not in TLB.
233
     */
396 jermar 234
    if (index.p) {
235
        printf("TLB entry not found.\n");
394 jermar 236
        goto fail;
396 jermar 237
    }
394 jermar 238
 
239
    pte = find_mapping_and_check(badvaddr);
240
    if (!pte)
241
        goto fail;
242
 
243
    /*
244
     * Fail if the page is not writable.
245
     */
246
    if (!pte->w)
247
        goto fail;
248
 
249
    /*
250
     * Read the faulting TLB entry.
251
     */
252
    tlbr();
253
 
254
    /*
255
     * Record access and write to PTE.
256
     */
257
    pte->a = 1;
403 jermar 258
    pte->lo.d = 1;
394 jermar 259
 
403 jermar 260
    prepare_entry_lo(&lo, pte->lo.g, pte->lo.v, pte->w, pte->lo.c, pte->lo.pfn);
394 jermar 261
 
262
    /*
263
     * The entry is to be updated in TLB.
264
     */
265
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
396 jermar 266
        cp0_entry_lo0_write(lo.value);
394 jermar 267
    else
396 jermar 268
        cp0_entry_lo1_write(lo.value);
612 jermar 269
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
394 jermar 270
    tlbwi();
271
 
703 jermar 272
    spinlock_unlock(&AS->lock);
394 jermar 273
    return;
274
 
275
fail:
703 jermar 276
    spinlock_unlock(&AS->lock);
391 jermar 277
    tlb_modified_fail(pstate);
278
}
279
 
280
void tlb_refill_fail(struct exception_regdump *pstate)
281
{
324 palkovsky 282
    char *symbol = "";
283
    char *sym2 = "";
284
 
332 palkovsky 285
    char *s = get_symtab_entry(pstate->epc);
286
    if (s)
287
        symbol = s;
288
    s = get_symtab_entry(pstate->ra);
289
    if (s)
290
        sym2 = s;
391 jermar 291
    panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), pstate->epc, symbol, sym2);
1 jermar 292
}
293
 
391 jermar 294
 
295
void tlb_invalid_fail(struct exception_regdump *pstate)
1 jermar 296
{
268 palkovsky 297
    char *symbol = "";
298
 
332 palkovsky 299
    char *s = get_symtab_entry(pstate->epc);
300
    if (s)
301
        symbol = s;
394 jermar 302
    panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
1 jermar 303
}
304
 
391 jermar 305
void tlb_modified_fail(struct exception_regdump *pstate)
389 jermar 306
{
307
    char *symbol = "";
308
 
309
    char *s = get_symtab_entry(pstate->epc);
310
    if (s)
311
        symbol = s;
394 jermar 312
    panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), pstate->epc, symbol);
389 jermar 313
}
314
 
394 jermar 315
/** Try to find PTE for faulting address
316
 *
317
 * Try to find PTE for faulting address.
703 jermar 318
 * The AS->lock must be held on entry to this function.
394 jermar 319
 *
320
 * @param badvaddr Faulting virtual address.
321
 *
322
 * @return PTE on success, NULL otherwise.
323
 */
324
pte_t *find_mapping_and_check(__address badvaddr)
325
{
396 jermar 326
    entry_hi_t hi;
394 jermar 327
    pte_t *pte;
328
 
396 jermar 329
    hi.value = cp0_entry_hi_read();
394 jermar 330
 
331
    /*
332
     * Handler cannot succeed if the ASIDs don't match.
333
     */
703 jermar 334
    if (hi.asid != AS->asid) {
335
        printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
394 jermar 336
        return NULL;
396 jermar 337
    }
703 jermar 338
 
394 jermar 339
    /*
703 jermar 340
     * Check if the mapping exists in page tables.
341
     */
756 jermar 342
    pte = page_mapping_find(AS, badvaddr);
703 jermar 343
    if (pte && pte->lo.v) {
344
        /*
345
         * Mapping found in page tables.
346
         * Immediately succeed.
347
         */
348
        return pte;
349
    } else {
350
        /*
351
         * Mapping not found in page tables.
352
         * Resort to higher-level page fault handler.
353
         */
354
        if (as_page_fault(badvaddr)) {
355
            /*
356
             * The higher-level page fault handler succeeded,
357
             * The mapping ought to be in place.
358
             */
756 jermar 359
            pte = page_mapping_find(AS, badvaddr);
703 jermar 360
            ASSERT(pte && pte->lo.v);
361
            return pte;
362
        }
363
    }
364
 
365
    /*
394 jermar 366
     * Handler cannot succeed if badvaddr has no mapping.
367
     */
396 jermar 368
    if (!pte) {
764 palkovsky 369
        printf("No such mapping: %P.\n", badvaddr);
394 jermar 370
        return NULL;
396 jermar 371
    }
394 jermar 372
 
373
    /*
374
     * Handler cannot succeed if the mapping is marked as invalid.
375
     */
403 jermar 376
    if (!pte->lo.v) {
396 jermar 377
        printf("Invalid mapping.\n");
394 jermar 378
        return NULL;
396 jermar 379
    }
394 jermar 380
 
381
    return pte;
382
}
383
 
396 jermar 384
void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, int c, __address pfn)
394 jermar 385
{
399 jermar 386
    lo->value = 0;
394 jermar 387
    lo->g = g;
388
    lo->v = v;
389
    lo->d = d;
390
    lo->c = c;
391
    lo->pfn = pfn;
392
}
399 jermar 393
 
394
void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
395
{
396
    hi->value = (((addr/PAGE_SIZE)/2)*PAGE_SIZE*2);
397
    hi->asid = asid;
398
}
569 jermar 399
 
594 jermar 400
/** Print contents of TLB. */
569 jermar 401
void tlb_print(void)
402
{
612 jermar 403
    page_mask_t mask;
594 jermar 404
    entry_lo_t lo0, lo1;
704 jermar 405
    entry_hi_t hi, hi_save;
594 jermar 406
    int i;
407
 
704 jermar 408
    hi_save.value = cp0_entry_hi_read();
409
 
594 jermar 410
    printf("TLB:\n");
411
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
412
        cp0_index_write(i);
413
        tlbr();
414
 
612 jermar 415
        mask.value = cp0_pagemask_read();
594 jermar 416
        hi.value = cp0_entry_hi_read();
417
        lo0.value = cp0_entry_lo0_read();
418
        lo1.value = cp0_entry_lo1_read();
419
 
612 jermar 420
        printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
421
               "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
422
               i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
594 jermar 423
               lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
424
    }
704 jermar 425
 
426
    cp0_entry_hi_write(hi_save.value);
569 jermar 427
}
598 jermar 428
 
618 jermar 429
/** Invalidate all not wired TLB entries. */
598 jermar 430
void tlb_invalidate_all(void)
431
{
599 jermar 432
    ipl_t ipl;
433
    entry_lo_t lo0, lo1;
704 jermar 434
    entry_hi_t hi_save;
598 jermar 435
    int i;
436
 
704 jermar 437
    hi_save.value = cp0_entry_hi_read();
599 jermar 438
    ipl = interrupts_disable();
598 jermar 439
 
618 jermar 440
    for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
598 jermar 441
        cp0_index_write(i);
599 jermar 442
        tlbr();
443
 
444
        lo0.value = cp0_entry_lo0_read();
445
        lo1.value = cp0_entry_lo1_read();
446
 
447
        lo0.v = 0;
448
        lo1.v = 0;
449
 
450
        cp0_entry_lo0_write(lo0.value);
451
        cp0_entry_lo1_write(lo1.value);
452
 
598 jermar 453
        tlbwi();
454
    }
599 jermar 455
 
456
    interrupts_restore(ipl);
704 jermar 457
    cp0_entry_hi_write(hi_save.value);
598 jermar 458
}
459
 
460
/** Invalidate all TLB entries belonging to specified address space.
461
 *
462
 * @param asid Address space identifier.
463
 */
464
void tlb_invalidate_asid(asid_t asid)
465
{
599 jermar 466
    ipl_t ipl;
467
    entry_lo_t lo0, lo1;
704 jermar 468
    entry_hi_t hi, hi_save;
598 jermar 469
    int i;
470
 
599 jermar 471
    ASSERT(asid != ASID_INVALID);
472
 
704 jermar 473
    hi_save.value = cp0_entry_hi_read();
599 jermar 474
    ipl = interrupts_disable();
475
 
598 jermar 476
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
477
        cp0_index_write(i);
478
        tlbr();
479
 
599 jermar 480
        hi.value = cp0_entry_hi_read();
481
 
598 jermar 482
        if (hi.asid == asid) {
599 jermar 483
            lo0.value = cp0_entry_lo0_read();
484
            lo1.value = cp0_entry_lo1_read();
485
 
486
            lo0.v = 0;
487
            lo1.v = 0;
488
 
489
            cp0_entry_lo0_write(lo0.value);
490
            cp0_entry_lo1_write(lo1.value);
491
 
598 jermar 492
            tlbwi();
493
        }
494
    }
599 jermar 495
 
496
    interrupts_restore(ipl);
704 jermar 497
    cp0_entry_hi_write(hi_save.value);
598 jermar 498
}
499
 
727 jermar 500
/** Invalidate TLB entries for specified page range belonging to specified address space.
598 jermar 501
 *
502
 * @param asid Address space identifier.
727 jermar 503
 * @param page First page whose TLB entry is to be invalidated.
504
 * @param cnt Number of entries to invalidate.
598 jermar 505
 */
727 jermar 506
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
598 jermar 507
{
727 jermar 508
    int i;
599 jermar 509
    ipl_t ipl;
510
    entry_lo_t lo0, lo1;
704 jermar 511
    entry_hi_t hi, hi_save;
598 jermar 512
    tlb_index_t index;
513
 
599 jermar 514
    ASSERT(asid != ASID_INVALID);
515
 
704 jermar 516
    hi_save.value = cp0_entry_hi_read();
599 jermar 517
    ipl = interrupts_disable();
518
 
727 jermar 519
    for (i = 0; i < cnt; i++) {
520
        hi.value = 0;
521
        prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
522
        cp0_entry_hi_write(hi.value);
599 jermar 523
 
727 jermar 524
        tlbp();
525
        index.value = cp0_index_read();
598 jermar 526
 
727 jermar 527
        if (!index.p) {
528
            /* Entry was found, index register contains valid index. */
529
            tlbr();
599 jermar 530
 
727 jermar 531
            lo0.value = cp0_entry_lo0_read();
532
            lo1.value = cp0_entry_lo1_read();
599 jermar 533
 
727 jermar 534
            lo0.v = 0;
535
            lo1.v = 0;
599 jermar 536
 
727 jermar 537
            cp0_entry_lo0_write(lo0.value);
538
            cp0_entry_lo1_write(lo1.value);
599 jermar 539
 
727 jermar 540
            tlbwi();
541
        }
598 jermar 542
    }
599 jermar 543
 
544
    interrupts_restore(ipl);
704 jermar 545
    cp0_entry_hi_write(hi_save.value);
598 jermar 546
}