Subversion Repositories HelenOS

Rev

Rev 831 | Rev 983 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1 jermar 1
/*
319 jermar 2
 * Copyright (C) 2003-2004 Jakub Jermar
1 jermar 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
29
#include <arch/mm/tlb.h>
727 jermar 30
#include <mm/asid.h>
1 jermar 31
#include <mm/tlb.h>
391 jermar 32
#include <mm/page.h>
703 jermar 33
#include <mm/as.h>
1 jermar 34
#include <arch/cp0.h>
35
#include <panic.h>
36
#include <arch.h>
268 palkovsky 37
#include <symtab.h>
391 jermar 38
#include <synch/spinlock.h>
39
#include <print.h>
396 jermar 40
#include <debug.h>
268 palkovsky 41
 
958 jermar 42
static void tlb_refill_fail(istate_t *istate);
43
static void tlb_invalid_fail(istate_t *istate);
44
static void tlb_modified_fail(istate_t *istate);
391 jermar 45
 
394 jermar 46
static pte_t *find_mapping_and_check(__address badvaddr);
399 jermar 47
 
831 jermar 48
static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
399 jermar 49
static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
394 jermar 50
 
391 jermar 51
/** Initialize TLB
52
 *
53
 * Initialize TLB.
54
 * Invalidate all entries and mark wired entries.
55
 */
569 jermar 56
void tlb_arch_init(void)
389 jermar 57
{
599 jermar 58
    int i;
59
 
389 jermar 60
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
599 jermar 61
    cp0_entry_hi_write(0);
62
    cp0_entry_lo0_write(0);
63
    cp0_entry_lo1_write(0);
389 jermar 64
 
599 jermar 65
    /* Clear and initialize TLB. */
66
 
67
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
68
        cp0_index_write(i);
69
        tlbwi();
70
    }
612 jermar 71
 
598 jermar 72
 
389 jermar 73
    /*
74
     * The kernel is going to make use of some wired
391 jermar 75
     * entries (e.g. mapping kernel stacks in kseg3).
389 jermar 76
     */
77
    cp0_wired_write(TLB_WIRED);
78
}
79
 
391 jermar 80
/** Process TLB Refill Exception
81
 *
82
 * Process TLB Refill Exception.
83
 *
958 jermar 84
 * @param istate Interrupted register context.
391 jermar 85
 */
958 jermar 86
void tlb_refill(istate_t *istate)
1 jermar 87
{
396 jermar 88
    entry_lo_t lo;
399 jermar 89
    entry_hi_t hi; 
391 jermar 90
    __address badvaddr;
91
    pte_t *pte;
397 jermar 92
 
391 jermar 93
    badvaddr = cp0_badvaddr_read();
397 jermar 94
 
703 jermar 95
    spinlock_lock(&AS->lock);      
399 jermar 96
 
394 jermar 97
    pte = find_mapping_and_check(badvaddr);
98
    if (!pte)
391 jermar 99
        goto fail;
100
 
101
    /*
394 jermar 102
     * Record access to PTE.
391 jermar 103
     */
394 jermar 104
    pte->a = 1;
391 jermar 105
 
703 jermar 106
    prepare_entry_hi(&hi, AS->asid, badvaddr);
831 jermar 107
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
394 jermar 108
 
391 jermar 109
    /*
110
     * New entry is to be inserted into TLB
111
     */
399 jermar 112
    cp0_entry_hi_write(hi.value);
391 jermar 113
    if ((badvaddr/PAGE_SIZE) % 2 == 0) {
396 jermar 114
        cp0_entry_lo0_write(lo.value);
391 jermar 115
        cp0_entry_lo1_write(0);
116
    }
117
    else {
118
        cp0_entry_lo0_write(0);
396 jermar 119
        cp0_entry_lo1_write(lo.value);
391 jermar 120
    }
612 jermar 121
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
391 jermar 122
    tlbwr();
123
 
703 jermar 124
    spinlock_unlock(&AS->lock);
391 jermar 125
    return;
126
 
127
fail:
703 jermar 128
    spinlock_unlock(&AS->lock);
958 jermar 129
    tlb_refill_fail(istate);
391 jermar 130
}
131
 
394 jermar 132
/** Process TLB Invalid Exception
133
 *
134
 * Process TLB Invalid Exception.
135
 *
958 jermar 136
 * @param istate Interrupted register context.
394 jermar 137
 */
958 jermar 138
void tlb_invalid(istate_t *istate)
391 jermar 139
{
396 jermar 140
    tlb_index_t index;
394 jermar 141
    __address badvaddr;
396 jermar 142
    entry_lo_t lo;
399 jermar 143
    entry_hi_t hi;
394 jermar 144
    pte_t *pte;
145
 
146
    badvaddr = cp0_badvaddr_read();
147
 
148
    /*
149
     * Locate the faulting entry in TLB.
150
     */
399 jermar 151
    hi.value = cp0_entry_hi_read();
152
    prepare_entry_hi(&hi, hi.asid, badvaddr);
153
    cp0_entry_hi_write(hi.value);
394 jermar 154
    tlbp();
396 jermar 155
    index.value = cp0_index_read();
394 jermar 156
 
703 jermar 157
    spinlock_lock(&AS->lock);  
394 jermar 158
 
159
    /*
160
     * Fail if the entry is not in TLB.
161
     */
396 jermar 162
    if (index.p) {
163
        printf("TLB entry not found.\n");
394 jermar 164
        goto fail;
396 jermar 165
    }
394 jermar 166
 
167
    pte = find_mapping_and_check(badvaddr);
168
    if (!pte)
169
        goto fail;
170
 
171
    /*
172
     * Read the faulting TLB entry.
173
     */
174
    tlbr();
175
 
176
    /*
177
     * Record access to PTE.
178
     */
179
    pte->a = 1;
180
 
831 jermar 181
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
394 jermar 182
 
183
    /*
184
     * The entry is to be updated in TLB.
185
     */
186
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
396 jermar 187
        cp0_entry_lo0_write(lo.value);
394 jermar 188
    else
396 jermar 189
        cp0_entry_lo1_write(lo.value);
612 jermar 190
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
394 jermar 191
    tlbwi();
192
 
703 jermar 193
    spinlock_unlock(&AS->lock);
394 jermar 194
    return;
195
 
196
fail:
703 jermar 197
    spinlock_unlock(&AS->lock);
958 jermar 198
    tlb_invalid_fail(istate);
391 jermar 199
}
200
 
394 jermar 201
/** Process TLB Modified Exception
202
 *
203
 * Process TLB Modified Exception.
204
 *
958 jermar 205
 * @param istate Interrupted register context.
394 jermar 206
 */
958 jermar 207
void tlb_modified(istate_t *istate)
391 jermar 208
{
396 jermar 209
    tlb_index_t index;
394 jermar 210
    __address badvaddr;
396 jermar 211
    entry_lo_t lo;
399 jermar 212
    entry_hi_t hi;
394 jermar 213
    pte_t *pte;
214
 
215
    badvaddr = cp0_badvaddr_read();
216
 
217
    /*
218
     * Locate the faulting entry in TLB.
219
     */
399 jermar 220
    hi.value = cp0_entry_hi_read();
221
    prepare_entry_hi(&hi, hi.asid, badvaddr);
222
    cp0_entry_hi_write(hi.value);
394 jermar 223
    tlbp();
396 jermar 224
    index.value = cp0_index_read();
394 jermar 225
 
703 jermar 226
    spinlock_lock(&AS->lock);  
394 jermar 227
 
228
    /*
229
     * Fail if the entry is not in TLB.
230
     */
396 jermar 231
    if (index.p) {
232
        printf("TLB entry not found.\n");
394 jermar 233
        goto fail;
396 jermar 234
    }
394 jermar 235
 
236
    pte = find_mapping_and_check(badvaddr);
237
    if (!pte)
238
        goto fail;
239
 
240
    /*
241
     * Fail if the page is not writable.
242
     */
243
    if (!pte->w)
244
        goto fail;
245
 
246
    /*
247
     * Read the faulting TLB entry.
248
     */
249
    tlbr();
250
 
251
    /*
252
     * Record access and write to PTE.
253
     */
254
    pte->a = 1;
831 jermar 255
    pte->d = 1;
394 jermar 256
 
831 jermar 257
    prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
394 jermar 258
 
259
    /*
260
     * The entry is to be updated in TLB.
261
     */
262
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
396 jermar 263
        cp0_entry_lo0_write(lo.value);
394 jermar 264
    else
396 jermar 265
        cp0_entry_lo1_write(lo.value);
612 jermar 266
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
394 jermar 267
    tlbwi();
268
 
703 jermar 269
    spinlock_unlock(&AS->lock);
394 jermar 270
    return;
271
 
272
fail:
703 jermar 273
    spinlock_unlock(&AS->lock);
958 jermar 274
    tlb_modified_fail(istate);
391 jermar 275
}
276
 
958 jermar 277
void tlb_refill_fail(istate_t *istate)
391 jermar 278
{
324 palkovsky 279
    char *symbol = "";
280
    char *sym2 = "";
281
 
958 jermar 282
    char *s = get_symtab_entry(istate->epc);
332 palkovsky 283
    if (s)
284
        symbol = s;
958 jermar 285
    s = get_symtab_entry(istate->ra);
332 palkovsky 286
    if (s)
287
        sym2 = s;
958 jermar 288
    panic("%X: TLB Refill Exception at %X(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
1 jermar 289
}
290
 
391 jermar 291
 
958 jermar 292
void tlb_invalid_fail(istate_t *istate)
1 jermar 293
{
268 palkovsky 294
    char *symbol = "";
295
 
958 jermar 296
    char *s = get_symtab_entry(istate->epc);
332 palkovsky 297
    if (s)
298
        symbol = s;
958 jermar 299
    panic("%X: TLB Invalid Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
1 jermar 300
}
301
 
958 jermar 302
void tlb_modified_fail(istate_t *istate)
389 jermar 303
{
304
    char *symbol = "";
305
 
958 jermar 306
    char *s = get_symtab_entry(istate->epc);
389 jermar 307
    if (s)
308
        symbol = s;
958 jermar 309
    panic("%X: TLB Modified Exception at %X(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
389 jermar 310
}
311
 
394 jermar 312
/** Try to find PTE for faulting address
313
 *
314
 * Try to find PTE for faulting address.
703 jermar 315
 * The AS->lock must be held on entry to this function.
394 jermar 316
 *
317
 * @param badvaddr Faulting virtual address.
318
 *
319
 * @return PTE on success, NULL otherwise.
320
 */
321
pte_t *find_mapping_and_check(__address badvaddr)
322
{
396 jermar 323
    entry_hi_t hi;
394 jermar 324
    pte_t *pte;
325
 
396 jermar 326
    hi.value = cp0_entry_hi_read();
394 jermar 327
 
328
    /*
329
     * Handler cannot succeed if the ASIDs don't match.
330
     */
703 jermar 331
    if (hi.asid != AS->asid) {
332
        printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
394 jermar 333
        return NULL;
396 jermar 334
    }
703 jermar 335
 
394 jermar 336
    /*
703 jermar 337
     * Check if the mapping exists in page tables.
338
     */
756 jermar 339
    pte = page_mapping_find(AS, badvaddr);
831 jermar 340
    if (pte && pte->p) {
703 jermar 341
        /*
342
         * Mapping found in page tables.
343
         * Immediately succeed.
344
         */
345
        return pte;
346
    } else {
347
        /*
348
         * Mapping not found in page tables.
349
         * Resort to higher-level page fault handler.
350
         */
351
        if (as_page_fault(badvaddr)) {
352
            /*
353
             * The higher-level page fault handler succeeded,
354
             * The mapping ought to be in place.
355
             */
756 jermar 356
            pte = page_mapping_find(AS, badvaddr);
831 jermar 357
            ASSERT(pte && pte->p);
703 jermar 358
            return pte;
359
        }
360
    }
361
 
362
    /*
394 jermar 363
     * Handler cannot succeed if badvaddr has no mapping.
364
     */
396 jermar 365
    if (!pte) {
765 jermar 366
        printf("No such mapping.\n");
394 jermar 367
        return NULL;
396 jermar 368
    }
394 jermar 369
 
370
    /*
371
     * Handler cannot succeed if the mapping is marked as invalid.
372
     */
831 jermar 373
    if (!pte->p) {
396 jermar 374
        printf("Invalid mapping.\n");
394 jermar 375
        return NULL;
396 jermar 376
    }
394 jermar 377
 
378
    return pte;
379
}
380
 
831 jermar 381
void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
394 jermar 382
{
399 jermar 383
    lo->value = 0;
394 jermar 384
    lo->g = g;
385
    lo->v = v;
386
    lo->d = d;
831 jermar 387
    lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
394 jermar 388
    lo->pfn = pfn;
389
}
399 jermar 390
 
391
void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
392
{
393
    hi->value = (((addr/PAGE_SIZE)/2)*PAGE_SIZE*2);
394
    hi->asid = asid;
395
}
569 jermar 396
 
594 jermar 397
/** Print contents of TLB. */
569 jermar 398
void tlb_print(void)
399
{
612 jermar 400
    page_mask_t mask;
594 jermar 401
    entry_lo_t lo0, lo1;
704 jermar 402
    entry_hi_t hi, hi_save;
594 jermar 403
    int i;
404
 
704 jermar 405
    hi_save.value = cp0_entry_hi_read();
406
 
594 jermar 407
    printf("TLB:\n");
408
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
409
        cp0_index_write(i);
410
        tlbr();
411
 
612 jermar 412
        mask.value = cp0_pagemask_read();
594 jermar 413
        hi.value = cp0_entry_hi_read();
414
        lo0.value = cp0_entry_lo0_read();
415
        lo1.value = cp0_entry_lo1_read();
416
 
612 jermar 417
        printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%B, pfn[0]=%d\n"
418
               "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%B, pfn[1]=%d\n",
419
               i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
594 jermar 420
               lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
421
    }
704 jermar 422
 
423
    cp0_entry_hi_write(hi_save.value);
569 jermar 424
}
598 jermar 425
 
618 jermar 426
/** Invalidate all not wired TLB entries. */
598 jermar 427
void tlb_invalidate_all(void)
428
{
599 jermar 429
    ipl_t ipl;
430
    entry_lo_t lo0, lo1;
704 jermar 431
    entry_hi_t hi_save;
598 jermar 432
    int i;
433
 
704 jermar 434
    hi_save.value = cp0_entry_hi_read();
599 jermar 435
    ipl = interrupts_disable();
598 jermar 436
 
618 jermar 437
    for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
598 jermar 438
        cp0_index_write(i);
599 jermar 439
        tlbr();
440
 
441
        lo0.value = cp0_entry_lo0_read();
442
        lo1.value = cp0_entry_lo1_read();
443
 
444
        lo0.v = 0;
445
        lo1.v = 0;
446
 
447
        cp0_entry_lo0_write(lo0.value);
448
        cp0_entry_lo1_write(lo1.value);
449
 
598 jermar 450
        tlbwi();
451
    }
599 jermar 452
 
453
    interrupts_restore(ipl);
704 jermar 454
    cp0_entry_hi_write(hi_save.value);
598 jermar 455
}
456
 
457
/** Invalidate all TLB entries belonging to specified address space.
458
 *
459
 * @param asid Address space identifier.
460
 */
461
void tlb_invalidate_asid(asid_t asid)
462
{
599 jermar 463
    ipl_t ipl;
464
    entry_lo_t lo0, lo1;
704 jermar 465
    entry_hi_t hi, hi_save;
598 jermar 466
    int i;
467
 
599 jermar 468
    ASSERT(asid != ASID_INVALID);
469
 
704 jermar 470
    hi_save.value = cp0_entry_hi_read();
599 jermar 471
    ipl = interrupts_disable();
472
 
598 jermar 473
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
474
        cp0_index_write(i);
475
        tlbr();
476
 
599 jermar 477
        hi.value = cp0_entry_hi_read();
478
 
598 jermar 479
        if (hi.asid == asid) {
599 jermar 480
            lo0.value = cp0_entry_lo0_read();
481
            lo1.value = cp0_entry_lo1_read();
482
 
483
            lo0.v = 0;
484
            lo1.v = 0;
485
 
486
            cp0_entry_lo0_write(lo0.value);
487
            cp0_entry_lo1_write(lo1.value);
488
 
598 jermar 489
            tlbwi();
490
        }
491
    }
599 jermar 492
 
493
    interrupts_restore(ipl);
704 jermar 494
    cp0_entry_hi_write(hi_save.value);
598 jermar 495
}
496
 
727 jermar 497
/** Invalidate TLB entries for specified page range belonging to specified address space.
598 jermar 498
 *
499
 * @param asid Address space identifier.
727 jermar 500
 * @param page First page whose TLB entry is to be invalidated.
501
 * @param cnt Number of entries to invalidate.
598 jermar 502
 */
727 jermar 503
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
598 jermar 504
{
727 jermar 505
    int i;
599 jermar 506
    ipl_t ipl;
507
    entry_lo_t lo0, lo1;
704 jermar 508
    entry_hi_t hi, hi_save;
598 jermar 509
    tlb_index_t index;
510
 
599 jermar 511
    ASSERT(asid != ASID_INVALID);
512
 
704 jermar 513
    hi_save.value = cp0_entry_hi_read();
599 jermar 514
    ipl = interrupts_disable();
515
 
727 jermar 516
    for (i = 0; i < cnt; i++) {
517
        hi.value = 0;
518
        prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
519
        cp0_entry_hi_write(hi.value);
599 jermar 520
 
727 jermar 521
        tlbp();
522
        index.value = cp0_index_read();
598 jermar 523
 
727 jermar 524
        if (!index.p) {
525
            /* Entry was found, index register contains valid index. */
526
            tlbr();
599 jermar 527
 
727 jermar 528
            lo0.value = cp0_entry_lo0_read();
529
            lo1.value = cp0_entry_lo1_read();
599 jermar 530
 
727 jermar 531
            lo0.v = 0;
532
            lo1.v = 0;
599 jermar 533
 
727 jermar 534
            cp0_entry_lo0_write(lo0.value);
535
            cp0_entry_lo1_write(lo1.value);
599 jermar 536
 
727 jermar 537
            tlbwi();
538
        }
598 jermar 539
    }
599 jermar 540
 
541
    interrupts_restore(ipl);
704 jermar 542
    cp0_entry_hi_write(hi_save.value);
598 jermar 543
}