Subversion Repositories HelenOS

Rev

Rev 2787 | Rev 3597 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 2787 Rev 3425
Line 45... Line 45...
45
#include <print.h>
45
#include <print.h>
46
#include <debug.h>
46
#include <debug.h>
47
#include <align.h>
47
#include <align.h>
48
#include <interrupt.h>
48
#include <interrupt.h>
49
 
49
 
50
static void tlb_refill_fail(istate_t *istate);
50
static void tlb_refill_fail(istate_t *);
51
static void tlb_invalid_fail(istate_t *istate);
51
static void tlb_invalid_fail(istate_t *);
52
static void tlb_modified_fail(istate_t *istate);
52
static void tlb_modified_fail(istate_t *);
53
 
53
 
54
static pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc);
54
static pte_t *find_mapping_and_check(uintptr_t, int, istate_t *, int *);
55
 
55
 
56
static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn);
-
 
57
static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr);
-
 
58
 
-
 
59
/** Initialize TLB
56
/** Initialize TLB.
60
 *
57
 *
61
 * Initialize TLB.
-
 
62
 * Invalidate all entries and mark wired entries.
58
 * Invalidate all entries and mark wired entries.
63
 */
59
 */
64
void tlb_arch_init(void)
60
void tlb_arch_init(void)
65
{
61
{
66
    int i;
62
    int i;
Line 74... Line 70...
74
   
70
   
75
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
71
    for (i = 0; i < TLB_ENTRY_COUNT; i++) {
76
        cp0_index_write(i);
72
        cp0_index_write(i);
77
        tlbwi();
73
        tlbwi();
78
    }
74
    }
79
 
-
 
80
       
75
       
81
    /*
76
    /*
82
     * The kernel is going to make use of some wired
77
     * The kernel is going to make use of some wired
83
     * entries (e.g. mapping kernel stacks in kseg3).
78
     * entries (e.g. mapping kernel stacks in kseg3).
84
     */
79
     */
85
    cp0_wired_write(TLB_WIRED);
80
    cp0_wired_write(TLB_WIRED);
86
}
81
}
87
 
82
 
88
/** Process TLB Refill Exception
83
/** Process TLB Refill Exception.
89
 *
-
 
90
 * Process TLB Refill Exception.
-
 
91
 *
84
 *
92
 * @param istate Interrupted register context.
85
 * @param istate    Interrupted register context.
93
 */
86
 */
94
void tlb_refill(istate_t *istate)
87
void tlb_refill(istate_t *istate)
95
{
88
{
96
    entry_lo_t lo;
89
    entry_lo_t lo;
97
    entry_hi_t hi;
90
    entry_hi_t hi;
Line 129... Line 122...
129
    /*
122
    /*
130
     * Record access to PTE.
123
     * Record access to PTE.
131
     */
124
     */
132
    pte->a = 1;
125
    pte->a = 1;
133
 
126
 
134
    prepare_entry_hi(&hi, asid, badvaddr);
127
    tlb_prepare_entry_hi(&hi, asid, badvaddr);
135
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
128
    tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
-
 
129
        pte->pfn);
136
 
130
 
137
    /*
131
    /*
138
     * New entry is to be inserted into TLB
132
     * New entry is to be inserted into TLB
139
     */
133
     */
140
    cp0_entry_hi_write(hi.value);
134
    cp0_entry_hi_write(hi.value);
141
    if ((badvaddr/PAGE_SIZE) % 2 == 0) {
135
    if ((badvaddr / PAGE_SIZE) % 2 == 0) {
142
        cp0_entry_lo0_write(lo.value);
136
        cp0_entry_lo0_write(lo.value);
143
        cp0_entry_lo1_write(0);
137
        cp0_entry_lo1_write(0);
144
    }
138
    }
145
    else {
139
    else {
146
        cp0_entry_lo0_write(0);
140
        cp0_entry_lo0_write(0);
Line 155... Line 149...
155
fail:
149
fail:
156
    page_table_unlock(AS, true);
150
    page_table_unlock(AS, true);
157
    tlb_refill_fail(istate);
151
    tlb_refill_fail(istate);
158
}
152
}
159
 
153
 
160
/** Process TLB Invalid Exception
154
/** Process TLB Invalid Exception.
161
 *
155
 *
162
 * Process TLB Invalid Exception.
-
 
163
 *
-
 
164
 * @param istate Interrupted register context.
156
 * @param istate    Interrupted register context.
165
 */
157
 */
166
void tlb_invalid(istate_t *istate)
158
void tlb_invalid(istate_t *istate)
167
{
159
{
168
    tlb_index_t index;
160
    tlb_index_t index;
169
    uintptr_t badvaddr;
161
    uintptr_t badvaddr;
Line 176... Line 168...
176
 
168
 
177
    /*
169
    /*
178
     * Locate the faulting entry in TLB.
170
     * Locate the faulting entry in TLB.
179
     */
171
     */
180
    hi.value = cp0_entry_hi_read();
172
    hi.value = cp0_entry_hi_read();
181
    prepare_entry_hi(&hi, hi.asid, badvaddr);
173
    tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
182
    cp0_entry_hi_write(hi.value);
174
    cp0_entry_hi_write(hi.value);
183
    tlbp();
175
    tlbp();
184
    index.value = cp0_index_read();
176
    index.value = cp0_index_read();
185
 
177
 
186
    page_table_lock(AS, true); 
178
    page_table_lock(AS, true); 
Line 219... Line 211...
219
    /*
211
    /*
220
     * Record access to PTE.
212
     * Record access to PTE.
221
     */
213
     */
222
    pte->a = 1;
214
    pte->a = 1;
223
 
215
 
224
    prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
216
    tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable,
-
 
217
        pte->pfn);
225
 
218
 
226
    /*
219
    /*
227
     * The entry is to be updated in TLB.
220
     * The entry is to be updated in TLB.
228
     */
221
     */
229
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
222
    if ((badvaddr / PAGE_SIZE) % 2 == 0)
230
        cp0_entry_lo0_write(lo.value);
223
        cp0_entry_lo0_write(lo.value);
231
    else
224
    else
232
        cp0_entry_lo1_write(lo.value);
225
        cp0_entry_lo1_write(lo.value);
233
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
226
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
234
    tlbwi();
227
    tlbwi();
Line 239... Line 232...
239
fail:
232
fail:
240
    page_table_unlock(AS, true);
233
    page_table_unlock(AS, true);
241
    tlb_invalid_fail(istate);
234
    tlb_invalid_fail(istate);
242
}
235
}
243
 
236
 
244
/** Process TLB Modified Exception
237
/** Process TLB Modified Exception.
245
 *
-
 
246
 * Process TLB Modified Exception.
-
 
247
 *
238
 *
248
 * @param istate Interrupted register context.
239
 * @param istate    Interrupted register context.
249
 */
240
 */
250
void tlb_modified(istate_t *istate)
241
void tlb_modified(istate_t *istate)
251
{
242
{
252
    tlb_index_t index;
243
    tlb_index_t index;
253
    uintptr_t badvaddr;
244
    uintptr_t badvaddr;
Line 260... Line 251...
260
 
251
 
261
    /*
252
    /*
262
     * Locate the faulting entry in TLB.
253
     * Locate the faulting entry in TLB.
263
     */
254
     */
264
    hi.value = cp0_entry_hi_read();
255
    hi.value = cp0_entry_hi_read();
265
    prepare_entry_hi(&hi, hi.asid, badvaddr);
256
    tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
266
    cp0_entry_hi_write(hi.value);
257
    cp0_entry_hi_write(hi.value);
267
    tlbp();
258
    tlbp();
268
    index.value = cp0_index_read();
259
    index.value = cp0_index_read();
269
 
260
 
270
    page_table_lock(AS, true); 
261
    page_table_lock(AS, true); 
Line 294... Line 285...
294
            panic("unexpected pfrc (%d)\n", pfrc);
285
            panic("unexpected pfrc (%d)\n", pfrc);
295
        }
286
        }
296
    }
287
    }
297
 
288
 
298
    /*
289
    /*
299
     * Fail if the page is not writable.
-
 
300
     */
-
 
301
    if (!pte->w)
-
 
302
        goto fail;
-
 
303
 
-
 
304
    /*
-
 
305
     * Read the faulting TLB entry.
290
     * Read the faulting TLB entry.
306
     */
291
     */
307
    tlbr();
292
    tlbr();
308
 
293
 
309
    /*
294
    /*
310
     * Record access and write to PTE.
295
     * Record access and write to PTE.
311
     */
296
     */
312
    pte->a = 1;
297
    pte->a = 1;
313
    pte->d = 1;
298
    pte->d = 1;
314
 
299
 
315
    prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
300
    tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable,
-
 
301
        pte->pfn);
316
 
302
 
317
    /*
303
    /*
318
     * The entry is to be updated in TLB.
304
     * The entry is to be updated in TLB.
319
     */
305
     */
320
    if ((badvaddr/PAGE_SIZE) % 2 == 0)
306
    if ((badvaddr / PAGE_SIZE) % 2 == 0)
321
        cp0_entry_lo0_write(lo.value);
307
        cp0_entry_lo0_write(lo.value);
322
    else
308
    else
323
        cp0_entry_lo1_write(lo.value);
309
        cp0_entry_lo1_write(lo.value);
324
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
310
    cp0_pagemask_write(TLB_PAGE_MASK_16K);
325
    tlbwi();
311
    tlbwi();
Line 342... Line 328...
342
        symbol = s;
328
        symbol = s;
343
    s = get_symtab_entry(istate->ra);
329
    s = get_symtab_entry(istate->ra);
344
    if (s)
330
    if (s)
345
        sym2 = s;
331
        sym2 = s;
346
 
332
 
347
    fault_if_from_uspace(istate, "TLB Refill Exception on %p", cp0_badvaddr_read());
333
    fault_if_from_uspace(istate, "TLB Refill Exception on %p",
-
 
334
        cp0_badvaddr_read());
348
    panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
335
    panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(),
-
 
336
        istate->epc, symbol, sym2);
349
}
337
}
350
 
338
 
351
 
339
 
352
void tlb_invalid_fail(istate_t *istate)
340
void tlb_invalid_fail(istate_t *istate)
353
{
341
{
354
    char *symbol = "";
342
    char *symbol = "";
355
 
343
 
356
    char *s = get_symtab_entry(istate->epc);
344
    char *s = get_symtab_entry(istate->epc);
357
    if (s)
345
    if (s)
358
        symbol = s;
346
        symbol = s;
359
    fault_if_from_uspace(istate, "TLB Invalid Exception on %p", cp0_badvaddr_read());
347
    fault_if_from_uspace(istate, "TLB Invalid Exception on %p",
-
 
348
        cp0_badvaddr_read());
360
    panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
349
    panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(),
-
 
350
        istate->epc, symbol);
361
}
351
}
362
 
352
 
363
void tlb_modified_fail(istate_t *istate)
353
void tlb_modified_fail(istate_t *istate)
364
{
354
{
365
    char *symbol = "";
355
    char *symbol = "";
366
 
356
 
367
    char *s = get_symtab_entry(istate->epc);
357
    char *s = get_symtab_entry(istate->epc);
368
    if (s)
358
    if (s)
369
        symbol = s;
359
        symbol = s;
370
    fault_if_from_uspace(istate, "TLB Modified Exception on %p", cp0_badvaddr_read());
360
    fault_if_from_uspace(istate, "TLB Modified Exception on %p",
-
 
361
        cp0_badvaddr_read());
371
    panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
362
    panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(),
-
 
363
        istate->epc, symbol);
372
}
364
}
373
 
365
 
374
/** Try to find PTE for faulting address
366
/** Try to find PTE for faulting address.
375
 *
367
 *
376
 * Try to find PTE for faulting address.
-
 
377
 * The AS->lock must be held on entry to this function.
368
 * The AS->lock must be held on entry to this function.
378
 *
369
 *
379
 * @param badvaddr Faulting virtual address.
370
 * @param badvaddr  Faulting virtual address.
380
 * @param access Access mode that caused the fault.
371
 * @param access    Access mode that caused the fault.
381
 * @param istate Pointer to interrupted state.
372
 * @param istate    Pointer to interrupted state.
382
 * @param pfrc Pointer to variable where as_page_fault() return code will be stored.
373
 * @param pfrc      Pointer to variable where as_page_fault() return code
-
 
374
 *          will be stored.
383
 *
375
 *
384
 * @return PTE on success, NULL otherwise.
376
 * @return      PTE on success, NULL otherwise.
385
 */
377
 */
-
 
378
pte_t *
386
pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc)
379
find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate,
-
 
380
    int *pfrc)
387
{
381
{
388
    entry_hi_t hi;
382
    entry_hi_t hi;
389
    pte_t *pte;
383
    pte_t *pte;
390
 
384
 
391
    hi.value = cp0_entry_hi_read();
385
    hi.value = cp0_entry_hi_read();
Line 400... Line 394...
400
 
394
 
401
    /*
395
    /*
402
     * Check if the mapping exists in page tables.
396
     * Check if the mapping exists in page tables.
403
     */
397
     */
404
    pte = page_mapping_find(AS, badvaddr);
398
    pte = page_mapping_find(AS, badvaddr);
405
    if (pte && pte->p) {
399
    if (pte && pte->p && (pte->w || access != PF_ACCESS_WRITE)) {
406
        /*
400
        /*
407
         * Mapping found in page tables.
401
         * Mapping found in page tables.
408
         * Immediately succeed.
402
         * Immediately succeed.
409
         */
403
         */
410
        return pte;
404
        return pte;
Line 423... Line 417...
423
             * The mapping ought to be in place.
417
             * The mapping ought to be in place.
424
             */
418
             */
425
            page_table_lock(AS, true);
419
            page_table_lock(AS, true);
426
            pte = page_mapping_find(AS, badvaddr);
420
            pte = page_mapping_find(AS, badvaddr);
427
            ASSERT(pte && pte->p);
421
            ASSERT(pte && pte->p);
-
 
422
            ASSERT(pte->w || access != PF_ACCESS_WRITE);
428
            return pte;
423
            return pte;
429
            break;
424
            break;
430
        case AS_PF_DEFER:
425
        case AS_PF_DEFER:
431
            page_table_lock(AS, true);
426
            page_table_lock(AS, true);
432
            *pfrc = AS_PF_DEFER;
427
            *pfrc = AS_PF_DEFER;
Line 443... Line 438...
443
        }
438
        }
444
       
439
       
445
    }
440
    }
446
}
441
}
447
 
442
 
-
 
443
void
448
void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn)
444
tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable,
-
 
445
    uintptr_t pfn)
449
{
446
{
450
    lo->value = 0;
447
    lo->value = 0;
451
    lo->g = g;
448
    lo->g = g;
452
    lo->v = v;
449
    lo->v = v;
453
    lo->d = d;
450
    lo->d = d;
454
    lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
451
    lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
455
    lo->pfn = pfn;
452
    lo->pfn = pfn;
456
}
453
}
457
 
454
 
458
void prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
455
void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
459
{
456
{
460
    hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
457
    hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
461
    hi->asid = asid;
458
    hi->asid = asid;
462
}
459
}
463
 
460
 
Line 482... Line 479...
482
        hi.value = cp0_entry_hi_read();
479
        hi.value = cp0_entry_hi_read();
483
        lo0.value = cp0_entry_lo0_read();
480
        lo0.value = cp0_entry_lo0_read();
484
        lo1.value = cp0_entry_lo1_read();
481
        lo1.value = cp0_entry_lo1_read();
485
       
482
       
486
        printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
483
        printf("%-2u %-4u %#6x %#4x %1u %1u %1u %1u %#6x\n",
487
            i, hi.asid, hi.vpn2, mask.mask,
484
            i, hi.asid, hi.vpn2, mask.mask,
488
            lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
485
            lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
489
        printf("                    %1u %1u %1u %1u %#6x\n",
486
        printf("                    %1u %1u %1u %1u %#6x\n",
490
            lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
487
            lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
491
    }
488
    }
492
   
489
   
493
    cp0_entry_hi_write(hi_save.value);
490
    cp0_entry_hi_write(hi_save.value);
494
}
491
}
495
 
492
 
Line 562... Line 559...
562
   
559
   
563
    interrupts_restore(ipl);
560
    interrupts_restore(ipl);
564
    cp0_entry_hi_write(hi_save.value);
561
    cp0_entry_hi_write(hi_save.value);
565
}
562
}
566
 
563
 
567
/** Invalidate TLB entries for specified page range belonging to specified address space.
564
/** Invalidate TLB entries for specified page range belonging to specified
-
 
565
 * address space.
568
 *
566
 *
569
 * @param asid Address space identifier.
567
 * @param asid      Address space identifier.
570
 * @param page First page whose TLB entry is to be invalidated.
568
 * @param page      First page whose TLB entry is to be invalidated.
571
 * @param cnt Number of entries to invalidate.
569
 * @param cnt       Number of entries to invalidate.
572
 */
570
 */
573
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
571
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
574
{
572
{
575
    unsigned int i;
573
    unsigned int i;
576
    ipl_t ipl;
574
    ipl_t ipl;
Line 583... Line 581...
583
    hi_save.value = cp0_entry_hi_read();
581
    hi_save.value = cp0_entry_hi_read();
584
    ipl = interrupts_disable();
582
    ipl = interrupts_disable();
585
 
583
 
586
    for (i = 0; i < cnt + 1; i += 2) {
584
    for (i = 0; i < cnt + 1; i += 2) {
587
        hi.value = 0;
585
        hi.value = 0;
588
        prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
586
        tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
589
        cp0_entry_hi_write(hi.value);
587
        cp0_entry_hi_write(hi.value);
590
 
588
 
591
        tlbp();
589
        tlbp();
592
        index.value = cp0_index_read();
590
        index.value = cp0_index_read();
593
 
591
 
594
        if (!index.p) {
592
        if (!index.p) {
-
 
593
            /*
595
            /* Entry was found, index register contains valid index. */
594
             * Entry was found, index register contains valid
-
 
595
             * index.
-
 
596
             */
596
            tlbr();
597
            tlbr();
597
 
598
 
598
            lo0.value = cp0_entry_lo0_read();
599
            lo0.value = cp0_entry_lo0_read();
599
            lo1.value = cp0_entry_lo1_read();
600
            lo1.value = cp0_entry_lo1_read();
600
 
601