Subversion Repositories HelenOS

Rev

Rev 3857 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1215 decky 1
/*
2071 jermar 2
 * Copyright (c) 2006 Martin Decky
1215 decky 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
3856 decky 29
/** @addtogroup ppc32mm
1702 cejka 30
 * @{
31
 */
32
/** @file
33
 */
34
 
1215 decky 35
#include <mm/tlb.h>
1730 decky 36
#include <arch/mm/tlb.h>
37
#include <arch/interrupt.h>
3594 svoboda 38
#include <interrupt.h>
1730 decky 39
#include <mm/as.h>
40
#include <arch.h>
41
#include <print.h>
3838 decky 42
#include <macros.h>
1215 decky 43
 
4132 svoboda 44
#ifdef CONFIG_SYMTAB
45
#include <symtab.h>
46
#endif
1215 decky 47
 
3837 decky 48
static unsigned int seed = 10;
49
static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
50
 
51
 
3856 decky 52
#define TLB_FLUSH \
53
    "tlbie %0\n" \
54
    "addi %0, %0, 0x1000\n"
55
 
56
 
1730 decky 57
/** Try to find PTE for faulting address
1215 decky 58
 *
1730 decky 59
 * Try to find PTE for faulting address.
60
 * The as->lock must be held on entry to this function
61
 * if lock is true.
1215 decky 62
 *
3193 jermar 63
 * @param as        Address space.
64
 * @param lock      Lock/unlock the address space.
65
 * @param badvaddr  Faulting virtual address.
66
 * @param access    Access mode that caused the fault.
67
 * @param istate    Pointer to interrupted state.
68
 * @param pfrc      Pointer to variable where as_page_fault() return code
69
 *          will be stored.
70
 * @return      PTE on success, NULL otherwise.
1730 decky 71
 *
1215 decky 72
 */
3193 jermar 73
static pte_t *
74
find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
75
    istate_t *istate, int *pfrc)
1730 decky 76
{
77
    /*
78
     * Check if the mapping exists in page tables.
79
     */
80
    pte_t *pte = page_mapping_find(as, badvaddr);
3830 decky 81
    if ((pte) && (pte->present)) {
1730 decky 82
        /*
83
         * Mapping found in page tables.
84
         * Immediately succeed.
85
         */
86
        return pte;
87
    } else {
88
        int rc;
89
 
90
        /*
91
         * Mapping not found in page tables.
92
         * Resort to higher-level page fault handler.
93
         */
94
        page_table_unlock(as, lock);
95
        switch (rc = as_page_fault(badvaddr, access, istate)) {
3193 jermar 96
        case AS_PF_OK:
97
            /*
98
             * The higher-level page fault handler succeeded,
99
             * The mapping ought to be in place.
100
             */
101
            page_table_lock(as, lock);
102
            pte = page_mapping_find(as, badvaddr);
3830 decky 103
            ASSERT((pte) && (pte->present));
3193 jermar 104
            *pfrc = 0;
105
            return pte;
106
        case AS_PF_DEFER:
107
            page_table_lock(as, lock);
108
            *pfrc = rc;
109
            return NULL;
110
        case AS_PF_FAULT:
111
            page_table_lock(as, lock);
112
            *pfrc = rc;
113
            return NULL;
114
        default:
3790 svoboda 115
            panic("Unexpected rc (%d).", rc);
1730 decky 116
        }  
117
    }
118
}
119
 
120
 
1780 jermar 121
static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
1730 decky 122
{
123
    char *symbol = "";
124
    char *sym2 = "";
125
 
4132 svoboda 126
#ifdef CONFIG_SYMTAB
3833 decky 127
    char *str = get_symtab_entry(istate->pc);
128
    if (str)
129
        symbol = str;
130
    str = get_symtab_entry(istate->lr);
131
    if (str)
132
        sym2 = str;
4132 svoboda 133
#endif
3594 svoboda 134
 
135
    fault_if_from_uspace(istate,
3833 decky 136
        "PHT Refill Exception on %p.", badvaddr);
3790 svoboda 137
    panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
3193 jermar 138
        istate->pc, symbol, sym2);
1730 decky 139
}
140
 
141
 
3830 decky 142
static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
1730 decky 143
{
1780 jermar 144
    uint32_t page = (vaddr >> 12) & 0xffff;
145
    uint32_t api = (vaddr >> 22) & 0x3f;
1730 decky 146
 
1780 jermar 147
    uint32_t vsid;
1730 decky 148
    asm volatile (
149
        "mfsrin %0, %1\n"
150
        : "=r" (vsid)
151
        : "r" (vaddr)
152
    );
153
 
1780 jermar 154
    uint32_t sdr1;
1730 decky 155
    asm volatile (
156
        "mfsdr1 %0\n"
157
        : "=r" (sdr1)
158
    );
159
    phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
160
 
161
    /* Primary hash (xor) */
1780 jermar 162
    uint32_t h = 0;
163
    uint32_t hash = vsid ^ page;
164
    uint32_t base = (hash & 0x3ff) << 3;
165
    uint32_t i;
1730 decky 166
    bool found = false;
167
 
3839 decky 168
    /* Find colliding PTE in PTEG */
1730 decky 169
    for (i = 0; i < 8; i++) {
3839 decky 170
        if ((phte[base + i].v)
171
            && (phte[base + i].vsid == vsid)
3836 decky 172
            && (phte[base + i].api == api)
3839 decky 173
            && (phte[base + i].h == 0)) {
1730 decky 174
            found = true;
175
            break;
176
        }
177
    }
178
 
179
    if (!found) {
3839 decky 180
        /* Find unused PTE in PTEG */
181
        for (i = 0; i < 8; i++) {
182
            if (!phte[base + i].v) {
183
                found = true;
184
                break;
185
            }
186
        }
187
    }
188
 
189
    if (!found) {
1730 decky 190
        /* Secondary hash (not) */
1780 jermar 191
        uint32_t base2 = (~hash & 0x3ff) << 3;
1730 decky 192
 
3839 decky 193
        /* Find colliding PTE in PTEG */
1730 decky 194
        for (i = 0; i < 8; i++) {
3839 decky 195
            if ((phte[base2 + i].v)
196
                && (phte[base2 + i].vsid == vsid)
3836 decky 197
                && (phte[base2 + i].api == api)
3839 decky 198
                && (phte[base2 + i].h == 1)) {
1730 decky 199
                found = true;
200
                base = base2;
201
                h = 1;
202
                break;
203
            }
204
        }
205
 
3839 decky 206
        if (!found) {
207
            /* Find unused PTE in PTEG */
208
            for (i = 0; i < 8; i++) {
209
                if (!phte[base2 + i].v) {
210
                    found = true;
211
                    base = base2;
212
                    h = 1;
213
                    break;
214
                }
215
            }
216
        }
217
 
3836 decky 218
        if (!found)
3837 decky 219
            i = RANDI(seed) % 8;
1730 decky 220
    }
221
 
222
    phte[base + i].v = 1;
223
    phte[base + i].vsid = vsid;
224
    phte[base + i].h = h;
225
    phte[base + i].api = api;
3830 decky 226
    phte[base + i].rpn = pte->pfn;
1730 decky 227
    phte[base + i].r = 0;
228
    phte[base + i].c = 0;
3830 decky 229
    phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
1730 decky 230
    phte[base + i].pp = 2; // FIXME
231
}
232
 
233
 
3857 decky 234
/** Process Instruction/Data Storage Exception
1730 decky 235
 *
3857 decky 236
 * @param n      Exception vector number.
237
 * @param istate Interrupted register context.
1730 decky 238
 *
239
 */
240
void pht_refill(int n, istate_t *istate)
241
{
1780 jermar 242
    uintptr_t badvaddr;
1730 decky 243
    pte_t *pte;
244
    int pfrc;
245
    as_t *as;
246
    bool lock;
247
 
248
    if (AS == NULL) {
249
        as = AS_KERNEL;
250
        lock = false;
251
    } else {
252
        as = AS;
253
        lock = true;
254
    }
255
 
3833 decky 256
    if (n == VECTOR_DATA_STORAGE)
257
        badvaddr = istate->dar;
258
    else
1730 decky 259
        badvaddr = istate->pc;
260
 
261
    page_table_lock(as, lock);
262
 
3193 jermar 263
    pte = find_mapping_and_check(as, lock, badvaddr,
264
        PF_ACCESS_READ /* FIXME */, istate, &pfrc);
1730 decky 265
    if (!pte) {
266
        switch (pfrc) {
3193 jermar 267
        case AS_PF_FAULT:
268
            goto fail;
269
            break;
270
        case AS_PF_DEFER:
271
            /*
272
             * The page fault came during copy_from_uspace()
273
             * or copy_to_uspace().
274
             */
275
            page_table_unlock(as, lock);
276
            return;
277
        default:
3790 svoboda 278
            panic("Unexpected pfrc (%d).", pfrc);
1730 decky 279
        }
280
    }
281
 
3830 decky 282
    pte->accessed = 1; /* Record access to PTE */
283
    pht_insert(badvaddr, pte);
1730 decky 284
 
285
    page_table_unlock(as, lock);
286
    return;
287
 
288
fail:
289
    page_table_unlock(as, lock);
290
    pht_refill_fail(badvaddr, istate);
291
}
292
 
293
 
3857 decky 294
/** Process Instruction/Data Storage Exception in Real Mode
1730 decky 295
 *
3857 decky 296
 * @param n      Exception vector number.
297
 * @param istate Interrupted register context.
1730 decky 298
 *
299
 */
3837 decky 300
bool pht_refill_real(int n, istate_t *istate)
1730 decky 301
{
1780 jermar 302
    uintptr_t badvaddr;
1730 decky 303
 
3833 decky 304
    if (n == VECTOR_DATA_STORAGE)
305
        badvaddr = istate->dar;
306
    else
1730 decky 307
        badvaddr = istate->pc;
308
 
1780 jermar 309
    uint32_t physmem;
1730 decky 310
    asm volatile (
311
        "mfsprg3 %0\n"
312
        : "=r" (physmem)
313
    );
314
 
3837 decky 315
    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
316
        return false;
317
 
318
    uint32_t page = (badvaddr >> 12) & 0xffff;
319
    uint32_t api = (badvaddr >> 22) & 0x3f;
320
 
321
    uint32_t vsid;
322
    asm volatile (
323
        "mfsrin %0, %1\n"
324
        : "=r" (vsid)
325
        : "r" (badvaddr)
326
    );
327
 
328
    uint32_t sdr1;
329
    asm volatile (
330
        "mfsdr1 %0\n"
331
        : "=r" (sdr1)
332
    );
333
    phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
334
 
335
    /* Primary hash (xor) */
336
    uint32_t h = 0;
337
    uint32_t hash = vsid ^ page;
338
    uint32_t base = (hash & 0x3ff) << 3;
339
    uint32_t i;
340
    bool found = false;
341
 
3839 decky 342
    /* Find colliding PTE in PTEG */
3837 decky 343
    for (i = 0; i < 8; i++) {
3839 decky 344
        if ((phte_real[base + i].v)
345
            && (phte_real[base + i].vsid == vsid)
3837 decky 346
            && (phte_real[base + i].api == api)
3839 decky 347
            && (phte_real[base + i].h == 0)) {
3837 decky 348
            found = true;
349
            break;
350
        }
1730 decky 351
    }
352
 
3837 decky 353
    if (!found) {
3839 decky 354
        /* Find unused PTE in PTEG */
355
        for (i = 0; i < 8; i++) {
356
            if (!phte_real[base + i].v) {
357
                found = true;
358
                break;
359
            }
360
        }
361
    }
362
 
363
    if (!found) {
3837 decky 364
        /* Secondary hash (not) */
365
        uint32_t base2 = (~hash & 0x3ff) << 3;
366
 
3839 decky 367
        /* Find colliding PTE in PTEG */
3837 decky 368
        for (i = 0; i < 8; i++) {
3839 decky 369
            if ((phte_real[base2 + i].v)
370
                && (phte_real[base2 + i].vsid == vsid)
3837 decky 371
                && (phte_real[base2 + i].api == api)
3839 decky 372
                && (phte_real[base2 + i].h == 1)) {
3837 decky 373
                found = true;
374
                base = base2;
375
                h = 1;
376
                break;
377
            }
378
        }
379
 
380
        if (!found) {
3839 decky 381
            /* Find unused PTE in PTEG */
382
            for (i = 0; i < 8; i++) {
383
                if (!phte_real[base2 + i].v) {
384
                    found = true;
385
                    base = base2;
386
                    h = 1;
387
                    break;
388
                }
389
            }
390
        }
391
 
392
        if (!found) {
3837 decky 393
            /* Use secondary hash to avoid collisions
394
               with usual PHT refill handler. */
395
            i = RANDI(seed_real) % 8;
396
            base = base2;
397
            h = 1;
398
        }
399
    }
400
 
401
    phte_real[base + i].v = 1;
402
    phte_real[base + i].vsid = vsid;
403
    phte_real[base + i].h = h;
404
    phte_real[base + i].api = api;
405
    phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
406
    phte_real[base + i].r = 0;
407
    phte_real[base + i].c = 0;
408
    phte_real[base + i].wimg = 0;
409
    phte_real[base + i].pp = 2; // FIXME
410
 
411
    return true;
1730 decky 412
}
413
 
414
 
3857 decky 415
/** Process ITLB/DTLB Miss Exception in Real Mode
416
 *
417
 *
418
 */
419
void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
420
{
421
    uint32_t badvaddr = tlbmiss & 0xfffffffc;
422
 
423
    uint32_t physmem;
424
    asm volatile (
425
        "mfsprg3 %0\n"
426
        : "=r" (physmem)
427
    );
428
 
429
    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
430
        return; // FIXME
431
 
432
    ptelo.rpn = KA2PA(badvaddr) >> 12;
433
    ptelo.wimg = 0;
434
    ptelo.pp = 2; // FIXME
435
 
436
    uint32_t index = 0;
437
    asm volatile (
438
        "mtspr 981, %0\n"
439
        "mtspr 982, %1\n"
440
        "tlbld %2\n"
441
        "tlbli %2\n"
442
        : "=r" (index)
443
        : "r" (ptehi),
444
          "r" (ptelo)
445
    );
446
}
447
 
448
 
1215 decky 449
void tlb_arch_init(void)
450
{
1384 decky 451
    tlb_invalidate_all();
452
}
453
 
454
 
455
void tlb_invalidate_all(void)
456
{
3856 decky 457
    uint32_t index;
1269 decky 458
    asm volatile (
3856 decky 459
        "li %0, 0\n"
460
        "sync\n"
461
 
462
        TLB_FLUSH
463
        TLB_FLUSH
464
        TLB_FLUSH
465
        TLB_FLUSH
466
        TLB_FLUSH
467
        TLB_FLUSH
468
        TLB_FLUSH
469
        TLB_FLUSH
470
 
471
        TLB_FLUSH
472
        TLB_FLUSH
473
        TLB_FLUSH
474
        TLB_FLUSH
475
        TLB_FLUSH
476
        TLB_FLUSH
477
        TLB_FLUSH
478
        TLB_FLUSH
479
 
480
        TLB_FLUSH
481
        TLB_FLUSH
482
        TLB_FLUSH
483
        TLB_FLUSH
484
        TLB_FLUSH
485
        TLB_FLUSH
486
        TLB_FLUSH
487
        TLB_FLUSH
488
 
489
        TLB_FLUSH
490
        TLB_FLUSH
491
        TLB_FLUSH
492
        TLB_FLUSH
493
        TLB_FLUSH
494
        TLB_FLUSH
495
        TLB_FLUSH
496
        TLB_FLUSH
497
 
498
        TLB_FLUSH
499
        TLB_FLUSH
500
        TLB_FLUSH
501
        TLB_FLUSH
502
        TLB_FLUSH
503
        TLB_FLUSH
504
        TLB_FLUSH
505
        TLB_FLUSH
506
 
507
        TLB_FLUSH
508
        TLB_FLUSH
509
        TLB_FLUSH
510
        TLB_FLUSH
511
        TLB_FLUSH
512
        TLB_FLUSH
513
        TLB_FLUSH
514
        TLB_FLUSH
515
 
516
        TLB_FLUSH
517
        TLB_FLUSH
518
        TLB_FLUSH
519
        TLB_FLUSH
520
        TLB_FLUSH
521
        TLB_FLUSH
522
        TLB_FLUSH
523
        TLB_FLUSH
524
 
525
        TLB_FLUSH
526
        TLB_FLUSH
527
        TLB_FLUSH
528
        TLB_FLUSH
529
        TLB_FLUSH
530
        TLB_FLUSH
531
        TLB_FLUSH
532
        TLB_FLUSH
533
 
534
        "eieio\n"
1384 decky 535
        "tlbsync\n"
3856 decky 536
        "sync\n"
537
        : "=r" (index)
1269 decky 538
    );
1215 decky 539
}
540
 
541
 
1384 decky 542
void tlb_invalidate_asid(asid_t asid)
1328 decky 543
{
1780 jermar 544
    uint32_t sdr1;
1758 decky 545
    asm volatile (
546
        "mfsdr1 %0\n"
547
        : "=r" (sdr1)
548
    );
549
    phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
550
 
1780 jermar 551
    uint32_t i;
1758 decky 552
    for (i = 0; i < 8192; i++) {
3193 jermar 553
        if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
554
            (phte[i].vsid < ((asid << 4) + 16)))
1758 decky 555
            phte[i].v = 0;
556
    }
1384 decky 557
    tlb_invalidate_all();
1328 decky 558
}
559
 
1730 decky 560
 
1780 jermar 561
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
1384 decky 562
{
1730 decky 563
    // TODO
1384 decky 564
    tlb_invalidate_all();
565
}
1328 decky 566
 
1384 decky 567
 
1736 decky 568
#define PRINT_BAT(name, ureg, lreg) \
569
    asm volatile ( \
570
        "mfspr %0," #ureg "\n" \
571
        "mfspr %1," #lreg "\n" \
572
        : "=r" (upper), "=r" (lower) \
573
    ); \
574
    mask = (upper & 0x1ffc) >> 2; \
575
    if (upper & 3) { \
1780 jermar 576
        uint32_t tmp = mask; \
1736 decky 577
        length = 128; \
578
        while (tmp) { \
579
            if ((tmp & 1) == 0) { \
580
                printf("ibat[0]: error in mask\n"); \
581
                break; \
582
            } \
583
            length <<= 1; \
584
            tmp >>= 1; \
585
        } \
586
    } else \
587
        length = 0; \
3193 jermar 588
    printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
589
        sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
590
        lower & 0xffff0000, length, mask, \
591
        ((upper >> 1) & 1) ? " supervisor" : "", \
592
        (upper & 1) ? " user" : "");
1736 decky 593
 
594
 
1215 decky 595
void tlb_print(void)
596
{
1780 jermar 597
    uint32_t sr;
1733 decky 598
 
599
    for (sr = 0; sr < 16; sr++) {
1780 jermar 600
        uint32_t vsid;
1733 decky 601
        asm volatile (
602
            "mfsrin %0, %1\n"
603
            : "=r" (vsid)
604
            : "r" (sr << 28)
605
        );
3837 decky 606
        printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
3193 jermar 607
            sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
608
            ((vsid >> 30) & 1) ? " supervisor" : "",
609
            ((vsid >> 29) & 1) ? " user" : "");
1733 decky 610
    }
1736 decky 611
 
1780 jermar 612
    uint32_t upper;
613
    uint32_t lower;
614
    uint32_t mask;
615
    uint32_t length;
1736 decky 616
 
617
    PRINT_BAT("ibat[0]", 528, 529);
618
    PRINT_BAT("ibat[1]", 530, 531);
619
    PRINT_BAT("ibat[2]", 532, 533);
620
    PRINT_BAT("ibat[3]", 534, 535);
621
 
622
    PRINT_BAT("dbat[0]", 536, 537);
623
    PRINT_BAT("dbat[1]", 538, 539);
624
    PRINT_BAT("dbat[2]", 540, 541);
625
    PRINT_BAT("dbat[3]", 542, 543);
1215 decky 626
}
1702 cejka 627
 
1730 decky 628
/** @}
1702 cejka 629
 */