Subversion Repositories HelenOS

Rev

Rev 4137 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1215 decky 1
/*
2071 jermar 2
 * Copyright (c) 2006 Martin Decky
1215 decky 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
3856 decky 29
/** @addtogroup ppc32mm
1702 cejka 30
 * @{
31
 */
32
/** @file
33
 */
34
 
1215 decky 35
#include <mm/tlb.h>
1730 decky 36
#include <arch/mm/tlb.h>
37
#include <arch/interrupt.h>
3594 svoboda 38
#include <interrupt.h>
1730 decky 39
#include <mm/as.h>
40
#include <arch.h>
41
#include <print.h>
3838 decky 42
#include <macros.h>
4132 svoboda 43
#include <symtab.h>
1215 decky 44
 
3837 decky 45
static unsigned int seed = 10;
46
static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
47
 
48
 
3856 decky 49
#define TLB_FLUSH \
50
    "tlbie %0\n" \
51
    "addi %0, %0, 0x1000\n"
52
 
53
 
1730 decky 54
/** Try to find PTE for faulting address
1215 decky 55
 *
1730 decky 56
 * Try to find PTE for faulting address.
57
 * The as->lock must be held on entry to this function
58
 * if lock is true.
1215 decky 59
 *
3193 jermar 60
 * @param as        Address space.
61
 * @param lock      Lock/unlock the address space.
62
 * @param badvaddr  Faulting virtual address.
63
 * @param access    Access mode that caused the fault.
64
 * @param istate    Pointer to interrupted state.
65
 * @param pfrc      Pointer to variable where as_page_fault() return code
66
 *          will be stored.
67
 * @return      PTE on success, NULL otherwise.
1730 decky 68
 *
1215 decky 69
 */
3193 jermar 70
static pte_t *
71
find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
72
    istate_t *istate, int *pfrc)
1730 decky 73
{
74
    /*
75
     * Check if the mapping exists in page tables.
76
     */
77
    pte_t *pte = page_mapping_find(as, badvaddr);
3830 decky 78
    if ((pte) && (pte->present)) {
1730 decky 79
        /*
80
         * Mapping found in page tables.
81
         * Immediately succeed.
82
         */
83
        return pte;
84
    } else {
85
        int rc;
86
 
87
        /*
88
         * Mapping not found in page tables.
89
         * Resort to higher-level page fault handler.
90
         */
91
        page_table_unlock(as, lock);
92
        switch (rc = as_page_fault(badvaddr, access, istate)) {
3193 jermar 93
        case AS_PF_OK:
94
            /*
95
             * The higher-level page fault handler succeeded,
96
             * The mapping ought to be in place.
97
             */
98
            page_table_lock(as, lock);
99
            pte = page_mapping_find(as, badvaddr);
3830 decky 100
            ASSERT((pte) && (pte->present));
3193 jermar 101
            *pfrc = 0;
102
            return pte;
103
        case AS_PF_DEFER:
104
            page_table_lock(as, lock);
105
            *pfrc = rc;
106
            return NULL;
107
        case AS_PF_FAULT:
108
            page_table_lock(as, lock);
109
            *pfrc = rc;
110
            return NULL;
111
        default:
3790 svoboda 112
            panic("Unexpected rc (%d).", rc);
1730 decky 113
        }  
114
    }
115
}
116
 
117
 
1780 jermar 118
static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
1730 decky 119
{
4137 svoboda 120
    char *symbol;
121
    char *sym2;
1730 decky 122
 
4137 svoboda 123
    symbol = symtab_fmt_name_lookup(istate->pc);
124
    sym2 = symtab_fmt_name_lookup(istate->lr);
3594 svoboda 125
 
126
    fault_if_from_uspace(istate,
3833 decky 127
        "PHT Refill Exception on %p.", badvaddr);
3790 svoboda 128
    panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
3193 jermar 129
        istate->pc, symbol, sym2);
1730 decky 130
}
131
 
132
 
3830 decky 133
static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
1730 decky 134
{
1780 jermar 135
    uint32_t page = (vaddr >> 12) & 0xffff;
136
    uint32_t api = (vaddr >> 22) & 0x3f;
1730 decky 137
 
1780 jermar 138
    uint32_t vsid;
1730 decky 139
    asm volatile (
140
        "mfsrin %0, %1\n"
141
        : "=r" (vsid)
142
        : "r" (vaddr)
143
    );
144
 
1780 jermar 145
    uint32_t sdr1;
1730 decky 146
    asm volatile (
147
        "mfsdr1 %0\n"
148
        : "=r" (sdr1)
149
    );
150
    phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
151
 
152
    /* Primary hash (xor) */
1780 jermar 153
    uint32_t h = 0;
154
    uint32_t hash = vsid ^ page;
155
    uint32_t base = (hash & 0x3ff) << 3;
156
    uint32_t i;
1730 decky 157
    bool found = false;
158
 
3839 decky 159
    /* Find colliding PTE in PTEG */
1730 decky 160
    for (i = 0; i < 8; i++) {
3839 decky 161
        if ((phte[base + i].v)
162
            && (phte[base + i].vsid == vsid)
3836 decky 163
            && (phte[base + i].api == api)
3839 decky 164
            && (phte[base + i].h == 0)) {
1730 decky 165
            found = true;
166
            break;
167
        }
168
    }
169
 
170
    if (!found) {
3839 decky 171
        /* Find unused PTE in PTEG */
172
        for (i = 0; i < 8; i++) {
173
            if (!phte[base + i].v) {
174
                found = true;
175
                break;
176
            }
177
        }
178
    }
179
 
180
    if (!found) {
1730 decky 181
        /* Secondary hash (not) */
1780 jermar 182
        uint32_t base2 = (~hash & 0x3ff) << 3;
1730 decky 183
 
3839 decky 184
        /* Find colliding PTE in PTEG */
1730 decky 185
        for (i = 0; i < 8; i++) {
3839 decky 186
            if ((phte[base2 + i].v)
187
                && (phte[base2 + i].vsid == vsid)
3836 decky 188
                && (phte[base2 + i].api == api)
3839 decky 189
                && (phte[base2 + i].h == 1)) {
1730 decky 190
                found = true;
191
                base = base2;
192
                h = 1;
193
                break;
194
            }
195
        }
196
 
3839 decky 197
        if (!found) {
198
            /* Find unused PTE in PTEG */
199
            for (i = 0; i < 8; i++) {
200
                if (!phte[base2 + i].v) {
201
                    found = true;
202
                    base = base2;
203
                    h = 1;
204
                    break;
205
                }
206
            }
207
        }
208
 
3836 decky 209
        if (!found)
3837 decky 210
            i = RANDI(seed) % 8;
1730 decky 211
    }
212
 
213
    phte[base + i].v = 1;
214
    phte[base + i].vsid = vsid;
215
    phte[base + i].h = h;
216
    phte[base + i].api = api;
3830 decky 217
    phte[base + i].rpn = pte->pfn;
1730 decky 218
    phte[base + i].r = 0;
219
    phte[base + i].c = 0;
3830 decky 220
    phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
1730 decky 221
    phte[base + i].pp = 2; // FIXME
222
}
223
 
224
 
3857 decky 225
/** Process Instruction/Data Storage Exception
1730 decky 226
 *
3857 decky 227
 * @param n      Exception vector number.
228
 * @param istate Interrupted register context.
1730 decky 229
 *
230
 */
231
void pht_refill(int n, istate_t *istate)
232
{
1780 jermar 233
    uintptr_t badvaddr;
1730 decky 234
    pte_t *pte;
235
    int pfrc;
236
    as_t *as;
237
    bool lock;
238
 
239
    if (AS == NULL) {
240
        as = AS_KERNEL;
241
        lock = false;
242
    } else {
243
        as = AS;
244
        lock = true;
245
    }
246
 
3833 decky 247
    if (n == VECTOR_DATA_STORAGE)
248
        badvaddr = istate->dar;
249
    else
1730 decky 250
        badvaddr = istate->pc;
251
 
252
    page_table_lock(as, lock);
253
 
3193 jermar 254
    pte = find_mapping_and_check(as, lock, badvaddr,
255
        PF_ACCESS_READ /* FIXME */, istate, &pfrc);
1730 decky 256
    if (!pte) {
257
        switch (pfrc) {
3193 jermar 258
        case AS_PF_FAULT:
259
            goto fail;
260
            break;
261
        case AS_PF_DEFER:
262
            /*
263
             * The page fault came during copy_from_uspace()
264
             * or copy_to_uspace().
265
             */
266
            page_table_unlock(as, lock);
267
            return;
268
        default:
3790 svoboda 269
            panic("Unexpected pfrc (%d).", pfrc);
1730 decky 270
        }
271
    }
272
 
3830 decky 273
    pte->accessed = 1; /* Record access to PTE */
274
    pht_insert(badvaddr, pte);
1730 decky 275
 
276
    page_table_unlock(as, lock);
277
    return;
278
 
279
fail:
280
    page_table_unlock(as, lock);
281
    pht_refill_fail(badvaddr, istate);
282
}
283
 
284
 
3857 decky 285
/** Process Instruction/Data Storage Exception in Real Mode
1730 decky 286
 *
3857 decky 287
 * @param n      Exception vector number.
288
 * @param istate Interrupted register context.
1730 decky 289
 *
290
 */
3837 decky 291
bool pht_refill_real(int n, istate_t *istate)
1730 decky 292
{
1780 jermar 293
    uintptr_t badvaddr;
1730 decky 294
 
3833 decky 295
    if (n == VECTOR_DATA_STORAGE)
296
        badvaddr = istate->dar;
297
    else
1730 decky 298
        badvaddr = istate->pc;
299
 
1780 jermar 300
    uint32_t physmem;
1730 decky 301
    asm volatile (
302
        "mfsprg3 %0\n"
303
        : "=r" (physmem)
304
    );
305
 
3837 decky 306
    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
307
        return false;
308
 
309
    uint32_t page = (badvaddr >> 12) & 0xffff;
310
    uint32_t api = (badvaddr >> 22) & 0x3f;
311
 
312
    uint32_t vsid;
313
    asm volatile (
314
        "mfsrin %0, %1\n"
315
        : "=r" (vsid)
316
        : "r" (badvaddr)
317
    );
318
 
319
    uint32_t sdr1;
320
    asm volatile (
321
        "mfsdr1 %0\n"
322
        : "=r" (sdr1)
323
    );
324
    phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
325
 
326
    /* Primary hash (xor) */
327
    uint32_t h = 0;
328
    uint32_t hash = vsid ^ page;
329
    uint32_t base = (hash & 0x3ff) << 3;
330
    uint32_t i;
331
    bool found = false;
332
 
3839 decky 333
    /* Find colliding PTE in PTEG */
3837 decky 334
    for (i = 0; i < 8; i++) {
3839 decky 335
        if ((phte_real[base + i].v)
336
            && (phte_real[base + i].vsid == vsid)
3837 decky 337
            && (phte_real[base + i].api == api)
3839 decky 338
            && (phte_real[base + i].h == 0)) {
3837 decky 339
            found = true;
340
            break;
341
        }
1730 decky 342
    }
343
 
3837 decky 344
    if (!found) {
3839 decky 345
        /* Find unused PTE in PTEG */
346
        for (i = 0; i < 8; i++) {
347
            if (!phte_real[base + i].v) {
348
                found = true;
349
                break;
350
            }
351
        }
352
    }
353
 
354
    if (!found) {
3837 decky 355
        /* Secondary hash (not) */
356
        uint32_t base2 = (~hash & 0x3ff) << 3;
357
 
3839 decky 358
        /* Find colliding PTE in PTEG */
3837 decky 359
        for (i = 0; i < 8; i++) {
3839 decky 360
            if ((phte_real[base2 + i].v)
361
                && (phte_real[base2 + i].vsid == vsid)
3837 decky 362
                && (phte_real[base2 + i].api == api)
3839 decky 363
                && (phte_real[base2 + i].h == 1)) {
3837 decky 364
                found = true;
365
                base = base2;
366
                h = 1;
367
                break;
368
            }
369
        }
370
 
371
        if (!found) {
3839 decky 372
            /* Find unused PTE in PTEG */
373
            for (i = 0; i < 8; i++) {
374
                if (!phte_real[base2 + i].v) {
375
                    found = true;
376
                    base = base2;
377
                    h = 1;
378
                    break;
379
                }
380
            }
381
        }
382
 
383
        if (!found) {
3837 decky 384
            /* Use secondary hash to avoid collisions
385
               with usual PHT refill handler. */
386
            i = RANDI(seed_real) % 8;
387
            base = base2;
388
            h = 1;
389
        }
390
    }
391
 
392
    phte_real[base + i].v = 1;
393
    phte_real[base + i].vsid = vsid;
394
    phte_real[base + i].h = h;
395
    phte_real[base + i].api = api;
396
    phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
397
    phte_real[base + i].r = 0;
398
    phte_real[base + i].c = 0;
399
    phte_real[base + i].wimg = 0;
400
    phte_real[base + i].pp = 2; // FIXME
401
 
402
    return true;
1730 decky 403
}
404
 
405
 
3857 decky 406
/** Process ITLB/DTLB Miss Exception in Real Mode
407
 *
408
 *
409
 */
410
void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
411
{
412
    uint32_t badvaddr = tlbmiss & 0xfffffffc;
413
 
414
    uint32_t physmem;
415
    asm volatile (
416
        "mfsprg3 %0\n"
417
        : "=r" (physmem)
418
    );
419
 
420
    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
421
        return; // FIXME
422
 
423
    ptelo.rpn = KA2PA(badvaddr) >> 12;
424
    ptelo.wimg = 0;
425
    ptelo.pp = 2; // FIXME
426
 
427
    uint32_t index = 0;
428
    asm volatile (
429
        "mtspr 981, %0\n"
430
        "mtspr 982, %1\n"
431
        "tlbld %2\n"
432
        "tlbli %2\n"
433
        : "=r" (index)
434
        : "r" (ptehi),
435
          "r" (ptelo)
436
    );
437
}
438
 
439
 
1215 decky 440
void tlb_arch_init(void)
441
{
1384 decky 442
    tlb_invalidate_all();
443
}
444
 
445
 
446
void tlb_invalidate_all(void)
447
{
3856 decky 448
    uint32_t index;
1269 decky 449
    asm volatile (
3856 decky 450
        "li %0, 0\n"
451
        "sync\n"
452
 
453
        TLB_FLUSH
454
        TLB_FLUSH
455
        TLB_FLUSH
456
        TLB_FLUSH
457
        TLB_FLUSH
458
        TLB_FLUSH
459
        TLB_FLUSH
460
        TLB_FLUSH
461
 
462
        TLB_FLUSH
463
        TLB_FLUSH
464
        TLB_FLUSH
465
        TLB_FLUSH
466
        TLB_FLUSH
467
        TLB_FLUSH
468
        TLB_FLUSH
469
        TLB_FLUSH
470
 
471
        TLB_FLUSH
472
        TLB_FLUSH
473
        TLB_FLUSH
474
        TLB_FLUSH
475
        TLB_FLUSH
476
        TLB_FLUSH
477
        TLB_FLUSH
478
        TLB_FLUSH
479
 
480
        TLB_FLUSH
481
        TLB_FLUSH
482
        TLB_FLUSH
483
        TLB_FLUSH
484
        TLB_FLUSH
485
        TLB_FLUSH
486
        TLB_FLUSH
487
        TLB_FLUSH
488
 
489
        TLB_FLUSH
490
        TLB_FLUSH
491
        TLB_FLUSH
492
        TLB_FLUSH
493
        TLB_FLUSH
494
        TLB_FLUSH
495
        TLB_FLUSH
496
        TLB_FLUSH
497
 
498
        TLB_FLUSH
499
        TLB_FLUSH
500
        TLB_FLUSH
501
        TLB_FLUSH
502
        TLB_FLUSH
503
        TLB_FLUSH
504
        TLB_FLUSH
505
        TLB_FLUSH
506
 
507
        TLB_FLUSH
508
        TLB_FLUSH
509
        TLB_FLUSH
510
        TLB_FLUSH
511
        TLB_FLUSH
512
        TLB_FLUSH
513
        TLB_FLUSH
514
        TLB_FLUSH
515
 
516
        TLB_FLUSH
517
        TLB_FLUSH
518
        TLB_FLUSH
519
        TLB_FLUSH
520
        TLB_FLUSH
521
        TLB_FLUSH
522
        TLB_FLUSH
523
        TLB_FLUSH
524
 
525
        "eieio\n"
1384 decky 526
        "tlbsync\n"
3856 decky 527
        "sync\n"
528
        : "=r" (index)
1269 decky 529
    );
1215 decky 530
}
531
 
532
 
1384 decky 533
void tlb_invalidate_asid(asid_t asid)
1328 decky 534
{
1780 jermar 535
    uint32_t sdr1;
1758 decky 536
    asm volatile (
537
        "mfsdr1 %0\n"
538
        : "=r" (sdr1)
539
    );
540
    phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
541
 
1780 jermar 542
    uint32_t i;
1758 decky 543
    for (i = 0; i < 8192; i++) {
3193 jermar 544
        if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
545
            (phte[i].vsid < ((asid << 4) + 16)))
1758 decky 546
            phte[i].v = 0;
547
    }
1384 decky 548
    tlb_invalidate_all();
1328 decky 549
}
550
 
1730 decky 551
 
4490 decky 552
void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
1384 decky 553
{
1730 decky 554
    // TODO
1384 decky 555
    tlb_invalidate_all();
556
}
1328 decky 557
 
1384 decky 558
 
1736 decky 559
#define PRINT_BAT(name, ureg, lreg) \
560
    asm volatile ( \
561
        "mfspr %0," #ureg "\n" \
562
        "mfspr %1," #lreg "\n" \
563
        : "=r" (upper), "=r" (lower) \
564
    ); \
565
    mask = (upper & 0x1ffc) >> 2; \
566
    if (upper & 3) { \
1780 jermar 567
        uint32_t tmp = mask; \
1736 decky 568
        length = 128; \
569
        while (tmp) { \
570
            if ((tmp & 1) == 0) { \
571
                printf("ibat[0]: error in mask\n"); \
572
                break; \
573
            } \
574
            length <<= 1; \
575
            tmp >>= 1; \
576
        } \
577
    } else \
578
        length = 0; \
3193 jermar 579
    printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
580
        sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
581
        lower & 0xffff0000, length, mask, \
582
        ((upper >> 1) & 1) ? " supervisor" : "", \
583
        (upper & 1) ? " user" : "");
1736 decky 584
 
585
 
1215 decky 586
void tlb_print(void)
587
{
1780 jermar 588
    uint32_t sr;
1733 decky 589
 
590
    for (sr = 0; sr < 16; sr++) {
1780 jermar 591
        uint32_t vsid;
1733 decky 592
        asm volatile (
593
            "mfsrin %0, %1\n"
594
            : "=r" (vsid)
595
            : "r" (sr << 28)
596
        );
3837 decky 597
        printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
3193 jermar 598
            sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
599
            ((vsid >> 30) & 1) ? " supervisor" : "",
600
            ((vsid >> 29) & 1) ? " user" : "");
1733 decky 601
    }
1736 decky 602
 
1780 jermar 603
    uint32_t upper;
604
    uint32_t lower;
605
    uint32_t mask;
606
    uint32_t length;
1736 decky 607
 
608
    PRINT_BAT("ibat[0]", 528, 529);
609
    PRINT_BAT("ibat[1]", 530, 531);
610
    PRINT_BAT("ibat[2]", 532, 533);
611
    PRINT_BAT("ibat[3]", 534, 535);
612
 
613
    PRINT_BAT("dbat[0]", 536, 537);
614
    PRINT_BAT("dbat[1]", 538, 539);
615
    PRINT_BAT("dbat[2]", 540, 541);
616
    PRINT_BAT("dbat[3]", 542, 543);
1215 decky 617
}
1702 cejka 618
 
1730 decky 619
/** @}
1702 cejka 620
 */