Subversion Repositories HelenOS

Rev

Rev 3022 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1215 decky 1
/*
2071 jermar 2
 * Copyright (c) 2006 Martin Decky
1215 decky 3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
4055 trochtova 29
/** @addtogroup ppc32mm
1702 cejka 30
 * @{
31
 */
32
/** @file
33
 */
34
 
1215 decky 35
#include <mm/tlb.h>
1730 decky 36
#include <arch/mm/tlb.h>
37
#include <arch/interrupt.h>
4055 trochtova 38
#include <interrupt.h>
1730 decky 39
#include <mm/as.h>
40
#include <arch.h>
41
#include <print.h>
42
#include <symtab.h>
4055 trochtova 43
#include <macros.h>
1215 decky 44
 
45
 
4055 trochtova 46
static unsigned int seed = 10;
47
static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
48
 
49
 
50
#define TLB_FLUSH \
51
    "tlbie %0\n" \
52
    "addi %0, %0, 0x1000\n"
53
 
54
 
1730 decky 55
/** Try to find PTE for faulting address
1215 decky 56
 *
1730 decky 57
 * Try to find PTE for faulting address.
58
 * The as->lock must be held on entry to this function
59
 * if lock is true.
1215 decky 60
 *
4055 trochtova 61
 * @param as        Address space.
62
 * @param lock      Lock/unlock the address space.
63
 * @param badvaddr  Faulting virtual address.
64
 * @param access    Access mode that caused the fault.
65
 * @param istate    Pointer to interrupted state.
66
 * @param pfrc      Pointer to variable where as_page_fault() return code
67
 *          will be stored.
68
 * @return      PTE on success, NULL otherwise.
1730 decky 69
 *
1215 decky 70
 */
4055 trochtova 71
static pte_t *
72
find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
73
    istate_t *istate, int *pfrc)
1730 decky 74
{
75
    /*
76
     * Check if the mapping exists in page tables.
77
     */
78
    pte_t *pte = page_mapping_find(as, badvaddr);
4055 trochtova 79
    if ((pte) && (pte->present)) {
1730 decky 80
        /*
81
         * Mapping found in page tables.
82
         * Immediately succeed.
83
         */
84
        return pte;
85
    } else {
86
        int rc;
87
 
88
        /*
89
         * Mapping not found in page tables.
90
         * Resort to higher-level page fault handler.
91
         */
92
        page_table_unlock(as, lock);
93
        switch (rc = as_page_fault(badvaddr, access, istate)) {
4055 trochtova 94
        case AS_PF_OK:
95
            /*
96
             * The higher-level page fault handler succeeded,
97
             * The mapping ought to be in place.
98
             */
99
            page_table_lock(as, lock);
100
            pte = page_mapping_find(as, badvaddr);
101
            ASSERT((pte) && (pte->present));
102
            *pfrc = 0;
103
            return pte;
104
        case AS_PF_DEFER:
105
            page_table_lock(as, lock);
106
            *pfrc = rc;
107
            return NULL;
108
        case AS_PF_FAULT:
109
            page_table_lock(as, lock);
110
            *pfrc = rc;
111
            return NULL;
112
        default:
113
            panic("Unexpected rc (%d).", rc);
1730 decky 114
        }  
115
    }
116
}
117
 
118
 
1780 jermar 119
static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
1730 decky 120
{
121
    char *symbol = "";
122
    char *sym2 = "";
123
 
4055 trochtova 124
    char *str = get_symtab_entry(istate->pc);
125
    if (str)
126
        symbol = str;
127
    str = get_symtab_entry(istate->lr);
128
    if (str)
129
        sym2 = str;
130
 
131
    fault_if_from_uspace(istate,
132
        "PHT Refill Exception on %p.", badvaddr);
133
    panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
134
        istate->pc, symbol, sym2);
1730 decky 135
}
136
 
137
 
4055 trochtova 138
static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
1730 decky 139
{
1780 jermar 140
    uint32_t page = (vaddr >> 12) & 0xffff;
141
    uint32_t api = (vaddr >> 22) & 0x3f;
1730 decky 142
 
1780 jermar 143
    uint32_t vsid;
1730 decky 144
    asm volatile (
145
        "mfsrin %0, %1\n"
146
        : "=r" (vsid)
147
        : "r" (vaddr)
148
    );
149
 
1780 jermar 150
    uint32_t sdr1;
1730 decky 151
    asm volatile (
152
        "mfsdr1 %0\n"
153
        : "=r" (sdr1)
154
    );
155
    phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
156
 
157
    /* Primary hash (xor) */
1780 jermar 158
    uint32_t h = 0;
159
    uint32_t hash = vsid ^ page;
160
    uint32_t base = (hash & 0x3ff) << 3;
161
    uint32_t i;
1730 decky 162
    bool found = false;
163
 
4055 trochtova 164
    /* Find colliding PTE in PTEG */
1730 decky 165
    for (i = 0; i < 8; i++) {
4055 trochtova 166
        if ((phte[base + i].v)
167
            && (phte[base + i].vsid == vsid)
168
            && (phte[base + i].api == api)
169
            && (phte[base + i].h == 0)) {
1730 decky 170
            found = true;
171
            break;
172
        }
173
    }
174
 
175
    if (!found) {
4055 trochtova 176
        /* Find unused PTE in PTEG */
177
        for (i = 0; i < 8; i++) {
178
            if (!phte[base + i].v) {
179
                found = true;
180
                break;
181
            }
182
        }
183
    }
184
 
185
    if (!found) {
1730 decky 186
        /* Secondary hash (not) */
1780 jermar 187
        uint32_t base2 = (~hash & 0x3ff) << 3;
1730 decky 188
 
4055 trochtova 189
        /* Find colliding PTE in PTEG */
1730 decky 190
        for (i = 0; i < 8; i++) {
4055 trochtova 191
            if ((phte[base2 + i].v)
192
                && (phte[base2 + i].vsid == vsid)
193
                && (phte[base2 + i].api == api)
194
                && (phte[base2 + i].h == 1)) {
1730 decky 195
                found = true;
196
                base = base2;
197
                h = 1;
198
                break;
199
            }
200
        }
201
 
202
        if (!found) {
4055 trochtova 203
            /* Find unused PTE in PTEG */
204
            for (i = 0; i < 8; i++) {
205
                if (!phte[base2 + i].v) {
206
                    found = true;
207
                    base = base2;
208
                    h = 1;
209
                    break;
210
                }
211
            }
1730 decky 212
        }
4055 trochtova 213
 
214
        if (!found)
215
            i = RANDI(seed) % 8;
1730 decky 216
    }
217
 
218
    phte[base + i].v = 1;
219
    phte[base + i].vsid = vsid;
220
    phte[base + i].h = h;
221
    phte[base + i].api = api;
4055 trochtova 222
    phte[base + i].rpn = pte->pfn;
1730 decky 223
    phte[base + i].r = 0;
224
    phte[base + i].c = 0;
4055 trochtova 225
    phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
1730 decky 226
    phte[base + i].pp = 2; // FIXME
227
}
228
 
229
 
4055 trochtova 230
/** Process Instruction/Data Storage Exception
231
 *
232
 * @param n      Exception vector number.
233
 * @param istate Interrupted register context.
234
 *
235
 */
236
void pht_refill(int n, istate_t *istate)
1730 decky 237
{
4055 trochtova 238
    uintptr_t badvaddr;
239
    pte_t *pte;
240
    int pfrc;
241
    as_t *as;
242
    bool lock;
1730 decky 243
 
4055 trochtova 244
    if (AS == NULL) {
245
        as = AS_KERNEL;
246
        lock = false;
247
    } else {
248
        as = AS;
249
        lock = true;
250
    }
251
 
252
    if (n == VECTOR_DATA_STORAGE)
253
        badvaddr = istate->dar;
254
    else
255
        badvaddr = istate->pc;
256
 
257
    page_table_lock(as, lock);
258
 
259
    pte = find_mapping_and_check(as, lock, badvaddr,
260
        PF_ACCESS_READ /* FIXME */, istate, &pfrc);
261
    if (!pte) {
262
        switch (pfrc) {
263
        case AS_PF_FAULT:
264
            goto fail;
265
            break;
266
        case AS_PF_DEFER:
267
            /*
268
             * The page fault came during copy_from_uspace()
269
             * or copy_to_uspace().
270
             */
271
            page_table_unlock(as, lock);
272
            return;
273
        default:
274
            panic("Unexpected pfrc (%d).", pfrc);
275
        }
276
    }
277
 
278
    pte->accessed = 1; /* Record access to PTE */
279
    pht_insert(badvaddr, pte);
280
 
281
    page_table_unlock(as, lock);
282
    return;
283
 
284
fail:
285
    page_table_unlock(as, lock);
286
    pht_refill_fail(badvaddr, istate);
287
}
288
 
289
 
290
/** Process Instruction/Data Storage Exception in Real Mode
291
 *
292
 * @param n      Exception vector number.
293
 * @param istate Interrupted register context.
294
 *
295
 */
296
bool pht_refill_real(int n, istate_t *istate)
297
{
298
    uintptr_t badvaddr;
299
 
300
    if (n == VECTOR_DATA_STORAGE)
301
        badvaddr = istate->dar;
302
    else
303
        badvaddr = istate->pc;
304
 
305
    uint32_t physmem;
306
    asm volatile (
307
        "mfsprg3 %0\n"
308
        : "=r" (physmem)
309
    );
310
 
311
    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
312
        return false;
313
 
314
    uint32_t page = (badvaddr >> 12) & 0xffff;
315
    uint32_t api = (badvaddr >> 22) & 0x3f;
316
 
1780 jermar 317
    uint32_t vsid;
1730 decky 318
    asm volatile (
319
        "mfsrin %0, %1\n"
320
        : "=r" (vsid)
4055 trochtova 321
        : "r" (badvaddr)
1730 decky 322
    );
323
 
1780 jermar 324
    uint32_t sdr1;
1730 decky 325
    asm volatile (
326
        "mfsdr1 %0\n"
327
        : "=r" (sdr1)
328
    );
4055 trochtova 329
    phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
1730 decky 330
 
331
    /* Primary hash (xor) */
1780 jermar 332
    uint32_t h = 0;
333
    uint32_t hash = vsid ^ page;
334
    uint32_t base = (hash & 0x3ff) << 3;
335
    uint32_t i;
1730 decky 336
    bool found = false;
337
 
4055 trochtova 338
    /* Find colliding PTE in PTEG */
1730 decky 339
    for (i = 0; i < 8; i++) {
4055 trochtova 340
        if ((phte_real[base + i].v)
341
            && (phte_real[base + i].vsid == vsid)
342
            && (phte_real[base + i].api == api)
343
            && (phte_real[base + i].h == 0)) {
1730 decky 344
            found = true;
345
            break;
346
        }
347
    }
348
 
349
    if (!found) {
4055 trochtova 350
        /* Find unused PTE in PTEG */
351
        for (i = 0; i < 8; i++) {
352
            if (!phte_real[base + i].v) {
353
                found = true;
354
                break;
355
            }
356
        }
357
    }
358
 
359
    if (!found) {
1730 decky 360
        /* Secondary hash (not) */
1780 jermar 361
        uint32_t base2 = (~hash & 0x3ff) << 3;
1730 decky 362
 
4055 trochtova 363
        /* Find colliding PTE in PTEG */
1730 decky 364
        for (i = 0; i < 8; i++) {
4055 trochtova 365
            if ((phte_real[base2 + i].v)
366
                && (phte_real[base2 + i].vsid == vsid)
367
                && (phte_real[base2 + i].api == api)
368
                && (phte_real[base2 + i].h == 1)) {
1730 decky 369
                found = true;
370
                base = base2;
371
                h = 1;
372
                break;
373
            }
374
        }
375
 
376
        if (!found) {
4055 trochtova 377
            /* Find unused PTE in PTEG */
378
            for (i = 0; i < 8; i++) {
379
                if (!phte_real[base2 + i].v) {
380
                    found = true;
381
                    base = base2;
382
                    h = 1;
383
                    break;
384
                }
385
            }
1730 decky 386
        }
387
 
4055 trochtova 388
        if (!found) {
389
            /* Use secondary hash to avoid collisions
390
               with usual PHT refill handler. */
391
            i = RANDI(seed_real) % 8;
392
            base = base2;
393
            h = 1;
1730 decky 394
        }
395
    }
396
 
4055 trochtova 397
    phte_real[base + i].v = 1;
398
    phte_real[base + i].vsid = vsid;
399
    phte_real[base + i].h = h;
400
    phte_real[base + i].api = api;
401
    phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
402
    phte_real[base + i].r = 0;
403
    phte_real[base + i].c = 0;
404
    phte_real[base + i].wimg = 0;
405
    phte_real[base + i].pp = 2; // FIXME
1730 decky 406
 
4055 trochtova 407
    return true;
1730 decky 408
}
409
 
410
 
4055 trochtova 411
/** Process ITLB/DTLB Miss Exception in Real Mode
1730 decky 412
 *
413
 *
414
 */
4055 trochtova 415
void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
1730 decky 416
{
4055 trochtova 417
    uint32_t badvaddr = tlbmiss & 0xfffffffc;
1730 decky 418
 
1780 jermar 419
    uint32_t physmem;
1730 decky 420
    asm volatile (
421
        "mfsprg3 %0\n"
422
        : "=r" (physmem)
423
    );
424
 
4055 trochtova 425
    if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
426
        return; // FIXME
1730 decky 427
 
4055 trochtova 428
    ptelo.rpn = KA2PA(badvaddr) >> 12;
429
    ptelo.wimg = 0;
430
    ptelo.pp = 2; // FIXME
431
 
432
    uint32_t index = 0;
433
    asm volatile (
434
        "mtspr 981, %0\n"
435
        "mtspr 982, %1\n"
436
        "tlbld %2\n"
437
        "tlbli %2\n"
438
        : "=r" (index)
439
        : "r" (ptehi),
440
          "r" (ptelo)
441
    );
1730 decky 442
}
443
 
444
 
1215 decky 445
void tlb_arch_init(void)
446
{
1384 decky 447
    tlb_invalidate_all();
448
}
449
 
450
 
451
void tlb_invalidate_all(void)
452
{
4055 trochtova 453
    uint32_t index;
1269 decky 454
    asm volatile (
4055 trochtova 455
        "li %0, 0\n"
456
        "sync\n"
457
 
458
        TLB_FLUSH
459
        TLB_FLUSH
460
        TLB_FLUSH
461
        TLB_FLUSH
462
        TLB_FLUSH
463
        TLB_FLUSH
464
        TLB_FLUSH
465
        TLB_FLUSH
466
 
467
        TLB_FLUSH
468
        TLB_FLUSH
469
        TLB_FLUSH
470
        TLB_FLUSH
471
        TLB_FLUSH
472
        TLB_FLUSH
473
        TLB_FLUSH
474
        TLB_FLUSH
475
 
476
        TLB_FLUSH
477
        TLB_FLUSH
478
        TLB_FLUSH
479
        TLB_FLUSH
480
        TLB_FLUSH
481
        TLB_FLUSH
482
        TLB_FLUSH
483
        TLB_FLUSH
484
 
485
        TLB_FLUSH
486
        TLB_FLUSH
487
        TLB_FLUSH
488
        TLB_FLUSH
489
        TLB_FLUSH
490
        TLB_FLUSH
491
        TLB_FLUSH
492
        TLB_FLUSH
493
 
494
        TLB_FLUSH
495
        TLB_FLUSH
496
        TLB_FLUSH
497
        TLB_FLUSH
498
        TLB_FLUSH
499
        TLB_FLUSH
500
        TLB_FLUSH
501
        TLB_FLUSH
502
 
503
        TLB_FLUSH
504
        TLB_FLUSH
505
        TLB_FLUSH
506
        TLB_FLUSH
507
        TLB_FLUSH
508
        TLB_FLUSH
509
        TLB_FLUSH
510
        TLB_FLUSH
511
 
512
        TLB_FLUSH
513
        TLB_FLUSH
514
        TLB_FLUSH
515
        TLB_FLUSH
516
        TLB_FLUSH
517
        TLB_FLUSH
518
        TLB_FLUSH
519
        TLB_FLUSH
520
 
521
        TLB_FLUSH
522
        TLB_FLUSH
523
        TLB_FLUSH
524
        TLB_FLUSH
525
        TLB_FLUSH
526
        TLB_FLUSH
527
        TLB_FLUSH
528
        TLB_FLUSH
529
 
530
        "eieio\n"
1384 decky 531
        "tlbsync\n"
4055 trochtova 532
        "sync\n"
533
        : "=r" (index)
1269 decky 534
    );
1215 decky 535
}
536
 
537
 
1384 decky 538
void tlb_invalidate_asid(asid_t asid)
1328 decky 539
{
1780 jermar 540
    uint32_t sdr1;
1758 decky 541
    asm volatile (
542
        "mfsdr1 %0\n"
543
        : "=r" (sdr1)
544
    );
545
    phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
546
 
1780 jermar 547
    uint32_t i;
1758 decky 548
    for (i = 0; i < 8192; i++) {
4055 trochtova 549
        if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
550
            (phte[i].vsid < ((asid << 4) + 16)))
1758 decky 551
            phte[i].v = 0;
552
    }
1384 decky 553
    tlb_invalidate_all();
1328 decky 554
}
555
 
1730 decky 556
 
1780 jermar 557
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
1384 decky 558
{
1730 decky 559
    // TODO
1384 decky 560
    tlb_invalidate_all();
561
}
1328 decky 562
 
1384 decky 563
 
1736 decky 564
#define PRINT_BAT(name, ureg, lreg) \
565
    asm volatile ( \
566
        "mfspr %0," #ureg "\n" \
567
        "mfspr %1," #lreg "\n" \
568
        : "=r" (upper), "=r" (lower) \
569
    ); \
570
    mask = (upper & 0x1ffc) >> 2; \
571
    if (upper & 3) { \
1780 jermar 572
        uint32_t tmp = mask; \
1736 decky 573
        length = 128; \
574
        while (tmp) { \
575
            if ((tmp & 1) == 0) { \
576
                printf("ibat[0]: error in mask\n"); \
577
                break; \
578
            } \
579
            length <<= 1; \
580
            tmp >>= 1; \
581
        } \
582
    } else \
583
        length = 0; \
4055 trochtova 584
    printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
585
        sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
586
        lower & 0xffff0000, length, mask, \
587
        ((upper >> 1) & 1) ? " supervisor" : "", \
588
        (upper & 1) ? " user" : "");
1736 decky 589
 
590
 
1215 decky 591
void tlb_print(void)
592
{
1780 jermar 593
    uint32_t sr;
1733 decky 594
 
595
    for (sr = 0; sr < 16; sr++) {
1780 jermar 596
        uint32_t vsid;
1733 decky 597
        asm volatile (
598
            "mfsrin %0, %1\n"
599
            : "=r" (vsid)
600
            : "r" (sr << 28)
601
        );
4055 trochtova 602
        printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
603
            sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
604
            ((vsid >> 30) & 1) ? " supervisor" : "",
605
            ((vsid >> 29) & 1) ? " user" : "");
1733 decky 606
    }
1736 decky 607
 
1780 jermar 608
    uint32_t upper;
609
    uint32_t lower;
610
    uint32_t mask;
611
    uint32_t length;
1736 decky 612
 
613
    PRINT_BAT("ibat[0]", 528, 529);
614
    PRINT_BAT("ibat[1]", 530, 531);
615
    PRINT_BAT("ibat[2]", 532, 533);
616
    PRINT_BAT("ibat[3]", 534, 535);
617
 
618
    PRINT_BAT("dbat[0]", 536, 537);
619
    PRINT_BAT("dbat[1]", 538, 539);
620
    PRINT_BAT("dbat[2]", 540, 541);
621
    PRINT_BAT("dbat[3]", 542, 543);
1215 decky 622
}
1702 cejka 623
 
1730 decky 624
/** @}
1702 cejka 625
 */