Subversion Repositories HelenOS-historic

Rev

Rev 945 | Rev 993 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
740 jermar 1
/*
2
 * Copyright (C) 2006 Jakub Jermar
3
 * All rights reserved.
4
 *
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
7
 * are met:
8
 *
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
28
 
29
/*
30
 * TLB management.
31
 */
32
 
33
#include <mm/tlb.h>
901 jermar 34
#include <mm/asid.h>
902 jermar 35
#include <mm/page.h>
36
#include <mm/as.h>
818 vana 37
#include <arch/mm/tlb.h>
901 jermar 38
#include <arch/mm/page.h>
819 vana 39
#include <arch/barrier.h>
900 jermar 40
#include <arch/interrupt.h>
928 vana 41
#include <arch/pal/pal.h>
42
#include <arch/asm.h>
899 jermar 43
#include <typedefs.h>
900 jermar 44
#include <panic.h>
902 jermar 45
#include <arch.h>
740 jermar 46
 
944 vana 47
 
48
 
756 jermar 49
/** Invalidate all TLB entries. */
740 jermar 50
void tlb_invalidate_all(void)
51
{
928 vana 52
        __address adr;
53
        __u32 count1,count2,stride1,stride2;
54
 
55
        int i,j;
56
 
57
        adr=PAL_PTCE_INFO_BASE();
58
        count1=PAL_PTCE_INFO_COUNT1();
59
        count2=PAL_PTCE_INFO_COUNT2();
60
        stride1=PAL_PTCE_INFO_STRIDE1();
61
        stride2=PAL_PTCE_INFO_STRIDE2();
62
 
63
        interrupts_disable();
64
 
65
        for(i=0;i<count1;i++)
66
        {
67
            for(j=0;j<count2;j++)
68
            {
69
                asm volatile
70
                (
71
                    "ptc.e %0;;"
72
                    :
73
                    :"r" (adr)
74
                );
75
                adr+=stride2;
76
            }
77
            adr+=stride1;
78
        }
79
 
80
        interrupts_enable();
81
 
82
        srlz_d();
83
        srlz_i();
740 jermar 84
}
85
 
86
/** Invalidate entries belonging to an address space.
87
 *
88
 * @param asid Address space identifier.
89
 */
90
void tlb_invalidate_asid(asid_t asid)
91
{
92
    /* TODO */
935 vana 93
    tlb_invalidate_all();
740 jermar 94
}
818 vana 95
 
935 vana 96
 
947 vana 97
void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
935 vana 98
{
99
 
100
 
944 vana 101
    region_register rr;
102
    bool restore_rr = false;
103
    int b=0;
104
    int c=cnt;
105
 
947 vana 106
    __address va;
107
    va=page;
108
 
944 vana 109
    rr.word = rr_read(VA2VRN(va));
110
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
111
        /*
112
         * The selected region register does not contain required RID.
113
         * Save the old content of the register and replace the RID.
114
         */
115
        region_register rr0;
116
 
117
        rr0 = rr;
118
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
119
        rr_write(VA2VRN(va), rr0.word);
120
        srlz_d();
121
        srlz_i();
122
    }
123
 
124
    while(c>>=1)    b++;
125
    b>>=1;
126
    __u64 ps;
127
 
128
    switch(b)
129
    {
130
        case 0: /*cnt 1-3*/
131
        {
132
            ps=PAGE_WIDTH;
133
            break;
134
        }
135
        case 1: /*cnt 4-15*/
136
        {
947 vana 137
            /*cnt=((cnt-1)/4)+1;*/
944 vana 138
            ps=PAGE_WIDTH+2;
139
            va&=~((1<<ps)-1);
140
            break;
141
        }
142
        case 2: /*cnt 16-63*/
143
        {
947 vana 144
            /*cnt=((cnt-1)/16)+1;*/
944 vana 145
            ps=PAGE_WIDTH+4;
146
            va&=~((1<<ps)-1);
147
            break;
148
        }
149
        case 3: /*cnt 64-255*/
150
        {
947 vana 151
            /*cnt=((cnt-1)/64)+1;*/
944 vana 152
            ps=PAGE_WIDTH+6;
153
            va&=~((1<<ps)-1);
154
            break;
155
        }
156
        case 4: /*cnt 256-1023*/
157
        {
947 vana 158
            /*cnt=((cnt-1)/256)+1;*/
944 vana 159
            ps=PAGE_WIDTH+8;
160
            va&=~((1<<ps)-1);
161
            break;
162
        }
163
        case 5: /*cnt 1024-4095*/
164
        {
947 vana 165
            /*cnt=((cnt-1)/1024)+1;*/
944 vana 166
            ps=PAGE_WIDTH+10;
167
            va&=~((1<<ps)-1);
168
            break;
169
        }
170
        case 6: /*cnt 4096-16383*/
171
        {
947 vana 172
            /*cnt=((cnt-1)/4096)+1;*/
944 vana 173
            ps=PAGE_WIDTH+12;
174
            va&=~((1<<ps)-1);
175
            break;
176
        }
177
        case 7: /*cnt 16384-65535*/
178
        case 8: /*cnt 65536-(256K-1)*/
179
        {
947 vana 180
            /*cnt=((cnt-1)/16384)+1;*/
944 vana 181
            ps=PAGE_WIDTH+14;
182
            va&=~((1<<ps)-1);
183
            break;
184
        }
185
        default:
186
        {
947 vana 187
            /*cnt=((cnt-1)/(16384*16))+1;*/
944 vana 188
            ps=PAGE_WIDTH+18;
189
            va&=~((1<<ps)-1);
190
            break;
191
        }
192
    }
947 vana 193
    /*cnt+=(page!=va);*/
194
    for(;va<(page+cnt*(PAGE_SIZE));va+=(1<<ps)) {
195
        __asm__ volatile
196
        (
197
            "ptc.l %0,%1;;"
198
            :
199
            : "r"(va), "r"(ps<<2)
200
        );
944 vana 201
    }
202
    srlz_d();
203
    srlz_i();
204
 
205
 
206
    if (restore_rr) {
207
        rr_write(VA2VRN(va), rr.word);
208
        srlz_d();
209
        srlz_i();
210
    }
211
 
212
 
935 vana 213
}
214
 
215
 
899 jermar 216
/** Insert data into data translation cache.
217
 *
218
 * @param va Virtual page address.
219
 * @param asid Address space identifier.
220
 * @param entry The rest of TLB entry as required by TLB insertion format.
221
 */
919 jermar 222
void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
223
{
899 jermar 224
    tc_mapping_insert(va, asid, entry, true);
225
}
818 vana 226
 
899 jermar 227
/** Insert data into instruction translation cache.
228
 *
229
 * @param va Virtual page address.
230
 * @param asid Address space identifier.
231
 * @param entry The rest of TLB entry as required by TLB insertion format.
232
 */
919 jermar 233
void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
234
{
899 jermar 235
    tc_mapping_insert(va, asid, entry, false);
236
}
818 vana 237
 
899 jermar 238
/** Insert data into instruction or data translation cache.
239
 *
240
 * @param va Virtual page address.
241
 * @param asid Address space identifier.
242
 * @param entry The rest of TLB entry as required by TLB insertion format.
243
 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
244
 */
245
void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
818 vana 246
{
247
    region_register rr;
899 jermar 248
    bool restore_rr = false;
818 vana 249
 
901 jermar 250
    rr.word = rr_read(VA2VRN(va));
251
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
899 jermar 252
        /*
253
         * The selected region register does not contain required RID.
254
         * Save the old content of the register and replace the RID.
255
         */
256
        region_register rr0;
818 vana 257
 
899 jermar 258
        rr0 = rr;
901 jermar 259
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
260
        rr_write(VA2VRN(va), rr0.word);
899 jermar 261
        srlz_d();
262
        srlz_i();
818 vana 263
    }
899 jermar 264
 
265
    __asm__ volatile (
266
        "mov r8=psr;;\n"
900 jermar 267
        "rsm %0;;\n"            /* PSR_IC_MASK */
899 jermar 268
        "srlz.d;;\n"
269
        "srlz.i;;\n"
270
        "mov cr.ifa=%1\n"       /* va */
271
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
272
        "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
273
        "(p6) itc.i %3;;\n"
274
        "(p7) itc.d %3;;\n"
275
        "mov psr.l=r8;;\n"
276
        "srlz.d;;\n"
277
        :
900 jermar 278
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
279
        : "p6", "p7", "r8"
899 jermar 280
    );
281
 
282
    if (restore_rr) {
901 jermar 283
        rr_write(VA2VRN(va), rr.word);
819 vana 284
        srlz_d();
899 jermar 285
        srlz_i();
818 vana 286
    }
899 jermar 287
}
818 vana 288
 
899 jermar 289
/** Insert data into instruction translation register.
290
 *
291
 * @param va Virtual page address.
292
 * @param asid Address space identifier.
293
 * @param entry The rest of TLB entry as required by TLB insertion format.
294
 * @param tr Translation register.
295
 */
296
void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
297
{
298
    tr_mapping_insert(va, asid, entry, false, tr);
299
}
818 vana 300
 
899 jermar 301
/** Insert data into data translation register.
302
 *
303
 * @param va Virtual page address.
304
 * @param asid Address space identifier.
305
 * @param entry The rest of TLB entry as required by TLB insertion format.
306
 * @param tr Translation register.
307
 */
308
void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
309
{
310
    tr_mapping_insert(va, asid, entry, true, tr);
818 vana 311
}
312
 
899 jermar 313
/** Insert data into instruction or data translation register.
314
 *
315
 * @param va Virtual page address.
316
 * @param asid Address space identifier.
317
 * @param entry The rest of TLB entry as required by TLB insertion format.
318
 * @param dtc If true, insert into data translation register, use instruction translation register otherwise.
319
 * @param tr Translation register.
320
 */
321
void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
818 vana 322
{
323
    region_register rr;
899 jermar 324
    bool restore_rr = false;
818 vana 325
 
901 jermar 326
    rr.word = rr_read(VA2VRN(va));
327
    if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
899 jermar 328
        /*
329
         * The selected region register does not contain required RID.
330
         * Save the old content of the register and replace the RID.
331
         */
332
        region_register rr0;
818 vana 333
 
899 jermar 334
        rr0 = rr;
901 jermar 335
        rr0.map.rid = ASID2RID(asid, VA2VRN(va));
336
        rr_write(VA2VRN(va), rr0.word);
899 jermar 337
        srlz_d();
338
        srlz_i();
339
    }
818 vana 340
 
899 jermar 341
    __asm__ volatile (
342
        "mov r8=psr;;\n"
900 jermar 343
        "rsm %0;;\n"            /* PSR_IC_MASK */
899 jermar 344
        "srlz.d;;\n"
345
        "srlz.i;;\n"
346
        "mov cr.ifa=%1\n"           /* va */         
347
        "mov cr.itir=%2;;\n"        /* entry.word[1] */
348
        "cmp.eq p6,p7=%5,r0;;\n"    /* decide between itr and dtr */
349
        "(p6) itr.i itr[%4]=%3;;\n"
350
        "(p7) itr.d dtr[%4]=%3;;\n"
351
        "mov psr.l=r8;;\n"
352
        "srlz.d;;\n"
353
        :
900 jermar 354
        : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
355
        : "p6", "p7", "r8"
899 jermar 356
    );
357
 
358
    if (restore_rr) {
901 jermar 359
        rr_write(VA2VRN(va), rr.word);
819 vana 360
        srlz_d();
899 jermar 361
        srlz_i();
818 vana 362
    }
899 jermar 363
}
818 vana 364
 
901 jermar 365
/** Insert data into DTLB.
366
 *
367
 * @param va Virtual page address.
368
 * @param asid Address space identifier.
369
 * @param entry The rest of TLB entry as required by TLB insertion format.
370
 * @param dtr If true, insert into data translation register, use data translation cache otherwise.
371
 * @param tr Translation register if dtr is true, ignored otherwise.
372
 */
902 jermar 373
void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
901 jermar 374
{
375
    tlb_entry_t entry;
376
 
377
    entry.word[0] = 0;
378
    entry.word[1] = 0;
379
 
380
    entry.p = true;         /* present */
381
    entry.ma = MA_WRITEBACK;
382
    entry.a = true;         /* already accessed */
383
    entry.d = true;         /* already dirty */
384
    entry.pl = PL_KERNEL;
385
    entry.ar = AR_READ | AR_WRITE;
386
    entry.ppn = frame >> PPN_SHIFT;
387
    entry.ps = PAGE_WIDTH;
388
 
389
    if (dtr)
390
        dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
391
    else
392
        dtc_mapping_insert(page, ASID_KERNEL, entry);
393
}
394
 
902 jermar 395
/** Copy content of PTE into data translation cache.
396
 *
397
 * @param t PTE.
398
 */
399
void dtc_pte_copy(pte_t *t)
400
{
401
    tlb_entry_t entry;
402
 
403
    entry.word[0] = 0;
404
    entry.word[1] = 0;
405
 
406
    entry.p = t->p;
407
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
408
    entry.a = t->a;
409
    entry.d = t->d;
410
    entry.pl = t->k ? PL_KERNEL : PL_USER;
411
    entry.ar = t->w ? AR_WRITE : AR_READ;
412
    entry.ppn = t->frame >> PPN_SHIFT;
413
    entry.ps = PAGE_WIDTH;
414
 
415
    dtc_mapping_insert(t->page, t->as->asid, entry);
416
}
417
 
418
/** Copy content of PTE into instruction translation cache.
419
 *
420
 * @param t PTE.
421
 */
422
void itc_pte_copy(pte_t *t)
423
{
424
    tlb_entry_t entry;
425
 
426
    entry.word[0] = 0;
427
    entry.word[1] = 0;
428
 
429
    ASSERT(t->x);
430
 
431
    entry.p = t->p;
432
    entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
433
    entry.a = t->a;
434
    entry.pl = t->k ? PL_KERNEL : PL_USER;
435
    entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
436
    entry.ppn = t->frame >> PPN_SHIFT;
437
    entry.ps = PAGE_WIDTH;
438
 
439
    itc_mapping_insert(t->page, t->as->asid, entry);
440
}
441
 
442
/** Instruction TLB fault handler for faults with VHPT turned off.
443
 *
444
 * @param vector Interruption vector.
445
 * @param pstate Structure with saved interruption state.
446
 */
900 jermar 447
void alternate_instruction_tlb_fault(__u64 vector, struct exception_regdump *pstate)
899 jermar 448
{
902 jermar 449
    region_register rr;
450
    __address va;
451
    pte_t *t;
452
 
453
    va = pstate->cr_ifa;    /* faulting address */
454
    t = page_mapping_find(AS, va);
455
    if (t) {
456
        /*
457
         * The mapping was found in software page hash table.
458
         * Insert it into data translation cache.
459
         */
460
        itc_pte_copy(t);
461
    } else {
462
        /*
463
         * Forward the page fault to address space page fault handler.
464
         */
465
        if (!as_page_fault(va)) {
466
            panic("%s: va=%P, rid=%d\n", __FUNCTION__, pstate->cr_ifa, rr.map.rid);
467
        }
468
    }
899 jermar 469
}
818 vana 470
 
902 jermar 471
/** Data TLB fault handler for faults with VHPT turned off.
901 jermar 472
 *
473
 * @param vector Interruption vector.
474
 * @param pstate Structure with saved interruption state.
475
 */
900 jermar 476
void alternate_data_tlb_fault(__u64 vector, struct exception_regdump *pstate)
899 jermar 477
{
901 jermar 478
    region_register rr;
479
    rid_t rid;
480
    __address va;
902 jermar 481
    pte_t *t;
901 jermar 482
 
483
    va = pstate->cr_ifa;    /* faulting address */
484
    rr.word = rr_read(VA2VRN(va));
485
    rid = rr.map.rid;
486
    if (RID2ASID(rid) == ASID_KERNEL) {
487
        if (VA2VRN(va) == VRN_KERNEL) {
488
            /*
489
             * Provide KA2PA(identity) mapping for faulting piece of
490
             * kernel address space.
491
             */
902 jermar 492
            dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
901 jermar 493
            return;
494
        }
495
    }
919 jermar 496
 
902 jermar 497
    t = page_mapping_find(AS, va);
498
    if (t) {
499
        /*
500
         * The mapping was found in software page hash table.
501
         * Insert it into data translation cache.
502
         */
503
        dtc_pte_copy(t);
504
    } else {
505
        /*
506
         * Forward the page fault to address space page fault handler.
507
         */
508
        if (!as_page_fault(va)) {
509
            panic("%s: va=%P, rid=%d\n", __FUNCTION__, pstate->cr_ifa, rr.map.rid);
510
        }
511
    }
818 vana 512
}
513
 
902 jermar 514
/** Data nested TLB fault handler.
515
 *
516
 * This fault should not occur.
517
 *
518
 * @param vector Interruption vector.
519
 * @param pstate Structure with saved interruption state.
520
 */
900 jermar 521
void data_nested_tlb_fault(__u64 vector, struct exception_regdump *pstate)
899 jermar 522
{
523
    panic("%s\n", __FUNCTION__);
524
}
818 vana 525
 
902 jermar 526
/** Data Dirty bit fault handler.
527
 *
528
 * @param vector Interruption vector.
529
 * @param pstate Structure with saved interruption state.
530
 */
900 jermar 531
void data_dirty_bit_fault(__u64 vector, struct exception_regdump *pstate)
819 vana 532
{
902 jermar 533
    pte_t *t;
534
 
535
    t = page_mapping_find(AS, pstate->cr_ifa);
536
    ASSERT(t && t->p);
537
    if (t && t->p) {
538
        /*
539
         * Update the Dirty bit in page tables and reinsert
540
         * the mapping into DTC.
541
         */
542
        t->d = true;
543
        dtc_pte_copy(t);
544
    }
899 jermar 545
}
819 vana 546
 
902 jermar 547
/** Instruction access bit fault handler.
548
 *
549
 * @param vector Interruption vector.
550
 * @param pstate Structure with saved interruption state.
551
 */
900 jermar 552
void instruction_access_bit_fault(__u64 vector, struct exception_regdump *pstate)
899 jermar 553
{
902 jermar 554
    pte_t *t;
555
 
556
    t = page_mapping_find(AS, pstate->cr_ifa);
557
    ASSERT(t && t->p);
558
    if (t && t->p) {
559
        /*
560
         * Update the Accessed bit in page tables and reinsert
561
         * the mapping into ITC.
562
         */
563
        t->a = true;
564
        itc_pte_copy(t);
565
    }
899 jermar 566
}
819 vana 567
 
902 jermar 568
/** Data access bit fault handler.
569
 *
570
 * @param vector Interruption vector.
571
 * @param pstate Structure with saved interruption state.
572
 */
900 jermar 573
void data_access_bit_fault(__u64 vector, struct exception_regdump *pstate)
899 jermar 574
{
902 jermar 575
    pte_t *t;
576
 
577
    t = page_mapping_find(AS, pstate->cr_ifa);
578
    ASSERT(t && t->p);
579
    if (t && t->p) {
580
        /*
581
         * Update the Accessed bit in page tables and reinsert
582
         * the mapping into DTC.
583
         */
584
        t->a = true;
585
        dtc_pte_copy(t);
586
    }
819 vana 587
}
588
 
902 jermar 589
/** Page not present fault handler.
590
 *
591
 * @param vector Interruption vector.
592
 * @param pstate Structure with saved interruption state.
593
 */
900 jermar 594
void page_not_present(__u64 vector, struct exception_regdump *pstate)
819 vana 595
{
902 jermar 596
    region_register rr;
597
    __address va;
598
    pte_t *t;
599
 
600
    va = pstate->cr_ifa;    /* faulting address */
601
    t = page_mapping_find(AS, va);
602
    ASSERT(t);
603
 
604
    if (t->p) {
605
        /*
606
         * If the Present bit is set in page hash table, just copy it
607
         * and update ITC/DTC.
608
         */
609
        if (t->x)
610
            itc_pte_copy(t);
611
        else
612
            dtc_pte_copy(t);
613
    } else {
614
        if (!as_page_fault(va)) {
615
            panic("%s: va=%P, rid=%d\n", __FUNCTION__, pstate->cr_ifa, rr.map.rid);
616
        }
617
    }
819 vana 618
}