Subversion Repositories HelenOS

Rev

Rev 3672 | Rev 3790 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 3672 Rev 3788
1
/*
1
/*
2
 * Copyright (c) 2005 Jakub Jermar
2
 * Copyright (c) 2005 Jakub Jermar
3
 * All rights reserved.
3
 * All rights reserved.
4
 *
4
 *
5
 * Redistribution and use in source and binary forms, with or without
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
6
 * modification, are permitted provided that the following conditions
7
 * are met:
7
 * are met:
8
 *
8
 *
9
 * - Redistributions of source code must retain the above copyright
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
15
 *   derived from this software without specific prior written permission.
16
 *
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
27
 */
28
 
28
 
29
/** @addtogroup sparc64mm  
29
/** @addtogroup sparc64mm  
30
 * @{
30
 * @{
31
 */
31
 */
32
/** @file
32
/** @file
33
 */
33
 */
34
 
34
 
35
#include <arch/mm/tlb.h>
35
#include <arch/mm/tlb.h>
36
#include <mm/tlb.h>
36
#include <mm/tlb.h>
37
#include <mm/as.h>
37
#include <mm/as.h>
38
#include <mm/asid.h>
38
#include <mm/asid.h>
39
#include <arch/mm/frame.h>
39
#include <arch/mm/frame.h>
40
#include <arch/mm/page.h>
40
#include <arch/mm/page.h>
41
#include <arch/mm/mmu.h>
41
#include <arch/mm/mmu.h>
42
#include <arch/interrupt.h>
42
#include <arch/interrupt.h>
43
#include <interrupt.h>
43
#include <interrupt.h>
44
#include <arch.h>
44
#include <arch.h>
45
#include <print.h>
45
#include <print.h>
46
#include <arch/types.h>
46
#include <arch/types.h>
47
#include <config.h>
47
#include <config.h>
48
#include <arch/trap/trap.h>
48
#include <arch/trap/trap.h>
49
#include <arch/trap/exception.h>
49
#include <arch/trap/exception.h>
50
#include <panic.h>
50
#include <panic.h>
51
#include <arch/asm.h>
51
#include <arch/asm.h>
52
 
52
 
53
#ifdef CONFIG_TSB
53
#ifdef CONFIG_TSB
54
#include <arch/mm/tsb.h>
54
#include <arch/mm/tsb.h>
55
#endif
55
#endif
56
 
56
 
57
static void dtlb_pte_copy(pte_t *, index_t, bool);
57
static void dtlb_pte_copy(pte_t *, index_t, bool);
58
static void itlb_pte_copy(pte_t *, index_t);
58
static void itlb_pte_copy(pte_t *, index_t);
59
static void do_fast_instruction_access_mmu_miss_fault(istate_t *, const char *);
59
static void do_fast_instruction_access_mmu_miss_fault(istate_t *, const char *);
60
static void do_fast_data_access_mmu_miss_fault(istate_t *, tlb_tag_access_reg_t,
60
static void do_fast_data_access_mmu_miss_fault(istate_t *, tlb_tag_access_reg_t,
61
    const char *);
61
    const char *);
62
static void do_fast_data_access_protection_fault(istate_t *,
62
static void do_fast_data_access_protection_fault(istate_t *,
63
    tlb_tag_access_reg_t, const char *);
63
    tlb_tag_access_reg_t, const char *);
64
 
64
 
65
char *context_encoding[] = {
65
char *context_encoding[] = {
66
    "Primary",
66
    "Primary",
67
    "Secondary",
67
    "Secondary",
68
    "Nucleus",
68
    "Nucleus",
69
    "Reserved"
69
    "Reserved"
70
};
70
};
71
 
71
 
72
void tlb_arch_init(void)
72
void tlb_arch_init(void)
73
{
73
{
74
    /*
74
    /*
75
     * Invalidate all non-locked DTLB and ITLB entries.
75
     * Invalidate all non-locked DTLB and ITLB entries.
76
     */
76
     */
77
    tlb_invalidate_all();
77
    tlb_invalidate_all();
78
 
78
 
79
    /*
79
    /*
80
     * Clear both SFSRs.
80
     * Clear both SFSRs.
81
     */
81
     */
82
    dtlb_sfsr_write(0);
82
    dtlb_sfsr_write(0);
83
    itlb_sfsr_write(0);
83
    itlb_sfsr_write(0);
84
}
84
}
85
 
85
 
86
/** Insert privileged mapping into DMMU TLB.
86
/** Insert privileged mapping into DMMU TLB.
87
 *
87
 *
88
 * @param page      Virtual page address.
88
 * @param page      Virtual page address.
89
 * @param frame     Physical frame address.
89
 * @param frame     Physical frame address.
90
 * @param pagesize  Page size.
90
 * @param pagesize  Page size.
91
 * @param locked    True for permanent mappings, false otherwise.
91
 * @param locked    True for permanent mappings, false otherwise.
92
 * @param cacheable True if the mapping is cacheable, false otherwise.
92
 * @param cacheable True if the mapping is cacheable, false otherwise.
93
 */
93
 */
94
void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize,
94
void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize,
95
    bool locked, bool cacheable)
95
    bool locked, bool cacheable)
96
{
96
{
97
    tlb_tag_access_reg_t tag;
97
    tlb_tag_access_reg_t tag;
98
    tlb_data_t data;
98
    tlb_data_t data;
99
    page_address_t pg;
99
    page_address_t pg;
100
    frame_address_t fr;
100
    frame_address_t fr;
101
 
101
 
102
    pg.address = page;
102
    pg.address = page;
103
    fr.address = frame;
103
    fr.address = frame;
104
 
104
 
105
    tag.context = ASID_KERNEL;
105
    tag.context = ASID_KERNEL;
106
    tag.vpn = pg.vpn;
106
    tag.vpn = pg.vpn;
107
 
107
 
108
    dtlb_tag_access_write(tag.value);
108
    dtlb_tag_access_write(tag.value);
109
 
109
 
110
    data.value = 0;
110
    data.value = 0;
111
    data.v = true;
111
    data.v = true;
112
    data.size = pagesize;
112
    data.size = pagesize;
113
    data.pfn = fr.pfn;
113
    data.pfn = fr.pfn;
114
    data.l = locked;
114
    data.l = locked;
115
    data.cp = cacheable;
115
    data.cp = cacheable;
116
#ifdef CONFIG_VIRT_IDX_DCACHE
116
#ifdef CONFIG_VIRT_IDX_DCACHE
117
    data.cv = cacheable;
117
    data.cv = cacheable;
118
#endif /* CONFIG_VIRT_IDX_DCACHE */
118
#endif /* CONFIG_VIRT_IDX_DCACHE */
119
    data.p = true;
119
    data.p = true;
120
    data.w = true;
120
    data.w = true;
121
    data.g = false;
121
    data.g = false;
122
 
122
 
123
    dtlb_data_in_write(data.value);
123
    dtlb_data_in_write(data.value);
124
}
124
}
125
 
125
 
126
/** Copy PTE to TLB.
126
/** Copy PTE to TLB.
127
 *
127
 *
128
 * @param t         Page Table Entry to be copied.
128
 * @param t         Page Table Entry to be copied.
129
 * @param index     Zero if lower 8K-subpage, one if higher 8K-subpage.
129
 * @param index     Zero if lower 8K-subpage, one if higher 8K-subpage.
130
 * @param ro        If true, the entry will be created read-only, regardless
130
 * @param ro        If true, the entry will be created read-only, regardless
131
 *          of its w field.
131
 *          of its w field.
132
 */
132
 */
133
void dtlb_pte_copy(pte_t *t, index_t index, bool ro)
133
void dtlb_pte_copy(pte_t *t, index_t index, bool ro)
134
{
134
{
135
    tlb_tag_access_reg_t tag;
135
    tlb_tag_access_reg_t tag;
136
    tlb_data_t data;
136
    tlb_data_t data;
137
    page_address_t pg;
137
    page_address_t pg;
138
    frame_address_t fr;
138
    frame_address_t fr;
139
 
139
 
140
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
140
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
141
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
141
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
142
 
142
 
143
    tag.value = 0;
143
    tag.value = 0;
144
    tag.context = t->as->asid;
144
    tag.context = t->as->asid;
145
    tag.vpn = pg.vpn;
145
    tag.vpn = pg.vpn;
146
 
146
 
147
    dtlb_tag_access_write(tag.value);
147
    dtlb_tag_access_write(tag.value);
148
 
148
 
149
    data.value = 0;
149
    data.value = 0;
150
    data.v = true;
150
    data.v = true;
151
    data.size = PAGESIZE_8K;
151
    data.size = PAGESIZE_8K;
152
    data.pfn = fr.pfn;
152
    data.pfn = fr.pfn;
153
    data.l = false;
153
    data.l = false;
154
    data.cp = t->c;
154
    data.cp = t->c;
155
#ifdef CONFIG_VIRT_IDX_DCACHE
155
#ifdef CONFIG_VIRT_IDX_DCACHE
156
    data.cv = t->c;
156
    data.cv = t->c;
157
#endif /* CONFIG_VIRT_IDX_DCACHE */
157
#endif /* CONFIG_VIRT_IDX_DCACHE */
158
    data.p = t->k;      /* p like privileged */
158
    data.p = t->k;      /* p like privileged */
159
    data.w = ro ? false : t->w;
159
    data.w = ro ? false : t->w;
160
    data.g = t->g;
160
    data.g = t->g;
161
 
161
 
162
    dtlb_data_in_write(data.value);
162
    dtlb_data_in_write(data.value);
163
}
163
}
164
 
164
 
165
/** Copy PTE to ITLB.
165
/** Copy PTE to ITLB.
166
 *
166
 *
167
 * @param t     Page Table Entry to be copied.
167
 * @param t     Page Table Entry to be copied.
168
 * @param index     Zero if lower 8K-subpage, one if higher 8K-subpage.
168
 * @param index     Zero if lower 8K-subpage, one if higher 8K-subpage.
169
 */
169
 */
170
void itlb_pte_copy(pte_t *t, index_t index)
170
void itlb_pte_copy(pte_t *t, index_t index)
171
{
171
{
172
    tlb_tag_access_reg_t tag;
172
    tlb_tag_access_reg_t tag;
173
    tlb_data_t data;
173
    tlb_data_t data;
174
    page_address_t pg;
174
    page_address_t pg;
175
    frame_address_t fr;
175
    frame_address_t fr;
176
 
176
 
177
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
177
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
178
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
178
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
179
 
179
 
180
    tag.value = 0;
180
    tag.value = 0;
181
    tag.context = t->as->asid;
181
    tag.context = t->as->asid;
182
    tag.vpn = pg.vpn;
182
    tag.vpn = pg.vpn;
183
   
183
   
184
    itlb_tag_access_write(tag.value);
184
    itlb_tag_access_write(tag.value);
185
   
185
   
186
    data.value = 0;
186
    data.value = 0;
187
    data.v = true;
187
    data.v = true;
188
    data.size = PAGESIZE_8K;
188
    data.size = PAGESIZE_8K;
189
    data.pfn = fr.pfn;
189
    data.pfn = fr.pfn;
190
    data.l = false;
190
    data.l = false;
191
    data.cp = t->c;
191
    data.cp = t->c;
192
    data.p = t->k;      /* p like privileged */
192
    data.p = t->k;      /* p like privileged */
193
    data.w = false;
193
    data.w = false;
194
    data.g = t->g;
194
    data.g = t->g;
195
   
195
   
196
    itlb_data_in_write(data.value);
196
    itlb_data_in_write(data.value);
197
}
197
}
198
 
198
 
199
/** ITLB miss handler. */
199
/** ITLB miss handler. */
200
void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate)
200
void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate)
201
{
201
{
202
    uintptr_t va = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
202
    uintptr_t va = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
203
    index_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE;
203
    index_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE;
204
    pte_t *t;
204
    pte_t *t;
205
 
205
 
206
    page_table_lock(AS, true);
206
    page_table_lock(AS, true);
207
    t = page_mapping_find(AS, va);
207
    t = page_mapping_find(AS, va);
208
    if (t && PTE_EXECUTABLE(t)) {
208
    if (t && PTE_EXECUTABLE(t)) {
209
        /*
209
        /*
210
         * The mapping was found in the software page hash table.
210
         * The mapping was found in the software page hash table.
211
         * Insert it into ITLB.
211
         * Insert it into ITLB.
212
         */
212
         */
213
        t->a = true;
213
        t->a = true;
214
        itlb_pte_copy(t, index);
214
        itlb_pte_copy(t, index);
215
#ifdef CONFIG_TSB
215
#ifdef CONFIG_TSB
216
        itsb_pte_copy(t, index);
216
        itsb_pte_copy(t, index);
217
#endif
217
#endif
218
        page_table_unlock(AS, true);
218
        page_table_unlock(AS, true);
219
    } else {
219
    } else {
220
        /*
220
        /*
221
         * Forward the page fault to the address space page fault
221
         * Forward the page fault to the address space page fault
222
         * handler.
222
         * handler.
223
         */    
223
         */    
224
        page_table_unlock(AS, true);
224
        page_table_unlock(AS, true);
225
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
225
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
226
            do_fast_instruction_access_mmu_miss_fault(istate,
226
            do_fast_instruction_access_mmu_miss_fault(istate,
227
                __func__);
227
                __func__);
228
        }
228
        }
229
    }
229
    }
230
}
230
}
231
 
231
 
232
/** DTLB miss handler.
232
/** DTLB miss handler.
233
 *
233
 *
234
 * Note that some faults (e.g. kernel faults) were already resolved by the
234
 * Note that some faults (e.g. kernel faults) were already resolved by the
235
 * low-level, assembly language part of the fast_data_access_mmu_miss handler.
235
 * low-level, assembly language part of the fast_data_access_mmu_miss handler.
236
 *
236
 *
237
 * @param tag       Content of the TLB Tag Access register as it existed
237
 * @param tag       Content of the TLB Tag Access register as it existed
238
 *          when the trap happened. This is to prevent confusion
238
 *          when the trap happened. This is to prevent confusion
239
 *          created by clobbered Tag Access register during a nested
239
 *          created by clobbered Tag Access register during a nested
240
 *          DTLB miss.
240
 *          DTLB miss.
241
 * @param istate    Interrupted state saved on the stack.
241
 * @param istate    Interrupted state saved on the stack.
242
 */
242
 */
243
void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate)
243
void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate)
244
{
244
{
245
    uintptr_t va;
245
    uintptr_t va;
246
    index_t index;
246
    index_t index;
247
    pte_t *t;
247
    pte_t *t;
248
 
248
 
249
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
249
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
250
    index = tag.vpn % MMU_PAGES_PER_PAGE;
250
    index = tag.vpn % MMU_PAGES_PER_PAGE;
251
 
251
 
252
    if (tag.context == ASID_KERNEL) {
252
    if (tag.context == ASID_KERNEL) {
253
        if (!tag.vpn) {
253
        if (!tag.vpn) {
254
            /* NULL access in kernel */
254
            /* NULL access in kernel */
255
            do_fast_data_access_mmu_miss_fault(istate, tag,
255
            do_fast_data_access_mmu_miss_fault(istate, tag,
256
                __func__);
256
                __func__);
257
        }
257
        }
258
        do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected "
258
        do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected "
259
            "kernel page fault.");
259
            "kernel page fault.");
260
    }
260
    }
261
 
261
 
262
    page_table_lock(AS, true);
262
    page_table_lock(AS, true);
263
    t = page_mapping_find(AS, va);
263
    t = page_mapping_find(AS, va);
264
    if (t) {
264
    if (t) {
265
        /*
265
        /*
266
         * The mapping was found in the software page hash table.
266
         * The mapping was found in the software page hash table.
267
         * Insert it into DTLB.
267
         * Insert it into DTLB.
268
         */
268
         */
269
        t->a = true;
269
        t->a = true;
270
        dtlb_pte_copy(t, index, true);
270
        dtlb_pte_copy(t, index, true);
271
#ifdef CONFIG_TSB
271
#ifdef CONFIG_TSB
272
        dtsb_pte_copy(t, index, true);
272
        dtsb_pte_copy(t, index, true);
273
#endif
273
#endif
274
        page_table_unlock(AS, true);
274
        page_table_unlock(AS, true);
275
    } else {
275
    } else {
276
        /*
276
        /*
277
         * Forward the page fault to the address space page fault
277
         * Forward the page fault to the address space page fault
278
         * handler.
278
         * handler.
279
         */    
279
         */    
280
        page_table_unlock(AS, true);
280
        page_table_unlock(AS, true);
281
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
281
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
282
            do_fast_data_access_mmu_miss_fault(istate, tag,
282
            do_fast_data_access_mmu_miss_fault(istate, tag,
283
                __func__);
283
                __func__);
284
        }
284
        }
285
    }
285
    }
286
}
286
}
287
 
287
 
288
/** DTLB protection fault handler.
288
/** DTLB protection fault handler.
289
 *
289
 *
290
 * @param tag       Content of the TLB Tag Access register as it existed
290
 * @param tag       Content of the TLB Tag Access register as it existed
291
 *          when the trap happened. This is to prevent confusion
291
 *          when the trap happened. This is to prevent confusion
292
 *          created by clobbered Tag Access register during a nested
292
 *          created by clobbered Tag Access register during a nested
293
 *          DTLB miss.
293
 *          DTLB miss.
294
 * @param istate    Interrupted state saved on the stack.
294
 * @param istate    Interrupted state saved on the stack.
295
 */
295
 */
296
void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate)
296
void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate)
297
{
297
{
298
    uintptr_t va;
298
    uintptr_t va;
299
    index_t index;
299
    index_t index;
300
    pte_t *t;
300
    pte_t *t;
301
 
301
 
302
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
302
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
303
    index = tag.vpn % MMU_PAGES_PER_PAGE;   /* 16K-page emulation */
303
    index = tag.vpn % MMU_PAGES_PER_PAGE;   /* 16K-page emulation */
304
 
304
 
305
    page_table_lock(AS, true);
305
    page_table_lock(AS, true);
306
    t = page_mapping_find(AS, va);
306
    t = page_mapping_find(AS, va);
307
    if (t && PTE_WRITABLE(t)) {
307
    if (t && PTE_WRITABLE(t)) {
308
        /*
308
        /*
309
         * The mapping was found in the software page hash table and is
309
         * The mapping was found in the software page hash table and is
310
         * writable. Demap the old mapping and insert an updated mapping
310
         * writable. Demap the old mapping and insert an updated mapping
311
         * into DTLB.
311
         * into DTLB.
312
         */
312
         */
313
        t->a = true;
313
        t->a = true;
314
        t->d = true;
314
        t->d = true;
315
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY,
315
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY,
316
            va + index * MMU_PAGE_SIZE);
316
            va + index * MMU_PAGE_SIZE);
317
        dtlb_pte_copy(t, index, false);
317
        dtlb_pte_copy(t, index, false);
318
#ifdef CONFIG_TSB
318
#ifdef CONFIG_TSB
319
        dtsb_pte_copy(t, index, false);
319
        dtsb_pte_copy(t, index, false);
320
#endif
320
#endif
321
        page_table_unlock(AS, true);
321
        page_table_unlock(AS, true);
322
    } else {
322
    } else {
323
        /*
323
        /*
324
         * Forward the page fault to the address space page fault
324
         * Forward the page fault to the address space page fault
325
         * handler.
325
         * handler.
326
         */    
326
         */    
327
        page_table_unlock(AS, true);
327
        page_table_unlock(AS, true);
328
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
328
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
329
            do_fast_data_access_protection_fault(istate, tag,
329
            do_fast_data_access_protection_fault(istate, tag,
330
                __func__);
330
                __func__);
331
        }
331
        }
332
    }
332
    }
333
}
333
}
334
 
334
 
335
/** Print TLB entry (for debugging purposes).
335
/** Print TLB entry (for debugging purposes).
336
 *
336
 *
337
 * The diag field has been left out in order to make this function more generic
337
 * The diag field has been left out in order to make this function more generic
338
 * (there is no diag field in US3 architeture).
338
 * (there is no diag field in US3 architeture).
339
 *
339
 *
340
 * @param i     TLB entry number
340
 * @param i     TLB entry number
341
 * @param t     TLB entry tag
341
 * @param t     TLB entry tag
342
 * @param d     TLB entry data
342
 * @param d     TLB entry data
343
 */
343
 */
344
static void print_tlb_entry(int i, tlb_tag_read_reg_t t, tlb_data_t d)
344
static void print_tlb_entry(int i, tlb_tag_read_reg_t t, tlb_data_t d)
345
{
345
{
346
    printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, "
346
    printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, "
347
        "ie=%d, soft2=%#x, pfn=%#x, soft=%#x, l=%d, "
347
        "ie=%d, soft2=%#x, pfn=%#x, soft=%#x, l=%d, "
348
        "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn,
348
        "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn,
349
        t.context, d.v, d.size, d.nfo, d.ie, d.soft2,
349
        t.context, d.v, d.size, d.nfo, d.ie, d.soft2,
350
        d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
350
        d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
351
}
351
}
352
 
352
 
353
#if defined (US)
353
#if defined (US)
354
 
354
 
355
/** Print contents of both TLBs. */
355
/** Print contents of both TLBs. */
356
void tlb_print(void)
356
void tlb_print(void)
357
{
357
{
358
    int i;
358
    int i;
359
    tlb_data_t d;
359
    tlb_data_t d;
360
    tlb_tag_read_reg_t t;
360
    tlb_tag_read_reg_t t;
361
   
361
   
362
    printf("I-TLB contents:\n");
362
    printf("I-TLB contents:\n");
363
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
363
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
364
        d.value = itlb_data_access_read(i);
364
        d.value = itlb_data_access_read(i);
365
        t.value = itlb_tag_read_read(i);
365
        t.value = itlb_tag_read_read(i);
366
        print_tlb_entry(i, t, d);
366
        print_tlb_entry(i, t, d);
367
    }
367
    }
368
 
368
 
369
    printf("D-TLB contents:\n");
369
    printf("D-TLB contents:\n");
370
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
370
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
371
        d.value = dtlb_data_access_read(i);
371
        d.value = dtlb_data_access_read(i);
372
        t.value = dtlb_tag_read_read(i);
372
        t.value = dtlb_tag_read_read(i);
373
        print_tlb_entry(i, t, d);
373
        print_tlb_entry(i, t, d);
374
    }
374
    }
375
}
375
}
376
 
376
 
377
#elif defined (US3)
377
#elif defined (US3)
378
 
378
 
379
/** Print contents of all TLBs. */
379
/** Print contents of all TLBs. */
380
void tlb_print(void)
380
void tlb_print(void)
381
{
381
{
382
    int i;
382
    int i;
383
    tlb_data_t d;
383
    tlb_data_t d;
384
    tlb_tag_read_reg_t t;
384
    tlb_tag_read_reg_t t;
385
   
385
   
386
    printf("TLB_ISMALL contents:\n");
386
    printf("TLB_ISMALL contents:\n");
387
    for (i = 0; i < tlb_ismall_size(); i++) {
387
    for (i = 0; i < tlb_ismall_size(); i++) {
388
        d.value = dtlb_data_access_read(TLB_ISMALL, i);
388
        d.value = dtlb_data_access_read(TLB_ISMALL, i);
389
        t.value = dtlb_tag_read_read(TLB_ISMALL, i);
389
        t.value = dtlb_tag_read_read(TLB_ISMALL, i);
390
        print_tlb_entry(i, t, d);
390
        print_tlb_entry(i, t, d);
391
    }
391
    }
392
   
392
   
393
    printf("TLB_IBIG contents:\n");
393
    printf("TLB_IBIG contents:\n");
394
    for (i = 0; i < tlb_ibig_size(); i++) {
394
    for (i = 0; i < tlb_ibig_size(); i++) {
395
        d.value = dtlb_data_access_read(TLB_IBIG, i);
395
        d.value = dtlb_data_access_read(TLB_IBIG, i);
396
        t.value = dtlb_tag_read_read(TLB_IBIG, i);
396
        t.value = dtlb_tag_read_read(TLB_IBIG, i);
397
        print_tlb_entry(i, t, d);
397
        print_tlb_entry(i, t, d);
398
    }
398
    }
399
   
399
   
400
    printf("TLB_DSMALL contents:\n");
400
    printf("TLB_DSMALL contents:\n");
401
    for (i = 0; i < tlb_dsmall_size(); i++) {
401
    for (i = 0; i < tlb_dsmall_size(); i++) {
402
        d.value = dtlb_data_access_read(TLB_DSMALL, i);
402
        d.value = dtlb_data_access_read(TLB_DSMALL, i);
403
        t.value = dtlb_tag_read_read(TLB_DSMALL, i);
403
        t.value = dtlb_tag_read_read(TLB_DSMALL, i);
404
        print_tlb_entry(i, t, d);
404
        print_tlb_entry(i, t, d);
405
    }
405
    }
406
   
406
   
407
    printf("TLB_DBIG_1 contents:\n");
407
    printf("TLB_DBIG_1 contents:\n");
408
    for (i = 0; i < tlb_dbig_size(); i++) {
408
    for (i = 0; i < tlb_dbig_size(); i++) {
409
        d.value = dtlb_data_access_read(TLB_DBIG_0, i);
409
        d.value = dtlb_data_access_read(TLB_DBIG_0, i);
410
        t.value = dtlb_tag_read_read(TLB_DBIG_0, i);
410
        t.value = dtlb_tag_read_read(TLB_DBIG_0, i);
411
        print_tlb_entry(i, t, d);
411
        print_tlb_entry(i, t, d);
412
    }
412
    }
413
   
413
   
414
    printf("TLB_DBIG_2 contents:\n");
414
    printf("TLB_DBIG_2 contents:\n");
415
    for (i = 0; i < tlb_dbig_size(); i++) {
415
    for (i = 0; i < tlb_dbig_size(); i++) {
416
        d.value = dtlb_data_access_read(TLB_DBIG_1, i);
416
        d.value = dtlb_data_access_read(TLB_DBIG_1, i);
417
        t.value = dtlb_tag_read_read(TLB_DBIG_1, i);
417
        t.value = dtlb_tag_read_read(TLB_DBIG_1, i);
418
        print_tlb_entry(i, t, d);
418
        print_tlb_entry(i, t, d);
419
    }
419
    }
420
}
420
}
421
 
421
 
422
#endif
422
#endif
423
 
423
 
424
void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
424
void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
425
    const char *str)
425
    const char *str)
426
{
426
{
427
    fault_if_from_uspace(istate, "%s\n", str);
427
    fault_if_from_uspace(istate, "%s", str);
428
    dump_istate(istate);
428
    dump_istate(istate);
429
    panic("%s\n", str);
429
    panic("%s\n", str);
430
}
430
}
431
 
431
 
432
void do_fast_data_access_mmu_miss_fault(istate_t *istate,
432
void do_fast_data_access_mmu_miss_fault(istate_t *istate,
433
    tlb_tag_access_reg_t tag, const char *str)
433
    tlb_tag_access_reg_t tag, const char *str)
434
{
434
{
435
    uintptr_t va;
435
    uintptr_t va;
436
 
436
 
437
    va = tag.vpn << MMU_PAGE_WIDTH;
437
    va = tag.vpn << MMU_PAGE_WIDTH;
438
    if (tag.context) {
438
    if (tag.context) {
439
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va,
439
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)", str, va,
440
            tag.context);
440
            tag.context);
441
    }
441
    }
442
    dump_istate(istate);
442
    dump_istate(istate);
443
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
443
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
444
    panic("%s\n", str);
444
    panic("%s\n", str);
445
}
445
}
446
 
446
 
447
void do_fast_data_access_protection_fault(istate_t *istate,
447
void do_fast_data_access_protection_fault(istate_t *istate,
448
    tlb_tag_access_reg_t tag, const char *str)
448
    tlb_tag_access_reg_t tag, const char *str)
449
{
449
{
450
    uintptr_t va;
450
    uintptr_t va;
451
 
451
 
452
    va = tag.vpn << MMU_PAGE_WIDTH;
452
    va = tag.vpn << MMU_PAGE_WIDTH;
453
 
453
 
454
    if (tag.context) {
454
    if (tag.context) {
455
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va,
455
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)", str, va,
456
            tag.context);
456
            tag.context);
457
    }
457
    }
458
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
458
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
459
    dump_istate(istate);
459
    dump_istate(istate);
460
    panic("%s\n", str);
460
    panic("%s\n", str);
461
}
461
}
462
 
462
 
463
void dump_sfsr_and_sfar(void)
463
void dump_sfsr_and_sfar(void)
464
{
464
{
465
    tlb_sfsr_reg_t sfsr;
465
    tlb_sfsr_reg_t sfsr;
466
    uintptr_t sfar;
466
    uintptr_t sfar;
467
 
467
 
468
    sfsr.value = dtlb_sfsr_read();
468
    sfsr.value = dtlb_sfsr_read();
469
    sfar = dtlb_sfar_read();
469
    sfar = dtlb_sfar_read();
470
   
470
   
471
#if defined (US)
471
#if defined (US)
472
    printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
472
    printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
473
        "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
473
        "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
474
        sfsr.ow, sfsr.fv);
474
        sfsr.ow, sfsr.fv);
475
#elif defined (US3)
475
#elif defined (US3)
476
    printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
476
    printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
477
        "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
477
        "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
478
        sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
478
        sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
479
#endif
479
#endif
480
       
480
       
481
    printf("DTLB SFAR: address=%p\n", sfar);
481
    printf("DTLB SFAR: address=%p\n", sfar);
482
   
482
   
483
    dtlb_sfsr_write(0);
483
    dtlb_sfsr_write(0);
484
}
484
}
485
 
485
 
486
#if defined (US3)
486
#if defined (US3)
487
/** Invalidates given TLB entry if and only if it is non-locked or global.
487
/** Invalidates given TLB entry if and only if it is non-locked or global.
488
 *
488
 *
489
 * @param tlb       TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1,
489
 * @param tlb       TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1,
490
 *          TLB_ISMALL, TLB_IBIG).
490
 *          TLB_ISMALL, TLB_IBIG).
491
 * @param entry     Entry index within the given TLB.
491
 * @param entry     Entry index within the given TLB.
492
 */
492
 */
493
static void tlb_invalidate_entry(int tlb, index_t entry)
493
static void tlb_invalidate_entry(int tlb, index_t entry)
494
{
494
{
495
    tlb_data_t d;
495
    tlb_data_t d;
496
    tlb_tag_read_reg_t t;
496
    tlb_tag_read_reg_t t;
497
   
497
   
498
    if (tlb == TLB_DSMALL || tlb == TLB_DBIG_0 || tlb == TLB_DBIG_1) {
498
    if (tlb == TLB_DSMALL || tlb == TLB_DBIG_0 || tlb == TLB_DBIG_1) {
499
        d.value = dtlb_data_access_read(tlb, entry);
499
        d.value = dtlb_data_access_read(tlb, entry);
500
        if (!d.l || d.g) {
500
        if (!d.l || d.g) {
501
            t.value = dtlb_tag_read_read(tlb, entry);
501
            t.value = dtlb_tag_read_read(tlb, entry);
502
            d.v = false;
502
            d.v = false;
503
            dtlb_tag_access_write(t.value);
503
            dtlb_tag_access_write(t.value);
504
            dtlb_data_access_write(tlb, entry, d.value);
504
            dtlb_data_access_write(tlb, entry, d.value);
505
        }
505
        }
506
    } else if (tlb == TLB_ISMALL || tlb == TLB_IBIG) {
506
    } else if (tlb == TLB_ISMALL || tlb == TLB_IBIG) {
507
        d.value = itlb_data_access_read(tlb, entry);
507
        d.value = itlb_data_access_read(tlb, entry);
508
        if (!d.l || d.g) {
508
        if (!d.l || d.g) {
509
            t.value = itlb_tag_read_read(tlb, entry);
509
            t.value = itlb_tag_read_read(tlb, entry);
510
            d.v = false;
510
            d.v = false;
511
            itlb_tag_access_write(t.value);
511
            itlb_tag_access_write(t.value);
512
            itlb_data_access_write(tlb, entry, d.value);
512
            itlb_data_access_write(tlb, entry, d.value);
513
        }
513
        }
514
    }
514
    }
515
}
515
}
516
#endif
516
#endif
517
 
517
 
518
/** Invalidate all unlocked ITLB and DTLB entries. */
518
/** Invalidate all unlocked ITLB and DTLB entries. */
519
void tlb_invalidate_all(void)
519
void tlb_invalidate_all(void)
520
{
520
{
521
    int i;
521
    int i;
522
   
522
   
523
    /*
523
    /*
524
     * Walk all ITLB and DTLB entries and remove all unlocked mappings.
524
     * Walk all ITLB and DTLB entries and remove all unlocked mappings.
525
     *
525
     *
526
     * The kernel doesn't use global mappings so any locked global mappings
526
     * The kernel doesn't use global mappings so any locked global mappings
527
     * found must have been created by someone else. Their only purpose now
527
     * found must have been created by someone else. Their only purpose now
528
     * is to collide with proper mappings. Invalidate immediately. It should
528
     * is to collide with proper mappings. Invalidate immediately. It should
529
     * be safe to invalidate them as late as now.
529
     * be safe to invalidate them as late as now.
530
     */
530
     */
531
 
531
 
532
#if defined (US)
532
#if defined (US)
533
    tlb_data_t d;
533
    tlb_data_t d;
534
    tlb_tag_read_reg_t t;
534
    tlb_tag_read_reg_t t;
535
 
535
 
536
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
536
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
537
        d.value = itlb_data_access_read(i);
537
        d.value = itlb_data_access_read(i);
538
        if (!d.l || d.g) {
538
        if (!d.l || d.g) {
539
            t.value = itlb_tag_read_read(i);
539
            t.value = itlb_tag_read_read(i);
540
            d.v = false;
540
            d.v = false;
541
            itlb_tag_access_write(t.value);
541
            itlb_tag_access_write(t.value);
542
            itlb_data_access_write(i, d.value);
542
            itlb_data_access_write(i, d.value);
543
        }
543
        }
544
    }
544
    }
545
 
545
 
546
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
546
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
547
        d.value = dtlb_data_access_read(i);
547
        d.value = dtlb_data_access_read(i);
548
        if (!d.l || d.g) {
548
        if (!d.l || d.g) {
549
            t.value = dtlb_tag_read_read(i);
549
            t.value = dtlb_tag_read_read(i);
550
            d.v = false;
550
            d.v = false;
551
            dtlb_tag_access_write(t.value);
551
            dtlb_tag_access_write(t.value);
552
            dtlb_data_access_write(i, d.value);
552
            dtlb_data_access_write(i, d.value);
553
        }
553
        }
554
    }
554
    }
555
 
555
 
556
#elif defined (US3)
556
#elif defined (US3)
557
 
557
 
558
    for (i = 0; i < tlb_ismall_size(); i++)
558
    for (i = 0; i < tlb_ismall_size(); i++)
559
        tlb_invalidate_entry(TLB_ISMALL, i);
559
        tlb_invalidate_entry(TLB_ISMALL, i);
560
    for (i = 0; i < tlb_ibig_size(); i++)
560
    for (i = 0; i < tlb_ibig_size(); i++)
561
        tlb_invalidate_entry(TLB_IBIG, i);
561
        tlb_invalidate_entry(TLB_IBIG, i);
562
    for (i = 0; i < tlb_dsmall_size(); i++)
562
    for (i = 0; i < tlb_dsmall_size(); i++)
563
        tlb_invalidate_entry(TLB_DSMALL, i);
563
        tlb_invalidate_entry(TLB_DSMALL, i);
564
    for (i = 0; i < tlb_dbig_size(); i++)
564
    for (i = 0; i < tlb_dbig_size(); i++)
565
        tlb_invalidate_entry(TLB_DBIG_0, i);
565
        tlb_invalidate_entry(TLB_DBIG_0, i);
566
    for (i = 0; i < tlb_dbig_size(); i++)
566
    for (i = 0; i < tlb_dbig_size(); i++)
567
        tlb_invalidate_entry(TLB_DBIG_1, i);
567
        tlb_invalidate_entry(TLB_DBIG_1, i);
568
#endif
568
#endif
569
 
569
 
570
}
570
}
571
 
571
 
572
/** Invalidate all ITLB and DTLB entries that belong to specified ASID
572
/** Invalidate all ITLB and DTLB entries that belong to specified ASID
573
 * (Context).
573
 * (Context).
574
 *
574
 *
575
 * @param asid Address Space ID.
575
 * @param asid Address Space ID.
576
 */
576
 */
577
void tlb_invalidate_asid(asid_t asid)
577
void tlb_invalidate_asid(asid_t asid)
578
{
578
{
579
    tlb_context_reg_t pc_save, ctx;
579
    tlb_context_reg_t pc_save, ctx;
580
   
580
   
581
    /* switch to nucleus because we are mapped by the primary context */
581
    /* switch to nucleus because we are mapped by the primary context */
582
    nucleus_enter();
582
    nucleus_enter();
583
   
583
   
584
    ctx.v = pc_save.v = mmu_primary_context_read();
584
    ctx.v = pc_save.v = mmu_primary_context_read();
585
    ctx.context = asid;
585
    ctx.context = asid;
586
    mmu_primary_context_write(ctx.v);
586
    mmu_primary_context_write(ctx.v);
587
   
587
   
588
    itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
588
    itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
589
    dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
589
    dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
590
   
590
   
591
    mmu_primary_context_write(pc_save.v);
591
    mmu_primary_context_write(pc_save.v);
592
   
592
   
593
    nucleus_leave();
593
    nucleus_leave();
594
}
594
}
595
 
595
 
596
/** Invalidate all ITLB and DTLB entries for specified page range in specified
596
/** Invalidate all ITLB and DTLB entries for specified page range in specified
597
 * address space.
597
 * address space.
598
 *
598
 *
599
 * @param asid      Address Space ID.
599
 * @param asid      Address Space ID.
600
 * @param page      First page which to sweep out from ITLB and DTLB.
600
 * @param page      First page which to sweep out from ITLB and DTLB.
601
 * @param cnt       Number of ITLB and DTLB entries to invalidate.
601
 * @param cnt       Number of ITLB and DTLB entries to invalidate.
602
 */
602
 */
603
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
603
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
604
{
604
{
605
    unsigned int i;
605
    unsigned int i;
606
    tlb_context_reg_t pc_save, ctx;
606
    tlb_context_reg_t pc_save, ctx;
607
   
607
   
608
    /* switch to nucleus because we are mapped by the primary context */
608
    /* switch to nucleus because we are mapped by the primary context */
609
    nucleus_enter();
609
    nucleus_enter();
610
   
610
   
611
    ctx.v = pc_save.v = mmu_primary_context_read();
611
    ctx.v = pc_save.v = mmu_primary_context_read();
612
    ctx.context = asid;
612
    ctx.context = asid;
613
    mmu_primary_context_write(ctx.v);
613
    mmu_primary_context_write(ctx.v);
614
   
614
   
615
    for (i = 0; i < cnt * MMU_PAGES_PER_PAGE; i++) {
615
    for (i = 0; i < cnt * MMU_PAGES_PER_PAGE; i++) {
616
        itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
616
        itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
617
            page + i * MMU_PAGE_SIZE);
617
            page + i * MMU_PAGE_SIZE);
618
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
618
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
619
            page + i * MMU_PAGE_SIZE);
619
            page + i * MMU_PAGE_SIZE);
620
    }
620
    }
621
   
621
   
622
    mmu_primary_context_write(pc_save.v);
622
    mmu_primary_context_write(pc_save.v);
623
   
623
   
624
    nucleus_leave();
624
    nucleus_leave();
625
}
625
}
626
 
626
 
627
/** @}
627
/** @}
628
 */
628
 */
629
 
629