Subversion Repositories HelenOS

Rev

Rev 2787 | Rev 4377 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 2787 Rev 3675
1
/*
1
/*
2
 * Copyright (c) 2005 Jakub Jermar
2
 * Copyright (c) 2005 Jakub Jermar
3
 * All rights reserved.
3
 * All rights reserved.
4
 *
4
 *
5
 * Redistribution and use in source and binary forms, with or without
5
 * Redistribution and use in source and binary forms, with or without
6
 * modification, are permitted provided that the following conditions
6
 * modification, are permitted provided that the following conditions
7
 * are met:
7
 * are met:
8
 *
8
 *
9
 * - Redistributions of source code must retain the above copyright
9
 * - Redistributions of source code must retain the above copyright
10
 *   notice, this list of conditions and the following disclaimer.
10
 *   notice, this list of conditions and the following disclaimer.
11
 * - Redistributions in binary form must reproduce the above copyright
11
 * - Redistributions in binary form must reproduce the above copyright
12
 *   notice, this list of conditions and the following disclaimer in the
12
 *   notice, this list of conditions and the following disclaimer in the
13
 *   documentation and/or other materials provided with the distribution.
13
 *   documentation and/or other materials provided with the distribution.
14
 * - The name of the author may not be used to endorse or promote products
14
 * - The name of the author may not be used to endorse or promote products
15
 *   derived from this software without specific prior written permission.
15
 *   derived from this software without specific prior written permission.
16
 *
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
 */
27
 */
28
 
28
 
29
/** @addtogroup sparc64mm  
29
/** @addtogroup sparc64mm  
30
 * @{
30
 * @{
31
 */
31
 */
32
/** @file
32
/** @file
33
 */
33
 */
34
 
34
 
35
#include <arch/mm/tlb.h>
35
#include <arch/mm/tlb.h>
36
#include <mm/tlb.h>
36
#include <mm/tlb.h>
37
#include <mm/as.h>
37
#include <mm/as.h>
38
#include <mm/asid.h>
38
#include <mm/asid.h>
39
#include <arch/mm/frame.h>
39
#include <arch/mm/frame.h>
40
#include <arch/mm/page.h>
40
#include <arch/mm/page.h>
41
#include <arch/mm/mmu.h>
41
#include <arch/mm/mmu.h>
42
#include <arch/interrupt.h>
42
#include <arch/interrupt.h>
43
#include <interrupt.h>
43
#include <interrupt.h>
44
#include <arch.h>
44
#include <arch.h>
45
#include <print.h>
45
#include <print.h>
46
#include <arch/types.h>
46
#include <arch/types.h>
47
#include <config.h>
47
#include <config.h>
48
#include <arch/trap/trap.h>
48
#include <arch/trap/trap.h>
49
#include <arch/trap/exception.h>
49
#include <arch/trap/exception.h>
50
#include <panic.h>
50
#include <panic.h>
51
#include <arch/asm.h>
51
#include <arch/asm.h>
52
 
52
 
53
#ifdef CONFIG_TSB
53
#ifdef CONFIG_TSB
54
#include <arch/mm/tsb.h>
54
#include <arch/mm/tsb.h>
55
#endif
55
#endif
56
 
56
 
57
static void dtlb_pte_copy(pte_t *t, index_t index, bool ro);
57
static void dtlb_pte_copy(pte_t *, index_t, bool);
58
static void itlb_pte_copy(pte_t *t, index_t index);
58
static void itlb_pte_copy(pte_t *, index_t);
59
static void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
59
static void do_fast_instruction_access_mmu_miss_fault(istate_t *, const char *);
60
    const char *str);
-
 
61
static void do_fast_data_access_mmu_miss_fault(istate_t *istate,
60
static void do_fast_data_access_mmu_miss_fault(istate_t *, tlb_tag_access_reg_t,
62
    tlb_tag_access_reg_t tag, const char *str);
61
    const char *);
63
static void do_fast_data_access_protection_fault(istate_t *istate,
62
static void do_fast_data_access_protection_fault(istate_t *,
64
    tlb_tag_access_reg_t tag, const char *str);
63
    tlb_tag_access_reg_t, const char *);
65
 
64
 
66
char *context_encoding[] = {
65
char *context_encoding[] = {
67
    "Primary",
66
    "Primary",
68
    "Secondary",
67
    "Secondary",
69
    "Nucleus",
68
    "Nucleus",
70
    "Reserved"
69
    "Reserved"
71
};
70
};
72
 
71
 
73
void tlb_arch_init(void)
72
void tlb_arch_init(void)
74
{
73
{
75
    /*
74
    /*
76
     * Invalidate all non-locked DTLB and ITLB entries.
75
     * Invalidate all non-locked DTLB and ITLB entries.
77
     */
76
     */
78
    tlb_invalidate_all();
77
    tlb_invalidate_all();
79
 
78
 
80
    /*
79
    /*
81
     * Clear both SFSRs.
80
     * Clear both SFSRs.
82
     */
81
     */
83
    dtlb_sfsr_write(0);
82
    dtlb_sfsr_write(0);
84
    itlb_sfsr_write(0);
83
    itlb_sfsr_write(0);
85
}
84
}
86
 
85
 
87
/** Insert privileged mapping into DMMU TLB.
86
/** Insert privileged mapping into DMMU TLB.
88
 *
87
 *
89
 * @param page Virtual page address.
88
 * @param page      Virtual page address.
90
 * @param frame Physical frame address.
89
 * @param frame     Physical frame address.
91
 * @param pagesize Page size.
90
 * @param pagesize  Page size.
92
 * @param locked True for permanent mappings, false otherwise.
91
 * @param locked    True for permanent mappings, false otherwise.
93
 * @param cacheable True if the mapping is cacheable, false otherwise.
92
 * @param cacheable True if the mapping is cacheable, false otherwise.
94
 */
93
 */
95
void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize,
94
void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize,
96
    bool locked, bool cacheable)
95
    bool locked, bool cacheable)
97
{
96
{
98
    tlb_tag_access_reg_t tag;
97
    tlb_tag_access_reg_t tag;
99
    tlb_data_t data;
98
    tlb_data_t data;
100
    page_address_t pg;
99
    page_address_t pg;
101
    frame_address_t fr;
100
    frame_address_t fr;
102
 
101
 
103
    pg.address = page;
102
    pg.address = page;
104
    fr.address = frame;
103
    fr.address = frame;
105
 
104
 
106
    tag.value = ASID_KERNEL;
105
    tag.context = ASID_KERNEL;
107
    tag.vpn = pg.vpn;
106
    tag.vpn = pg.vpn;
108
 
107
 
109
    dtlb_tag_access_write(tag.value);
108
    dtlb_tag_access_write(tag.value);
110
 
109
 
111
    data.value = 0;
110
    data.value = 0;
112
    data.v = true;
111
    data.v = true;
113
    data.size = pagesize;
112
    data.size = pagesize;
114
    data.pfn = fr.pfn;
113
    data.pfn = fr.pfn;
115
    data.l = locked;
114
    data.l = locked;
116
    data.cp = cacheable;
115
    data.cp = cacheable;
117
#ifdef CONFIG_VIRT_IDX_DCACHE
116
#ifdef CONFIG_VIRT_IDX_DCACHE
118
    data.cv = cacheable;
117
    data.cv = cacheable;
119
#endif /* CONFIG_VIRT_IDX_DCACHE */
118
#endif /* CONFIG_VIRT_IDX_DCACHE */
120
    data.p = true;
119
    data.p = true;
121
    data.w = true;
120
    data.w = true;
122
    data.g = false;
121
    data.g = false;
123
 
122
 
124
    dtlb_data_in_write(data.value);
123
    dtlb_data_in_write(data.value);
125
}
124
}
126
 
125
 
127
/** Copy PTE to TLB.
126
/** Copy PTE to TLB.
128
 *
127
 *
129
 * @param t     Page Table Entry to be copied.
128
 * @param t         Page Table Entry to be copied.
130
 * @param index Zero if lower 8K-subpage, one if higher 8K-subpage.
129
 * @param index     Zero if lower 8K-subpage, one if higher 8K-subpage.
131
 * @param ro    If true, the entry will be created read-only, regardless of its
130
 * @param ro        If true, the entry will be created read-only, regardless
132
 *      w field.
131
 *          of its w field.
133
 */
132
 */
134
void dtlb_pte_copy(pte_t *t, index_t index, bool ro)
133
void dtlb_pte_copy(pte_t *t, index_t index, bool ro)
135
{
134
{
136
    tlb_tag_access_reg_t tag;
135
    tlb_tag_access_reg_t tag;
137
    tlb_data_t data;
136
    tlb_data_t data;
138
    page_address_t pg;
137
    page_address_t pg;
139
    frame_address_t fr;
138
    frame_address_t fr;
140
 
139
 
141
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
140
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
142
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
141
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
143
 
142
 
144
    tag.value = 0;
143
    tag.value = 0;
145
    tag.context = t->as->asid;
144
    tag.context = t->as->asid;
146
    tag.vpn = pg.vpn;
145
    tag.vpn = pg.vpn;
147
 
146
 
148
    dtlb_tag_access_write(tag.value);
147
    dtlb_tag_access_write(tag.value);
149
 
148
 
150
    data.value = 0;
149
    data.value = 0;
151
    data.v = true;
150
    data.v = true;
152
    data.size = PAGESIZE_8K;
151
    data.size = PAGESIZE_8K;
153
    data.pfn = fr.pfn;
152
    data.pfn = fr.pfn;
154
    data.l = false;
153
    data.l = false;
155
    data.cp = t->c;
154
    data.cp = t->c;
156
#ifdef CONFIG_VIRT_IDX_DCACHE
155
#ifdef CONFIG_VIRT_IDX_DCACHE
157
    data.cv = t->c;
156
    data.cv = t->c;
158
#endif /* CONFIG_VIRT_IDX_DCACHE */
157
#endif /* CONFIG_VIRT_IDX_DCACHE */
159
    data.p = t->k;      /* p like privileged */
158
    data.p = t->k;      /* p like privileged */
160
    data.w = ro ? false : t->w;
159
    data.w = ro ? false : t->w;
161
    data.g = t->g;
160
    data.g = t->g;
162
 
161
 
163
    dtlb_data_in_write(data.value);
162
    dtlb_data_in_write(data.value);
164
}
163
}
165
 
164
 
166
/** Copy PTE to ITLB.
165
/** Copy PTE to ITLB.
167
 *
166
 *
168
 * @param t     Page Table Entry to be copied.
167
 * @param t     Page Table Entry to be copied.
169
 * @param index Zero if lower 8K-subpage, one if higher 8K-subpage.
168
 * @param index     Zero if lower 8K-subpage, one if higher 8K-subpage.
170
 */
169
 */
171
void itlb_pte_copy(pte_t *t, index_t index)
170
void itlb_pte_copy(pte_t *t, index_t index)
172
{
171
{
173
    tlb_tag_access_reg_t tag;
172
    tlb_tag_access_reg_t tag;
174
    tlb_data_t data;
173
    tlb_data_t data;
175
    page_address_t pg;
174
    page_address_t pg;
176
    frame_address_t fr;
175
    frame_address_t fr;
177
 
176
 
178
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
177
    pg.address = t->page + (index << MMU_PAGE_WIDTH);
179
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
178
    fr.address = t->frame + (index << MMU_PAGE_WIDTH);
180
 
179
 
181
    tag.value = 0;
180
    tag.value = 0;
182
    tag.context = t->as->asid;
181
    tag.context = t->as->asid;
183
    tag.vpn = pg.vpn;
182
    tag.vpn = pg.vpn;
184
   
183
   
185
    itlb_tag_access_write(tag.value);
184
    itlb_tag_access_write(tag.value);
186
   
185
   
187
    data.value = 0;
186
    data.value = 0;
188
    data.v = true;
187
    data.v = true;
189
    data.size = PAGESIZE_8K;
188
    data.size = PAGESIZE_8K;
190
    data.pfn = fr.pfn;
189
    data.pfn = fr.pfn;
191
    data.l = false;
190
    data.l = false;
192
    data.cp = t->c;
191
    data.cp = t->c;
193
    data.p = t->k;      /* p like privileged */
192
    data.p = t->k;      /* p like privileged */
194
    data.w = false;
193
    data.w = false;
195
    data.g = t->g;
194
    data.g = t->g;
196
   
195
   
197
    itlb_data_in_write(data.value);
196
    itlb_data_in_write(data.value);
198
}
197
}
199
 
198
 
200
/** ITLB miss handler. */
199
/** ITLB miss handler. */
201
void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate)
200
void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate)
202
{
201
{
203
    uintptr_t va = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
202
    uintptr_t va = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
204
    index_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE;
203
    index_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE;
205
    pte_t *t;
204
    pte_t *t;
206
 
205
 
207
    page_table_lock(AS, true);
206
    page_table_lock(AS, true);
208
    t = page_mapping_find(AS, va);
207
    t = page_mapping_find(AS, va);
209
    if (t && PTE_EXECUTABLE(t)) {
208
    if (t && PTE_EXECUTABLE(t)) {
210
        /*
209
        /*
211
         * The mapping was found in the software page hash table.
210
         * The mapping was found in the software page hash table.
212
         * Insert it into ITLB.
211
         * Insert it into ITLB.
213
         */
212
         */
214
        t->a = true;
213
        t->a = true;
215
        itlb_pte_copy(t, index);
214
        itlb_pte_copy(t, index);
216
#ifdef CONFIG_TSB
215
#ifdef CONFIG_TSB
217
        itsb_pte_copy(t, index);
216
        itsb_pte_copy(t, index);
218
#endif
217
#endif
219
        page_table_unlock(AS, true);
218
        page_table_unlock(AS, true);
220
    } else {
219
    } else {
221
        /*
220
        /*
222
         * Forward the page fault to the address space page fault
221
         * Forward the page fault to the address space page fault
223
         * handler.
222
         * handler.
224
         */    
223
         */    
225
        page_table_unlock(AS, true);
224
        page_table_unlock(AS, true);
226
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
225
        if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
227
            do_fast_instruction_access_mmu_miss_fault(istate,
226
            do_fast_instruction_access_mmu_miss_fault(istate,
228
                __func__);
227
                __func__);
229
        }
228
        }
230
    }
229
    }
231
}
230
}
232
 
231
 
233
/** DTLB miss handler.
232
/** DTLB miss handler.
234
 *
233
 *
235
 * Note that some faults (e.g. kernel faults) were already resolved by the
234
 * Note that some faults (e.g. kernel faults) were already resolved by the
236
 * low-level, assembly language part of the fast_data_access_mmu_miss handler.
235
 * low-level, assembly language part of the fast_data_access_mmu_miss handler.
237
 *
236
 *
238
 * @param tag Content of the TLB Tag Access register as it existed when the
237
 * @param tag       Content of the TLB Tag Access register as it existed
239
 *    trap happened. This is to prevent confusion created by clobbered
238
 *          when the trap happened. This is to prevent confusion
240
 *    Tag Access register during a nested DTLB miss.
239
 *          created by clobbered Tag Access register during a nested
-
 
240
 *          DTLB miss.
241
 * @param istate Interrupted state saved on the stack.
241
 * @param istate    Interrupted state saved on the stack.
242
 */
242
 */
243
void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate)
243
void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate)
244
{
244
{
245
    uintptr_t va;
245
    uintptr_t va;
246
    index_t index;
246
    index_t index;
247
    pte_t *t;
247
    pte_t *t;
248
 
248
 
249
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
249
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
250
    index = tag.vpn % MMU_PAGES_PER_PAGE;
250
    index = tag.vpn % MMU_PAGES_PER_PAGE;
251
 
251
 
252
    if (tag.context == ASID_KERNEL) {
252
    if (tag.context == ASID_KERNEL) {
253
        if (!tag.vpn) {
253
        if (!tag.vpn) {
254
            /* NULL access in kernel */
254
            /* NULL access in kernel */
255
            do_fast_data_access_mmu_miss_fault(istate, tag,
255
            do_fast_data_access_mmu_miss_fault(istate, tag,
256
                __func__);
256
                __func__);
257
        }
257
        }
258
        do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected "
258
        do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected "
259
            "kernel page fault.");
259
            "kernel page fault.");
260
    }
260
    }
261
 
261
 
262
    page_table_lock(AS, true);
262
    page_table_lock(AS, true);
263
    t = page_mapping_find(AS, va);
263
    t = page_mapping_find(AS, va);
264
    if (t) {
264
    if (t) {
265
        /*
265
        /*
266
         * The mapping was found in the software page hash table.
266
         * The mapping was found in the software page hash table.
267
         * Insert it into DTLB.
267
         * Insert it into DTLB.
268
         */
268
         */
269
        t->a = true;
269
        t->a = true;
270
        dtlb_pte_copy(t, index, true);
270
        dtlb_pte_copy(t, index, true);
271
#ifdef CONFIG_TSB
271
#ifdef CONFIG_TSB
272
        dtsb_pte_copy(t, index, true);
272
        dtsb_pte_copy(t, index, true);
273
#endif
273
#endif
274
        page_table_unlock(AS, true);
274
        page_table_unlock(AS, true);
275
    } else {
275
    } else {
276
        /*
276
        /*
277
         * Forward the page fault to the address space page fault
277
         * Forward the page fault to the address space page fault
278
         * handler.
278
         * handler.
279
         */    
279
         */    
280
        page_table_unlock(AS, true);
280
        page_table_unlock(AS, true);
281
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
281
        if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
282
            do_fast_data_access_mmu_miss_fault(istate, tag,
282
            do_fast_data_access_mmu_miss_fault(istate, tag,
283
                __func__);
283
                __func__);
284
        }
284
        }
285
    }
285
    }
286
}
286
}
287
 
287
 
288
/** DTLB protection fault handler.
288
/** DTLB protection fault handler.
289
 *
289
 *
290
 * @param tag Content of the TLB Tag Access register as it existed when the
290
 * @param tag       Content of the TLB Tag Access register as it existed
291
 *    trap happened. This is to prevent confusion created by clobbered
291
 *          when the trap happened. This is to prevent confusion
292
 *    Tag Access register during a nested DTLB miss.
292
 *          created by clobbered Tag Access register during a nested
-
 
293
 *          DTLB miss.
293
 * @param istate Interrupted state saved on the stack.
294
 * @param istate    Interrupted state saved on the stack.
294
 */
295
 */
295
void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate)
296
void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate)
296
{
297
{
297
    uintptr_t va;
298
    uintptr_t va;
298
    index_t index;
299
    index_t index;
299
    pte_t *t;
300
    pte_t *t;
300
 
301
 
301
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
302
    va = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
302
    index = tag.vpn % MMU_PAGES_PER_PAGE;   /* 16K-page emulation */
303
    index = tag.vpn % MMU_PAGES_PER_PAGE;   /* 16K-page emulation */
303
 
304
 
304
    page_table_lock(AS, true);
305
    page_table_lock(AS, true);
305
    t = page_mapping_find(AS, va);
306
    t = page_mapping_find(AS, va);
306
    if (t && PTE_WRITABLE(t)) {
307
    if (t && PTE_WRITABLE(t)) {
307
        /*
308
        /*
308
         * The mapping was found in the software page hash table and is
309
         * The mapping was found in the software page hash table and is
309
         * writable. Demap the old mapping and insert an updated mapping
310
         * writable. Demap the old mapping and insert an updated mapping
310
         * into DTLB.
311
         * into DTLB.
311
         */
312
         */
312
        t->a = true;
313
        t->a = true;
313
        t->d = true;
314
        t->d = true;
314
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY,
315
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY,
315
            va + index * MMU_PAGE_SIZE);
316
            va + index * MMU_PAGE_SIZE);
316
        dtlb_pte_copy(t, index, false);
317
        dtlb_pte_copy(t, index, false);
317
#ifdef CONFIG_TSB
318
#ifdef CONFIG_TSB
318
        dtsb_pte_copy(t, index, false);
319
        dtsb_pte_copy(t, index, false);
319
#endif
320
#endif
320
        page_table_unlock(AS, true);
321
        page_table_unlock(AS, true);
321
    } else {
322
    } else {
322
        /*
323
        /*
323
         * Forward the page fault to the address space page fault
324
         * Forward the page fault to the address space page fault
324
         * handler.
325
         * handler.
325
         */    
326
         */    
326
        page_table_unlock(AS, true);
327
        page_table_unlock(AS, true);
327
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
328
        if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
328
            do_fast_data_access_protection_fault(istate, tag,
329
            do_fast_data_access_protection_fault(istate, tag,
329
                __func__);
330
                __func__);
330
        }
331
        }
331
    }
332
    }
332
}
333
}
333
 
334
 
-
 
335
/** Print TLB entry (for debugging purposes).
-
 
336
 *
-
 
337
 * The diag field has been left out in order to make this function more generic
-
 
338
 * (there is no diag field in US3 architeture).
-
 
339
 *
-
 
340
 * @param i     TLB entry number
-
 
341
 * @param t     TLB entry tag
-
 
342
 * @param d     TLB entry data
-
 
343
 */
-
 
344
static void print_tlb_entry(int i, tlb_tag_read_reg_t t, tlb_data_t d)
-
 
345
{
-
 
346
    printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, "
-
 
347
        "ie=%d, soft2=%#x, pfn=%#x, soft=%#x, l=%d, "
-
 
348
        "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn,
-
 
349
        t.context, d.v, d.size, d.nfo, d.ie, d.soft2,
-
 
350
        d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
-
 
351
}
-
 
352
 
-
 
353
#if defined (US)
-
 
354
 
334
/** Print contents of both TLBs. */
355
/** Print contents of both TLBs. */
335
void tlb_print(void)
356
void tlb_print(void)
336
{
357
{
337
    int i;
358
    int i;
338
    tlb_data_t d;
359
    tlb_data_t d;
339
    tlb_tag_read_reg_t t;
360
    tlb_tag_read_reg_t t;
340
   
361
   
341
    printf("I-TLB contents:\n");
362
    printf("I-TLB contents:\n");
342
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
363
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
343
        d.value = itlb_data_access_read(i);
364
        d.value = itlb_data_access_read(i);
344
        t.value = itlb_tag_read_read(i);
365
        t.value = itlb_tag_read_read(i);
345
 
-
 
346
        printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, "
-
 
347
            "ie=%d, soft2=%#x, diag=%#x, pfn=%#x, soft=%#x, l=%d, "
-
 
348
            "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn,
-
 
349
            t.context, d.v, d.size, d.nfo, d.ie, d.soft2, d.diag,
366
        print_tlb_entry(i, t, d);
350
            d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
-
 
351
    }
367
    }
352
 
368
 
353
    printf("D-TLB contents:\n");
369
    printf("D-TLB contents:\n");
354
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
370
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
355
        d.value = dtlb_data_access_read(i);
371
        d.value = dtlb_data_access_read(i);
356
        t.value = dtlb_tag_read_read(i);
372
        t.value = dtlb_tag_read_read(i);
357
       
-
 
358
        printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, "
-
 
359
            "ie=%d, soft2=%#x, diag=%#x, pfn=%#x, soft=%#x, l=%d, "
-
 
360
            "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn,
-
 
361
            t.context, d.v, d.size, d.nfo, d.ie, d.soft2, d.diag,
373
        print_tlb_entry(i, t, d);
362
            d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
-
 
363
    }
374
    }
-
 
375
}
-
 
376
 
-
 
377
#elif defined (US3)
364
 
378
 
-
 
379
/** Print contents of all TLBs. */
-
 
380
void tlb_print(void)
-
 
381
{
-
 
382
    int i;
-
 
383
    tlb_data_t d;
-
 
384
    tlb_tag_read_reg_t t;
-
 
385
   
-
 
386
    printf("TLB_ISMALL contents:\n");
-
 
387
    for (i = 0; i < tlb_ismall_size(); i++) {
-
 
388
        d.value = dtlb_data_access_read(TLB_ISMALL, i);
-
 
389
        t.value = dtlb_tag_read_read(TLB_ISMALL, i);
-
 
390
        print_tlb_entry(i, t, d);
-
 
391
    }
-
 
392
   
-
 
393
    printf("TLB_IBIG contents:\n");
-
 
394
    for (i = 0; i < tlb_ibig_size(); i++) {
-
 
395
        d.value = dtlb_data_access_read(TLB_IBIG, i);
-
 
396
        t.value = dtlb_tag_read_read(TLB_IBIG, i);
-
 
397
        print_tlb_entry(i, t, d);
-
 
398
    }
-
 
399
   
-
 
400
    printf("TLB_DSMALL contents:\n");
-
 
401
    for (i = 0; i < tlb_dsmall_size(); i++) {
-
 
402
        d.value = dtlb_data_access_read(TLB_DSMALL, i);
-
 
403
        t.value = dtlb_tag_read_read(TLB_DSMALL, i);
-
 
404
        print_tlb_entry(i, t, d);
-
 
405
    }
-
 
406
   
-
 
407
    printf("TLB_DBIG_1 contents:\n");
-
 
408
    for (i = 0; i < tlb_dbig_size(); i++) {
-
 
409
        d.value = dtlb_data_access_read(TLB_DBIG_0, i);
-
 
410
        t.value = dtlb_tag_read_read(TLB_DBIG_0, i);
-
 
411
        print_tlb_entry(i, t, d);
-
 
412
    }
-
 
413
   
-
 
414
    printf("TLB_DBIG_2 contents:\n");
-
 
415
    for (i = 0; i < tlb_dbig_size(); i++) {
-
 
416
        d.value = dtlb_data_access_read(TLB_DBIG_1, i);
-
 
417
        t.value = dtlb_tag_read_read(TLB_DBIG_1, i);
-
 
418
        print_tlb_entry(i, t, d);
-
 
419
    }
365
}
420
}
366
 
421
 
-
 
422
#endif
-
 
423
 
367
void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
424
void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
368
    const char *str)
425
    const char *str)
369
{
426
{
370
    fault_if_from_uspace(istate, "%s\n", str);
427
    fault_if_from_uspace(istate, "%s\n", str);
371
    dump_istate(istate);
428
    dump_istate(istate);
372
    panic("%s\n", str);
429
    panic("%s\n", str);
373
}
430
}
374
 
431
 
375
void do_fast_data_access_mmu_miss_fault(istate_t *istate,
432
void do_fast_data_access_mmu_miss_fault(istate_t *istate,
376
    tlb_tag_access_reg_t tag, const char *str)
433
    tlb_tag_access_reg_t tag, const char *str)
377
{
434
{
378
    uintptr_t va;
435
    uintptr_t va;
379
 
436
 
380
    va = tag.vpn << MMU_PAGE_WIDTH;
437
    va = tag.vpn << MMU_PAGE_WIDTH;
381
    if (tag.context) {
438
    if (tag.context) {
382
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va,
439
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va,
383
            tag.context);
440
            tag.context);
384
    }
441
    }
385
    dump_istate(istate);
442
    dump_istate(istate);
386
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
443
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
387
    panic("%s\n", str);
444
    panic("%s\n", str);
388
}
445
}
389
 
446
 
390
void do_fast_data_access_protection_fault(istate_t *istate,
447
void do_fast_data_access_protection_fault(istate_t *istate,
391
    tlb_tag_access_reg_t tag, const char *str)
448
    tlb_tag_access_reg_t tag, const char *str)
392
{
449
{
393
    uintptr_t va;
450
    uintptr_t va;
394
 
451
 
395
    va = tag.vpn << MMU_PAGE_WIDTH;
452
    va = tag.vpn << MMU_PAGE_WIDTH;
396
 
453
 
397
    if (tag.context) {
454
    if (tag.context) {
398
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va,
455
        fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va,
399
            tag.context);
456
            tag.context);
400
    }
457
    }
401
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
458
    printf("Faulting page: %p, ASID=%d\n", va, tag.context);
402
    dump_istate(istate);
459
    dump_istate(istate);
403
    panic("%s\n", str);
460
    panic("%s\n", str);
404
}
461
}
405
 
462
 
406
void dump_sfsr_and_sfar(void)
463
void dump_sfsr_and_sfar(void)
407
{
464
{
408
    tlb_sfsr_reg_t sfsr;
465
    tlb_sfsr_reg_t sfsr;
409
    uintptr_t sfar;
466
    uintptr_t sfar;
410
 
467
 
411
    sfsr.value = dtlb_sfsr_read();
468
    sfsr.value = dtlb_sfsr_read();
412
    sfar = dtlb_sfar_read();
469
    sfar = dtlb_sfar_read();
413
   
470
   
-
 
471
#if defined (US)
414
    printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
472
    printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
415
        "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
473
        "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
416
        sfsr.ow, sfsr.fv);
474
        sfsr.ow, sfsr.fv);
-
 
475
#elif defined (US3)
-
 
476
    printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
-
 
477
        "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
-
 
478
        sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
-
 
479
#endif
-
 
480
       
417
    printf("DTLB SFAR: address=%p\n", sfar);
481
    printf("DTLB SFAR: address=%p\n", sfar);
418
   
482
   
419
    dtlb_sfsr_write(0);
483
    dtlb_sfsr_write(0);
420
}
484
}
421
 
485
 
-
 
486
#if defined (US3)
422
/** Invalidate all unlocked ITLB and DTLB entries. */
487
/** Invalidates given TLB entry if and only if it is non-locked or global.
-
 
488
 *
-
 
489
 * @param tlb       TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1,
-
 
490
 *          TLB_ISMALL, TLB_IBIG).
-
 
491
 * @param entry     Entry index within the given TLB.
-
 
492
 */
423
void tlb_invalidate_all(void)
493
static void tlb_invalidate_entry(int tlb, index_t entry)
424
{
494
{
425
    int i;
-
 
426
    tlb_data_t d;
495
    tlb_data_t d;
427
    tlb_tag_read_reg_t t;
496
    tlb_tag_read_reg_t t;
-
 
497
   
-
 
498
    if (tlb == TLB_DSMALL || tlb == TLB_DBIG_0 || tlb == TLB_DBIG_1) {
-
 
499
        d.value = dtlb_data_access_read(tlb, entry);
-
 
500
        if (!d.l || d.g) {
-
 
501
            t.value = dtlb_tag_read_read(tlb, entry);
-
 
502
            d.v = false;
-
 
503
            dtlb_tag_access_write(t.value);
-
 
504
            dtlb_data_access_write(tlb, entry, d.value);
-
 
505
        }
-
 
506
    } else if (tlb == TLB_ISMALL || tlb == TLB_IBIG) {
-
 
507
        d.value = itlb_data_access_read(tlb, entry);
-
 
508
        if (!d.l || d.g) {
-
 
509
            t.value = itlb_tag_read_read(tlb, entry);
-
 
510
            d.v = false;
-
 
511
            itlb_tag_access_write(t.value);
-
 
512
            itlb_data_access_write(tlb, entry, d.value);
-
 
513
        }
-
 
514
    }
-
 
515
}
-
 
516
#endif
428
 
517
 
-
 
518
/** Invalidate all unlocked ITLB and DTLB entries. */
-
 
519
void tlb_invalidate_all(void)
-
 
520
{
-
 
521
    int i;
-
 
522
   
429
    /*
523
    /*
430
     * Walk all ITLB and DTLB entries and remove all unlocked mappings.
524
     * Walk all ITLB and DTLB entries and remove all unlocked mappings.
431
     *
525
     *
432
     * The kernel doesn't use global mappings so any locked global mappings
526
     * The kernel doesn't use global mappings so any locked global mappings
433
     * found  must have been created by someone else. Their only purpose now
527
     * found must have been created by someone else. Their only purpose now
434
     * is to collide with proper mappings. Invalidate immediately. It should
528
     * is to collide with proper mappings. Invalidate immediately. It should
435
     * be safe to invalidate them as late as now.
529
     * be safe to invalidate them as late as now.
436
     */
530
     */
437
 
531
 
-
 
532
#if defined (US)
-
 
533
    tlb_data_t d;
-
 
534
    tlb_tag_read_reg_t t;
-
 
535
 
438
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
536
    for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
439
        d.value = itlb_data_access_read(i);
537
        d.value = itlb_data_access_read(i);
440
        if (!d.l || d.g) {
538
        if (!d.l || d.g) {
441
            t.value = itlb_tag_read_read(i);
539
            t.value = itlb_tag_read_read(i);
442
            d.v = false;
540
            d.v = false;
443
            itlb_tag_access_write(t.value);
541
            itlb_tag_access_write(t.value);
444
            itlb_data_access_write(i, d.value);
542
            itlb_data_access_write(i, d.value);
445
        }
543
        }
446
    }
544
    }
447
   
545
 
448
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
546
    for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
449
        d.value = dtlb_data_access_read(i);
547
        d.value = dtlb_data_access_read(i);
450
        if (!d.l || d.g) {
548
        if (!d.l || d.g) {
451
            t.value = dtlb_tag_read_read(i);
549
            t.value = dtlb_tag_read_read(i);
452
            d.v = false;
550
            d.v = false;
453
            dtlb_tag_access_write(t.value);
551
            dtlb_tag_access_write(t.value);
454
            dtlb_data_access_write(i, d.value);
552
            dtlb_data_access_write(i, d.value);
455
        }
553
        }
456
    }
554
    }
-
 
555
 
-
 
556
#elif defined (US3)
457
   
557
 
-
 
558
    for (i = 0; i < tlb_ismall_size(); i++)
-
 
559
        tlb_invalidate_entry(TLB_ISMALL, i);
-
 
560
    for (i = 0; i < tlb_ibig_size(); i++)
-
 
561
        tlb_invalidate_entry(TLB_IBIG, i);
-
 
562
    for (i = 0; i < tlb_dsmall_size(); i++)
-
 
563
        tlb_invalidate_entry(TLB_DSMALL, i);
-
 
564
    for (i = 0; i < tlb_dbig_size(); i++)
-
 
565
        tlb_invalidate_entry(TLB_DBIG_0, i);
-
 
566
    for (i = 0; i < tlb_dbig_size(); i++)
-
 
567
        tlb_invalidate_entry(TLB_DBIG_1, i);
-
 
568
#endif
-
 
569
 
458
}
570
}
459
 
571
 
460
/** Invalidate all ITLB and DTLB entries that belong to specified ASID
572
/** Invalidate all ITLB and DTLB entries that belong to specified ASID
461
 * (Context).
573
 * (Context).
462
 *
574
 *
463
 * @param asid Address Space ID.
575
 * @param asid Address Space ID.
464
 */
576
 */
465
void tlb_invalidate_asid(asid_t asid)
577
void tlb_invalidate_asid(asid_t asid)
466
{
578
{
467
    tlb_context_reg_t pc_save, ctx;
579
    tlb_context_reg_t pc_save, ctx;
468
   
580
   
469
    /* switch to nucleus because we are mapped by the primary context */
581
    /* switch to nucleus because we are mapped by the primary context */
470
    nucleus_enter();
582
    nucleus_enter();
471
   
583
   
472
    ctx.v = pc_save.v = mmu_primary_context_read();
584
    ctx.v = pc_save.v = mmu_primary_context_read();
473
    ctx.context = asid;
585
    ctx.context = asid;
474
    mmu_primary_context_write(ctx.v);
586
    mmu_primary_context_write(ctx.v);
475
   
587
   
476
    itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
588
    itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
477
    dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
589
    dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
478
   
590
   
479
    mmu_primary_context_write(pc_save.v);
591
    mmu_primary_context_write(pc_save.v);
480
   
592
   
481
    nucleus_leave();
593
    nucleus_leave();
482
}
594
}
483
 
595
 
484
/** Invalidate all ITLB and DTLB entries for specified page range in specified
596
/** Invalidate all ITLB and DTLB entries for specified page range in specified
485
 * address space.
597
 * address space.
486
 *
598
 *
487
 * @param asid Address Space ID.
599
 * @param asid      Address Space ID.
488
 * @param page First page which to sweep out from ITLB and DTLB.
600
 * @param page      First page which to sweep out from ITLB and DTLB.
489
 * @param cnt Number of ITLB and DTLB entries to invalidate.
601
 * @param cnt       Number of ITLB and DTLB entries to invalidate.
490
 */
602
 */
491
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
603
void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
492
{
604
{
493
    unsigned int i;
605
    unsigned int i;
494
    tlb_context_reg_t pc_save, ctx;
606
    tlb_context_reg_t pc_save, ctx;
495
   
607
   
496
    /* switch to nucleus because we are mapped by the primary context */
608
    /* switch to nucleus because we are mapped by the primary context */
497
    nucleus_enter();
609
    nucleus_enter();
498
   
610
   
499
    ctx.v = pc_save.v = mmu_primary_context_read();
611
    ctx.v = pc_save.v = mmu_primary_context_read();
500
    ctx.context = asid;
612
    ctx.context = asid;
501
    mmu_primary_context_write(ctx.v);
613
    mmu_primary_context_write(ctx.v);
502
   
614
   
503
    for (i = 0; i < cnt * MMU_PAGES_PER_PAGE; i++) {
615
    for (i = 0; i < cnt * MMU_PAGES_PER_PAGE; i++) {
504
        itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
616
        itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
505
            page + i * MMU_PAGE_SIZE);
617
            page + i * MMU_PAGE_SIZE);
506
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
618
        dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
507
            page + i * MMU_PAGE_SIZE);
619
            page + i * MMU_PAGE_SIZE);
508
    }
620
    }
509
   
621
   
510
    mmu_primary_context_write(pc_save.v);
622
    mmu_primary_context_write(pc_save.v);
511
   
623
   
512
    nucleus_leave();
624
    nucleus_leave();
513
}
625
}
514
 
626
 
515
/** @}
627
/** @}
516
 */
628
 */
517
 
629