47,6 → 47,8 |
#include <arch/mm/page.h> |
#include <arch/stack.h> |
#include <arch/sun4v/regdef.h> |
#include <arch/sun4v/arch.h> |
#include <arch/sun4v/cpu.h> |
|
#define TABLE_SIZE TRAP_TABLE_SIZE |
#define ENTRY_SIZE TRAP_TABLE_ENTRY_SIZE |
292,7 → 294,7 |
.org trap_table + TT_FAST_DATA_ACCESS_PROTECTION*ENTRY_SIZE |
.global fast_data_access_protection_handler_tl0 |
fast_data_access_protection_handler_tl0: |
/*FAST_DATA_ACCESS_PROTECTION_HANDLER 0*/ |
FAST_DATA_ACCESS_PROTECTION_HANDLER 0 |
|
/* TT = 0x80, TL = 0, spill_0_normal handler */ |
.org trap_table + TT_SPILL_0_NORMAL*ENTRY_SIZE |
524,42 → 526,27 |
restored |
.endm |
|
#define NOT(x) ((x) == 0) |
|
/* |
* Preemptible trap handler for handling traps from kernel. |
* Perform all the actions of the preemptible trap handler which are common |
* for trapping from kernel and trapping from userspace, including call of the |
* higher level service routine. |
* |
* Important note: |
* This macro must be inserted between the "2:" and "4:" labels. The |
* inserting code must be aware of the usage of all the registers |
* contained in this macro. |
*/ |
.macro PREEMPTIBLE_HANDLER_KERNEL |
|
/* |
* ASSERT(%tl == 1) |
*/ |
rdpr %tl, %g3 |
cmp %g3, 1 |
be 1f |
nop |
0: ba 0b ! this is for debugging, if we ever get here |
nop ! it will be easy to find |
|
/* prevent unnecessary CLEANWIN exceptions */ |
wrpr %g0, NWINDOWS - 1, %cleanwin |
1: |
/* |
* Prevent SAVE instruction from causing a spill exception. If the |
* CANSAVE register is zero, explicitly spill register window |
* at CWP + 2. |
*/ |
|
rdpr %cansave, %g3 |
brnz %g3, 2f |
nop |
INLINE_SPILL %g3, %g4 |
|
2: |
/* ask for new register window */ |
save %sp, -PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE, %sp |
|
.macro MIDDLE_PART is_syscall |
/* copy higher level routine's address and its argument */ |
mov %g1, %l0 |
.if NOT(\is_syscall) |
mov %g2, %o0 |
.else |
! store the syscall number on the stack as 7th argument |
stx %g2, [%sp + STACK_WINDOW_SAVE_AREA_SIZE + STACK_BIAS + STACK_ARG6] |
.endif |
|
/* |
* Save TSTATE, TPC and TNPC aside. |
590,9 → 577,18 |
/* g1 -> l1, ..., g7 -> l7 */ |
SAVE_GLOBALS |
|
.if NOT(\is_syscall) |
/* call higher-level service routine, pass istate as its 2nd parameter */ |
call %l0 |
add %sp, PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE + STACK_BIAS + SAVED_TNPC, %o1 |
.else |
/* Call the higher-level syscall handler. */ |
wrpr %g0, PSTATE_PRIV_BIT | PSTATE_PEF_BIT | PSTATE_IE_BIT, %pstate |
call syscall_handler |
nop |
/* copy the value returned by the syscall */ |
mov %o0, %i0 |
.endif |
|
/* l1 -> g1, ..., l7 -> g7 */ |
RESTORE_GLOBALS |
663,7 → 659,44 |
ldx [%sp + PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE + STACK_BIAS + SAVED_I5], %i5 |
ldx [%sp + PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE + STACK_BIAS + SAVED_I6], %i6 |
ldx [%sp + PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE + STACK_BIAS + SAVED_I7], %i7 |
.endm |
|
/* |
* Preemptible trap handler for handling traps from kernel. |
*/ |
.macro PREEMPTIBLE_HANDLER_KERNEL |
|
/* |
* ASSERT(%tl == 1) |
*/ |
rdpr %tl, %g3 |
cmp %g3, 1 |
be 1f |
nop |
0: ba 0b ! this is for debugging, if we ever get here |
nop ! it will be easy to find |
|
1: |
/* prevent unnecessary CLEANWIN exceptions */ |
wrpr %g0, NWINDOWS - 1, %cleanwin |
|
/* |
* Prevent SAVE instruction from causing a spill exception. If the |
* CANSAVE register is zero, explicitly spill register window |
* at CWP + 2. |
*/ |
|
rdpr %cansave, %g3 |
brnz %g3, 2f |
nop |
INLINE_SPILL %g3, %g4 |
|
2: |
/* ask for new register window */ |
save %sp, -PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE, %sp |
|
MIDDLE_PART 0 |
|
4: |
/* |
* Prevent RESTORE instruction from causing a fill exception. If the |
681,9 → 714,232 |
retry |
.endm |
|
/* |
* Spills the window at CWP + 2 to the userspace window buffer. This macro |
* is to be used before doing SAVE when the spill trap is undesirable. |
* |
* Parameters: |
* tmpreg1 global register to be used for scratching purposes |
* tmpreg2 global register to be used for scratching purposes |
* tmpreg3 global register to be used for scratching purposes |
*/ |
.macro INLINE_SPILL_TO_WBUF tmpreg1, tmpreg2, tmpreg3 |
! CWP := CWP + 2 |
rdpr %cwp, \tmpreg2 |
add \tmpreg2, 2, \tmpreg1 |
and \tmpreg1, NWINDOWS - 1, \tmpreg1 ! modulo NWINDOWS |
wrpr \tmpreg1, %cwp |
|
! spill to userspace window buffer |
SAVE_TO_USPACE_WBUF \tmpreg3, \tmpreg1 |
|
#define NOT(x) ((x) == 0) |
! CWP := CWP - 2 |
wrpr \tmpreg2, %cwp |
|
saved |
.endm |
|
/* |
* Preemptible handler for handling traps from userspace. |
*/ |
.macro PREEMPTIBLE_HANDLER_USPACE is_syscall |
/* |
* One of the ways this handler can be invoked is after a nested MMU trap from |
* either spill_1_normal or fill_1_normal traps. Both of these traps manipulate |
* the CWP register. We deal with the situation by simulating the MMU trap |
* on TL=1 and restart the respective SAVE or RESTORE instruction once the MMU |
* trap is resolved. However, because we are in the wrong window from the |
* perspective of the MMU trap, we need to synchronize CWP with CWP from TL=0. |
*/ |
.if NOT(\is_syscall) |
rdpr %tstate, %g3 |
and %g3, TSTATE_CWP_MASK, %g4 |
wrpr %g4, 0, %cwp ! resynchronize CWP |
.endif |
|
/* prevent unnecessary CLEANWIN exceptions */ |
wrpr %g0, NWINDOWS - 1, %cleanwin |
|
/* |
* Prevent SAVE instruction from causing a spill exception. If the |
* CANSAVE register is zero, explicitly spill register window |
* at CWP + 2. |
*/ |
rdpr %cansave, %g3 |
brnz %g3, 2f |
nop |
INLINE_SPILL_TO_WBUF %g3, %g4, %g7 |
|
2: |
get_kstack_wbuf_ptr %g3, %g4 |
ldx [%g4], %g6 |
save %g6, -PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE, %sp |
|
.if \is_syscall |
/* Copy arguments for the syscall to the new window. */ |
mov %i0, %o0 |
mov %i1, %o1 |
mov %i2, %o2 |
mov %i3, %o3 |
mov %i4, %o4 |
mov %i5, %o5 |
.endif |
|
mov VA_PRIMARY_CONTEXT_REG, %l0 |
stxa %g0, [%l0] ASI_PRIMARY_CONTEXT_REG |
rd %pc, %l0 |
flush %l0 |
|
/* Mark the CANRESTORE windows as OTHER windows. */ |
rdpr %canrestore, %l0 |
wrpr %l0, %otherwin |
wrpr %g0, %canrestore |
|
/* |
* Other window spills will go to the userspace window buffer |
* and normal spills will go to the kernel stack. |
*/ |
wrpr %g0, WSTATE_OTHER(0) | WSTATE_NORMAL(0), %wstate |
|
MIDDLE_PART \is_syscall |
|
4: |
/* |
* Spills and fills will be processed by the {spill,fill}_1_normal |
* handlers. |
*/ |
wrpr %g0, WSTATE_OTHER(0) | WSTATE_NORMAL(1), %wstate |
|
/* |
* Set primary context according to secondary context. |
*/ |
wr %g0, ASI_SECONDARY_CONTEXT_REG, %asi |
ldxa [VA_SECONDARY_CONTEXT_REG] %asi, %g1 |
wr %g0, ASI_PRIMARY_CONTEXT_REG, %asi |
stxa %g1, [VA_PRIMARY_CONTEXT_REG] %asi |
rd %pc, %g1 |
flush %g1 |
|
/* Restoring userspace windows: */ |
|
/* Save address of the userspace window buffer to the %g7 register. */ |
get_kstack_wbuf_ptr %g1, %g5 |
ldx [%g5 + 8], %g7 |
|
rdpr %cwp, %g1 |
rdpr %otherwin, %g2 |
|
/* |
* Skip all OTHERWIN windows and descend to the first window |
* in the userspace window buffer. |
*/ |
sub %g1, %g2, %g3 |
dec %g3 |
and %g3, NWINDOWS - 1, %g3 |
wrpr %g3, 0, %cwp |
|
/* |
* CWP is now in the window last saved in the userspace window buffer. |
* Fill all windows stored in the buffer. |
*/ |
clr %g4 |
5: andcc %g7, UWB_ALIGNMENT - 1, %g0 ! alignment check |
bz 6f ! %g7 is UWB_ALIGNMENT-aligned, no more windows to refill |
nop |
|
add %g7, -STACK_WINDOW_SAVE_AREA_SIZE, %g7 |
ldx [%g7 + L0_OFFSET], %l0 |
ldx [%g7 + L1_OFFSET], %l1 |
ldx [%g7 + L2_OFFSET], %l2 |
ldx [%g7 + L3_OFFSET], %l3 |
ldx [%g7 + L4_OFFSET], %l4 |
ldx [%g7 + L5_OFFSET], %l5 |
ldx [%g7 + L6_OFFSET], %l6 |
ldx [%g7 + L7_OFFSET], %l7 |
ldx [%g7 + I0_OFFSET], %i0 |
ldx [%g7 + I1_OFFSET], %i1 |
ldx [%g7 + I2_OFFSET], %i2 |
ldx [%g7 + I3_OFFSET], %i3 |
ldx [%g7 + I4_OFFSET], %i4 |
ldx [%g7 + I5_OFFSET], %i5 |
ldx [%g7 + I6_OFFSET], %i6 |
ldx [%g7 + I7_OFFSET], %i7 |
|
dec %g3 |
and %g3, NWINDOWS - 1, %g3 |
wrpr %g3, 0, %cwp ! switch to the preceeding window |
|
ba 5b |
inc %g4 |
|
6: |
/* Save changes of the address of the userspace window buffer. */ |
stx %g7, [%g5 + 8] |
|
/* |
* Switch back to the proper current window and adjust |
* OTHERWIN, CANRESTORE, CANSAVE and CLEANWIN. |
*/ |
wrpr %g1, 0, %cwp |
add %g4, %g2, %g2 |
cmp %g2, NWINDOWS - 2 |
bg 8f ! fix the CANRESTORE=NWINDOWS-1 anomaly |
mov NWINDOWS - 2, %g1 ! use dealy slot for both cases |
sub %g1, %g2, %g1 |
|
wrpr %g0, 0, %otherwin |
wrpr %g1, 0, %cansave ! NWINDOWS - 2 - CANRESTORE |
wrpr %g2, 0, %canrestore ! OTHERWIN + windows in the buffer |
wrpr %g2, 0, %cleanwin ! avoid information leak |
|
7: |
restore |
|
.if \is_syscall |
done |
.else |
retry |
.endif |
|
8: |
/* |
* We got here in order to avoid inconsistency of the window state registers. |
* If the: |
* |
* save %g6, -PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE, %sp |
* |
* instruction trapped and spilled a register window into the userspace |
* window buffer, we have just restored NWINDOWS - 1 register windows. |
* However, CANRESTORE can be only NWINDOW - 2 at most. |
* |
* The solution is to manually switch to (CWP - 1) mod NWINDOWS |
* and set the window state registers so that: |
* |
* CANRESTORE = NWINDOWS - 2 |
* CLEANWIN = NWINDOWS - 2 |
* CANSAVE = 0 |
* OTHERWIN = 0 |
* |
* The RESTORE instruction is therfore to be skipped. |
*/ |
wrpr %g0, 0, %otherwin |
wrpr %g0, 0, %cansave |
wrpr %g1, 0, %canrestore |
wrpr %g1, 0, %cleanwin |
|
rdpr %cwp, %g1 |
dec %g1 |
and %g1, NWINDOWS - 1, %g1 |
wrpr %g1, 0, %cwp ! CWP-- |
|
.if \is_syscall |
done |
.else |
retry |
.endif |
|
.endm |
|
/* Preemptible trap handler for TL=1. |
* |
* This trap handler makes arrangements to make calling of scheduler() from |
691,7 → 947,19 |
* handlers. |
*/ |
.macro PREEMPTIBLE_HANDLER_TEMPLATE is_syscall |
rdpr %tstate, %g3 |
and %g3, TSTATE_PRIV_BIT, %g3 |
brz %g3, 100f ! trapping from userspace |
nop |
|
PREEMPTIBLE_HANDLER_KERNEL |
ba 101f |
nop |
|
100: |
PREEMPTIBLE_HANDLER_USPACE \is_syscall |
|
101: |
.endm |
|
.global preemptible_handler |