Rev 3948 | Rev 4021 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed
| Rev 3948 | Rev 4017 | ||
|---|---|---|---|
| Line 24... | Line 24... | ||
| 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | */ |
27 | */ |
| 28 | 28 | ||
| 29 | /** @addtogroup amd64 |
29 | /** @addtogroup amd64 |
| 30 | * @{ |
30 | * @{ |
| 31 | */ |
31 | */ |
| 32 | /** @file |
32 | /** @file |
| 33 | */ |
33 | */ |
| 34 | 34 | ||
| Line 43... | Line 43... | ||
| 43 | /** Return base address of current stack. |
43 | /** Return base address of current stack. |
| 44 | * |
44 | * |
| 45 | * Return the base address of the current stack. |
45 | * Return the base address of the current stack. |
| 46 | * The stack is assumed to be STACK_SIZE bytes long. |
46 | * The stack is assumed to be STACK_SIZE bytes long. |
| 47 | * The stack must start on page boundary. |
47 | * The stack must start on page boundary. |
| - | 48 | * |
|
| 48 | */ |
49 | */ |
| 49 | static inline uintptr_t get_stack_base(void) |
50 | static inline uintptr_t get_stack_base(void) |
| 50 | { |
51 | { |
| 51 | uintptr_t v; |
52 | uintptr_t v; |
| 52 | 53 | ||
| - | 54 | asm volatile ( |
|
| - | 55 | "andq %%rsp, %[v]\n" |
|
| - | 56 | : [v] "=r" (v) |
|
| 53 | asm volatile ("andq %%rsp, %0\n" : "=r" (v) : "0" (~((uint64_t)STACK_SIZE-1))); |
57 | : "0" (~((uint64_t) STACK_SIZE-1)) |
| - | 58 | ); |
|
| 54 | 59 | ||
| 55 | return v; |
60 | return v; |
| 56 | } |
61 | } |
| 57 | 62 | ||
| 58 | static inline void cpu_sleep(void) |
63 | static inline void cpu_sleep(void) |
| Line 70... | Line 75... | ||
| 70 | * |
75 | * |
| 71 | * Get byte from port |
76 | * Get byte from port |
| 72 | * |
77 | * |
| 73 | * @param port Port to read from |
78 | * @param port Port to read from |
| 74 | * @return Value read |
79 | * @return Value read |
| - | 80 | * |
|
| 75 | */ |
81 | */ |
| 76 | static inline uint8_t pio_read_8(ioport8_t *port) |
82 | static inline uint8_t pio_read_8(ioport8_t *port) |
| 77 | { |
83 | { |
| 78 | uint8_t val; |
84 | uint8_t val; |
| 79 | 85 | ||
| - | 86 | asm volatile ( |
|
| 80 | asm volatile ("inb %w1, %b0 \n" : "=a" (val) : "d" (port)); |
87 | "inb %w[port], %b[val]\n" |
| - | 88 | : [val] "=a" (val) |
|
| - | 89 | : [port] "d" (port) |
|
| - | 90 | ); |
|
| - | 91 | ||
| 81 | return val; |
92 | return val; |
| 82 | } |
93 | } |
| 83 | 94 | ||
| 84 | /** Word from port |
95 | /** Word from port |
| 85 | * |
96 | * |
| 86 | * Get word from port |
97 | * Get word from port |
| 87 | * |
98 | * |
| 88 | * @param port Port to read from |
99 | * @param port Port to read from |
| 89 | * @return Value read |
100 | * @return Value read |
| - | 101 | * |
|
| 90 | */ |
102 | */ |
| 91 | static inline uint16_t pio_read_16(ioport16_t *port) |
103 | static inline uint16_t pio_read_16(ioport16_t *port) |
| 92 | { |
104 | { |
| 93 | uint16_t val; |
105 | uint16_t val; |
| 94 | 106 | ||
| - | 107 | asm volatile ( |
|
| 95 | asm volatile ("inw %w1, %w0 \n" : "=a" (val) : "d" (port)); |
108 | "inw %w[port], %w[val]\n" |
| - | 109 | : [val] "=a" (val) |
|
| - | 110 | : [port] "d" (port) |
|
| - | 111 | ); |
|
| - | 112 | ||
| 96 | return val; |
113 | return val; |
| 97 | } |
114 | } |
| 98 | 115 | ||
| 99 | /** Double word from port |
116 | /** Double word from port |
| 100 | * |
117 | * |
| 101 | * Get double word from port |
118 | * Get double word from port |
| 102 | * |
119 | * |
| 103 | * @param port Port to read from |
120 | * @param port Port to read from |
| 104 | * @return Value read |
121 | * @return Value read |
| - | 122 | * |
|
| 105 | */ |
123 | */ |
| 106 | static inline uint32_t pio_read_32(ioport32_t *port) |
124 | static inline uint32_t pio_read_32(ioport32_t *port) |
| 107 | { |
125 | { |
| 108 | uint32_t val; |
126 | uint32_t val; |
| 109 | 127 | ||
| - | 128 | asm volatile ( |
|
| 110 | asm volatile ("inl %w1, %0 \n" : "=a" (val) : "d" (port)); |
129 | "inl %w[port], %[val]\n" |
| - | 130 | : [val] "=a" (val) |
|
| - | 131 | : [port] "d" (port) |
|
| - | 132 | ); |
|
| - | 133 | ||
| 111 | return val; |
134 | return val; |
| 112 | } |
135 | } |
| 113 | 136 | ||
| 114 | /** Byte to port |
137 | /** Byte to port |
| 115 | * |
138 | * |
| 116 | * Output byte to port |
139 | * Output byte to port |
| 117 | * |
140 | * |
| 118 | * @param port Port to write to |
141 | * @param port Port to write to |
| 119 | * @param val Value to write |
142 | * @param val Value to write |
| - | 143 | * |
|
| 120 | */ |
144 | */ |
| 121 | static inline void pio_write_8(ioport8_t *port, uint8_t val) |
145 | static inline void pio_write_8(ioport8_t *port, uint8_t val) |
| 122 | { |
146 | { |
| - | 147 | asm volatile ( |
|
| - | 148 | "outb %b[val], %w[port]\n" |
|
| 123 | asm volatile ("outb %b0, %w1\n" : : "a" (val), "d" (port)); |
149 | :: [val] "a" (val), [port] "d" (port) |
| - | 150 | ); |
|
| 124 | } |
151 | } |
| 125 | 152 | ||
| 126 | /** Word to port |
153 | /** Word to port |
| 127 | * |
154 | * |
| 128 | * Output word to port |
155 | * Output word to port |
| 129 | * |
156 | * |
| 130 | * @param port Port to write to |
157 | * @param port Port to write to |
| 131 | * @param val Value to write |
158 | * @param val Value to write |
| - | 159 | * |
|
| 132 | */ |
160 | */ |
| 133 | static inline void pio_write_16(ioport16_t *port, uint16_t val) |
161 | static inline void pio_write_16(ioport16_t *port, uint16_t val) |
| 134 | { |
162 | { |
| - | 163 | asm volatile ( |
|
| - | 164 | "outw %w[val], %w[port]\n" |
|
| 135 | asm volatile ("outw %w0, %w1\n" : : "a" (val), "d" (port)); |
165 | :: [val] "a" (val), [port] "d" (port) |
| - | 166 | ); |
|
| 136 | } |
167 | } |
| 137 | 168 | ||
| 138 | /** Double word to port |
169 | /** Double word to port |
| 139 | * |
170 | * |
| 140 | * Output double word to port |
171 | * Output double word to port |
| 141 | * |
172 | * |
| 142 | * @param port Port to write to |
173 | * @param port Port to write to |
| 143 | * @param val Value to write |
174 | * @param val Value to write |
| - | 175 | * |
|
| 144 | */ |
176 | */ |
| 145 | static inline void pio_write_32(ioport32_t *port, uint32_t val) |
177 | static inline void pio_write_32(ioport32_t *port, uint32_t val) |
| 146 | { |
178 | { |
| - | 179 | asm volatile ( |
|
| - | 180 | "outl %[val], %w[port]\n" |
|
| 147 | asm volatile ("outl %0, %w1\n" : : "a" (val), "d" (port)); |
181 | :: [val] "a" (val), [port] "d" (port) |
| - | 182 | ); |
|
| 148 | } |
183 | } |
| 149 | 184 | ||
| 150 | /** Swap Hidden part of GS register with visible one */ |
185 | /** Swap Hidden part of GS register with visible one */ |
| 151 | static inline void swapgs(void) |
186 | static inline void swapgs(void) |
| 152 | { |
187 | { |
| Line 157... | Line 192... | ||
| 157 | * |
192 | * |
| 158 | * Enable interrupts and return previous |
193 | * Enable interrupts and return previous |
| 159 | * value of EFLAGS. |
194 | * value of EFLAGS. |
| 160 | * |
195 | * |
| 161 | * @return Old interrupt priority level. |
196 | * @return Old interrupt priority level. |
| - | 197 | * |
|
| 162 | */ |
198 | */ |
| 163 | static inline ipl_t interrupts_enable(void) { |
199 | static inline ipl_t interrupts_enable(void) { |
| 164 | ipl_t v; |
200 | ipl_t v; |
| - | 201 | ||
| 165 | __asm__ volatile ( |
202 | asm volatile ( |
| 166 | "pushfq\n" |
203 | "pushfq\n" |
| 167 | "popq %0\n" |
204 | "popq %[v]\n" |
| 168 | "sti\n" |
205 | "sti\n" |
| 169 | : "=r" (v) |
206 | : [v] "=r" (v) |
| 170 | ); |
207 | ); |
| - | 208 | ||
| 171 | return v; |
209 | return v; |
| 172 | } |
210 | } |
| 173 | 211 | ||
| 174 | /** Disable interrupts. |
212 | /** Disable interrupts. |
| 175 | * |
213 | * |
| 176 | * Disable interrupts and return previous |
214 | * Disable interrupts and return previous |
| 177 | * value of EFLAGS. |
215 | * value of EFLAGS. |
| 178 | * |
216 | * |
| 179 | * @return Old interrupt priority level. |
217 | * @return Old interrupt priority level. |
| - | 218 | * |
|
| 180 | */ |
219 | */ |
| 181 | static inline ipl_t interrupts_disable(void) { |
220 | static inline ipl_t interrupts_disable(void) { |
| 182 | ipl_t v; |
221 | ipl_t v; |
| - | 222 | ||
| 183 | __asm__ volatile ( |
223 | asm volatile ( |
| 184 | "pushfq\n" |
224 | "pushfq\n" |
| 185 | "popq %0\n" |
225 | "popq %[v]\n" |
| 186 | "cli\n" |
226 | "cli\n" |
| 187 | : "=r" (v) |
227 | : [v] "=r" (v) |
| 188 | ); |
228 | ); |
| - | 229 | ||
| 189 | return v; |
230 | return v; |
| 190 | } |
231 | } |
| 191 | 232 | ||
| 192 | /** Restore interrupt priority level. |
233 | /** Restore interrupt priority level. |
| 193 | * |
234 | * |
| 194 | * Restore EFLAGS. |
235 | * Restore EFLAGS. |
| 195 | * |
236 | * |
| 196 | * @param ipl Saved interrupt priority level. |
237 | * @param ipl Saved interrupt priority level. |
| - | 238 | * |
|
| 197 | */ |
239 | */ |
| 198 | static inline void interrupts_restore(ipl_t ipl) { |
240 | static inline void interrupts_restore(ipl_t ipl) { |
| 199 | __asm__ volatile ( |
241 | asm volatile ( |
| 200 | "pushq %0\n" |
242 | "pushq %[ipl]\n" |
| 201 | "popfq\n" |
243 | "popfq\n" |
| 202 | : : "r" (ipl) |
244 | :: [ipl] "r" (ipl) |
| 203 | ); |
245 | ); |
| 204 | } |
246 | } |
| 205 | 247 | ||
| 206 | /** Return interrupt priority level. |
248 | /** Return interrupt priority level. |
| 207 | * |
249 | * |
| 208 | * Return EFLAFS. |
250 | * Return EFLAFS. |
| 209 | * |
251 | * |
| 210 | * @return Current interrupt priority level. |
252 | * @return Current interrupt priority level. |
| - | 253 | * |
|
| 211 | */ |
254 | */ |
| 212 | static inline ipl_t interrupts_read(void) { |
255 | static inline ipl_t interrupts_read(void) { |
| 213 | ipl_t v; |
256 | ipl_t v; |
| - | 257 | ||
| 214 | __asm__ volatile ( |
258 | asm volatile ( |
| 215 | "pushfq\n" |
259 | "pushfq\n" |
| 216 | "popq %0\n" |
260 | "popq %[v]\n" |
| 217 | : "=r" (v) |
261 | : [v] "=r" (v) |
| 218 | ); |
262 | ); |
| - | 263 | ||
| 219 | return v; |
264 | return v; |
| 220 | } |
265 | } |
| 221 | 266 | ||
| 222 | /** Write to MSR */ |
267 | /** Write to MSR */ |
| 223 | static inline void write_msr(uint32_t msr, uint64_t value) |
268 | static inline void write_msr(uint32_t msr, uint64_t value) |
| 224 | { |
269 | { |
| 225 | __asm__ volatile ( |
270 | asm volatile ( |
| - | 271 | "wrmsr\n" |
|
| 226 | "wrmsr;" : : "c" (msr), |
272 | :: "c" (msr), |
| 227 | "a" ((uint32_t)(value)), |
273 | "a" ((uint32_t) (value)), |
| 228 | "d" ((uint32_t)(value >> 32)) |
274 | "d" ((uint32_t) (value >> 32)) |
| 229 | ); |
275 | ); |
| 230 | } |
276 | } |
| 231 | 277 | ||
| 232 | static inline unative_t read_msr(uint32_t msr) |
278 | static inline unative_t read_msr(uint32_t msr) |
| 233 | { |
279 | { |
| 234 | uint32_t ax, dx; |
280 | uint32_t ax, dx; |
| 235 | 281 | ||
| 236 | __asm__ volatile ( |
282 | asm volatile ( |
| - | 283 | "rdmsr\n" |
|
| 237 | "rdmsr;" : "=a"(ax), "=d"(dx) : "c" (msr) |
284 | : "=a" (ax), "=d" (dx) |
| - | 285 | : "c" (msr) |
|
| 238 | ); |
286 | ); |
| - | 287 | ||
| 239 | return ((uint64_t)dx << 32) | ax; |
288 | return ((uint64_t) dx << 32) | ax; |
| 240 | } |
289 | } |
| 241 | 290 | ||
| 242 | 291 | ||
| 243 | /** Enable local APIC |
292 | /** Enable local APIC |
| 244 | * |
293 | * |
| 245 | * Enable local APIC in MSR. |
294 | * Enable local APIC in MSR. |
| - | 295 | * |
|
| 246 | */ |
296 | */ |
| 247 | static inline void enable_l_apic_in_msr() |
297 | static inline void enable_l_apic_in_msr() |
| 248 | { |
298 | { |
| 249 | __asm__ volatile ( |
299 | asm volatile ( |
| 250 | "movl $0x1b, %%ecx\n" |
300 | "movl $0x1b, %%ecx\n" |
| 251 | "rdmsr\n" |
301 | "rdmsr\n" |
| 252 | "orl $(1<<11),%%eax\n" |
302 | "orl $(1 << 11),%%eax\n" |
| 253 | "orl $(0xfee00000),%%eax\n" |
303 | "orl $(0xfee00000),%%eax\n" |
| 254 | "wrmsr\n" |
304 | "wrmsr\n" |
| 255 | : |
- | |
| 256 | : |
- | |
| 257 | :"%eax","%ecx","%edx" |
305 | ::: "%eax","%ecx","%edx" |
| 258 | ); |
306 | ); |
| 259 | } |
307 | } |
| 260 | 308 | ||
| 261 | static inline uintptr_t * get_ip() |
309 | static inline uintptr_t * get_ip() |
| 262 | { |
310 | { |
| 263 | uintptr_t *ip; |
311 | uintptr_t *ip; |
| 264 | 312 | ||
| 265 | __asm__ volatile ( |
313 | asm volatile ( |
| 266 | "mov %%rip, %0" |
314 | "mov %%rip, %[ip]" |
| 267 | : "=r" (ip) |
315 | : [ip] "=r" (ip) |
| 268 | ); |
316 | ); |
| - | 317 | ||
| 269 | return ip; |
318 | return ip; |
| 270 | } |
319 | } |
| 271 | 320 | ||
| 272 | /** Invalidate TLB Entry. |
321 | /** Invalidate TLB Entry. |
| 273 | * |
322 | * |
| 274 | * @param addr Address on a page whose TLB entry is to be invalidated. |
323 | * @param addr Address on a page whose TLB entry is to be invalidated. |
| - | 324 | * |
|
| 275 | */ |
325 | */ |
| 276 | static inline void invlpg(uintptr_t addr) |
326 | static inline void invlpg(uintptr_t addr) |
| 277 | { |
327 | { |
| - | 328 | asm volatile ( |
|
| - | 329 | "invlpg %[addr]\n" |
|
| 278 | __asm__ volatile ("invlpg %0\n" :: "m" (*((unative_t *)addr))); |
330 | :: [addr] "m" (*((unative_t *) addr)) |
| - | 331 | ); |
|
| 279 | } |
332 | } |
| 280 | 333 | ||
| 281 | /** Load GDTR register from memory. |
334 | /** Load GDTR register from memory. |
| 282 | * |
335 | * |
| 283 | * @param gdtr_reg Address of memory from where to load GDTR. |
336 | * @param gdtr_reg Address of memory from where to load GDTR. |
| - | 337 | * |
|
| 284 | */ |
338 | */ |
| 285 | static inline void gdtr_load(struct ptr_16_64 *gdtr_reg) |
339 | static inline void gdtr_load(struct ptr_16_64 *gdtr_reg) |
| 286 | { |
340 | { |
| - | 341 | asm volatile ( |
|
| - | 342 | "lgdtq %[gdtr_reg]\n" |
|
| 287 | __asm__ volatile ("lgdtq %0\n" : : "m" (*gdtr_reg)); |
343 | :: [gdtr_reg] "m" (*gdtr_reg) |
| - | 344 | ); |
|
| 288 | } |
345 | } |
| 289 | 346 | ||
| 290 | /** Store GDTR register to memory. |
347 | /** Store GDTR register to memory. |
| 291 | * |
348 | * |
| 292 | * @param gdtr_reg Address of memory to where to load GDTR. |
349 | * @param gdtr_reg Address of memory to where to load GDTR. |
| - | 350 | * |
|
| 293 | */ |
351 | */ |
| 294 | static inline void gdtr_store(struct ptr_16_64 *gdtr_reg) |
352 | static inline void gdtr_store(struct ptr_16_64 *gdtr_reg) |
| 295 | { |
353 | { |
| - | 354 | asm volatile ( |
|
| - | 355 | "sgdtq %[gdtr_reg]\n" |
|
| 296 | __asm__ volatile ("sgdtq %0\n" : : "m" (*gdtr_reg)); |
356 | :: [gdtr_reg] "m" (*gdtr_reg) |
| - | 357 | ); |
|
| 297 | } |
358 | } |
| 298 | 359 | ||
| 299 | /** Load IDTR register from memory. |
360 | /** Load IDTR register from memory. |
| 300 | * |
361 | * |
| 301 | * @param idtr_reg Address of memory from where to load IDTR. |
362 | * @param idtr_reg Address of memory from where to load IDTR. |
| - | 363 | * |
|
| 302 | */ |
364 | */ |
| 303 | static inline void idtr_load(struct ptr_16_64 *idtr_reg) |
365 | static inline void idtr_load(struct ptr_16_64 *idtr_reg) |
| 304 | { |
366 | { |
| - | 367 | asm volatile ( |
|
| - | 368 | "lidtq %[idtr_reg]\n" |
|
| 305 | __asm__ volatile ("lidtq %0\n" : : "m" (*idtr_reg)); |
369 | :: [idtr_reg] "m" (*idtr_reg)); |
| 306 | } |
370 | } |
| 307 | 371 | ||
| 308 | /** Load TR from descriptor table. |
372 | /** Load TR from descriptor table. |
| 309 | * |
373 | * |
| 310 | * @param sel Selector specifying descriptor of TSS segment. |
374 | * @param sel Selector specifying descriptor of TSS segment. |
| - | 375 | * |
|
| 311 | */ |
376 | */ |
| 312 | static inline void tr_load(uint16_t sel) |
377 | static inline void tr_load(uint16_t sel) |
| 313 | { |
378 | { |
| - | 379 | asm volatile ( |
|
| - | 380 | "ltr %[sel]" |
|
| 314 | __asm__ volatile ("ltr %0" : : "r" (sel)); |
381 | :: [sel] "r" (sel) |
| - | 382 | ); |
|
| 315 | } |
383 | } |
| 316 | 384 | ||
| 317 | #define GEN_READ_REG(reg) static inline unative_t read_ ##reg (void) \ |
385 | #define GEN_READ_REG(reg) static inline unative_t read_ ##reg (void) \ |
| 318 | { \ |
386 | { \ |
| 319 | unative_t res; \ |
387 | unative_t res; \ |
| - | 388 | asm volatile ( \ |
|
| 320 | __asm__ volatile ("movq %%" #reg ", %0" : "=r" (res) ); \ |
389 | "movq %%" #reg ", %[res]" \ |
| - | 390 | : [res] "=r" (res) \ |
|
| - | 391 | ); \ |
|
| 321 | return res; \ |
392 | return res; \ |
| 322 | } |
393 | } |
| 323 | 394 | ||
| 324 | #define GEN_WRITE_REG(reg) static inline void write_ ##reg (unative_t regn) \ |
395 | #define GEN_WRITE_REG(reg) static inline void write_ ##reg (unative_t regn) \ |
| 325 | { \ |
396 | { \ |
| - | 397 | asm volatile ( \ |
|
| 326 | __asm__ volatile ("movq %0, %%" #reg : : "r" (regn)); \ |
398 | "movq %[regn], %%" #reg \ |
| - | 399 | :: [regn] "r" (regn) \ |
|
| 327 | } |
400 | ); \ |
| - | 401 | } |
|
| 328 | 402 | ||
| 329 | GEN_READ_REG(cr0) |
403 | GEN_READ_REG(cr0) |
| 330 | GEN_READ_REG(cr2) |
404 | GEN_READ_REG(cr2) |
| 331 | GEN_READ_REG(cr3) |
405 | GEN_READ_REG(cr3) |
| 332 | GEN_WRITE_REG(cr3) |
406 | GEN_WRITE_REG(cr3) |