1 #ifdef LISP_FEATURE_SB_THREAD
2 #include <mach/mach_init.h>
3 #endif
4
5 #include "thread.h"
6 #include "validate.h"
7 #include "runtime.h"
8 #include "interrupt.h"
9 #include "x86-64-darwin-os.h"
10 #include "genesis/fdefn.h"
11
12 #include <mach/mach.h>
13 #include <mach/mach_error.h>
14 #include <mach/mach_types.h>
15 #include <mach/sync_policy.h>
16 #include <mach/machine/thread_state.h>
17 #include <mach/machine/thread_status.h>
18 #include <sys/_types.h>
19 #include <sys/ucontext.h>
20 #include <pthread.h>
21 #include <assert.h>
22 #include <stdlib.h>
23 #include <stdio.h>
24
25 #if __DARWIN_UNIX03
26 #include <sys/_structs.h>
27 #endif
28
29 #if __DARWIN_UNIX03
30
31 typedef struct __darwin_ucontext darwin_ucontext;
32 typedef struct __darwin_mcontext64 darwin_mcontext;
33
34 #define rip __rip
35 #define rsp __rsp
36 #define rbp __rbp
37 #define rax __rax
38 #define rbx __rbx
39 #define rcx __rcx
40 #define rdx __rdx
41 #define rsi __rsi
42 #define rdi __rdi
43 #define r8 __r8
44 #define r9 __r9
45 #define faultvaddr __faultvaddr
46 #define ss __ss
47 #define es __es
48 #define fs __fs
49
50 #define fpu_fcw __fpu_fcw
51 #define fpu_mxcsr __fpu_mxcsr
52
53 #else
54
55 typedef struct ucontext darwin_ucontext;
56 typedef struct mcontext darwin_mcontext;
57
58 #endif
59
60 #ifdef LISP_FEATURE_SB_THREAD
61 pthread_mutex_t mach_exception_lock = PTHREAD_MUTEX_INITIALIZER;
62 #endif
63
64 #ifdef LISP_FEATURE_MACH_EXCEPTION_HANDLER
65
66 void sigill_handler(int signal, siginfo_t *siginfo, os_context_t *context);
67 void sigtrap_handler(int signal, siginfo_t *siginfo, os_context_t *context);
68 void memory_fault_handler(int signal, siginfo_t *siginfo,
69 os_context_t *context);
70
71 /* This executes in the faulting thread as part of the signal
72 * emulation. It is passed a context with the uc_mcontext field
73 * pointing to a valid block of memory. */
build_fake_signal_context(darwin_ucontext * context,x86_thread_state64_t * thread_state,x86_float_state64_t * float_state)74 void build_fake_signal_context(darwin_ucontext *context,
75 x86_thread_state64_t *thread_state,
76 x86_float_state64_t *float_state) {
77 thread_sigmask(0, NULL, &context->uc_sigmask);
78 context->uc_mcontext->ss = *thread_state;
79 context->uc_mcontext->fs = *float_state;
80 }
81
82 /* This executes in the faulting thread as part of the signal
83 * emulation. It is effectively the inverse operation from above. */
update_thread_state_from_context(x86_thread_state64_t * thread_state,x86_float_state64_t * float_state,darwin_ucontext * context)84 void update_thread_state_from_context(x86_thread_state64_t *thread_state,
85 x86_float_state64_t *float_state,
86 darwin_ucontext *context) {
87 *thread_state = context->uc_mcontext->ss;
88 *float_state = context->uc_mcontext->fs;
89 thread_sigmask(SIG_SETMASK, &context->uc_sigmask, NULL);
90 }
91
92 /* Modify a context to push new data on its stack. */
push_context(u64 data,x86_thread_state64_t * context)93 void push_context(u64 data, x86_thread_state64_t *context)
94 {
95 u64 *stack_pointer;
96
97 stack_pointer = (u64*) context->rsp;
98 *(--stack_pointer) = data;
99 context->rsp = (u64) stack_pointer;
100 }
101
align_context_stack(x86_thread_state64_t * context)102 void align_context_stack(x86_thread_state64_t *context)
103 {
104 /* 16byte align the stack (provided that the stack is, as it
105 * should be, 8byte aligned. */
106 while (context->rsp & 15) push_context(0, context);
107 }
108
109 /* Stack allocation starts with a context that has a mod-4 ESP value
110 * and needs to leave a context with a mod-16 ESP that will restore
111 * the old ESP value and other register state when activated. The
112 * first part of this is the recovery trampoline, which loads ESP from
113 * EBP, pops EBP, and returns. */
114 asm(".globl _stack_allocation_recover; \
115 .align 4; \
116 _stack_allocation_recover: \
117 lea -48(%rbp), %rsp; \
118 pop %rsi; \
119 pop %rdi; \
120 pop %rdx; \
121 pop %rcx; \
122 pop %r8; \
123 pop %r9; \
124 pop %rbp; \
125 ret;");
126
open_stack_allocation(x86_thread_state64_t * context)127 void open_stack_allocation(x86_thread_state64_t *context)
128 {
129 void stack_allocation_recover(void);
130
131 push_context(context->rip, context);
132 push_context(context->rbp, context);
133 context->rbp = context->rsp;
134
135 push_context(context->r9, context);
136 push_context(context->r8, context);
137 push_context(context->rcx, context);
138 push_context(context->rdx, context);
139 push_context(context->rsi, context);
140 push_context(context->rdi, context);
141
142 context->rip = (u64) stack_allocation_recover;
143
144 align_context_stack(context);
145 }
146
147 /* Stack allocation of data starts with a context with a mod-16 ESP
148 * value and reserves some space on it by manipulating the ESP
149 * register. */
stack_allocate(x86_thread_state64_t * context,size_t size)150 void *stack_allocate(x86_thread_state64_t *context, size_t size)
151 {
152 /* round up size to 16byte multiple */
153 size = (size + 15) & -16;
154
155 context->rsp = ((u64)context->rsp) - size;
156
157 return (void *)context->rsp;
158 }
159
160 /* Arranging to invoke a C function is tricky, as we have to assume
161 * cdecl calling conventions (caller removes args) and x86/darwin
162 * alignment requirements. The simplest way to arrange this,
163 * actually, is to open a new stack allocation.
164 * WARNING!!! THIS DOES NOT PRESERVE REGISTERS! */
call_c_function_in_context(x86_thread_state64_t * context,void * function,int nargs,...)165 void call_c_function_in_context(x86_thread_state64_t *context,
166 void *function,
167 int nargs,
168 ...)
169 {
170 va_list ap;
171 int i;
172 u64 *stack_pointer;
173
174 /* Set up to restore stack on exit. */
175 open_stack_allocation(context);
176
177 /* Have to keep stack 16byte aligned on x86/darwin. */
178 for (i = (1 & -nargs); i; i--) {
179 push_context(0, context);
180 }
181
182 context->rsp = ((u64)context->rsp) - nargs * 8;
183 stack_pointer = (u64 *)context->rsp;
184
185 va_start(ap, nargs);
186 if (nargs > 0) context->rdi = va_arg(ap, u64);
187 if (nargs > 1) context->rsi = va_arg(ap, u64);
188 if (nargs > 2) context->rdx = va_arg(ap, u64);
189 if (nargs > 3) context->rcx = va_arg(ap, u64);
190 if (nargs > 4) context->r8 = va_arg(ap, u64);
191 if (nargs > 5) context->r9 = va_arg(ap, u64);
192 for (i = 6; i < nargs; i++) {
193 stack_pointer[i] = va_arg(ap, u64);
194 }
195 va_end(ap);
196
197 push_context(context->rip, context);
198 context->rip = (u64) function;
199 }
200
signal_emulation_wrapper(x86_thread_state64_t * thread_state,x86_float_state64_t * float_state,int signal,siginfo_t * siginfo,void (* handler)(int,siginfo_t *,void *))201 void signal_emulation_wrapper(x86_thread_state64_t *thread_state,
202 x86_float_state64_t *float_state,
203 int signal,
204 siginfo_t *siginfo,
205 void (*handler)(int, siginfo_t *, void *))
206 {
207
208 darwin_ucontext context;
209 darwin_mcontext regs;
210
211 context.uc_mcontext = ®s;
212
213 /* when BSD signals are fired, they mask they signals in sa_mask
214 which always seem to be the blockable_sigset, for us, so we
215 need to:
216 1) save the current sigmask
217 2) block blockable signals
218 3) call the signal handler
219 4) restore the sigmask */
220
221 build_fake_signal_context(&context, thread_state, float_state);
222
223 block_blockable_signals(0);
224
225 handler(signal, siginfo, &context);
226
227 update_thread_state_from_context(thread_state, float_state, &context);
228
229 /* Trap to restore the signal context. */
230 asm volatile (".quad 0xffffffffffff0b0f"
231 : : "a" (thread_state), "b" (float_state));
232 }
233
234 #if defined DUMP_CONTEXT
dump_context(x86_thread_state64_t * context)235 void dump_context(x86_thread_state64_t *context)
236 {
237 int i;
238 u64 *stack_pointer;
239
240 printf("rax: %08lx rcx: %08lx rdx: %08lx rbx: %08lx\n",
241 context->rax, context->rcx, context->rdx, context->rbx);
242 printf("rsp: %08lx rbp: %08lx rsi: %08lx rdi: %08lx\n",
243 context->rsp, context->rbp, context->rsi, context->rdi);
244 printf("rip: %08lx eflags: %08lx\n",
245 context->rip, context->rflags);
246 printf("cs: %04hx ds: %04hx es: %04hx "
247 "ss: %04hx fs: %04hx gs: %04hx\n",
248 context->cs, context->ds, context->rs,
249 context->ss, context->fs, context->gs);
250
251 stack_pointer = (u64 *)context->rsp;
252 for (i = 0; i < 48; i+=4) {
253 printf("%08x: %08x %08x %08x %08x\n",
254 context->rsp + (i * 4),
255 stack_pointer[i],
256 stack_pointer[i+1],
257 stack_pointer[i+2],
258 stack_pointer[i+3]);
259 }
260 }
261 #endif
262
263 void
control_stack_exhausted_handler(int signal,siginfo_t * siginfo,os_context_t * context)264 control_stack_exhausted_handler(int signal, siginfo_t *siginfo,
265 os_context_t *context) {
266 extern void unblock_signals_in_context_and_maybe_warn(os_context_t*);
267 unblock_signals_in_context_and_maybe_warn(context);
268 arrange_return_to_lisp_function
269 (context, StaticSymbolFunction(CONTROL_STACK_EXHAUSTED_ERROR));
270 }
271
272 void
undefined_alien_handler(int signal,siginfo_t * siginfo,os_context_t * context)273 undefined_alien_handler(int signal, siginfo_t *siginfo, os_context_t *context) {
274 arrange_return_to_lisp_function
275 (context, StaticSymbolFunction(UNDEFINED_ALIEN_VARIABLE_ERROR));
276 }
277
278 kern_return_t
catch_exception_raise(mach_port_t exception_port,mach_port_t thread,mach_port_t task,exception_type_t exception,exception_data_t code_vector,mach_msg_type_number_t code_count)279 catch_exception_raise(mach_port_t exception_port,
280 mach_port_t thread,
281 mach_port_t task,
282 exception_type_t exception,
283 exception_data_t code_vector,
284 mach_msg_type_number_t code_count)
285 {
286 kern_return_t ret, dealloc_ret;
287 int signal;
288 siginfo_t* siginfo;
289
290 #ifdef LISP_FEATURE_SB_THREAD
291 thread_mutex_lock(&mach_exception_lock);
292 #endif
293
294 x86_thread_state64_t thread_state;
295 mach_msg_type_number_t thread_state_count = x86_THREAD_STATE64_COUNT;
296
297 x86_float_state64_t float_state;
298 mach_msg_type_number_t float_state_count = x86_FLOAT_STATE64_COUNT;
299
300 x86_exception_state64_t exception_state;
301 mach_msg_type_number_t exception_state_count = x86_EXCEPTION_STATE64_COUNT;
302
303 x86_thread_state64_t backup_thread_state;
304 x86_thread_state64_t *target_thread_state;
305 x86_float_state64_t *target_float_state;
306
307 os_vm_address_t addr;
308
309 struct thread *th;
310
311 FSHOW((stderr,"/entering catch_exception_raise with exception: %d\n", exception));
312 if (mach_port_get_context(mach_task_self(), exception_port, (mach_vm_address_t *)&th)
313 != KERN_SUCCESS) {
314 lose("Can't find the thread for an exception %p", exception_port);
315 }
316
317 switch (exception) {
318
319 case EXC_BAD_ACCESS:
320 signal = SIGBUS;
321 ret = thread_get_state(thread,
322 x86_THREAD_STATE64,
323 (thread_state_t)&thread_state,
324 &thread_state_count);
325 ret = thread_get_state(thread,
326 x86_FLOAT_STATE64,
327 (thread_state_t)&float_state,
328 &float_state_count);
329 ret = thread_get_state(thread,
330 x86_EXCEPTION_STATE64,
331 (thread_state_t)&exception_state,
332 &exception_state_count);
333
334 if (code_count && code_vector[0] == EXC_I386_GPFLT) {
335 /* This can happen for addresses larger than 48 bits,
336 resulting in bogus faultvaddr. */
337 addr = NULL;
338 } else {
339 addr = (void*)exception_state.faultvaddr;
340 }
341
342 /* note the os_context hackery here. When the signal handler returns,
343 * it won't go back to what it was doing ... */
344 if(addr >= CONTROL_STACK_GUARD_PAGE(th) &&
345 addr < CONTROL_STACK_GUARD_PAGE(th) + os_vm_page_size) {
346 /* We hit the end of the control stack: disable guard page
347 * protection so the error handler has some headroom, protect the
348 * previous page so that we can catch returns from the guard page
349 * and restore it. */
350 lower_thread_control_stack_guard_page(th);
351
352 backup_thread_state = thread_state;
353 open_stack_allocation(&thread_state);
354 /* Reserve a 256 byte zone for signal handlers
355 * to use on the interrupted thread stack.
356 */
357 stack_allocate(&thread_state, 256);
358
359 /* Save thread state */
360 target_thread_state =
361 stack_allocate(&thread_state, sizeof(*target_thread_state));
362 (*target_thread_state) = backup_thread_state;
363
364 /* Save float state */
365 target_float_state =
366 stack_allocate(&thread_state, sizeof(*target_float_state));
367 (*target_float_state) = float_state;
368
369 /* Set up siginfo */
370 siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
371 /* what do we need to put in our fake siginfo? It looks like
372 * the x86 code only uses si_signo and si_adrr. */
373 siginfo->si_signo = signal;
374 siginfo->si_addr = addr;
375
376 call_c_function_in_context(&thread_state,
377 signal_emulation_wrapper,
378 5,
379 target_thread_state,
380 target_float_state,
381 signal,
382 siginfo,
383 control_stack_exhausted_handler);
384 }
385 else if(addr >= CONTROL_STACK_RETURN_GUARD_PAGE(th) &&
386 addr < CONTROL_STACK_RETURN_GUARD_PAGE(th) + os_vm_page_size) {
387 /* We're returning from the guard page: reprotect it, and
388 * unprotect this one. This works even if we somehow missed
389 * the return-guard-page, and hit it on our way to new
390 * exhaustion instead. */
391 reset_thread_control_stack_guard_page(th);
392 }
393 else if (addr >= undefined_alien_address &&
394 addr < undefined_alien_address + os_vm_page_size) {
395 backup_thread_state = thread_state;
396 open_stack_allocation(&thread_state);
397 stack_allocate(&thread_state, 256);
398
399 /* Save thread state */
400 target_thread_state =
401 stack_allocate(&thread_state, sizeof(*target_thread_state));
402 (*target_thread_state) = backup_thread_state;
403
404 target_float_state =
405 stack_allocate(&thread_state, sizeof(*target_float_state));
406 (*target_float_state) = float_state;
407
408 /* Set up siginfo */
409 siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
410 /* what do we need to put in our fake siginfo? It looks like
411 * the x86 code only uses si_signo and si_adrr. */
412 siginfo->si_signo = signal;
413 siginfo->si_addr = addr;
414
415 call_c_function_in_context(&thread_state,
416 signal_emulation_wrapper,
417 5,
418 target_thread_state,
419 target_float_state,
420 signal,
421 siginfo,
422 undefined_alien_handler);
423 } else {
424
425 backup_thread_state = thread_state;
426 open_stack_allocation(&thread_state);
427 stack_allocate(&thread_state, 256);
428
429 /* Save thread state */
430 target_thread_state =
431 stack_allocate(&thread_state, sizeof(*target_thread_state));
432 (*target_thread_state) = backup_thread_state;
433
434 target_float_state =
435 stack_allocate(&thread_state, sizeof(*target_float_state));
436 (*target_float_state) = float_state;
437
438 /* Set up siginfo */
439 siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
440 /* what do we need to put in our fake siginfo? It looks like
441 * the x86 code only uses si_signo and si_adrr. */
442 siginfo->si_signo = signal;
443 siginfo->si_addr = addr;
444
445 call_c_function_in_context(&thread_state,
446 signal_emulation_wrapper,
447 5,
448 target_thread_state,
449 target_float_state,
450 signal,
451 siginfo,
452 memory_fault_handler);
453 }
454 ret = thread_set_state(thread,
455 x86_THREAD_STATE64,
456 (thread_state_t)&thread_state,
457 thread_state_count);
458
459 ret = thread_set_state(thread,
460 x86_FLOAT_STATE64,
461 (thread_state_t)&float_state,
462 float_state_count);
463 #ifdef LISP_FEATURE_SB_THREAD
464 thread_mutex_unlock(&mach_exception_lock);
465 #endif
466 ret = KERN_SUCCESS;
467 break;
468
469 case EXC_BAD_INSTRUCTION:
470
471 ret = thread_get_state(thread,
472 x86_THREAD_STATE64,
473 (thread_state_t)&thread_state,
474 &thread_state_count);
475 ret = thread_get_state(thread,
476 x86_FLOAT_STATE64,
477 (thread_state_t)&float_state,
478 &float_state_count);
479 ret = thread_get_state(thread,
480 x86_EXCEPTION_STATE64,
481 (thread_state_t)&exception_state,
482 &exception_state_count);
483 if (0xffffffffffff0b0f == *((u64 *)thread_state.rip)) {
484 /* fake sigreturn. */
485
486 /* When we get here, thread_state.rax is a pointer to a
487 * thread_state to restore. */
488 /* thread_state = *((thread_state_t *)thread_state.rax); */
489
490 ret = thread_set_state(thread,
491 x86_THREAD_STATE64,
492 (thread_state_t) thread_state.rax,
493 /* &thread_state, */
494 thread_state_count);
495
496 ret = thread_set_state(thread,
497 x86_FLOAT_STATE64,
498 (thread_state_t) thread_state.rbx,
499 /* &thread_state, */
500 float_state_count);
501 } else {
502
503 backup_thread_state = thread_state;
504 open_stack_allocation(&thread_state);
505 stack_allocate(&thread_state, 256);
506
507 /* Save thread state */
508 target_thread_state =
509 stack_allocate(&thread_state, sizeof(*target_thread_state));
510 (*target_thread_state) = backup_thread_state;
511
512 target_float_state =
513 stack_allocate(&thread_state, sizeof(*target_float_state));
514 (*target_float_state) = float_state;
515
516 /* Set up siginfo */
517 siginfo = stack_allocate(&thread_state, sizeof(*siginfo));
518 /* what do we need to put in our fake siginfo? It looks like
519 * the x86 code only uses si_signo and si_adrr. */
520 if (*((unsigned short *)target_thread_state->rip) == 0x0b0f) {
521 signal = SIGTRAP;
522 siginfo->si_signo = signal;
523 siginfo->si_addr = (void*)exception_state.faultvaddr;
524 target_thread_state->rip += 2;
525 call_c_function_in_context(&thread_state,
526 signal_emulation_wrapper,
527 5,
528 target_thread_state,
529 target_float_state,
530 signal,
531 siginfo,
532 sigtrap_handler);
533 } else {
534 signal = SIGILL;
535 siginfo->si_signo = signal;
536 siginfo->si_addr = (void*)exception_state.faultvaddr;
537
538 call_c_function_in_context(&thread_state,
539 signal_emulation_wrapper,
540 5,
541 target_thread_state,
542 target_float_state,
543 signal,
544 siginfo,
545 sigill_handler);
546 }
547 ret = thread_set_state(thread,
548 x86_THREAD_STATE64,
549 (thread_state_t)&thread_state,
550 thread_state_count);
551 ret = thread_set_state(thread,
552 x86_FLOAT_STATE64,
553 (thread_state_t)&float_state,
554 float_state_count);
555 }
556 #ifdef LISP_FEATURE_SB_THREAD
557 thread_mutex_unlock(&mach_exception_lock);
558 #endif
559 ret = KERN_SUCCESS;
560 break;
561
562 default:
563 #ifdef LISP_FEATURE_SB_THREAD
564 thread_mutex_unlock(&mach_exception_lock);
565 #endif
566 ret = KERN_INVALID_RIGHT;
567 }
568
569 dealloc_ret = mach_port_deallocate (mach_task_self(), thread);
570 if (dealloc_ret) {
571 lose("mach_port_deallocate (thread) failed with return_code %d\n", dealloc_ret);
572 }
573
574 dealloc_ret = mach_port_deallocate (mach_task_self(), task);
575 if (dealloc_ret) {
576 lose("mach_port_deallocate (task) failed with return_code %d\n", dealloc_ret);
577 }
578
579 return ret;
580 }
581
582 void
os_restore_fp_control(os_context_t * context)583 os_restore_fp_control(os_context_t *context)
584 {
585 /* KLUDGE: The x87 FPU control word is some nasty bitfield struct
586 * thing. Rather than deal with that, just grab it as a 16-bit
587 * integer. */
588 unsigned short fpu_control_word =
589 *((unsigned short *)&context->uc_mcontext->fs.fpu_fcw);
590 /* reset exception flags and restore control flags on SSE2 FPU */
591 unsigned int temp = (context->uc_mcontext->fs.fpu_mxcsr) & ~0x3F;
592 asm ("ldmxcsr %0" : : "m" (temp));
593 /* same for x87 FPU. */
594 asm ("fldcw %0" : : "m" (fpu_control_word));
595 }
596
597 #endif
598