1 /*
2 ** FFI C callback handling.
3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
4 */
5 
6 #include "lj_obj.h"
7 
8 #if LJ_HASFFI
9 
10 #include "lj_gc.h"
11 #include "lj_err.h"
12 #include "lj_tab.h"
13 #include "lj_state.h"
14 #include "lj_frame.h"
15 #include "lj_ctype.h"
16 #include "lj_cconv.h"
17 #include "lj_ccall.h"
18 #include "lj_ccallback.h"
19 #include "lj_target.h"
20 #include "lj_mcode.h"
21 #include "lj_trace.h"
22 #include "lj_vm.h"
23 
24 /* -- Target-specific handling of callback slots -------------------------- */
25 
26 #define CALLBACK_MCODE_SIZE	(LJ_PAGESIZE * LJ_NUM_CBPAGE)
27 
28 #if LJ_OS_NOJIT
29 
30 /* Callbacks disabled. */
31 #define CALLBACK_SLOT2OFS(slot)	(0*(slot))
32 #define CALLBACK_OFS2SLOT(ofs)	(0*(ofs))
33 #define CALLBACK_MAX_SLOT	0
34 
35 #elif LJ_TARGET_X86ORX64
36 
37 #define CALLBACK_MCODE_HEAD	(LJ_64 ? 8 : 0)
38 #define CALLBACK_MCODE_GROUP	(-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
39 
40 #define CALLBACK_SLOT2OFS(slot) \
41   (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
42 
CALLBACK_OFS2SLOT(MSize ofs)43 static MSize CALLBACK_OFS2SLOT(MSize ofs)
44 {
45   MSize group;
46   ofs -= CALLBACK_MCODE_HEAD;
47   group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
48   return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
49 }
50 
51 #define CALLBACK_MAX_SLOT \
52   (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
53 
54 #elif LJ_TARGET_ARM
55 
56 #define CALLBACK_MCODE_HEAD		32
57 
58 #elif LJ_TARGET_ARM64
59 
60 #define CALLBACK_MCODE_HEAD		32
61 
62 #elif LJ_TARGET_PPC
63 
64 #define CALLBACK_MCODE_HEAD		24
65 
66 #elif LJ_TARGET_MIPS32
67 
68 #define CALLBACK_MCODE_HEAD		20
69 
70 #elif LJ_TARGET_MIPS64
71 
72 #define CALLBACK_MCODE_HEAD		52
73 
74 #else
75 
76 /* Missing support for this architecture. */
77 #define CALLBACK_SLOT2OFS(slot)	(0*(slot))
78 #define CALLBACK_OFS2SLOT(ofs)	(0*(ofs))
79 #define CALLBACK_MAX_SLOT	0
80 
81 #endif
82 
83 #ifndef CALLBACK_SLOT2OFS
84 #define CALLBACK_SLOT2OFS(slot)		(CALLBACK_MCODE_HEAD + 8*(slot))
85 #define CALLBACK_OFS2SLOT(ofs)		(((ofs)-CALLBACK_MCODE_HEAD)/8)
86 #define CALLBACK_MAX_SLOT		(CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
87 #endif
88 
89 /* Convert callback slot number to callback function pointer. */
callback_slot2ptr(CTState * cts,MSize slot)90 static void *callback_slot2ptr(CTState *cts, MSize slot)
91 {
92   return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
93 }
94 
95 /* Convert callback function pointer to slot number. */
lj_ccallback_ptr2slot(CTState * cts,void * p)96 MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
97 {
98   uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
99   if (ofs < CALLBACK_MCODE_SIZE) {
100     MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
101     if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
102       return slot;
103   }
104   return ~0u;  /* Not a known callback function pointer. */
105 }
106 
107 /* Initialize machine code for callback function pointers. */
108 #if LJ_OS_NOJIT
109 /* Disabled callback support. */
110 #define callback_mcode_init(g, p)	UNUSED(p)
111 #elif LJ_TARGET_X86ORX64
callback_mcode_init(global_State * g,uint8_t * page)112 static void callback_mcode_init(global_State *g, uint8_t *page)
113 {
114   uint8_t *p = page;
115   uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
116   MSize slot;
117 #if LJ_64
118   *(void **)p = target; p += 8;
119 #endif
120   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
121     /* mov al, slot; jmp group */
122     *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
123     if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
124       /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
125       *p++ = XI_PUSH + RID_EBP;
126       *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
127 #if LJ_GC64
128       *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
129       *(uint64_t *)p = (uint64_t)(g); p += 8;
130 #else
131       *p++ = XI_MOVri | RID_EBP;
132       *(int32_t *)p = i32ptr(g); p += 4;
133 #endif
134 #if LJ_64
135       /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
136       *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
137       *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
138 #else
139       /* jmp lj_vm_ffi_callback. */
140       *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
141 #endif
142     } else {
143       *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
144     }
145   }
146   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
147 }
148 #elif LJ_TARGET_ARM
callback_mcode_init(global_State * g,uint32_t * page)149 static void callback_mcode_init(global_State *g, uint32_t *page)
150 {
151   uint32_t *p = page;
152   void *target = (void *)lj_vm_ffi_callback;
153   MSize slot;
154   /* This must match with the saveregs macro in buildvm_arm.dasc. */
155   *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
156   *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
157   *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
158   *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
159   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
160   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
161   *p++ = u32ptr(g);
162   *p++ = u32ptr(target);
163   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
164     *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
165     *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
166     p++;
167   }
168   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
169 }
170 #elif LJ_TARGET_ARM64
callback_mcode_init(global_State * g,uint32_t * page)171 static void callback_mcode_init(global_State *g, uint32_t *page)
172 {
173   uint32_t *p = page;
174   void *target = (void *)lj_vm_ffi_callback;
175   MSize slot;
176   *p++ = A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4);
177   *p++ = A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5);
178   *p++ = A64I_BR | A64F_N(RID_X11);
179   *p++ = A64I_NOP;
180   ((void **)p)[0] = target;
181   ((void **)p)[1] = g;
182   p += 4;
183   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
184     *p++ = A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot);
185     *p = A64I_B | A64F_S26((page-p) & 0x03ffffffu);
186     p++;
187   }
188   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
189 }
190 #elif LJ_TARGET_PPC
callback_mcode_init(global_State * g,uint32_t * page)191 static void callback_mcode_init(global_State *g, uint32_t *page)
192 {
193   uint32_t *p = page;
194   void *target = (void *)lj_vm_ffi_callback;
195   MSize slot;
196   *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
197   *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
198   *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
199   *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
200   *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
201   *p++ = PPCI_BCTR;
202   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
203     *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
204     *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
205     p++;
206   }
207   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
208 }
209 #elif LJ_TARGET_MIPS
callback_mcode_init(global_State * g,uint32_t * page)210 static void callback_mcode_init(global_State *g, uint32_t *page)
211 {
212   uint32_t *p = page;
213   uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
214   uintptr_t ug = (uintptr_t)(void *)g;
215   MSize slot;
216 #if LJ_TARGET_MIPS32
217   *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16);
218   *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16);
219 #else
220   *p++ = MIPSI_LUI  | MIPSF_T(RID_R3) | (target >> 48);
221   *p++ = MIPSI_LUI  | MIPSF_T(RID_R2) | (ug >> 48);
222   *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff);
223   *p++ = MIPSI_ORI  | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff);
224   *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
225   *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
226   *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff);
227   *p++ = MIPSI_ORI  | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff);
228   *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
229   *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
230 #endif
231   *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff);
232   *p++ = MIPSI_JR | MIPSF_S(RID_R3);
233   *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff);
234   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
235     *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
236     p++;
237     *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
238   }
239   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
240 }
241 #else
242 /* Missing support for this architecture. */
243 #define callback_mcode_init(g, p)	UNUSED(p)
244 #endif
245 
246 /* -- Machine code management --------------------------------------------- */
247 
248 #if LJ_TARGET_WINDOWS
249 
250 #define WIN32_LEAN_AND_MEAN
251 #include <windows.h>
252 
253 #elif LJ_TARGET_POSIX
254 
255 #include <sys/mman.h>
256 #ifndef MAP_ANONYMOUS
257 #define MAP_ANONYMOUS   MAP_ANON
258 #endif
259 
260 #endif
261 
262 /* Allocate and initialize area for callback function pointers. */
callback_mcode_new(CTState * cts)263 static void callback_mcode_new(CTState *cts)
264 {
265   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
266   void *p;
267   if (CALLBACK_MAX_SLOT == 0)
268     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
269 #if LJ_TARGET_WINDOWS
270   p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
271   if (!p)
272     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
273 #elif LJ_TARGET_POSIX
274   p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS,
275 	   -1, 0);
276   if (p == MAP_FAILED)
277     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
278 #else
279   /* Fallback allocator. Fails if memory is not executable by default. */
280   p = lj_mem_new(cts->L, sz);
281 #endif
282   cts->cb.mcode = p;
283   callback_mcode_init(cts->g, p);
284   lj_mcode_sync(p, (char *)p + sz);
285 #if LJ_TARGET_WINDOWS
286   {
287     DWORD oprot;
288     VirtualProtect(p, sz, PAGE_EXECUTE_READ, &oprot);
289   }
290 #elif LJ_TARGET_POSIX
291   mprotect(p, sz, (PROT_READ|PROT_EXEC));
292 #endif
293 }
294 
295 /* Free area for callback function pointers. */
lj_ccallback_mcode_free(CTState * cts)296 void lj_ccallback_mcode_free(CTState *cts)
297 {
298   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
299   void *p = cts->cb.mcode;
300   if (p == NULL) return;
301 #if LJ_TARGET_WINDOWS
302   VirtualFree(p, 0, MEM_RELEASE);
303   UNUSED(sz);
304 #elif LJ_TARGET_POSIX
305   munmap(p, sz);
306 #else
307   lj_mem_free(cts->g, p, sz);
308 #endif
309 }
310 
311 /* -- C callback entry ---------------------------------------------------- */
312 
313 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
314 #if LJ_TARGET_X86
315 
316 #define CALLBACK_HANDLE_REGARG \
317   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
318     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
319       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
320     } else if (ngpr + 1 <= maxgpr) { \
321       sp = &cts->cb.gpr[ngpr]; \
322       ngpr += n; \
323       goto done; \
324     } \
325   }
326 
327 #elif LJ_TARGET_X64 && LJ_ABI_WIN
328 
329 /* Windows/x64 argument registers are strictly positional (use ngpr). */
330 #define CALLBACK_HANDLE_REGARG \
331   if (isfp) { \
332     if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
333   } else { \
334     if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
335   }
336 
337 #elif LJ_TARGET_X64
338 
339 #define CALLBACK_HANDLE_REGARG \
340   if (isfp) { \
341     if (nfpr + n <= CCALL_NARG_FPR) { \
342       sp = &cts->cb.fpr[nfpr]; \
343       nfpr += n; \
344       goto done; \
345     } \
346   } else { \
347     if (ngpr + n <= maxgpr) { \
348       sp = &cts->cb.gpr[ngpr]; \
349       ngpr += n; \
350       goto done; \
351     } \
352   }
353 
354 #elif LJ_TARGET_ARM
355 
356 #if LJ_ABI_SOFTFP
357 
358 #define CALLBACK_HANDLE_REGARG_FP1	UNUSED(isfp);
359 #define CALLBACK_HANDLE_REGARG_FP2
360 
361 #else
362 
363 #define CALLBACK_HANDLE_REGARG_FP1 \
364   if (isfp) { \
365     if (n == 1) { \
366       if (fprodd) { \
367 	sp = &cts->cb.fpr[fprodd-1]; \
368 	fprodd = 0; \
369 	goto done; \
370       } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
371 	sp = &cts->cb.fpr[nfpr++]; \
372 	fprodd = nfpr; \
373 	goto done; \
374       } \
375     } else { \
376       if (nfpr + 1 <= CCALL_NARG_FPR) { \
377 	sp = &cts->cb.fpr[nfpr++]; \
378 	goto done; \
379       } \
380     } \
381     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
382   } else {
383 
384 #define CALLBACK_HANDLE_REGARG_FP2	}
385 
386 #endif
387 
388 #define CALLBACK_HANDLE_REGARG \
389   CALLBACK_HANDLE_REGARG_FP1 \
390   if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
391   if (ngpr + n <= maxgpr) { \
392     sp = &cts->cb.gpr[ngpr]; \
393     ngpr += n; \
394     goto done; \
395   } CALLBACK_HANDLE_REGARG_FP2
396 
397 #elif LJ_TARGET_ARM64
398 
399 #define CALLBACK_HANDLE_REGARG \
400   if (isfp) { \
401     if (nfpr + n <= CCALL_NARG_FPR) { \
402       sp = &cts->cb.fpr[nfpr]; \
403       nfpr += n; \
404       goto done; \
405     } else { \
406       nfpr = CCALL_NARG_FPR;  /* Prevent reordering. */ \
407     } \
408   } else { \
409     if (!LJ_TARGET_IOS && n > 1) \
410       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
411     if (ngpr + n <= maxgpr) { \
412       sp = &cts->cb.gpr[ngpr]; \
413       ngpr += n; \
414       goto done; \
415     } else { \
416       ngpr = CCALL_NARG_GPR;  /* Prevent reordering. */ \
417     } \
418   }
419 
420 #elif LJ_TARGET_PPC
421 
422 #define CALLBACK_HANDLE_REGARG \
423   if (isfp) { \
424     if (nfpr + 1 <= CCALL_NARG_FPR) { \
425       sp = &cts->cb.fpr[nfpr++]; \
426       cta = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
427       goto done; \
428     } \
429   } else {  /* Try to pass argument in GPRs. */ \
430     if (n > 1) { \
431       lua_assert(ctype_isinteger(cta->info) && n == 2);  /* int64_t. */ \
432       ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
433     } \
434     if (ngpr + n <= maxgpr) { \
435       sp = &cts->cb.gpr[ngpr]; \
436       ngpr += n; \
437       goto done; \
438     } \
439   }
440 
441 #define CALLBACK_HANDLE_RET \
442   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
443     *(double *)dp = *(float *)dp;  /* FPRs always hold doubles. */
444 
445 #elif LJ_TARGET_MIPS32
446 
447 #define CALLBACK_HANDLE_GPR \
448   if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
449   if (ngpr + n <= maxgpr) { \
450     sp = &cts->cb.gpr[ngpr]; \
451     ngpr += n; \
452     goto done; \
453   }
454 
455 #if !LJ_ABI_SOFTFP	/* MIPS32 hard-float */
456 #define CALLBACK_HANDLE_REGARG \
457   if (isfp && nfpr < CCALL_NARG_FPR) {  /* Try to pass argument in FPRs. */ \
458     sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
459     nfpr++; ngpr += n; \
460     goto done; \
461   } else {  /* Try to pass argument in GPRs. */ \
462     nfpr = CCALL_NARG_FPR; \
463     CALLBACK_HANDLE_GPR \
464   }
465 #else			/* MIPS32 soft-float */
466 #define CALLBACK_HANDLE_REGARG \
467   CALLBACK_HANDLE_GPR \
468   UNUSED(isfp);
469 #endif
470 
471 #define CALLBACK_HANDLE_RET \
472   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
473     ((float *)dp)[1] = *(float *)dp;
474 
475 #elif LJ_TARGET_MIPS64
476 
477 #if !LJ_ABI_SOFTFP	/* MIPS64 hard-float */
478 #define CALLBACK_HANDLE_REGARG \
479   if (ngpr + n <= maxgpr) { \
480     sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
481     ngpr += n; \
482     goto done; \
483   }
484 #else			/* MIPS64 soft-float */
485 #define CALLBACK_HANDLE_REGARG \
486   if (ngpr + n <= maxgpr) { \
487     UNUSED(isfp); \
488     sp = (void*) &cts->cb.gpr[ngpr]; \
489     ngpr += n; \
490     goto done; \
491   }
492 #endif
493 
494 #define CALLBACK_HANDLE_RET \
495   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
496     ((float *)dp)[1] = *(float *)dp;
497 
498 #else
499 #error "Missing calling convention definitions for this architecture"
500 #endif
501 
502 /* Convert and push callback arguments to Lua stack. */
callback_conv_args(CTState * cts,lua_State * L)503 static void callback_conv_args(CTState *cts, lua_State *L)
504 {
505   TValue *o = L->top;
506   intptr_t *stack = cts->cb.stack;
507   MSize slot = cts->cb.slot;
508   CTypeID id = 0, rid, fid;
509   int gcsteps = 0;
510   CType *ct;
511   GCfunc *fn;
512   int fntp;
513   MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
514 #if CCALL_NARG_FPR
515   MSize nfpr = 0;
516 #if LJ_TARGET_ARM
517   MSize fprodd = 0;
518 #endif
519 #endif
520 
521   if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
522     ct = ctype_get(cts, id);
523     rid = ctype_cid(ct->info);  /* Return type. x86: +(spadj<<16). */
524     fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
525     fntp = LJ_TFUNC;
526   } else {  /* Must set up frame first, before throwing the error. */
527     ct = NULL;
528     rid = 0;
529     fn = (GCfunc *)L;
530     fntp = LJ_TTHREAD;
531   }
532   /* Continuation returns from callback. */
533   if (LJ_FR2) {
534     (o++)->u64 = LJ_CONT_FFI_CALLBACK;
535     (o++)->u64 = rid;
536     o++;
537   } else {
538     o->u32.lo = LJ_CONT_FFI_CALLBACK;
539     o->u32.hi = rid;
540     o++;
541   }
542   setframe_gc(o, obj2gco(fn), fntp);
543   setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
544   L->top = L->base = ++o;
545   if (!ct)
546     lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
547   if (isluafunc(fn))
548     setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
549   lj_state_checkstack(L, LUA_MINSTACK);  /* May throw. */
550   o = L->base;  /* Might have been reallocated. */
551 
552 #if LJ_TARGET_X86
553   /* x86 has several different calling conventions. */
554   switch (ctype_cconv(ct->info)) {
555   case CTCC_FASTCALL: maxgpr = 2; break;
556   case CTCC_THISCALL: maxgpr = 1; break;
557   default: maxgpr = 0; break;
558   }
559 #endif
560 
561   fid = ct->sib;
562   while (fid) {
563     CType *ctf = ctype_get(cts, fid);
564     if (!ctype_isattrib(ctf->info)) {
565       CType *cta;
566       void *sp;
567       CTSize sz;
568       int isfp;
569       MSize n;
570       lua_assert(ctype_isfield(ctf->info));
571       cta = ctype_rawchild(cts, ctf);
572       isfp = ctype_isfp(cta->info);
573       sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
574       n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
575 
576       CALLBACK_HANDLE_REGARG  /* Handle register arguments. */
577 
578       /* Otherwise pass argument on stack. */
579       if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
580 	nsp = (nsp + 1) & ~1u;  /* Align 64 bit argument on stack. */
581       sp = &stack[nsp];
582       nsp += n;
583 
584     done:
585       if (LJ_BE && cta->size < CTSIZE_PTR
586 #if LJ_TARGET_MIPS64
587 	  && !(isfp && nsp)
588 #endif
589 	 )
590 	sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
591       gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
592     }
593     fid = ctf->sib;
594   }
595   L->top = o;
596 #if LJ_TARGET_X86
597   /* Store stack adjustment for returns from non-cdecl callbacks. */
598   if (ctype_cconv(ct->info) != CTCC_CDECL) {
599 #if LJ_FR2
600     (L->base-3)->u64 |= (nsp << (16+2));
601 #else
602     (L->base-2)->u32.hi |= (nsp << (16+2));
603 #endif
604   }
605 #endif
606   while (gcsteps-- > 0)
607     lj_gc_check(L);
608 }
609 
610 /* Convert Lua object to callback result. */
callback_conv_result(CTState * cts,lua_State * L,TValue * o)611 static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
612 {
613 #if LJ_FR2
614   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
615 #else
616   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
617 #endif
618 #if LJ_TARGET_X86
619   cts->cb.gpr[2] = 0;
620 #endif
621   if (!ctype_isvoid(ctr->info)) {
622     uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
623 #if CCALL_NUM_FPR
624     if (ctype_isfp(ctr->info))
625       dp = (uint8_t *)&cts->cb.fpr[0];
626 #endif
627     lj_cconv_ct_tv(cts, ctr, dp, o, 0);
628 #ifdef CALLBACK_HANDLE_RET
629     CALLBACK_HANDLE_RET
630 #endif
631     /* Extend returned integers to (at least) 32 bits. */
632     if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
633       if (ctr->info & CTF_UNSIGNED)
634 	*(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
635 					   (uint32_t)*(uint16_t *)dp;
636       else
637 	*(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
638 					  (int32_t)*(int16_t *)dp;
639     }
640 #if LJ_TARGET_MIPS64
641     /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
642     if (ctr->size <= 4 &&
643 	(LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
644       *(int64_t *)dp = (int64_t)*(int32_t *)dp;
645 #endif
646 #if LJ_TARGET_X86
647     if (ctype_isfp(ctr->info))
648       cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
649 #endif
650   }
651 }
652 
653 /* Enter callback. */
lj_ccallback_enter(CTState * cts,void * cf)654 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
655 {
656   lua_State *L = cts->L;
657   global_State *g = cts->g;
658   lua_assert(L != NULL);
659   if (tvref(g->jit_base)) {
660     setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
661     if (g->panic) g->panic(L);
662     exit(EXIT_FAILURE);
663   }
664   lj_trace_abort(g);  /* Never record across callback. */
665   /* Setup C frame. */
666   cframe_prev(cf) = L->cframe;
667   setcframe_L(cf, L);
668   cframe_errfunc(cf) = -1;
669   cframe_nres(cf) = 0;
670   L->cframe = cf;
671   callback_conv_args(cts, L);
672   return L;  /* Now call the function on this stack. */
673 }
674 
675 /* Leave callback. */
lj_ccallback_leave(CTState * cts,TValue * o)676 void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
677 {
678   lua_State *L = cts->L;
679   GCfunc *fn;
680   TValue *obase = L->base;
681   L->base = L->top;  /* Keep continuation frame for throwing errors. */
682   if (o >= L->base) {
683     /* PC of RET* is lost. Point to last line for result conv. errors. */
684     fn = curr_func(L);
685     if (isluafunc(fn)) {
686       GCproto *pt = funcproto(fn);
687       setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
688     }
689   }
690   callback_conv_result(cts, L, o);
691   /* Finally drop C frame and continuation frame. */
692   L->top -= 2+2*LJ_FR2;
693   L->base = obase;
694   L->cframe = cframe_prev(L->cframe);
695   cts->cb.slot = 0;  /* Blacklist C function that called the callback. */
696 }
697 
698 /* -- C callback management ----------------------------------------------- */
699 
700 /* Get an unused slot in the callback slot table. */
callback_slot_new(CTState * cts,CType * ct)701 static MSize callback_slot_new(CTState *cts, CType *ct)
702 {
703   CTypeID id = ctype_typeid(cts, ct);
704   CTypeID1 *cbid = cts->cb.cbid;
705   MSize top;
706   for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
707     if (LJ_LIKELY(cbid[top] == 0))
708       goto found;
709 #if CALLBACK_MAX_SLOT
710   if (top >= CALLBACK_MAX_SLOT)
711 #endif
712     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
713   if (!cts->cb.mcode)
714     callback_mcode_new(cts);
715   lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
716   cts->cb.cbid = cbid;
717   memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
718 found:
719   cbid[top] = id;
720   cts->cb.topid = top+1;
721   return top;
722 }
723 
724 /* Check for function pointer and supported argument/result types. */
callback_checkfunc(CTState * cts,CType * ct)725 static CType *callback_checkfunc(CTState *cts, CType *ct)
726 {
727   int narg = 0;
728   if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
729     return NULL;
730   ct = ctype_rawchild(cts, ct);
731   if (ctype_isfunc(ct->info)) {
732     CType *ctr = ctype_rawchild(cts, ct);
733     CTypeID fid = ct->sib;
734     if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
735 	  ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
736       return NULL;
737     if ((ct->info & CTF_VARARG))
738       return NULL;
739     while (fid) {
740       CType *ctf = ctype_get(cts, fid);
741       if (!ctype_isattrib(ctf->info)) {
742 	CType *cta;
743 	lua_assert(ctype_isfield(ctf->info));
744 	cta = ctype_rawchild(cts, ctf);
745 	if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
746 	      (ctype_isnum(cta->info) && cta->size <= 8)) ||
747 	    ++narg >= LUA_MINSTACK-3)
748 	  return NULL;
749       }
750       fid = ctf->sib;
751     }
752     return ct;
753   }
754   return NULL;
755 }
756 
757 /* Create a new callback and return the callback function pointer. */
lj_ccallback_new(CTState * cts,CType * ct,GCfunc * fn)758 void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
759 {
760   ct = callback_checkfunc(cts, ct);
761   if (ct) {
762     MSize slot = callback_slot_new(cts, ct);
763     GCtab *t = cts->miscmap;
764     setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
765     lj_gc_anybarriert(cts->L, t);
766     return callback_slot2ptr(cts, slot);
767   }
768   return NULL;  /* Bad conversion. */
769 }
770 
771 #endif
772