1 /*
2 ** FFI C call handling.
3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #include "lj_obj.h"
7
8 #if LJ_HASFFI
9
10 #include "lj_gc.h"
11 #include "lj_err.h"
12 #include "lj_str.h"
13 #include "lj_tab.h"
14 #include "lj_ctype.h"
15 #include "lj_cconv.h"
16 #include "lj_cdata.h"
17 #include "lj_ccall.h"
18 #include "lj_trace.h"
19
20 /* Target-specific handling of register arguments. */
21 #if LJ_TARGET_X86
22 /* -- x86 calling conventions --------------------------------------------- */
23
24 #if LJ_ABI_WIN
25
26 #define CCALL_HANDLE_STRUCTRET \
27 /* Return structs bigger than 8 by reference (on stack only). */ \
28 cc->retref = (sz > 8); \
29 if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
30
31 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
32
33 #else
34
35 #if LJ_TARGET_OSX
36
37 #define CCALL_HANDLE_STRUCTRET \
38 /* Return structs of size 1, 2, 4 or 8 in registers. */ \
39 cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
40 if (cc->retref) { \
41 if (ngpr < maxgpr) \
42 cc->gpr[ngpr++] = (GPRArg)dp; \
43 else \
44 cc->stack[nsp++] = (GPRArg)dp; \
45 } else { /* Struct with single FP field ends up in FPR. */ \
46 cc->resx87 = ccall_classify_struct(cts, ctr); \
47 }
48
49 #define CCALL_HANDLE_STRUCTRET2 \
50 if (cc->resx87) sp = (uint8_t *)&cc->fpr[0]; \
51 memcpy(dp, sp, ctr->size);
52
53 #else
54
55 #define CCALL_HANDLE_STRUCTRET \
56 cc->retref = 1; /* Return all structs by reference (in reg or on stack). */ \
57 if (ngpr < maxgpr) \
58 cc->gpr[ngpr++] = (GPRArg)dp; \
59 else \
60 cc->stack[nsp++] = (GPRArg)dp;
61
62 #endif
63
64 #define CCALL_HANDLE_COMPLEXRET \
65 /* Return complex float in GPRs and complex double by reference. */ \
66 cc->retref = (sz > 8); \
67 if (cc->retref) { \
68 if (ngpr < maxgpr) \
69 cc->gpr[ngpr++] = (GPRArg)dp; \
70 else \
71 cc->stack[nsp++] = (GPRArg)dp; \
72 }
73
74 #endif
75
76 #define CCALL_HANDLE_COMPLEXRET2 \
77 if (!cc->retref) \
78 *(int64_t *)dp = *(int64_t *)sp; /* Copy complex float from GPRs. */
79
80 #define CCALL_HANDLE_STRUCTARG \
81 ngpr = maxgpr; /* Pass all structs by value on the stack. */
82
83 #define CCALL_HANDLE_COMPLEXARG \
84 isfp = 1; /* Pass complex by value on stack. */
85
86 #define CCALL_HANDLE_REGARG \
87 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
88 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
89 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
90 } else if (ngpr + 1 <= maxgpr) { \
91 dp = &cc->gpr[ngpr]; \
92 ngpr += n; \
93 goto done; \
94 } \
95 }
96
97 #elif LJ_TARGET_X64 && LJ_ABI_WIN
98 /* -- Windows/x64 calling conventions ------------------------------------- */
99
100 #define CCALL_HANDLE_STRUCTRET \
101 /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
102 cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
103 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
104
105 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
106
107 #define CCALL_HANDLE_COMPLEXRET2 \
108 if (!cc->retref) \
109 *(int64_t *)dp = *(int64_t *)sp; /* Copy complex float from GPRs. */
110
111 #define CCALL_HANDLE_STRUCTARG \
112 /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
113 if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
114 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
115 sz = CTSIZE_PTR; /* Pass all other structs by reference. */ \
116 }
117
118 #define CCALL_HANDLE_COMPLEXARG \
119 /* Pass complex float in a GPR and complex double by reference. */ \
120 if (sz != 2*sizeof(float)) { \
121 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
122 sz = CTSIZE_PTR; \
123 }
124
125 /* Windows/x64 argument registers are strictly positional (use ngpr). */
126 #define CCALL_HANDLE_REGARG \
127 if (isfp) { \
128 if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
129 } else { \
130 if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
131 }
132
133 #elif LJ_TARGET_X64
134 /* -- POSIX/x64 calling conventions --------------------------------------- */
135
136 #define CCALL_HANDLE_STRUCTRET \
137 int rcl[2]; rcl[0] = rcl[1] = 0; \
138 if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
139 cc->retref = 1; /* Return struct by reference. */ \
140 cc->gpr[ngpr++] = (GPRArg)dp; \
141 } else { \
142 cc->retref = 0; /* Return small structs in registers. */ \
143 }
144
145 #define CCALL_HANDLE_STRUCTRET2 \
146 int rcl[2]; rcl[0] = rcl[1] = 0; \
147 ccall_classify_struct(cts, ctr, rcl, 0); \
148 ccall_struct_ret(cc, rcl, dp, ctr->size);
149
150 #define CCALL_HANDLE_COMPLEXRET \
151 /* Complex values are returned in one or two FPRs. */ \
152 cc->retref = 0;
153
154 #define CCALL_HANDLE_COMPLEXRET2 \
155 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPR. */ \
156 *(int64_t *)dp = cc->fpr[0].l[0]; \
157 } else { /* Copy non-contiguous complex double from FPRs. */ \
158 ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
159 ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
160 }
161
162 #define CCALL_HANDLE_STRUCTARG \
163 int rcl[2]; rcl[0] = rcl[1] = 0; \
164 if (!ccall_classify_struct(cts, d, rcl, 0)) { \
165 cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
166 if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
167 nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
168 continue; \
169 } /* Pass all other structs by value on stack. */
170
171 #define CCALL_HANDLE_COMPLEXARG \
172 isfp = 2; /* Pass complex in FPRs or on stack. Needs postprocessing. */
173
174 #define CCALL_HANDLE_REGARG \
175 if (isfp) { /* Try to pass argument in FPRs. */ \
176 int n2 = ctype_isvector(d->info) ? 1 : n; \
177 if (nfpr + n2 <= CCALL_NARG_FPR) { \
178 dp = &cc->fpr[nfpr]; \
179 nfpr += n2; \
180 goto done; \
181 } \
182 } else { /* Try to pass argument in GPRs. */ \
183 /* Note that reordering is explicitly allowed in the x64 ABI. */ \
184 if (n <= 2 && ngpr + n <= maxgpr) { \
185 dp = &cc->gpr[ngpr]; \
186 ngpr += n; \
187 goto done; \
188 } \
189 }
190
191 #elif LJ_TARGET_ARM
192 /* -- ARM calling conventions --------------------------------------------- */
193
194 #if LJ_ABI_SOFTFP
195
196 #define CCALL_HANDLE_STRUCTRET \
197 /* Return structs of size <= 4 in a GPR. */ \
198 cc->retref = !(sz <= 4); \
199 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
200
201 #define CCALL_HANDLE_COMPLEXRET \
202 cc->retref = 1; /* Return all complex values by reference. */ \
203 cc->gpr[ngpr++] = (GPRArg)dp;
204
205 #define CCALL_HANDLE_COMPLEXRET2 \
206 UNUSED(dp); /* Nothing to do. */
207
208 #define CCALL_HANDLE_STRUCTARG \
209 /* Pass all structs by value in registers and/or on the stack. */
210
211 #define CCALL_HANDLE_COMPLEXARG \
212 /* Pass complex by value in 2 or 4 GPRs. */
213
214 #define CCALL_HANDLE_REGARG_FP1
215 #define CCALL_HANDLE_REGARG_FP2
216
217 #else
218
219 #define CCALL_HANDLE_STRUCTRET \
220 cc->retref = !ccall_classify_struct(cts, ctr, ct); \
221 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
222
223 #define CCALL_HANDLE_STRUCTRET2 \
224 if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
225 memcpy(dp, sp, ctr->size);
226
227 #define CCALL_HANDLE_COMPLEXRET \
228 if (!(ct->info & CTF_VARARG)) cc->retref = 0; /* Return complex in FPRs. */
229
230 #define CCALL_HANDLE_COMPLEXRET2 \
231 if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
232
233 #define CCALL_HANDLE_STRUCTARG \
234 isfp = (ccall_classify_struct(cts, d, ct) > 1);
235 /* Pass all structs by value in registers and/or on the stack. */
236
237 #define CCALL_HANDLE_COMPLEXARG \
238 isfp = 1; /* Pass complex by value in FPRs or on stack. */
239
240 #define CCALL_HANDLE_REGARG_FP1 \
241 if (isfp && !(ct->info & CTF_VARARG)) { \
242 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
243 if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
244 dp = &cc->fpr[nfpr]; \
245 nfpr += (n >> 1); \
246 goto done; \
247 } \
248 } else { \
249 if (sz > 1 && fprodd != nfpr) fprodd = 0; \
250 if (fprodd) { \
251 if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
252 dp = (void *)&cc->fpr[fprodd-1].f[1]; \
253 nfpr += (n >> 1); \
254 if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
255 goto done; \
256 } \
257 } else { \
258 if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
259 dp = (void *)&cc->fpr[nfpr]; \
260 nfpr += (n >> 1); \
261 if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
262 goto done; \
263 } \
264 } \
265 } \
266 fprodd = 0; /* No reordering after the first FP value is on stack. */ \
267 } else {
268
269 #define CCALL_HANDLE_REGARG_FP2 }
270
271 #endif
272
273 #define CCALL_HANDLE_REGARG \
274 CCALL_HANDLE_REGARG_FP1 \
275 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
276 if (ngpr < maxgpr) \
277 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
278 } \
279 if (ngpr < maxgpr) { \
280 dp = &cc->gpr[ngpr]; \
281 if (ngpr + n > maxgpr) { \
282 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
283 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
284 ngpr = maxgpr; \
285 } else { \
286 ngpr += n; \
287 } \
288 goto done; \
289 } CCALL_HANDLE_REGARG_FP2
290
291 #define CCALL_HANDLE_RET \
292 if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
293
294 #elif LJ_TARGET_PPC
295 /* -- PPC calling conventions --------------------------------------------- */
296
297 #define CCALL_HANDLE_STRUCTRET \
298 cc->retref = 1; /* Return all structs by reference. */ \
299 cc->gpr[ngpr++] = (GPRArg)dp;
300
301 #define CCALL_HANDLE_COMPLEXRET \
302 /* Complex values are returned in 2 or 4 GPRs. */ \
303 cc->retref = 0;
304
305 #define CCALL_HANDLE_COMPLEXRET2 \
306 memcpy(dp, sp, ctr->size); /* Copy complex from GPRs. */
307
308 #define CCALL_HANDLE_STRUCTARG \
309 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
310 sz = CTSIZE_PTR; /* Pass all structs by reference. */
311
312 #define CCALL_HANDLE_COMPLEXARG \
313 /* Pass complex by value in 2 or 4 GPRs. */
314
315 #define CCALL_HANDLE_REGARG \
316 if (isfp) { /* Try to pass argument in FPRs. */ \
317 if (nfpr + 1 <= CCALL_NARG_FPR) { \
318 dp = &cc->fpr[nfpr]; \
319 nfpr += 1; \
320 d = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
321 goto done; \
322 } \
323 } else { /* Try to pass argument in GPRs. */ \
324 if (n > 1) { \
325 lua_assert(n == 2 || n == 4); /* int64_t or complex (float). */ \
326 if (ctype_isinteger(d->info)) \
327 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
328 else if (ngpr + n > maxgpr) \
329 ngpr = maxgpr; /* Prevent reordering. */ \
330 } \
331 if (ngpr + n <= maxgpr) { \
332 dp = &cc->gpr[ngpr]; \
333 ngpr += n; \
334 goto done; \
335 } \
336 }
337
338 #define CCALL_HANDLE_RET \
339 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
340 ctr = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */
341
342 #elif LJ_TARGET_PPCSPE
343 /* -- PPC/SPE calling conventions ----------------------------------------- */
344
345 #define CCALL_HANDLE_STRUCTRET \
346 cc->retref = 1; /* Return all structs by reference. */ \
347 cc->gpr[ngpr++] = (GPRArg)dp;
348
349 #define CCALL_HANDLE_COMPLEXRET \
350 /* Complex values are returned in 2 or 4 GPRs. */ \
351 cc->retref = 0;
352
353 #define CCALL_HANDLE_COMPLEXRET2 \
354 memcpy(dp, sp, ctr->size); /* Copy complex from GPRs. */
355
356 #define CCALL_HANDLE_STRUCTARG \
357 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
358 sz = CTSIZE_PTR; /* Pass all structs by reference. */
359
360 #define CCALL_HANDLE_COMPLEXARG \
361 /* Pass complex by value in 2 or 4 GPRs. */
362
363 /* PPC/SPE has a softfp ABI. */
364 #define CCALL_HANDLE_REGARG \
365 if (n > 1) { /* Doesn't fit in a single GPR? */ \
366 lua_assert(n == 2 || n == 4); /* int64_t, double or complex (float). */ \
367 if (n == 2) \
368 ngpr = (ngpr + 1u) & ~1u; /* Only align 64 bit value to regpair. */ \
369 else if (ngpr + n > maxgpr) \
370 ngpr = maxgpr; /* Prevent reordering. */ \
371 } \
372 if (ngpr + n <= maxgpr) { \
373 dp = &cc->gpr[ngpr]; \
374 ngpr += n; \
375 goto done; \
376 }
377
378 #elif LJ_TARGET_MIPS
379 /* -- MIPS calling conventions -------------------------------------------- */
380
381 #define CCALL_HANDLE_STRUCTRET \
382 cc->retref = 1; /* Return all structs by reference. */ \
383 cc->gpr[ngpr++] = (GPRArg)dp;
384
385 #define CCALL_HANDLE_COMPLEXRET \
386 /* Complex values are returned in 1 or 2 FPRs. */ \
387 cc->retref = 0;
388
389 #define CCALL_HANDLE_COMPLEXRET2 \
390 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
391 ((float *)dp)[0] = cc->fpr[0].f; \
392 ((float *)dp)[1] = cc->fpr[1].f; \
393 } else { /* Copy complex double from FPRs. */ \
394 ((double *)dp)[0] = cc->fpr[0].d; \
395 ((double *)dp)[1] = cc->fpr[1].d; \
396 }
397
398 #define CCALL_HANDLE_STRUCTARG \
399 /* Pass all structs by value in registers and/or on the stack. */
400
401 #define CCALL_HANDLE_COMPLEXARG \
402 /* Pass complex by value in 2 or 4 GPRs. */
403
404 #define CCALL_HANDLE_REGARG \
405 if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
406 /* Try to pass argument in FPRs. */ \
407 dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
408 nfpr++; ngpr += n; \
409 goto done; \
410 } else { /* Try to pass argument in GPRs. */ \
411 nfpr = CCALL_NARG_FPR; \
412 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
413 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
414 if (ngpr < maxgpr) { \
415 dp = &cc->gpr[ngpr]; \
416 if (ngpr + n > maxgpr) { \
417 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
418 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
419 ngpr = maxgpr; \
420 } else { \
421 ngpr += n; \
422 } \
423 goto done; \
424 } \
425 }
426
427 #define CCALL_HANDLE_RET \
428 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
429 sp = (uint8_t *)&cc->fpr[0].f;
430
431 #else
432 #error "Missing calling convention definitions for this architecture"
433 #endif
434
435 #ifndef CCALL_HANDLE_STRUCTRET2
436 #define CCALL_HANDLE_STRUCTRET2 \
437 memcpy(dp, sp, ctr->size); /* Copy struct return value from GPRs. */
438 #endif
439
440 /* -- x86 OSX ABI struct classification ----------------------------------- */
441
442 #if LJ_TARGET_X86 && LJ_TARGET_OSX
443
444 /* Check for struct with single FP field. */
ccall_classify_struct(CTState * cts,CType * ct)445 static int ccall_classify_struct(CTState *cts, CType *ct)
446 {
447 CTSize sz = ct->size;
448 if (!(sz == sizeof(float) || sz == sizeof(double))) return 0;
449 if ((ct->info & CTF_UNION)) return 0;
450 while (ct->sib) {
451 ct = ctype_get(cts, ct->sib);
452 if (ctype_isfield(ct->info)) {
453 CType *sct = ctype_rawchild(cts, ct);
454 if (ctype_isfp(sct->info)) {
455 if (sct->size == sz)
456 return (sz >> 2); /* Return 1 for float or 2 for double. */
457 } else if (ctype_isstruct(sct->info)) {
458 if (sct->size)
459 return ccall_classify_struct(cts, sct);
460 } else {
461 break;
462 }
463 } else if (ctype_isbitfield(ct->info)) {
464 break;
465 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
466 CType *sct = ctype_rawchild(cts, ct);
467 if (sct->size)
468 return ccall_classify_struct(cts, sct);
469 }
470 }
471 return 0;
472 }
473
474 #endif
475
476 /* -- x64 struct classification ------------------------------------------- */
477
478 #if LJ_TARGET_X64 && !LJ_ABI_WIN
479
480 /* Register classes for x64 struct classification. */
481 #define CCALL_RCL_INT 1
482 #define CCALL_RCL_SSE 2
483 #define CCALL_RCL_MEM 4
484 /* NYI: classify vectors. */
485
486 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
487
488 /* Classify a C type. */
ccall_classify_ct(CTState * cts,CType * ct,int * rcl,CTSize ofs)489 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
490 {
491 if (ctype_isarray(ct->info)) {
492 CType *cct = ctype_rawchild(cts, ct);
493 CTSize eofs, esz = cct->size, asz = ct->size;
494 for (eofs = 0; eofs < asz; eofs += esz)
495 ccall_classify_ct(cts, cct, rcl, ofs+eofs);
496 } else if (ctype_isstruct(ct->info)) {
497 ccall_classify_struct(cts, ct, rcl, ofs);
498 } else {
499 int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
500 lua_assert(ctype_hassize(ct->info));
501 if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM; /* Unaligned. */
502 rcl[(ofs >= 8)] |= cl;
503 }
504 }
505
506 /* Recursively classify a struct based on its fields. */
ccall_classify_struct(CTState * cts,CType * ct,int * rcl,CTSize ofs)507 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
508 {
509 if (ct->size > 16) return CCALL_RCL_MEM; /* Too big, gets memory class. */
510 while (ct->sib) {
511 CTSize fofs;
512 ct = ctype_get(cts, ct->sib);
513 fofs = ofs+ct->size;
514 if (ctype_isfield(ct->info))
515 ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
516 else if (ctype_isbitfield(ct->info))
517 rcl[(fofs >= 8)] |= CCALL_RCL_INT; /* NYI: unaligned bitfields? */
518 else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
519 ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
520 }
521 return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM); /* Memory class? */
522 }
523
524 /* Try to split up a small struct into registers. */
ccall_struct_reg(CCallState * cc,GPRArg * dp,int * rcl)525 static int ccall_struct_reg(CCallState *cc, GPRArg *dp, int *rcl)
526 {
527 MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
528 uint32_t i;
529 for (i = 0; i < 2; i++) {
530 lua_assert(!(rcl[i] & CCALL_RCL_MEM));
531 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
532 if (ngpr >= CCALL_NARG_GPR) return 1; /* Register overflow. */
533 cc->gpr[ngpr++] = dp[i];
534 } else if ((rcl[i] & CCALL_RCL_SSE)) {
535 if (nfpr >= CCALL_NARG_FPR) return 1; /* Register overflow. */
536 cc->fpr[nfpr++].l[0] = dp[i];
537 }
538 }
539 cc->ngpr = ngpr; cc->nfpr = nfpr;
540 return 0; /* Ok. */
541 }
542
543 /* Pass a small struct argument. */
ccall_struct_arg(CCallState * cc,CTState * cts,CType * d,int * rcl,TValue * o,int narg)544 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
545 TValue *o, int narg)
546 {
547 GPRArg dp[2];
548 dp[0] = dp[1] = 0;
549 /* Convert to temp. struct. */
550 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
551 if (ccall_struct_reg(cc, dp, rcl)) { /* Register overflow? Pass on stack. */
552 MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
553 if (nsp + n > CCALL_MAXSTACK) return 1; /* Too many arguments. */
554 cc->nsp = nsp + n;
555 memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
556 }
557 return 0; /* Ok. */
558 }
559
560 /* Combine returned small struct. */
ccall_struct_ret(CCallState * cc,int * rcl,uint8_t * dp,CTSize sz)561 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
562 {
563 GPRArg sp[2];
564 MSize ngpr = 0, nfpr = 0;
565 uint32_t i;
566 for (i = 0; i < 2; i++) {
567 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
568 sp[i] = cc->gpr[ngpr++];
569 } else if ((rcl[i] & CCALL_RCL_SSE)) {
570 sp[i] = cc->fpr[nfpr++].l[0];
571 }
572 }
573 memcpy(dp, sp, sz);
574 }
575 #endif
576
577 /* -- ARM hard-float ABI struct classification ---------------------------- */
578
579 #if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
580
581 /* Classify a struct based on its fields. */
ccall_classify_struct(CTState * cts,CType * ct,CType * ctf)582 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
583 {
584 CTSize sz = ct->size;
585 unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
586 if ((ctf->info & CTF_VARARG)) goto noth;
587 while (ct->sib) {
588 CType *sct;
589 ct = ctype_get(cts, ct->sib);
590 if (ctype_isfield(ct->info)) {
591 sct = ctype_rawchild(cts, ct);
592 if (ctype_isfp(sct->info)) {
593 r |= sct->size;
594 if (!isu) n++; else if (n == 0) n = 1;
595 } else if (ctype_iscomplex(sct->info)) {
596 r |= (sct->size >> 1);
597 if (!isu) n += 2; else if (n < 2) n = 2;
598 } else if (ctype_isstruct(sct->info)) {
599 goto substruct;
600 } else {
601 goto noth;
602 }
603 } else if (ctype_isbitfield(ct->info)) {
604 goto noth;
605 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
606 sct = ctype_rawchild(cts, ct);
607 substruct:
608 if (sct->size > 0) {
609 unsigned int s = ccall_classify_struct(cts, sct, ctf);
610 if (s <= 1) goto noth;
611 r |= (s & 255);
612 if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
613 }
614 }
615 }
616 if ((r == 4 || r == 8) && n <= 4)
617 return r + (n << 8);
618 noth: /* Not a homogeneous float/double aggregate. */
619 return (sz <= 4); /* Return structs of size <= 4 in a GPR. */
620 }
621
622 #endif
623
624 /* -- Common C call handling ---------------------------------------------- */
625
626 /* Infer the destination CTypeID for a vararg argument. */
lj_ccall_ctid_vararg(CTState * cts,cTValue * o)627 CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
628 {
629 if (tvisnumber(o)) {
630 return CTID_DOUBLE;
631 } else if (tviscdata(o)) {
632 CTypeID id = cdataV(o)->ctypeid;
633 CType *s = ctype_get(cts, id);
634 if (ctype_isrefarray(s->info)) {
635 return lj_ctype_intern(cts,
636 CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
637 } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
638 /* NYI: how to pass a struct by value in a vararg argument? */
639 return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
640 } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
641 return CTID_DOUBLE;
642 } else {
643 return id;
644 }
645 } else if (tvisstr(o)) {
646 return CTID_P_CCHAR;
647 } else if (tvisbool(o)) {
648 return CTID_BOOL;
649 } else {
650 return CTID_P_VOID;
651 }
652 }
653
654 /* Setup arguments for C call. */
ccall_set_args(lua_State * L,CTState * cts,CType * ct,CCallState * cc)655 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
656 CCallState *cc)
657 {
658 int gcsteps = 0;
659 TValue *o, *top = L->top;
660 CTypeID fid;
661 CType *ctr;
662 MSize maxgpr, ngpr = 0, nsp = 0, narg;
663 #if CCALL_NARG_FPR
664 MSize nfpr = 0;
665 #if LJ_TARGET_ARM
666 MSize fprodd = 0;
667 #endif
668 #endif
669
670 /* Clear unused regs to get some determinism in case of misdeclaration. */
671 memset(cc->gpr, 0, sizeof(cc->gpr));
672 #if CCALL_NUM_FPR
673 memset(cc->fpr, 0, sizeof(cc->fpr));
674 #endif
675
676 #if LJ_TARGET_X86
677 /* x86 has several different calling conventions. */
678 cc->resx87 = 0;
679 switch (ctype_cconv(ct->info)) {
680 case CTCC_FASTCALL: maxgpr = 2; break;
681 case CTCC_THISCALL: maxgpr = 1; break;
682 default: maxgpr = 0; break;
683 }
684 #else
685 maxgpr = CCALL_NARG_GPR;
686 #endif
687
688 /* Perform required setup for some result types. */
689 ctr = ctype_rawchild(cts, ct);
690 if (ctype_isvector(ctr->info)) {
691 if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
692 goto err_nyi;
693 } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
694 /* Preallocate cdata object and anchor it after arguments. */
695 CTSize sz = ctr->size;
696 GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
697 void *dp = cdataptr(cd);
698 setcdataV(L, L->top++, cd);
699 if (ctype_isstruct(ctr->info)) {
700 CCALL_HANDLE_STRUCTRET
701 } else {
702 CCALL_HANDLE_COMPLEXRET
703 }
704 #if LJ_TARGET_X86
705 } else if (ctype_isfp(ctr->info)) {
706 cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
707 #endif
708 }
709
710 /* Skip initial attributes. */
711 fid = ct->sib;
712 while (fid) {
713 CType *ctf = ctype_get(cts, fid);
714 if (!ctype_isattrib(ctf->info)) break;
715 fid = ctf->sib;
716 }
717
718 /* Walk through all passed arguments. */
719 for (o = L->base+1, narg = 1; o < top; o++, narg++) {
720 CTypeID did;
721 CType *d;
722 CTSize sz;
723 MSize n, isfp = 0, isva = 0;
724 void *dp, *rp = NULL;
725
726 if (fid) { /* Get argument type from field. */
727 CType *ctf = ctype_get(cts, fid);
728 fid = ctf->sib;
729 lua_assert(ctype_isfield(ctf->info));
730 did = ctype_cid(ctf->info);
731 } else {
732 if (!(ct->info & CTF_VARARG))
733 lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too many arguments. */
734 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
735 isva = 1;
736 }
737 d = ctype_raw(cts, did);
738 sz = d->size;
739
740 /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
741 if (ctype_isnum(d->info)) {
742 if (sz > 8) goto err_nyi;
743 if ((d->info & CTF_FP))
744 isfp = 1;
745 } else if (ctype_isvector(d->info)) {
746 if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
747 isfp = 1;
748 else
749 goto err_nyi;
750 } else if (ctype_isstruct(d->info)) {
751 CCALL_HANDLE_STRUCTARG
752 } else if (ctype_iscomplex(d->info)) {
753 CCALL_HANDLE_COMPLEXARG
754 } else {
755 sz = CTSIZE_PTR;
756 }
757 sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
758 n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */
759
760 CCALL_HANDLE_REGARG /* Handle register arguments. */
761
762 /* Otherwise pass argument on stack. */
763 if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
764 MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
765 nsp = (nsp + align) & ~align; /* Align argument on stack. */
766 }
767 if (nsp + n > CCALL_MAXSTACK) { /* Too many arguments. */
768 err_nyi:
769 lj_err_caller(L, LJ_ERR_FFI_NYICALL);
770 }
771 dp = &cc->stack[nsp];
772 nsp += n;
773 isva = 0;
774
775 done:
776 if (rp) { /* Pass by reference. */
777 gcsteps++;
778 *(void **)dp = rp;
779 dp = rp;
780 }
781 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
782 /* Extend passed integers to 32 bits at least. */
783 if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
784 if (d->info & CTF_UNSIGNED)
785 *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
786 (uint32_t)*(uint16_t *)dp;
787 else
788 *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
789 (int32_t)*(int16_t *)dp;
790 }
791 #if LJ_TARGET_X64 && LJ_ABI_WIN
792 if (isva) { /* Windows/x64 mirrors varargs in both register sets. */
793 if (nfpr == ngpr)
794 cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
795 else
796 cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
797 }
798 #else
799 UNUSED(isva);
800 #endif
801 #if LJ_TARGET_X64 && !LJ_ABI_WIN
802 if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
803 cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1]; /* Split complex double. */
804 cc->fpr[nfpr-2].d[1] = 0;
805 }
806 #else
807 UNUSED(isfp);
808 #endif
809 }
810 if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too few arguments. */
811
812 #if LJ_TARGET_X64 || LJ_TARGET_PPC
813 cc->nfpr = nfpr; /* Required for vararg functions. */
814 #endif
815 cc->nsp = nsp;
816 cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
817 if (nsp > CCALL_SPS_FREE)
818 cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
819 return gcsteps;
820 }
821
822 /* Get results from C call. */
ccall_get_results(lua_State * L,CTState * cts,CType * ct,CCallState * cc,int * ret)823 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
824 CCallState *cc, int *ret)
825 {
826 CType *ctr = ctype_rawchild(cts, ct);
827 uint8_t *sp = (uint8_t *)&cc->gpr[0];
828 if (ctype_isvoid(ctr->info)) {
829 *ret = 0; /* Zero results. */
830 return 0; /* No additional GC step. */
831 }
832 *ret = 1; /* One result. */
833 if (ctype_isstruct(ctr->info)) {
834 /* Return cdata object which is already on top of stack. */
835 if (!cc->retref) {
836 void *dp = cdataptr(cdataV(L->top-1)); /* Use preallocated object. */
837 CCALL_HANDLE_STRUCTRET2
838 }
839 return 1; /* One GC step. */
840 }
841 if (ctype_iscomplex(ctr->info)) {
842 /* Return cdata object which is already on top of stack. */
843 void *dp = cdataptr(cdataV(L->top-1)); /* Use preallocated object. */
844 CCALL_HANDLE_COMPLEXRET2
845 return 1; /* One GC step. */
846 }
847 if (LJ_BE && ctype_isinteger_or_bool(ctr->info) && ctr->size < CTSIZE_PTR)
848 sp += (CTSIZE_PTR - ctr->size);
849 #if CCALL_NUM_FPR
850 if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
851 sp = (uint8_t *)&cc->fpr[0];
852 #endif
853 #ifdef CCALL_HANDLE_RET
854 CCALL_HANDLE_RET
855 #endif
856 /* No reference types end up here, so there's no need for the CTypeID. */
857 lua_assert(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)));
858 return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
859 }
860
861 /* Call C function. */
lj_ccall_func(lua_State * L,GCcdata * cd)862 int lj_ccall_func(lua_State *L, GCcdata *cd)
863 {
864 CTState *cts = ctype_cts(L);
865 CType *ct = ctype_raw(cts, cd->ctypeid);
866 CTSize sz = CTSIZE_PTR;
867 if (ctype_isptr(ct->info)) {
868 sz = ct->size;
869 ct = ctype_rawchild(cts, ct);
870 }
871 if (ctype_isfunc(ct->info)) {
872 CCallState cc;
873 int gcsteps, ret;
874 cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
875 gcsteps = ccall_set_args(L, cts, ct, &cc);
876 ct = (CType *)((intptr_t)ct-(intptr_t)cts->tab);
877 cts->cb.slot = ~0u;
878 lj_vm_ffi_call(&cc);
879 if (cts->cb.slot != ~0u) { /* Blacklist function that called a callback. */
880 TValue tv;
881 setlightudV(&tv, (void *)cc.func);
882 setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
883 }
884 ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab); /* May be reallocated. */
885 gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
886 #if LJ_TARGET_X86 && LJ_ABI_WIN
887 /* Automatically detect __stdcall and fix up C function declaration. */
888 if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
889 CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
890 lj_trace_abort(G(L));
891 }
892 #endif
893 while (gcsteps-- > 0)
894 lj_gc_check(L);
895 return ret;
896 }
897 return -1; /* Not a function. */
898 }
899
900 #endif
901