1 /*
2 ** FFI C call handling.
3 ** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #include "lj_obj.h"
7
8 #if LJ_HASFFI
9
10 #include "lj_gc.h"
11 #include "lj_err.h"
12 #include "lj_tab.h"
13 #include "lj_ctype.h"
14 #include "lj_cconv.h"
15 #include "lj_cdata.h"
16 #include "lj_ccall.h"
17 #include "lj_trace.h"
18
19 /* Target-specific handling of register arguments. */
20 #if LJ_TARGET_X86
21 /* -- x86 calling conventions --------------------------------------------- */
22
23 #if LJ_ABI_WIN
24
25 #define CCALL_HANDLE_STRUCTRET \
26 /* Return structs bigger than 8 by reference (on stack only). */ \
27 cc->retref = (sz > 8); \
28 if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
29
30 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
31
32 #else
33
34 #if LJ_TARGET_OSX
35
36 #define CCALL_HANDLE_STRUCTRET \
37 /* Return structs of size 1, 2, 4 or 8 in registers. */ \
38 cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
39 if (cc->retref) { \
40 if (ngpr < maxgpr) \
41 cc->gpr[ngpr++] = (GPRArg)dp; \
42 else \
43 cc->stack[nsp++] = (GPRArg)dp; \
44 } else { /* Struct with single FP field ends up in FPR. */ \
45 cc->resx87 = ccall_classify_struct(cts, ctr); \
46 }
47
48 #define CCALL_HANDLE_STRUCTRET2 \
49 if (cc->resx87) sp = (uint8_t *)&cc->fpr[0]; \
50 memcpy(dp, sp, ctr->size);
51
52 #else
53
54 #define CCALL_HANDLE_STRUCTRET \
55 cc->retref = 1; /* Return all structs by reference (in reg or on stack). */ \
56 if (ngpr < maxgpr) \
57 cc->gpr[ngpr++] = (GPRArg)dp; \
58 else \
59 cc->stack[nsp++] = (GPRArg)dp;
60
61 #endif
62
63 #define CCALL_HANDLE_COMPLEXRET \
64 /* Return complex float in GPRs and complex double by reference. */ \
65 cc->retref = (sz > 8); \
66 if (cc->retref) { \
67 if (ngpr < maxgpr) \
68 cc->gpr[ngpr++] = (GPRArg)dp; \
69 else \
70 cc->stack[nsp++] = (GPRArg)dp; \
71 }
72
73 #endif
74
75 #define CCALL_HANDLE_COMPLEXRET2 \
76 if (!cc->retref) \
77 *(int64_t *)dp = *(int64_t *)sp; /* Copy complex float from GPRs. */
78
79 #define CCALL_HANDLE_STRUCTARG \
80 ngpr = maxgpr; /* Pass all structs by value on the stack. */
81
82 #define CCALL_HANDLE_COMPLEXARG \
83 isfp = 1; /* Pass complex by value on stack. */
84
85 #define CCALL_HANDLE_REGARG \
86 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
87 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
88 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
89 } else if (ngpr + 1 <= maxgpr) { \
90 dp = &cc->gpr[ngpr]; \
91 ngpr += n; \
92 goto done; \
93 } \
94 }
95
96 #elif LJ_TARGET_X64 && LJ_ABI_WIN
97 /* -- Windows/x64 calling conventions ------------------------------------- */
98
99 #define CCALL_HANDLE_STRUCTRET \
100 /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
101 cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
102 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
103
104 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
105
106 #define CCALL_HANDLE_COMPLEXRET2 \
107 if (!cc->retref) \
108 *(int64_t *)dp = *(int64_t *)sp; /* Copy complex float from GPRs. */
109
110 #define CCALL_HANDLE_STRUCTARG \
111 /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
112 if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
113 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
114 sz = CTSIZE_PTR; /* Pass all other structs by reference. */ \
115 }
116
117 #define CCALL_HANDLE_COMPLEXARG \
118 /* Pass complex float in a GPR and complex double by reference. */ \
119 if (sz != 2*sizeof(float)) { \
120 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
121 sz = CTSIZE_PTR; \
122 }
123
124 /* Windows/x64 argument registers are strictly positional (use ngpr). */
125 #define CCALL_HANDLE_REGARG \
126 if (isfp) { \
127 if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
128 } else { \
129 if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
130 }
131
132 #elif LJ_TARGET_X64
133 /* -- POSIX/x64 calling conventions --------------------------------------- */
134
135 #define CCALL_HANDLE_STRUCTRET \
136 int rcl[2]; rcl[0] = rcl[1] = 0; \
137 if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
138 cc->retref = 1; /* Return struct by reference. */ \
139 cc->gpr[ngpr++] = (GPRArg)dp; \
140 } else { \
141 cc->retref = 0; /* Return small structs in registers. */ \
142 }
143
144 #define CCALL_HANDLE_STRUCTRET2 \
145 int rcl[2]; rcl[0] = rcl[1] = 0; \
146 ccall_classify_struct(cts, ctr, rcl, 0); \
147 ccall_struct_ret(cc, rcl, dp, ctr->size);
148
149 #define CCALL_HANDLE_COMPLEXRET \
150 /* Complex values are returned in one or two FPRs. */ \
151 cc->retref = 0;
152
153 #define CCALL_HANDLE_COMPLEXRET2 \
154 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPR. */ \
155 *(int64_t *)dp = cc->fpr[0].l[0]; \
156 } else { /* Copy non-contiguous complex double from FPRs. */ \
157 ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
158 ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
159 }
160
161 #define CCALL_HANDLE_STRUCTARG \
162 int rcl[2]; rcl[0] = rcl[1] = 0; \
163 if (!ccall_classify_struct(cts, d, rcl, 0)) { \
164 cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
165 if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
166 nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
167 continue; \
168 } /* Pass all other structs by value on stack. */
169
170 #define CCALL_HANDLE_COMPLEXARG \
171 isfp = 2; /* Pass complex in FPRs or on stack. Needs postprocessing. */
172
173 #define CCALL_HANDLE_REGARG \
174 if (isfp) { /* Try to pass argument in FPRs. */ \
175 int n2 = ctype_isvector(d->info) ? 1 : n; \
176 if (nfpr + n2 <= CCALL_NARG_FPR) { \
177 dp = &cc->fpr[nfpr]; \
178 nfpr += n2; \
179 goto done; \
180 } \
181 } else { /* Try to pass argument in GPRs. */ \
182 /* Note that reordering is explicitly allowed in the x64 ABI. */ \
183 if (n <= 2 && ngpr + n <= maxgpr) { \
184 dp = &cc->gpr[ngpr]; \
185 ngpr += n; \
186 goto done; \
187 } \
188 }
189
190 #elif LJ_TARGET_ARM
191 /* -- ARM calling conventions --------------------------------------------- */
192
193 #if LJ_ABI_SOFTFP
194
195 #define CCALL_HANDLE_STRUCTRET \
196 /* Return structs of size <= 4 in a GPR. */ \
197 cc->retref = !(sz <= 4); \
198 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
199
200 #define CCALL_HANDLE_COMPLEXRET \
201 cc->retref = 1; /* Return all complex values by reference. */ \
202 cc->gpr[ngpr++] = (GPRArg)dp;
203
204 #define CCALL_HANDLE_COMPLEXRET2 \
205 UNUSED(dp); /* Nothing to do. */
206
207 #define CCALL_HANDLE_STRUCTARG \
208 /* Pass all structs by value in registers and/or on the stack. */
209
210 #define CCALL_HANDLE_COMPLEXARG \
211 /* Pass complex by value in 2 or 4 GPRs. */
212
213 #define CCALL_HANDLE_REGARG_FP1
214 #define CCALL_HANDLE_REGARG_FP2
215
216 #else
217
218 #define CCALL_HANDLE_STRUCTRET \
219 cc->retref = !ccall_classify_struct(cts, ctr, ct); \
220 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
221
222 #define CCALL_HANDLE_STRUCTRET2 \
223 if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
224 memcpy(dp, sp, ctr->size);
225
226 #define CCALL_HANDLE_COMPLEXRET \
227 if (!(ct->info & CTF_VARARG)) cc->retref = 0; /* Return complex in FPRs. */
228
229 #define CCALL_HANDLE_COMPLEXRET2 \
230 if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
231
232 #define CCALL_HANDLE_STRUCTARG \
233 isfp = (ccall_classify_struct(cts, d, ct) > 1);
234 /* Pass all structs by value in registers and/or on the stack. */
235
236 #define CCALL_HANDLE_COMPLEXARG \
237 isfp = 1; /* Pass complex by value in FPRs or on stack. */
238
239 #define CCALL_HANDLE_REGARG_FP1 \
240 if (isfp && !(ct->info & CTF_VARARG)) { \
241 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
242 if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
243 dp = &cc->fpr[nfpr]; \
244 nfpr += (n >> 1); \
245 goto done; \
246 } \
247 } else { \
248 if (sz > 1 && fprodd != nfpr) fprodd = 0; \
249 if (fprodd) { \
250 if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
251 dp = (void *)&cc->fpr[fprodd-1].f[1]; \
252 nfpr += (n >> 1); \
253 if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
254 goto done; \
255 } \
256 } else { \
257 if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
258 dp = (void *)&cc->fpr[nfpr]; \
259 nfpr += (n >> 1); \
260 if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
261 goto done; \
262 } \
263 } \
264 } \
265 fprodd = 0; /* No reordering after the first FP value is on stack. */ \
266 } else {
267
268 #define CCALL_HANDLE_REGARG_FP2 }
269
270 #endif
271
272 #define CCALL_HANDLE_REGARG \
273 CCALL_HANDLE_REGARG_FP1 \
274 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
275 if (ngpr < maxgpr) \
276 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
277 } \
278 if (ngpr < maxgpr) { \
279 dp = &cc->gpr[ngpr]; \
280 if (ngpr + n > maxgpr) { \
281 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
282 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
283 ngpr = maxgpr; \
284 } else { \
285 ngpr += n; \
286 } \
287 goto done; \
288 } CCALL_HANDLE_REGARG_FP2
289
290 #define CCALL_HANDLE_RET \
291 if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
292
293 #elif LJ_TARGET_ARM64
294 /* -- ARM64 calling conventions ------------------------------------------- */
295
296 #define CCALL_HANDLE_STRUCTRET \
297 cc->retref = !ccall_classify_struct(cts, ctr); \
298 if (cc->retref) cc->retp = dp;
299
300 #define CCALL_HANDLE_STRUCTRET2 \
301 unsigned int cl = ccall_classify_struct(cts, ctr); \
302 if ((cl & 4)) { /* Combine float HFA from separate registers. */ \
303 CTSize i = (cl >> 8) - 1; \
304 do { ((uint32_t *)dp)[i] = cc->fpr[i].lo; } while (i--); \
305 } else { \
306 if (cl > 1) sp = (uint8_t *)&cc->fpr[0]; \
307 memcpy(dp, sp, ctr->size); \
308 }
309
310 #define CCALL_HANDLE_COMPLEXRET \
311 /* Complex values are returned in one or two FPRs. */ \
312 cc->retref = 0;
313
314 #define CCALL_HANDLE_COMPLEXRET2 \
315 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
316 ((float *)dp)[0] = cc->fpr[0].f; \
317 ((float *)dp)[1] = cc->fpr[1].f; \
318 } else { /* Copy complex double from FPRs. */ \
319 ((double *)dp)[0] = cc->fpr[0].d; \
320 ((double *)dp)[1] = cc->fpr[1].d; \
321 }
322
323 #define CCALL_HANDLE_STRUCTARG \
324 unsigned int cl = ccall_classify_struct(cts, d); \
325 if (cl == 0) { /* Pass struct by reference. */ \
326 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
327 sz = CTSIZE_PTR; \
328 } else if (cl > 1) { /* Pass struct in FPRs or on stack. */ \
329 isfp = (cl & 4) ? 2 : 1; \
330 } /* else: Pass struct in GPRs or on stack. */
331
332 #define CCALL_HANDLE_COMPLEXARG \
333 /* Pass complex by value in separate (!) FPRs or on stack. */ \
334 isfp = sz == 2*sizeof(float) ? 2 : 1;
335
336 #define CCALL_HANDLE_REGARG \
337 if (LJ_TARGET_OSX && isva) { \
338 /* IOS: All variadic arguments are on the stack. */ \
339 } else if (isfp) { /* Try to pass argument in FPRs. */ \
340 int n2 = ctype_isvector(d->info) ? 1 : \
341 isfp == 1 ? n : (d->size >> (4-isfp)); \
342 if (nfpr + n2 <= CCALL_NARG_FPR) { \
343 dp = &cc->fpr[nfpr]; \
344 nfpr += n2; \
345 goto done; \
346 } else { \
347 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
348 if (LJ_TARGET_OSX && d->size < 8) goto err_nyi; \
349 } \
350 } else { /* Try to pass argument in GPRs. */ \
351 if (!LJ_TARGET_OSX && (d->info & CTF_ALIGN) > CTALIGN_PTR) \
352 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
353 if (ngpr + n <= maxgpr) { \
354 dp = &cc->gpr[ngpr]; \
355 ngpr += n; \
356 goto done; \
357 } else { \
358 ngpr = maxgpr; /* Prevent reordering. */ \
359 if (LJ_TARGET_OSX && d->size < 8) goto err_nyi; \
360 } \
361 }
362
363 #if LJ_BE
364 #define CCALL_HANDLE_RET \
365 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
366 sp = (uint8_t *)&cc->fpr[0].f;
367 #endif
368
369
370 #elif LJ_TARGET_PPC
371 /* -- PPC calling conventions --------------------------------------------- */
372
373 #if LJ_ARCH_BITS == 64
374
375 #if LJ_ARCH_PPC_ELFV2
376
377 #define CCALL_HANDLE_STRUCTRET \
378 if (sz > 16 && ccall_classify_fp(cts, ctr) <= 0) { \
379 cc->retref = 1; /* Return by reference. */ \
380 cc->gpr[ngpr++] = (GPRArg)dp; \
381 }
382
383 #define CCALL_HANDLE_STRUCTRET2 \
384 int isfp = ccall_classify_fp(cts, ctr); \
385 int i; \
386 if (isfp == FTYPE_FLOAT) { \
387 for (i = 0; i < ctr->size / 4; i++) \
388 ((float *)dp)[i] = cc->fpr[i]; \
389 } else if (isfp == FTYPE_DOUBLE) { \
390 for (i = 0; i < ctr->size / 8; i++) \
391 ((double *)dp)[i] = cc->fpr[i]; \
392 } else { \
393 if (ctr->size < 8 && LJ_BE) { \
394 sp += 8 - ctr->size; \
395 } \
396 memcpy(dp, sp, ctr->size); \
397 }
398
399 #else
400
401 #define CCALL_HANDLE_STRUCTRET \
402 cc->retref = 1; /* Return all structs by reference. */ \
403 cc->gpr[ngpr++] = (GPRArg)dp;
404
405 #endif
406
407 #define CCALL_HANDLE_COMPLEXRET \
408 /* Complex values are returned in 2 or 4 GPRs. */ \
409 cc->retref = 0;
410
411 #define CCALL_HANDLE_STRUCTARG
412
413 #define CCALL_HANDLE_COMPLEXRET2 \
414 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
415 ((float *)dp)[0] = cc->fpr[0]; \
416 ((float *)dp)[1] = cc->fpr[1]; \
417 } else { /* Copy complex double from FPRs. */ \
418 ((double *)dp)[0] = cc->fpr[0]; \
419 ((double *)dp)[1] = cc->fpr[1]; \
420 }
421
422 #define CCALL_HANDLE_COMPLEXARG \
423 isfp = 1; \
424 if (d->size == sizeof(float) * 2) { \
425 d = ctype_get(cts, CTID_COMPLEX_DOUBLE); \
426 isf32 = 1; \
427 }
428
429 #define CCALL_HANDLE_REGARG \
430 if (isfp && d->size == sizeof(float)) { \
431 d = ctype_get(cts, CTID_DOUBLE); \
432 isf32 = 1; \
433 } \
434 if (ngpr < maxgpr) { \
435 dp = &cc->gpr[ngpr]; \
436 ngpr += n; \
437 if (ngpr > maxgpr) { \
438 nsp += ngpr - 8; \
439 ngpr = 8; \
440 if (nsp > CCALL_MAXSTACK) { \
441 goto err_nyi; \
442 } \
443 } \
444 goto done; \
445 }
446
447 #else
448
449 #define CCALL_HANDLE_STRUCTRET \
450 cc->retref = 1; /* Return all structs by reference. */ \
451 cc->gpr[ngpr++] = (GPRArg)dp;
452
453 #define CCALL_HANDLE_COMPLEXRET \
454 /* Complex values are returned in 2 or 4 GPRs. */ \
455 cc->retref = 0;
456
457 #define CCALL_HANDLE_STRUCTARG \
458 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
459 sz = CTSIZE_PTR; /* Pass all structs by reference. */
460
461 #define CCALL_HANDLE_COMPLEXRET2 \
462 memcpy(dp, sp, ctr->size); /* Copy complex from GPRs. */
463
464 #define CCALL_HANDLE_COMPLEXARG \
465 /* Pass complex by value in 2 or 4 GPRs. */
466
467 #define CCALL_HANDLE_GPR \
468 /* Try to pass argument in GPRs. */ \
469 if (n > 1) { \
470 /* int64_t or complex (float). */ \
471 lj_assertL(n == 2 || n == 4, "bad GPR size %d", n); \
472 if (ctype_isinteger(d->info) || ctype_isfp(d->info)) \
473 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
474 else if (ngpr + n > maxgpr) \
475 ngpr = maxgpr; /* Prevent reordering. */ \
476 } \
477 if (ngpr + n <= maxgpr) { \
478 dp = &cc->gpr[ngpr]; \
479 ngpr += n; \
480 goto done; \
481 } \
482
483 #if LJ_ABI_SOFTFP
484 #define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
485 #else
486 #define CCALL_HANDLE_REGARG \
487 if (isfp) { /* Try to pass argument in FPRs. */ \
488 if (nfpr + 1 <= CCALL_NARG_FPR) { \
489 dp = &cc->fpr[nfpr]; \
490 nfpr += 1; \
491 d = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
492 goto done; \
493 } \
494 } else { \
495 CCALL_HANDLE_GPR \
496 }
497 #endif
498
499 #endif
500
501 #if !LJ_ABI_SOFTFP
502 #define CCALL_HANDLE_RET \
503 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
504 ctr = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */
505 #endif
506
507 #elif LJ_TARGET_MIPS32
508 /* -- MIPS o32 calling conventions ---------------------------------------- */
509
510 #define CCALL_HANDLE_STRUCTRET \
511 cc->retref = 1; /* Return all structs by reference. */ \
512 cc->gpr[ngpr++] = (GPRArg)dp;
513
514 #define CCALL_HANDLE_COMPLEXRET \
515 /* Complex values are returned in 1 or 2 FPRs. */ \
516 cc->retref = 0;
517
518 #if LJ_ABI_SOFTFP
519 #define CCALL_HANDLE_COMPLEXRET2 \
520 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from GPRs. */ \
521 ((intptr_t *)dp)[0] = cc->gpr[0]; \
522 ((intptr_t *)dp)[1] = cc->gpr[1]; \
523 } else { /* Copy complex double from GPRs. */ \
524 ((intptr_t *)dp)[0] = cc->gpr[0]; \
525 ((intptr_t *)dp)[1] = cc->gpr[1]; \
526 ((intptr_t *)dp)[2] = cc->gpr[2]; \
527 ((intptr_t *)dp)[3] = cc->gpr[3]; \
528 }
529 #else
530 #define CCALL_HANDLE_COMPLEXRET2 \
531 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
532 ((float *)dp)[0] = cc->fpr[0].f; \
533 ((float *)dp)[1] = cc->fpr[1].f; \
534 } else { /* Copy complex double from FPRs. */ \
535 ((double *)dp)[0] = cc->fpr[0].d; \
536 ((double *)dp)[1] = cc->fpr[1].d; \
537 }
538 #endif
539
540 #define CCALL_HANDLE_STRUCTARG \
541 /* Pass all structs by value in registers and/or on the stack. */
542
543 #define CCALL_HANDLE_COMPLEXARG \
544 /* Pass complex by value in 2 or 4 GPRs. */
545
546 #define CCALL_HANDLE_GPR \
547 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
548 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
549 if (ngpr < maxgpr) { \
550 dp = &cc->gpr[ngpr]; \
551 if (ngpr + n > maxgpr) { \
552 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
553 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
554 ngpr = maxgpr; \
555 } else { \
556 ngpr += n; \
557 } \
558 goto done; \
559 }
560
561 #if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
562 #define CCALL_HANDLE_REGARG \
563 if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
564 /* Try to pass argument in FPRs. */ \
565 dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
566 nfpr++; ngpr += n; \
567 goto done; \
568 } else { /* Try to pass argument in GPRs. */ \
569 nfpr = CCALL_NARG_FPR; \
570 CCALL_HANDLE_GPR \
571 }
572 #else /* MIPS32 soft-float */
573 #define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
574 #endif
575
576 #if !LJ_ABI_SOFTFP
577 /* On MIPS64 soft-float, position of float return values is endian-dependant. */
578 #define CCALL_HANDLE_RET \
579 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
580 sp = (uint8_t *)&cc->fpr[0].f;
581 #endif
582
583 #elif LJ_TARGET_MIPS64
584 /* -- MIPS n64 calling conventions ---------------------------------------- */
585
586 #define CCALL_HANDLE_STRUCTRET \
587 cc->retref = !(sz <= 16); \
588 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
589
590 #define CCALL_HANDLE_STRUCTRET2 \
591 ccall_copy_struct(cc, ctr, dp, sp, ccall_classify_struct(cts, ctr, ct));
592
593 #define CCALL_HANDLE_COMPLEXRET \
594 /* Complex values are returned in 1 or 2 FPRs. */ \
595 cc->retref = 0;
596
597 #if LJ_ABI_SOFTFP /* MIPS64 soft-float */
598
599 #define CCALL_HANDLE_COMPLEXRET2 \
600 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from GPRs. */ \
601 ((intptr_t *)dp)[0] = cc->gpr[0]; \
602 } else { /* Copy complex double from GPRs. */ \
603 ((intptr_t *)dp)[0] = cc->gpr[0]; \
604 ((intptr_t *)dp)[1] = cc->gpr[1]; \
605 }
606
607 #define CCALL_HANDLE_COMPLEXARG \
608 /* Pass complex by value in 2 or 4 GPRs. */
609
610 /* Position of soft-float 'float' return value depends on endianess. */
611 #define CCALL_HANDLE_RET \
612 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
613 sp = (uint8_t *)cc->gpr + LJ_ENDIAN_SELECT(0, 4);
614
615 #else /* MIPS64 hard-float */
616
617 #define CCALL_HANDLE_COMPLEXRET2 \
618 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
619 ((float *)dp)[0] = cc->fpr[0].f; \
620 ((float *)dp)[1] = cc->fpr[1].f; \
621 } else { /* Copy complex double from FPRs. */ \
622 ((double *)dp)[0] = cc->fpr[0].d; \
623 ((double *)dp)[1] = cc->fpr[1].d; \
624 }
625
626 #define CCALL_HANDLE_COMPLEXARG \
627 if (sz == 2*sizeof(float)) { \
628 isfp = 2; \
629 if (ngpr < maxgpr) \
630 sz *= 2; \
631 }
632
633 #define CCALL_HANDLE_RET \
634 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
635 sp = (uint8_t *)&cc->fpr[0].f;
636
637 #endif
638
639 #define CCALL_HANDLE_STRUCTARG \
640 /* Pass all structs by value in registers and/or on the stack. */
641
642 #define CCALL_HANDLE_REGARG \
643 if (ngpr < maxgpr) { \
644 dp = &cc->gpr[ngpr]; \
645 if (ngpr + n > maxgpr) { \
646 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
647 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
648 ngpr = maxgpr; \
649 } else { \
650 ngpr += n; \
651 } \
652 goto done; \
653 }
654
655 #elif LJ_TARGET_S390X
656 /* -- POSIX/s390x calling conventions --------------------------------------- */
657
658 #define CCALL_HANDLE_STRUCTRET \
659 cc->retref = 1; /* Return all structs by reference. */ \
660 cc->gpr[ngpr++] = (GPRArg)dp;
661
662 #define CCALL_HANDLE_COMPLEXRET \
663 cc->retref = 1; /* Return all complex values by reference. */ \
664 cc->gpr[ngpr++] = (GPRArg)dp;
665
666 #define CCALL_HANDLE_COMPLEXRET2 \
667 UNUSED(dp); /* Nothing to do. */
668
669 #define CCALL_HANDLE_STRUCTARG \
670 /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
671 if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
672 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
673 sz = CTSIZE_PTR; /* Pass all other structs by reference. */ \
674 }
675
676 #define CCALL_HANDLE_COMPLEXARG \
677 /* Pass complex numbers by reference. */ \
678 /* TODO: not sure why this is different to structs. */ \
679 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
680 sz = CTSIZE_PTR; \
681
682 #define CCALL_HANDLE_REGARG \
683 if (isfp) { \
684 if (nfpr < CCALL_NARG_FPR) { dp = &cc->fpr[nfpr++]; goto done; } \
685 } else { \
686 if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
687 }
688
689 #else
690 #error "Missing calling convention definitions for this architecture"
691 #endif
692
693 #ifndef CCALL_HANDLE_STRUCTRET2
694 #define CCALL_HANDLE_STRUCTRET2 \
695 memcpy(dp, sp, ctr->size); /* Copy struct return value from GPRs. */
696 #endif
697
698 /* -- x86 OSX ABI struct classification ----------------------------------- */
699
700 #if LJ_TARGET_X86 && LJ_TARGET_OSX
701
702 /* Check for struct with single FP field. */
ccall_classify_struct(CTState * cts,CType * ct)703 static int ccall_classify_struct(CTState *cts, CType *ct)
704 {
705 CTSize sz = ct->size;
706 if (!(sz == sizeof(float) || sz == sizeof(double))) return 0;
707 if ((ct->info & CTF_UNION)) return 0;
708 while (ct->sib) {
709 ct = ctype_get(cts, ct->sib);
710 if (ctype_isfield(ct->info)) {
711 CType *sct = ctype_rawchild(cts, ct);
712 if (ctype_isfp(sct->info)) {
713 if (sct->size == sz)
714 return (sz >> 2); /* Return 1 for float or 2 for double. */
715 } else if (ctype_isstruct(sct->info)) {
716 if (sct->size)
717 return ccall_classify_struct(cts, sct);
718 } else {
719 break;
720 }
721 } else if (ctype_isbitfield(ct->info)) {
722 break;
723 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
724 CType *sct = ctype_rawchild(cts, ct);
725 if (sct->size)
726 return ccall_classify_struct(cts, sct);
727 }
728 }
729 return 0;
730 }
731
732 #endif
733
734 /* -- x64 struct classification ------------------------------------------- */
735
736 #if LJ_TARGET_X64 && !LJ_ABI_WIN
737
738 /* Register classes for x64 struct classification. */
739 #define CCALL_RCL_INT 1
740 #define CCALL_RCL_SSE 2
741 #define CCALL_RCL_MEM 4
742 /* NYI: classify vectors. */
743
744 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
745
746 /* Classify a C type. */
ccall_classify_ct(CTState * cts,CType * ct,int * rcl,CTSize ofs)747 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
748 {
749 if (ctype_isarray(ct->info)) {
750 CType *cct = ctype_rawchild(cts, ct);
751 CTSize eofs, esz = cct->size, asz = ct->size;
752 for (eofs = 0; eofs < asz; eofs += esz)
753 ccall_classify_ct(cts, cct, rcl, ofs+eofs);
754 } else if (ctype_isstruct(ct->info)) {
755 ccall_classify_struct(cts, ct, rcl, ofs);
756 } else {
757 int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
758 lj_assertCTS(ctype_hassize(ct->info),
759 "classify ctype %08x without size", ct->info);
760 if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM; /* Unaligned. */
761 rcl[(ofs >= 8)] |= cl;
762 }
763 }
764
765 /* Recursively classify a struct based on its fields. */
ccall_classify_struct(CTState * cts,CType * ct,int * rcl,CTSize ofs)766 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
767 {
768 if (ct->size > 16) return CCALL_RCL_MEM; /* Too big, gets memory class. */
769 while (ct->sib) {
770 CTSize fofs;
771 ct = ctype_get(cts, ct->sib);
772 fofs = ofs+ct->size;
773 if (ctype_isfield(ct->info))
774 ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
775 else if (ctype_isbitfield(ct->info))
776 rcl[(fofs >= 8)] |= CCALL_RCL_INT; /* NYI: unaligned bitfields? */
777 else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
778 ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
779 }
780 return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM); /* Memory class? */
781 }
782
783 /* Try to split up a small struct into registers. */
ccall_struct_reg(CCallState * cc,CTState * cts,GPRArg * dp,int * rcl)784 static int ccall_struct_reg(CCallState *cc, CTState *cts, GPRArg *dp, int *rcl)
785 {
786 MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
787 uint32_t i;
788 UNUSED(cts);
789 for (i = 0; i < 2; i++) {
790 lj_assertCTS(!(rcl[i] & CCALL_RCL_MEM), "pass mem struct in reg");
791 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
792 if (ngpr >= CCALL_NARG_GPR) return 1; /* Register overflow. */
793 cc->gpr[ngpr++] = dp[i];
794 } else if ((rcl[i] & CCALL_RCL_SSE)) {
795 if (nfpr >= CCALL_NARG_FPR) return 1; /* Register overflow. */
796 cc->fpr[nfpr++].l[0] = dp[i];
797 }
798 }
799 cc->ngpr = ngpr; cc->nfpr = nfpr;
800 return 0; /* Ok. */
801 }
802
803 /* Pass a small struct argument. */
ccall_struct_arg(CCallState * cc,CTState * cts,CType * d,int * rcl,TValue * o,int narg)804 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
805 TValue *o, int narg)
806 {
807 GPRArg dp[2];
808 dp[0] = dp[1] = 0;
809 /* Convert to temp. struct. */
810 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
811 if (ccall_struct_reg(cc, cts, dp, rcl)) {
812 /* Register overflow? Pass on stack. */
813 MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
814 if (nsp + n > CCALL_MAXSTACK) return 1; /* Too many arguments. */
815 cc->nsp = nsp + n;
816 memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
817 }
818 return 0; /* Ok. */
819 }
820
821 /* Combine returned small struct. */
ccall_struct_ret(CCallState * cc,int * rcl,uint8_t * dp,CTSize sz)822 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
823 {
824 GPRArg sp[2];
825 MSize ngpr = 0, nfpr = 0;
826 uint32_t i;
827 for (i = 0; i < 2; i++) {
828 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
829 sp[i] = cc->gpr[ngpr++];
830 } else if ((rcl[i] & CCALL_RCL_SSE)) {
831 sp[i] = cc->fpr[nfpr++].l[0];
832 }
833 }
834 memcpy(dp, sp, sz);
835 }
836 #endif
837
838 /* -- ARM hard-float ABI struct classification ---------------------------- */
839
840 #if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
841
842 /* Classify a struct based on its fields. */
ccall_classify_struct(CTState * cts,CType * ct,CType * ctf)843 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
844 {
845 CTSize sz = ct->size;
846 unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
847 if ((ctf->info & CTF_VARARG)) goto noth;
848 while (ct->sib) {
849 CType *sct;
850 ct = ctype_get(cts, ct->sib);
851 if (ctype_isfield(ct->info)) {
852 sct = ctype_rawchild(cts, ct);
853 if (ctype_isfp(sct->info)) {
854 r |= sct->size;
855 if (!isu) n++; else if (n == 0) n = 1;
856 } else if (ctype_iscomplex(sct->info)) {
857 r |= (sct->size >> 1);
858 if (!isu) n += 2; else if (n < 2) n = 2;
859 } else if (ctype_isstruct(sct->info)) {
860 goto substruct;
861 } else {
862 goto noth;
863 }
864 } else if (ctype_isbitfield(ct->info)) {
865 goto noth;
866 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
867 sct = ctype_rawchild(cts, ct);
868 substruct:
869 if (sct->size > 0) {
870 unsigned int s = ccall_classify_struct(cts, sct, ctf);
871 if (s <= 1) goto noth;
872 r |= (s & 255);
873 if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
874 }
875 }
876 }
877 if ((r == 4 || r == 8) && n <= 4)
878 return r + (n << 8);
879 noth: /* Not a homogeneous float/double aggregate. */
880 return (sz <= 4); /* Return structs of size <= 4 in a GPR. */
881 }
882
883 #endif
884
885 /* -- ARM64 ABI struct classification ------------------------------------- */
886
887 #if LJ_TARGET_ARM64
888
889 /* Classify a struct based on its fields. */
ccall_classify_struct(CTState * cts,CType * ct)890 static unsigned int ccall_classify_struct(CTState *cts, CType *ct)
891 {
892 CTSize sz = ct->size;
893 unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
894 while (ct->sib) {
895 CType *sct;
896 ct = ctype_get(cts, ct->sib);
897 if (ctype_isfield(ct->info)) {
898 sct = ctype_rawchild(cts, ct);
899 if (ctype_isfp(sct->info)) {
900 r |= sct->size;
901 if (!isu) n++; else if (n == 0) n = 1;
902 } else if (ctype_iscomplex(sct->info)) {
903 r |= (sct->size >> 1);
904 if (!isu) n += 2; else if (n < 2) n = 2;
905 } else if (ctype_isstruct(sct->info)) {
906 goto substruct;
907 } else {
908 goto noth;
909 }
910 } else if (ctype_isbitfield(ct->info)) {
911 goto noth;
912 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
913 sct = ctype_rawchild(cts, ct);
914 substruct:
915 if (sct->size > 0) {
916 unsigned int s = ccall_classify_struct(cts, sct);
917 if (s <= 1) goto noth;
918 r |= (s & 255);
919 if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
920 }
921 }
922 }
923 if ((r == 4 || r == 8) && n <= 4)
924 return r + (n << 8);
925 noth: /* Not a homogeneous float/double aggregate. */
926 return (sz <= 16); /* Return structs of size <= 16 in GPRs. */
927 }
928
929 #endif
930
931 /* -- PowerPC64 ELFv2 ABI struct classification ------------------- */
932
933 #if LJ_ARCH_PPC_ELFV2
934
935 #define FTYPE_FLOAT 1
936 #define FTYPE_DOUBLE 2
937
ccall_classify_fp(CTState * cts,CType * ct)938 static unsigned int ccall_classify_fp(CTState *cts, CType *ct) {
939 if (ctype_isfp(ct->info)) {
940 if (ct->size == sizeof(float))
941 return FTYPE_FLOAT;
942 else
943 return FTYPE_DOUBLE;
944 } else if (ctype_iscomplex(ct->info)) {
945 if (ct->size == sizeof(float) * 2)
946 return FTYPE_FLOAT;
947 else
948 return FTYPE_DOUBLE;
949 } else if (ctype_isstruct(ct->info)) {
950 int res = -1;
951 int sz = ct->size;
952 while (ct->sib) {
953 ct = ctype_get(cts, ct->sib);
954 if (ctype_isfield(ct->info)) {
955 int sub = ccall_classify_fp(cts, ctype_rawchild(cts, ct));
956 if (res == -1)
957 res = sub;
958 if (sub != -1 && sub != res)
959 return 0;
960 } else if (ctype_isbitfield(ct->info) ||
961 ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
962 return 0;
963 }
964 }
965 if (res > 0 && sz > res * 4 * 8)
966 return 0;
967 return res;
968 } else {
969 return 0;
970 }
971 }
972
973 #endif
974
975 /* -- MIPS64 ABI struct classification ---------------------------- */
976
977 #if LJ_TARGET_MIPS64
978
979 #define FTYPE_FLOAT 1
980 #define FTYPE_DOUBLE 2
981
982 /* Classify FP fields (max. 2) and their types. */
ccall_classify_struct(CTState * cts,CType * ct,CType * ctf)983 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
984 {
985 int n = 0, ft = 0;
986 if ((ctf->info & CTF_VARARG) || (ct->info & CTF_UNION))
987 goto noth;
988 while (ct->sib) {
989 CType *sct;
990 ct = ctype_get(cts, ct->sib);
991 if (n == 2) {
992 goto noth;
993 } else if (ctype_isfield(ct->info)) {
994 sct = ctype_rawchild(cts, ct);
995 if (ctype_isfp(sct->info)) {
996 ft |= (sct->size == 4 ? FTYPE_FLOAT : FTYPE_DOUBLE) << 2*n;
997 n++;
998 } else {
999 goto noth;
1000 }
1001 } else if (ctype_isbitfield(ct->info) ||
1002 ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
1003 goto noth;
1004 }
1005 }
1006 if (n <= 2)
1007 return ft;
1008 noth: /* Not a homogeneous float/double aggregate. */
1009 return 0; /* Struct is in GPRs. */
1010 }
1011
ccall_copy_struct(CCallState * cc,CType * ctr,void * dp,void * sp,int ft)1012 static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp,
1013 int ft)
1014 {
1015 if (LJ_ABI_SOFTFP ? ft :
1016 ((ft & 3) == FTYPE_FLOAT || (ft >> 2) == FTYPE_FLOAT)) {
1017 int i, ofs = 0;
1018 for (i = 0; ft != 0; i++, ft >>= 2) {
1019 if ((ft & 3) == FTYPE_FLOAT) {
1020 #if LJ_ABI_SOFTFP
1021 /* The 2nd FP struct result is in CARG1 (gpr[2]) and not CRET2. */
1022 memcpy((uint8_t *)dp + ofs,
1023 (uint8_t *)&cc->gpr[2*i] + LJ_ENDIAN_SELECT(0, 4), 4);
1024 #else
1025 *(float *)((uint8_t *)dp + ofs) = cc->fpr[i].f;
1026 #endif
1027 ofs += 4;
1028 } else {
1029 ofs = (ofs + 7) & ~7; /* 64 bit alignment. */
1030 #if LJ_ABI_SOFTFP
1031 *(intptr_t *)((uint8_t *)dp + ofs) = cc->gpr[2*i];
1032 #else
1033 *(double *)((uint8_t *)dp + ofs) = cc->fpr[i].d;
1034 #endif
1035 ofs += 8;
1036 }
1037 }
1038 } else {
1039 #if !LJ_ABI_SOFTFP
1040 if (ft) sp = (uint8_t *)&cc->fpr[0];
1041 #endif
1042 memcpy(dp, sp, ctr->size);
1043 }
1044 }
1045
1046 #endif
1047
1048 /* -- Common C call handling ---------------------------------------------- */
1049
1050 /* Infer the destination CTypeID for a vararg argument. */
lj_ccall_ctid_vararg(CTState * cts,cTValue * o)1051 CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
1052 {
1053 if (tvisnumber(o)) {
1054 return CTID_DOUBLE;
1055 } else if (tviscdata(o)) {
1056 CTypeID id = cdataV(o)->ctypeid;
1057 CType *s = ctype_get(cts, id);
1058 if (ctype_isrefarray(s->info)) {
1059 return lj_ctype_intern(cts,
1060 CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
1061 } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
1062 /* NYI: how to pass a struct by value in a vararg argument? */
1063 return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
1064 } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
1065 return CTID_DOUBLE;
1066 } else {
1067 return id;
1068 }
1069 } else if (tvisstr(o)) {
1070 return CTID_P_CCHAR;
1071 } else if (tvisbool(o)) {
1072 return CTID_BOOL;
1073 } else {
1074 return CTID_P_VOID;
1075 }
1076 }
1077
1078 /* Setup arguments for C call. */
ccall_set_args(lua_State * L,CTState * cts,CType * ct,CCallState * cc)1079 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
1080 CCallState *cc)
1081 {
1082 int gcsteps = 0;
1083 TValue *o, *top = L->top;
1084 CTypeID fid;
1085 CType *ctr;
1086 MSize maxgpr, ngpr = 0, nsp = 0, narg;
1087 #if CCALL_NARG_FPR
1088 MSize nfpr = 0;
1089 #if LJ_TARGET_ARM
1090 MSize fprodd = 0;
1091 #endif
1092 #endif
1093
1094 /* Clear unused regs to get some determinism in case of misdeclaration. */
1095 memset(cc->gpr, 0, sizeof(cc->gpr));
1096 #if CCALL_NUM_FPR
1097 memset(cc->fpr, 0, sizeof(cc->fpr));
1098 #endif
1099
1100 #if LJ_TARGET_X86
1101 /* x86 has several different calling conventions. */
1102 cc->resx87 = 0;
1103 switch (ctype_cconv(ct->info)) {
1104 case CTCC_FASTCALL: maxgpr = 2; break;
1105 case CTCC_THISCALL: maxgpr = 1; break;
1106 default: maxgpr = 0; break;
1107 }
1108 #else
1109 maxgpr = CCALL_NARG_GPR;
1110 #endif
1111
1112 /* Perform required setup for some result types. */
1113 ctr = ctype_rawchild(cts, ct);
1114 if (ctype_isvector(ctr->info)) {
1115 if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
1116 goto err_nyi;
1117 } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
1118 /* Preallocate cdata object and anchor it after arguments. */
1119 CTSize sz = ctr->size;
1120 GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
1121 void *dp = cdataptr(cd);
1122 setcdataV(L, L->top++, cd);
1123 if (ctype_isstruct(ctr->info)) {
1124 CCALL_HANDLE_STRUCTRET
1125 } else {
1126 CCALL_HANDLE_COMPLEXRET
1127 }
1128 #if LJ_TARGET_X86
1129 } else if (ctype_isfp(ctr->info)) {
1130 cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
1131 #endif
1132 }
1133
1134 /* Skip initial attributes. */
1135 fid = ct->sib;
1136 while (fid) {
1137 CType *ctf = ctype_get(cts, fid);
1138 if (!ctype_isattrib(ctf->info)) break;
1139 fid = ctf->sib;
1140 }
1141
1142 /* Walk through all passed arguments. */
1143 for (o = L->base+1, narg = 1; o < top; o++, narg++) {
1144 CTypeID did;
1145 CType *d;
1146 CTSize sz;
1147 MSize n, isfp = 0, isva = 0;
1148 void *dp, *rp = NULL;
1149 #if LJ_TARGET_PPC && LJ_ARCH_BITS == 64
1150 int isf32 = 0;
1151 #endif
1152
1153 #if LJ_TARGET_S390X
1154 uint32_t onstack = 0;
1155 #endif
1156
1157 if (fid) { /* Get argument type from field. */
1158 CType *ctf = ctype_get(cts, fid);
1159 fid = ctf->sib;
1160 lj_assertL(ctype_isfield(ctf->info), "field expected");
1161 did = ctype_cid(ctf->info);
1162 } else {
1163 if (!(ct->info & CTF_VARARG))
1164 lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too many arguments. */
1165 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
1166 isva = 1;
1167 }
1168 d = ctype_raw(cts, did);
1169 sz = d->size;
1170
1171 /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
1172 if (ctype_isnum(d->info)) {
1173 if (sz > 8) goto err_nyi;
1174 if ((d->info & CTF_FP))
1175 isfp = 1;
1176 } else if (ctype_isvector(d->info)) {
1177 if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
1178 isfp = 1;
1179 else
1180 goto err_nyi;
1181 } else if (ctype_isstruct(d->info)) {
1182 CCALL_HANDLE_STRUCTARG
1183 } else if (ctype_iscomplex(d->info)) {
1184 CCALL_HANDLE_COMPLEXARG
1185 } else {
1186 sz = CTSIZE_PTR;
1187 }
1188 sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
1189 n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */
1190
1191 CCALL_HANDLE_REGARG /* Handle register arguments. */
1192
1193 /* Otherwise pass argument on stack. */
1194 #if LJ_TARGET_S390X
1195 onstack = 1;
1196 #endif
1197 if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
1198 MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
1199 nsp = (nsp + align) & ~align; /* Align argument on stack. */
1200 }
1201 if (nsp + n > CCALL_MAXSTACK) { /* Too many arguments. */
1202 err_nyi:
1203 lj_err_caller(L, LJ_ERR_FFI_NYICALL);
1204 }
1205 dp = &cc->stack[nsp];
1206 nsp += n;
1207 isva = 0;
1208
1209 done:
1210 if (rp) { /* Pass by reference. */
1211 gcsteps++;
1212 *(void **)dp = rp;
1213 dp = rp;
1214 }
1215 #if LJ_TARGET_PPC && LJ_ARCH_BITS == 64 && LJ_BE
1216 if (ctype_isstruct(d->info) && sz < CTSIZE_PTR) {
1217 dp = (char *)dp + (CTSIZE_PTR - sz);
1218 }
1219 #endif
1220 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
1221 #if LJ_TARGET_PPC && LJ_ARCH_BITS == 64
1222 if (isfp) {
1223 int i;
1224 for (i = 0; i < d->size / 8 && nfpr < CCALL_NARG_FPR; i++)
1225 cc->fpr[nfpr++] = ((double *)dp)[i];
1226 }
1227 if (isf32) {
1228 int i;
1229 for (i = 0; i < d->size / 8; i++)
1230 ((float *)dp)[i*2] = ((double *)dp)[i];
1231 }
1232 #endif
1233 #if LJ_ARCH_PPC_ELFV2
1234 if (ctype_isstruct(d->info)) {
1235 isfp = ccall_classify_fp(cts, d);
1236 int i;
1237 if (isfp == FTYPE_FLOAT) {
1238 for (i = 0; i < d->size / 4 && nfpr < CCALL_NARG_FPR; i++)
1239 cc->fpr[nfpr++] = ((float *)dp)[i];
1240 } else if (isfp == FTYPE_DOUBLE) {
1241 for (i = 0; i < d->size / 8 && nfpr < CCALL_NARG_FPR; i++)
1242 cc->fpr[nfpr++] = ((double *)dp)[i];
1243 }
1244 }
1245 #endif
1246 /* Extend passed integers to 32 bits at least. */
1247 if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
1248 if (d->info & CTF_UNSIGNED)
1249 *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
1250 (uint32_t)*(uint16_t *)dp;
1251 else
1252 *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
1253 (int32_t)*(int16_t *)dp;
1254 }
1255 #if LJ_TARGET_ARM64 && LJ_BE
1256 if (isfp && d->size == sizeof(float))
1257 ((float *)dp)[1] = ((float *)dp)[0]; /* Floats occupy high slot. */
1258 #endif
1259 #if LJ_TARGET_PPC && LJ_ARCH_BITS == 64
1260 if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info))
1261 && d->size <= 4) {
1262 if (d->info & CTF_UNSIGNED)
1263 *(uint64_t *)dp = (uint64_t)*(uint32_t *)dp;
1264 else
1265 *(int64_t *)dp = (int64_t)*(int32_t *)dp;
1266 }
1267 #endif
1268 #if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
1269 if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info)
1270 #if LJ_TARGET_MIPS64
1271 || (isfp && nsp == 0)
1272 #endif
1273 ) && d->size <= 4) {
1274 *(int64_t *)dp = (int64_t)*(int32_t *)dp; /* Sign-extend to 64 bit. */
1275 }
1276 #endif
1277 #if LJ_TARGET_S390X
1278 /* Arguments need to be sign-/zero-extended to 64-bits. */
1279 if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info) ||
1280 (isfp && onstack)) && d->size <= 4) {
1281 if (d->info & CTF_UNSIGNED || isfp)
1282 *(uint64_t *)dp = (uint64_t)*(uint32_t *)dp;
1283 else
1284 *(int64_t *)dp = (int64_t)*(int32_t *)dp;
1285 }
1286 #endif
1287 #if LJ_TARGET_X64 && LJ_ABI_WIN
1288 if (isva) { /* Windows/x64 mirrors varargs in both register sets. */
1289 if (nfpr == ngpr)
1290 cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
1291 else
1292 cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
1293 }
1294 #else
1295 UNUSED(isva);
1296 #endif
1297 #if LJ_TARGET_X64 && !LJ_ABI_WIN
1298 if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
1299 cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1]; /* Split complex double. */
1300 cc->fpr[nfpr-2].d[1] = 0;
1301 }
1302 #elif LJ_TARGET_ARM64 || (LJ_TARGET_MIPS64 && !LJ_ABI_SOFTFP)
1303 if (isfp == 2 && (uint8_t *)dp < (uint8_t *)cc->stack) {
1304 /* Split float HFA or complex float into separate registers. */
1305 CTSize i = (sz >> 2) - 1;
1306 do { ((uint64_t *)dp)[i] = ((uint32_t *)dp)[i]; } while (i--);
1307 }
1308 #else
1309 UNUSED(isfp);
1310 #endif
1311 }
1312 if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too few arguments. */
1313
1314 #if LJ_TARGET_X64 || (LJ_TARGET_PPC && !LJ_ABI_SOFTFP)
1315 cc->nfpr = nfpr; /* Required for vararg functions. */
1316 #endif
1317 cc->nsp = nsp;
1318 cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
1319 if (nsp > CCALL_SPS_FREE)
1320 cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
1321 return gcsteps;
1322 }
1323
1324 /* Get results from C call. */
ccall_get_results(lua_State * L,CTState * cts,CType * ct,CCallState * cc,int * ret)1325 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
1326 CCallState *cc, int *ret)
1327 {
1328 CType *ctr = ctype_rawchild(cts, ct);
1329 uint8_t *sp = (uint8_t *)&cc->gpr[0];
1330 if (ctype_isvoid(ctr->info)) {
1331 *ret = 0; /* Zero results. */
1332 return 0; /* No additional GC step. */
1333 }
1334 *ret = 1; /* One result. */
1335 if (ctype_isstruct(ctr->info)) {
1336 /* Return cdata object which is already on top of stack. */
1337 if (!cc->retref) {
1338 void *dp = cdataptr(cdataV(L->top-1)); /* Use preallocated object. */
1339 CCALL_HANDLE_STRUCTRET2
1340 }
1341 return 1; /* One GC step. */
1342 }
1343 if (ctype_iscomplex(ctr->info)) {
1344 /* Return cdata object which is already on top of stack. */
1345 void *dp = cdataptr(cdataV(L->top-1)); /* Use preallocated object. */
1346 CCALL_HANDLE_COMPLEXRET2
1347 return 1; /* One GC step. */
1348 }
1349 if (LJ_BE && ctr->size < CTSIZE_PTR &&
1350 (ctype_isinteger_or_bool(ctr->info) || ctype_isenum(ctr->info)))
1351 sp += (CTSIZE_PTR - ctr->size);
1352 #if CCALL_NUM_FPR
1353 if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
1354 sp = (uint8_t *)&cc->fpr[0];
1355 #endif
1356 #ifdef CCALL_HANDLE_RET
1357 CCALL_HANDLE_RET
1358 #endif
1359 /* No reference types end up here, so there's no need for the CTypeID. */
1360 lj_assertL(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)),
1361 "unexpected reference ctype");
1362 return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
1363 }
1364
1365 /* Call C function. */
lj_ccall_func(lua_State * L,GCcdata * cd)1366 int lj_ccall_func(lua_State *L, GCcdata *cd)
1367 {
1368 CTState *cts = ctype_cts(L);
1369 CType *ct = ctype_raw(cts, cd->ctypeid);
1370 CTSize sz = CTSIZE_PTR;
1371 if (ctype_isptr(ct->info)) {
1372 sz = ct->size;
1373 ct = ctype_rawchild(cts, ct);
1374 }
1375 if (ctype_isfunc(ct->info)) {
1376 CCallState cc;
1377 int gcsteps, ret;
1378 cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
1379 gcsteps = ccall_set_args(L, cts, ct, &cc);
1380 ct = (CType *)((intptr_t)ct-(intptr_t)cts->tab);
1381 cts->cb.slot = ~0u;
1382 lj_vm_ffi_call(&cc);
1383 if (cts->cb.slot != ~0u) { /* Blacklist function that called a callback. */
1384 TValue tv;
1385 tv.u64 = ((uintptr_t)(void *)cc.func >> 2) | U64x(800000000, 00000000);
1386 setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
1387 }
1388 ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab); /* May be reallocated. */
1389 gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
1390 #if LJ_TARGET_X86 && LJ_ABI_WIN
1391 /* Automatically detect __stdcall and fix up C function declaration. */
1392 if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
1393 CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
1394 lj_trace_abort(G(L));
1395 }
1396 #endif
1397 while (gcsteps-- > 0)
1398 lj_gc_check(L);
1399 return ret;
1400 }
1401 return -1; /* Not a function. */
1402 }
1403
1404 #endif
1405