1 /*
2 ** Trace recorder for C data operations.
3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #define lj_ffrecord_c
7 #define LUA_CORE
8
9 #include "lj_obj.h"
10
11 #if LJ_HASJIT && LJ_HASFFI
12
13 #include "lj_err.h"
14 #include "lj_tab.h"
15 #include "lj_frame.h"
16 #include "lj_ctype.h"
17 #include "lj_cdata.h"
18 #include "lj_cparse.h"
19 #include "lj_cconv.h"
20 #include "lj_carith.h"
21 #include "lj_clib.h"
22 #include "lj_ccall.h"
23 #include "lj_ff.h"
24 #include "lj_ir.h"
25 #include "lj_jit.h"
26 #include "lj_ircall.h"
27 #include "lj_iropt.h"
28 #include "lj_trace.h"
29 #include "lj_record.h"
30 #include "lj_ffrecord.h"
31 #include "lj_snap.h"
32 #include "lj_crecord.h"
33 #include "lj_dispatch.h"
34 #include "lj_strfmt.h"
35
36 /* Some local macros to save typing. Undef'd at the end. */
37 #define IR(ref) (&J->cur.ir[(ref)])
38
39 /* Pass IR on to next optimization in chain (FOLD). */
40 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
41
42 #define emitconv(a, dt, st, flags) \
43 emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
44
45 /* -- C type checks ------------------------------------------------------- */
46
argv2cdata(jit_State * J,TRef tr,cTValue * o)47 static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
48 {
49 GCcdata *cd;
50 TRef trtypeid;
51 if (!tref_iscdata(tr))
52 lj_trace_err(J, LJ_TRERR_BADTYPE);
53 cd = cdataV(o);
54 /* Specialize to the CTypeID. */
55 trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
56 emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
57 return cd;
58 }
59
60 /* Specialize to the CTypeID held by a cdata constructor. */
crec_constructor(jit_State * J,GCcdata * cd,TRef tr)61 static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
62 {
63 CTypeID id;
64 lua_assert(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID);
65 id = *(CTypeID *)cdataptr(cd);
66 tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
67 emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
68 return id;
69 }
70
argv2ctype(jit_State * J,TRef tr,cTValue * o)71 static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
72 {
73 if (tref_isstr(tr)) {
74 GCstr *s = strV(o);
75 CPState cp;
76 CTypeID oldtop;
77 /* Specialize to the string containing the C type declaration. */
78 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
79 cp.L = J->L;
80 cp.cts = ctype_ctsG(J2G(J));
81 oldtop = cp.cts->top;
82 cp.srcname = strdata(s);
83 cp.p = strdata(s);
84 cp.param = NULL;
85 cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
86 if (lj_cparse(&cp) || cp.cts->top > oldtop) /* Avoid new struct defs. */
87 lj_trace_err(J, LJ_TRERR_BADTYPE);
88 return cp.val.id;
89 } else {
90 GCcdata *cd = argv2cdata(J, tr, o);
91 return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
92 cd->ctypeid;
93 }
94 }
95
96 /* Convert CType to IRType (if possible). */
crec_ct2irt(CTState * cts,CType * ct)97 static IRType crec_ct2irt(CTState *cts, CType *ct)
98 {
99 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
100 if (LJ_LIKELY(ctype_isnum(ct->info))) {
101 if ((ct->info & CTF_FP)) {
102 if (ct->size == sizeof(double))
103 return IRT_NUM;
104 else if (ct->size == sizeof(float))
105 return IRT_FLOAT;
106 } else {
107 uint32_t b = lj_fls(ct->size);
108 if (b <= 3)
109 return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
110 }
111 } else if (ctype_isptr(ct->info)) {
112 return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
113 } else if (ctype_iscomplex(ct->info)) {
114 if (ct->size == 2*sizeof(double))
115 return IRT_NUM;
116 else if (ct->size == 2*sizeof(float))
117 return IRT_FLOAT;
118 }
119 return IRT_CDATA;
120 }
121
122 /* -- Optimized memory fill and copy -------------------------------------- */
123
124 /* Maximum length and unroll of inlined copy/fill. */
125 #define CREC_COPY_MAXUNROLL 16
126 #define CREC_COPY_MAXLEN 128
127
128 #define CREC_FILL_MAXUNROLL 16
129
130 /* Number of windowed registers used for optimized memory copy. */
131 #if LJ_TARGET_X86
132 #define CREC_COPY_REGWIN 2
133 #elif LJ_TARGET_PPC || LJ_TARGET_MIPS
134 #define CREC_COPY_REGWIN 8
135 #else
136 #define CREC_COPY_REGWIN 4
137 #endif
138
139 /* List of memory offsets for copy/fill. */
140 typedef struct CRecMemList {
141 CTSize ofs; /* Offset in bytes. */
142 IRType tp; /* Type of load/store. */
143 TRef trofs; /* TRef of interned offset. */
144 TRef trval; /* TRef of load value. */
145 } CRecMemList;
146
147 /* Generate copy list for element-wise struct copy. */
crec_copy_struct(CRecMemList * ml,CTState * cts,CType * ct)148 static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
149 {
150 CTypeID fid = ct->sib;
151 MSize mlp = 0;
152 while (fid) {
153 CType *df = ctype_get(cts, fid);
154 fid = df->sib;
155 if (ctype_isfield(df->info)) {
156 CType *cct;
157 IRType tp;
158 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
159 cct = ctype_rawchild(cts, df); /* Field type. */
160 tp = crec_ct2irt(cts, cct);
161 if (tp == IRT_CDATA) return 0; /* NYI: aggregates. */
162 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
163 ml[mlp].ofs = df->size;
164 ml[mlp].tp = tp;
165 mlp++;
166 if (ctype_iscomplex(cct->info)) {
167 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
168 ml[mlp].ofs = df->size + (cct->size >> 1);
169 ml[mlp].tp = tp;
170 mlp++;
171 }
172 } else if (!ctype_isconstval(df->info)) {
173 /* NYI: bitfields and sub-structures. */
174 return 0;
175 }
176 }
177 return mlp;
178 }
179
180 /* Generate unrolled copy list, from highest to lowest step size/alignment. */
crec_copy_unroll(CRecMemList * ml,CTSize len,CTSize step,IRType tp)181 static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
182 IRType tp)
183 {
184 CTSize ofs = 0;
185 MSize mlp = 0;
186 if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
187 do {
188 while (ofs + step <= len) {
189 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
190 ml[mlp].ofs = ofs;
191 ml[mlp].tp = tp;
192 mlp++;
193 ofs += step;
194 }
195 step >>= 1;
196 tp -= 2;
197 } while (ofs < len);
198 return mlp;
199 }
200
201 /*
202 ** Emit copy list with windowed loads/stores.
203 ** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
204 */
crec_copy_emit(jit_State * J,CRecMemList * ml,MSize mlp,TRef trdst,TRef trsrc)205 static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
206 TRef trdst, TRef trsrc)
207 {
208 MSize i, j, rwin = 0;
209 for (i = 0, j = 0; i < mlp; ) {
210 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
211 TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
212 ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
213 ml[i].trofs = trofs;
214 i++;
215 rwin += (LJ_SOFTFP && ml[i].tp == IRT_NUM) ? 2 : 1;
216 if (rwin >= CREC_COPY_REGWIN || i >= mlp) { /* Flush buffered stores. */
217 rwin = 0;
218 for ( ; j < i; j++) {
219 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
220 emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
221 }
222 }
223 }
224 }
225
226 /* Optimized memory copy. */
crec_copy(jit_State * J,TRef trdst,TRef trsrc,TRef trlen,CType * ct)227 static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
228 CType *ct)
229 {
230 if (tref_isk(trlen)) { /* Length must be constant. */
231 CRecMemList ml[CREC_COPY_MAXUNROLL];
232 MSize mlp = 0;
233 CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
234 IRType tp = IRT_CDATA;
235 int needxbar = 0;
236 if (len == 0) return; /* Shortcut. */
237 if (len > CREC_COPY_MAXLEN) goto fallback;
238 if (ct) {
239 CTState *cts = ctype_ctsG(J2G(J));
240 lua_assert(ctype_isarray(ct->info) || ctype_isstruct(ct->info));
241 if (ctype_isarray(ct->info)) {
242 CType *cct = ctype_rawchild(cts, ct);
243 tp = crec_ct2irt(cts, cct);
244 if (tp == IRT_CDATA) goto rawcopy;
245 step = lj_ir_type_size[tp];
246 lua_assert((len & (step-1)) == 0);
247 } else if ((ct->info & CTF_UNION)) {
248 step = (1u << ctype_align(ct->info));
249 goto rawcopy;
250 } else {
251 mlp = crec_copy_struct(ml, cts, ct);
252 goto emitcopy;
253 }
254 } else {
255 rawcopy:
256 needxbar = 1;
257 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
258 step = CTSIZE_PTR;
259 }
260 mlp = crec_copy_unroll(ml, len, step, tp);
261 emitcopy:
262 if (mlp) {
263 crec_copy_emit(J, ml, mlp, trdst, trsrc);
264 if (needxbar)
265 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
266 return;
267 }
268 }
269 fallback:
270 /* Call memcpy. Always needs a barrier to disable alias analysis. */
271 lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
272 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
273 }
274
275 /* Generate unrolled fill list, from highest to lowest step size/alignment. */
crec_fill_unroll(CRecMemList * ml,CTSize len,CTSize step)276 static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
277 {
278 CTSize ofs = 0;
279 MSize mlp = 0;
280 IRType tp = IRT_U8 + 2*lj_fls(step);
281 do {
282 while (ofs + step <= len) {
283 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
284 ml[mlp].ofs = ofs;
285 ml[mlp].tp = tp;
286 mlp++;
287 ofs += step;
288 }
289 step >>= 1;
290 tp -= 2;
291 } while (ofs < len);
292 return mlp;
293 }
294
295 /*
296 ** Emit stores for fill list.
297 ** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
298 */
crec_fill_emit(jit_State * J,CRecMemList * ml,MSize mlp,TRef trdst,TRef trfill)299 static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
300 TRef trdst, TRef trfill)
301 {
302 MSize i;
303 for (i = 0; i < mlp; i++) {
304 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
305 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
306 emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
307 }
308 }
309
310 /* Optimized memory fill. */
crec_fill(jit_State * J,TRef trdst,TRef trlen,TRef trfill,CTSize step)311 static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
312 CTSize step)
313 {
314 if (tref_isk(trlen)) { /* Length must be constant. */
315 CRecMemList ml[CREC_FILL_MAXUNROLL];
316 MSize mlp;
317 CTSize len = (CTSize)IR(tref_ref(trlen))->i;
318 if (len == 0) return; /* Shortcut. */
319 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
320 step = CTSIZE_PTR;
321 if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
322 mlp = crec_fill_unroll(ml, len, step);
323 if (!mlp) goto fallback;
324 if (tref_isk(trfill) || ml[0].tp != IRT_U8)
325 trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
326 if (ml[0].tp != IRT_U8) { /* Scatter U8 to U16/U32/U64. */
327 if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
328 if (tref_isk(trfill)) /* Pointless on x64 with zero-extended regs. */
329 trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
330 trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
331 lj_ir_kint64(J, U64x(01010101,01010101)));
332 } else {
333 trfill = emitir(IRTI(IR_MUL), trfill,
334 lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
335 }
336 }
337 crec_fill_emit(J, ml, mlp, trdst, trfill);
338 } else {
339 fallback:
340 /* Call memset. Always needs a barrier to disable alias analysis. */
341 lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen); /* Note: arg order! */
342 }
343 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
344 }
345
346 /* -- Convert C type to C type -------------------------------------------- */
347
348 /*
349 ** This code mirrors the code in lj_cconv.c. It performs the same steps
350 ** for the trace recorder that lj_cconv.c does for the interpreter.
351 **
352 ** One major difference is that we can get away with much fewer checks
353 ** here. E.g. checks for casts, constness or correct types can often be
354 ** omitted, even if they might fail. The interpreter subsequently throws
355 ** an error, which aborts the trace.
356 **
357 ** All operations are specialized to their C types, so the on-trace
358 ** outcome must be the same as the outcome in the interpreter. If the
359 ** interpreter doesn't throw an error, then the trace is correct, too.
360 ** Care must be taken not to generate invalid (temporary) IR or to
361 ** trigger asserts.
362 */
363
364 /* Determine whether a passed number or cdata number is non-zero. */
crec_isnonzero(CType * s,void * p)365 static int crec_isnonzero(CType *s, void *p)
366 {
367 if (p == (void *)0)
368 return 0;
369 if (p == (void *)1)
370 return 1;
371 if ((s->info & CTF_FP)) {
372 if (s->size == sizeof(float))
373 return (*(float *)p != 0);
374 else
375 return (*(double *)p != 0);
376 } else {
377 if (s->size == 1)
378 return (*(uint8_t *)p != 0);
379 else if (s->size == 2)
380 return (*(uint16_t *)p != 0);
381 else if (s->size == 4)
382 return (*(uint32_t *)p != 0);
383 else
384 return (*(uint64_t *)p != 0);
385 }
386 }
387
crec_ct_ct(jit_State * J,CType * d,CType * s,TRef dp,TRef sp,void * svisnz)388 static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
389 void *svisnz)
390 {
391 IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
392 IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
393 CTSize dsize = d->size, ssize = s->size;
394 CTInfo dinfo = d->info, sinfo = s->info;
395
396 if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
397 goto err_conv;
398
399 /*
400 ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
401 ** numbers up to 8 bytes. Otherwise sp holds a pointer.
402 */
403
404 switch (cconv_idx2(dinfo, sinfo)) {
405 /* Destination is a bool. */
406 case CCX(B, B):
407 goto xstore; /* Source operand is already normalized. */
408 case CCX(B, I):
409 case CCX(B, F):
410 if (st != IRT_CDATA) {
411 /* Specialize to the result of a comparison against 0. */
412 TRef zero = (st == IRT_NUM || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
413 (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
414 lj_ir_kint(J, 0);
415 int isnz = crec_isnonzero(s, svisnz);
416 emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
417 sp = lj_ir_kint(J, isnz);
418 goto xstore;
419 }
420 goto err_nyi;
421
422 /* Destination is an integer. */
423 case CCX(I, B):
424 case CCX(I, I):
425 conv_I_I:
426 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
427 /* Extend 32 to 64 bit integer. */
428 if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
429 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
430 (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
431 else if (dsize < 8 && ssize == 8) /* Truncate from 64 bit integer. */
432 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
433 else if (st == IRT_INT)
434 sp = lj_opt_narrow_toint(J, sp);
435 xstore:
436 if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
437 if (dp == 0) return sp;
438 emitir(IRT(IR_XSTORE, dt), dp, sp);
439 break;
440 case CCX(I, C):
441 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
442 /* fallthrough */
443 case CCX(I, F):
444 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
445 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
446 goto xstore;
447 case CCX(I, P):
448 case CCX(I, A):
449 sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
450 ssize = CTSIZE_PTR;
451 st = IRT_UINTP;
452 if (((dsize ^ ssize) & 8) == 0) { /* Must insert no-op type conversion. */
453 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
454 goto xstore;
455 }
456 goto conv_I_I;
457
458 /* Destination is a floating-point number. */
459 case CCX(F, B):
460 case CCX(F, I):
461 conv_F_I:
462 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
463 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
464 goto xstore;
465 case CCX(F, C):
466 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
467 /* fallthrough */
468 case CCX(F, F):
469 conv_F_F:
470 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
471 if (dt != st) sp = emitconv(sp, dt, st, 0);
472 goto xstore;
473
474 /* Destination is a complex number. */
475 case CCX(C, I):
476 case CCX(C, F):
477 { /* Clear im. */
478 TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
479 emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
480 }
481 /* Convert to re. */
482 if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
483
484 case CCX(C, C):
485 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
486 {
487 TRef re, im, ptr;
488 re = emitir(IRT(IR_XLOAD, st), sp, 0);
489 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
490 im = emitir(IRT(IR_XLOAD, st), ptr, 0);
491 if (dt != st) {
492 re = emitconv(re, dt, st, 0);
493 im = emitconv(im, dt, st, 0);
494 }
495 emitir(IRT(IR_XSTORE, dt), dp, re);
496 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
497 emitir(IRT(IR_XSTORE, dt), ptr, im);
498 }
499 break;
500
501 /* Destination is a vector. */
502 case CCX(V, I):
503 case CCX(V, F):
504 case CCX(V, C):
505 case CCX(V, V):
506 goto err_nyi;
507
508 /* Destination is a pointer. */
509 case CCX(P, P):
510 case CCX(P, A):
511 case CCX(P, S):
512 /* There are only 32 bit pointers/addresses on 32 bit machines.
513 ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
514 */
515 goto xstore;
516 case CCX(P, I):
517 if (st == IRT_CDATA) goto err_nyi;
518 if (!LJ_64 && ssize == 8) /* Truncate from 64 bit integer. */
519 sp = emitconv(sp, IRT_U32, st, 0);
520 goto xstore;
521 case CCX(P, F):
522 if (st == IRT_CDATA) goto err_nyi;
523 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
524 sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
525 st, IRCONV_ANY);
526 goto xstore;
527
528 /* Destination is an array. */
529 case CCX(A, A):
530 /* Destination is a struct/union. */
531 case CCX(S, S):
532 if (dp == 0) goto err_conv;
533 crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
534 break;
535
536 default:
537 err_conv:
538 err_nyi:
539 lj_trace_err(J, LJ_TRERR_NYICONV);
540 break;
541 }
542 return 0;
543 }
544
545 /* -- Convert C type to TValue (load) ------------------------------------- */
546
crec_tv_ct(jit_State * J,CType * s,CTypeID sid,TRef sp)547 static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
548 {
549 CTState *cts = ctype_ctsG(J2G(J));
550 IRType t = crec_ct2irt(cts, s);
551 CTInfo sinfo = s->info;
552 if (ctype_isnum(sinfo)) {
553 TRef tr;
554 if (t == IRT_CDATA)
555 goto err_nyi; /* NYI: copyval of >64 bit integers. */
556 tr = emitir(IRT(IR_XLOAD, t), sp, 0);
557 if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */
558 return emitconv(tr, IRT_NUM, t, 0);
559 } else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */
560 sp = tr;
561 lj_needsplit(J);
562 } else if ((sinfo & CTF_BOOL)) {
563 /* Assume not equal to zero. Fixup and emit pending guard later. */
564 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
565 J->postproc = LJ_POST_FIXGUARD;
566 return TREF_TRUE;
567 } else {
568 return tr;
569 }
570 } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
571 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Box pointers and enums. */
572 } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
573 cts->L = J->L;
574 sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */
575 } else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */
576 ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
577 TRef ptr, tr1, tr2, dp;
578 dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
579 tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
580 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
581 tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
582 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
583 emitir(IRT(IR_XSTORE, t), ptr, tr1);
584 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
585 emitir(IRT(IR_XSTORE, t), ptr, tr2);
586 return dp;
587 } else {
588 /* NYI: copyval of vectors. */
589 err_nyi:
590 lj_trace_err(J, LJ_TRERR_NYICONV);
591 }
592 /* Box pointer, ref, enum or 64 bit integer. */
593 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
594 }
595
596 /* -- Convert TValue to C type (store) ------------------------------------ */
597
crec_ct_tv(jit_State * J,CType * d,TRef dp,TRef sp,cTValue * sval)598 static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
599 {
600 CTState *cts = ctype_ctsG(J2G(J));
601 CTypeID sid = CTID_P_VOID;
602 void *svisnz = 0;
603 CType *s;
604 if (LJ_LIKELY(tref_isinteger(sp))) {
605 sid = CTID_INT32;
606 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
607 } else if (tref_isnum(sp)) {
608 sid = CTID_DOUBLE;
609 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
610 } else if (tref_isbool(sp)) {
611 sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
612 sid = CTID_BOOL;
613 } else if (tref_isnil(sp)) {
614 sp = lj_ir_kptr(J, NULL);
615 } else if (tref_isudata(sp)) {
616 GCudata *ud = udataV(sval);
617 if (ud->udtype == UDTYPE_IO_FILE) {
618 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
619 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
620 sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE);
621 } else {
622 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
623 }
624 } else if (tref_isstr(sp)) {
625 if (ctype_isenum(d->info)) { /* Match string against enum constant. */
626 GCstr *str = strV(sval);
627 CTSize ofs;
628 CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
629 /* Specialize to the name of the enum constant. */
630 emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
631 if (cct && ctype_isconstval(cct->info)) {
632 lua_assert(ctype_child(cts, cct)->size == 4);
633 svisnz = (void *)(intptr_t)(ofs != 0);
634 sp = lj_ir_kint(J, (int32_t)ofs);
635 sid = ctype_cid(cct->info);
636 } /* else: interpreter will throw. */
637 } else if (ctype_isrefarray(d->info)) { /* Copy string to array. */
638 lj_trace_err(J, LJ_TRERR_BADTYPE); /* NYI */
639 } else { /* Otherwise pass the string data as a const char[]. */
640 /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
641 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
642 sid = CTID_A_CCHAR;
643 }
644 } else if (tref_islightud(sp)) {
645 #if LJ_64
646 sp = emitir(IRT(IR_BAND, IRT_P64), sp,
647 lj_ir_kint64(J, U64x(00007fff,ffffffff)));
648 #endif
649 } else { /* NYI: tref_istab(sp). */
650 IRType t;
651 sid = argv2cdata(J, sp, sval)->ctypeid;
652 s = ctype_raw(cts, sid);
653 svisnz = cdataptr(cdataV(sval));
654 if (ctype_isfunc(s->info)) {
655 sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
656 s = ctype_get(cts, sid);
657 t = IRT_PTR;
658 } else {
659 t = crec_ct2irt(cts, s);
660 }
661 if (ctype_isptr(s->info)) {
662 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
663 if (ctype_isref(s->info)) {
664 svisnz = *(void **)svisnz;
665 s = ctype_rawchild(cts, s);
666 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
667 t = crec_ct2irt(cts, s);
668 } else {
669 goto doconv;
670 }
671 } else if (t == IRT_I64 || t == IRT_U64) {
672 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
673 lj_needsplit(J);
674 goto doconv;
675 } else if (t == IRT_INT || t == IRT_U32) {
676 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
677 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
678 goto doconv;
679 } else {
680 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
681 }
682 if (ctype_isnum(s->info) && t != IRT_CDATA)
683 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Load number value. */
684 goto doconv;
685 }
686 s = ctype_get(cts, sid);
687 doconv:
688 if (ctype_isenum(d->info)) d = ctype_child(cts, d);
689 return crec_ct_ct(J, d, s, dp, sp, svisnz);
690 }
691
692 /* -- C data metamethods -------------------------------------------------- */
693
694 /* This would be rather difficult in FOLD, so do it here:
695 ** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
696 ** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
697 */
crec_reassoc_ofs(jit_State * J,TRef tr,ptrdiff_t * ofsp,MSize sz)698 static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
699 {
700 IRIns *ir = IR(tref_ref(tr));
701 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
702 (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
703 IRIns *irk = IR(ir->op2);
704 ptrdiff_t k;
705 if (LJ_64 && irk->o == IR_KINT64)
706 k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
707 else
708 k = (ptrdiff_t)irk->i * sz;
709 if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
710 tr = ir->op1; /* Not a TRef, but the caller doesn't care. */
711 }
712 return tr;
713 }
714
715 /* Tailcall to function. */
crec_tailcall(jit_State * J,RecordFFData * rd,cTValue * tv)716 static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
717 {
718 TRef kfunc = lj_ir_kfunc(J, funcV(tv));
719 #if LJ_FR2
720 J->base[-2] = kfunc;
721 J->base[-1] = TREF_FRAME;
722 #else
723 J->base[-1] = kfunc | TREF_FRAME;
724 #endif
725 rd->nres = -1; /* Pending tailcall. */
726 }
727
728 /* Record ctype __index/__newindex metamethods. */
crec_index_meta(jit_State * J,CTState * cts,CType * ct,RecordFFData * rd)729 static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
730 RecordFFData *rd)
731 {
732 CTypeID id = ctype_typeid(cts, ct);
733 cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
734 if (!tv)
735 lj_trace_err(J, LJ_TRERR_BADTYPE);
736 if (tvisfunc(tv)) {
737 crec_tailcall(J, rd, tv);
738 } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
739 /* Specialize to result of __index lookup. */
740 cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
741 J->base[0] = lj_record_constify(J, o);
742 if (!J->base[0])
743 lj_trace_err(J, LJ_TRERR_BADTYPE);
744 /* Always specialize to the key. */
745 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
746 } else {
747 /* NYI: resolving of non-function metamethods. */
748 /* NYI: non-string keys for __index table. */
749 /* NYI: stores to __newindex table. */
750 lj_trace_err(J, LJ_TRERR_BADTYPE);
751 }
752 }
753
754 /* Record bitfield load/store. */
crec_index_bf(jit_State * J,RecordFFData * rd,TRef ptr,CTInfo info)755 static void crec_index_bf(jit_State *J, RecordFFData *rd, TRef ptr, CTInfo info)
756 {
757 IRType t = IRT_I8 + 2*lj_fls(ctype_bitcsz(info)) + ((info&CTF_UNSIGNED)?1:0);
758 TRef tr = emitir(IRT(IR_XLOAD, t), ptr, 0);
759 CTSize pos = ctype_bitpos(info), bsz = ctype_bitbsz(info), shift = 32 - bsz;
760 lua_assert(t <= IRT_U32); /* NYI: 64 bit bitfields. */
761 if (rd->data == 0) { /* __index metamethod. */
762 if ((info & CTF_BOOL)) {
763 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << pos))));
764 /* Assume not equal to zero. Fixup and emit pending guard later. */
765 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
766 J->postproc = LJ_POST_FIXGUARD;
767 tr = TREF_TRUE;
768 } else if (!(info & CTF_UNSIGNED)) {
769 tr = emitir(IRTI(IR_BSHL), tr, lj_ir_kint(J, shift - pos));
770 tr = emitir(IRTI(IR_BSAR), tr, lj_ir_kint(J, shift));
771 } else {
772 lua_assert(bsz < 32); /* Full-size fields cannot end up here. */
773 tr = emitir(IRTI(IR_BSHR), tr, lj_ir_kint(J, pos));
774 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << bsz)-1)));
775 /* We can omit the U32 to NUM conversion, since bsz < 32. */
776 }
777 J->base[0] = tr;
778 } else { /* __newindex metamethod. */
779 CTState *cts = ctype_ctsG(J2G(J));
780 CType *ct = ctype_get(cts,
781 (info & CTF_BOOL) ? CTID_BOOL :
782 (info & CTF_UNSIGNED) ? CTID_UINT32 : CTID_INT32);
783 int32_t mask = (int32_t)(((1u << bsz)-1) << pos);
784 TRef sp = crec_ct_tv(J, ct, 0, J->base[2], &rd->argv[2]);
785 sp = emitir(IRTI(IR_BSHL), sp, lj_ir_kint(J, pos));
786 /* Use of the target type avoids forwarding conversions. */
787 sp = emitir(IRT(IR_BAND, t), sp, lj_ir_kint(J, mask));
788 tr = emitir(IRT(IR_BAND, t), tr, lj_ir_kint(J, (int32_t)~mask));
789 tr = emitir(IRT(IR_BOR, t), tr, sp);
790 emitir(IRT(IR_XSTORE, t), ptr, tr);
791 rd->nres = 0;
792 J->needsnap = 1;
793 }
794 }
795
recff_cdata_index(jit_State * J,RecordFFData * rd)796 void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
797 {
798 TRef idx, ptr = J->base[0];
799 ptrdiff_t ofs = sizeof(GCcdata);
800 GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
801 CTState *cts = ctype_ctsG(J2G(J));
802 CType *ct = ctype_raw(cts, cd->ctypeid);
803 CTypeID sid = 0;
804
805 /* Resolve pointer or reference for cdata object. */
806 if (ctype_isptr(ct->info)) {
807 IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
808 if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
809 ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
810 ofs = 0;
811 ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
812 }
813
814 again:
815 idx = J->base[1];
816 if (tref_isnumber(idx)) {
817 idx = lj_opt_narrow_cindex(J, idx);
818 if (ctype_ispointer(ct->info)) {
819 CTSize sz;
820 integer_key:
821 if ((ct->info & CTF_COMPLEX))
822 idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
823 sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
824 idx = crec_reassoc_ofs(J, idx, &ofs, sz);
825 #if LJ_TARGET_ARM || LJ_TARGET_PPC
826 /* Hoist base add to allow fusion of index/shift into operands. */
827 if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
828 #if LJ_TARGET_ARM
829 && (sz == 1 || sz == 4)
830 #endif
831 ) {
832 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
833 ofs = 0;
834 }
835 #endif
836 idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
837 ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
838 }
839 } else if (tref_iscdata(idx)) {
840 GCcdata *cdk = cdataV(&rd->argv[1]);
841 CType *ctk = ctype_raw(cts, cdk->ctypeid);
842 IRType t = crec_ct2irt(cts, ctk);
843 if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
844 if (ctk->size == 8) {
845 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
846 } else if (ctk->size == 4) {
847 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
848 } else {
849 idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
850 lj_ir_kintp(J, sizeof(GCcdata)));
851 idx = emitir(IRT(IR_XLOAD, t), idx, 0);
852 }
853 if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
854 idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
855 if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
856 idx = emitconv(idx, IRT_INTP, t, 0);
857 lj_needsplit(J);
858 }
859 goto integer_key;
860 }
861 } else if (tref_isstr(idx)) {
862 GCstr *name = strV(&rd->argv[1]);
863 if (cd && cd->ctypeid == CTID_CTYPEID)
864 ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
865 if (ctype_isstruct(ct->info)) {
866 CTSize fofs;
867 CType *fct;
868 fct = lj_ctype_getfield(cts, ct, name, &fofs);
869 if (fct) {
870 ofs += (ptrdiff_t)fofs;
871 /* Always specialize to the field name. */
872 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
873 if (ctype_isconstval(fct->info)) {
874 if (fct->size >= 0x80000000u &&
875 (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
876 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
877 return;
878 }
879 J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
880 return; /* Interpreter will throw for newindex. */
881 } else if (ctype_isbitfield(fct->info)) {
882 if (ofs)
883 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
884 crec_index_bf(J, rd, ptr, fct->info);
885 return;
886 } else {
887 lua_assert(ctype_isfield(fct->info));
888 sid = ctype_cid(fct->info);
889 }
890 }
891 } else if (ctype_iscomplex(ct->info)) {
892 if (name->len == 2 &&
893 ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
894 (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
895 /* Always specialize to the field name. */
896 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
897 if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
898 sid = ctype_cid(ct->info);
899 }
900 }
901 }
902 if (!sid) {
903 if (ctype_isptr(ct->info)) { /* Automatically perform '->'. */
904 CType *cct = ctype_rawchild(cts, ct);
905 if (ctype_isstruct(cct->info)) {
906 ct = cct;
907 cd = NULL;
908 if (tref_isstr(idx)) goto again;
909 }
910 }
911 crec_index_meta(J, cts, ct, rd);
912 return;
913 }
914
915 if (ofs)
916 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
917
918 /* Resolve reference for field. */
919 ct = ctype_get(cts, sid);
920 if (ctype_isref(ct->info)) {
921 ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
922 sid = ctype_cid(ct->info);
923 ct = ctype_get(cts, sid);
924 }
925
926 while (ctype_isattrib(ct->info))
927 ct = ctype_child(cts, ct); /* Skip attributes. */
928
929 if (rd->data == 0) { /* __index metamethod. */
930 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
931 } else { /* __newindex metamethod. */
932 rd->nres = 0;
933 J->needsnap = 1;
934 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
935 }
936 }
937
938 /* Record setting a finalizer. */
crec_finalizer(jit_State * J,TRef trcd,TRef trfin,cTValue * fin)939 static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
940 {
941 if (tvisgcv(fin)) {
942 if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
943 } else if (tvisnil(fin)) {
944 trfin = lj_ir_kptr(J, NULL);
945 } else {
946 lj_trace_err(J, LJ_TRERR_BADTYPE);
947 }
948 lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
949 trfin, lj_ir_kint(J, (int32_t)itype(fin)));
950 J->needsnap = 1;
951 }
952
953 /* Record cdata allocation. */
crec_alloc(jit_State * J,RecordFFData * rd,CTypeID id)954 static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
955 {
956 CTState *cts = ctype_ctsG(J2G(J));
957 CTSize sz;
958 CTInfo info = lj_ctype_info(cts, id, &sz);
959 CType *d = ctype_raw(cts, id);
960 TRef trcd, trid = lj_ir_kint(J, id);
961 cTValue *fin;
962 /* Use special instruction to box pointer or 32/64 bit integer. */
963 if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
964 TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
965 ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
966 sz == 4 ? lj_ir_kint(J, 0) :
967 (lj_needsplit(J), lj_ir_kint64(J, 0));
968 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
969 return;
970 } else {
971 TRef trsz = TREF_NIL;
972 if ((info & CTF_VLA)) { /* Calculate VLA/VLS size at runtime. */
973 CTSize sz0, sz1;
974 if (!J->base[1] || J->base[2])
975 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init VLA/VLS. */
976 trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
977 J->base[1], &rd->argv[1]);
978 sz0 = lj_ctype_vlsize(cts, d, 0);
979 sz1 = lj_ctype_vlsize(cts, d, 1);
980 trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
981 trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
982 J->base[1] = 0; /* Simplify logic below. */
983 } else if (ctype_align(info) > CT_MEMALIGN) {
984 trsz = lj_ir_kint(J, sz);
985 }
986 trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
987 if (sz > 128 || (info & CTF_VLA)) {
988 TRef dp;
989 CTSize align;
990 special: /* Only handle bulk zero-fill for large/VLA/VLS types. */
991 if (J->base[1])
992 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init large/VLA/VLS types. */
993 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
994 if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
995 align = ctype_align(info);
996 if (align < CT_MEMALIGN) align = CT_MEMALIGN;
997 crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
998 } else if (J->base[1] && !J->base[2] &&
999 !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
1000 goto single_init;
1001 } else if (ctype_isarray(d->info)) {
1002 CType *dc = ctype_rawchild(cts, d); /* Array element type. */
1003 CTSize ofs, esize = dc->size;
1004 TRef sp = 0;
1005 TValue tv;
1006 TValue *sval = &tv;
1007 MSize i;
1008 tv.u64 = 0;
1009 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
1010 esize * CREC_FILL_MAXUNROLL < sz)
1011 goto special;
1012 for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
1013 TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1014 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
1015 if (J->base[i]) {
1016 sp = J->base[i];
1017 sval = &rd->argv[i];
1018 i++;
1019 } else if (i != 2) {
1020 sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
1021 }
1022 crec_ct_tv(J, dc, dp, sp, sval);
1023 }
1024 } else if (ctype_isstruct(d->info)) {
1025 CTypeID fid = d->sib;
1026 MSize i = 1;
1027 while (fid) {
1028 CType *df = ctype_get(cts, fid);
1029 fid = df->sib;
1030 if (ctype_isfield(df->info)) {
1031 CType *dc;
1032 TRef sp, dp;
1033 TValue tv;
1034 TValue *sval = &tv;
1035 setintV(&tv, 0);
1036 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
1037 dc = ctype_rawchild(cts, df); /* Field type. */
1038 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1039 ctype_isenum(dc->info)))
1040 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init aggregates. */
1041 if (J->base[i]) {
1042 sp = J->base[i];
1043 sval = &rd->argv[i];
1044 i++;
1045 } else {
1046 sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
1047 }
1048 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1049 lj_ir_kintp(J, df->size + sizeof(GCcdata)));
1050 crec_ct_tv(J, dc, dp, sp, sval);
1051 } else if (!ctype_isconstval(df->info)) {
1052 /* NYI: init bitfields and sub-structures. */
1053 lj_trace_err(J, LJ_TRERR_NYICONV);
1054 }
1055 }
1056 } else {
1057 TRef dp;
1058 single_init:
1059 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
1060 if (J->base[1]) {
1061 crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
1062 } else {
1063 TValue tv;
1064 tv.u64 = 0;
1065 crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
1066 }
1067 }
1068 }
1069 J->base[0] = trcd;
1070 /* Handle __gc metamethod. */
1071 fin = lj_ctype_meta(cts, id, MM_gc);
1072 if (fin)
1073 crec_finalizer(J, trcd, 0, fin);
1074 }
1075
1076 /* Record argument conversions. */
crec_call_args(jit_State * J,RecordFFData * rd,CTState * cts,CType * ct)1077 static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1078 CTState *cts, CType *ct)
1079 {
1080 TRef args[CCI_NARGS_MAX];
1081 CTypeID fid;
1082 MSize i, n;
1083 TRef tr, *base;
1084 cTValue *o;
1085 #if LJ_TARGET_X86
1086 #if LJ_ABI_WIN
1087 TRef *arg0 = NULL, *arg1 = NULL;
1088 #endif
1089 int ngpr = 0;
1090 if (ctype_cconv(ct->info) == CTCC_THISCALL)
1091 ngpr = 1;
1092 else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
1093 ngpr = 2;
1094 #endif
1095
1096 /* Skip initial attributes. */
1097 fid = ct->sib;
1098 while (fid) {
1099 CType *ctf = ctype_get(cts, fid);
1100 if (!ctype_isattrib(ctf->info)) break;
1101 fid = ctf->sib;
1102 }
1103 args[0] = TREF_NIL;
1104 for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1105 CTypeID did;
1106 CType *d;
1107
1108 if (n >= CCI_NARGS_MAX)
1109 lj_trace_err(J, LJ_TRERR_NYICALL);
1110
1111 if (fid) { /* Get argument type from field. */
1112 CType *ctf = ctype_get(cts, fid);
1113 fid = ctf->sib;
1114 lua_assert(ctype_isfield(ctf->info));
1115 did = ctype_cid(ctf->info);
1116 } else {
1117 if (!(ct->info & CTF_VARARG))
1118 lj_trace_err(J, LJ_TRERR_NYICALL); /* Too many arguments. */
1119 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
1120 }
1121 d = ctype_raw(cts, did);
1122 if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1123 ctype_isenum(d->info)))
1124 lj_trace_err(J, LJ_TRERR_NYICALL);
1125 tr = crec_ct_tv(J, d, 0, *base, o);
1126 if (ctype_isinteger_or_bool(d->info)) {
1127 if (d->size < 4) {
1128 if ((d->info & CTF_UNSIGNED))
1129 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1130 else
1131 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1132 }
1133 } else if (LJ_SOFTFP && ctype_isfp(d->info) && d->size > 4) {
1134 lj_needsplit(J);
1135 }
1136 #if LJ_TARGET_X86
1137 /* 64 bit args must not end up in registers for fastcall/thiscall. */
1138 #if LJ_ABI_WIN
1139 if (!ctype_isfp(d->info)) {
1140 /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1141 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1142 if (ngpr) {
1143 arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1144 if (ngpr) {
1145 arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1146 }
1147 }
1148 } else {
1149 if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1150 if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1151 if (ngpr) ngpr--;
1152 }
1153 }
1154 #else
1155 if (!ctype_isfp(d->info) && ngpr) {
1156 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1157 /* No reordering for other x86 ABIs. Simply add alignment args. */
1158 do { args[n++] = TREF_NIL; } while (--ngpr);
1159 } else {
1160 ngpr--;
1161 }
1162 }
1163 #endif
1164 #endif
1165 args[n] = tr;
1166 }
1167 tr = args[0];
1168 for (i = 1; i < n; i++)
1169 tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1170 return tr;
1171 }
1172
1173 /* Create a snapshot for the caller, simulating a 'false' return value. */
crec_snap_caller(jit_State * J)1174 static void crec_snap_caller(jit_State *J)
1175 {
1176 lua_State *L = J->L;
1177 TValue *base = L->base, *top = L->top;
1178 const BCIns *pc = J->pc;
1179 TRef ftr = J->base[-1-LJ_FR2];
1180 ptrdiff_t delta;
1181 if (!frame_islua(base-1) || J->framedepth <= 0)
1182 lj_trace_err(J, LJ_TRERR_NYICALL);
1183 J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
1184 L->top = base; L->base = base - delta;
1185 J->base[-1-LJ_FR2] = TREF_FALSE;
1186 J->base -= delta; J->baseslot -= (BCReg)delta;
1187 J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
1188 lj_snap_add(J);
1189 L->base = base; L->top = top;
1190 J->framedepth++; J->maxslot = 1;
1191 J->base += delta; J->baseslot += (BCReg)delta;
1192 J->base[-1-LJ_FR2] = ftr; J->pc = pc;
1193 }
1194
1195 /* Record function call. */
crec_call(jit_State * J,RecordFFData * rd,GCcdata * cd)1196 static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1197 {
1198 CTState *cts = ctype_ctsG(J2G(J));
1199 CType *ct = ctype_raw(cts, cd->ctypeid);
1200 IRType tp = IRT_PTR;
1201 if (ctype_isptr(ct->info)) {
1202 tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1203 ct = ctype_rawchild(cts, ct);
1204 }
1205 if (ctype_isfunc(ct->info)) {
1206 TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1207 CType *ctr = ctype_rawchild(cts, ct);
1208 IRType t = crec_ct2irt(cts, ctr);
1209 TRef tr;
1210 TValue tv;
1211 /* Check for blacklisted C functions that might call a callback. */
1212 setlightudV(&tv,
1213 cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
1214 if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1215 lj_trace_err(J, LJ_TRERR_BLACKL);
1216 if (ctype_isvoid(ctr->info)) {
1217 t = IRT_NIL;
1218 rd->nres = 0;
1219 } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
1220 ctype_isenum(ctr->info)) || t == IRT_CDATA) {
1221 lj_trace_err(J, LJ_TRERR_NYICALL);
1222 }
1223 if ((ct->info & CTF_VARARG)
1224 #if LJ_TARGET_X86
1225 || ctype_cconv(ct->info) != CTCC_CDECL
1226 #endif
1227 )
1228 func = emitir(IRT(IR_CARG, IRT_NIL), func,
1229 lj_ir_kint(J, ctype_typeid(cts, ct)));
1230 tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1231 if (ctype_isbool(ctr->info)) {
1232 if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1233 /* Don't check result if ignored. */
1234 tr = TREF_NIL;
1235 } else {
1236 crec_snap_caller(J);
1237 #if LJ_TARGET_X86ORX64
1238 /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1239 lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1240 #else
1241 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1242 #endif
1243 J->postproc = LJ_POST_FIXGUARDSNAP;
1244 tr = TREF_TRUE;
1245 }
1246 } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1247 t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
1248 TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
1249 tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1250 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1251 } else if (t == IRT_FLOAT || t == IRT_U32) {
1252 tr = emitconv(tr, IRT_NUM, t, 0);
1253 } else if (t == IRT_I8 || t == IRT_I16) {
1254 tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1255 } else if (t == IRT_U8 || t == IRT_U16) {
1256 tr = emitconv(tr, IRT_INT, t, 0);
1257 }
1258 J->base[0] = tr;
1259 J->needsnap = 1;
1260 return 1;
1261 }
1262 return 0;
1263 }
1264
recff_cdata_call(jit_State * J,RecordFFData * rd)1265 void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1266 {
1267 CTState *cts = ctype_ctsG(J2G(J));
1268 GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1269 CTypeID id = cd->ctypeid;
1270 CType *ct;
1271 cTValue *tv;
1272 MMS mm = MM_call;
1273 if (id == CTID_CTYPEID) {
1274 id = crec_constructor(J, cd, J->base[0]);
1275 mm = MM_new;
1276 } else if (crec_call(J, rd, cd)) {
1277 return;
1278 }
1279 /* Record ctype __call/__new metamethod. */
1280 ct = ctype_raw(cts, id);
1281 tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1282 if (tv) {
1283 if (tvisfunc(tv)) {
1284 crec_tailcall(J, rd, tv);
1285 return;
1286 }
1287 } else if (mm == MM_new) {
1288 crec_alloc(J, rd, id);
1289 return;
1290 }
1291 /* No metamethod or NYI: non-function metamethods. */
1292 lj_trace_err(J, LJ_TRERR_BADTYPE);
1293 }
1294
crec_arith_int64(jit_State * J,TRef * sp,CType ** s,MMS mm)1295 static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1296 {
1297 if (sp[0] && sp[1] && ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1298 IRType dt;
1299 CTypeID id;
1300 TRef tr;
1301 MSize i;
1302 IROp op;
1303 lj_needsplit(J);
1304 if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1305 ((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1306 dt = IRT_U64; id = CTID_UINT64;
1307 } else {
1308 dt = IRT_I64; id = CTID_INT64;
1309 if (mm < MM_add &&
1310 !((s[0]->info | s[1]->info) & CTF_FP) &&
1311 s[0]->size == 4 && s[1]->size == 4) { /* Try to narrow comparison. */
1312 if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1313 (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1314 dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1315 goto comp;
1316 } else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1317 dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1318 goto comp;
1319 }
1320 }
1321 }
1322 for (i = 0; i < 2; i++) {
1323 IRType st = tref_type(sp[i]);
1324 if (st == IRT_NUM || st == IRT_FLOAT)
1325 sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
1326 else if (!(st == IRT_I64 || st == IRT_U64))
1327 sp[i] = emitconv(sp[i], dt, IRT_INT,
1328 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1329 }
1330 if (mm < MM_add) {
1331 comp:
1332 /* Assume true comparison. Fixup and emit pending guard later. */
1333 if (mm == MM_eq) {
1334 op = IR_EQ;
1335 } else {
1336 op = mm == MM_lt ? IR_LT : IR_LE;
1337 if (dt == IRT_U32 || dt == IRT_U64)
1338 op += (IR_ULT-IR_LT);
1339 }
1340 lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1341 J->postproc = LJ_POST_FIXGUARD;
1342 return TREF_TRUE;
1343 } else {
1344 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1345 }
1346 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1347 }
1348 return 0;
1349 }
1350
crec_arith_ptr(jit_State * J,TRef * sp,CType ** s,MMS mm)1351 static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1352 {
1353 CTState *cts = ctype_ctsG(J2G(J));
1354 CType *ctp = s[0];
1355 if (!(sp[0] && sp[1])) return 0;
1356 if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1357 if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1358 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1359 if (mm == MM_sub) { /* Pointer difference. */
1360 TRef tr;
1361 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1362 if (sz == 0 || (sz & (sz-1)) != 0)
1363 return 0; /* NYI: integer division. */
1364 tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1365 tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1366 #if LJ_64
1367 tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1368 #endif
1369 return tr;
1370 } else { /* Pointer comparison (unsigned). */
1371 /* Assume true comparison. Fixup and emit pending guard later. */
1372 IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1373 lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1374 J->postproc = LJ_POST_FIXGUARD;
1375 return TREF_TRUE;
1376 }
1377 }
1378 if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1379 return 0;
1380 } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1381 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1382 TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr; /* Swap pointer and index. */
1383 ctp = s[1];
1384 } else {
1385 return 0;
1386 }
1387 {
1388 TRef tr = sp[1];
1389 IRType t = tref_type(tr);
1390 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1391 CTypeID id;
1392 #if LJ_64
1393 if (t == IRT_NUM || t == IRT_FLOAT)
1394 tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
1395 else if (!(t == IRT_I64 || t == IRT_U64))
1396 tr = emitconv(tr, IRT_INTP, IRT_INT,
1397 ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1398 #else
1399 if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1400 tr = emitconv(tr, IRT_INTP, t,
1401 (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
1402 }
1403 #endif
1404 tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1405 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1406 id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1407 CTSIZE_PTR);
1408 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1409 }
1410 }
1411
1412 /* Record ctype arithmetic metamethods. */
crec_arith_meta(jit_State * J,TRef * sp,CType ** s,CTState * cts,RecordFFData * rd)1413 static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1414 RecordFFData *rd)
1415 {
1416 cTValue *tv = NULL;
1417 if (J->base[0]) {
1418 if (tviscdata(&rd->argv[0])) {
1419 CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1420 CType *ct = ctype_raw(cts, id);
1421 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1422 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1423 }
1424 if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1425 CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1426 CType *ct = ctype_raw(cts, id);
1427 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1428 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1429 }
1430 }
1431 if (tv) {
1432 if (tvisfunc(tv)) {
1433 crec_tailcall(J, rd, tv);
1434 return 0;
1435 } /* NYI: non-function metamethods. */
1436 } else if ((MMS)rd->data == MM_eq) { /* Fallback cdata pointer comparison. */
1437 if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
1438 /* Assume true comparison. Fixup and emit pending guard later. */
1439 lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
1440 J->postproc = LJ_POST_FIXGUARD;
1441 return TREF_TRUE;
1442 } else {
1443 return TREF_FALSE;
1444 }
1445 }
1446 lj_trace_err(J, LJ_TRERR_BADTYPE);
1447 return 0;
1448 }
1449
recff_cdata_arith(jit_State * J,RecordFFData * rd)1450 void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1451 {
1452 CTState *cts = ctype_ctsG(J2G(J));
1453 TRef sp[2];
1454 CType *s[2];
1455 MSize i;
1456 for (i = 0; i < 2; i++) {
1457 TRef tr = J->base[i];
1458 CType *ct = ctype_get(cts, CTID_DOUBLE);
1459 if (!tr) {
1460 lj_trace_err(J, LJ_TRERR_BADTYPE);
1461 } else if (tref_iscdata(tr)) {
1462 CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1463 IRType t;
1464 ct = ctype_raw(cts, id);
1465 t = crec_ct2irt(cts, ct);
1466 if (ctype_isptr(ct->info)) { /* Resolve pointer or reference. */
1467 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1468 if (ctype_isref(ct->info)) {
1469 ct = ctype_rawchild(cts, ct);
1470 t = crec_ct2irt(cts, ct);
1471 }
1472 } else if (t == IRT_I64 || t == IRT_U64) {
1473 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1474 lj_needsplit(J);
1475 goto ok;
1476 } else if (t == IRT_INT || t == IRT_U32) {
1477 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1478 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1479 goto ok;
1480 } else if (ctype_isfunc(ct->info)) {
1481 tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1482 ct = ctype_get(cts,
1483 lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1484 goto ok;
1485 } else {
1486 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1487 }
1488 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1489 if (ctype_isnum(ct->info)) {
1490 if (t == IRT_CDATA) {
1491 tr = 0;
1492 } else {
1493 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1494 tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1495 }
1496 }
1497 } else if (tref_isnil(tr)) {
1498 tr = lj_ir_kptr(J, NULL);
1499 ct = ctype_get(cts, CTID_P_VOID);
1500 } else if (tref_isinteger(tr)) {
1501 ct = ctype_get(cts, CTID_INT32);
1502 } else if (tref_isstr(tr)) {
1503 TRef tr2 = J->base[1-i];
1504 CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1505 ct = ctype_raw(cts, id);
1506 if (ctype_isenum(ct->info)) { /* Match string against enum constant. */
1507 GCstr *str = strV(&rd->argv[i]);
1508 CTSize ofs;
1509 CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1510 if (cct && ctype_isconstval(cct->info)) {
1511 /* Specialize to the name of the enum constant. */
1512 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1513 ct = ctype_child(cts, cct);
1514 tr = lj_ir_kint(J, (int32_t)ofs);
1515 } else { /* Interpreter will throw or return false. */
1516 ct = ctype_get(cts, CTID_P_VOID);
1517 }
1518 } else if (ctype_isptr(ct->info)) {
1519 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
1520 } else {
1521 ct = ctype_get(cts, CTID_P_VOID);
1522 }
1523 } else if (!tref_isnum(tr)) {
1524 tr = 0;
1525 ct = ctype_get(cts, CTID_P_VOID);
1526 }
1527 ok:
1528 s[i] = ct;
1529 sp[i] = tr;
1530 }
1531 {
1532 TRef tr;
1533 if (!(tr = crec_arith_int64(J, sp, s, (MMS)rd->data)) &&
1534 !(tr = crec_arith_ptr(J, sp, s, (MMS)rd->data)) &&
1535 !(tr = crec_arith_meta(J, sp, s, cts, rd)))
1536 return;
1537 J->base[0] = tr;
1538 /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1539 if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1540 !irt_isguard(J->guardemit)) {
1541 const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1542 if (bc_op(*pc) <= BC_ISNEP) {
1543 J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
1544 J->postproc = LJ_POST_FIXCOMP;
1545 }
1546 }
1547 }
1548 }
1549
1550 /* -- C library namespace metamethods ------------------------------------- */
1551
recff_clib_index(jit_State * J,RecordFFData * rd)1552 void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1553 {
1554 CTState *cts = ctype_ctsG(J2G(J));
1555 if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1556 udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1557 CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1558 GCstr *name = strV(&rd->argv[1]);
1559 CType *ct;
1560 CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1561 cTValue *tv = lj_tab_getstr(cl->cache, name);
1562 rd->nres = rd->data;
1563 if (id && tv && !tvisnil(tv)) {
1564 /* Specialize to the symbol name and make the result a constant. */
1565 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1566 if (ctype_isconstval(ct->info)) {
1567 if (ct->size >= 0x80000000u &&
1568 (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1569 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1570 else
1571 J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1572 } else if (ctype_isextern(ct->info)) {
1573 CTypeID sid = ctype_cid(ct->info);
1574 void *sp = *(void **)cdataptr(cdataV(tv));
1575 TRef ptr;
1576 ct = ctype_raw(cts, sid);
1577 if (LJ_64 && !checkptr32(sp))
1578 ptr = lj_ir_kintp(J, (uintptr_t)sp);
1579 else
1580 ptr = lj_ir_kptr(J, sp);
1581 if (rd->data) {
1582 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1583 } else {
1584 J->needsnap = 1;
1585 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1586 }
1587 } else {
1588 J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1589 }
1590 } else {
1591 lj_trace_err(J, LJ_TRERR_NOCACHE);
1592 }
1593 } /* else: interpreter will throw. */
1594 }
1595
1596 /* -- FFI library functions ----------------------------------------------- */
1597
crec_toint(jit_State * J,CTState * cts,TRef sp,TValue * sval)1598 static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1599 {
1600 return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1601 }
1602
recff_ffi_new(jit_State * J,RecordFFData * rd)1603 void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1604 {
1605 crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1606 }
1607
recff_ffi_errno(jit_State * J,RecordFFData * rd)1608 void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1609 {
1610 UNUSED(rd);
1611 if (J->base[0])
1612 lj_trace_err(J, LJ_TRERR_NYICALL);
1613 J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1614 }
1615
recff_ffi_string(jit_State * J,RecordFFData * rd)1616 void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1617 {
1618 CTState *cts = ctype_ctsG(J2G(J));
1619 TRef tr = J->base[0];
1620 if (tr) {
1621 TRef trlen = J->base[1];
1622 if (!tref_isnil(trlen)) {
1623 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1624 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1625 } else {
1626 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1627 trlen = lj_ir_call(J, IRCALL_strlen, tr);
1628 }
1629 J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1630 } /* else: interpreter will throw. */
1631 }
1632
recff_ffi_copy(jit_State * J,RecordFFData * rd)1633 void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1634 {
1635 CTState *cts = ctype_ctsG(J2G(J));
1636 TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1637 if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1638 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1639 trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1640 if (trlen) {
1641 trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1642 } else {
1643 trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1644 trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1645 }
1646 rd->nres = 0;
1647 crec_copy(J, trdst, trsrc, trlen, NULL);
1648 } /* else: interpreter will throw. */
1649 }
1650
recff_ffi_fill(jit_State * J,RecordFFData * rd)1651 void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1652 {
1653 CTState *cts = ctype_ctsG(J2G(J));
1654 TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1655 if (trdst && trlen) {
1656 CTSize step = 1;
1657 if (tviscdata(&rd->argv[0])) { /* Get alignment of original destination. */
1658 CTSize sz;
1659 CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1660 if (ctype_isptr(ct->info))
1661 ct = ctype_rawchild(cts, ct);
1662 step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1663 }
1664 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1665 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1666 if (trfill)
1667 trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1668 else
1669 trfill = lj_ir_kint(J, 0);
1670 rd->nres = 0;
1671 crec_fill(J, trdst, trlen, trfill, step);
1672 } /* else: interpreter will throw. */
1673 }
1674
recff_ffi_typeof(jit_State * J,RecordFFData * rd)1675 void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1676 {
1677 if (tref_iscdata(J->base[0])) {
1678 TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1679 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1680 lj_ir_kint(J, CTID_CTYPEID), trid);
1681 } else {
1682 setfuncV(J->L, &J->errinfo, J->fn);
1683 lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1684 }
1685 }
1686
recff_ffi_istype(jit_State * J,RecordFFData * rd)1687 void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1688 {
1689 argv2ctype(J, J->base[0], &rd->argv[0]);
1690 if (tref_iscdata(J->base[1])) {
1691 argv2ctype(J, J->base[1], &rd->argv[1]);
1692 J->postproc = LJ_POST_FIXBOOL;
1693 J->base[0] = TREF_TRUE;
1694 } else {
1695 J->base[0] = TREF_FALSE;
1696 }
1697 }
1698
recff_ffi_abi(jit_State * J,RecordFFData * rd)1699 void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1700 {
1701 if (tref_isstr(J->base[0])) {
1702 /* Specialize to the ABI string to make the boolean result a constant. */
1703 emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1704 J->postproc = LJ_POST_FIXBOOL;
1705 J->base[0] = TREF_TRUE;
1706 } else {
1707 lj_trace_err(J, LJ_TRERR_BADTYPE);
1708 }
1709 }
1710
1711 /* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
recff_ffi_xof(jit_State * J,RecordFFData * rd)1712 void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1713 {
1714 CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1715 if (rd->data == FF_ffi_sizeof) {
1716 CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1717 if (ctype_isvltype(ct->info))
1718 lj_trace_err(J, LJ_TRERR_BADTYPE);
1719 } else if (rd->data == FF_ffi_offsetof) { /* Specialize to the field name. */
1720 if (!tref_isstr(J->base[1]))
1721 lj_trace_err(J, LJ_TRERR_BADTYPE);
1722 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1723 rd->nres = 3; /* Just in case. */
1724 }
1725 J->postproc = LJ_POST_FIXCONST;
1726 J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1727 }
1728
recff_ffi_gc(jit_State * J,RecordFFData * rd)1729 void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1730 {
1731 argv2cdata(J, J->base[0], &rd->argv[0]);
1732 if (!J->base[1])
1733 lj_trace_err(J, LJ_TRERR_BADTYPE);
1734 crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
1735 }
1736
1737 /* -- 64 bit bit.* library functions -------------------------------------- */
1738
1739 /* Determine bit operation type from argument type. */
crec_bit64_type(CTState * cts,cTValue * tv)1740 static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
1741 {
1742 if (tviscdata(tv)) {
1743 CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
1744 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1745 if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
1746 CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
1747 return CTID_UINT64; /* Use uint64_t, since it has the highest rank. */
1748 return CTID_INT64; /* Otherwise use int64_t. */
1749 }
1750 return 0; /* Use regular 32 bit ops. */
1751 }
1752
recff_bit64_tobit(jit_State * J,RecordFFData * rd)1753 void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
1754 {
1755 CTState *cts = ctype_ctsG(J2G(J));
1756 TRef tr = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1757 J->base[0], &rd->argv[0]);
1758 if (!tref_isinteger(tr))
1759 tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
1760 J->base[0] = tr;
1761 }
1762
recff_bit64_unary(jit_State * J,RecordFFData * rd)1763 int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
1764 {
1765 CTState *cts = ctype_ctsG(J2G(J));
1766 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1767 if (id) {
1768 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1769 tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
1770 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1771 return 1;
1772 }
1773 return 0;
1774 }
1775
recff_bit64_nary(jit_State * J,RecordFFData * rd)1776 int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
1777 {
1778 CTState *cts = ctype_ctsG(J2G(J));
1779 CTypeID id = 0;
1780 MSize i;
1781 for (i = 0; J->base[i] != 0; i++) {
1782 CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
1783 if (id < aid) id = aid; /* Determine highest type rank of all arguments. */
1784 }
1785 if (id) {
1786 CType *ct = ctype_get(cts, id);
1787 uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
1788 TRef tr = crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
1789 for (i = 1; J->base[i] != 0; i++) {
1790 TRef tr2 = crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
1791 tr = emitir(ot, tr, tr2);
1792 }
1793 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1794 return 1;
1795 }
1796 return 0;
1797 }
1798
recff_bit64_shift(jit_State * J,RecordFFData * rd)1799 int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
1800 {
1801 CTState *cts = ctype_ctsG(J2G(J));
1802 CTypeID id;
1803 TRef tsh = 0;
1804 if (J->base[0] && tref_iscdata(J->base[1])) {
1805 tsh = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1806 J->base[1], &rd->argv[1]);
1807 if (!tref_isinteger(tsh))
1808 tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
1809 J->base[1] = tsh;
1810 }
1811 id = crec_bit64_type(cts, &rd->argv[0]);
1812 if (id) {
1813 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1814 uint32_t op = rd->data;
1815 if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
1816 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
1817 !tref_isk(tsh))
1818 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
1819 #ifdef LJ_TARGET_UNIFYROT
1820 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
1821 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
1822 tsh = emitir(IRTI(IR_NEG), tsh, tsh);
1823 }
1824 #endif
1825 tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
1826 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1827 return 1;
1828 }
1829 return 0;
1830 }
1831
recff_bit64_tohex(jit_State * J,RecordFFData * rd,TRef hdr)1832 TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
1833 {
1834 CTState *cts = ctype_ctsG(J2G(J));
1835 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1836 TRef tr, trsf = J->base[1];
1837 SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
1838 int32_t n;
1839 if (trsf) {
1840 CTypeID id2 = 0;
1841 n = (int32_t)lj_carith_check64(J->L, 2, &id2);
1842 if (id2)
1843 trsf = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
1844 else
1845 trsf = lj_opt_narrow_tobit(J, trsf);
1846 emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n)); /* Specialize to n. */
1847 } else {
1848 n = id ? 16 : 8;
1849 }
1850 if (n < 0) { n = -n; sf |= STRFMT_F_UPPER; }
1851 sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
1852 if (id) {
1853 tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1854 if (n < 16)
1855 tr = emitir(IRT(IR_BAND, IRT_U64), tr,
1856 lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
1857 } else {
1858 tr = lj_opt_narrow_tobit(J, J->base[0]);
1859 if (n < 8)
1860 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
1861 tr = emitconv(tr, IRT_U64, IRT_INT, 0); /* No sign-extension. */
1862 lj_needsplit(J);
1863 }
1864 return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
1865 }
1866
1867 /* -- Miscellaneous library functions ------------------------------------- */
1868
lj_crecord_tonumber(jit_State * J,RecordFFData * rd)1869 void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1870 {
1871 CTState *cts = ctype_ctsG(J2G(J));
1872 CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1873 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1874 if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1875 if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1876 !(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1877 d = ctype_get(cts, CTID_INT32);
1878 else
1879 d = ctype_get(cts, CTID_DOUBLE);
1880 J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1881 } else {
1882 J->base[0] = TREF_NIL;
1883 }
1884 }
1885
1886 #undef IR
1887 #undef emitir
1888 #undef emitconv
1889
1890 #endif
1891