1 /*
2 ** Fast function call recorder.
3 ** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #define lj_ffrecord_c
7 #define LUA_CORE
8
9 #include "lj_obj.h"
10
11 #if LJ_HASJIT
12
13 #include "lj_err.h"
14 #include "lj_buf.h"
15 #include "lj_str.h"
16 #include "lj_tab.h"
17 #include "lj_frame.h"
18 #include "lj_bc.h"
19 #include "lj_ff.h"
20 #include "lj_ir.h"
21 #include "lj_jit.h"
22 #include "lj_ircall.h"
23 #include "lj_iropt.h"
24 #include "lj_trace.h"
25 #include "lj_record.h"
26 #include "lj_ffrecord.h"
27 #include "lj_crecord.h"
28 #include "lj_dispatch.h"
29 #include "lj_vm.h"
30 #include "lj_strscan.h"
31 #include "lj_strfmt.h"
32 #include "lj_cdata.h"
33 #include "lj_serialize.h"
34
35 /* Some local macros to save typing. Undef'd at the end. */
36 #define IR(ref) (&J->cur.ir[(ref)])
37
38 /* Pass IR on to next optimization in chain (FOLD). */
39 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
40
41 /* -- Fast function recording handlers ------------------------------------ */
42
43 /* Conventions for fast function call handlers:
44 **
45 ** The argument slots start at J->base[0]. All of them are guaranteed to be
46 ** valid and type-specialized references. J->base[J->maxslot] is set to 0
47 ** as a sentinel. The runtime argument values start at rd->argv[0].
48 **
49 ** In general fast functions should check for presence of all of their
50 ** arguments and for the correct argument types. Some simplifications
51 ** are allowed if the interpreter throws instead. But even if recording
52 ** is aborted, the generated IR must be consistent (no zero-refs).
53 **
54 ** The number of results in rd->nres is set to 1. Handlers that return
55 ** a different number of results need to override it. A negative value
56 ** prevents return processing (e.g. for pending calls).
57 **
58 ** Results need to be stored starting at J->base[0]. Return processing
59 ** moves them to the right slots later.
60 **
61 ** The per-ffid auxiliary data is the value of the 2nd part of the
62 ** LJLIB_REC() annotation. This allows handling similar functionality
63 ** in a common handler.
64 */
65
66 /* Type of handler to record a fast function. */
67 typedef void (LJ_FASTCALL *RecordFunc)(jit_State *J, RecordFFData *rd);
68
69 /* Get runtime value of int argument. */
argv2int(jit_State * J,TValue * o)70 static int32_t argv2int(jit_State *J, TValue *o)
71 {
72 if (!lj_strscan_numberobj(o))
73 lj_trace_err(J, LJ_TRERR_BADTYPE);
74 return tvisint(o) ? intV(o) : lj_num2int(numV(o));
75 }
76
77 /* Get runtime value of string argument. */
argv2str(jit_State * J,TValue * o)78 static GCstr *argv2str(jit_State *J, TValue *o)
79 {
80 if (LJ_LIKELY(tvisstr(o))) {
81 return strV(o);
82 } else {
83 GCstr *s;
84 if (!tvisnumber(o))
85 lj_trace_err(J, LJ_TRERR_BADTYPE);
86 s = lj_strfmt_number(J->L, o);
87 setstrV(J->L, o, s);
88 return s;
89 }
90 }
91
92 /* Return number of results wanted by caller. */
results_wanted(jit_State * J)93 static ptrdiff_t results_wanted(jit_State *J)
94 {
95 TValue *frame = J->L->base-1;
96 if (frame_islua(frame))
97 return (ptrdiff_t)bc_b(frame_pc(frame)[-1]) - 1;
98 else
99 return -1;
100 }
101
102 /* Trace stitching: add continuation below frame to start a new trace. */
recff_stitch(jit_State * J)103 static void recff_stitch(jit_State *J)
104 {
105 ASMFunction cont = lj_cont_stitch;
106 lua_State *L = J->L;
107 TValue *base = L->base;
108 BCReg nslot = J->maxslot + 1 + LJ_FR2;
109 TValue *nframe = base + 1 + LJ_FR2;
110 const BCIns *pc = frame_pc(base-1);
111 TValue *pframe = frame_prevl(base-1);
112
113 /* Check for this now. Throwing in lj_record_stop messes up the stack. */
114 if (J->cur.nsnap >= (MSize)J->param[JIT_P_maxsnap])
115 lj_trace_err(J, LJ_TRERR_SNAPOV);
116
117 /* Move func + args up in Lua stack and insert continuation. */
118 memmove(&base[1], &base[-1-LJ_FR2], sizeof(TValue)*nslot);
119 setframe_ftsz(nframe, ((char *)nframe - (char *)pframe) + FRAME_CONT);
120 setcont(base-LJ_FR2, cont);
121 setframe_pc(base, pc);
122 setnilV(base-1-LJ_FR2); /* Incorrect, but rec_check_slots() won't run anymore. */
123 L->base += 2 + LJ_FR2;
124 L->top += 2 + LJ_FR2;
125
126 /* Ditto for the IR. */
127 memmove(&J->base[1], &J->base[-1-LJ_FR2], sizeof(TRef)*nslot);
128 #if LJ_FR2
129 J->base[2] = TREF_FRAME;
130 J->base[-1] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
131 J->base[0] = lj_ir_k64(J, IR_KNUM, u64ptr(pc)) | TREF_CONT;
132 #else
133 J->base[0] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
134 #endif
135 J->ktrace = tref_ref((J->base[-1-LJ_FR2] = lj_ir_ktrace(J)));
136 J->base += 2 + LJ_FR2;
137 J->baseslot += 2 + LJ_FR2;
138 J->framedepth++;
139
140 lj_record_stop(J, LJ_TRLINK_STITCH, 0);
141
142 /* Undo Lua stack changes. */
143 memmove(&base[-1-LJ_FR2], &base[1], sizeof(TValue)*nslot);
144 setframe_pc(base-1, pc);
145 L->base -= 2 + LJ_FR2;
146 L->top -= 2 + LJ_FR2;
147 }
148
149 /* Fallback handler for fast functions that are not recorded (yet). */
recff_nyi(jit_State * J,RecordFFData * rd)150 static void LJ_FASTCALL recff_nyi(jit_State *J, RecordFFData *rd)
151 {
152 if (J->cur.nins < (IRRef)J->param[JIT_P_minstitch] + REF_BASE) {
153 lj_trace_err_info(J, LJ_TRERR_TRACEUV);
154 } else {
155 /* Can only stitch from Lua call. */
156 if (J->framedepth && frame_islua(J->L->base-1)) {
157 BCOp op = bc_op(*frame_pc(J->L->base-1));
158 /* Stitched trace cannot start with *M op with variable # of args. */
159 if (!(op == BC_CALLM || op == BC_CALLMT ||
160 op == BC_RETM || op == BC_TSETM)) {
161 switch (J->fn->c.ffid) {
162 case FF_error:
163 case FF_debug_sethook:
164 case FF_jit_flush:
165 break; /* Don't stitch across special builtins. */
166 default:
167 recff_stitch(J); /* Use trace stitching. */
168 rd->nres = -1;
169 return;
170 }
171 }
172 }
173 /* Otherwise stop trace and return to interpreter. */
174 lj_record_stop(J, LJ_TRLINK_RETURN, 0);
175 rd->nres = -1;
176 }
177 }
178
179 /* Fallback handler for unsupported variants of fast functions. */
180 #define recff_nyiu recff_nyi
181
182 /* Must stop the trace for classic C functions with arbitrary side-effects. */
183 #define recff_c recff_nyi
184
185 /* Emit BUFHDR for the global temporary buffer. */
recff_bufhdr(jit_State * J)186 static TRef recff_bufhdr(jit_State *J)
187 {
188 return emitir(IRT(IR_BUFHDR, IRT_PGC),
189 lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
190 }
191
192 /* Emit TMPREF. */
recff_tmpref(jit_State * J,TRef tr,int mode)193 static TRef recff_tmpref(jit_State *J, TRef tr, int mode)
194 {
195 if (!LJ_DUALNUM && tref_isinteger(tr))
196 tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
197 return emitir(IRT(IR_TMPREF, IRT_PGC), tr, mode);
198 }
199
200 /* -- Base library fast functions ----------------------------------------- */
201
recff_assert(jit_State * J,RecordFFData * rd)202 static void LJ_FASTCALL recff_assert(jit_State *J, RecordFFData *rd)
203 {
204 /* Arguments already specialized. The interpreter throws for nil/false. */
205 rd->nres = J->maxslot; /* Pass through all arguments. */
206 }
207
recff_type(jit_State * J,RecordFFData * rd)208 static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd)
209 {
210 /* Arguments already specialized. Result is a constant string. Neat, huh? */
211 uint32_t t;
212 if (tvisnumber(&rd->argv[0]))
213 t = ~LJ_TNUMX;
214 else if (LJ_64 && !LJ_GC64 && tvislightud(&rd->argv[0]))
215 t = ~LJ_TLIGHTUD;
216 else
217 t = ~itype(&rd->argv[0]);
218 J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[t]));
219 UNUSED(rd);
220 }
221
recff_getmetatable(jit_State * J,RecordFFData * rd)222 static void LJ_FASTCALL recff_getmetatable(jit_State *J, RecordFFData *rd)
223 {
224 TRef tr = J->base[0];
225 if (tr) {
226 RecordIndex ix;
227 ix.tab = tr;
228 copyTV(J->L, &ix.tabv, &rd->argv[0]);
229 if (lj_record_mm_lookup(J, &ix, MM_metatable))
230 J->base[0] = ix.mobj;
231 else
232 J->base[0] = ix.mt;
233 } /* else: Interpreter will throw. */
234 }
235
recff_setmetatable(jit_State * J,RecordFFData * rd)236 static void LJ_FASTCALL recff_setmetatable(jit_State *J, RecordFFData *rd)
237 {
238 TRef tr = J->base[0];
239 TRef mt = J->base[1];
240 if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) {
241 TRef fref, mtref;
242 RecordIndex ix;
243 ix.tab = tr;
244 copyTV(J->L, &ix.tabv, &rd->argv[0]);
245 lj_record_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable. */
246 fref = emitir(IRT(IR_FREF, IRT_PGC), tr, IRFL_TAB_META);
247 mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt;
248 emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref);
249 if (!tref_isnil(mt))
250 emitir(IRT(IR_TBAR, IRT_TAB), tr, 0);
251 J->base[0] = tr;
252 J->needsnap = 1;
253 } /* else: Interpreter will throw. */
254 }
255
recff_rawget(jit_State * J,RecordFFData * rd)256 static void LJ_FASTCALL recff_rawget(jit_State *J, RecordFFData *rd)
257 {
258 RecordIndex ix;
259 ix.tab = J->base[0]; ix.key = J->base[1];
260 if (tref_istab(ix.tab) && ix.key) {
261 ix.val = 0; ix.idxchain = 0;
262 settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
263 copyTV(J->L, &ix.keyv, &rd->argv[1]);
264 J->base[0] = lj_record_idx(J, &ix);
265 } /* else: Interpreter will throw. */
266 }
267
recff_rawset(jit_State * J,RecordFFData * rd)268 static void LJ_FASTCALL recff_rawset(jit_State *J, RecordFFData *rd)
269 {
270 RecordIndex ix;
271 ix.tab = J->base[0]; ix.key = J->base[1]; ix.val = J->base[2];
272 if (tref_istab(ix.tab) && ix.key && ix.val) {
273 ix.idxchain = 0;
274 settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
275 copyTV(J->L, &ix.keyv, &rd->argv[1]);
276 copyTV(J->L, &ix.valv, &rd->argv[2]);
277 lj_record_idx(J, &ix);
278 /* Pass through table at J->base[0] as result. */
279 } /* else: Interpreter will throw. */
280 }
281
recff_rawequal(jit_State * J,RecordFFData * rd)282 static void LJ_FASTCALL recff_rawequal(jit_State *J, RecordFFData *rd)
283 {
284 TRef tra = J->base[0];
285 TRef trb = J->base[1];
286 if (tra && trb) {
287 int diff = lj_record_objcmp(J, tra, trb, &rd->argv[0], &rd->argv[1]);
288 J->base[0] = diff ? TREF_FALSE : TREF_TRUE;
289 } /* else: Interpreter will throw. */
290 }
291
292 #if LJ_52
recff_rawlen(jit_State * J,RecordFFData * rd)293 static void LJ_FASTCALL recff_rawlen(jit_State *J, RecordFFData *rd)
294 {
295 TRef tr = J->base[0];
296 if (tref_isstr(tr))
297 J->base[0] = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN);
298 else if (tref_istab(tr))
299 J->base[0] = emitir(IRTI(IR_ALEN), tr, TREF_NIL);
300 /* else: Interpreter will throw. */
301 UNUSED(rd);
302 }
303 #endif
304
305 /* Determine mode of select() call. */
lj_ffrecord_select_mode(jit_State * J,TRef tr,TValue * tv)306 int32_t lj_ffrecord_select_mode(jit_State *J, TRef tr, TValue *tv)
307 {
308 if (tref_isstr(tr) && *strVdata(tv) == '#') { /* select('#', ...) */
309 if (strV(tv)->len == 1) {
310 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, strV(tv)));
311 } else {
312 TRef trptr = emitir(IRT(IR_STRREF, IRT_PGC), tr, lj_ir_kint(J, 0));
313 TRef trchar = emitir(IRT(IR_XLOAD, IRT_U8), trptr, IRXLOAD_READONLY);
314 emitir(IRTGI(IR_EQ), trchar, lj_ir_kint(J, '#'));
315 }
316 return 0;
317 } else { /* select(n, ...) */
318 int32_t start = argv2int(J, tv);
319 if (start == 0) lj_trace_err(J, LJ_TRERR_BADTYPE); /* A bit misleading. */
320 return start;
321 }
322 }
323
recff_select(jit_State * J,RecordFFData * rd)324 static void LJ_FASTCALL recff_select(jit_State *J, RecordFFData *rd)
325 {
326 TRef tr = J->base[0];
327 if (tr) {
328 ptrdiff_t start = lj_ffrecord_select_mode(J, tr, &rd->argv[0]);
329 if (start == 0) { /* select('#', ...) */
330 J->base[0] = lj_ir_kint(J, J->maxslot - 1);
331 } else if (tref_isk(tr)) { /* select(k, ...) */
332 ptrdiff_t n = (ptrdiff_t)J->maxslot;
333 if (start < 0) start += n;
334 else if (start > n) start = n;
335 if (start >= 1) {
336 ptrdiff_t i;
337 rd->nres = n - start;
338 for (i = 0; i < n - start; i++)
339 J->base[i] = J->base[start+i];
340 } /* else: Interpreter will throw. */
341 } else {
342 recff_nyiu(J, rd);
343 return;
344 }
345 } /* else: Interpreter will throw. */
346 }
347
recff_tonumber(jit_State * J,RecordFFData * rd)348 static void LJ_FASTCALL recff_tonumber(jit_State *J, RecordFFData *rd)
349 {
350 TRef tr = J->base[0];
351 TRef base = J->base[1];
352 if (tr && !tref_isnil(base)) {
353 base = lj_opt_narrow_toint(J, base);
354 if (!tref_isk(base) || IR(tref_ref(base))->i != 10) {
355 recff_nyiu(J, rd);
356 return;
357 }
358 }
359 if (tref_isnumber_str(tr)) {
360 if (tref_isstr(tr)) {
361 TValue tmp;
362 if (!lj_strscan_num(strV(&rd->argv[0]), &tmp)) {
363 recff_nyiu(J, rd); /* Would need an inverted STRTO for this case. */
364 return;
365 }
366 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
367 }
368 #if LJ_HASFFI
369 } else if (tref_iscdata(tr)) {
370 lj_crecord_tonumber(J, rd);
371 return;
372 #endif
373 } else {
374 tr = TREF_NIL;
375 }
376 J->base[0] = tr;
377 UNUSED(rd);
378 }
379
recff_metacall_cp(lua_State * L,lua_CFunction dummy,void * ud)380 static TValue *recff_metacall_cp(lua_State *L, lua_CFunction dummy, void *ud)
381 {
382 jit_State *J = (jit_State *)ud;
383 lj_record_tailcall(J, 0, 1);
384 UNUSED(L); UNUSED(dummy);
385 return NULL;
386 }
387
recff_metacall(jit_State * J,RecordFFData * rd,MMS mm)388 static int recff_metacall(jit_State *J, RecordFFData *rd, MMS mm)
389 {
390 RecordIndex ix;
391 ix.tab = J->base[0];
392 copyTV(J->L, &ix.tabv, &rd->argv[0]);
393 if (lj_record_mm_lookup(J, &ix, mm)) { /* Has metamethod? */
394 int errcode;
395 TValue argv0;
396 /* Temporarily insert metamethod below object. */
397 J->base[1+LJ_FR2] = J->base[0];
398 J->base[0] = ix.mobj;
399 copyTV(J->L, &argv0, &rd->argv[0]);
400 copyTV(J->L, &rd->argv[1+LJ_FR2], &rd->argv[0]);
401 copyTV(J->L, &rd->argv[0], &ix.mobjv);
402 /* Need to protect lj_record_tailcall because it may throw. */
403 errcode = lj_vm_cpcall(J->L, NULL, J, recff_metacall_cp);
404 /* Always undo Lua stack changes to avoid confusing the interpreter. */
405 copyTV(J->L, &rd->argv[0], &argv0);
406 if (errcode)
407 lj_err_throw(J->L, errcode); /* Propagate errors. */
408 rd->nres = -1; /* Pending call. */
409 return 1; /* Tailcalled to metamethod. */
410 }
411 return 0;
412 }
413
recff_tostring(jit_State * J,RecordFFData * rd)414 static void LJ_FASTCALL recff_tostring(jit_State *J, RecordFFData *rd)
415 {
416 TRef tr = J->base[0];
417 if (tref_isstr(tr)) {
418 /* Ignore __tostring in the string base metatable. */
419 /* Pass on result in J->base[0]. */
420 } else if (tr && !recff_metacall(J, rd, MM_tostring)) {
421 if (tref_isnumber(tr)) {
422 J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr,
423 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
424 } else if (tref_ispri(tr)) {
425 J->base[0] = lj_ir_kstr(J, lj_strfmt_obj(J->L, &rd->argv[0]));
426 } else {
427 recff_nyiu(J, rd);
428 return;
429 }
430 }
431 }
432
recff_ipairs_aux(jit_State * J,RecordFFData * rd)433 static void LJ_FASTCALL recff_ipairs_aux(jit_State *J, RecordFFData *rd)
434 {
435 RecordIndex ix;
436 ix.tab = J->base[0];
437 if (tref_istab(ix.tab)) {
438 if (!tvisnumber(&rd->argv[1])) /* No support for string coercion. */
439 lj_trace_err(J, LJ_TRERR_BADTYPE);
440 setintV(&ix.keyv, numberVint(&rd->argv[1])+1);
441 settabV(J->L, &ix.tabv, tabV(&rd->argv[0]));
442 ix.val = 0; ix.idxchain = 0;
443 ix.key = lj_opt_narrow_toint(J, J->base[1]);
444 J->base[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1));
445 J->base[1] = lj_record_idx(J, &ix);
446 rd->nres = tref_isnil(J->base[1]) ? 0 : 2;
447 } /* else: Interpreter will throw. */
448 }
449
recff_xpairs(jit_State * J,RecordFFData * rd)450 static void LJ_FASTCALL recff_xpairs(jit_State *J, RecordFFData *rd)
451 {
452 TRef tr = J->base[0];
453 if (!((LJ_52 || (LJ_HASFFI && tref_iscdata(tr))) &&
454 recff_metacall(J, rd, MM_pairs + rd->data))) {
455 if (tref_istab(tr)) {
456 J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0]));
457 J->base[1] = tr;
458 J->base[2] = rd->data ? lj_ir_kint(J, 0) : TREF_NIL;
459 rd->nres = 3;
460 } /* else: Interpreter will throw. */
461 }
462 }
463
recff_pcall(jit_State * J,RecordFFData * rd)464 static void LJ_FASTCALL recff_pcall(jit_State *J, RecordFFData *rd)
465 {
466 if (J->maxslot >= 1) {
467 #if LJ_FR2
468 /* Shift function arguments up. */
469 memmove(J->base + 1, J->base, sizeof(TRef) * J->maxslot);
470 #endif
471 lj_record_call(J, 0, J->maxslot - 1);
472 rd->nres = -1; /* Pending call. */
473 J->needsnap = 1; /* Start catching on-trace errors. */
474 } /* else: Interpreter will throw. */
475 }
476
recff_xpcall_cp(lua_State * L,lua_CFunction dummy,void * ud)477 static TValue *recff_xpcall_cp(lua_State *L, lua_CFunction dummy, void *ud)
478 {
479 jit_State *J = (jit_State *)ud;
480 lj_record_call(J, 1, J->maxslot - 2);
481 UNUSED(L); UNUSED(dummy);
482 return NULL;
483 }
484
recff_xpcall(jit_State * J,RecordFFData * rd)485 static void LJ_FASTCALL recff_xpcall(jit_State *J, RecordFFData *rd)
486 {
487 if (J->maxslot >= 2) {
488 TValue argv0, argv1;
489 TRef tmp;
490 int errcode;
491 /* Swap function and traceback. */
492 tmp = J->base[0]; J->base[0] = J->base[1]; J->base[1] = tmp;
493 copyTV(J->L, &argv0, &rd->argv[0]);
494 copyTV(J->L, &argv1, &rd->argv[1]);
495 copyTV(J->L, &rd->argv[0], &argv1);
496 copyTV(J->L, &rd->argv[1], &argv0);
497 #if LJ_FR2
498 /* Shift function arguments up. */
499 memmove(J->base + 2, J->base + 1, sizeof(TRef) * (J->maxslot-1));
500 #endif
501 /* Need to protect lj_record_call because it may throw. */
502 errcode = lj_vm_cpcall(J->L, NULL, J, recff_xpcall_cp);
503 /* Always undo Lua stack swap to avoid confusing the interpreter. */
504 copyTV(J->L, &rd->argv[0], &argv0);
505 copyTV(J->L, &rd->argv[1], &argv1);
506 if (errcode)
507 lj_err_throw(J->L, errcode); /* Propagate errors. */
508 rd->nres = -1; /* Pending call. */
509 J->needsnap = 1; /* Start catching on-trace errors. */
510 } /* else: Interpreter will throw. */
511 }
512
recff_getfenv(jit_State * J,RecordFFData * rd)513 static void LJ_FASTCALL recff_getfenv(jit_State *J, RecordFFData *rd)
514 {
515 TRef tr = J->base[0];
516 /* Only support getfenv(0) for now. */
517 if (tref_isint(tr) && tref_isk(tr) && IR(tref_ref(tr))->i == 0) {
518 TRef trl = emitir(IRT(IR_LREF, IRT_THREAD), 0, 0);
519 J->base[0] = emitir(IRT(IR_FLOAD, IRT_TAB), trl, IRFL_THREAD_ENV);
520 return;
521 }
522 recff_nyiu(J, rd);
523 }
524
recff_next(jit_State * J,RecordFFData * rd)525 static void LJ_FASTCALL recff_next(jit_State *J, RecordFFData *rd)
526 {
527 #if LJ_BE
528 /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
529 ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
530 */
531 recff_nyi(J, rd);
532 #else
533 TRef tab = J->base[0];
534 if (tref_istab(tab)) {
535 RecordIndex ix;
536 cTValue *keyv;
537 ix.tab = tab;
538 if (tref_isnil(J->base[1])) { /* Shortcut for start of traversal. */
539 ix.key = lj_ir_kint(J, 0);
540 keyv = niltvg(J2G(J));
541 } else {
542 TRef tmp = recff_tmpref(J, J->base[1], IRTMPREF_IN1);
543 ix.key = lj_ir_call(J, IRCALL_lj_tab_keyindex, tab, tmp);
544 keyv = &rd->argv[1];
545 }
546 copyTV(J->L, &ix.tabv, &rd->argv[0]);
547 ix.keyv.u32.lo = lj_tab_keyindex(tabV(&ix.tabv), keyv);
548 /* Omit the value, if not used by the caller. */
549 ix.idxchain = (J->framedepth && frame_islua(J->L->base-1) &&
550 bc_b(frame_pc(J->L->base-1)[-1])-1 < 2);
551 ix.mobj = 0; /* We don't need the next index. */
552 rd->nres = lj_record_next(J, &ix);
553 J->base[0] = ix.key;
554 J->base[1] = ix.val;
555 } /* else: Interpreter will throw. */
556 #endif
557 }
558
559 /* -- Math library fast functions ----------------------------------------- */
560
recff_math_abs(jit_State * J,RecordFFData * rd)561 static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd)
562 {
563 TRef tr = lj_ir_tonum(J, J->base[0]);
564 J->base[0] = emitir(IRTN(IR_ABS), tr, lj_ir_ksimd(J, LJ_KSIMD_ABS));
565 UNUSED(rd);
566 }
567
568 /* Record rounding functions math.floor and math.ceil. */
recff_math_round(jit_State * J,RecordFFData * rd)569 static void LJ_FASTCALL recff_math_round(jit_State *J, RecordFFData *rd)
570 {
571 TRef tr = J->base[0];
572 if (!tref_isinteger(tr)) { /* Pass through integers unmodified. */
573 tr = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, tr), rd->data);
574 /* Result is integral (or NaN/Inf), but may not fit an int32_t. */
575 if (LJ_DUALNUM) { /* Try to narrow using a guarded conversion to int. */
576 lua_Number n = lj_vm_foldfpm(numberVnum(&rd->argv[0]), rd->data);
577 if (n == (lua_Number)lj_num2int(n))
578 tr = emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_CHECK);
579 }
580 J->base[0] = tr;
581 }
582 }
583
584 /* Record unary math.* functions, mapped to IR_FPMATH opcode. */
recff_math_unary(jit_State * J,RecordFFData * rd)585 static void LJ_FASTCALL recff_math_unary(jit_State *J, RecordFFData *rd)
586 {
587 J->base[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, J->base[0]), rd->data);
588 }
589
590 /* Record math.log. */
recff_math_log(jit_State * J,RecordFFData * rd)591 static void LJ_FASTCALL recff_math_log(jit_State *J, RecordFFData *rd)
592 {
593 TRef tr = lj_ir_tonum(J, J->base[0]);
594 if (J->base[1]) {
595 #ifdef LUAJIT_NO_LOG2
596 uint32_t fpm = IRFPM_LOG;
597 #else
598 uint32_t fpm = IRFPM_LOG2;
599 #endif
600 TRef trb = lj_ir_tonum(J, J->base[1]);
601 tr = emitir(IRTN(IR_FPMATH), tr, fpm);
602 trb = emitir(IRTN(IR_FPMATH), trb, fpm);
603 trb = emitir(IRTN(IR_DIV), lj_ir_knum_one(J), trb);
604 tr = emitir(IRTN(IR_MUL), tr, trb);
605 } else {
606 tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_LOG);
607 }
608 J->base[0] = tr;
609 UNUSED(rd);
610 }
611
612 /* Record math.atan2. */
recff_math_atan2(jit_State * J,RecordFFData * rd)613 static void LJ_FASTCALL recff_math_atan2(jit_State *J, RecordFFData *rd)
614 {
615 TRef tr = lj_ir_tonum(J, J->base[0]);
616 TRef tr2 = lj_ir_tonum(J, J->base[1]);
617 J->base[0] = lj_ir_call(J, IRCALL_atan2, tr, tr2);
618 UNUSED(rd);
619 }
620
621 /* Record math.ldexp. */
recff_math_ldexp(jit_State * J,RecordFFData * rd)622 static void LJ_FASTCALL recff_math_ldexp(jit_State *J, RecordFFData *rd)
623 {
624 TRef tr = lj_ir_tonum(J, J->base[0]);
625 #if LJ_TARGET_X86ORX64
626 TRef tr2 = lj_ir_tonum(J, J->base[1]);
627 #else
628 TRef tr2 = lj_opt_narrow_toint(J, J->base[1]);
629 #endif
630 J->base[0] = emitir(IRTN(IR_LDEXP), tr, tr2);
631 UNUSED(rd);
632 }
633
recff_math_call(jit_State * J,RecordFFData * rd)634 static void LJ_FASTCALL recff_math_call(jit_State *J, RecordFFData *rd)
635 {
636 TRef tr = lj_ir_tonum(J, J->base[0]);
637 J->base[0] = emitir(IRTN(IR_CALLN), tr, rd->data);
638 }
639
recff_math_pow(jit_State * J,RecordFFData * rd)640 static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd)
641 {
642 J->base[0] = lj_opt_narrow_pow(J, J->base[0], J->base[1],
643 &rd->argv[0], &rd->argv[1]);
644 UNUSED(rd);
645 }
646
recff_math_minmax(jit_State * J,RecordFFData * rd)647 static void LJ_FASTCALL recff_math_minmax(jit_State *J, RecordFFData *rd)
648 {
649 TRef tr = lj_ir_tonumber(J, J->base[0]);
650 uint32_t op = rd->data;
651 BCReg i;
652 for (i = 1; J->base[i] != 0; i++) {
653 TRef tr2 = lj_ir_tonumber(J, J->base[i]);
654 IRType t = IRT_INT;
655 if (!(tref_isinteger(tr) && tref_isinteger(tr2))) {
656 if (tref_isinteger(tr)) tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
657 if (tref_isinteger(tr2)) tr2 = emitir(IRTN(IR_CONV), tr2, IRCONV_NUM_INT);
658 t = IRT_NUM;
659 }
660 tr = emitir(IRT(op, t), tr, tr2);
661 }
662 J->base[0] = tr;
663 }
664
recff_math_random(jit_State * J,RecordFFData * rd)665 static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd)
666 {
667 GCudata *ud = udataV(&J->fn->c.upvalue[0]);
668 TRef tr, one;
669 lj_ir_kgc(J, obj2gco(ud), IRT_UDATA); /* Prevent collection. */
670 tr = lj_ir_call(J, IRCALL_lj_prng_u64d, lj_ir_kptr(J, uddata(ud)));
671 one = lj_ir_knum_one(J);
672 tr = emitir(IRTN(IR_SUB), tr, one);
673 if (J->base[0]) {
674 TRef tr1 = lj_ir_tonum(J, J->base[0]);
675 if (J->base[1]) { /* d = floor(d*(r2-r1+1.0)) + r1 */
676 TRef tr2 = lj_ir_tonum(J, J->base[1]);
677 tr2 = emitir(IRTN(IR_SUB), tr2, tr1);
678 tr2 = emitir(IRTN(IR_ADD), tr2, one);
679 tr = emitir(IRTN(IR_MUL), tr, tr2);
680 tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_FLOOR);
681 tr = emitir(IRTN(IR_ADD), tr, tr1);
682 } else { /* d = floor(d*r1) + 1.0 */
683 tr = emitir(IRTN(IR_MUL), tr, tr1);
684 tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_FLOOR);
685 tr = emitir(IRTN(IR_ADD), tr, one);
686 }
687 }
688 J->base[0] = tr;
689 UNUSED(rd);
690 }
691
692 /* -- Bit library fast functions ------------------------------------------ */
693
694 /* Record bit.tobit. */
recff_bit_tobit(jit_State * J,RecordFFData * rd)695 static void LJ_FASTCALL recff_bit_tobit(jit_State *J, RecordFFData *rd)
696 {
697 TRef tr = J->base[0];
698 #if LJ_HASFFI
699 if (tref_iscdata(tr)) { recff_bit64_tobit(J, rd); return; }
700 #endif
701 J->base[0] = lj_opt_narrow_tobit(J, tr);
702 UNUSED(rd);
703 }
704
705 /* Record unary bit.bnot, bit.bswap. */
recff_bit_unary(jit_State * J,RecordFFData * rd)706 static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd)
707 {
708 #if LJ_HASFFI
709 if (recff_bit64_unary(J, rd))
710 return;
711 #endif
712 J->base[0] = emitir(IRTI(rd->data), lj_opt_narrow_tobit(J, J->base[0]), 0);
713 }
714
715 /* Record N-ary bit.band, bit.bor, bit.bxor. */
recff_bit_nary(jit_State * J,RecordFFData * rd)716 static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd)
717 {
718 #if LJ_HASFFI
719 if (recff_bit64_nary(J, rd))
720 return;
721 #endif
722 {
723 TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
724 uint32_t ot = IRTI(rd->data);
725 BCReg i;
726 for (i = 1; J->base[i] != 0; i++)
727 tr = emitir(ot, tr, lj_opt_narrow_tobit(J, J->base[i]));
728 J->base[0] = tr;
729 }
730 }
731
732 /* Record bit shifts. */
recff_bit_shift(jit_State * J,RecordFFData * rd)733 static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd)
734 {
735 #if LJ_HASFFI
736 if (recff_bit64_shift(J, rd))
737 return;
738 #endif
739 {
740 TRef tr = lj_opt_narrow_tobit(J, J->base[0]);
741 TRef tsh = lj_opt_narrow_tobit(J, J->base[1]);
742 IROp op = (IROp)rd->data;
743 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
744 !tref_isk(tsh))
745 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31));
746 #ifdef LJ_TARGET_UNIFYROT
747 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
748 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
749 tsh = emitir(IRTI(IR_NEG), tsh, tsh);
750 }
751 #endif
752 J->base[0] = emitir(IRTI(op), tr, tsh);
753 }
754 }
755
recff_bit_tohex(jit_State * J,RecordFFData * rd)756 static void LJ_FASTCALL recff_bit_tohex(jit_State *J, RecordFFData *rd)
757 {
758 #if LJ_HASFFI
759 TRef hdr = recff_bufhdr(J);
760 TRef tr = recff_bit64_tohex(J, rd, hdr);
761 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
762 #else
763 recff_nyiu(J, rd); /* Don't bother working around this NYI. */
764 #endif
765 }
766
767 /* -- String library fast functions --------------------------------------- */
768
769 /* Specialize to relative starting position for string. */
recff_string_start(jit_State * J,GCstr * s,int32_t * st,TRef tr,TRef trlen,TRef tr0)770 static TRef recff_string_start(jit_State *J, GCstr *s, int32_t *st, TRef tr,
771 TRef trlen, TRef tr0)
772 {
773 int32_t start = *st;
774 if (start < 0) {
775 emitir(IRTGI(IR_LT), tr, tr0);
776 tr = emitir(IRTI(IR_ADD), trlen, tr);
777 start = start + (int32_t)s->len;
778 emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), tr, tr0);
779 if (start < 0) {
780 tr = tr0;
781 start = 0;
782 }
783 } else if (start == 0) {
784 emitir(IRTGI(IR_EQ), tr, tr0);
785 tr = tr0;
786 } else {
787 tr = emitir(IRTI(IR_ADD), tr, lj_ir_kint(J, -1));
788 emitir(IRTGI(IR_GE), tr, tr0);
789 start--;
790 }
791 *st = start;
792 return tr;
793 }
794
795 /* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */
recff_string_range(jit_State * J,RecordFFData * rd)796 static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd)
797 {
798 TRef trstr = lj_ir_tostr(J, J->base[0]);
799 TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
800 TRef tr0 = lj_ir_kint(J, 0);
801 TRef trstart, trend;
802 GCstr *str = argv2str(J, &rd->argv[0]);
803 int32_t start, end;
804 if (rd->data) { /* string.sub(str, start [,end]) */
805 start = argv2int(J, &rd->argv[1]);
806 trstart = lj_opt_narrow_toint(J, J->base[1]);
807 trend = J->base[2];
808 if (tref_isnil(trend)) {
809 trend = lj_ir_kint(J, -1);
810 end = -1;
811 } else {
812 trend = lj_opt_narrow_toint(J, trend);
813 end = argv2int(J, &rd->argv[2]);
814 }
815 } else { /* string.byte(str, [,start [,end]]) */
816 if (tref_isnil(J->base[1])) {
817 start = 1;
818 trstart = lj_ir_kint(J, 1);
819 } else {
820 start = argv2int(J, &rd->argv[1]);
821 trstart = lj_opt_narrow_toint(J, J->base[1]);
822 }
823 if (J->base[1] && !tref_isnil(J->base[2])) {
824 trend = lj_opt_narrow_toint(J, J->base[2]);
825 end = argv2int(J, &rd->argv[2]);
826 } else {
827 trend = trstart;
828 end = start;
829 }
830 }
831 if (end < 0) {
832 emitir(IRTGI(IR_LT), trend, tr0);
833 trend = emitir(IRTI(IR_ADD), emitir(IRTI(IR_ADD), trlen, trend),
834 lj_ir_kint(J, 1));
835 end = end+(int32_t)str->len+1;
836 } else if ((MSize)end <= str->len) {
837 emitir(IRTGI(IR_ULE), trend, trlen);
838 } else {
839 emitir(IRTGI(IR_UGT), trend, trlen);
840 end = (int32_t)str->len;
841 trend = trlen;
842 }
843 trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
844 if (rd->data) { /* Return string.sub result. */
845 if (end - start >= 0) {
846 /* Also handle empty range here, to avoid extra traces. */
847 TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart);
848 emitir(IRTGI(IR_GE), trslen, tr0);
849 trptr = emitir(IRT(IR_STRREF, IRT_PGC), trstr, trstart);
850 J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen);
851 } else { /* Range underflow: return empty string. */
852 emitir(IRTGI(IR_LT), trend, trstart);
853 J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty);
854 }
855 } else { /* Return string.byte result(s). */
856 ptrdiff_t i, len = end - start;
857 if (len > 0) {
858 TRef trslen = emitir(IRTI(IR_SUB), trend, trstart);
859 emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, (int32_t)len));
860 if (J->baseslot + len > LJ_MAX_JSLOTS)
861 lj_trace_err_info(J, LJ_TRERR_STACKOV);
862 rd->nres = len;
863 for (i = 0; i < len; i++) {
864 TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, (int32_t)i));
865 tmp = emitir(IRT(IR_STRREF, IRT_PGC), trstr, tmp);
866 J->base[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY);
867 }
868 } else { /* Empty range or range underflow: return no results. */
869 emitir(IRTGI(IR_LE), trend, trstart);
870 rd->nres = 0;
871 }
872 }
873 }
874
recff_string_char(jit_State * J,RecordFFData * rd)875 static void LJ_FASTCALL recff_string_char(jit_State *J, RecordFFData *rd)
876 {
877 TRef k255 = lj_ir_kint(J, 255);
878 BCReg i;
879 for (i = 0; J->base[i] != 0; i++) { /* Convert char values to strings. */
880 TRef tr = lj_opt_narrow_toint(J, J->base[i]);
881 emitir(IRTGI(IR_ULE), tr, k255);
882 J->base[i] = emitir(IRT(IR_TOSTR, IRT_STR), tr, IRTOSTR_CHAR);
883 }
884 if (i > 1) { /* Concatenate the strings, if there's more than one. */
885 TRef hdr = recff_bufhdr(J), tr = hdr;
886 for (i = 0; J->base[i] != 0; i++)
887 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, J->base[i]);
888 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
889 } else if (i == 0) {
890 J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty);
891 }
892 UNUSED(rd);
893 }
894
recff_string_rep(jit_State * J,RecordFFData * rd)895 static void LJ_FASTCALL recff_string_rep(jit_State *J, RecordFFData *rd)
896 {
897 TRef str = lj_ir_tostr(J, J->base[0]);
898 TRef rep = lj_opt_narrow_toint(J, J->base[1]);
899 TRef hdr, tr, str2 = 0;
900 if (!tref_isnil(J->base[2])) {
901 TRef sep = lj_ir_tostr(J, J->base[2]);
902 int32_t vrep = argv2int(J, &rd->argv[1]);
903 emitir(IRTGI(vrep > 1 ? IR_GT : IR_LE), rep, lj_ir_kint(J, 1));
904 if (vrep > 1) {
905 TRef hdr2 = recff_bufhdr(J);
906 TRef tr2 = emitir(IRTG(IR_BUFPUT, IRT_PGC), hdr2, sep);
907 tr2 = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr2, str);
908 str2 = emitir(IRTG(IR_BUFSTR, IRT_STR), tr2, hdr2);
909 }
910 }
911 tr = hdr = recff_bufhdr(J);
912 if (str2) {
913 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, str);
914 str = str2;
915 rep = emitir(IRTI(IR_ADD), rep, lj_ir_kint(J, -1));
916 }
917 tr = lj_ir_call(J, IRCALL_lj_buf_putstr_rep, tr, str, rep);
918 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
919 }
920
recff_string_op(jit_State * J,RecordFFData * rd)921 static void LJ_FASTCALL recff_string_op(jit_State *J, RecordFFData *rd)
922 {
923 TRef str = lj_ir_tostr(J, J->base[0]);
924 TRef hdr = recff_bufhdr(J);
925 TRef tr = lj_ir_call(J, rd->data, hdr, str);
926 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
927 }
928
recff_string_find(jit_State * J,RecordFFData * rd)929 static void LJ_FASTCALL recff_string_find(jit_State *J, RecordFFData *rd)
930 {
931 TRef trstr = lj_ir_tostr(J, J->base[0]);
932 TRef trpat = lj_ir_tostr(J, J->base[1]);
933 TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN);
934 TRef tr0 = lj_ir_kint(J, 0);
935 TRef trstart;
936 GCstr *str = argv2str(J, &rd->argv[0]);
937 GCstr *pat = argv2str(J, &rd->argv[1]);
938 int32_t start;
939 J->needsnap = 1;
940 if (tref_isnil(J->base[2])) {
941 trstart = lj_ir_kint(J, 1);
942 start = 1;
943 } else {
944 trstart = lj_opt_narrow_toint(J, J->base[2]);
945 start = argv2int(J, &rd->argv[2]);
946 }
947 trstart = recff_string_start(J, str, &start, trstart, trlen, tr0);
948 if ((MSize)start <= str->len) {
949 emitir(IRTGI(IR_ULE), trstart, trlen);
950 } else {
951 emitir(IRTGI(IR_UGT), trstart, trlen);
952 #if LJ_52
953 J->base[0] = TREF_NIL;
954 return;
955 #else
956 trstart = trlen;
957 start = str->len;
958 #endif
959 }
960 /* Fixed arg or no pattern matching chars? (Specialized to pattern string.) */
961 if ((J->base[2] && tref_istruecond(J->base[3])) ||
962 (emitir(IRTG(IR_EQ, IRT_STR), trpat, lj_ir_kstr(J, pat)),
963 !lj_str_haspattern(pat))) { /* Search for fixed string. */
964 TRef trsptr = emitir(IRT(IR_STRREF, IRT_PGC), trstr, trstart);
965 TRef trpptr = emitir(IRT(IR_STRREF, IRT_PGC), trpat, tr0);
966 TRef trslen = emitir(IRTI(IR_SUB), trlen, trstart);
967 TRef trplen = emitir(IRTI(IR_FLOAD), trpat, IRFL_STR_LEN);
968 TRef tr = lj_ir_call(J, IRCALL_lj_str_find, trsptr, trpptr, trslen, trplen);
969 TRef trp0 = lj_ir_kkptr(J, NULL);
970 if (lj_str_find(strdata(str)+(MSize)start, strdata(pat),
971 str->len-(MSize)start, pat->len)) {
972 TRef pos;
973 emitir(IRTG(IR_NE, IRT_PGC), tr, trp0);
974 /* Recompute offset. trsptr may not point into trstr after folding. */
975 pos = emitir(IRTI(IR_ADD), emitir(IRTI(IR_SUB), tr, trsptr), trstart);
976 J->base[0] = emitir(IRTI(IR_ADD), pos, lj_ir_kint(J, 1));
977 J->base[1] = emitir(IRTI(IR_ADD), pos, trplen);
978 rd->nres = 2;
979 } else {
980 emitir(IRTG(IR_EQ, IRT_PGC), tr, trp0);
981 J->base[0] = TREF_NIL;
982 }
983 } else { /* Search for pattern. */
984 recff_nyiu(J, rd);
985 return;
986 }
987 }
988
recff_format(jit_State * J,RecordFFData * rd,TRef hdr,int sbufx)989 static void recff_format(jit_State *J, RecordFFData *rd, TRef hdr, int sbufx)
990 {
991 ptrdiff_t arg = sbufx;
992 TRef tr = hdr, trfmt = lj_ir_tostr(J, J->base[arg]);
993 GCstr *fmt = argv2str(J, &rd->argv[arg]);
994 FormatState fs;
995 SFormat sf;
996 /* Specialize to the format string. */
997 emitir(IRTG(IR_EQ, IRT_STR), trfmt, lj_ir_kstr(J, fmt));
998 lj_strfmt_init(&fs, strdata(fmt), fmt->len);
999 while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) { /* Parse format. */
1000 TRef tra = sf == STRFMT_LIT ? 0 : J->base[++arg];
1001 TRef trsf = lj_ir_kint(J, (int32_t)sf);
1002 IRCallID id;
1003 switch (STRFMT_TYPE(sf)) {
1004 case STRFMT_LIT:
1005 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr,
1006 lj_ir_kstr(J, lj_str_new(J->L, fs.str, fs.len)));
1007 break;
1008 case STRFMT_INT:
1009 id = IRCALL_lj_strfmt_putfnum_int;
1010 handle_int:
1011 if (!tref_isinteger(tra)) {
1012 #if LJ_HASFFI
1013 if (tref_iscdata(tra)) {
1014 tra = lj_crecord_loadiu64(J, tra, &rd->argv[arg]);
1015 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfxint, tr, trsf, tra);
1016 break;
1017 }
1018 #endif
1019 goto handle_num;
1020 }
1021 if (sf == STRFMT_INT) { /* Shortcut for plain %d. */
1022 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr,
1023 emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_INT));
1024 } else {
1025 #if LJ_HASFFI
1026 tra = emitir(IRT(IR_CONV, IRT_U64), tra,
1027 (IRT_INT|(IRT_U64<<5)|IRCONV_SEXT));
1028 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfxint, tr, trsf, tra);
1029 lj_needsplit(J);
1030 #else
1031 recff_nyiu(J, rd); /* Don't bother working around this NYI. */
1032 return;
1033 #endif
1034 }
1035 break;
1036 case STRFMT_UINT:
1037 id = IRCALL_lj_strfmt_putfnum_uint;
1038 goto handle_int;
1039 case STRFMT_NUM:
1040 id = IRCALL_lj_strfmt_putfnum;
1041 handle_num:
1042 tra = lj_ir_tonum(J, tra);
1043 tr = lj_ir_call(J, id, tr, trsf, tra);
1044 if (LJ_SOFTFP32) lj_needsplit(J);
1045 break;
1046 case STRFMT_STR:
1047 if (!tref_isstr(tra)) {
1048 recff_nyiu(J, rd); /* NYI: __tostring and non-string types for %s. */
1049 /* NYI: also buffers. */
1050 return;
1051 }
1052 if (sf == STRFMT_STR) /* Shortcut for plain %s. */
1053 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, tra);
1054 else if ((sf & STRFMT_T_QUOTED))
1055 tr = lj_ir_call(J, IRCALL_lj_strfmt_putquoted, tr, tra);
1056 else
1057 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfstr, tr, trsf, tra);
1058 break;
1059 case STRFMT_CHAR:
1060 tra = lj_opt_narrow_toint(J, tra);
1061 if (sf == STRFMT_CHAR) /* Shortcut for plain %c. */
1062 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr,
1063 emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_CHAR));
1064 else
1065 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfchar, tr, trsf, tra);
1066 break;
1067 case STRFMT_PTR: /* NYI */
1068 case STRFMT_ERR:
1069 default:
1070 recff_nyiu(J, rd);
1071 return;
1072 }
1073 }
1074 if (sbufx) {
1075 emitir(IRT(IR_USE, IRT_NIL), tr, 0);
1076 } else {
1077 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
1078 }
1079 }
1080
recff_string_format(jit_State * J,RecordFFData * rd)1081 static void LJ_FASTCALL recff_string_format(jit_State *J, RecordFFData *rd)
1082 {
1083 recff_format(J, rd, recff_bufhdr(J), 0);
1084 }
1085
1086 /* -- Buffer library fast functions --------------------------------------- */
1087
1088 #if LJ_HASBUFFER
1089
recff_sbufx_get_L(jit_State * J,TRef ud)1090 static LJ_AINLINE TRef recff_sbufx_get_L(jit_State *J, TRef ud)
1091 {
1092 return emitir(IRT(IR_FLOAD, IRT_PGC), ud, IRFL_SBUF_L);
1093 }
1094
recff_sbufx_set_L(jit_State * J,TRef ud,TRef val)1095 static LJ_AINLINE void recff_sbufx_set_L(jit_State *J, TRef ud, TRef val)
1096 {
1097 TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ud, IRFL_SBUF_L);
1098 emitir(IRT(IR_FSTORE, IRT_PGC), fref, val);
1099 }
1100
recff_sbufx_get_ptr(jit_State * J,TRef ud,IRFieldID fl)1101 static LJ_AINLINE TRef recff_sbufx_get_ptr(jit_State *J, TRef ud, IRFieldID fl)
1102 {
1103 return emitir(IRT(IR_FLOAD, IRT_PTR), ud, fl);
1104 }
1105
recff_sbufx_set_ptr(jit_State * J,TRef ud,IRFieldID fl,TRef val)1106 static LJ_AINLINE void recff_sbufx_set_ptr(jit_State *J, TRef ud, IRFieldID fl, TRef val)
1107 {
1108 TRef fref = emitir(IRT(IR_FREF, IRT_PTR), ud, fl);
1109 emitir(IRT(IR_FSTORE, IRT_PTR), fref, val);
1110 }
1111
recff_sbufx_len(jit_State * J,TRef trr,TRef trw)1112 static LJ_AINLINE TRef recff_sbufx_len(jit_State *J, TRef trr, TRef trw)
1113 {
1114 TRef len = emitir(IRT(IR_SUB, IRT_INTP), trw, trr);
1115 if (LJ_64)
1116 len = emitir(IRTI(IR_CONV), len, (IRT_INT<<5)|IRT_INTP|IRCONV_NONE);
1117 return len;
1118 }
1119
1120 /* Emit typecheck for string buffer. */
recff_sbufx_check(jit_State * J,RecordFFData * rd,int arg)1121 static TRef recff_sbufx_check(jit_State *J, RecordFFData *rd, int arg)
1122 {
1123 TRef trtype, ud = J->base[arg];
1124 if (!tvisbuf(&rd->argv[arg])) lj_trace_err(J, LJ_TRERR_BADTYPE);
1125 trtype = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE);
1126 emitir(IRTGI(IR_EQ), trtype, lj_ir_kint(J, UDTYPE_BUFFER));
1127 J->needsnap = 1;
1128 return ud;
1129 }
1130
1131 /* Emit BUFHDR for write to extended string buffer. */
recff_sbufx_write(jit_State * J,TRef ud)1132 static TRef recff_sbufx_write(jit_State *J, TRef ud)
1133 {
1134 TRef trbuf = emitir(IRT(IR_ADD, IRT_PGC), ud, lj_ir_kint(J, sizeof(GCudata)));
1135 return emitir(IRT(IR_BUFHDR, IRT_PGC), trbuf, IRBUFHDR_WRITE);
1136 }
1137
1138 /* Check for integer in range for the buffer API. */
recff_sbufx_checkint(jit_State * J,RecordFFData * rd,int arg)1139 static TRef recff_sbufx_checkint(jit_State *J, RecordFFData *rd, int arg)
1140 {
1141 TRef tr = J->base[arg];
1142 TRef trlim = lj_ir_kint(J, LJ_MAX_BUF);
1143 if (tref_isinteger(tr)) {
1144 emitir(IRTGI(IR_ULE), tr, trlim);
1145 } else if (tref_isnum(tr)) {
1146 tr = emitir(IRTI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_ANY);
1147 emitir(IRTGI(IR_ULE), tr, trlim);
1148 #if LJ_HASFFI
1149 } else if (tref_iscdata(tr)) {
1150 tr = lj_crecord_loadiu64(J, tr, &rd->argv[arg]);
1151 emitir(IRTG(IR_ULE, IRT_U64), tr, lj_ir_kint64(J, LJ_MAX_BUF));
1152 tr = emitir(IRTI(IR_CONV), tr, (IRT_INT<<5)|IRT_I64|IRCONV_NONE);
1153 #else
1154 UNUSED(rd);
1155 #endif
1156 } else {
1157 lj_trace_err(J, LJ_TRERR_BADTYPE);
1158 }
1159 return tr;
1160 }
1161
recff_buffer_method_reset(jit_State * J,RecordFFData * rd)1162 static void LJ_FASTCALL recff_buffer_method_reset(jit_State *J, RecordFFData *rd)
1163 {
1164 TRef ud = recff_sbufx_check(J, rd, 0);
1165 SBufExt *sbx = bufV(&rd->argv[0]);
1166 int iscow = (int)sbufiscow(sbx);
1167 TRef trl = recff_sbufx_get_L(J, ud);
1168 TRef trcow = emitir(IRT(IR_BAND, IRT_IGC), trl, lj_ir_kint(J, SBUF_FLAG_COW));
1169 TRef zero = lj_ir_kint(J, 0);
1170 emitir(IRTG(iscow ? IR_NE : IR_EQ, IRT_IGC), trcow, zero);
1171 if (iscow) {
1172 trl = emitir(IRT(IR_BXOR, IRT_IGC), trl,
1173 LJ_GC64 ? lj_ir_kint64(J, SBUF_FLAG_COW) :
1174 lj_ir_kint(J, SBUF_FLAG_COW));
1175 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_W, zero);
1176 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_E, zero);
1177 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_B, zero);
1178 recff_sbufx_set_L(J, ud, trl);
1179 emitir(IRT(IR_FSTORE, IRT_PGC),
1180 emitir(IRT(IR_FREF, IRT_PGC), ud, IRFL_SBUF_REF), zero);
1181 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, zero);
1182 } else {
1183 TRef trb = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_B);
1184 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_W, trb);
1185 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trb);
1186 }
1187 }
1188
recff_buffer_method_skip(jit_State * J,RecordFFData * rd)1189 static void LJ_FASTCALL recff_buffer_method_skip(jit_State *J, RecordFFData *rd)
1190 {
1191 TRef ud = recff_sbufx_check(J, rd, 0);
1192 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1193 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1194 TRef len = recff_sbufx_len(J, trr, trw);
1195 TRef trn = recff_sbufx_checkint(J, rd, 1);
1196 len = emitir(IRTI(IR_MIN), len, trn);
1197 trr = emitir(IRT(IR_ADD, IRT_PTR), trr, len);
1198 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trr);
1199 }
1200
recff_buffer_method_set(jit_State * J,RecordFFData * rd)1201 static void LJ_FASTCALL recff_buffer_method_set(jit_State *J, RecordFFData *rd)
1202 {
1203 TRef ud = recff_sbufx_check(J, rd, 0);
1204 TRef trbuf = recff_sbufx_write(J, ud);
1205 TRef tr = J->base[1];
1206 if (tref_isstr(tr)) {
1207 TRef trp = emitir(IRT(IR_STRREF, IRT_PGC), tr, lj_ir_kint(J, 0));
1208 TRef len = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN);
1209 lj_ir_call(J, IRCALL_lj_bufx_set, trbuf, trp, len, tr);
1210 #if LJ_HASFFI
1211 } else if (tref_iscdata(tr)) {
1212 TRef trp = lj_crecord_topcvoid(J, tr, &rd->argv[1]);
1213 TRef len = recff_sbufx_checkint(J, rd, 2);
1214 lj_ir_call(J, IRCALL_lj_bufx_set, trbuf, trp, len, tr);
1215 #endif
1216 } /* else: Interpreter will throw. */
1217 }
1218
recff_buffer_method_put(jit_State * J,RecordFFData * rd)1219 static void LJ_FASTCALL recff_buffer_method_put(jit_State *J, RecordFFData *rd)
1220 {
1221 TRef ud = recff_sbufx_check(J, rd, 0);
1222 TRef trbuf = recff_sbufx_write(J, ud);
1223 TRef tr;
1224 ptrdiff_t arg;
1225 if (!J->base[1]) return;
1226 for (arg = 1; (tr = J->base[arg]); arg++) {
1227 if (tref_isstr(tr)) {
1228 trbuf = emitir(IRTG(IR_BUFPUT, IRT_PGC), trbuf, tr);
1229 } else if (tref_isnumber(tr)) {
1230 trbuf = emitir(IRTG(IR_BUFPUT, IRT_PGC), trbuf,
1231 emitir(IRT(IR_TOSTR, IRT_STR), tr,
1232 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT));
1233 } else if (tref_isudata(tr)) {
1234 TRef ud2 = recff_sbufx_check(J, rd, arg);
1235 TRef trr = recff_sbufx_get_ptr(J, ud2, IRFL_SBUF_R);
1236 TRef trw = recff_sbufx_get_ptr(J, ud2, IRFL_SBUF_W);
1237 TRef len = recff_sbufx_len(J, trr, trw);
1238 emitir(IRTG(IR_NE, IRT_PGC), ud, ud2);
1239 trbuf = lj_ir_call(J, IRCALL_lj_buf_putmem, trbuf, trr, len);
1240 } else {
1241 recff_nyiu(J, rd);
1242 }
1243 }
1244 emitir(IRT(IR_USE, IRT_NIL), trbuf, 0);
1245 }
1246
recff_buffer_method_putf(jit_State * J,RecordFFData * rd)1247 static void LJ_FASTCALL recff_buffer_method_putf(jit_State *J, RecordFFData *rd)
1248 {
1249 TRef ud = recff_sbufx_check(J, rd, 0);
1250 TRef trbuf = recff_sbufx_write(J, ud);
1251 recff_format(J, rd, trbuf, 1);
1252 }
1253
recff_buffer_method_get(jit_State * J,RecordFFData * rd)1254 static void LJ_FASTCALL recff_buffer_method_get(jit_State *J, RecordFFData *rd)
1255 {
1256 TRef ud = recff_sbufx_check(J, rd, 0);
1257 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1258 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1259 TRef tr;
1260 ptrdiff_t arg;
1261 if (!J->base[1]) { J->base[1] = TREF_NIL; J->base[2] = 0; }
1262 for (arg = 0; (tr = J->base[arg+1]); arg++) {
1263 TRef len = recff_sbufx_len(J, trr, trw);
1264 if (tref_isnil(tr)) {
1265 J->base[arg] = emitir(IRT(IR_XSNEW, IRT_STR), trr, len);
1266 trr = trw;
1267 } else {
1268 TRef trn = recff_sbufx_checkint(J, rd, arg+1);
1269 TRef tru;
1270 len = emitir(IRTI(IR_MIN), len, trn);
1271 tru = emitir(IRT(IR_ADD, IRT_PTR), trr, len);
1272 J->base[arg] = emitir(IRT(IR_XSNEW, IRT_STR), trr, len);
1273 trr = tru; /* Doing the ADD before the SNEW generates better code. */
1274 }
1275 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trr);
1276 }
1277 rd->nres = arg;
1278 }
1279
recff_buffer_method___tostring(jit_State * J,RecordFFData * rd)1280 static void LJ_FASTCALL recff_buffer_method___tostring(jit_State *J, RecordFFData *rd)
1281 {
1282 TRef ud = recff_sbufx_check(J, rd, 0);
1283 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1284 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1285 J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), trr, recff_sbufx_len(J, trr, trw));
1286 }
1287
recff_buffer_method___len(jit_State * J,RecordFFData * rd)1288 static void LJ_FASTCALL recff_buffer_method___len(jit_State *J, RecordFFData *rd)
1289 {
1290 TRef ud = recff_sbufx_check(J, rd, 0);
1291 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1292 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1293 J->base[0] = recff_sbufx_len(J, trr, trw);
1294 }
1295
1296 #if LJ_HASFFI
recff_buffer_method_putcdata(jit_State * J,RecordFFData * rd)1297 static void LJ_FASTCALL recff_buffer_method_putcdata(jit_State *J, RecordFFData *rd)
1298 {
1299 TRef ud = recff_sbufx_check(J, rd, 0);
1300 TRef trbuf = recff_sbufx_write(J, ud);
1301 TRef tr = lj_crecord_topcvoid(J, J->base[1], &rd->argv[1]);
1302 TRef len = recff_sbufx_checkint(J, rd, 2);
1303 trbuf = lj_ir_call(J, IRCALL_lj_buf_putmem, trbuf, tr, len);
1304 emitir(IRT(IR_USE, IRT_NIL), trbuf, 0);
1305 }
1306
recff_buffer_method_reserve(jit_State * J,RecordFFData * rd)1307 static void LJ_FASTCALL recff_buffer_method_reserve(jit_State *J, RecordFFData *rd)
1308 {
1309 TRef ud = recff_sbufx_check(J, rd, 0);
1310 TRef trbuf = recff_sbufx_write(J, ud);
1311 TRef trsz = recff_sbufx_checkint(J, rd, 1);
1312 J->base[1] = lj_ir_call(J, IRCALL_lj_bufx_more, trbuf, trsz);
1313 J->base[0] = lj_crecord_topuint8(J, recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W));
1314 rd->nres = 2;
1315 }
1316
recff_buffer_method_commit(jit_State * J,RecordFFData * rd)1317 static void LJ_FASTCALL recff_buffer_method_commit(jit_State *J, RecordFFData *rd)
1318 {
1319 TRef ud = recff_sbufx_check(J, rd, 0);
1320 TRef len = recff_sbufx_checkint(J, rd, 1);
1321 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1322 TRef tre = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_E);
1323 TRef left = emitir(IRT(IR_SUB, IRT_INTP), tre, trw);
1324 if (LJ_64)
1325 left = emitir(IRTI(IR_CONV), left, (IRT_INT<<5)|IRT_INTP|IRCONV_NONE);
1326 emitir(IRTGI(IR_ULE), len, left);
1327 trw = emitir(IRT(IR_ADD, IRT_PTR), trw, len);
1328 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_W, trw);
1329 }
1330
recff_buffer_method_ref(jit_State * J,RecordFFData * rd)1331 static void LJ_FASTCALL recff_buffer_method_ref(jit_State *J, RecordFFData *rd)
1332 {
1333 TRef ud = recff_sbufx_check(J, rd, 0);
1334 TRef trr = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_R);
1335 TRef trw = recff_sbufx_get_ptr(J, ud, IRFL_SBUF_W);
1336 J->base[0] = lj_crecord_topuint8(J, trr);
1337 J->base[1] = recff_sbufx_len(J, trr, trw);
1338 rd->nres = 2;
1339 }
1340 #endif
1341
recff_buffer_method_encode(jit_State * J,RecordFFData * rd)1342 static void LJ_FASTCALL recff_buffer_method_encode(jit_State *J, RecordFFData *rd)
1343 {
1344 TRef ud = recff_sbufx_check(J, rd, 0);
1345 TRef trbuf = recff_sbufx_write(J, ud);
1346 TRef tmp = recff_tmpref(J, J->base[1], IRTMPREF_IN1);
1347 lj_ir_call(J, IRCALL_lj_serialize_put, trbuf, tmp);
1348 /* No IR_USE needed, since the call is a store. */
1349 }
1350
recff_buffer_method_decode(jit_State * J,RecordFFData * rd)1351 static void LJ_FASTCALL recff_buffer_method_decode(jit_State *J, RecordFFData *rd)
1352 {
1353 TRef ud = recff_sbufx_check(J, rd, 0);
1354 TRef trbuf = recff_sbufx_write(J, ud);
1355 TRef tmp = recff_tmpref(J, TREF_NIL, IRTMPREF_OUT1);
1356 TRef trr = lj_ir_call(J, IRCALL_lj_serialize_get, trbuf, tmp);
1357 IRType t = (IRType)lj_serialize_peektype(bufV(&rd->argv[0]));
1358 /* No IR_USE needed, since the call is a store. */
1359 J->base[0] = lj_record_vload(J, tmp, 0, t);
1360 /* The sbx->r store must be after the VLOAD type check, in case it fails. */
1361 recff_sbufx_set_ptr(J, ud, IRFL_SBUF_R, trr);
1362 }
1363
recff_buffer_encode(jit_State * J,RecordFFData * rd)1364 static void LJ_FASTCALL recff_buffer_encode(jit_State *J, RecordFFData *rd)
1365 {
1366 TRef tmp = recff_tmpref(J, J->base[0], IRTMPREF_IN1);
1367 J->base[0] = lj_ir_call(J, IRCALL_lj_serialize_encode, tmp);
1368 /* IR_USE needed for IR_CALLA, because the encoder may throw non-OOM. */
1369 emitir(IRT(IR_USE, IRT_NIL), J->base[0], 0);
1370 UNUSED(rd);
1371 }
1372
recff_buffer_decode(jit_State * J,RecordFFData * rd)1373 static void LJ_FASTCALL recff_buffer_decode(jit_State *J, RecordFFData *rd)
1374 {
1375 if (tvisstr(&rd->argv[0])) {
1376 GCstr *str = strV(&rd->argv[0]);
1377 SBufExt sbx;
1378 IRType t;
1379 TRef tmp = recff_tmpref(J, TREF_NIL, IRTMPREF_OUT1);
1380 TRef tr = lj_ir_call(J, IRCALL_lj_serialize_decode, tmp, J->base[0]);
1381 /* IR_USE needed for IR_CALLA, because the decoder may throw non-OOM.
1382 ** That's why IRCALL_lj_serialize_decode needs a fake INT result.
1383 */
1384 emitir(IRT(IR_USE, IRT_NIL), tr, 0);
1385 memset(&sbx, 0, sizeof(SBufExt));
1386 lj_bufx_set_cow(J->L, &sbx, strdata(str), str->len);
1387 t = (IRType)lj_serialize_peektype(&sbx);
1388 J->base[0] = lj_record_vload(J, tmp, 0, t);
1389 } /* else: Interpreter will throw. */
1390 }
1391
1392 #endif
1393
1394 /* -- Table library fast functions ---------------------------------------- */
1395
recff_table_insert(jit_State * J,RecordFFData * rd)1396 static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd)
1397 {
1398 RecordIndex ix;
1399 ix.tab = J->base[0];
1400 ix.val = J->base[1];
1401 rd->nres = 0;
1402 if (tref_istab(ix.tab) && ix.val) {
1403 if (!J->base[2]) { /* Simple push: t[#t+1] = v */
1404 TRef trlen = emitir(IRTI(IR_ALEN), ix.tab, TREF_NIL);
1405 GCtab *t = tabV(&rd->argv[0]);
1406 ix.key = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1407 settabV(J->L, &ix.tabv, t);
1408 setintV(&ix.keyv, lj_tab_len(t) + 1);
1409 ix.idxchain = 0;
1410 lj_record_idx(J, &ix); /* Set new value. */
1411 } else { /* Complex case: insert in the middle. */
1412 recff_nyiu(J, rd);
1413 return;
1414 }
1415 } /* else: Interpreter will throw. */
1416 }
1417
recff_table_concat(jit_State * J,RecordFFData * rd)1418 static void LJ_FASTCALL recff_table_concat(jit_State *J, RecordFFData *rd)
1419 {
1420 TRef tab = J->base[0];
1421 if (tref_istab(tab)) {
1422 TRef sep = !tref_isnil(J->base[1]) ?
1423 lj_ir_tostr(J, J->base[1]) : lj_ir_knull(J, IRT_STR);
1424 TRef tri = (J->base[1] && !tref_isnil(J->base[2])) ?
1425 lj_opt_narrow_toint(J, J->base[2]) : lj_ir_kint(J, 1);
1426 TRef tre = (J->base[1] && J->base[2] && !tref_isnil(J->base[3])) ?
1427 lj_opt_narrow_toint(J, J->base[3]) :
1428 emitir(IRTI(IR_ALEN), tab, TREF_NIL);
1429 TRef hdr = recff_bufhdr(J);
1430 TRef tr = lj_ir_call(J, IRCALL_lj_buf_puttab, hdr, tab, sep, tri, tre);
1431 emitir(IRTG(IR_NE, IRT_PTR), tr, lj_ir_kptr(J, NULL));
1432 J->base[0] = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
1433 } /* else: Interpreter will throw. */
1434 UNUSED(rd);
1435 }
1436
recff_table_new(jit_State * J,RecordFFData * rd)1437 static void LJ_FASTCALL recff_table_new(jit_State *J, RecordFFData *rd)
1438 {
1439 TRef tra = lj_opt_narrow_toint(J, J->base[0]);
1440 TRef trh = lj_opt_narrow_toint(J, J->base[1]);
1441 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_new_ah, tra, trh);
1442 UNUSED(rd);
1443 }
1444
recff_table_clear(jit_State * J,RecordFFData * rd)1445 static void LJ_FASTCALL recff_table_clear(jit_State *J, RecordFFData *rd)
1446 {
1447 TRef tr = J->base[0];
1448 if (tref_istab(tr)) {
1449 rd->nres = 0;
1450 lj_ir_call(J, IRCALL_lj_tab_clear, tr);
1451 J->needsnap = 1;
1452 } /* else: Interpreter will throw. */
1453 }
1454
recff_table_clone(jit_State * J,RecordFFData * rd)1455 static void LJ_FASTCALL recff_table_clone(jit_State *J, RecordFFData *rd)
1456 {
1457 TRef src = J->base[0];
1458 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_clone, src);
1459 UNUSED(rd);
1460 }
1461
recff_table_isarray(jit_State * J,RecordFFData * rd)1462 static void LJ_FASTCALL recff_table_isarray(jit_State *J, RecordFFData *rd)
1463 {
1464 TRef src = J->base[0];
1465 if (LJ_LIKELY(tref_istab(src))) {
1466 TRef trres = lj_ir_call(J, IRCALL_lj_tab_isarray, src);
1467 GCtab *t = tabV(&rd->argv[0]);
1468 int isarr = lj_tab_isarray(t);
1469 TRef tr0 = lj_ir_kint(J, 0);
1470 emitir(isarr ? IRTGI(IR_NE) : IRTGI(IR_EQ), trres, tr0);
1471 J->base[0] = isarr ? TREF_TRUE : TREF_FALSE;
1472 } /* else: Interpreter will throw. */
1473 }
1474
recff_table_nkeys(jit_State * J,RecordFFData * rd)1475 static void LJ_FASTCALL recff_table_nkeys(jit_State *J, RecordFFData *rd)
1476 {
1477 TRef src = J->base[0];
1478 if (LJ_LIKELY(tref_istab(src))) {
1479 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_nkeys, src);
1480 } /* else: Interpreter will throw. */
1481 }
1482
recff_table_isempty(jit_State * J,RecordFFData * rd)1483 static void LJ_FASTCALL recff_table_isempty(jit_State *J, RecordFFData *rd)
1484 {
1485 TRef src = J->base[0];
1486 if (LJ_LIKELY(tref_istab(src))) {
1487 TRef trres = lj_ir_call(J, IRCALL_lj_tab_isempty, src);
1488 GCtab *t = tabV(&rd->argv[0]);
1489 int isempty = lj_tab_isempty(t);
1490 TRef tr0 = lj_ir_kint(J, 0);
1491 emitir(isempty ? IRTGI(IR_NE) : IRTGI(IR_EQ), trres, tr0);
1492 J->base[0] = isempty ? TREF_TRUE : TREF_FALSE;
1493 } /* else: Interpreter will throw. */
1494 }
1495
1496 /* -- thread library fast functions ------------------------------------------ */
1497
1498 #if LJ_HASFFI
recff_thread_exdata(jit_State * J,RecordFFData * rd)1499 void LJ_FASTCALL recff_thread_exdata(jit_State *J, RecordFFData *rd)
1500 {
1501 TRef tr = J->base[0];
1502 if (!tr) {
1503 TRef trl = emitir(IRT(IR_LREF, IRT_THREAD), 0, 0);
1504 TRef trp = emitir(IRT(IR_FLOAD, IRT_PTR), trl, IRFL_THREAD_EXDATA);
1505 TRef trid = lj_ir_kint(J, CTID_P_VOID);
1506 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, trp);
1507 return;
1508 }
1509 recff_nyiu(J, rd); /* this case is too rare to be interesting */
1510 }
1511
recff_thread_exdata2(jit_State * J,RecordFFData * rd)1512 void LJ_FASTCALL recff_thread_exdata2(jit_State *J, RecordFFData *rd)
1513 {
1514 TRef tr = J->base[0];
1515 if (!tr) {
1516 TRef trl = emitir(IRT(IR_LREF, IRT_THREAD), 0, 0);
1517 TRef trp = emitir(IRT(IR_FLOAD, IRT_PTR), trl, IRFL_THREAD_EXDATA2);
1518 TRef trid = lj_ir_kint(J, CTID_P_VOID);
1519 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, trp);
1520 return;
1521 }
1522 recff_nyiu(J, rd); /* this case is too rare to be interesting */
1523 }
1524 #endif
1525
1526 /* -- I/O library fast functions ------------------------------------------ */
1527
1528 /* Get FILE* for I/O function. Any I/O error aborts recording, so there's
1529 ** no need to encode the alternate cases for any of the guards.
1530 */
recff_io_fp(jit_State * J,TRef * udp,int32_t id)1531 static TRef recff_io_fp(jit_State *J, TRef *udp, int32_t id)
1532 {
1533 TRef tr, ud, fp;
1534 if (id) { /* io.func() */
1535 ud = lj_ir_ggfload(J, IRT_UDATA, GG_OFS(g.gcroot[id]));
1536 } else { /* fp:method() */
1537 ud = J->base[0];
1538 if (!tref_isudata(ud))
1539 lj_trace_err(J, LJ_TRERR_BADTYPE);
1540 tr = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE);
1541 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
1542 }
1543 *udp = ud;
1544 fp = emitir(IRT(IR_FLOAD, IRT_PTR), ud, IRFL_UDATA_FILE);
1545 emitir(IRTG(IR_NE, IRT_PTR), fp, lj_ir_knull(J, IRT_PTR));
1546 return fp;
1547 }
1548
recff_io_write(jit_State * J,RecordFFData * rd)1549 static void LJ_FASTCALL recff_io_write(jit_State *J, RecordFFData *rd)
1550 {
1551 TRef ud, fp = recff_io_fp(J, &ud, rd->data);
1552 TRef zero = lj_ir_kint(J, 0);
1553 TRef one = lj_ir_kint(J, 1);
1554 ptrdiff_t i = rd->data == 0 ? 1 : 0;
1555 for (; J->base[i]; i++) {
1556 TRef str = lj_ir_tostr(J, J->base[i]);
1557 TRef buf = emitir(IRT(IR_STRREF, IRT_PGC), str, zero);
1558 TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN);
1559 if (tref_isk(len) && IR(tref_ref(len))->i == 1) {
1560 IRIns *irs = IR(tref_ref(str));
1561 TRef tr = (irs->o == IR_TOSTR && irs->op2 == IRTOSTR_CHAR) ?
1562 irs->op1 :
1563 emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY);
1564 tr = lj_ir_call(J, IRCALL_fputc, tr, fp);
1565 if (results_wanted(J) != 0) /* Check result only if not ignored. */
1566 emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1));
1567 } else {
1568 TRef tr = lj_ir_call(J, IRCALL_fwrite, buf, one, len, fp);
1569 if (results_wanted(J) != 0) /* Check result only if not ignored. */
1570 emitir(IRTGI(IR_EQ), tr, len);
1571 }
1572 }
1573 J->base[0] = LJ_52 ? ud : TREF_TRUE;
1574 }
1575
recff_io_flush(jit_State * J,RecordFFData * rd)1576 static void LJ_FASTCALL recff_io_flush(jit_State *J, RecordFFData *rd)
1577 {
1578 TRef ud, fp = recff_io_fp(J, &ud, rd->data);
1579 TRef tr = lj_ir_call(J, IRCALL_fflush, fp);
1580 if (results_wanted(J) != 0) /* Check result only if not ignored. */
1581 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
1582 J->base[0] = TREF_TRUE;
1583 }
1584
1585 /* -- Debug library fast functions ---------------------------------------- */
1586
recff_debug_getmetatable(jit_State * J,RecordFFData * rd)1587 static void LJ_FASTCALL recff_debug_getmetatable(jit_State *J, RecordFFData *rd)
1588 {
1589 GCtab *mt;
1590 TRef mtref;
1591 TRef tr = J->base[0];
1592 if (tref_istab(tr)) {
1593 mt = tabref(tabV(&rd->argv[0])->metatable);
1594 mtref = emitir(IRT(IR_FLOAD, IRT_TAB), tr, IRFL_TAB_META);
1595 } else if (tref_isudata(tr)) {
1596 mt = tabref(udataV(&rd->argv[0])->metatable);
1597 mtref = emitir(IRT(IR_FLOAD, IRT_TAB), tr, IRFL_UDATA_META);
1598 } else {
1599 mt = tabref(basemt_obj(J2G(J), &rd->argv[0]));
1600 J->base[0] = mt ? lj_ir_ktab(J, mt) : TREF_NIL;
1601 return;
1602 }
1603 emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
1604 J->base[0] = mt ? mtref : TREF_NIL;
1605 }
1606
1607 /* -- Record calls to fast functions -------------------------------------- */
1608
1609 #include "lj_recdef.h"
1610
recdef_lookup(GCfunc * fn)1611 static uint32_t recdef_lookup(GCfunc *fn)
1612 {
1613 if (fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0]))
1614 return recff_idmap[fn->c.ffid];
1615 else
1616 return 0;
1617 }
1618
1619 /* Record entry to a fast function or C function. */
lj_ffrecord_func(jit_State * J)1620 void lj_ffrecord_func(jit_State *J)
1621 {
1622 RecordFFData rd;
1623 uint32_t m = recdef_lookup(J->fn);
1624 rd.data = m & 0xff;
1625 rd.nres = 1; /* Default is one result. */
1626 rd.argv = J->L->base;
1627 J->base[J->maxslot] = 0; /* Mark end of arguments. */
1628 (recff_func[m >> 8])(J, &rd); /* Call recff_* handler. */
1629 if (rd.nres >= 0) {
1630 if (J->postproc == LJ_POST_NONE) J->postproc = LJ_POST_FFRETRY;
1631 lj_record_ret(J, 0, rd.nres);
1632 }
1633 }
1634
1635 #undef IR
1636 #undef emitir
1637
1638 #endif
1639