1 /*
2 ** MIPS IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 /* -- Register allocator extensions --------------------------------------- */
7
8 /* Allocate a register with a hint. */
ra_hintalloc(ASMState * as,IRRef ref,Reg hint,RegSet allow)9 static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
10 {
11 Reg r = IR(ref)->r;
12 if (ra_noreg(r)) {
13 if (!ra_hashint(r) && !iscrossref(as, ref))
14 ra_sethint(IR(ref)->r, hint); /* Propagate register hint. */
15 r = ra_allocref(as, ref, allow);
16 }
17 ra_noweak(as, r);
18 return r;
19 }
20
21 /* Allocate a register or RID_ZERO. */
ra_alloc1z(ASMState * as,IRRef ref,RegSet allow)22 static Reg ra_alloc1z(ASMState *as, IRRef ref, RegSet allow)
23 {
24 Reg r = IR(ref)->r;
25 if (ra_noreg(r)) {
26 if (!(allow & RSET_FPR) && irref_isk(ref) && IR(ref)->i == 0)
27 return RID_ZERO;
28 r = ra_allocref(as, ref, allow);
29 } else {
30 ra_noweak(as, r);
31 }
32 return r;
33 }
34
35 /* Allocate two source registers for three-operand instructions. */
ra_alloc2(ASMState * as,IRIns * ir,RegSet allow)36 static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
37 {
38 IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
39 Reg left = irl->r, right = irr->r;
40 if (ra_hasreg(left)) {
41 ra_noweak(as, left);
42 if (ra_noreg(right))
43 right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
44 else
45 ra_noweak(as, right);
46 } else if (ra_hasreg(right)) {
47 ra_noweak(as, right);
48 left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
49 } else if (ra_hashint(right)) {
50 right = ra_alloc1z(as, ir->op2, allow);
51 left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
52 } else {
53 left = ra_alloc1z(as, ir->op1, allow);
54 right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
55 }
56 return left | (right << 8);
57 }
58
59 /* -- Guard handling ------------------------------------------------------ */
60
61 /* Need some spare long-range jump slots, for out-of-range branches. */
62 #define MIPS_SPAREJUMP 4
63
64 /* Setup spare long-range jump slots per mcarea. */
asm_sparejump_setup(ASMState * as)65 static void asm_sparejump_setup(ASMState *as)
66 {
67 MCode *mxp = as->mcbot;
68 /* Assumes sizeof(MCLink) == 8. */
69 if (((uintptr_t)mxp & (LJ_PAGESIZE-1)) == 8) {
70 lua_assert(MIPSI_NOP == 0);
71 memset(mxp+2, 0, MIPS_SPAREJUMP*8);
72 mxp += MIPS_SPAREJUMP*2;
73 lua_assert(mxp < as->mctop);
74 lj_mcode_sync(as->mcbot, mxp);
75 lj_mcode_commitbot(as->J, mxp);
76 as->mcbot = mxp;
77 as->mclim = as->mcbot + MCLIM_REDZONE;
78 }
79 }
80
81 /* Setup exit stub after the end of each trace. */
asm_exitstub_setup(ASMState * as)82 static void asm_exitstub_setup(ASMState *as)
83 {
84 MCode *mxp = as->mctop;
85 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
86 *--mxp = MIPSI_LI|MIPSF_T(RID_TMP)|as->T->traceno;
87 *--mxp = MIPSI_J|((((uintptr_t)(void *)lj_vm_exit_handler)>>2)&0x03ffffffu);
88 lua_assert(((uintptr_t)mxp ^ (uintptr_t)(void *)lj_vm_exit_handler)>>28 == 0);
89 *--mxp = MIPSI_SW|MIPSF_T(RID_TMP)|MIPSF_S(RID_SP)|0;
90 as->mctop = mxp;
91 }
92
93 /* Keep this in-sync with exitstub_trace_addr(). */
94 #define asm_exitstub_addr(as) ((as)->mctop)
95
96 /* Emit conditional branch to exit for guard. */
asm_guard(ASMState * as,MIPSIns mi,Reg rs,Reg rt)97 static void asm_guard(ASMState *as, MIPSIns mi, Reg rs, Reg rt)
98 {
99 MCode *target = asm_exitstub_addr(as);
100 MCode *p = as->mcp;
101 if (LJ_UNLIKELY(p == as->invmcp)) {
102 as->invmcp = NULL;
103 as->loopinv = 1;
104 as->mcp = p+1;
105 mi = mi ^ ((mi>>28) == 1 ? 0x04000000u : 0x00010000u); /* Invert cond. */
106 target = p; /* Patch target later in asm_loop_fixup. */
107 }
108 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
109 emit_branch(as, mi, rs, rt, target);
110 }
111
112 /* -- Operand fusion ------------------------------------------------------ */
113
114 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
115 #define CONFLICT_SEARCH_LIM 31
116
117 /* Check if there's no conflicting instruction between curins and ref. */
noconflict(ASMState * as,IRRef ref,IROp conflict)118 static int noconflict(ASMState *as, IRRef ref, IROp conflict)
119 {
120 IRIns *ir = as->ir;
121 IRRef i = as->curins;
122 if (i > ref + CONFLICT_SEARCH_LIM)
123 return 0; /* Give up, ref is too far away. */
124 while (--i > ref)
125 if (ir[i].o == conflict)
126 return 0; /* Conflict found. */
127 return 1; /* Ok, no conflict. */
128 }
129
130 /* Fuse the array base of colocated arrays. */
asm_fuseabase(ASMState * as,IRRef ref)131 static int32_t asm_fuseabase(ASMState *as, IRRef ref)
132 {
133 IRIns *ir = IR(ref);
134 if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
135 !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
136 return (int32_t)sizeof(GCtab);
137 return 0;
138 }
139
140 /* Fuse array/hash/upvalue reference into register+offset operand. */
asm_fuseahuref(ASMState * as,IRRef ref,int32_t * ofsp,RegSet allow)141 static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
142 {
143 IRIns *ir = IR(ref);
144 if (ra_noreg(ir->r)) {
145 if (ir->o == IR_AREF) {
146 if (mayfuse(as, ref)) {
147 if (irref_isk(ir->op2)) {
148 IRRef tab = IR(ir->op1)->op1;
149 int32_t ofs = asm_fuseabase(as, tab);
150 IRRef refa = ofs ? tab : ir->op1;
151 ofs += 8*IR(ir->op2)->i;
152 if (checki16(ofs)) {
153 *ofsp = ofs;
154 return ra_alloc1(as, refa, allow);
155 }
156 }
157 }
158 } else if (ir->o == IR_HREFK) {
159 if (mayfuse(as, ref)) {
160 int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
161 if (checki16(ofs)) {
162 *ofsp = ofs;
163 return ra_alloc1(as, ir->op1, allow);
164 }
165 }
166 } else if (ir->o == IR_UREFC) {
167 if (irref_isk(ir->op1)) {
168 GCfunc *fn = ir_kfunc(IR(ir->op1));
169 int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv);
170 int32_t jgl = (intptr_t)J2G(as->J);
171 if ((uint32_t)(ofs-jgl) < 65536) {
172 *ofsp = ofs-jgl-32768;
173 return RID_JGL;
174 } else {
175 *ofsp = (int16_t)ofs;
176 return ra_allock(as, ofs-(int16_t)ofs, allow);
177 }
178 }
179 }
180 }
181 *ofsp = 0;
182 return ra_alloc1(as, ref, allow);
183 }
184
185 /* Fuse XLOAD/XSTORE reference into load/store operand. */
asm_fusexref(ASMState * as,MIPSIns mi,Reg rt,IRRef ref,RegSet allow,int32_t ofs)186 static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
187 RegSet allow, int32_t ofs)
188 {
189 IRIns *ir = IR(ref);
190 Reg base;
191 if (ra_noreg(ir->r) && canfuse(as, ir)) {
192 if (ir->o == IR_ADD) {
193 int32_t ofs2;
194 if (irref_isk(ir->op2) && (ofs2 = ofs + IR(ir->op2)->i, checki16(ofs2))) {
195 ref = ir->op1;
196 ofs = ofs2;
197 }
198 } else if (ir->o == IR_STRREF) {
199 int32_t ofs2 = 65536;
200 lua_assert(ofs == 0);
201 ofs = (int32_t)sizeof(GCstr);
202 if (irref_isk(ir->op2)) {
203 ofs2 = ofs + IR(ir->op2)->i;
204 ref = ir->op1;
205 } else if (irref_isk(ir->op1)) {
206 ofs2 = ofs + IR(ir->op1)->i;
207 ref = ir->op2;
208 }
209 if (!checki16(ofs2)) {
210 /* NYI: Fuse ADD with constant. */
211 Reg right, left = ra_alloc2(as, ir, allow);
212 right = (left >> 8); left &= 255;
213 emit_hsi(as, mi, rt, RID_TMP, ofs);
214 emit_dst(as, MIPSI_ADDU, RID_TMP, left, right);
215 return;
216 }
217 ofs = ofs2;
218 }
219 }
220 base = ra_alloc1(as, ref, allow);
221 emit_hsi(as, mi, rt, base, ofs);
222 }
223
224 /* -- Calls --------------------------------------------------------------- */
225
226 /* Generate a call to a C function. */
asm_gencall(ASMState * as,const CCallInfo * ci,IRRef * args)227 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
228 {
229 uint32_t n, nargs = CCI_NARGS(ci);
230 int32_t ofs = 16;
231 Reg gpr, fpr = REGARG_FIRSTFPR;
232 if ((void *)ci->func)
233 emit_call(as, (void *)ci->func);
234 for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
235 as->cost[gpr] = REGCOST(~0u, ASMREF_L);
236 gpr = REGARG_FIRSTGPR;
237 for (n = 0; n < nargs; n++) { /* Setup args. */
238 IRRef ref = args[n];
239 if (ref) {
240 IRIns *ir = IR(ref);
241 if (irt_isfp(ir->t) && fpr <= REGARG_LASTFPR &&
242 !(ci->flags & CCI_VARARG)) {
243 lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */
244 ra_leftov(as, fpr, ref);
245 fpr += 2;
246 gpr += irt_isnum(ir->t) ? 2 : 1;
247 } else {
248 fpr = REGARG_LASTFPR+1;
249 if (irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
250 if (gpr <= REGARG_LASTGPR) {
251 lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */
252 if (irt_isfp(ir->t)) {
253 RegSet of = as->freeset;
254 Reg r;
255 /* Workaround to protect argument GPRs from being used for remat. */
256 as->freeset &= ~RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1);
257 r = ra_alloc1(as, ref, RSET_FPR);
258 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
259 if (irt_isnum(ir->t)) {
260 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1);
261 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r);
262 lua_assert(rset_test(as->freeset, gpr+1)); /* Already evicted. */
263 gpr += 2;
264 } else if (irt_isfloat(ir->t)) {
265 emit_tg(as, MIPSI_MFC1, gpr, r);
266 gpr++;
267 }
268 } else {
269 ra_leftov(as, gpr, ref);
270 gpr++;
271 }
272 } else {
273 Reg r = ra_alloc1z(as, ref, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
274 if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
275 emit_spstore(as, ir, r, ofs);
276 ofs += irt_isnum(ir->t) ? 8 : 4;
277 }
278 }
279 } else {
280 fpr = REGARG_LASTFPR+1;
281 if (gpr <= REGARG_LASTGPR)
282 gpr++;
283 else
284 ofs += 4;
285 }
286 checkmclim(as);
287 }
288 }
289
290 /* Setup result reg/sp for call. Evict scratch regs. */
asm_setupresult(ASMState * as,IRIns * ir,const CCallInfo * ci)291 static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
292 {
293 RegSet drop = RSET_SCRATCH;
294 int hiop = ((ir+1)->o == IR_HIOP);
295 if ((ci->flags & CCI_NOFPRCLOBBER))
296 drop &= ~RSET_FPR;
297 if (ra_hasreg(ir->r))
298 rset_clear(drop, ir->r); /* Dest reg handled below. */
299 if (hiop && ra_hasreg((ir+1)->r))
300 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
301 ra_evictset(as, drop); /* Evictions must be performed first. */
302 if (ra_used(ir)) {
303 lua_assert(!irt_ispri(ir->t));
304 if (irt_isfp(ir->t)) {
305 if ((ci->flags & CCI_CASTU64)) {
306 int32_t ofs = sps_scale(ir->s);
307 Reg dest = ir->r;
308 if (ra_hasreg(dest)) {
309 ra_free(as, dest);
310 ra_modified(as, dest);
311 emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1);
312 emit_tg(as, MIPSI_MTC1, RID_RETLO, dest);
313 }
314 if (ofs) {
315 emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0));
316 emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4));
317 }
318 } else {
319 ra_destreg(as, ir, RID_FPRET);
320 }
321 } else if (hiop) {
322 ra_destpair(as, ir);
323 } else {
324 ra_destreg(as, ir, RID_RET);
325 }
326 }
327 }
328
asm_call(ASMState * as,IRIns * ir)329 static void asm_call(ASMState *as, IRIns *ir)
330 {
331 IRRef args[CCI_NARGS_MAX];
332 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
333 asm_collectargs(as, ir, ci, args);
334 asm_setupresult(as, ir, ci);
335 asm_gencall(as, ci, args);
336 }
337
asm_callx(ASMState * as,IRIns * ir)338 static void asm_callx(ASMState *as, IRIns *ir)
339 {
340 IRRef args[CCI_NARGS_MAX*2];
341 CCallInfo ci;
342 IRRef func;
343 IRIns *irf;
344 ci.flags = asm_callx_flags(as, ir);
345 asm_collectargs(as, ir, &ci, args);
346 asm_setupresult(as, ir, &ci);
347 func = ir->op2; irf = IR(func);
348 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
349 if (irref_isk(func)) { /* Call to constant address. */
350 ci.func = (ASMFunction)(void *)(irf->i);
351 } else { /* Need specific register for indirect calls. */
352 Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR));
353 MCode *p = as->mcp;
354 if (r == RID_CFUNCADDR)
355 *--p = MIPSI_NOP;
356 else
357 *--p = MIPSI_MOVE | MIPSF_D(RID_CFUNCADDR) | MIPSF_S(r);
358 *--p = MIPSI_JALR | MIPSF_S(r);
359 as->mcp = p;
360 ci.func = (ASMFunction)(void *)0;
361 }
362 asm_gencall(as, &ci, args);
363 }
364
asm_callid(ASMState * as,IRIns * ir,IRCallID id)365 static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
366 {
367 const CCallInfo *ci = &lj_ir_callinfo[id];
368 IRRef args[2];
369 args[0] = ir->op1;
370 args[1] = ir->op2;
371 asm_setupresult(as, ir, ci);
372 asm_gencall(as, ci, args);
373 }
374
asm_callround(ASMState * as,IRIns * ir,IRCallID id)375 static void asm_callround(ASMState *as, IRIns *ir, IRCallID id)
376 {
377 /* The modified regs must match with the *.dasc implementation. */
378 RegSet drop = RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_FPRET)|
379 RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(REGARG_FIRSTFPR);
380 if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
381 ra_evictset(as, drop);
382 ra_destreg(as, ir, RID_FPRET);
383 emit_call(as, (void *)lj_ir_callinfo[id].func);
384 ra_leftov(as, REGARG_FIRSTFPR, ir->op1);
385 }
386
387 /* -- Returns ------------------------------------------------------------- */
388
389 /* Return to lower frame. Guard that it goes to the right spot. */
asm_retf(ASMState * as,IRIns * ir)390 static void asm_retf(ASMState *as, IRIns *ir)
391 {
392 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
393 void *pc = ir_kptr(IR(ir->op2));
394 int32_t delta = 1+bc_a(*((const BCIns *)pc - 1));
395 as->topslot -= (BCReg)delta;
396 if ((int32_t)as->topslot < 0) as->topslot = 0;
397 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
398 emit_setgl(as, base, jit_base);
399 emit_addptr(as, base, -8*delta);
400 asm_guard(as, MIPSI_BNE, RID_TMP,
401 ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base)));
402 emit_tsi(as, MIPSI_LW, RID_TMP, base, -8);
403 }
404
405 /* -- Type conversions ---------------------------------------------------- */
406
asm_tointg(ASMState * as,IRIns * ir,Reg left)407 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
408 {
409 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
410 Reg dest = ra_dest(as, ir, RSET_GPR);
411 asm_guard(as, MIPSI_BC1F, 0, 0);
412 emit_fgh(as, MIPSI_C_EQ_D, 0, tmp, left);
413 emit_fg(as, MIPSI_CVT_D_W, tmp, tmp);
414 emit_tg(as, MIPSI_MFC1, dest, tmp);
415 emit_fg(as, MIPSI_CVT_W_D, tmp, left);
416 }
417
asm_tobit(ASMState * as,IRIns * ir)418 static void asm_tobit(ASMState *as, IRIns *ir)
419 {
420 RegSet allow = RSET_FPR;
421 Reg dest = ra_dest(as, ir, RSET_GPR);
422 Reg left = ra_alloc1(as, ir->op1, allow);
423 Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
424 Reg tmp = ra_scratch(as, rset_clear(allow, right));
425 emit_tg(as, MIPSI_MFC1, dest, tmp);
426 emit_fgh(as, MIPSI_ADD_D, tmp, left, right);
427 }
428
asm_conv(ASMState * as,IRIns * ir)429 static void asm_conv(ASMState *as, IRIns *ir)
430 {
431 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
432 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
433 IRRef lref = ir->op1;
434 lua_assert(irt_type(ir->t) != st);
435 lua_assert(!(irt_isint64(ir->t) ||
436 (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */
437 if (irt_isfp(ir->t)) {
438 Reg dest = ra_dest(as, ir, RSET_FPR);
439 if (stfp) { /* FP to FP conversion. */
440 emit_fg(as, st == IRT_NUM ? MIPSI_CVT_S_D : MIPSI_CVT_D_S,
441 dest, ra_alloc1(as, lref, RSET_FPR));
442 } else if (st == IRT_U32) { /* U32 to FP conversion. */
443 /* y = (x ^ 0x8000000) + 2147483648.0 */
444 Reg left = ra_alloc1(as, lref, RSET_GPR);
445 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
446 emit_fgh(as, irt_isfloat(ir->t) ? MIPSI_ADD_S : MIPSI_ADD_D,
447 dest, dest, tmp);
448 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
449 dest, dest);
450 if (irt_isfloat(ir->t))
451 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
452 (void *)lj_ir_k64_find(as->J, U64x(4f000000,4f000000)),
453 RSET_GPR);
454 else
455 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
456 (void *)lj_ir_k64_find(as->J, U64x(41e00000,00000000)),
457 RSET_GPR);
458 emit_tg(as, MIPSI_MTC1, RID_TMP, dest);
459 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left);
460 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
461 } else { /* Integer to FP conversion. */
462 Reg left = ra_alloc1(as, lref, RSET_GPR);
463 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
464 dest, dest);
465 emit_tg(as, MIPSI_MTC1, left, dest);
466 }
467 } else if (stfp) { /* FP to integer conversion. */
468 if (irt_isguard(ir->t)) {
469 /* Checked conversions are only supported from number to int. */
470 lua_assert(irt_isint(ir->t) && st == IRT_NUM);
471 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
472 } else {
473 Reg dest = ra_dest(as, ir, RSET_GPR);
474 Reg left = ra_alloc1(as, lref, RSET_FPR);
475 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
476 if (irt_isu32(ir->t)) {
477 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
478 emit_dst(as, MIPSI_XOR, dest, dest, RID_TMP);
479 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
480 emit_tg(as, MIPSI_MFC1, dest, tmp);
481 emit_fg(as, st == IRT_FLOAT ? MIPSI_FLOOR_W_S : MIPSI_FLOOR_W_D,
482 tmp, tmp);
483 emit_fgh(as, st == IRT_FLOAT ? MIPSI_SUB_S : MIPSI_SUB_D,
484 tmp, left, tmp);
485 if (st == IRT_FLOAT)
486 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
487 (void *)lj_ir_k64_find(as->J, U64x(4f000000,4f000000)),
488 RSET_GPR);
489 else
490 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
491 (void *)lj_ir_k64_find(as->J, U64x(41e00000,00000000)),
492 RSET_GPR);
493 } else {
494 emit_tg(as, MIPSI_MFC1, dest, tmp);
495 emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D,
496 tmp, left);
497 }
498 }
499 } else {
500 Reg dest = ra_dest(as, ir, RSET_GPR);
501 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
502 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
503 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
504 if ((ir->op2 & IRCONV_SEXT)) {
505 if ((as->flags & JIT_F_MIPS32R2)) {
506 emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left);
507 } else {
508 uint32_t shift = st == IRT_I8 ? 24 : 16;
509 emit_dta(as, MIPSI_SRA, dest, dest, shift);
510 emit_dta(as, MIPSI_SLL, dest, left, shift);
511 }
512 } else {
513 emit_tsi(as, MIPSI_ANDI, dest, left,
514 (int32_t)(st == IRT_U8 ? 0xff : 0xffff));
515 }
516 } else { /* 32/64 bit integer conversions. */
517 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
518 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
519 }
520 }
521 }
522
523 #if LJ_HASFFI
asm_conv64(ASMState * as,IRIns * ir)524 static void asm_conv64(ASMState *as, IRIns *ir)
525 {
526 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
527 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
528 IRCallID id;
529 const CCallInfo *ci;
530 IRRef args[2];
531 args[LJ_BE?0:1] = ir->op1;
532 args[LJ_BE?1:0] = (ir-1)->op1;
533 if (st == IRT_NUM || st == IRT_FLOAT) {
534 id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
535 ir--;
536 } else {
537 id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
538 }
539 ci = &lj_ir_callinfo[id];
540 asm_setupresult(as, ir, ci);
541 asm_gencall(as, ci, args);
542 }
543 #endif
544
asm_strto(ASMState * as,IRIns * ir)545 static void asm_strto(ASMState *as, IRIns *ir)
546 {
547 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
548 IRRef args[2];
549 RegSet drop = RSET_SCRATCH;
550 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */
551 ra_evictset(as, drop);
552 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO); /* Test return status. */
553 args[0] = ir->op1; /* GCstr *str */
554 args[1] = ASMREF_TMP1; /* TValue *n */
555 asm_gencall(as, ci, args);
556 /* Store the result to the spill slot or temp slots. */
557 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1),
558 RID_SP, sps_scale(ir->s));
559 }
560
561 /* Get pointer to TValue. */
asm_tvptr(ASMState * as,Reg dest,IRRef ref)562 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
563 {
564 IRIns *ir = IR(ref);
565 if (irt_isnum(ir->t)) {
566 if (irref_isk(ref)) /* Use the number constant itself as a TValue. */
567 ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
568 else /* Otherwise force a spill and use the spill slot. */
569 emit_tsi(as, MIPSI_ADDIU, dest, RID_SP, ra_spill(as, ir));
570 } else {
571 /* Otherwise use g->tmptv to hold the TValue. */
572 RegSet allow = rset_exclude(RSET_GPR, dest);
573 Reg type;
574 emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, offsetof(global_State, tmptv)-32768);
575 if (!irt_ispri(ir->t)) {
576 Reg src = ra_alloc1(as, ref, allow);
577 emit_setgl(as, src, tmptv.gcr);
578 }
579 type = ra_allock(as, irt_toitype(ir->t), allow);
580 emit_setgl(as, type, tmptv.it);
581 }
582 }
583
asm_tostr(ASMState * as,IRIns * ir)584 static void asm_tostr(ASMState *as, IRIns *ir)
585 {
586 IRRef args[2];
587 args[0] = ASMREF_L;
588 as->gcsteps++;
589 if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
590 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
591 args[1] = ASMREF_TMP1; /* const lua_Number * */
592 asm_setupresult(as, ir, ci); /* GCstr * */
593 asm_gencall(as, ci, args);
594 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
595 } else {
596 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
597 args[1] = ir->op1; /* int32_t k */
598 asm_setupresult(as, ir, ci); /* GCstr * */
599 asm_gencall(as, ci, args);
600 }
601 }
602
603 /* -- Memory references --------------------------------------------------- */
604
asm_aref(ASMState * as,IRIns * ir)605 static void asm_aref(ASMState *as, IRIns *ir)
606 {
607 Reg dest = ra_dest(as, ir, RSET_GPR);
608 Reg idx, base;
609 if (irref_isk(ir->op2)) {
610 IRRef tab = IR(ir->op1)->op1;
611 int32_t ofs = asm_fuseabase(as, tab);
612 IRRef refa = ofs ? tab : ir->op1;
613 ofs += 8*IR(ir->op2)->i;
614 if (checki16(ofs)) {
615 base = ra_alloc1(as, refa, RSET_GPR);
616 emit_tsi(as, MIPSI_ADDIU, dest, base, ofs);
617 return;
618 }
619 }
620 base = ra_alloc1(as, ir->op1, RSET_GPR);
621 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
622 emit_dst(as, MIPSI_ADDU, dest, RID_TMP, base);
623 emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3);
624 }
625
626 /* Inlined hash lookup. Specialized for key type and for const keys.
627 ** The equivalent C code is:
628 ** Node *n = hashkey(t, key);
629 ** do {
630 ** if (lj_obj_equal(&n->key, key)) return &n->val;
631 ** } while ((n = nextnode(n)));
632 ** return niltv(L);
633 */
asm_href(ASMState * as,IRIns * ir)634 static void asm_href(ASMState *as, IRIns *ir)
635 {
636 RegSet allow = RSET_GPR;
637 int destused = ra_used(ir);
638 Reg dest = ra_dest(as, ir, allow);
639 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
640 Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2;
641 IRRef refkey = ir->op2;
642 IRIns *irkey = IR(refkey);
643 IRType1 kt = irkey->t;
644 uint32_t khash;
645 MCLabel l_end, l_loop, l_next;
646
647 rset_clear(allow, tab);
648 if (irt_isnum(kt)) {
649 key = ra_alloc1(as, refkey, RSET_FPR);
650 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
651 } else if (!irt_ispri(kt)) {
652 key = ra_alloc1(as, refkey, allow);
653 rset_clear(allow, key);
654 type = ra_allock(as, irt_toitype(irkey->t), allow);
655 rset_clear(allow, type);
656 }
657 tmp2 = ra_scratch(as, allow);
658 rset_clear(allow, tmp2);
659
660 /* Key not found in chain: load niltv. */
661 l_end = emit_label(as);
662 if (destused)
663 emit_loada(as, dest, niltvg(J2G(as->J)));
664 else
665 *--as->mcp = MIPSI_NOP;
666 /* Follow hash chain until the end. */
667 emit_move(as, dest, tmp1);
668 l_loop = --as->mcp;
669 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, next));
670 l_next = emit_label(as);
671
672 /* Type and value comparison. */
673 if (irt_isnum(kt)) {
674 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
675 emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
676 emit_tg(as, MIPSI_MFC1, tmp1, key+1);
677 emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next);
678 emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM);
679 emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
680 } else {
681 if (irt_ispri(kt)) {
682 emit_branch(as, MIPSI_BEQ, tmp1, type, l_end);
683 } else {
684 emit_branch(as, MIPSI_BEQ, tmp2, key, l_end);
685 emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
686 emit_branch(as, MIPSI_BNE, tmp1, type, l_next);
687 }
688 }
689 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it));
690 *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
691
692 /* Load main position relative to tab->node into dest. */
693 khash = irref_isk(refkey) ? ir_khash(irkey) : 1;
694 if (khash == 0) {
695 emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node));
696 } else {
697 Reg tmphash = tmp1;
698 if (irref_isk(refkey))
699 tmphash = ra_allock(as, khash, allow);
700 emit_dst(as, MIPSI_ADDU, dest, dest, tmp1);
701 lua_assert(sizeof(Node) == 24);
702 emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1);
703 emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3);
704 emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5);
705 emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash);
706 emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node));
707 emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
708 if (irref_isk(refkey)) {
709 /* Nothing to do. */
710 } else if (irt_isstr(kt)) {
711 emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, hash));
712 } else { /* Must match with hash*() in lj_tab.c. */
713 emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2);
714 emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31);
715 emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2);
716 emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31);
717 emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest);
718 if (irt_isnum(kt)) {
719 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
720 if ((as->flags & JIT_F_MIPS32R2)) {
721 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
722 } else {
723 emit_dst(as, MIPSI_OR, dest, dest, tmp1);
724 emit_dta(as, MIPSI_SLL, tmp1, tmp1, HASH_ROT1);
725 emit_dta(as, MIPSI_SRL, dest, tmp1, (-HASH_ROT1)&31);
726 }
727 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
728 emit_tg(as, MIPSI_MFC1, tmp2, key);
729 emit_tg(as, MIPSI_MFC1, tmp1, key+1);
730 } else {
731 emit_dst(as, MIPSI_XOR, tmp2, key, tmp1);
732 emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31);
733 emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow));
734 }
735 }
736 }
737 }
738
asm_hrefk(ASMState * as,IRIns * ir)739 static void asm_hrefk(ASMState *as, IRIns *ir)
740 {
741 IRIns *kslot = IR(ir->op2);
742 IRIns *irkey = IR(kslot->op1);
743 int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
744 int32_t kofs = ofs + (int32_t)offsetof(Node, key);
745 Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
746 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
747 Reg key = RID_NONE, type = RID_TMP, idx = node;
748 RegSet allow = rset_exclude(RSET_GPR, node);
749 int32_t lo, hi;
750 lua_assert(ofs % sizeof(Node) == 0);
751 if (ofs > 32736) {
752 idx = dest;
753 rset_clear(allow, dest);
754 kofs = (int32_t)offsetof(Node, key);
755 } else if (ra_hasreg(dest)) {
756 emit_tsi(as, MIPSI_ADDIU, dest, node, ofs);
757 }
758 if (!irt_ispri(irkey->t)) {
759 key = ra_scratch(as, allow);
760 rset_clear(allow, key);
761 }
762 if (irt_isnum(irkey->t)) {
763 lo = (int32_t)ir_knum(irkey)->u32.lo;
764 hi = (int32_t)ir_knum(irkey)->u32.hi;
765 } else {
766 lo = irkey->i;
767 hi = irt_toitype(irkey->t);
768 if (!ra_hasreg(key))
769 goto nolo;
770 }
771 asm_guard(as, MIPSI_BNE, key, lo ? ra_allock(as, lo, allow) : RID_ZERO);
772 nolo:
773 asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO);
774 if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0));
775 emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4));
776 if (ofs > 32736)
777 emit_tsi(as, MIPSI_ADDU, dest, node, ra_allock(as, ofs, allow));
778 }
779
asm_newref(ASMState * as,IRIns * ir)780 static void asm_newref(ASMState *as, IRIns *ir)
781 {
782 if (ir->r != RID_SINK) {
783 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
784 IRRef args[3];
785 args[0] = ASMREF_L; /* lua_State *L */
786 args[1] = ir->op1; /* GCtab *t */
787 args[2] = ASMREF_TMP1; /* cTValue *key */
788 asm_setupresult(as, ir, ci); /* TValue * */
789 asm_gencall(as, ci, args);
790 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
791 }
792 }
793
asm_uref(ASMState * as,IRIns * ir)794 static void asm_uref(ASMState *as, IRIns *ir)
795 {
796 /* NYI: Check that UREFO is still open and not aliasing a slot. */
797 Reg dest = ra_dest(as, ir, RSET_GPR);
798 if (irref_isk(ir->op1)) {
799 GCfunc *fn = ir_kfunc(IR(ir->op1));
800 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
801 emit_lsptr(as, MIPSI_LW, dest, v, RSET_GPR);
802 } else {
803 Reg uv = ra_scratch(as, RSET_GPR);
804 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
805 if (ir->o == IR_UREFC) {
806 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
807 emit_tsi(as, MIPSI_ADDIU, dest, uv, (int32_t)offsetof(GCupval, tv));
808 emit_tsi(as, MIPSI_LBU, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
809 } else {
810 emit_tsi(as, MIPSI_LW, dest, uv, (int32_t)offsetof(GCupval, v));
811 }
812 emit_tsi(as, MIPSI_LW, uv, func,
813 (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8));
814 }
815 }
816
asm_fref(ASMState * as,IRIns * ir)817 static void asm_fref(ASMState *as, IRIns *ir)
818 {
819 UNUSED(as); UNUSED(ir);
820 lua_assert(!ra_used(ir));
821 }
822
asm_strref(ASMState * as,IRIns * ir)823 static void asm_strref(ASMState *as, IRIns *ir)
824 {
825 Reg dest = ra_dest(as, ir, RSET_GPR);
826 IRRef ref = ir->op2, refk = ir->op1;
827 int32_t ofs = (int32_t)sizeof(GCstr);
828 Reg r;
829 if (irref_isk(ref)) {
830 IRRef tmp = refk; refk = ref; ref = tmp;
831 } else if (!irref_isk(refk)) {
832 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
833 IRIns *irr = IR(ir->op2);
834 if (ra_hasreg(irr->r)) {
835 ra_noweak(as, irr->r);
836 right = irr->r;
837 } else if (mayfuse(as, irr->op2) &&
838 irr->o == IR_ADD && irref_isk(irr->op2) &&
839 checki16(ofs + IR(irr->op2)->i)) {
840 ofs += IR(irr->op2)->i;
841 right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left));
842 } else {
843 right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left));
844 }
845 emit_tsi(as, MIPSI_ADDIU, dest, dest, ofs);
846 emit_dst(as, MIPSI_ADDU, dest, left, right);
847 return;
848 }
849 r = ra_alloc1(as, ref, RSET_GPR);
850 ofs += IR(refk)->i;
851 if (checki16(ofs))
852 emit_tsi(as, MIPSI_ADDIU, dest, r, ofs);
853 else
854 emit_dst(as, MIPSI_ADDU, dest, r,
855 ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
856 }
857
858 /* -- Loads and stores ---------------------------------------------------- */
859
asm_fxloadins(IRIns * ir)860 static MIPSIns asm_fxloadins(IRIns *ir)
861 {
862 switch (irt_type(ir->t)) {
863 case IRT_I8: return MIPSI_LB;
864 case IRT_U8: return MIPSI_LBU;
865 case IRT_I16: return MIPSI_LH;
866 case IRT_U16: return MIPSI_LHU;
867 case IRT_NUM: return MIPSI_LDC1;
868 case IRT_FLOAT: return MIPSI_LWC1;
869 default: return MIPSI_LW;
870 }
871 }
872
asm_fxstoreins(IRIns * ir)873 static MIPSIns asm_fxstoreins(IRIns *ir)
874 {
875 switch (irt_type(ir->t)) {
876 case IRT_I8: case IRT_U8: return MIPSI_SB;
877 case IRT_I16: case IRT_U16: return MIPSI_SH;
878 case IRT_NUM: return MIPSI_SDC1;
879 case IRT_FLOAT: return MIPSI_SWC1;
880 default: return MIPSI_SW;
881 }
882 }
883
asm_fload(ASMState * as,IRIns * ir)884 static void asm_fload(ASMState *as, IRIns *ir)
885 {
886 Reg dest = ra_dest(as, ir, RSET_GPR);
887 Reg idx = ra_alloc1(as, ir->op1, RSET_GPR);
888 MIPSIns mi = asm_fxloadins(ir);
889 int32_t ofs;
890 if (ir->op2 == IRFL_TAB_ARRAY) {
891 ofs = asm_fuseabase(as, ir->op1);
892 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
893 emit_tsi(as, MIPSI_ADDIU, dest, idx, ofs);
894 return;
895 }
896 }
897 ofs = field_ofs[ir->op2];
898 lua_assert(!irt_isfp(ir->t));
899 emit_tsi(as, mi, dest, idx, ofs);
900 }
901
asm_fstore(ASMState * as,IRIns * ir)902 static void asm_fstore(ASMState *as, IRIns *ir)
903 {
904 if (ir->r != RID_SINK) {
905 Reg src = ra_alloc1z(as, ir->op2, RSET_GPR);
906 IRIns *irf = IR(ir->op1);
907 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
908 int32_t ofs = field_ofs[irf->op2];
909 MIPSIns mi = asm_fxstoreins(ir);
910 lua_assert(!irt_isfp(ir->t));
911 emit_tsi(as, mi, src, idx, ofs);
912 }
913 }
914
asm_xload(ASMState * as,IRIns * ir)915 static void asm_xload(ASMState *as, IRIns *ir)
916 {
917 Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
918 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED));
919 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
920 }
921
asm_xstore(ASMState * as,IRIns * ir,int32_t ofs)922 static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
923 {
924 if (ir->r != RID_SINK) {
925 Reg src = ra_alloc1z(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
926 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
927 rset_exclude(RSET_GPR, src), ofs);
928 }
929 }
930
asm_ahuvload(ASMState * as,IRIns * ir)931 static void asm_ahuvload(ASMState *as, IRIns *ir)
932 {
933 IRType1 t = ir->t;
934 Reg dest = RID_NONE, type = RID_TMP, idx;
935 RegSet allow = RSET_GPR;
936 int32_t ofs = 0;
937 if (ra_used(ir)) {
938 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
939 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR);
940 rset_clear(allow, dest);
941 }
942 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
943 rset_clear(allow, idx);
944 if (irt_isnum(t)) {
945 asm_guard(as, MIPSI_BEQ, type, RID_ZERO);
946 emit_tsi(as, MIPSI_SLTIU, type, type, (int32_t)LJ_TISNUM);
947 if (ra_hasreg(dest))
948 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
949 } else {
950 asm_guard(as, MIPSI_BNE, type, ra_allock(as, irt_toitype(t), allow));
951 if (ra_hasreg(dest)) emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0));
952 }
953 emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4));
954 }
955
asm_ahustore(ASMState * as,IRIns * ir)956 static void asm_ahustore(ASMState *as, IRIns *ir)
957 {
958 RegSet allow = RSET_GPR;
959 Reg idx, src = RID_NONE, type = RID_NONE;
960 int32_t ofs = 0;
961 if (ir->r == RID_SINK)
962 return;
963 if (irt_isnum(ir->t)) {
964 src = ra_alloc1(as, ir->op2, RSET_FPR);
965 } else {
966 if (!irt_ispri(ir->t)) {
967 src = ra_alloc1(as, ir->op2, allow);
968 rset_clear(allow, src);
969 }
970 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
971 rset_clear(allow, type);
972 }
973 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
974 if (irt_isnum(ir->t)) {
975 emit_hsi(as, MIPSI_SDC1, src, idx, ofs);
976 } else {
977 if (ra_hasreg(src))
978 emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0));
979 emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4));
980 }
981 }
982
asm_sload(ASMState * as,IRIns * ir)983 static void asm_sload(ASMState *as, IRIns *ir)
984 {
985 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
986 IRType1 t = ir->t;
987 Reg dest = RID_NONE, type = RID_NONE, base;
988 RegSet allow = RSET_GPR;
989 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */
990 lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK));
991 lua_assert(!irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)));
992 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
993 dest = ra_scratch(as, RSET_FPR);
994 asm_tointg(as, ir, dest);
995 t.irt = IRT_NUM; /* Continue with a regular number type check. */
996 } else if (ra_used(ir)) {
997 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
998 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR);
999 rset_clear(allow, dest);
1000 base = ra_alloc1(as, REF_BASE, allow);
1001 rset_clear(allow, base);
1002 if ((ir->op2 & IRSLOAD_CONVERT)) {
1003 if (irt_isint(t)) {
1004 Reg tmp = ra_scratch(as, RSET_FPR);
1005 emit_tg(as, MIPSI_MFC1, dest, tmp);
1006 emit_fg(as, MIPSI_CVT_W_D, tmp, tmp);
1007 dest = tmp;
1008 t.irt = IRT_NUM; /* Check for original type. */
1009 } else {
1010 Reg tmp = ra_scratch(as, RSET_GPR);
1011 emit_fg(as, MIPSI_CVT_D_W, dest, dest);
1012 emit_tg(as, MIPSI_MTC1, tmp, dest);
1013 dest = tmp;
1014 t.irt = IRT_INT; /* Check for original type. */
1015 }
1016 }
1017 goto dotypecheck;
1018 }
1019 base = ra_alloc1(as, REF_BASE, allow);
1020 rset_clear(allow, base);
1021 dotypecheck:
1022 if (irt_isnum(t)) {
1023 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1024 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1025 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)LJ_TISNUM);
1026 type = RID_TMP;
1027 }
1028 if (ra_hasreg(dest)) emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1029 } else {
1030 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1031 Reg ktype = ra_allock(as, irt_toitype(t), allow);
1032 asm_guard(as, MIPSI_BNE, RID_TMP, ktype);
1033 type = RID_TMP;
1034 }
1035 if (ra_hasreg(dest)) emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0));
1036 }
1037 if (ra_hasreg(type)) emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4));
1038 }
1039
1040 /* -- Allocations --------------------------------------------------------- */
1041
1042 #if LJ_HASFFI
asm_cnew(ASMState * as,IRIns * ir)1043 static void asm_cnew(ASMState *as, IRIns *ir)
1044 {
1045 CTState *cts = ctype_ctsG(J2G(as->J));
1046 CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
1047 CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
1048 lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
1049 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1050 IRRef args[2];
1051 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1052 RegSet drop = RSET_SCRATCH;
1053 lua_assert(sz != CTSIZE_INVALID);
1054
1055 args[0] = ASMREF_L; /* lua_State *L */
1056 args[1] = ASMREF_TMP1; /* MSize size */
1057 as->gcsteps++;
1058
1059 if (ra_hasreg(ir->r))
1060 rset_clear(drop, ir->r); /* Dest reg handled below. */
1061 ra_evictset(as, drop);
1062 if (ra_used(ir))
1063 ra_destreg(as, ir, RID_RET); /* GCcdata * */
1064
1065 /* Initialize immutable cdata object. */
1066 if (ir->o == IR_CNEWI) {
1067 int32_t ofs = sizeof(GCcdata);
1068 lua_assert(sz == 4 || sz == 8);
1069 if (sz == 8) {
1070 ofs += 4;
1071 lua_assert((ir+1)->o == IR_HIOP);
1072 if (LJ_LE) ir++;
1073 }
1074 for (;;) {
1075 Reg r = ra_alloc1z(as, ir->op2, allow);
1076 emit_tsi(as, MIPSI_SW, r, RID_RET, ofs);
1077 rset_clear(allow, r);
1078 if (ofs == sizeof(GCcdata)) break;
1079 ofs -= 4; if (LJ_BE) ir++; else ir--;
1080 }
1081 }
1082 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1083 emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1084 emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1085 emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA);
1086 emit_ti(as, MIPSI_LI, RID_TMP, ctypeid); /* Lower 16 bit used. Sign-ext ok. */
1087 asm_gencall(as, ci, args);
1088 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1089 ra_releasetmp(as, ASMREF_TMP1));
1090 }
1091 #else
1092 #define asm_cnew(as, ir) ((void)0)
1093 #endif
1094
1095 /* -- Write barriers ------------------------------------------------------ */
1096
asm_tbar(ASMState * as,IRIns * ir)1097 static void asm_tbar(ASMState *as, IRIns *ir)
1098 {
1099 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1100 Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1101 Reg link = RID_TMP;
1102 MCLabel l_end = emit_label(as);
1103 emit_tsi(as, MIPSI_SW, link, tab, (int32_t)offsetof(GCtab, gclist));
1104 emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked));
1105 emit_setgl(as, tab, gc.grayagain);
1106 emit_getgl(as, link, gc.grayagain);
1107 emit_dst(as, MIPSI_XOR, mark, mark, RID_TMP); /* Clear black bit. */
1108 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1109 emit_tsi(as, MIPSI_ANDI, RID_TMP, mark, LJ_GC_BLACK);
1110 emit_tsi(as, MIPSI_LBU, mark, tab, (int32_t)offsetof(GCtab, marked));
1111 }
1112
asm_obar(ASMState * as,IRIns * ir)1113 static void asm_obar(ASMState *as, IRIns *ir)
1114 {
1115 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1116 IRRef args[2];
1117 MCLabel l_end;
1118 Reg obj, val, tmp;
1119 /* No need for other object barriers (yet). */
1120 lua_assert(IR(ir->op1)->o == IR_UREFC);
1121 ra_evictset(as, RSET_SCRATCH);
1122 l_end = emit_label(as);
1123 args[0] = ASMREF_TMP1; /* global_State *g */
1124 args[1] = ir->op1; /* TValue *tv */
1125 asm_gencall(as, ci, args);
1126 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1127 obj = IR(ir->op1)->r;
1128 tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1129 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1130 emit_tsi(as, MIPSI_ANDI, tmp, tmp, LJ_GC_BLACK);
1131 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1132 emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, LJ_GC_WHITES);
1133 val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1134 emit_tsi(as, MIPSI_LBU, tmp, obj,
1135 (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1136 emit_tsi(as, MIPSI_LBU, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1137 }
1138
1139 /* -- Arithmetic and logic operations ------------------------------------- */
1140
asm_fparith(ASMState * as,IRIns * ir,MIPSIns mi)1141 static void asm_fparith(ASMState *as, IRIns *ir, MIPSIns mi)
1142 {
1143 Reg dest = ra_dest(as, ir, RSET_FPR);
1144 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1145 right = (left >> 8); left &= 255;
1146 emit_fgh(as, mi, dest, left, right);
1147 }
1148
asm_fpunary(ASMState * as,IRIns * ir,MIPSIns mi)1149 static void asm_fpunary(ASMState *as, IRIns *ir, MIPSIns mi)
1150 {
1151 Reg dest = ra_dest(as, ir, RSET_FPR);
1152 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1153 emit_fg(as, mi, dest, left);
1154 }
1155
asm_fpjoin_pow(ASMState * as,IRIns * ir)1156 static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
1157 {
1158 IRIns *irp = IR(ir->op1);
1159 if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
1160 IRIns *irpp = IR(irp->op1);
1161 if (irpp == ir-2 && irpp->o == IR_FPMATH &&
1162 irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
1163 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
1164 IRRef args[2];
1165 args[0] = irpp->op1;
1166 args[1] = irp->op2;
1167 asm_setupresult(as, ir, ci);
1168 asm_gencall(as, ci, args);
1169 return 1;
1170 }
1171 }
1172 return 0;
1173 }
1174
asm_add(ASMState * as,IRIns * ir)1175 static void asm_add(ASMState *as, IRIns *ir)
1176 {
1177 if (irt_isnum(ir->t)) {
1178 asm_fparith(as, ir, MIPSI_ADD_D);
1179 } else {
1180 Reg dest = ra_dest(as, ir, RSET_GPR);
1181 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1182 if (irref_isk(ir->op2)) {
1183 int32_t k = IR(ir->op2)->i;
1184 if (checki16(k)) {
1185 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1186 return;
1187 }
1188 }
1189 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1190 emit_dst(as, MIPSI_ADDU, dest, left, right);
1191 }
1192 }
1193
asm_sub(ASMState * as,IRIns * ir)1194 static void asm_sub(ASMState *as, IRIns *ir)
1195 {
1196 if (irt_isnum(ir->t)) {
1197 asm_fparith(as, ir, MIPSI_SUB_D);
1198 } else {
1199 Reg dest = ra_dest(as, ir, RSET_GPR);
1200 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1201 right = (left >> 8); left &= 255;
1202 emit_dst(as, MIPSI_SUBU, dest, left, right);
1203 }
1204 }
1205
asm_mul(ASMState * as,IRIns * ir)1206 static void asm_mul(ASMState *as, IRIns *ir)
1207 {
1208 if (irt_isnum(ir->t)) {
1209 asm_fparith(as, ir, MIPSI_MUL_D);
1210 } else {
1211 Reg dest = ra_dest(as, ir, RSET_GPR);
1212 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1213 right = (left >> 8); left &= 255;
1214 emit_dst(as, MIPSI_MUL, dest, left, right);
1215 }
1216 }
1217
asm_neg(ASMState * as,IRIns * ir)1218 static void asm_neg(ASMState *as, IRIns *ir)
1219 {
1220 if (irt_isnum(ir->t)) {
1221 asm_fpunary(as, ir, MIPSI_NEG_D);
1222 } else {
1223 Reg dest = ra_dest(as, ir, RSET_GPR);
1224 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1225 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1226 }
1227 }
1228
asm_arithov(ASMState * as,IRIns * ir)1229 static void asm_arithov(ASMState *as, IRIns *ir)
1230 {
1231 Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
1232 if (irref_isk(ir->op2)) {
1233 int k = IR(ir->op2)->i;
1234 if (ir->o == IR_SUBOV) k = -k;
1235 if (checki16(k)) { /* (dest < left) == (k >= 0 ? 1 : 0) */
1236 left = ra_alloc1(as, ir->op1, RSET_GPR);
1237 asm_guard(as, k >= 0 ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1238 emit_dst(as, MIPSI_SLT, RID_TMP, dest, dest == left ? RID_TMP : left);
1239 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1240 if (dest == left) emit_move(as, RID_TMP, left);
1241 return;
1242 }
1243 }
1244 left = ra_alloc2(as, ir, RSET_GPR);
1245 right = (left >> 8); left &= 255;
1246 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1247 right), dest));
1248 asm_guard(as, MIPSI_BLTZ, RID_TMP, 0);
1249 emit_dst(as, MIPSI_AND, RID_TMP, RID_TMP, tmp);
1250 if (ir->o == IR_ADDOV) { /* ((dest^left) & (dest^right)) < 0 */
1251 emit_dst(as, MIPSI_XOR, RID_TMP, dest, dest == right ? RID_TMP : right);
1252 } else { /* ((dest^left) & (dest^~right)) < 0 */
1253 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, dest);
1254 emit_dst(as, MIPSI_NOR, RID_TMP, dest == right ? RID_TMP : right, RID_ZERO);
1255 }
1256 emit_dst(as, MIPSI_XOR, tmp, dest, dest == left ? RID_TMP : left);
1257 emit_dst(as, ir->o == IR_ADDOV ? MIPSI_ADDU : MIPSI_SUBU, dest, left, right);
1258 if (dest == left || dest == right)
1259 emit_move(as, RID_TMP, dest == left ? left : right);
1260 }
1261
asm_mulov(ASMState * as,IRIns * ir)1262 static void asm_mulov(ASMState *as, IRIns *ir)
1263 {
1264 #if LJ_DUALNUM
1265 #error "NYI: MULOV"
1266 #else
1267 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused in single-number mode. */
1268 #endif
1269 }
1270
1271 #if LJ_HASFFI
asm_add64(ASMState * as,IRIns * ir)1272 static void asm_add64(ASMState *as, IRIns *ir)
1273 {
1274 Reg dest = ra_dest(as, ir, RSET_GPR);
1275 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1276 if (irref_isk(ir->op2)) {
1277 int32_t k = IR(ir->op2)->i;
1278 if (k == 0) {
1279 emit_dst(as, MIPSI_ADDU, dest, left, RID_TMP);
1280 goto loarith;
1281 } else if (checki16(k)) {
1282 emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1283 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1284 goto loarith;
1285 }
1286 }
1287 emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1288 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1289 emit_dst(as, MIPSI_ADDU, dest, left, right);
1290 loarith:
1291 ir--;
1292 dest = ra_dest(as, ir, RSET_GPR);
1293 left = ra_alloc1(as, ir->op1, RSET_GPR);
1294 if (irref_isk(ir->op2)) {
1295 int32_t k = IR(ir->op2)->i;
1296 if (k == 0) {
1297 if (dest != left)
1298 emit_move(as, dest, left);
1299 return;
1300 } else if (checki16(k)) {
1301 if (dest == left) {
1302 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, left));
1303 emit_move(as, dest, tmp);
1304 dest = tmp;
1305 }
1306 emit_dst(as, MIPSI_SLTU, RID_TMP, dest, left);
1307 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1308 return;
1309 }
1310 }
1311 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1312 if (dest == left && dest == right) {
1313 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1314 emit_move(as, dest, tmp);
1315 dest = tmp;
1316 }
1317 emit_dst(as, MIPSI_SLTU, RID_TMP, dest, dest == left ? right : left);
1318 emit_dst(as, MIPSI_ADDU, dest, left, right);
1319 }
1320
asm_sub64(ASMState * as,IRIns * ir)1321 static void asm_sub64(ASMState *as, IRIns *ir)
1322 {
1323 Reg dest = ra_dest(as, ir, RSET_GPR);
1324 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1325 right = (left >> 8); left &= 255;
1326 emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1327 emit_dst(as, MIPSI_SUBU, dest, left, right);
1328 ir--;
1329 dest = ra_dest(as, ir, RSET_GPR);
1330 left = ra_alloc2(as, ir, RSET_GPR);
1331 right = (left >> 8); left &= 255;
1332 if (dest == left) {
1333 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1334 emit_move(as, dest, tmp);
1335 dest = tmp;
1336 }
1337 emit_dst(as, MIPSI_SLTU, RID_TMP, left, dest);
1338 emit_dst(as, MIPSI_SUBU, dest, left, right);
1339 }
1340
asm_neg64(ASMState * as,IRIns * ir)1341 static void asm_neg64(ASMState *as, IRIns *ir)
1342 {
1343 Reg dest = ra_dest(as, ir, RSET_GPR);
1344 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1345 emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1346 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1347 ir--;
1348 dest = ra_dest(as, ir, RSET_GPR);
1349 left = ra_alloc1(as, ir->op1, RSET_GPR);
1350 emit_dst(as, MIPSI_SLTU, RID_TMP, RID_ZERO, dest);
1351 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1352 }
1353 #endif
1354
asm_bitnot(ASMState * as,IRIns * ir)1355 static void asm_bitnot(ASMState *as, IRIns *ir)
1356 {
1357 Reg left, right, dest = ra_dest(as, ir, RSET_GPR);
1358 IRIns *irl = IR(ir->op1);
1359 if (mayfuse(as, ir->op1) && irl->o == IR_BOR) {
1360 left = ra_alloc2(as, irl, RSET_GPR);
1361 right = (left >> 8); left &= 255;
1362 } else {
1363 left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1364 right = RID_ZERO;
1365 }
1366 emit_dst(as, MIPSI_NOR, dest, left, right);
1367 }
1368
asm_bitswap(ASMState * as,IRIns * ir)1369 static void asm_bitswap(ASMState *as, IRIns *ir)
1370 {
1371 Reg dest = ra_dest(as, ir, RSET_GPR);
1372 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1373 if ((as->flags & JIT_F_MIPS32R2)) {
1374 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
1375 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
1376 } else {
1377 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), dest));
1378 emit_dst(as, MIPSI_OR, dest, dest, tmp);
1379 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
1380 emit_tsi(as, MIPSI_ANDI, dest, dest, 0xff00);
1381 emit_dta(as, MIPSI_SLL, RID_TMP, RID_TMP, 8);
1382 emit_dta(as, MIPSI_SRL, dest, left, 8);
1383 emit_tsi(as, MIPSI_ANDI, RID_TMP, left, 0xff00);
1384 emit_dst(as, MIPSI_OR, tmp, tmp, RID_TMP);
1385 emit_dta(as, MIPSI_SRL, tmp, left, 24);
1386 emit_dta(as, MIPSI_SLL, RID_TMP, left, 24);
1387 }
1388 }
1389
asm_bitop(ASMState * as,IRIns * ir,MIPSIns mi,MIPSIns mik)1390 static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1391 {
1392 Reg dest = ra_dest(as, ir, RSET_GPR);
1393 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1394 if (irref_isk(ir->op2)) {
1395 int32_t k = IR(ir->op2)->i;
1396 if (checku16(k)) {
1397 emit_tsi(as, mik, dest, left, k);
1398 return;
1399 }
1400 }
1401 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1402 emit_dst(as, mi, dest, left, right);
1403 }
1404
asm_bitshift(ASMState * as,IRIns * ir,MIPSIns mi,MIPSIns mik)1405 static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1406 {
1407 Reg dest = ra_dest(as, ir, RSET_GPR);
1408 if (irref_isk(ir->op2)) { /* Constant shifts. */
1409 uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
1410 emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR), shift);
1411 } else {
1412 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1413 right = (left >> 8); left &= 255;
1414 emit_dst(as, mi, dest, right, left); /* Shift amount is in rs. */
1415 }
1416 }
1417
asm_bitror(ASMState * as,IRIns * ir)1418 static void asm_bitror(ASMState *as, IRIns *ir)
1419 {
1420 if ((as->flags & JIT_F_MIPS32R2)) {
1421 asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
1422 } else {
1423 Reg dest = ra_dest(as, ir, RSET_GPR);
1424 if (irref_isk(ir->op2)) { /* Constant shifts. */
1425 uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
1426 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1427 emit_rotr(as, dest, left, RID_TMP, shift);
1428 } else {
1429 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1430 right = (left >> 8); left &= 255;
1431 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
1432 emit_dst(as, MIPSI_SRLV, dest, right, left);
1433 emit_dst(as, MIPSI_SLLV, RID_TMP, RID_TMP, left);
1434 emit_dst(as, MIPSI_SUBU, RID_TMP, ra_allock(as, 32, RSET_GPR), right);
1435 }
1436 }
1437 }
1438
asm_min_max(ASMState * as,IRIns * ir,int ismax)1439 static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1440 {
1441 if (irt_isnum(ir->t)) {
1442 Reg dest = ra_dest(as, ir, RSET_FPR);
1443 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1444 right = (left >> 8); left &= 255;
1445 if (dest == left) {
1446 emit_fg(as, MIPSI_MOVT_D, dest, right);
1447 } else {
1448 emit_fg(as, MIPSI_MOVF_D, dest, left);
1449 if (dest != right) emit_fg(as, MIPSI_MOV_D, dest, right);
1450 }
1451 emit_fgh(as, MIPSI_C_OLT_D, 0, ismax ? left : right, ismax ? right : left);
1452 } else {
1453 Reg dest = ra_dest(as, ir, RSET_GPR);
1454 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1455 right = (left >> 8); left &= 255;
1456 if (dest == left) {
1457 emit_dst(as, MIPSI_MOVN, dest, right, RID_TMP);
1458 } else {
1459 emit_dst(as, MIPSI_MOVZ, dest, left, RID_TMP);
1460 if (dest != right) emit_move(as, dest, right);
1461 }
1462 emit_dst(as, MIPSI_SLT, RID_TMP,
1463 ismax ? left : right, ismax ? right : left);
1464 }
1465 }
1466
1467 /* -- Comparisons --------------------------------------------------------- */
1468
asm_comp(ASMState * as,IRIns * ir)1469 static void asm_comp(ASMState *as, IRIns *ir)
1470 {
1471 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1472 IROp op = ir->o;
1473 if (irt_isnum(ir->t)) {
1474 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1475 right = (left >> 8); left &= 255;
1476 asm_guard(as, (op&1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
1477 emit_fgh(as, MIPSI_C_OLT_D + ((op&3) ^ ((op>>2)&1)), 0, left, right);
1478 } else {
1479 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1480 if (op == IR_ABC) op = IR_UGT;
1481 if ((op&4) == 0 && irref_isk(ir->op2) && IR(ir->op2)->i == 0) {
1482 MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) :
1483 ((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ);
1484 asm_guard(as, mi, left, 0);
1485 } else {
1486 if (irref_isk(ir->op2)) {
1487 int32_t k = IR(ir->op2)->i;
1488 if ((op&2)) k++;
1489 if (checki16(k)) {
1490 asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1491 emit_tsi(as, (op&4) ? MIPSI_SLTIU : MIPSI_SLTI,
1492 RID_TMP, left, k);
1493 return;
1494 }
1495 }
1496 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1497 asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1498 emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT,
1499 RID_TMP, (op&2) ? right : left, (op&2) ? left : right);
1500 }
1501 }
1502 }
1503
asm_compeq(ASMState * as,IRIns * ir)1504 static void asm_compeq(ASMState *as, IRIns *ir)
1505 {
1506 Reg right, left = ra_alloc2(as, ir, irt_isnum(ir->t) ? RSET_FPR : RSET_GPR);
1507 right = (left >> 8); left &= 255;
1508 if (irt_isnum(ir->t)) {
1509 asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
1510 emit_fgh(as, MIPSI_C_EQ_D, 0, left, right);
1511 } else {
1512 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, left, right);
1513 }
1514 }
1515
1516 #if LJ_HASFFI
1517 /* 64 bit integer comparisons. */
asm_comp64(ASMState * as,IRIns * ir)1518 static void asm_comp64(ASMState *as, IRIns *ir)
1519 {
1520 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1521 IROp op = (ir-1)->o;
1522 MCLabel l_end;
1523 Reg rightlo, leftlo, righthi, lefthi = ra_alloc2(as, ir, RSET_GPR);
1524 righthi = (lefthi >> 8); lefthi &= 255;
1525 leftlo = ra_alloc2(as, ir-1,
1526 rset_exclude(rset_exclude(RSET_GPR, lefthi), righthi));
1527 rightlo = (leftlo >> 8); leftlo &= 255;
1528 asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1529 l_end = emit_label(as);
1530 if (lefthi != righthi)
1531 emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT, RID_TMP,
1532 (op&2) ? righthi : lefthi, (op&2) ? lefthi : righthi);
1533 emit_dst(as, MIPSI_SLTU, RID_TMP,
1534 (op&2) ? rightlo : leftlo, (op&2) ? leftlo : rightlo);
1535 if (lefthi != righthi)
1536 emit_branch(as, MIPSI_BEQ, lefthi, righthi, l_end);
1537 }
1538
asm_comp64eq(ASMState * as,IRIns * ir)1539 static void asm_comp64eq(ASMState *as, IRIns *ir)
1540 {
1541 Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
1542 right = (left >> 8); left &= 255;
1543 asm_guard(as, ((ir-1)->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_TMP, RID_ZERO);
1544 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1545 emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
1546 emit_dst(as, MIPSI_XOR, tmp, left, right);
1547 left = ra_alloc2(as, ir-1, RSET_GPR);
1548 right = (left >> 8); left &= 255;
1549 emit_dst(as, MIPSI_XOR, RID_TMP, left, right);
1550 }
1551 #endif
1552
1553 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1554
1555 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
asm_hiop(ASMState * as,IRIns * ir)1556 static void asm_hiop(ASMState *as, IRIns *ir)
1557 {
1558 #if LJ_HASFFI
1559 /* HIOP is marked as a store because it needs its own DCE logic. */
1560 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1561 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1562 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
1563 as->curins--; /* Always skip the CONV. */
1564 if (usehi || uselo)
1565 asm_conv64(as, ir);
1566 return;
1567 } else if ((ir-1)->o < IR_EQ) { /* 64 bit integer comparisons. ORDER IR. */
1568 as->curins--; /* Always skip the loword comparison. */
1569 asm_comp64(as, ir);
1570 return;
1571 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
1572 as->curins--; /* Always skip the loword comparison. */
1573 asm_comp64eq(as, ir);
1574 return;
1575 } else if ((ir-1)->o == IR_XSTORE) {
1576 as->curins--; /* Handle both stores here. */
1577 if ((ir-1)->r != RID_SINK) {
1578 asm_xstore(as, ir, LJ_LE ? 4 : 0);
1579 asm_xstore(as, ir-1, LJ_LE ? 0 : 4);
1580 }
1581 return;
1582 }
1583 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1584 switch ((ir-1)->o) {
1585 case IR_ADD: as->curins--; asm_add64(as, ir); break;
1586 case IR_SUB: as->curins--; asm_sub64(as, ir); break;
1587 case IR_NEG: as->curins--; asm_neg64(as, ir); break;
1588 case IR_CALLN:
1589 case IR_CALLXS:
1590 if (!uselo)
1591 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
1592 break;
1593 case IR_CNEWI:
1594 /* Nothing to do here. Handled by lo op itself. */
1595 break;
1596 default: lua_assert(0); break;
1597 }
1598 #else
1599 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused without FFI. */
1600 #endif
1601 }
1602
1603 /* -- Stack handling ------------------------------------------------------ */
1604
1605 /* Check Lua stack size for overflow. Use exit handler as fallback. */
asm_stack_check(ASMState * as,BCReg topslot,IRIns * irp,RegSet allow,ExitNo exitno)1606 static void asm_stack_check(ASMState *as, BCReg topslot,
1607 IRIns *irp, RegSet allow, ExitNo exitno)
1608 {
1609 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1610 Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
1611 ExitNo oldsnap = as->snapno;
1612 rset_clear(allow, pbase);
1613 tmp = allow ? rset_pickbot(allow) :
1614 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
1615 as->snapno = exitno;
1616 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
1617 as->snapno = oldsnap;
1618 if (allow == RSET_EMPTY) /* Restore temp. register. */
1619 emit_tsi(as, MIPSI_LW, tmp, RID_SP, 0);
1620 else
1621 ra_modified(as, tmp);
1622 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot));
1623 emit_dst(as, MIPSI_SUBU, RID_TMP, tmp, pbase);
1624 emit_tsi(as, MIPSI_LW, tmp, tmp, offsetof(lua_State, maxstack));
1625 if (pbase == RID_TMP)
1626 emit_getgl(as, RID_TMP, jit_base);
1627 emit_getgl(as, tmp, jit_L);
1628 if (allow == RSET_EMPTY) /* Spill temp. register. */
1629 emit_tsi(as, MIPSI_SW, tmp, RID_SP, 0);
1630 }
1631
1632 /* Restore Lua stack from on-trace state. */
asm_stack_restore(ASMState * as,SnapShot * snap)1633 static void asm_stack_restore(ASMState *as, SnapShot *snap)
1634 {
1635 SnapEntry *map = &as->T->snapmap[snap->mapofs];
1636 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1];
1637 MSize n, nent = snap->nent;
1638 /* Store the value of all modified slots to the Lua stack. */
1639 for (n = 0; n < nent; n++) {
1640 SnapEntry sn = map[n];
1641 BCReg s = snap_slot(sn);
1642 int32_t ofs = 8*((int32_t)s-1);
1643 IRRef ref = snap_ref(sn);
1644 IRIns *ir = IR(ref);
1645 if ((sn & SNAP_NORESTORE))
1646 continue;
1647 if (irt_isnum(ir->t)) {
1648 Reg src = ra_alloc1(as, ref, RSET_FPR);
1649 emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs);
1650 } else {
1651 Reg type;
1652 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1653 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
1654 if (!irt_ispri(ir->t)) {
1655 Reg src = ra_alloc1(as, ref, allow);
1656 rset_clear(allow, src);
1657 emit_tsi(as, MIPSI_SW, src, RID_BASE, ofs+(LJ_BE?4:0));
1658 }
1659 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1660 if (s == 0) continue; /* Do not overwrite link to previous frame. */
1661 type = ra_allock(as, (int32_t)(*flinks--), allow);
1662 } else {
1663 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1664 }
1665 emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4));
1666 }
1667 checkmclim(as);
1668 }
1669 lua_assert(map + nent == flinks);
1670 }
1671
1672 /* -- GC handling --------------------------------------------------------- */
1673
1674 /* Check GC threshold and do one or more GC steps. */
asm_gc_check(ASMState * as)1675 static void asm_gc_check(ASMState *as)
1676 {
1677 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
1678 IRRef args[2];
1679 MCLabel l_end;
1680 Reg tmp;
1681 ra_evictset(as, RSET_SCRATCH);
1682 l_end = emit_label(as);
1683 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1684 /* Assumes asm_snap_prep() already done. */
1685 asm_guard(as, MIPSI_BNE, RID_RET, RID_ZERO);
1686 args[0] = ASMREF_TMP1; /* global_State *g */
1687 args[1] = ASMREF_TMP2; /* MSize steps */
1688 asm_gencall(as, ci, args);
1689 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1690 tmp = ra_releasetmp(as, ASMREF_TMP2);
1691 emit_loadi(as, tmp, as->gcsteps);
1692 /* Jump around GC step if GC total < GC threshold. */
1693 emit_branch(as, MIPSI_BNE, RID_TMP, RID_ZERO, l_end);
1694 emit_dst(as, MIPSI_SLTU, RID_TMP, RID_TMP, tmp);
1695 emit_getgl(as, tmp, gc.threshold);
1696 emit_getgl(as, RID_TMP, gc.total);
1697 as->gcsteps = 0;
1698 checkmclim(as);
1699 }
1700
1701 /* -- Loop handling ------------------------------------------------------- */
1702
1703 /* Fixup the loop branch. */
asm_loop_fixup(ASMState * as)1704 static void asm_loop_fixup(ASMState *as)
1705 {
1706 MCode *p = as->mctop;
1707 MCode *target = as->mcp;
1708 p[-1] = MIPSI_NOP;
1709 if (as->loopinv) { /* Inverted loop branch? */
1710 /* asm_guard already inverted the cond branch. Only patch the target. */
1711 p[-3] |= ((target-p+2) & 0x0000ffffu);
1712 } else {
1713 p[-2] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
1714 }
1715 }
1716
1717 /* -- Head of trace ------------------------------------------------------- */
1718
1719 /* Coalesce BASE register for a root trace. */
asm_head_root_base(ASMState * as)1720 static void asm_head_root_base(ASMState *as)
1721 {
1722 IRIns *ir = IR(REF_BASE);
1723 Reg r = ir->r;
1724 if (as->loopinv) as->mctop--;
1725 if (ra_hasreg(r)) {
1726 ra_free(as, r);
1727 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1728 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
1729 if (r != RID_BASE)
1730 emit_move(as, r, RID_BASE);
1731 }
1732 }
1733
1734 /* Coalesce BASE register for a side trace. */
asm_head_side_base(ASMState * as,IRIns * irp,RegSet allow)1735 static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow)
1736 {
1737 IRIns *ir = IR(REF_BASE);
1738 Reg r = ir->r;
1739 if (as->loopinv) as->mctop--;
1740 if (ra_hasreg(r)) {
1741 ra_free(as, r);
1742 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1743 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
1744 if (irp->r == r) {
1745 rset_clear(allow, r); /* Mark same BASE register as coalesced. */
1746 } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
1747 rset_clear(allow, irp->r);
1748 emit_move(as, r, irp->r); /* Move from coalesced parent reg. */
1749 } else {
1750 emit_getgl(as, r, jit_base); /* Otherwise reload BASE. */
1751 }
1752 }
1753 return allow;
1754 }
1755
1756 /* -- Tail of trace ------------------------------------------------------- */
1757
1758 /* Fixup the tail code. */
asm_tail_fixup(ASMState * as,TraceNo lnk)1759 static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1760 {
1761 MCode *target = lnk ? traceref(as->J,lnk)->mcode : (MCode *)lj_vm_exit_interp;
1762 int32_t spadj = as->T->spadjust;
1763 MCode *p = as->mctop-1;
1764 *p = spadj ? (MIPSI_ADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
1765 p[-1] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
1766 }
1767
1768 /* Prepare tail of code. */
asm_tail_prep(ASMState * as)1769 static void asm_tail_prep(ASMState *as)
1770 {
1771 as->mcp = as->mctop-2; /* Leave room for branch plus nop or stack adj. */
1772 as->invmcp = as->loopref ? as->mcp : NULL;
1773 }
1774
1775 /* -- Instruction dispatch ------------------------------------------------ */
1776
1777 /* Assemble a single instruction. */
asm_ir(ASMState * as,IRIns * ir)1778 static void asm_ir(ASMState *as, IRIns *ir)
1779 {
1780 switch ((IROp)ir->o) {
1781 /* Miscellaneous ops. */
1782 case IR_LOOP: asm_loop(as); break;
1783 case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
1784 case IR_USE:
1785 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
1786 case IR_PHI: asm_phi(as, ir); break;
1787 case IR_HIOP: asm_hiop(as, ir); break;
1788 case IR_GCSTEP: asm_gcstep(as, ir); break;
1789
1790 /* Guarded assertions. */
1791 case IR_EQ: case IR_NE: asm_compeq(as, ir); break;
1792 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
1793 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
1794 case IR_ABC:
1795 asm_comp(as, ir);
1796 break;
1797
1798 case IR_RETF: asm_retf(as, ir); break;
1799
1800 /* Bit ops. */
1801 case IR_BNOT: asm_bitnot(as, ir); break;
1802 case IR_BSWAP: asm_bitswap(as, ir); break;
1803
1804 case IR_BAND: asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI); break;
1805 case IR_BOR: asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI); break;
1806 case IR_BXOR: asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI); break;
1807
1808 case IR_BSHL: asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL); break;
1809 case IR_BSHR: asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL); break;
1810 case IR_BSAR: asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA); break;
1811 case IR_BROL: lua_assert(0); break;
1812 case IR_BROR: asm_bitror(as, ir); break;
1813
1814 /* Arithmetic ops. */
1815 case IR_ADD: asm_add(as, ir); break;
1816 case IR_SUB: asm_sub(as, ir); break;
1817 case IR_MUL: asm_mul(as, ir); break;
1818 case IR_DIV: asm_fparith(as, ir, MIPSI_DIV_D); break;
1819 case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
1820 case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
1821 case IR_NEG: asm_neg(as, ir); break;
1822
1823 case IR_ABS: asm_fpunary(as, ir, MIPSI_ABS_D); break;
1824 case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
1825 case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
1826 case IR_MIN: asm_min_max(as, ir, 0); break;
1827 case IR_MAX: asm_min_max(as, ir, 1); break;
1828 case IR_FPMATH:
1829 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
1830 break;
1831 if (ir->op2 <= IRFPM_TRUNC)
1832 asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
1833 else if (ir->op2 == IRFPM_SQRT)
1834 asm_fpunary(as, ir, MIPSI_SQRT_D);
1835 else
1836 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1837 break;
1838
1839 /* Overflow-checking arithmetic ops. */
1840 case IR_ADDOV: asm_arithov(as, ir); break;
1841 case IR_SUBOV: asm_arithov(as, ir); break;
1842 case IR_MULOV: asm_mulov(as, ir); break;
1843
1844 /* Memory references. */
1845 case IR_AREF: asm_aref(as, ir); break;
1846 case IR_HREF: asm_href(as, ir); break;
1847 case IR_HREFK: asm_hrefk(as, ir); break;
1848 case IR_NEWREF: asm_newref(as, ir); break;
1849 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
1850 case IR_FREF: asm_fref(as, ir); break;
1851 case IR_STRREF: asm_strref(as, ir); break;
1852
1853 /* Loads and stores. */
1854 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
1855 asm_ahuvload(as, ir);
1856 break;
1857 case IR_FLOAD: asm_fload(as, ir); break;
1858 case IR_XLOAD: asm_xload(as, ir); break;
1859 case IR_SLOAD: asm_sload(as, ir); break;
1860
1861 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
1862 case IR_FSTORE: asm_fstore(as, ir); break;
1863 case IR_XSTORE: asm_xstore(as, ir, 0); break;
1864
1865 /* Allocations. */
1866 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
1867 case IR_TNEW: asm_tnew(as, ir); break;
1868 case IR_TDUP: asm_tdup(as, ir); break;
1869 case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
1870
1871 /* Write barriers. */
1872 case IR_TBAR: asm_tbar(as, ir); break;
1873 case IR_OBAR: asm_obar(as, ir); break;
1874
1875 /* Type conversions. */
1876 case IR_CONV: asm_conv(as, ir); break;
1877 case IR_TOBIT: asm_tobit(as, ir); break;
1878 case IR_TOSTR: asm_tostr(as, ir); break;
1879 case IR_STRTO: asm_strto(as, ir); break;
1880
1881 /* Calls. */
1882 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
1883 case IR_CALLXS: asm_callx(as, ir); break;
1884 case IR_CARG: break;
1885
1886 default:
1887 setintV(&as->J->errinfo, ir->o);
1888 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
1889 break;
1890 }
1891 }
1892
1893 /* -- Trace setup --------------------------------------------------------- */
1894
1895 /* Ensure there are enough stack slots for call arguments. */
asm_setup_call_slots(ASMState * as,IRIns * ir,const CCallInfo * ci)1896 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
1897 {
1898 IRRef args[CCI_NARGS_MAX*2];
1899 uint32_t i, nargs = (int)CCI_NARGS(ci);
1900 int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
1901 asm_collectargs(as, ir, ci, args);
1902 for (i = 0; i < nargs; i++) {
1903 if (args[i] && irt_isfp(IR(args[i])->t) &&
1904 nfpr > 0 && !(ci->flags & CCI_VARARG)) {
1905 nfpr--;
1906 ngpr -= irt_isnum(IR(args[i])->t) ? 2 : 1;
1907 } else if (args[i] && irt_isnum(IR(args[i])->t)) {
1908 nfpr = 0;
1909 ngpr = ngpr & ~1;
1910 if (ngpr > 0) ngpr -= 2; else nslots = (nslots+3) & ~1;
1911 } else {
1912 nfpr = 0;
1913 if (ngpr > 0) ngpr--; else nslots++;
1914 }
1915 }
1916 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
1917 as->evenspill = nslots;
1918 return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
1919 }
1920
asm_setup_target(ASMState * as)1921 static void asm_setup_target(ASMState *as)
1922 {
1923 asm_sparejump_setup(as);
1924 asm_exitstub_setup(as);
1925 }
1926
1927 /* -- Trace patching ------------------------------------------------------ */
1928
1929 /* Patch exit jumps of existing machine code to a new target. */
lj_asm_patchexit(jit_State * J,GCtrace * T,ExitNo exitno,MCode * target)1930 void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
1931 {
1932 MCode *p = T->mcode;
1933 MCode *pe = (MCode *)((char *)p + T->szmcode);
1934 MCode *px = exitstub_trace_addr(T, exitno);
1935 MCode *cstart = NULL, *cstop = NULL;
1936 MCode *mcarea = lj_mcode_patch(J, p, 0);
1937 MCode exitload = MIPSI_LI | MIPSF_T(RID_TMP) | exitno;
1938 MCode tjump = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
1939 for (p++; p < pe; p++) {
1940 if (*p == exitload) { /* Look for load of exit number. */
1941 if (((p[-1] ^ (px-p)) & 0xffffu) == 0) { /* Look for exitstub branch. */
1942 ptrdiff_t delta = target - p;
1943 if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */
1944 patchbranch:
1945 p[-1] = (p[-1] & 0xffff0000u) | (delta & 0xffffu);
1946 *p = MIPSI_NOP; /* Replace the load of the exit number. */
1947 cstop = p;
1948 if (!cstart) cstart = p-1;
1949 } else { /* Branch out of range. Use spare jump slot in mcarea. */
1950 int i;
1951 for (i = 2; i < 2+MIPS_SPAREJUMP*2; i += 2) {
1952 if (mcarea[i] == tjump) {
1953 delta = mcarea+i - p;
1954 goto patchbranch;
1955 } else if (mcarea[i] == MIPSI_NOP) {
1956 mcarea[i] = tjump;
1957 cstart = mcarea+i;
1958 delta = mcarea+i - p;
1959 goto patchbranch;
1960 }
1961 }
1962 /* Ignore jump slot overflow. Child trace is simply not attached. */
1963 }
1964 } else if (p+1 == pe) {
1965 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
1966 lua_assert(p[1] == MIPSI_NOP);
1967 p[1] = tjump;
1968 *p = MIPSI_NOP; /* Replace the load of the exit number. */
1969 cstop = p+2;
1970 if (!cstart) cstart = p+1;
1971 }
1972 }
1973 }
1974 if (cstart) lj_mcode_sync(cstart, cstop);
1975 lj_mcode_patch(J, mcarea, 1);
1976 }
1977
1978