1 /*
2 ** MIPS IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 /* -- Register allocator extensions --------------------------------------- */
7
8 /* Allocate a register with a hint. */
ra_hintalloc(ASMState * as,IRRef ref,Reg hint,RegSet allow)9 static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
10 {
11 Reg r = IR(ref)->r;
12 if (ra_noreg(r)) {
13 if (!ra_hashint(r) && !iscrossref(as, ref))
14 ra_sethint(IR(ref)->r, hint); /* Propagate register hint. */
15 r = ra_allocref(as, ref, allow);
16 }
17 ra_noweak(as, r);
18 return r;
19 }
20
21 /* Allocate a register or RID_ZERO. */
ra_alloc1z(ASMState * as,IRRef ref,RegSet allow)22 static Reg ra_alloc1z(ASMState *as, IRRef ref, RegSet allow)
23 {
24 Reg r = IR(ref)->r;
25 if (ra_noreg(r)) {
26 if (!(allow & RSET_FPR) && irref_isk(ref) && IR(ref)->i == 0)
27 return RID_ZERO;
28 r = ra_allocref(as, ref, allow);
29 } else {
30 ra_noweak(as, r);
31 }
32 return r;
33 }
34
35 /* Allocate two source registers for three-operand instructions. */
ra_alloc2(ASMState * as,IRIns * ir,RegSet allow)36 static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
37 {
38 IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
39 Reg left = irl->r, right = irr->r;
40 if (ra_hasreg(left)) {
41 ra_noweak(as, left);
42 if (ra_noreg(right))
43 right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
44 else
45 ra_noweak(as, right);
46 } else if (ra_hasreg(right)) {
47 ra_noweak(as, right);
48 left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
49 } else if (ra_hashint(right)) {
50 right = ra_alloc1z(as, ir->op2, allow);
51 left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
52 } else {
53 left = ra_alloc1z(as, ir->op1, allow);
54 right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
55 }
56 return left | (right << 8);
57 }
58
59 /* -- Guard handling ------------------------------------------------------ */
60
61 /* Need some spare long-range jump slots, for out-of-range branches. */
62 #define MIPS_SPAREJUMP 4
63
64 /* Setup spare long-range jump slots per mcarea. */
asm_sparejump_setup(ASMState * as)65 static void asm_sparejump_setup(ASMState *as)
66 {
67 MCode *mxp = as->mcbot;
68 /* Assumes sizeof(MCLink) == 8. */
69 if (((uintptr_t)mxp & (LJ_PAGESIZE-1)) == 8) {
70 lua_assert(MIPSI_NOP == 0);
71 memset(mxp+2, 0, MIPS_SPAREJUMP*8);
72 mxp += MIPS_SPAREJUMP*2;
73 lua_assert(mxp < as->mctop);
74 lj_mcode_sync(as->mcbot, mxp);
75 lj_mcode_commitbot(as->J, mxp);
76 as->mcbot = mxp;
77 as->mclim = as->mcbot + MCLIM_REDZONE;
78 }
79 }
80
81 /* Setup exit stub after the end of each trace. */
asm_exitstub_setup(ASMState * as)82 static void asm_exitstub_setup(ASMState *as)
83 {
84 MCode *mxp = as->mctop;
85 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
86 *--mxp = MIPSI_LI|MIPSF_T(RID_TMP)|as->T->traceno;
87 *--mxp = MIPSI_J|((((uintptr_t)(void *)lj_vm_exit_handler)>>2)&0x03ffffffu);
88 lua_assert(((uintptr_t)mxp ^ (uintptr_t)(void *)lj_vm_exit_handler)>>28 == 0);
89 *--mxp = MIPSI_SW|MIPSF_T(RID_TMP)|MIPSF_S(RID_SP)|0;
90 as->mctop = mxp;
91 }
92
93 /* Keep this in-sync with exitstub_trace_addr(). */
94 #define asm_exitstub_addr(as) ((as)->mctop)
95
96 /* Emit conditional branch to exit for guard. */
asm_guard(ASMState * as,MIPSIns mi,Reg rs,Reg rt)97 static void asm_guard(ASMState *as, MIPSIns mi, Reg rs, Reg rt)
98 {
99 MCode *target = asm_exitstub_addr(as);
100 MCode *p = as->mcp;
101 if (LJ_UNLIKELY(p == as->invmcp)) {
102 as->invmcp = NULL;
103 as->loopinv = 1;
104 as->mcp = p+1;
105 mi = mi ^ ((mi>>28) == 1 ? 0x04000000u : 0x00010000u); /* Invert cond. */
106 target = p; /* Patch target later in asm_loop_fixup. */
107 }
108 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
109 emit_branch(as, mi, rs, rt, target);
110 }
111
112 /* -- Operand fusion ------------------------------------------------------ */
113
114 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
115 #define CONFLICT_SEARCH_LIM 31
116
117 /* Check if there's no conflicting instruction between curins and ref. */
noconflict(ASMState * as,IRRef ref,IROp conflict)118 static int noconflict(ASMState *as, IRRef ref, IROp conflict)
119 {
120 IRIns *ir = as->ir;
121 IRRef i = as->curins;
122 if (i > ref + CONFLICT_SEARCH_LIM)
123 return 0; /* Give up, ref is too far away. */
124 while (--i > ref)
125 if (ir[i].o == conflict)
126 return 0; /* Conflict found. */
127 return 1; /* Ok, no conflict. */
128 }
129
130 /* Fuse the array base of colocated arrays. */
asm_fuseabase(ASMState * as,IRRef ref)131 static int32_t asm_fuseabase(ASMState *as, IRRef ref)
132 {
133 IRIns *ir = IR(ref);
134 if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
135 !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
136 return (int32_t)sizeof(GCtab);
137 return 0;
138 }
139
140 /* Fuse array/hash/upvalue reference into register+offset operand. */
asm_fuseahuref(ASMState * as,IRRef ref,int32_t * ofsp,RegSet allow)141 static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
142 {
143 IRIns *ir = IR(ref);
144 if (ra_noreg(ir->r)) {
145 if (ir->o == IR_AREF) {
146 if (mayfuse(as, ref)) {
147 if (irref_isk(ir->op2)) {
148 IRRef tab = IR(ir->op1)->op1;
149 int32_t ofs = asm_fuseabase(as, tab);
150 IRRef refa = ofs ? tab : ir->op1;
151 ofs += 8*IR(ir->op2)->i;
152 if (checki16(ofs)) {
153 *ofsp = ofs;
154 return ra_alloc1(as, refa, allow);
155 }
156 }
157 }
158 } else if (ir->o == IR_HREFK) {
159 if (mayfuse(as, ref)) {
160 int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
161 if (checki16(ofs)) {
162 *ofsp = ofs;
163 return ra_alloc1(as, ir->op1, allow);
164 }
165 }
166 } else if (ir->o == IR_UREFC) {
167 if (irref_isk(ir->op1)) {
168 GCfunc *fn = ir_kfunc(IR(ir->op1));
169 int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv);
170 int32_t jgl = (intptr_t)J2G(as->J);
171 if ((uint32_t)(ofs-jgl) < 65536) {
172 *ofsp = ofs-jgl-32768;
173 return RID_JGL;
174 } else {
175 *ofsp = (int16_t)ofs;
176 return ra_allock(as, ofs-(int16_t)ofs, allow);
177 }
178 }
179 }
180 }
181 *ofsp = 0;
182 return ra_alloc1(as, ref, allow);
183 }
184
185 /* Fuse XLOAD/XSTORE reference into load/store operand. */
asm_fusexref(ASMState * as,MIPSIns mi,Reg rt,IRRef ref,RegSet allow,int32_t ofs)186 static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
187 RegSet allow, int32_t ofs)
188 {
189 IRIns *ir = IR(ref);
190 Reg base;
191 if (ra_noreg(ir->r) && canfuse(as, ir)) {
192 if (ir->o == IR_ADD) {
193 int32_t ofs2;
194 if (irref_isk(ir->op2) && (ofs2 = ofs + IR(ir->op2)->i, checki16(ofs2))) {
195 ref = ir->op1;
196 ofs = ofs2;
197 }
198 } else if (ir->o == IR_STRREF) {
199 int32_t ofs2 = 65536;
200 lua_assert(ofs == 0);
201 ofs = (int32_t)sizeof(GCstr);
202 if (irref_isk(ir->op2)) {
203 ofs2 = ofs + IR(ir->op2)->i;
204 ref = ir->op1;
205 } else if (irref_isk(ir->op1)) {
206 ofs2 = ofs + IR(ir->op1)->i;
207 ref = ir->op2;
208 }
209 if (!checki16(ofs2)) {
210 /* NYI: Fuse ADD with constant. */
211 Reg right, left = ra_alloc2(as, ir, allow);
212 right = (left >> 8); left &= 255;
213 emit_hsi(as, mi, rt, RID_TMP, ofs);
214 emit_dst(as, MIPSI_ADDU, RID_TMP, left, right);
215 return;
216 }
217 ofs = ofs2;
218 }
219 }
220 base = ra_alloc1(as, ref, allow);
221 emit_hsi(as, mi, rt, base, ofs);
222 }
223
224 /* -- Calls --------------------------------------------------------------- */
225
226 /* Generate a call to a C function. */
asm_gencall(ASMState * as,const CCallInfo * ci,IRRef * args)227 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
228 {
229 uint32_t n, nargs = CCI_XNARGS(ci);
230 int32_t ofs = 16;
231 #if LJ_SOFTFP
232 Reg gpr = REGARG_FIRSTGPR;
233 #else
234 Reg gpr, fpr = REGARG_FIRSTFPR;
235 #endif
236 if ((void *)ci->func)
237 emit_call(as, (void *)ci->func, 1);
238 #if !LJ_SOFTFP
239 for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
240 as->cost[gpr] = REGCOST(~0u, ASMREF_L);
241 gpr = REGARG_FIRSTGPR;
242 #endif
243 for (n = 0; n < nargs; n++) { /* Setup args. */
244 IRRef ref = args[n];
245 if (ref) {
246 IRIns *ir = IR(ref);
247 #if !LJ_SOFTFP
248 if (irt_isfp(ir->t) && fpr <= REGARG_LASTFPR &&
249 !(ci->flags & CCI_VARARG)) {
250 lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */
251 ra_leftov(as, fpr, ref);
252 fpr += 2;
253 gpr += irt_isnum(ir->t) ? 2 : 1;
254 } else
255 #endif
256 {
257 #if !LJ_SOFTFP
258 fpr = REGARG_LASTFPR+1;
259 #endif
260 if (irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
261 if (gpr <= REGARG_LASTGPR) {
262 lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */
263 #if !LJ_SOFTFP
264 if (irt_isfp(ir->t)) {
265 RegSet of = as->freeset;
266 Reg r;
267 /* Workaround to protect argument GPRs from being used for remat. */
268 as->freeset &= ~RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1);
269 r = ra_alloc1(as, ref, RSET_FPR);
270 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
271 if (irt_isnum(ir->t)) {
272 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1);
273 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r);
274 lua_assert(rset_test(as->freeset, gpr+1)); /* Already evicted. */
275 gpr += 2;
276 } else if (irt_isfloat(ir->t)) {
277 emit_tg(as, MIPSI_MFC1, gpr, r);
278 gpr++;
279 }
280 } else
281 #endif
282 {
283 ra_leftov(as, gpr, ref);
284 gpr++;
285 }
286 } else {
287 Reg r = ra_alloc1z(as, ref, !LJ_SOFTFP && irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
288 if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
289 emit_spstore(as, ir, r, ofs);
290 ofs += irt_isnum(ir->t) ? 8 : 4;
291 }
292 }
293 } else {
294 #if !LJ_SOFTFP
295 fpr = REGARG_LASTFPR+1;
296 #endif
297 if (gpr <= REGARG_LASTGPR)
298 gpr++;
299 else
300 ofs += 4;
301 }
302 checkmclim(as);
303 }
304 }
305
306 /* Setup result reg/sp for call. Evict scratch regs. */
asm_setupresult(ASMState * as,IRIns * ir,const CCallInfo * ci)307 static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
308 {
309 RegSet drop = RSET_SCRATCH;
310 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
311 #if !LJ_SOFTFP
312 if ((ci->flags & CCI_NOFPRCLOBBER))
313 drop &= ~RSET_FPR;
314 #endif
315 if (ra_hasreg(ir->r))
316 rset_clear(drop, ir->r); /* Dest reg handled below. */
317 if (hiop && ra_hasreg((ir+1)->r))
318 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
319 ra_evictset(as, drop); /* Evictions must be performed first. */
320 if (ra_used(ir)) {
321 lua_assert(!irt_ispri(ir->t));
322 if (!LJ_SOFTFP && irt_isfp(ir->t)) {
323 if ((ci->flags & CCI_CASTU64)) {
324 int32_t ofs = sps_scale(ir->s);
325 Reg dest = ir->r;
326 if (ra_hasreg(dest)) {
327 ra_free(as, dest);
328 ra_modified(as, dest);
329 emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1);
330 emit_tg(as, MIPSI_MTC1, RID_RETLO, dest);
331 }
332 if (ofs) {
333 emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0));
334 emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4));
335 }
336 } else {
337 ra_destreg(as, ir, RID_FPRET);
338 }
339 } else if (hiop) {
340 ra_destpair(as, ir);
341 } else {
342 ra_destreg(as, ir, RID_RET);
343 }
344 }
345 }
346
asm_callx(ASMState * as,IRIns * ir)347 static void asm_callx(ASMState *as, IRIns *ir)
348 {
349 IRRef args[CCI_NARGS_MAX*2];
350 CCallInfo ci;
351 IRRef func;
352 IRIns *irf;
353 ci.flags = asm_callx_flags(as, ir);
354 asm_collectargs(as, ir, &ci, args);
355 asm_setupresult(as, ir, &ci);
356 func = ir->op2; irf = IR(func);
357 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
358 if (irref_isk(func)) { /* Call to constant address. */
359 ci.func = (ASMFunction)(void *)(irf->i);
360 } else { /* Need specific register for indirect calls. */
361 Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR));
362 MCode *p = as->mcp;
363 if (r == RID_CFUNCADDR)
364 *--p = MIPSI_NOP;
365 else
366 *--p = MIPSI_MOVE | MIPSF_D(RID_CFUNCADDR) | MIPSF_S(r);
367 *--p = MIPSI_JALR | MIPSF_S(r);
368 as->mcp = p;
369 ci.func = (ASMFunction)(void *)0;
370 }
371 asm_gencall(as, &ci, args);
372 }
373
374 #if !LJ_SOFTFP
asm_callround(ASMState * as,IRIns * ir,IRCallID id)375 static void asm_callround(ASMState *as, IRIns *ir, IRCallID id)
376 {
377 /* The modified regs must match with the *.dasc implementation. */
378 RegSet drop = RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_FPRET)|
379 RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(REGARG_FIRSTFPR);
380 if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
381 ra_evictset(as, drop);
382 ra_destreg(as, ir, RID_FPRET);
383 emit_call(as, (void *)lj_ir_callinfo[id].func, 0);
384 ra_leftov(as, REGARG_FIRSTFPR, ir->op1);
385 }
386 #endif
387
388 /* -- Returns ------------------------------------------------------------- */
389
390 /* Return to lower frame. Guard that it goes to the right spot. */
asm_retf(ASMState * as,IRIns * ir)391 static void asm_retf(ASMState *as, IRIns *ir)
392 {
393 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
394 void *pc = ir_kptr(IR(ir->op2));
395 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
396 as->topslot -= (BCReg)delta;
397 if ((int32_t)as->topslot < 0) as->topslot = 0;
398 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
399 emit_setgl(as, base, jit_base);
400 emit_addptr(as, base, -8*delta);
401 asm_guard(as, MIPSI_BNE, RID_TMP,
402 ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base)));
403 emit_tsi(as, MIPSI_LW, RID_TMP, base, -8);
404 }
405
406 /* -- Type conversions ---------------------------------------------------- */
407
408 #if !LJ_SOFTFP
asm_tointg(ASMState * as,IRIns * ir,Reg left)409 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
410 {
411 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
412 Reg dest = ra_dest(as, ir, RSET_GPR);
413 asm_guard(as, MIPSI_BC1F, 0, 0);
414 emit_fgh(as, MIPSI_C_EQ_D, 0, tmp, left);
415 emit_fg(as, MIPSI_CVT_D_W, tmp, tmp);
416 emit_tg(as, MIPSI_MFC1, dest, tmp);
417 emit_fg(as, MIPSI_CVT_W_D, tmp, left);
418 }
419
asm_tobit(ASMState * as,IRIns * ir)420 static void asm_tobit(ASMState *as, IRIns *ir)
421 {
422 RegSet allow = RSET_FPR;
423 Reg dest = ra_dest(as, ir, RSET_GPR);
424 Reg left = ra_alloc1(as, ir->op1, allow);
425 Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
426 Reg tmp = ra_scratch(as, rset_clear(allow, right));
427 emit_tg(as, MIPSI_MFC1, dest, tmp);
428 emit_fgh(as, MIPSI_ADD_D, tmp, left, right);
429 }
430 #endif
431
asm_conv(ASMState * as,IRIns * ir)432 static void asm_conv(ASMState *as, IRIns *ir)
433 {
434 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
435 #if !LJ_SOFTFP
436 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
437 #endif
438 IRRef lref = ir->op1;
439 lua_assert(!(irt_isint64(ir->t) ||
440 (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */
441 #if LJ_SOFTFP
442 /* FP conversions are handled by SPLIT. */
443 lua_assert(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT));
444 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
445 #else
446 lua_assert(irt_type(ir->t) != st);
447 if (irt_isfp(ir->t)) {
448 Reg dest = ra_dest(as, ir, RSET_FPR);
449 if (stfp) { /* FP to FP conversion. */
450 emit_fg(as, st == IRT_NUM ? MIPSI_CVT_S_D : MIPSI_CVT_D_S,
451 dest, ra_alloc1(as, lref, RSET_FPR));
452 } else if (st == IRT_U32) { /* U32 to FP conversion. */
453 /* y = (x ^ 0x8000000) + 2147483648.0 */
454 Reg left = ra_alloc1(as, lref, RSET_GPR);
455 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
456 emit_fgh(as, irt_isfloat(ir->t) ? MIPSI_ADD_S : MIPSI_ADD_D,
457 dest, dest, tmp);
458 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
459 dest, dest);
460 if (irt_isfloat(ir->t))
461 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
462 (void *)lj_ir_k64_find(as->J, U64x(4f000000,4f000000)),
463 RSET_GPR);
464 else
465 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
466 (void *)lj_ir_k64_find(as->J, U64x(41e00000,00000000)),
467 RSET_GPR);
468 emit_tg(as, MIPSI_MTC1, RID_TMP, dest);
469 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left);
470 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
471 } else { /* Integer to FP conversion. */
472 Reg left = ra_alloc1(as, lref, RSET_GPR);
473 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
474 dest, dest);
475 emit_tg(as, MIPSI_MTC1, left, dest);
476 }
477 } else if (stfp) { /* FP to integer conversion. */
478 if (irt_isguard(ir->t)) {
479 /* Checked conversions are only supported from number to int. */
480 lua_assert(irt_isint(ir->t) && st == IRT_NUM);
481 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
482 } else {
483 Reg dest = ra_dest(as, ir, RSET_GPR);
484 Reg left = ra_alloc1(as, lref, RSET_FPR);
485 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
486 if (irt_isu32(ir->t)) {
487 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
488 emit_dst(as, MIPSI_XOR, dest, dest, RID_TMP);
489 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
490 emit_tg(as, MIPSI_MFC1, dest, tmp);
491 emit_fg(as, st == IRT_FLOAT ? MIPSI_FLOOR_W_S : MIPSI_FLOOR_W_D,
492 tmp, tmp);
493 emit_fgh(as, st == IRT_FLOAT ? MIPSI_SUB_S : MIPSI_SUB_D,
494 tmp, left, tmp);
495 if (st == IRT_FLOAT)
496 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
497 (void *)lj_ir_k64_find(as->J, U64x(4f000000,4f000000)),
498 RSET_GPR);
499 else
500 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
501 (void *)lj_ir_k64_find(as->J, U64x(41e00000,00000000)),
502 RSET_GPR);
503 } else {
504 emit_tg(as, MIPSI_MFC1, dest, tmp);
505 emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D,
506 tmp, left);
507 }
508 }
509 } else
510 #endif
511 {
512 Reg dest = ra_dest(as, ir, RSET_GPR);
513 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
514 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
515 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
516 if ((ir->op2 & IRCONV_SEXT)) {
517 if ((as->flags & JIT_F_MIPS32R2)) {
518 emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left);
519 } else {
520 uint32_t shift = st == IRT_I8 ? 24 : 16;
521 emit_dta(as, MIPSI_SRA, dest, dest, shift);
522 emit_dta(as, MIPSI_SLL, dest, left, shift);
523 }
524 } else {
525 emit_tsi(as, MIPSI_ANDI, dest, left,
526 (int32_t)(st == IRT_U8 ? 0xff : 0xffff));
527 }
528 } else { /* 32/64 bit integer conversions. */
529 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
530 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
531 }
532 }
533 }
534
asm_strto(ASMState * as,IRIns * ir)535 static void asm_strto(ASMState *as, IRIns *ir)
536 {
537 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
538 IRRef args[2];
539 int32_t ofs = 0;
540 #if LJ_SOFTFP
541 ra_evictset(as, RSET_SCRATCH);
542 if (ra_used(ir)) {
543 if (ra_hasspill(ir->s) && ra_hasspill((ir+1)->s) &&
544 (ir->s & 1) == LJ_BE && (ir->s ^ 1) == (ir+1)->s) {
545 int i;
546 for (i = 0; i < 2; i++) {
547 Reg r = (ir+i)->r;
548 if (ra_hasreg(r)) {
549 ra_free(as, r);
550 ra_modified(as, r);
551 emit_spload(as, ir+i, r, sps_scale((ir+i)->s));
552 }
553 }
554 ofs = sps_scale(ir->s & ~1);
555 } else {
556 Reg rhi = ra_dest(as, ir+1, RSET_GPR);
557 Reg rlo = ra_dest(as, ir, rset_exclude(RSET_GPR, rhi));
558 emit_tsi(as, MIPSI_LW, rhi, RID_SP, ofs+(LJ_BE?0:4));
559 emit_tsi(as, MIPSI_LW, rlo, RID_SP, ofs+(LJ_BE?4:0));
560 }
561 }
562 #else
563 RegSet drop = RSET_SCRATCH;
564 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */
565 ra_evictset(as, drop);
566 ofs = sps_scale(ir->s);
567 #endif
568 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO); /* Test return status. */
569 args[0] = ir->op1; /* GCstr *str */
570 args[1] = ASMREF_TMP1; /* TValue *n */
571 asm_gencall(as, ci, args);
572 /* Store the result to the spill slot or temp slots. */
573 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1),
574 RID_SP, ofs);
575 }
576
577 /* -- Memory references --------------------------------------------------- */
578
579 /* Get pointer to TValue. */
asm_tvptr(ASMState * as,Reg dest,IRRef ref)580 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
581 {
582 IRIns *ir = IR(ref);
583 if (irt_isnum(ir->t)) {
584 if (irref_isk(ref)) /* Use the number constant itself as a TValue. */
585 ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
586 else /* Otherwise force a spill and use the spill slot. */
587 emit_tsi(as, MIPSI_ADDIU, dest, RID_SP, ra_spill(as, ir));
588 } else {
589 /* Otherwise use g->tmptv to hold the TValue. */
590 RegSet allow = rset_exclude(RSET_GPR, dest);
591 Reg type;
592 emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, (int32_t)(offsetof(global_State, tmptv)-32768));
593 if (!irt_ispri(ir->t)) {
594 Reg src = ra_alloc1(as, ref, allow);
595 emit_setgl(as, src, tmptv.gcr);
596 }
597 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
598 type = ra_alloc1(as, ref+1, allow);
599 else
600 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
601 emit_setgl(as, type, tmptv.it);
602 }
603 }
604
asm_aref(ASMState * as,IRIns * ir)605 static void asm_aref(ASMState *as, IRIns *ir)
606 {
607 Reg dest = ra_dest(as, ir, RSET_GPR);
608 Reg idx, base;
609 if (irref_isk(ir->op2)) {
610 IRRef tab = IR(ir->op1)->op1;
611 int32_t ofs = asm_fuseabase(as, tab);
612 IRRef refa = ofs ? tab : ir->op1;
613 ofs += 8*IR(ir->op2)->i;
614 if (checki16(ofs)) {
615 base = ra_alloc1(as, refa, RSET_GPR);
616 emit_tsi(as, MIPSI_ADDIU, dest, base, ofs);
617 return;
618 }
619 }
620 base = ra_alloc1(as, ir->op1, RSET_GPR);
621 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
622 emit_dst(as, MIPSI_ADDU, dest, RID_TMP, base);
623 emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3);
624 }
625
626 /* Inlined hash lookup. Specialized for key type and for const keys.
627 ** The equivalent C code is:
628 ** Node *n = hashkey(t, key);
629 ** do {
630 ** if (lj_obj_equal(&n->key, key)) return &n->val;
631 ** } while ((n = nextnode(n)));
632 ** return niltv(L);
633 */
asm_href(ASMState * as,IRIns * ir,IROp merge)634 static void asm_href(ASMState *as, IRIns *ir, IROp merge)
635 {
636 RegSet allow = RSET_GPR;
637 int destused = ra_used(ir);
638 Reg dest = ra_dest(as, ir, allow);
639 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
640 Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2;
641 IRRef refkey = ir->op2;
642 IRIns *irkey = IR(refkey);
643 IRType1 kt = irkey->t;
644 uint32_t khash;
645 MCLabel l_end, l_loop, l_next;
646
647 rset_clear(allow, tab);
648 #if LJ_SOFTFP
649 if (!irref_isk(refkey)) {
650 key = ra_alloc1(as, refkey, allow);
651 rset_clear(allow, key);
652 if (irkey[1].o == IR_HIOP) {
653 if (ra_hasreg((irkey+1)->r)) {
654 type = tmpnum = (irkey+1)->r;
655 tmp1 = ra_scratch(as, allow);
656 rset_clear(allow, tmp1);
657 ra_noweak(as, tmpnum);
658 } else {
659 type = tmpnum = ra_allocref(as, refkey+1, allow);
660 }
661 rset_clear(allow, tmpnum);
662 } else {
663 type = ra_allock(as, (int32_t)irt_toitype(irkey->t), allow);
664 rset_clear(allow, type);
665 }
666 }
667 #else
668 if (irt_isnum(kt)) {
669 key = ra_alloc1(as, refkey, RSET_FPR);
670 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
671 } else if (!irt_ispri(kt)) {
672 key = ra_alloc1(as, refkey, allow);
673 rset_clear(allow, key);
674 type = ra_allock(as, (int32_t)irt_toitype(irkey->t), allow);
675 rset_clear(allow, type);
676 }
677 #endif
678 tmp2 = ra_scratch(as, allow);
679 rset_clear(allow, tmp2);
680
681 /* Key not found in chain: jump to exit (if merged) or load niltv. */
682 l_end = emit_label(as);
683 as->invmcp = NULL;
684 if (merge == IR_NE)
685 asm_guard(as, MIPSI_B, RID_ZERO, RID_ZERO);
686 else if (destused)
687 emit_loada(as, dest, niltvg(J2G(as->J)));
688 /* Follow hash chain until the end. */
689 emit_move(as, dest, tmp2);
690 l_loop = --as->mcp;
691 emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, next));
692 l_next = emit_label(as);
693
694 /* Type and value comparison. */
695 if (merge == IR_EQ) { /* Must match asm_guard(). */
696 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
697 l_end = asm_exitstub_addr(as);
698 }
699 if (!LJ_SOFTFP && irt_isnum(kt)) {
700 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
701 emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
702 *--as->mcp = MIPSI_NOP; /* Avoid NaN comparison overhead. */
703 emit_branch(as, MIPSI_BEQ, tmp2, RID_ZERO, l_next);
704 emit_tsi(as, MIPSI_SLTIU, tmp2, tmp2, (int32_t)LJ_TISNUM);
705 emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
706 } else {
707 if (irt_ispri(kt)) {
708 emit_branch(as, MIPSI_BEQ, tmp2, type, l_end);
709 } else {
710 emit_branch(as, MIPSI_BEQ, tmp1, key, l_end);
711 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.gcr));
712 emit_branch(as, MIPSI_BNE, tmp2, type, l_next);
713 }
714 }
715 emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.it));
716 *l_loop = MIPSI_BNE | MIPSF_S(tmp2) | ((as->mcp-l_loop-1) & 0xffffu);
717
718 /* Load main position relative to tab->node into dest. */
719 khash = irref_isk(refkey) ? ir_khash(irkey) : 1;
720 if (khash == 0) {
721 emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node));
722 } else {
723 Reg tmphash = tmp1;
724 if (irref_isk(refkey))
725 tmphash = ra_allock(as, khash, allow);
726 emit_dst(as, MIPSI_ADDU, dest, dest, tmp1);
727 lua_assert(sizeof(Node) == 24);
728 emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1);
729 emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3);
730 emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5);
731 emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash);
732 emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node));
733 emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
734 if (irref_isk(refkey)) {
735 /* Nothing to do. */
736 } else if (irt_isstr(kt)) {
737 emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, hash));
738 } else { /* Must match with hash*() in lj_tab.c. */
739 emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2);
740 emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31);
741 emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2);
742 emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31);
743 emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest);
744 if (LJ_SOFTFP ? (irkey[1].o == IR_HIOP) : irt_isnum(kt)) {
745 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
746 if ((as->flags & JIT_F_MIPS32R2)) {
747 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
748 } else {
749 emit_dst(as, MIPSI_OR, dest, dest, tmp1);
750 emit_dta(as, MIPSI_SLL, tmp1, tmp1, HASH_ROT1);
751 emit_dta(as, MIPSI_SRL, dest, tmp1, (-HASH_ROT1)&31);
752 }
753 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
754 #if LJ_SOFTFP
755 emit_ds(as, MIPSI_MOVE, tmp1, type);
756 emit_ds(as, MIPSI_MOVE, tmp2, key);
757 #else
758 emit_tg(as, MIPSI_MFC1, tmp2, key);
759 emit_tg(as, MIPSI_MFC1, tmp1, key+1);
760 #endif
761 } else {
762 emit_dst(as, MIPSI_XOR, tmp2, key, tmp1);
763 emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31);
764 emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow));
765 }
766 }
767 }
768 }
769
asm_hrefk(ASMState * as,IRIns * ir)770 static void asm_hrefk(ASMState *as, IRIns *ir)
771 {
772 IRIns *kslot = IR(ir->op2);
773 IRIns *irkey = IR(kslot->op1);
774 int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
775 int32_t kofs = ofs + (int32_t)offsetof(Node, key);
776 Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
777 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
778 Reg key = RID_NONE, type = RID_TMP, idx = node;
779 RegSet allow = rset_exclude(RSET_GPR, node);
780 int32_t lo, hi;
781 lua_assert(ofs % sizeof(Node) == 0);
782 if (ofs > 32736) {
783 idx = dest;
784 rset_clear(allow, dest);
785 kofs = (int32_t)offsetof(Node, key);
786 } else if (ra_hasreg(dest)) {
787 emit_tsi(as, MIPSI_ADDIU, dest, node, ofs);
788 }
789 if (!irt_ispri(irkey->t)) {
790 key = ra_scratch(as, allow);
791 rset_clear(allow, key);
792 }
793 if (irt_isnum(irkey->t)) {
794 lo = (int32_t)ir_knum(irkey)->u32.lo;
795 hi = (int32_t)ir_knum(irkey)->u32.hi;
796 } else {
797 lo = irkey->i;
798 hi = irt_toitype(irkey->t);
799 if (!ra_hasreg(key))
800 goto nolo;
801 }
802 asm_guard(as, MIPSI_BNE, key, lo ? ra_allock(as, lo, allow) : RID_ZERO);
803 nolo:
804 asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO);
805 if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0));
806 emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4));
807 if (ofs > 32736)
808 emit_tsi(as, MIPSI_ADDU, dest, node, ra_allock(as, ofs, allow));
809 }
810
asm_uref(ASMState * as,IRIns * ir)811 static void asm_uref(ASMState *as, IRIns *ir)
812 {
813 /* NYI: Check that UREFO is still open and not aliasing a slot. */
814 Reg dest = ra_dest(as, ir, RSET_GPR);
815 if (irref_isk(ir->op1)) {
816 GCfunc *fn = ir_kfunc(IR(ir->op1));
817 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
818 emit_lsptr(as, MIPSI_LW, dest, v, RSET_GPR);
819 } else {
820 Reg uv = ra_scratch(as, RSET_GPR);
821 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
822 if (ir->o == IR_UREFC) {
823 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
824 emit_tsi(as, MIPSI_ADDIU, dest, uv, (int32_t)offsetof(GCupval, tv));
825 emit_tsi(as, MIPSI_LBU, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
826 } else {
827 emit_tsi(as, MIPSI_LW, dest, uv, (int32_t)offsetof(GCupval, v));
828 }
829 emit_tsi(as, MIPSI_LW, uv, func,
830 (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8));
831 }
832 }
833
asm_fref(ASMState * as,IRIns * ir)834 static void asm_fref(ASMState *as, IRIns *ir)
835 {
836 UNUSED(as); UNUSED(ir);
837 lua_assert(!ra_used(ir));
838 }
839
asm_strref(ASMState * as,IRIns * ir)840 static void asm_strref(ASMState *as, IRIns *ir)
841 {
842 Reg dest = ra_dest(as, ir, RSET_GPR);
843 IRRef ref = ir->op2, refk = ir->op1;
844 int32_t ofs = (int32_t)sizeof(GCstr);
845 Reg r;
846 if (irref_isk(ref)) {
847 IRRef tmp = refk; refk = ref; ref = tmp;
848 } else if (!irref_isk(refk)) {
849 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
850 IRIns *irr = IR(ir->op2);
851 if (ra_hasreg(irr->r)) {
852 ra_noweak(as, irr->r);
853 right = irr->r;
854 } else if (mayfuse(as, irr->op2) &&
855 irr->o == IR_ADD && irref_isk(irr->op2) &&
856 checki16(ofs + IR(irr->op2)->i)) {
857 ofs += IR(irr->op2)->i;
858 right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left));
859 } else {
860 right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left));
861 }
862 emit_tsi(as, MIPSI_ADDIU, dest, dest, ofs);
863 emit_dst(as, MIPSI_ADDU, dest, left, right);
864 return;
865 }
866 r = ra_alloc1(as, ref, RSET_GPR);
867 ofs += IR(refk)->i;
868 if (checki16(ofs))
869 emit_tsi(as, MIPSI_ADDIU, dest, r, ofs);
870 else
871 emit_dst(as, MIPSI_ADDU, dest, r,
872 ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
873 }
874
875 /* -- Loads and stores ---------------------------------------------------- */
876
asm_fxloadins(IRIns * ir)877 static MIPSIns asm_fxloadins(IRIns *ir)
878 {
879 switch (irt_type(ir->t)) {
880 case IRT_I8: return MIPSI_LB;
881 case IRT_U8: return MIPSI_LBU;
882 case IRT_I16: return MIPSI_LH;
883 case IRT_U16: return MIPSI_LHU;
884 case IRT_NUM: lua_assert(!LJ_SOFTFP); return MIPSI_LDC1;
885 case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_LWC1;
886 default: return MIPSI_LW;
887 }
888 }
889
asm_fxstoreins(IRIns * ir)890 static MIPSIns asm_fxstoreins(IRIns *ir)
891 {
892 switch (irt_type(ir->t)) {
893 case IRT_I8: case IRT_U8: return MIPSI_SB;
894 case IRT_I16: case IRT_U16: return MIPSI_SH;
895 case IRT_NUM: lua_assert(!LJ_SOFTFP); return MIPSI_SDC1;
896 case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_SWC1;
897 default: return MIPSI_SW;
898 }
899 }
900
asm_fload(ASMState * as,IRIns * ir)901 static void asm_fload(ASMState *as, IRIns *ir)
902 {
903 Reg dest = ra_dest(as, ir, RSET_GPR);
904 Reg idx = ra_alloc1(as, ir->op1, RSET_GPR);
905 MIPSIns mi = asm_fxloadins(ir);
906 int32_t ofs;
907 if (ir->op2 == IRFL_TAB_ARRAY) {
908 ofs = asm_fuseabase(as, ir->op1);
909 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
910 emit_tsi(as, MIPSI_ADDIU, dest, idx, ofs);
911 return;
912 }
913 }
914 ofs = field_ofs[ir->op2];
915 lua_assert(!irt_isfp(ir->t));
916 emit_tsi(as, mi, dest, idx, ofs);
917 }
918
asm_fstore(ASMState * as,IRIns * ir)919 static void asm_fstore(ASMState *as, IRIns *ir)
920 {
921 if (ir->r != RID_SINK) {
922 Reg src = ra_alloc1z(as, ir->op2, RSET_GPR);
923 IRIns *irf = IR(ir->op1);
924 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
925 int32_t ofs = field_ofs[irf->op2];
926 MIPSIns mi = asm_fxstoreins(ir);
927 lua_assert(!irt_isfp(ir->t));
928 emit_tsi(as, mi, src, idx, ofs);
929 }
930 }
931
asm_xload(ASMState * as,IRIns * ir)932 static void asm_xload(ASMState *as, IRIns *ir)
933 {
934 Reg dest = ra_dest(as, ir,
935 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
936 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED));
937 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
938 }
939
asm_xstore_(ASMState * as,IRIns * ir,int32_t ofs)940 static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
941 {
942 if (ir->r != RID_SINK) {
943 Reg src = ra_alloc1z(as, ir->op2,
944 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
945 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
946 rset_exclude(RSET_GPR, src), ofs);
947 }
948 }
949
950 #define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
951
asm_ahuvload(ASMState * as,IRIns * ir)952 static void asm_ahuvload(ASMState *as, IRIns *ir)
953 {
954 int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
955 IRType t = hiop ? IRT_NUM : irt_type(ir->t);
956 Reg dest = RID_NONE, type = RID_TMP, idx;
957 RegSet allow = RSET_GPR;
958 int32_t ofs = 0;
959 if (hiop && ra_used(ir+1)) {
960 type = ra_dest(as, ir+1, allow);
961 rset_clear(allow, type);
962 }
963 if (ra_used(ir)) {
964 lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
965 irt_isint(ir->t) || irt_isaddr(ir->t));
966 dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow);
967 rset_clear(allow, dest);
968 }
969 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
970 rset_clear(allow, idx);
971 if (t == IRT_NUM) {
972 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
973 emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
974 } else {
975 asm_guard(as, MIPSI_BNE, type, ra_allock(as, irt_toitype_(t), allow));
976 }
977 if (ra_hasreg(dest)) {
978 if (!LJ_SOFTFP && t == IRT_NUM)
979 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
980 else
981 emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0));
982 }
983 emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4));
984 }
985
asm_ahustore(ASMState * as,IRIns * ir)986 static void asm_ahustore(ASMState *as, IRIns *ir)
987 {
988 RegSet allow = RSET_GPR;
989 Reg idx, src = RID_NONE, type = RID_NONE;
990 int32_t ofs = 0;
991 if (ir->r == RID_SINK)
992 return;
993 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
994 src = ra_alloc1(as, ir->op2, RSET_FPR);
995 } else {
996 int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
997 if (!irt_ispri(ir->t)) {
998 src = ra_alloc1(as, ir->op2, allow);
999 rset_clear(allow, src);
1000 }
1001 if (hiop)
1002 type = ra_alloc1(as, (ir+1)->op2, allow);
1003 else
1004 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1005 rset_clear(allow, type);
1006 }
1007 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1008 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1009 emit_hsi(as, MIPSI_SDC1, src, idx, ofs);
1010 } else {
1011 if (ra_hasreg(src))
1012 emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0));
1013 emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4));
1014 }
1015 }
1016
asm_sload(ASMState * as,IRIns * ir)1017 static void asm_sload(ASMState *as, IRIns *ir)
1018 {
1019 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1020 int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
1021 IRType t = hiop ? IRT_NUM : irt_type(ir->t);
1022 Reg dest = RID_NONE, type = RID_NONE, base;
1023 RegSet allow = RSET_GPR;
1024 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */
1025 lua_assert(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK));
1026 #if LJ_SOFTFP
1027 lua_assert(!(ir->op2 & IRSLOAD_CONVERT)); /* Handled by LJ_SOFTFP SPLIT. */
1028 if (hiop && ra_used(ir+1)) {
1029 type = ra_dest(as, ir+1, allow);
1030 rset_clear(allow, type);
1031 }
1032 #else
1033 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(ir->t) && t == IRT_INT) {
1034 dest = ra_scratch(as, RSET_FPR);
1035 asm_tointg(as, ir, dest);
1036 t = IRT_NUM; /* Continue with a regular number type check. */
1037 } else
1038 #endif
1039 if (ra_used(ir)) {
1040 lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
1041 irt_isint(ir->t) || irt_isaddr(ir->t));
1042 dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow);
1043 rset_clear(allow, dest);
1044 base = ra_alloc1(as, REF_BASE, allow);
1045 rset_clear(allow, base);
1046 if (!LJ_SOFTFP && (ir->op2 & IRSLOAD_CONVERT)) {
1047 if (t == IRT_INT) {
1048 Reg tmp = ra_scratch(as, RSET_FPR);
1049 emit_tg(as, MIPSI_MFC1, dest, tmp);
1050 emit_fg(as, MIPSI_TRUNC_W_D, tmp, tmp);
1051 dest = tmp;
1052 t = IRT_NUM; /* Check for original type. */
1053 } else {
1054 Reg tmp = ra_scratch(as, RSET_GPR);
1055 emit_fg(as, MIPSI_CVT_D_W, dest, dest);
1056 emit_tg(as, MIPSI_MTC1, tmp, dest);
1057 dest = tmp;
1058 t = IRT_INT; /* Check for original type. */
1059 }
1060 }
1061 goto dotypecheck;
1062 }
1063 base = ra_alloc1(as, REF_BASE, allow);
1064 rset_clear(allow, base);
1065 dotypecheck:
1066 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1067 if (ra_noreg(type)) {
1068 if (ofs < 256 && ra_hasreg(dest) && (dest & 1) == 0 &&
1069 rset_test((as->freeset & allow), dest+1)) {
1070 type = dest+1;
1071 ra_modified(as, type);
1072 } else {
1073 type = RID_TMP;
1074 }
1075 }
1076 if (t == IRT_NUM) {
1077 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1078 emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
1079 } else {
1080 Reg ktype = ra_allock(as, irt_toitype_(t), allow);
1081 asm_guard(as, MIPSI_BNE, type, ktype);
1082 }
1083 }
1084 if (ra_hasreg(dest)) {
1085 if (!LJ_SOFTFP && t == IRT_NUM)
1086 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1087 else
1088 emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0));
1089 }
1090 if (ra_hasreg(type))
1091 emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4));
1092 }
1093
1094 /* -- Allocations --------------------------------------------------------- */
1095
1096 #if LJ_HASFFI
asm_cnew(ASMState * as,IRIns * ir)1097 static void asm_cnew(ASMState *as, IRIns *ir)
1098 {
1099 CTState *cts = ctype_ctsG(J2G(as->J));
1100 CTypeID id = (CTypeID)IR(ir->op1)->i;
1101 CTSize sz;
1102 CTInfo info = lj_ctype_info(cts, id, &sz);
1103 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1104 IRRef args[4];
1105 RegSet drop = RSET_SCRATCH;
1106 lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
1107
1108 as->gcsteps++;
1109 if (ra_hasreg(ir->r))
1110 rset_clear(drop, ir->r); /* Dest reg handled below. */
1111 ra_evictset(as, drop);
1112 if (ra_used(ir))
1113 ra_destreg(as, ir, RID_RET); /* GCcdata * */
1114
1115 /* Initialize immutable cdata object. */
1116 if (ir->o == IR_CNEWI) {
1117 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1118 int32_t ofs = sizeof(GCcdata);
1119 lua_assert(sz == 4 || sz == 8);
1120 if (sz == 8) {
1121 ofs += 4;
1122 lua_assert((ir+1)->o == IR_HIOP);
1123 if (LJ_LE) ir++;
1124 }
1125 for (;;) {
1126 Reg r = ra_alloc1z(as, ir->op2, allow);
1127 emit_tsi(as, MIPSI_SW, r, RID_RET, ofs);
1128 rset_clear(allow, r);
1129 if (ofs == sizeof(GCcdata)) break;
1130 ofs -= 4; if (LJ_BE) ir++; else ir--;
1131 }
1132 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1133 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1134 args[0] = ASMREF_L; /* lua_State *L */
1135 args[1] = ir->op1; /* CTypeID id */
1136 args[2] = ir->op2; /* CTSize sz */
1137 args[3] = ASMREF_TMP1; /* CTSize align */
1138 asm_gencall(as, ci, args);
1139 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1140 return;
1141 }
1142
1143 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1144 emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1145 emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1146 emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA);
1147 emit_ti(as, MIPSI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */
1148 args[0] = ASMREF_L; /* lua_State *L */
1149 args[1] = ASMREF_TMP1; /* MSize size */
1150 asm_gencall(as, ci, args);
1151 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1152 ra_releasetmp(as, ASMREF_TMP1));
1153 }
1154 #else
1155 #define asm_cnew(as, ir) ((void)0)
1156 #endif
1157
1158 /* -- Write barriers ------------------------------------------------------ */
1159
asm_tbar(ASMState * as,IRIns * ir)1160 static void asm_tbar(ASMState *as, IRIns *ir)
1161 {
1162 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1163 Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1164 Reg link = RID_TMP;
1165 MCLabel l_end = emit_label(as);
1166 emit_tsi(as, MIPSI_SW, link, tab, (int32_t)offsetof(GCtab, gclist));
1167 emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked));
1168 emit_setgl(as, tab, gc.grayagain);
1169 emit_getgl(as, link, gc.grayagain);
1170 emit_dst(as, MIPSI_XOR, mark, mark, RID_TMP); /* Clear black bit. */
1171 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1172 emit_tsi(as, MIPSI_ANDI, RID_TMP, mark, LJ_GC_BLACK);
1173 emit_tsi(as, MIPSI_LBU, mark, tab, (int32_t)offsetof(GCtab, marked));
1174 }
1175
asm_obar(ASMState * as,IRIns * ir)1176 static void asm_obar(ASMState *as, IRIns *ir)
1177 {
1178 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1179 IRRef args[2];
1180 MCLabel l_end;
1181 Reg obj, val, tmp;
1182 /* No need for other object barriers (yet). */
1183 lua_assert(IR(ir->op1)->o == IR_UREFC);
1184 ra_evictset(as, RSET_SCRATCH);
1185 l_end = emit_label(as);
1186 args[0] = ASMREF_TMP1; /* global_State *g */
1187 args[1] = ir->op1; /* TValue *tv */
1188 asm_gencall(as, ci, args);
1189 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1190 obj = IR(ir->op1)->r;
1191 tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1192 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1193 emit_tsi(as, MIPSI_ANDI, tmp, tmp, LJ_GC_BLACK);
1194 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1195 emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, LJ_GC_WHITES);
1196 val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1197 emit_tsi(as, MIPSI_LBU, tmp, obj,
1198 (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1199 emit_tsi(as, MIPSI_LBU, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1200 }
1201
1202 /* -- Arithmetic and logic operations ------------------------------------- */
1203
1204 #if !LJ_SOFTFP
asm_fparith(ASMState * as,IRIns * ir,MIPSIns mi)1205 static void asm_fparith(ASMState *as, IRIns *ir, MIPSIns mi)
1206 {
1207 Reg dest = ra_dest(as, ir, RSET_FPR);
1208 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1209 right = (left >> 8); left &= 255;
1210 emit_fgh(as, mi, dest, left, right);
1211 }
1212
asm_fpunary(ASMState * as,IRIns * ir,MIPSIns mi)1213 static void asm_fpunary(ASMState *as, IRIns *ir, MIPSIns mi)
1214 {
1215 Reg dest = ra_dest(as, ir, RSET_FPR);
1216 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1217 emit_fg(as, mi, dest, left);
1218 }
1219
asm_fpmath(ASMState * as,IRIns * ir)1220 static void asm_fpmath(ASMState *as, IRIns *ir)
1221 {
1222 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
1223 return;
1224 if (ir->op2 <= IRFPM_TRUNC)
1225 asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
1226 else if (ir->op2 == IRFPM_SQRT)
1227 asm_fpunary(as, ir, MIPSI_SQRT_D);
1228 else
1229 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1230 }
1231 #endif
1232
asm_add(ASMState * as,IRIns * ir)1233 static void asm_add(ASMState *as, IRIns *ir)
1234 {
1235 #if !LJ_SOFTFP
1236 if (irt_isnum(ir->t)) {
1237 asm_fparith(as, ir, MIPSI_ADD_D);
1238 } else
1239 #endif
1240 {
1241 Reg dest = ra_dest(as, ir, RSET_GPR);
1242 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1243 if (irref_isk(ir->op2)) {
1244 int32_t k = IR(ir->op2)->i;
1245 if (checki16(k)) {
1246 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1247 return;
1248 }
1249 }
1250 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1251 emit_dst(as, MIPSI_ADDU, dest, left, right);
1252 }
1253 }
1254
asm_sub(ASMState * as,IRIns * ir)1255 static void asm_sub(ASMState *as, IRIns *ir)
1256 {
1257 #if !LJ_SOFTFP
1258 if (irt_isnum(ir->t)) {
1259 asm_fparith(as, ir, MIPSI_SUB_D);
1260 } else
1261 #endif
1262 {
1263 Reg dest = ra_dest(as, ir, RSET_GPR);
1264 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1265 right = (left >> 8); left &= 255;
1266 emit_dst(as, MIPSI_SUBU, dest, left, right);
1267 }
1268 }
1269
asm_mul(ASMState * as,IRIns * ir)1270 static void asm_mul(ASMState *as, IRIns *ir)
1271 {
1272 #if !LJ_SOFTFP
1273 if (irt_isnum(ir->t)) {
1274 asm_fparith(as, ir, MIPSI_MUL_D);
1275 } else
1276 #endif
1277 {
1278 Reg dest = ra_dest(as, ir, RSET_GPR);
1279 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1280 right = (left >> 8); left &= 255;
1281 emit_dst(as, MIPSI_MUL, dest, left, right);
1282 }
1283 }
1284
1285 #define asm_div(as, ir) asm_fparith(as, ir, MIPSI_DIV_D)
1286 #define asm_mod(as, ir) asm_callid(as, ir, IRCALL_lj_vm_modi)
1287 #define asm_pow(as, ir) asm_callid(as, ir, IRCALL_lj_vm_powi)
1288
asm_neg(ASMState * as,IRIns * ir)1289 static void asm_neg(ASMState *as, IRIns *ir)
1290 {
1291 #if !LJ_SOFTFP
1292 if (irt_isnum(ir->t)) {
1293 asm_fpunary(as, ir, MIPSI_NEG_D);
1294 } else
1295 #endif
1296 {
1297 Reg dest = ra_dest(as, ir, RSET_GPR);
1298 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1299 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1300 }
1301 }
1302
1303 #define asm_abs(as, ir) asm_fpunary(as, ir, MIPSI_ABS_D)
1304 #define asm_atan2(as, ir) asm_callid(as, ir, IRCALL_atan2)
1305 #define asm_ldexp(as, ir) asm_callid(as, ir, IRCALL_ldexp)
1306
asm_arithov(ASMState * as,IRIns * ir)1307 static void asm_arithov(ASMState *as, IRIns *ir)
1308 {
1309 Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
1310 if (irref_isk(ir->op2)) {
1311 int k = IR(ir->op2)->i;
1312 if (ir->o == IR_SUBOV) k = -k;
1313 if (checki16(k)) { /* (dest < left) == (k >= 0 ? 1 : 0) */
1314 left = ra_alloc1(as, ir->op1, RSET_GPR);
1315 asm_guard(as, k >= 0 ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1316 emit_dst(as, MIPSI_SLT, RID_TMP, dest, dest == left ? RID_TMP : left);
1317 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1318 if (dest == left) emit_move(as, RID_TMP, left);
1319 return;
1320 }
1321 }
1322 left = ra_alloc2(as, ir, RSET_GPR);
1323 right = (left >> 8); left &= 255;
1324 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1325 right), dest));
1326 asm_guard(as, MIPSI_BLTZ, RID_TMP, 0);
1327 emit_dst(as, MIPSI_AND, RID_TMP, RID_TMP, tmp);
1328 if (ir->o == IR_ADDOV) { /* ((dest^left) & (dest^right)) < 0 */
1329 emit_dst(as, MIPSI_XOR, RID_TMP, dest, dest == right ? RID_TMP : right);
1330 } else { /* ((dest^left) & (dest^~right)) < 0 */
1331 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, dest);
1332 emit_dst(as, MIPSI_NOR, RID_TMP, dest == right ? RID_TMP : right, RID_ZERO);
1333 }
1334 emit_dst(as, MIPSI_XOR, tmp, dest, dest == left ? RID_TMP : left);
1335 emit_dst(as, ir->o == IR_ADDOV ? MIPSI_ADDU : MIPSI_SUBU, dest, left, right);
1336 if (dest == left || dest == right)
1337 emit_move(as, RID_TMP, dest == left ? left : right);
1338 }
1339
1340 #define asm_addov(as, ir) asm_arithov(as, ir)
1341 #define asm_subov(as, ir) asm_arithov(as, ir)
1342
asm_mulov(ASMState * as,IRIns * ir)1343 static void asm_mulov(ASMState *as, IRIns *ir)
1344 {
1345 Reg dest = ra_dest(as, ir, RSET_GPR);
1346 Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
1347 right = (left >> 8); left &= 255;
1348 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1349 right), dest));
1350 asm_guard(as, MIPSI_BNE, RID_TMP, tmp);
1351 emit_dta(as, MIPSI_SRA, RID_TMP, dest, 31);
1352 emit_dst(as, MIPSI_MFHI, tmp, 0, 0);
1353 emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1354 emit_dst(as, MIPSI_MULT, 0, left, right);
1355 }
1356
1357 #if LJ_HASFFI
asm_add64(ASMState * as,IRIns * ir)1358 static void asm_add64(ASMState *as, IRIns *ir)
1359 {
1360 Reg dest = ra_dest(as, ir, RSET_GPR);
1361 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1362 if (irref_isk(ir->op2)) {
1363 int32_t k = IR(ir->op2)->i;
1364 if (k == 0) {
1365 emit_dst(as, MIPSI_ADDU, dest, left, RID_TMP);
1366 goto loarith;
1367 } else if (checki16(k)) {
1368 emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1369 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1370 goto loarith;
1371 }
1372 }
1373 emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1374 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1375 emit_dst(as, MIPSI_ADDU, dest, left, right);
1376 loarith:
1377 ir--;
1378 dest = ra_dest(as, ir, RSET_GPR);
1379 left = ra_alloc1(as, ir->op1, RSET_GPR);
1380 if (irref_isk(ir->op2)) {
1381 int32_t k = IR(ir->op2)->i;
1382 if (k == 0) {
1383 if (dest != left)
1384 emit_move(as, dest, left);
1385 return;
1386 } else if (checki16(k)) {
1387 if (dest == left) {
1388 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, left));
1389 emit_move(as, dest, tmp);
1390 dest = tmp;
1391 }
1392 emit_dst(as, MIPSI_SLTU, RID_TMP, dest, left);
1393 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1394 return;
1395 }
1396 }
1397 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1398 if (dest == left && dest == right) {
1399 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1400 emit_move(as, dest, tmp);
1401 dest = tmp;
1402 }
1403 emit_dst(as, MIPSI_SLTU, RID_TMP, dest, dest == left ? right : left);
1404 emit_dst(as, MIPSI_ADDU, dest, left, right);
1405 }
1406
asm_sub64(ASMState * as,IRIns * ir)1407 static void asm_sub64(ASMState *as, IRIns *ir)
1408 {
1409 Reg dest = ra_dest(as, ir, RSET_GPR);
1410 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1411 right = (left >> 8); left &= 255;
1412 emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1413 emit_dst(as, MIPSI_SUBU, dest, left, right);
1414 ir--;
1415 dest = ra_dest(as, ir, RSET_GPR);
1416 left = ra_alloc2(as, ir, RSET_GPR);
1417 right = (left >> 8); left &= 255;
1418 if (dest == left) {
1419 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1420 emit_move(as, dest, tmp);
1421 dest = tmp;
1422 }
1423 emit_dst(as, MIPSI_SLTU, RID_TMP, left, dest);
1424 emit_dst(as, MIPSI_SUBU, dest, left, right);
1425 }
1426
asm_neg64(ASMState * as,IRIns * ir)1427 static void asm_neg64(ASMState *as, IRIns *ir)
1428 {
1429 Reg dest = ra_dest(as, ir, RSET_GPR);
1430 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1431 emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1432 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1433 ir--;
1434 dest = ra_dest(as, ir, RSET_GPR);
1435 left = ra_alloc1(as, ir->op1, RSET_GPR);
1436 emit_dst(as, MIPSI_SLTU, RID_TMP, RID_ZERO, dest);
1437 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1438 }
1439 #endif
1440
asm_bnot(ASMState * as,IRIns * ir)1441 static void asm_bnot(ASMState *as, IRIns *ir)
1442 {
1443 Reg left, right, dest = ra_dest(as, ir, RSET_GPR);
1444 IRIns *irl = IR(ir->op1);
1445 if (mayfuse(as, ir->op1) && irl->o == IR_BOR) {
1446 left = ra_alloc2(as, irl, RSET_GPR);
1447 right = (left >> 8); left &= 255;
1448 } else {
1449 left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1450 right = RID_ZERO;
1451 }
1452 emit_dst(as, MIPSI_NOR, dest, left, right);
1453 }
1454
asm_bswap(ASMState * as,IRIns * ir)1455 static void asm_bswap(ASMState *as, IRIns *ir)
1456 {
1457 Reg dest = ra_dest(as, ir, RSET_GPR);
1458 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1459 if ((as->flags & JIT_F_MIPS32R2)) {
1460 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
1461 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
1462 } else {
1463 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), dest));
1464 emit_dst(as, MIPSI_OR, dest, dest, tmp);
1465 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
1466 emit_tsi(as, MIPSI_ANDI, dest, dest, 0xff00);
1467 emit_dta(as, MIPSI_SLL, RID_TMP, RID_TMP, 8);
1468 emit_dta(as, MIPSI_SRL, dest, left, 8);
1469 emit_tsi(as, MIPSI_ANDI, RID_TMP, left, 0xff00);
1470 emit_dst(as, MIPSI_OR, tmp, tmp, RID_TMP);
1471 emit_dta(as, MIPSI_SRL, tmp, left, 24);
1472 emit_dta(as, MIPSI_SLL, RID_TMP, left, 24);
1473 }
1474 }
1475
asm_bitop(ASMState * as,IRIns * ir,MIPSIns mi,MIPSIns mik)1476 static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1477 {
1478 Reg dest = ra_dest(as, ir, RSET_GPR);
1479 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1480 if (irref_isk(ir->op2)) {
1481 int32_t k = IR(ir->op2)->i;
1482 if (checku16(k)) {
1483 emit_tsi(as, mik, dest, left, k);
1484 return;
1485 }
1486 }
1487 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1488 emit_dst(as, mi, dest, left, right);
1489 }
1490
1491 #define asm_band(as, ir) asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI)
1492 #define asm_bor(as, ir) asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI)
1493 #define asm_bxor(as, ir) asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI)
1494
asm_bitshift(ASMState * as,IRIns * ir,MIPSIns mi,MIPSIns mik)1495 static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1496 {
1497 Reg dest = ra_dest(as, ir, RSET_GPR);
1498 if (irref_isk(ir->op2)) { /* Constant shifts. */
1499 uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
1500 emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR), shift);
1501 } else {
1502 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1503 right = (left >> 8); left &= 255;
1504 emit_dst(as, mi, dest, right, left); /* Shift amount is in rs. */
1505 }
1506 }
1507
1508 #define asm_bshl(as, ir) asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL)
1509 #define asm_bshr(as, ir) asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL)
1510 #define asm_bsar(as, ir) asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA)
1511 #define asm_brol(as, ir) lua_assert(0)
1512
asm_bror(ASMState * as,IRIns * ir)1513 static void asm_bror(ASMState *as, IRIns *ir)
1514 {
1515 if ((as->flags & JIT_F_MIPS32R2)) {
1516 asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
1517 } else {
1518 Reg dest = ra_dest(as, ir, RSET_GPR);
1519 if (irref_isk(ir->op2)) { /* Constant shifts. */
1520 uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
1521 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1522 emit_rotr(as, dest, left, RID_TMP, shift);
1523 } else {
1524 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1525 right = (left >> 8); left &= 255;
1526 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
1527 emit_dst(as, MIPSI_SRLV, dest, right, left);
1528 emit_dst(as, MIPSI_SLLV, RID_TMP, RID_TMP, left);
1529 emit_dst(as, MIPSI_SUBU, RID_TMP, ra_allock(as, 32, RSET_GPR), right);
1530 }
1531 }
1532 }
1533
1534 #if LJ_SOFTFP
asm_sfpmin_max(ASMState * as,IRIns * ir)1535 static void asm_sfpmin_max(ASMState *as, IRIns *ir)
1536 {
1537 CCallInfo ci = lj_ir_callinfo[(IROp)ir->o == IR_MIN ? IRCALL_lj_vm_sfmin : IRCALL_lj_vm_sfmax];
1538 IRRef args[4];
1539 args[0^LJ_BE] = ir->op1;
1540 args[1^LJ_BE] = (ir+1)->op1;
1541 args[2^LJ_BE] = ir->op2;
1542 args[3^LJ_BE] = (ir+1)->op2;
1543 asm_setupresult(as, ir, &ci);
1544 emit_call(as, (void *)ci.func, 0);
1545 ci.func = NULL;
1546 asm_gencall(as, &ci, args);
1547 }
1548 #endif
1549
asm_min_max(ASMState * as,IRIns * ir,int ismax)1550 static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1551 {
1552 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1553 Reg dest = ra_dest(as, ir, RSET_FPR);
1554 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1555 right = (left >> 8); left &= 255;
1556 if (dest == left) {
1557 emit_fg(as, MIPSI_MOVT_D, dest, right);
1558 } else {
1559 emit_fg(as, MIPSI_MOVF_D, dest, left);
1560 if (dest != right) emit_fg(as, MIPSI_MOV_D, dest, right);
1561 }
1562 emit_fgh(as, MIPSI_C_OLT_D, 0, ismax ? left : right, ismax ? right : left);
1563 } else {
1564 Reg dest = ra_dest(as, ir, RSET_GPR);
1565 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1566 right = (left >> 8); left &= 255;
1567 if (dest == left) {
1568 emit_dst(as, MIPSI_MOVN, dest, right, RID_TMP);
1569 } else {
1570 emit_dst(as, MIPSI_MOVZ, dest, left, RID_TMP);
1571 if (dest != right) emit_move(as, dest, right);
1572 }
1573 emit_dst(as, MIPSI_SLT, RID_TMP,
1574 ismax ? left : right, ismax ? right : left);
1575 }
1576 }
1577
1578 #define asm_min(as, ir) asm_min_max(as, ir, 0)
1579 #define asm_max(as, ir) asm_min_max(as, ir, 1)
1580
1581 /* -- Comparisons --------------------------------------------------------- */
1582
1583 #if LJ_SOFTFP
1584 /* SFP comparisons. */
asm_sfpcomp(ASMState * as,IRIns * ir)1585 static void asm_sfpcomp(ASMState *as, IRIns *ir)
1586 {
1587 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp];
1588 RegSet drop = RSET_SCRATCH;
1589 Reg r;
1590 IRRef args[4];
1591 args[LJ_LE ? 0 : 1] = ir->op1; args[LJ_LE ? 1 : 0] = (ir+1)->op1;
1592 args[LJ_LE ? 2 : 3] = ir->op2; args[LJ_LE ? 3 : 2] = (ir+1)->op2;
1593
1594 for (r = REGARG_FIRSTGPR; r <= REGARG_FIRSTGPR+3; r++) {
1595 if (!rset_test(as->freeset, r) &&
1596 regcost_ref(as->cost[r]) == args[r-REGARG_FIRSTGPR])
1597 rset_clear(drop, r);
1598 }
1599 ra_evictset(as, drop);
1600
1601 asm_setupresult(as, ir, ci);
1602
1603 switch ((IROp)ir->o) {
1604 case IR_LT:
1605 asm_guard(as, MIPSI_BGEZ, RID_RET, 0);
1606 break;
1607 case IR_ULT:
1608 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
1609 emit_loadi(as, RID_TMP, 1);
1610 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO);
1611 break;
1612 case IR_GE:
1613 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
1614 emit_loadi(as, RID_TMP, 2);
1615 asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
1616 break;
1617 case IR_LE:
1618 asm_guard(as, MIPSI_BGTZ, RID_RET, 0);
1619 break;
1620 case IR_GT:
1621 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
1622 emit_loadi(as, RID_TMP, 2);
1623 asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
1624 break;
1625 case IR_UGE:
1626 asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
1627 break;
1628 case IR_ULE:
1629 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
1630 emit_loadi(as, RID_TMP, 1);
1631 break;
1632 case IR_UGT: case IR_ABC:
1633 asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
1634 break;
1635 case IR_EQ: case IR_NE:
1636 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_RET, RID_ZERO);
1637 default:
1638 break;
1639 }
1640 asm_gencall(as, ci, args);
1641 }
1642 #endif
1643
asm_comp(ASMState * as,IRIns * ir)1644 static void asm_comp(ASMState *as, IRIns *ir)
1645 {
1646 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1647 IROp op = ir->o;
1648 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1649 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1650 right = (left >> 8); left &= 255;
1651 asm_guard(as, (op&1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
1652 emit_fgh(as, MIPSI_C_OLT_D + ((op&3) ^ ((op>>2)&1)), 0, left, right);
1653 } else {
1654 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1655 if (op == IR_ABC) op = IR_UGT;
1656 if ((op&4) == 0 && irref_isk(ir->op2) && IR(ir->op2)->i == 0) {
1657 MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) :
1658 ((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ);
1659 asm_guard(as, mi, left, 0);
1660 } else {
1661 if (irref_isk(ir->op2)) {
1662 int32_t k = IR(ir->op2)->i;
1663 if ((op&2)) k++;
1664 if (checki16(k)) {
1665 asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1666 emit_tsi(as, (op&4) ? MIPSI_SLTIU : MIPSI_SLTI,
1667 RID_TMP, left, k);
1668 return;
1669 }
1670 }
1671 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1672 asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1673 emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT,
1674 RID_TMP, (op&2) ? right : left, (op&2) ? left : right);
1675 }
1676 }
1677 }
1678
asm_equal(ASMState * as,IRIns * ir)1679 static void asm_equal(ASMState *as, IRIns *ir)
1680 {
1681 Reg right, left = ra_alloc2(as, ir, (!LJ_SOFTFP && irt_isnum(ir->t)) ? RSET_FPR : RSET_GPR);
1682 right = (left >> 8); left &= 255;
1683 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1684 asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
1685 emit_fgh(as, MIPSI_C_EQ_D, 0, left, right);
1686 } else {
1687 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, left, right);
1688 }
1689 }
1690
1691 #if LJ_HASFFI
1692 /* 64 bit integer comparisons. */
asm_comp64(ASMState * as,IRIns * ir)1693 static void asm_comp64(ASMState *as, IRIns *ir)
1694 {
1695 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
1696 IROp op = (ir-1)->o;
1697 MCLabel l_end;
1698 Reg rightlo, leftlo, righthi, lefthi = ra_alloc2(as, ir, RSET_GPR);
1699 righthi = (lefthi >> 8); lefthi &= 255;
1700 leftlo = ra_alloc2(as, ir-1,
1701 rset_exclude(rset_exclude(RSET_GPR, lefthi), righthi));
1702 rightlo = (leftlo >> 8); leftlo &= 255;
1703 asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1704 l_end = emit_label(as);
1705 if (lefthi != righthi)
1706 emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT, RID_TMP,
1707 (op&2) ? righthi : lefthi, (op&2) ? lefthi : righthi);
1708 emit_dst(as, MIPSI_SLTU, RID_TMP,
1709 (op&2) ? rightlo : leftlo, (op&2) ? leftlo : rightlo);
1710 if (lefthi != righthi)
1711 emit_branch(as, MIPSI_BEQ, lefthi, righthi, l_end);
1712 }
1713
asm_comp64eq(ASMState * as,IRIns * ir)1714 static void asm_comp64eq(ASMState *as, IRIns *ir)
1715 {
1716 Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
1717 right = (left >> 8); left &= 255;
1718 asm_guard(as, ((ir-1)->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_TMP, RID_ZERO);
1719 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1720 emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
1721 emit_dst(as, MIPSI_XOR, tmp, left, right);
1722 left = ra_alloc2(as, ir-1, RSET_GPR);
1723 right = (left >> 8); left &= 255;
1724 emit_dst(as, MIPSI_XOR, RID_TMP, left, right);
1725 }
1726 #endif
1727
1728 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1729
1730 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
asm_hiop(ASMState * as,IRIns * ir)1731 static void asm_hiop(ASMState *as, IRIns *ir)
1732 {
1733 #if LJ_HASFFI || LJ_SOFTFP
1734 /* HIOP is marked as a store because it needs its own DCE logic. */
1735 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1736 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1737 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
1738 as->curins--; /* Always skip the CONV. */
1739 #if LJ_HASFFI && !LJ_SOFTFP
1740 if (usehi || uselo)
1741 asm_conv64(as, ir);
1742 return;
1743 #endif
1744 } else if ((ir-1)->o < IR_EQ) { /* 64 bit integer comparisons. ORDER IR. */
1745 as->curins--; /* Always skip the loword comparison. */
1746 #if LJ_SOFTFP
1747 if (!irt_isint(ir->t)) {
1748 asm_sfpcomp(as, ir-1);
1749 return;
1750 }
1751 #endif
1752 #if LJ_HASFFI
1753 asm_comp64(as, ir);
1754 #endif
1755 return;
1756 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
1757 as->curins--; /* Always skip the loword comparison. */
1758 #if LJ_SOFTFP
1759 if (!irt_isint(ir->t)) {
1760 asm_sfpcomp(as, ir-1);
1761 return;
1762 }
1763 #endif
1764 #if LJ_HASFFI
1765 asm_comp64eq(as, ir);
1766 #endif
1767 return;
1768 #if LJ_SOFTFP
1769 } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) {
1770 as->curins--; /* Always skip the loword min/max. */
1771 if (uselo || usehi)
1772 asm_sfpmin_max(as, ir-1);
1773 return;
1774 #endif
1775 } else if ((ir-1)->o == IR_XSTORE) {
1776 as->curins--; /* Handle both stores here. */
1777 if ((ir-1)->r != RID_SINK) {
1778 asm_xstore_(as, ir, LJ_LE ? 4 : 0);
1779 asm_xstore_(as, ir-1, LJ_LE ? 0 : 4);
1780 }
1781 return;
1782 }
1783 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1784 switch ((ir-1)->o) {
1785 #if LJ_HASFFI
1786 case IR_ADD: as->curins--; asm_add64(as, ir); break;
1787 case IR_SUB: as->curins--; asm_sub64(as, ir); break;
1788 case IR_NEG: as->curins--; asm_neg64(as, ir); break;
1789 #endif
1790 #if LJ_SOFTFP
1791 case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
1792 case IR_STRTO:
1793 if (!uselo)
1794 ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */
1795 break;
1796 #endif
1797 case IR_CALLN:
1798 case IR_CALLS:
1799 case IR_CALLXS:
1800 if (!uselo)
1801 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
1802 break;
1803 #if LJ_SOFTFP
1804 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR:
1805 #endif
1806 case IR_CNEWI:
1807 /* Nothing to do here. Handled by lo op itself. */
1808 break;
1809 default: lua_assert(0); break;
1810 }
1811 #else
1812 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused without FFI. */
1813 #endif
1814 }
1815
1816 /* -- Profiling ----------------------------------------------------------- */
1817
asm_prof(ASMState * as,IRIns * ir)1818 static void asm_prof(ASMState *as, IRIns *ir)
1819 {
1820 UNUSED(ir);
1821 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
1822 emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, HOOK_PROFILE);
1823 emit_lsglptr(as, MIPSI_LBU, RID_TMP,
1824 (int32_t)offsetof(global_State, hookmask));
1825 }
1826
1827 /* -- Stack handling ------------------------------------------------------ */
1828
1829 /* Check Lua stack size for overflow. Use exit handler as fallback. */
asm_stack_check(ASMState * as,BCReg topslot,IRIns * irp,RegSet allow,ExitNo exitno)1830 static void asm_stack_check(ASMState *as, BCReg topslot,
1831 IRIns *irp, RegSet allow, ExitNo exitno)
1832 {
1833 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1834 Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
1835 ExitNo oldsnap = as->snapno;
1836 rset_clear(allow, pbase);
1837 tmp = allow ? rset_pickbot(allow) :
1838 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
1839 as->snapno = exitno;
1840 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
1841 as->snapno = oldsnap;
1842 if (allow == RSET_EMPTY) /* Restore temp. register. */
1843 emit_tsi(as, MIPSI_LW, tmp, RID_SP, 0);
1844 else
1845 ra_modified(as, tmp);
1846 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot));
1847 emit_dst(as, MIPSI_SUBU, RID_TMP, tmp, pbase);
1848 emit_tsi(as, MIPSI_LW, tmp, tmp, offsetof(lua_State, maxstack));
1849 if (pbase == RID_TMP)
1850 emit_getgl(as, RID_TMP, jit_base);
1851 emit_getgl(as, tmp, cur_L);
1852 if (allow == RSET_EMPTY) /* Spill temp. register. */
1853 emit_tsi(as, MIPSI_SW, tmp, RID_SP, 0);
1854 }
1855
1856 /* Restore Lua stack from on-trace state. */
asm_stack_restore(ASMState * as,SnapShot * snap)1857 static void asm_stack_restore(ASMState *as, SnapShot *snap)
1858 {
1859 SnapEntry *map = &as->T->snapmap[snap->mapofs];
1860 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1];
1861 MSize n, nent = snap->nent;
1862 /* Store the value of all modified slots to the Lua stack. */
1863 for (n = 0; n < nent; n++) {
1864 SnapEntry sn = map[n];
1865 BCReg s = snap_slot(sn);
1866 int32_t ofs = 8*((int32_t)s-1);
1867 IRRef ref = snap_ref(sn);
1868 IRIns *ir = IR(ref);
1869 if ((sn & SNAP_NORESTORE))
1870 continue;
1871 if (irt_isnum(ir->t)) {
1872 #if LJ_SOFTFP
1873 Reg tmp;
1874 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1875 lua_assert(irref_isk(ref)); /* LJ_SOFTFP: must be a number constant. */
1876 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, allow);
1877 emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?4:0));
1878 if (rset_test(as->freeset, tmp+1)) allow = RID2RSET(tmp+1);
1879 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, allow);
1880 emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?0:4));
1881 #else
1882 Reg src = ra_alloc1(as, ref, RSET_FPR);
1883 emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs);
1884 #endif
1885 } else {
1886 Reg type;
1887 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1888 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
1889 if (!irt_ispri(ir->t)) {
1890 Reg src = ra_alloc1(as, ref, allow);
1891 rset_clear(allow, src);
1892 emit_tsi(as, MIPSI_SW, src, RID_BASE, ofs+(LJ_BE?4:0));
1893 }
1894 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1895 if (s == 0) continue; /* Do not overwrite link to previous frame. */
1896 type = ra_allock(as, (int32_t)(*flinks--), allow);
1897 #if LJ_SOFTFP
1898 } else if ((sn & SNAP_SOFTFPNUM)) {
1899 type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPR, RID_BASE));
1900 #endif
1901 } else {
1902 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1903 }
1904 emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4));
1905 }
1906 checkmclim(as);
1907 }
1908 lua_assert(map + nent == flinks);
1909 }
1910
1911 /* -- GC handling --------------------------------------------------------- */
1912
1913 /* Check GC threshold and do one or more GC steps. */
asm_gc_check(ASMState * as)1914 static void asm_gc_check(ASMState *as)
1915 {
1916 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
1917 IRRef args[2];
1918 MCLabel l_end;
1919 Reg tmp;
1920 ra_evictset(as, RSET_SCRATCH);
1921 l_end = emit_label(as);
1922 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1923 /* Assumes asm_snap_prep() already done. */
1924 asm_guard(as, MIPSI_BNE, RID_RET, RID_ZERO);
1925 args[0] = ASMREF_TMP1; /* global_State *g */
1926 args[1] = ASMREF_TMP2; /* MSize steps */
1927 asm_gencall(as, ci, args);
1928 emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1929 tmp = ra_releasetmp(as, ASMREF_TMP2);
1930 emit_loadi(as, tmp, as->gcsteps);
1931 /* Jump around GC step if GC total < GC threshold. */
1932 emit_branch(as, MIPSI_BNE, RID_TMP, RID_ZERO, l_end);
1933 emit_dst(as, MIPSI_SLTU, RID_TMP, RID_TMP, tmp);
1934 emit_getgl(as, tmp, gc.threshold);
1935 emit_getgl(as, RID_TMP, gc.total);
1936 as->gcsteps = 0;
1937 checkmclim(as);
1938 }
1939
1940 /* -- Loop handling ------------------------------------------------------- */
1941
1942 /* Fixup the loop branch. */
asm_loop_fixup(ASMState * as)1943 static void asm_loop_fixup(ASMState *as)
1944 {
1945 MCode *p = as->mctop;
1946 MCode *target = as->mcp;
1947 p[-1] = MIPSI_NOP;
1948 if (as->loopinv) { /* Inverted loop branch? */
1949 /* asm_guard already inverted the cond branch. Only patch the target. */
1950 p[-3] |= ((target-p+2) & 0x0000ffffu);
1951 } else {
1952 p[-2] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
1953 }
1954 }
1955
1956 /* -- Head of trace ------------------------------------------------------- */
1957
1958 /* Coalesce BASE register for a root trace. */
asm_head_root_base(ASMState * as)1959 static void asm_head_root_base(ASMState *as)
1960 {
1961 IRIns *ir = IR(REF_BASE);
1962 Reg r = ir->r;
1963 if (as->loopinv) as->mctop--;
1964 if (ra_hasreg(r)) {
1965 ra_free(as, r);
1966 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1967 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
1968 if (r != RID_BASE)
1969 emit_move(as, r, RID_BASE);
1970 }
1971 }
1972
1973 /* Coalesce BASE register for a side trace. */
asm_head_side_base(ASMState * as,IRIns * irp,RegSet allow)1974 static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow)
1975 {
1976 IRIns *ir = IR(REF_BASE);
1977 Reg r = ir->r;
1978 if (as->loopinv) as->mctop--;
1979 if (ra_hasreg(r)) {
1980 ra_free(as, r);
1981 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1982 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
1983 if (irp->r == r) {
1984 rset_clear(allow, r); /* Mark same BASE register as coalesced. */
1985 } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
1986 rset_clear(allow, irp->r);
1987 emit_move(as, r, irp->r); /* Move from coalesced parent reg. */
1988 } else {
1989 emit_getgl(as, r, jit_base); /* Otherwise reload BASE. */
1990 }
1991 }
1992 return allow;
1993 }
1994
1995 /* -- Tail of trace ------------------------------------------------------- */
1996
1997 /* Fixup the tail code. */
asm_tail_fixup(ASMState * as,TraceNo lnk)1998 static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1999 {
2000 MCode *target = lnk ? traceref(as->J,lnk)->mcode : (MCode *)lj_vm_exit_interp;
2001 int32_t spadj = as->T->spadjust;
2002 MCode *p = as->mctop-1;
2003 *p = spadj ? (MIPSI_ADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
2004 p[-1] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2005 }
2006
2007 /* Prepare tail of code. */
asm_tail_prep(ASMState * as)2008 static void asm_tail_prep(ASMState *as)
2009 {
2010 as->mcp = as->mctop-2; /* Leave room for branch plus nop or stack adj. */
2011 as->invmcp = as->loopref ? as->mcp : NULL;
2012 }
2013
2014 /* -- Trace setup --------------------------------------------------------- */
2015
2016 /* Ensure there are enough stack slots for call arguments. */
asm_setup_call_slots(ASMState * as,IRIns * ir,const CCallInfo * ci)2017 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2018 {
2019 IRRef args[CCI_NARGS_MAX*2];
2020 uint32_t i, nargs = CCI_XNARGS(ci);
2021 int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2022 asm_collectargs(as, ir, ci, args);
2023 for (i = 0; i < nargs; i++) {
2024 if (!LJ_SOFTFP && args[i] && irt_isfp(IR(args[i])->t) &&
2025 nfpr > 0 && !(ci->flags & CCI_VARARG)) {
2026 nfpr--;
2027 ngpr -= irt_isnum(IR(args[i])->t) ? 2 : 1;
2028 } else if (!LJ_SOFTFP && args[i] && irt_isnum(IR(args[i])->t)) {
2029 nfpr = 0;
2030 ngpr = ngpr & ~1;
2031 if (ngpr > 0) ngpr -= 2; else nslots = (nslots+3) & ~1;
2032 } else {
2033 nfpr = 0;
2034 if (ngpr > 0) ngpr--; else nslots++;
2035 }
2036 }
2037 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
2038 as->evenspill = nslots;
2039 return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
2040 }
2041
asm_setup_target(ASMState * as)2042 static void asm_setup_target(ASMState *as)
2043 {
2044 asm_sparejump_setup(as);
2045 asm_exitstub_setup(as);
2046 }
2047
2048 /* -- Trace patching ------------------------------------------------------ */
2049
2050 /* Patch exit jumps of existing machine code to a new target. */
lj_asm_patchexit(jit_State * J,GCtrace * T,ExitNo exitno,MCode * target)2051 void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2052 {
2053 MCode *p = T->mcode;
2054 MCode *pe = (MCode *)((char *)p + T->szmcode);
2055 MCode *px = exitstub_trace_addr(T, exitno);
2056 MCode *cstart = NULL, *cstop = NULL;
2057 MCode *mcarea = lj_mcode_patch(J, p, 0);
2058 MCode exitload = MIPSI_LI | MIPSF_T(RID_TMP) | exitno;
2059 MCode tjump = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2060 for (p++; p < pe; p++) {
2061 if (*p == exitload) { /* Look for load of exit number. */
2062 if (((p[-1] ^ (px-p)) & 0xffffu) == 0) { /* Look for exitstub branch. */
2063 ptrdiff_t delta = target - p;
2064 if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */
2065 patchbranch:
2066 p[-1] = (p[-1] & 0xffff0000u) | (delta & 0xffffu);
2067 *p = MIPSI_NOP; /* Replace the load of the exit number. */
2068 cstop = p;
2069 if (!cstart) cstart = p-1;
2070 } else { /* Branch out of range. Use spare jump slot in mcarea. */
2071 int i;
2072 for (i = 2; i < 2+MIPS_SPAREJUMP*2; i += 2) {
2073 if (mcarea[i] == tjump) {
2074 delta = mcarea+i - p;
2075 goto patchbranch;
2076 } else if (mcarea[i] == MIPSI_NOP) {
2077 mcarea[i] = tjump;
2078 cstart = mcarea+i;
2079 delta = mcarea+i - p;
2080 goto patchbranch;
2081 }
2082 }
2083 /* Ignore jump slot overflow. Child trace is simply not attached. */
2084 }
2085 } else if (p+1 == pe) {
2086 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
2087 lua_assert(p[1] == MIPSI_NOP);
2088 p[1] = tjump;
2089 *p = MIPSI_NOP; /* Replace the load of the exit number. */
2090 cstop = p+2;
2091 if (!cstart) cstart = p+1;
2092 }
2093 }
2094 }
2095 if (cstart) lj_mcode_sync(cstart, cstop);
2096 lj_mcode_patch(J, mcarea, 1);
2097 }
2098
2099