1 /*
2 ** MIPS IR assembler (SSA IR -> machine code).
3 ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 /* -- Register allocator extensions --------------------------------------- */
7
8 /* Allocate a register with a hint. */
ra_hintalloc(ASMState * as,IRRef ref,Reg hint,RegSet allow)9 static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
10 {
11 Reg r = IR(ref)->r;
12 if (ra_noreg(r)) {
13 if (!ra_hashint(r) && !iscrossref(as, ref))
14 ra_sethint(IR(ref)->r, hint); /* Propagate register hint. */
15 r = ra_allocref(as, ref, allow);
16 }
17 ra_noweak(as, r);
18 return r;
19 }
20
21 /* Allocate a register or RID_ZERO. */
ra_alloc1z(ASMState * as,IRRef ref,RegSet allow)22 static Reg ra_alloc1z(ASMState *as, IRRef ref, RegSet allow)
23 {
24 Reg r = IR(ref)->r;
25 if (ra_noreg(r)) {
26 if (!(allow & RSET_FPR) && irref_isk(ref) && get_kval(IR(ref)) == 0)
27 return RID_ZERO;
28 r = ra_allocref(as, ref, allow);
29 } else {
30 ra_noweak(as, r);
31 }
32 return r;
33 }
34
35 /* Allocate two source registers for three-operand instructions. */
ra_alloc2(ASMState * as,IRIns * ir,RegSet allow)36 static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
37 {
38 IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
39 Reg left = irl->r, right = irr->r;
40 if (ra_hasreg(left)) {
41 ra_noweak(as, left);
42 if (ra_noreg(right))
43 right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
44 else
45 ra_noweak(as, right);
46 } else if (ra_hasreg(right)) {
47 ra_noweak(as, right);
48 left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
49 } else if (ra_hashint(right)) {
50 right = ra_alloc1z(as, ir->op2, allow);
51 left = ra_alloc1z(as, ir->op1, rset_exclude(allow, right));
52 } else {
53 left = ra_alloc1z(as, ir->op1, allow);
54 right = ra_alloc1z(as, ir->op2, rset_exclude(allow, left));
55 }
56 return left | (right << 8);
57 }
58
59 /* -- Guard handling ------------------------------------------------------ */
60
61 /* Need some spare long-range jump slots, for out-of-range branches. */
62 #define MIPS_SPAREJUMP 4
63
64 /* Setup spare long-range jump slots per mcarea. */
asm_sparejump_setup(ASMState * as)65 static void asm_sparejump_setup(ASMState *as)
66 {
67 MCode *mxp = as->mcbot;
68 /* Assumes sizeof(MCLink) == 8. */
69 if (((uintptr_t)mxp & (LJ_PAGESIZE-1)) == 8) {
70 lua_assert(MIPSI_NOP == 0);
71 memset(mxp+2, 0, MIPS_SPAREJUMP*8);
72 mxp += MIPS_SPAREJUMP*2;
73 lua_assert(mxp < as->mctop);
74 lj_mcode_sync(as->mcbot, mxp);
75 lj_mcode_commitbot(as->J, mxp);
76 as->mcbot = mxp;
77 as->mclim = as->mcbot + MCLIM_REDZONE;
78 }
79 }
80
81 /* Setup exit stub after the end of each trace. */
asm_exitstub_setup(ASMState * as)82 static void asm_exitstub_setup(ASMState *as)
83 {
84 MCode *mxp = as->mctop;
85 /* sw TMP, 0(sp); j ->vm_exit_handler; li TMP, traceno */
86 *--mxp = MIPSI_LI|MIPSF_T(RID_TMP)|as->T->traceno;
87 *--mxp = MIPSI_J|((((uintptr_t)(void *)lj_vm_exit_handler)>>2)&0x03ffffffu);
88 lua_assert(((uintptr_t)mxp ^ (uintptr_t)(void *)lj_vm_exit_handler)>>28 == 0);
89 *--mxp = MIPSI_SW|MIPSF_T(RID_TMP)|MIPSF_S(RID_SP)|0;
90 as->mctop = mxp;
91 }
92
93 /* Keep this in-sync with exitstub_trace_addr(). */
94 #define asm_exitstub_addr(as) ((as)->mctop)
95
96 /* Emit conditional branch to exit for guard. */
asm_guard(ASMState * as,MIPSIns mi,Reg rs,Reg rt)97 static void asm_guard(ASMState *as, MIPSIns mi, Reg rs, Reg rt)
98 {
99 MCode *target = asm_exitstub_addr(as);
100 MCode *p = as->mcp;
101 if (LJ_UNLIKELY(p == as->invmcp)) {
102 as->invmcp = NULL;
103 as->loopinv = 1;
104 as->mcp = p+1;
105 mi = mi ^ ((mi>>28) == 1 ? 0x04000000u : 0x00010000u); /* Invert cond. */
106 target = p; /* Patch target later in asm_loop_fixup. */
107 }
108 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
109 emit_branch(as, mi, rs, rt, target);
110 }
111
112 /* -- Operand fusion ------------------------------------------------------ */
113
114 /* Limit linear search to this distance. Avoids O(n^2) behavior. */
115 #define CONFLICT_SEARCH_LIM 31
116
117 /* Check if there's no conflicting instruction between curins and ref. */
noconflict(ASMState * as,IRRef ref,IROp conflict)118 static int noconflict(ASMState *as, IRRef ref, IROp conflict)
119 {
120 IRIns *ir = as->ir;
121 IRRef i = as->curins;
122 if (i > ref + CONFLICT_SEARCH_LIM)
123 return 0; /* Give up, ref is too far away. */
124 while (--i > ref)
125 if (ir[i].o == conflict)
126 return 0; /* Conflict found. */
127 return 1; /* Ok, no conflict. */
128 }
129
130 /* Fuse the array base of colocated arrays. */
asm_fuseabase(ASMState * as,IRRef ref)131 static int32_t asm_fuseabase(ASMState *as, IRRef ref)
132 {
133 IRIns *ir = IR(ref);
134 if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
135 !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
136 return (int32_t)sizeof(GCtab);
137 return 0;
138 }
139
140 /* Fuse array/hash/upvalue reference into register+offset operand. */
asm_fuseahuref(ASMState * as,IRRef ref,int32_t * ofsp,RegSet allow)141 static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
142 {
143 IRIns *ir = IR(ref);
144 if (ra_noreg(ir->r)) {
145 if (ir->o == IR_AREF) {
146 if (mayfuse(as, ref)) {
147 if (irref_isk(ir->op2)) {
148 IRRef tab = IR(ir->op1)->op1;
149 int32_t ofs = asm_fuseabase(as, tab);
150 IRRef refa = ofs ? tab : ir->op1;
151 ofs += 8*IR(ir->op2)->i;
152 if (checki16(ofs)) {
153 *ofsp = ofs;
154 return ra_alloc1(as, refa, allow);
155 }
156 }
157 }
158 } else if (ir->o == IR_HREFK) {
159 if (mayfuse(as, ref)) {
160 int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
161 if (checki16(ofs)) {
162 *ofsp = ofs;
163 return ra_alloc1(as, ir->op1, allow);
164 }
165 }
166 } else if (ir->o == IR_UREFC) {
167 if (irref_isk(ir->op1)) {
168 GCfunc *fn = ir_kfunc(IR(ir->op1));
169 intptr_t ofs = (intptr_t)&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv;
170 intptr_t jgl = (intptr_t)J2G(as->J);
171 if ((uintptr_t)(ofs-jgl) < 65536) {
172 *ofsp = ofs-jgl-32768;
173 return RID_JGL;
174 } else {
175 *ofsp = (int16_t)ofs;
176 return ra_allock(as, ofs-(int16_t)ofs, allow);
177 }
178 }
179 }
180 }
181 *ofsp = 0;
182 return ra_alloc1(as, ref, allow);
183 }
184
185 /* Fuse XLOAD/XSTORE reference into load/store operand. */
asm_fusexref(ASMState * as,MIPSIns mi,Reg rt,IRRef ref,RegSet allow,int32_t ofs)186 static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
187 RegSet allow, int32_t ofs)
188 {
189 IRIns *ir = IR(ref);
190 Reg base;
191 if (ra_noreg(ir->r) && canfuse(as, ir)) {
192 if (ir->o == IR_ADD) {
193 intptr_t ofs2;
194 if (irref_isk(ir->op2) && (ofs2 = ofs + get_kval(IR(ir->op2)),
195 checki16(ofs2))) {
196 ref = ir->op1;
197 ofs = (int32_t)ofs2;
198 }
199 } else if (ir->o == IR_STRREF) {
200 intptr_t ofs2 = 65536;
201 lua_assert(ofs == 0);
202 ofs = (int32_t)sizeof(GCstr);
203 if (irref_isk(ir->op2)) {
204 ofs2 = ofs + get_kval(IR(ir->op2));
205 ref = ir->op1;
206 } else if (irref_isk(ir->op1)) {
207 ofs2 = ofs + get_kval(IR(ir->op1));
208 ref = ir->op2;
209 }
210 if (!checki16(ofs2)) {
211 /* NYI: Fuse ADD with constant. */
212 Reg right, left = ra_alloc2(as, ir, allow);
213 right = (left >> 8); left &= 255;
214 emit_hsi(as, mi, rt, RID_TMP, ofs);
215 emit_dst(as, MIPSI_AADDU, RID_TMP, left, right);
216 return;
217 }
218 ofs = ofs2;
219 }
220 }
221 base = ra_alloc1(as, ref, allow);
222 emit_hsi(as, mi, rt, base, ofs);
223 }
224
225 /* -- Calls --------------------------------------------------------------- */
226
227 /* Generate a call to a C function. */
asm_gencall(ASMState * as,const CCallInfo * ci,IRRef * args)228 static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
229 {
230 uint32_t n, nargs = CCI_XNARGS(ci);
231 int32_t ofs = LJ_32 ? 16 : 0;
232 #if LJ_SOFTFP
233 Reg gpr = REGARG_FIRSTGPR;
234 #else
235 Reg gpr, fpr = REGARG_FIRSTFPR;
236 #endif
237 if ((void *)ci->func)
238 emit_call(as, (void *)ci->func, 1);
239 #if !LJ_SOFTFP
240 for (gpr = REGARG_FIRSTGPR; gpr <= REGARG_LASTGPR; gpr++)
241 as->cost[gpr] = REGCOST(~0u, ASMREF_L);
242 gpr = REGARG_FIRSTGPR;
243 #endif
244 for (n = 0; n < nargs; n++) { /* Setup args. */
245 IRRef ref = args[n];
246 if (ref) {
247 IRIns *ir = IR(ref);
248 #if !LJ_SOFTFP
249 if (irt_isfp(ir->t) && fpr <= REGARG_LASTFPR &&
250 !(ci->flags & CCI_VARARG)) {
251 lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */
252 ra_leftov(as, fpr, ref);
253 fpr += LJ_32 ? 2 : 1;
254 gpr += (LJ_32 && irt_isnum(ir->t)) ? 2 : 1;
255 } else
256 #endif
257 {
258 #if LJ_32 && !LJ_SOFTFP
259 fpr = REGARG_LASTFPR+1;
260 #endif
261 if (LJ_32 && irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
262 if (gpr <= REGARG_LASTGPR) {
263 lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */
264 #if !LJ_SOFTFP
265 if (irt_isfp(ir->t)) {
266 RegSet of = as->freeset;
267 Reg r;
268 /* Workaround to protect argument GPRs from being used for remat. */
269 as->freeset &= ~RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1);
270 r = ra_alloc1(as, ref, RSET_FPR);
271 as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
272 if (irt_isnum(ir->t)) {
273 #if LJ_32
274 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1);
275 emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r);
276 lua_assert(rset_test(as->freeset, gpr+1)); /* Already evicted. */
277 gpr += 2;
278 #else
279 emit_tg(as, MIPSI_DMFC1, gpr, r);
280 gpr++; fpr++;
281 #endif
282 } else if (irt_isfloat(ir->t)) {
283 emit_tg(as, MIPSI_MFC1, gpr, r);
284 gpr++;
285 #if LJ_64
286 fpr++;
287 #endif
288 }
289 } else
290 #endif
291 {
292 ra_leftov(as, gpr, ref);
293 gpr++;
294 #if LJ_64
295 fpr++;
296 #endif
297 }
298 } else {
299 Reg r = ra_alloc1z(as, ref, !LJ_SOFTFP && irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
300 #if LJ_32
301 if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
302 emit_spstore(as, ir, r, ofs);
303 ofs += irt_isnum(ir->t) ? 8 : 4;
304 #else
305 emit_spstore(as, ir, r, ofs + ((LJ_BE && (LJ_SOFTFP || r < RID_MAX_GPR) && !irt_is64(ir->t)) ? 4 : 0));
306 ofs += 8;
307 #endif
308 }
309 }
310 } else {
311 #if !LJ_SOFTFP
312 fpr = REGARG_LASTFPR+1;
313 #endif
314 if (gpr <= REGARG_LASTGPR) {
315 gpr++;
316 #if LJ_64
317 fpr++;
318 #endif
319 } else {
320 ofs += LJ_32 ? 4 : 8;
321 }
322 }
323 checkmclim(as);
324 }
325 }
326
327 /* Setup result reg/sp for call. Evict scratch regs. */
asm_setupresult(ASMState * as,IRIns * ir,const CCallInfo * ci)328 static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
329 {
330 RegSet drop = RSET_SCRATCH;
331 #if LJ_32
332 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
333 #endif
334 #if !LJ_SOFTFP
335 if ((ci->flags & CCI_NOFPRCLOBBER))
336 drop &= ~RSET_FPR;
337 #endif
338 if (ra_hasreg(ir->r))
339 rset_clear(drop, ir->r); /* Dest reg handled below. */
340 #if LJ_32
341 if (hiop && ra_hasreg((ir+1)->r))
342 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
343 #endif
344 ra_evictset(as, drop); /* Evictions must be performed first. */
345 if (ra_used(ir)) {
346 lua_assert(!irt_ispri(ir->t));
347 if (!LJ_SOFTFP && irt_isfp(ir->t)) {
348 if ((ci->flags & CCI_CASTU64)) {
349 int32_t ofs = sps_scale(ir->s);
350 Reg dest = ir->r;
351 if (ra_hasreg(dest)) {
352 ra_free(as, dest);
353 ra_modified(as, dest);
354 #if LJ_32
355 emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1);
356 emit_tg(as, MIPSI_MTC1, RID_RETLO, dest);
357 #else
358 emit_tg(as, MIPSI_DMTC1, RID_RET, dest);
359 #endif
360 }
361 if (ofs) {
362 #if LJ_32
363 emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0));
364 emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4));
365 #else
366 emit_tsi(as, MIPSI_SD, RID_RET, RID_SP, ofs);
367 #endif
368 }
369 } else {
370 ra_destreg(as, ir, RID_FPRET);
371 }
372 #if LJ_32
373 } else if (hiop) {
374 ra_destpair(as, ir);
375 #endif
376 } else {
377 ra_destreg(as, ir, RID_RET);
378 }
379 }
380 }
381
asm_callx(ASMState * as,IRIns * ir)382 static void asm_callx(ASMState *as, IRIns *ir)
383 {
384 IRRef args[CCI_NARGS_MAX*2];
385 CCallInfo ci;
386 IRRef func;
387 IRIns *irf;
388 ci.flags = asm_callx_flags(as, ir);
389 asm_collectargs(as, ir, &ci, args);
390 asm_setupresult(as, ir, &ci);
391 func = ir->op2; irf = IR(func);
392 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
393 if (irref_isk(func)) { /* Call to constant address. */
394 ci.func = (ASMFunction)(void *)get_kval(irf);
395 } else { /* Need specific register for indirect calls. */
396 Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR));
397 MCode *p = as->mcp;
398 if (r == RID_CFUNCADDR)
399 *--p = MIPSI_NOP;
400 else
401 *--p = MIPSI_MOVE | MIPSF_D(RID_CFUNCADDR) | MIPSF_S(r);
402 *--p = MIPSI_JALR | MIPSF_S(r);
403 as->mcp = p;
404 ci.func = (ASMFunction)(void *)0;
405 }
406 asm_gencall(as, &ci, args);
407 }
408
409 #if !LJ_SOFTFP
asm_callround(ASMState * as,IRIns * ir,IRCallID id)410 static void asm_callround(ASMState *as, IRIns *ir, IRCallID id)
411 {
412 /* The modified regs must match with the *.dasc implementation. */
413 RegSet drop = RID2RSET(RID_R1)|RID2RSET(RID_R12)|RID2RSET(RID_FPRET)|
414 RID2RSET(RID_F2)|RID2RSET(RID_F4)|RID2RSET(REGARG_FIRSTFPR);
415 if (ra_hasreg(ir->r)) rset_clear(drop, ir->r);
416 ra_evictset(as, drop);
417 ra_destreg(as, ir, RID_FPRET);
418 emit_call(as, (void *)lj_ir_callinfo[id].func, 0);
419 ra_leftov(as, REGARG_FIRSTFPR, ir->op1);
420 }
421 #endif
422
423 /* -- Returns ------------------------------------------------------------- */
424
425 /* Return to lower frame. Guard that it goes to the right spot. */
asm_retf(ASMState * as,IRIns * ir)426 static void asm_retf(ASMState *as, IRIns *ir)
427 {
428 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
429 void *pc = ir_kptr(IR(ir->op2));
430 int32_t delta = 1+LJ_FR2+bc_a(*((const BCIns *)pc - 1));
431 as->topslot -= (BCReg)delta;
432 if ((int32_t)as->topslot < 0) as->topslot = 0;
433 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
434 emit_setgl(as, base, jit_base);
435 emit_addptr(as, base, -8*delta);
436 asm_guard(as, MIPSI_BNE, RID_TMP,
437 ra_allock(as, igcptr(pc), rset_exclude(RSET_GPR, base)));
438 emit_tsi(as, MIPSI_AL, RID_TMP, base, -8);
439 }
440
441 /* -- Type conversions ---------------------------------------------------- */
442
443 #if !LJ_SOFTFP
asm_tointg(ASMState * as,IRIns * ir,Reg left)444 static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
445 {
446 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
447 Reg dest = ra_dest(as, ir, RSET_GPR);
448 asm_guard(as, MIPSI_BC1F, 0, 0);
449 emit_fgh(as, MIPSI_C_EQ_D, 0, tmp, left);
450 emit_fg(as, MIPSI_CVT_D_W, tmp, tmp);
451 emit_tg(as, MIPSI_MFC1, dest, tmp);
452 emit_fg(as, MIPSI_CVT_W_D, tmp, left);
453 }
454
asm_tobit(ASMState * as,IRIns * ir)455 static void asm_tobit(ASMState *as, IRIns *ir)
456 {
457 RegSet allow = RSET_FPR;
458 Reg dest = ra_dest(as, ir, RSET_GPR);
459 Reg left = ra_alloc1(as, ir->op1, allow);
460 Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
461 Reg tmp = ra_scratch(as, rset_clear(allow, right));
462 emit_tg(as, MIPSI_MFC1, dest, tmp);
463 emit_fgh(as, MIPSI_ADD_D, tmp, left, right);
464 }
465 #endif
466
asm_conv(ASMState * as,IRIns * ir)467 static void asm_conv(ASMState *as, IRIns *ir)
468 {
469 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
470 #if !LJ_SOFTFP
471 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
472 #endif
473 #if LJ_64
474 int st64 = (st == IRT_I64 || st == IRT_U64 || st == IRT_P64);
475 #endif
476 IRRef lref = ir->op1;
477 #if LJ_32
478 lua_assert(!(irt_isint64(ir->t) ||
479 (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */
480 #endif
481 #if LJ_32 && LJ_SOFTFP
482 /* FP conversions are handled by SPLIT. */
483 lua_assert(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT));
484 /* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
485 #else
486 lua_assert(irt_type(ir->t) != st);
487 if (irt_isfp(ir->t)) {
488 Reg dest = ra_dest(as, ir, RSET_FPR);
489 if (stfp) { /* FP to FP conversion. */
490 emit_fg(as, st == IRT_NUM ? MIPSI_CVT_S_D : MIPSI_CVT_D_S,
491 dest, ra_alloc1(as, lref, RSET_FPR));
492 } else if (st == IRT_U32) { /* U32 to FP conversion. */
493 /* y = (x ^ 0x8000000) + 2147483648.0 */
494 Reg left = ra_alloc1(as, lref, RSET_GPR);
495 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
496 if (irt_isfloat(ir->t))
497 emit_fg(as, MIPSI_CVT_S_D, dest, dest);
498 /* Must perform arithmetic with doubles to keep the precision. */
499 emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
500 emit_fg(as, MIPSI_CVT_D_W, dest, dest);
501 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
502 (void *)&as->J->k64[LJ_K64_2P31], RSET_GPR);
503 emit_tg(as, MIPSI_MTC1, RID_TMP, dest);
504 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left);
505 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
506 #if LJ_64
507 } else if(st == IRT_U64) { /* U64 to FP conversion. */
508 /* if (x >= 1u<<63) y = (double)(int64_t)(x&(1u<<63)-1) + pow(2.0, 63) */
509 Reg left = ra_alloc1(as, lref, RSET_GPR);
510 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
511 MCLabel l_end = emit_label(as);
512 if (irt_isfloat(ir->t)) {
513 emit_fgh(as, MIPSI_ADD_S, dest, dest, tmp);
514 emit_lsptr(as, MIPSI_LWC1, (tmp & 31), (void *)&as->J->k32[LJ_K32_2P63],
515 rset_exclude(RSET_GPR, left));
516 emit_fg(as, MIPSI_CVT_S_L, dest, dest);
517 } else {
518 emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
519 emit_lsptr(as, MIPSI_LDC1, (tmp & 31), (void *)&as->J->k64[LJ_K64_2P63],
520 rset_exclude(RSET_GPR, left));
521 emit_fg(as, MIPSI_CVT_D_L, dest, dest);
522 }
523 emit_branch(as, MIPSI_BGEZ, left, RID_ZERO, l_end);
524 emit_tg(as, MIPSI_DMTC1, RID_TMP, dest);
525 emit_tsml(as, MIPSI_DEXTM, RID_TMP, left, 30, 0);
526 #endif
527 } else { /* Integer to FP conversion. */
528 Reg left = ra_alloc1(as, lref, RSET_GPR);
529 #if LJ_32
530 emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
531 dest, dest);
532 emit_tg(as, MIPSI_MTC1, left, dest);
533 #else
534 MIPSIns mi = irt_isfloat(ir->t) ?
535 (st64 ? MIPSI_CVT_S_L : MIPSI_CVT_S_W) :
536 (st64 ? MIPSI_CVT_D_L : MIPSI_CVT_D_W);
537 emit_fg(as, mi, dest, dest);
538 emit_tg(as, st64 ? MIPSI_DMTC1 : MIPSI_MTC1, left, dest);
539 #endif
540 }
541 } else if (stfp) { /* FP to integer conversion. */
542 if (irt_isguard(ir->t)) {
543 /* Checked conversions are only supported from number to int. */
544 lua_assert(irt_isint(ir->t) && st == IRT_NUM);
545 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
546 } else {
547 Reg dest = ra_dest(as, ir, RSET_GPR);
548 Reg left = ra_alloc1(as, lref, RSET_FPR);
549 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
550 if (irt_isu32(ir->t)) { /* FP to U32 conversion. */
551 /* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
552 emit_dst(as, MIPSI_XOR, dest, dest, RID_TMP);
553 emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
554 emit_tg(as, MIPSI_MFC1, dest, tmp);
555 emit_fg(as, st == IRT_FLOAT ? MIPSI_FLOOR_W_S : MIPSI_FLOOR_W_D,
556 tmp, tmp);
557 emit_fgh(as, st == IRT_FLOAT ? MIPSI_SUB_S : MIPSI_SUB_D,
558 tmp, left, tmp);
559 if (st == IRT_FLOAT)
560 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
561 (void *)&as->J->k32[LJ_K32_2P31], RSET_GPR);
562 else
563 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
564 (void *)&as->J->k64[LJ_K64_2P31], RSET_GPR);
565 #if LJ_64
566 } else if (irt_isu64(ir->t)) { /* FP to U64 conversion. */
567 MCLabel l_end;
568 emit_tg(as, MIPSI_DMFC1, dest, tmp);
569 l_end = emit_label(as);
570 /* For inputs >= 2^63 add -2^64 and convert again. */
571 if (st == IRT_NUM) {
572 emit_fg(as, MIPSI_TRUNC_L_D, tmp, tmp);
573 emit_fgh(as, MIPSI_ADD_D, tmp, left, tmp);
574 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
575 (void *)&as->J->k64[LJ_K64_M2P64],
576 rset_exclude(RSET_GPR, dest));
577 emit_fg(as, MIPSI_TRUNC_L_D, tmp, left); /* Delay slot. */
578 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
579 emit_fgh(as, MIPSI_C_OLT_D, 0, left, tmp);
580 emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
581 (void *)&as->J->k64[LJ_K64_2P63],
582 rset_exclude(RSET_GPR, dest));
583 } else {
584 emit_fg(as, MIPSI_TRUNC_L_S, tmp, tmp);
585 emit_fgh(as, MIPSI_ADD_S, tmp, left, tmp);
586 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
587 (void *)&as->J->k32[LJ_K32_M2P64],
588 rset_exclude(RSET_GPR, dest));
589 emit_fg(as, MIPSI_TRUNC_L_S, tmp, left); /* Delay slot. */
590 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
591 emit_fgh(as, MIPSI_C_OLT_S, 0, left, tmp);
592 emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
593 (void *)&as->J->k32[LJ_K32_2P63],
594 rset_exclude(RSET_GPR, dest));
595 }
596 #endif
597 } else {
598 #if LJ_32
599 emit_tg(as, MIPSI_MFC1, dest, tmp);
600 emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D,
601 tmp, left);
602 #else
603 MIPSIns mi = irt_is64(ir->t) ?
604 (st == IRT_NUM ? MIPSI_TRUNC_L_D : MIPSI_TRUNC_L_S) :
605 (st == IRT_NUM ? MIPSI_TRUNC_W_D : MIPSI_TRUNC_W_S);
606 emit_tg(as, irt_is64(ir->t) ? MIPSI_DMFC1 : MIPSI_MFC1, dest, left);
607 emit_fg(as, mi, left, left);
608 #endif
609 }
610 }
611 } else
612 #endif
613 {
614 Reg dest = ra_dest(as, ir, RSET_GPR);
615 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
616 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
617 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
618 if ((ir->op2 & IRCONV_SEXT)) {
619 if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
620 emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left);
621 } else {
622 uint32_t shift = st == IRT_I8 ? 24 : 16;
623 emit_dta(as, MIPSI_SRA, dest, dest, shift);
624 emit_dta(as, MIPSI_SLL, dest, left, shift);
625 }
626 } else {
627 emit_tsi(as, MIPSI_ANDI, dest, left,
628 (int32_t)(st == IRT_U8 ? 0xff : 0xffff));
629 }
630 } else { /* 32/64 bit integer conversions. */
631 #if LJ_32
632 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
633 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
634 #else
635 if (irt_is64(ir->t)) {
636 if (st64) {
637 /* 64/64 bit no-op (cast)*/
638 ra_leftov(as, dest, lref);
639 } else {
640 Reg left = ra_alloc1(as, lref, RSET_GPR);
641 if ((ir->op2 & IRCONV_SEXT)) { /* 32 to 64 bit sign extension. */
642 emit_dta(as, MIPSI_SLL, dest, left, 0);
643 } else { /* 32 to 64 bit zero extension. */
644 emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
645 }
646 }
647 } else {
648 if (st64) {
649 /* This is either a 32 bit reg/reg mov which zeroes the hiword
650 ** or a load of the loword from a 64 bit address.
651 */
652 Reg left = ra_alloc1(as, lref, RSET_GPR);
653 emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
654 } else { /* 32/32 bit no-op (cast). */
655 /* Do nothing, but may need to move regs. */
656 ra_leftov(as, dest, lref);
657 }
658 }
659 #endif
660 }
661 }
662 }
663
asm_strto(ASMState * as,IRIns * ir)664 static void asm_strto(ASMState *as, IRIns *ir)
665 {
666 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
667 IRRef args[2];
668 int32_t ofs = 0;
669 #if LJ_SOFTFP
670 ra_evictset(as, RSET_SCRATCH);
671 if (ra_used(ir)) {
672 if (ra_hasspill(ir->s) && ra_hasspill((ir+1)->s) &&
673 (ir->s & 1) == LJ_BE && (ir->s ^ 1) == (ir+1)->s) {
674 int i;
675 for (i = 0; i < 2; i++) {
676 Reg r = (ir+i)->r;
677 if (ra_hasreg(r)) {
678 ra_free(as, r);
679 ra_modified(as, r);
680 emit_spload(as, ir+i, r, sps_scale((ir+i)->s));
681 }
682 }
683 ofs = sps_scale(ir->s & ~1);
684 } else {
685 Reg rhi = ra_dest(as, ir+1, RSET_GPR);
686 Reg rlo = ra_dest(as, ir, rset_exclude(RSET_GPR, rhi));
687 emit_tsi(as, MIPSI_LW, rhi, RID_SP, ofs+(LJ_BE?0:4));
688 emit_tsi(as, MIPSI_LW, rlo, RID_SP, ofs+(LJ_BE?4:0));
689 }
690 }
691 #else
692 RegSet drop = RSET_SCRATCH;
693 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */
694 ra_evictset(as, drop);
695 ofs = sps_scale(ir->s);
696 #endif
697 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO); /* Test return status. */
698 args[0] = ir->op1; /* GCstr *str */
699 args[1] = ASMREF_TMP1; /* TValue *n */
700 asm_gencall(as, ci, args);
701 /* Store the result to the spill slot or temp slots. */
702 emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1),
703 RID_SP, ofs);
704 }
705
706 /* -- Memory references --------------------------------------------------- */
707
708 #if LJ_64
709 /* Store tagged value for ref at base+ofs. */
asm_tvstore64(ASMState * as,Reg base,int32_t ofs,IRRef ref)710 static void asm_tvstore64(ASMState *as, Reg base, int32_t ofs, IRRef ref)
711 {
712 RegSet allow = rset_exclude(RSET_GPR, base);
713 IRIns *ir = IR(ref);
714 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
715 if (irref_isk(ref)) {
716 TValue k;
717 lj_ir_kvalue(as->J->L, &k, ir);
718 emit_tsi(as, MIPSI_SD, ra_allock(as, (int64_t)k.u64, allow), base, ofs);
719 } else {
720 Reg src = ra_alloc1(as, ref, allow);
721 Reg type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47,
722 rset_exclude(allow, src));
723 emit_tsi(as, MIPSI_SD, RID_TMP, base, ofs);
724 if (irt_isinteger(ir->t)) {
725 emit_dst(as, MIPSI_DADDU, RID_TMP, RID_TMP, type);
726 emit_tsml(as, MIPSI_DEXT, RID_TMP, src, 31, 0);
727 } else {
728 emit_dst(as, MIPSI_DADDU, RID_TMP, src, type);
729 }
730 }
731 }
732 #endif
733
734 /* Get pointer to TValue. */
asm_tvptr(ASMState * as,Reg dest,IRRef ref)735 static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
736 {
737 IRIns *ir = IR(ref);
738 if (irt_isnum(ir->t)) {
739 if (irref_isk(ref)) /* Use the number constant itself as a TValue. */
740 ra_allockreg(as, igcptr(ir_knum(ir)), dest);
741 else /* Otherwise force a spill and use the spill slot. */
742 emit_tsi(as, MIPSI_AADDIU, dest, RID_SP, ra_spill(as, ir));
743 } else {
744 /* Otherwise use g->tmptv to hold the TValue. */
745 #if LJ_32
746 RegSet allow = rset_exclude(RSET_GPR, dest);
747 Reg type;
748 emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, (int32_t)(offsetof(global_State, tmptv)-32768));
749 if (!irt_ispri(ir->t)) {
750 Reg src = ra_alloc1(as, ref, allow);
751 emit_setgl(as, src, tmptv.gcr);
752 }
753 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
754 type = ra_alloc1(as, ref+1, allow);
755 else
756 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
757 emit_setgl(as, type, tmptv.it);
758 #else
759 asm_tvstore64(as, dest, 0, ref);
760 emit_tsi(as, MIPSI_DADDIU, dest, RID_JGL,
761 (int32_t)(offsetof(global_State, tmptv)-32768));
762 #endif
763 }
764 }
765
asm_aref(ASMState * as,IRIns * ir)766 static void asm_aref(ASMState *as, IRIns *ir)
767 {
768 Reg dest = ra_dest(as, ir, RSET_GPR);
769 Reg idx, base;
770 if (irref_isk(ir->op2)) {
771 IRRef tab = IR(ir->op1)->op1;
772 int32_t ofs = asm_fuseabase(as, tab);
773 IRRef refa = ofs ? tab : ir->op1;
774 ofs += 8*IR(ir->op2)->i;
775 if (checki16(ofs)) {
776 base = ra_alloc1(as, refa, RSET_GPR);
777 emit_tsi(as, MIPSI_AADDIU, dest, base, ofs);
778 return;
779 }
780 }
781 base = ra_alloc1(as, ir->op1, RSET_GPR);
782 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
783 emit_dst(as, MIPSI_AADDU, dest, RID_TMP, base);
784 emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3);
785 }
786
787 /* Inlined hash lookup. Specialized for key type and for const keys.
788 ** The equivalent C code is:
789 ** Node *n = hashkey(t, key);
790 ** do {
791 ** if (lj_obj_equal(&n->key, key)) return &n->val;
792 ** } while ((n = nextnode(n)));
793 ** return niltv(L);
794 */
asm_href(ASMState * as,IRIns * ir,IROp merge)795 static void asm_href(ASMState *as, IRIns *ir, IROp merge)
796 {
797 RegSet allow = RSET_GPR;
798 int destused = ra_used(ir);
799 Reg dest = ra_dest(as, ir, allow);
800 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
801 Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2;
802 IRRef refkey = ir->op2;
803 IRIns *irkey = IR(refkey);
804 int isk = irref_isk(refkey);
805 IRType1 kt = irkey->t;
806 uint32_t khash;
807 MCLabel l_end, l_loop, l_next;
808
809 rset_clear(allow, tab);
810 #if LJ_32 && LJ_SOFTFP
811 if (!isk) {
812 key = ra_alloc1(as, refkey, allow);
813 rset_clear(allow, key);
814 if (irkey[1].o == IR_HIOP) {
815 if (ra_hasreg((irkey+1)->r)) {
816 type = tmpnum = (irkey+1)->r;
817 tmp1 = ra_scratch(as, allow);
818 rset_clear(allow, tmp1);
819 ra_noweak(as, tmpnum);
820 } else {
821 type = tmpnum = ra_allocref(as, refkey+1, allow);
822 }
823 rset_clear(allow, tmpnum);
824 } else {
825 type = ra_allock(as, (int32_t)irt_toitype(irkey->t), allow);
826 rset_clear(allow, type);
827 }
828 }
829 #else
830 if (irt_isnum(kt)) {
831 key = ra_alloc1(as, refkey, RSET_FPR);
832 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
833 } else if (!irt_ispri(kt)) {
834 key = ra_alloc1(as, refkey, allow);
835 rset_clear(allow, key);
836 #if LJ_32
837 type = ra_allock(as, (int32_t)irt_toitype(irkey->t), allow);
838 rset_clear(allow, type);
839 #endif
840 }
841 #endif
842 tmp2 = ra_scratch(as, allow);
843 rset_clear(allow, tmp2);
844
845 /* Key not found in chain: jump to exit (if merged) or load niltv. */
846 l_end = emit_label(as);
847 as->invmcp = NULL;
848 if (merge == IR_NE)
849 asm_guard(as, MIPSI_B, RID_ZERO, RID_ZERO);
850 else if (destused)
851 emit_loada(as, dest, niltvg(J2G(as->J)));
852 /* Follow hash chain until the end. */
853 emit_move(as, dest, tmp1);
854 l_loop = --as->mcp;
855 emit_tsi(as, MIPSI_AL, tmp1, dest, (int32_t)offsetof(Node, next));
856 l_next = emit_label(as);
857
858 /* Type and value comparison. */
859 if (merge == IR_EQ) { /* Must match asm_guard(). */
860 emit_ti(as, MIPSI_LI, RID_TMP, as->snapno);
861 l_end = asm_exitstub_addr(as);
862 }
863 if (!LJ_SOFTFP && irt_isnum(kt)) {
864 emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
865 emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
866 *--as->mcp = MIPSI_NOP; /* Avoid NaN comparison overhead. */
867 emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next);
868 emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM);
869 #if LJ_32
870 emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
871 } else {
872 if (irt_ispri(kt)) {
873 emit_branch(as, MIPSI_BEQ, tmp1, type, l_end);
874 } else {
875 emit_branch(as, MIPSI_BEQ, tmp2, key, l_end);
876 emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
877 emit_branch(as, MIPSI_BNE, tmp1, type, l_next);
878 }
879 }
880 emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it));
881 *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
882 #else
883 emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 15);
884 emit_tg(as, MIPSI_DMTC1, tmp1, tmpnum);
885 emit_tsi(as, MIPSI_LD, tmp1, dest, (int32_t)offsetof(Node, key.u64));
886 } else if (irt_isaddr(kt)) {
887 Reg refk = tmp2;
888 if (isk) {
889 int64_t k = ((int64_t)irt_toitype(irkey->t) << 47) | irkey[1].tv.u64;
890 refk = ra_allock(as, k, allow);
891 rset_clear(allow, refk);
892 }
893 emit_branch(as, MIPSI_BEQ, tmp1, refk, l_end);
894 emit_tsi(as, MIPSI_LD, tmp1, dest, offsetof(Node, key));
895 } else {
896 Reg pri = ra_allock(as, ~((int64_t)~irt_toitype(ir->t) << 47), allow);
897 rset_clear(allow, pri);
898 lua_assert(irt_ispri(kt) && !irt_isnil(kt));
899 emit_branch(as, MIPSI_BEQ, tmp1, pri, l_end);
900 emit_tsi(as, MIPSI_LD, tmp1, dest, offsetof(Node, key));
901 }
902 *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
903 if (!isk && irt_isaddr(kt)) {
904 type = ra_allock(as, (int64_t)irt_toitype(kt) << 47, allow);
905 emit_dst(as, MIPSI_DADDU, tmp2, key, type);
906 rset_clear(allow, type);
907 }
908 #endif
909
910 /* Load main position relative to tab->node into dest. */
911 khash = isk ? ir_khash(irkey) : 1;
912 if (khash == 0) {
913 emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
914 } else {
915 Reg tmphash = tmp1;
916 if (isk)
917 tmphash = ra_allock(as, khash, allow);
918 emit_dst(as, MIPSI_AADDU, dest, dest, tmp1);
919 lua_assert(sizeof(Node) == 24);
920 emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1);
921 emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3);
922 emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5);
923 emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash);
924 emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
925 emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
926 if (isk) {
927 /* Nothing to do. */
928 } else if (irt_isstr(kt)) {
929 emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, hash));
930 } else { /* Must match with hash*() in lj_tab.c. */
931 emit_dst(as, MIPSI_SUBU, tmp1, tmp1, tmp2);
932 emit_rotr(as, tmp2, tmp2, dest, (-HASH_ROT3)&31);
933 emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2);
934 emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31);
935 emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest);
936 #if LJ_32
937 if (LJ_SOFTFP ? (irkey[1].o == IR_HIOP) : irt_isnum(kt)) {
938 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
939 if ((as->flags & JIT_F_MIPSXXR2)) {
940 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
941 } else {
942 emit_dst(as, MIPSI_OR, dest, dest, tmp1);
943 emit_dta(as, MIPSI_SLL, tmp1, tmp1, HASH_ROT1);
944 emit_dta(as, MIPSI_SRL, dest, tmp1, (-HASH_ROT1)&31);
945 }
946 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
947 #if LJ_SOFTFP
948 emit_ds(as, MIPSI_MOVE, tmp1, type);
949 emit_ds(as, MIPSI_MOVE, tmp2, key);
950 #else
951 emit_tg(as, MIPSI_MFC1, tmp2, key);
952 emit_tg(as, MIPSI_MFC1, tmp1, key+1);
953 #endif
954 } else {
955 emit_dst(as, MIPSI_XOR, tmp2, key, tmp1);
956 emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31);
957 emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow));
958 }
959 #else
960 emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
961 emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
962 if (irt_isnum(kt)) {
963 emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
964 emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 0);
965 emit_dta(as, MIPSI_SLL, tmp2, LJ_SOFTFP ? key : tmp1, 0);
966 #if !LJ_SOFTFP
967 emit_tg(as, MIPSI_DMFC1, tmp1, key);
968 #endif
969 } else {
970 checkmclim(as);
971 emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 0);
972 emit_dta(as, MIPSI_SLL, tmp2, key, 0);
973 emit_dst(as, MIPSI_DADDU, tmp1, key, type);
974 }
975 #endif
976 }
977 }
978 }
979
asm_hrefk(ASMState * as,IRIns * ir)980 static void asm_hrefk(ASMState *as, IRIns *ir)
981 {
982 IRIns *kslot = IR(ir->op2);
983 IRIns *irkey = IR(kslot->op1);
984 int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
985 int32_t kofs = ofs + (int32_t)offsetof(Node, key);
986 Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
987 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
988 RegSet allow = rset_exclude(RSET_GPR, node);
989 Reg idx = node;
990 #if LJ_32
991 Reg key = RID_NONE, type = RID_TMP;
992 int32_t lo, hi;
993 #else
994 Reg key = ra_scratch(as, allow);
995 int64_t k;
996 #endif
997 lua_assert(ofs % sizeof(Node) == 0);
998 if (ofs > 32736) {
999 idx = dest;
1000 rset_clear(allow, dest);
1001 kofs = (int32_t)offsetof(Node, key);
1002 } else if (ra_hasreg(dest)) {
1003 emit_tsi(as, MIPSI_AADDIU, dest, node, ofs);
1004 }
1005 #if LJ_32
1006 if (!irt_ispri(irkey->t)) {
1007 key = ra_scratch(as, allow);
1008 rset_clear(allow, key);
1009 }
1010 if (irt_isnum(irkey->t)) {
1011 lo = (int32_t)ir_knum(irkey)->u32.lo;
1012 hi = (int32_t)ir_knum(irkey)->u32.hi;
1013 } else {
1014 lo = irkey->i;
1015 hi = irt_toitype(irkey->t);
1016 if (!ra_hasreg(key))
1017 goto nolo;
1018 }
1019 asm_guard(as, MIPSI_BNE, key, lo ? ra_allock(as, lo, allow) : RID_ZERO);
1020 nolo:
1021 asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO);
1022 if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0));
1023 emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4));
1024 #else
1025 if (irt_ispri(irkey->t)) {
1026 lua_assert(!irt_isnil(irkey->t));
1027 k = ~((int64_t)~irt_toitype(irkey->t) << 47);
1028 } else if (irt_isnum(irkey->t)) {
1029 k = (int64_t)ir_knum(irkey)->u64;
1030 } else {
1031 k = ((int64_t)irt_toitype(irkey->t) << 47) | (int64_t)ir_kgc(irkey);
1032 }
1033 asm_guard(as, MIPSI_BNE, key, ra_allock(as, k, allow));
1034 emit_tsi(as, MIPSI_LD, key, idx, kofs);
1035 #endif
1036 if (ofs > 32736)
1037 emit_tsi(as, MIPSI_AADDU, dest, node, ra_allock(as, ofs, allow));
1038 }
1039
asm_uref(ASMState * as,IRIns * ir)1040 static void asm_uref(ASMState *as, IRIns *ir)
1041 {
1042 Reg dest = ra_dest(as, ir, RSET_GPR);
1043 if (irref_isk(ir->op1)) {
1044 GCfunc *fn = ir_kfunc(IR(ir->op1));
1045 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
1046 emit_lsptr(as, MIPSI_AL, dest, v, RSET_GPR);
1047 } else {
1048 Reg uv = ra_scratch(as, RSET_GPR);
1049 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
1050 if (ir->o == IR_UREFC) {
1051 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1052 emit_tsi(as, MIPSI_AADDIU, dest, uv, (int32_t)offsetof(GCupval, tv));
1053 emit_tsi(as, MIPSI_LBU, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
1054 } else {
1055 emit_tsi(as, MIPSI_AL, dest, uv, (int32_t)offsetof(GCupval, v));
1056 }
1057 emit_tsi(as, MIPSI_AL, uv, func, (int32_t)offsetof(GCfuncL, uvptr) +
1058 (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
1059 }
1060 }
1061
asm_fref(ASMState * as,IRIns * ir)1062 static void asm_fref(ASMState *as, IRIns *ir)
1063 {
1064 UNUSED(as); UNUSED(ir);
1065 lua_assert(!ra_used(ir));
1066 }
1067
asm_strref(ASMState * as,IRIns * ir)1068 static void asm_strref(ASMState *as, IRIns *ir)
1069 {
1070 #if LJ_32
1071 Reg dest = ra_dest(as, ir, RSET_GPR);
1072 IRRef ref = ir->op2, refk = ir->op1;
1073 int32_t ofs = (int32_t)sizeof(GCstr);
1074 Reg r;
1075 if (irref_isk(ref)) {
1076 IRRef tmp = refk; refk = ref; ref = tmp;
1077 } else if (!irref_isk(refk)) {
1078 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1079 IRIns *irr = IR(ir->op2);
1080 if (ra_hasreg(irr->r)) {
1081 ra_noweak(as, irr->r);
1082 right = irr->r;
1083 } else if (mayfuse(as, irr->op2) &&
1084 irr->o == IR_ADD && irref_isk(irr->op2) &&
1085 checki16(ofs + IR(irr->op2)->i)) {
1086 ofs += IR(irr->op2)->i;
1087 right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left));
1088 } else {
1089 right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left));
1090 }
1091 emit_tsi(as, MIPSI_ADDIU, dest, dest, ofs);
1092 emit_dst(as, MIPSI_ADDU, dest, left, right);
1093 return;
1094 }
1095 r = ra_alloc1(as, ref, RSET_GPR);
1096 ofs += IR(refk)->i;
1097 if (checki16(ofs))
1098 emit_tsi(as, MIPSI_ADDIU, dest, r, ofs);
1099 else
1100 emit_dst(as, MIPSI_ADDU, dest, r,
1101 ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
1102 #else
1103 RegSet allow = RSET_GPR;
1104 Reg dest = ra_dest(as, ir, allow);
1105 Reg base = ra_alloc1(as, ir->op1, allow);
1106 IRIns *irr = IR(ir->op2);
1107 int32_t ofs = sizeof(GCstr);
1108 rset_clear(allow, base);
1109 if (irref_isk(ir->op2) && checki16(ofs + irr->i)) {
1110 emit_tsi(as, MIPSI_DADDIU, dest, base, ofs + irr->i);
1111 } else {
1112 emit_tsi(as, MIPSI_DADDIU, dest, dest, ofs);
1113 emit_dst(as, MIPSI_DADDU, dest, base, ra_alloc1(as, ir->op2, allow));
1114 }
1115 #endif
1116 }
1117
1118 /* -- Loads and stores ---------------------------------------------------- */
1119
asm_fxloadins(IRIns * ir)1120 static MIPSIns asm_fxloadins(IRIns *ir)
1121 {
1122 switch (irt_type(ir->t)) {
1123 case IRT_I8: return MIPSI_LB;
1124 case IRT_U8: return MIPSI_LBU;
1125 case IRT_I16: return MIPSI_LH;
1126 case IRT_U16: return MIPSI_LHU;
1127 case IRT_NUM: lua_assert(!LJ_SOFTFP); return MIPSI_LDC1;
1128 case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_LWC1;
1129 default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_LD : MIPSI_LW;
1130 }
1131 }
1132
asm_fxstoreins(IRIns * ir)1133 static MIPSIns asm_fxstoreins(IRIns *ir)
1134 {
1135 switch (irt_type(ir->t)) {
1136 case IRT_I8: case IRT_U8: return MIPSI_SB;
1137 case IRT_I16: case IRT_U16: return MIPSI_SH;
1138 case IRT_NUM: lua_assert(!LJ_SOFTFP); return MIPSI_SDC1;
1139 case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_SWC1;
1140 default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_SD : MIPSI_SW;
1141 }
1142 }
1143
asm_fload(ASMState * as,IRIns * ir)1144 static void asm_fload(ASMState *as, IRIns *ir)
1145 {
1146 Reg dest = ra_dest(as, ir, RSET_GPR);
1147 MIPSIns mi = asm_fxloadins(ir);
1148 Reg idx;
1149 int32_t ofs;
1150 if (ir->op1 == REF_NIL) {
1151 idx = RID_JGL;
1152 ofs = (ir->op2 << 2) - 32768 - GG_OFS(g);
1153 } else {
1154 idx = ra_alloc1(as, ir->op1, RSET_GPR);
1155 if (ir->op2 == IRFL_TAB_ARRAY) {
1156 ofs = asm_fuseabase(as, ir->op1);
1157 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
1158 emit_tsi(as, MIPSI_AADDIU, dest, idx, ofs);
1159 return;
1160 }
1161 }
1162 ofs = field_ofs[ir->op2];
1163 }
1164 lua_assert(!irt_isfp(ir->t));
1165 emit_tsi(as, mi, dest, idx, ofs);
1166 }
1167
asm_fstore(ASMState * as,IRIns * ir)1168 static void asm_fstore(ASMState *as, IRIns *ir)
1169 {
1170 if (ir->r != RID_SINK) {
1171 Reg src = ra_alloc1z(as, ir->op2, RSET_GPR);
1172 IRIns *irf = IR(ir->op1);
1173 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
1174 int32_t ofs = field_ofs[irf->op2];
1175 MIPSIns mi = asm_fxstoreins(ir);
1176 lua_assert(!irt_isfp(ir->t));
1177 emit_tsi(as, mi, src, idx, ofs);
1178 }
1179 }
1180
asm_xload(ASMState * as,IRIns * ir)1181 static void asm_xload(ASMState *as, IRIns *ir)
1182 {
1183 Reg dest = ra_dest(as, ir,
1184 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1185 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED));
1186 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
1187 }
1188
asm_xstore_(ASMState * as,IRIns * ir,int32_t ofs)1189 static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
1190 {
1191 if (ir->r != RID_SINK) {
1192 Reg src = ra_alloc1z(as, ir->op2,
1193 (!LJ_SOFTFP && irt_isfp(ir->t)) ? RSET_FPR : RSET_GPR);
1194 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
1195 rset_exclude(RSET_GPR, src), ofs);
1196 }
1197 }
1198
1199 #define asm_xstore(as, ir) asm_xstore_(as, ir, 0)
1200
asm_ahuvload(ASMState * as,IRIns * ir)1201 static void asm_ahuvload(ASMState *as, IRIns *ir)
1202 {
1203 int hiop = (LJ_32 && LJ_SOFTFP && (ir+1)->o == IR_HIOP);
1204 Reg dest = RID_NONE, type = RID_TMP, idx;
1205 RegSet allow = RSET_GPR;
1206 int32_t ofs = 0;
1207 IRType1 t = ir->t;
1208 if (hiop) {
1209 t.irt = IRT_NUM;
1210 if (ra_used(ir+1)) {
1211 type = ra_dest(as, ir+1, allow);
1212 rset_clear(allow, type);
1213 }
1214 }
1215 if (ra_used(ir)) {
1216 lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
1217 irt_isint(ir->t) || irt_isaddr(ir->t));
1218 dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
1219 rset_clear(allow, dest);
1220 #if LJ_64
1221 if (irt_isaddr(t))
1222 emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
1223 else if (irt_isint(t))
1224 emit_dta(as, MIPSI_SLL, dest, dest, 0);
1225 #endif
1226 }
1227 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1228 rset_clear(allow, idx);
1229 if (irt_isnum(t)) {
1230 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1231 emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
1232 } else {
1233 asm_guard(as, MIPSI_BNE, type,
1234 ra_allock(as, (int32_t)irt_toitype(t), allow));
1235 }
1236 #if LJ_32
1237 if (ra_hasreg(dest)) {
1238 if (!LJ_SOFTFP && irt_isnum(t))
1239 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
1240 else
1241 emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0));
1242 }
1243 emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4));
1244 #else
1245 if (ra_hasreg(dest)) {
1246 if (!LJ_SOFTFP && irt_isnum(t)) {
1247 emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
1248 dest = type;
1249 }
1250 } else {
1251 dest = type;
1252 }
1253 emit_dta(as, MIPSI_DSRA32, type, dest, 15);
1254 emit_tsi(as, MIPSI_LD, dest, idx, ofs);
1255 #endif
1256 }
1257
asm_ahustore(ASMState * as,IRIns * ir)1258 static void asm_ahustore(ASMState *as, IRIns *ir)
1259 {
1260 RegSet allow = RSET_GPR;
1261 Reg idx, src = RID_NONE, type = RID_NONE;
1262 int32_t ofs = 0;
1263 if (ir->r == RID_SINK)
1264 return;
1265 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1266 src = ra_alloc1(as, ir->op2, RSET_FPR);
1267 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1268 emit_hsi(as, MIPSI_SDC1, src, idx, ofs);
1269 } else {
1270 #if LJ_32
1271 if (!irt_ispri(ir->t)) {
1272 src = ra_alloc1(as, ir->op2, allow);
1273 rset_clear(allow, src);
1274 }
1275 if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
1276 type = ra_alloc1(as, (ir+1)->op2, allow);
1277 else
1278 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1279 rset_clear(allow, type);
1280 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1281 if (ra_hasreg(src))
1282 emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0));
1283 emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4));
1284 #else
1285 Reg tmp = RID_TMP;
1286 if (irt_ispri(ir->t)) {
1287 tmp = ra_allock(as, ~((int64_t)~irt_toitype(ir->t) << 47), allow);
1288 rset_clear(allow, tmp);
1289 } else {
1290 src = ra_alloc1(as, ir->op2, allow);
1291 rset_clear(allow, src);
1292 type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47, allow);
1293 rset_clear(allow, type);
1294 }
1295 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
1296 emit_tsi(as, MIPSI_SD, tmp, idx, ofs);
1297 if (ra_hasreg(src)) {
1298 if (irt_isinteger(ir->t)) {
1299 emit_dst(as, MIPSI_DADDU, tmp, tmp, type);
1300 emit_tsml(as, MIPSI_DEXT, tmp, src, 31, 0);
1301 } else {
1302 emit_dst(as, MIPSI_DADDU, tmp, src, type);
1303 }
1304 }
1305 #endif
1306 }
1307 }
1308
asm_sload(ASMState * as,IRIns * ir)1309 static void asm_sload(ASMState *as, IRIns *ir)
1310 {
1311 Reg dest = RID_NONE, type = RID_NONE, base;
1312 RegSet allow = RSET_GPR;
1313 IRType1 t = ir->t;
1314 #if LJ_32
1315 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
1316 int hiop = (LJ_32 && LJ_SOFTFP && (ir+1)->o == IR_HIOP);
1317 if (hiop)
1318 t.irt = IRT_NUM;
1319 #else
1320 int32_t ofs = 8*((int32_t)ir->op1-2);
1321 #endif
1322 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */
1323 lua_assert(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK));
1324 #if LJ_32 && LJ_SOFTFP
1325 lua_assert(!(ir->op2 & IRSLOAD_CONVERT)); /* Handled by LJ_SOFTFP SPLIT. */
1326 if (hiop && ra_used(ir+1)) {
1327 type = ra_dest(as, ir+1, allow);
1328 rset_clear(allow, type);
1329 }
1330 #else
1331 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1332 dest = ra_scratch(as, RSET_FPR);
1333 asm_tointg(as, ir, dest);
1334 t.irt = IRT_NUM; /* Continue with a regular number type check. */
1335 } else
1336 #endif
1337 if (ra_used(ir)) {
1338 lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
1339 irt_isint(ir->t) || irt_isaddr(ir->t));
1340 dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
1341 rset_clear(allow, dest);
1342 base = ra_alloc1(as, REF_BASE, allow);
1343 rset_clear(allow, base);
1344 if (!LJ_SOFTFP && (ir->op2 & IRSLOAD_CONVERT)) {
1345 if (irt_isint(t)) {
1346 Reg tmp = ra_scratch(as, RSET_FPR);
1347 emit_tg(as, MIPSI_MFC1, dest, tmp);
1348 emit_fg(as, MIPSI_TRUNC_W_D, tmp, tmp);
1349 dest = tmp;
1350 t.irt = IRT_NUM; /* Check for original type. */
1351 } else {
1352 Reg tmp = ra_scratch(as, RSET_GPR);
1353 emit_fg(as, MIPSI_CVT_D_W, dest, dest);
1354 emit_tg(as, MIPSI_MTC1, tmp, dest);
1355 dest = tmp;
1356 t.irt = IRT_INT; /* Check for original type. */
1357 }
1358 }
1359 #if LJ_64
1360 else if (irt_isaddr(t)) {
1361 /* Clear type from pointers. */
1362 emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
1363 } else if (irt_isint(t) && (ir->op2 & IRSLOAD_TYPECHECK)) {
1364 /* Sign-extend integers. */
1365 emit_dta(as, MIPSI_SLL, dest, dest, 0);
1366 }
1367 #endif
1368 goto dotypecheck;
1369 }
1370 base = ra_alloc1(as, REF_BASE, allow);
1371 rset_clear(allow, base);
1372 dotypecheck:
1373 #if LJ_32
1374 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1375 if (ra_noreg(type))
1376 type = RID_TMP;
1377 if (irt_isnum(t)) {
1378 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1379 emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
1380 } else {
1381 Reg ktype = ra_allock(as, irt_toitype(t), allow);
1382 asm_guard(as, MIPSI_BNE, type, ktype);
1383 }
1384 }
1385 if (ra_hasreg(dest)) {
1386 if (!LJ_SOFTFP && irt_isnum(t))
1387 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1388 else
1389 emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0));
1390 }
1391 if (ra_hasreg(type))
1392 emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4));
1393 #else
1394 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1395 type = dest < RID_MAX_GPR ? dest : RID_TMP;
1396 if (irt_ispri(t)) {
1397 asm_guard(as, MIPSI_BNE, type,
1398 ra_allock(as, ~((int64_t)~irt_toitype(t) << 47) , allow));
1399 } else {
1400 if (irt_isnum(t)) {
1401 asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
1402 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)LJ_TISNUM);
1403 if (ra_hasreg(dest))
1404 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1405 } else {
1406 asm_guard(as, MIPSI_BNE, RID_TMP,
1407 ra_allock(as, (int32_t)irt_toitype(t), allow));
1408 }
1409 emit_dta(as, MIPSI_DSRA32, RID_TMP, type, 15);
1410 }
1411 emit_tsi(as, MIPSI_LD, type, base, ofs);
1412 } else if (ra_hasreg(dest)) {
1413 if (irt_isnum(t))
1414 emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
1415 else
1416 emit_tsi(as, irt_isint(t) ? MIPSI_LW : MIPSI_LD, dest, base,
1417 ofs ^ ((LJ_BE && irt_isint(t)) ? 4 : 0));
1418 }
1419 #endif
1420 }
1421
1422 /* -- Allocations --------------------------------------------------------- */
1423
1424 #if LJ_HASFFI
asm_cnew(ASMState * as,IRIns * ir)1425 static void asm_cnew(ASMState *as, IRIns *ir)
1426 {
1427 CTState *cts = ctype_ctsG(J2G(as->J));
1428 CTypeID id = (CTypeID)IR(ir->op1)->i;
1429 CTSize sz;
1430 CTInfo info = lj_ctype_info(cts, id, &sz);
1431 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1432 IRRef args[4];
1433 RegSet drop = RSET_SCRATCH;
1434 lua_assert(sz != CTSIZE_INVALID || (ir->o == IR_CNEW && ir->op2 != REF_NIL));
1435
1436 as->gcsteps++;
1437 if (ra_hasreg(ir->r))
1438 rset_clear(drop, ir->r); /* Dest reg handled below. */
1439 ra_evictset(as, drop);
1440 if (ra_used(ir))
1441 ra_destreg(as, ir, RID_RET); /* GCcdata * */
1442
1443 /* Initialize immutable cdata object. */
1444 if (ir->o == IR_CNEWI) {
1445 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1446 #if LJ_32
1447 int32_t ofs = sizeof(GCcdata);
1448 if (sz == 8) {
1449 ofs += 4;
1450 lua_assert((ir+1)->o == IR_HIOP);
1451 if (LJ_LE) ir++;
1452 }
1453 for (;;) {
1454 Reg r = ra_alloc1z(as, ir->op2, allow);
1455 emit_tsi(as, MIPSI_SW, r, RID_RET, ofs);
1456 rset_clear(allow, r);
1457 if (ofs == sizeof(GCcdata)) break;
1458 ofs -= 4; if (LJ_BE) ir++; else ir--;
1459 }
1460 #else
1461 emit_tsi(as, MIPSI_SD, ra_alloc1(as, ir->op2, allow),
1462 RID_RET, sizeof(GCcdata));
1463 #endif
1464 lua_assert(sz == 4 || sz == 8);
1465 } else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
1466 ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
1467 args[0] = ASMREF_L; /* lua_State *L */
1468 args[1] = ir->op1; /* CTypeID id */
1469 args[2] = ir->op2; /* CTSize sz */
1470 args[3] = ASMREF_TMP1; /* CTSize align */
1471 asm_gencall(as, ci, args);
1472 emit_loadi(as, ra_releasetmp(as, ASMREF_TMP1), (int32_t)ctype_align(info));
1473 return;
1474 }
1475
1476 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1477 emit_tsi(as, MIPSI_SB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1478 emit_tsi(as, MIPSI_SH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1479 emit_ti(as, MIPSI_LI, RID_RET+1, ~LJ_TCDATA);
1480 emit_ti(as, MIPSI_LI, RID_TMP, id); /* Lower 16 bit used. Sign-ext ok. */
1481 args[0] = ASMREF_L; /* lua_State *L */
1482 args[1] = ASMREF_TMP1; /* MSize size */
1483 asm_gencall(as, ci, args);
1484 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1485 ra_releasetmp(as, ASMREF_TMP1));
1486 }
1487 #else
1488 #define asm_cnew(as, ir) ((void)0)
1489 #endif
1490
1491 /* -- Write barriers ------------------------------------------------------ */
1492
asm_tbar(ASMState * as,IRIns * ir)1493 static void asm_tbar(ASMState *as, IRIns *ir)
1494 {
1495 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1496 Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1497 Reg link = RID_TMP;
1498 MCLabel l_end = emit_label(as);
1499 emit_tsi(as, MIPSI_AS, link, tab, (int32_t)offsetof(GCtab, gclist));
1500 emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked));
1501 emit_setgl(as, tab, gc.grayagain);
1502 emit_getgl(as, link, gc.grayagain);
1503 emit_dst(as, MIPSI_XOR, mark, mark, RID_TMP); /* Clear black bit. */
1504 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1505 emit_tsi(as, MIPSI_ANDI, RID_TMP, mark, LJ_GC_BLACK);
1506 emit_tsi(as, MIPSI_LBU, mark, tab, (int32_t)offsetof(GCtab, marked));
1507 }
1508
asm_obar(ASMState * as,IRIns * ir)1509 static void asm_obar(ASMState *as, IRIns *ir)
1510 {
1511 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1512 IRRef args[2];
1513 MCLabel l_end;
1514 Reg obj, val, tmp;
1515 /* No need for other object barriers (yet). */
1516 lua_assert(IR(ir->op1)->o == IR_UREFC);
1517 ra_evictset(as, RSET_SCRATCH);
1518 l_end = emit_label(as);
1519 args[0] = ASMREF_TMP1; /* global_State *g */
1520 args[1] = ir->op1; /* TValue *tv */
1521 asm_gencall(as, ci, args);
1522 emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1523 obj = IR(ir->op1)->r;
1524 tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1525 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1526 emit_tsi(as, MIPSI_ANDI, tmp, tmp, LJ_GC_BLACK);
1527 emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
1528 emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, LJ_GC_WHITES);
1529 val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1530 emit_tsi(as, MIPSI_LBU, tmp, obj,
1531 (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1532 emit_tsi(as, MIPSI_LBU, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1533 }
1534
1535 /* -- Arithmetic and logic operations ------------------------------------- */
1536
1537 #if !LJ_SOFTFP
asm_fparith(ASMState * as,IRIns * ir,MIPSIns mi)1538 static void asm_fparith(ASMState *as, IRIns *ir, MIPSIns mi)
1539 {
1540 Reg dest = ra_dest(as, ir, RSET_FPR);
1541 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1542 right = (left >> 8); left &= 255;
1543 emit_fgh(as, mi, dest, left, right);
1544 }
1545
asm_fpunary(ASMState * as,IRIns * ir,MIPSIns mi)1546 static void asm_fpunary(ASMState *as, IRIns *ir, MIPSIns mi)
1547 {
1548 Reg dest = ra_dest(as, ir, RSET_FPR);
1549 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1550 emit_fg(as, mi, dest, left);
1551 }
1552
asm_fpmath(ASMState * as,IRIns * ir)1553 static void asm_fpmath(ASMState *as, IRIns *ir)
1554 {
1555 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
1556 return;
1557 if (ir->op2 <= IRFPM_TRUNC)
1558 asm_callround(as, ir, IRCALL_lj_vm_floor + ir->op2);
1559 else if (ir->op2 == IRFPM_SQRT)
1560 asm_fpunary(as, ir, MIPSI_SQRT_D);
1561 else
1562 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
1563 }
1564 #endif
1565
asm_add(ASMState * as,IRIns * ir)1566 static void asm_add(ASMState *as, IRIns *ir)
1567 {
1568 IRType1 t = ir->t;
1569 #if !LJ_SOFTFP
1570 if (irt_isnum(t)) {
1571 asm_fparith(as, ir, MIPSI_ADD_D);
1572 } else
1573 #endif
1574 {
1575 Reg dest = ra_dest(as, ir, RSET_GPR);
1576 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1577 if (irref_isk(ir->op2)) {
1578 intptr_t k = get_kval(IR(ir->op2));
1579 if (checki16(k)) {
1580 emit_tsi(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDIU : MIPSI_ADDIU, dest,
1581 left, k);
1582 return;
1583 }
1584 }
1585 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1586 emit_dst(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDU : MIPSI_ADDU, dest,
1587 left, right);
1588 }
1589 }
1590
asm_sub(ASMState * as,IRIns * ir)1591 static void asm_sub(ASMState *as, IRIns *ir)
1592 {
1593 #if !LJ_SOFTFP
1594 if (irt_isnum(ir->t)) {
1595 asm_fparith(as, ir, MIPSI_SUB_D);
1596 } else
1597 #endif
1598 {
1599 Reg dest = ra_dest(as, ir, RSET_GPR);
1600 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1601 right = (left >> 8); left &= 255;
1602 emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
1603 left, right);
1604 }
1605 }
1606
asm_mul(ASMState * as,IRIns * ir)1607 static void asm_mul(ASMState *as, IRIns *ir)
1608 {
1609 #if !LJ_SOFTFP
1610 if (irt_isnum(ir->t)) {
1611 asm_fparith(as, ir, MIPSI_MUL_D);
1612 } else
1613 #endif
1614 {
1615 Reg dest = ra_dest(as, ir, RSET_GPR);
1616 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1617 right = (left >> 8); left &= 255;
1618 if (LJ_64 && irt_is64(ir->t)) {
1619 emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1620 emit_dst(as, MIPSI_DMULT, 0, left, right);
1621 } else {
1622 emit_dst(as, MIPSI_MUL, dest, left, right);
1623 }
1624 }
1625 }
1626
asm_mod(ASMState * as,IRIns * ir)1627 static void asm_mod(ASMState *as, IRIns *ir)
1628 {
1629 #if LJ_64 && LJ_HASFFI
1630 if (!irt_isint(ir->t))
1631 asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
1632 IRCALL_lj_carith_modu64);
1633 else
1634 #endif
1635 asm_callid(as, ir, IRCALL_lj_vm_modi);
1636 }
1637
1638 #if !LJ_SOFTFP
asm_pow(ASMState * as,IRIns * ir)1639 static void asm_pow(ASMState *as, IRIns *ir)
1640 {
1641 #if LJ_64 && LJ_HASFFI
1642 if (!irt_isnum(ir->t))
1643 asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
1644 IRCALL_lj_carith_powu64);
1645 else
1646 #endif
1647 asm_callid(as, ir, IRCALL_lj_vm_powi);
1648 }
1649
asm_div(ASMState * as,IRIns * ir)1650 static void asm_div(ASMState *as, IRIns *ir)
1651 {
1652 #if LJ_64 && LJ_HASFFI
1653 if (!irt_isnum(ir->t))
1654 asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
1655 IRCALL_lj_carith_divu64);
1656 else
1657 #endif
1658 asm_fparith(as, ir, MIPSI_DIV_D);
1659 }
1660 #endif
1661
asm_neg(ASMState * as,IRIns * ir)1662 static void asm_neg(ASMState *as, IRIns *ir)
1663 {
1664 #if !LJ_SOFTFP
1665 if (irt_isnum(ir->t)) {
1666 asm_fpunary(as, ir, MIPSI_NEG_D);
1667 } else
1668 #endif
1669 {
1670 Reg dest = ra_dest(as, ir, RSET_GPR);
1671 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1672 emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
1673 RID_ZERO, left);
1674 }
1675 }
1676
1677 #define asm_abs(as, ir) asm_fpunary(as, ir, MIPSI_ABS_D)
1678 #define asm_atan2(as, ir) asm_callid(as, ir, IRCALL_atan2)
1679 #define asm_ldexp(as, ir) asm_callid(as, ir, IRCALL_ldexp)
1680
asm_arithov(ASMState * as,IRIns * ir)1681 static void asm_arithov(ASMState *as, IRIns *ir)
1682 {
1683 Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
1684 lua_assert(!irt_is64(ir->t));
1685 if (irref_isk(ir->op2)) {
1686 int k = IR(ir->op2)->i;
1687 if (ir->o == IR_SUBOV) k = -k;
1688 if (checki16(k)) { /* (dest < left) == (k >= 0 ? 1 : 0) */
1689 left = ra_alloc1(as, ir->op1, RSET_GPR);
1690 asm_guard(as, k >= 0 ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
1691 emit_dst(as, MIPSI_SLT, RID_TMP, dest, dest == left ? RID_TMP : left);
1692 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1693 if (dest == left) emit_move(as, RID_TMP, left);
1694 return;
1695 }
1696 }
1697 left = ra_alloc2(as, ir, RSET_GPR);
1698 right = (left >> 8); left &= 255;
1699 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1700 right), dest));
1701 asm_guard(as, MIPSI_BLTZ, RID_TMP, 0);
1702 emit_dst(as, MIPSI_AND, RID_TMP, RID_TMP, tmp);
1703 if (ir->o == IR_ADDOV) { /* ((dest^left) & (dest^right)) < 0 */
1704 emit_dst(as, MIPSI_XOR, RID_TMP, dest, dest == right ? RID_TMP : right);
1705 } else { /* ((dest^left) & (dest^~right)) < 0 */
1706 emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, dest);
1707 emit_dst(as, MIPSI_NOR, RID_TMP, dest == right ? RID_TMP : right, RID_ZERO);
1708 }
1709 emit_dst(as, MIPSI_XOR, tmp, dest, dest == left ? RID_TMP : left);
1710 emit_dst(as, ir->o == IR_ADDOV ? MIPSI_ADDU : MIPSI_SUBU, dest, left, right);
1711 if (dest == left || dest == right)
1712 emit_move(as, RID_TMP, dest == left ? left : right);
1713 }
1714
1715 #define asm_addov(as, ir) asm_arithov(as, ir)
1716 #define asm_subov(as, ir) asm_arithov(as, ir)
1717
asm_mulov(ASMState * as,IRIns * ir)1718 static void asm_mulov(ASMState *as, IRIns *ir)
1719 {
1720 Reg dest = ra_dest(as, ir, RSET_GPR);
1721 Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
1722 right = (left >> 8); left &= 255;
1723 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR, left),
1724 right), dest));
1725 asm_guard(as, MIPSI_BNE, RID_TMP, tmp);
1726 emit_dta(as, MIPSI_SRA, RID_TMP, dest, 31);
1727 emit_dst(as, MIPSI_MFHI, tmp, 0, 0);
1728 emit_dst(as, MIPSI_MFLO, dest, 0, 0);
1729 emit_dst(as, MIPSI_MULT, 0, left, right);
1730 }
1731
1732 #if LJ_32 && LJ_HASFFI
asm_add64(ASMState * as,IRIns * ir)1733 static void asm_add64(ASMState *as, IRIns *ir)
1734 {
1735 Reg dest = ra_dest(as, ir, RSET_GPR);
1736 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1737 if (irref_isk(ir->op2)) {
1738 int32_t k = IR(ir->op2)->i;
1739 if (k == 0) {
1740 emit_dst(as, MIPSI_ADDU, dest, left, RID_TMP);
1741 goto loarith;
1742 } else if (checki16(k)) {
1743 emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1744 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1745 goto loarith;
1746 }
1747 }
1748 emit_dst(as, MIPSI_ADDU, dest, dest, RID_TMP);
1749 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1750 emit_dst(as, MIPSI_ADDU, dest, left, right);
1751 loarith:
1752 ir--;
1753 dest = ra_dest(as, ir, RSET_GPR);
1754 left = ra_alloc1(as, ir->op1, RSET_GPR);
1755 if (irref_isk(ir->op2)) {
1756 int32_t k = IR(ir->op2)->i;
1757 if (k == 0) {
1758 if (dest != left)
1759 emit_move(as, dest, left);
1760 return;
1761 } else if (checki16(k)) {
1762 if (dest == left) {
1763 Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, left));
1764 emit_move(as, dest, tmp);
1765 dest = tmp;
1766 }
1767 emit_dst(as, MIPSI_SLTU, RID_TMP, dest, left);
1768 emit_tsi(as, MIPSI_ADDIU, dest, left, k);
1769 return;
1770 }
1771 }
1772 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1773 if (dest == left && dest == right) {
1774 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1775 emit_move(as, dest, tmp);
1776 dest = tmp;
1777 }
1778 emit_dst(as, MIPSI_SLTU, RID_TMP, dest, dest == left ? right : left);
1779 emit_dst(as, MIPSI_ADDU, dest, left, right);
1780 }
1781
asm_sub64(ASMState * as,IRIns * ir)1782 static void asm_sub64(ASMState *as, IRIns *ir)
1783 {
1784 Reg dest = ra_dest(as, ir, RSET_GPR);
1785 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1786 right = (left >> 8); left &= 255;
1787 emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1788 emit_dst(as, MIPSI_SUBU, dest, left, right);
1789 ir--;
1790 dest = ra_dest(as, ir, RSET_GPR);
1791 left = ra_alloc2(as, ir, RSET_GPR);
1792 right = (left >> 8); left &= 255;
1793 if (dest == left) {
1794 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
1795 emit_move(as, dest, tmp);
1796 dest = tmp;
1797 }
1798 emit_dst(as, MIPSI_SLTU, RID_TMP, left, dest);
1799 emit_dst(as, MIPSI_SUBU, dest, left, right);
1800 }
1801
asm_neg64(ASMState * as,IRIns * ir)1802 static void asm_neg64(ASMState *as, IRIns *ir)
1803 {
1804 Reg dest = ra_dest(as, ir, RSET_GPR);
1805 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1806 emit_dst(as, MIPSI_SUBU, dest, dest, RID_TMP);
1807 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1808 ir--;
1809 dest = ra_dest(as, ir, RSET_GPR);
1810 left = ra_alloc1(as, ir->op1, RSET_GPR);
1811 emit_dst(as, MIPSI_SLTU, RID_TMP, RID_ZERO, dest);
1812 emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
1813 }
1814 #endif
1815
asm_bnot(ASMState * as,IRIns * ir)1816 static void asm_bnot(ASMState *as, IRIns *ir)
1817 {
1818 Reg left, right, dest = ra_dest(as, ir, RSET_GPR);
1819 IRIns *irl = IR(ir->op1);
1820 if (mayfuse(as, ir->op1) && irl->o == IR_BOR) {
1821 left = ra_alloc2(as, irl, RSET_GPR);
1822 right = (left >> 8); left &= 255;
1823 } else {
1824 left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1825 right = RID_ZERO;
1826 }
1827 emit_dst(as, MIPSI_NOR, dest, left, right);
1828 }
1829
asm_bswap(ASMState * as,IRIns * ir)1830 static void asm_bswap(ASMState *as, IRIns *ir)
1831 {
1832 Reg dest = ra_dest(as, ir, RSET_GPR);
1833 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1834 #if LJ_32
1835 if ((as->flags & JIT_F_MIPSXXR2)) {
1836 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
1837 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
1838 } else {
1839 Reg tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), dest));
1840 emit_dst(as, MIPSI_OR, dest, dest, tmp);
1841 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
1842 emit_tsi(as, MIPSI_ANDI, dest, dest, 0xff00);
1843 emit_dta(as, MIPSI_SLL, RID_TMP, RID_TMP, 8);
1844 emit_dta(as, MIPSI_SRL, dest, left, 8);
1845 emit_tsi(as, MIPSI_ANDI, RID_TMP, left, 0xff00);
1846 emit_dst(as, MIPSI_OR, tmp, tmp, RID_TMP);
1847 emit_dta(as, MIPSI_SRL, tmp, left, 24);
1848 emit_dta(as, MIPSI_SLL, RID_TMP, left, 24);
1849 }
1850 #else
1851 if (irt_is64(ir->t)) {
1852 emit_dst(as, MIPSI_DSHD, dest, 0, RID_TMP);
1853 emit_dst(as, MIPSI_DSBH, RID_TMP, 0, left);
1854 } else {
1855 emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
1856 emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
1857 }
1858 #endif
1859 }
1860
asm_bitop(ASMState * as,IRIns * ir,MIPSIns mi,MIPSIns mik)1861 static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1862 {
1863 Reg dest = ra_dest(as, ir, RSET_GPR);
1864 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1865 if (irref_isk(ir->op2)) {
1866 intptr_t k = get_kval(IR(ir->op2));
1867 if (checku16(k)) {
1868 emit_tsi(as, mik, dest, left, k);
1869 return;
1870 }
1871 }
1872 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1873 emit_dst(as, mi, dest, left, right);
1874 }
1875
1876 #define asm_band(as, ir) asm_bitop(as, ir, MIPSI_AND, MIPSI_ANDI)
1877 #define asm_bor(as, ir) asm_bitop(as, ir, MIPSI_OR, MIPSI_ORI)
1878 #define asm_bxor(as, ir) asm_bitop(as, ir, MIPSI_XOR, MIPSI_XORI)
1879
asm_bitshift(ASMState * as,IRIns * ir,MIPSIns mi,MIPSIns mik)1880 static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
1881 {
1882 Reg dest = ra_dest(as, ir, RSET_GPR);
1883 if (irref_isk(ir->op2)) { /* Constant shifts. */
1884 uint32_t shift = (uint32_t)IR(ir->op2)->i;
1885 if (LJ_64 && irt_is64(ir->t)) mik |= (shift & 32) ? MIPSI_D32 : MIPSI_D;
1886 emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR),
1887 (shift & 31));
1888 } else {
1889 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1890 right = (left >> 8); left &= 255;
1891 if (LJ_64 && irt_is64(ir->t)) mi |= MIPSI_DV;
1892 emit_dst(as, mi, dest, right, left); /* Shift amount is in rs. */
1893 }
1894 }
1895
1896 #define asm_bshl(as, ir) asm_bitshift(as, ir, MIPSI_SLLV, MIPSI_SLL)
1897 #define asm_bshr(as, ir) asm_bitshift(as, ir, MIPSI_SRLV, MIPSI_SRL)
1898 #define asm_bsar(as, ir) asm_bitshift(as, ir, MIPSI_SRAV, MIPSI_SRA)
1899 #define asm_brol(as, ir) lua_assert(0)
1900
asm_bror(ASMState * as,IRIns * ir)1901 static void asm_bror(ASMState *as, IRIns *ir)
1902 {
1903 if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
1904 asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
1905 } else {
1906 Reg dest = ra_dest(as, ir, RSET_GPR);
1907 if (irref_isk(ir->op2)) { /* Constant shifts. */
1908 uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
1909 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1910 emit_rotr(as, dest, left, RID_TMP, shift);
1911 } else {
1912 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1913 right = (left >> 8); left &= 255;
1914 emit_dst(as, MIPSI_OR, dest, dest, RID_TMP);
1915 emit_dst(as, MIPSI_SRLV, dest, right, left);
1916 emit_dst(as, MIPSI_SLLV, RID_TMP, RID_TMP, left);
1917 emit_dst(as, MIPSI_SUBU, RID_TMP, ra_allock(as, 32, RSET_GPR), right);
1918 }
1919 }
1920 }
1921
1922 #if LJ_32 && LJ_SOFTFP
asm_sfpmin_max(ASMState * as,IRIns * ir)1923 static void asm_sfpmin_max(ASMState *as, IRIns *ir)
1924 {
1925 CCallInfo ci = lj_ir_callinfo[(IROp)ir->o == IR_MIN ? IRCALL_lj_vm_sfmin : IRCALL_lj_vm_sfmax];
1926 IRRef args[4];
1927 args[0^LJ_BE] = ir->op1;
1928 args[1^LJ_BE] = (ir+1)->op1;
1929 args[2^LJ_BE] = ir->op2;
1930 args[3^LJ_BE] = (ir+1)->op2;
1931 asm_setupresult(as, ir, &ci);
1932 emit_call(as, (void *)ci.func, 0);
1933 ci.func = NULL;
1934 asm_gencall(as, &ci, args);
1935 }
1936 #endif
1937
asm_min_max(ASMState * as,IRIns * ir,int ismax)1938 static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1939 {
1940 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
1941 Reg dest = ra_dest(as, ir, RSET_FPR);
1942 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1943 right = (left >> 8); left &= 255;
1944 if (dest == left) {
1945 emit_fg(as, MIPSI_MOVT_D, dest, right);
1946 } else {
1947 emit_fg(as, MIPSI_MOVF_D, dest, left);
1948 if (dest != right) emit_fg(as, MIPSI_MOV_D, dest, right);
1949 }
1950 emit_fgh(as, MIPSI_C_OLT_D, 0, ismax ? left : right, ismax ? right : left);
1951 } else {
1952 Reg dest = ra_dest(as, ir, RSET_GPR);
1953 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1954 right = (left >> 8); left &= 255;
1955 if (dest == left) {
1956 emit_dst(as, MIPSI_MOVN, dest, right, RID_TMP);
1957 } else {
1958 emit_dst(as, MIPSI_MOVZ, dest, left, RID_TMP);
1959 if (dest != right) emit_move(as, dest, right);
1960 }
1961 emit_dst(as, MIPSI_SLT, RID_TMP,
1962 ismax ? left : right, ismax ? right : left);
1963 }
1964 }
1965
1966 #define asm_min(as, ir) asm_min_max(as, ir, 0)
1967 #define asm_max(as, ir) asm_min_max(as, ir, 1)
1968
1969 /* -- Comparisons --------------------------------------------------------- */
1970
1971 #if LJ_32 && LJ_SOFTFP
1972 /* SFP comparisons. */
asm_sfpcomp(ASMState * as,IRIns * ir)1973 static void asm_sfpcomp(ASMState *as, IRIns *ir)
1974 {
1975 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_softfp_cmp];
1976 RegSet drop = RSET_SCRATCH;
1977 Reg r;
1978 IRRef args[4];
1979 args[LJ_LE ? 0 : 1] = ir->op1; args[LJ_LE ? 1 : 0] = (ir+1)->op1;
1980 args[LJ_LE ? 2 : 3] = ir->op2; args[LJ_LE ? 3 : 2] = (ir+1)->op2;
1981
1982 for (r = REGARG_FIRSTGPR; r <= REGARG_FIRSTGPR+3; r++) {
1983 if (!rset_test(as->freeset, r) &&
1984 regcost_ref(as->cost[r]) == args[r-REGARG_FIRSTGPR])
1985 rset_clear(drop, r);
1986 }
1987 ra_evictset(as, drop);
1988
1989 asm_setupresult(as, ir, ci);
1990
1991 switch ((IROp)ir->o) {
1992 case IR_LT:
1993 asm_guard(as, MIPSI_BGEZ, RID_RET, 0);
1994 break;
1995 case IR_ULT:
1996 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
1997 emit_loadi(as, RID_TMP, 1);
1998 asm_guard(as, MIPSI_BEQ, RID_RET, RID_ZERO);
1999 break;
2000 case IR_GE:
2001 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2002 emit_loadi(as, RID_TMP, 2);
2003 asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
2004 break;
2005 case IR_LE:
2006 asm_guard(as, MIPSI_BGTZ, RID_RET, 0);
2007 break;
2008 case IR_GT:
2009 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2010 emit_loadi(as, RID_TMP, 2);
2011 asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
2012 break;
2013 case IR_UGE:
2014 asm_guard(as, MIPSI_BLTZ, RID_RET, 0);
2015 break;
2016 case IR_ULE:
2017 asm_guard(as, MIPSI_BEQ, RID_RET, RID_TMP);
2018 emit_loadi(as, RID_TMP, 1);
2019 break;
2020 case IR_UGT: case IR_ABC:
2021 asm_guard(as, MIPSI_BLEZ, RID_RET, 0);
2022 break;
2023 case IR_EQ: case IR_NE:
2024 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_RET, RID_ZERO);
2025 default:
2026 break;
2027 }
2028 asm_gencall(as, ci, args);
2029 }
2030 #endif
2031
asm_comp(ASMState * as,IRIns * ir)2032 static void asm_comp(ASMState *as, IRIns *ir)
2033 {
2034 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
2035 IROp op = ir->o;
2036 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
2037 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
2038 right = (left >> 8); left &= 255;
2039 asm_guard(as, (op&1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
2040 emit_fgh(as, MIPSI_C_OLT_D + ((op&3) ^ ((op>>2)&1)), 0, left, right);
2041 } else {
2042 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
2043 if (op == IR_ABC) op = IR_UGT;
2044 if ((op&4) == 0 && irref_isk(ir->op2) && get_kval(IR(ir->op2)) == 0) {
2045 MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) :
2046 ((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ);
2047 asm_guard(as, mi, left, 0);
2048 } else {
2049 if (irref_isk(ir->op2)) {
2050 intptr_t k = get_kval(IR(ir->op2));
2051 if ((op&2)) k++;
2052 if (checki16(k)) {
2053 asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
2054 emit_tsi(as, (op&4) ? MIPSI_SLTIU : MIPSI_SLTI,
2055 RID_TMP, left, k);
2056 return;
2057 }
2058 }
2059 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
2060 asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
2061 emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT,
2062 RID_TMP, (op&2) ? right : left, (op&2) ? left : right);
2063 }
2064 }
2065 }
2066
asm_equal(ASMState * as,IRIns * ir)2067 static void asm_equal(ASMState *as, IRIns *ir)
2068 {
2069 Reg right, left = ra_alloc2(as, ir, (!LJ_SOFTFP && irt_isnum(ir->t)) ?
2070 RSET_FPR : RSET_GPR);
2071 right = (left >> 8); left &= 255;
2072 if (!LJ_SOFTFP && irt_isnum(ir->t)) {
2073 asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
2074 emit_fgh(as, MIPSI_C_EQ_D, 0, left, right);
2075 } else {
2076 asm_guard(as, (ir->o & 1) ? MIPSI_BEQ : MIPSI_BNE, left, right);
2077 }
2078 }
2079
2080 #if LJ_32 && LJ_HASFFI
2081 /* 64 bit integer comparisons. */
asm_comp64(ASMState * as,IRIns * ir)2082 static void asm_comp64(ASMState *as, IRIns *ir)
2083 {
2084 /* ORDER IR: LT GE LE GT ULT UGE ULE UGT. */
2085 IROp op = (ir-1)->o;
2086 MCLabel l_end;
2087 Reg rightlo, leftlo, righthi, lefthi = ra_alloc2(as, ir, RSET_GPR);
2088 righthi = (lefthi >> 8); lefthi &= 255;
2089 leftlo = ra_alloc2(as, ir-1,
2090 rset_exclude(rset_exclude(RSET_GPR, lefthi), righthi));
2091 rightlo = (leftlo >> 8); leftlo &= 255;
2092 asm_guard(as, ((op^(op>>1))&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
2093 l_end = emit_label(as);
2094 if (lefthi != righthi)
2095 emit_dst(as, (op&4) ? MIPSI_SLTU : MIPSI_SLT, RID_TMP,
2096 (op&2) ? righthi : lefthi, (op&2) ? lefthi : righthi);
2097 emit_dst(as, MIPSI_SLTU, RID_TMP,
2098 (op&2) ? rightlo : leftlo, (op&2) ? leftlo : rightlo);
2099 if (lefthi != righthi)
2100 emit_branch(as, MIPSI_BEQ, lefthi, righthi, l_end);
2101 }
2102
asm_comp64eq(ASMState * as,IRIns * ir)2103 static void asm_comp64eq(ASMState *as, IRIns *ir)
2104 {
2105 Reg tmp, right, left = ra_alloc2(as, ir, RSET_GPR);
2106 right = (left >> 8); left &= 255;
2107 asm_guard(as, ((ir-1)->o & 1) ? MIPSI_BEQ : MIPSI_BNE, RID_TMP, RID_ZERO);
2108 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR, left), right));
2109 emit_dst(as, MIPSI_OR, RID_TMP, RID_TMP, tmp);
2110 emit_dst(as, MIPSI_XOR, tmp, left, right);
2111 left = ra_alloc2(as, ir-1, RSET_GPR);
2112 right = (left >> 8); left &= 255;
2113 emit_dst(as, MIPSI_XOR, RID_TMP, left, right);
2114 }
2115 #endif
2116
2117 /* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
2118
2119 /* Hiword op of a split 64 bit op. Previous op must be the loword op. */
asm_hiop(ASMState * as,IRIns * ir)2120 static void asm_hiop(ASMState *as, IRIns *ir)
2121 {
2122 #if LJ_32 && (LJ_HASFFI || LJ_SOFTFP)
2123 /* HIOP is marked as a store because it needs its own DCE logic. */
2124 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
2125 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
2126 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
2127 as->curins--; /* Always skip the CONV. */
2128 #if LJ_HASFFI && !LJ_SOFTFP
2129 if (usehi || uselo)
2130 asm_conv64(as, ir);
2131 return;
2132 #endif
2133 } else if ((ir-1)->o < IR_EQ) { /* 64 bit integer comparisons. ORDER IR. */
2134 as->curins--; /* Always skip the loword comparison. */
2135 #if LJ_SOFTFP
2136 if (!irt_isint(ir->t)) {
2137 asm_sfpcomp(as, ir-1);
2138 return;
2139 }
2140 #endif
2141 #if LJ_HASFFI
2142 asm_comp64(as, ir);
2143 #endif
2144 return;
2145 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
2146 as->curins--; /* Always skip the loword comparison. */
2147 #if LJ_SOFTFP
2148 if (!irt_isint(ir->t)) {
2149 asm_sfpcomp(as, ir-1);
2150 return;
2151 }
2152 #endif
2153 #if LJ_HASFFI
2154 asm_comp64eq(as, ir);
2155 #endif
2156 return;
2157 #if LJ_SOFTFP
2158 } else if ((ir-1)->o == IR_MIN || (ir-1)->o == IR_MAX) {
2159 as->curins--; /* Always skip the loword min/max. */
2160 if (uselo || usehi)
2161 asm_sfpmin_max(as, ir-1);
2162 return;
2163 #endif
2164 } else if ((ir-1)->o == IR_XSTORE) {
2165 as->curins--; /* Handle both stores here. */
2166 if ((ir-1)->r != RID_SINK) {
2167 asm_xstore_(as, ir, LJ_LE ? 4 : 0);
2168 asm_xstore_(as, ir-1, LJ_LE ? 0 : 4);
2169 }
2170 return;
2171 }
2172 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
2173 switch ((ir-1)->o) {
2174 #if LJ_HASFFI
2175 case IR_ADD: as->curins--; asm_add64(as, ir); break;
2176 case IR_SUB: as->curins--; asm_sub64(as, ir); break;
2177 case IR_NEG: as->curins--; asm_neg64(as, ir); break;
2178 #endif
2179 #if LJ_SOFTFP
2180 case IR_SLOAD: case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2181 case IR_STRTO:
2182 if (!uselo)
2183 ra_allocref(as, ir->op1, RSET_GPR); /* Mark lo op as used. */
2184 break;
2185 #endif
2186 case IR_CALLN:
2187 case IR_CALLS:
2188 case IR_CALLXS:
2189 if (!uselo)
2190 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
2191 break;
2192 #if LJ_SOFTFP
2193 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: case IR_TOSTR:
2194 #endif
2195 case IR_CNEWI:
2196 /* Nothing to do here. Handled by lo op itself. */
2197 break;
2198 default: lua_assert(0); break;
2199 }
2200 #else
2201 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused without FFI. */
2202 #endif
2203 }
2204
2205 /* -- Profiling ----------------------------------------------------------- */
2206
asm_prof(ASMState * as,IRIns * ir)2207 static void asm_prof(ASMState *as, IRIns *ir)
2208 {
2209 UNUSED(ir);
2210 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
2211 emit_tsi(as, MIPSI_ANDI, RID_TMP, RID_TMP, HOOK_PROFILE);
2212 emit_lsglptr(as, MIPSI_LBU, RID_TMP,
2213 (int32_t)offsetof(global_State, hookmask));
2214 }
2215
2216 /* -- Stack handling ------------------------------------------------------ */
2217
2218 /* Check Lua stack size for overflow. Use exit handler as fallback. */
asm_stack_check(ASMState * as,BCReg topslot,IRIns * irp,RegSet allow,ExitNo exitno)2219 static void asm_stack_check(ASMState *as, BCReg topslot,
2220 IRIns *irp, RegSet allow, ExitNo exitno)
2221 {
2222 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
2223 Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
2224 ExitNo oldsnap = as->snapno;
2225 rset_clear(allow, pbase);
2226 #if LJ_32
2227 tmp = allow ? rset_pickbot(allow) :
2228 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
2229 #else
2230 tmp = allow ? rset_pickbot(allow) : RID_RET;
2231 #endif
2232 as->snapno = exitno;
2233 asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
2234 as->snapno = oldsnap;
2235 if (allow == RSET_EMPTY) /* Restore temp. register. */
2236 emit_tsi(as, MIPSI_AL, tmp, RID_SP, 0);
2237 else
2238 ra_modified(as, tmp);
2239 emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot));
2240 emit_dst(as, MIPSI_ASUBU, RID_TMP, tmp, pbase);
2241 emit_tsi(as, MIPSI_AL, tmp, tmp, offsetof(lua_State, maxstack));
2242 if (pbase == RID_TMP)
2243 emit_getgl(as, RID_TMP, jit_base);
2244 emit_getgl(as, tmp, cur_L);
2245 if (allow == RSET_EMPTY) /* Spill temp. register. */
2246 emit_tsi(as, MIPSI_AS, tmp, RID_SP, 0);
2247 }
2248
2249 /* Restore Lua stack from on-trace state. */
asm_stack_restore(ASMState * as,SnapShot * snap)2250 static void asm_stack_restore(ASMState *as, SnapShot *snap)
2251 {
2252 SnapEntry *map = &as->T->snapmap[snap->mapofs];
2253 #if LJ_32 || defined(LUA_USE_ASSERT)
2254 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
2255 #endif
2256 MSize n, nent = snap->nent;
2257 /* Store the value of all modified slots to the Lua stack. */
2258 for (n = 0; n < nent; n++) {
2259 SnapEntry sn = map[n];
2260 BCReg s = snap_slot(sn);
2261 int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
2262 IRRef ref = snap_ref(sn);
2263 IRIns *ir = IR(ref);
2264 if ((sn & SNAP_NORESTORE))
2265 continue;
2266 if (irt_isnum(ir->t)) {
2267 #if LJ_SOFTFP
2268 Reg tmp;
2269 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2270 lua_assert(irref_isk(ref)); /* LJ_SOFTFP: must be a number constant. */
2271 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.lo, allow);
2272 emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?4:0));
2273 if (rset_test(as->freeset, tmp+1)) allow = RID2RSET(tmp+1);
2274 tmp = ra_allock(as, (int32_t)ir_knum(ir)->u32.hi, allow);
2275 emit_tsi(as, MIPSI_SW, tmp, RID_BASE, ofs+(LJ_BE?0:4));
2276 #else
2277 Reg src = ra_alloc1(as, ref, RSET_FPR);
2278 emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs);
2279 #endif
2280 } else {
2281 #if LJ_32
2282 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
2283 Reg type;
2284 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
2285 if (!irt_ispri(ir->t)) {
2286 Reg src = ra_alloc1(as, ref, allow);
2287 rset_clear(allow, src);
2288 emit_tsi(as, MIPSI_SW, src, RID_BASE, ofs+(LJ_BE?4:0));
2289 }
2290 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
2291 if (s == 0) continue; /* Do not overwrite link to previous frame. */
2292 type = ra_allock(as, (int32_t)(*flinks--), allow);
2293 #if LJ_SOFTFP
2294 } else if ((sn & SNAP_SOFTFPNUM)) {
2295 type = ra_alloc1(as, ref+1, rset_exclude(RSET_GPR, RID_BASE));
2296 #endif
2297 } else {
2298 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
2299 }
2300 emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4));
2301 #else
2302 asm_tvstore64(as, RID_BASE, ofs, ref);
2303 #endif
2304 }
2305 checkmclim(as);
2306 }
2307 lua_assert(map + nent == flinks);
2308 }
2309
2310 /* -- GC handling --------------------------------------------------------- */
2311
2312 /* Check GC threshold and do one or more GC steps. */
asm_gc_check(ASMState * as)2313 static void asm_gc_check(ASMState *as)
2314 {
2315 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
2316 IRRef args[2];
2317 MCLabel l_end;
2318 Reg tmp;
2319 ra_evictset(as, RSET_SCRATCH);
2320 l_end = emit_label(as);
2321 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
2322 /* Assumes asm_snap_prep() already done. */
2323 asm_guard(as, MIPSI_BNE, RID_RET, RID_ZERO);
2324 args[0] = ASMREF_TMP1; /* global_State *g */
2325 args[1] = ASMREF_TMP2; /* MSize steps */
2326 asm_gencall(as, ci, args);
2327 emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
2328 tmp = ra_releasetmp(as, ASMREF_TMP2);
2329 emit_loadi(as, tmp, as->gcsteps);
2330 /* Jump around GC step if GC total < GC threshold. */
2331 emit_branch(as, MIPSI_BNE, RID_TMP, RID_ZERO, l_end);
2332 emit_dst(as, MIPSI_SLTU, RID_TMP, RID_TMP, tmp);
2333 emit_getgl(as, tmp, gc.threshold);
2334 emit_getgl(as, RID_TMP, gc.total);
2335 as->gcsteps = 0;
2336 checkmclim(as);
2337 }
2338
2339 /* -- Loop handling ------------------------------------------------------- */
2340
2341 /* Fixup the loop branch. */
asm_loop_fixup(ASMState * as)2342 static void asm_loop_fixup(ASMState *as)
2343 {
2344 MCode *p = as->mctop;
2345 MCode *target = as->mcp;
2346 p[-1] = MIPSI_NOP;
2347 if (as->loopinv) { /* Inverted loop branch? */
2348 /* asm_guard already inverted the cond branch. Only patch the target. */
2349 p[-3] |= ((target-p+2) & 0x0000ffffu);
2350 } else {
2351 p[-2] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2352 }
2353 }
2354
2355 /* -- Head of trace ------------------------------------------------------- */
2356
2357 /* Coalesce BASE register for a root trace. */
asm_head_root_base(ASMState * as)2358 static void asm_head_root_base(ASMState *as)
2359 {
2360 IRIns *ir = IR(REF_BASE);
2361 Reg r = ir->r;
2362 if (as->loopinv) as->mctop--;
2363 if (ra_hasreg(r)) {
2364 ra_free(as, r);
2365 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2366 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
2367 if (r != RID_BASE)
2368 emit_move(as, r, RID_BASE);
2369 }
2370 }
2371
2372 /* Coalesce BASE register for a side trace. */
asm_head_side_base(ASMState * as,IRIns * irp,RegSet allow)2373 static RegSet asm_head_side_base(ASMState *as, IRIns *irp, RegSet allow)
2374 {
2375 IRIns *ir = IR(REF_BASE);
2376 Reg r = ir->r;
2377 if (as->loopinv) as->mctop--;
2378 if (ra_hasreg(r)) {
2379 ra_free(as, r);
2380 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
2381 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
2382 if (irp->r == r) {
2383 rset_clear(allow, r); /* Mark same BASE register as coalesced. */
2384 } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
2385 rset_clear(allow, irp->r);
2386 emit_move(as, r, irp->r); /* Move from coalesced parent reg. */
2387 } else {
2388 emit_getgl(as, r, jit_base); /* Otherwise reload BASE. */
2389 }
2390 }
2391 return allow;
2392 }
2393
2394 /* -- Tail of trace ------------------------------------------------------- */
2395
2396 /* Fixup the tail code. */
asm_tail_fixup(ASMState * as,TraceNo lnk)2397 static void asm_tail_fixup(ASMState *as, TraceNo lnk)
2398 {
2399 MCode *target = lnk ? traceref(as->J,lnk)->mcode : (MCode *)lj_vm_exit_interp;
2400 int32_t spadj = as->T->spadjust;
2401 MCode *p = as->mctop-1;
2402 *p = spadj ? (MIPSI_AADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
2403 p[-1] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2404 }
2405
2406 /* Prepare tail of code. */
asm_tail_prep(ASMState * as)2407 static void asm_tail_prep(ASMState *as)
2408 {
2409 as->mcp = as->mctop-2; /* Leave room for branch plus nop or stack adj. */
2410 as->invmcp = as->loopref ? as->mcp : NULL;
2411 }
2412
2413 /* -- Trace setup --------------------------------------------------------- */
2414
2415 /* Ensure there are enough stack slots for call arguments. */
asm_setup_call_slots(ASMState * as,IRIns * ir,const CCallInfo * ci)2416 static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2417 {
2418 IRRef args[CCI_NARGS_MAX*2];
2419 uint32_t i, nargs = CCI_XNARGS(ci);
2420 #if LJ_32
2421 int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2422 #else
2423 int nslots = 0, ngpr = REGARG_NUMGPR;
2424 #endif
2425 asm_collectargs(as, ir, ci, args);
2426 for (i = 0; i < nargs; i++) {
2427 #if LJ_32
2428 if (!LJ_SOFTFP && args[i] && irt_isfp(IR(args[i])->t) &&
2429 nfpr > 0 && !(ci->flags & CCI_VARARG)) {
2430 nfpr--;
2431 ngpr -= irt_isnum(IR(args[i])->t) ? 2 : 1;
2432 } else if (!LJ_SOFTFP && args[i] && irt_isnum(IR(args[i])->t)) {
2433 nfpr = 0;
2434 ngpr = ngpr & ~1;
2435 if (ngpr > 0) ngpr -= 2; else nslots = (nslots+3) & ~1;
2436 } else {
2437 nfpr = 0;
2438 if (ngpr > 0) ngpr--; else nslots++;
2439 }
2440 #else
2441 if (ngpr > 0) ngpr--; else nslots += 2;
2442 #endif
2443 }
2444 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
2445 as->evenspill = nslots;
2446 return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
2447 }
2448
asm_setup_target(ASMState * as)2449 static void asm_setup_target(ASMState *as)
2450 {
2451 asm_sparejump_setup(as);
2452 asm_exitstub_setup(as);
2453 }
2454
2455 /* -- Trace patching ------------------------------------------------------ */
2456
2457 /* Patch exit jumps of existing machine code to a new target. */
lj_asm_patchexit(jit_State * J,GCtrace * T,ExitNo exitno,MCode * target)2458 void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2459 {
2460 MCode *p = T->mcode;
2461 MCode *pe = (MCode *)((char *)p + T->szmcode);
2462 MCode *px = exitstub_trace_addr(T, exitno);
2463 MCode *cstart = NULL, *cstop = NULL;
2464 MCode *mcarea = lj_mcode_patch(J, p, 0);
2465 MCode exitload = MIPSI_LI | MIPSF_T(RID_TMP) | exitno;
2466 MCode tjump = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
2467 for (p++; p < pe; p++) {
2468 if (*p == exitload) { /* Look for load of exit number. */
2469 if (((p[-1] ^ (px-p)) & 0xffffu) == 0) { /* Look for exitstub branch. */
2470 ptrdiff_t delta = target - p;
2471 if (((delta + 0x8000) >> 16) == 0) { /* Patch in-range branch. */
2472 patchbranch:
2473 p[-1] = (p[-1] & 0xffff0000u) | (delta & 0xffffu);
2474 *p = MIPSI_NOP; /* Replace the load of the exit number. */
2475 cstop = p;
2476 if (!cstart) cstart = p-1;
2477 } else { /* Branch out of range. Use spare jump slot in mcarea. */
2478 int i;
2479 for (i = 2; i < 2+MIPS_SPAREJUMP*2; i += 2) {
2480 if (mcarea[i] == tjump) {
2481 delta = mcarea+i - p;
2482 goto patchbranch;
2483 } else if (mcarea[i] == MIPSI_NOP) {
2484 mcarea[i] = tjump;
2485 cstart = mcarea+i;
2486 delta = mcarea+i - p;
2487 goto patchbranch;
2488 }
2489 }
2490 /* Ignore jump slot overflow. Child trace is simply not attached. */
2491 }
2492 } else if (p+1 == pe) {
2493 /* Patch NOP after code for inverted loop branch. Use of J is ok. */
2494 lua_assert(p[1] == MIPSI_NOP);
2495 p[1] = tjump;
2496 *p = MIPSI_NOP; /* Replace the load of the exit number. */
2497 cstop = p+2;
2498 if (!cstart) cstart = p+1;
2499 }
2500 }
2501 }
2502 if (cstart) lj_mcode_sync(cstart, cstop);
2503 lj_mcode_patch(J, mcarea, 1);
2504 }
2505
2506