1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/arm/MacroAssembler-arm.h"
8
9 #include "mozilla/Attributes.h"
10 #include "mozilla/Casting.h"
11 #include "mozilla/DebugOnly.h"
12 #include "mozilla/MathAlgorithms.h"
13
14 #include "jit/arm/Simulator-arm.h"
15 #include "jit/AtomicOp.h"
16 #include "jit/AtomicOperations.h"
17 #include "jit/Bailouts.h"
18 #include "jit/BaselineFrame.h"
19 #include "jit/JitFrames.h"
20 #include "jit/MacroAssembler.h"
21 #include "jit/MoveEmitter.h"
22
23 #include "jit/MacroAssembler-inl.h"
24
25 using namespace js;
26 using namespace jit;
27
28 using mozilla::Abs;
29 using mozilla::BitwiseCast;
30 using mozilla::IsPositiveZero;
31
isValueDTRDCandidate(ValueOperand & val)32 bool isValueDTRDCandidate(ValueOperand& val) {
33 // In order to be used for a DTRD memory function, the two target registers
34 // need to be a) Adjacent, with the tag larger than the payload, and b)
35 // Aligned to a multiple of two.
36 if ((val.typeReg().code() != (val.payloadReg().code() + 1))) return false;
37 if ((val.payloadReg().code() & 1) != 0) return false;
38 return true;
39 }
40
convertBoolToInt32(Register source,Register dest)41 void MacroAssemblerARM::convertBoolToInt32(Register source, Register dest) {
42 // Note that C++ bool is only 1 byte, so zero extend it to clear the
43 // higher-order bits.
44 as_and(dest, source, Imm8(0xff));
45 }
46
convertInt32ToDouble(Register src,FloatRegister dest_)47 void MacroAssemblerARM::convertInt32ToDouble(Register src,
48 FloatRegister dest_) {
49 // Direct conversions aren't possible.
50 VFPRegister dest = VFPRegister(dest_);
51 as_vxfer(src, InvalidReg, dest.sintOverlay(), CoreToFloat);
52 as_vcvt(dest, dest.sintOverlay());
53 }
54
convertInt32ToDouble(const Address & src,FloatRegister dest)55 void MacroAssemblerARM::convertInt32ToDouble(const Address& src,
56 FloatRegister dest) {
57 ScratchDoubleScope scratch(asMasm());
58 SecondScratchRegisterScope scratch2(asMasm());
59 ma_vldr(src, scratch, scratch2);
60 as_vcvt(dest, VFPRegister(scratch).sintOverlay());
61 }
62
convertInt32ToDouble(const BaseIndex & src,FloatRegister dest)63 void MacroAssemblerARM::convertInt32ToDouble(const BaseIndex& src,
64 FloatRegister dest) {
65 Register base = src.base;
66 uint32_t scale = Imm32::ShiftOf(src.scale).value;
67
68 ScratchRegisterScope scratch(asMasm());
69 SecondScratchRegisterScope scratch2(asMasm());
70
71 if (src.offset != 0) {
72 ma_add(base, Imm32(src.offset), scratch, scratch2);
73 base = scratch;
74 }
75 ma_ldr(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), scratch);
76 convertInt32ToDouble(scratch, dest);
77 }
78
convertUInt32ToDouble(Register src,FloatRegister dest_)79 void MacroAssemblerARM::convertUInt32ToDouble(Register src,
80 FloatRegister dest_) {
81 // Direct conversions aren't possible.
82 VFPRegister dest = VFPRegister(dest_);
83 as_vxfer(src, InvalidReg, dest.uintOverlay(), CoreToFloat);
84 as_vcvt(dest, dest.uintOverlay());
85 }
86
87 static const double TO_DOUBLE_HIGH_SCALE = 0x100000000;
88
convertUInt32ToFloat32(Register src,FloatRegister dest_)89 void MacroAssemblerARM::convertUInt32ToFloat32(Register src,
90 FloatRegister dest_) {
91 // Direct conversions aren't possible.
92 VFPRegister dest = VFPRegister(dest_);
93 as_vxfer(src, InvalidReg, dest.uintOverlay(), CoreToFloat);
94 as_vcvt(VFPRegister(dest).singleOverlay(), dest.uintOverlay());
95 }
96
convertDoubleToFloat32(FloatRegister src,FloatRegister dest,Condition c)97 void MacroAssemblerARM::convertDoubleToFloat32(FloatRegister src,
98 FloatRegister dest,
99 Condition c) {
100 as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src), false, c);
101 }
102
103 // Checks whether a double is representable as a 32-bit integer. If so, the
104 // integer is written to the output register. Otherwise, a bailout is taken to
105 // the given snapshot. This function overwrites the scratch float register.
convertDoubleToInt32(FloatRegister src,Register dest,Label * fail,bool negativeZeroCheck)106 void MacroAssemblerARM::convertDoubleToInt32(FloatRegister src, Register dest,
107 Label* fail,
108 bool negativeZeroCheck) {
109 // Convert the floating point value to an integer, if it did not fit, then
110 // when we convert it *back* to a float, it will have a different value,
111 // which we can test.
112 ScratchDoubleScope scratchDouble(asMasm());
113 ScratchRegisterScope scratch(asMasm());
114
115 FloatRegister scratchSIntReg = scratchDouble.sintOverlay();
116
117 ma_vcvt_F64_I32(src, scratchSIntReg);
118 // Move the value into the dest register.
119 ma_vxfer(scratchSIntReg, dest);
120 ma_vcvt_I32_F64(scratchSIntReg, scratchDouble);
121 ma_vcmp(src, scratchDouble);
122 as_vmrs(pc);
123 ma_b(fail, Assembler::VFP_NotEqualOrUnordered);
124
125 if (negativeZeroCheck) {
126 as_cmp(dest, Imm8(0));
127 // Test and bail for -0.0, when integer result is 0. Move the top word
128 // of the double into the output reg, if it is non-zero, then the
129 // original value was -0.0.
130 as_vxfer(dest, InvalidReg, src, FloatToCore, Assembler::Equal, 1);
131 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::Equal);
132 ma_b(fail, Assembler::Equal);
133 }
134 }
135
136 // Checks whether a float32 is representable as a 32-bit integer. If so, the
137 // integer is written to the output register. Otherwise, a bailout is taken to
138 // the given snapshot. This function overwrites the scratch float register.
convertFloat32ToInt32(FloatRegister src,Register dest,Label * fail,bool negativeZeroCheck)139 void MacroAssemblerARM::convertFloat32ToInt32(FloatRegister src, Register dest,
140 Label* fail,
141 bool negativeZeroCheck) {
142 // Converting the floating point value to an integer and then converting it
143 // back to a float32 would not work, as float to int32 conversions are
144 // clamping (e.g. float(INT32_MAX + 1) would get converted into INT32_MAX
145 // and then back to float(INT32_MAX + 1)). If this ever happens, we just
146 // bail out.
147 ScratchFloat32Scope scratchFloat(asMasm());
148 ScratchRegisterScope scratch(asMasm());
149
150 FloatRegister ScratchSIntReg = scratchFloat.sintOverlay();
151 ma_vcvt_F32_I32(src, ScratchSIntReg);
152
153 // Store the result
154 ma_vxfer(ScratchSIntReg, dest);
155
156 ma_vcvt_I32_F32(ScratchSIntReg, scratchFloat);
157 ma_vcmp(src, scratchFloat);
158 as_vmrs(pc);
159 ma_b(fail, Assembler::VFP_NotEqualOrUnordered);
160
161 // Bail out in the clamped cases.
162 ma_cmp(dest, Imm32(0x7fffffff), scratch);
163 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
164 ma_b(fail, Assembler::Equal);
165
166 if (negativeZeroCheck) {
167 as_cmp(dest, Imm8(0));
168 // Test and bail for -0.0, when integer result is 0. Move the float into
169 // the output reg, and if it is non-zero then the original value was
170 // -0.0
171 as_vxfer(dest, InvalidReg, VFPRegister(src).singleOverlay(), FloatToCore,
172 Assembler::Equal, 0);
173 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::Equal);
174 ma_b(fail, Assembler::Equal);
175 }
176 }
177
convertFloat32ToDouble(FloatRegister src,FloatRegister dest)178 void MacroAssemblerARM::convertFloat32ToDouble(FloatRegister src,
179 FloatRegister dest) {
180 MOZ_ASSERT(dest.isDouble());
181 MOZ_ASSERT(src.isSingle());
182 as_vcvt(VFPRegister(dest), VFPRegister(src).singleOverlay());
183 }
184
convertInt32ToFloat32(Register src,FloatRegister dest)185 void MacroAssemblerARM::convertInt32ToFloat32(Register src,
186 FloatRegister dest) {
187 // Direct conversions aren't possible.
188 as_vxfer(src, InvalidReg, dest.sintOverlay(), CoreToFloat);
189 as_vcvt(dest.singleOverlay(), dest.sintOverlay());
190 }
191
convertInt32ToFloat32(const Address & src,FloatRegister dest)192 void MacroAssemblerARM::convertInt32ToFloat32(const Address& src,
193 FloatRegister dest) {
194 ScratchFloat32Scope scratch(asMasm());
195 SecondScratchRegisterScope scratch2(asMasm());
196 ma_vldr(src, scratch, scratch2);
197 as_vcvt(dest, VFPRegister(scratch).sintOverlay());
198 }
199
alu_dbl(Register src1,Imm32 imm,Register dest,ALUOp op,SBit s,Condition c)200 bool MacroAssemblerARM::alu_dbl(Register src1, Imm32 imm, Register dest,
201 ALUOp op, SBit s, Condition c) {
202 if ((s == SetCC && !condsAreSafe(op)) || !can_dbl(op)) return false;
203
204 ALUOp interop = getDestVariant(op);
205 Imm8::TwoImm8mData both = Imm8::EncodeTwoImms(imm.value);
206 if (both.fst().invalid()) return false;
207
208 // For the most part, there is no good reason to set the condition codes for
209 // the first instruction. We can do better things if the second instruction
210 // doesn't have a dest, such as check for overflow by doing first operation
211 // don't do second operation if first operation overflowed. This preserves
212 // the overflow condition code. Unfortunately, it is horribly brittle.
213 as_alu(dest, src1, Operand2(both.fst()), interop, LeaveCC, c);
214 as_alu(dest, dest, Operand2(both.snd()), op, s, c);
215 return true;
216 }
217
ma_alu(Register src1,Imm32 imm,Register dest,AutoRegisterScope & scratch,ALUOp op,SBit s,Condition c)218 void MacroAssemblerARM::ma_alu(Register src1, Imm32 imm, Register dest,
219 AutoRegisterScope& scratch, ALUOp op, SBit s,
220 Condition c) {
221 // ma_mov should be used for moves.
222 MOZ_ASSERT(op != OpMov);
223 MOZ_ASSERT(op != OpMvn);
224 MOZ_ASSERT(src1 != scratch);
225
226 // As it turns out, if you ask for a compare-like instruction you *probably*
227 // want it to set condition codes.
228 MOZ_ASSERT_IF(dest == InvalidReg, s == SetCC);
229
230 // The operator gives us the ability to determine how this can be used.
231 Imm8 imm8 = Imm8(imm.value);
232 // One instruction: If we can encode it using an imm8m, then do so.
233 if (!imm8.invalid()) {
234 as_alu(dest, src1, imm8, op, s, c);
235 return;
236 }
237
238 // One instruction, negated:
239 Imm32 negImm = imm;
240 Register negDest;
241 ALUOp negOp = ALUNeg(op, dest, scratch, &negImm, &negDest);
242 Imm8 negImm8 = Imm8(negImm.value);
243 // 'add r1, r2, -15' can be replaced with 'sub r1, r2, 15'.
244 // The dest can be replaced (InvalidReg => scratch).
245 // This is useful if we wish to negate tst. tst has an invalid (aka not
246 // used) dest, but its negation bic requires a dest.
247 if (negOp != OpInvalid && !negImm8.invalid()) {
248 as_alu(negDest, src1, negImm8, negOp, s, c);
249 return;
250 }
251
252 // Start by attempting to generate a two instruction form. Some things
253 // cannot be made into two-inst forms correctly. Namely, adds dest, src,
254 // 0xffff. Since we want the condition codes (and don't know which ones
255 // will be checked), we need to assume that the overflow flag will be
256 // checked and add{,s} dest, src, 0xff00; add{,s} dest, dest, 0xff is not
257 // guaranteed to set the overflof flag the same as the (theoretical) one
258 // instruction variant.
259 if (alu_dbl(src1, imm, dest, op, s, c)) return;
260
261 // And try with its negative.
262 if (negOp != OpInvalid && alu_dbl(src1, negImm, negDest, negOp, s, c)) return;
263
264 ma_mov(imm, scratch, c);
265 as_alu(dest, src1, O2Reg(scratch), op, s, c);
266 }
267
ma_alu(Register src1,Operand op2,Register dest,ALUOp op,SBit s,Assembler::Condition c)268 void MacroAssemblerARM::ma_alu(Register src1, Operand op2, Register dest,
269 ALUOp op, SBit s, Assembler::Condition c) {
270 MOZ_ASSERT(op2.tag() == Operand::Tag::OP2);
271 as_alu(dest, src1, op2.toOp2(), op, s, c);
272 }
273
ma_alu(Register src1,Operand2 op2,Register dest,ALUOp op,SBit s,Condition c)274 void MacroAssemblerARM::ma_alu(Register src1, Operand2 op2, Register dest,
275 ALUOp op, SBit s, Condition c) {
276 as_alu(dest, src1, op2, op, s, c);
277 }
278
ma_nop()279 void MacroAssemblerARM::ma_nop() { as_nop(); }
280
ma_movPatchable(Imm32 imm_,Register dest,Assembler::Condition c)281 BufferOffset MacroAssemblerARM::ma_movPatchable(Imm32 imm_, Register dest,
282 Assembler::Condition c) {
283 int32_t imm = imm_.value;
284 if (HasMOVWT()) {
285 BufferOffset offset = as_movw(dest, Imm16(imm & 0xffff), c);
286 as_movt(dest, Imm16(imm >> 16 & 0xffff), c);
287 return offset;
288 } else {
289 return as_Imm32Pool(dest, imm, c);
290 }
291 }
292
ma_movPatchable(ImmPtr imm,Register dest,Assembler::Condition c)293 BufferOffset MacroAssemblerARM::ma_movPatchable(ImmPtr imm, Register dest,
294 Assembler::Condition c) {
295 return ma_movPatchable(Imm32(int32_t(imm.value)), dest, c);
296 }
297
298 /* static */
299 template <class Iter>
ma_mov_patch(Imm32 imm32,Register dest,Assembler::Condition c,RelocStyle rs,Iter iter)300 void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
301 Assembler::Condition c, RelocStyle rs,
302 Iter iter) {
303 // The current instruction must be an actual instruction,
304 // not automatically-inserted boilerplate.
305 MOZ_ASSERT(iter.cur());
306 MOZ_ASSERT(iter.cur() == iter.maybeSkipAutomaticInstructions());
307
308 int32_t imm = imm32.value;
309 switch (rs) {
310 case L_MOVWT:
311 Assembler::as_movw_patch(dest, Imm16(imm & 0xffff), c, iter.cur());
312 Assembler::as_movt_patch(dest, Imm16(imm >> 16 & 0xffff), c, iter.next());
313 break;
314 case L_LDR:
315 Assembler::WritePoolEntry(iter.cur(), c, imm);
316 break;
317 }
318 }
319
320 template void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
321 Assembler::Condition c,
322 RelocStyle rs,
323 InstructionIterator iter);
324 template void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
325 Assembler::Condition c,
326 RelocStyle rs,
327 BufferInstructionIterator iter);
328
ma_mov(Register src,Register dest,SBit s,Assembler::Condition c)329 void MacroAssemblerARM::ma_mov(Register src, Register dest, SBit s,
330 Assembler::Condition c) {
331 if (s == SetCC || dest != src) as_mov(dest, O2Reg(src), s, c);
332 }
333
ma_mov(Imm32 imm,Register dest,Assembler::Condition c)334 void MacroAssemblerARM::ma_mov(Imm32 imm, Register dest,
335 Assembler::Condition c) {
336 // Try mov with Imm8 operand.
337 Imm8 imm8 = Imm8(imm.value);
338 if (!imm8.invalid()) {
339 as_alu(dest, InvalidReg, imm8, OpMov, LeaveCC, c);
340 return;
341 }
342
343 // Try mvn with Imm8 operand.
344 Imm8 negImm8 = Imm8(~imm.value);
345 if (!negImm8.invalid()) {
346 as_alu(dest, InvalidReg, negImm8, OpMvn, LeaveCC, c);
347 return;
348 }
349
350 // Try movw/movt.
351 if (HasMOVWT()) {
352 // ARMv7 supports movw/movt. movw zero-extends its 16 bit argument,
353 // so we can set the register this way. movt leaves the bottom 16
354 // bits in tact, so we always need a movw.
355 as_movw(dest, Imm16(imm.value & 0xffff), c);
356 if (uint32_t(imm.value) >> 16)
357 as_movt(dest, Imm16(uint32_t(imm.value) >> 16), c);
358 return;
359 }
360
361 // If we don't have movw/movt, we need a load.
362 as_Imm32Pool(dest, imm.value, c);
363 }
364
ma_mov(ImmWord imm,Register dest,Assembler::Condition c)365 void MacroAssemblerARM::ma_mov(ImmWord imm, Register dest,
366 Assembler::Condition c) {
367 ma_mov(Imm32(imm.value), dest, c);
368 }
369
ma_mov(ImmGCPtr ptr,Register dest)370 void MacroAssemblerARM::ma_mov(ImmGCPtr ptr, Register dest) {
371 BufferOffset offset =
372 ma_movPatchable(Imm32(uintptr_t(ptr.value)), dest, Always);
373 writeDataRelocation(offset, ptr);
374 }
375
376 // Shifts (just a move with a shifting op2)
ma_lsl(Imm32 shift,Register src,Register dst)377 void MacroAssemblerARM::ma_lsl(Imm32 shift, Register src, Register dst) {
378 as_mov(dst, lsl(src, shift.value));
379 }
380
ma_lsr(Imm32 shift,Register src,Register dst)381 void MacroAssemblerARM::ma_lsr(Imm32 shift, Register src, Register dst) {
382 as_mov(dst, lsr(src, shift.value));
383 }
384
ma_asr(Imm32 shift,Register src,Register dst)385 void MacroAssemblerARM::ma_asr(Imm32 shift, Register src, Register dst) {
386 as_mov(dst, asr(src, shift.value));
387 }
388
ma_ror(Imm32 shift,Register src,Register dst)389 void MacroAssemblerARM::ma_ror(Imm32 shift, Register src, Register dst) {
390 as_mov(dst, ror(src, shift.value));
391 }
392
ma_rol(Imm32 shift,Register src,Register dst)393 void MacroAssemblerARM::ma_rol(Imm32 shift, Register src, Register dst) {
394 as_mov(dst, rol(src, shift.value));
395 }
396
397 // Shifts (just a move with a shifting op2)
ma_lsl(Register shift,Register src,Register dst)398 void MacroAssemblerARM::ma_lsl(Register shift, Register src, Register dst) {
399 as_mov(dst, lsl(src, shift));
400 }
401
ma_lsr(Register shift,Register src,Register dst)402 void MacroAssemblerARM::ma_lsr(Register shift, Register src, Register dst) {
403 as_mov(dst, lsr(src, shift));
404 }
405
ma_asr(Register shift,Register src,Register dst)406 void MacroAssemblerARM::ma_asr(Register shift, Register src, Register dst) {
407 as_mov(dst, asr(src, shift));
408 }
409
ma_ror(Register shift,Register src,Register dst)410 void MacroAssemblerARM::ma_ror(Register shift, Register src, Register dst) {
411 as_mov(dst, ror(src, shift));
412 }
413
ma_rol(Register shift,Register src,Register dst,AutoRegisterScope & scratch)414 void MacroAssemblerARM::ma_rol(Register shift, Register src, Register dst,
415 AutoRegisterScope& scratch) {
416 as_rsb(scratch, shift, Imm8(32));
417 as_mov(dst, ror(src, scratch));
418 }
419
420 // Move not (dest <- ~src)
ma_mvn(Register src1,Register dest,SBit s,Assembler::Condition c)421 void MacroAssemblerARM::ma_mvn(Register src1, Register dest, SBit s,
422 Assembler::Condition c) {
423 as_alu(dest, InvalidReg, O2Reg(src1), OpMvn, s, c);
424 }
425
426 // Negate (dest <- -src), src is a register, rather than a general op2.
ma_neg(Register src1,Register dest,SBit s,Assembler::Condition c)427 void MacroAssemblerARM::ma_neg(Register src1, Register dest, SBit s,
428 Assembler::Condition c) {
429 as_rsb(dest, src1, Imm8(0), s, c);
430 }
431
432 // And.
ma_and(Register src,Register dest,SBit s,Assembler::Condition c)433 void MacroAssemblerARM::ma_and(Register src, Register dest, SBit s,
434 Assembler::Condition c) {
435 ma_and(dest, src, dest);
436 }
437
ma_and(Register src1,Register src2,Register dest,SBit s,Assembler::Condition c)438 void MacroAssemblerARM::ma_and(Register src1, Register src2, Register dest,
439 SBit s, Assembler::Condition c) {
440 as_and(dest, src1, O2Reg(src2), s, c);
441 }
442
ma_and(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)443 void MacroAssemblerARM::ma_and(Imm32 imm, Register dest,
444 AutoRegisterScope& scratch, SBit s,
445 Assembler::Condition c) {
446 ma_alu(dest, imm, dest, scratch, OpAnd, s, c);
447 }
448
ma_and(Imm32 imm,Register src1,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)449 void MacroAssemblerARM::ma_and(Imm32 imm, Register src1, Register dest,
450 AutoRegisterScope& scratch, SBit s,
451 Assembler::Condition c) {
452 ma_alu(src1, imm, dest, scratch, OpAnd, s, c);
453 }
454
455 // Bit clear (dest <- dest & ~imm) or (dest <- src1 & ~src2).
ma_bic(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)456 void MacroAssemblerARM::ma_bic(Imm32 imm, Register dest,
457 AutoRegisterScope& scratch, SBit s,
458 Assembler::Condition c) {
459 ma_alu(dest, imm, dest, scratch, OpBic, s, c);
460 }
461
462 // Exclusive or.
ma_eor(Register src,Register dest,SBit s,Assembler::Condition c)463 void MacroAssemblerARM::ma_eor(Register src, Register dest, SBit s,
464 Assembler::Condition c) {
465 ma_eor(dest, src, dest, s, c);
466 }
467
ma_eor(Register src1,Register src2,Register dest,SBit s,Assembler::Condition c)468 void MacroAssemblerARM::ma_eor(Register src1, Register src2, Register dest,
469 SBit s, Assembler::Condition c) {
470 as_eor(dest, src1, O2Reg(src2), s, c);
471 }
472
ma_eor(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)473 void MacroAssemblerARM::ma_eor(Imm32 imm, Register dest,
474 AutoRegisterScope& scratch, SBit s,
475 Assembler::Condition c) {
476 ma_alu(dest, imm, dest, scratch, OpEor, s, c);
477 }
478
ma_eor(Imm32 imm,Register src1,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)479 void MacroAssemblerARM::ma_eor(Imm32 imm, Register src1, Register dest,
480 AutoRegisterScope& scratch, SBit s,
481 Assembler::Condition c) {
482 ma_alu(src1, imm, dest, scratch, OpEor, s, c);
483 }
484
485 // Or.
ma_orr(Register src,Register dest,SBit s,Assembler::Condition c)486 void MacroAssemblerARM::ma_orr(Register src, Register dest, SBit s,
487 Assembler::Condition c) {
488 ma_orr(dest, src, dest, s, c);
489 }
490
ma_orr(Register src1,Register src2,Register dest,SBit s,Assembler::Condition c)491 void MacroAssemblerARM::ma_orr(Register src1, Register src2, Register dest,
492 SBit s, Assembler::Condition c) {
493 as_orr(dest, src1, O2Reg(src2), s, c);
494 }
495
ma_orr(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)496 void MacroAssemblerARM::ma_orr(Imm32 imm, Register dest,
497 AutoRegisterScope& scratch, SBit s,
498 Assembler::Condition c) {
499 ma_alu(dest, imm, dest, scratch, OpOrr, s, c);
500 }
501
ma_orr(Imm32 imm,Register src1,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)502 void MacroAssemblerARM::ma_orr(Imm32 imm, Register src1, Register dest,
503 AutoRegisterScope& scratch, SBit s,
504 Assembler::Condition c) {
505 ma_alu(src1, imm, dest, scratch, OpOrr, s, c);
506 }
507
508 // Arithmetic-based ops.
509 // Add with carry.
ma_adc(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)510 void MacroAssemblerARM::ma_adc(Imm32 imm, Register dest,
511 AutoRegisterScope& scratch, SBit s,
512 Condition c) {
513 ma_alu(dest, imm, dest, scratch, OpAdc, s, c);
514 }
515
ma_adc(Register src,Register dest,SBit s,Condition c)516 void MacroAssemblerARM::ma_adc(Register src, Register dest, SBit s,
517 Condition c) {
518 as_alu(dest, dest, O2Reg(src), OpAdc, s, c);
519 }
520
ma_adc(Register src1,Register src2,Register dest,SBit s,Condition c)521 void MacroAssemblerARM::ma_adc(Register src1, Register src2, Register dest,
522 SBit s, Condition c) {
523 as_alu(dest, src1, O2Reg(src2), OpAdc, s, c);
524 }
525
526 // Add.
ma_add(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)527 void MacroAssemblerARM::ma_add(Imm32 imm, Register dest,
528 AutoRegisterScope& scratch, SBit s,
529 Condition c) {
530 ma_alu(dest, imm, dest, scratch, OpAdd, s, c);
531 }
532
ma_add(Register src1,Register dest,SBit s,Condition c)533 void MacroAssemblerARM::ma_add(Register src1, Register dest, SBit s,
534 Condition c) {
535 ma_alu(dest, O2Reg(src1), dest, OpAdd, s, c);
536 }
537
ma_add(Register src1,Register src2,Register dest,SBit s,Condition c)538 void MacroAssemblerARM::ma_add(Register src1, Register src2, Register dest,
539 SBit s, Condition c) {
540 as_alu(dest, src1, O2Reg(src2), OpAdd, s, c);
541 }
542
ma_add(Register src1,Operand op,Register dest,SBit s,Condition c)543 void MacroAssemblerARM::ma_add(Register src1, Operand op, Register dest, SBit s,
544 Condition c) {
545 ma_alu(src1, op, dest, OpAdd, s, c);
546 }
547
ma_add(Register src1,Imm32 op,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)548 void MacroAssemblerARM::ma_add(Register src1, Imm32 op, Register dest,
549 AutoRegisterScope& scratch, SBit s,
550 Condition c) {
551 ma_alu(src1, op, dest, scratch, OpAdd, s, c);
552 }
553
554 // Subtract with carry.
ma_sbc(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)555 void MacroAssemblerARM::ma_sbc(Imm32 imm, Register dest,
556 AutoRegisterScope& scratch, SBit s,
557 Condition c) {
558 ma_alu(dest, imm, dest, scratch, OpSbc, s, c);
559 }
560
ma_sbc(Register src1,Register dest,SBit s,Condition c)561 void MacroAssemblerARM::ma_sbc(Register src1, Register dest, SBit s,
562 Condition c) {
563 as_alu(dest, dest, O2Reg(src1), OpSbc, s, c);
564 }
565
ma_sbc(Register src1,Register src2,Register dest,SBit s,Condition c)566 void MacroAssemblerARM::ma_sbc(Register src1, Register src2, Register dest,
567 SBit s, Condition c) {
568 as_alu(dest, src1, O2Reg(src2), OpSbc, s, c);
569 }
570
571 // Subtract.
ma_sub(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)572 void MacroAssemblerARM::ma_sub(Imm32 imm, Register dest,
573 AutoRegisterScope& scratch, SBit s,
574 Condition c) {
575 ma_alu(dest, imm, dest, scratch, OpSub, s, c);
576 }
577
ma_sub(Register src1,Register dest,SBit s,Condition c)578 void MacroAssemblerARM::ma_sub(Register src1, Register dest, SBit s,
579 Condition c) {
580 ma_alu(dest, Operand(src1), dest, OpSub, s, c);
581 }
582
ma_sub(Register src1,Register src2,Register dest,SBit s,Condition c)583 void MacroAssemblerARM::ma_sub(Register src1, Register src2, Register dest,
584 SBit s, Condition c) {
585 ma_alu(src1, Operand(src2), dest, OpSub, s, c);
586 }
587
ma_sub(Register src1,Operand op,Register dest,SBit s,Condition c)588 void MacroAssemblerARM::ma_sub(Register src1, Operand op, Register dest, SBit s,
589 Condition c) {
590 ma_alu(src1, op, dest, OpSub, s, c);
591 }
592
ma_sub(Register src1,Imm32 op,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)593 void MacroAssemblerARM::ma_sub(Register src1, Imm32 op, Register dest,
594 AutoRegisterScope& scratch, SBit s,
595 Condition c) {
596 ma_alu(src1, op, dest, scratch, OpSub, s, c);
597 }
598
599 // Reverse subtract.
ma_rsb(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)600 void MacroAssemblerARM::ma_rsb(Imm32 imm, Register dest,
601 AutoRegisterScope& scratch, SBit s,
602 Condition c) {
603 ma_alu(dest, imm, dest, scratch, OpRsb, s, c);
604 }
605
ma_rsb(Register src1,Register dest,SBit s,Condition c)606 void MacroAssemblerARM::ma_rsb(Register src1, Register dest, SBit s,
607 Condition c) {
608 as_alu(dest, src1, O2Reg(dest), OpRsb, s, c);
609 }
610
ma_rsb(Register src1,Register src2,Register dest,SBit s,Condition c)611 void MacroAssemblerARM::ma_rsb(Register src1, Register src2, Register dest,
612 SBit s, Condition c) {
613 as_alu(dest, src1, O2Reg(src2), OpRsb, s, c);
614 }
615
ma_rsb(Register src1,Imm32 op2,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)616 void MacroAssemblerARM::ma_rsb(Register src1, Imm32 op2, Register dest,
617 AutoRegisterScope& scratch, SBit s,
618 Condition c) {
619 ma_alu(src1, op2, dest, scratch, OpRsb, s, c);
620 }
621
622 // Reverse subtract with carry.
ma_rsc(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)623 void MacroAssemblerARM::ma_rsc(Imm32 imm, Register dest,
624 AutoRegisterScope& scratch, SBit s,
625 Condition c) {
626 ma_alu(dest, imm, dest, scratch, OpRsc, s, c);
627 }
628
ma_rsc(Register src1,Register dest,SBit s,Condition c)629 void MacroAssemblerARM::ma_rsc(Register src1, Register dest, SBit s,
630 Condition c) {
631 as_alu(dest, dest, O2Reg(src1), OpRsc, s, c);
632 }
633
ma_rsc(Register src1,Register src2,Register dest,SBit s,Condition c)634 void MacroAssemblerARM::ma_rsc(Register src1, Register src2, Register dest,
635 SBit s, Condition c) {
636 as_alu(dest, src1, O2Reg(src2), OpRsc, s, c);
637 }
638
639 // Compares/tests.
640 // Compare negative (sets condition codes as src1 + src2 would).
ma_cmn(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)641 void MacroAssemblerARM::ma_cmn(Register src1, Imm32 imm,
642 AutoRegisterScope& scratch, Condition c) {
643 ma_alu(src1, imm, InvalidReg, scratch, OpCmn, SetCC, c);
644 }
645
ma_cmn(Register src1,Register src2,Condition c)646 void MacroAssemblerARM::ma_cmn(Register src1, Register src2, Condition c) {
647 as_alu(InvalidReg, src2, O2Reg(src1), OpCmn, SetCC, c);
648 }
649
ma_cmn(Register src1,Operand op,Condition c)650 void MacroAssemblerARM::ma_cmn(Register src1, Operand op, Condition c) {
651 MOZ_CRASH("Feature NYI");
652 }
653
654 // Compare (src - src2).
ma_cmp(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)655 void MacroAssemblerARM::ma_cmp(Register src1, Imm32 imm,
656 AutoRegisterScope& scratch, Condition c) {
657 ma_alu(src1, imm, InvalidReg, scratch, OpCmp, SetCC, c);
658 }
659
ma_cmp(Register src1,ImmTag tag,Condition c)660 void MacroAssemblerARM::ma_cmp(Register src1, ImmTag tag, Condition c) {
661 // ImmTag comparisons can always be done without use of a scratch register.
662 Imm8 negtag = Imm8(-tag.value);
663 MOZ_ASSERT(!negtag.invalid());
664 as_cmn(src1, negtag, c);
665 }
666
ma_cmp(Register src1,ImmWord ptr,AutoRegisterScope & scratch,Condition c)667 void MacroAssemblerARM::ma_cmp(Register src1, ImmWord ptr,
668 AutoRegisterScope& scratch, Condition c) {
669 ma_cmp(src1, Imm32(ptr.value), scratch, c);
670 }
671
ma_cmp(Register src1,ImmGCPtr ptr,AutoRegisterScope & scratch,Condition c)672 void MacroAssemblerARM::ma_cmp(Register src1, ImmGCPtr ptr,
673 AutoRegisterScope& scratch, Condition c) {
674 ma_mov(ptr, scratch);
675 ma_cmp(src1, scratch, c);
676 }
677
ma_cmp(Register src1,Operand op,AutoRegisterScope & scratch,AutoRegisterScope & scratch2,Condition c)678 void MacroAssemblerARM::ma_cmp(Register src1, Operand op,
679 AutoRegisterScope& scratch,
680 AutoRegisterScope& scratch2, Condition c) {
681 switch (op.tag()) {
682 case Operand::Tag::OP2:
683 as_cmp(src1, op.toOp2(), c);
684 break;
685 case Operand::Tag::MEM:
686 ma_ldr(op.toAddress(), scratch, scratch2);
687 as_cmp(src1, O2Reg(scratch), c);
688 break;
689 default:
690 MOZ_CRASH("trying to compare FP and integer registers");
691 }
692 }
693
ma_cmp(Register src1,Register src2,Condition c)694 void MacroAssemblerARM::ma_cmp(Register src1, Register src2, Condition c) {
695 as_cmp(src1, O2Reg(src2), c);
696 }
697
698 // Test for equality, (src1 ^ src2).
ma_teq(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)699 void MacroAssemblerARM::ma_teq(Register src1, Imm32 imm,
700 AutoRegisterScope& scratch, Condition c) {
701 ma_alu(src1, imm, InvalidReg, scratch, OpTeq, SetCC, c);
702 }
703
ma_teq(Register src1,Register src2,Condition c)704 void MacroAssemblerARM::ma_teq(Register src1, Register src2, Condition c) {
705 as_tst(src1, O2Reg(src2), c);
706 }
707
ma_teq(Register src1,Operand op,Condition c)708 void MacroAssemblerARM::ma_teq(Register src1, Operand op, Condition c) {
709 as_teq(src1, op.toOp2(), c);
710 }
711
712 // Test (src1 & src2).
ma_tst(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)713 void MacroAssemblerARM::ma_tst(Register src1, Imm32 imm,
714 AutoRegisterScope& scratch, Condition c) {
715 ma_alu(src1, imm, InvalidReg, scratch, OpTst, SetCC, c);
716 }
717
ma_tst(Register src1,Register src2,Condition c)718 void MacroAssemblerARM::ma_tst(Register src1, Register src2, Condition c) {
719 as_tst(src1, O2Reg(src2), c);
720 }
721
ma_tst(Register src1,Operand op,Condition c)722 void MacroAssemblerARM::ma_tst(Register src1, Operand op, Condition c) {
723 as_tst(src1, op.toOp2(), c);
724 }
725
ma_mul(Register src1,Register src2,Register dest)726 void MacroAssemblerARM::ma_mul(Register src1, Register src2, Register dest) {
727 as_mul(dest, src1, src2);
728 }
729
ma_mul(Register src1,Imm32 imm,Register dest,AutoRegisterScope & scratch)730 void MacroAssemblerARM::ma_mul(Register src1, Imm32 imm, Register dest,
731 AutoRegisterScope& scratch) {
732 ma_mov(imm, scratch);
733 as_mul(dest, src1, scratch);
734 }
735
ma_check_mul(Register src1,Register src2,Register dest,AutoRegisterScope & scratch,Condition cond)736 Assembler::Condition MacroAssemblerARM::ma_check_mul(Register src1,
737 Register src2,
738 Register dest,
739 AutoRegisterScope& scratch,
740 Condition cond) {
741 // TODO: this operation is illegal on armv6 and earlier
742 // if src2 == scratch or src2 == dest.
743 if (cond == Equal || cond == NotEqual) {
744 as_smull(scratch, dest, src1, src2, SetCC);
745 return cond;
746 }
747
748 if (cond == Overflow) {
749 as_smull(scratch, dest, src1, src2);
750 as_cmp(scratch, asr(dest, 31));
751 return NotEqual;
752 }
753
754 MOZ_CRASH("Condition NYI");
755 }
756
ma_check_mul(Register src1,Imm32 imm,Register dest,AutoRegisterScope & scratch,Condition cond)757 Assembler::Condition MacroAssemblerARM::ma_check_mul(Register src1, Imm32 imm,
758 Register dest,
759 AutoRegisterScope& scratch,
760 Condition cond) {
761 ma_mov(imm, scratch);
762
763 if (cond == Equal || cond == NotEqual) {
764 as_smull(scratch, dest, scratch, src1, SetCC);
765 return cond;
766 }
767
768 if (cond == Overflow) {
769 as_smull(scratch, dest, scratch, src1);
770 as_cmp(scratch, asr(dest, 31));
771 return NotEqual;
772 }
773
774 MOZ_CRASH("Condition NYI");
775 }
776
ma_umull(Register src1,Imm32 imm,Register destHigh,Register destLow,AutoRegisterScope & scratch)777 void MacroAssemblerARM::ma_umull(Register src1, Imm32 imm, Register destHigh,
778 Register destLow, AutoRegisterScope& scratch) {
779 ma_mov(imm, scratch);
780 as_umull(destHigh, destLow, src1, scratch);
781 }
782
ma_umull(Register src1,Register src2,Register destHigh,Register destLow)783 void MacroAssemblerARM::ma_umull(Register src1, Register src2,
784 Register destHigh, Register destLow) {
785 as_umull(destHigh, destLow, src1, src2);
786 }
787
ma_mod_mask(Register src,Register dest,Register hold,Register tmp,AutoRegisterScope & scratch,AutoRegisterScope & scratch2,int32_t shift)788 void MacroAssemblerARM::ma_mod_mask(Register src, Register dest, Register hold,
789 Register tmp, AutoRegisterScope& scratch,
790 AutoRegisterScope& scratch2,
791 int32_t shift) {
792 // We wish to compute x % (1<<y) - 1 for a known constant, y.
793 //
794 // 1. Let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit dividend as
795 // a number in base b, namely c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
796 //
797 // 2. Since both addition and multiplication commute with modulus:
798 // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
799 // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
800 //
801 // 3. Since b == C + 1, b % C == 1, and b^n % C == 1 the whole thing
802 // simplifies to: c_0 + c_1 + c_2 ... c_n % C
803 //
804 // Each c_n can easily be computed by a shift/bitextract, and the modulus
805 // can be maintained by simply subtracting by C whenever the number gets
806 // over C.
807 int32_t mask = (1 << shift) - 1;
808 Label head;
809
810 // Register 'hold' holds -1 if the value was negative, 1 otherwise. The
811 // scratch reg holds the remaining bits that have not been processed lr
812 // serves as a temporary location to store extracted bits into as well as
813 // holding the trial subtraction as a temp value dest is the accumulator
814 // (and holds the final result)
815 //
816 // Move the whole value into tmp, setting the codition codes so we can muck
817 // with them later.
818 as_mov(tmp, O2Reg(src), SetCC);
819 // Zero out the dest.
820 ma_mov(Imm32(0), dest);
821 // Set the hold appropriately.
822 ma_mov(Imm32(1), hold);
823 ma_mov(Imm32(-1), hold, Signed);
824 as_rsb(tmp, tmp, Imm8(0), SetCC, Signed);
825
826 // Begin the main loop.
827 bind(&head);
828 {
829 // Extract the bottom bits.
830 ma_and(Imm32(mask), tmp, scratch, scratch2);
831 // Add those bits to the accumulator.
832 ma_add(scratch, dest, dest);
833 // Do a trial subtraction, this is the same operation as cmp, but we store
834 // the dest.
835 ma_sub(dest, Imm32(mask), scratch, scratch2, SetCC);
836 // If (sum - C) > 0, store sum - C back into sum, thus performing a modulus.
837 ma_mov(scratch, dest, LeaveCC, NotSigned);
838 // Get rid of the bits that we extracted before, and set the condition
839 // codes.
840 as_mov(tmp, lsr(tmp, shift), SetCC);
841 // If the shift produced zero, finish, otherwise, continue in the loop.
842 ma_b(&head, NonZero);
843 }
844
845 // Check the hold to see if we need to negate the result. Hold can only be
846 // 1 or -1, so this will never set the 0 flag.
847 as_cmp(hold, Imm8(0));
848 // If the hold was non-zero, negate the result to be in line with what JS
849 // wants this will set the condition codes if we try to negate.
850 as_rsb(dest, dest, Imm8(0), SetCC, Signed);
851 // Since the Zero flag is not set by the compare, we can *only* set the Zero
852 // flag in the rsb, so Zero is set iff we negated zero (e.g. the result of
853 // the computation was -0.0).
854 }
855
ma_smod(Register num,Register div,Register dest,AutoRegisterScope & scratch)856 void MacroAssemblerARM::ma_smod(Register num, Register div, Register dest,
857 AutoRegisterScope& scratch) {
858 as_sdiv(scratch, num, div);
859 as_mls(dest, num, scratch, div);
860 }
861
ma_umod(Register num,Register div,Register dest,AutoRegisterScope & scratch)862 void MacroAssemblerARM::ma_umod(Register num, Register div, Register dest,
863 AutoRegisterScope& scratch) {
864 as_udiv(scratch, num, div);
865 as_mls(dest, num, scratch, div);
866 }
867
868 // Division
ma_sdiv(Register num,Register div,Register dest,Condition cond)869 void MacroAssemblerARM::ma_sdiv(Register num, Register div, Register dest,
870 Condition cond) {
871 as_sdiv(dest, num, div, cond);
872 }
873
ma_udiv(Register num,Register div,Register dest,Condition cond)874 void MacroAssemblerARM::ma_udiv(Register num, Register div, Register dest,
875 Condition cond) {
876 as_udiv(dest, num, div, cond);
877 }
878
879 // Miscellaneous instructions.
ma_clz(Register src,Register dest,Condition cond)880 void MacroAssemblerARM::ma_clz(Register src, Register dest, Condition cond) {
881 as_clz(dest, src, cond);
882 }
883
ma_ctz(Register src,Register dest,AutoRegisterScope & scratch)884 void MacroAssemblerARM::ma_ctz(Register src, Register dest,
885 AutoRegisterScope& scratch) {
886 // int c = __clz(a & -a);
887 // return a ? 31 - c : c;
888 as_rsb(scratch, src, Imm8(0), SetCC);
889 as_and(dest, src, O2Reg(scratch), LeaveCC);
890 as_clz(dest, dest);
891 as_rsb(dest, dest, Imm8(0x1F), LeaveCC, Assembler::NotEqual);
892 }
893
894 // Memory.
895 // Shortcut for when we know we're transferring 32 bits of data.
ma_dtr(LoadStore ls,Register rn,Imm32 offset,Register rt,AutoRegisterScope & scratch,Index mode,Assembler::Condition cc)896 void MacroAssemblerARM::ma_dtr(LoadStore ls, Register rn, Imm32 offset,
897 Register rt, AutoRegisterScope& scratch,
898 Index mode, Assembler::Condition cc) {
899 ma_dataTransferN(ls, 32, true, rn, offset, rt, scratch, mode, cc);
900 }
901
ma_dtr(LoadStore ls,Register rt,const Address & addr,AutoRegisterScope & scratch,Index mode,Condition cc)902 void MacroAssemblerARM::ma_dtr(LoadStore ls, Register rt, const Address& addr,
903 AutoRegisterScope& scratch, Index mode,
904 Condition cc) {
905 ma_dataTransferN(ls, 32, true, addr.base, Imm32(addr.offset), rt, scratch,
906 mode, cc);
907 }
908
ma_str(Register rt,DTRAddr addr,Index mode,Condition cc)909 void MacroAssemblerARM::ma_str(Register rt, DTRAddr addr, Index mode,
910 Condition cc) {
911 as_dtr(IsStore, 32, mode, rt, addr, cc);
912 }
913
ma_str(Register rt,const Address & addr,AutoRegisterScope & scratch,Index mode,Condition cc)914 void MacroAssemblerARM::ma_str(Register rt, const Address& addr,
915 AutoRegisterScope& scratch, Index mode,
916 Condition cc) {
917 ma_dtr(IsStore, rt, addr, scratch, mode, cc);
918 }
919
ma_strd(Register rt,DebugOnly<Register> rt2,EDtrAddr addr,Index mode,Condition cc)920 void MacroAssemblerARM::ma_strd(Register rt, DebugOnly<Register> rt2,
921 EDtrAddr addr, Index mode, Condition cc) {
922 MOZ_ASSERT((rt.code() & 1) == 0);
923 MOZ_ASSERT(rt2.value.code() == rt.code() + 1);
924 as_extdtr(IsStore, 64, true, mode, rt, addr, cc);
925 }
926
ma_ldr(DTRAddr addr,Register rt,Index mode,Condition cc)927 void MacroAssemblerARM::ma_ldr(DTRAddr addr, Register rt, Index mode,
928 Condition cc) {
929 as_dtr(IsLoad, 32, mode, rt, addr, cc);
930 }
931
ma_ldr(const Address & addr,Register rt,AutoRegisterScope & scratch,Index mode,Condition cc)932 void MacroAssemblerARM::ma_ldr(const Address& addr, Register rt,
933 AutoRegisterScope& scratch, Index mode,
934 Condition cc) {
935 ma_dtr(IsLoad, rt, addr, scratch, mode, cc);
936 }
937
ma_ldrb(DTRAddr addr,Register rt,Index mode,Condition cc)938 void MacroAssemblerARM::ma_ldrb(DTRAddr addr, Register rt, Index mode,
939 Condition cc) {
940 as_dtr(IsLoad, 8, mode, rt, addr, cc);
941 }
942
ma_ldrsh(EDtrAddr addr,Register rt,Index mode,Condition cc)943 void MacroAssemblerARM::ma_ldrsh(EDtrAddr addr, Register rt, Index mode,
944 Condition cc) {
945 as_extdtr(IsLoad, 16, true, mode, rt, addr, cc);
946 }
947
ma_ldrh(EDtrAddr addr,Register rt,Index mode,Condition cc)948 void MacroAssemblerARM::ma_ldrh(EDtrAddr addr, Register rt, Index mode,
949 Condition cc) {
950 as_extdtr(IsLoad, 16, false, mode, rt, addr, cc);
951 }
952
ma_ldrsb(EDtrAddr addr,Register rt,Index mode,Condition cc)953 void MacroAssemblerARM::ma_ldrsb(EDtrAddr addr, Register rt, Index mode,
954 Condition cc) {
955 as_extdtr(IsLoad, 8, true, mode, rt, addr, cc);
956 }
957
ma_ldrd(EDtrAddr addr,Register rt,DebugOnly<Register> rt2,Index mode,Condition cc)958 void MacroAssemblerARM::ma_ldrd(EDtrAddr addr, Register rt,
959 DebugOnly<Register> rt2, Index mode,
960 Condition cc) {
961 MOZ_ASSERT((rt.code() & 1) == 0);
962 MOZ_ASSERT(rt2.value.code() == rt.code() + 1);
963 MOZ_ASSERT(addr.maybeOffsetRegister() !=
964 rt); // Undefined behavior if rm == rt/rt2.
965 MOZ_ASSERT(addr.maybeOffsetRegister() != rt2);
966 as_extdtr(IsLoad, 64, true, mode, rt, addr, cc);
967 }
968
ma_strh(Register rt,EDtrAddr addr,Index mode,Condition cc)969 void MacroAssemblerARM::ma_strh(Register rt, EDtrAddr addr, Index mode,
970 Condition cc) {
971 as_extdtr(IsStore, 16, false, mode, rt, addr, cc);
972 }
973
ma_strb(Register rt,DTRAddr addr,Index mode,Condition cc)974 void MacroAssemblerARM::ma_strb(Register rt, DTRAddr addr, Index mode,
975 Condition cc) {
976 as_dtr(IsStore, 8, mode, rt, addr, cc);
977 }
978
979 // Specialty for moving N bits of data, where n == 8,16,32,64.
ma_dataTransferN(LoadStore ls,int size,bool IsSigned,Register rn,Register rm,Register rt,AutoRegisterScope & scratch,Index mode,Assembler::Condition cc,Scale scale)980 BufferOffset MacroAssemblerARM::ma_dataTransferN(
981 LoadStore ls, int size, bool IsSigned, Register rn, Register rm,
982 Register rt, AutoRegisterScope& scratch, Index mode,
983 Assembler::Condition cc, Scale scale) {
984 MOZ_ASSERT(size == 8 || size == 16 || size == 32 || size == 64);
985
986 if (size == 32 || (size == 8 && !IsSigned))
987 return as_dtr(ls, size, mode, rt,
988 DTRAddr(rn, DtrRegImmShift(rm, LSL, scale)), cc);
989
990 if (scale != TimesOne) {
991 ma_lsl(Imm32(scale), rm, scratch);
992 rm = scratch;
993 }
994
995 return as_extdtr(ls, size, IsSigned, mode, rt, EDtrAddr(rn, EDtrOffReg(rm)),
996 cc);
997 }
998
999 // No scratch register is required if scale is TimesOne.
ma_dataTransferN(LoadStore ls,int size,bool IsSigned,Register rn,Register rm,Register rt,Index mode,Assembler::Condition cc)1000 BufferOffset MacroAssemblerARM::ma_dataTransferN(LoadStore ls, int size,
1001 bool IsSigned, Register rn,
1002 Register rm, Register rt,
1003 Index mode,
1004 Assembler::Condition cc) {
1005 MOZ_ASSERT(size == 8 || size == 16 || size == 32 || size == 64);
1006 if (size == 32 || (size == 8 && !IsSigned))
1007 return as_dtr(ls, size, mode, rt,
1008 DTRAddr(rn, DtrRegImmShift(rm, LSL, TimesOne)), cc);
1009 return as_extdtr(ls, size, IsSigned, mode, rt, EDtrAddr(rn, EDtrOffReg(rm)),
1010 cc);
1011 }
1012
ma_dataTransferN(LoadStore ls,int size,bool IsSigned,Register rn,Imm32 offset,Register rt,AutoRegisterScope & scratch,Index mode,Assembler::Condition cc)1013 BufferOffset MacroAssemblerARM::ma_dataTransferN(LoadStore ls, int size,
1014 bool IsSigned, Register rn,
1015 Imm32 offset, Register rt,
1016 AutoRegisterScope& scratch,
1017 Index mode,
1018 Assembler::Condition cc) {
1019 MOZ_ASSERT(!(ls == IsLoad && mode == PostIndex && rt == pc),
1020 "Large-offset PostIndex loading into PC requires special logic: "
1021 "see ma_popn_pc().");
1022
1023 int off = offset.value;
1024
1025 // We can encode this as a standard ldr.
1026 if (size == 32 || (size == 8 && !IsSigned)) {
1027 if (off < 4096 && off > -4096) {
1028 // This encodes as a single instruction, Emulating mode's behavior
1029 // in a multi-instruction sequence is not necessary.
1030 return as_dtr(ls, size, mode, rt, DTRAddr(rn, DtrOffImm(off)), cc);
1031 }
1032
1033 // We cannot encode this offset in a single ldr. For mode == index,
1034 // try to encode it as |add scratch, base, imm; ldr dest, [scratch,
1035 // +offset]|. This does not wark for mode == PreIndex or mode == PostIndex.
1036 // PreIndex is simple, just do the add into the base register first,
1037 // then do a PreIndex'ed load. PostIndexed loads can be tricky.
1038 // Normally, doing the load with an index of 0, then doing an add would
1039 // work, but if the destination is the PC, you don't get to execute the
1040 // instruction after the branch, which will lead to the base register
1041 // not being updated correctly. Explicitly handle this case, without
1042 // doing anything fancy, then handle all of the other cases.
1043
1044 // mode == Offset
1045 // add scratch, base, offset_hi
1046 // ldr dest, [scratch, +offset_lo]
1047 //
1048 // mode == PreIndex
1049 // add base, base, offset_hi
1050 // ldr dest, [base, +offset_lo]!
1051
1052 int bottom = off & 0xfff;
1053 int neg_bottom = 0x1000 - bottom;
1054
1055 MOZ_ASSERT(rn != scratch);
1056 MOZ_ASSERT(mode != PostIndex);
1057
1058 // At this point, both off - bottom and off + neg_bottom will be
1059 // reasonable-ish quantities.
1060 //
1061 // Note a neg_bottom of 0x1000 can not be encoded as an immediate
1062 // negative offset in the instruction and this occurs when bottom is
1063 // zero, so this case is guarded against below.
1064 if (off < 0) {
1065 Operand2 sub_off = Imm8(-(off - bottom)); // sub_off = bottom - off
1066 if (!sub_off.invalid()) {
1067 // - sub_off = off - bottom
1068 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1069 return as_dtr(ls, size, Offset, rt, DTRAddr(scratch, DtrOffImm(bottom)),
1070 cc);
1071 }
1072
1073 // sub_off = -neg_bottom - off
1074 sub_off = Imm8(-(off + neg_bottom));
1075 if (!sub_off.invalid() && bottom != 0) {
1076 // Guarded against by: bottom != 0
1077 MOZ_ASSERT(neg_bottom < 0x1000);
1078 // - sub_off = neg_bottom + off
1079 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1080 return as_dtr(ls, size, Offset, rt,
1081 DTRAddr(scratch, DtrOffImm(-neg_bottom)), cc);
1082 }
1083 } else {
1084 // sub_off = off - bottom
1085 Operand2 sub_off = Imm8(off - bottom);
1086 if (!sub_off.invalid()) {
1087 // sub_off = off - bottom
1088 as_add(scratch, rn, sub_off, LeaveCC, cc);
1089 return as_dtr(ls, size, Offset, rt, DTRAddr(scratch, DtrOffImm(bottom)),
1090 cc);
1091 }
1092
1093 // sub_off = neg_bottom + off
1094 sub_off = Imm8(off + neg_bottom);
1095 if (!sub_off.invalid() && bottom != 0) {
1096 // Guarded against by: bottom != 0
1097 MOZ_ASSERT(neg_bottom < 0x1000);
1098 // sub_off = neg_bottom + off
1099 as_add(scratch, rn, sub_off, LeaveCC, cc);
1100 return as_dtr(ls, size, Offset, rt,
1101 DTRAddr(scratch, DtrOffImm(-neg_bottom)), cc);
1102 }
1103 }
1104
1105 ma_mov(offset, scratch);
1106 return as_dtr(ls, size, mode, rt,
1107 DTRAddr(rn, DtrRegImmShift(scratch, LSL, 0)));
1108 } else {
1109 // Should attempt to use the extended load/store instructions.
1110 if (off < 256 && off > -256)
1111 return as_extdtr(ls, size, IsSigned, mode, rt,
1112 EDtrAddr(rn, EDtrOffImm(off)), cc);
1113
1114 // We cannot encode this offset in a single extldr. Try to encode it as
1115 // an add scratch, base, imm; extldr dest, [scratch, +offset].
1116 int bottom = off & 0xff;
1117 int neg_bottom = 0x100 - bottom;
1118 // At this point, both off - bottom and off + neg_bottom will be
1119 // reasonable-ish quantities.
1120 //
1121 // Note a neg_bottom of 0x100 can not be encoded as an immediate
1122 // negative offset in the instruction and this occurs when bottom is
1123 // zero, so this case is guarded against below.
1124 if (off < 0) {
1125 // sub_off = bottom - off
1126 Operand2 sub_off = Imm8(-(off - bottom));
1127 if (!sub_off.invalid()) {
1128 // - sub_off = off - bottom
1129 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1130 return as_extdtr(ls, size, IsSigned, Offset, rt,
1131 EDtrAddr(scratch, EDtrOffImm(bottom)), cc);
1132 }
1133 // sub_off = -neg_bottom - off
1134 sub_off = Imm8(-(off + neg_bottom));
1135 if (!sub_off.invalid() && bottom != 0) {
1136 // Guarded against by: bottom != 0
1137 MOZ_ASSERT(neg_bottom < 0x100);
1138 // - sub_off = neg_bottom + off
1139 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1140 return as_extdtr(ls, size, IsSigned, Offset, rt,
1141 EDtrAddr(scratch, EDtrOffImm(-neg_bottom)), cc);
1142 }
1143 } else {
1144 // sub_off = off - bottom
1145 Operand2 sub_off = Imm8(off - bottom);
1146 if (!sub_off.invalid()) {
1147 // sub_off = off - bottom
1148 as_add(scratch, rn, sub_off, LeaveCC, cc);
1149 return as_extdtr(ls, size, IsSigned, Offset, rt,
1150 EDtrAddr(scratch, EDtrOffImm(bottom)), cc);
1151 }
1152 // sub_off = neg_bottom + off
1153 sub_off = Imm8(off + neg_bottom);
1154 if (!sub_off.invalid() && bottom != 0) {
1155 // Guarded against by: bottom != 0
1156 MOZ_ASSERT(neg_bottom < 0x100);
1157 // sub_off = neg_bottom + off
1158 as_add(scratch, rn, sub_off, LeaveCC, cc);
1159 return as_extdtr(ls, size, IsSigned, Offset, rt,
1160 EDtrAddr(scratch, EDtrOffImm(-neg_bottom)), cc);
1161 }
1162 }
1163 ma_mov(offset, scratch);
1164 return as_extdtr(ls, size, IsSigned, mode, rt,
1165 EDtrAddr(rn, EDtrOffReg(scratch)), cc);
1166 }
1167 }
1168
ma_pop(Register r)1169 void MacroAssemblerARM::ma_pop(Register r) {
1170 as_dtr(IsLoad, 32, PostIndex, r, DTRAddr(sp, DtrOffImm(4)));
1171 }
1172
ma_popn_pc(Imm32 n,AutoRegisterScope & scratch,AutoRegisterScope & scratch2)1173 void MacroAssemblerARM::ma_popn_pc(Imm32 n, AutoRegisterScope& scratch,
1174 AutoRegisterScope& scratch2) {
1175 // pc <- [sp]; sp += n
1176 int32_t nv = n.value;
1177
1178 if (nv < 4096 && nv >= -4096) {
1179 as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(nv)));
1180 } else {
1181 ma_mov(sp, scratch);
1182 ma_add(Imm32(n), sp, scratch2);
1183 as_dtr(IsLoad, 32, Offset, pc, DTRAddr(scratch, DtrOffImm(0)));
1184 }
1185 }
1186
ma_push(Register r)1187 void MacroAssemblerARM::ma_push(Register r) {
1188 MOZ_ASSERT(r != sp, "Use ma_push_sp().");
1189 as_dtr(IsStore, 32, PreIndex, r, DTRAddr(sp, DtrOffImm(-4)));
1190 }
1191
ma_push_sp(Register r,AutoRegisterScope & scratch)1192 void MacroAssemblerARM::ma_push_sp(Register r, AutoRegisterScope& scratch) {
1193 // Pushing sp is not well-defined: use two instructions.
1194 MOZ_ASSERT(r == sp);
1195 ma_mov(sp, scratch);
1196 as_dtr(IsStore, 32, PreIndex, scratch, DTRAddr(sp, DtrOffImm(-4)));
1197 }
1198
ma_vpop(VFPRegister r)1199 void MacroAssemblerARM::ma_vpop(VFPRegister r) {
1200 startFloatTransferM(IsLoad, sp, IA, WriteBack);
1201 transferFloatReg(r);
1202 finishFloatTransfer();
1203 }
1204
ma_vpush(VFPRegister r)1205 void MacroAssemblerARM::ma_vpush(VFPRegister r) {
1206 startFloatTransferM(IsStore, sp, DB, WriteBack);
1207 transferFloatReg(r);
1208 finishFloatTransfer();
1209 }
1210
1211 // Barriers
ma_dmb(BarrierOption option)1212 void MacroAssemblerARM::ma_dmb(BarrierOption option) {
1213 if (HasDMBDSBISB())
1214 as_dmb(option);
1215 else
1216 as_dmb_trap();
1217 }
1218
ma_dsb(BarrierOption option)1219 void MacroAssemblerARM::ma_dsb(BarrierOption option) {
1220 if (HasDMBDSBISB())
1221 as_dsb(option);
1222 else
1223 as_dsb_trap();
1224 }
1225
1226 // Branches when done from within arm-specific code.
ma_b(Label * dest,Assembler::Condition c)1227 BufferOffset MacroAssemblerARM::ma_b(Label* dest, Assembler::Condition c) {
1228 return as_b(dest, c);
1229 }
1230
ma_b(wasm::OldTrapDesc target,Assembler::Condition c)1231 BufferOffset MacroAssemblerARM::ma_b(wasm::OldTrapDesc target,
1232 Assembler::Condition c) {
1233 return as_b(target, c);
1234 }
1235
ma_bx(Register dest,Assembler::Condition c)1236 void MacroAssemblerARM::ma_bx(Register dest, Assembler::Condition c) {
1237 as_bx(dest, c);
1238 }
1239
ma_b(void * target,Assembler::Condition c)1240 void MacroAssemblerARM::ma_b(void* target, Assembler::Condition c) {
1241 // An immediate pool is used for easier patching.
1242 as_Imm32Pool(pc, uint32_t(target), c);
1243 }
1244
1245 // This is almost NEVER necessary: we'll basically never be calling a label,
1246 // except possibly in the crazy bailout-table case.
ma_bl(Label * dest,Assembler::Condition c)1247 void MacroAssemblerARM::ma_bl(Label* dest, Assembler::Condition c) {
1248 as_bl(dest, c);
1249 }
1250
ma_blx(Register reg,Assembler::Condition c)1251 void MacroAssemblerARM::ma_blx(Register reg, Assembler::Condition c) {
1252 as_blx(reg, c);
1253 }
1254
1255 // VFP/ALU
ma_vadd(FloatRegister src1,FloatRegister src2,FloatRegister dst)1256 void MacroAssemblerARM::ma_vadd(FloatRegister src1, FloatRegister src2,
1257 FloatRegister dst) {
1258 as_vadd(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1259 }
1260
ma_vadd_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1261 void MacroAssemblerARM::ma_vadd_f32(FloatRegister src1, FloatRegister src2,
1262 FloatRegister dst) {
1263 as_vadd(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1264 VFPRegister(src2).singleOverlay());
1265 }
1266
ma_vsub(FloatRegister src1,FloatRegister src2,FloatRegister dst)1267 void MacroAssemblerARM::ma_vsub(FloatRegister src1, FloatRegister src2,
1268 FloatRegister dst) {
1269 as_vsub(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1270 }
1271
ma_vsub_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1272 void MacroAssemblerARM::ma_vsub_f32(FloatRegister src1, FloatRegister src2,
1273 FloatRegister dst) {
1274 as_vsub(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1275 VFPRegister(src2).singleOverlay());
1276 }
1277
ma_vmul(FloatRegister src1,FloatRegister src2,FloatRegister dst)1278 void MacroAssemblerARM::ma_vmul(FloatRegister src1, FloatRegister src2,
1279 FloatRegister dst) {
1280 as_vmul(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1281 }
1282
ma_vmul_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1283 void MacroAssemblerARM::ma_vmul_f32(FloatRegister src1, FloatRegister src2,
1284 FloatRegister dst) {
1285 as_vmul(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1286 VFPRegister(src2).singleOverlay());
1287 }
1288
ma_vdiv(FloatRegister src1,FloatRegister src2,FloatRegister dst)1289 void MacroAssemblerARM::ma_vdiv(FloatRegister src1, FloatRegister src2,
1290 FloatRegister dst) {
1291 as_vdiv(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1292 }
1293
ma_vdiv_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1294 void MacroAssemblerARM::ma_vdiv_f32(FloatRegister src1, FloatRegister src2,
1295 FloatRegister dst) {
1296 as_vdiv(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1297 VFPRegister(src2).singleOverlay());
1298 }
1299
ma_vmov(FloatRegister src,FloatRegister dest,Condition cc)1300 void MacroAssemblerARM::ma_vmov(FloatRegister src, FloatRegister dest,
1301 Condition cc) {
1302 as_vmov(dest, src, cc);
1303 }
1304
ma_vmov_f32(FloatRegister src,FloatRegister dest,Condition cc)1305 void MacroAssemblerARM::ma_vmov_f32(FloatRegister src, FloatRegister dest,
1306 Condition cc) {
1307 as_vmov(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1308 cc);
1309 }
1310
ma_vneg(FloatRegister src,FloatRegister dest,Condition cc)1311 void MacroAssemblerARM::ma_vneg(FloatRegister src, FloatRegister dest,
1312 Condition cc) {
1313 as_vneg(dest, src, cc);
1314 }
1315
ma_vneg_f32(FloatRegister src,FloatRegister dest,Condition cc)1316 void MacroAssemblerARM::ma_vneg_f32(FloatRegister src, FloatRegister dest,
1317 Condition cc) {
1318 as_vneg(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1319 cc);
1320 }
1321
ma_vabs(FloatRegister src,FloatRegister dest,Condition cc)1322 void MacroAssemblerARM::ma_vabs(FloatRegister src, FloatRegister dest,
1323 Condition cc) {
1324 as_vabs(dest, src, cc);
1325 }
1326
ma_vabs_f32(FloatRegister src,FloatRegister dest,Condition cc)1327 void MacroAssemblerARM::ma_vabs_f32(FloatRegister src, FloatRegister dest,
1328 Condition cc) {
1329 as_vabs(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1330 cc);
1331 }
1332
ma_vsqrt(FloatRegister src,FloatRegister dest,Condition cc)1333 void MacroAssemblerARM::ma_vsqrt(FloatRegister src, FloatRegister dest,
1334 Condition cc) {
1335 as_vsqrt(dest, src, cc);
1336 }
1337
ma_vsqrt_f32(FloatRegister src,FloatRegister dest,Condition cc)1338 void MacroAssemblerARM::ma_vsqrt_f32(FloatRegister src, FloatRegister dest,
1339 Condition cc) {
1340 as_vsqrt(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1341 cc);
1342 }
1343
DoubleHighWord(double d)1344 static inline uint32_t DoubleHighWord(double d) {
1345 return static_cast<uint32_t>(BitwiseCast<uint64_t>(d) >> 32);
1346 }
1347
DoubleLowWord(double d)1348 static inline uint32_t DoubleLowWord(double d) {
1349 return static_cast<uint32_t>(BitwiseCast<uint64_t>(d)) & uint32_t(0xffffffff);
1350 }
1351
ma_vimm(double value,FloatRegister dest,Condition cc)1352 void MacroAssemblerARM::ma_vimm(double value, FloatRegister dest,
1353 Condition cc) {
1354 if (HasVFPv3()) {
1355 if (DoubleLowWord(value) == 0) {
1356 if (DoubleHighWord(value) == 0) {
1357 // To zero a register, load 1.0, then execute dN <- dN - dN
1358 as_vimm(dest, VFPImm::One, cc);
1359 as_vsub(dest, dest, dest, cc);
1360 return;
1361 }
1362
1363 VFPImm enc(DoubleHighWord(value));
1364 if (enc.isValid()) {
1365 as_vimm(dest, enc, cc);
1366 return;
1367 }
1368 }
1369 }
1370 // Fall back to putting the value in a pool.
1371 as_FImm64Pool(dest, value, cc);
1372 }
1373
ma_vimm_f32(float value,FloatRegister dest,Condition cc)1374 void MacroAssemblerARM::ma_vimm_f32(float value, FloatRegister dest,
1375 Condition cc) {
1376 VFPRegister vd = VFPRegister(dest).singleOverlay();
1377 if (HasVFPv3()) {
1378 if (IsPositiveZero(value)) {
1379 // To zero a register, load 1.0, then execute sN <- sN - sN.
1380 as_vimm(vd, VFPImm::One, cc);
1381 as_vsub(vd, vd, vd, cc);
1382 return;
1383 }
1384
1385 // Note that the vimm immediate float32 instruction encoding differs
1386 // from the vimm immediate double encoding, but this difference matches
1387 // the difference in the floating point formats, so it is possible to
1388 // convert the float32 to a double and then use the double encoding
1389 // paths. It is still necessary to firstly check that the double low
1390 // word is zero because some float32 numbers set these bits and this can
1391 // not be ignored.
1392 double doubleValue(value);
1393 if (DoubleLowWord(doubleValue) == 0) {
1394 VFPImm enc(DoubleHighWord(doubleValue));
1395 if (enc.isValid()) {
1396 as_vimm(vd, enc, cc);
1397 return;
1398 }
1399 }
1400 }
1401
1402 // Fall back to putting the value in a pool.
1403 as_FImm32Pool(vd, value, cc);
1404 }
1405
ma_vcmp(FloatRegister src1,FloatRegister src2,Condition cc)1406 void MacroAssemblerARM::ma_vcmp(FloatRegister src1, FloatRegister src2,
1407 Condition cc) {
1408 as_vcmp(VFPRegister(src1), VFPRegister(src2), cc);
1409 }
1410
ma_vcmp_f32(FloatRegister src1,FloatRegister src2,Condition cc)1411 void MacroAssemblerARM::ma_vcmp_f32(FloatRegister src1, FloatRegister src2,
1412 Condition cc) {
1413 as_vcmp(VFPRegister(src1).singleOverlay(), VFPRegister(src2).singleOverlay(),
1414 cc);
1415 }
1416
ma_vcmpz(FloatRegister src1,Condition cc)1417 void MacroAssemblerARM::ma_vcmpz(FloatRegister src1, Condition cc) {
1418 as_vcmpz(VFPRegister(src1), cc);
1419 }
1420
ma_vcmpz_f32(FloatRegister src1,Condition cc)1421 void MacroAssemblerARM::ma_vcmpz_f32(FloatRegister src1, Condition cc) {
1422 as_vcmpz(VFPRegister(src1).singleOverlay(), cc);
1423 }
1424
ma_vcvt_F64_I32(FloatRegister src,FloatRegister dest,Condition cc)1425 void MacroAssemblerARM::ma_vcvt_F64_I32(FloatRegister src, FloatRegister dest,
1426 Condition cc) {
1427 MOZ_ASSERT(src.isDouble());
1428 MOZ_ASSERT(dest.isSInt());
1429 as_vcvt(dest, src, false, cc);
1430 }
1431
ma_vcvt_F64_U32(FloatRegister src,FloatRegister dest,Condition cc)1432 void MacroAssemblerARM::ma_vcvt_F64_U32(FloatRegister src, FloatRegister dest,
1433 Condition cc) {
1434 MOZ_ASSERT(src.isDouble());
1435 MOZ_ASSERT(dest.isUInt());
1436 as_vcvt(dest, src, false, cc);
1437 }
1438
ma_vcvt_I32_F64(FloatRegister src,FloatRegister dest,Condition cc)1439 void MacroAssemblerARM::ma_vcvt_I32_F64(FloatRegister src, FloatRegister dest,
1440 Condition cc) {
1441 MOZ_ASSERT(src.isSInt());
1442 MOZ_ASSERT(dest.isDouble());
1443 as_vcvt(dest, src, false, cc);
1444 }
1445
ma_vcvt_U32_F64(FloatRegister src,FloatRegister dest,Condition cc)1446 void MacroAssemblerARM::ma_vcvt_U32_F64(FloatRegister src, FloatRegister dest,
1447 Condition cc) {
1448 MOZ_ASSERT(src.isUInt());
1449 MOZ_ASSERT(dest.isDouble());
1450 as_vcvt(dest, src, false, cc);
1451 }
1452
ma_vcvt_F32_I32(FloatRegister src,FloatRegister dest,Condition cc)1453 void MacroAssemblerARM::ma_vcvt_F32_I32(FloatRegister src, FloatRegister dest,
1454 Condition cc) {
1455 MOZ_ASSERT(src.isSingle());
1456 MOZ_ASSERT(dest.isSInt());
1457 as_vcvt(VFPRegister(dest).sintOverlay(), VFPRegister(src).singleOverlay(),
1458 false, cc);
1459 }
1460
ma_vcvt_F32_U32(FloatRegister src,FloatRegister dest,Condition cc)1461 void MacroAssemblerARM::ma_vcvt_F32_U32(FloatRegister src, FloatRegister dest,
1462 Condition cc) {
1463 MOZ_ASSERT(src.isSingle());
1464 MOZ_ASSERT(dest.isUInt());
1465 as_vcvt(VFPRegister(dest).uintOverlay(), VFPRegister(src).singleOverlay(),
1466 false, cc);
1467 }
1468
ma_vcvt_I32_F32(FloatRegister src,FloatRegister dest,Condition cc)1469 void MacroAssemblerARM::ma_vcvt_I32_F32(FloatRegister src, FloatRegister dest,
1470 Condition cc) {
1471 MOZ_ASSERT(src.isSInt());
1472 MOZ_ASSERT(dest.isSingle());
1473 as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).sintOverlay(),
1474 false, cc);
1475 }
1476
ma_vcvt_U32_F32(FloatRegister src,FloatRegister dest,Condition cc)1477 void MacroAssemblerARM::ma_vcvt_U32_F32(FloatRegister src, FloatRegister dest,
1478 Condition cc) {
1479 MOZ_ASSERT(src.isUInt());
1480 MOZ_ASSERT(dest.isSingle());
1481 as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).uintOverlay(),
1482 false, cc);
1483 }
1484
ma_vxfer(FloatRegister src,Register dest,Condition cc)1485 void MacroAssemblerARM::ma_vxfer(FloatRegister src, Register dest,
1486 Condition cc) {
1487 as_vxfer(dest, InvalidReg, VFPRegister(src).singleOverlay(), FloatToCore, cc);
1488 }
1489
ma_vxfer(FloatRegister src,Register dest1,Register dest2,Condition cc)1490 void MacroAssemblerARM::ma_vxfer(FloatRegister src, Register dest1,
1491 Register dest2, Condition cc) {
1492 as_vxfer(dest1, dest2, VFPRegister(src), FloatToCore, cc);
1493 }
1494
ma_vxfer(Register src,FloatRegister dest,Condition cc)1495 void MacroAssemblerARM::ma_vxfer(Register src, FloatRegister dest,
1496 Condition cc) {
1497 as_vxfer(src, InvalidReg, VFPRegister(dest).singleOverlay(), CoreToFloat, cc);
1498 }
1499
ma_vxfer(Register src1,Register src2,FloatRegister dest,Condition cc)1500 void MacroAssemblerARM::ma_vxfer(Register src1, Register src2,
1501 FloatRegister dest, Condition cc) {
1502 as_vxfer(src1, src2, VFPRegister(dest), CoreToFloat, cc);
1503 }
1504
ma_vdtr(LoadStore ls,const Address & addr,VFPRegister rt,AutoRegisterScope & scratch,Condition cc)1505 BufferOffset MacroAssemblerARM::ma_vdtr(LoadStore ls, const Address& addr,
1506 VFPRegister rt,
1507 AutoRegisterScope& scratch,
1508 Condition cc) {
1509 int off = addr.offset;
1510 MOZ_ASSERT((off & 3) == 0);
1511 Register base = addr.base;
1512 if (off > -1024 && off < 1024)
1513 return as_vdtr(ls, rt, Operand(addr).toVFPAddr(), cc);
1514
1515 // We cannot encode this offset in a a single ldr. Try to encode it as an
1516 // add scratch, base, imm; ldr dest, [scratch, +offset].
1517 int bottom = off & (0xff << 2);
1518 int neg_bottom = (0x100 << 2) - bottom;
1519 // At this point, both off - bottom and off + neg_bottom will be
1520 // reasonable-ish quantities.
1521 //
1522 // Note a neg_bottom of 0x400 can not be encoded as an immediate negative
1523 // offset in the instruction and this occurs when bottom is zero, so this
1524 // case is guarded against below.
1525 if (off < 0) {
1526 // sub_off = bottom - off
1527 Operand2 sub_off = Imm8(-(off - bottom));
1528 if (!sub_off.invalid()) {
1529 // - sub_off = off - bottom
1530 as_sub(scratch, base, sub_off, LeaveCC, cc);
1531 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(bottom)), cc);
1532 }
1533 // sub_off = -neg_bottom - off
1534 sub_off = Imm8(-(off + neg_bottom));
1535 if (!sub_off.invalid() && bottom != 0) {
1536 // Guarded against by: bottom != 0
1537 MOZ_ASSERT(neg_bottom < 0x400);
1538 // - sub_off = neg_bottom + off
1539 as_sub(scratch, base, sub_off, LeaveCC, cc);
1540 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(-neg_bottom)), cc);
1541 }
1542 } else {
1543 // sub_off = off - bottom
1544 Operand2 sub_off = Imm8(off - bottom);
1545 if (!sub_off.invalid()) {
1546 // sub_off = off - bottom
1547 as_add(scratch, base, sub_off, LeaveCC, cc);
1548 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(bottom)), cc);
1549 }
1550 // sub_off = neg_bottom + off
1551 sub_off = Imm8(off + neg_bottom);
1552 if (!sub_off.invalid() && bottom != 0) {
1553 // Guarded against by: bottom != 0
1554 MOZ_ASSERT(neg_bottom < 0x400);
1555 // sub_off = neg_bottom + off
1556 as_add(scratch, base, sub_off, LeaveCC, cc);
1557 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(-neg_bottom)), cc);
1558 }
1559 }
1560
1561 // Safe to use scratch as dest, since ma_add() overwrites dest at the end
1562 // and can't use it as internal scratch since it may also == base.
1563 ma_add(base, Imm32(off), scratch, scratch, LeaveCC, cc);
1564 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(0)), cc);
1565 }
1566
ma_vldr(VFPAddr addr,VFPRegister dest,Condition cc)1567 BufferOffset MacroAssemblerARM::ma_vldr(VFPAddr addr, VFPRegister dest,
1568 Condition cc) {
1569 return as_vdtr(IsLoad, dest, addr, cc);
1570 }
1571
ma_vldr(const Address & addr,VFPRegister dest,AutoRegisterScope & scratch,Condition cc)1572 BufferOffset MacroAssemblerARM::ma_vldr(const Address& addr, VFPRegister dest,
1573 AutoRegisterScope& scratch,
1574 Condition cc) {
1575 return ma_vdtr(IsLoad, addr, dest, scratch, cc);
1576 }
1577
ma_vldr(VFPRegister src,Register base,Register index,AutoRegisterScope & scratch,int32_t shift,Condition cc)1578 BufferOffset MacroAssemblerARM::ma_vldr(VFPRegister src, Register base,
1579 Register index,
1580 AutoRegisterScope& scratch,
1581 int32_t shift, Condition cc) {
1582 as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
1583 return as_vdtr(IsLoad, src, Operand(Address(scratch, 0)).toVFPAddr(), cc);
1584 }
1585
ma_vstr(VFPRegister src,VFPAddr addr,Condition cc)1586 BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, VFPAddr addr,
1587 Condition cc) {
1588 return as_vdtr(IsStore, src, addr, cc);
1589 }
1590
ma_vstr(VFPRegister src,const Address & addr,AutoRegisterScope & scratch,Condition cc)1591 BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, const Address& addr,
1592 AutoRegisterScope& scratch,
1593 Condition cc) {
1594 return ma_vdtr(IsStore, addr, src, scratch, cc);
1595 }
1596
ma_vstr(VFPRegister src,Register base,Register index,AutoRegisterScope & scratch,AutoRegisterScope & scratch2,int32_t shift,int32_t offset,Condition cc)1597 BufferOffset MacroAssemblerARM::ma_vstr(
1598 VFPRegister src, Register base, Register index, AutoRegisterScope& scratch,
1599 AutoRegisterScope& scratch2, int32_t shift, int32_t offset, Condition cc) {
1600 as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
1601 return ma_vstr(src, Address(scratch, offset), scratch2, cc);
1602 }
1603
1604 // Without an offset, no second scratch register is necessary.
ma_vstr(VFPRegister src,Register base,Register index,AutoRegisterScope & scratch,int32_t shift,Condition cc)1605 BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, Register base,
1606 Register index,
1607 AutoRegisterScope& scratch,
1608 int32_t shift, Condition cc) {
1609 as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
1610 return as_vdtr(IsStore, src, Operand(Address(scratch, 0)).toVFPAddr(), cc);
1611 }
1612
buildOOLFakeExitFrame(void * fakeReturnAddr)1613 bool MacroAssemblerARMCompat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
1614 DebugOnly<uint32_t> initialDepth = asMasm().framePushed();
1615 uint32_t descriptor = MakeFrameDescriptor(
1616 asMasm().framePushed(), JitFrame_IonJS, ExitFrameLayout::Size());
1617
1618 asMasm().Push(Imm32(descriptor)); // descriptor_
1619 asMasm().Push(ImmPtr(fakeReturnAddr));
1620
1621 return true;
1622 }
1623
move32(Imm32 imm,Register dest)1624 void MacroAssemblerARMCompat::move32(Imm32 imm, Register dest) {
1625 ma_mov(imm, dest);
1626 }
1627
move32(Register src,Register dest)1628 void MacroAssemblerARMCompat::move32(Register src, Register dest) {
1629 ma_mov(src, dest);
1630 }
1631
movePtr(Register src,Register dest)1632 void MacroAssemblerARMCompat::movePtr(Register src, Register dest) {
1633 ma_mov(src, dest);
1634 }
1635
movePtr(ImmWord imm,Register dest)1636 void MacroAssemblerARMCompat::movePtr(ImmWord imm, Register dest) {
1637 ma_mov(Imm32(imm.value), dest);
1638 }
1639
movePtr(ImmGCPtr imm,Register dest)1640 void MacroAssemblerARMCompat::movePtr(ImmGCPtr imm, Register dest) {
1641 ma_mov(imm, dest);
1642 }
1643
movePtr(ImmPtr imm,Register dest)1644 void MacroAssemblerARMCompat::movePtr(ImmPtr imm, Register dest) {
1645 movePtr(ImmWord(uintptr_t(imm.value)), dest);
1646 }
1647
movePtr(wasm::SymbolicAddress imm,Register dest)1648 void MacroAssemblerARMCompat::movePtr(wasm::SymbolicAddress imm,
1649 Register dest) {
1650 append(wasm::SymbolicAccess(CodeOffset(currentOffset()), imm));
1651 ma_movPatchable(Imm32(-1), dest, Always);
1652 }
1653
load8ZeroExtend(const Address & address,Register dest)1654 void MacroAssemblerARMCompat::load8ZeroExtend(const Address& address,
1655 Register dest) {
1656 ScratchRegisterScope scratch(asMasm());
1657 ma_dataTransferN(IsLoad, 8, false, address.base, Imm32(address.offset), dest,
1658 scratch);
1659 }
1660
load8ZeroExtend(const BaseIndex & src,Register dest)1661 void MacroAssemblerARMCompat::load8ZeroExtend(const BaseIndex& src,
1662 Register dest) {
1663 Register base = src.base;
1664 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1665
1666 ScratchRegisterScope scratch(asMasm());
1667 SecondScratchRegisterScope scratch2(asMasm());
1668
1669 if (src.offset == 0) {
1670 ma_ldrb(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), dest);
1671 } else {
1672 ma_add(base, Imm32(src.offset), scratch, scratch2);
1673 ma_ldrb(DTRAddr(scratch, DtrRegImmShift(src.index, LSL, scale)), dest);
1674 }
1675 }
1676
load8SignExtend(const Address & address,Register dest)1677 void MacroAssemblerARMCompat::load8SignExtend(const Address& address,
1678 Register dest) {
1679 ScratchRegisterScope scratch(asMasm());
1680 ma_dataTransferN(IsLoad, 8, true, address.base, Imm32(address.offset), dest,
1681 scratch);
1682 }
1683
load8SignExtend(const BaseIndex & src,Register dest)1684 void MacroAssemblerARMCompat::load8SignExtend(const BaseIndex& src,
1685 Register dest) {
1686 Register index = src.index;
1687
1688 ScratchRegisterScope scratch(asMasm());
1689 SecondScratchRegisterScope scratch2(asMasm());
1690
1691 // ARMv7 does not have LSL on an index register with an extended load.
1692 if (src.scale != TimesOne) {
1693 ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
1694 index = scratch;
1695 }
1696
1697 if (src.offset != 0) {
1698 if (index != scratch) {
1699 ma_mov(index, scratch);
1700 index = scratch;
1701 }
1702 ma_add(Imm32(src.offset), index, scratch2);
1703 }
1704 ma_ldrsb(EDtrAddr(src.base, EDtrOffReg(index)), dest);
1705 }
1706
load16ZeroExtend(const Address & address,Register dest)1707 void MacroAssemblerARMCompat::load16ZeroExtend(const Address& address,
1708 Register dest) {
1709 ScratchRegisterScope scratch(asMasm());
1710 ma_dataTransferN(IsLoad, 16, false, address.base, Imm32(address.offset), dest,
1711 scratch);
1712 }
1713
load16ZeroExtend(const BaseIndex & src,Register dest)1714 void MacroAssemblerARMCompat::load16ZeroExtend(const BaseIndex& src,
1715 Register dest) {
1716 Register index = src.index;
1717
1718 ScratchRegisterScope scratch(asMasm());
1719 SecondScratchRegisterScope scratch2(asMasm());
1720
1721 // ARMv7 does not have LSL on an index register with an extended load.
1722 if (src.scale != TimesOne) {
1723 ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
1724 index = scratch;
1725 }
1726
1727 if (src.offset != 0) {
1728 if (index != scratch) {
1729 ma_mov(index, scratch);
1730 index = scratch;
1731 }
1732 ma_add(Imm32(src.offset), index, scratch2);
1733 }
1734 ma_ldrh(EDtrAddr(src.base, EDtrOffReg(index)), dest);
1735 }
1736
load16SignExtend(const Address & address,Register dest)1737 void MacroAssemblerARMCompat::load16SignExtend(const Address& address,
1738 Register dest) {
1739 ScratchRegisterScope scratch(asMasm());
1740 ma_dataTransferN(IsLoad, 16, true, address.base, Imm32(address.offset), dest,
1741 scratch);
1742 }
1743
load16SignExtend(const BaseIndex & src,Register dest)1744 void MacroAssemblerARMCompat::load16SignExtend(const BaseIndex& src,
1745 Register dest) {
1746 Register index = src.index;
1747
1748 ScratchRegisterScope scratch(asMasm());
1749 SecondScratchRegisterScope scratch2(asMasm());
1750
1751 // We don't have LSL on index register yet.
1752 if (src.scale != TimesOne) {
1753 ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
1754 index = scratch;
1755 }
1756
1757 if (src.offset != 0) {
1758 if (index != scratch) {
1759 ma_mov(index, scratch);
1760 index = scratch;
1761 }
1762 ma_add(Imm32(src.offset), index, scratch2);
1763 }
1764 ma_ldrsh(EDtrAddr(src.base, EDtrOffReg(index)), dest);
1765 }
1766
load32(const Address & address,Register dest)1767 void MacroAssemblerARMCompat::load32(const Address& address, Register dest) {
1768 loadPtr(address, dest);
1769 }
1770
load32(const BaseIndex & address,Register dest)1771 void MacroAssemblerARMCompat::load32(const BaseIndex& address, Register dest) {
1772 loadPtr(address, dest);
1773 }
1774
load32(AbsoluteAddress address,Register dest)1775 void MacroAssemblerARMCompat::load32(AbsoluteAddress address, Register dest) {
1776 loadPtr(address, dest);
1777 }
1778
loadPtr(const Address & address,Register dest)1779 void MacroAssemblerARMCompat::loadPtr(const Address& address, Register dest) {
1780 ScratchRegisterScope scratch(asMasm());
1781 ma_ldr(address, dest, scratch);
1782 }
1783
loadPtr(const BaseIndex & src,Register dest)1784 void MacroAssemblerARMCompat::loadPtr(const BaseIndex& src, Register dest) {
1785 Register base = src.base;
1786 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1787
1788 ScratchRegisterScope scratch(asMasm());
1789 SecondScratchRegisterScope scratch2(asMasm());
1790
1791 if (src.offset != 0) {
1792 ma_add(base, Imm32(src.offset), scratch, scratch2);
1793 ma_ldr(DTRAddr(scratch, DtrRegImmShift(src.index, LSL, scale)), dest);
1794 } else {
1795 ma_ldr(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), dest);
1796 }
1797 }
1798
loadPtr(AbsoluteAddress address,Register dest)1799 void MacroAssemblerARMCompat::loadPtr(AbsoluteAddress address, Register dest) {
1800 MOZ_ASSERT(dest != pc); // Use dest as a scratch register.
1801 movePtr(ImmWord(uintptr_t(address.addr)), dest);
1802 loadPtr(Address(dest, 0), dest);
1803 }
1804
loadPtr(wasm::SymbolicAddress address,Register dest)1805 void MacroAssemblerARMCompat::loadPtr(wasm::SymbolicAddress address,
1806 Register dest) {
1807 MOZ_ASSERT(dest != pc); // Use dest as a scratch register.
1808 movePtr(address, dest);
1809 loadPtr(Address(dest, 0), dest);
1810 }
1811
loadPrivate(const Address & address,Register dest)1812 void MacroAssemblerARMCompat::loadPrivate(const Address& address,
1813 Register dest) {
1814 ScratchRegisterScope scratch(asMasm());
1815 ma_ldr(ToPayload(address), dest, scratch);
1816 }
1817
loadDouble(const Address & address,FloatRegister dest)1818 void MacroAssemblerARMCompat::loadDouble(const Address& address,
1819 FloatRegister dest) {
1820 ScratchRegisterScope scratch(asMasm());
1821 ma_vldr(address, dest, scratch);
1822 }
1823
loadDouble(const BaseIndex & src,FloatRegister dest)1824 void MacroAssemblerARMCompat::loadDouble(const BaseIndex& src,
1825 FloatRegister dest) {
1826 // VFP instructions don't even support register Base + register Index modes,
1827 // so just add the index, then handle the offset like normal.
1828 Register base = src.base;
1829 Register index = src.index;
1830 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1831 int32_t offset = src.offset;
1832
1833 ScratchRegisterScope scratch(asMasm());
1834 SecondScratchRegisterScope scratch2(asMasm());
1835
1836 as_add(scratch, base, lsl(index, scale));
1837 ma_vldr(Address(scratch, offset), dest, scratch2);
1838 }
1839
loadFloatAsDouble(const Address & address,FloatRegister dest)1840 void MacroAssemblerARMCompat::loadFloatAsDouble(const Address& address,
1841 FloatRegister dest) {
1842 ScratchRegisterScope scratch(asMasm());
1843
1844 VFPRegister rt = dest;
1845 ma_vldr(address, rt.singleOverlay(), scratch);
1846 as_vcvt(rt, rt.singleOverlay());
1847 }
1848
loadFloatAsDouble(const BaseIndex & src,FloatRegister dest)1849 void MacroAssemblerARMCompat::loadFloatAsDouble(const BaseIndex& src,
1850 FloatRegister dest) {
1851 // VFP instructions don't even support register Base + register Index modes,
1852 // so just add the index, then handle the offset like normal.
1853 Register base = src.base;
1854 Register index = src.index;
1855 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1856 int32_t offset = src.offset;
1857 VFPRegister rt = dest;
1858
1859 ScratchRegisterScope scratch(asMasm());
1860 SecondScratchRegisterScope scratch2(asMasm());
1861
1862 as_add(scratch, base, lsl(index, scale));
1863 ma_vldr(Address(scratch, offset), rt.singleOverlay(), scratch2);
1864 as_vcvt(rt, rt.singleOverlay());
1865 }
1866
loadFloat32(const Address & address,FloatRegister dest)1867 void MacroAssemblerARMCompat::loadFloat32(const Address& address,
1868 FloatRegister dest) {
1869 ScratchRegisterScope scratch(asMasm());
1870 ma_vldr(address, VFPRegister(dest).singleOverlay(), scratch);
1871 }
1872
loadFloat32(const BaseIndex & src,FloatRegister dest)1873 void MacroAssemblerARMCompat::loadFloat32(const BaseIndex& src,
1874 FloatRegister dest) {
1875 // VFP instructions don't even support register Base + register Index modes,
1876 // so just add the index, then handle the offset like normal.
1877 Register base = src.base;
1878 Register index = src.index;
1879 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1880 int32_t offset = src.offset;
1881
1882 ScratchRegisterScope scratch(asMasm());
1883 SecondScratchRegisterScope scratch2(asMasm());
1884
1885 as_add(scratch, base, lsl(index, scale));
1886 ma_vldr(Address(scratch, offset), VFPRegister(dest).singleOverlay(),
1887 scratch2);
1888 }
1889
store8(Imm32 imm,const Address & address)1890 void MacroAssemblerARMCompat::store8(Imm32 imm, const Address& address) {
1891 SecondScratchRegisterScope scratch2(asMasm());
1892 ma_mov(imm, scratch2);
1893 store8(scratch2, address);
1894 }
1895
store8(Register src,const Address & address)1896 void MacroAssemblerARMCompat::store8(Register src, const Address& address) {
1897 ScratchRegisterScope scratch(asMasm());
1898 ma_dataTransferN(IsStore, 8, false, address.base, Imm32(address.offset), src,
1899 scratch);
1900 }
1901
store8(Imm32 imm,const BaseIndex & dest)1902 void MacroAssemblerARMCompat::store8(Imm32 imm, const BaseIndex& dest) {
1903 Register base = dest.base;
1904 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
1905
1906 ScratchRegisterScope scratch(asMasm());
1907 SecondScratchRegisterScope scratch2(asMasm());
1908
1909 if (dest.offset != 0) {
1910 ma_add(base, Imm32(dest.offset), scratch, scratch2);
1911 ma_mov(imm, scratch2);
1912 ma_strb(scratch2, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
1913 } else {
1914 ma_mov(imm, scratch2);
1915 ma_strb(scratch2, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
1916 }
1917 }
1918
store8(Register src,const BaseIndex & dest)1919 void MacroAssemblerARMCompat::store8(Register src, const BaseIndex& dest) {
1920 Register base = dest.base;
1921 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
1922
1923 ScratchRegisterScope scratch(asMasm());
1924 SecondScratchRegisterScope scratch2(asMasm());
1925
1926 if (dest.offset != 0) {
1927 ma_add(base, Imm32(dest.offset), scratch, scratch2);
1928 ma_strb(src, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
1929 } else {
1930 ma_strb(src, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
1931 }
1932 }
1933
store16(Imm32 imm,const Address & address)1934 void MacroAssemblerARMCompat::store16(Imm32 imm, const Address& address) {
1935 SecondScratchRegisterScope scratch2(asMasm());
1936 ma_mov(imm, scratch2);
1937 store16(scratch2, address);
1938 }
1939
store16(Register src,const Address & address)1940 void MacroAssemblerARMCompat::store16(Register src, const Address& address) {
1941 ScratchRegisterScope scratch(asMasm());
1942 ma_dataTransferN(IsStore, 16, false, address.base, Imm32(address.offset), src,
1943 scratch);
1944 }
1945
store16(Imm32 imm,const BaseIndex & dest)1946 void MacroAssemblerARMCompat::store16(Imm32 imm, const BaseIndex& dest) {
1947 Register index = dest.index;
1948
1949 ScratchRegisterScope scratch(asMasm());
1950 SecondScratchRegisterScope scratch2(asMasm());
1951
1952 // We don't have LSL on index register yet.
1953 if (dest.scale != TimesOne) {
1954 ma_lsl(Imm32::ShiftOf(dest.scale), index, scratch);
1955 index = scratch;
1956 }
1957
1958 if (dest.offset != 0) {
1959 ma_add(index, Imm32(dest.offset), scratch, scratch2);
1960 index = scratch;
1961 }
1962
1963 ma_mov(imm, scratch2);
1964 ma_strh(scratch2, EDtrAddr(dest.base, EDtrOffReg(index)));
1965 }
1966
store16(Register src,const BaseIndex & address)1967 void MacroAssemblerARMCompat::store16(Register src, const BaseIndex& address) {
1968 Register index = address.index;
1969
1970 ScratchRegisterScope scratch(asMasm());
1971 SecondScratchRegisterScope scratch2(asMasm());
1972
1973 // We don't have LSL on index register yet.
1974 if (address.scale != TimesOne) {
1975 ma_lsl(Imm32::ShiftOf(address.scale), index, scratch);
1976 index = scratch;
1977 }
1978
1979 if (address.offset != 0) {
1980 ma_add(index, Imm32(address.offset), scratch, scratch2);
1981 index = scratch;
1982 }
1983 ma_strh(src, EDtrAddr(address.base, EDtrOffReg(index)));
1984 }
1985
store32(Register src,AbsoluteAddress address)1986 void MacroAssemblerARMCompat::store32(Register src, AbsoluteAddress address) {
1987 storePtr(src, address);
1988 }
1989
store32(Register src,const Address & address)1990 void MacroAssemblerARMCompat::store32(Register src, const Address& address) {
1991 storePtr(src, address);
1992 }
1993
store32(Imm32 src,const Address & address)1994 void MacroAssemblerARMCompat::store32(Imm32 src, const Address& address) {
1995 ScratchRegisterScope scratch(asMasm());
1996 SecondScratchRegisterScope scratch2(asMasm());
1997 move32(src, scratch);
1998 ma_str(scratch, address, scratch2);
1999 }
2000
store32(Imm32 imm,const BaseIndex & dest)2001 void MacroAssemblerARMCompat::store32(Imm32 imm, const BaseIndex& dest) {
2002 Register base = dest.base;
2003 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
2004
2005 ScratchRegisterScope scratch(asMasm());
2006 SecondScratchRegisterScope scratch2(asMasm());
2007
2008 if (dest.offset != 0) {
2009 ma_add(base, Imm32(dest.offset), scratch, scratch2);
2010 ma_mov(imm, scratch2);
2011 ma_str(scratch2, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
2012 } else {
2013 ma_mov(imm, scratch);
2014 ma_str(scratch, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
2015 }
2016 }
2017
store32(Register src,const BaseIndex & dest)2018 void MacroAssemblerARMCompat::store32(Register src, const BaseIndex& dest) {
2019 Register base = dest.base;
2020 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
2021
2022 ScratchRegisterScope scratch(asMasm());
2023 SecondScratchRegisterScope scratch2(asMasm());
2024
2025 if (dest.offset != 0) {
2026 ma_add(base, Imm32(dest.offset), scratch, scratch2);
2027 ma_str(src, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
2028 } else {
2029 ma_str(src, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
2030 }
2031 }
2032
storePtr(ImmWord imm,const Address & address)2033 void MacroAssemblerARMCompat::storePtr(ImmWord imm, const Address& address) {
2034 store32(Imm32(imm.value), address);
2035 }
2036
storePtr(ImmWord imm,const BaseIndex & address)2037 void MacroAssemblerARMCompat::storePtr(ImmWord imm, const BaseIndex& address) {
2038 store32(Imm32(imm.value), address);
2039 }
2040
storePtr(ImmPtr imm,const Address & address)2041 void MacroAssemblerARMCompat::storePtr(ImmPtr imm, const Address& address) {
2042 store32(Imm32(uintptr_t(imm.value)), address);
2043 }
2044
storePtr(ImmPtr imm,const BaseIndex & address)2045 void MacroAssemblerARMCompat::storePtr(ImmPtr imm, const BaseIndex& address) {
2046 store32(Imm32(uintptr_t(imm.value)), address);
2047 }
2048
storePtr(ImmGCPtr imm,const Address & address)2049 void MacroAssemblerARMCompat::storePtr(ImmGCPtr imm, const Address& address) {
2050 ScratchRegisterScope scratch(asMasm());
2051 SecondScratchRegisterScope scratch2(asMasm());
2052 ma_mov(imm, scratch);
2053 ma_str(scratch, address, scratch2);
2054 }
2055
storePtr(ImmGCPtr imm,const BaseIndex & address)2056 void MacroAssemblerARMCompat::storePtr(ImmGCPtr imm, const BaseIndex& address) {
2057 Register base = address.base;
2058 uint32_t scale = Imm32::ShiftOf(address.scale).value;
2059
2060 ScratchRegisterScope scratch(asMasm());
2061 SecondScratchRegisterScope scratch2(asMasm());
2062
2063 if (address.offset != 0) {
2064 ma_add(base, Imm32(address.offset), scratch, scratch2);
2065 ma_mov(imm, scratch2);
2066 ma_str(scratch2,
2067 DTRAddr(scratch, DtrRegImmShift(address.index, LSL, scale)));
2068 } else {
2069 ma_mov(imm, scratch);
2070 ma_str(scratch, DTRAddr(base, DtrRegImmShift(address.index, LSL, scale)));
2071 }
2072 }
2073
storePtr(Register src,const Address & address)2074 void MacroAssemblerARMCompat::storePtr(Register src, const Address& address) {
2075 SecondScratchRegisterScope scratch2(asMasm());
2076 ma_str(src, address, scratch2);
2077 }
2078
storePtr(Register src,const BaseIndex & address)2079 void MacroAssemblerARMCompat::storePtr(Register src, const BaseIndex& address) {
2080 store32(src, address);
2081 }
2082
storePtr(Register src,AbsoluteAddress dest)2083 void MacroAssemblerARMCompat::storePtr(Register src, AbsoluteAddress dest) {
2084 ScratchRegisterScope scratch(asMasm());
2085 movePtr(ImmWord(uintptr_t(dest.addr)), scratch);
2086 ma_str(src, DTRAddr(scratch, DtrOffImm(0)));
2087 }
2088
2089 // Note: this function clobbers the input register.
clampDoubleToUint8(FloatRegister input,Register output)2090 void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
2091 if (HasVFPv3()) {
2092 Label notSplit;
2093 {
2094 ScratchDoubleScope scratchDouble(*this);
2095 MOZ_ASSERT(input != scratchDouble);
2096 loadConstantDouble(0.5, scratchDouble);
2097
2098 ma_vadd(input, scratchDouble, scratchDouble);
2099 // Convert the double into an unsigned fixed point value with 24 bits of
2100 // precision. The resulting number will look like 0xII.DDDDDD
2101 as_vcvtFixed(scratchDouble, false, 24, true);
2102 }
2103
2104 // Move the fixed point value into an integer register.
2105 {
2106 ScratchFloat32Scope scratchFloat(*this);
2107 as_vxfer(output, InvalidReg, scratchFloat.uintOverlay(), FloatToCore);
2108 }
2109
2110 ScratchRegisterScope scratch(*this);
2111
2112 // See if this value *might* have been an exact integer after adding
2113 // 0.5. This tests the 1/2 through 1/16,777,216th places, but 0.5 needs
2114 // to be tested out to the 1/140,737,488,355,328th place.
2115 ma_tst(output, Imm32(0x00ffffff), scratch);
2116 // Convert to a uint8 by shifting out all of the fraction bits.
2117 ma_lsr(Imm32(24), output, output);
2118 // If any of the bottom 24 bits were non-zero, then we're good, since
2119 // this number can't be exactly XX.0
2120 ma_b(¬Split, NonZero);
2121 as_vxfer(scratch, InvalidReg, input, FloatToCore);
2122 as_cmp(scratch, Imm8(0));
2123 // If the lower 32 bits of the double were 0, then this was an exact number,
2124 // and it should be even.
2125 as_bic(output, output, Imm8(1), LeaveCC, Zero);
2126 bind(¬Split);
2127 } else {
2128 ScratchDoubleScope scratchDouble(*this);
2129 MOZ_ASSERT(input != scratchDouble);
2130 loadConstantDouble(0.5, scratchDouble);
2131
2132 Label outOfRange;
2133 ma_vcmpz(input);
2134 // Do the add, in place so we can reference it later.
2135 ma_vadd(input, scratchDouble, input);
2136 // Do the conversion to an integer.
2137 as_vcvt(VFPRegister(scratchDouble).uintOverlay(), VFPRegister(input));
2138 // Copy the converted value out.
2139 as_vxfer(output, InvalidReg, scratchDouble, FloatToCore);
2140 as_vmrs(pc);
2141 ma_mov(Imm32(0), output, Overflow); // NaN => 0
2142 ma_b(&outOfRange, Overflow); // NaN
2143 as_cmp(output, Imm8(0xff));
2144 ma_mov(Imm32(0xff), output, Above);
2145 ma_b(&outOfRange, Above);
2146 // Convert it back to see if we got the same value back.
2147 as_vcvt(scratchDouble, VFPRegister(scratchDouble).uintOverlay());
2148 // Do the check.
2149 as_vcmp(scratchDouble, input);
2150 as_vmrs(pc);
2151 as_bic(output, output, Imm8(1), LeaveCC, Zero);
2152 bind(&outOfRange);
2153 }
2154 }
2155
cmp32(Register lhs,Imm32 rhs)2156 void MacroAssemblerARMCompat::cmp32(Register lhs, Imm32 rhs) {
2157 ScratchRegisterScope scratch(asMasm());
2158 ma_cmp(lhs, rhs, scratch);
2159 }
2160
cmp32(Register lhs,Register rhs)2161 void MacroAssemblerARMCompat::cmp32(Register lhs, Register rhs) {
2162 ma_cmp(lhs, rhs);
2163 }
2164
cmpPtr(Register lhs,ImmWord rhs)2165 void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmWord rhs) {
2166 cmp32(lhs, Imm32(rhs.value));
2167 }
2168
cmpPtr(Register lhs,ImmPtr rhs)2169 void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmPtr rhs) {
2170 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
2171 }
2172
cmpPtr(Register lhs,Register rhs)2173 void MacroAssemblerARMCompat::cmpPtr(Register lhs, Register rhs) {
2174 ma_cmp(lhs, rhs);
2175 }
2176
cmpPtr(Register lhs,ImmGCPtr rhs)2177 void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmGCPtr rhs) {
2178 ScratchRegisterScope scratch(asMasm());
2179 ma_cmp(lhs, rhs, scratch);
2180 }
2181
cmpPtr(Register lhs,Imm32 rhs)2182 void MacroAssemblerARMCompat::cmpPtr(Register lhs, Imm32 rhs) {
2183 cmp32(lhs, rhs);
2184 }
2185
cmpPtr(const Address & lhs,Register rhs)2186 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, Register rhs) {
2187 ScratchRegisterScope scratch(asMasm());
2188 SecondScratchRegisterScope scratch2(asMasm());
2189 ma_ldr(lhs, scratch, scratch2);
2190 ma_cmp(scratch, rhs);
2191 }
2192
cmpPtr(const Address & lhs,ImmWord rhs)2193 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmWord rhs) {
2194 ScratchRegisterScope scratch(asMasm());
2195 SecondScratchRegisterScope scratch2(asMasm());
2196 ma_ldr(lhs, scratch, scratch2);
2197 ma_cmp(scratch, Imm32(rhs.value), scratch2);
2198 }
2199
cmpPtr(const Address & lhs,ImmPtr rhs)2200 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmPtr rhs) {
2201 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
2202 }
2203
cmpPtr(const Address & lhs,ImmGCPtr rhs)2204 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmGCPtr rhs) {
2205 ScratchRegisterScope scratch(asMasm());
2206 SecondScratchRegisterScope scratch2(asMasm());
2207 ma_ldr(lhs, scratch, scratch2);
2208 ma_cmp(scratch, rhs, scratch2);
2209 }
2210
cmpPtr(const Address & lhs,Imm32 rhs)2211 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, Imm32 rhs) {
2212 ScratchRegisterScope scratch(asMasm());
2213 SecondScratchRegisterScope scratch2(asMasm());
2214 ma_ldr(lhs, scratch, scratch2);
2215 ma_cmp(scratch, rhs, scratch2);
2216 }
2217
setStackArg(Register reg,uint32_t arg)2218 void MacroAssemblerARMCompat::setStackArg(Register reg, uint32_t arg) {
2219 ScratchRegisterScope scratch(asMasm());
2220 ma_dataTransferN(IsStore, 32, true, sp, Imm32(arg * sizeof(intptr_t)), reg,
2221 scratch);
2222 }
2223
minMaxDouble(FloatRegister srcDest,FloatRegister second,bool canBeNaN,bool isMax)2224 void MacroAssemblerARMCompat::minMaxDouble(FloatRegister srcDest,
2225 FloatRegister second, bool canBeNaN,
2226 bool isMax) {
2227 FloatRegister first = srcDest;
2228
2229 Label nan, equal, returnSecond, done;
2230
2231 Assembler::Condition cond = isMax ? Assembler::VFP_LessThanOrEqual
2232 : Assembler::VFP_GreaterThanOrEqual;
2233
2234 compareDouble(first, second);
2235 // First or second is NaN, result is NaN.
2236 ma_b(&nan, Assembler::VFP_Unordered);
2237 // Make sure we handle -0 and 0 right.
2238 ma_b(&equal, Assembler::VFP_Equal);
2239 ma_b(&returnSecond, cond);
2240 ma_b(&done);
2241
2242 // Check for zero.
2243 bind(&equal);
2244 compareDouble(first, NoVFPRegister);
2245 // First wasn't 0 or -0, so just return it.
2246 ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
2247 // So now both operands are either -0 or 0.
2248 if (isMax) {
2249 // -0 + -0 = -0 and -0 + 0 = 0.
2250 ma_vadd(second, first, first);
2251 } else {
2252 ma_vneg(first, first);
2253 ma_vsub(first, second, first);
2254 ma_vneg(first, first);
2255 }
2256 ma_b(&done);
2257
2258 bind(&nan);
2259 // If the first argument is the NaN, return it; otherwise return the second
2260 // operand.
2261 compareDouble(first, first);
2262 ma_vmov(first, srcDest, Assembler::VFP_Unordered);
2263 ma_b(&done, Assembler::VFP_Unordered);
2264
2265 bind(&returnSecond);
2266 ma_vmov(second, srcDest);
2267
2268 bind(&done);
2269 }
2270
minMaxFloat32(FloatRegister srcDest,FloatRegister second,bool canBeNaN,bool isMax)2271 void MacroAssemblerARMCompat::minMaxFloat32(FloatRegister srcDest,
2272 FloatRegister second, bool canBeNaN,
2273 bool isMax) {
2274 FloatRegister first = srcDest;
2275
2276 Label nan, equal, returnSecond, done;
2277
2278 Assembler::Condition cond = isMax ? Assembler::VFP_LessThanOrEqual
2279 : Assembler::VFP_GreaterThanOrEqual;
2280
2281 compareFloat(first, second);
2282 // First or second is NaN, result is NaN.
2283 ma_b(&nan, Assembler::VFP_Unordered);
2284 // Make sure we handle -0 and 0 right.
2285 ma_b(&equal, Assembler::VFP_Equal);
2286 ma_b(&returnSecond, cond);
2287 ma_b(&done);
2288
2289 // Check for zero.
2290 bind(&equal);
2291 compareFloat(first, NoVFPRegister);
2292 // First wasn't 0 or -0, so just return it.
2293 ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
2294 // So now both operands are either -0 or 0.
2295 if (isMax) {
2296 // -0 + -0 = -0 and -0 + 0 = 0.
2297 ma_vadd_f32(second, first, first);
2298 } else {
2299 ma_vneg_f32(first, first);
2300 ma_vsub_f32(first, second, first);
2301 ma_vneg_f32(first, first);
2302 }
2303 ma_b(&done);
2304
2305 bind(&nan);
2306 // See comment in minMaxDouble.
2307 compareFloat(first, first);
2308 ma_vmov_f32(first, srcDest, Assembler::VFP_Unordered);
2309 ma_b(&done, Assembler::VFP_Unordered);
2310
2311 bind(&returnSecond);
2312 ma_vmov_f32(second, srcDest);
2313
2314 bind(&done);
2315 }
2316
compareDouble(FloatRegister lhs,FloatRegister rhs)2317 void MacroAssemblerARMCompat::compareDouble(FloatRegister lhs,
2318 FloatRegister rhs) {
2319 // Compare the doubles, setting vector status flags.
2320 if (rhs.isMissing())
2321 ma_vcmpz(lhs);
2322 else
2323 ma_vcmp(lhs, rhs);
2324
2325 // Move vector status bits to normal status flags.
2326 as_vmrs(pc);
2327 }
2328
compareFloat(FloatRegister lhs,FloatRegister rhs)2329 void MacroAssemblerARMCompat::compareFloat(FloatRegister lhs,
2330 FloatRegister rhs) {
2331 // Compare the doubles, setting vector status flags.
2332 if (rhs.isMissing())
2333 as_vcmpz(VFPRegister(lhs).singleOverlay());
2334 else
2335 as_vcmp(VFPRegister(lhs).singleOverlay(), VFPRegister(rhs).singleOverlay());
2336
2337 // Move vector status bits to normal status flags.
2338 as_vmrs(pc);
2339 }
2340
testInt32(Assembler::Condition cond,const ValueOperand & value)2341 Assembler::Condition MacroAssemblerARMCompat::testInt32(
2342 Assembler::Condition cond, const ValueOperand& value) {
2343 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2344 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_INT32));
2345 return cond;
2346 }
2347
testBoolean(Assembler::Condition cond,const ValueOperand & value)2348 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2349 Assembler::Condition cond, const ValueOperand& value) {
2350 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2351 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_BOOLEAN));
2352 return cond;
2353 }
2354
testDouble(Assembler::Condition cond,const ValueOperand & value)2355 Assembler::Condition MacroAssemblerARMCompat::testDouble(
2356 Assembler::Condition cond, const ValueOperand& value) {
2357 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2358 Assembler::Condition actual = (cond == Equal) ? Below : AboveOrEqual;
2359 ScratchRegisterScope scratch(asMasm());
2360 ma_cmp(value.typeReg(), ImmTag(JSVAL_TAG_CLEAR), scratch);
2361 return actual;
2362 }
2363
testNull(Assembler::Condition cond,const ValueOperand & value)2364 Assembler::Condition MacroAssemblerARMCompat::testNull(
2365 Assembler::Condition cond, const ValueOperand& value) {
2366 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2367 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_NULL));
2368 return cond;
2369 }
2370
testUndefined(Assembler::Condition cond,const ValueOperand & value)2371 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2372 Assembler::Condition cond, const ValueOperand& value) {
2373 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2374 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_UNDEFINED));
2375 return cond;
2376 }
2377
testString(Assembler::Condition cond,const ValueOperand & value)2378 Assembler::Condition MacroAssemblerARMCompat::testString(
2379 Assembler::Condition cond, const ValueOperand& value) {
2380 return testString(cond, value.typeReg());
2381 }
2382
testSymbol(Assembler::Condition cond,const ValueOperand & value)2383 Assembler::Condition MacroAssemblerARMCompat::testSymbol(
2384 Assembler::Condition cond, const ValueOperand& value) {
2385 return testSymbol(cond, value.typeReg());
2386 }
2387
testObject(Assembler::Condition cond,const ValueOperand & value)2388 Assembler::Condition MacroAssemblerARMCompat::testObject(
2389 Assembler::Condition cond, const ValueOperand& value) {
2390 return testObject(cond, value.typeReg());
2391 }
2392
testNumber(Assembler::Condition cond,const ValueOperand & value)2393 Assembler::Condition MacroAssemblerARMCompat::testNumber(
2394 Assembler::Condition cond, const ValueOperand& value) {
2395 return testNumber(cond, value.typeReg());
2396 }
2397
testMagic(Assembler::Condition cond,const ValueOperand & value)2398 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2399 Assembler::Condition cond, const ValueOperand& value) {
2400 return testMagic(cond, value.typeReg());
2401 }
2402
testPrimitive(Assembler::Condition cond,const ValueOperand & value)2403 Assembler::Condition MacroAssemblerARMCompat::testPrimitive(
2404 Assembler::Condition cond, const ValueOperand& value) {
2405 return testPrimitive(cond, value.typeReg());
2406 }
2407
2408 // Register-based tests.
testInt32(Assembler::Condition cond,Register tag)2409 Assembler::Condition MacroAssemblerARMCompat::testInt32(
2410 Assembler::Condition cond, Register tag) {
2411 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2412 ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
2413 return cond;
2414 }
2415
testBoolean(Assembler::Condition cond,Register tag)2416 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2417 Assembler::Condition cond, Register tag) {
2418 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2419 ma_cmp(tag, ImmTag(JSVAL_TAG_BOOLEAN));
2420 return cond;
2421 }
2422
testNull(Assembler::Condition cond,Register tag)2423 Assembler::Condition MacroAssemblerARMCompat::testNull(
2424 Assembler::Condition cond, Register tag) {
2425 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2426 ma_cmp(tag, ImmTag(JSVAL_TAG_NULL));
2427 return cond;
2428 }
2429
testUndefined(Assembler::Condition cond,Register tag)2430 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2431 Assembler::Condition cond, Register tag) {
2432 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2433 ma_cmp(tag, ImmTag(JSVAL_TAG_UNDEFINED));
2434 return cond;
2435 }
2436
testString(Assembler::Condition cond,Register tag)2437 Assembler::Condition MacroAssemblerARMCompat::testString(
2438 Assembler::Condition cond, Register tag) {
2439 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2440 ma_cmp(tag, ImmTag(JSVAL_TAG_STRING));
2441 return cond;
2442 }
2443
testSymbol(Assembler::Condition cond,Register tag)2444 Assembler::Condition MacroAssemblerARMCompat::testSymbol(
2445 Assembler::Condition cond, Register tag) {
2446 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2447 ma_cmp(tag, ImmTag(JSVAL_TAG_SYMBOL));
2448 return cond;
2449 }
2450
testObject(Assembler::Condition cond,Register tag)2451 Assembler::Condition MacroAssemblerARMCompat::testObject(
2452 Assembler::Condition cond, Register tag) {
2453 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2454 ma_cmp(tag, ImmTag(JSVAL_TAG_OBJECT));
2455 return cond;
2456 }
2457
testMagic(Assembler::Condition cond,Register tag)2458 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2459 Assembler::Condition cond, Register tag) {
2460 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2461 ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
2462 return cond;
2463 }
2464
testPrimitive(Assembler::Condition cond,Register tag)2465 Assembler::Condition MacroAssemblerARMCompat::testPrimitive(
2466 Assembler::Condition cond, Register tag) {
2467 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2468 ma_cmp(tag, ImmTag(JSVAL_UPPER_EXCL_TAG_OF_PRIMITIVE_SET));
2469 return cond == Equal ? Below : AboveOrEqual;
2470 }
2471
testGCThing(Assembler::Condition cond,const Address & address)2472 Assembler::Condition MacroAssemblerARMCompat::testGCThing(
2473 Assembler::Condition cond, const Address& address) {
2474 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2475 ScratchRegisterScope scratch(asMasm());
2476 extractTag(address, scratch);
2477 ma_cmp(scratch, ImmTag(JSVAL_LOWER_INCL_TAG_OF_GCTHING_SET));
2478 return cond == Equal ? AboveOrEqual : Below;
2479 }
2480
testMagic(Assembler::Condition cond,const Address & address)2481 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2482 Assembler::Condition cond, const Address& address) {
2483 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2484 ScratchRegisterScope scratch(asMasm());
2485 extractTag(address, scratch);
2486 ma_cmp(scratch, ImmTag(JSVAL_TAG_MAGIC));
2487 return cond;
2488 }
2489
testInt32(Assembler::Condition cond,const Address & address)2490 Assembler::Condition MacroAssemblerARMCompat::testInt32(
2491 Assembler::Condition cond, const Address& address) {
2492 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2493 ScratchRegisterScope scratch(asMasm());
2494 extractTag(address, scratch);
2495 ma_cmp(scratch, ImmTag(JSVAL_TAG_INT32));
2496 return cond;
2497 }
2498
testDouble(Condition cond,const Address & address)2499 Assembler::Condition MacroAssemblerARMCompat::testDouble(
2500 Condition cond, const Address& address) {
2501 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2502 ScratchRegisterScope scratch(asMasm());
2503 extractTag(address, scratch);
2504 return testDouble(cond, scratch);
2505 }
2506
testBoolean(Condition cond,const Address & address)2507 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2508 Condition cond, const Address& address) {
2509 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2510 ScratchRegisterScope scratch(asMasm());
2511 extractTag(address, scratch);
2512 return testBoolean(cond, scratch);
2513 }
2514
testNull(Condition cond,const Address & address)2515 Assembler::Condition MacroAssemblerARMCompat::testNull(Condition cond,
2516 const Address& address) {
2517 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2518 ScratchRegisterScope scratch(asMasm());
2519 extractTag(address, scratch);
2520 return testNull(cond, scratch);
2521 }
2522
testUndefined(Condition cond,const Address & address)2523 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2524 Condition cond, const Address& address) {
2525 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2526 ScratchRegisterScope scratch(asMasm());
2527 extractTag(address, scratch);
2528 return testUndefined(cond, scratch);
2529 }
2530
testString(Condition cond,const Address & address)2531 Assembler::Condition MacroAssemblerARMCompat::testString(
2532 Condition cond, const Address& address) {
2533 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2534 ScratchRegisterScope scratch(asMasm());
2535 extractTag(address, scratch);
2536 return testString(cond, scratch);
2537 }
2538
testSymbol(Condition cond,const Address & address)2539 Assembler::Condition MacroAssemblerARMCompat::testSymbol(
2540 Condition cond, const Address& address) {
2541 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2542 ScratchRegisterScope scratch(asMasm());
2543 extractTag(address, scratch);
2544 return testSymbol(cond, scratch);
2545 }
2546
testObject(Condition cond,const Address & address)2547 Assembler::Condition MacroAssemblerARMCompat::testObject(
2548 Condition cond, const Address& address) {
2549 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2550 ScratchRegisterScope scratch(asMasm());
2551 extractTag(address, scratch);
2552 return testObject(cond, scratch);
2553 }
2554
testNumber(Condition cond,const Address & address)2555 Assembler::Condition MacroAssemblerARMCompat::testNumber(
2556 Condition cond, const Address& address) {
2557 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2558 ScratchRegisterScope scratch(asMasm());
2559 extractTag(address, scratch);
2560 return testNumber(cond, scratch);
2561 }
2562
testDouble(Condition cond,Register tag)2563 Assembler::Condition MacroAssemblerARMCompat::testDouble(Condition cond,
2564 Register tag) {
2565 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2566 Condition actual = (cond == Equal) ? Below : AboveOrEqual;
2567 ma_cmp(tag, ImmTag(JSVAL_TAG_CLEAR));
2568 return actual;
2569 }
2570
testNumber(Condition cond,Register tag)2571 Assembler::Condition MacroAssemblerARMCompat::testNumber(Condition cond,
2572 Register tag) {
2573 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2574 ma_cmp(tag, ImmTag(JSVAL_UPPER_INCL_TAG_OF_NUMBER_SET));
2575 return cond == Equal ? BelowOrEqual : Above;
2576 }
2577
testUndefined(Condition cond,const BaseIndex & src)2578 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2579 Condition cond, const BaseIndex& src) {
2580 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2581 ScratchRegisterScope scratch(asMasm());
2582 extractTag(src, scratch);
2583 ma_cmp(scratch, ImmTag(JSVAL_TAG_UNDEFINED));
2584 return cond;
2585 }
2586
testNull(Condition cond,const BaseIndex & src)2587 Assembler::Condition MacroAssemblerARMCompat::testNull(Condition cond,
2588 const BaseIndex& src) {
2589 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2590 ScratchRegisterScope scratch(asMasm());
2591 extractTag(src, scratch);
2592 ma_cmp(scratch, ImmTag(JSVAL_TAG_NULL));
2593 return cond;
2594 }
2595
testBoolean(Condition cond,const BaseIndex & src)2596 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2597 Condition cond, const BaseIndex& src) {
2598 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2599 ScratchRegisterScope scratch(asMasm());
2600 extractTag(src, scratch);
2601 ma_cmp(scratch, ImmTag(JSVAL_TAG_BOOLEAN));
2602 return cond;
2603 }
2604
testString(Condition cond,const BaseIndex & src)2605 Assembler::Condition MacroAssemblerARMCompat::testString(Condition cond,
2606 const BaseIndex& src) {
2607 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2608 ScratchRegisterScope scratch(asMasm());
2609 extractTag(src, scratch);
2610 ma_cmp(scratch, ImmTag(JSVAL_TAG_STRING));
2611 return cond;
2612 }
2613
testSymbol(Condition cond,const BaseIndex & src)2614 Assembler::Condition MacroAssemblerARMCompat::testSymbol(Condition cond,
2615 const BaseIndex& src) {
2616 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2617 ScratchRegisterScope scratch(asMasm());
2618 extractTag(src, scratch);
2619 ma_cmp(scratch, ImmTag(JSVAL_TAG_SYMBOL));
2620 return cond;
2621 }
2622
testInt32(Condition cond,const BaseIndex & src)2623 Assembler::Condition MacroAssemblerARMCompat::testInt32(Condition cond,
2624 const BaseIndex& src) {
2625 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2626 ScratchRegisterScope scratch(asMasm());
2627 extractTag(src, scratch);
2628 ma_cmp(scratch, ImmTag(JSVAL_TAG_INT32));
2629 return cond;
2630 }
2631
testObject(Condition cond,const BaseIndex & src)2632 Assembler::Condition MacroAssemblerARMCompat::testObject(Condition cond,
2633 const BaseIndex& src) {
2634 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2635 ScratchRegisterScope scratch(asMasm());
2636 extractTag(src, scratch);
2637 ma_cmp(scratch, ImmTag(JSVAL_TAG_OBJECT));
2638 return cond;
2639 }
2640
testDouble(Condition cond,const BaseIndex & src)2641 Assembler::Condition MacroAssemblerARMCompat::testDouble(Condition cond,
2642 const BaseIndex& src) {
2643 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2644 Assembler::Condition actual = (cond == Equal) ? Below : AboveOrEqual;
2645 ScratchRegisterScope scratch(asMasm());
2646 extractTag(src, scratch);
2647 ma_cmp(scratch, ImmTag(JSVAL_TAG_CLEAR));
2648 return actual;
2649 }
2650
testMagic(Condition cond,const BaseIndex & address)2651 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2652 Condition cond, const BaseIndex& address) {
2653 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2654 ScratchRegisterScope scratch(asMasm());
2655 extractTag(address, scratch);
2656 ma_cmp(scratch, ImmTag(JSVAL_TAG_MAGIC));
2657 return cond;
2658 }
2659
testGCThing(Condition cond,const BaseIndex & address)2660 Assembler::Condition MacroAssemblerARMCompat::testGCThing(
2661 Condition cond, const BaseIndex& address) {
2662 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2663 ScratchRegisterScope scratch(asMasm());
2664 extractTag(address, scratch);
2665 ma_cmp(scratch, ImmTag(JSVAL_LOWER_INCL_TAG_OF_GCTHING_SET));
2666 return cond == Equal ? AboveOrEqual : Below;
2667 }
2668
2669 // Unboxing code.
unboxNonDouble(const ValueOperand & operand,Register dest,JSValueType type)2670 void MacroAssemblerARMCompat::unboxNonDouble(const ValueOperand& operand,
2671 Register dest, JSValueType type) {
2672 auto movPayloadToDest = [&]() {
2673 if (operand.payloadReg() != dest)
2674 ma_mov(operand.payloadReg(), dest, LeaveCC);
2675 };
2676 if (!JitOptions.spectreValueMasking) {
2677 movPayloadToDest();
2678 return;
2679 }
2680
2681 // Spectre mitigation: We zero the payload if the tag does not match the
2682 // expected type and if this is a pointer type.
2683 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
2684 movPayloadToDest();
2685 return;
2686 }
2687
2688 // We zero the destination register and move the payload into it if
2689 // the tag corresponds to the given type.
2690 ma_cmp(operand.typeReg(), ImmType(type));
2691 movPayloadToDest();
2692 ma_mov(Imm32(0), dest, NotEqual);
2693 }
2694
unboxNonDouble(const Address & src,Register dest,JSValueType type)2695 void MacroAssemblerARMCompat::unboxNonDouble(const Address& src, Register dest,
2696 JSValueType type) {
2697 ScratchRegisterScope scratch(asMasm());
2698 if (!JitOptions.spectreValueMasking) {
2699 ma_ldr(ToPayload(src), dest, scratch);
2700 return;
2701 }
2702
2703 // Spectre mitigation: We zero the payload if the tag does not match the
2704 // expected type and if this is a pointer type.
2705 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
2706 ma_ldr(ToPayload(src), dest, scratch);
2707 return;
2708 }
2709
2710 // We zero the destination register and move the payload into it if
2711 // the tag corresponds to the given type.
2712 ma_ldr(ToType(src), scratch, scratch);
2713 ma_cmp(scratch, ImmType(type));
2714 ma_ldr(ToPayload(src), dest, scratch, Offset, Equal);
2715 ma_mov(Imm32(0), dest, NotEqual);
2716 }
2717
unboxNonDouble(const BaseIndex & src,Register dest,JSValueType type)2718 void MacroAssemblerARMCompat::unboxNonDouble(const BaseIndex& src,
2719 Register dest, JSValueType type) {
2720 SecondScratchRegisterScope scratch2(asMasm());
2721 ma_alu(src.base, lsl(src.index, src.scale), scratch2, OpAdd);
2722 Address value(scratch2, src.offset);
2723 unboxNonDouble(value, dest, type);
2724 }
2725
unboxDouble(const ValueOperand & operand,FloatRegister dest)2726 void MacroAssemblerARMCompat::unboxDouble(const ValueOperand& operand,
2727 FloatRegister dest) {
2728 MOZ_ASSERT(dest.isDouble());
2729 as_vxfer(operand.payloadReg(), operand.typeReg(), VFPRegister(dest),
2730 CoreToFloat);
2731 }
2732
unboxDouble(const Address & src,FloatRegister dest)2733 void MacroAssemblerARMCompat::unboxDouble(const Address& src,
2734 FloatRegister dest) {
2735 MOZ_ASSERT(dest.isDouble());
2736 ScratchRegisterScope scratch(asMasm());
2737 ma_vldr(src, dest, scratch);
2738 }
2739
unboxValue(const ValueOperand & src,AnyRegister dest,JSValueType type)2740 void MacroAssemblerARMCompat::unboxValue(const ValueOperand& src,
2741 AnyRegister dest, JSValueType type) {
2742 if (dest.isFloat()) {
2743 Label notInt32, end;
2744 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
2745 convertInt32ToDouble(src.payloadReg(), dest.fpu());
2746 ma_b(&end);
2747 bind(¬Int32);
2748 unboxDouble(src, dest.fpu());
2749 bind(&end);
2750 } else {
2751 unboxNonDouble(src, dest.gpr(), type);
2752 }
2753 }
2754
unboxPrivate(const ValueOperand & src,Register dest)2755 void MacroAssemblerARMCompat::unboxPrivate(const ValueOperand& src,
2756 Register dest) {
2757 ma_mov(src.payloadReg(), dest);
2758 }
2759
boxDouble(FloatRegister src,const ValueOperand & dest,FloatRegister)2760 void MacroAssemblerARMCompat::boxDouble(FloatRegister src,
2761 const ValueOperand& dest,
2762 FloatRegister) {
2763 as_vxfer(dest.payloadReg(), dest.typeReg(), VFPRegister(src), FloatToCore);
2764 }
2765
boxNonDouble(JSValueType type,Register src,const ValueOperand & dest)2766 void MacroAssemblerARMCompat::boxNonDouble(JSValueType type, Register src,
2767 const ValueOperand& dest) {
2768 if (src != dest.payloadReg()) ma_mov(src, dest.payloadReg());
2769 ma_mov(ImmType(type), dest.typeReg());
2770 }
2771
boolValueToDouble(const ValueOperand & operand,FloatRegister dest)2772 void MacroAssemblerARMCompat::boolValueToDouble(const ValueOperand& operand,
2773 FloatRegister dest) {
2774 VFPRegister d = VFPRegister(dest);
2775 loadConstantDouble(1.0, dest);
2776 as_cmp(operand.payloadReg(), Imm8(0));
2777 // If the source is 0, then subtract the dest from itself, producing 0.
2778 as_vsub(d, d, d, Equal);
2779 }
2780
int32ValueToDouble(const ValueOperand & operand,FloatRegister dest)2781 void MacroAssemblerARMCompat::int32ValueToDouble(const ValueOperand& operand,
2782 FloatRegister dest) {
2783 VFPRegister vfpdest = VFPRegister(dest);
2784 ScratchFloat32Scope scratch(asMasm());
2785
2786 // Transfer the integral value to a floating point register.
2787 as_vxfer(operand.payloadReg(), InvalidReg, scratch.sintOverlay(),
2788 CoreToFloat);
2789 // Convert the value to a double.
2790 as_vcvt(vfpdest, scratch.sintOverlay());
2791 }
2792
boolValueToFloat32(const ValueOperand & operand,FloatRegister dest)2793 void MacroAssemblerARMCompat::boolValueToFloat32(const ValueOperand& operand,
2794 FloatRegister dest) {
2795 VFPRegister d = VFPRegister(dest).singleOverlay();
2796 loadConstantFloat32(1.0, dest);
2797 as_cmp(operand.payloadReg(), Imm8(0));
2798 // If the source is 0, then subtract the dest from itself, producing 0.
2799 as_vsub(d, d, d, Equal);
2800 }
2801
int32ValueToFloat32(const ValueOperand & operand,FloatRegister dest)2802 void MacroAssemblerARMCompat::int32ValueToFloat32(const ValueOperand& operand,
2803 FloatRegister dest) {
2804 // Transfer the integral value to a floating point register.
2805 VFPRegister vfpdest = VFPRegister(dest).singleOverlay();
2806 as_vxfer(operand.payloadReg(), InvalidReg, vfpdest.sintOverlay(),
2807 CoreToFloat);
2808 // Convert the value to a float.
2809 as_vcvt(vfpdest, vfpdest.sintOverlay());
2810 }
2811
loadConstantFloat32(float f,FloatRegister dest)2812 void MacroAssemblerARMCompat::loadConstantFloat32(float f, FloatRegister dest) {
2813 ma_vimm_f32(f, dest);
2814 }
2815
loadInt32OrDouble(const Address & src,FloatRegister dest)2816 void MacroAssemblerARMCompat::loadInt32OrDouble(const Address& src,
2817 FloatRegister dest) {
2818 Label notInt32, end;
2819
2820 // If it's an int, convert to a double.
2821 {
2822 ScratchRegisterScope scratch(asMasm());
2823 SecondScratchRegisterScope scratch2(asMasm());
2824
2825 ma_ldr(ToType(src), scratch, scratch2);
2826 asMasm().branchTestInt32(Assembler::NotEqual, scratch, ¬Int32);
2827 ma_ldr(ToPayload(src), scratch, scratch2);
2828 convertInt32ToDouble(scratch, dest);
2829 ma_b(&end);
2830 }
2831
2832 // Not an int, just load as double.
2833 bind(¬Int32);
2834 {
2835 ScratchRegisterScope scratch(asMasm());
2836 ma_vldr(src, dest, scratch);
2837 }
2838 bind(&end);
2839 }
2840
loadInt32OrDouble(Register base,Register index,FloatRegister dest,int32_t shift)2841 void MacroAssemblerARMCompat::loadInt32OrDouble(Register base, Register index,
2842 FloatRegister dest,
2843 int32_t shift) {
2844 Label notInt32, end;
2845
2846 JS_STATIC_ASSERT(NUNBOX32_PAYLOAD_OFFSET == 0);
2847
2848 ScratchRegisterScope scratch(asMasm());
2849
2850 // If it's an int, convert it to double.
2851 ma_alu(base, lsl(index, shift), scratch, OpAdd);
2852
2853 // Since we only have one scratch register, we need to stomp over it with
2854 // the tag.
2855 ma_ldr(DTRAddr(scratch, DtrOffImm(NUNBOX32_TYPE_OFFSET)), scratch);
2856 asMasm().branchTestInt32(Assembler::NotEqual, scratch, ¬Int32);
2857
2858 // Implicitly requires NUNBOX32_PAYLOAD_OFFSET == 0: no offset provided
2859 ma_ldr(DTRAddr(base, DtrRegImmShift(index, LSL, shift)), scratch);
2860 convertInt32ToDouble(scratch, dest);
2861 ma_b(&end);
2862
2863 // Not an int, just load as double.
2864 bind(¬Int32);
2865 // First, recompute the offset that had been stored in the scratch register
2866 // since the scratch register was overwritten loading in the type.
2867 ma_alu(base, lsl(index, shift), scratch, OpAdd);
2868 ma_vldr(VFPAddr(scratch, VFPOffImm(0)), dest);
2869 bind(&end);
2870 }
2871
loadConstantDouble(double dp,FloatRegister dest)2872 void MacroAssemblerARMCompat::loadConstantDouble(double dp,
2873 FloatRegister dest) {
2874 ma_vimm(dp, dest);
2875 }
2876
2877 // Treat the value as a boolean, and set condition codes accordingly.
testInt32Truthy(bool truthy,const ValueOperand & operand)2878 Assembler::Condition MacroAssemblerARMCompat::testInt32Truthy(
2879 bool truthy, const ValueOperand& operand) {
2880 ma_tst(operand.payloadReg(), operand.payloadReg());
2881 return truthy ? NonZero : Zero;
2882 }
2883
testBooleanTruthy(bool truthy,const ValueOperand & operand)2884 Assembler::Condition MacroAssemblerARMCompat::testBooleanTruthy(
2885 bool truthy, const ValueOperand& operand) {
2886 ma_tst(operand.payloadReg(), operand.payloadReg());
2887 return truthy ? NonZero : Zero;
2888 }
2889
testDoubleTruthy(bool truthy,FloatRegister reg)2890 Assembler::Condition MacroAssemblerARMCompat::testDoubleTruthy(
2891 bool truthy, FloatRegister reg) {
2892 as_vcmpz(VFPRegister(reg));
2893 as_vmrs(pc);
2894 as_cmp(r0, O2Reg(r0), Overflow);
2895 return truthy ? NonZero : Zero;
2896 }
2897
extractObject(const Address & address,Register scratch)2898 Register MacroAssemblerARMCompat::extractObject(const Address& address,
2899 Register scratch) {
2900 SecondScratchRegisterScope scratch2(asMasm());
2901 ma_ldr(ToPayload(address), scratch, scratch2);
2902 return scratch;
2903 }
2904
extractTag(const Address & address,Register scratch)2905 Register MacroAssemblerARMCompat::extractTag(const Address& address,
2906 Register scratch) {
2907 SecondScratchRegisterScope scratch2(asMasm());
2908 ma_ldr(ToType(address), scratch, scratch2);
2909 return scratch;
2910 }
2911
extractTag(const BaseIndex & address,Register scratch)2912 Register MacroAssemblerARMCompat::extractTag(const BaseIndex& address,
2913 Register scratch) {
2914 ma_alu(address.base, lsl(address.index, address.scale), scratch, OpAdd,
2915 LeaveCC);
2916 return extractTag(Address(scratch, address.offset), scratch);
2917 }
2918
2919 /////////////////////////////////////////////////////////////////
2920 // X86/X64-common (ARM too now) interface.
2921 /////////////////////////////////////////////////////////////////
storeValue(ValueOperand val,const Address & dst)2922 void MacroAssemblerARMCompat::storeValue(ValueOperand val, const Address& dst) {
2923 SecondScratchRegisterScope scratch2(asMasm());
2924 ma_str(val.payloadReg(), ToPayload(dst), scratch2);
2925 ma_str(val.typeReg(), ToType(dst), scratch2);
2926 }
2927
storeValue(ValueOperand val,const BaseIndex & dest)2928 void MacroAssemblerARMCompat::storeValue(ValueOperand val,
2929 const BaseIndex& dest) {
2930 ScratchRegisterScope scratch(asMasm());
2931
2932 if (isValueDTRDCandidate(val) && Abs(dest.offset) <= 255) {
2933 Register tmpIdx;
2934 if (dest.offset == 0) {
2935 if (dest.scale == TimesOne) {
2936 tmpIdx = dest.index;
2937 } else {
2938 ma_lsl(Imm32(dest.scale), dest.index, scratch);
2939 tmpIdx = scratch;
2940 }
2941 ma_strd(val.payloadReg(), val.typeReg(),
2942 EDtrAddr(dest.base, EDtrOffReg(tmpIdx)));
2943 } else {
2944 ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
2945 ma_strd(val.payloadReg(), val.typeReg(),
2946 EDtrAddr(scratch, EDtrOffImm(dest.offset)));
2947 }
2948 } else {
2949 ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
2950 storeValue(val, Address(scratch, dest.offset));
2951 }
2952 }
2953
loadValue(const BaseIndex & addr,ValueOperand val)2954 void MacroAssemblerARMCompat::loadValue(const BaseIndex& addr,
2955 ValueOperand val) {
2956 ScratchRegisterScope scratch(asMasm());
2957
2958 if (isValueDTRDCandidate(val) && Abs(addr.offset) <= 255) {
2959 Register tmpIdx;
2960 if (addr.offset == 0) {
2961 if (addr.scale == TimesOne) {
2962 // If the offset register is the same as one of the destination
2963 // registers, LDRD's behavior is undefined. Use the scratch
2964 // register to avoid this.
2965 if (val.aliases(addr.index)) {
2966 ma_mov(addr.index, scratch);
2967 tmpIdx = scratch;
2968 } else {
2969 tmpIdx = addr.index;
2970 }
2971 } else {
2972 ma_lsl(Imm32(addr.scale), addr.index, scratch);
2973 tmpIdx = scratch;
2974 }
2975 ma_ldrd(EDtrAddr(addr.base, EDtrOffReg(tmpIdx)), val.payloadReg(),
2976 val.typeReg());
2977 } else {
2978 ma_alu(addr.base, lsl(addr.index, addr.scale), scratch, OpAdd);
2979 ma_ldrd(EDtrAddr(scratch, EDtrOffImm(addr.offset)), val.payloadReg(),
2980 val.typeReg());
2981 }
2982 } else {
2983 ma_alu(addr.base, lsl(addr.index, addr.scale), scratch, OpAdd);
2984 loadValue(Address(scratch, addr.offset), val);
2985 }
2986 }
2987
loadValue(Address src,ValueOperand val)2988 void MacroAssemblerARMCompat::loadValue(Address src, ValueOperand val) {
2989 Address payload = ToPayload(src);
2990 Address type = ToType(src);
2991
2992 // TODO: copy this code into a generic function that acts on all sequences
2993 // of memory accesses
2994 if (isValueDTRDCandidate(val)) {
2995 // If the value we want is in two consecutive registers starting with an
2996 // even register, they can be combined as a single ldrd.
2997 int offset = src.offset;
2998 if (offset < 256 && offset > -256) {
2999 ma_ldrd(EDtrAddr(src.base, EDtrOffImm(src.offset)), val.payloadReg(),
3000 val.typeReg());
3001 return;
3002 }
3003 }
3004 // If the value is lower than the type, then we may be able to use an ldm
3005 // instruction.
3006
3007 if (val.payloadReg().code() < val.typeReg().code()) {
3008 if (src.offset <= 4 && src.offset >= -8 && (src.offset & 3) == 0) {
3009 // Turns out each of the 4 value -8, -4, 0, 4 corresponds exactly
3010 // with one of LDM{DB, DA, IA, IB}
3011 DTMMode mode;
3012 switch (src.offset) {
3013 case -8:
3014 mode = DB;
3015 break;
3016 case -4:
3017 mode = DA;
3018 break;
3019 case 0:
3020 mode = IA;
3021 break;
3022 case 4:
3023 mode = IB;
3024 break;
3025 default:
3026 MOZ_CRASH("Bogus Offset for LoadValue as DTM");
3027 }
3028 startDataTransferM(IsLoad, src.base, mode);
3029 transferReg(val.payloadReg());
3030 transferReg(val.typeReg());
3031 finishDataTransfer();
3032 return;
3033 }
3034 }
3035 // Ensure that loading the payload does not erase the pointer to the Value
3036 // in memory.
3037 if (type.base != val.payloadReg()) {
3038 SecondScratchRegisterScope scratch2(asMasm());
3039 ma_ldr(payload, val.payloadReg(), scratch2);
3040 ma_ldr(type, val.typeReg(), scratch2);
3041 } else {
3042 SecondScratchRegisterScope scratch2(asMasm());
3043 ma_ldr(type, val.typeReg(), scratch2);
3044 ma_ldr(payload, val.payloadReg(), scratch2);
3045 }
3046 }
3047
tagValue(JSValueType type,Register payload,ValueOperand dest)3048 void MacroAssemblerARMCompat::tagValue(JSValueType type, Register payload,
3049 ValueOperand dest) {
3050 MOZ_ASSERT(dest.typeReg() != dest.payloadReg());
3051 if (payload != dest.payloadReg()) ma_mov(payload, dest.payloadReg());
3052 ma_mov(ImmType(type), dest.typeReg());
3053 }
3054
pushValue(ValueOperand val)3055 void MacroAssemblerARMCompat::pushValue(ValueOperand val) {
3056 ma_push(val.typeReg());
3057 ma_push(val.payloadReg());
3058 }
3059
pushValue(const Address & addr)3060 void MacroAssemblerARMCompat::pushValue(const Address& addr) {
3061 ScratchRegisterScope scratch(asMasm());
3062 SecondScratchRegisterScope scratch2(asMasm());
3063
3064 ma_ldr(ToType(addr), scratch, scratch2);
3065 ma_push(scratch);
3066 ma_ldr(ToPayloadAfterStackPush(addr), scratch, scratch2);
3067 ma_push(scratch);
3068 }
3069
popValue(ValueOperand val)3070 void MacroAssemblerARMCompat::popValue(ValueOperand val) {
3071 ma_pop(val.payloadReg());
3072 ma_pop(val.typeReg());
3073 }
3074
storePayload(const Value & val,const Address & dest)3075 void MacroAssemblerARMCompat::storePayload(const Value& val,
3076 const Address& dest) {
3077 ScratchRegisterScope scratch(asMasm());
3078 SecondScratchRegisterScope scratch2(asMasm());
3079
3080 if (val.isGCThing())
3081 ma_mov(ImmGCPtr(val.toGCThing()), scratch);
3082 else
3083 ma_mov(Imm32(val.toNunboxPayload()), scratch);
3084 ma_str(scratch, ToPayload(dest), scratch2);
3085 }
3086
storePayload(Register src,const Address & dest)3087 void MacroAssemblerARMCompat::storePayload(Register src, const Address& dest) {
3088 ScratchRegisterScope scratch(asMasm());
3089 ma_str(src, ToPayload(dest), scratch);
3090 }
3091
storePayload(const Value & val,const BaseIndex & dest)3092 void MacroAssemblerARMCompat::storePayload(const Value& val,
3093 const BaseIndex& dest) {
3094 unsigned shift = ScaleToShift(dest.scale);
3095
3096 ScratchRegisterScope scratch(asMasm());
3097 SecondScratchRegisterScope scratch2(asMasm());
3098
3099 if (val.isGCThing())
3100 ma_mov(ImmGCPtr(val.toGCThing()), scratch);
3101 else
3102 ma_mov(Imm32(val.toNunboxPayload()), scratch);
3103
3104 // If NUNBOX32_PAYLOAD_OFFSET is not zero, the memory operand [base + index
3105 // << shift + imm] cannot be encoded into a single instruction, and cannot
3106 // be integrated into the as_dtr call.
3107 JS_STATIC_ASSERT(NUNBOX32_PAYLOAD_OFFSET == 0);
3108
3109 // If an offset is used, modify the base so that a [base + index << shift]
3110 // instruction format can be used.
3111 if (dest.offset != 0)
3112 ma_add(dest.base, Imm32(dest.offset), dest.base, scratch2);
3113
3114 as_dtr(IsStore, 32, Offset, scratch,
3115 DTRAddr(dest.base, DtrRegImmShift(dest.index, LSL, shift)));
3116
3117 // Restore the original value of the base, if necessary.
3118 if (dest.offset != 0)
3119 ma_sub(dest.base, Imm32(dest.offset), dest.base, scratch);
3120 }
3121
storePayload(Register src,const BaseIndex & dest)3122 void MacroAssemblerARMCompat::storePayload(Register src,
3123 const BaseIndex& dest) {
3124 unsigned shift = ScaleToShift(dest.scale);
3125 MOZ_ASSERT(shift < 32);
3126
3127 ScratchRegisterScope scratch(asMasm());
3128
3129 // If NUNBOX32_PAYLOAD_OFFSET is not zero, the memory operand [base + index
3130 // << shift + imm] cannot be encoded into a single instruction, and cannot
3131 // be integrated into the as_dtr call.
3132 JS_STATIC_ASSERT(NUNBOX32_PAYLOAD_OFFSET == 0);
3133
3134 // Save/restore the base if the BaseIndex has an offset, as above.
3135 if (dest.offset != 0)
3136 ma_add(dest.base, Imm32(dest.offset), dest.base, scratch);
3137
3138 // Technically, shift > -32 can be handle by changing LSL to ASR, but should
3139 // never come up, and this is one less code path to get wrong.
3140 as_dtr(IsStore, 32, Offset, src,
3141 DTRAddr(dest.base, DtrRegImmShift(dest.index, LSL, shift)));
3142
3143 if (dest.offset != 0)
3144 ma_sub(dest.base, Imm32(dest.offset), dest.base, scratch);
3145 }
3146
storeTypeTag(ImmTag tag,const Address & dest)3147 void MacroAssemblerARMCompat::storeTypeTag(ImmTag tag, const Address& dest) {
3148 ScratchRegisterScope scratch(asMasm());
3149 SecondScratchRegisterScope scratch2(asMasm());
3150
3151 ma_mov(tag, scratch);
3152 ma_str(scratch, ToType(dest), scratch2);
3153 }
3154
storeTypeTag(ImmTag tag,const BaseIndex & dest)3155 void MacroAssemblerARMCompat::storeTypeTag(ImmTag tag, const BaseIndex& dest) {
3156 Register base = dest.base;
3157 Register index = dest.index;
3158 unsigned shift = ScaleToShift(dest.scale);
3159
3160 ScratchRegisterScope scratch(asMasm());
3161 SecondScratchRegisterScope scratch2(asMasm());
3162
3163 MOZ_ASSERT(base != scratch && base != scratch2);
3164 MOZ_ASSERT(index != scratch && index != scratch2);
3165
3166 ma_add(base, Imm32(dest.offset + NUNBOX32_TYPE_OFFSET), scratch2, scratch);
3167 ma_mov(tag, scratch);
3168 ma_str(scratch, DTRAddr(scratch2, DtrRegImmShift(index, LSL, shift)));
3169 }
3170
ma_call(ImmPtr dest)3171 void MacroAssemblerARM::ma_call(ImmPtr dest) {
3172 ma_movPatchable(dest, CallReg, Always);
3173 as_blx(CallReg);
3174 }
3175
breakpoint()3176 void MacroAssemblerARMCompat::breakpoint() { as_bkpt(); }
3177
simulatorStop(const char * msg)3178 void MacroAssemblerARMCompat::simulatorStop(const char* msg) {
3179 #ifdef JS_SIMULATOR_ARM
3180 MOZ_ASSERT(sizeof(char*) == 4);
3181 writeInst(0xefffffff);
3182 writeInst((int)msg);
3183 #endif
3184 }
3185
ensureDouble(const ValueOperand & source,FloatRegister dest,Label * failure)3186 void MacroAssemblerARMCompat::ensureDouble(const ValueOperand& source,
3187 FloatRegister dest, Label* failure) {
3188 Label isDouble, done;
3189 asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
3190 asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
3191
3192 convertInt32ToDouble(source.payloadReg(), dest);
3193 jump(&done);
3194
3195 bind(&isDouble);
3196 unboxDouble(source, dest);
3197
3198 bind(&done);
3199 }
3200
breakpoint(Condition cc)3201 void MacroAssemblerARMCompat::breakpoint(Condition cc) {
3202 ma_ldr(DTRAddr(r12, DtrRegImmShift(r12, LSL, 0, IsDown)), r12, Offset, cc);
3203 }
3204
checkStackAlignment()3205 void MacroAssemblerARMCompat::checkStackAlignment() {
3206 asMasm().assertStackAlignment(ABIStackAlignment);
3207 }
3208
handleFailureWithHandlerTail(void * handler,Label * profilerExitTail)3209 void MacroAssemblerARMCompat::handleFailureWithHandlerTail(
3210 void* handler, Label* profilerExitTail) {
3211 // Reserve space for exception information.
3212 int size = (sizeof(ResumeFromException) + 7) & ~7;
3213
3214 Imm8 size8(size);
3215 as_sub(sp, sp, size8);
3216 ma_mov(sp, r0);
3217
3218 // Call the handler.
3219 asMasm().setupUnalignedABICall(r1);
3220 asMasm().passABIArg(r0);
3221 asMasm().callWithABI(handler, MoveOp::GENERAL,
3222 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
3223
3224 Label entryFrame;
3225 Label catch_;
3226 Label finally;
3227 Label return_;
3228 Label bailout;
3229 Label wasm;
3230
3231 {
3232 ScratchRegisterScope scratch(asMasm());
3233 ma_ldr(Address(sp, offsetof(ResumeFromException, kind)), r0, scratch);
3234 }
3235
3236 asMasm().branch32(Assembler::Equal, r0,
3237 Imm32(ResumeFromException::RESUME_ENTRY_FRAME),
3238 &entryFrame);
3239 asMasm().branch32(Assembler::Equal, r0,
3240 Imm32(ResumeFromException::RESUME_CATCH), &catch_);
3241 asMasm().branch32(Assembler::Equal, r0,
3242 Imm32(ResumeFromException::RESUME_FINALLY), &finally);
3243 asMasm().branch32(Assembler::Equal, r0,
3244 Imm32(ResumeFromException::RESUME_FORCED_RETURN), &return_);
3245 asMasm().branch32(Assembler::Equal, r0,
3246 Imm32(ResumeFromException::RESUME_BAILOUT), &bailout);
3247 asMasm().branch32(Assembler::Equal, r0,
3248 Imm32(ResumeFromException::RESUME_WASM), &wasm);
3249
3250 breakpoint(); // Invalid kind.
3251
3252 // No exception handler. Load the error value, load the new stack pointer
3253 // and return from the entry frame.
3254 bind(&entryFrame);
3255 asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
3256 {
3257 ScratchRegisterScope scratch(asMasm());
3258 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3259 scratch);
3260 }
3261
3262 // We're going to be returning by the ion calling convention, which returns
3263 // by ??? (for now, I think ldr pc, [sp]!)
3264 as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
3265
3266 // If we found a catch handler, this must be a baseline frame. Restore state
3267 // and jump to the catch block.
3268 bind(&catch_);
3269 {
3270 ScratchRegisterScope scratch(asMasm());
3271 ma_ldr(Address(sp, offsetof(ResumeFromException, target)), r0, scratch);
3272 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3273 scratch);
3274 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3275 scratch);
3276 }
3277 jump(r0);
3278
3279 // If we found a finally block, this must be a baseline frame. Push two
3280 // values expected by JSOP_RETSUB: BooleanValue(true) and the exception.
3281 bind(&finally);
3282 ValueOperand exception = ValueOperand(r1, r2);
3283 loadValue(Operand(sp, offsetof(ResumeFromException, exception)), exception);
3284 {
3285 ScratchRegisterScope scratch(asMasm());
3286 ma_ldr(Address(sp, offsetof(ResumeFromException, target)), r0, scratch);
3287 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3288 scratch);
3289 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3290 scratch);
3291 }
3292
3293 pushValue(BooleanValue(true));
3294 pushValue(exception);
3295 jump(r0);
3296
3297 // Only used in debug mode. Return BaselineFrame->returnValue() to the
3298 // caller.
3299 bind(&return_);
3300 {
3301 ScratchRegisterScope scratch(asMasm());
3302 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3303 scratch);
3304 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3305 scratch);
3306 }
3307 loadValue(Address(r11, BaselineFrame::reverseOffsetOfReturnValue()),
3308 JSReturnOperand);
3309 ma_mov(r11, sp);
3310 pop(r11);
3311
3312 // If profiling is enabled, then update the lastProfilingFrame to refer to
3313 // caller frame before returning.
3314 {
3315 Label skipProfilingInstrumentation;
3316 // Test if profiler enabled.
3317 AbsoluteAddress addressOfEnabled(
3318 GetJitContext()->runtime->geckoProfiler().addressOfEnabled());
3319 asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
3320 &skipProfilingInstrumentation);
3321 jump(profilerExitTail);
3322 bind(&skipProfilingInstrumentation);
3323 }
3324
3325 ret();
3326
3327 // If we are bailing out to baseline to handle an exception, jump to the
3328 // bailout tail stub.
3329 bind(&bailout);
3330 {
3331 ScratchRegisterScope scratch(asMasm());
3332 ma_ldr(Address(sp, offsetof(ResumeFromException, bailoutInfo)), r2,
3333 scratch);
3334 ma_mov(Imm32(BAILOUT_RETURN_OK), r0);
3335 ma_ldr(Address(sp, offsetof(ResumeFromException, target)), r1, scratch);
3336 }
3337 jump(r1);
3338
3339 // If we are throwing and the innermost frame was a wasm frame, reset SP and
3340 // FP; SP is pointing to the unwound return address to the wasm entry, so
3341 // we can just ret().
3342 bind(&wasm);
3343 {
3344 ScratchRegisterScope scratch(asMasm());
3345 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3346 scratch);
3347 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3348 scratch);
3349 }
3350 as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
3351 }
3352
testStringTruthy(bool truthy,const ValueOperand & value)3353 Assembler::Condition MacroAssemblerARMCompat::testStringTruthy(
3354 bool truthy, const ValueOperand& value) {
3355 Register string = value.payloadReg();
3356 ScratchRegisterScope scratch(asMasm());
3357 SecondScratchRegisterScope scratch2(asMasm());
3358
3359 ma_dtr(IsLoad, string, Imm32(JSString::offsetOfLength()), scratch, scratch2);
3360 as_cmp(scratch, Imm8(0));
3361 return truthy ? Assembler::NotEqual : Assembler::Equal;
3362 }
3363
floor(FloatRegister input,Register output,Label * bail)3364 void MacroAssemblerARMCompat::floor(FloatRegister input, Register output,
3365 Label* bail) {
3366 Label handleZero;
3367 Label handleNeg;
3368 Label fin;
3369
3370 ScratchDoubleScope scratchDouble(asMasm());
3371
3372 compareDouble(input, NoVFPRegister);
3373 ma_b(&handleZero, Assembler::Equal);
3374 ma_b(&handleNeg, Assembler::Signed);
3375 // NaN is always a bail condition, just bail directly.
3376 ma_b(bail, Assembler::Overflow);
3377
3378 // The argument is a positive number, truncation is the path to glory. Since
3379 // it is known to be > 0.0, explicitly convert to a larger range, then a
3380 // value that rounds to INT_MAX is explicitly different from an argument
3381 // that clamps to INT_MAX.
3382 ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
3383 ma_vxfer(scratchDouble.uintOverlay(), output);
3384 ma_mov(output, output, SetCC);
3385 ma_b(bail, Signed);
3386 ma_b(&fin);
3387
3388 bind(&handleZero);
3389 // Move the top word of the double into the output reg, if it is non-zero,
3390 // then the original value was -0.0.
3391 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3392 as_cmp(output, Imm8(0));
3393 ma_b(bail, NonZero);
3394 ma_b(&fin);
3395
3396 bind(&handleNeg);
3397 // Negative case, negate, then start dancing.
3398 ma_vneg(input, input);
3399 ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
3400 ma_vxfer(scratchDouble.uintOverlay(), output);
3401 ma_vcvt_U32_F64(scratchDouble.uintOverlay(), scratchDouble);
3402 compareDouble(scratchDouble, input);
3403 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3404 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3405 // result will still be a negative number.
3406 as_rsb(output, output, Imm8(0), SetCC);
3407 // Flip the negated input back to its original value.
3408 ma_vneg(input, input);
3409 // If the result looks non-negative, then this value didn't actually fit
3410 // into the int range, and special handling is required. Zero is also caught
3411 // by this case, but floor of a negative number should never be zero.
3412 ma_b(bail, NotSigned);
3413
3414 bind(&fin);
3415 }
3416
floorf(FloatRegister input,Register output,Label * bail)3417 void MacroAssemblerARMCompat::floorf(FloatRegister input, Register output,
3418 Label* bail) {
3419 Label handleZero;
3420 Label handleNeg;
3421 Label fin;
3422 compareFloat(input, NoVFPRegister);
3423 ma_b(&handleZero, Assembler::Equal);
3424 ma_b(&handleNeg, Assembler::Signed);
3425 // NaN is always a bail condition, just bail directly.
3426 ma_b(bail, Assembler::Overflow);
3427
3428 // The argument is a positive number, truncation is the path to glory; Since
3429 // it is known to be > 0.0, explicitly convert to a larger range, then a
3430 // value that rounds to INT_MAX is explicitly different from an argument
3431 // that clamps to INT_MAX.
3432 {
3433 ScratchFloat32Scope scratch(asMasm());
3434 ma_vcvt_F32_U32(input, scratch.uintOverlay());
3435 ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
3436 }
3437 ma_mov(output, output, SetCC);
3438 ma_b(bail, Signed);
3439 ma_b(&fin);
3440
3441 bind(&handleZero);
3442 // Move the top word of the double into the output reg, if it is non-zero,
3443 // then the original value was -0.0.
3444 as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
3445 Always, 0);
3446 as_cmp(output, Imm8(0));
3447 ma_b(bail, NonZero);
3448 ma_b(&fin);
3449
3450 bind(&handleNeg);
3451 // Negative case, negate, then start dancing.
3452 {
3453 ScratchFloat32Scope scratch(asMasm());
3454 ma_vneg_f32(input, input);
3455 ma_vcvt_F32_U32(input, scratch.uintOverlay());
3456 ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
3457 ma_vcvt_U32_F32(scratch.uintOverlay(), scratch);
3458 compareFloat(scratch, input);
3459 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3460 }
3461 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3462 // result will still be a negative number.
3463 as_rsb(output, output, Imm8(0), SetCC);
3464 // Flip the negated input back to its original value.
3465 ma_vneg_f32(input, input);
3466 // If the result looks non-negative, then this value didn't actually fit
3467 // into the int range, and special handling is required. Zero is also caught
3468 // by this case, but floor of a negative number should never be zero.
3469 ma_b(bail, NotSigned);
3470
3471 bind(&fin);
3472 }
3473
ceil(FloatRegister input,Register output,Label * bail)3474 void MacroAssemblerARMCompat::ceil(FloatRegister input, Register output,
3475 Label* bail) {
3476 Label handleZero;
3477 Label handlePos;
3478 Label fin;
3479
3480 compareDouble(input, NoVFPRegister);
3481 // NaN is always a bail condition, just bail directly.
3482 ma_b(bail, Assembler::Overflow);
3483 ma_b(&handleZero, Assembler::Equal);
3484 ma_b(&handlePos, Assembler::NotSigned);
3485
3486 ScratchDoubleScope scratchDouble(asMasm());
3487
3488 // We are in the ]-Inf; 0[ range
3489 // If we are in the ]-1; 0[ range => bailout
3490 loadConstantDouble(-1.0, scratchDouble);
3491 compareDouble(input, scratchDouble);
3492 ma_b(bail, Assembler::GreaterThan);
3493
3494 // We are in the ]-Inf; -1] range: ceil(x) == -floor(-x) and floor can be
3495 // computed with direct truncation here (x > 0).
3496 ma_vneg(input, scratchDouble);
3497 FloatRegister ScratchUIntReg = scratchDouble.uintOverlay();
3498 ma_vcvt_F64_U32(scratchDouble, ScratchUIntReg);
3499 ma_vxfer(ScratchUIntReg, output);
3500 ma_neg(output, output, SetCC);
3501 ma_b(bail, NotSigned);
3502 ma_b(&fin);
3503
3504 // Test for 0.0 / -0.0: if the top word of the input double is not zero,
3505 // then it was -0 and we need to bail out.
3506 bind(&handleZero);
3507 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3508 as_cmp(output, Imm8(0));
3509 ma_b(bail, NonZero);
3510 ma_b(&fin);
3511
3512 // We are in the ]0; +inf] range: truncate integer values, maybe add 1 for
3513 // non integer values, maybe bail if overflow.
3514 bind(&handlePos);
3515 ma_vcvt_F64_U32(input, ScratchUIntReg);
3516 ma_vxfer(ScratchUIntReg, output);
3517 ma_vcvt_U32_F64(ScratchUIntReg, scratchDouble);
3518 compareDouble(scratchDouble, input);
3519 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3520 // Bail out if the add overflowed or the result is non positive.
3521 ma_mov(output, output, SetCC);
3522 ma_b(bail, Signed);
3523 ma_b(bail, Zero);
3524
3525 bind(&fin);
3526 }
3527
ceilf(FloatRegister input,Register output,Label * bail)3528 void MacroAssemblerARMCompat::ceilf(FloatRegister input, Register output,
3529 Label* bail) {
3530 Label handleZero;
3531 Label handlePos;
3532 Label fin;
3533
3534 compareFloat(input, NoVFPRegister);
3535 // NaN is always a bail condition, just bail directly.
3536 ma_b(bail, Assembler::Overflow);
3537 ma_b(&handleZero, Assembler::Equal);
3538 ma_b(&handlePos, Assembler::NotSigned);
3539
3540 // We are in the ]-Inf; 0[ range
3541 // If we are in the ]-1; 0[ range => bailout
3542 {
3543 ScratchFloat32Scope scratch(asMasm());
3544 loadConstantFloat32(-1.f, scratch);
3545 compareFloat(input, scratch);
3546 ma_b(bail, Assembler::GreaterThan);
3547 }
3548
3549 // We are in the ]-Inf; -1] range: ceil(x) == -floor(-x) and floor can be
3550 // computed with direct truncation here (x > 0).
3551 {
3552 ScratchDoubleScope scratchDouble(asMasm());
3553 FloatRegister scratchFloat = scratchDouble.asSingle();
3554 FloatRegister scratchUInt = scratchDouble.uintOverlay();
3555
3556 ma_vneg_f32(input, scratchFloat);
3557 ma_vcvt_F32_U32(scratchFloat, scratchUInt);
3558 ma_vxfer(scratchUInt, output);
3559 ma_neg(output, output, SetCC);
3560 ma_b(bail, NotSigned);
3561 ma_b(&fin);
3562 }
3563
3564 // Test for 0.0 / -0.0: if the top word of the input double is not zero,
3565 // then it was -0 and we need to bail out.
3566 bind(&handleZero);
3567 as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
3568 Always, 0);
3569 as_cmp(output, Imm8(0));
3570 ma_b(bail, NonZero);
3571 ma_b(&fin);
3572
3573 // We are in the ]0; +inf] range: truncate integer values, maybe add 1 for
3574 // non integer values, maybe bail if overflow.
3575 bind(&handlePos);
3576 {
3577 ScratchDoubleScope scratchDouble(asMasm());
3578 FloatRegister scratchFloat = scratchDouble.asSingle();
3579 FloatRegister scratchUInt = scratchDouble.uintOverlay();
3580
3581 ma_vcvt_F32_U32(input, scratchUInt);
3582 ma_vxfer(scratchUInt, output);
3583 ma_vcvt_U32_F32(scratchUInt, scratchFloat);
3584 compareFloat(scratchFloat, input);
3585 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3586
3587 // Bail on overflow or non-positive result.
3588 ma_mov(output, output, SetCC);
3589 ma_b(bail, Signed);
3590 ma_b(bail, Zero);
3591 }
3592
3593 bind(&fin);
3594 }
3595
toggledJump(Label * label)3596 CodeOffset MacroAssemblerARMCompat::toggledJump(Label* label) {
3597 // Emit a B that can be toggled to a CMP. See ToggleToJmp(), ToggleToCmp().
3598 BufferOffset b = ma_b(label, Always);
3599 CodeOffset ret(b.getOffset());
3600 return ret;
3601 }
3602
toggledCall(JitCode * target,bool enabled)3603 CodeOffset MacroAssemblerARMCompat::toggledCall(JitCode* target, bool enabled) {
3604 BufferOffset bo = nextOffset();
3605 addPendingJump(bo, ImmPtr(target->raw()), Relocation::JITCODE);
3606 ScratchRegisterScope scratch(asMasm());
3607 ma_movPatchable(ImmPtr(target->raw()), scratch, Always);
3608 if (enabled)
3609 ma_blx(scratch);
3610 else
3611 ma_nop();
3612 return CodeOffset(bo.getOffset());
3613 }
3614
round(FloatRegister input,Register output,Label * bail,FloatRegister tmp)3615 void MacroAssemblerARMCompat::round(FloatRegister input, Register output,
3616 Label* bail, FloatRegister tmp) {
3617 Label handleZero;
3618 Label handleNeg;
3619 Label fin;
3620
3621 ScratchDoubleScope scratchDouble(asMasm());
3622
3623 // Do a compare based on the original value, then do most other things based
3624 // on the shifted value.
3625 ma_vcmpz(input);
3626 // Since we already know the sign bit, flip all numbers to be positive,
3627 // stored in tmp.
3628 ma_vabs(input, tmp);
3629 as_vmrs(pc);
3630 ma_b(&handleZero, Assembler::Equal);
3631 ma_b(&handleNeg, Assembler::Signed);
3632 // NaN is always a bail condition, just bail directly.
3633 ma_b(bail, Assembler::Overflow);
3634
3635 // The argument is a positive number, truncation is the path to glory; Since
3636 // it is known to be > 0.0, explicitly convert to a larger range, then a
3637 // value that rounds to INT_MAX is explicitly different from an argument
3638 // that clamps to INT_MAX.
3639
3640 // Add the biggest number less than 0.5 (not 0.5, because adding that to
3641 // the biggest number less than 0.5 would undesirably round up to 1), and
3642 // store the result into tmp.
3643 loadConstantDouble(GetBiggestNumberLessThan(0.5), scratchDouble);
3644 ma_vadd(scratchDouble, tmp, tmp);
3645
3646 ma_vcvt_F64_U32(tmp, scratchDouble.uintOverlay());
3647 ma_vxfer(VFPRegister(scratchDouble).uintOverlay(), output);
3648 ma_mov(output, output, SetCC);
3649 ma_b(bail, Signed);
3650 ma_b(&fin);
3651
3652 bind(&handleZero);
3653 // Move the top word of the double into the output reg, if it is non-zero,
3654 // then the original value was -0.0
3655 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3656 as_cmp(output, Imm8(0));
3657 ma_b(bail, NonZero);
3658 ma_b(&fin);
3659
3660 bind(&handleNeg);
3661 // Negative case, negate, then start dancing. This number may be positive,
3662 // since we added 0.5.
3663
3664 // Add 0.5 to negative numbers, store the result into tmp
3665 loadConstantDouble(0.5, scratchDouble);
3666 ma_vadd(scratchDouble, tmp, tmp);
3667
3668 ma_vcvt_F64_U32(tmp, scratchDouble.uintOverlay());
3669 ma_vxfer(VFPRegister(scratchDouble).uintOverlay(), output);
3670
3671 // -output is now a correctly rounded value, unless the original value was
3672 // exactly halfway between two integers, at which point, it has been rounded
3673 // away from zero, when it should be rounded towards \infty.
3674 ma_vcvt_U32_F64(scratchDouble.uintOverlay(), scratchDouble);
3675 compareDouble(scratchDouble, tmp);
3676 as_sub(output, output, Imm8(1), LeaveCC, Equal);
3677 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3678 // result will still be a negative number.
3679 as_rsb(output, output, Imm8(0), SetCC);
3680
3681 // If the result looks non-negative, then this value didn't actually fit
3682 // into the int range, and special handling is required, or it was zero,
3683 // which means the result is actually -0.0 which also requires special
3684 // handling.
3685 ma_b(bail, NotSigned);
3686
3687 bind(&fin);
3688 }
3689
roundf(FloatRegister input,Register output,Label * bail,FloatRegister tmp)3690 void MacroAssemblerARMCompat::roundf(FloatRegister input, Register output,
3691 Label* bail, FloatRegister tmp) {
3692 Label handleZero;
3693 Label handleNeg;
3694 Label fin;
3695
3696 ScratchFloat32Scope scratchFloat(asMasm());
3697
3698 // Do a compare based on the original value, then do most other things based
3699 // on the shifted value.
3700 compareFloat(input, NoVFPRegister);
3701 ma_b(&handleZero, Assembler::Equal);
3702 ma_b(&handleNeg, Assembler::Signed);
3703
3704 // NaN is always a bail condition, just bail directly.
3705 ma_b(bail, Assembler::Overflow);
3706
3707 // The argument is a positive number, truncation is the path to glory; Since
3708 // it is known to be > 0.0, explicitly convert to a larger range, then a
3709 // value that rounds to INT_MAX is explicitly different from an argument
3710 // that clamps to INT_MAX.
3711
3712 // Add the biggest number less than 0.5f (not 0.5f, because adding that to
3713 // the biggest number less than 0.5f would undesirably round up to 1), and
3714 // store the result into tmp.
3715 loadConstantFloat32(GetBiggestNumberLessThan(0.5f), scratchFloat);
3716 ma_vadd_f32(scratchFloat, input, tmp);
3717
3718 // Note: it doesn't matter whether x + .5 === x or not here, as it doesn't
3719 // affect the semantics of the float to unsigned conversion (in particular,
3720 // we are not applying any fixup after the operation).
3721 ma_vcvt_F32_U32(tmp, scratchFloat.uintOverlay());
3722 ma_vxfer(VFPRegister(scratchFloat).uintOverlay(), output);
3723 ma_mov(output, output, SetCC);
3724 ma_b(bail, Signed);
3725 ma_b(&fin);
3726
3727 bind(&handleZero);
3728
3729 // Move the whole float32 into the output reg, if it is non-zero, then the
3730 // original value was -0.0.
3731 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 0);
3732 as_cmp(output, Imm8(0));
3733 ma_b(bail, NonZero);
3734 ma_b(&fin);
3735
3736 bind(&handleNeg);
3737
3738 // Add 0.5 to negative numbers, storing the result into tmp.
3739 ma_vneg_f32(input, tmp);
3740 loadConstantFloat32(0.5f, scratchFloat);
3741 ma_vadd_f32(tmp, scratchFloat, scratchFloat);
3742
3743 // Adding 0.5 to a float input has chances to yield the wrong result, if
3744 // the input is too large. In this case, skip the -1 adjustment made below.
3745 compareFloat(scratchFloat, tmp);
3746
3747 // Negative case, negate, then start dancing. This number may be positive,
3748 // since we added 0.5.
3749 // /!\ The conditional jump afterwards depends on these two instructions
3750 // *not* setting the status flags. They need to not change after the
3751 // comparison above.
3752 ma_vcvt_F32_U32(scratchFloat, tmp.uintOverlay());
3753 ma_vxfer(VFPRegister(tmp).uintOverlay(), output);
3754
3755 Label flipSign;
3756 ma_b(&flipSign, Equal);
3757
3758 // -output is now a correctly rounded value, unless the original value was
3759 // exactly halfway between two integers, at which point, it has been rounded
3760 // away from zero, when it should be rounded towards \infty.
3761 ma_vcvt_U32_F32(tmp.uintOverlay(), tmp);
3762 compareFloat(tmp, scratchFloat);
3763 as_sub(output, output, Imm8(1), LeaveCC, Equal);
3764
3765 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3766 // result will still be a negative number.
3767 bind(&flipSign);
3768 as_rsb(output, output, Imm8(0), SetCC);
3769
3770 // If the result looks non-negative, then this value didn't actually fit
3771 // into the int range, and special handling is required, or it was zero,
3772 // which means the result is actually -0.0 which also requires special
3773 // handling.
3774 ma_b(bail, NotSigned);
3775
3776 bind(&fin);
3777 }
3778
jumpWithPatch(RepatchLabel * label,Condition cond,Label * documentation)3779 CodeOffsetJump MacroAssemblerARMCompat::jumpWithPatch(RepatchLabel* label,
3780 Condition cond,
3781 Label* documentation) {
3782 ARMBuffer::PoolEntry pe;
3783 BufferOffset bo =
3784 as_BranchPool(0xdeadbeef, label, refLabel(documentation), &pe, cond);
3785 // Fill in a new CodeOffset with both the load and the pool entry that the
3786 // instruction loads from.
3787 CodeOffsetJump ret(bo.getOffset(), pe.index());
3788 return ret;
3789 }
3790
profilerEnterFrame(Register framePtr,Register scratch)3791 void MacroAssemblerARMCompat::profilerEnterFrame(Register framePtr,
3792 Register scratch) {
3793 asMasm().loadJSContext(scratch);
3794 loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
3795 storePtr(framePtr,
3796 Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
3797 storePtr(ImmPtr(nullptr),
3798 Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
3799 }
3800
profilerExitFrame()3801 void MacroAssemblerARMCompat::profilerExitFrame() {
3802 jump(GetJitContext()->runtime->jitRuntime()->getProfilerExitFrameTail());
3803 }
3804
asMasm()3805 MacroAssembler& MacroAssemblerARM::asMasm() {
3806 return *static_cast<MacroAssembler*>(this);
3807 }
3808
asMasm() const3809 const MacroAssembler& MacroAssemblerARM::asMasm() const {
3810 return *static_cast<const MacroAssembler*>(this);
3811 }
3812
asMasm()3813 MacroAssembler& MacroAssemblerARMCompat::asMasm() {
3814 return *static_cast<MacroAssembler*>(this);
3815 }
3816
asMasm() const3817 const MacroAssembler& MacroAssemblerARMCompat::asMasm() const {
3818 return *static_cast<const MacroAssembler*>(this);
3819 }
3820
subFromStackPtr(Imm32 imm32)3821 void MacroAssembler::subFromStackPtr(Imm32 imm32) {
3822 ScratchRegisterScope scratch(*this);
3823 if (imm32.value) ma_sub(imm32, sp, scratch);
3824 }
3825
3826 //{{{ check_macroassembler_style
3827 // ===============================================================
3828 // MacroAssembler high-level usage.
3829
flush()3830 void MacroAssembler::flush() { Assembler::flush(); }
3831
comment(const char * msg)3832 void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
3833
3834 // ===============================================================
3835 // Stack manipulation functions.
3836
PushRegsInMask(LiveRegisterSet set)3837 void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
3838 int32_t diffF = set.fpus().getPushSizeInBytes();
3839 int32_t diffG = set.gprs().size() * sizeof(intptr_t);
3840
3841 if (set.gprs().size() > 1) {
3842 adjustFrame(diffG);
3843 startDataTransferM(IsStore, StackPointer, DB, WriteBack);
3844 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
3845 ++iter) {
3846 diffG -= sizeof(intptr_t);
3847 transferReg(*iter);
3848 }
3849 finishDataTransfer();
3850 } else {
3851 reserveStack(diffG);
3852 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
3853 ++iter) {
3854 diffG -= sizeof(intptr_t);
3855 storePtr(*iter, Address(StackPointer, diffG));
3856 }
3857 }
3858 MOZ_ASSERT(diffG == 0);
3859
3860 adjustFrame(diffF);
3861 diffF += transferMultipleByRuns(set.fpus(), IsStore, StackPointer, DB);
3862 MOZ_ASSERT(diffF == 0);
3863 }
3864
storeRegsInMask(LiveRegisterSet set,Address dest,Register scratch)3865 void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
3866 Register scratch) {
3867 int32_t diffF = set.fpus().getPushSizeInBytes();
3868 int32_t diffG = set.gprs().size() * sizeof(intptr_t);
3869
3870 MOZ_ASSERT(dest.offset >= diffF + diffG);
3871
3872 if (set.gprs().size() > 1) {
3873 computeEffectiveAddress(dest, scratch);
3874
3875 startDataTransferM(IsStore, scratch, DB, WriteBack);
3876 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
3877 ++iter) {
3878 diffG -= sizeof(intptr_t);
3879 dest.offset -= sizeof(intptr_t);
3880 transferReg(*iter);
3881 }
3882 finishDataTransfer();
3883 } else {
3884 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
3885 ++iter) {
3886 diffG -= sizeof(intptr_t);
3887 dest.offset -= sizeof(intptr_t);
3888 storePtr(*iter, dest);
3889 }
3890 }
3891 MOZ_ASSERT(diffG == 0);
3892
3893 if (diffF > 0) {
3894 computeEffectiveAddress(dest, scratch);
3895 diffF += transferMultipleByRuns(set.fpus(), IsStore, scratch, DB);
3896 }
3897
3898 MOZ_ASSERT(diffF == 0);
3899 }
3900
PopRegsInMaskIgnore(LiveRegisterSet set,LiveRegisterSet ignore)3901 void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
3902 LiveRegisterSet ignore) {
3903 int32_t diffG = set.gprs().size() * sizeof(intptr_t);
3904 int32_t diffF = set.fpus().getPushSizeInBytes();
3905 const int32_t reservedG = diffG;
3906 const int32_t reservedF = diffF;
3907
3908 // ARM can load multiple registers at once, but only if we want back all
3909 // the registers we previously saved to the stack.
3910 if (ignore.emptyFloat()) {
3911 diffF -= transferMultipleByRuns(set.fpus(), IsLoad, StackPointer, IA);
3912 adjustFrame(-reservedF);
3913 } else {
3914 LiveFloatRegisterSet fpset(set.fpus().reduceSetForPush());
3915 LiveFloatRegisterSet fpignore(ignore.fpus().reduceSetForPush());
3916 for (FloatRegisterBackwardIterator iter(fpset); iter.more(); ++iter) {
3917 diffF -= (*iter).size();
3918 if (!fpignore.has(*iter)) loadDouble(Address(StackPointer, diffF), *iter);
3919 }
3920 freeStack(reservedF);
3921 }
3922 MOZ_ASSERT(diffF == 0);
3923
3924 if (set.gprs().size() > 1 && ignore.emptyGeneral()) {
3925 startDataTransferM(IsLoad, StackPointer, IA, WriteBack);
3926 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
3927 ++iter) {
3928 diffG -= sizeof(intptr_t);
3929 transferReg(*iter);
3930 }
3931 finishDataTransfer();
3932 adjustFrame(-reservedG);
3933 } else {
3934 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
3935 ++iter) {
3936 diffG -= sizeof(intptr_t);
3937 if (!ignore.has(*iter)) loadPtr(Address(StackPointer, diffG), *iter);
3938 }
3939 freeStack(reservedG);
3940 }
3941 MOZ_ASSERT(diffG == 0);
3942 }
3943
Push(Register reg)3944 void MacroAssembler::Push(Register reg) {
3945 push(reg);
3946 adjustFrame(sizeof(intptr_t));
3947 }
3948
Push(const Imm32 imm)3949 void MacroAssembler::Push(const Imm32 imm) {
3950 push(imm);
3951 adjustFrame(sizeof(intptr_t));
3952 }
3953
Push(const ImmWord imm)3954 void MacroAssembler::Push(const ImmWord imm) {
3955 push(imm);
3956 adjustFrame(sizeof(intptr_t));
3957 }
3958
Push(const ImmPtr imm)3959 void MacroAssembler::Push(const ImmPtr imm) {
3960 Push(ImmWord(uintptr_t(imm.value)));
3961 }
3962
Push(const ImmGCPtr ptr)3963 void MacroAssembler::Push(const ImmGCPtr ptr) {
3964 push(ptr);
3965 adjustFrame(sizeof(intptr_t));
3966 }
3967
Push(FloatRegister reg)3968 void MacroAssembler::Push(FloatRegister reg) {
3969 VFPRegister r = VFPRegister(reg);
3970 ma_vpush(VFPRegister(reg));
3971 adjustFrame(r.size());
3972 }
3973
Pop(Register reg)3974 void MacroAssembler::Pop(Register reg) {
3975 ma_pop(reg);
3976 adjustFrame(-sizeof(intptr_t));
3977 }
3978
Pop(FloatRegister reg)3979 void MacroAssembler::Pop(FloatRegister reg) {
3980 ma_vpop(reg);
3981 adjustFrame(-reg.size());
3982 }
3983
Pop(const ValueOperand & val)3984 void MacroAssembler::Pop(const ValueOperand& val) {
3985 popValue(val);
3986 adjustFrame(-sizeof(Value));
3987 }
3988
PopStackPtr()3989 void MacroAssembler::PopStackPtr() {
3990 as_dtr(IsLoad, 32, Offset, sp, DTRAddr(sp, DtrOffImm(0)));
3991 adjustFrame(-sizeof(intptr_t));
3992 }
3993
3994 // ===============================================================
3995 // Simple call functions.
3996
call(Register reg)3997 CodeOffset MacroAssembler::call(Register reg) {
3998 as_blx(reg);
3999 return CodeOffset(currentOffset());
4000 }
4001
call(Label * label)4002 CodeOffset MacroAssembler::call(Label* label) {
4003 // For now, assume that it'll be nearby.
4004 as_bl(label, Always);
4005 return CodeOffset(currentOffset());
4006 }
4007
call(ImmWord imm)4008 void MacroAssembler::call(ImmWord imm) { call(ImmPtr((void*)imm.value)); }
4009
call(ImmPtr imm)4010 void MacroAssembler::call(ImmPtr imm) {
4011 BufferOffset bo = m_buffer.nextOffset();
4012 addPendingJump(bo, imm, Relocation::HARDCODED);
4013 ma_call(imm);
4014 }
4015
call(wasm::SymbolicAddress imm)4016 void MacroAssembler::call(wasm::SymbolicAddress imm) {
4017 movePtr(imm, CallReg);
4018 call(CallReg);
4019 }
4020
call(const Address & addr)4021 void MacroAssembler::call(const Address& addr) {
4022 loadPtr(addr, CallReg);
4023 call(CallReg);
4024 }
4025
call(JitCode * c)4026 void MacroAssembler::call(JitCode* c) {
4027 BufferOffset bo = m_buffer.nextOffset();
4028 addPendingJump(bo, ImmPtr(c->raw()), Relocation::JITCODE);
4029 ScratchRegisterScope scratch(*this);
4030 ma_movPatchable(ImmPtr(c->raw()), scratch, Always);
4031 callJitNoProfiler(scratch);
4032 }
4033
callWithPatch()4034 CodeOffset MacroAssembler::callWithPatch() {
4035 // The caller ensures that the call is always in range using thunks (below)
4036 // as necessary.
4037 as_bl(BOffImm(), Always, /* documentation */ nullptr);
4038 return CodeOffset(currentOffset());
4039 }
4040
patchCall(uint32_t callerOffset,uint32_t calleeOffset)4041 void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
4042 BufferOffset inst(callerOffset - 4);
4043 BOffImm off = BufferOffset(calleeOffset).diffB<BOffImm>(inst);
4044 MOZ_RELEASE_ASSERT(!off.isInvalid(),
4045 "Failed to insert necessary far jump islands");
4046 as_bl(off, Always, inst);
4047 }
4048
farJumpWithPatch()4049 CodeOffset MacroAssembler::farJumpWithPatch() {
4050 static_assert(32 * 1024 * 1024 - JumpImmediateRange >
4051 wasm::MaxFuncs * 3 * sizeof(Instruction),
4052 "always enough space for thunks");
4053
4054 // The goal of the thunk is to be able to jump to any address without the
4055 // usual 32MiB branch range limitation. Additionally, to make the thunk
4056 // simple to use, the thunk does not use the constant pool or require
4057 // patching an absolute address. Instead, a relative offset is used which
4058 // can be patched during compilation.
4059
4060 // Inhibit pools since these three words must be contiguous so that the offset
4061 // calculations below are valid.
4062 AutoForbidPools afp(this, 3);
4063
4064 // When pc is used, the read value is the address of the instruction + 8.
4065 // This is exactly the address of the uint32 word we want to load.
4066 ScratchRegisterScope scratch(*this);
4067 ma_ldr(DTRAddr(pc, DtrOffImm(0)), scratch);
4068
4069 // Branch by making pc the destination register.
4070 ma_add(pc, scratch, pc, LeaveCC, Always);
4071
4072 // Allocate space which will be patched by patchFarJump().
4073 CodeOffset farJump(currentOffset());
4074 writeInst(UINT32_MAX);
4075
4076 return farJump;
4077 }
4078
patchFarJump(CodeOffset farJump,uint32_t targetOffset)4079 void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
4080 uint32_t* u32 =
4081 reinterpret_cast<uint32_t*>(editSrc(BufferOffset(farJump.offset())));
4082 MOZ_ASSERT(*u32 == UINT32_MAX);
4083
4084 uint32_t addOffset = farJump.offset() - 4;
4085 MOZ_ASSERT(editSrc(BufferOffset(addOffset))->is<InstALU>());
4086
4087 // When pc is read as the operand of the add, its value is the address of
4088 // the add instruction + 8.
4089 *u32 = (targetOffset - addOffset) - 8;
4090 }
4091
repatchFarJump(uint8_t * code,uint32_t farJumpOffset,uint32_t targetOffset)4092 void MacroAssembler::repatchFarJump(uint8_t* code, uint32_t farJumpOffset,
4093 uint32_t targetOffset) {
4094 uint32_t* u32 = reinterpret_cast<uint32_t*>(code + farJumpOffset);
4095
4096 uint32_t addOffset = farJumpOffset - 4;
4097 MOZ_ASSERT(reinterpret_cast<Instruction*>(code + addOffset)->is<InstALU>());
4098
4099 *u32 = (targetOffset - addOffset) - 8;
4100 }
4101
nopPatchableToNearJump()4102 CodeOffset MacroAssembler::nopPatchableToNearJump() {
4103 // Inhibit pools so that the offset points precisely to the nop.
4104 AutoForbidPools afp(this, 1);
4105
4106 CodeOffset offset(currentOffset());
4107 ma_nop();
4108 return offset;
4109 }
4110
patchNopToNearJump(uint8_t * jump,uint8_t * target)4111 void MacroAssembler::patchNopToNearJump(uint8_t* jump, uint8_t* target) {
4112 MOZ_ASSERT(reinterpret_cast<Instruction*>(jump)->is<InstNOP>());
4113 new (jump) InstBImm(BOffImm(target - jump), Assembler::Always);
4114 }
4115
patchNearJumpToNop(uint8_t * jump)4116 void MacroAssembler::patchNearJumpToNop(uint8_t* jump) {
4117 MOZ_ASSERT(reinterpret_cast<Instruction*>(jump)->is<InstBImm>());
4118 new (jump) InstNOP();
4119 }
4120
nopPatchableToCall(const wasm::CallSiteDesc & desc)4121 CodeOffset MacroAssembler::nopPatchableToCall(const wasm::CallSiteDesc& desc) {
4122 CodeOffset offset(currentOffset());
4123 ma_nop();
4124 append(desc, CodeOffset(currentOffset()));
4125 return offset;
4126 }
4127
patchNopToCall(uint8_t * call,uint8_t * target)4128 void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
4129 uint8_t* inst = call - 4;
4130 MOZ_ASSERT(reinterpret_cast<Instruction*>(inst)->is<InstBLImm>() ||
4131 reinterpret_cast<Instruction*>(inst)->is<InstNOP>());
4132
4133 new (inst) InstBLImm(BOffImm(target - inst), Assembler::Always);
4134 }
4135
patchCallToNop(uint8_t * call)4136 void MacroAssembler::patchCallToNop(uint8_t* call) {
4137 uint8_t* inst = call - 4;
4138 MOZ_ASSERT(reinterpret_cast<Instruction*>(inst)->is<InstBLImm>() ||
4139 reinterpret_cast<Instruction*>(inst)->is<InstNOP>());
4140 new (inst) InstNOP();
4141 }
4142
pushReturnAddress()4143 void MacroAssembler::pushReturnAddress() { push(lr); }
4144
popReturnAddress()4145 void MacroAssembler::popReturnAddress() { pop(lr); }
4146
4147 // ===============================================================
4148 // ABI function calls.
4149
setupUnalignedABICall(Register scratch)4150 void MacroAssembler::setupUnalignedABICall(Register scratch) {
4151 MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls");
4152 setupABICall();
4153 dynamicAlignment_ = true;
4154
4155 ma_mov(sp, scratch);
4156 // Force sp to be aligned.
4157 as_bic(sp, sp, Imm8(ABIStackAlignment - 1));
4158 ma_push(scratch);
4159 }
4160
callWithABIPre(uint32_t * stackAdjust,bool callFromWasm)4161 void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
4162 MOZ_ASSERT(inCall_);
4163 uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
4164
4165 if (dynamicAlignment_) {
4166 // sizeof(intptr_t) accounts for the saved stack pointer pushed by
4167 // setupUnalignedABICall.
4168 stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
4169 ABIStackAlignment);
4170 } else {
4171 uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
4172 stackForCall += ComputeByteAlignment(
4173 stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
4174 }
4175
4176 *stackAdjust = stackForCall;
4177 reserveStack(stackForCall);
4178
4179 // Position all arguments.
4180 {
4181 enoughMemory_ = enoughMemory_ && moveResolver_.resolve();
4182 if (!enoughMemory_) return;
4183
4184 MoveEmitter emitter(*this);
4185 emitter.emit(moveResolver_);
4186 emitter.finish();
4187 }
4188
4189 assertStackAlignment(ABIStackAlignment);
4190
4191 // Save the lr register if we need to preserve it.
4192 if (secondScratchReg_ != lr) ma_mov(lr, secondScratchReg_);
4193 }
4194
callWithABIPost(uint32_t stackAdjust,MoveOp::Type result,bool callFromWasm)4195 void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
4196 bool callFromWasm) {
4197 if (secondScratchReg_ != lr) ma_mov(secondScratchReg_, lr);
4198
4199 // Calls to native functions in wasm pass through a thunk which already
4200 // fixes up the return value for us.
4201 if (!callFromWasm && !UseHardFpABI()) {
4202 switch (result) {
4203 case MoveOp::DOUBLE:
4204 // Move double from r0/r1 to ReturnFloatReg.
4205 ma_vxfer(r0, r1, ReturnDoubleReg);
4206 break;
4207 case MoveOp::FLOAT32:
4208 // Move float32 from r0 to ReturnFloatReg.
4209 ma_vxfer(r0, ReturnFloat32Reg);
4210 break;
4211 case MoveOp::GENERAL:
4212 break;
4213 default:
4214 MOZ_CRASH("unexpected callWithABI result");
4215 }
4216 }
4217
4218 freeStack(stackAdjust);
4219
4220 if (dynamicAlignment_) {
4221 // While the x86 supports pop esp, on ARM that isn't well defined, so
4222 // just do it manually.
4223 as_dtr(IsLoad, 32, Offset, sp, DTRAddr(sp, DtrOffImm(0)));
4224 }
4225
4226 #ifdef DEBUG
4227 MOZ_ASSERT(inCall_);
4228 inCall_ = false;
4229 #endif
4230 }
4231
callWithABINoProfiler(Register fun,MoveOp::Type result)4232 void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
4233 // Load the callee in r12, as above.
4234 ma_mov(fun, r12);
4235 uint32_t stackAdjust;
4236 callWithABIPre(&stackAdjust);
4237 call(r12);
4238 callWithABIPost(stackAdjust, result);
4239 }
4240
callWithABINoProfiler(const Address & fun,MoveOp::Type result)4241 void MacroAssembler::callWithABINoProfiler(const Address& fun,
4242 MoveOp::Type result) {
4243 // Load the callee in r12, no instruction between the ldr and call should
4244 // clobber it. Note that we can't use fun.base because it may be one of the
4245 // IntArg registers clobbered before the call.
4246 {
4247 ScratchRegisterScope scratch(*this);
4248 ma_ldr(fun, r12, scratch);
4249 }
4250 uint32_t stackAdjust;
4251 callWithABIPre(&stackAdjust);
4252 call(r12);
4253 callWithABIPost(stackAdjust, result);
4254 }
4255
4256 // ===============================================================
4257 // Jit Frames.
4258
pushFakeReturnAddress(Register scratch)4259 uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
4260 // On ARM any references to the pc, adds an additional 8 to it, which
4261 // correspond to 2 instructions of 4 bytes. Thus we use an additional nop
4262 // to pad until we reach the pushed pc.
4263 //
4264 // Note: In practice this should not be necessary, as this fake return
4265 // address is never used for resuming any execution. Thus theoriticaly we
4266 // could just do a Push(pc), and ignore the nop as well as the pool.
4267 enterNoPool(2);
4268 DebugOnly<uint32_t> offsetBeforePush = currentOffset();
4269 Push(pc); // actually pushes $pc + 8.
4270 ma_nop();
4271 uint32_t pseudoReturnOffset = currentOffset();
4272 leaveNoPool();
4273
4274 MOZ_ASSERT_IF(!oom(), pseudoReturnOffset - offsetBeforePush == 8);
4275 return pseudoReturnOffset;
4276 }
4277
4278 // ===============================================================
4279 // Move instructions
4280
moveValue(const TypedOrValueRegister & src,const ValueOperand & dest)4281 void MacroAssembler::moveValue(const TypedOrValueRegister& src,
4282 const ValueOperand& dest) {
4283 if (src.hasValue()) {
4284 moveValue(src.valueReg(), dest);
4285 return;
4286 }
4287
4288 MIRType type = src.type();
4289 AnyRegister reg = src.typedReg();
4290
4291 if (!IsFloatingPointType(type)) {
4292 mov(ImmWord(MIRTypeToTag(type)), dest.typeReg());
4293 if (reg.gpr() != dest.payloadReg()) mov(reg.gpr(), dest.payloadReg());
4294 return;
4295 }
4296
4297 ScratchDoubleScope scratch(*this);
4298 FloatRegister freg = reg.fpu();
4299 if (type == MIRType::Float32) {
4300 convertFloat32ToDouble(freg, ScratchFloat32Reg);
4301 freg = ScratchFloat32Reg;
4302 }
4303 ma_vxfer(freg, dest.payloadReg(), dest.typeReg());
4304 }
4305
moveValue(const ValueOperand & src,const ValueOperand & dest)4306 void MacroAssembler::moveValue(const ValueOperand& src,
4307 const ValueOperand& dest) {
4308 Register s0 = src.typeReg();
4309 Register s1 = src.payloadReg();
4310 Register d0 = dest.typeReg();
4311 Register d1 = dest.payloadReg();
4312
4313 // Either one or both of the source registers could be the same as a
4314 // destination register.
4315 if (s1 == d0) {
4316 if (s0 == d1) {
4317 // If both are, this is just a swap of two registers.
4318 ScratchRegisterScope scratch(*this);
4319 MOZ_ASSERT(d1 != scratch);
4320 MOZ_ASSERT(d0 != scratch);
4321 ma_mov(d1, scratch);
4322 ma_mov(d0, d1);
4323 ma_mov(scratch, d0);
4324 return;
4325 }
4326 // If only one is, copy that source first.
4327 mozilla::Swap(s0, s1);
4328 mozilla::Swap(d0, d1);
4329 }
4330
4331 if (s0 != d0) ma_mov(s0, d0);
4332 if (s1 != d1) ma_mov(s1, d1);
4333 }
4334
moveValue(const Value & src,const ValueOperand & dest)4335 void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
4336 ma_mov(Imm32(src.toNunboxTag()), dest.typeReg());
4337 if (src.isGCThing())
4338 ma_mov(ImmGCPtr(src.toGCThing()), dest.payloadReg());
4339 else
4340 ma_mov(Imm32(src.toNunboxPayload()), dest.payloadReg());
4341 }
4342
4343 // ===============================================================
4344 // Branch functions
4345
loadStoreBuffer(Register ptr,Register buffer)4346 void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
4347 ma_lsr(Imm32(gc::ChunkShift), ptr, buffer);
4348 ma_lsl(Imm32(gc::ChunkShift), buffer, buffer);
4349 load32(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
4350 }
4351
branchPtrInNurseryChunk(Condition cond,Register ptr,Register temp,Label * label)4352 void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
4353 Register temp, Label* label) {
4354 Maybe<SecondScratchRegisterScope> scratch2;
4355 if (temp == Register::Invalid()) {
4356 scratch2.emplace(*this);
4357 temp = scratch2.ref();
4358 }
4359
4360 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4361 MOZ_ASSERT(ptr != temp);
4362
4363 ma_lsr(Imm32(gc::ChunkShift), ptr, temp);
4364 ma_lsl(Imm32(gc::ChunkShift), temp, temp);
4365 load32(Address(temp, gc::ChunkLocationOffset), temp);
4366 branch32(cond, temp, Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
4367 }
4368
branchValueIsNurseryCell(Condition cond,const Address & address,Register temp,Label * label)4369 void MacroAssembler::branchValueIsNurseryCell(Condition cond,
4370 const Address& address,
4371 Register temp, Label* label) {
4372 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4373 Label done, checkAddress;
4374
4375 Register tag = temp;
4376 extractTag(address, tag);
4377 branchTestObject(Assembler::Equal, tag, &checkAddress);
4378 branchTestString(Assembler::NotEqual, tag,
4379 cond == Assembler::Equal ? &done : label);
4380
4381 bind(&checkAddress);
4382 loadPtr(ToPayload(address), temp);
4383 SecondScratchRegisterScope scratch2(*this);
4384 branchPtrInNurseryChunk(cond, temp, scratch2, label);
4385
4386 bind(&done);
4387 }
4388
branchValueIsNurseryCell(Condition cond,ValueOperand value,Register temp,Label * label)4389 void MacroAssembler::branchValueIsNurseryCell(Condition cond,
4390 ValueOperand value, Register temp,
4391 Label* label) {
4392 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4393 Label done, checkAddress;
4394
4395 branchTestObject(Assembler::Equal, value.typeReg(), &checkAddress);
4396 branchTestString(Assembler::NotEqual, value.typeReg(),
4397 cond == Assembler::Equal ? &done : label);
4398
4399 bind(&checkAddress);
4400 branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
4401
4402 bind(&done);
4403 }
4404
branchValueIsNurseryObject(Condition cond,ValueOperand value,Register temp,Label * label)4405 void MacroAssembler::branchValueIsNurseryObject(Condition cond,
4406 ValueOperand value,
4407 Register temp, Label* label) {
4408 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4409 Label done;
4410
4411 branchTestObject(Assembler::NotEqual, value,
4412 cond == Assembler::Equal ? &done : label);
4413 branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
4414
4415 bind(&done);
4416 }
4417
branchTestValue(Condition cond,const ValueOperand & lhs,const Value & rhs,Label * label)4418 void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
4419 const Value& rhs, Label* label) {
4420 MOZ_ASSERT(cond == Equal || cond == NotEqual);
4421 // If cond == NotEqual, branch when a.payload != b.payload || a.tag !=
4422 // b.tag. If the payloads are equal, compare the tags. If the payloads are
4423 // not equal, short circuit true (NotEqual).
4424 //
4425 // If cand == Equal, branch when a.payload == b.payload && a.tag == b.tag.
4426 // If the payloads are equal, compare the tags. If the payloads are not
4427 // equal, short circuit false (NotEqual).
4428 ScratchRegisterScope scratch(*this);
4429
4430 if (rhs.isGCThing())
4431 ma_cmp(lhs.payloadReg(), ImmGCPtr(rhs.toGCThing()), scratch);
4432 else
4433 ma_cmp(lhs.payloadReg(), Imm32(rhs.toNunboxPayload()), scratch);
4434 ma_cmp(lhs.typeReg(), Imm32(rhs.toNunboxTag()), scratch, Equal);
4435 ma_b(label, cond);
4436 }
4437
4438 // ========================================================================
4439 // Memory access primitives.
4440 template <typename T>
storeUnboxedValue(const ConstantOrRegister & value,MIRType valueType,const T & dest,MIRType slotType)4441 void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
4442 MIRType valueType, const T& dest,
4443 MIRType slotType) {
4444 if (valueType == MIRType::Double) {
4445 storeDouble(value.reg().typedReg().fpu(), dest);
4446 return;
4447 }
4448
4449 // Store the type tag if needed.
4450 if (valueType != slotType)
4451 storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), dest);
4452
4453 // Store the payload.
4454 if (value.constant())
4455 storePayload(value.value(), dest);
4456 else
4457 storePayload(value.reg().typedReg().gpr(), dest);
4458 }
4459
4460 template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
4461 MIRType valueType,
4462 const Address& dest,
4463 MIRType slotType);
4464 template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
4465 MIRType valueType,
4466 const BaseIndex& dest,
4467 MIRType slotType);
4468
wasmTrapInstruction()4469 CodeOffset MacroAssembler::wasmTrapInstruction() {
4470 return CodeOffset(as_illegal_trap().getOffset());
4471 }
4472
wasmTruncateDoubleToUInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4473 void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
4474 Register output,
4475 bool isSaturating,
4476 Label* oolEntry) {
4477 wasmTruncateToInt32(input, output, MIRType::Double, /* isUnsigned= */ true,
4478 isSaturating, oolEntry);
4479 }
4480
wasmTruncateDoubleToInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4481 void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
4482 Register output,
4483 bool isSaturating,
4484 Label* oolEntry) {
4485 wasmTruncateToInt32(input, output, MIRType::Double, /* isUnsigned= */ false,
4486 isSaturating, oolEntry);
4487 }
4488
wasmTruncateFloat32ToUInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4489 void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
4490 Register output,
4491 bool isSaturating,
4492 Label* oolEntry) {
4493 wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ true,
4494 isSaturating, oolEntry);
4495 }
4496
wasmTruncateFloat32ToInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4497 void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
4498 Register output,
4499 bool isSaturating,
4500 Label* oolEntry) {
4501 wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ false,
4502 isSaturating, oolEntry);
4503 }
4504
oolWasmTruncateCheckF32ToI32(FloatRegister input,Register output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4505 void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
4506 Register output,
4507 TruncFlags flags,
4508 wasm::BytecodeOffset off,
4509 Label* rejoin) {
4510 outOfLineWasmTruncateToIntCheck(input, MIRType::Float32, MIRType::Int32,
4511 flags, rejoin, off);
4512 }
4513
oolWasmTruncateCheckF64ToI32(FloatRegister input,Register output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4514 void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
4515 Register output,
4516 TruncFlags flags,
4517 wasm::BytecodeOffset off,
4518 Label* rejoin) {
4519 outOfLineWasmTruncateToIntCheck(input, MIRType::Double, MIRType::Int32, flags,
4520 rejoin, off);
4521 }
4522
oolWasmTruncateCheckF32ToI64(FloatRegister input,Register64 output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4523 void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
4524 Register64 output,
4525 TruncFlags flags,
4526 wasm::BytecodeOffset off,
4527 Label* rejoin) {
4528 outOfLineWasmTruncateToIntCheck(input, MIRType::Float32, MIRType::Int64,
4529 flags, rejoin, off);
4530 }
4531
oolWasmTruncateCheckF64ToI64(FloatRegister input,Register64 output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4532 void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
4533 Register64 output,
4534 TruncFlags flags,
4535 wasm::BytecodeOffset off,
4536 Label* rejoin) {
4537 outOfLineWasmTruncateToIntCheck(input, MIRType::Double, MIRType::Int64, flags,
4538 rejoin, off);
4539 }
4540
wasmLoad(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,AnyRegister output)4541 void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
4542 Register memoryBase, Register ptr,
4543 Register ptrScratch, AnyRegister output) {
4544 wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output,
4545 Register64::Invalid());
4546 }
4547
wasmLoadI64(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,Register64 output)4548 void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
4549 Register memoryBase, Register ptr,
4550 Register ptrScratch, Register64 output) {
4551 MOZ_ASSERT_IF(access.isAtomic(), access.byteSize() <= 4);
4552 wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(), output);
4553 }
4554
wasmStore(const wasm::MemoryAccessDesc & access,AnyRegister value,Register memoryBase,Register ptr,Register ptrScratch)4555 void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
4556 AnyRegister value, Register memoryBase,
4557 Register ptr, Register ptrScratch) {
4558 wasmStoreImpl(access, value, Register64::Invalid(), memoryBase, ptr,
4559 ptrScratch);
4560 }
4561
wasmStoreI64(const wasm::MemoryAccessDesc & access,Register64 value,Register memoryBase,Register ptr,Register ptrScratch)4562 void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
4563 Register64 value, Register memoryBase,
4564 Register ptr, Register ptrScratch) {
4565 MOZ_ASSERT(!access.isAtomic());
4566 wasmStoreImpl(access, AnyRegister(), value, memoryBase, ptr, ptrScratch);
4567 }
4568
wasmUnalignedLoad(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,Register output,Register tmp)4569 void MacroAssembler::wasmUnalignedLoad(const wasm::MemoryAccessDesc& access,
4570 Register memoryBase, Register ptr,
4571 Register ptrScratch, Register output,
4572 Register tmp) {
4573 wasmUnalignedLoadImpl(access, memoryBase, ptr, ptrScratch,
4574 AnyRegister(output), Register64::Invalid(), tmp,
4575 Register::Invalid(), Register::Invalid());
4576 }
4577
wasmUnalignedLoadFP(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,FloatRegister outFP,Register tmp1,Register tmp2,Register tmp3)4578 void MacroAssembler::wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access,
4579 Register memoryBase, Register ptr,
4580 Register ptrScratch,
4581 FloatRegister outFP, Register tmp1,
4582 Register tmp2, Register tmp3) {
4583 wasmUnalignedLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(outFP),
4584 Register64::Invalid(), tmp1, tmp2, tmp3);
4585 }
4586
wasmUnalignedLoadI64(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,Register64 out64,Register tmp)4587 void MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
4588 Register memoryBase, Register ptr,
4589 Register ptrScratch, Register64 out64,
4590 Register tmp) {
4591 wasmUnalignedLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(),
4592 out64, tmp, Register::Invalid(), Register::Invalid());
4593 }
4594
wasmUnalignedStore(const wasm::MemoryAccessDesc & access,Register value,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)4595 void MacroAssembler::wasmUnalignedStore(const wasm::MemoryAccessDesc& access,
4596 Register value, Register memoryBase,
4597 Register ptr, Register ptrScratch,
4598 Register tmp) {
4599 MOZ_ASSERT(tmp == Register::Invalid());
4600 wasmUnalignedStoreImpl(access, FloatRegister(), Register64::Invalid(),
4601 memoryBase, ptr, ptrScratch, value);
4602 }
4603
wasmUnalignedStoreFP(const wasm::MemoryAccessDesc & access,FloatRegister floatVal,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)4604 void MacroAssembler::wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access,
4605 FloatRegister floatVal,
4606 Register memoryBase, Register ptr,
4607 Register ptrScratch, Register tmp) {
4608 wasmUnalignedStoreImpl(access, floatVal, Register64::Invalid(), memoryBase,
4609 ptr, ptrScratch, tmp);
4610 }
4611
wasmUnalignedStoreI64(const wasm::MemoryAccessDesc & access,Register64 val64,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)4612 void MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
4613 Register64 val64,
4614 Register memoryBase, Register ptr,
4615 Register ptrScratch, Register tmp) {
4616 wasmUnalignedStoreImpl(access, FloatRegister(), val64, memoryBase, ptr,
4617 ptrScratch, tmp);
4618 }
4619
4620 // ========================================================================
4621 // Primitive atomic operations.
4622
ComputePointerForAtomic(MacroAssembler & masm,const BaseIndex & src,Register r)4623 static Register ComputePointerForAtomic(MacroAssembler& masm,
4624 const BaseIndex& src, Register r) {
4625 Register base = src.base;
4626 Register index = src.index;
4627 uint32_t scale = Imm32::ShiftOf(src.scale).value;
4628 int32_t offset = src.offset;
4629
4630 ScratchRegisterScope scratch(masm);
4631
4632 masm.as_add(r, base, lsl(index, scale));
4633 if (offset != 0) masm.ma_add(r, Imm32(offset), r, scratch);
4634 return r;
4635 }
4636
ComputePointerForAtomic(MacroAssembler & masm,const Address & src,Register r)4637 static Register ComputePointerForAtomic(MacroAssembler& masm,
4638 const Address& src, Register r) {
4639 ScratchRegisterScope scratch(masm);
4640 if (src.offset == 0) return src.base;
4641 masm.ma_add(src.base, Imm32(src.offset), r, scratch);
4642 return r;
4643 }
4644
4645 // General algorithm:
4646 //
4647 // ... ptr, <addr> ; compute address of item
4648 // dmb
4649 // L0 ldrex* output, [ptr]
4650 // sxt* output, output, 0 ; sign-extend if applicable
4651 // *xt* tmp, oldval, 0 ; sign-extend or zero-extend if applicable
4652 // cmp output, tmp
4653 // bne L1 ; failed - values are different
4654 // strex* tmp, newval, [ptr]
4655 // cmp tmp, 1
4656 // beq L0 ; failed - location is dirty, retry
4657 // L1 dmb
4658 //
4659 // Discussion here: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html.
4660 // However note that that discussion uses 'isb' as the trailing fence.
4661 // I've not quite figured out why, and I've gone with dmb here which
4662 // is safe. Also see the LLVM source, which uses 'dmb ish' generally.
4663 // (Apple's Swift CPU apparently handles ish in a non-default, faster
4664 // way.)
4665
4666 template <typename T>
CompareExchange(MacroAssembler & masm,Scalar::Type type,const Synchronization & sync,const T & mem,Register oldval,Register newval,Register output)4667 static void CompareExchange(MacroAssembler& masm, Scalar::Type type,
4668 const Synchronization& sync, const T& mem,
4669 Register oldval, Register newval, Register output) {
4670 bool signExtend = Scalar::isSignedIntType(type);
4671 unsigned nbytes = Scalar::byteSize(type);
4672
4673 MOZ_ASSERT(nbytes <= 4);
4674
4675 Label again;
4676 Label done;
4677
4678 SecondScratchRegisterScope scratch2(masm);
4679 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
4680
4681 ScratchRegisterScope scratch(masm);
4682
4683 masm.memoryBarrierBefore(sync);
4684
4685 masm.bind(&again);
4686 switch (nbytes) {
4687 case 1:
4688 masm.as_ldrexb(output, ptr);
4689 if (signExtend) {
4690 masm.as_sxtb(output, output, 0);
4691 masm.as_sxtb(scratch, oldval, 0);
4692 } else {
4693 masm.as_uxtb(scratch, oldval, 0);
4694 }
4695 break;
4696 case 2:
4697 masm.as_ldrexh(output, ptr);
4698 if (signExtend) {
4699 masm.as_sxth(output, output, 0);
4700 masm.as_sxth(scratch, oldval, 0);
4701 } else {
4702 masm.as_uxth(scratch, oldval, 0);
4703 }
4704 break;
4705 case 4:
4706 masm.as_ldrex(output, ptr);
4707 break;
4708 }
4709 if (nbytes < 4)
4710 masm.as_cmp(output, O2Reg(scratch));
4711 else
4712 masm.as_cmp(output, O2Reg(oldval));
4713 masm.as_b(&done, MacroAssembler::NotEqual);
4714 switch (nbytes) {
4715 case 1:
4716 masm.as_strexb(scratch, newval, ptr);
4717 break;
4718 case 2:
4719 masm.as_strexh(scratch, newval, ptr);
4720 break;
4721 case 4:
4722 masm.as_strex(scratch, newval, ptr);
4723 break;
4724 }
4725 masm.as_cmp(scratch, Imm8(1));
4726 masm.as_b(&again, MacroAssembler::Equal);
4727 masm.bind(&done);
4728
4729 masm.memoryBarrierAfter(sync);
4730 }
4731
compareExchange(Scalar::Type type,const Synchronization & sync,const Address & address,Register oldval,Register newval,Register output)4732 void MacroAssembler::compareExchange(Scalar::Type type,
4733 const Synchronization& sync,
4734 const Address& address, Register oldval,
4735 Register newval, Register output) {
4736 CompareExchange(*this, type, sync, address, oldval, newval, output);
4737 }
4738
compareExchange(Scalar::Type type,const Synchronization & sync,const BaseIndex & address,Register oldval,Register newval,Register output)4739 void MacroAssembler::compareExchange(Scalar::Type type,
4740 const Synchronization& sync,
4741 const BaseIndex& address, Register oldval,
4742 Register newval, Register output) {
4743 CompareExchange(*this, type, sync, address, oldval, newval, output);
4744 }
4745
4746 template <typename T>
AtomicExchange(MacroAssembler & masm,Scalar::Type type,const Synchronization & sync,const T & mem,Register value,Register output)4747 static void AtomicExchange(MacroAssembler& masm, Scalar::Type type,
4748 const Synchronization& sync, const T& mem,
4749 Register value, Register output) {
4750 bool signExtend = Scalar::isSignedIntType(type);
4751 unsigned nbytes = Scalar::byteSize(type);
4752
4753 MOZ_ASSERT(nbytes <= 4);
4754
4755 // Bug 1077321: We may further optimize for ARMv8 (AArch32) here.
4756 Label again;
4757 Label done;
4758
4759 SecondScratchRegisterScope scratch2(masm);
4760 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
4761
4762 ScratchRegisterScope scratch(masm);
4763
4764 masm.memoryBarrierBefore(sync);
4765
4766 masm.bind(&again);
4767 switch (nbytes) {
4768 case 1:
4769 masm.as_ldrexb(output, ptr);
4770 if (signExtend) masm.as_sxtb(output, output, 0);
4771 masm.as_strexb(scratch, value, ptr);
4772 break;
4773 case 2:
4774 masm.as_ldrexh(output, ptr);
4775 if (signExtend) masm.as_sxth(output, output, 0);
4776 masm.as_strexh(scratch, value, ptr);
4777 break;
4778 case 4:
4779 masm.as_ldrex(output, ptr);
4780 masm.as_strex(scratch, value, ptr);
4781 break;
4782 }
4783 masm.as_cmp(scratch, Imm8(1));
4784 masm.as_b(&again, MacroAssembler::Equal);
4785 masm.bind(&done);
4786
4787 masm.memoryBarrierAfter(sync);
4788 }
4789
atomicExchange(Scalar::Type type,const Synchronization & sync,const Address & address,Register value,Register output)4790 void MacroAssembler::atomicExchange(Scalar::Type type,
4791 const Synchronization& sync,
4792 const Address& address, Register value,
4793 Register output) {
4794 AtomicExchange(*this, type, sync, address, value, output);
4795 }
4796
atomicExchange(Scalar::Type type,const Synchronization & sync,const BaseIndex & address,Register value,Register output)4797 void MacroAssembler::atomicExchange(Scalar::Type type,
4798 const Synchronization& sync,
4799 const BaseIndex& address, Register value,
4800 Register output) {
4801 AtomicExchange(*this, type, sync, address, value, output);
4802 }
4803
4804 // General algorithm:
4805 //
4806 // ... ptr, <addr> ; compute address of item
4807 // dmb
4808 // L0 ldrex* output, [ptr]
4809 // sxt* output, output, 0 ; sign-extend if applicable
4810 // OP tmp, output, value ; compute value to store
4811 // strex* tmp2, tmp, [ptr] ; tmp2 required by strex
4812 // cmp tmp2, 1
4813 // beq L0 ; failed - location is dirty, retry
4814 // dmb ; ordering barrier required
4815 //
4816 // Also see notes above at compareExchange re the barrier strategy.
4817 //
4818 // Observe that the value being operated into the memory element need
4819 // not be sign-extended because no OP will make use of bits to the
4820 // left of the bits indicated by the width of the element, and neither
4821 // output nor the bits stored are affected by OP.
4822
4823 template <typename T>
AtomicFetchOp(MacroAssembler & masm,Scalar::Type type,const Synchronization & sync,AtomicOp op,const Register & value,const T & mem,Register flagTemp,Register output)4824 static void AtomicFetchOp(MacroAssembler& masm, Scalar::Type type,
4825 const Synchronization& sync, AtomicOp op,
4826 const Register& value, const T& mem,
4827 Register flagTemp, Register output) {
4828 bool signExtend = Scalar::isSignedIntType(type);
4829 unsigned nbytes = Scalar::byteSize(type);
4830
4831 MOZ_ASSERT(nbytes <= 4);
4832 MOZ_ASSERT(flagTemp != InvalidReg);
4833 MOZ_ASSERT(output != value);
4834
4835 Label again;
4836
4837 SecondScratchRegisterScope scratch2(masm);
4838 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
4839
4840 masm.memoryBarrierBefore(sync);
4841
4842 ScratchRegisterScope scratch(masm);
4843
4844 masm.bind(&again);
4845 switch (nbytes) {
4846 case 1:
4847 masm.as_ldrexb(output, ptr);
4848 if (signExtend) masm.as_sxtb(output, output, 0);
4849 break;
4850 case 2:
4851 masm.as_ldrexh(output, ptr);
4852 if (signExtend) masm.as_sxth(output, output, 0);
4853 break;
4854 case 4:
4855 masm.as_ldrex(output, ptr);
4856 break;
4857 }
4858 switch (op) {
4859 case AtomicFetchAddOp:
4860 masm.as_add(scratch, output, O2Reg(value));
4861 break;
4862 case AtomicFetchSubOp:
4863 masm.as_sub(scratch, output, O2Reg(value));
4864 break;
4865 case AtomicFetchAndOp:
4866 masm.as_and(scratch, output, O2Reg(value));
4867 break;
4868 case AtomicFetchOrOp:
4869 masm.as_orr(scratch, output, O2Reg(value));
4870 break;
4871 case AtomicFetchXorOp:
4872 masm.as_eor(scratch, output, O2Reg(value));
4873 break;
4874 }
4875 // Rd must differ from the two other arguments to strex.
4876 switch (nbytes) {
4877 case 1:
4878 masm.as_strexb(flagTemp, scratch, ptr);
4879 break;
4880 case 2:
4881 masm.as_strexh(flagTemp, scratch, ptr);
4882 break;
4883 case 4:
4884 masm.as_strex(flagTemp, scratch, ptr);
4885 break;
4886 }
4887 masm.as_cmp(flagTemp, Imm8(1));
4888 masm.as_b(&again, MacroAssembler::Equal);
4889
4890 masm.memoryBarrierAfter(sync);
4891 }
4892
atomicFetchOp(Scalar::Type type,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register temp,Register output)4893 void MacroAssembler::atomicFetchOp(Scalar::Type type,
4894 const Synchronization& sync, AtomicOp op,
4895 Register value, const Address& mem,
4896 Register temp, Register output) {
4897 AtomicFetchOp(*this, type, sync, op, value, mem, temp, output);
4898 }
4899
atomicFetchOp(Scalar::Type type,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register temp,Register output)4900 void MacroAssembler::atomicFetchOp(Scalar::Type type,
4901 const Synchronization& sync, AtomicOp op,
4902 Register value, const BaseIndex& mem,
4903 Register temp, Register output) {
4904 AtomicFetchOp(*this, type, sync, op, value, mem, temp, output);
4905 }
4906
4907 // Uses both scratch registers, one for the address and one for a temp,
4908 // but needs two temps for strex:
4909 //
4910 // ... ptr, <addr> ; compute address of item
4911 // dmb
4912 // L0 ldrex* temp, [ptr]
4913 // OP temp, temp, value ; compute value to store
4914 // strex* temp2, temp, [ptr]
4915 // cmp temp2, 1
4916 // beq L0 ; failed - location is dirty, retry
4917 // dmb ; ordering barrier required
4918
4919 template <typename T>
AtomicEffectOp(MacroAssembler & masm,Scalar::Type type,const Synchronization & sync,AtomicOp op,const Register & value,const T & mem,Register flagTemp)4920 static void AtomicEffectOp(MacroAssembler& masm, Scalar::Type type,
4921 const Synchronization& sync, AtomicOp op,
4922 const Register& value, const T& mem,
4923 Register flagTemp) {
4924 unsigned nbytes = Scalar::byteSize(type);
4925
4926 MOZ_ASSERT(nbytes <= 4);
4927 MOZ_ASSERT(flagTemp != InvalidReg);
4928
4929 Label again;
4930
4931 SecondScratchRegisterScope scratch2(masm);
4932 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
4933
4934 masm.memoryBarrierBefore(sync);
4935
4936 ScratchRegisterScope scratch(masm);
4937
4938 masm.bind(&again);
4939 switch (nbytes) {
4940 case 1:
4941 masm.as_ldrexb(scratch, ptr);
4942 break;
4943 case 2:
4944 masm.as_ldrexh(scratch, ptr);
4945 break;
4946 case 4:
4947 masm.as_ldrex(scratch, ptr);
4948 break;
4949 }
4950 switch (op) {
4951 case AtomicFetchAddOp:
4952 masm.as_add(scratch, scratch, O2Reg(value));
4953 break;
4954 case AtomicFetchSubOp:
4955 masm.as_sub(scratch, scratch, O2Reg(value));
4956 break;
4957 case AtomicFetchAndOp:
4958 masm.as_and(scratch, scratch, O2Reg(value));
4959 break;
4960 case AtomicFetchOrOp:
4961 masm.as_orr(scratch, scratch, O2Reg(value));
4962 break;
4963 case AtomicFetchXorOp:
4964 masm.as_eor(scratch, scratch, O2Reg(value));
4965 break;
4966 }
4967 // Rd must differ from the two other arguments to strex.
4968 switch (nbytes) {
4969 case 1:
4970 masm.as_strexb(flagTemp, scratch, ptr);
4971 break;
4972 case 2:
4973 masm.as_strexh(flagTemp, scratch, ptr);
4974 break;
4975 case 4:
4976 masm.as_strex(flagTemp, scratch, ptr);
4977 break;
4978 }
4979 masm.as_cmp(flagTemp, Imm8(1));
4980 masm.as_b(&again, MacroAssembler::Equal);
4981
4982 masm.memoryBarrierAfter(sync);
4983 }
4984
atomicEffectOp(Scalar::Type type,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register flagTemp)4985 void MacroAssembler::atomicEffectOp(Scalar::Type type,
4986 const Synchronization& sync, AtomicOp op,
4987 Register value, const Address& mem,
4988 Register flagTemp) {
4989 AtomicEffectOp(*this, type, sync, op, value, mem, flagTemp);
4990 }
4991
atomicEffectOp(Scalar::Type type,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register flagTemp)4992 void MacroAssembler::atomicEffectOp(Scalar::Type type,
4993 const Synchronization& sync, AtomicOp op,
4994 Register value, const BaseIndex& mem,
4995 Register flagTemp) {
4996 AtomicEffectOp(*this, type, sync, op, value, mem, flagTemp);
4997 }
4998
4999 template <typename T>
AtomicLoad64(MacroAssembler & masm,const Synchronization & sync,const T & mem,Register64 temp,Register64 output)5000 static void AtomicLoad64(MacroAssembler& masm, const Synchronization& sync,
5001 const T& mem, Register64 temp, Register64 output) {
5002 MOZ_ASSERT(temp.low == InvalidReg && temp.high == InvalidReg);
5003 MOZ_ASSERT((output.low.code() & 1) == 0);
5004 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5005
5006 masm.memoryBarrierBefore(sync);
5007
5008 SecondScratchRegisterScope scratch2(masm);
5009 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5010
5011 masm.as_ldrexd(output.low, output.high, ptr);
5012 masm.as_clrex();
5013
5014 masm.memoryBarrierAfter(sync);
5015 }
5016
atomicLoad64(const Synchronization & sync,const Address & mem,Register64 temp,Register64 output)5017 void MacroAssembler::atomicLoad64(const Synchronization& sync,
5018 const Address& mem, Register64 temp,
5019 Register64 output) {
5020 AtomicLoad64(*this, sync, mem, temp, output);
5021 }
5022
atomicLoad64(const Synchronization & sync,const BaseIndex & mem,Register64 temp,Register64 output)5023 void MacroAssembler::atomicLoad64(const Synchronization& sync,
5024 const BaseIndex& mem, Register64 temp,
5025 Register64 output) {
5026 AtomicLoad64(*this, sync, mem, temp, output);
5027 }
5028
5029 template <typename T>
CompareExchange64(MacroAssembler & masm,const Synchronization & sync,const T & mem,Register64 expect,Register64 replace,Register64 output)5030 static void CompareExchange64(MacroAssembler& masm, const Synchronization& sync,
5031 const T& mem, Register64 expect,
5032 Register64 replace, Register64 output) {
5033 MOZ_ASSERT(expect != replace && replace != output && output != expect);
5034
5035 MOZ_ASSERT((replace.low.code() & 1) == 0);
5036 MOZ_ASSERT(replace.low.code() + 1 == replace.high.code());
5037
5038 MOZ_ASSERT((output.low.code() & 1) == 0);
5039 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5040
5041 Label again;
5042 Label done;
5043
5044 SecondScratchRegisterScope scratch2(masm);
5045 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5046
5047 masm.memoryBarrierBefore(sync);
5048
5049 masm.bind(&again);
5050 masm.as_ldrexd(output.low, output.high, ptr);
5051
5052 masm.as_cmp(output.low, O2Reg(expect.low));
5053 masm.as_cmp(output.high, O2Reg(expect.high), MacroAssembler::Equal);
5054 masm.as_b(&done, MacroAssembler::NotEqual);
5055
5056 ScratchRegisterScope scratch(masm);
5057
5058 // Rd (temp) must differ from the two other arguments to strex.
5059 masm.as_strexd(scratch, replace.low, replace.high, ptr);
5060 masm.as_cmp(scratch, Imm8(1));
5061 masm.as_b(&again, MacroAssembler::Equal);
5062 masm.bind(&done);
5063
5064 masm.memoryBarrierAfter(sync);
5065 }
5066
compareExchange64(const Synchronization & sync,const Address & mem,Register64 expect,Register64 replace,Register64 output)5067 void MacroAssembler::compareExchange64(const Synchronization& sync,
5068 const Address& mem, Register64 expect,
5069 Register64 replace, Register64 output) {
5070 CompareExchange64(*this, sync, mem, expect, replace, output);
5071 }
5072
compareExchange64(const Synchronization & sync,const BaseIndex & mem,Register64 expect,Register64 replace,Register64 output)5073 void MacroAssembler::compareExchange64(const Synchronization& sync,
5074 const BaseIndex& mem, Register64 expect,
5075 Register64 replace, Register64 output) {
5076 CompareExchange64(*this, sync, mem, expect, replace, output);
5077 }
5078
5079 template <typename T>
AtomicExchange64(MacroAssembler & masm,const Synchronization & sync,const T & mem,Register64 value,Register64 output)5080 static void AtomicExchange64(MacroAssembler& masm, const Synchronization& sync,
5081 const T& mem, Register64 value,
5082 Register64 output) {
5083 MOZ_ASSERT(output != value);
5084
5085 MOZ_ASSERT((value.low.code() & 1) == 0);
5086 MOZ_ASSERT(value.low.code() + 1 == value.high.code());
5087
5088 MOZ_ASSERT((output.low.code() & 1) == 0);
5089 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5090
5091 Label again;
5092
5093 SecondScratchRegisterScope scratch2(masm);
5094 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5095
5096 masm.memoryBarrierBefore(sync);
5097
5098 masm.bind(&again);
5099 masm.as_ldrexd(output.low, output.high, ptr);
5100
5101 ScratchRegisterScope scratch(masm);
5102
5103 masm.as_strexd(scratch, value.low, value.high, ptr);
5104 masm.as_cmp(scratch, Imm8(1));
5105 masm.as_b(&again, MacroAssembler::Equal);
5106
5107 masm.memoryBarrierAfter(sync);
5108 }
5109
atomicExchange64(const Synchronization & sync,const Address & mem,Register64 value,Register64 output)5110 void MacroAssembler::atomicExchange64(const Synchronization& sync,
5111 const Address& mem, Register64 value,
5112 Register64 output) {
5113 AtomicExchange64(*this, sync, mem, value, output);
5114 }
5115
atomicExchange64(const Synchronization & sync,const BaseIndex & mem,Register64 value,Register64 output)5116 void MacroAssembler::atomicExchange64(const Synchronization& sync,
5117 const BaseIndex& mem, Register64 value,
5118 Register64 output) {
5119 AtomicExchange64(*this, sync, mem, value, output);
5120 }
5121
5122 template <typename T>
AtomicFetchOp64(MacroAssembler & masm,const Synchronization & sync,AtomicOp op,Register64 value,const T & mem,Register64 temp,Register64 output)5123 static void AtomicFetchOp64(MacroAssembler& masm, const Synchronization& sync,
5124 AtomicOp op, Register64 value, const T& mem,
5125 Register64 temp, Register64 output) {
5126 MOZ_ASSERT(temp.low != InvalidReg && temp.high != InvalidReg);
5127 MOZ_ASSERT(output != value);
5128
5129 MOZ_ASSERT((temp.low.code() & 1) == 0);
5130 MOZ_ASSERT(temp.low.code() + 1 == temp.high.code());
5131
5132 // We could avoid this pair requirement but in that case we would end up
5133 // with two moves in the loop to preserve the loaded value in output. The
5134 // prize would be less register spilling around this op since the pair
5135 // requirement will tend to force more spilling.
5136
5137 MOZ_ASSERT((output.low.code() & 1) == 0);
5138 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5139
5140 Label again;
5141
5142 SecondScratchRegisterScope scratch2(masm);
5143 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5144
5145 masm.memoryBarrierBefore(sync);
5146
5147 masm.bind(&again);
5148 masm.as_ldrexd(output.low, output.high, ptr);
5149 switch (op) {
5150 case AtomicFetchAddOp:
5151 masm.as_add(temp.low, output.low, O2Reg(value.low), SetCC);
5152 masm.as_adc(temp.high, output.high, O2Reg(value.high));
5153 break;
5154 case AtomicFetchSubOp:
5155 masm.as_sub(temp.low, output.low, O2Reg(value.low), SetCC);
5156 masm.as_sbc(temp.high, output.high, O2Reg(value.high));
5157 break;
5158 case AtomicFetchAndOp:
5159 masm.as_and(temp.low, output.low, O2Reg(value.low));
5160 masm.as_and(temp.high, output.high, O2Reg(value.high));
5161 break;
5162 case AtomicFetchOrOp:
5163 masm.as_orr(temp.low, output.low, O2Reg(value.low));
5164 masm.as_orr(temp.high, output.high, O2Reg(value.high));
5165 break;
5166 case AtomicFetchXorOp:
5167 masm.as_eor(temp.low, output.low, O2Reg(value.low));
5168 masm.as_eor(temp.high, output.high, O2Reg(value.high));
5169 break;
5170 }
5171
5172 ScratchRegisterScope scratch(masm);
5173
5174 // Rd (temp) must differ from the two other arguments to strex.
5175 masm.as_strexd(scratch, temp.low, temp.high, ptr);
5176 masm.as_cmp(scratch, Imm8(1));
5177 masm.as_b(&again, MacroAssembler::Equal);
5178
5179 masm.memoryBarrierAfter(sync);
5180 }
5181
atomicFetchOp64(const Synchronization & sync,AtomicOp op,Register64 value,const Address & mem,Register64 temp,Register64 output)5182 void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
5183 Register64 value, const Address& mem,
5184 Register64 temp, Register64 output) {
5185 AtomicFetchOp64(*this, sync, op, value, mem, temp, output);
5186 }
5187
atomicFetchOp64(const Synchronization & sync,AtomicOp op,Register64 value,const BaseIndex & mem,Register64 temp,Register64 output)5188 void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
5189 Register64 value, const BaseIndex& mem,
5190 Register64 temp, Register64 output) {
5191 AtomicFetchOp64(*this, sync, op, value, mem, temp, output);
5192 }
5193
5194 // ========================================================================
5195 // JS atomic operations.
5196
5197 template <typename T>
CompareExchangeJS(MacroAssembler & masm,Scalar::Type arrayType,const Synchronization & sync,const T & mem,Register oldval,Register newval,Register temp,AnyRegister output)5198 static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
5199 const Synchronization& sync, const T& mem,
5200 Register oldval, Register newval, Register temp,
5201 AnyRegister output) {
5202 if (arrayType == Scalar::Uint32) {
5203 masm.compareExchange(arrayType, sync, mem, oldval, newval, temp);
5204 masm.convertUInt32ToDouble(temp, output.fpu());
5205 } else {
5206 masm.compareExchange(arrayType, sync, mem, oldval, newval, output.gpr());
5207 }
5208 }
5209
compareExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const Address & mem,Register oldval,Register newval,Register temp,AnyRegister output)5210 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
5211 const Synchronization& sync,
5212 const Address& mem, Register oldval,
5213 Register newval, Register temp,
5214 AnyRegister output) {
5215 CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
5216 }
5217
compareExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const BaseIndex & mem,Register oldval,Register newval,Register temp,AnyRegister output)5218 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
5219 const Synchronization& sync,
5220 const BaseIndex& mem, Register oldval,
5221 Register newval, Register temp,
5222 AnyRegister output) {
5223 CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
5224 }
5225
5226 template <typename T>
AtomicExchangeJS(MacroAssembler & masm,Scalar::Type arrayType,const Synchronization & sync,const T & mem,Register value,Register temp,AnyRegister output)5227 static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
5228 const Synchronization& sync, const T& mem,
5229 Register value, Register temp,
5230 AnyRegister output) {
5231 if (arrayType == Scalar::Uint32) {
5232 masm.atomicExchange(arrayType, sync, mem, value, temp);
5233 masm.convertUInt32ToDouble(temp, output.fpu());
5234 } else {
5235 masm.atomicExchange(arrayType, sync, mem, value, output.gpr());
5236 }
5237 }
5238
atomicExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const Address & mem,Register value,Register temp,AnyRegister output)5239 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
5240 const Synchronization& sync,
5241 const Address& mem, Register value,
5242 Register temp, AnyRegister output) {
5243 AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
5244 }
5245
atomicExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const BaseIndex & mem,Register value,Register temp,AnyRegister output)5246 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
5247 const Synchronization& sync,
5248 const BaseIndex& mem, Register value,
5249 Register temp, AnyRegister output) {
5250 AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
5251 }
5252
5253 template <typename T>
AtomicFetchOpJS(MacroAssembler & masm,Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const T & mem,Register temp1,Register temp2,AnyRegister output)5254 static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
5255 const Synchronization& sync, AtomicOp op,
5256 Register value, const T& mem, Register temp1,
5257 Register temp2, AnyRegister output) {
5258 if (arrayType == Scalar::Uint32) {
5259 masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1);
5260 masm.convertUInt32ToDouble(temp1, output.fpu());
5261 } else {
5262 masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr());
5263 }
5264 }
5265
atomicFetchOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register temp1,Register temp2,AnyRegister output)5266 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
5267 const Synchronization& sync, AtomicOp op,
5268 Register value, const Address& mem,
5269 Register temp1, Register temp2,
5270 AnyRegister output) {
5271 AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
5272 }
5273
atomicFetchOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register temp1,Register temp2,AnyRegister output)5274 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
5275 const Synchronization& sync, AtomicOp op,
5276 Register value, const BaseIndex& mem,
5277 Register temp1, Register temp2,
5278 AnyRegister output) {
5279 AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
5280 }
5281
atomicEffectOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register temp)5282 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
5283 const Synchronization& sync, AtomicOp op,
5284 Register value, const BaseIndex& mem,
5285 Register temp) {
5286 atomicEffectOp(arrayType, sync, op, value, mem, temp);
5287 }
5288
atomicEffectOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register temp)5289 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
5290 const Synchronization& sync, AtomicOp op,
5291 Register value, const Address& mem,
5292 Register temp) {
5293 atomicEffectOp(arrayType, sync, op, value, mem, temp);
5294 }
5295
5296 // ========================================================================
5297 // Convert floating point.
5298
convertUInt64ToDoubleNeedsTemp()5299 bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
5300
convertUInt64ToDouble(Register64 src,FloatRegister dest,Register temp)5301 void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
5302 Register temp) {
5303 MOZ_ASSERT(temp == Register::Invalid());
5304 ScratchDoubleScope scratchDouble(*this);
5305
5306 convertUInt32ToDouble(src.high, dest);
5307 {
5308 ScratchRegisterScope scratch(*this);
5309 movePtr(ImmPtr(&TO_DOUBLE_HIGH_SCALE), scratch);
5310 ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), scratchDouble);
5311 }
5312 mulDouble(scratchDouble, dest);
5313 convertUInt32ToDouble(src.low, scratchDouble);
5314 addDouble(scratchDouble, dest);
5315 }
5316
5317 // ========================================================================
5318 // Spectre Mitigations.
5319
speculationBarrier()5320 void MacroAssembler::speculationBarrier() {
5321 // Spectre mitigation recommended by ARM for cases where csel/cmov cannot be
5322 // used.
5323 as_csdb();
5324 }
5325
5326 //}}} check_macroassembler_style
5327
wasmTruncateToInt32(FloatRegister input,Register output,MIRType fromType,bool isUnsigned,bool isSaturating,Label * oolEntry)5328 void MacroAssemblerARM::wasmTruncateToInt32(FloatRegister input,
5329 Register output, MIRType fromType,
5330 bool isUnsigned, bool isSaturating,
5331 Label* oolEntry) {
5332 // vcvt* converts NaN into 0, so check for NaNs here.
5333 if (!isSaturating) {
5334 if (fromType == MIRType::Double)
5335 asMasm().compareDouble(input, input);
5336 else if (fromType == MIRType::Float32)
5337 asMasm().compareFloat(input, input);
5338 else
5339 MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
5340
5341 ma_b(oolEntry, Assembler::VFP_Unordered);
5342 }
5343
5344 ScratchDoubleScope scratchScope(asMasm());
5345 ScratchRegisterScope scratchReg(asMasm());
5346 FloatRegister scratch = scratchScope.uintOverlay();
5347
5348 // ARM conversion instructions clamp the value to ensure it fits within the
5349 // target's type bounds, so every time we see those, we need to check the
5350 // input.
5351 if (isUnsigned) {
5352 if (fromType == MIRType::Double)
5353 ma_vcvt_F64_U32(input, scratch);
5354 else if (fromType == MIRType::Float32)
5355 ma_vcvt_F32_U32(input, scratch);
5356 else
5357 MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
5358
5359 ma_vxfer(scratch, output);
5360
5361 if (!isSaturating) {
5362 // int32_t(UINT32_MAX) == -1.
5363 ma_cmp(output, Imm32(-1), scratchReg);
5364 as_cmp(output, Imm8(0), Assembler::NotEqual);
5365 ma_b(oolEntry, Assembler::Equal);
5366 }
5367
5368 return;
5369 }
5370
5371 scratch = scratchScope.sintOverlay();
5372
5373 if (fromType == MIRType::Double)
5374 ma_vcvt_F64_I32(input, scratch);
5375 else if (fromType == MIRType::Float32)
5376 ma_vcvt_F32_I32(input, scratch);
5377 else
5378 MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
5379
5380 ma_vxfer(scratch, output);
5381
5382 if (!isSaturating) {
5383 ma_cmp(output, Imm32(INT32_MAX), scratchReg);
5384 ma_cmp(output, Imm32(INT32_MIN), scratchReg, Assembler::NotEqual);
5385 ma_b(oolEntry, Assembler::Equal);
5386 }
5387 }
5388
outOfLineWasmTruncateToIntCheck(FloatRegister input,MIRType fromType,MIRType toType,TruncFlags flags,Label * rejoin,wasm::BytecodeOffset trapOffset)5389 void MacroAssemblerARM::outOfLineWasmTruncateToIntCheck(
5390 FloatRegister input, MIRType fromType, MIRType toType, TruncFlags flags,
5391 Label* rejoin, wasm::BytecodeOffset trapOffset) {
5392 // On ARM, saturating truncation codegen handles saturating itself rather
5393 // than relying on out-of-line fixup code.
5394 if (flags & TRUNC_SATURATING) return;
5395
5396 bool isUnsigned = flags & TRUNC_UNSIGNED;
5397 ScratchDoubleScope scratchScope(asMasm());
5398 FloatRegister scratch;
5399
5400 // Eagerly take care of NaNs.
5401 Label inputIsNaN;
5402 if (fromType == MIRType::Double)
5403 asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
5404 &inputIsNaN);
5405 else if (fromType == MIRType::Float32)
5406 asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
5407 else
5408 MOZ_CRASH("unexpected type in visitOutOfLineWasmTruncateCheck");
5409
5410 // Handle special values.
5411 Label fail;
5412
5413 // By default test for the following inputs and bail:
5414 // signed: ] -Inf, INTXX_MIN - 1.0 ] and [ INTXX_MAX + 1.0 : +Inf [
5415 // unsigned: ] -Inf, -1.0 ] and [ UINTXX_MAX + 1.0 : +Inf [
5416 // Note: we cannot always represent those exact values. As a result
5417 // this changes the actual comparison a bit.
5418 double minValue, maxValue;
5419 Assembler::DoubleCondition minCond = Assembler::DoubleLessThanOrEqual;
5420 Assembler::DoubleCondition maxCond = Assembler::DoubleGreaterThanOrEqual;
5421 if (toType == MIRType::Int64) {
5422 if (isUnsigned) {
5423 minValue = -1;
5424 maxValue = double(UINT64_MAX) + 1.0;
5425 } else {
5426 // In the float32/double range there exists no value between
5427 // INT64_MIN and INT64_MIN - 1.0. Making INT64_MIN the lower-bound.
5428 minValue = double(INT64_MIN);
5429 minCond = Assembler::DoubleLessThan;
5430 maxValue = double(INT64_MAX) + 1.0;
5431 }
5432 } else {
5433 if (isUnsigned) {
5434 minValue = -1;
5435 maxValue = double(UINT32_MAX) + 1.0;
5436 } else {
5437 if (fromType == MIRType::Float32) {
5438 // In the float32 range there exists no value between
5439 // INT32_MIN and INT32_MIN - 1.0. Making INT32_MIN the lower-bound.
5440 minValue = double(INT32_MIN);
5441 minCond = Assembler::DoubleLessThan;
5442 } else {
5443 minValue = double(INT32_MIN) - 1.0;
5444 }
5445 maxValue = double(INT32_MAX) + 1.0;
5446 }
5447 }
5448
5449 if (fromType == MIRType::Double) {
5450 scratch = scratchScope.doubleOverlay();
5451 asMasm().loadConstantDouble(minValue, scratch);
5452 asMasm().branchDouble(minCond, input, scratch, &fail);
5453
5454 asMasm().loadConstantDouble(maxValue, scratch);
5455 asMasm().branchDouble(maxCond, input, scratch, &fail);
5456 } else {
5457 MOZ_ASSERT(fromType == MIRType::Float32);
5458 scratch = scratchScope.singleOverlay();
5459 asMasm().loadConstantFloat32(float(minValue), scratch);
5460 asMasm().branchFloat(minCond, input, scratch, &fail);
5461
5462 asMasm().loadConstantFloat32(float(maxValue), scratch);
5463 asMasm().branchFloat(maxCond, input, scratch, &fail);
5464 }
5465
5466 // We had an actual correct value, get back to where we were.
5467 ma_b(rejoin);
5468
5469 // Handle errors.
5470 bind(&fail);
5471 asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
5472
5473 bind(&inputIsNaN);
5474 asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
5475 }
5476
wasmLoadImpl(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,AnyRegister output,Register64 out64)5477 void MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
5478 Register memoryBase, Register ptr,
5479 Register ptrScratch, AnyRegister output,
5480 Register64 out64) {
5481 MOZ_ASSERT(ptr == ptrScratch);
5482
5483 uint32_t offset = access.offset();
5484 MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
5485
5486 Scalar::Type type = access.type();
5487
5488 // Maybe add the offset.
5489 if (offset || type == Scalar::Int64) {
5490 ScratchRegisterScope scratch(asMasm());
5491 if (offset) ma_add(Imm32(offset), ptr, scratch);
5492 }
5493
5494 bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 ||
5495 type == Scalar::Int32 || type == Scalar::Int64;
5496 unsigned byteSize = access.byteSize();
5497
5498 asMasm().memoryBarrierBefore(access.sync());
5499
5500 uint32_t framePushed = asMasm().framePushed();
5501 BufferOffset load;
5502 if (out64 != Register64::Invalid()) {
5503 if (type == Scalar::Int64) {
5504 MOZ_ASSERT(INT64LOW_OFFSET == 0);
5505
5506 load = ma_dataTransferN(IsLoad, 32, /* signed = */ false, memoryBase, ptr,
5507 out64.low);
5508 append(access, load.getOffset(), framePushed);
5509
5510 as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
5511
5512 load =
5513 ma_dataTransferN(IsLoad, 32, isSigned, memoryBase, ptr, out64.high);
5514 append(access, load.getOffset(), framePushed);
5515 } else {
5516 load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, memoryBase, ptr,
5517 out64.low);
5518 append(access, load.getOffset(), framePushed);
5519
5520 if (isSigned)
5521 ma_asr(Imm32(31), out64.low, out64.high);
5522 else
5523 ma_mov(Imm32(0), out64.high);
5524 }
5525 } else {
5526 bool isFloat = output.isFloat();
5527 if (isFloat) {
5528 MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle());
5529 ScratchRegisterScope scratch(asMasm());
5530 ma_add(memoryBase, ptr, scratch);
5531
5532 load = ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), output.fpu());
5533 append(access, load.getOffset(), framePushed);
5534 } else {
5535 load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, memoryBase, ptr,
5536 output.gpr());
5537 append(access, load.getOffset(), framePushed);
5538 }
5539 }
5540
5541 asMasm().memoryBarrierAfter(access.sync());
5542 }
5543
wasmStoreImpl(const wasm::MemoryAccessDesc & access,AnyRegister value,Register64 val64,Register memoryBase,Register ptr,Register ptrScratch)5544 void MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
5545 AnyRegister value, Register64 val64,
5546 Register memoryBase, Register ptr,
5547 Register ptrScratch) {
5548 MOZ_ASSERT(ptr == ptrScratch);
5549
5550 uint32_t offset = access.offset();
5551 MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
5552
5553 unsigned byteSize = access.byteSize();
5554 Scalar::Type type = access.type();
5555
5556 // Maybe add the offset.
5557 if (offset || type == Scalar::Int64) {
5558 ScratchRegisterScope scratch(asMasm());
5559 if (offset) ma_add(Imm32(offset), ptr, scratch);
5560 }
5561
5562 asMasm().memoryBarrierAfter(access.sync());
5563
5564 uint32_t framePushed = asMasm().framePushed();
5565
5566 BufferOffset store;
5567 if (type == Scalar::Int64) {
5568 MOZ_ASSERT(INT64LOW_OFFSET == 0);
5569
5570 store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false,
5571 memoryBase, ptr, val64.low);
5572 append(access, store.getOffset(), framePushed);
5573
5574 as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
5575
5576 store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true,
5577 memoryBase, ptr, val64.high);
5578 append(access, store.getOffset(), framePushed);
5579 } else {
5580 if (value.isFloat()) {
5581 ScratchRegisterScope scratch(asMasm());
5582 FloatRegister val = value.fpu();
5583 MOZ_ASSERT((byteSize == 4) == val.isSingle());
5584 ma_add(memoryBase, ptr, scratch);
5585
5586 store = ma_vstr(val, Operand(Address(scratch, 0)).toVFPAddr());
5587 append(access, store.getOffset(), framePushed);
5588 } else {
5589 bool isSigned = type == Scalar::Uint32 ||
5590 type == Scalar::Int32; // see AsmJSStoreHeap;
5591 Register val = value.gpr();
5592
5593 store = ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned,
5594 memoryBase, ptr, val);
5595 append(access, store.getOffset(), framePushed);
5596 }
5597 }
5598
5599 asMasm().memoryBarrierAfter(access.sync());
5600 }
5601
wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,AnyRegister outAny,Register64 out64,Register tmp,Register tmp2,Register tmp3)5602 void MacroAssemblerARM::wasmUnalignedLoadImpl(
5603 const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
5604 Register ptrScratch, AnyRegister outAny, Register64 out64, Register tmp,
5605 Register tmp2, Register tmp3) {
5606 MOZ_ASSERT(ptr == ptrScratch);
5607 MOZ_ASSERT_IF(
5608 access.type() != Scalar::Float32 && access.type() != Scalar::Float64,
5609 tmp2 == Register::Invalid() && tmp3 == Register::Invalid());
5610 MOZ_ASSERT_IF(access.type() == Scalar::Float32,
5611 tmp2 != Register::Invalid() && tmp3 == Register::Invalid());
5612 MOZ_ASSERT_IF(access.type() == Scalar::Float64,
5613 tmp2 != Register::Invalid() && tmp3 != Register::Invalid());
5614
5615 uint32_t offset = access.offset();
5616 MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
5617
5618 if (offset) {
5619 ScratchRegisterScope scratch(asMasm());
5620 ma_add(Imm32(offset), ptr, scratch);
5621 }
5622
5623 // Add memoryBase to ptr, so we can use base+index addressing in the byte
5624 // loads.
5625 ma_add(memoryBase, ptr);
5626
5627 unsigned byteSize = access.byteSize();
5628 Scalar::Type type = access.type();
5629 bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 ||
5630 type == Scalar::Int32 || type == Scalar::Int64;
5631
5632 Register low;
5633 if (out64 != Register64::Invalid())
5634 low = out64.low;
5635 else if (outAny.isFloat())
5636 low = tmp2;
5637 else
5638 low = outAny.gpr();
5639
5640 MOZ_ASSERT(low != tmp);
5641 MOZ_ASSERT(low != ptr);
5642
5643 asMasm().memoryBarrierBefore(access.sync());
5644
5645 emitUnalignedLoad(isSigned, Min(byteSize, 4u), ptr, tmp, low);
5646
5647 if (out64 != Register64::Invalid()) {
5648 if (type == Scalar::Int64) {
5649 MOZ_ASSERT(byteSize == 8);
5650 emitUnalignedLoad(isSigned, 4, ptr, tmp, out64.high, /* offset */ 4);
5651 } else {
5652 MOZ_ASSERT(byteSize <= 4);
5653 // Propagate sign.
5654 if (isSigned)
5655 ma_asr(Imm32(31), out64.low, out64.high);
5656 else
5657 ma_mov(Imm32(0), out64.high);
5658 }
5659 } else if (outAny.isFloat()) {
5660 FloatRegister output = outAny.fpu();
5661 if (byteSize == 4) {
5662 MOZ_ASSERT(output.isSingle());
5663 ma_vxfer(low, output);
5664 } else {
5665 MOZ_ASSERT(byteSize == 8);
5666 MOZ_ASSERT(output.isDouble());
5667 Register high = tmp3;
5668 emitUnalignedLoad(/* signed */ false, 4, ptr, tmp, high, /* offset */ 4);
5669 ma_vxfer(low, high, output);
5670 }
5671 }
5672
5673 asMasm().memoryBarrierAfter(access.sync());
5674 }
5675
wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc & access,FloatRegister floatValue,Register64 val64,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)5676 void MacroAssemblerARM::wasmUnalignedStoreImpl(
5677 const wasm::MemoryAccessDesc& access, FloatRegister floatValue,
5678 Register64 val64, Register memoryBase, Register ptr, Register ptrScratch,
5679 Register tmp) {
5680 MOZ_ASSERT(ptr == ptrScratch);
5681 // They can't both be valid, but they can both be invalid.
5682 MOZ_ASSERT_IF(!floatValue.isInvalid(), val64 == Register64::Invalid());
5683 MOZ_ASSERT_IF(val64 != Register64::Invalid(), floatValue.isInvalid());
5684
5685 uint32_t offset = access.offset();
5686 MOZ_ASSERT(offset < wasm::OffsetGuardLimit);
5687
5688 unsigned byteSize = access.byteSize();
5689
5690 if (offset) {
5691 ScratchRegisterScope scratch(asMasm());
5692 ma_add(Imm32(offset), ptr, scratch);
5693 }
5694
5695 // Add memoryBase to ptr, so we can use base+index addressing in the byte
5696 // loads.
5697 ma_add(memoryBase, ptr);
5698
5699 asMasm().memoryBarrierAfter(access.sync());
5700
5701 if (val64 != Register64::Invalid()) {
5702 if (val64.low != tmp) ma_mov(val64.low, tmp);
5703 } else if (!floatValue.isInvalid()) {
5704 ma_vxfer(floatValue, tmp);
5705 }
5706 // Otherwise, tmp has the integer value to store.
5707
5708 emitUnalignedStore(Min(byteSize, 4u), ptr, tmp);
5709
5710 if (byteSize > 4) {
5711 if (val64 != Register64::Invalid()) {
5712 if (val64.high != tmp) ma_mov(val64.high, tmp);
5713 } else {
5714 MOZ_ASSERT(!floatValue.isInvalid());
5715 MOZ_ASSERT(floatValue.isDouble());
5716 ScratchRegisterScope scratch(asMasm());
5717 ma_vxfer(floatValue, scratch, tmp);
5718 }
5719 emitUnalignedStore(4, ptr, tmp, /* offset */ 4);
5720 }
5721
5722 asMasm().memoryBarrierAfter(access.sync());
5723 }
5724
emitUnalignedLoad(bool isSigned,unsigned byteSize,Register ptr,Register tmp,Register dest,unsigned offset)5725 void MacroAssemblerARM::emitUnalignedLoad(bool isSigned, unsigned byteSize,
5726 Register ptr, Register tmp,
5727 Register dest, unsigned offset) {
5728 // Preconditions.
5729 MOZ_ASSERT(ptr != tmp);
5730 MOZ_ASSERT(ptr != dest);
5731 MOZ_ASSERT(tmp != dest);
5732 MOZ_ASSERT(byteSize <= 4);
5733
5734 ScratchRegisterScope scratch(asMasm());
5735
5736 for (unsigned i = 0; i < byteSize; i++) {
5737 // Only the last byte load shall be signed, if needed.
5738 bool signedByteLoad = isSigned && (i == byteSize - 1);
5739 ma_dataTransferN(IsLoad, 8, signedByteLoad, ptr, Imm32(offset + i),
5740 i ? tmp : dest, scratch);
5741 if (i) as_orr(dest, dest, lsl(tmp, 8 * i));
5742 }
5743 }
5744
emitUnalignedStore(unsigned byteSize,Register ptr,Register val,unsigned offset)5745 void MacroAssemblerARM::emitUnalignedStore(unsigned byteSize, Register ptr,
5746 Register val, unsigned offset) {
5747 // Preconditions.
5748 MOZ_ASSERT(ptr != val);
5749 MOZ_ASSERT(byteSize <= 4);
5750
5751 ScratchRegisterScope scratch(asMasm());
5752
5753 for (unsigned i = 0; i < byteSize; i++) {
5754 ma_dataTransferN(IsStore, 8 /* bits */, /* signed */ false, ptr,
5755 Imm32(offset + i), val, scratch);
5756 if (i < byteSize - 1) ma_lsr(Imm32(8), val, val);
5757 }
5758 }
5759