1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/arm/MacroAssembler-arm.h"
8
9 #include "mozilla/Attributes.h"
10 #include "mozilla/Casting.h"
11 #include "mozilla/DebugOnly.h"
12 #include "mozilla/MathAlgorithms.h"
13 #include "mozilla/Maybe.h"
14
15 #include "jit/arm/Simulator-arm.h"
16 #include "jit/AtomicOp.h"
17 #include "jit/AtomicOperations.h"
18 #include "jit/Bailouts.h"
19 #include "jit/BaselineFrame.h"
20 #include "jit/JitFrames.h"
21 #include "jit/MacroAssembler.h"
22 #include "jit/MoveEmitter.h"
23 #include "util/Memory.h"
24 #include "vm/JitActivation.h" // js::jit::JitActivation
25
26 #include "jit/MacroAssembler-inl.h"
27
28 using namespace js;
29 using namespace jit;
30
31 using mozilla::Abs;
32 using mozilla::BitwiseCast;
33 using mozilla::DebugOnly;
34 using mozilla::IsPositiveZero;
35 using mozilla::Maybe;
36
isValueDTRDCandidate(ValueOperand & val)37 bool isValueDTRDCandidate(ValueOperand& val) {
38 // In order to be used for a DTRD memory function, the two target registers
39 // need to be a) Adjacent, with the tag larger than the payload, and b)
40 // Aligned to a multiple of two.
41 if ((val.typeReg().code() != (val.payloadReg().code() + 1))) {
42 return false;
43 }
44 if ((val.payloadReg().code() & 1) != 0) {
45 return false;
46 }
47 return true;
48 }
49
convertBoolToInt32(Register source,Register dest)50 void MacroAssemblerARM::convertBoolToInt32(Register source, Register dest) {
51 // Note that C++ bool is only 1 byte, so zero extend it to clear the
52 // higher-order bits.
53 as_and(dest, source, Imm8(0xff));
54 }
55
convertInt32ToDouble(Register src,FloatRegister dest_)56 void MacroAssemblerARM::convertInt32ToDouble(Register src,
57 FloatRegister dest_) {
58 // Direct conversions aren't possible.
59 VFPRegister dest = VFPRegister(dest_);
60 as_vxfer(src, InvalidReg, dest.sintOverlay(), CoreToFloat);
61 as_vcvt(dest, dest.sintOverlay());
62 }
63
convertInt32ToDouble(const Address & src,FloatRegister dest)64 void MacroAssemblerARM::convertInt32ToDouble(const Address& src,
65 FloatRegister dest) {
66 ScratchDoubleScope scratch(asMasm());
67 SecondScratchRegisterScope scratch2(asMasm());
68 ma_vldr(src, scratch, scratch2);
69 as_vcvt(dest, VFPRegister(scratch).sintOverlay());
70 }
71
convertInt32ToDouble(const BaseIndex & src,FloatRegister dest)72 void MacroAssemblerARM::convertInt32ToDouble(const BaseIndex& src,
73 FloatRegister dest) {
74 Register base = src.base;
75 uint32_t scale = Imm32::ShiftOf(src.scale).value;
76
77 ScratchRegisterScope scratch(asMasm());
78 SecondScratchRegisterScope scratch2(asMasm());
79
80 if (src.offset != 0) {
81 ma_add(base, Imm32(src.offset), scratch, scratch2);
82 base = scratch;
83 }
84 ma_ldr(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), scratch);
85 convertInt32ToDouble(scratch, dest);
86 }
87
convertUInt32ToDouble(Register src,FloatRegister dest_)88 void MacroAssemblerARM::convertUInt32ToDouble(Register src,
89 FloatRegister dest_) {
90 // Direct conversions aren't possible.
91 VFPRegister dest = VFPRegister(dest_);
92 as_vxfer(src, InvalidReg, dest.uintOverlay(), CoreToFloat);
93 as_vcvt(dest, dest.uintOverlay());
94 }
95
96 static const double TO_DOUBLE_HIGH_SCALE = 0x100000000;
97
convertUInt32ToFloat32(Register src,FloatRegister dest_)98 void MacroAssemblerARM::convertUInt32ToFloat32(Register src,
99 FloatRegister dest_) {
100 // Direct conversions aren't possible.
101 VFPRegister dest = VFPRegister(dest_);
102 as_vxfer(src, InvalidReg, dest.uintOverlay(), CoreToFloat);
103 as_vcvt(VFPRegister(dest).singleOverlay(), dest.uintOverlay());
104 }
105
convertDoubleToFloat32(FloatRegister src,FloatRegister dest,Condition c)106 void MacroAssemblerARM::convertDoubleToFloat32(FloatRegister src,
107 FloatRegister dest,
108 Condition c) {
109 as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src), false, c);
110 }
111
112 // Checks whether a double is representable as a 32-bit integer. If so, the
113 // integer is written to the output register. Otherwise, a bailout is taken to
114 // the given snapshot. This function overwrites the scratch float register.
convertDoubleToInt32(FloatRegister src,Register dest,Label * fail,bool negativeZeroCheck)115 void MacroAssemblerARM::convertDoubleToInt32(FloatRegister src, Register dest,
116 Label* fail,
117 bool negativeZeroCheck) {
118 // Convert the floating point value to an integer, if it did not fit, then
119 // when we convert it *back* to a float, it will have a different value,
120 // which we can test.
121 ScratchDoubleScope scratchDouble(asMasm());
122 ScratchRegisterScope scratch(asMasm());
123
124 FloatRegister scratchSIntReg = scratchDouble.sintOverlay();
125
126 ma_vcvt_F64_I32(src, scratchSIntReg);
127 // Move the value into the dest register.
128 ma_vxfer(scratchSIntReg, dest);
129 ma_vcvt_I32_F64(scratchSIntReg, scratchDouble);
130 ma_vcmp(src, scratchDouble);
131 as_vmrs(pc);
132 ma_b(fail, Assembler::VFP_NotEqualOrUnordered);
133
134 if (negativeZeroCheck) {
135 as_cmp(dest, Imm8(0));
136 // Test and bail for -0.0, when integer result is 0. Move the top word
137 // of the double into the output reg, if it is non-zero, then the
138 // original value was -0.0.
139 as_vxfer(dest, InvalidReg, src, FloatToCore, Assembler::Equal, 1);
140 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::Equal);
141 ma_b(fail, Assembler::Equal);
142 }
143 }
144
145 // Checks whether a float32 is representable as a 32-bit integer. If so, the
146 // integer is written to the output register. Otherwise, a bailout is taken to
147 // the given snapshot. This function overwrites the scratch float register.
convertFloat32ToInt32(FloatRegister src,Register dest,Label * fail,bool negativeZeroCheck)148 void MacroAssemblerARM::convertFloat32ToInt32(FloatRegister src, Register dest,
149 Label* fail,
150 bool negativeZeroCheck) {
151 // Converting the floating point value to an integer and then converting it
152 // back to a float32 would not work, as float to int32 conversions are
153 // clamping (e.g. float(INT32_MAX + 1) would get converted into INT32_MAX
154 // and then back to float(INT32_MAX + 1)). If this ever happens, we just
155 // bail out.
156 ScratchFloat32Scope scratchFloat(asMasm());
157 ScratchRegisterScope scratch(asMasm());
158
159 FloatRegister ScratchSIntReg = scratchFloat.sintOverlay();
160 ma_vcvt_F32_I32(src, ScratchSIntReg);
161
162 // Store the result
163 ma_vxfer(ScratchSIntReg, dest);
164
165 ma_vcvt_I32_F32(ScratchSIntReg, scratchFloat);
166 ma_vcmp(src, scratchFloat);
167 as_vmrs(pc);
168 ma_b(fail, Assembler::VFP_NotEqualOrUnordered);
169
170 // Bail out in the clamped cases.
171 ma_cmp(dest, Imm32(0x7fffffff), scratch);
172 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
173 ma_b(fail, Assembler::Equal);
174
175 if (negativeZeroCheck) {
176 as_cmp(dest, Imm8(0));
177 // Test and bail for -0.0, when integer result is 0. Move the float into
178 // the output reg, and if it is non-zero then the original value was
179 // -0.0
180 as_vxfer(dest, InvalidReg, VFPRegister(src).singleOverlay(), FloatToCore,
181 Assembler::Equal, 0);
182 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::Equal);
183 ma_b(fail, Assembler::Equal);
184 }
185 }
186
convertFloat32ToDouble(FloatRegister src,FloatRegister dest)187 void MacroAssemblerARM::convertFloat32ToDouble(FloatRegister src,
188 FloatRegister dest) {
189 MOZ_ASSERT(dest.isDouble());
190 MOZ_ASSERT(src.isSingle());
191 as_vcvt(VFPRegister(dest), VFPRegister(src).singleOverlay());
192 }
193
convertInt32ToFloat32(Register src,FloatRegister dest)194 void MacroAssemblerARM::convertInt32ToFloat32(Register src,
195 FloatRegister dest) {
196 // Direct conversions aren't possible.
197 as_vxfer(src, InvalidReg, dest.sintOverlay(), CoreToFloat);
198 as_vcvt(dest.singleOverlay(), dest.sintOverlay());
199 }
200
convertInt32ToFloat32(const Address & src,FloatRegister dest)201 void MacroAssemblerARM::convertInt32ToFloat32(const Address& src,
202 FloatRegister dest) {
203 ScratchFloat32Scope scratch(asMasm());
204 SecondScratchRegisterScope scratch2(asMasm());
205 ma_vldr(src, scratch, scratch2);
206 as_vcvt(dest, VFPRegister(scratch).sintOverlay());
207 }
208
alu_dbl(Register src1,Imm32 imm,Register dest,ALUOp op,SBit s,Condition c)209 bool MacroAssemblerARM::alu_dbl(Register src1, Imm32 imm, Register dest,
210 ALUOp op, SBit s, Condition c) {
211 if ((s == SetCC && !condsAreSafe(op)) || !can_dbl(op)) {
212 return false;
213 }
214
215 ALUOp interop = getDestVariant(op);
216 Imm8::TwoImm8mData both = Imm8::EncodeTwoImms(imm.value);
217 if (both.fst().invalid()) {
218 return false;
219 }
220
221 // For the most part, there is no good reason to set the condition codes for
222 // the first instruction. We can do better things if the second instruction
223 // doesn't have a dest, such as check for overflow by doing first operation
224 // don't do second operation if first operation overflowed. This preserves
225 // the overflow condition code. Unfortunately, it is horribly brittle.
226 as_alu(dest, src1, Operand2(both.fst()), interop, LeaveCC, c);
227 as_alu(dest, dest, Operand2(both.snd()), op, s, c);
228 return true;
229 }
230
ma_alu(Register src1,Imm32 imm,Register dest,AutoRegisterScope & scratch,ALUOp op,SBit s,Condition c)231 void MacroAssemblerARM::ma_alu(Register src1, Imm32 imm, Register dest,
232 AutoRegisterScope& scratch, ALUOp op, SBit s,
233 Condition c) {
234 // ma_mov should be used for moves.
235 MOZ_ASSERT(op != OpMov);
236 MOZ_ASSERT(op != OpMvn);
237 MOZ_ASSERT(src1 != scratch);
238
239 // As it turns out, if you ask for a compare-like instruction you *probably*
240 // want it to set condition codes.
241 MOZ_ASSERT_IF(dest == InvalidReg, s == SetCC);
242
243 // The operator gives us the ability to determine how this can be used.
244 Imm8 imm8 = Imm8(imm.value);
245 // One instruction: If we can encode it using an imm8m, then do so.
246 if (!imm8.invalid()) {
247 as_alu(dest, src1, imm8, op, s, c);
248 return;
249 }
250
251 // One instruction, negated:
252 Imm32 negImm = imm;
253 Register negDest;
254 ALUOp negOp = ALUNeg(op, dest, scratch, &negImm, &negDest);
255 Imm8 negImm8 = Imm8(negImm.value);
256 // 'add r1, r2, -15' can be replaced with 'sub r1, r2, 15'.
257 // The dest can be replaced (InvalidReg => scratch).
258 // This is useful if we wish to negate tst. tst has an invalid (aka not
259 // used) dest, but its negation bic requires a dest.
260 if (negOp != OpInvalid && !negImm8.invalid()) {
261 as_alu(negDest, src1, negImm8, negOp, s, c);
262 return;
263 }
264
265 // Start by attempting to generate a two instruction form. Some things
266 // cannot be made into two-inst forms correctly. Namely, adds dest, src,
267 // 0xffff. Since we want the condition codes (and don't know which ones
268 // will be checked), we need to assume that the overflow flag will be
269 // checked and add{,s} dest, src, 0xff00; add{,s} dest, dest, 0xff is not
270 // guaranteed to set the overflof flag the same as the (theoretical) one
271 // instruction variant.
272 if (alu_dbl(src1, imm, dest, op, s, c)) {
273 return;
274 }
275
276 // And try with its negative.
277 if (negOp != OpInvalid && alu_dbl(src1, negImm, negDest, negOp, s, c)) {
278 return;
279 }
280
281 ma_mov(imm, scratch, c);
282 as_alu(dest, src1, O2Reg(scratch), op, s, c);
283 }
284
ma_alu(Register src1,Operand op2,Register dest,ALUOp op,SBit s,Assembler::Condition c)285 void MacroAssemblerARM::ma_alu(Register src1, Operand op2, Register dest,
286 ALUOp op, SBit s, Assembler::Condition c) {
287 MOZ_ASSERT(op2.tag() == Operand::Tag::OP2);
288 as_alu(dest, src1, op2.toOp2(), op, s, c);
289 }
290
ma_alu(Register src1,Operand2 op2,Register dest,ALUOp op,SBit s,Condition c)291 void MacroAssemblerARM::ma_alu(Register src1, Operand2 op2, Register dest,
292 ALUOp op, SBit s, Condition c) {
293 as_alu(dest, src1, op2, op, s, c);
294 }
295
ma_nop()296 void MacroAssemblerARM::ma_nop() { as_nop(); }
297
ma_movPatchable(Imm32 imm_,Register dest,Assembler::Condition c)298 BufferOffset MacroAssemblerARM::ma_movPatchable(Imm32 imm_, Register dest,
299 Assembler::Condition c) {
300 int32_t imm = imm_.value;
301 if (HasMOVWT()) {
302 BufferOffset offset = as_movw(dest, Imm16(imm & 0xffff), c);
303 as_movt(dest, Imm16(imm >> 16 & 0xffff), c);
304 return offset;
305 } else {
306 return as_Imm32Pool(dest, imm, c);
307 }
308 }
309
ma_movPatchable(ImmPtr imm,Register dest,Assembler::Condition c)310 BufferOffset MacroAssemblerARM::ma_movPatchable(ImmPtr imm, Register dest,
311 Assembler::Condition c) {
312 return ma_movPatchable(Imm32(int32_t(imm.value)), dest, c);
313 }
314
315 /* static */
316 template <class Iter>
ma_mov_patch(Imm32 imm32,Register dest,Assembler::Condition c,RelocStyle rs,Iter iter)317 void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
318 Assembler::Condition c, RelocStyle rs,
319 Iter iter) {
320 // The current instruction must be an actual instruction,
321 // not automatically-inserted boilerplate.
322 MOZ_ASSERT(iter.cur());
323 MOZ_ASSERT(iter.cur() == iter.maybeSkipAutomaticInstructions());
324
325 int32_t imm = imm32.value;
326 switch (rs) {
327 case L_MOVWT:
328 Assembler::as_movw_patch(dest, Imm16(imm & 0xffff), c, iter.cur());
329 Assembler::as_movt_patch(dest, Imm16(imm >> 16 & 0xffff), c, iter.next());
330 break;
331 case L_LDR:
332 Assembler::WritePoolEntry(iter.cur(), c, imm);
333 break;
334 }
335 }
336
337 template void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
338 Assembler::Condition c,
339 RelocStyle rs,
340 InstructionIterator iter);
341 template void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
342 Assembler::Condition c,
343 RelocStyle rs,
344 BufferInstructionIterator iter);
345
ma_mov(Register src,Register dest,SBit s,Assembler::Condition c)346 void MacroAssemblerARM::ma_mov(Register src, Register dest, SBit s,
347 Assembler::Condition c) {
348 if (s == SetCC || dest != src) {
349 as_mov(dest, O2Reg(src), s, c);
350 }
351 }
352
ma_mov(Imm32 imm,Register dest,Assembler::Condition c)353 void MacroAssemblerARM::ma_mov(Imm32 imm, Register dest,
354 Assembler::Condition c) {
355 // Try mov with Imm8 operand.
356 Imm8 imm8 = Imm8(imm.value);
357 if (!imm8.invalid()) {
358 as_alu(dest, InvalidReg, imm8, OpMov, LeaveCC, c);
359 return;
360 }
361
362 // Try mvn with Imm8 operand.
363 Imm8 negImm8 = Imm8(~imm.value);
364 if (!negImm8.invalid()) {
365 as_alu(dest, InvalidReg, negImm8, OpMvn, LeaveCC, c);
366 return;
367 }
368
369 // Try movw/movt.
370 if (HasMOVWT()) {
371 // ARMv7 supports movw/movt. movw zero-extends its 16 bit argument,
372 // so we can set the register this way. movt leaves the bottom 16
373 // bits in tact, so we always need a movw.
374 as_movw(dest, Imm16(imm.value & 0xffff), c);
375 if (uint32_t(imm.value) >> 16) {
376 as_movt(dest, Imm16(uint32_t(imm.value) >> 16), c);
377 }
378 return;
379 }
380
381 // If we don't have movw/movt, we need a load.
382 as_Imm32Pool(dest, imm.value, c);
383 }
384
ma_mov(ImmWord imm,Register dest,Assembler::Condition c)385 void MacroAssemblerARM::ma_mov(ImmWord imm, Register dest,
386 Assembler::Condition c) {
387 ma_mov(Imm32(imm.value), dest, c);
388 }
389
ma_mov(ImmGCPtr ptr,Register dest)390 void MacroAssemblerARM::ma_mov(ImmGCPtr ptr, Register dest) {
391 BufferOffset offset =
392 ma_movPatchable(Imm32(uintptr_t(ptr.value)), dest, Always);
393 writeDataRelocation(offset, ptr);
394 }
395
396 // Shifts (just a move with a shifting op2)
ma_lsl(Imm32 shift,Register src,Register dst)397 void MacroAssemblerARM::ma_lsl(Imm32 shift, Register src, Register dst) {
398 as_mov(dst, lsl(src, shift.value));
399 }
400
ma_lsr(Imm32 shift,Register src,Register dst)401 void MacroAssemblerARM::ma_lsr(Imm32 shift, Register src, Register dst) {
402 as_mov(dst, lsr(src, shift.value));
403 }
404
ma_asr(Imm32 shift,Register src,Register dst)405 void MacroAssemblerARM::ma_asr(Imm32 shift, Register src, Register dst) {
406 as_mov(dst, asr(src, shift.value));
407 }
408
ma_ror(Imm32 shift,Register src,Register dst)409 void MacroAssemblerARM::ma_ror(Imm32 shift, Register src, Register dst) {
410 as_mov(dst, ror(src, shift.value));
411 }
412
ma_rol(Imm32 shift,Register src,Register dst)413 void MacroAssemblerARM::ma_rol(Imm32 shift, Register src, Register dst) {
414 as_mov(dst, rol(src, shift.value));
415 }
416
417 // Shifts (just a move with a shifting op2)
ma_lsl(Register shift,Register src,Register dst)418 void MacroAssemblerARM::ma_lsl(Register shift, Register src, Register dst) {
419 as_mov(dst, lsl(src, shift));
420 }
421
ma_lsr(Register shift,Register src,Register dst)422 void MacroAssemblerARM::ma_lsr(Register shift, Register src, Register dst) {
423 as_mov(dst, lsr(src, shift));
424 }
425
ma_asr(Register shift,Register src,Register dst)426 void MacroAssemblerARM::ma_asr(Register shift, Register src, Register dst) {
427 as_mov(dst, asr(src, shift));
428 }
429
ma_ror(Register shift,Register src,Register dst)430 void MacroAssemblerARM::ma_ror(Register shift, Register src, Register dst) {
431 as_mov(dst, ror(src, shift));
432 }
433
ma_rol(Register shift,Register src,Register dst,AutoRegisterScope & scratch)434 void MacroAssemblerARM::ma_rol(Register shift, Register src, Register dst,
435 AutoRegisterScope& scratch) {
436 as_rsb(scratch, shift, Imm8(32));
437 as_mov(dst, ror(src, scratch));
438 }
439
440 // Move not (dest <- ~src)
ma_mvn(Register src1,Register dest,SBit s,Assembler::Condition c)441 void MacroAssemblerARM::ma_mvn(Register src1, Register dest, SBit s,
442 Assembler::Condition c) {
443 as_alu(dest, InvalidReg, O2Reg(src1), OpMvn, s, c);
444 }
445
446 // Negate (dest <- -src), src is a register, rather than a general op2.
ma_neg(Register src1,Register dest,SBit s,Assembler::Condition c)447 void MacroAssemblerARM::ma_neg(Register src1, Register dest, SBit s,
448 Assembler::Condition c) {
449 as_rsb(dest, src1, Imm8(0), s, c);
450 }
451
452 // And.
ma_and(Register src,Register dest,SBit s,Assembler::Condition c)453 void MacroAssemblerARM::ma_and(Register src, Register dest, SBit s,
454 Assembler::Condition c) {
455 ma_and(dest, src, dest);
456 }
457
ma_and(Register src1,Register src2,Register dest,SBit s,Assembler::Condition c)458 void MacroAssemblerARM::ma_and(Register src1, Register src2, Register dest,
459 SBit s, Assembler::Condition c) {
460 as_and(dest, src1, O2Reg(src2), s, c);
461 }
462
ma_and(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)463 void MacroAssemblerARM::ma_and(Imm32 imm, Register dest,
464 AutoRegisterScope& scratch, SBit s,
465 Assembler::Condition c) {
466 ma_alu(dest, imm, dest, scratch, OpAnd, s, c);
467 }
468
ma_and(Imm32 imm,Register src1,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)469 void MacroAssemblerARM::ma_and(Imm32 imm, Register src1, Register dest,
470 AutoRegisterScope& scratch, SBit s,
471 Assembler::Condition c) {
472 ma_alu(src1, imm, dest, scratch, OpAnd, s, c);
473 }
474
475 // Bit clear (dest <- dest & ~imm) or (dest <- src1 & ~src2).
ma_bic(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)476 void MacroAssemblerARM::ma_bic(Imm32 imm, Register dest,
477 AutoRegisterScope& scratch, SBit s,
478 Assembler::Condition c) {
479 ma_alu(dest, imm, dest, scratch, OpBic, s, c);
480 }
481
482 // Exclusive or.
ma_eor(Register src,Register dest,SBit s,Assembler::Condition c)483 void MacroAssemblerARM::ma_eor(Register src, Register dest, SBit s,
484 Assembler::Condition c) {
485 ma_eor(dest, src, dest, s, c);
486 }
487
ma_eor(Register src1,Register src2,Register dest,SBit s,Assembler::Condition c)488 void MacroAssemblerARM::ma_eor(Register src1, Register src2, Register dest,
489 SBit s, Assembler::Condition c) {
490 as_eor(dest, src1, O2Reg(src2), s, c);
491 }
492
ma_eor(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)493 void MacroAssemblerARM::ma_eor(Imm32 imm, Register dest,
494 AutoRegisterScope& scratch, SBit s,
495 Assembler::Condition c) {
496 ma_alu(dest, imm, dest, scratch, OpEor, s, c);
497 }
498
ma_eor(Imm32 imm,Register src1,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)499 void MacroAssemblerARM::ma_eor(Imm32 imm, Register src1, Register dest,
500 AutoRegisterScope& scratch, SBit s,
501 Assembler::Condition c) {
502 ma_alu(src1, imm, dest, scratch, OpEor, s, c);
503 }
504
505 // Or.
ma_orr(Register src,Register dest,SBit s,Assembler::Condition c)506 void MacroAssemblerARM::ma_orr(Register src, Register dest, SBit s,
507 Assembler::Condition c) {
508 ma_orr(dest, src, dest, s, c);
509 }
510
ma_orr(Register src1,Register src2,Register dest,SBit s,Assembler::Condition c)511 void MacroAssemblerARM::ma_orr(Register src1, Register src2, Register dest,
512 SBit s, Assembler::Condition c) {
513 as_orr(dest, src1, O2Reg(src2), s, c);
514 }
515
ma_orr(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)516 void MacroAssemblerARM::ma_orr(Imm32 imm, Register dest,
517 AutoRegisterScope& scratch, SBit s,
518 Assembler::Condition c) {
519 ma_alu(dest, imm, dest, scratch, OpOrr, s, c);
520 }
521
ma_orr(Imm32 imm,Register src1,Register dest,AutoRegisterScope & scratch,SBit s,Assembler::Condition c)522 void MacroAssemblerARM::ma_orr(Imm32 imm, Register src1, Register dest,
523 AutoRegisterScope& scratch, SBit s,
524 Assembler::Condition c) {
525 ma_alu(src1, imm, dest, scratch, OpOrr, s, c);
526 }
527
528 // Arithmetic-based ops.
529 // Add with carry.
ma_adc(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)530 void MacroAssemblerARM::ma_adc(Imm32 imm, Register dest,
531 AutoRegisterScope& scratch, SBit s,
532 Condition c) {
533 ma_alu(dest, imm, dest, scratch, OpAdc, s, c);
534 }
535
ma_adc(Register src,Register dest,SBit s,Condition c)536 void MacroAssemblerARM::ma_adc(Register src, Register dest, SBit s,
537 Condition c) {
538 as_alu(dest, dest, O2Reg(src), OpAdc, s, c);
539 }
540
ma_adc(Register src1,Register src2,Register dest,SBit s,Condition c)541 void MacroAssemblerARM::ma_adc(Register src1, Register src2, Register dest,
542 SBit s, Condition c) {
543 as_alu(dest, src1, O2Reg(src2), OpAdc, s, c);
544 }
545
546 // Add.
ma_add(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)547 void MacroAssemblerARM::ma_add(Imm32 imm, Register dest,
548 AutoRegisterScope& scratch, SBit s,
549 Condition c) {
550 ma_alu(dest, imm, dest, scratch, OpAdd, s, c);
551 }
552
ma_add(Register src1,Register dest,SBit s,Condition c)553 void MacroAssemblerARM::ma_add(Register src1, Register dest, SBit s,
554 Condition c) {
555 ma_alu(dest, O2Reg(src1), dest, OpAdd, s, c);
556 }
557
ma_add(Register src1,Register src2,Register dest,SBit s,Condition c)558 void MacroAssemblerARM::ma_add(Register src1, Register src2, Register dest,
559 SBit s, Condition c) {
560 as_alu(dest, src1, O2Reg(src2), OpAdd, s, c);
561 }
562
ma_add(Register src1,Operand op,Register dest,SBit s,Condition c)563 void MacroAssemblerARM::ma_add(Register src1, Operand op, Register dest, SBit s,
564 Condition c) {
565 ma_alu(src1, op, dest, OpAdd, s, c);
566 }
567
ma_add(Register src1,Imm32 op,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)568 void MacroAssemblerARM::ma_add(Register src1, Imm32 op, Register dest,
569 AutoRegisterScope& scratch, SBit s,
570 Condition c) {
571 ma_alu(src1, op, dest, scratch, OpAdd, s, c);
572 }
573
574 // Subtract with carry.
ma_sbc(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)575 void MacroAssemblerARM::ma_sbc(Imm32 imm, Register dest,
576 AutoRegisterScope& scratch, SBit s,
577 Condition c) {
578 ma_alu(dest, imm, dest, scratch, OpSbc, s, c);
579 }
580
ma_sbc(Register src1,Register dest,SBit s,Condition c)581 void MacroAssemblerARM::ma_sbc(Register src1, Register dest, SBit s,
582 Condition c) {
583 as_alu(dest, dest, O2Reg(src1), OpSbc, s, c);
584 }
585
ma_sbc(Register src1,Register src2,Register dest,SBit s,Condition c)586 void MacroAssemblerARM::ma_sbc(Register src1, Register src2, Register dest,
587 SBit s, Condition c) {
588 as_alu(dest, src1, O2Reg(src2), OpSbc, s, c);
589 }
590
591 // Subtract.
ma_sub(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)592 void MacroAssemblerARM::ma_sub(Imm32 imm, Register dest,
593 AutoRegisterScope& scratch, SBit s,
594 Condition c) {
595 ma_alu(dest, imm, dest, scratch, OpSub, s, c);
596 }
597
ma_sub(Register src1,Register dest,SBit s,Condition c)598 void MacroAssemblerARM::ma_sub(Register src1, Register dest, SBit s,
599 Condition c) {
600 ma_alu(dest, Operand(src1), dest, OpSub, s, c);
601 }
602
ma_sub(Register src1,Register src2,Register dest,SBit s,Condition c)603 void MacroAssemblerARM::ma_sub(Register src1, Register src2, Register dest,
604 SBit s, Condition c) {
605 ma_alu(src1, Operand(src2), dest, OpSub, s, c);
606 }
607
ma_sub(Register src1,Operand op,Register dest,SBit s,Condition c)608 void MacroAssemblerARM::ma_sub(Register src1, Operand op, Register dest, SBit s,
609 Condition c) {
610 ma_alu(src1, op, dest, OpSub, s, c);
611 }
612
ma_sub(Register src1,Imm32 op,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)613 void MacroAssemblerARM::ma_sub(Register src1, Imm32 op, Register dest,
614 AutoRegisterScope& scratch, SBit s,
615 Condition c) {
616 ma_alu(src1, op, dest, scratch, OpSub, s, c);
617 }
618
619 // Reverse subtract.
ma_rsb(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)620 void MacroAssemblerARM::ma_rsb(Imm32 imm, Register dest,
621 AutoRegisterScope& scratch, SBit s,
622 Condition c) {
623 ma_alu(dest, imm, dest, scratch, OpRsb, s, c);
624 }
625
ma_rsb(Register src1,Register dest,SBit s,Condition c)626 void MacroAssemblerARM::ma_rsb(Register src1, Register dest, SBit s,
627 Condition c) {
628 as_alu(dest, src1, O2Reg(dest), OpRsb, s, c);
629 }
630
ma_rsb(Register src1,Register src2,Register dest,SBit s,Condition c)631 void MacroAssemblerARM::ma_rsb(Register src1, Register src2, Register dest,
632 SBit s, Condition c) {
633 as_alu(dest, src1, O2Reg(src2), OpRsb, s, c);
634 }
635
ma_rsb(Register src1,Imm32 op2,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)636 void MacroAssemblerARM::ma_rsb(Register src1, Imm32 op2, Register dest,
637 AutoRegisterScope& scratch, SBit s,
638 Condition c) {
639 ma_alu(src1, op2, dest, scratch, OpRsb, s, c);
640 }
641
642 // Reverse subtract with carry.
ma_rsc(Imm32 imm,Register dest,AutoRegisterScope & scratch,SBit s,Condition c)643 void MacroAssemblerARM::ma_rsc(Imm32 imm, Register dest,
644 AutoRegisterScope& scratch, SBit s,
645 Condition c) {
646 ma_alu(dest, imm, dest, scratch, OpRsc, s, c);
647 }
648
ma_rsc(Register src1,Register dest,SBit s,Condition c)649 void MacroAssemblerARM::ma_rsc(Register src1, Register dest, SBit s,
650 Condition c) {
651 as_alu(dest, dest, O2Reg(src1), OpRsc, s, c);
652 }
653
ma_rsc(Register src1,Register src2,Register dest,SBit s,Condition c)654 void MacroAssemblerARM::ma_rsc(Register src1, Register src2, Register dest,
655 SBit s, Condition c) {
656 as_alu(dest, src1, O2Reg(src2), OpRsc, s, c);
657 }
658
659 // Compares/tests.
660 // Compare negative (sets condition codes as src1 + src2 would).
ma_cmn(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)661 void MacroAssemblerARM::ma_cmn(Register src1, Imm32 imm,
662 AutoRegisterScope& scratch, Condition c) {
663 ma_alu(src1, imm, InvalidReg, scratch, OpCmn, SetCC, c);
664 }
665
ma_cmn(Register src1,Register src2,Condition c)666 void MacroAssemblerARM::ma_cmn(Register src1, Register src2, Condition c) {
667 as_alu(InvalidReg, src2, O2Reg(src1), OpCmn, SetCC, c);
668 }
669
ma_cmn(Register src1,Operand op,Condition c)670 void MacroAssemblerARM::ma_cmn(Register src1, Operand op, Condition c) {
671 MOZ_CRASH("Feature NYI");
672 }
673
674 // Compare (src - src2).
ma_cmp(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)675 void MacroAssemblerARM::ma_cmp(Register src1, Imm32 imm,
676 AutoRegisterScope& scratch, Condition c) {
677 ma_alu(src1, imm, InvalidReg, scratch, OpCmp, SetCC, c);
678 }
679
ma_cmp(Register src1,ImmTag tag,Condition c)680 void MacroAssemblerARM::ma_cmp(Register src1, ImmTag tag, Condition c) {
681 // ImmTag comparisons can always be done without use of a scratch register.
682 Imm8 negtag = Imm8(-tag.value);
683 MOZ_ASSERT(!negtag.invalid());
684 as_cmn(src1, negtag, c);
685 }
686
ma_cmp(Register src1,ImmWord ptr,AutoRegisterScope & scratch,Condition c)687 void MacroAssemblerARM::ma_cmp(Register src1, ImmWord ptr,
688 AutoRegisterScope& scratch, Condition c) {
689 ma_cmp(src1, Imm32(ptr.value), scratch, c);
690 }
691
ma_cmp(Register src1,ImmGCPtr ptr,AutoRegisterScope & scratch,Condition c)692 void MacroAssemblerARM::ma_cmp(Register src1, ImmGCPtr ptr,
693 AutoRegisterScope& scratch, Condition c) {
694 ma_mov(ptr, scratch);
695 ma_cmp(src1, scratch, c);
696 }
697
ma_cmp(Register src1,Operand op,AutoRegisterScope & scratch,AutoRegisterScope & scratch2,Condition c)698 void MacroAssemblerARM::ma_cmp(Register src1, Operand op,
699 AutoRegisterScope& scratch,
700 AutoRegisterScope& scratch2, Condition c) {
701 switch (op.tag()) {
702 case Operand::Tag::OP2:
703 as_cmp(src1, op.toOp2(), c);
704 break;
705 case Operand::Tag::MEM:
706 ma_ldr(op.toAddress(), scratch, scratch2);
707 as_cmp(src1, O2Reg(scratch), c);
708 break;
709 default:
710 MOZ_CRASH("trying to compare FP and integer registers");
711 }
712 }
713
ma_cmp(Register src1,Register src2,Condition c)714 void MacroAssemblerARM::ma_cmp(Register src1, Register src2, Condition c) {
715 as_cmp(src1, O2Reg(src2), c);
716 }
717
718 // Test for equality, (src1 ^ src2).
ma_teq(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)719 void MacroAssemblerARM::ma_teq(Register src1, Imm32 imm,
720 AutoRegisterScope& scratch, Condition c) {
721 ma_alu(src1, imm, InvalidReg, scratch, OpTeq, SetCC, c);
722 }
723
ma_teq(Register src1,Register src2,Condition c)724 void MacroAssemblerARM::ma_teq(Register src1, Register src2, Condition c) {
725 as_tst(src1, O2Reg(src2), c);
726 }
727
ma_teq(Register src1,Operand op,Condition c)728 void MacroAssemblerARM::ma_teq(Register src1, Operand op, Condition c) {
729 as_teq(src1, op.toOp2(), c);
730 }
731
732 // Test (src1 & src2).
ma_tst(Register src1,Imm32 imm,AutoRegisterScope & scratch,Condition c)733 void MacroAssemblerARM::ma_tst(Register src1, Imm32 imm,
734 AutoRegisterScope& scratch, Condition c) {
735 ma_alu(src1, imm, InvalidReg, scratch, OpTst, SetCC, c);
736 }
737
ma_tst(Register src1,Register src2,Condition c)738 void MacroAssemblerARM::ma_tst(Register src1, Register src2, Condition c) {
739 as_tst(src1, O2Reg(src2), c);
740 }
741
ma_tst(Register src1,Operand op,Condition c)742 void MacroAssemblerARM::ma_tst(Register src1, Operand op, Condition c) {
743 as_tst(src1, op.toOp2(), c);
744 }
745
ma_mul(Register src1,Register src2,Register dest)746 void MacroAssemblerARM::ma_mul(Register src1, Register src2, Register dest) {
747 as_mul(dest, src1, src2);
748 }
749
ma_mul(Register src1,Imm32 imm,Register dest,AutoRegisterScope & scratch)750 void MacroAssemblerARM::ma_mul(Register src1, Imm32 imm, Register dest,
751 AutoRegisterScope& scratch) {
752 ma_mov(imm, scratch);
753 as_mul(dest, src1, scratch);
754 }
755
ma_check_mul(Register src1,Register src2,Register dest,AutoRegisterScope & scratch,Condition cond)756 Assembler::Condition MacroAssemblerARM::ma_check_mul(Register src1,
757 Register src2,
758 Register dest,
759 AutoRegisterScope& scratch,
760 Condition cond) {
761 // TODO: this operation is illegal on armv6 and earlier
762 // if src2 == scratch or src2 == dest.
763 if (cond == Equal || cond == NotEqual) {
764 as_smull(scratch, dest, src1, src2, SetCC);
765 return cond;
766 }
767
768 if (cond == Overflow) {
769 as_smull(scratch, dest, src1, src2);
770 as_cmp(scratch, asr(dest, 31));
771 return NotEqual;
772 }
773
774 MOZ_CRASH("Condition NYI");
775 }
776
ma_check_mul(Register src1,Imm32 imm,Register dest,AutoRegisterScope & scratch,Condition cond)777 Assembler::Condition MacroAssemblerARM::ma_check_mul(Register src1, Imm32 imm,
778 Register dest,
779 AutoRegisterScope& scratch,
780 Condition cond) {
781 ma_mov(imm, scratch);
782
783 if (cond == Equal || cond == NotEqual) {
784 as_smull(scratch, dest, scratch, src1, SetCC);
785 return cond;
786 }
787
788 if (cond == Overflow) {
789 as_smull(scratch, dest, scratch, src1);
790 as_cmp(scratch, asr(dest, 31));
791 return NotEqual;
792 }
793
794 MOZ_CRASH("Condition NYI");
795 }
796
ma_umull(Register src1,Imm32 imm,Register destHigh,Register destLow,AutoRegisterScope & scratch)797 void MacroAssemblerARM::ma_umull(Register src1, Imm32 imm, Register destHigh,
798 Register destLow, AutoRegisterScope& scratch) {
799 ma_mov(imm, scratch);
800 as_umull(destHigh, destLow, src1, scratch);
801 }
802
ma_umull(Register src1,Register src2,Register destHigh,Register destLow)803 void MacroAssemblerARM::ma_umull(Register src1, Register src2,
804 Register destHigh, Register destLow) {
805 as_umull(destHigh, destLow, src1, src2);
806 }
807
ma_mod_mask(Register src,Register dest,Register hold,Register tmp,AutoRegisterScope & scratch,AutoRegisterScope & scratch2,int32_t shift)808 void MacroAssemblerARM::ma_mod_mask(Register src, Register dest, Register hold,
809 Register tmp, AutoRegisterScope& scratch,
810 AutoRegisterScope& scratch2,
811 int32_t shift) {
812 // We wish to compute x % (1<<y) - 1 for a known constant, y.
813 //
814 // 1. Let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit dividend as
815 // a number in base b, namely c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
816 //
817 // 2. Since both addition and multiplication commute with modulus:
818 // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
819 // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
820 //
821 // 3. Since b == C + 1, b % C == 1, and b^n % C == 1 the whole thing
822 // simplifies to: c_0 + c_1 + c_2 ... c_n % C
823 //
824 // Each c_n can easily be computed by a shift/bitextract, and the modulus
825 // can be maintained by simply subtracting by C whenever the number gets
826 // over C.
827 int32_t mask = (1 << shift) - 1;
828 Label head;
829
830 // Register 'hold' holds -1 if the value was negative, 1 otherwise. The
831 // scratch reg holds the remaining bits that have not been processed lr
832 // serves as a temporary location to store extracted bits into as well as
833 // holding the trial subtraction as a temp value dest is the accumulator
834 // (and holds the final result)
835 //
836 // Move the whole value into tmp, setting the codition codes so we can muck
837 // with them later.
838 as_mov(tmp, O2Reg(src), SetCC);
839 // Zero out the dest.
840 ma_mov(Imm32(0), dest);
841 // Set the hold appropriately.
842 ma_mov(Imm32(1), hold);
843 ma_mov(Imm32(-1), hold, Signed);
844 as_rsb(tmp, tmp, Imm8(0), SetCC, Signed);
845
846 // Begin the main loop.
847 bind(&head);
848 {
849 // Extract the bottom bits.
850 ma_and(Imm32(mask), tmp, scratch, scratch2);
851 // Add those bits to the accumulator.
852 ma_add(scratch, dest, dest);
853 // Do a trial subtraction, this is the same operation as cmp, but we store
854 // the dest.
855 ma_sub(dest, Imm32(mask), scratch, scratch2, SetCC);
856 // If (sum - C) > 0, store sum - C back into sum, thus performing a modulus.
857 ma_mov(scratch, dest, LeaveCC, NotSigned);
858 // Get rid of the bits that we extracted before, and set the condition
859 // codes.
860 as_mov(tmp, lsr(tmp, shift), SetCC);
861 // If the shift produced zero, finish, otherwise, continue in the loop.
862 ma_b(&head, NonZero);
863 }
864
865 // Check the hold to see if we need to negate the result. Hold can only be
866 // 1 or -1, so this will never set the 0 flag.
867 as_cmp(hold, Imm8(0));
868 // If the hold was non-zero, negate the result to be in line with what JS
869 // wants this will set the condition codes if we try to negate.
870 as_rsb(dest, dest, Imm8(0), SetCC, Signed);
871 // Since the Zero flag is not set by the compare, we can *only* set the Zero
872 // flag in the rsb, so Zero is set iff we negated zero (e.g. the result of
873 // the computation was -0.0).
874 }
875
ma_smod(Register num,Register div,Register dest,AutoRegisterScope & scratch)876 void MacroAssemblerARM::ma_smod(Register num, Register div, Register dest,
877 AutoRegisterScope& scratch) {
878 as_sdiv(scratch, num, div);
879 as_mls(dest, num, scratch, div);
880 }
881
ma_umod(Register num,Register div,Register dest,AutoRegisterScope & scratch)882 void MacroAssemblerARM::ma_umod(Register num, Register div, Register dest,
883 AutoRegisterScope& scratch) {
884 as_udiv(scratch, num, div);
885 as_mls(dest, num, scratch, div);
886 }
887
888 // Division
ma_sdiv(Register num,Register div,Register dest,Condition cond)889 void MacroAssemblerARM::ma_sdiv(Register num, Register div, Register dest,
890 Condition cond) {
891 as_sdiv(dest, num, div, cond);
892 }
893
ma_udiv(Register num,Register div,Register dest,Condition cond)894 void MacroAssemblerARM::ma_udiv(Register num, Register div, Register dest,
895 Condition cond) {
896 as_udiv(dest, num, div, cond);
897 }
898
899 // Miscellaneous instructions.
ma_clz(Register src,Register dest,Condition cond)900 void MacroAssemblerARM::ma_clz(Register src, Register dest, Condition cond) {
901 as_clz(dest, src, cond);
902 }
903
ma_ctz(Register src,Register dest,AutoRegisterScope & scratch)904 void MacroAssemblerARM::ma_ctz(Register src, Register dest,
905 AutoRegisterScope& scratch) {
906 // int c = __clz(a & -a);
907 // return a ? 31 - c : c;
908 as_rsb(scratch, src, Imm8(0), SetCC);
909 as_and(dest, src, O2Reg(scratch), LeaveCC);
910 as_clz(dest, dest);
911 as_rsb(dest, dest, Imm8(0x1F), LeaveCC, Assembler::NotEqual);
912 }
913
914 // Memory.
915 // Shortcut for when we know we're transferring 32 bits of data.
ma_dtr(LoadStore ls,Register rn,Imm32 offset,Register rt,AutoRegisterScope & scratch,Index mode,Assembler::Condition cc)916 void MacroAssemblerARM::ma_dtr(LoadStore ls, Register rn, Imm32 offset,
917 Register rt, AutoRegisterScope& scratch,
918 Index mode, Assembler::Condition cc) {
919 ma_dataTransferN(ls, 32, true, rn, offset, rt, scratch, mode, cc);
920 }
921
ma_dtr(LoadStore ls,Register rt,const Address & addr,AutoRegisterScope & scratch,Index mode,Condition cc)922 void MacroAssemblerARM::ma_dtr(LoadStore ls, Register rt, const Address& addr,
923 AutoRegisterScope& scratch, Index mode,
924 Condition cc) {
925 ma_dataTransferN(ls, 32, true, addr.base, Imm32(addr.offset), rt, scratch,
926 mode, cc);
927 }
928
ma_str(Register rt,DTRAddr addr,Index mode,Condition cc)929 void MacroAssemblerARM::ma_str(Register rt, DTRAddr addr, Index mode,
930 Condition cc) {
931 as_dtr(IsStore, 32, mode, rt, addr, cc);
932 }
933
ma_str(Register rt,const Address & addr,AutoRegisterScope & scratch,Index mode,Condition cc)934 void MacroAssemblerARM::ma_str(Register rt, const Address& addr,
935 AutoRegisterScope& scratch, Index mode,
936 Condition cc) {
937 ma_dtr(IsStore, rt, addr, scratch, mode, cc);
938 }
939
ma_strd(Register rt,DebugOnly<Register> rt2,EDtrAddr addr,Index mode,Condition cc)940 void MacroAssemblerARM::ma_strd(Register rt, DebugOnly<Register> rt2,
941 EDtrAddr addr, Index mode, Condition cc) {
942 MOZ_ASSERT((rt.code() & 1) == 0);
943 MOZ_ASSERT(rt2.value.code() == rt.code() + 1);
944 as_extdtr(IsStore, 64, true, mode, rt, addr, cc);
945 }
946
ma_ldr(DTRAddr addr,Register rt,Index mode,Condition cc)947 void MacroAssemblerARM::ma_ldr(DTRAddr addr, Register rt, Index mode,
948 Condition cc) {
949 as_dtr(IsLoad, 32, mode, rt, addr, cc);
950 }
951
ma_ldr(const Address & addr,Register rt,AutoRegisterScope & scratch,Index mode,Condition cc)952 void MacroAssemblerARM::ma_ldr(const Address& addr, Register rt,
953 AutoRegisterScope& scratch, Index mode,
954 Condition cc) {
955 ma_dtr(IsLoad, rt, addr, scratch, mode, cc);
956 }
957
ma_ldrb(DTRAddr addr,Register rt,Index mode,Condition cc)958 void MacroAssemblerARM::ma_ldrb(DTRAddr addr, Register rt, Index mode,
959 Condition cc) {
960 as_dtr(IsLoad, 8, mode, rt, addr, cc);
961 }
962
ma_ldrsh(EDtrAddr addr,Register rt,Index mode,Condition cc)963 void MacroAssemblerARM::ma_ldrsh(EDtrAddr addr, Register rt, Index mode,
964 Condition cc) {
965 as_extdtr(IsLoad, 16, true, mode, rt, addr, cc);
966 }
967
ma_ldrh(EDtrAddr addr,Register rt,Index mode,Condition cc)968 void MacroAssemblerARM::ma_ldrh(EDtrAddr addr, Register rt, Index mode,
969 Condition cc) {
970 as_extdtr(IsLoad, 16, false, mode, rt, addr, cc);
971 }
972
ma_ldrsb(EDtrAddr addr,Register rt,Index mode,Condition cc)973 void MacroAssemblerARM::ma_ldrsb(EDtrAddr addr, Register rt, Index mode,
974 Condition cc) {
975 as_extdtr(IsLoad, 8, true, mode, rt, addr, cc);
976 }
977
ma_ldrd(EDtrAddr addr,Register rt,DebugOnly<Register> rt2,Index mode,Condition cc)978 void MacroAssemblerARM::ma_ldrd(EDtrAddr addr, Register rt,
979 DebugOnly<Register> rt2, Index mode,
980 Condition cc) {
981 MOZ_ASSERT((rt.code() & 1) == 0);
982 MOZ_ASSERT(rt2.value.code() == rt.code() + 1);
983 MOZ_ASSERT(addr.maybeOffsetRegister() !=
984 rt); // Undefined behavior if rm == rt/rt2.
985 MOZ_ASSERT(addr.maybeOffsetRegister() != rt2);
986 as_extdtr(IsLoad, 64, true, mode, rt, addr, cc);
987 }
988
ma_strh(Register rt,EDtrAddr addr,Index mode,Condition cc)989 void MacroAssemblerARM::ma_strh(Register rt, EDtrAddr addr, Index mode,
990 Condition cc) {
991 as_extdtr(IsStore, 16, false, mode, rt, addr, cc);
992 }
993
ma_strb(Register rt,DTRAddr addr,Index mode,Condition cc)994 void MacroAssemblerARM::ma_strb(Register rt, DTRAddr addr, Index mode,
995 Condition cc) {
996 as_dtr(IsStore, 8, mode, rt, addr, cc);
997 }
998
999 // Specialty for moving N bits of data, where n == 8,16,32,64.
ma_dataTransferN(LoadStore ls,int size,bool IsSigned,Register rn,Register rm,Register rt,AutoRegisterScope & scratch,Index mode,Assembler::Condition cc,Scale scale)1000 BufferOffset MacroAssemblerARM::ma_dataTransferN(
1001 LoadStore ls, int size, bool IsSigned, Register rn, Register rm,
1002 Register rt, AutoRegisterScope& scratch, Index mode,
1003 Assembler::Condition cc, Scale scale) {
1004 MOZ_ASSERT(size == 8 || size == 16 || size == 32 || size == 64);
1005
1006 if (size == 32 || (size == 8 && !IsSigned)) {
1007 return as_dtr(ls, size, mode, rt,
1008 DTRAddr(rn, DtrRegImmShift(rm, LSL, scale)), cc);
1009 }
1010
1011 if (scale != TimesOne) {
1012 ma_lsl(Imm32(scale), rm, scratch);
1013 rm = scratch;
1014 }
1015
1016 return as_extdtr(ls, size, IsSigned, mode, rt, EDtrAddr(rn, EDtrOffReg(rm)),
1017 cc);
1018 }
1019
1020 // No scratch register is required if scale is TimesOne.
ma_dataTransferN(LoadStore ls,int size,bool IsSigned,Register rn,Register rm,Register rt,Index mode,Assembler::Condition cc)1021 BufferOffset MacroAssemblerARM::ma_dataTransferN(LoadStore ls, int size,
1022 bool IsSigned, Register rn,
1023 Register rm, Register rt,
1024 Index mode,
1025 Assembler::Condition cc) {
1026 MOZ_ASSERT(size == 8 || size == 16 || size == 32 || size == 64);
1027 if (size == 32 || (size == 8 && !IsSigned)) {
1028 return as_dtr(ls, size, mode, rt,
1029 DTRAddr(rn, DtrRegImmShift(rm, LSL, TimesOne)), cc);
1030 }
1031 return as_extdtr(ls, size, IsSigned, mode, rt, EDtrAddr(rn, EDtrOffReg(rm)),
1032 cc);
1033 }
1034
ma_dataTransferN(LoadStore ls,int size,bool IsSigned,Register rn,Imm32 offset,Register rt,AutoRegisterScope & scratch,Index mode,Assembler::Condition cc)1035 BufferOffset MacroAssemblerARM::ma_dataTransferN(LoadStore ls, int size,
1036 bool IsSigned, Register rn,
1037 Imm32 offset, Register rt,
1038 AutoRegisterScope& scratch,
1039 Index mode,
1040 Assembler::Condition cc) {
1041 MOZ_ASSERT(!(ls == IsLoad && mode == PostIndex && rt == pc),
1042 "Large-offset PostIndex loading into PC requires special logic: "
1043 "see ma_popn_pc().");
1044
1045 int off = offset.value;
1046
1047 // We can encode this as a standard ldr.
1048 if (size == 32 || (size == 8 && !IsSigned)) {
1049 if (off < 4096 && off > -4096) {
1050 // This encodes as a single instruction, Emulating mode's behavior
1051 // in a multi-instruction sequence is not necessary.
1052 return as_dtr(ls, size, mode, rt, DTRAddr(rn, DtrOffImm(off)), cc);
1053 }
1054
1055 // We cannot encode this offset in a single ldr. For mode == index,
1056 // try to encode it as |add scratch, base, imm; ldr dest, [scratch,
1057 // +offset]|. This does not wark for mode == PreIndex or mode == PostIndex.
1058 // PreIndex is simple, just do the add into the base register first,
1059 // then do a PreIndex'ed load. PostIndexed loads can be tricky.
1060 // Normally, doing the load with an index of 0, then doing an add would
1061 // work, but if the destination is the PC, you don't get to execute the
1062 // instruction after the branch, which will lead to the base register
1063 // not being updated correctly. Explicitly handle this case, without
1064 // doing anything fancy, then handle all of the other cases.
1065
1066 // mode == Offset
1067 // add scratch, base, offset_hi
1068 // ldr dest, [scratch, +offset_lo]
1069 //
1070 // mode == PreIndex
1071 // add base, base, offset_hi
1072 // ldr dest, [base, +offset_lo]!
1073
1074 int bottom = off & 0xfff;
1075 int neg_bottom = 0x1000 - bottom;
1076
1077 MOZ_ASSERT(rn != scratch);
1078 MOZ_ASSERT(mode != PostIndex);
1079
1080 // At this point, both off - bottom and off + neg_bottom will be
1081 // reasonable-ish quantities.
1082 //
1083 // Note a neg_bottom of 0x1000 can not be encoded as an immediate
1084 // negative offset in the instruction and this occurs when bottom is
1085 // zero, so this case is guarded against below.
1086 if (off < 0) {
1087 Operand2 sub_off = Imm8(-(off - bottom)); // sub_off = bottom - off
1088 if (!sub_off.invalid()) {
1089 // - sub_off = off - bottom
1090 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1091 return as_dtr(ls, size, Offset, rt, DTRAddr(scratch, DtrOffImm(bottom)),
1092 cc);
1093 }
1094
1095 // sub_off = -neg_bottom - off
1096 sub_off = Imm8(-(off + neg_bottom));
1097 if (!sub_off.invalid() && bottom != 0) {
1098 // Guarded against by: bottom != 0
1099 MOZ_ASSERT(neg_bottom < 0x1000);
1100 // - sub_off = neg_bottom + off
1101 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1102 return as_dtr(ls, size, Offset, rt,
1103 DTRAddr(scratch, DtrOffImm(-neg_bottom)), cc);
1104 }
1105 } else {
1106 // sub_off = off - bottom
1107 Operand2 sub_off = Imm8(off - bottom);
1108 if (!sub_off.invalid()) {
1109 // sub_off = off - bottom
1110 as_add(scratch, rn, sub_off, LeaveCC, cc);
1111 return as_dtr(ls, size, Offset, rt, DTRAddr(scratch, DtrOffImm(bottom)),
1112 cc);
1113 }
1114
1115 // sub_off = neg_bottom + off
1116 sub_off = Imm8(off + neg_bottom);
1117 if (!sub_off.invalid() && bottom != 0) {
1118 // Guarded against by: bottom != 0
1119 MOZ_ASSERT(neg_bottom < 0x1000);
1120 // sub_off = neg_bottom + off
1121 as_add(scratch, rn, sub_off, LeaveCC, cc);
1122 return as_dtr(ls, size, Offset, rt,
1123 DTRAddr(scratch, DtrOffImm(-neg_bottom)), cc);
1124 }
1125 }
1126
1127 ma_mov(offset, scratch);
1128 return as_dtr(ls, size, mode, rt,
1129 DTRAddr(rn, DtrRegImmShift(scratch, LSL, 0)));
1130 } else {
1131 // Should attempt to use the extended load/store instructions.
1132 if (off < 256 && off > -256) {
1133 return as_extdtr(ls, size, IsSigned, mode, rt,
1134 EDtrAddr(rn, EDtrOffImm(off)), cc);
1135 }
1136
1137 // We cannot encode this offset in a single extldr. Try to encode it as
1138 // an add scratch, base, imm; extldr dest, [scratch, +offset].
1139 int bottom = off & 0xff;
1140 int neg_bottom = 0x100 - bottom;
1141 // At this point, both off - bottom and off + neg_bottom will be
1142 // reasonable-ish quantities.
1143 //
1144 // Note a neg_bottom of 0x100 can not be encoded as an immediate
1145 // negative offset in the instruction and this occurs when bottom is
1146 // zero, so this case is guarded against below.
1147 if (off < 0) {
1148 // sub_off = bottom - off
1149 Operand2 sub_off = Imm8(-(off - bottom));
1150 if (!sub_off.invalid()) {
1151 // - sub_off = off - bottom
1152 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1153 return as_extdtr(ls, size, IsSigned, Offset, rt,
1154 EDtrAddr(scratch, EDtrOffImm(bottom)), cc);
1155 }
1156 // sub_off = -neg_bottom - off
1157 sub_off = Imm8(-(off + neg_bottom));
1158 if (!sub_off.invalid() && bottom != 0) {
1159 // Guarded against by: bottom != 0
1160 MOZ_ASSERT(neg_bottom < 0x100);
1161 // - sub_off = neg_bottom + off
1162 as_sub(scratch, rn, sub_off, LeaveCC, cc);
1163 return as_extdtr(ls, size, IsSigned, Offset, rt,
1164 EDtrAddr(scratch, EDtrOffImm(-neg_bottom)), cc);
1165 }
1166 } else {
1167 // sub_off = off - bottom
1168 Operand2 sub_off = Imm8(off - bottom);
1169 if (!sub_off.invalid()) {
1170 // sub_off = off - bottom
1171 as_add(scratch, rn, sub_off, LeaveCC, cc);
1172 return as_extdtr(ls, size, IsSigned, Offset, rt,
1173 EDtrAddr(scratch, EDtrOffImm(bottom)), cc);
1174 }
1175 // sub_off = neg_bottom + off
1176 sub_off = Imm8(off + neg_bottom);
1177 if (!sub_off.invalid() && bottom != 0) {
1178 // Guarded against by: bottom != 0
1179 MOZ_ASSERT(neg_bottom < 0x100);
1180 // sub_off = neg_bottom + off
1181 as_add(scratch, rn, sub_off, LeaveCC, cc);
1182 return as_extdtr(ls, size, IsSigned, Offset, rt,
1183 EDtrAddr(scratch, EDtrOffImm(-neg_bottom)), cc);
1184 }
1185 }
1186 ma_mov(offset, scratch);
1187 return as_extdtr(ls, size, IsSigned, mode, rt,
1188 EDtrAddr(rn, EDtrOffReg(scratch)), cc);
1189 }
1190 }
1191
ma_pop(Register r)1192 void MacroAssemblerARM::ma_pop(Register r) {
1193 as_dtr(IsLoad, 32, PostIndex, r, DTRAddr(sp, DtrOffImm(4)));
1194 }
1195
ma_popn_pc(Imm32 n,AutoRegisterScope & scratch,AutoRegisterScope & scratch2)1196 void MacroAssemblerARM::ma_popn_pc(Imm32 n, AutoRegisterScope& scratch,
1197 AutoRegisterScope& scratch2) {
1198 // pc <- [sp]; sp += n
1199 int32_t nv = n.value;
1200
1201 if (nv < 4096 && nv >= -4096) {
1202 as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(nv)));
1203 } else {
1204 ma_mov(sp, scratch);
1205 ma_add(Imm32(n), sp, scratch2);
1206 as_dtr(IsLoad, 32, Offset, pc, DTRAddr(scratch, DtrOffImm(0)));
1207 }
1208 }
1209
ma_push(Register r)1210 void MacroAssemblerARM::ma_push(Register r) {
1211 MOZ_ASSERT(r != sp, "Use ma_push_sp().");
1212 as_dtr(IsStore, 32, PreIndex, r, DTRAddr(sp, DtrOffImm(-4)));
1213 }
1214
ma_push_sp(Register r,AutoRegisterScope & scratch)1215 void MacroAssemblerARM::ma_push_sp(Register r, AutoRegisterScope& scratch) {
1216 // Pushing sp is not well-defined: use two instructions.
1217 MOZ_ASSERT(r == sp);
1218 ma_mov(sp, scratch);
1219 as_dtr(IsStore, 32, PreIndex, scratch, DTRAddr(sp, DtrOffImm(-4)));
1220 }
1221
ma_vpop(VFPRegister r)1222 void MacroAssemblerARM::ma_vpop(VFPRegister r) {
1223 startFloatTransferM(IsLoad, sp, IA, WriteBack);
1224 transferFloatReg(r);
1225 finishFloatTransfer();
1226 }
1227
ma_vpush(VFPRegister r)1228 void MacroAssemblerARM::ma_vpush(VFPRegister r) {
1229 startFloatTransferM(IsStore, sp, DB, WriteBack);
1230 transferFloatReg(r);
1231 finishFloatTransfer();
1232 }
1233
1234 // Barriers
ma_dmb(BarrierOption option)1235 void MacroAssemblerARM::ma_dmb(BarrierOption option) {
1236 if (HasDMBDSBISB()) {
1237 as_dmb(option);
1238 } else {
1239 as_dmb_trap();
1240 }
1241 }
1242
ma_dsb(BarrierOption option)1243 void MacroAssemblerARM::ma_dsb(BarrierOption option) {
1244 if (HasDMBDSBISB()) {
1245 as_dsb(option);
1246 } else {
1247 as_dsb_trap();
1248 }
1249 }
1250
1251 // Branches when done from within arm-specific code.
ma_b(Label * dest,Assembler::Condition c)1252 BufferOffset MacroAssemblerARM::ma_b(Label* dest, Assembler::Condition c) {
1253 return as_b(dest, c);
1254 }
1255
ma_bx(Register dest,Assembler::Condition c)1256 void MacroAssemblerARM::ma_bx(Register dest, Assembler::Condition c) {
1257 as_bx(dest, c);
1258 }
1259
ma_b(void * target,Assembler::Condition c)1260 void MacroAssemblerARM::ma_b(void* target, Assembler::Condition c) {
1261 // An immediate pool is used for easier patching.
1262 as_Imm32Pool(pc, uint32_t(target), c);
1263 }
1264
1265 // This is almost NEVER necessary: we'll basically never be calling a label,
1266 // except possibly in the crazy bailout-table case.
ma_bl(Label * dest,Assembler::Condition c)1267 void MacroAssemblerARM::ma_bl(Label* dest, Assembler::Condition c) {
1268 as_bl(dest, c);
1269 }
1270
ma_blx(Register reg,Assembler::Condition c)1271 void MacroAssemblerARM::ma_blx(Register reg, Assembler::Condition c) {
1272 as_blx(reg, c);
1273 }
1274
1275 // VFP/ALU
ma_vadd(FloatRegister src1,FloatRegister src2,FloatRegister dst)1276 void MacroAssemblerARM::ma_vadd(FloatRegister src1, FloatRegister src2,
1277 FloatRegister dst) {
1278 as_vadd(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1279 }
1280
ma_vadd_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1281 void MacroAssemblerARM::ma_vadd_f32(FloatRegister src1, FloatRegister src2,
1282 FloatRegister dst) {
1283 as_vadd(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1284 VFPRegister(src2).singleOverlay());
1285 }
1286
ma_vsub(FloatRegister src1,FloatRegister src2,FloatRegister dst)1287 void MacroAssemblerARM::ma_vsub(FloatRegister src1, FloatRegister src2,
1288 FloatRegister dst) {
1289 as_vsub(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1290 }
1291
ma_vsub_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1292 void MacroAssemblerARM::ma_vsub_f32(FloatRegister src1, FloatRegister src2,
1293 FloatRegister dst) {
1294 as_vsub(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1295 VFPRegister(src2).singleOverlay());
1296 }
1297
ma_vmul(FloatRegister src1,FloatRegister src2,FloatRegister dst)1298 void MacroAssemblerARM::ma_vmul(FloatRegister src1, FloatRegister src2,
1299 FloatRegister dst) {
1300 as_vmul(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1301 }
1302
ma_vmul_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1303 void MacroAssemblerARM::ma_vmul_f32(FloatRegister src1, FloatRegister src2,
1304 FloatRegister dst) {
1305 as_vmul(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1306 VFPRegister(src2).singleOverlay());
1307 }
1308
ma_vdiv(FloatRegister src1,FloatRegister src2,FloatRegister dst)1309 void MacroAssemblerARM::ma_vdiv(FloatRegister src1, FloatRegister src2,
1310 FloatRegister dst) {
1311 as_vdiv(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
1312 }
1313
ma_vdiv_f32(FloatRegister src1,FloatRegister src2,FloatRegister dst)1314 void MacroAssemblerARM::ma_vdiv_f32(FloatRegister src1, FloatRegister src2,
1315 FloatRegister dst) {
1316 as_vdiv(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
1317 VFPRegister(src2).singleOverlay());
1318 }
1319
ma_vmov(FloatRegister src,FloatRegister dest,Condition cc)1320 void MacroAssemblerARM::ma_vmov(FloatRegister src, FloatRegister dest,
1321 Condition cc) {
1322 as_vmov(dest, src, cc);
1323 }
1324
ma_vmov_f32(FloatRegister src,FloatRegister dest,Condition cc)1325 void MacroAssemblerARM::ma_vmov_f32(FloatRegister src, FloatRegister dest,
1326 Condition cc) {
1327 as_vmov(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1328 cc);
1329 }
1330
ma_vneg(FloatRegister src,FloatRegister dest,Condition cc)1331 void MacroAssemblerARM::ma_vneg(FloatRegister src, FloatRegister dest,
1332 Condition cc) {
1333 as_vneg(dest, src, cc);
1334 }
1335
ma_vneg_f32(FloatRegister src,FloatRegister dest,Condition cc)1336 void MacroAssemblerARM::ma_vneg_f32(FloatRegister src, FloatRegister dest,
1337 Condition cc) {
1338 as_vneg(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1339 cc);
1340 }
1341
ma_vabs(FloatRegister src,FloatRegister dest,Condition cc)1342 void MacroAssemblerARM::ma_vabs(FloatRegister src, FloatRegister dest,
1343 Condition cc) {
1344 as_vabs(dest, src, cc);
1345 }
1346
ma_vabs_f32(FloatRegister src,FloatRegister dest,Condition cc)1347 void MacroAssemblerARM::ma_vabs_f32(FloatRegister src, FloatRegister dest,
1348 Condition cc) {
1349 as_vabs(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1350 cc);
1351 }
1352
ma_vsqrt(FloatRegister src,FloatRegister dest,Condition cc)1353 void MacroAssemblerARM::ma_vsqrt(FloatRegister src, FloatRegister dest,
1354 Condition cc) {
1355 as_vsqrt(dest, src, cc);
1356 }
1357
ma_vsqrt_f32(FloatRegister src,FloatRegister dest,Condition cc)1358 void MacroAssemblerARM::ma_vsqrt_f32(FloatRegister src, FloatRegister dest,
1359 Condition cc) {
1360 as_vsqrt(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
1361 cc);
1362 }
1363
DoubleHighWord(double d)1364 static inline uint32_t DoubleHighWord(double d) {
1365 return static_cast<uint32_t>(BitwiseCast<uint64_t>(d) >> 32);
1366 }
1367
DoubleLowWord(double d)1368 static inline uint32_t DoubleLowWord(double d) {
1369 return static_cast<uint32_t>(BitwiseCast<uint64_t>(d)) & uint32_t(0xffffffff);
1370 }
1371
ma_vimm(double value,FloatRegister dest,Condition cc)1372 void MacroAssemblerARM::ma_vimm(double value, FloatRegister dest,
1373 Condition cc) {
1374 if (HasVFPv3()) {
1375 if (DoubleLowWord(value) == 0) {
1376 if (DoubleHighWord(value) == 0) {
1377 // To zero a register, load 1.0, then execute dN <- dN - dN
1378 as_vimm(dest, VFPImm::One, cc);
1379 as_vsub(dest, dest, dest, cc);
1380 return;
1381 }
1382
1383 VFPImm enc(DoubleHighWord(value));
1384 if (enc.isValid()) {
1385 as_vimm(dest, enc, cc);
1386 return;
1387 }
1388 }
1389 }
1390 // Fall back to putting the value in a pool.
1391 as_FImm64Pool(dest, value, cc);
1392 }
1393
ma_vimm_f32(float value,FloatRegister dest,Condition cc)1394 void MacroAssemblerARM::ma_vimm_f32(float value, FloatRegister dest,
1395 Condition cc) {
1396 VFPRegister vd = VFPRegister(dest).singleOverlay();
1397 if (HasVFPv3()) {
1398 if (IsPositiveZero(value)) {
1399 // To zero a register, load 1.0, then execute sN <- sN - sN.
1400 as_vimm(vd, VFPImm::One, cc);
1401 as_vsub(vd, vd, vd, cc);
1402 return;
1403 }
1404
1405 // Note that the vimm immediate float32 instruction encoding differs
1406 // from the vimm immediate double encoding, but this difference matches
1407 // the difference in the floating point formats, so it is possible to
1408 // convert the float32 to a double and then use the double encoding
1409 // paths. It is still necessary to firstly check that the double low
1410 // word is zero because some float32 numbers set these bits and this can
1411 // not be ignored.
1412 double doubleValue(value);
1413 if (DoubleLowWord(doubleValue) == 0) {
1414 VFPImm enc(DoubleHighWord(doubleValue));
1415 if (enc.isValid()) {
1416 as_vimm(vd, enc, cc);
1417 return;
1418 }
1419 }
1420 }
1421
1422 // Fall back to putting the value in a pool.
1423 as_FImm32Pool(vd, value, cc);
1424 }
1425
ma_vcmp(FloatRegister src1,FloatRegister src2,Condition cc)1426 void MacroAssemblerARM::ma_vcmp(FloatRegister src1, FloatRegister src2,
1427 Condition cc) {
1428 as_vcmp(VFPRegister(src1), VFPRegister(src2), cc);
1429 }
1430
ma_vcmp_f32(FloatRegister src1,FloatRegister src2,Condition cc)1431 void MacroAssemblerARM::ma_vcmp_f32(FloatRegister src1, FloatRegister src2,
1432 Condition cc) {
1433 as_vcmp(VFPRegister(src1).singleOverlay(), VFPRegister(src2).singleOverlay(),
1434 cc);
1435 }
1436
ma_vcmpz(FloatRegister src1,Condition cc)1437 void MacroAssemblerARM::ma_vcmpz(FloatRegister src1, Condition cc) {
1438 as_vcmpz(VFPRegister(src1), cc);
1439 }
1440
ma_vcmpz_f32(FloatRegister src1,Condition cc)1441 void MacroAssemblerARM::ma_vcmpz_f32(FloatRegister src1, Condition cc) {
1442 as_vcmpz(VFPRegister(src1).singleOverlay(), cc);
1443 }
1444
ma_vcvt_F64_I32(FloatRegister src,FloatRegister dest,Condition cc)1445 void MacroAssemblerARM::ma_vcvt_F64_I32(FloatRegister src, FloatRegister dest,
1446 Condition cc) {
1447 MOZ_ASSERT(src.isDouble());
1448 MOZ_ASSERT(dest.isSInt());
1449 as_vcvt(dest, src, false, cc);
1450 }
1451
ma_vcvt_F64_U32(FloatRegister src,FloatRegister dest,Condition cc)1452 void MacroAssemblerARM::ma_vcvt_F64_U32(FloatRegister src, FloatRegister dest,
1453 Condition cc) {
1454 MOZ_ASSERT(src.isDouble());
1455 MOZ_ASSERT(dest.isUInt());
1456 as_vcvt(dest, src, false, cc);
1457 }
1458
ma_vcvt_I32_F64(FloatRegister src,FloatRegister dest,Condition cc)1459 void MacroAssemblerARM::ma_vcvt_I32_F64(FloatRegister src, FloatRegister dest,
1460 Condition cc) {
1461 MOZ_ASSERT(src.isSInt());
1462 MOZ_ASSERT(dest.isDouble());
1463 as_vcvt(dest, src, false, cc);
1464 }
1465
ma_vcvt_U32_F64(FloatRegister src,FloatRegister dest,Condition cc)1466 void MacroAssemblerARM::ma_vcvt_U32_F64(FloatRegister src, FloatRegister dest,
1467 Condition cc) {
1468 MOZ_ASSERT(src.isUInt());
1469 MOZ_ASSERT(dest.isDouble());
1470 as_vcvt(dest, src, false, cc);
1471 }
1472
ma_vcvt_F32_I32(FloatRegister src,FloatRegister dest,Condition cc)1473 void MacroAssemblerARM::ma_vcvt_F32_I32(FloatRegister src, FloatRegister dest,
1474 Condition cc) {
1475 MOZ_ASSERT(src.isSingle());
1476 MOZ_ASSERT(dest.isSInt());
1477 as_vcvt(VFPRegister(dest).sintOverlay(), VFPRegister(src).singleOverlay(),
1478 false, cc);
1479 }
1480
ma_vcvt_F32_U32(FloatRegister src,FloatRegister dest,Condition cc)1481 void MacroAssemblerARM::ma_vcvt_F32_U32(FloatRegister src, FloatRegister dest,
1482 Condition cc) {
1483 MOZ_ASSERT(src.isSingle());
1484 MOZ_ASSERT(dest.isUInt());
1485 as_vcvt(VFPRegister(dest).uintOverlay(), VFPRegister(src).singleOverlay(),
1486 false, cc);
1487 }
1488
ma_vcvt_I32_F32(FloatRegister src,FloatRegister dest,Condition cc)1489 void MacroAssemblerARM::ma_vcvt_I32_F32(FloatRegister src, FloatRegister dest,
1490 Condition cc) {
1491 MOZ_ASSERT(src.isSInt());
1492 MOZ_ASSERT(dest.isSingle());
1493 as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).sintOverlay(),
1494 false, cc);
1495 }
1496
ma_vcvt_U32_F32(FloatRegister src,FloatRegister dest,Condition cc)1497 void MacroAssemblerARM::ma_vcvt_U32_F32(FloatRegister src, FloatRegister dest,
1498 Condition cc) {
1499 MOZ_ASSERT(src.isUInt());
1500 MOZ_ASSERT(dest.isSingle());
1501 as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).uintOverlay(),
1502 false, cc);
1503 }
1504
ma_vxfer(FloatRegister src,Register dest,Condition cc)1505 void MacroAssemblerARM::ma_vxfer(FloatRegister src, Register dest,
1506 Condition cc) {
1507 as_vxfer(dest, InvalidReg, VFPRegister(src).singleOverlay(), FloatToCore, cc);
1508 }
1509
ma_vxfer(FloatRegister src,Register dest1,Register dest2,Condition cc)1510 void MacroAssemblerARM::ma_vxfer(FloatRegister src, Register dest1,
1511 Register dest2, Condition cc) {
1512 as_vxfer(dest1, dest2, VFPRegister(src), FloatToCore, cc);
1513 }
1514
ma_vxfer(Register src,FloatRegister dest,Condition cc)1515 void MacroAssemblerARM::ma_vxfer(Register src, FloatRegister dest,
1516 Condition cc) {
1517 as_vxfer(src, InvalidReg, VFPRegister(dest).singleOverlay(), CoreToFloat, cc);
1518 }
1519
ma_vxfer(Register src1,Register src2,FloatRegister dest,Condition cc)1520 void MacroAssemblerARM::ma_vxfer(Register src1, Register src2,
1521 FloatRegister dest, Condition cc) {
1522 as_vxfer(src1, src2, VFPRegister(dest), CoreToFloat, cc);
1523 }
1524
ma_vdtr(LoadStore ls,const Address & addr,VFPRegister rt,AutoRegisterScope & scratch,Condition cc)1525 BufferOffset MacroAssemblerARM::ma_vdtr(LoadStore ls, const Address& addr,
1526 VFPRegister rt,
1527 AutoRegisterScope& scratch,
1528 Condition cc) {
1529 int off = addr.offset;
1530 MOZ_ASSERT((off & 3) == 0);
1531 Register base = addr.base;
1532 if (off > -1024 && off < 1024) {
1533 return as_vdtr(ls, rt, Operand(addr).toVFPAddr(), cc);
1534 }
1535
1536 // We cannot encode this offset in a a single ldr. Try to encode it as an
1537 // add scratch, base, imm; ldr dest, [scratch, +offset].
1538 int bottom = off & (0xff << 2);
1539 int neg_bottom = (0x100 << 2) - bottom;
1540 // At this point, both off - bottom and off + neg_bottom will be
1541 // reasonable-ish quantities.
1542 //
1543 // Note a neg_bottom of 0x400 can not be encoded as an immediate negative
1544 // offset in the instruction and this occurs when bottom is zero, so this
1545 // case is guarded against below.
1546 if (off < 0) {
1547 // sub_off = bottom - off
1548 Operand2 sub_off = Imm8(-(off - bottom));
1549 if (!sub_off.invalid()) {
1550 // - sub_off = off - bottom
1551 as_sub(scratch, base, sub_off, LeaveCC, cc);
1552 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(bottom)), cc);
1553 }
1554 // sub_off = -neg_bottom - off
1555 sub_off = Imm8(-(off + neg_bottom));
1556 if (!sub_off.invalid() && bottom != 0) {
1557 // Guarded against by: bottom != 0
1558 MOZ_ASSERT(neg_bottom < 0x400);
1559 // - sub_off = neg_bottom + off
1560 as_sub(scratch, base, sub_off, LeaveCC, cc);
1561 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(-neg_bottom)), cc);
1562 }
1563 } else {
1564 // sub_off = off - bottom
1565 Operand2 sub_off = Imm8(off - bottom);
1566 if (!sub_off.invalid()) {
1567 // sub_off = off - bottom
1568 as_add(scratch, base, sub_off, LeaveCC, cc);
1569 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(bottom)), cc);
1570 }
1571 // sub_off = neg_bottom + off
1572 sub_off = Imm8(off + neg_bottom);
1573 if (!sub_off.invalid() && bottom != 0) {
1574 // Guarded against by: bottom != 0
1575 MOZ_ASSERT(neg_bottom < 0x400);
1576 // sub_off = neg_bottom + off
1577 as_add(scratch, base, sub_off, LeaveCC, cc);
1578 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(-neg_bottom)), cc);
1579 }
1580 }
1581
1582 // Safe to use scratch as dest, since ma_add() overwrites dest at the end
1583 // and can't use it as internal scratch since it may also == base.
1584 ma_add(base, Imm32(off), scratch, scratch, LeaveCC, cc);
1585 return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(0)), cc);
1586 }
1587
ma_vldr(VFPAddr addr,VFPRegister dest,Condition cc)1588 BufferOffset MacroAssemblerARM::ma_vldr(VFPAddr addr, VFPRegister dest,
1589 Condition cc) {
1590 return as_vdtr(IsLoad, dest, addr, cc);
1591 }
1592
ma_vldr(const Address & addr,VFPRegister dest,AutoRegisterScope & scratch,Condition cc)1593 BufferOffset MacroAssemblerARM::ma_vldr(const Address& addr, VFPRegister dest,
1594 AutoRegisterScope& scratch,
1595 Condition cc) {
1596 return ma_vdtr(IsLoad, addr, dest, scratch, cc);
1597 }
1598
ma_vldr(VFPRegister src,Register base,Register index,AutoRegisterScope & scratch,int32_t shift,Condition cc)1599 BufferOffset MacroAssemblerARM::ma_vldr(VFPRegister src, Register base,
1600 Register index,
1601 AutoRegisterScope& scratch,
1602 int32_t shift, Condition cc) {
1603 as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
1604 return as_vdtr(IsLoad, src, Operand(Address(scratch, 0)).toVFPAddr(), cc);
1605 }
1606
ma_vstr(VFPRegister src,VFPAddr addr,Condition cc)1607 BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, VFPAddr addr,
1608 Condition cc) {
1609 return as_vdtr(IsStore, src, addr, cc);
1610 }
1611
ma_vstr(VFPRegister src,const Address & addr,AutoRegisterScope & scratch,Condition cc)1612 BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, const Address& addr,
1613 AutoRegisterScope& scratch,
1614 Condition cc) {
1615 return ma_vdtr(IsStore, addr, src, scratch, cc);
1616 }
1617
ma_vstr(VFPRegister src,Register base,Register index,AutoRegisterScope & scratch,AutoRegisterScope & scratch2,int32_t shift,int32_t offset,Condition cc)1618 BufferOffset MacroAssemblerARM::ma_vstr(
1619 VFPRegister src, Register base, Register index, AutoRegisterScope& scratch,
1620 AutoRegisterScope& scratch2, int32_t shift, int32_t offset, Condition cc) {
1621 as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
1622 return ma_vstr(src, Address(scratch, offset), scratch2, cc);
1623 }
1624
1625 // Without an offset, no second scratch register is necessary.
ma_vstr(VFPRegister src,Register base,Register index,AutoRegisterScope & scratch,int32_t shift,Condition cc)1626 BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, Register base,
1627 Register index,
1628 AutoRegisterScope& scratch,
1629 int32_t shift, Condition cc) {
1630 as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
1631 return as_vdtr(IsStore, src, Operand(Address(scratch, 0)).toVFPAddr(), cc);
1632 }
1633
buildOOLFakeExitFrame(void * fakeReturnAddr)1634 bool MacroAssemblerARMCompat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
1635 uint32_t descriptor = MakeFrameDescriptor(
1636 asMasm().framePushed(), FrameType::IonJS, ExitFrameLayout::Size());
1637
1638 asMasm().Push(Imm32(descriptor)); // descriptor_
1639 asMasm().Push(ImmPtr(fakeReturnAddr));
1640
1641 return true;
1642 }
1643
move32(Imm32 imm,Register dest)1644 void MacroAssemblerARMCompat::move32(Imm32 imm, Register dest) {
1645 ma_mov(imm, dest);
1646 }
1647
move32(Register src,Register dest)1648 void MacroAssemblerARMCompat::move32(Register src, Register dest) {
1649 ma_mov(src, dest);
1650 }
1651
movePtr(Register src,Register dest)1652 void MacroAssemblerARMCompat::movePtr(Register src, Register dest) {
1653 ma_mov(src, dest);
1654 }
1655
movePtr(ImmWord imm,Register dest)1656 void MacroAssemblerARMCompat::movePtr(ImmWord imm, Register dest) {
1657 ma_mov(Imm32(imm.value), dest);
1658 }
1659
movePtr(ImmGCPtr imm,Register dest)1660 void MacroAssemblerARMCompat::movePtr(ImmGCPtr imm, Register dest) {
1661 ma_mov(imm, dest);
1662 }
1663
movePtr(ImmPtr imm,Register dest)1664 void MacroAssemblerARMCompat::movePtr(ImmPtr imm, Register dest) {
1665 movePtr(ImmWord(uintptr_t(imm.value)), dest);
1666 }
1667
movePtr(wasm::SymbolicAddress imm,Register dest)1668 void MacroAssemblerARMCompat::movePtr(wasm::SymbolicAddress imm,
1669 Register dest) {
1670 append(wasm::SymbolicAccess(CodeOffset(currentOffset()), imm));
1671 ma_movPatchable(Imm32(-1), dest, Always);
1672 }
1673
load8ZeroExtend(const Address & address,Register dest)1674 void MacroAssemblerARMCompat::load8ZeroExtend(const Address& address,
1675 Register dest) {
1676 ScratchRegisterScope scratch(asMasm());
1677 ma_dataTransferN(IsLoad, 8, false, address.base, Imm32(address.offset), dest,
1678 scratch);
1679 }
1680
load8ZeroExtend(const BaseIndex & src,Register dest)1681 void MacroAssemblerARMCompat::load8ZeroExtend(const BaseIndex& src,
1682 Register dest) {
1683 Register base = src.base;
1684 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1685
1686 ScratchRegisterScope scratch(asMasm());
1687 SecondScratchRegisterScope scratch2(asMasm());
1688
1689 if (src.offset == 0) {
1690 ma_ldrb(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), dest);
1691 } else {
1692 ma_add(base, Imm32(src.offset), scratch, scratch2);
1693 ma_ldrb(DTRAddr(scratch, DtrRegImmShift(src.index, LSL, scale)), dest);
1694 }
1695 }
1696
load8SignExtend(const Address & address,Register dest)1697 void MacroAssemblerARMCompat::load8SignExtend(const Address& address,
1698 Register dest) {
1699 ScratchRegisterScope scratch(asMasm());
1700 ma_dataTransferN(IsLoad, 8, true, address.base, Imm32(address.offset), dest,
1701 scratch);
1702 }
1703
load8SignExtend(const BaseIndex & src,Register dest)1704 void MacroAssemblerARMCompat::load8SignExtend(const BaseIndex& src,
1705 Register dest) {
1706 Register index = src.index;
1707
1708 ScratchRegisterScope scratch(asMasm());
1709 SecondScratchRegisterScope scratch2(asMasm());
1710
1711 // ARMv7 does not have LSL on an index register with an extended load.
1712 if (src.scale != TimesOne) {
1713 ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
1714 index = scratch;
1715 }
1716
1717 if (src.offset != 0) {
1718 if (index != scratch) {
1719 ma_mov(index, scratch);
1720 index = scratch;
1721 }
1722 ma_add(Imm32(src.offset), index, scratch2);
1723 }
1724 ma_ldrsb(EDtrAddr(src.base, EDtrOffReg(index)), dest);
1725 }
1726
load16ZeroExtend(const Address & address,Register dest)1727 void MacroAssemblerARMCompat::load16ZeroExtend(const Address& address,
1728 Register dest) {
1729 ScratchRegisterScope scratch(asMasm());
1730 ma_dataTransferN(IsLoad, 16, false, address.base, Imm32(address.offset), dest,
1731 scratch);
1732 }
1733
load16ZeroExtend(const BaseIndex & src,Register dest)1734 void MacroAssemblerARMCompat::load16ZeroExtend(const BaseIndex& src,
1735 Register dest) {
1736 Register index = src.index;
1737
1738 ScratchRegisterScope scratch(asMasm());
1739 SecondScratchRegisterScope scratch2(asMasm());
1740
1741 // ARMv7 does not have LSL on an index register with an extended load.
1742 if (src.scale != TimesOne) {
1743 ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
1744 index = scratch;
1745 }
1746
1747 if (src.offset != 0) {
1748 if (index != scratch) {
1749 ma_mov(index, scratch);
1750 index = scratch;
1751 }
1752 ma_add(Imm32(src.offset), index, scratch2);
1753 }
1754 ma_ldrh(EDtrAddr(src.base, EDtrOffReg(index)), dest);
1755 }
1756
load16SignExtend(const Address & address,Register dest)1757 void MacroAssemblerARMCompat::load16SignExtend(const Address& address,
1758 Register dest) {
1759 ScratchRegisterScope scratch(asMasm());
1760 ma_dataTransferN(IsLoad, 16, true, address.base, Imm32(address.offset), dest,
1761 scratch);
1762 }
1763
load16SignExtend(const BaseIndex & src,Register dest)1764 void MacroAssemblerARMCompat::load16SignExtend(const BaseIndex& src,
1765 Register dest) {
1766 Register index = src.index;
1767
1768 ScratchRegisterScope scratch(asMasm());
1769 SecondScratchRegisterScope scratch2(asMasm());
1770
1771 // We don't have LSL on index register yet.
1772 if (src.scale != TimesOne) {
1773 ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
1774 index = scratch;
1775 }
1776
1777 if (src.offset != 0) {
1778 if (index != scratch) {
1779 ma_mov(index, scratch);
1780 index = scratch;
1781 }
1782 ma_add(Imm32(src.offset), index, scratch2);
1783 }
1784 ma_ldrsh(EDtrAddr(src.base, EDtrOffReg(index)), dest);
1785 }
1786
load32(const Address & address,Register dest)1787 void MacroAssemblerARMCompat::load32(const Address& address, Register dest) {
1788 loadPtr(address, dest);
1789 }
1790
load32(const BaseIndex & address,Register dest)1791 void MacroAssemblerARMCompat::load32(const BaseIndex& address, Register dest) {
1792 loadPtr(address, dest);
1793 }
1794
load32(AbsoluteAddress address,Register dest)1795 void MacroAssemblerARMCompat::load32(AbsoluteAddress address, Register dest) {
1796 loadPtr(address, dest);
1797 }
1798
loadPtr(const Address & address,Register dest)1799 void MacroAssemblerARMCompat::loadPtr(const Address& address, Register dest) {
1800 ScratchRegisterScope scratch(asMasm());
1801 ma_ldr(address, dest, scratch);
1802 }
1803
loadPtr(const BaseIndex & src,Register dest)1804 void MacroAssemblerARMCompat::loadPtr(const BaseIndex& src, Register dest) {
1805 Register base = src.base;
1806 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1807
1808 ScratchRegisterScope scratch(asMasm());
1809 SecondScratchRegisterScope scratch2(asMasm());
1810
1811 if (src.offset != 0) {
1812 ma_add(base, Imm32(src.offset), scratch, scratch2);
1813 ma_ldr(DTRAddr(scratch, DtrRegImmShift(src.index, LSL, scale)), dest);
1814 } else {
1815 ma_ldr(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), dest);
1816 }
1817 }
1818
loadPtr(AbsoluteAddress address,Register dest)1819 void MacroAssemblerARMCompat::loadPtr(AbsoluteAddress address, Register dest) {
1820 MOZ_ASSERT(dest != pc); // Use dest as a scratch register.
1821 movePtr(ImmWord(uintptr_t(address.addr)), dest);
1822 loadPtr(Address(dest, 0), dest);
1823 }
1824
loadPtr(wasm::SymbolicAddress address,Register dest)1825 void MacroAssemblerARMCompat::loadPtr(wasm::SymbolicAddress address,
1826 Register dest) {
1827 MOZ_ASSERT(dest != pc); // Use dest as a scratch register.
1828 movePtr(address, dest);
1829 loadPtr(Address(dest, 0), dest);
1830 }
1831
loadPrivate(const Address & address,Register dest)1832 void MacroAssemblerARMCompat::loadPrivate(const Address& address,
1833 Register dest) {
1834 ScratchRegisterScope scratch(asMasm());
1835 ma_ldr(ToPayload(address), dest, scratch);
1836 }
1837
loadDouble(const Address & address,FloatRegister dest)1838 void MacroAssemblerARMCompat::loadDouble(const Address& address,
1839 FloatRegister dest) {
1840 ScratchRegisterScope scratch(asMasm());
1841 ma_vldr(address, dest, scratch);
1842 }
1843
loadDouble(const BaseIndex & src,FloatRegister dest)1844 void MacroAssemblerARMCompat::loadDouble(const BaseIndex& src,
1845 FloatRegister dest) {
1846 // VFP instructions don't even support register Base + register Index modes,
1847 // so just add the index, then handle the offset like normal.
1848 Register base = src.base;
1849 Register index = src.index;
1850 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1851 int32_t offset = src.offset;
1852
1853 ScratchRegisterScope scratch(asMasm());
1854 SecondScratchRegisterScope scratch2(asMasm());
1855
1856 as_add(scratch, base, lsl(index, scale));
1857 ma_vldr(Address(scratch, offset), dest, scratch2);
1858 }
1859
loadFloatAsDouble(const Address & address,FloatRegister dest)1860 void MacroAssemblerARMCompat::loadFloatAsDouble(const Address& address,
1861 FloatRegister dest) {
1862 ScratchRegisterScope scratch(asMasm());
1863
1864 VFPRegister rt = dest;
1865 ma_vldr(address, rt.singleOverlay(), scratch);
1866 as_vcvt(rt, rt.singleOverlay());
1867 }
1868
loadFloatAsDouble(const BaseIndex & src,FloatRegister dest)1869 void MacroAssemblerARMCompat::loadFloatAsDouble(const BaseIndex& src,
1870 FloatRegister dest) {
1871 // VFP instructions don't even support register Base + register Index modes,
1872 // so just add the index, then handle the offset like normal.
1873 Register base = src.base;
1874 Register index = src.index;
1875 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1876 int32_t offset = src.offset;
1877 VFPRegister rt = dest;
1878
1879 ScratchRegisterScope scratch(asMasm());
1880 SecondScratchRegisterScope scratch2(asMasm());
1881
1882 as_add(scratch, base, lsl(index, scale));
1883 ma_vldr(Address(scratch, offset), rt.singleOverlay(), scratch2);
1884 as_vcvt(rt, rt.singleOverlay());
1885 }
1886
loadFloat32(const Address & address,FloatRegister dest)1887 void MacroAssemblerARMCompat::loadFloat32(const Address& address,
1888 FloatRegister dest) {
1889 ScratchRegisterScope scratch(asMasm());
1890 ma_vldr(address, VFPRegister(dest).singleOverlay(), scratch);
1891 }
1892
loadFloat32(const BaseIndex & src,FloatRegister dest)1893 void MacroAssemblerARMCompat::loadFloat32(const BaseIndex& src,
1894 FloatRegister dest) {
1895 // VFP instructions don't even support register Base + register Index modes,
1896 // so just add the index, then handle the offset like normal.
1897 Register base = src.base;
1898 Register index = src.index;
1899 uint32_t scale = Imm32::ShiftOf(src.scale).value;
1900 int32_t offset = src.offset;
1901
1902 ScratchRegisterScope scratch(asMasm());
1903 SecondScratchRegisterScope scratch2(asMasm());
1904
1905 as_add(scratch, base, lsl(index, scale));
1906 ma_vldr(Address(scratch, offset), VFPRegister(dest).singleOverlay(),
1907 scratch2);
1908 }
1909
store8(Imm32 imm,const Address & address)1910 void MacroAssemblerARMCompat::store8(Imm32 imm, const Address& address) {
1911 SecondScratchRegisterScope scratch2(asMasm());
1912 ma_mov(imm, scratch2);
1913 store8(scratch2, address);
1914 }
1915
store8(Register src,const Address & address)1916 void MacroAssemblerARMCompat::store8(Register src, const Address& address) {
1917 ScratchRegisterScope scratch(asMasm());
1918 ma_dataTransferN(IsStore, 8, false, address.base, Imm32(address.offset), src,
1919 scratch);
1920 }
1921
store8(Imm32 imm,const BaseIndex & dest)1922 void MacroAssemblerARMCompat::store8(Imm32 imm, const BaseIndex& dest) {
1923 Register base = dest.base;
1924 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
1925
1926 ScratchRegisterScope scratch(asMasm());
1927 SecondScratchRegisterScope scratch2(asMasm());
1928
1929 if (dest.offset != 0) {
1930 ma_add(base, Imm32(dest.offset), scratch, scratch2);
1931 ma_mov(imm, scratch2);
1932 ma_strb(scratch2, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
1933 } else {
1934 ma_mov(imm, scratch2);
1935 ma_strb(scratch2, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
1936 }
1937 }
1938
store8(Register src,const BaseIndex & dest)1939 void MacroAssemblerARMCompat::store8(Register src, const BaseIndex& dest) {
1940 Register base = dest.base;
1941 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
1942
1943 ScratchRegisterScope scratch(asMasm());
1944 SecondScratchRegisterScope scratch2(asMasm());
1945
1946 if (dest.offset != 0) {
1947 ma_add(base, Imm32(dest.offset), scratch, scratch2);
1948 ma_strb(src, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
1949 } else {
1950 ma_strb(src, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
1951 }
1952 }
1953
store16(Imm32 imm,const Address & address)1954 void MacroAssemblerARMCompat::store16(Imm32 imm, const Address& address) {
1955 SecondScratchRegisterScope scratch2(asMasm());
1956 ma_mov(imm, scratch2);
1957 store16(scratch2, address);
1958 }
1959
store16(Register src,const Address & address)1960 void MacroAssemblerARMCompat::store16(Register src, const Address& address) {
1961 ScratchRegisterScope scratch(asMasm());
1962 ma_dataTransferN(IsStore, 16, false, address.base, Imm32(address.offset), src,
1963 scratch);
1964 }
1965
store16(Imm32 imm,const BaseIndex & dest)1966 void MacroAssemblerARMCompat::store16(Imm32 imm, const BaseIndex& dest) {
1967 Register index = dest.index;
1968
1969 ScratchRegisterScope scratch(asMasm());
1970 SecondScratchRegisterScope scratch2(asMasm());
1971
1972 // We don't have LSL on index register yet.
1973 if (dest.scale != TimesOne) {
1974 ma_lsl(Imm32::ShiftOf(dest.scale), index, scratch);
1975 index = scratch;
1976 }
1977
1978 if (dest.offset != 0) {
1979 ma_add(index, Imm32(dest.offset), scratch, scratch2);
1980 index = scratch;
1981 }
1982
1983 ma_mov(imm, scratch2);
1984 ma_strh(scratch2, EDtrAddr(dest.base, EDtrOffReg(index)));
1985 }
1986
store16(Register src,const BaseIndex & address)1987 void MacroAssemblerARMCompat::store16(Register src, const BaseIndex& address) {
1988 Register index = address.index;
1989
1990 ScratchRegisterScope scratch(asMasm());
1991 SecondScratchRegisterScope scratch2(asMasm());
1992
1993 // We don't have LSL on index register yet.
1994 if (address.scale != TimesOne) {
1995 ma_lsl(Imm32::ShiftOf(address.scale), index, scratch);
1996 index = scratch;
1997 }
1998
1999 if (address.offset != 0) {
2000 ma_add(index, Imm32(address.offset), scratch, scratch2);
2001 index = scratch;
2002 }
2003 ma_strh(src, EDtrAddr(address.base, EDtrOffReg(index)));
2004 }
2005
store32(Register src,AbsoluteAddress address)2006 void MacroAssemblerARMCompat::store32(Register src, AbsoluteAddress address) {
2007 storePtr(src, address);
2008 }
2009
store32(Register src,const Address & address)2010 void MacroAssemblerARMCompat::store32(Register src, const Address& address) {
2011 storePtr(src, address);
2012 }
2013
store32(Imm32 src,const Address & address)2014 void MacroAssemblerARMCompat::store32(Imm32 src, const Address& address) {
2015 ScratchRegisterScope scratch(asMasm());
2016 SecondScratchRegisterScope scratch2(asMasm());
2017 move32(src, scratch);
2018 ma_str(scratch, address, scratch2);
2019 }
2020
store32(Imm32 imm,const BaseIndex & dest)2021 void MacroAssemblerARMCompat::store32(Imm32 imm, const BaseIndex& dest) {
2022 Register base = dest.base;
2023 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
2024
2025 ScratchRegisterScope scratch(asMasm());
2026 SecondScratchRegisterScope scratch2(asMasm());
2027
2028 if (dest.offset != 0) {
2029 ma_add(base, Imm32(dest.offset), scratch, scratch2);
2030 ma_mov(imm, scratch2);
2031 ma_str(scratch2, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
2032 } else {
2033 ma_mov(imm, scratch);
2034 ma_str(scratch, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
2035 }
2036 }
2037
store32(Register src,const BaseIndex & dest)2038 void MacroAssemblerARMCompat::store32(Register src, const BaseIndex& dest) {
2039 Register base = dest.base;
2040 uint32_t scale = Imm32::ShiftOf(dest.scale).value;
2041
2042 ScratchRegisterScope scratch(asMasm());
2043 SecondScratchRegisterScope scratch2(asMasm());
2044
2045 if (dest.offset != 0) {
2046 ma_add(base, Imm32(dest.offset), scratch, scratch2);
2047 ma_str(src, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
2048 } else {
2049 ma_str(src, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
2050 }
2051 }
2052
storePtr(ImmWord imm,const Address & address)2053 void MacroAssemblerARMCompat::storePtr(ImmWord imm, const Address& address) {
2054 store32(Imm32(imm.value), address);
2055 }
2056
storePtr(ImmWord imm,const BaseIndex & address)2057 void MacroAssemblerARMCompat::storePtr(ImmWord imm, const BaseIndex& address) {
2058 store32(Imm32(imm.value), address);
2059 }
2060
storePtr(ImmPtr imm,const Address & address)2061 void MacroAssemblerARMCompat::storePtr(ImmPtr imm, const Address& address) {
2062 store32(Imm32(uintptr_t(imm.value)), address);
2063 }
2064
storePtr(ImmPtr imm,const BaseIndex & address)2065 void MacroAssemblerARMCompat::storePtr(ImmPtr imm, const BaseIndex& address) {
2066 store32(Imm32(uintptr_t(imm.value)), address);
2067 }
2068
storePtr(ImmGCPtr imm,const Address & address)2069 void MacroAssemblerARMCompat::storePtr(ImmGCPtr imm, const Address& address) {
2070 ScratchRegisterScope scratch(asMasm());
2071 SecondScratchRegisterScope scratch2(asMasm());
2072 ma_mov(imm, scratch);
2073 ma_str(scratch, address, scratch2);
2074 }
2075
storePtr(ImmGCPtr imm,const BaseIndex & address)2076 void MacroAssemblerARMCompat::storePtr(ImmGCPtr imm, const BaseIndex& address) {
2077 Register base = address.base;
2078 uint32_t scale = Imm32::ShiftOf(address.scale).value;
2079
2080 ScratchRegisterScope scratch(asMasm());
2081 SecondScratchRegisterScope scratch2(asMasm());
2082
2083 if (address.offset != 0) {
2084 ma_add(base, Imm32(address.offset), scratch, scratch2);
2085 ma_mov(imm, scratch2);
2086 ma_str(scratch2,
2087 DTRAddr(scratch, DtrRegImmShift(address.index, LSL, scale)));
2088 } else {
2089 ma_mov(imm, scratch);
2090 ma_str(scratch, DTRAddr(base, DtrRegImmShift(address.index, LSL, scale)));
2091 }
2092 }
2093
storePtr(Register src,const Address & address)2094 void MacroAssemblerARMCompat::storePtr(Register src, const Address& address) {
2095 SecondScratchRegisterScope scratch2(asMasm());
2096 ma_str(src, address, scratch2);
2097 }
2098
storePtr(Register src,const BaseIndex & address)2099 void MacroAssemblerARMCompat::storePtr(Register src, const BaseIndex& address) {
2100 store32(src, address);
2101 }
2102
storePtr(Register src,AbsoluteAddress dest)2103 void MacroAssemblerARMCompat::storePtr(Register src, AbsoluteAddress dest) {
2104 ScratchRegisterScope scratch(asMasm());
2105 movePtr(ImmWord(uintptr_t(dest.addr)), scratch);
2106 ma_str(src, DTRAddr(scratch, DtrOffImm(0)));
2107 }
2108
2109 // Note: this function clobbers the input register.
clampDoubleToUint8(FloatRegister input,Register output)2110 void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
2111 if (HasVFPv3()) {
2112 Label notSplit;
2113 {
2114 ScratchDoubleScope scratchDouble(*this);
2115 MOZ_ASSERT(input != scratchDouble);
2116 loadConstantDouble(0.5, scratchDouble);
2117
2118 ma_vadd(input, scratchDouble, scratchDouble);
2119 // Convert the double into an unsigned fixed point value with 24 bits of
2120 // precision. The resulting number will look like 0xII.DDDDDD
2121 as_vcvtFixed(scratchDouble, false, 24, true);
2122 }
2123
2124 // Move the fixed point value into an integer register.
2125 {
2126 ScratchFloat32Scope scratchFloat(*this);
2127 as_vxfer(output, InvalidReg, scratchFloat.uintOverlay(), FloatToCore);
2128 }
2129
2130 ScratchRegisterScope scratch(*this);
2131
2132 // See if this value *might* have been an exact integer after adding
2133 // 0.5. This tests the 1/2 through 1/16,777,216th places, but 0.5 needs
2134 // to be tested out to the 1/140,737,488,355,328th place.
2135 ma_tst(output, Imm32(0x00ffffff), scratch);
2136 // Convert to a uint8 by shifting out all of the fraction bits.
2137 ma_lsr(Imm32(24), output, output);
2138 // If any of the bottom 24 bits were non-zero, then we're good, since
2139 // this number can't be exactly XX.0
2140 ma_b(¬Split, NonZero);
2141 as_vxfer(scratch, InvalidReg, input, FloatToCore);
2142 as_cmp(scratch, Imm8(0));
2143 // If the lower 32 bits of the double were 0, then this was an exact number,
2144 // and it should be even.
2145 as_bic(output, output, Imm8(1), LeaveCC, Zero);
2146 bind(¬Split);
2147 } else {
2148 ScratchDoubleScope scratchDouble(*this);
2149 MOZ_ASSERT(input != scratchDouble);
2150 loadConstantDouble(0.5, scratchDouble);
2151
2152 Label outOfRange;
2153 ma_vcmpz(input);
2154 // Do the add, in place so we can reference it later.
2155 ma_vadd(input, scratchDouble, input);
2156 // Do the conversion to an integer.
2157 as_vcvt(VFPRegister(scratchDouble).uintOverlay(), VFPRegister(input));
2158 // Copy the converted value out.
2159 as_vxfer(output, InvalidReg, scratchDouble, FloatToCore);
2160 as_vmrs(pc);
2161 ma_mov(Imm32(0), output, Overflow); // NaN => 0
2162 ma_b(&outOfRange, Overflow); // NaN
2163 as_cmp(output, Imm8(0xff));
2164 ma_mov(Imm32(0xff), output, Above);
2165 ma_b(&outOfRange, Above);
2166 // Convert it back to see if we got the same value back.
2167 as_vcvt(scratchDouble, VFPRegister(scratchDouble).uintOverlay());
2168 // Do the check.
2169 as_vcmp(scratchDouble, input);
2170 as_vmrs(pc);
2171 as_bic(output, output, Imm8(1), LeaveCC, Zero);
2172 bind(&outOfRange);
2173 }
2174 }
2175
cmp32(Register lhs,Imm32 rhs)2176 void MacroAssemblerARMCompat::cmp32(Register lhs, Imm32 rhs) {
2177 ScratchRegisterScope scratch(asMasm());
2178 ma_cmp(lhs, rhs, scratch);
2179 }
2180
cmp32(Register lhs,Register rhs)2181 void MacroAssemblerARMCompat::cmp32(Register lhs, Register rhs) {
2182 ma_cmp(lhs, rhs);
2183 }
2184
cmp32(const Address & lhs,Imm32 rhs)2185 void MacroAssemblerARMCompat::cmp32(const Address& lhs, Imm32 rhs) {
2186 ScratchRegisterScope scratch(asMasm());
2187 SecondScratchRegisterScope scratch2(asMasm());
2188 ma_ldr(lhs, scratch, scratch2);
2189 ma_cmp(scratch, rhs, scratch2);
2190 }
2191
cmp32(const Address & lhs,Register rhs)2192 void MacroAssemblerARMCompat::cmp32(const Address& lhs, Register rhs) {
2193 ScratchRegisterScope scratch(asMasm());
2194 SecondScratchRegisterScope scratch2(asMasm());
2195 ma_ldr(lhs, scratch, scratch2);
2196 ma_cmp(scratch, rhs);
2197 }
2198
cmpPtr(Register lhs,ImmWord rhs)2199 void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmWord rhs) {
2200 cmp32(lhs, Imm32(rhs.value));
2201 }
2202
cmpPtr(Register lhs,ImmPtr rhs)2203 void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmPtr rhs) {
2204 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
2205 }
2206
cmpPtr(Register lhs,Register rhs)2207 void MacroAssemblerARMCompat::cmpPtr(Register lhs, Register rhs) {
2208 ma_cmp(lhs, rhs);
2209 }
2210
cmpPtr(Register lhs,ImmGCPtr rhs)2211 void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmGCPtr rhs) {
2212 ScratchRegisterScope scratch(asMasm());
2213 ma_cmp(lhs, rhs, scratch);
2214 }
2215
cmpPtr(Register lhs,Imm32 rhs)2216 void MacroAssemblerARMCompat::cmpPtr(Register lhs, Imm32 rhs) {
2217 cmp32(lhs, rhs);
2218 }
2219
cmpPtr(const Address & lhs,Register rhs)2220 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, Register rhs) {
2221 ScratchRegisterScope scratch(asMasm());
2222 SecondScratchRegisterScope scratch2(asMasm());
2223 ma_ldr(lhs, scratch, scratch2);
2224 ma_cmp(scratch, rhs);
2225 }
2226
cmpPtr(const Address & lhs,ImmWord rhs)2227 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmWord rhs) {
2228 ScratchRegisterScope scratch(asMasm());
2229 SecondScratchRegisterScope scratch2(asMasm());
2230 ma_ldr(lhs, scratch, scratch2);
2231 ma_cmp(scratch, Imm32(rhs.value), scratch2);
2232 }
2233
cmpPtr(const Address & lhs,ImmPtr rhs)2234 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmPtr rhs) {
2235 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
2236 }
2237
cmpPtr(const Address & lhs,ImmGCPtr rhs)2238 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmGCPtr rhs) {
2239 ScratchRegisterScope scratch(asMasm());
2240 SecondScratchRegisterScope scratch2(asMasm());
2241 ma_ldr(lhs, scratch, scratch2);
2242 ma_cmp(scratch, rhs, scratch2);
2243 }
2244
cmpPtr(const Address & lhs,Imm32 rhs)2245 void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, Imm32 rhs) {
2246 ScratchRegisterScope scratch(asMasm());
2247 SecondScratchRegisterScope scratch2(asMasm());
2248 ma_ldr(lhs, scratch, scratch2);
2249 ma_cmp(scratch, rhs, scratch2);
2250 }
2251
setStackArg(Register reg,uint32_t arg)2252 void MacroAssemblerARMCompat::setStackArg(Register reg, uint32_t arg) {
2253 ScratchRegisterScope scratch(asMasm());
2254 ma_dataTransferN(IsStore, 32, true, sp, Imm32(arg * sizeof(intptr_t)), reg,
2255 scratch);
2256 }
2257
minMaxDouble(FloatRegister srcDest,FloatRegister second,bool canBeNaN,bool isMax)2258 void MacroAssemblerARMCompat::minMaxDouble(FloatRegister srcDest,
2259 FloatRegister second, bool canBeNaN,
2260 bool isMax) {
2261 FloatRegister first = srcDest;
2262
2263 Label nan, equal, returnSecond, done;
2264
2265 Assembler::Condition cond = isMax ? Assembler::VFP_LessThanOrEqual
2266 : Assembler::VFP_GreaterThanOrEqual;
2267
2268 compareDouble(first, second);
2269 // First or second is NaN, result is NaN.
2270 ma_b(&nan, Assembler::VFP_Unordered);
2271 // Make sure we handle -0 and 0 right.
2272 ma_b(&equal, Assembler::VFP_Equal);
2273 ma_b(&returnSecond, cond);
2274 ma_b(&done);
2275
2276 // Check for zero.
2277 bind(&equal);
2278 compareDouble(first, NoVFPRegister);
2279 // First wasn't 0 or -0, so just return it.
2280 ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
2281 // So now both operands are either -0 or 0.
2282 if (isMax) {
2283 // -0 + -0 = -0 and -0 + 0 = 0.
2284 ma_vadd(second, first, first);
2285 } else {
2286 ma_vneg(first, first);
2287 ma_vsub(first, second, first);
2288 ma_vneg(first, first);
2289 }
2290 ma_b(&done);
2291
2292 bind(&nan);
2293 // If the first argument is the NaN, return it; otherwise return the second
2294 // operand.
2295 compareDouble(first, first);
2296 ma_vmov(first, srcDest, Assembler::VFP_Unordered);
2297 ma_b(&done, Assembler::VFP_Unordered);
2298
2299 bind(&returnSecond);
2300 ma_vmov(second, srcDest);
2301
2302 bind(&done);
2303 }
2304
minMaxFloat32(FloatRegister srcDest,FloatRegister second,bool canBeNaN,bool isMax)2305 void MacroAssemblerARMCompat::minMaxFloat32(FloatRegister srcDest,
2306 FloatRegister second, bool canBeNaN,
2307 bool isMax) {
2308 FloatRegister first = srcDest;
2309
2310 Label nan, equal, returnSecond, done;
2311
2312 Assembler::Condition cond = isMax ? Assembler::VFP_LessThanOrEqual
2313 : Assembler::VFP_GreaterThanOrEqual;
2314
2315 compareFloat(first, second);
2316 // First or second is NaN, result is NaN.
2317 ma_b(&nan, Assembler::VFP_Unordered);
2318 // Make sure we handle -0 and 0 right.
2319 ma_b(&equal, Assembler::VFP_Equal);
2320 ma_b(&returnSecond, cond);
2321 ma_b(&done);
2322
2323 // Check for zero.
2324 bind(&equal);
2325 compareFloat(first, NoVFPRegister);
2326 // First wasn't 0 or -0, so just return it.
2327 ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
2328 // So now both operands are either -0 or 0.
2329 if (isMax) {
2330 // -0 + -0 = -0 and -0 + 0 = 0.
2331 ma_vadd_f32(second, first, first);
2332 } else {
2333 ma_vneg_f32(first, first);
2334 ma_vsub_f32(first, second, first);
2335 ma_vneg_f32(first, first);
2336 }
2337 ma_b(&done);
2338
2339 bind(&nan);
2340 // See comment in minMaxDouble.
2341 compareFloat(first, first);
2342 ma_vmov_f32(first, srcDest, Assembler::VFP_Unordered);
2343 ma_b(&done, Assembler::VFP_Unordered);
2344
2345 bind(&returnSecond);
2346 ma_vmov_f32(second, srcDest);
2347
2348 bind(&done);
2349 }
2350
compareDouble(FloatRegister lhs,FloatRegister rhs)2351 void MacroAssemblerARMCompat::compareDouble(FloatRegister lhs,
2352 FloatRegister rhs) {
2353 // Compare the doubles, setting vector status flags.
2354 if (rhs.isMissing()) {
2355 ma_vcmpz(lhs);
2356 } else {
2357 ma_vcmp(lhs, rhs);
2358 }
2359
2360 // Move vector status bits to normal status flags.
2361 as_vmrs(pc);
2362 }
2363
compareFloat(FloatRegister lhs,FloatRegister rhs)2364 void MacroAssemblerARMCompat::compareFloat(FloatRegister lhs,
2365 FloatRegister rhs) {
2366 // Compare the doubles, setting vector status flags.
2367 if (rhs.isMissing()) {
2368 as_vcmpz(VFPRegister(lhs).singleOverlay());
2369 } else {
2370 as_vcmp(VFPRegister(lhs).singleOverlay(), VFPRegister(rhs).singleOverlay());
2371 }
2372
2373 // Move vector status bits to normal status flags.
2374 as_vmrs(pc);
2375 }
2376
testInt32(Assembler::Condition cond,const ValueOperand & value)2377 Assembler::Condition MacroAssemblerARMCompat::testInt32(
2378 Assembler::Condition cond, const ValueOperand& value) {
2379 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2380 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_INT32));
2381 return cond;
2382 }
2383
testBoolean(Assembler::Condition cond,const ValueOperand & value)2384 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2385 Assembler::Condition cond, const ValueOperand& value) {
2386 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2387 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_BOOLEAN));
2388 return cond;
2389 }
2390
testDouble(Assembler::Condition cond,const ValueOperand & value)2391 Assembler::Condition MacroAssemblerARMCompat::testDouble(
2392 Assembler::Condition cond, const ValueOperand& value) {
2393 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2394 Assembler::Condition actual = (cond == Equal) ? Below : AboveOrEqual;
2395 ScratchRegisterScope scratch(asMasm());
2396 ma_cmp(value.typeReg(), ImmTag(JSVAL_TAG_CLEAR), scratch);
2397 return actual;
2398 }
2399
testNull(Assembler::Condition cond,const ValueOperand & value)2400 Assembler::Condition MacroAssemblerARMCompat::testNull(
2401 Assembler::Condition cond, const ValueOperand& value) {
2402 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2403 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_NULL));
2404 return cond;
2405 }
2406
testUndefined(Assembler::Condition cond,const ValueOperand & value)2407 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2408 Assembler::Condition cond, const ValueOperand& value) {
2409 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2410 ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_UNDEFINED));
2411 return cond;
2412 }
2413
testString(Assembler::Condition cond,const ValueOperand & value)2414 Assembler::Condition MacroAssemblerARMCompat::testString(
2415 Assembler::Condition cond, const ValueOperand& value) {
2416 return testString(cond, value.typeReg());
2417 }
2418
testSymbol(Assembler::Condition cond,const ValueOperand & value)2419 Assembler::Condition MacroAssemblerARMCompat::testSymbol(
2420 Assembler::Condition cond, const ValueOperand& value) {
2421 return testSymbol(cond, value.typeReg());
2422 }
2423
testBigInt(Assembler::Condition cond,const ValueOperand & value)2424 Assembler::Condition MacroAssemblerARMCompat::testBigInt(
2425 Assembler::Condition cond, const ValueOperand& value) {
2426 return testBigInt(cond, value.typeReg());
2427 }
2428
testObject(Assembler::Condition cond,const ValueOperand & value)2429 Assembler::Condition MacroAssemblerARMCompat::testObject(
2430 Assembler::Condition cond, const ValueOperand& value) {
2431 return testObject(cond, value.typeReg());
2432 }
2433
testNumber(Assembler::Condition cond,const ValueOperand & value)2434 Assembler::Condition MacroAssemblerARMCompat::testNumber(
2435 Assembler::Condition cond, const ValueOperand& value) {
2436 return testNumber(cond, value.typeReg());
2437 }
2438
testMagic(Assembler::Condition cond,const ValueOperand & value)2439 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2440 Assembler::Condition cond, const ValueOperand& value) {
2441 return testMagic(cond, value.typeReg());
2442 }
2443
testPrimitive(Assembler::Condition cond,const ValueOperand & value)2444 Assembler::Condition MacroAssemblerARMCompat::testPrimitive(
2445 Assembler::Condition cond, const ValueOperand& value) {
2446 return testPrimitive(cond, value.typeReg());
2447 }
2448
testGCThing(Assembler::Condition cond,const ValueOperand & value)2449 Assembler::Condition MacroAssemblerARMCompat::testGCThing(
2450 Assembler::Condition cond, const ValueOperand& value) {
2451 return testGCThing(cond, value.typeReg());
2452 }
2453
2454 // Register-based tests.
testInt32(Assembler::Condition cond,Register tag)2455 Assembler::Condition MacroAssemblerARMCompat::testInt32(
2456 Assembler::Condition cond, Register tag) {
2457 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2458 ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
2459 return cond;
2460 }
2461
testBoolean(Assembler::Condition cond,Register tag)2462 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2463 Assembler::Condition cond, Register tag) {
2464 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2465 ma_cmp(tag, ImmTag(JSVAL_TAG_BOOLEAN));
2466 return cond;
2467 }
2468
testNull(Assembler::Condition cond,Register tag)2469 Assembler::Condition MacroAssemblerARMCompat::testNull(
2470 Assembler::Condition cond, Register tag) {
2471 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2472 ma_cmp(tag, ImmTag(JSVAL_TAG_NULL));
2473 return cond;
2474 }
2475
testUndefined(Assembler::Condition cond,Register tag)2476 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2477 Assembler::Condition cond, Register tag) {
2478 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2479 ma_cmp(tag, ImmTag(JSVAL_TAG_UNDEFINED));
2480 return cond;
2481 }
2482
testString(Assembler::Condition cond,Register tag)2483 Assembler::Condition MacroAssemblerARMCompat::testString(
2484 Assembler::Condition cond, Register tag) {
2485 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2486 ma_cmp(tag, ImmTag(JSVAL_TAG_STRING));
2487 return cond;
2488 }
2489
testSymbol(Assembler::Condition cond,Register tag)2490 Assembler::Condition MacroAssemblerARMCompat::testSymbol(
2491 Assembler::Condition cond, Register tag) {
2492 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2493 ma_cmp(tag, ImmTag(JSVAL_TAG_SYMBOL));
2494 return cond;
2495 }
2496
testBigInt(Assembler::Condition cond,Register tag)2497 Assembler::Condition MacroAssemblerARMCompat::testBigInt(
2498 Assembler::Condition cond, Register tag) {
2499 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2500 ma_cmp(tag, ImmTag(JSVAL_TAG_BIGINT));
2501 return cond;
2502 }
2503
testObject(Assembler::Condition cond,Register tag)2504 Assembler::Condition MacroAssemblerARMCompat::testObject(
2505 Assembler::Condition cond, Register tag) {
2506 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2507 ma_cmp(tag, ImmTag(JSVAL_TAG_OBJECT));
2508 return cond;
2509 }
2510
testMagic(Assembler::Condition cond,Register tag)2511 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2512 Assembler::Condition cond, Register tag) {
2513 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2514 ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
2515 return cond;
2516 }
2517
testPrimitive(Assembler::Condition cond,Register tag)2518 Assembler::Condition MacroAssemblerARMCompat::testPrimitive(
2519 Assembler::Condition cond, Register tag) {
2520 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2521 ma_cmp(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
2522 return cond == Equal ? Below : AboveOrEqual;
2523 }
2524
testGCThing(Assembler::Condition cond,Register tag)2525 Assembler::Condition MacroAssemblerARMCompat::testGCThing(
2526 Assembler::Condition cond, Register tag) {
2527 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2528 ma_cmp(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
2529 return cond == Equal ? AboveOrEqual : Below;
2530 }
2531
testGCThing(Assembler::Condition cond,const Address & address)2532 Assembler::Condition MacroAssemblerARMCompat::testGCThing(
2533 Assembler::Condition cond, const Address& address) {
2534 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2535 ScratchRegisterScope scratch(asMasm());
2536 Register tag = extractTag(address, scratch);
2537 ma_cmp(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
2538 return cond == Equal ? AboveOrEqual : Below;
2539 }
2540
testMagic(Assembler::Condition cond,const Address & address)2541 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2542 Assembler::Condition cond, const Address& address) {
2543 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2544 ScratchRegisterScope scratch(asMasm());
2545 Register tag = extractTag(address, scratch);
2546 ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
2547 return cond;
2548 }
2549
testInt32(Assembler::Condition cond,const Address & address)2550 Assembler::Condition MacroAssemblerARMCompat::testInt32(
2551 Assembler::Condition cond, const Address& address) {
2552 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2553 ScratchRegisterScope scratch(asMasm());
2554 Register tag = extractTag(address, scratch);
2555 ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
2556 return cond;
2557 }
2558
testDouble(Condition cond,const Address & address)2559 Assembler::Condition MacroAssemblerARMCompat::testDouble(
2560 Condition cond, const Address& address) {
2561 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2562 ScratchRegisterScope scratch(asMasm());
2563 Register tag = extractTag(address, scratch);
2564 return testDouble(cond, tag);
2565 }
2566
testBoolean(Condition cond,const Address & address)2567 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2568 Condition cond, const Address& address) {
2569 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2570 ScratchRegisterScope scratch(asMasm());
2571 Register tag = extractTag(address, scratch);
2572 return testBoolean(cond, tag);
2573 }
2574
testNull(Condition cond,const Address & address)2575 Assembler::Condition MacroAssemblerARMCompat::testNull(Condition cond,
2576 const Address& address) {
2577 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2578 ScratchRegisterScope scratch(asMasm());
2579 Register tag = extractTag(address, scratch);
2580 return testNull(cond, tag);
2581 }
2582
testUndefined(Condition cond,const Address & address)2583 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2584 Condition cond, const Address& address) {
2585 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2586 ScratchRegisterScope scratch(asMasm());
2587 Register tag = extractTag(address, scratch);
2588 return testUndefined(cond, tag);
2589 }
2590
testString(Condition cond,const Address & address)2591 Assembler::Condition MacroAssemblerARMCompat::testString(
2592 Condition cond, const Address& address) {
2593 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2594 ScratchRegisterScope scratch(asMasm());
2595 Register tag = extractTag(address, scratch);
2596 return testString(cond, tag);
2597 }
2598
testSymbol(Condition cond,const Address & address)2599 Assembler::Condition MacroAssemblerARMCompat::testSymbol(
2600 Condition cond, const Address& address) {
2601 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2602 ScratchRegisterScope scratch(asMasm());
2603 Register tag = extractTag(address, scratch);
2604 return testSymbol(cond, tag);
2605 }
2606
testBigInt(Condition cond,const Address & address)2607 Assembler::Condition MacroAssemblerARMCompat::testBigInt(
2608 Condition cond, const Address& address) {
2609 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2610 ScratchRegisterScope scratch(asMasm());
2611 Register tag = extractTag(address, scratch);
2612 return testBigInt(cond, tag);
2613 }
2614
testObject(Condition cond,const Address & address)2615 Assembler::Condition MacroAssemblerARMCompat::testObject(
2616 Condition cond, const Address& address) {
2617 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2618 ScratchRegisterScope scratch(asMasm());
2619 Register tag = extractTag(address, scratch);
2620 return testObject(cond, tag);
2621 }
2622
testNumber(Condition cond,const Address & address)2623 Assembler::Condition MacroAssemblerARMCompat::testNumber(
2624 Condition cond, const Address& address) {
2625 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2626 ScratchRegisterScope scratch(asMasm());
2627 Register tag = extractTag(address, scratch);
2628 return testNumber(cond, tag);
2629 }
2630
testDouble(Condition cond,Register tag)2631 Assembler::Condition MacroAssemblerARMCompat::testDouble(Condition cond,
2632 Register tag) {
2633 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
2634 Condition actual = (cond == Equal) ? Below : AboveOrEqual;
2635 ma_cmp(tag, ImmTag(JSVAL_TAG_CLEAR));
2636 return actual;
2637 }
2638
testNumber(Condition cond,Register tag)2639 Assembler::Condition MacroAssemblerARMCompat::testNumber(Condition cond,
2640 Register tag) {
2641 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2642 ma_cmp(tag, ImmTag(JS::detail::ValueUpperInclNumberTag));
2643 return cond == Equal ? BelowOrEqual : Above;
2644 }
2645
testUndefined(Condition cond,const BaseIndex & src)2646 Assembler::Condition MacroAssemblerARMCompat::testUndefined(
2647 Condition cond, const BaseIndex& src) {
2648 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2649 ScratchRegisterScope scratch(asMasm());
2650 Register tag = extractTag(src, scratch);
2651 ma_cmp(tag, ImmTag(JSVAL_TAG_UNDEFINED));
2652 return cond;
2653 }
2654
testNull(Condition cond,const BaseIndex & src)2655 Assembler::Condition MacroAssemblerARMCompat::testNull(Condition cond,
2656 const BaseIndex& src) {
2657 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2658 ScratchRegisterScope scratch(asMasm());
2659 Register tag = extractTag(src, scratch);
2660 ma_cmp(tag, ImmTag(JSVAL_TAG_NULL));
2661 return cond;
2662 }
2663
testBoolean(Condition cond,const BaseIndex & src)2664 Assembler::Condition MacroAssemblerARMCompat::testBoolean(
2665 Condition cond, const BaseIndex& src) {
2666 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2667 ScratchRegisterScope scratch(asMasm());
2668 Register tag = extractTag(src, scratch);
2669 ma_cmp(tag, ImmTag(JSVAL_TAG_BOOLEAN));
2670 return cond;
2671 }
2672
testString(Condition cond,const BaseIndex & src)2673 Assembler::Condition MacroAssemblerARMCompat::testString(Condition cond,
2674 const BaseIndex& src) {
2675 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2676 ScratchRegisterScope scratch(asMasm());
2677 Register tag = extractTag(src, scratch);
2678 ma_cmp(tag, ImmTag(JSVAL_TAG_STRING));
2679 return cond;
2680 }
2681
testSymbol(Condition cond,const BaseIndex & src)2682 Assembler::Condition MacroAssemblerARMCompat::testSymbol(Condition cond,
2683 const BaseIndex& src) {
2684 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2685 ScratchRegisterScope scratch(asMasm());
2686 Register tag = extractTag(src, scratch);
2687 ma_cmp(tag, ImmTag(JSVAL_TAG_SYMBOL));
2688 return cond;
2689 }
2690
testBigInt(Condition cond,const BaseIndex & src)2691 Assembler::Condition MacroAssemblerARMCompat::testBigInt(Condition cond,
2692 const BaseIndex& src) {
2693 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2694 ScratchRegisterScope scratch(asMasm());
2695 Register tag = extractTag(src, scratch);
2696 ma_cmp(tag, ImmTag(JSVAL_TAG_BIGINT));
2697 return cond;
2698 }
2699
testInt32(Condition cond,const BaseIndex & src)2700 Assembler::Condition MacroAssemblerARMCompat::testInt32(Condition cond,
2701 const BaseIndex& src) {
2702 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2703 ScratchRegisterScope scratch(asMasm());
2704 Register tag = extractTag(src, scratch);
2705 ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
2706 return cond;
2707 }
2708
testObject(Condition cond,const BaseIndex & src)2709 Assembler::Condition MacroAssemblerARMCompat::testObject(Condition cond,
2710 const BaseIndex& src) {
2711 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2712 ScratchRegisterScope scratch(asMasm());
2713 Register tag = extractTag(src, scratch);
2714 ma_cmp(tag, ImmTag(JSVAL_TAG_OBJECT));
2715 return cond;
2716 }
2717
testDouble(Condition cond,const BaseIndex & src)2718 Assembler::Condition MacroAssemblerARMCompat::testDouble(Condition cond,
2719 const BaseIndex& src) {
2720 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2721 Assembler::Condition actual = (cond == Equal) ? Below : AboveOrEqual;
2722 ScratchRegisterScope scratch(asMasm());
2723 Register tag = extractTag(src, scratch);
2724 ma_cmp(tag, ImmTag(JSVAL_TAG_CLEAR));
2725 return actual;
2726 }
2727
testMagic(Condition cond,const BaseIndex & address)2728 Assembler::Condition MacroAssemblerARMCompat::testMagic(
2729 Condition cond, const BaseIndex& address) {
2730 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2731 ScratchRegisterScope scratch(asMasm());
2732 Register tag = extractTag(address, scratch);
2733 ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
2734 return cond;
2735 }
2736
testGCThing(Condition cond,const BaseIndex & address)2737 Assembler::Condition MacroAssemblerARMCompat::testGCThing(
2738 Condition cond, const BaseIndex& address) {
2739 MOZ_ASSERT(cond == Equal || cond == NotEqual);
2740 ScratchRegisterScope scratch(asMasm());
2741 Register tag = extractTag(address, scratch);
2742 ma_cmp(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
2743 return cond == Equal ? AboveOrEqual : Below;
2744 }
2745
2746 // Unboxing code.
unboxNonDouble(const ValueOperand & operand,Register dest,JSValueType type)2747 void MacroAssemblerARMCompat::unboxNonDouble(const ValueOperand& operand,
2748 Register dest, JSValueType type) {
2749 auto movPayloadToDest = [&]() {
2750 if (operand.payloadReg() != dest) {
2751 ma_mov(operand.payloadReg(), dest, LeaveCC);
2752 }
2753 };
2754 if (!JitOptions.spectreValueMasking) {
2755 movPayloadToDest();
2756 return;
2757 }
2758
2759 // Spectre mitigation: We zero the payload if the tag does not match the
2760 // expected type and if this is a pointer type.
2761 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
2762 movPayloadToDest();
2763 return;
2764 }
2765
2766 // We zero the destination register and move the payload into it if
2767 // the tag corresponds to the given type.
2768 ma_cmp(operand.typeReg(), ImmType(type));
2769 movPayloadToDest();
2770 ma_mov(Imm32(0), dest, NotEqual);
2771 }
2772
unboxNonDouble(const Address & src,Register dest,JSValueType type)2773 void MacroAssemblerARMCompat::unboxNonDouble(const Address& src, Register dest,
2774 JSValueType type) {
2775 ScratchRegisterScope scratch(asMasm());
2776 if (!JitOptions.spectreValueMasking) {
2777 ma_ldr(ToPayload(src), dest, scratch);
2778 return;
2779 }
2780
2781 // Spectre mitigation: We zero the payload if the tag does not match the
2782 // expected type and if this is a pointer type.
2783 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
2784 ma_ldr(ToPayload(src), dest, scratch);
2785 return;
2786 }
2787
2788 // We zero the destination register and move the payload into it if
2789 // the tag corresponds to the given type.
2790 ma_ldr(ToType(src), scratch, scratch);
2791 ma_cmp(scratch, ImmType(type));
2792 ma_ldr(ToPayload(src), dest, scratch, Offset, Equal);
2793 ma_mov(Imm32(0), dest, NotEqual);
2794 }
2795
unboxNonDouble(const BaseIndex & src,Register dest,JSValueType type)2796 void MacroAssemblerARMCompat::unboxNonDouble(const BaseIndex& src,
2797 Register dest, JSValueType type) {
2798 SecondScratchRegisterScope scratch2(asMasm());
2799 ma_alu(src.base, lsl(src.index, src.scale), scratch2, OpAdd);
2800 Address value(scratch2, src.offset);
2801 unboxNonDouble(value, dest, type);
2802 }
2803
unboxDouble(const ValueOperand & operand,FloatRegister dest)2804 void MacroAssemblerARMCompat::unboxDouble(const ValueOperand& operand,
2805 FloatRegister dest) {
2806 MOZ_ASSERT(dest.isDouble());
2807 as_vxfer(operand.payloadReg(), operand.typeReg(), VFPRegister(dest),
2808 CoreToFloat);
2809 }
2810
unboxDouble(const Address & src,FloatRegister dest)2811 void MacroAssemblerARMCompat::unboxDouble(const Address& src,
2812 FloatRegister dest) {
2813 MOZ_ASSERT(dest.isDouble());
2814 loadDouble(src, dest);
2815 }
2816
unboxDouble(const BaseIndex & src,FloatRegister dest)2817 void MacroAssemblerARMCompat::unboxDouble(const BaseIndex& src,
2818 FloatRegister dest) {
2819 MOZ_ASSERT(dest.isDouble());
2820 loadDouble(src, dest);
2821 }
2822
unboxValue(const ValueOperand & src,AnyRegister dest,JSValueType type)2823 void MacroAssemblerARMCompat::unboxValue(const ValueOperand& src,
2824 AnyRegister dest, JSValueType type) {
2825 if (dest.isFloat()) {
2826 Label notInt32, end;
2827 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
2828 convertInt32ToDouble(src.payloadReg(), dest.fpu());
2829 ma_b(&end);
2830 bind(¬Int32);
2831 unboxDouble(src, dest.fpu());
2832 bind(&end);
2833 } else {
2834 unboxNonDouble(src, dest.gpr(), type);
2835 }
2836 }
2837
boxDouble(FloatRegister src,const ValueOperand & dest,FloatRegister)2838 void MacroAssemblerARMCompat::boxDouble(FloatRegister src,
2839 const ValueOperand& dest,
2840 FloatRegister) {
2841 as_vxfer(dest.payloadReg(), dest.typeReg(), VFPRegister(src), FloatToCore);
2842 }
2843
boxNonDouble(JSValueType type,Register src,const ValueOperand & dest)2844 void MacroAssemblerARMCompat::boxNonDouble(JSValueType type, Register src,
2845 const ValueOperand& dest) {
2846 if (src != dest.payloadReg()) {
2847 ma_mov(src, dest.payloadReg());
2848 }
2849 ma_mov(ImmType(type), dest.typeReg());
2850 }
2851
boolValueToDouble(const ValueOperand & operand,FloatRegister dest)2852 void MacroAssemblerARMCompat::boolValueToDouble(const ValueOperand& operand,
2853 FloatRegister dest) {
2854 VFPRegister d = VFPRegister(dest);
2855 loadConstantDouble(1.0, dest);
2856 as_cmp(operand.payloadReg(), Imm8(0));
2857 // If the source is 0, then subtract the dest from itself, producing 0.
2858 as_vsub(d, d, d, Equal);
2859 }
2860
int32ValueToDouble(const ValueOperand & operand,FloatRegister dest)2861 void MacroAssemblerARMCompat::int32ValueToDouble(const ValueOperand& operand,
2862 FloatRegister dest) {
2863 // Transfer the integral value to a floating point register.
2864 VFPRegister vfpdest = VFPRegister(dest);
2865 as_vxfer(operand.payloadReg(), InvalidReg, vfpdest.sintOverlay(),
2866 CoreToFloat);
2867 // Convert the value to a double.
2868 as_vcvt(vfpdest, vfpdest.sintOverlay());
2869 }
2870
boolValueToFloat32(const ValueOperand & operand,FloatRegister dest)2871 void MacroAssemblerARMCompat::boolValueToFloat32(const ValueOperand& operand,
2872 FloatRegister dest) {
2873 VFPRegister d = VFPRegister(dest).singleOverlay();
2874 loadConstantFloat32(1.0, dest);
2875 as_cmp(operand.payloadReg(), Imm8(0));
2876 // If the source is 0, then subtract the dest from itself, producing 0.
2877 as_vsub(d, d, d, Equal);
2878 }
2879
int32ValueToFloat32(const ValueOperand & operand,FloatRegister dest)2880 void MacroAssemblerARMCompat::int32ValueToFloat32(const ValueOperand& operand,
2881 FloatRegister dest) {
2882 // Transfer the integral value to a floating point register.
2883 VFPRegister vfpdest = VFPRegister(dest).singleOverlay();
2884 as_vxfer(operand.payloadReg(), InvalidReg, vfpdest.sintOverlay(),
2885 CoreToFloat);
2886 // Convert the value to a float.
2887 as_vcvt(vfpdest, vfpdest.sintOverlay());
2888 }
2889
loadConstantFloat32(float f,FloatRegister dest)2890 void MacroAssemblerARMCompat::loadConstantFloat32(float f, FloatRegister dest) {
2891 ma_vimm_f32(f, dest);
2892 }
2893
loadInt32OrDouble(const Address & src,FloatRegister dest)2894 void MacroAssemblerARMCompat::loadInt32OrDouble(const Address& src,
2895 FloatRegister dest) {
2896 Label notInt32, end;
2897
2898 // If it's an int, convert to a double.
2899 {
2900 ScratchRegisterScope scratch(asMasm());
2901 SecondScratchRegisterScope scratch2(asMasm());
2902
2903 ma_ldr(ToType(src), scratch, scratch2);
2904 asMasm().branchTestInt32(Assembler::NotEqual, scratch, ¬Int32);
2905 ma_ldr(ToPayload(src), scratch, scratch2);
2906 convertInt32ToDouble(scratch, dest);
2907 ma_b(&end);
2908 }
2909
2910 // Not an int, just load as double.
2911 bind(¬Int32);
2912 {
2913 ScratchRegisterScope scratch(asMasm());
2914 ma_vldr(src, dest, scratch);
2915 }
2916 bind(&end);
2917 }
2918
loadInt32OrDouble(Register base,Register index,FloatRegister dest,int32_t shift)2919 void MacroAssemblerARMCompat::loadInt32OrDouble(Register base, Register index,
2920 FloatRegister dest,
2921 int32_t shift) {
2922 Label notInt32, end;
2923
2924 static_assert(NUNBOX32_PAYLOAD_OFFSET == 0);
2925
2926 ScratchRegisterScope scratch(asMasm());
2927
2928 // If it's an int, convert it to double.
2929 ma_alu(base, lsl(index, shift), scratch, OpAdd);
2930
2931 // Since we only have one scratch register, we need to stomp over it with
2932 // the tag.
2933 ma_ldr(DTRAddr(scratch, DtrOffImm(NUNBOX32_TYPE_OFFSET)), scratch);
2934 asMasm().branchTestInt32(Assembler::NotEqual, scratch, ¬Int32);
2935
2936 // Implicitly requires NUNBOX32_PAYLOAD_OFFSET == 0: no offset provided
2937 ma_ldr(DTRAddr(base, DtrRegImmShift(index, LSL, shift)), scratch);
2938 convertInt32ToDouble(scratch, dest);
2939 ma_b(&end);
2940
2941 // Not an int, just load as double.
2942 bind(¬Int32);
2943 // First, recompute the offset that had been stored in the scratch register
2944 // since the scratch register was overwritten loading in the type.
2945 ma_alu(base, lsl(index, shift), scratch, OpAdd);
2946 ma_vldr(VFPAddr(scratch, VFPOffImm(0)), dest);
2947 bind(&end);
2948 }
2949
loadConstantDouble(double dp,FloatRegister dest)2950 void MacroAssemblerARMCompat::loadConstantDouble(double dp,
2951 FloatRegister dest) {
2952 ma_vimm(dp, dest);
2953 }
2954
2955 // Treat the value as a boolean, and set condition codes accordingly.
testInt32Truthy(bool truthy,const ValueOperand & operand)2956 Assembler::Condition MacroAssemblerARMCompat::testInt32Truthy(
2957 bool truthy, const ValueOperand& operand) {
2958 ma_tst(operand.payloadReg(), operand.payloadReg());
2959 return truthy ? NonZero : Zero;
2960 }
2961
testBooleanTruthy(bool truthy,const ValueOperand & operand)2962 Assembler::Condition MacroAssemblerARMCompat::testBooleanTruthy(
2963 bool truthy, const ValueOperand& operand) {
2964 ma_tst(operand.payloadReg(), operand.payloadReg());
2965 return truthy ? NonZero : Zero;
2966 }
2967
testDoubleTruthy(bool truthy,FloatRegister reg)2968 Assembler::Condition MacroAssemblerARMCompat::testDoubleTruthy(
2969 bool truthy, FloatRegister reg) {
2970 as_vcmpz(VFPRegister(reg));
2971 as_vmrs(pc);
2972 as_cmp(r0, O2Reg(r0), Overflow);
2973 return truthy ? NonZero : Zero;
2974 }
2975
extractObject(const Address & address,Register scratch)2976 Register MacroAssemblerARMCompat::extractObject(const Address& address,
2977 Register scratch) {
2978 SecondScratchRegisterScope scratch2(asMasm());
2979 ma_ldr(ToPayload(address), scratch, scratch2);
2980 return scratch;
2981 }
2982
extractTag(const Address & address,Register scratch)2983 Register MacroAssemblerARMCompat::extractTag(const Address& address,
2984 Register scratch) {
2985 SecondScratchRegisterScope scratch2(asMasm());
2986 ma_ldr(ToType(address), scratch, scratch2);
2987 return scratch;
2988 }
2989
extractTag(const BaseIndex & address,Register scratch)2990 Register MacroAssemblerARMCompat::extractTag(const BaseIndex& address,
2991 Register scratch) {
2992 ma_alu(address.base, lsl(address.index, address.scale), scratch, OpAdd,
2993 LeaveCC);
2994 return extractTag(Address(scratch, address.offset), scratch);
2995 }
2996
2997 /////////////////////////////////////////////////////////////////
2998 // X86/X64-common (ARM too now) interface.
2999 /////////////////////////////////////////////////////////////////
storeValue(ValueOperand val,const Address & dst)3000 void MacroAssemblerARMCompat::storeValue(ValueOperand val, const Address& dst) {
3001 SecondScratchRegisterScope scratch2(asMasm());
3002 ma_str(val.payloadReg(), ToPayload(dst), scratch2);
3003 ma_str(val.typeReg(), ToType(dst), scratch2);
3004 }
3005
storeValue(ValueOperand val,const BaseIndex & dest)3006 void MacroAssemblerARMCompat::storeValue(ValueOperand val,
3007 const BaseIndex& dest) {
3008 ScratchRegisterScope scratch(asMasm());
3009
3010 if (isValueDTRDCandidate(val) && Abs(dest.offset) <= 255) {
3011 Register tmpIdx;
3012 if (dest.offset == 0) {
3013 if (dest.scale == TimesOne) {
3014 tmpIdx = dest.index;
3015 } else {
3016 ma_lsl(Imm32(dest.scale), dest.index, scratch);
3017 tmpIdx = scratch;
3018 }
3019 ma_strd(val.payloadReg(), val.typeReg(),
3020 EDtrAddr(dest.base, EDtrOffReg(tmpIdx)));
3021 } else {
3022 ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
3023 ma_strd(val.payloadReg(), val.typeReg(),
3024 EDtrAddr(scratch, EDtrOffImm(dest.offset)));
3025 }
3026 } else {
3027 ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
3028 storeValue(val, Address(scratch, dest.offset));
3029 }
3030 }
3031
loadValue(const BaseIndex & addr,ValueOperand val)3032 void MacroAssemblerARMCompat::loadValue(const BaseIndex& addr,
3033 ValueOperand val) {
3034 ScratchRegisterScope scratch(asMasm());
3035
3036 if (isValueDTRDCandidate(val) && Abs(addr.offset) <= 255) {
3037 Register tmpIdx;
3038 if (addr.offset == 0) {
3039 if (addr.scale == TimesOne) {
3040 // If the offset register is the same as one of the destination
3041 // registers, LDRD's behavior is undefined. Use the scratch
3042 // register to avoid this.
3043 if (val.aliases(addr.index)) {
3044 ma_mov(addr.index, scratch);
3045 tmpIdx = scratch;
3046 } else {
3047 tmpIdx = addr.index;
3048 }
3049 } else {
3050 ma_lsl(Imm32(addr.scale), addr.index, scratch);
3051 tmpIdx = scratch;
3052 }
3053 ma_ldrd(EDtrAddr(addr.base, EDtrOffReg(tmpIdx)), val.payloadReg(),
3054 val.typeReg());
3055 } else {
3056 ma_alu(addr.base, lsl(addr.index, addr.scale), scratch, OpAdd);
3057 ma_ldrd(EDtrAddr(scratch, EDtrOffImm(addr.offset)), val.payloadReg(),
3058 val.typeReg());
3059 }
3060 } else {
3061 ma_alu(addr.base, lsl(addr.index, addr.scale), scratch, OpAdd);
3062 loadValue(Address(scratch, addr.offset), val);
3063 }
3064 }
3065
loadValue(Address src,ValueOperand val)3066 void MacroAssemblerARMCompat::loadValue(Address src, ValueOperand val) {
3067 // TODO: copy this code into a generic function that acts on all sequences
3068 // of memory accesses
3069 if (isValueDTRDCandidate(val)) {
3070 // If the value we want is in two consecutive registers starting with an
3071 // even register, they can be combined as a single ldrd.
3072 int offset = src.offset;
3073 if (offset < 256 && offset > -256) {
3074 ma_ldrd(EDtrAddr(src.base, EDtrOffImm(src.offset)), val.payloadReg(),
3075 val.typeReg());
3076 return;
3077 }
3078 }
3079 // If the value is lower than the type, then we may be able to use an ldm
3080 // instruction.
3081
3082 if (val.payloadReg().code() < val.typeReg().code()) {
3083 if (src.offset <= 4 && src.offset >= -8 && (src.offset & 3) == 0) {
3084 // Turns out each of the 4 value -8, -4, 0, 4 corresponds exactly
3085 // with one of LDM{DB, DA, IA, IB}
3086 DTMMode mode;
3087 switch (src.offset) {
3088 case -8:
3089 mode = DB;
3090 break;
3091 case -4:
3092 mode = DA;
3093 break;
3094 case 0:
3095 mode = IA;
3096 break;
3097 case 4:
3098 mode = IB;
3099 break;
3100 default:
3101 MOZ_CRASH("Bogus Offset for LoadValue as DTM");
3102 }
3103 startDataTransferM(IsLoad, src.base, mode);
3104 transferReg(val.payloadReg());
3105 transferReg(val.typeReg());
3106 finishDataTransfer();
3107 return;
3108 }
3109 }
3110
3111 loadUnalignedValue(src, val);
3112 }
3113
loadUnalignedValue(const Address & src,ValueOperand dest)3114 void MacroAssemblerARMCompat::loadUnalignedValue(const Address& src,
3115 ValueOperand dest) {
3116 Address payload = ToPayload(src);
3117 Address type = ToType(src);
3118
3119 // Ensure that loading the payload does not erase the pointer to the Value
3120 // in memory.
3121 if (type.base != dest.payloadReg()) {
3122 SecondScratchRegisterScope scratch2(asMasm());
3123 ma_ldr(payload, dest.payloadReg(), scratch2);
3124 ma_ldr(type, dest.typeReg(), scratch2);
3125 } else {
3126 SecondScratchRegisterScope scratch2(asMasm());
3127 ma_ldr(type, dest.typeReg(), scratch2);
3128 ma_ldr(payload, dest.payloadReg(), scratch2);
3129 }
3130 }
3131
tagValue(JSValueType type,Register payload,ValueOperand dest)3132 void MacroAssemblerARMCompat::tagValue(JSValueType type, Register payload,
3133 ValueOperand dest) {
3134 MOZ_ASSERT(dest.typeReg() != dest.payloadReg());
3135 if (payload != dest.payloadReg()) {
3136 ma_mov(payload, dest.payloadReg());
3137 }
3138 ma_mov(ImmType(type), dest.typeReg());
3139 }
3140
pushValue(ValueOperand val)3141 void MacroAssemblerARMCompat::pushValue(ValueOperand val) {
3142 ma_push(val.typeReg());
3143 ma_push(val.payloadReg());
3144 }
3145
pushValue(const Address & addr)3146 void MacroAssemblerARMCompat::pushValue(const Address& addr) {
3147 ScratchRegisterScope scratch(asMasm());
3148 SecondScratchRegisterScope scratch2(asMasm());
3149
3150 ma_ldr(ToType(addr), scratch, scratch2);
3151 ma_push(scratch);
3152 ma_ldr(ToPayloadAfterStackPush(addr), scratch, scratch2);
3153 ma_push(scratch);
3154 }
3155
popValue(ValueOperand val)3156 void MacroAssemblerARMCompat::popValue(ValueOperand val) {
3157 ma_pop(val.payloadReg());
3158 ma_pop(val.typeReg());
3159 }
3160
storePayload(const Value & val,const Address & dest)3161 void MacroAssemblerARMCompat::storePayload(const Value& val,
3162 const Address& dest) {
3163 ScratchRegisterScope scratch(asMasm());
3164 SecondScratchRegisterScope scratch2(asMasm());
3165
3166 if (val.isGCThing()) {
3167 ma_mov(ImmGCPtr(val.toGCThing()), scratch);
3168 } else {
3169 ma_mov(Imm32(val.toNunboxPayload()), scratch);
3170 }
3171 ma_str(scratch, ToPayload(dest), scratch2);
3172 }
3173
storePayload(Register src,const Address & dest)3174 void MacroAssemblerARMCompat::storePayload(Register src, const Address& dest) {
3175 ScratchRegisterScope scratch(asMasm());
3176 ma_str(src, ToPayload(dest), scratch);
3177 }
3178
storePayload(const Value & val,const BaseIndex & dest)3179 void MacroAssemblerARMCompat::storePayload(const Value& val,
3180 const BaseIndex& dest) {
3181 unsigned shift = ScaleToShift(dest.scale);
3182
3183 ScratchRegisterScope scratch(asMasm());
3184 SecondScratchRegisterScope scratch2(asMasm());
3185
3186 if (val.isGCThing()) {
3187 ma_mov(ImmGCPtr(val.toGCThing()), scratch);
3188 } else {
3189 ma_mov(Imm32(val.toNunboxPayload()), scratch);
3190 }
3191
3192 // If NUNBOX32_PAYLOAD_OFFSET is not zero, the memory operand [base + index
3193 // << shift + imm] cannot be encoded into a single instruction, and cannot
3194 // be integrated into the as_dtr call.
3195 static_assert(NUNBOX32_PAYLOAD_OFFSET == 0);
3196
3197 // If an offset is used, modify the base so that a [base + index << shift]
3198 // instruction format can be used.
3199 if (dest.offset != 0) {
3200 ma_add(dest.base, Imm32(dest.offset), dest.base, scratch2);
3201 }
3202
3203 as_dtr(IsStore, 32, Offset, scratch,
3204 DTRAddr(dest.base, DtrRegImmShift(dest.index, LSL, shift)));
3205
3206 // Restore the original value of the base, if necessary.
3207 if (dest.offset != 0) {
3208 ma_sub(dest.base, Imm32(dest.offset), dest.base, scratch);
3209 }
3210 }
3211
storePayload(Register src,const BaseIndex & dest)3212 void MacroAssemblerARMCompat::storePayload(Register src,
3213 const BaseIndex& dest) {
3214 unsigned shift = ScaleToShift(dest.scale);
3215 MOZ_ASSERT(shift < 32);
3216
3217 ScratchRegisterScope scratch(asMasm());
3218
3219 // If NUNBOX32_PAYLOAD_OFFSET is not zero, the memory operand [base + index
3220 // << shift + imm] cannot be encoded into a single instruction, and cannot
3221 // be integrated into the as_dtr call.
3222 static_assert(NUNBOX32_PAYLOAD_OFFSET == 0);
3223
3224 // Save/restore the base if the BaseIndex has an offset, as above.
3225 if (dest.offset != 0) {
3226 ma_add(dest.base, Imm32(dest.offset), dest.base, scratch);
3227 }
3228
3229 // Technically, shift > -32 can be handle by changing LSL to ASR, but should
3230 // never come up, and this is one less code path to get wrong.
3231 as_dtr(IsStore, 32, Offset, src,
3232 DTRAddr(dest.base, DtrRegImmShift(dest.index, LSL, shift)));
3233
3234 if (dest.offset != 0) {
3235 ma_sub(dest.base, Imm32(dest.offset), dest.base, scratch);
3236 }
3237 }
3238
storeTypeTag(ImmTag tag,const Address & dest)3239 void MacroAssemblerARMCompat::storeTypeTag(ImmTag tag, const Address& dest) {
3240 ScratchRegisterScope scratch(asMasm());
3241 SecondScratchRegisterScope scratch2(asMasm());
3242
3243 ma_mov(tag, scratch);
3244 ma_str(scratch, ToType(dest), scratch2);
3245 }
3246
storeTypeTag(ImmTag tag,const BaseIndex & dest)3247 void MacroAssemblerARMCompat::storeTypeTag(ImmTag tag, const BaseIndex& dest) {
3248 Register base = dest.base;
3249 Register index = dest.index;
3250 unsigned shift = ScaleToShift(dest.scale);
3251
3252 ScratchRegisterScope scratch(asMasm());
3253 SecondScratchRegisterScope scratch2(asMasm());
3254
3255 MOZ_ASSERT(base != scratch && base != scratch2);
3256 MOZ_ASSERT(index != scratch && index != scratch2);
3257
3258 ma_add(base, Imm32(dest.offset + NUNBOX32_TYPE_OFFSET), scratch2, scratch);
3259 ma_mov(tag, scratch);
3260 ma_str(scratch, DTRAddr(scratch2, DtrRegImmShift(index, LSL, shift)));
3261 }
3262
ma_call(ImmPtr dest)3263 void MacroAssemblerARM::ma_call(ImmPtr dest) {
3264 ma_movPatchable(dest, CallReg, Always);
3265 as_blx(CallReg);
3266 }
3267
breakpoint()3268 void MacroAssemblerARMCompat::breakpoint() { as_bkpt(); }
3269
simulatorStop(const char * msg)3270 void MacroAssemblerARMCompat::simulatorStop(const char* msg) {
3271 #ifdef JS_SIMULATOR_ARM
3272 MOZ_ASSERT(sizeof(char*) == 4);
3273 writeInst(0xefffffff);
3274 writeInst((int)msg);
3275 #endif
3276 }
3277
ensureDouble(const ValueOperand & source,FloatRegister dest,Label * failure)3278 void MacroAssemblerARMCompat::ensureDouble(const ValueOperand& source,
3279 FloatRegister dest, Label* failure) {
3280 Label isDouble, done;
3281 asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
3282 asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
3283
3284 convertInt32ToDouble(source.payloadReg(), dest);
3285 jump(&done);
3286
3287 bind(&isDouble);
3288 unboxDouble(source, dest);
3289
3290 bind(&done);
3291 }
3292
breakpoint(Condition cc)3293 void MacroAssemblerARMCompat::breakpoint(Condition cc) {
3294 ma_ldr(DTRAddr(r12, DtrRegImmShift(r12, LSL, 0, IsDown)), r12, Offset, cc);
3295 }
3296
checkStackAlignment()3297 void MacroAssemblerARMCompat::checkStackAlignment() {
3298 asMasm().assertStackAlignment(ABIStackAlignment);
3299 }
3300
handleFailureWithHandlerTail(void * handler,Label * profilerExitTail)3301 void MacroAssemblerARMCompat::handleFailureWithHandlerTail(
3302 void* handler, Label* profilerExitTail) {
3303 // Reserve space for exception information.
3304 int size = (sizeof(ResumeFromException) + 7) & ~7;
3305
3306 Imm8 size8(size);
3307 as_sub(sp, sp, size8);
3308 ma_mov(sp, r0);
3309
3310 // Call the handler.
3311 asMasm().setupUnalignedABICall(r1);
3312 asMasm().passABIArg(r0);
3313 asMasm().callWithABI(handler, MoveOp::GENERAL,
3314 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
3315
3316 Label entryFrame;
3317 Label catch_;
3318 Label finally;
3319 Label return_;
3320 Label bailout;
3321 Label wasm;
3322
3323 {
3324 ScratchRegisterScope scratch(asMasm());
3325 ma_ldr(Address(sp, offsetof(ResumeFromException, kind)), r0, scratch);
3326 }
3327
3328 asMasm().branch32(Assembler::Equal, r0,
3329 Imm32(ResumeFromException::RESUME_ENTRY_FRAME),
3330 &entryFrame);
3331 asMasm().branch32(Assembler::Equal, r0,
3332 Imm32(ResumeFromException::RESUME_CATCH), &catch_);
3333 asMasm().branch32(Assembler::Equal, r0,
3334 Imm32(ResumeFromException::RESUME_FINALLY), &finally);
3335 asMasm().branch32(Assembler::Equal, r0,
3336 Imm32(ResumeFromException::RESUME_FORCED_RETURN), &return_);
3337 asMasm().branch32(Assembler::Equal, r0,
3338 Imm32(ResumeFromException::RESUME_BAILOUT), &bailout);
3339 asMasm().branch32(Assembler::Equal, r0,
3340 Imm32(ResumeFromException::RESUME_WASM), &wasm);
3341
3342 breakpoint(); // Invalid kind.
3343
3344 // No exception handler. Load the error value, load the new stack pointer
3345 // and return from the entry frame.
3346 bind(&entryFrame);
3347 asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
3348 {
3349 ScratchRegisterScope scratch(asMasm());
3350 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3351 scratch);
3352 }
3353
3354 // We're going to be returning by the ion calling convention, which returns
3355 // by ??? (for now, I think ldr pc, [sp]!)
3356 as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
3357
3358 // If we found a catch handler, this must be a baseline frame. Restore state
3359 // and jump to the catch block.
3360 bind(&catch_);
3361 {
3362 ScratchRegisterScope scratch(asMasm());
3363 ma_ldr(Address(sp, offsetof(ResumeFromException, target)), r0, scratch);
3364 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3365 scratch);
3366 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3367 scratch);
3368 }
3369 jump(r0);
3370
3371 // If we found a finally block, this must be a baseline frame. Push two
3372 // values expected by JSOp::Retsub: BooleanValue(true) and the exception.
3373 bind(&finally);
3374 ValueOperand exception = ValueOperand(r1, r2);
3375 loadValue(Operand(sp, offsetof(ResumeFromException, exception)), exception);
3376 {
3377 ScratchRegisterScope scratch(asMasm());
3378 ma_ldr(Address(sp, offsetof(ResumeFromException, target)), r0, scratch);
3379 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3380 scratch);
3381 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3382 scratch);
3383 }
3384
3385 pushValue(BooleanValue(true));
3386 pushValue(exception);
3387 jump(r0);
3388
3389 // Only used in debug mode. Return BaselineFrame->returnValue() to the
3390 // caller.
3391 bind(&return_);
3392 {
3393 ScratchRegisterScope scratch(asMasm());
3394 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3395 scratch);
3396 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3397 scratch);
3398 }
3399 loadValue(Address(r11, BaselineFrame::reverseOffsetOfReturnValue()),
3400 JSReturnOperand);
3401 ma_mov(r11, sp);
3402 pop(r11);
3403
3404 // If profiling is enabled, then update the lastProfilingFrame to refer to
3405 // caller frame before returning.
3406 {
3407 Label skipProfilingInstrumentation;
3408 // Test if profiler enabled.
3409 AbsoluteAddress addressOfEnabled(
3410 GetJitContext()->runtime->geckoProfiler().addressOfEnabled());
3411 asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
3412 &skipProfilingInstrumentation);
3413 jump(profilerExitTail);
3414 bind(&skipProfilingInstrumentation);
3415 }
3416
3417 ret();
3418
3419 // If we are bailing out to baseline to handle an exception, jump to the
3420 // bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
3421 bind(&bailout);
3422 {
3423 ScratchRegisterScope scratch(asMasm());
3424 ma_ldr(Address(sp, offsetof(ResumeFromException, bailoutInfo)), r2,
3425 scratch);
3426 ma_mov(Imm32(1), ReturnReg);
3427 ma_ldr(Address(sp, offsetof(ResumeFromException, target)), r1, scratch);
3428 }
3429 jump(r1);
3430
3431 // If we are throwing and the innermost frame was a wasm frame, reset SP and
3432 // FP; SP is pointing to the unwound return address to the wasm entry, so
3433 // we can just ret().
3434 bind(&wasm);
3435 {
3436 ScratchRegisterScope scratch(asMasm());
3437 ma_ldr(Address(sp, offsetof(ResumeFromException, framePointer)), r11,
3438 scratch);
3439 ma_ldr(Address(sp, offsetof(ResumeFromException, stackPointer)), sp,
3440 scratch);
3441 }
3442 as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
3443 }
3444
testStringTruthy(bool truthy,const ValueOperand & value)3445 Assembler::Condition MacroAssemblerARMCompat::testStringTruthy(
3446 bool truthy, const ValueOperand& value) {
3447 Register string = value.payloadReg();
3448 ScratchRegisterScope scratch(asMasm());
3449 SecondScratchRegisterScope scratch2(asMasm());
3450
3451 ma_dtr(IsLoad, string, Imm32(JSString::offsetOfLength()), scratch, scratch2);
3452 as_cmp(scratch, Imm8(0));
3453 return truthy ? Assembler::NotEqual : Assembler::Equal;
3454 }
3455
testBigIntTruthy(bool truthy,const ValueOperand & value)3456 Assembler::Condition MacroAssemblerARMCompat::testBigIntTruthy(
3457 bool truthy, const ValueOperand& value) {
3458 Register bi = value.payloadReg();
3459 ScratchRegisterScope scratch(asMasm());
3460 SecondScratchRegisterScope scratch2(asMasm());
3461
3462 ma_dtr(IsLoad, bi, Imm32(BigInt::offsetOfDigitLength()), scratch, scratch2);
3463 as_cmp(scratch, Imm8(0));
3464 return truthy ? Assembler::NotEqual : Assembler::Equal;
3465 }
3466
floor(FloatRegister input,Register output,Label * bail)3467 void MacroAssemblerARMCompat::floor(FloatRegister input, Register output,
3468 Label* bail) {
3469 Label handleZero;
3470 Label handleNeg;
3471 Label fin;
3472
3473 ScratchDoubleScope scratchDouble(asMasm());
3474
3475 compareDouble(input, NoVFPRegister);
3476 ma_b(&handleZero, Assembler::Equal);
3477 ma_b(&handleNeg, Assembler::Signed);
3478 // NaN is always a bail condition, just bail directly.
3479 ma_b(bail, Assembler::Overflow);
3480
3481 // The argument is a positive number, truncation is the path to glory. Since
3482 // it is known to be > 0.0, explicitly convert to a larger range, then a
3483 // value that rounds to INT_MAX is explicitly different from an argument
3484 // that clamps to INT_MAX.
3485 ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
3486 ma_vxfer(scratchDouble.uintOverlay(), output);
3487 ma_mov(output, output, SetCC);
3488 ma_b(bail, Signed);
3489 ma_b(&fin);
3490
3491 bind(&handleZero);
3492 // Move the top word of the double into the output reg, if it is non-zero,
3493 // then the original value was -0.0.
3494 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3495 as_cmp(output, Imm8(0));
3496 ma_b(bail, NonZero);
3497 ma_b(&fin);
3498
3499 bind(&handleNeg);
3500 // Negative case, negate, then start dancing.
3501 ma_vneg(input, input);
3502 ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
3503 ma_vxfer(scratchDouble.uintOverlay(), output);
3504 ma_vcvt_U32_F64(scratchDouble.uintOverlay(), scratchDouble);
3505 compareDouble(scratchDouble, input);
3506 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3507 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3508 // result will still be a negative number.
3509 as_rsb(output, output, Imm8(0), SetCC);
3510 // Flip the negated input back to its original value.
3511 ma_vneg(input, input);
3512 // If the result looks non-negative, then this value didn't actually fit
3513 // into the int range, and special handling is required. Zero is also caught
3514 // by this case, but floor of a negative number should never be zero.
3515 ma_b(bail, NotSigned);
3516
3517 bind(&fin);
3518 }
3519
floorf(FloatRegister input,Register output,Label * bail)3520 void MacroAssemblerARMCompat::floorf(FloatRegister input, Register output,
3521 Label* bail) {
3522 Label handleZero;
3523 Label handleNeg;
3524 Label fin;
3525 compareFloat(input, NoVFPRegister);
3526 ma_b(&handleZero, Assembler::Equal);
3527 ma_b(&handleNeg, Assembler::Signed);
3528 // NaN is always a bail condition, just bail directly.
3529 ma_b(bail, Assembler::Overflow);
3530
3531 // The argument is a positive number, truncation is the path to glory; Since
3532 // it is known to be > 0.0, explicitly convert to a larger range, then a
3533 // value that rounds to INT_MAX is explicitly different from an argument
3534 // that clamps to INT_MAX.
3535 {
3536 ScratchFloat32Scope scratch(asMasm());
3537 ma_vcvt_F32_U32(input, scratch.uintOverlay());
3538 ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
3539 }
3540 ma_mov(output, output, SetCC);
3541 ma_b(bail, Signed);
3542 ma_b(&fin);
3543
3544 bind(&handleZero);
3545 // Move the top word of the double into the output reg, if it is non-zero,
3546 // then the original value was -0.0.
3547 as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
3548 Always, 0);
3549 as_cmp(output, Imm8(0));
3550 ma_b(bail, NonZero);
3551 ma_b(&fin);
3552
3553 bind(&handleNeg);
3554 // Negative case, negate, then start dancing.
3555 {
3556 ScratchFloat32Scope scratch(asMasm());
3557 ma_vneg_f32(input, input);
3558 ma_vcvt_F32_U32(input, scratch.uintOverlay());
3559 ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
3560 ma_vcvt_U32_F32(scratch.uintOverlay(), scratch);
3561 compareFloat(scratch, input);
3562 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3563 }
3564 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3565 // result will still be a negative number.
3566 as_rsb(output, output, Imm8(0), SetCC);
3567 // Flip the negated input back to its original value.
3568 ma_vneg_f32(input, input);
3569 // If the result looks non-negative, then this value didn't actually fit
3570 // into the int range, and special handling is required. Zero is also caught
3571 // by this case, but floor of a negative number should never be zero.
3572 ma_b(bail, NotSigned);
3573
3574 bind(&fin);
3575 }
3576
ceil(FloatRegister input,Register output,Label * bail)3577 void MacroAssemblerARMCompat::ceil(FloatRegister input, Register output,
3578 Label* bail) {
3579 Label handleZero;
3580 Label handlePos;
3581 Label fin;
3582
3583 compareDouble(input, NoVFPRegister);
3584 // NaN is always a bail condition, just bail directly.
3585 ma_b(bail, Assembler::Overflow);
3586 ma_b(&handleZero, Assembler::Equal);
3587 ma_b(&handlePos, Assembler::NotSigned);
3588
3589 ScratchDoubleScope scratchDouble(asMasm());
3590
3591 // We are in the ]-Inf; 0[ range
3592 // If we are in the ]-1; 0[ range => bailout
3593 loadConstantDouble(-1.0, scratchDouble);
3594 compareDouble(input, scratchDouble);
3595 ma_b(bail, Assembler::GreaterThan);
3596
3597 // We are in the ]-Inf; -1] range: ceil(x) == -floor(-x) and floor can be
3598 // computed with direct truncation here (x > 0).
3599 ma_vneg(input, scratchDouble);
3600 FloatRegister ScratchUIntReg = scratchDouble.uintOverlay();
3601 ma_vcvt_F64_U32(scratchDouble, ScratchUIntReg);
3602 ma_vxfer(ScratchUIntReg, output);
3603 ma_neg(output, output, SetCC);
3604 ma_b(bail, NotSigned);
3605 ma_b(&fin);
3606
3607 // Test for 0.0 / -0.0: if the top word of the input double is not zero,
3608 // then it was -0 and we need to bail out.
3609 bind(&handleZero);
3610 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3611 as_cmp(output, Imm8(0));
3612 ma_b(bail, NonZero);
3613 ma_b(&fin);
3614
3615 // We are in the ]0; +inf] range: truncate integer values, maybe add 1 for
3616 // non integer values, maybe bail if overflow.
3617 bind(&handlePos);
3618 ma_vcvt_F64_U32(input, ScratchUIntReg);
3619 ma_vxfer(ScratchUIntReg, output);
3620 ma_vcvt_U32_F64(ScratchUIntReg, scratchDouble);
3621 compareDouble(scratchDouble, input);
3622 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3623 // Bail out if the add overflowed or the result is non positive.
3624 ma_mov(output, output, SetCC);
3625 ma_b(bail, Signed);
3626 ma_b(bail, Zero);
3627
3628 bind(&fin);
3629 }
3630
ceilf(FloatRegister input,Register output,Label * bail)3631 void MacroAssemblerARMCompat::ceilf(FloatRegister input, Register output,
3632 Label* bail) {
3633 Label handleZero;
3634 Label handlePos;
3635 Label fin;
3636
3637 compareFloat(input, NoVFPRegister);
3638 // NaN is always a bail condition, just bail directly.
3639 ma_b(bail, Assembler::Overflow);
3640 ma_b(&handleZero, Assembler::Equal);
3641 ma_b(&handlePos, Assembler::NotSigned);
3642
3643 // We are in the ]-Inf; 0[ range
3644 // If we are in the ]-1; 0[ range => bailout
3645 {
3646 ScratchFloat32Scope scratch(asMasm());
3647 loadConstantFloat32(-1.f, scratch);
3648 compareFloat(input, scratch);
3649 ma_b(bail, Assembler::GreaterThan);
3650 }
3651
3652 // We are in the ]-Inf; -1] range: ceil(x) == -floor(-x) and floor can be
3653 // computed with direct truncation here (x > 0).
3654 {
3655 ScratchDoubleScope scratchDouble(asMasm());
3656 FloatRegister scratchFloat = scratchDouble.asSingle();
3657 FloatRegister scratchUInt = scratchDouble.uintOverlay();
3658
3659 ma_vneg_f32(input, scratchFloat);
3660 ma_vcvt_F32_U32(scratchFloat, scratchUInt);
3661 ma_vxfer(scratchUInt, output);
3662 ma_neg(output, output, SetCC);
3663 ma_b(bail, NotSigned);
3664 ma_b(&fin);
3665 }
3666
3667 // Test for 0.0 / -0.0: if the top word of the input double is not zero,
3668 // then it was -0 and we need to bail out.
3669 bind(&handleZero);
3670 as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
3671 Always, 0);
3672 as_cmp(output, Imm8(0));
3673 ma_b(bail, NonZero);
3674 ma_b(&fin);
3675
3676 // We are in the ]0; +inf] range: truncate integer values, maybe add 1 for
3677 // non integer values, maybe bail if overflow.
3678 bind(&handlePos);
3679 {
3680 ScratchDoubleScope scratchDouble(asMasm());
3681 FloatRegister scratchFloat = scratchDouble.asSingle();
3682 FloatRegister scratchUInt = scratchDouble.uintOverlay();
3683
3684 ma_vcvt_F32_U32(input, scratchUInt);
3685 ma_vxfer(scratchUInt, output);
3686 ma_vcvt_U32_F32(scratchUInt, scratchFloat);
3687 compareFloat(scratchFloat, input);
3688 as_add(output, output, Imm8(1), LeaveCC, NotEqual);
3689
3690 // Bail on overflow or non-positive result.
3691 ma_mov(output, output, SetCC);
3692 ma_b(bail, Signed);
3693 ma_b(bail, Zero);
3694 }
3695
3696 bind(&fin);
3697 }
3698
toggledJump(Label * label)3699 CodeOffset MacroAssemblerARMCompat::toggledJump(Label* label) {
3700 // Emit a B that can be toggled to a CMP. See ToggleToJmp(), ToggleToCmp().
3701 BufferOffset b = ma_b(label, Always);
3702 CodeOffset ret(b.getOffset());
3703 return ret;
3704 }
3705
toggledCall(JitCode * target,bool enabled)3706 CodeOffset MacroAssemblerARMCompat::toggledCall(JitCode* target, bool enabled) {
3707 BufferOffset bo = nextOffset();
3708 addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
3709 ScratchRegisterScope scratch(asMasm());
3710 ma_movPatchable(ImmPtr(target->raw()), scratch, Always);
3711 if (enabled) {
3712 ma_blx(scratch);
3713 } else {
3714 ma_nop();
3715 }
3716 return CodeOffset(bo.getOffset());
3717 }
3718
round(FloatRegister input,Register output,Label * bail,FloatRegister tmp)3719 void MacroAssemblerARMCompat::round(FloatRegister input, Register output,
3720 Label* bail, FloatRegister tmp) {
3721 Label handleZero;
3722 Label handleNeg;
3723 Label fin;
3724
3725 ScratchDoubleScope scratchDouble(asMasm());
3726
3727 // Do a compare based on the original value, then do most other things based
3728 // on the shifted value.
3729 ma_vcmpz(input);
3730 // Since we already know the sign bit, flip all numbers to be positive,
3731 // stored in tmp.
3732 ma_vabs(input, tmp);
3733 as_vmrs(pc);
3734 ma_b(&handleZero, Assembler::Equal);
3735 ma_b(&handleNeg, Assembler::Signed);
3736 // NaN is always a bail condition, just bail directly.
3737 ma_b(bail, Assembler::Overflow);
3738
3739 // The argument is a positive number, truncation is the path to glory; Since
3740 // it is known to be > 0.0, explicitly convert to a larger range, then a
3741 // value that rounds to INT_MAX is explicitly different from an argument
3742 // that clamps to INT_MAX.
3743
3744 // Add the biggest number less than 0.5 (not 0.5, because adding that to
3745 // the biggest number less than 0.5 would undesirably round up to 1), and
3746 // store the result into tmp.
3747 loadConstantDouble(GetBiggestNumberLessThan(0.5), scratchDouble);
3748 ma_vadd(scratchDouble, tmp, tmp);
3749
3750 ma_vcvt_F64_U32(tmp, scratchDouble.uintOverlay());
3751 ma_vxfer(VFPRegister(scratchDouble).uintOverlay(), output);
3752 ma_mov(output, output, SetCC);
3753 ma_b(bail, Signed);
3754 ma_b(&fin);
3755
3756 bind(&handleZero);
3757 // Move the top word of the double into the output reg, if it is non-zero,
3758 // then the original value was -0.0
3759 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3760 as_cmp(output, Imm8(0));
3761 ma_b(bail, NonZero);
3762 ma_b(&fin);
3763
3764 bind(&handleNeg);
3765 // Negative case, negate, then start dancing. This number may be positive,
3766 // since we added 0.5.
3767
3768 // Add 0.5 to negative numbers, store the result into tmp
3769 loadConstantDouble(0.5, scratchDouble);
3770 ma_vadd(scratchDouble, tmp, tmp);
3771
3772 ma_vcvt_F64_U32(tmp, scratchDouble.uintOverlay());
3773 ma_vxfer(VFPRegister(scratchDouble).uintOverlay(), output);
3774
3775 // -output is now a correctly rounded value, unless the original value was
3776 // exactly halfway between two integers, at which point, it has been rounded
3777 // away from zero, when it should be rounded towards \infty.
3778 ma_vcvt_U32_F64(scratchDouble.uintOverlay(), scratchDouble);
3779 compareDouble(scratchDouble, tmp);
3780 as_sub(output, output, Imm8(1), LeaveCC, Equal);
3781 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3782 // result will still be a negative number.
3783 as_rsb(output, output, Imm8(0), SetCC);
3784
3785 // If the result looks non-negative, then this value didn't actually fit
3786 // into the int range, and special handling is required, or it was zero,
3787 // which means the result is actually -0.0 which also requires special
3788 // handling.
3789 ma_b(bail, NotSigned);
3790
3791 bind(&fin);
3792 }
3793
roundf(FloatRegister input,Register output,Label * bail,FloatRegister tmp)3794 void MacroAssemblerARMCompat::roundf(FloatRegister input, Register output,
3795 Label* bail, FloatRegister tmp) {
3796 Label handleZero;
3797 Label handleNeg;
3798 Label fin;
3799
3800 ScratchFloat32Scope scratchFloat(asMasm());
3801
3802 // Do a compare based on the original value, then do most other things based
3803 // on the shifted value.
3804 compareFloat(input, NoVFPRegister);
3805 ma_b(&handleZero, Assembler::Equal);
3806 ma_b(&handleNeg, Assembler::Signed);
3807
3808 // NaN is always a bail condition, just bail directly.
3809 ma_b(bail, Assembler::Overflow);
3810
3811 // The argument is a positive number, truncation is the path to glory; Since
3812 // it is known to be > 0.0, explicitly convert to a larger range, then a
3813 // value that rounds to INT_MAX is explicitly different from an argument
3814 // that clamps to INT_MAX.
3815
3816 // Add the biggest number less than 0.5f (not 0.5f, because adding that to
3817 // the biggest number less than 0.5f would undesirably round up to 1), and
3818 // store the result into tmp.
3819 loadConstantFloat32(GetBiggestNumberLessThan(0.5f), scratchFloat);
3820 ma_vadd_f32(scratchFloat, input, tmp);
3821
3822 // Note: it doesn't matter whether x + .5 === x or not here, as it doesn't
3823 // affect the semantics of the float to unsigned conversion (in particular,
3824 // we are not applying any fixup after the operation).
3825 ma_vcvt_F32_U32(tmp, scratchFloat.uintOverlay());
3826 ma_vxfer(VFPRegister(scratchFloat).uintOverlay(), output);
3827 ma_mov(output, output, SetCC);
3828 ma_b(bail, Signed);
3829 ma_b(&fin);
3830
3831 bind(&handleZero);
3832
3833 // Move the whole float32 into the output reg, if it is non-zero, then the
3834 // original value was -0.0.
3835 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 0);
3836 as_cmp(output, Imm8(0));
3837 ma_b(bail, NonZero);
3838 ma_b(&fin);
3839
3840 bind(&handleNeg);
3841
3842 // Add 0.5 to negative numbers, storing the result into tmp.
3843 ma_vneg_f32(input, tmp);
3844 loadConstantFloat32(0.5f, scratchFloat);
3845 ma_vadd_f32(tmp, scratchFloat, scratchFloat);
3846
3847 // Adding 0.5 to a float input has chances to yield the wrong result, if
3848 // the input is too large. In this case, skip the -1 adjustment made below.
3849 compareFloat(scratchFloat, tmp);
3850
3851 // Negative case, negate, then start dancing. This number may be positive,
3852 // since we added 0.5.
3853 // /!\ The conditional jump afterwards depends on these two instructions
3854 // *not* setting the status flags. They need to not change after the
3855 // comparison above.
3856 ma_vcvt_F32_U32(scratchFloat, tmp.uintOverlay());
3857 ma_vxfer(VFPRegister(tmp).uintOverlay(), output);
3858
3859 Label flipSign;
3860 ma_b(&flipSign, Equal);
3861
3862 // -output is now a correctly rounded value, unless the original value was
3863 // exactly halfway between two integers, at which point, it has been rounded
3864 // away from zero, when it should be rounded towards \infty.
3865 ma_vcvt_U32_F32(tmp.uintOverlay(), tmp);
3866 compareFloat(tmp, scratchFloat);
3867 as_sub(output, output, Imm8(1), LeaveCC, Equal);
3868
3869 // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
3870 // result will still be a negative number.
3871 bind(&flipSign);
3872 as_rsb(output, output, Imm8(0), SetCC);
3873
3874 // If the result looks non-negative, then this value didn't actually fit
3875 // into the int range, and special handling is required, or it was zero,
3876 // which means the result is actually -0.0 which also requires special
3877 // handling.
3878 ma_b(bail, NotSigned);
3879
3880 bind(&fin);
3881 }
3882
trunc(FloatRegister input,Register output,Label * bail)3883 void MacroAssemblerARMCompat::trunc(FloatRegister input, Register output,
3884 Label* bail) {
3885 Label handleZero;
3886 Label handlePos;
3887 Label fin;
3888
3889 compareDouble(input, NoVFPRegister);
3890 // NaN is always a bail condition, just bail directly.
3891 ma_b(bail, Assembler::Overflow);
3892 ma_b(&handleZero, Assembler::Equal);
3893 ma_b(&handlePos, Assembler::NotSigned);
3894
3895 ScratchDoubleScope scratchDouble(asMasm());
3896
3897 // We are in the ]-Inf; 0[ range
3898 // If we are in the ]-1; 0[ range => bailout
3899 loadConstantDouble(-1.0, scratchDouble);
3900 compareDouble(input, scratchDouble);
3901 ma_b(bail, Assembler::GreaterThan);
3902
3903 // We are in the ]-Inf; -1] range: trunc(x) == -floor(-x) and floor can be
3904 // computed with direct truncation here (x > 0).
3905 ma_vneg(input, scratchDouble);
3906 ma_vcvt_F64_U32(scratchDouble, scratchDouble.uintOverlay());
3907 ma_vxfer(scratchDouble.uintOverlay(), output);
3908 ma_neg(output, output, SetCC);
3909 ma_b(bail, NotSigned);
3910 ma_b(&fin);
3911
3912 // Test for 0.0 / -0.0: if the top word of the input double is not zero,
3913 // then it was -0 and we need to bail out.
3914 bind(&handleZero);
3915 as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
3916 as_cmp(output, Imm8(0));
3917 ma_b(bail, NonZero);
3918 ma_b(&fin);
3919
3920 // We are in the ]0; +inf] range: truncation is the path to glory. Since
3921 // it is known to be > 0.0, explicitly convert to a larger range, then a
3922 // value that rounds to INT_MAX is explicitly different from an argument
3923 // that clamps to INT_MAX.
3924 bind(&handlePos);
3925 ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
3926 ma_vxfer(scratchDouble.uintOverlay(), output);
3927 ma_mov(output, output, SetCC);
3928 ma_b(bail, Signed);
3929
3930 bind(&fin);
3931 }
3932
truncf(FloatRegister input,Register output,Label * bail)3933 void MacroAssemblerARMCompat::truncf(FloatRegister input, Register output,
3934 Label* bail) {
3935 Label handleZero;
3936 Label handlePos;
3937 Label fin;
3938
3939 compareFloat(input, NoVFPRegister);
3940 // NaN is always a bail condition, just bail directly.
3941 ma_b(bail, Assembler::Overflow);
3942 ma_b(&handleZero, Assembler::Equal);
3943 ma_b(&handlePos, Assembler::NotSigned);
3944
3945 // We are in the ]-Inf; 0[ range
3946 // If we are in the ]-1; 0[ range => bailout
3947 {
3948 ScratchFloat32Scope scratch(asMasm());
3949 loadConstantFloat32(-1.f, scratch);
3950 compareFloat(input, scratch);
3951 ma_b(bail, Assembler::GreaterThan);
3952 }
3953
3954 // We are in the ]-Inf; -1] range: trunc(x) == -floor(-x) and floor can be
3955 // computed with direct truncation here (x > 0).
3956 {
3957 ScratchDoubleScope scratchDouble(asMasm());
3958 FloatRegister scratchFloat = scratchDouble.asSingle();
3959 FloatRegister scratchUInt = scratchDouble.uintOverlay();
3960
3961 ma_vneg_f32(input, scratchFloat);
3962 ma_vcvt_F32_U32(scratchFloat, scratchUInt);
3963 ma_vxfer(scratchUInt, output);
3964 ma_neg(output, output, SetCC);
3965 ma_b(bail, NotSigned);
3966 ma_b(&fin);
3967 }
3968
3969 // Test for 0.0 / -0.0: if the top word of the input double is not zero,
3970 // then it was -0 and we need to bail out.
3971 bind(&handleZero);
3972 as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
3973 Always, 0);
3974 as_cmp(output, Imm8(0));
3975 ma_b(bail, NonZero);
3976 ma_b(&fin);
3977
3978 // We are in the ]0; +inf] range: truncation is the path to glory; Since
3979 // it is known to be > 0.0, explicitly convert to a larger range, then a
3980 // value that rounds to INT_MAX is explicitly different from an argument
3981 bind(&handlePos);
3982 {
3983 // The argument is a positive number,
3984 // that clamps to INT_MAX.
3985 {
3986 ScratchFloat32Scope scratch(asMasm());
3987 ma_vcvt_F32_U32(input, scratch.uintOverlay());
3988 ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
3989 }
3990 ma_mov(output, output, SetCC);
3991 ma_b(bail, Signed);
3992 }
3993
3994 bind(&fin);
3995 }
3996
profilerEnterFrame(Register framePtr,Register scratch)3997 void MacroAssemblerARMCompat::profilerEnterFrame(Register framePtr,
3998 Register scratch) {
3999 asMasm().loadJSContext(scratch);
4000 loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
4001 storePtr(framePtr,
4002 Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
4003 storePtr(ImmPtr(nullptr),
4004 Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
4005 }
4006
profilerExitFrame()4007 void MacroAssemblerARMCompat::profilerExitFrame() {
4008 jump(GetJitContext()->runtime->jitRuntime()->getProfilerExitFrameTail());
4009 }
4010
asMasm()4011 MacroAssembler& MacroAssemblerARM::asMasm() {
4012 return *static_cast<MacroAssembler*>(this);
4013 }
4014
asMasm() const4015 const MacroAssembler& MacroAssemblerARM::asMasm() const {
4016 return *static_cast<const MacroAssembler*>(this);
4017 }
4018
asMasm()4019 MacroAssembler& MacroAssemblerARMCompat::asMasm() {
4020 return *static_cast<MacroAssembler*>(this);
4021 }
4022
asMasm() const4023 const MacroAssembler& MacroAssemblerARMCompat::asMasm() const {
4024 return *static_cast<const MacroAssembler*>(this);
4025 }
4026
subFromStackPtr(Imm32 imm32)4027 void MacroAssembler::subFromStackPtr(Imm32 imm32) {
4028 ScratchRegisterScope scratch(*this);
4029 if (imm32.value) {
4030 ma_sub(imm32, sp, scratch);
4031 }
4032 }
4033
4034 //{{{ check_macroassembler_style
4035 // ===============================================================
4036 // MacroAssembler high-level usage.
4037
flush()4038 void MacroAssembler::flush() { Assembler::flush(); }
4039
comment(const char * msg)4040 void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
4041
4042 // ===============================================================
4043 // Stack manipulation functions.
4044
PushRegsInMask(LiveRegisterSet set)4045 void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
4046 int32_t diffF = set.fpus().getPushSizeInBytes();
4047 int32_t diffG = set.gprs().size() * sizeof(intptr_t);
4048
4049 if (set.gprs().size() > 1) {
4050 adjustFrame(diffG);
4051 startDataTransferM(IsStore, StackPointer, DB, WriteBack);
4052 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
4053 ++iter) {
4054 diffG -= sizeof(intptr_t);
4055 transferReg(*iter);
4056 }
4057 finishDataTransfer();
4058 } else {
4059 reserveStack(diffG);
4060 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
4061 ++iter) {
4062 diffG -= sizeof(intptr_t);
4063 storePtr(*iter, Address(StackPointer, diffG));
4064 }
4065 }
4066 MOZ_ASSERT(diffG == 0);
4067
4068 // It's possible that the logic is just fine as it is if the reduced set
4069 // maps SIMD pairs to plain doubles and transferMultipleByRuns() stores
4070 // and loads doubles.
4071 #ifdef ENABLE_WASM_SIMD
4072 # error "Needs more careful logic if SIMD is enabled"
4073 #endif
4074
4075 adjustFrame(diffF);
4076 diffF += transferMultipleByRuns(set.fpus(), IsStore, StackPointer, DB);
4077 MOZ_ASSERT(diffF == 0);
4078 }
4079
storeRegsInMask(LiveRegisterSet set,Address dest,Register scratch)4080 void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
4081 Register scratch) {
4082 int32_t diffF = set.fpus().getPushSizeInBytes();
4083 int32_t diffG = set.gprs().size() * sizeof(intptr_t);
4084
4085 MOZ_ASSERT(dest.offset >= diffF + diffG);
4086
4087 if (set.gprs().size() > 1) {
4088 computeEffectiveAddress(dest, scratch);
4089
4090 startDataTransferM(IsStore, scratch, DB, WriteBack);
4091 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
4092 ++iter) {
4093 diffG -= sizeof(intptr_t);
4094 dest.offset -= sizeof(intptr_t);
4095 transferReg(*iter);
4096 }
4097 finishDataTransfer();
4098 } else {
4099 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
4100 ++iter) {
4101 diffG -= sizeof(intptr_t);
4102 dest.offset -= sizeof(intptr_t);
4103 storePtr(*iter, dest);
4104 }
4105 }
4106 MOZ_ASSERT(diffG == 0);
4107
4108 // See above.
4109 #ifdef ENABLE_WASM_SIMD
4110 # error "Needs more careful logic if SIMD is enabled"
4111 #endif
4112
4113 if (diffF > 0) {
4114 computeEffectiveAddress(dest, scratch);
4115 diffF += transferMultipleByRuns(set.fpus(), IsStore, scratch, DB);
4116 }
4117
4118 MOZ_ASSERT(diffF == 0);
4119 }
4120
PopRegsInMaskIgnore(LiveRegisterSet set,LiveRegisterSet ignore)4121 void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
4122 LiveRegisterSet ignore) {
4123 int32_t diffG = set.gprs().size() * sizeof(intptr_t);
4124 int32_t diffF = set.fpus().getPushSizeInBytes();
4125 const int32_t reservedG = diffG;
4126 const int32_t reservedF = diffF;
4127
4128 // See above.
4129 #ifdef ENABLE_WASM_SIMD
4130 # error "Needs more careful logic if SIMD is enabled"
4131 #endif
4132
4133 // ARM can load multiple registers at once, but only if we want back all
4134 // the registers we previously saved to the stack.
4135 if (ignore.emptyFloat()) {
4136 diffF -= transferMultipleByRuns(set.fpus(), IsLoad, StackPointer, IA);
4137 adjustFrame(-reservedF);
4138 } else {
4139 LiveFloatRegisterSet fpset(set.fpus().reduceSetForPush());
4140 LiveFloatRegisterSet fpignore(ignore.fpus().reduceSetForPush());
4141 for (FloatRegisterBackwardIterator iter(fpset); iter.more(); ++iter) {
4142 diffF -= (*iter).size();
4143 if (!fpignore.has(*iter)) {
4144 loadDouble(Address(StackPointer, diffF), *iter);
4145 }
4146 }
4147 freeStack(reservedF);
4148 }
4149 MOZ_ASSERT(diffF == 0);
4150
4151 if (set.gprs().size() > 1 && ignore.emptyGeneral()) {
4152 startDataTransferM(IsLoad, StackPointer, IA, WriteBack);
4153 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
4154 ++iter) {
4155 diffG -= sizeof(intptr_t);
4156 transferReg(*iter);
4157 }
4158 finishDataTransfer();
4159 adjustFrame(-reservedG);
4160 } else {
4161 for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
4162 ++iter) {
4163 diffG -= sizeof(intptr_t);
4164 if (!ignore.has(*iter)) {
4165 loadPtr(Address(StackPointer, diffG), *iter);
4166 }
4167 }
4168 freeStack(reservedG);
4169 }
4170 MOZ_ASSERT(diffG == 0);
4171 }
4172
Push(Register reg)4173 void MacroAssembler::Push(Register reg) {
4174 push(reg);
4175 adjustFrame(sizeof(intptr_t));
4176 }
4177
Push(const Imm32 imm)4178 void MacroAssembler::Push(const Imm32 imm) {
4179 push(imm);
4180 adjustFrame(sizeof(intptr_t));
4181 }
4182
Push(const ImmWord imm)4183 void MacroAssembler::Push(const ImmWord imm) {
4184 push(imm);
4185 adjustFrame(sizeof(intptr_t));
4186 }
4187
Push(const ImmPtr imm)4188 void MacroAssembler::Push(const ImmPtr imm) {
4189 Push(ImmWord(uintptr_t(imm.value)));
4190 }
4191
Push(const ImmGCPtr ptr)4192 void MacroAssembler::Push(const ImmGCPtr ptr) {
4193 push(ptr);
4194 adjustFrame(sizeof(intptr_t));
4195 }
4196
Push(FloatRegister reg)4197 void MacroAssembler::Push(FloatRegister reg) {
4198 VFPRegister r = VFPRegister(reg);
4199 ma_vpush(VFPRegister(reg));
4200 adjustFrame(r.size());
4201 }
4202
PushBoxed(FloatRegister reg)4203 void MacroAssembler::PushBoxed(FloatRegister reg) {
4204 MOZ_ASSERT(reg.isDouble());
4205 Push(reg);
4206 }
4207
Pop(Register reg)4208 void MacroAssembler::Pop(Register reg) {
4209 ma_pop(reg);
4210 adjustFrame(-sizeof(intptr_t));
4211 }
4212
Pop(FloatRegister reg)4213 void MacroAssembler::Pop(FloatRegister reg) {
4214 ma_vpop(reg);
4215 adjustFrame(-reg.size());
4216 }
4217
Pop(const ValueOperand & val)4218 void MacroAssembler::Pop(const ValueOperand& val) {
4219 popValue(val);
4220 adjustFrame(-sizeof(Value));
4221 }
4222
PopStackPtr()4223 void MacroAssembler::PopStackPtr() {
4224 as_dtr(IsLoad, 32, Offset, sp, DTRAddr(sp, DtrOffImm(0)));
4225 adjustFrame(-sizeof(intptr_t));
4226 }
4227
4228 // ===============================================================
4229 // Simple call functions.
4230
call(Register reg)4231 CodeOffset MacroAssembler::call(Register reg) {
4232 as_blx(reg);
4233 return CodeOffset(currentOffset());
4234 }
4235
call(Label * label)4236 CodeOffset MacroAssembler::call(Label* label) {
4237 // For now, assume that it'll be nearby.
4238 as_bl(label, Always);
4239 return CodeOffset(currentOffset());
4240 }
4241
call(ImmWord imm)4242 void MacroAssembler::call(ImmWord imm) { call(ImmPtr((void*)imm.value)); }
4243
call(ImmPtr imm)4244 void MacroAssembler::call(ImmPtr imm) {
4245 BufferOffset bo = m_buffer.nextOffset();
4246 addPendingJump(bo, imm, RelocationKind::HARDCODED);
4247 ma_call(imm);
4248 }
4249
call(wasm::SymbolicAddress imm)4250 CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
4251 movePtr(imm, CallReg);
4252 return call(CallReg);
4253 }
4254
call(const Address & addr)4255 void MacroAssembler::call(const Address& addr) {
4256 loadPtr(addr, CallReg);
4257 call(CallReg);
4258 }
4259
call(JitCode * c)4260 void MacroAssembler::call(JitCode* c) {
4261 BufferOffset bo = m_buffer.nextOffset();
4262 addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
4263 ScratchRegisterScope scratch(*this);
4264 ma_movPatchable(ImmPtr(c->raw()), scratch, Always);
4265 callJitNoProfiler(scratch);
4266 }
4267
callWithPatch()4268 CodeOffset MacroAssembler::callWithPatch() {
4269 // The caller ensures that the call is always in range using thunks (below)
4270 // as necessary.
4271 as_bl(BOffImm(), Always, /* documentation */ nullptr);
4272 return CodeOffset(currentOffset());
4273 }
4274
patchCall(uint32_t callerOffset,uint32_t calleeOffset)4275 void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
4276 BufferOffset inst(callerOffset - 4);
4277 BOffImm off = BufferOffset(calleeOffset).diffB<BOffImm>(inst);
4278 MOZ_RELEASE_ASSERT(!off.isInvalid(),
4279 "Failed to insert necessary far jump islands");
4280 as_bl(off, Always, inst);
4281 }
4282
farJumpWithPatch()4283 CodeOffset MacroAssembler::farJumpWithPatch() {
4284 static_assert(32 * 1024 * 1024 - JumpImmediateRange >
4285 wasm::MaxFuncs * 3 * sizeof(Instruction),
4286 "always enough space for thunks");
4287
4288 // The goal of the thunk is to be able to jump to any address without the
4289 // usual 32MiB branch range limitation. Additionally, to make the thunk
4290 // simple to use, the thunk does not use the constant pool or require
4291 // patching an absolute address. Instead, a relative offset is used which
4292 // can be patched during compilation.
4293
4294 // Inhibit pools since these three words must be contiguous so that the offset
4295 // calculations below are valid.
4296 AutoForbidPoolsAndNops afp(this, 3);
4297
4298 // When pc is used, the read value is the address of the instruction + 8.
4299 // This is exactly the address of the uint32 word we want to load.
4300 ScratchRegisterScope scratch(*this);
4301 ma_ldr(DTRAddr(pc, DtrOffImm(0)), scratch);
4302
4303 // Branch by making pc the destination register.
4304 ma_add(pc, scratch, pc, LeaveCC, Always);
4305
4306 // Allocate space which will be patched by patchFarJump().
4307 CodeOffset farJump(currentOffset());
4308 writeInst(UINT32_MAX);
4309
4310 return farJump;
4311 }
4312
patchFarJump(CodeOffset farJump,uint32_t targetOffset)4313 void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
4314 uint32_t* u32 =
4315 reinterpret_cast<uint32_t*>(editSrc(BufferOffset(farJump.offset())));
4316 MOZ_ASSERT(*u32 == UINT32_MAX);
4317
4318 uint32_t addOffset = farJump.offset() - 4;
4319 MOZ_ASSERT(editSrc(BufferOffset(addOffset))->is<InstALU>());
4320
4321 // When pc is read as the operand of the add, its value is the address of
4322 // the add instruction + 8.
4323 *u32 = (targetOffset - addOffset) - 8;
4324 }
4325
nopPatchableToCall()4326 CodeOffset MacroAssembler::nopPatchableToCall() {
4327 AutoForbidPoolsAndNops afp(this,
4328 /* max number of instructions in scope = */ 1);
4329 ma_nop();
4330 return CodeOffset(currentOffset());
4331 }
4332
patchNopToCall(uint8_t * call,uint8_t * target)4333 void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
4334 uint8_t* inst = call - 4;
4335 MOZ_ASSERT(reinterpret_cast<Instruction*>(inst)->is<InstBLImm>() ||
4336 reinterpret_cast<Instruction*>(inst)->is<InstNOP>());
4337
4338 new (inst) InstBLImm(BOffImm(target - inst), Assembler::Always);
4339 }
4340
patchCallToNop(uint8_t * call)4341 void MacroAssembler::patchCallToNop(uint8_t* call) {
4342 uint8_t* inst = call - 4;
4343 MOZ_ASSERT(reinterpret_cast<Instruction*>(inst)->is<InstBLImm>() ||
4344 reinterpret_cast<Instruction*>(inst)->is<InstNOP>());
4345 new (inst) InstNOP();
4346 }
4347
pushReturnAddress()4348 void MacroAssembler::pushReturnAddress() { push(lr); }
4349
popReturnAddress()4350 void MacroAssembler::popReturnAddress() { pop(lr); }
4351
4352 // ===============================================================
4353 // ABI function calls.
4354
setupUnalignedABICall(Register scratch)4355 void MacroAssembler::setupUnalignedABICall(Register scratch) {
4356 setupABICall();
4357 dynamicAlignment_ = true;
4358
4359 ma_mov(sp, scratch);
4360 // Force sp to be aligned.
4361 as_bic(sp, sp, Imm8(ABIStackAlignment - 1));
4362 ma_push(scratch);
4363 }
4364
callWithABIPre(uint32_t * stackAdjust,bool callFromWasm)4365 void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
4366 MOZ_ASSERT(inCall_);
4367 uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
4368
4369 if (dynamicAlignment_) {
4370 // sizeof(intptr_t) accounts for the saved stack pointer pushed by
4371 // setupUnalignedABICall.
4372 stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
4373 ABIStackAlignment);
4374 } else {
4375 uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
4376 stackForCall += ComputeByteAlignment(
4377 stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
4378 }
4379
4380 *stackAdjust = stackForCall;
4381 reserveStack(stackForCall);
4382
4383 // Position all arguments.
4384 {
4385 enoughMemory_ &= moveResolver_.resolve();
4386 if (!enoughMemory_) {
4387 return;
4388 }
4389
4390 MoveEmitter emitter(*this);
4391 emitter.emit(moveResolver_);
4392 emitter.finish();
4393 }
4394
4395 assertStackAlignment(ABIStackAlignment);
4396
4397 // Save the lr register if we need to preserve it.
4398 if (secondScratchReg_ != lr) {
4399 ma_mov(lr, secondScratchReg_);
4400 }
4401 }
4402
callWithABIPost(uint32_t stackAdjust,MoveOp::Type result,bool callFromWasm)4403 void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
4404 bool callFromWasm) {
4405 if (secondScratchReg_ != lr) {
4406 ma_mov(secondScratchReg_, lr);
4407 }
4408
4409 // Calls to native functions in wasm pass through a thunk which already
4410 // fixes up the return value for us.
4411 if (!callFromWasm && !UseHardFpABI()) {
4412 switch (result) {
4413 case MoveOp::DOUBLE:
4414 // Move double from r0/r1 to ReturnFloatReg.
4415 ma_vxfer(r0, r1, ReturnDoubleReg);
4416 break;
4417 case MoveOp::FLOAT32:
4418 // Move float32 from r0 to ReturnFloatReg.
4419 ma_vxfer(r0, ReturnFloat32Reg);
4420 break;
4421 case MoveOp::GENERAL:
4422 break;
4423 default:
4424 MOZ_CRASH("unexpected callWithABI result");
4425 }
4426 }
4427
4428 freeStack(stackAdjust);
4429
4430 if (dynamicAlignment_) {
4431 // While the x86 supports pop esp, on ARM that isn't well defined, so
4432 // just do it manually.
4433 as_dtr(IsLoad, 32, Offset, sp, DTRAddr(sp, DtrOffImm(0)));
4434 }
4435
4436 #ifdef DEBUG
4437 MOZ_ASSERT(inCall_);
4438 inCall_ = false;
4439 #endif
4440 }
4441
callWithABINoProfiler(Register fun,MoveOp::Type result)4442 void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
4443 // Load the callee in r12, as above.
4444 ma_mov(fun, r12);
4445 uint32_t stackAdjust;
4446 callWithABIPre(&stackAdjust);
4447 call(r12);
4448 callWithABIPost(stackAdjust, result);
4449 }
4450
callWithABINoProfiler(const Address & fun,MoveOp::Type result)4451 void MacroAssembler::callWithABINoProfiler(const Address& fun,
4452 MoveOp::Type result) {
4453 // Load the callee in r12, no instruction between the ldr and call should
4454 // clobber it. Note that we can't use fun.base because it may be one of the
4455 // IntArg registers clobbered before the call.
4456 {
4457 ScratchRegisterScope scratch(*this);
4458 ma_ldr(fun, r12, scratch);
4459 }
4460 uint32_t stackAdjust;
4461 callWithABIPre(&stackAdjust);
4462 call(r12);
4463 callWithABIPost(stackAdjust, result);
4464 }
4465
4466 // ===============================================================
4467 // Jit Frames.
4468
pushFakeReturnAddress(Register scratch)4469 uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
4470 // On ARM any references to the pc, adds an additional 8 to it, which
4471 // correspond to 2 instructions of 4 bytes. Thus we use an additional nop
4472 // to pad until we reach the pushed pc.
4473 //
4474 // Note: In practice this should not be necessary, as this fake return
4475 // address is never used for resuming any execution. Thus theoriticaly we
4476 // could just do a Push(pc), and ignore the nop as well as the pool.
4477 enterNoPool(2);
4478 DebugOnly<uint32_t> offsetBeforePush = currentOffset();
4479 Push(pc); // actually pushes $pc + 8.
4480 ma_nop();
4481 uint32_t pseudoReturnOffset = currentOffset();
4482 leaveNoPool();
4483
4484 MOZ_ASSERT_IF(!oom(), pseudoReturnOffset - offsetBeforePush == 8);
4485 return pseudoReturnOffset;
4486 }
4487
enterFakeExitFrameForWasm(Register cxreg,Register scratch,ExitFrameType type)4488 void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
4489 ExitFrameType type) {
4490 enterFakeExitFrame(cxreg, scratch, type);
4491 }
4492
4493 // ===============================================================
4494 // Move instructions
4495
moveValue(const TypedOrValueRegister & src,const ValueOperand & dest)4496 void MacroAssembler::moveValue(const TypedOrValueRegister& src,
4497 const ValueOperand& dest) {
4498 if (src.hasValue()) {
4499 moveValue(src.valueReg(), dest);
4500 return;
4501 }
4502
4503 MIRType type = src.type();
4504 AnyRegister reg = src.typedReg();
4505
4506 if (!IsFloatingPointType(type)) {
4507 mov(ImmWord(MIRTypeToTag(type)), dest.typeReg());
4508 if (reg.gpr() != dest.payloadReg()) {
4509 mov(reg.gpr(), dest.payloadReg());
4510 }
4511 return;
4512 }
4513
4514 ScratchFloat32Scope scratch(*this);
4515 FloatRegister freg = reg.fpu();
4516 if (type == MIRType::Float32) {
4517 convertFloat32ToDouble(freg, scratch);
4518 freg = scratch;
4519 }
4520 ma_vxfer(freg, dest.payloadReg(), dest.typeReg());
4521 }
4522
moveValue(const ValueOperand & src,const ValueOperand & dest)4523 void MacroAssembler::moveValue(const ValueOperand& src,
4524 const ValueOperand& dest) {
4525 Register s0 = src.typeReg();
4526 Register s1 = src.payloadReg();
4527 Register d0 = dest.typeReg();
4528 Register d1 = dest.payloadReg();
4529
4530 // Either one or both of the source registers could be the same as a
4531 // destination register.
4532 if (s1 == d0) {
4533 if (s0 == d1) {
4534 // If both are, this is just a swap of two registers.
4535 ScratchRegisterScope scratch(*this);
4536 MOZ_ASSERT(d1 != scratch);
4537 MOZ_ASSERT(d0 != scratch);
4538 ma_mov(d1, scratch);
4539 ma_mov(d0, d1);
4540 ma_mov(scratch, d0);
4541 return;
4542 }
4543 // If only one is, copy that source first.
4544 std::swap(s0, s1);
4545 std::swap(d0, d1);
4546 }
4547
4548 if (s0 != d0) {
4549 ma_mov(s0, d0);
4550 }
4551 if (s1 != d1) {
4552 ma_mov(s1, d1);
4553 }
4554 }
4555
moveValue(const Value & src,const ValueOperand & dest)4556 void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
4557 ma_mov(Imm32(src.toNunboxTag()), dest.typeReg());
4558 if (src.isGCThing()) {
4559 ma_mov(ImmGCPtr(src.toGCThing()), dest.payloadReg());
4560 } else {
4561 ma_mov(Imm32(src.toNunboxPayload()), dest.payloadReg());
4562 }
4563 }
4564
4565 // ===============================================================
4566 // Branch functions
4567
loadStoreBuffer(Register ptr,Register buffer)4568 void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
4569 ma_lsr(Imm32(gc::ChunkShift), ptr, buffer);
4570 ma_lsl(Imm32(gc::ChunkShift), buffer, buffer);
4571 load32(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
4572 }
4573
branchPtrInNurseryChunk(Condition cond,Register ptr,Register temp,Label * label)4574 void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
4575 Register temp, Label* label) {
4576 Maybe<SecondScratchRegisterScope> scratch2;
4577 if (temp == Register::Invalid()) {
4578 scratch2.emplace(*this);
4579 temp = scratch2.ref();
4580 }
4581
4582 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4583 MOZ_ASSERT(ptr != temp);
4584
4585 ma_lsr(Imm32(gc::ChunkShift), ptr, temp);
4586 ma_lsl(Imm32(gc::ChunkShift), temp, temp);
4587 load32(Address(temp, gc::ChunkLocationOffset), temp);
4588 branch32(cond, temp, Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
4589 }
4590
branchValueIsNurseryCell(Condition cond,const Address & address,Register temp,Label * label)4591 void MacroAssembler::branchValueIsNurseryCell(Condition cond,
4592 const Address& address,
4593 Register temp, Label* label) {
4594 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4595
4596 Label done;
4597
4598 branchTestGCThing(Assembler::NotEqual, address,
4599 cond == Assembler::Equal ? &done : label);
4600
4601 loadPtr(ToPayload(address), temp);
4602 SecondScratchRegisterScope scratch2(*this);
4603 branchPtrInNurseryChunk(cond, temp, scratch2, label);
4604
4605 bind(&done);
4606 }
4607
branchValueIsNurseryCell(Condition cond,ValueOperand value,Register temp,Label * label)4608 void MacroAssembler::branchValueIsNurseryCell(Condition cond,
4609 ValueOperand value, Register temp,
4610 Label* label) {
4611 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
4612
4613 Label done;
4614
4615 branchTestGCThing(Assembler::NotEqual, value,
4616 cond == Assembler::Equal ? &done : label);
4617 branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
4618
4619 bind(&done);
4620 }
4621
branchTestValue(Condition cond,const ValueOperand & lhs,const Value & rhs,Label * label)4622 void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
4623 const Value& rhs, Label* label) {
4624 MOZ_ASSERT(cond == Equal || cond == NotEqual);
4625 // If cond == NotEqual, branch when a.payload != b.payload || a.tag !=
4626 // b.tag. If the payloads are equal, compare the tags. If the payloads are
4627 // not equal, short circuit true (NotEqual).
4628 //
4629 // If cand == Equal, branch when a.payload == b.payload && a.tag == b.tag.
4630 // If the payloads are equal, compare the tags. If the payloads are not
4631 // equal, short circuit false (NotEqual).
4632 ScratchRegisterScope scratch(*this);
4633
4634 if (rhs.isGCThing()) {
4635 ma_cmp(lhs.payloadReg(), ImmGCPtr(rhs.toGCThing()), scratch);
4636 } else {
4637 ma_cmp(lhs.payloadReg(), Imm32(rhs.toNunboxPayload()), scratch);
4638 }
4639 ma_cmp(lhs.typeReg(), Imm32(rhs.toNunboxTag()), scratch, Equal);
4640 ma_b(label, cond);
4641 }
4642
4643 // ========================================================================
4644 // Memory access primitives.
4645 template <typename T>
storeUnboxedValue(const ConstantOrRegister & value,MIRType valueType,const T & dest,MIRType slotType)4646 void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
4647 MIRType valueType, const T& dest,
4648 MIRType slotType) {
4649 if (valueType == MIRType::Double) {
4650 storeDouble(value.reg().typedReg().fpu(), dest);
4651 return;
4652 }
4653
4654 // Store the type tag if needed.
4655 if (valueType != slotType) {
4656 storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), dest);
4657 }
4658
4659 // Store the payload.
4660 if (value.constant()) {
4661 storePayload(value.value(), dest);
4662 } else {
4663 storePayload(value.reg().typedReg().gpr(), dest);
4664 }
4665 }
4666
4667 template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
4668 MIRType valueType,
4669 const Address& dest,
4670 MIRType slotType);
4671 template void MacroAssembler::storeUnboxedValue(
4672 const ConstantOrRegister& value, MIRType valueType,
4673 const BaseObjectElementIndex& dest, MIRType slotType);
4674
wasmTrapInstruction()4675 CodeOffset MacroAssembler::wasmTrapInstruction() {
4676 return CodeOffset(as_illegal_trap().getOffset());
4677 }
4678
wasmBoundsCheck(Condition cond,Register index,Register boundsCheckLimit,Label * label)4679 void MacroAssembler::wasmBoundsCheck(Condition cond, Register index,
4680 Register boundsCheckLimit, Label* label) {
4681 as_cmp(index, O2Reg(boundsCheckLimit));
4682 as_b(label, cond);
4683 if (JitOptions.spectreIndexMasking) {
4684 ma_mov(boundsCheckLimit, index, LeaveCC, cond);
4685 }
4686 }
4687
wasmBoundsCheck(Condition cond,Register index,Address boundsCheckLimit,Label * label)4688 void MacroAssembler::wasmBoundsCheck(Condition cond, Register index,
4689 Address boundsCheckLimit, Label* label) {
4690 ScratchRegisterScope scratch(*this);
4691 MOZ_ASSERT(boundsCheckLimit.offset ==
4692 offsetof(wasm::TlsData, boundsCheckLimit));
4693 ma_ldr(DTRAddr(boundsCheckLimit.base, DtrOffImm(boundsCheckLimit.offset)),
4694 scratch);
4695 as_cmp(index, O2Reg(scratch));
4696 as_b(label, cond);
4697 if (JitOptions.spectreIndexMasking) {
4698 ma_mov(scratch, index, LeaveCC, cond);
4699 }
4700 }
4701
wasmTruncateDoubleToUInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4702 void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
4703 Register output,
4704 bool isSaturating,
4705 Label* oolEntry) {
4706 wasmTruncateToInt32(input, output, MIRType::Double, /* isUnsigned= */ true,
4707 isSaturating, oolEntry);
4708 }
4709
wasmTruncateDoubleToInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4710 void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
4711 Register output,
4712 bool isSaturating,
4713 Label* oolEntry) {
4714 wasmTruncateToInt32(input, output, MIRType::Double, /* isUnsigned= */ false,
4715 isSaturating, oolEntry);
4716 }
4717
wasmTruncateFloat32ToUInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4718 void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
4719 Register output,
4720 bool isSaturating,
4721 Label* oolEntry) {
4722 wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ true,
4723 isSaturating, oolEntry);
4724 }
4725
wasmTruncateFloat32ToInt32(FloatRegister input,Register output,bool isSaturating,Label * oolEntry)4726 void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
4727 Register output,
4728 bool isSaturating,
4729 Label* oolEntry) {
4730 wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ false,
4731 isSaturating, oolEntry);
4732 }
4733
oolWasmTruncateCheckF32ToI32(FloatRegister input,Register output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4734 void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
4735 Register output,
4736 TruncFlags flags,
4737 wasm::BytecodeOffset off,
4738 Label* rejoin) {
4739 outOfLineWasmTruncateToIntCheck(input, MIRType::Float32, MIRType::Int32,
4740 flags, rejoin, off);
4741 }
4742
oolWasmTruncateCheckF64ToI32(FloatRegister input,Register output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4743 void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
4744 Register output,
4745 TruncFlags flags,
4746 wasm::BytecodeOffset off,
4747 Label* rejoin) {
4748 outOfLineWasmTruncateToIntCheck(input, MIRType::Double, MIRType::Int32, flags,
4749 rejoin, off);
4750 }
4751
oolWasmTruncateCheckF32ToI64(FloatRegister input,Register64 output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4752 void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
4753 Register64 output,
4754 TruncFlags flags,
4755 wasm::BytecodeOffset off,
4756 Label* rejoin) {
4757 outOfLineWasmTruncateToIntCheck(input, MIRType::Float32, MIRType::Int64,
4758 flags, rejoin, off);
4759 }
4760
oolWasmTruncateCheckF64ToI64(FloatRegister input,Register64 output,TruncFlags flags,wasm::BytecodeOffset off,Label * rejoin)4761 void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
4762 Register64 output,
4763 TruncFlags flags,
4764 wasm::BytecodeOffset off,
4765 Label* rejoin) {
4766 outOfLineWasmTruncateToIntCheck(input, MIRType::Double, MIRType::Int64, flags,
4767 rejoin, off);
4768 }
4769
wasmLoad(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,AnyRegister output)4770 void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
4771 Register memoryBase, Register ptr,
4772 Register ptrScratch, AnyRegister output) {
4773 wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output,
4774 Register64::Invalid());
4775 }
4776
wasmLoadI64(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,Register64 output)4777 void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
4778 Register memoryBase, Register ptr,
4779 Register ptrScratch, Register64 output) {
4780 MOZ_ASSERT_IF(access.isAtomic(), access.byteSize() <= 4);
4781 wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(), output);
4782 }
4783
wasmStore(const wasm::MemoryAccessDesc & access,AnyRegister value,Register memoryBase,Register ptr,Register ptrScratch)4784 void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
4785 AnyRegister value, Register memoryBase,
4786 Register ptr, Register ptrScratch) {
4787 wasmStoreImpl(access, value, Register64::Invalid(), memoryBase, ptr,
4788 ptrScratch);
4789 }
4790
wasmStoreI64(const wasm::MemoryAccessDesc & access,Register64 value,Register memoryBase,Register ptr,Register ptrScratch)4791 void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
4792 Register64 value, Register memoryBase,
4793 Register ptr, Register ptrScratch) {
4794 MOZ_ASSERT(!access.isAtomic());
4795 wasmStoreImpl(access, AnyRegister(), value, memoryBase, ptr, ptrScratch);
4796 }
4797
wasmUnalignedLoad(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,Register output,Register tmp)4798 void MacroAssembler::wasmUnalignedLoad(const wasm::MemoryAccessDesc& access,
4799 Register memoryBase, Register ptr,
4800 Register ptrScratch, Register output,
4801 Register tmp) {
4802 wasmUnalignedLoadImpl(access, memoryBase, ptr, ptrScratch,
4803 AnyRegister(output), Register64::Invalid(), tmp,
4804 Register::Invalid(), Register::Invalid());
4805 }
4806
wasmUnalignedLoadFP(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,FloatRegister outFP,Register tmp1,Register tmp2,Register tmp3)4807 void MacroAssembler::wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access,
4808 Register memoryBase, Register ptr,
4809 Register ptrScratch,
4810 FloatRegister outFP, Register tmp1,
4811 Register tmp2, Register tmp3) {
4812 wasmUnalignedLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(outFP),
4813 Register64::Invalid(), tmp1, tmp2, tmp3);
4814 }
4815
wasmUnalignedLoadI64(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,Register64 out64,Register tmp)4816 void MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
4817 Register memoryBase, Register ptr,
4818 Register ptrScratch, Register64 out64,
4819 Register tmp) {
4820 wasmUnalignedLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(),
4821 out64, tmp, Register::Invalid(), Register::Invalid());
4822 }
4823
wasmUnalignedStore(const wasm::MemoryAccessDesc & access,Register value,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)4824 void MacroAssembler::wasmUnalignedStore(const wasm::MemoryAccessDesc& access,
4825 Register value, Register memoryBase,
4826 Register ptr, Register ptrScratch,
4827 Register tmp) {
4828 MOZ_ASSERT(tmp == Register::Invalid());
4829 wasmUnalignedStoreImpl(access, FloatRegister(), Register64::Invalid(),
4830 memoryBase, ptr, ptrScratch, value);
4831 }
4832
wasmUnalignedStoreFP(const wasm::MemoryAccessDesc & access,FloatRegister floatVal,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)4833 void MacroAssembler::wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access,
4834 FloatRegister floatVal,
4835 Register memoryBase, Register ptr,
4836 Register ptrScratch, Register tmp) {
4837 wasmUnalignedStoreImpl(access, floatVal, Register64::Invalid(), memoryBase,
4838 ptr, ptrScratch, tmp);
4839 }
4840
wasmUnalignedStoreI64(const wasm::MemoryAccessDesc & access,Register64 val64,Register memoryBase,Register ptr,Register ptrScratch,Register tmp)4841 void MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
4842 Register64 val64,
4843 Register memoryBase, Register ptr,
4844 Register ptrScratch, Register tmp) {
4845 wasmUnalignedStoreImpl(access, FloatRegister(), val64, memoryBase, ptr,
4846 ptrScratch, tmp);
4847 }
4848
4849 // ========================================================================
4850 // Primitive atomic operations.
4851
ComputePointerForAtomic(MacroAssembler & masm,const BaseIndex & src,Register r)4852 static Register ComputePointerForAtomic(MacroAssembler& masm,
4853 const BaseIndex& src, Register r) {
4854 Register base = src.base;
4855 Register index = src.index;
4856 uint32_t scale = Imm32::ShiftOf(src.scale).value;
4857 int32_t offset = src.offset;
4858
4859 ScratchRegisterScope scratch(masm);
4860
4861 masm.as_add(r, base, lsl(index, scale));
4862 if (offset != 0) {
4863 masm.ma_add(r, Imm32(offset), r, scratch);
4864 }
4865 return r;
4866 }
4867
ComputePointerForAtomic(MacroAssembler & masm,const Address & src,Register r)4868 static Register ComputePointerForAtomic(MacroAssembler& masm,
4869 const Address& src, Register r) {
4870 ScratchRegisterScope scratch(masm);
4871 if (src.offset == 0) {
4872 return src.base;
4873 }
4874 masm.ma_add(src.base, Imm32(src.offset), r, scratch);
4875 return r;
4876 }
4877
4878 // General algorithm:
4879 //
4880 // ... ptr, <addr> ; compute address of item
4881 // dmb
4882 // L0 ldrex* output, [ptr]
4883 // sxt* output, output, 0 ; sign-extend if applicable
4884 // *xt* tmp, oldval, 0 ; sign-extend or zero-extend if applicable
4885 // cmp output, tmp
4886 // bne L1 ; failed - values are different
4887 // strex* tmp, newval, [ptr]
4888 // cmp tmp, 1
4889 // beq L0 ; failed - location is dirty, retry
4890 // L1 dmb
4891 //
4892 // Discussion here: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html.
4893 // However note that that discussion uses 'isb' as the trailing fence.
4894 // I've not quite figured out why, and I've gone with dmb here which
4895 // is safe. Also see the LLVM source, which uses 'dmb ish' generally.
4896 // (Apple's Swift CPU apparently handles ish in a non-default, faster
4897 // way.)
4898
4899 template <typename T>
CompareExchange(MacroAssembler & masm,const wasm::MemoryAccessDesc * access,Scalar::Type type,const Synchronization & sync,const T & mem,Register oldval,Register newval,Register output)4900 static void CompareExchange(MacroAssembler& masm,
4901 const wasm::MemoryAccessDesc* access,
4902 Scalar::Type type, const Synchronization& sync,
4903 const T& mem, Register oldval, Register newval,
4904 Register output) {
4905 bool signExtend = Scalar::isSignedIntType(type);
4906 unsigned nbytes = Scalar::byteSize(type);
4907
4908 MOZ_ASSERT(nbytes <= 4);
4909
4910 Label again;
4911 Label done;
4912
4913 SecondScratchRegisterScope scratch2(masm);
4914 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
4915
4916 ScratchRegisterScope scratch(masm);
4917
4918 masm.memoryBarrierBefore(sync);
4919
4920 masm.bind(&again);
4921
4922 BufferOffset firstAccess;
4923 switch (nbytes) {
4924 case 1:
4925 firstAccess = masm.as_ldrexb(output, ptr);
4926 if (signExtend) {
4927 masm.as_sxtb(output, output, 0);
4928 masm.as_sxtb(scratch, oldval, 0);
4929 } else {
4930 masm.as_uxtb(scratch, oldval, 0);
4931 }
4932 break;
4933 case 2:
4934 firstAccess = masm.as_ldrexh(output, ptr);
4935 if (signExtend) {
4936 masm.as_sxth(output, output, 0);
4937 masm.as_sxth(scratch, oldval, 0);
4938 } else {
4939 masm.as_uxth(scratch, oldval, 0);
4940 }
4941 break;
4942 case 4:
4943 firstAccess = masm.as_ldrex(output, ptr);
4944 break;
4945 }
4946 if (access) {
4947 masm.append(*access, firstAccess.getOffset());
4948 }
4949
4950 if (nbytes < 4) {
4951 masm.as_cmp(output, O2Reg(scratch));
4952 } else {
4953 masm.as_cmp(output, O2Reg(oldval));
4954 }
4955 masm.as_b(&done, MacroAssembler::NotEqual);
4956 switch (nbytes) {
4957 case 1:
4958 masm.as_strexb(scratch, newval, ptr);
4959 break;
4960 case 2:
4961 masm.as_strexh(scratch, newval, ptr);
4962 break;
4963 case 4:
4964 masm.as_strex(scratch, newval, ptr);
4965 break;
4966 }
4967 masm.as_cmp(scratch, Imm8(1));
4968 masm.as_b(&again, MacroAssembler::Equal);
4969 masm.bind(&done);
4970
4971 masm.memoryBarrierAfter(sync);
4972 }
4973
compareExchange(Scalar::Type type,const Synchronization & sync,const Address & address,Register oldval,Register newval,Register output)4974 void MacroAssembler::compareExchange(Scalar::Type type,
4975 const Synchronization& sync,
4976 const Address& address, Register oldval,
4977 Register newval, Register output) {
4978 CompareExchange(*this, nullptr, type, sync, address, oldval, newval, output);
4979 }
4980
compareExchange(Scalar::Type type,const Synchronization & sync,const BaseIndex & address,Register oldval,Register newval,Register output)4981 void MacroAssembler::compareExchange(Scalar::Type type,
4982 const Synchronization& sync,
4983 const BaseIndex& address, Register oldval,
4984 Register newval, Register output) {
4985 CompareExchange(*this, nullptr, type, sync, address, oldval, newval, output);
4986 }
4987
wasmCompareExchange(const wasm::MemoryAccessDesc & access,const Address & mem,Register oldval,Register newval,Register output)4988 void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
4989 const Address& mem, Register oldval,
4990 Register newval, Register output) {
4991 CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
4992 newval, output);
4993 }
4994
wasmCompareExchange(const wasm::MemoryAccessDesc & access,const BaseIndex & mem,Register oldval,Register newval,Register output)4995 void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
4996 const BaseIndex& mem, Register oldval,
4997 Register newval, Register output) {
4998 CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
4999 newval, output);
5000 }
5001
5002 template <typename T>
AtomicExchange(MacroAssembler & masm,const wasm::MemoryAccessDesc * access,Scalar::Type type,const Synchronization & sync,const T & mem,Register value,Register output)5003 static void AtomicExchange(MacroAssembler& masm,
5004 const wasm::MemoryAccessDesc* access,
5005 Scalar::Type type, const Synchronization& sync,
5006 const T& mem, Register value, Register output) {
5007 bool signExtend = Scalar::isSignedIntType(type);
5008 unsigned nbytes = Scalar::byteSize(type);
5009
5010 MOZ_ASSERT(nbytes <= 4);
5011
5012 // Bug 1077321: We may further optimize for ARMv8 (AArch32) here.
5013 Label again;
5014 Label done;
5015
5016 SecondScratchRegisterScope scratch2(masm);
5017 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5018
5019 ScratchRegisterScope scratch(masm);
5020
5021 masm.memoryBarrierBefore(sync);
5022
5023 masm.bind(&again);
5024
5025 BufferOffset firstAccess;
5026 switch (nbytes) {
5027 case 1:
5028 firstAccess = masm.as_ldrexb(output, ptr);
5029 if (signExtend) {
5030 masm.as_sxtb(output, output, 0);
5031 }
5032 masm.as_strexb(scratch, value, ptr);
5033 break;
5034 case 2:
5035 firstAccess = masm.as_ldrexh(output, ptr);
5036 if (signExtend) {
5037 masm.as_sxth(output, output, 0);
5038 }
5039 masm.as_strexh(scratch, value, ptr);
5040 break;
5041 case 4:
5042 firstAccess = masm.as_ldrex(output, ptr);
5043 masm.as_strex(scratch, value, ptr);
5044 break;
5045 }
5046 if (access) {
5047 masm.append(*access, firstAccess.getOffset());
5048 }
5049
5050 masm.as_cmp(scratch, Imm8(1));
5051 masm.as_b(&again, MacroAssembler::Equal);
5052 masm.bind(&done);
5053
5054 masm.memoryBarrierAfter(sync);
5055 }
5056
atomicExchange(Scalar::Type type,const Synchronization & sync,const Address & address,Register value,Register output)5057 void MacroAssembler::atomicExchange(Scalar::Type type,
5058 const Synchronization& sync,
5059 const Address& address, Register value,
5060 Register output) {
5061 AtomicExchange(*this, nullptr, type, sync, address, value, output);
5062 }
5063
atomicExchange(Scalar::Type type,const Synchronization & sync,const BaseIndex & address,Register value,Register output)5064 void MacroAssembler::atomicExchange(Scalar::Type type,
5065 const Synchronization& sync,
5066 const BaseIndex& address, Register value,
5067 Register output) {
5068 AtomicExchange(*this, nullptr, type, sync, address, value, output);
5069 }
5070
wasmAtomicExchange(const wasm::MemoryAccessDesc & access,const Address & mem,Register value,Register output)5071 void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
5072 const Address& mem, Register value,
5073 Register output) {
5074 AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
5075 output);
5076 }
5077
wasmAtomicExchange(const wasm::MemoryAccessDesc & access,const BaseIndex & mem,Register value,Register output)5078 void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
5079 const BaseIndex& mem, Register value,
5080 Register output) {
5081 AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
5082 output);
5083 }
5084
5085 // General algorithm:
5086 //
5087 // ... ptr, <addr> ; compute address of item
5088 // dmb
5089 // L0 ldrex* output, [ptr]
5090 // sxt* output, output, 0 ; sign-extend if applicable
5091 // OP tmp, output, value ; compute value to store
5092 // strex* tmp2, tmp, [ptr] ; tmp2 required by strex
5093 // cmp tmp2, 1
5094 // beq L0 ; failed - location is dirty, retry
5095 // dmb ; ordering barrier required
5096 //
5097 // Also see notes above at compareExchange re the barrier strategy.
5098 //
5099 // Observe that the value being operated into the memory element need
5100 // not be sign-extended because no OP will make use of bits to the
5101 // left of the bits indicated by the width of the element, and neither
5102 // output nor the bits stored are affected by OP.
5103
5104 template <typename T>
AtomicFetchOp(MacroAssembler & masm,const wasm::MemoryAccessDesc * access,Scalar::Type type,const Synchronization & sync,AtomicOp op,const Register & value,const T & mem,Register flagTemp,Register output)5105 static void AtomicFetchOp(MacroAssembler& masm,
5106 const wasm::MemoryAccessDesc* access,
5107 Scalar::Type type, const Synchronization& sync,
5108 AtomicOp op, const Register& value, const T& mem,
5109 Register flagTemp, Register output) {
5110 bool signExtend = Scalar::isSignedIntType(type);
5111 unsigned nbytes = Scalar::byteSize(type);
5112
5113 MOZ_ASSERT(nbytes <= 4);
5114 MOZ_ASSERT(flagTemp != InvalidReg);
5115 MOZ_ASSERT(output != value);
5116
5117 Label again;
5118
5119 SecondScratchRegisterScope scratch2(masm);
5120 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5121
5122 masm.memoryBarrierBefore(sync);
5123
5124 ScratchRegisterScope scratch(masm);
5125
5126 masm.bind(&again);
5127
5128 BufferOffset firstAccess;
5129 switch (nbytes) {
5130 case 1:
5131 firstAccess = masm.as_ldrexb(output, ptr);
5132 if (signExtend) {
5133 masm.as_sxtb(output, output, 0);
5134 }
5135 break;
5136 case 2:
5137 firstAccess = masm.as_ldrexh(output, ptr);
5138 if (signExtend) {
5139 masm.as_sxth(output, output, 0);
5140 }
5141 break;
5142 case 4:
5143 firstAccess = masm.as_ldrex(output, ptr);
5144 break;
5145 }
5146 if (access) {
5147 masm.append(*access, firstAccess.getOffset());
5148 }
5149
5150 switch (op) {
5151 case AtomicFetchAddOp:
5152 masm.as_add(scratch, output, O2Reg(value));
5153 break;
5154 case AtomicFetchSubOp:
5155 masm.as_sub(scratch, output, O2Reg(value));
5156 break;
5157 case AtomicFetchAndOp:
5158 masm.as_and(scratch, output, O2Reg(value));
5159 break;
5160 case AtomicFetchOrOp:
5161 masm.as_orr(scratch, output, O2Reg(value));
5162 break;
5163 case AtomicFetchXorOp:
5164 masm.as_eor(scratch, output, O2Reg(value));
5165 break;
5166 }
5167 // Rd must differ from the two other arguments to strex.
5168 switch (nbytes) {
5169 case 1:
5170 masm.as_strexb(flagTemp, scratch, ptr);
5171 break;
5172 case 2:
5173 masm.as_strexh(flagTemp, scratch, ptr);
5174 break;
5175 case 4:
5176 masm.as_strex(flagTemp, scratch, ptr);
5177 break;
5178 }
5179 masm.as_cmp(flagTemp, Imm8(1));
5180 masm.as_b(&again, MacroAssembler::Equal);
5181
5182 masm.memoryBarrierAfter(sync);
5183 }
5184
atomicFetchOp(Scalar::Type type,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register temp,Register output)5185 void MacroAssembler::atomicFetchOp(Scalar::Type type,
5186 const Synchronization& sync, AtomicOp op,
5187 Register value, const Address& mem,
5188 Register temp, Register output) {
5189 AtomicFetchOp(*this, nullptr, type, sync, op, value, mem, temp, output);
5190 }
5191
atomicFetchOp(Scalar::Type type,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register temp,Register output)5192 void MacroAssembler::atomicFetchOp(Scalar::Type type,
5193 const Synchronization& sync, AtomicOp op,
5194 Register value, const BaseIndex& mem,
5195 Register temp, Register output) {
5196 AtomicFetchOp(*this, nullptr, type, sync, op, value, mem, temp, output);
5197 }
5198
wasmAtomicFetchOp(const wasm::MemoryAccessDesc & access,AtomicOp op,Register value,const Address & mem,Register temp,Register output)5199 void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
5200 AtomicOp op, Register value,
5201 const Address& mem, Register temp,
5202 Register output) {
5203 AtomicFetchOp(*this, &access, access.type(), access.sync(), op, value, mem,
5204 temp, output);
5205 }
5206
wasmAtomicFetchOp(const wasm::MemoryAccessDesc & access,AtomicOp op,Register value,const BaseIndex & mem,Register temp,Register output)5207 void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
5208 AtomicOp op, Register value,
5209 const BaseIndex& mem, Register temp,
5210 Register output) {
5211 AtomicFetchOp(*this, &access, access.type(), access.sync(), op, value, mem,
5212 temp, output);
5213 }
5214
5215 // Uses both scratch registers, one for the address and one for a temp,
5216 // but needs two temps for strex:
5217 //
5218 // ... ptr, <addr> ; compute address of item
5219 // dmb
5220 // L0 ldrex* temp, [ptr]
5221 // OP temp, temp, value ; compute value to store
5222 // strex* temp2, temp, [ptr]
5223 // cmp temp2, 1
5224 // beq L0 ; failed - location is dirty, retry
5225 // dmb ; ordering barrier required
5226
5227 template <typename T>
AtomicEffectOp(MacroAssembler & masm,const wasm::MemoryAccessDesc * access,Scalar::Type type,const Synchronization & sync,AtomicOp op,const Register & value,const T & mem,Register flagTemp)5228 static void AtomicEffectOp(MacroAssembler& masm,
5229 const wasm::MemoryAccessDesc* access,
5230 Scalar::Type type, const Synchronization& sync,
5231 AtomicOp op, const Register& value, const T& mem,
5232 Register flagTemp) {
5233 unsigned nbytes = Scalar::byteSize(type);
5234
5235 MOZ_ASSERT(nbytes <= 4);
5236 MOZ_ASSERT(flagTemp != InvalidReg);
5237
5238 Label again;
5239
5240 SecondScratchRegisterScope scratch2(masm);
5241 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5242
5243 masm.memoryBarrierBefore(sync);
5244
5245 ScratchRegisterScope scratch(masm);
5246
5247 masm.bind(&again);
5248
5249 BufferOffset firstAccess;
5250 switch (nbytes) {
5251 case 1:
5252 firstAccess = masm.as_ldrexb(scratch, ptr);
5253 break;
5254 case 2:
5255 firstAccess = masm.as_ldrexh(scratch, ptr);
5256 break;
5257 case 4:
5258 firstAccess = masm.as_ldrex(scratch, ptr);
5259 break;
5260 }
5261 if (access) {
5262 masm.append(*access, firstAccess.getOffset());
5263 }
5264
5265 switch (op) {
5266 case AtomicFetchAddOp:
5267 masm.as_add(scratch, scratch, O2Reg(value));
5268 break;
5269 case AtomicFetchSubOp:
5270 masm.as_sub(scratch, scratch, O2Reg(value));
5271 break;
5272 case AtomicFetchAndOp:
5273 masm.as_and(scratch, scratch, O2Reg(value));
5274 break;
5275 case AtomicFetchOrOp:
5276 masm.as_orr(scratch, scratch, O2Reg(value));
5277 break;
5278 case AtomicFetchXorOp:
5279 masm.as_eor(scratch, scratch, O2Reg(value));
5280 break;
5281 }
5282 // Rd must differ from the two other arguments to strex.
5283 switch (nbytes) {
5284 case 1:
5285 masm.as_strexb(flagTemp, scratch, ptr);
5286 break;
5287 case 2:
5288 masm.as_strexh(flagTemp, scratch, ptr);
5289 break;
5290 case 4:
5291 masm.as_strex(flagTemp, scratch, ptr);
5292 break;
5293 }
5294 masm.as_cmp(flagTemp, Imm8(1));
5295 masm.as_b(&again, MacroAssembler::Equal);
5296
5297 masm.memoryBarrierAfter(sync);
5298 }
5299
wasmAtomicEffectOp(const wasm::MemoryAccessDesc & access,AtomicOp op,Register value,const Address & mem,Register temp)5300 void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
5301 AtomicOp op, Register value,
5302 const Address& mem, Register temp) {
5303 AtomicEffectOp(*this, &access, access.type(), access.sync(), op, value, mem,
5304 temp);
5305 }
5306
wasmAtomicEffectOp(const wasm::MemoryAccessDesc & access,AtomicOp op,Register value,const BaseIndex & mem,Register temp)5307 void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
5308 AtomicOp op, Register value,
5309 const BaseIndex& mem, Register temp) {
5310 AtomicEffectOp(*this, &access, access.type(), access.sync(), op, value, mem,
5311 temp);
5312 }
5313
5314 template <typename T>
WasmAtomicLoad64(MacroAssembler & masm,const wasm::MemoryAccessDesc & access,const T & mem,Register64 temp,Register64 output)5315 static void WasmAtomicLoad64(MacroAssembler& masm,
5316 const wasm::MemoryAccessDesc& access, const T& mem,
5317 Register64 temp, Register64 output) {
5318 MOZ_ASSERT(temp.low == InvalidReg && temp.high == InvalidReg);
5319 MOZ_ASSERT((output.low.code() & 1) == 0);
5320 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5321
5322 masm.memoryBarrierBefore(access.sync());
5323
5324 SecondScratchRegisterScope scratch2(masm);
5325 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5326
5327 BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
5328 masm.append(access, load.getOffset());
5329 masm.as_clrex();
5330
5331 masm.memoryBarrierAfter(access.sync());
5332 }
5333
wasmAtomicLoad64(const wasm::MemoryAccessDesc & access,const Address & mem,Register64 temp,Register64 output)5334 void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
5335 const Address& mem, Register64 temp,
5336 Register64 output) {
5337 WasmAtomicLoad64(*this, access, mem, temp, output);
5338 }
5339
wasmAtomicLoad64(const wasm::MemoryAccessDesc & access,const BaseIndex & mem,Register64 temp,Register64 output)5340 void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
5341 const BaseIndex& mem, Register64 temp,
5342 Register64 output) {
5343 WasmAtomicLoad64(*this, access, mem, temp, output);
5344 }
5345
5346 template <typename T>
CompareExchange64(MacroAssembler & masm,const wasm::MemoryAccessDesc * access,const Synchronization & sync,const T & mem,Register64 expect,Register64 replace,Register64 output)5347 static void CompareExchange64(MacroAssembler& masm,
5348 const wasm::MemoryAccessDesc* access,
5349 const Synchronization& sync, const T& mem,
5350 Register64 expect, Register64 replace,
5351 Register64 output) {
5352 MOZ_ASSERT(expect != replace && replace != output && output != expect);
5353
5354 MOZ_ASSERT((replace.low.code() & 1) == 0);
5355 MOZ_ASSERT(replace.low.code() + 1 == replace.high.code());
5356
5357 MOZ_ASSERT((output.low.code() & 1) == 0);
5358 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5359
5360 Label again;
5361 Label done;
5362
5363 SecondScratchRegisterScope scratch2(masm);
5364 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5365
5366 masm.memoryBarrierBefore(sync);
5367
5368 masm.bind(&again);
5369 BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
5370 if (access) {
5371 masm.append(*access, load.getOffset());
5372 }
5373
5374 masm.as_cmp(output.low, O2Reg(expect.low));
5375 masm.as_cmp(output.high, O2Reg(expect.high), MacroAssembler::Equal);
5376 masm.as_b(&done, MacroAssembler::NotEqual);
5377
5378 ScratchRegisterScope scratch(masm);
5379
5380 // Rd (temp) must differ from the two other arguments to strex.
5381 masm.as_strexd(scratch, replace.low, replace.high, ptr);
5382 masm.as_cmp(scratch, Imm8(1));
5383 masm.as_b(&again, MacroAssembler::Equal);
5384 masm.bind(&done);
5385
5386 masm.memoryBarrierAfter(sync);
5387 }
5388
wasmCompareExchange64(const wasm::MemoryAccessDesc & access,const Address & mem,Register64 expect,Register64 replace,Register64 output)5389 void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
5390 const Address& mem,
5391 Register64 expect,
5392 Register64 replace,
5393 Register64 output) {
5394 CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
5395 output);
5396 }
5397
wasmCompareExchange64(const wasm::MemoryAccessDesc & access,const BaseIndex & mem,Register64 expect,Register64 replace,Register64 output)5398 void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
5399 const BaseIndex& mem,
5400 Register64 expect,
5401 Register64 replace,
5402 Register64 output) {
5403 CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
5404 output);
5405 }
5406
compareExchange64(const Synchronization & sync,const Address & mem,Register64 expect,Register64 replace,Register64 output)5407 void MacroAssembler::compareExchange64(const Synchronization& sync,
5408 const Address& mem, Register64 expect,
5409 Register64 replace, Register64 output) {
5410 CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
5411 }
5412
5413 template <typename T>
WasmAtomicExchange64(MacroAssembler & masm,const wasm::MemoryAccessDesc & access,const T & mem,Register64 value,Register64 output)5414 static void WasmAtomicExchange64(MacroAssembler& masm,
5415 const wasm::MemoryAccessDesc& access,
5416 const T& mem, Register64 value,
5417 Register64 output) {
5418 MOZ_ASSERT(output != value);
5419
5420 MOZ_ASSERT((value.low.code() & 1) == 0);
5421 MOZ_ASSERT(value.low.code() + 1 == value.high.code());
5422
5423 MOZ_ASSERT((output.low.code() & 1) == 0);
5424 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5425
5426 Label again;
5427
5428 SecondScratchRegisterScope scratch2(masm);
5429 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5430
5431 masm.memoryBarrierBefore(access.sync());
5432
5433 masm.bind(&again);
5434 BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
5435 masm.append(access, load.getOffset());
5436
5437 ScratchRegisterScope scratch(masm);
5438
5439 masm.as_strexd(scratch, value.low, value.high, ptr);
5440 masm.as_cmp(scratch, Imm8(1));
5441 masm.as_b(&again, MacroAssembler::Equal);
5442
5443 masm.memoryBarrierAfter(access.sync());
5444 }
5445
wasmAtomicExchange64(const wasm::MemoryAccessDesc & access,const Address & mem,Register64 value,Register64 output)5446 void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
5447 const Address& mem, Register64 value,
5448 Register64 output) {
5449 WasmAtomicExchange64(*this, access, mem, value, output);
5450 }
5451
wasmAtomicExchange64(const wasm::MemoryAccessDesc & access,const BaseIndex & mem,Register64 value,Register64 output)5452 void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
5453 const BaseIndex& mem,
5454 Register64 value, Register64 output) {
5455 WasmAtomicExchange64(*this, access, mem, value, output);
5456 }
5457
5458 template <typename T>
WasmAtomicFetchOp64(MacroAssembler & masm,const wasm::MemoryAccessDesc & access,AtomicOp op,Register64 value,const T & mem,Register64 temp,Register64 output)5459 static void WasmAtomicFetchOp64(MacroAssembler& masm,
5460 const wasm::MemoryAccessDesc& access,
5461 AtomicOp op, Register64 value, const T& mem,
5462 Register64 temp, Register64 output) {
5463 MOZ_ASSERT(temp.low != InvalidReg && temp.high != InvalidReg);
5464 MOZ_ASSERT(output != value);
5465
5466 MOZ_ASSERT((temp.low.code() & 1) == 0);
5467 MOZ_ASSERT(temp.low.code() + 1 == temp.high.code());
5468
5469 // We could avoid this pair requirement but in that case we would end up
5470 // with two moves in the loop to preserve the loaded value in output. The
5471 // prize would be less register spilling around this op since the pair
5472 // requirement will tend to force more spilling.
5473
5474 MOZ_ASSERT((output.low.code() & 1) == 0);
5475 MOZ_ASSERT(output.low.code() + 1 == output.high.code());
5476
5477 Label again;
5478
5479 SecondScratchRegisterScope scratch2(masm);
5480 Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
5481
5482 masm.memoryBarrierBefore(access.sync());
5483
5484 masm.bind(&again);
5485 BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
5486 masm.append(access, load.getOffset());
5487 switch (op) {
5488 case AtomicFetchAddOp:
5489 masm.as_add(temp.low, output.low, O2Reg(value.low), SetCC);
5490 masm.as_adc(temp.high, output.high, O2Reg(value.high));
5491 break;
5492 case AtomicFetchSubOp:
5493 masm.as_sub(temp.low, output.low, O2Reg(value.low), SetCC);
5494 masm.as_sbc(temp.high, output.high, O2Reg(value.high));
5495 break;
5496 case AtomicFetchAndOp:
5497 masm.as_and(temp.low, output.low, O2Reg(value.low));
5498 masm.as_and(temp.high, output.high, O2Reg(value.high));
5499 break;
5500 case AtomicFetchOrOp:
5501 masm.as_orr(temp.low, output.low, O2Reg(value.low));
5502 masm.as_orr(temp.high, output.high, O2Reg(value.high));
5503 break;
5504 case AtomicFetchXorOp:
5505 masm.as_eor(temp.low, output.low, O2Reg(value.low));
5506 masm.as_eor(temp.high, output.high, O2Reg(value.high));
5507 break;
5508 }
5509
5510 ScratchRegisterScope scratch(masm);
5511
5512 // Rd (temp) must differ from the two other arguments to strex.
5513 masm.as_strexd(scratch, temp.low, temp.high, ptr);
5514 masm.as_cmp(scratch, Imm8(1));
5515 masm.as_b(&again, MacroAssembler::Equal);
5516
5517 masm.memoryBarrierAfter(access.sync());
5518 }
5519
wasmAtomicFetchOp64(const wasm::MemoryAccessDesc & access,AtomicOp op,Register64 value,const Address & mem,Register64 temp,Register64 output)5520 void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
5521 AtomicOp op, Register64 value,
5522 const Address& mem, Register64 temp,
5523 Register64 output) {
5524 WasmAtomicFetchOp64(*this, access, op, value, mem, temp, output);
5525 }
5526
wasmAtomicFetchOp64(const wasm::MemoryAccessDesc & access,AtomicOp op,Register64 value,const BaseIndex & mem,Register64 temp,Register64 output)5527 void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
5528 AtomicOp op, Register64 value,
5529 const BaseIndex& mem, Register64 temp,
5530 Register64 output) {
5531 WasmAtomicFetchOp64(*this, access, op, value, mem, temp, output);
5532 }
5533
5534 // ========================================================================
5535 // JS atomic operations.
5536
5537 template <typename T>
CompareExchangeJS(MacroAssembler & masm,Scalar::Type arrayType,const Synchronization & sync,const T & mem,Register oldval,Register newval,Register temp,AnyRegister output)5538 static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
5539 const Synchronization& sync, const T& mem,
5540 Register oldval, Register newval, Register temp,
5541 AnyRegister output) {
5542 if (arrayType == Scalar::Uint32) {
5543 masm.compareExchange(arrayType, sync, mem, oldval, newval, temp);
5544 masm.convertUInt32ToDouble(temp, output.fpu());
5545 } else {
5546 masm.compareExchange(arrayType, sync, mem, oldval, newval, output.gpr());
5547 }
5548 }
5549
compareExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const Address & mem,Register oldval,Register newval,Register temp,AnyRegister output)5550 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
5551 const Synchronization& sync,
5552 const Address& mem, Register oldval,
5553 Register newval, Register temp,
5554 AnyRegister output) {
5555 CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
5556 }
5557
compareExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const BaseIndex & mem,Register oldval,Register newval,Register temp,AnyRegister output)5558 void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
5559 const Synchronization& sync,
5560 const BaseIndex& mem, Register oldval,
5561 Register newval, Register temp,
5562 AnyRegister output) {
5563 CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
5564 }
5565
5566 template <typename T>
AtomicExchangeJS(MacroAssembler & masm,Scalar::Type arrayType,const Synchronization & sync,const T & mem,Register value,Register temp,AnyRegister output)5567 static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
5568 const Synchronization& sync, const T& mem,
5569 Register value, Register temp,
5570 AnyRegister output) {
5571 if (arrayType == Scalar::Uint32) {
5572 masm.atomicExchange(arrayType, sync, mem, value, temp);
5573 masm.convertUInt32ToDouble(temp, output.fpu());
5574 } else {
5575 masm.atomicExchange(arrayType, sync, mem, value, output.gpr());
5576 }
5577 }
5578
atomicExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const Address & mem,Register value,Register temp,AnyRegister output)5579 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
5580 const Synchronization& sync,
5581 const Address& mem, Register value,
5582 Register temp, AnyRegister output) {
5583 AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
5584 }
5585
atomicExchangeJS(Scalar::Type arrayType,const Synchronization & sync,const BaseIndex & mem,Register value,Register temp,AnyRegister output)5586 void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
5587 const Synchronization& sync,
5588 const BaseIndex& mem, Register value,
5589 Register temp, AnyRegister output) {
5590 AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
5591 }
5592
5593 template <typename T>
AtomicFetchOpJS(MacroAssembler & masm,Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const T & mem,Register temp1,Register temp2,AnyRegister output)5594 static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
5595 const Synchronization& sync, AtomicOp op,
5596 Register value, const T& mem, Register temp1,
5597 Register temp2, AnyRegister output) {
5598 if (arrayType == Scalar::Uint32) {
5599 masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1);
5600 masm.convertUInt32ToDouble(temp1, output.fpu());
5601 } else {
5602 masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr());
5603 }
5604 }
5605
atomicFetchOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register temp1,Register temp2,AnyRegister output)5606 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
5607 const Synchronization& sync, AtomicOp op,
5608 Register value, const Address& mem,
5609 Register temp1, Register temp2,
5610 AnyRegister output) {
5611 AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
5612 }
5613
atomicFetchOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register temp1,Register temp2,AnyRegister output)5614 void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
5615 const Synchronization& sync, AtomicOp op,
5616 Register value, const BaseIndex& mem,
5617 Register temp1, Register temp2,
5618 AnyRegister output) {
5619 AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
5620 }
5621
atomicEffectOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const BaseIndex & mem,Register temp)5622 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
5623 const Synchronization& sync, AtomicOp op,
5624 Register value, const BaseIndex& mem,
5625 Register temp) {
5626 AtomicEffectOp(*this, nullptr, arrayType, sync, op, value, mem, temp);
5627 }
5628
atomicEffectOpJS(Scalar::Type arrayType,const Synchronization & sync,AtomicOp op,Register value,const Address & mem,Register temp)5629 void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
5630 const Synchronization& sync, AtomicOp op,
5631 Register value, const Address& mem,
5632 Register temp) {
5633 AtomicEffectOp(*this, nullptr, arrayType, sync, op, value, mem, temp);
5634 }
5635
5636 // ========================================================================
5637 // Convert floating point.
5638
convertUInt64ToDoubleNeedsTemp()5639 bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
5640
convertUInt64ToDouble(Register64 src,FloatRegister dest,Register temp)5641 void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
5642 Register temp) {
5643 MOZ_ASSERT(temp == Register::Invalid());
5644 ScratchDoubleScope scratchDouble(*this);
5645
5646 convertUInt32ToDouble(src.high, dest);
5647 {
5648 ScratchRegisterScope scratch(*this);
5649 movePtr(ImmPtr(&TO_DOUBLE_HIGH_SCALE), scratch);
5650 ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), scratchDouble);
5651 }
5652 mulDouble(scratchDouble, dest);
5653 convertUInt32ToDouble(src.low, scratchDouble);
5654 addDouble(scratchDouble, dest);
5655 }
5656
5657 extern "C" {
5658 extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int);
5659 extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int);
5660 }
5661
EmitRemainderOrQuotient(bool isRemainder,MacroAssembler & masm,Register rhs,Register lhsOutput,bool isSigned,const LiveRegisterSet & volatileLiveRegs)5662 inline void EmitRemainderOrQuotient(bool isRemainder, MacroAssembler& masm,
5663 Register rhs, Register lhsOutput,
5664 bool isSigned,
5665 const LiveRegisterSet& volatileLiveRegs) {
5666 // Currently this helper can't handle this situation.
5667 MOZ_ASSERT(lhsOutput != rhs);
5668
5669 if (HasIDIV()) {
5670 if (isRemainder) {
5671 masm.remainder32(rhs, lhsOutput, isSigned);
5672 } else {
5673 masm.quotient32(rhs, lhsOutput, isSigned);
5674 }
5675 } else {
5676 // Ensure that the output registers are saved and restored properly,
5677 MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal0));
5678 MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal1));
5679
5680 masm.PushRegsInMask(volatileLiveRegs);
5681 {
5682 ScratchRegisterScope scratch(masm);
5683 masm.setupUnalignedABICall(scratch);
5684 }
5685 masm.passABIArg(lhsOutput);
5686 masm.passABIArg(rhs);
5687 masm.callWithABI(isSigned ? JS_FUNC_TO_DATA_PTR(void*, __aeabi_uidivmod)
5688 : JS_FUNC_TO_DATA_PTR(void*, __aeabi_idivmod),
5689 MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
5690 if (isRemainder) {
5691 masm.mov(ReturnRegVal1, lhsOutput);
5692 } else {
5693 masm.mov(ReturnRegVal0, lhsOutput);
5694 }
5695
5696 LiveRegisterSet ignore;
5697 ignore.add(lhsOutput);
5698 masm.PopRegsInMaskIgnore(volatileLiveRegs, ignore);
5699 }
5700 }
5701
flexibleQuotient32(Register rhs,Register srcDest,bool isUnsigned,const LiveRegisterSet & volatileLiveRegs)5702 void MacroAssembler::flexibleQuotient32(
5703 Register rhs, Register srcDest, bool isUnsigned,
5704 const LiveRegisterSet& volatileLiveRegs) {
5705 EmitRemainderOrQuotient(false, *this, rhs, srcDest, isUnsigned,
5706 volatileLiveRegs);
5707 }
5708
flexibleRemainder32(Register rhs,Register srcDest,bool isUnsigned,const LiveRegisterSet & volatileLiveRegs)5709 void MacroAssembler::flexibleRemainder32(
5710 Register rhs, Register srcDest, bool isUnsigned,
5711 const LiveRegisterSet& volatileLiveRegs) {
5712 EmitRemainderOrQuotient(true, *this, rhs, srcDest, isUnsigned,
5713 volatileLiveRegs);
5714 }
5715
flexibleDivMod32(Register rhs,Register lhsOutput,Register remOutput,bool isUnsigned,const LiveRegisterSet & volatileLiveRegs)5716 void MacroAssembler::flexibleDivMod32(Register rhs, Register lhsOutput,
5717 Register remOutput, bool isUnsigned,
5718 const LiveRegisterSet& volatileLiveRegs) {
5719 // Currently this helper can't handle this situation.
5720 MOZ_ASSERT(lhsOutput != rhs);
5721
5722 if (HasIDIV()) {
5723 mov(lhsOutput, remOutput);
5724 remainder32(rhs, remOutput, isUnsigned);
5725 quotient32(rhs, lhsOutput, isUnsigned);
5726 } else {
5727 // Ensure that the output registers are saved and restored properly,
5728 MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal0));
5729 MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal1));
5730 PushRegsInMask(volatileLiveRegs);
5731
5732 {
5733 ScratchRegisterScope scratch(*this);
5734 setupUnalignedABICall(scratch);
5735 }
5736 passABIArg(lhsOutput);
5737 passABIArg(rhs);
5738 callWithABI(isUnsigned ? JS_FUNC_TO_DATA_PTR(void*, __aeabi_uidivmod)
5739 : JS_FUNC_TO_DATA_PTR(void*, __aeabi_idivmod),
5740 MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
5741 moveRegPair(ReturnRegVal0, ReturnRegVal1, lhsOutput, remOutput);
5742
5743 LiveRegisterSet ignore;
5744 ignore.add(remOutput);
5745 ignore.add(lhsOutput);
5746 PopRegsInMaskIgnore(volatileLiveRegs, ignore);
5747 }
5748 }
5749
moveNearAddressWithPatch(Register dest)5750 CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
5751 return movWithPatch(ImmPtr(nullptr), dest);
5752 }
5753
patchNearAddressMove(CodeLocationLabel loc,CodeLocationLabel target)5754 void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
5755 CodeLocationLabel target) {
5756 PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
5757 }
5758
5759 // ========================================================================
5760 // Spectre Mitigations.
5761
speculationBarrier()5762 void MacroAssembler::speculationBarrier() {
5763 // Spectre mitigation recommended by ARM for cases where csel/cmov cannot be
5764 // used.
5765 as_csdb();
5766 }
5767
floorFloat32ToInt32(FloatRegister src,Register dest,Label * fail)5768 void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
5769 Label* fail) {
5770 floorf(src, dest, fail);
5771 }
5772
floorDoubleToInt32(FloatRegister src,Register dest,Label * fail)5773 void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
5774 Label* fail) {
5775 floor(src, dest, fail);
5776 }
5777
ceilFloat32ToInt32(FloatRegister src,Register dest,Label * fail)5778 void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
5779 Label* fail) {
5780 ceilf(src, dest, fail);
5781 }
5782
ceilDoubleToInt32(FloatRegister src,Register dest,Label * fail)5783 void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
5784 Label* fail) {
5785 ceil(src, dest, fail);
5786 }
5787
roundFloat32ToInt32(FloatRegister src,Register dest,FloatRegister temp,Label * fail)5788 void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
5789 FloatRegister temp, Label* fail) {
5790 roundf(src, dest, fail, temp);
5791 }
5792
roundDoubleToInt32(FloatRegister src,Register dest,FloatRegister temp,Label * fail)5793 void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
5794 FloatRegister temp, Label* fail) {
5795 round(src, dest, fail, temp);
5796 }
5797
5798 //}}} check_macroassembler_style
5799
wasmTruncateToInt32(FloatRegister input,Register output,MIRType fromType,bool isUnsigned,bool isSaturating,Label * oolEntry)5800 void MacroAssemblerARM::wasmTruncateToInt32(FloatRegister input,
5801 Register output, MIRType fromType,
5802 bool isUnsigned, bool isSaturating,
5803 Label* oolEntry) {
5804 ScratchDoubleScope scratchScope(asMasm());
5805 ScratchRegisterScope scratchReg(asMasm());
5806 FloatRegister scratch = scratchScope.uintOverlay();
5807
5808 // ARM conversion instructions clamp the value to ensure it fits within the
5809 // target's type bounds, so every time we see those, we need to check the
5810 // input. A NaN check is not necessary because NaN is converted to zero and
5811 // on a zero result we branch out of line to do further processing anyway.
5812 if (isUnsigned) {
5813 if (fromType == MIRType::Double) {
5814 ma_vcvt_F64_U32(input, scratch);
5815 } else if (fromType == MIRType::Float32) {
5816 ma_vcvt_F32_U32(input, scratch);
5817 } else {
5818 MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
5819 }
5820
5821 ma_vxfer(scratch, output);
5822
5823 if (!isSaturating) {
5824 // int32_t(UINT32_MAX) == -1.
5825 ma_cmp(output, Imm32(-1), scratchReg);
5826 as_cmp(output, Imm8(0), Assembler::NotEqual);
5827 ma_b(oolEntry, Assembler::Equal);
5828 }
5829
5830 return;
5831 }
5832
5833 // vcvt* converts NaN into 0, so check for NaNs here.
5834 if (!isSaturating) {
5835 if (fromType == MIRType::Double) {
5836 asMasm().compareDouble(input, input);
5837 } else if (fromType == MIRType::Float32) {
5838 asMasm().compareFloat(input, input);
5839 } else {
5840 MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
5841 }
5842
5843 ma_b(oolEntry, Assembler::VFP_Unordered);
5844 }
5845
5846 scratch = scratchScope.sintOverlay();
5847
5848 if (fromType == MIRType::Double) {
5849 ma_vcvt_F64_I32(input, scratch);
5850 } else if (fromType == MIRType::Float32) {
5851 ma_vcvt_F32_I32(input, scratch);
5852 } else {
5853 MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
5854 }
5855
5856 ma_vxfer(scratch, output);
5857
5858 if (!isSaturating) {
5859 ma_cmp(output, Imm32(INT32_MAX), scratchReg);
5860 ma_cmp(output, Imm32(INT32_MIN), scratchReg, Assembler::NotEqual);
5861 ma_b(oolEntry, Assembler::Equal);
5862 }
5863 }
5864
outOfLineWasmTruncateToIntCheck(FloatRegister input,MIRType fromType,MIRType toType,TruncFlags flags,Label * rejoin,wasm::BytecodeOffset trapOffset)5865 void MacroAssemblerARM::outOfLineWasmTruncateToIntCheck(
5866 FloatRegister input, MIRType fromType, MIRType toType, TruncFlags flags,
5867 Label* rejoin, wasm::BytecodeOffset trapOffset) {
5868 // On ARM, saturating truncation codegen handles saturating itself rather
5869 // than relying on out-of-line fixup code.
5870 if (flags & TRUNC_SATURATING) {
5871 return;
5872 }
5873
5874 bool isUnsigned = flags & TRUNC_UNSIGNED;
5875 ScratchDoubleScope scratchScope(asMasm());
5876 FloatRegister scratch;
5877
5878 // Eagerly take care of NaNs.
5879 Label inputIsNaN;
5880 if (fromType == MIRType::Double) {
5881 asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
5882 &inputIsNaN);
5883 } else if (fromType == MIRType::Float32) {
5884 asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
5885 } else {
5886 MOZ_CRASH("unexpected type in visitOutOfLineWasmTruncateCheck");
5887 }
5888
5889 // Handle special values.
5890 Label fail;
5891
5892 // By default test for the following inputs and bail:
5893 // signed: ] -Inf, INTXX_MIN - 1.0 ] and [ INTXX_MAX + 1.0 : +Inf [
5894 // unsigned: ] -Inf, -1.0 ] and [ UINTXX_MAX + 1.0 : +Inf [
5895 // Note: we cannot always represent those exact values. As a result
5896 // this changes the actual comparison a bit.
5897 double minValue, maxValue;
5898 Assembler::DoubleCondition minCond = Assembler::DoubleLessThanOrEqual;
5899 Assembler::DoubleCondition maxCond = Assembler::DoubleGreaterThanOrEqual;
5900 if (toType == MIRType::Int64) {
5901 if (isUnsigned) {
5902 minValue = -1;
5903 maxValue = double(UINT64_MAX) + 1.0;
5904 } else {
5905 // In the float32/double range there exists no value between
5906 // INT64_MIN and INT64_MIN - 1.0. Making INT64_MIN the lower-bound.
5907 minValue = double(INT64_MIN);
5908 minCond = Assembler::DoubleLessThan;
5909 maxValue = double(INT64_MAX) + 1.0;
5910 }
5911 } else {
5912 if (isUnsigned) {
5913 minValue = -1;
5914 maxValue = double(UINT32_MAX) + 1.0;
5915 } else {
5916 if (fromType == MIRType::Float32) {
5917 // In the float32 range there exists no value between
5918 // INT32_MIN and INT32_MIN - 1.0. Making INT32_MIN the lower-bound.
5919 minValue = double(INT32_MIN);
5920 minCond = Assembler::DoubleLessThan;
5921 } else {
5922 minValue = double(INT32_MIN) - 1.0;
5923 }
5924 maxValue = double(INT32_MAX) + 1.0;
5925 }
5926 }
5927
5928 if (fromType == MIRType::Double) {
5929 scratch = scratchScope.doubleOverlay();
5930 asMasm().loadConstantDouble(minValue, scratch);
5931 asMasm().branchDouble(minCond, input, scratch, &fail);
5932
5933 asMasm().loadConstantDouble(maxValue, scratch);
5934 asMasm().branchDouble(maxCond, input, scratch, &fail);
5935 } else {
5936 MOZ_ASSERT(fromType == MIRType::Float32);
5937 scratch = scratchScope.singleOverlay();
5938 asMasm().loadConstantFloat32(float(minValue), scratch);
5939 asMasm().branchFloat(minCond, input, scratch, &fail);
5940
5941 asMasm().loadConstantFloat32(float(maxValue), scratch);
5942 asMasm().branchFloat(maxCond, input, scratch, &fail);
5943 }
5944
5945 // We had an actual correct value, get back to where we were.
5946 ma_b(rejoin);
5947
5948 // Handle errors.
5949 bind(&fail);
5950 asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
5951
5952 bind(&inputIsNaN);
5953 asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
5954 }
5955
wasmLoadImpl(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,AnyRegister output,Register64 out64)5956 void MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
5957 Register memoryBase, Register ptr,
5958 Register ptrScratch, AnyRegister output,
5959 Register64 out64) {
5960 MOZ_ASSERT(ptr == ptrScratch);
5961
5962 uint32_t offset = access.offset();
5963 MOZ_ASSERT(offset < wasm::MaxOffsetGuardLimit);
5964
5965 Scalar::Type type = access.type();
5966
5967 // Maybe add the offset.
5968 if (offset || type == Scalar::Int64) {
5969 ScratchRegisterScope scratch(asMasm());
5970 if (offset) {
5971 ma_add(Imm32(offset), ptr, scratch);
5972 }
5973 }
5974
5975 bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 ||
5976 type == Scalar::Int32 || type == Scalar::Int64;
5977 unsigned byteSize = access.byteSize();
5978
5979 asMasm().memoryBarrierBefore(access.sync());
5980
5981 BufferOffset load;
5982 if (out64 != Register64::Invalid()) {
5983 if (type == Scalar::Int64) {
5984 static_assert(INT64LOW_OFFSET == 0);
5985
5986 load = ma_dataTransferN(IsLoad, 32, /* signed = */ false, memoryBase, ptr,
5987 out64.low);
5988 append(access, load.getOffset());
5989
5990 as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
5991
5992 load =
5993 ma_dataTransferN(IsLoad, 32, isSigned, memoryBase, ptr, out64.high);
5994 append(access, load.getOffset());
5995 } else {
5996 load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, memoryBase, ptr,
5997 out64.low);
5998 append(access, load.getOffset());
5999
6000 if (isSigned) {
6001 ma_asr(Imm32(31), out64.low, out64.high);
6002 } else {
6003 ma_mov(Imm32(0), out64.high);
6004 }
6005 }
6006 } else {
6007 bool isFloat = output.isFloat();
6008 if (isFloat) {
6009 MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle());
6010 ScratchRegisterScope scratch(asMasm());
6011 ma_add(memoryBase, ptr, scratch);
6012
6013 // See HandleUnalignedTrap() in WasmSignalHandler.cpp. We depend on this
6014 // being a single, unconditional VLDR with a base pointer other than PC.
6015 load = ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), output.fpu());
6016 append(access, load.getOffset());
6017 } else {
6018 load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, memoryBase, ptr,
6019 output.gpr());
6020 append(access, load.getOffset());
6021 }
6022 }
6023
6024 asMasm().memoryBarrierAfter(access.sync());
6025 }
6026
wasmStoreImpl(const wasm::MemoryAccessDesc & access,AnyRegister value,Register64 val64,Register memoryBase,Register ptr,Register ptrScratch)6027 void MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
6028 AnyRegister value, Register64 val64,
6029 Register memoryBase, Register ptr,
6030 Register ptrScratch) {
6031 MOZ_ASSERT(ptr == ptrScratch);
6032
6033 uint32_t offset = access.offset();
6034 MOZ_ASSERT(offset < wasm::MaxOffsetGuardLimit);
6035
6036 unsigned byteSize = access.byteSize();
6037 Scalar::Type type = access.type();
6038
6039 // Maybe add the offset.
6040 if (offset || type == Scalar::Int64) {
6041 ScratchRegisterScope scratch(asMasm());
6042 if (offset) {
6043 ma_add(Imm32(offset), ptr, scratch);
6044 }
6045 }
6046
6047 asMasm().memoryBarrierAfter(access.sync());
6048
6049 BufferOffset store;
6050 if (type == Scalar::Int64) {
6051 static_assert(INT64LOW_OFFSET == 0);
6052
6053 store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false,
6054 memoryBase, ptr, val64.low);
6055 append(access, store.getOffset());
6056
6057 as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
6058
6059 store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true,
6060 memoryBase, ptr, val64.high);
6061 append(access, store.getOffset());
6062 } else {
6063 if (value.isFloat()) {
6064 ScratchRegisterScope scratch(asMasm());
6065 FloatRegister val = value.fpu();
6066 MOZ_ASSERT((byteSize == 4) == val.isSingle());
6067 ma_add(memoryBase, ptr, scratch);
6068
6069 // See HandleUnalignedTrap() in WasmSignalHandler.cpp. We depend on this
6070 // being a single, unconditional VLDR with a base pointer other than PC.
6071 store = ma_vstr(val, Operand(Address(scratch, 0)).toVFPAddr());
6072 append(access, store.getOffset());
6073 } else {
6074 bool isSigned = type == Scalar::Uint32 ||
6075 type == Scalar::Int32; // see AsmJSStoreHeap;
6076 Register val = value.gpr();
6077
6078 store = ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned,
6079 memoryBase, ptr, val);
6080 append(access, store.getOffset());
6081 }
6082 }
6083
6084 asMasm().memoryBarrierAfter(access.sync());
6085 }
6086
wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc & access,Register memoryBase,Register ptr,Register ptrScratch,AnyRegister outAny,Register64 out64,Register tmp,Register tmp2,Register tmp3)6087 void MacroAssemblerARM::wasmUnalignedLoadImpl(
6088 const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
6089 Register ptrScratch, AnyRegister outAny, Register64 out64, Register tmp,
6090 Register tmp2, Register tmp3) {
6091 MOZ_ASSERT(ptr == ptrScratch);
6092 MOZ_ASSERT(tmp != ptr);
6093 MOZ_ASSERT(!Assembler::SupportsFastUnalignedAccesses());
6094
6095 uint32_t offset = access.offset();
6096 MOZ_ASSERT(offset < wasm::MaxOffsetGuardLimit);
6097
6098 if (offset) {
6099 ScratchRegisterScope scratch(asMasm());
6100 ma_add(Imm32(offset), ptr, scratch);
6101 }
6102
6103 // Add memoryBase to ptr, so we can use base+index addressing in the byte
6104 // loads.
6105 ma_add(memoryBase, ptr);
6106
6107 unsigned byteSize = access.byteSize();
6108 MOZ_ASSERT(byteSize == 8 || byteSize == 4 || byteSize == 2);
6109
6110 Scalar::Type type = access.type();
6111 bool isSigned =
6112 type == Scalar::Int16 || type == Scalar::Int32 || type == Scalar::Int64;
6113
6114 // If it's a two-word load we must load the high word first to get signal
6115 // handling right.
6116
6117 asMasm().memoryBarrierBefore(access.sync());
6118
6119 switch (access.type()) {
6120 case Scalar::Float32: {
6121 MOZ_ASSERT(byteSize == 4);
6122 MOZ_ASSERT(tmp2 != Register::Invalid() && tmp3 == Register::Invalid());
6123 MOZ_ASSERT(outAny.fpu().isSingle());
6124 emitUnalignedLoad(&access, /*signed*/ false, /*size*/ 4, ptr, tmp, tmp2);
6125 ma_vxfer(tmp2, outAny.fpu());
6126 break;
6127 }
6128 case Scalar::Float64: {
6129 MOZ_ASSERT(byteSize == 8);
6130 MOZ_ASSERT(tmp2 != Register::Invalid() && tmp3 != Register::Invalid());
6131 MOZ_ASSERT(outAny.fpu().isDouble());
6132 emitUnalignedLoad(&access, /*signed=*/false, /*size=*/4, ptr, tmp, tmp3,
6133 /*offset=*/4);
6134 emitUnalignedLoad(nullptr, /*signed=*/false, /*size=*/4, ptr, tmp, tmp2);
6135 ma_vxfer(tmp2, tmp3, outAny.fpu());
6136 break;
6137 }
6138 case Scalar::Int64: {
6139 MOZ_ASSERT(byteSize == 8);
6140 MOZ_ASSERT(tmp2 == Register::Invalid() && tmp3 == Register::Invalid());
6141 MOZ_ASSERT(out64.high != ptr);
6142 emitUnalignedLoad(&access, /*signed=*/false, /*size=*/4, ptr, tmp,
6143 out64.high, /*offset=*/4);
6144 emitUnalignedLoad(nullptr, /*signed=*/false, /*size=*/4, ptr, tmp,
6145 out64.low);
6146 break;
6147 }
6148 case Scalar::Int16:
6149 case Scalar::Uint16:
6150 case Scalar::Int32:
6151 case Scalar::Uint32: {
6152 MOZ_ASSERT(byteSize <= 4);
6153 MOZ_ASSERT(tmp2 == Register::Invalid() && tmp3 == Register::Invalid());
6154 if (out64 != Register64::Invalid()) {
6155 emitUnalignedLoad(&access, isSigned, byteSize, ptr, tmp, out64.low);
6156 if (isSigned) {
6157 ma_asr(Imm32(31), out64.low, out64.high);
6158 } else {
6159 ma_mov(Imm32(0), out64.high);
6160 }
6161 } else {
6162 emitUnalignedLoad(&access, isSigned, byteSize, ptr, tmp, outAny.gpr());
6163 }
6164 break;
6165 }
6166 case Scalar::Int8:
6167 case Scalar::Uint8:
6168 default: {
6169 MOZ_CRASH("Bad type");
6170 }
6171 }
6172
6173 asMasm().memoryBarrierAfter(access.sync());
6174 }
6175
wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc & access,FloatRegister floatValue,Register64 val64,Register memoryBase,Register ptr,Register ptrScratch,Register valOrTmp)6176 void MacroAssemblerARM::wasmUnalignedStoreImpl(
6177 const wasm::MemoryAccessDesc& access, FloatRegister floatValue,
6178 Register64 val64, Register memoryBase, Register ptr, Register ptrScratch,
6179 Register valOrTmp) {
6180 MOZ_ASSERT(ptr == ptrScratch);
6181 // They can't both be valid, but they can both be invalid.
6182 MOZ_ASSERT(floatValue.isInvalid() || val64 == Register64::Invalid());
6183 // Don't try extremely clever optimizations.
6184 MOZ_ASSERT_IF(val64 != Register64::Invalid(),
6185 valOrTmp != val64.high && valOrTmp != val64.low);
6186
6187 uint32_t offset = access.offset();
6188 MOZ_ASSERT(offset < wasm::MaxOffsetGuardLimit);
6189
6190 unsigned byteSize = access.byteSize();
6191 MOZ_ASSERT(byteSize == 8 || byteSize == 4 || byteSize == 2);
6192
6193 if (offset) {
6194 ScratchRegisterScope scratch(asMasm());
6195 ma_add(Imm32(offset), ptr, scratch);
6196 }
6197
6198 // Add memoryBase to ptr, so we can use base+index addressing in the byte
6199 // loads.
6200 ma_add(memoryBase, ptr);
6201
6202 asMasm().memoryBarrierAfter(access.sync());
6203
6204 // If it's a two-word store we must store the high word first to get signal
6205 // handling right.
6206
6207 if (val64 != Register64::Invalid()) {
6208 if (byteSize == 8) {
6209 emitUnalignedStore(&access, /*size=*/4, ptr, val64.high, /*offset=*/4);
6210 emitUnalignedStore(nullptr, /*size=*/4, ptr, val64.low);
6211 } else {
6212 emitUnalignedStore(&access, byteSize, ptr, val64.low);
6213 }
6214 } else if (!floatValue.isInvalid()) {
6215 if (floatValue.isDouble()) {
6216 MOZ_ASSERT(byteSize == 8);
6217 ScratchRegisterScope scratch(asMasm());
6218 ma_vxfer(floatValue, scratch, valOrTmp);
6219 emitUnalignedStore(&access, /*size=*/4, ptr, valOrTmp, /*offset=*/4);
6220 emitUnalignedStore(nullptr, /*size=*/4, ptr, scratch);
6221 } else {
6222 MOZ_ASSERT(byteSize == 4);
6223 ma_vxfer(floatValue, valOrTmp);
6224 emitUnalignedStore(&access, /*size=*/4, ptr, valOrTmp);
6225 }
6226 } else {
6227 MOZ_ASSERT(byteSize == 2 || byteSize == 4);
6228 emitUnalignedStore(&access, byteSize, ptr, valOrTmp);
6229 }
6230
6231 asMasm().memoryBarrierAfter(access.sync());
6232 }
6233
emitUnalignedLoad(const wasm::MemoryAccessDesc * access,bool isSigned,unsigned byteSize,Register ptr,Register tmp,Register dest,unsigned offset)6234 void MacroAssemblerARM::emitUnalignedLoad(const wasm::MemoryAccessDesc* access,
6235 bool isSigned, unsigned byteSize,
6236 Register ptr, Register tmp,
6237 Register dest, unsigned offset) {
6238 // Preconditions.
6239 MOZ_ASSERT(ptr != tmp);
6240 MOZ_ASSERT(ptr != dest);
6241 MOZ_ASSERT(tmp != dest);
6242 MOZ_ASSERT(byteSize == 2 || byteSize == 4);
6243 MOZ_ASSERT(offset == 0 || offset == 4);
6244
6245 // The trap metadata is only valid for the first instruction, so we must
6246 // make the first instruction fault if any of them is going to fault. Hence
6247 // byte loads must be issued from high addresses toward low addresses (or we
6248 // must emit metadata for each load).
6249 //
6250 // So for a four-byte load from address x we will emit an eight-instruction
6251 // sequence:
6252 //
6253 // ldrsb [x+3], tmp // note signed load *if appropriate*
6254 // lsl dest, tmp lsl 24 // move high byte + sign bits into place;
6255 // // clear low bits
6256 // ldrb [x+2], tmp // note unsigned load
6257 // or dest, dest, tmp lsl 16 // add another byte
6258 // ldrb [x+1], tmp // ...
6259 // or dest, dest, tmp lsl 8
6260 // ldrb [x], tmp
6261 // or dest, dest, tmp
6262
6263 ScratchRegisterScope scratch(asMasm());
6264
6265 int i = byteSize - 1;
6266
6267 BufferOffset load = ma_dataTransferN(IsLoad, 8, isSigned, ptr,
6268 Imm32(offset + i), tmp, scratch);
6269 if (access) {
6270 append(*access, load.getOffset());
6271 }
6272 ma_lsl(Imm32(8 * i), tmp, dest);
6273 --i;
6274
6275 while (i >= 0) {
6276 ma_dataTransferN(IsLoad, 8, /*signed=*/false, ptr, Imm32(offset + i), tmp,
6277 scratch);
6278 as_orr(dest, dest, lsl(tmp, 8 * i));
6279 --i;
6280 }
6281 }
6282
emitUnalignedStore(const wasm::MemoryAccessDesc * access,unsigned byteSize,Register ptr,Register val,unsigned offset)6283 void MacroAssemblerARM::emitUnalignedStore(const wasm::MemoryAccessDesc* access,
6284 unsigned byteSize, Register ptr,
6285 Register val, unsigned offset) {
6286 // Preconditions.
6287 MOZ_ASSERT(ptr != val);
6288 MOZ_ASSERT(byteSize == 2 || byteSize == 4);
6289 MOZ_ASSERT(offset == 0 || offset == 4);
6290
6291 // See comments above. Here an additional motivation is that no side
6292 // effects should be observed if any of the stores would fault, so we *must*
6293 // go high-to-low, we can't emit metadata for individual stores in
6294 // low-to-high order.
6295 //
6296 // For a four-byte store to address x we will emit a seven-instruction
6297 // sequence:
6298 //
6299 // lsr scratch, val, 24
6300 // strb [x+3], scratch
6301 // lsr scratch, val, 16
6302 // strb [x+2], scratch
6303 // lsr scratch, val, 8
6304 // strb [x+1], scratch
6305 // strb [x], val
6306
6307 // `val` may be scratch in the case when we store doubles.
6308 SecondScratchRegisterScope scratch(asMasm());
6309
6310 for (int i = byteSize - 1; i > 0; i--) {
6311 ma_lsr(Imm32(i * 8), val, scratch);
6312 // Use as_dtr directly to avoid needing another scratch register; we can
6313 // do this because `offset` is known to be small.
6314 BufferOffset store = as_dtr(IsStore, 8, Offset, scratch,
6315 DTRAddr(ptr, DtrOffImm(offset + i)));
6316 if (i == (int)byteSize - 1 && access) {
6317 append(*access, store.getOffset());
6318 }
6319 }
6320 as_dtr(IsStore, 8, Offset, val, DTRAddr(ptr, DtrOffImm(offset)));
6321 }
6322