1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_arm64_MacroAssembler_arm64_inl_h
8 #define jit_arm64_MacroAssembler_arm64_inl_h
9
10 #include "jit/arm64/MacroAssembler-arm64.h"
11
12 namespace js {
13 namespace jit {
14
15 //{{{ check_macroassembler_style
16
move64(Register64 src,Register64 dest)17 void MacroAssembler::move64(Register64 src, Register64 dest) {
18 Mov(ARMRegister(dest.reg, 64), ARMRegister(src.reg, 64));
19 }
20
move64(Imm64 imm,Register64 dest)21 void MacroAssembler::move64(Imm64 imm, Register64 dest) {
22 Mov(ARMRegister(dest.reg, 64), imm.value);
23 }
24
moveFloat32ToGPR(FloatRegister src,Register dest)25 void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
26 Fmov(ARMRegister(dest, 32), ARMFPRegister(src, 32));
27 }
28
moveGPRToFloat32(Register src,FloatRegister dest)29 void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
30 Fmov(ARMFPRegister(dest, 32), ARMRegister(src, 32));
31 }
32
move8SignExtend(Register src,Register dest)33 void MacroAssembler::move8SignExtend(Register src, Register dest) {
34 Sxtb(ARMRegister(dest, 32), ARMRegister(src, 32));
35 }
36
move16SignExtend(Register src,Register dest)37 void MacroAssembler::move16SignExtend(Register src, Register dest) {
38 Sxth(ARMRegister(dest, 32), ARMRegister(src, 32));
39 }
40
moveDoubleToGPR64(FloatRegister src,Register64 dest)41 void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
42 Fmov(ARMRegister(dest.reg, 64), ARMFPRegister(src, 64));
43 }
44
moveGPR64ToDouble(Register64 src,FloatRegister dest)45 void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
46 Fmov(ARMFPRegister(dest, 64), ARMRegister(src.reg, 64));
47 }
48
move64To32(Register64 src,Register dest)49 void MacroAssembler::move64To32(Register64 src, Register dest) {
50 Mov(ARMRegister(dest, 32), ARMRegister(src.reg, 32));
51 }
52
move32To64ZeroExtend(Register src,Register64 dest)53 void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
54 Mov(ARMRegister(dest.reg, 32), ARMRegister(src, 32));
55 }
56
move8To64SignExtend(Register src,Register64 dest)57 void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
58 Sxtb(ARMRegister(dest.reg, 64), ARMRegister(src, 32));
59 }
60
move16To64SignExtend(Register src,Register64 dest)61 void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
62 Sxth(ARMRegister(dest.reg, 64), ARMRegister(src, 32));
63 }
64
move32To64SignExtend(Register src,Register64 dest)65 void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
66 Sxtw(ARMRegister(dest.reg, 64), ARMRegister(src, 32));
67 }
68
move32SignExtendToPtr(Register src,Register dest)69 void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
70 Sxtw(ARMRegister(dest, 64), ARMRegister(src, 32));
71 }
72
move32ZeroExtendToPtr(Register src,Register dest)73 void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
74 Mov(ARMRegister(dest, 32), ARMRegister(src, 32));
75 }
76
77 // ===============================================================
78 // Load instructions
79
load32SignExtendToPtr(const Address & src,Register dest)80 void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
81 load32(src, dest);
82 move32To64SignExtend(dest, Register64(dest));
83 }
84
loadAbiReturnAddress(Register dest)85 void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(lr, dest); }
86
87 // ===============================================================
88 // Logical instructions
89
not32(Register reg)90 void MacroAssembler::not32(Register reg) {
91 Orn(ARMRegister(reg, 32), vixl::wzr, ARMRegister(reg, 32));
92 }
93
notPtr(Register reg)94 void MacroAssembler::notPtr(Register reg) {
95 Orn(ARMRegister(reg, 64), vixl::xzr, ARMRegister(reg, 64));
96 }
97
and32(Register src,Register dest)98 void MacroAssembler::and32(Register src, Register dest) {
99 And(ARMRegister(dest, 32), ARMRegister(dest, 32),
100 Operand(ARMRegister(src, 32)));
101 }
102
and32(Imm32 imm,Register dest)103 void MacroAssembler::and32(Imm32 imm, Register dest) {
104 And(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
105 }
106
and32(Imm32 imm,Register src,Register dest)107 void MacroAssembler::and32(Imm32 imm, Register src, Register dest) {
108 And(ARMRegister(dest, 32), ARMRegister(src, 32), Operand(imm.value));
109 }
110
and32(Imm32 imm,const Address & dest)111 void MacroAssembler::and32(Imm32 imm, const Address& dest) {
112 vixl::UseScratchRegisterScope temps(this);
113 const ARMRegister scratch32 = temps.AcquireW();
114 MOZ_ASSERT(scratch32.asUnsized() != dest.base);
115 load32(dest, scratch32.asUnsized());
116 And(scratch32, scratch32, Operand(imm.value));
117 store32(scratch32.asUnsized(), dest);
118 }
119
and32(const Address & src,Register dest)120 void MacroAssembler::and32(const Address& src, Register dest) {
121 vixl::UseScratchRegisterScope temps(this);
122 const ARMRegister scratch32 = temps.AcquireW();
123 MOZ_ASSERT(scratch32.asUnsized() != src.base);
124 load32(src, scratch32.asUnsized());
125 And(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(scratch32));
126 }
127
andPtr(Register src,Register dest)128 void MacroAssembler::andPtr(Register src, Register dest) {
129 And(ARMRegister(dest, 64), ARMRegister(dest, 64),
130 Operand(ARMRegister(src, 64)));
131 }
132
andPtr(Imm32 imm,Register dest)133 void MacroAssembler::andPtr(Imm32 imm, Register dest) {
134 And(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
135 }
136
and64(Imm64 imm,Register64 dest)137 void MacroAssembler::and64(Imm64 imm, Register64 dest) {
138 And(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
139 }
140
and64(Register64 src,Register64 dest)141 void MacroAssembler::and64(Register64 src, Register64 dest) {
142 And(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64),
143 ARMRegister(src.reg, 64));
144 }
145
or64(Imm64 imm,Register64 dest)146 void MacroAssembler::or64(Imm64 imm, Register64 dest) {
147 Orr(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
148 }
149
or32(Imm32 imm,Register dest)150 void MacroAssembler::or32(Imm32 imm, Register dest) {
151 Orr(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
152 }
153
or32(Register src,Register dest)154 void MacroAssembler::or32(Register src, Register dest) {
155 Orr(ARMRegister(dest, 32), ARMRegister(dest, 32),
156 Operand(ARMRegister(src, 32)));
157 }
158
or32(Imm32 imm,const Address & dest)159 void MacroAssembler::or32(Imm32 imm, const Address& dest) {
160 vixl::UseScratchRegisterScope temps(this);
161 const ARMRegister scratch32 = temps.AcquireW();
162 MOZ_ASSERT(scratch32.asUnsized() != dest.base);
163 load32(dest, scratch32.asUnsized());
164 Orr(scratch32, scratch32, Operand(imm.value));
165 store32(scratch32.asUnsized(), dest);
166 }
167
orPtr(Register src,Register dest)168 void MacroAssembler::orPtr(Register src, Register dest) {
169 Orr(ARMRegister(dest, 64), ARMRegister(dest, 64),
170 Operand(ARMRegister(src, 64)));
171 }
172
orPtr(Imm32 imm,Register dest)173 void MacroAssembler::orPtr(Imm32 imm, Register dest) {
174 Orr(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
175 }
176
or64(Register64 src,Register64 dest)177 void MacroAssembler::or64(Register64 src, Register64 dest) {
178 orPtr(src.reg, dest.reg);
179 }
180
xor64(Register64 src,Register64 dest)181 void MacroAssembler::xor64(Register64 src, Register64 dest) {
182 xorPtr(src.reg, dest.reg);
183 }
184
xor32(Register src,Register dest)185 void MacroAssembler::xor32(Register src, Register dest) {
186 Eor(ARMRegister(dest, 32), ARMRegister(dest, 32),
187 Operand(ARMRegister(src, 32)));
188 }
189
xor32(Imm32 imm,Register dest)190 void MacroAssembler::xor32(Imm32 imm, Register dest) {
191 Eor(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
192 }
193
xor32(Imm32 imm,const Address & dest)194 void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
195 vixl::UseScratchRegisterScope temps(this);
196 const ARMRegister scratch32 = temps.AcquireW();
197 MOZ_ASSERT(scratch32.asUnsized() != dest.base);
198 load32(dest, scratch32.asUnsized());
199 Eor(scratch32, scratch32, Operand(imm.value));
200 store32(scratch32.asUnsized(), dest);
201 }
202
xor32(const Address & src,Register dest)203 void MacroAssembler::xor32(const Address& src, Register dest) {
204 vixl::UseScratchRegisterScope temps(this);
205 const ARMRegister scratch32 = temps.AcquireW();
206 MOZ_ASSERT(scratch32.asUnsized() != src.base);
207 load32(src, scratch32.asUnsized());
208 Eor(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(scratch32));
209 }
210
xorPtr(Register src,Register dest)211 void MacroAssembler::xorPtr(Register src, Register dest) {
212 Eor(ARMRegister(dest, 64), ARMRegister(dest, 64),
213 Operand(ARMRegister(src, 64)));
214 }
215
xorPtr(Imm32 imm,Register dest)216 void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
217 Eor(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
218 }
219
xor64(Imm64 imm,Register64 dest)220 void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
221 Eor(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
222 }
223
224 // ===============================================================
225 // Swap instructions
226
byteSwap16SignExtend(Register reg)227 void MacroAssembler::byteSwap16SignExtend(Register reg) {
228 rev16(ARMRegister(reg, 32), ARMRegister(reg, 32));
229 sxth(ARMRegister(reg, 32), ARMRegister(reg, 32));
230 }
231
byteSwap16ZeroExtend(Register reg)232 void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
233 rev16(ARMRegister(reg, 32), ARMRegister(reg, 32));
234 uxth(ARMRegister(reg, 32), ARMRegister(reg, 32));
235 }
236
byteSwap32(Register reg)237 void MacroAssembler::byteSwap32(Register reg) {
238 rev(ARMRegister(reg, 32), ARMRegister(reg, 32));
239 }
240
byteSwap64(Register64 reg)241 void MacroAssembler::byteSwap64(Register64 reg) {
242 rev(ARMRegister(reg.reg, 64), ARMRegister(reg.reg, 64));
243 }
244
245 // ===============================================================
246 // Arithmetic functions
247
add32(Register src,Register dest)248 void MacroAssembler::add32(Register src, Register dest) {
249 Add(ARMRegister(dest, 32), ARMRegister(dest, 32),
250 Operand(ARMRegister(src, 32)));
251 }
252
add32(Imm32 imm,Register dest)253 void MacroAssembler::add32(Imm32 imm, Register dest) {
254 Add(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
255 }
256
add32(Imm32 imm,const Address & dest)257 void MacroAssembler::add32(Imm32 imm, const Address& dest) {
258 vixl::UseScratchRegisterScope temps(this);
259 const ARMRegister scratch32 = temps.AcquireW();
260 MOZ_ASSERT(scratch32.asUnsized() != dest.base);
261
262 Ldr(scratch32, toMemOperand(dest));
263 Add(scratch32, scratch32, Operand(imm.value));
264 Str(scratch32, toMemOperand(dest));
265 }
266
addPtr(Register src,Register dest)267 void MacroAssembler::addPtr(Register src, Register dest) {
268 addPtr(src, dest, dest);
269 }
270
addPtr(Register src1,Register src2,Register dest)271 void MacroAssembler::addPtr(Register src1, Register src2, Register dest) {
272 Add(ARMRegister(dest, 64), ARMRegister(src1, 64),
273 Operand(ARMRegister(src2, 64)));
274 }
275
addPtr(Imm32 imm,Register dest)276 void MacroAssembler::addPtr(Imm32 imm, Register dest) {
277 addPtr(imm, dest, dest);
278 }
279
addPtr(Imm32 imm,Register src,Register dest)280 void MacroAssembler::addPtr(Imm32 imm, Register src, Register dest) {
281 Add(ARMRegister(dest, 64), ARMRegister(src, 64), Operand(imm.value));
282 }
283
addPtr(ImmWord imm,Register dest)284 void MacroAssembler::addPtr(ImmWord imm, Register dest) {
285 Add(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
286 }
287
addPtr(Imm32 imm,const Address & dest)288 void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
289 vixl::UseScratchRegisterScope temps(this);
290 const ARMRegister scratch64 = temps.AcquireX();
291 MOZ_ASSERT(scratch64.asUnsized() != dest.base);
292
293 Ldr(scratch64, toMemOperand(dest));
294 Add(scratch64, scratch64, Operand(imm.value));
295 Str(scratch64, toMemOperand(dest));
296 }
297
addPtr(const Address & src,Register dest)298 void MacroAssembler::addPtr(const Address& src, Register dest) {
299 vixl::UseScratchRegisterScope temps(this);
300 const ARMRegister scratch64 = temps.AcquireX();
301 MOZ_ASSERT(scratch64.asUnsized() != src.base);
302
303 Ldr(scratch64, toMemOperand(src));
304 Add(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(scratch64));
305 }
306
add64(Register64 src,Register64 dest)307 void MacroAssembler::add64(Register64 src, Register64 dest) {
308 addPtr(src.reg, dest.reg);
309 }
310
add64(Imm32 imm,Register64 dest)311 void MacroAssembler::add64(Imm32 imm, Register64 dest) {
312 Add(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
313 }
314
add64(Imm64 imm,Register64 dest)315 void MacroAssembler::add64(Imm64 imm, Register64 dest) {
316 Add(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
317 }
318
sub32FromStackPtrWithPatch(Register dest)319 CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
320 vixl::UseScratchRegisterScope temps(this);
321 const ARMRegister scratch = temps.AcquireX();
322 AutoForbidPoolsAndNops afp(this,
323 /* max number of instructions in scope = */ 3);
324 CodeOffset offs = CodeOffset(currentOffset());
325 movz(scratch, 0, 0);
326 movk(scratch, 0, 16);
327 Sub(ARMRegister(dest, 64), sp, scratch);
328 return offs;
329 }
330
patchSub32FromStackPtr(CodeOffset offset,Imm32 imm)331 void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
332 Instruction* i1 = getInstructionAt(BufferOffset(offset.offset()));
333 MOZ_ASSERT(i1->IsMovz());
334 i1->SetInstructionBits(i1->InstructionBits() |
335 ImmMoveWide(uint16_t(imm.value)));
336
337 Instruction* i2 = getInstructionAt(BufferOffset(offset.offset() + 4));
338 MOZ_ASSERT(i2->IsMovk());
339 i2->SetInstructionBits(i2->InstructionBits() |
340 ImmMoveWide(uint16_t(imm.value >> 16)));
341 }
342
addDouble(FloatRegister src,FloatRegister dest)343 void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
344 fadd(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
345 ARMFPRegister(src, 64));
346 }
347
addFloat32(FloatRegister src,FloatRegister dest)348 void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
349 fadd(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
350 ARMFPRegister(src, 32));
351 }
352
sub32(Imm32 imm,Register dest)353 void MacroAssembler::sub32(Imm32 imm, Register dest) {
354 Sub(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
355 }
356
sub32(Register src,Register dest)357 void MacroAssembler::sub32(Register src, Register dest) {
358 Sub(ARMRegister(dest, 32), ARMRegister(dest, 32),
359 Operand(ARMRegister(src, 32)));
360 }
361
sub32(const Address & src,Register dest)362 void MacroAssembler::sub32(const Address& src, Register dest) {
363 vixl::UseScratchRegisterScope temps(this);
364 const ARMRegister scratch32 = temps.AcquireW();
365 MOZ_ASSERT(scratch32.asUnsized() != src.base);
366 load32(src, scratch32.asUnsized());
367 Sub(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(scratch32));
368 }
369
subPtr(Register src,Register dest)370 void MacroAssembler::subPtr(Register src, Register dest) {
371 Sub(ARMRegister(dest, 64), ARMRegister(dest, 64),
372 Operand(ARMRegister(src, 64)));
373 }
374
subPtr(Register src,const Address & dest)375 void MacroAssembler::subPtr(Register src, const Address& dest) {
376 vixl::UseScratchRegisterScope temps(this);
377 const ARMRegister scratch64 = temps.AcquireX();
378 MOZ_ASSERT(scratch64.asUnsized() != dest.base);
379
380 Ldr(scratch64, toMemOperand(dest));
381 Sub(scratch64, scratch64, Operand(ARMRegister(src, 64)));
382 Str(scratch64, toMemOperand(dest));
383 }
384
subPtr(Imm32 imm,Register dest)385 void MacroAssembler::subPtr(Imm32 imm, Register dest) {
386 Sub(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
387 }
388
subPtr(const Address & addr,Register dest)389 void MacroAssembler::subPtr(const Address& addr, Register dest) {
390 vixl::UseScratchRegisterScope temps(this);
391 const ARMRegister scratch64 = temps.AcquireX();
392 MOZ_ASSERT(scratch64.asUnsized() != addr.base);
393
394 Ldr(scratch64, toMemOperand(addr));
395 Sub(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(scratch64));
396 }
397
sub64(Register64 src,Register64 dest)398 void MacroAssembler::sub64(Register64 src, Register64 dest) {
399 Sub(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64),
400 ARMRegister(src.reg, 64));
401 }
402
sub64(Imm64 imm,Register64 dest)403 void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
404 Sub(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
405 }
406
subDouble(FloatRegister src,FloatRegister dest)407 void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
408 fsub(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
409 ARMFPRegister(src, 64));
410 }
411
subFloat32(FloatRegister src,FloatRegister dest)412 void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
413 fsub(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
414 ARMFPRegister(src, 32));
415 }
416
mul32(Register rhs,Register srcDest)417 void MacroAssembler::mul32(Register rhs, Register srcDest) {
418 mul32(srcDest, rhs, srcDest, nullptr);
419 }
420
mul32(Imm32 imm,Register srcDest)421 void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
422 vixl::UseScratchRegisterScope temps(this);
423 const ARMRegister scratch32 = temps.AcquireW();
424
425 move32(imm, scratch32.asUnsized());
426 mul32(scratch32.asUnsized(), srcDest);
427 }
428
mul32(Register src1,Register src2,Register dest,Label * onOver)429 void MacroAssembler::mul32(Register src1, Register src2, Register dest,
430 Label* onOver) {
431 if (onOver) {
432 Smull(ARMRegister(dest, 64), ARMRegister(src1, 32), ARMRegister(src2, 32));
433 Cmp(ARMRegister(dest, 64), Operand(ARMRegister(dest, 32), vixl::SXTW));
434 B(onOver, NotEqual);
435
436 // Clear upper 32 bits.
437 Mov(ARMRegister(dest, 32), ARMRegister(dest, 32));
438 } else {
439 Mul(ARMRegister(dest, 32), ARMRegister(src1, 32), ARMRegister(src2, 32));
440 }
441 }
442
mulPtr(Register rhs,Register srcDest)443 void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
444 Mul(ARMRegister(srcDest, 64), ARMRegister(srcDest, 64), ARMRegister(rhs, 64));
445 }
446
mul64(Imm64 imm,const Register64 & dest)447 void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
448 vixl::UseScratchRegisterScope temps(this);
449 const ARMRegister scratch64 = temps.AcquireX();
450 MOZ_ASSERT(dest.reg != scratch64.asUnsized());
451 mov(ImmWord(imm.value), scratch64.asUnsized());
452 Mul(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), scratch64);
453 }
454
mul64(const Register64 & src,const Register64 & dest,const Register temp)455 void MacroAssembler::mul64(const Register64& src, const Register64& dest,
456 const Register temp) {
457 MOZ_ASSERT(temp == Register::Invalid());
458 Mul(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64),
459 ARMRegister(src.reg, 64));
460 }
461
mul64(const Register64 & src1,const Register64 & src2,const Register64 & dest)462 void MacroAssembler::mul64(const Register64& src1, const Register64& src2,
463 const Register64& dest) {
464 Mul(ARMRegister(dest.reg, 64), ARMRegister(src1.reg, 64),
465 ARMRegister(src2.reg, 64));
466 }
467
mul64(Imm64 src1,const Register64 & src2,const Register64 & dest)468 void MacroAssembler::mul64(Imm64 src1, const Register64& src2,
469 const Register64& dest) {
470 vixl::UseScratchRegisterScope temps(this);
471 const ARMRegister scratch64 = temps.AcquireX();
472 MOZ_ASSERT(dest.reg != scratch64.asUnsized());
473 mov(ImmWord(src1.value), scratch64.asUnsized());
474 Mul(ARMRegister(dest.reg, 64), ARMRegister(src2.reg, 64), scratch64);
475 }
476
mulBy3(Register src,Register dest)477 void MacroAssembler::mulBy3(Register src, Register dest) {
478 ARMRegister xdest(dest, 64);
479 ARMRegister xsrc(src, 64);
480 Add(xdest, xsrc, Operand(xsrc, vixl::LSL, 1));
481 }
482
mulFloat32(FloatRegister src,FloatRegister dest)483 void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
484 fmul(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
485 ARMFPRegister(src, 32));
486 }
487
mulDouble(FloatRegister src,FloatRegister dest)488 void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
489 fmul(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
490 ARMFPRegister(src, 64));
491 }
492
mulDoublePtr(ImmPtr imm,Register temp,FloatRegister dest)493 void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
494 FloatRegister dest) {
495 vixl::UseScratchRegisterScope temps(this);
496 const Register scratch = temps.AcquireX().asUnsized();
497 MOZ_ASSERT(temp != scratch);
498 movePtr(imm, scratch);
499 const ARMFPRegister scratchDouble = temps.AcquireD();
500 Ldr(scratchDouble, MemOperand(Address(scratch, 0)));
501 fmul(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64), scratchDouble);
502 }
503
quotient32(Register rhs,Register srcDest,bool isUnsigned)504 void MacroAssembler::quotient32(Register rhs, Register srcDest,
505 bool isUnsigned) {
506 if (isUnsigned) {
507 Udiv(ARMRegister(srcDest, 32), ARMRegister(srcDest, 32),
508 ARMRegister(rhs, 32));
509 } else {
510 Sdiv(ARMRegister(srcDest, 32), ARMRegister(srcDest, 32),
511 ARMRegister(rhs, 32));
512 }
513 }
514
515 // This does not deal with x % 0 or INT_MIN % -1, the caller needs to filter
516 // those cases when they may occur.
517
remainder32(Register rhs,Register srcDest,bool isUnsigned)518 void MacroAssembler::remainder32(Register rhs, Register srcDest,
519 bool isUnsigned) {
520 vixl::UseScratchRegisterScope temps(this);
521 ARMRegister scratch = temps.AcquireW();
522 if (isUnsigned) {
523 Udiv(scratch, ARMRegister(srcDest, 32), ARMRegister(rhs, 32));
524 } else {
525 Sdiv(scratch, ARMRegister(srcDest, 32), ARMRegister(rhs, 32));
526 }
527 Mul(scratch, scratch, ARMRegister(rhs, 32));
528 Sub(ARMRegister(srcDest, 32), ARMRegister(srcDest, 32), scratch);
529 }
530
divFloat32(FloatRegister src,FloatRegister dest)531 void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
532 fdiv(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
533 ARMFPRegister(src, 32));
534 }
535
divDouble(FloatRegister src,FloatRegister dest)536 void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
537 fdiv(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
538 ARMFPRegister(src, 64));
539 }
540
inc64(AbsoluteAddress dest)541 void MacroAssembler::inc64(AbsoluteAddress dest) {
542 vixl::UseScratchRegisterScope temps(this);
543 const ARMRegister scratchAddr64 = temps.AcquireX();
544 const ARMRegister scratch64 = temps.AcquireX();
545
546 Mov(scratchAddr64, uint64_t(dest.addr));
547 Ldr(scratch64, MemOperand(scratchAddr64, 0));
548 Add(scratch64, scratch64, Operand(1));
549 Str(scratch64, MemOperand(scratchAddr64, 0));
550 }
551
neg32(Register reg)552 void MacroAssembler::neg32(Register reg) {
553 Negs(ARMRegister(reg, 32), Operand(ARMRegister(reg, 32)));
554 }
555
neg64(Register64 reg)556 void MacroAssembler::neg64(Register64 reg) { negPtr(reg.reg); }
557
negPtr(Register reg)558 void MacroAssembler::negPtr(Register reg) {
559 Negs(ARMRegister(reg, 64), Operand(ARMRegister(reg, 64)));
560 }
561
negateFloat(FloatRegister reg)562 void MacroAssembler::negateFloat(FloatRegister reg) {
563 fneg(ARMFPRegister(reg, 32), ARMFPRegister(reg, 32));
564 }
565
negateDouble(FloatRegister reg)566 void MacroAssembler::negateDouble(FloatRegister reg) {
567 fneg(ARMFPRegister(reg, 64), ARMFPRegister(reg, 64));
568 }
569
abs32(Register src,Register dest)570 void MacroAssembler::abs32(Register src, Register dest) {
571 Cmp(ARMRegister(src, 32), wzr);
572 Cneg(ARMRegister(dest, 32), ARMRegister(src, 32), Assembler::LessThan);
573 }
574
absFloat32(FloatRegister src,FloatRegister dest)575 void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
576 fabs(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
577 }
578
absDouble(FloatRegister src,FloatRegister dest)579 void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
580 fabs(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
581 }
582
sqrtFloat32(FloatRegister src,FloatRegister dest)583 void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
584 fsqrt(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
585 }
586
sqrtDouble(FloatRegister src,FloatRegister dest)587 void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
588 fsqrt(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
589 }
590
minFloat32(FloatRegister other,FloatRegister srcDest,bool handleNaN)591 void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
592 bool handleNaN) {
593 MOZ_ASSERT(handleNaN); // Always true for wasm
594 fmin(ARMFPRegister(srcDest, 32), ARMFPRegister(srcDest, 32),
595 ARMFPRegister(other, 32));
596 }
597
minDouble(FloatRegister other,FloatRegister srcDest,bool handleNaN)598 void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
599 bool handleNaN) {
600 MOZ_ASSERT(handleNaN); // Always true for wasm
601 fmin(ARMFPRegister(srcDest, 64), ARMFPRegister(srcDest, 64),
602 ARMFPRegister(other, 64));
603 }
604
maxFloat32(FloatRegister other,FloatRegister srcDest,bool handleNaN)605 void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
606 bool handleNaN) {
607 MOZ_ASSERT(handleNaN); // Always true for wasm
608 fmax(ARMFPRegister(srcDest, 32), ARMFPRegister(srcDest, 32),
609 ARMFPRegister(other, 32));
610 }
611
maxDouble(FloatRegister other,FloatRegister srcDest,bool handleNaN)612 void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
613 bool handleNaN) {
614 MOZ_ASSERT(handleNaN); // Always true for wasm
615 fmax(ARMFPRegister(srcDest, 64), ARMFPRegister(srcDest, 64),
616 ARMFPRegister(other, 64));
617 }
618
619 // ===============================================================
620 // Shift functions
621
lshiftPtr(Imm32 imm,Register dest)622 void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
623 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
624 Lsl(ARMRegister(dest, 64), ARMRegister(dest, 64), imm.value);
625 }
626
lshiftPtr(Register shift,Register dest)627 void MacroAssembler::lshiftPtr(Register shift, Register dest) {
628 Lsl(ARMRegister(dest, 64), ARMRegister(dest, 64), ARMRegister(shift, 64));
629 }
630
lshift64(Imm32 imm,Register64 dest)631 void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
632 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
633 lshiftPtr(imm, dest.reg);
634 }
635
lshift64(Register shift,Register64 srcDest)636 void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
637 Lsl(ARMRegister(srcDest.reg, 64), ARMRegister(srcDest.reg, 64),
638 ARMRegister(shift, 64));
639 }
640
lshift32(Register shift,Register dest)641 void MacroAssembler::lshift32(Register shift, Register dest) {
642 Lsl(ARMRegister(dest, 32), ARMRegister(dest, 32), ARMRegister(shift, 32));
643 }
644
flexibleLshift32(Register src,Register dest)645 void MacroAssembler::flexibleLshift32(Register src, Register dest) {
646 lshift32(src, dest);
647 }
648
lshift32(Imm32 imm,Register dest)649 void MacroAssembler::lshift32(Imm32 imm, Register dest) {
650 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
651 Lsl(ARMRegister(dest, 32), ARMRegister(dest, 32), imm.value);
652 }
653
rshiftPtr(Imm32 imm,Register dest)654 void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
655 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
656 Lsr(ARMRegister(dest, 64), ARMRegister(dest, 64), imm.value);
657 }
658
rshiftPtr(Imm32 imm,Register src,Register dest)659 void MacroAssembler::rshiftPtr(Imm32 imm, Register src, Register dest) {
660 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
661 Lsr(ARMRegister(dest, 64), ARMRegister(src, 64), imm.value);
662 }
663
rshiftPtr(Register shift,Register dest)664 void MacroAssembler::rshiftPtr(Register shift, Register dest) {
665 Lsr(ARMRegister(dest, 64), ARMRegister(dest, 64), ARMRegister(shift, 64));
666 }
667
rshift32(Register shift,Register dest)668 void MacroAssembler::rshift32(Register shift, Register dest) {
669 Lsr(ARMRegister(dest, 32), ARMRegister(dest, 32), ARMRegister(shift, 32));
670 }
671
flexibleRshift32(Register src,Register dest)672 void MacroAssembler::flexibleRshift32(Register src, Register dest) {
673 rshift32(src, dest);
674 }
675
rshift32(Imm32 imm,Register dest)676 void MacroAssembler::rshift32(Imm32 imm, Register dest) {
677 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
678 Lsr(ARMRegister(dest, 32), ARMRegister(dest, 32), imm.value);
679 }
680
rshiftPtrArithmetic(Imm32 imm,Register dest)681 void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
682 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
683 Asr(ARMRegister(dest, 64), ARMRegister(dest, 64), imm.value);
684 }
685
rshift32Arithmetic(Register shift,Register dest)686 void MacroAssembler::rshift32Arithmetic(Register shift, Register dest) {
687 Asr(ARMRegister(dest, 32), ARMRegister(dest, 32), ARMRegister(shift, 32));
688 }
689
rshift32Arithmetic(Imm32 imm,Register dest)690 void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
691 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
692 Asr(ARMRegister(dest, 32), ARMRegister(dest, 32), imm.value);
693 }
694
flexibleRshift32Arithmetic(Register src,Register dest)695 void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
696 rshift32Arithmetic(src, dest);
697 }
698
rshift64(Imm32 imm,Register64 dest)699 void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
700 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
701 rshiftPtr(imm, dest.reg);
702 }
703
rshift64(Register shift,Register64 srcDest)704 void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
705 Lsr(ARMRegister(srcDest.reg, 64), ARMRegister(srcDest.reg, 64),
706 ARMRegister(shift, 64));
707 }
708
rshift64Arithmetic(Imm32 imm,Register64 dest)709 void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
710 Asr(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), imm.value);
711 }
712
rshift64Arithmetic(Register shift,Register64 srcDest)713 void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
714 Asr(ARMRegister(srcDest.reg, 64), ARMRegister(srcDest.reg, 64),
715 ARMRegister(shift, 64));
716 }
717
718 // ===============================================================
719 // Condition functions
720
cmp8Set(Condition cond,Address lhs,Imm32 rhs,Register dest)721 void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
722 Register dest) {
723 vixl::UseScratchRegisterScope temps(this);
724 Register scratch = temps.AcquireX().asUnsized();
725 MOZ_ASSERT(scratch != lhs.base);
726
727 switch (cond) {
728 case Assembler::Equal:
729 case Assembler::NotEqual:
730 case Assembler::Above:
731 case Assembler::AboveOrEqual:
732 case Assembler::Below:
733 case Assembler::BelowOrEqual:
734 load8ZeroExtend(lhs, scratch);
735 cmp32Set(cond, scratch, Imm32(uint8_t(rhs.value)), dest);
736 break;
737
738 case Assembler::GreaterThan:
739 case Assembler::GreaterThanOrEqual:
740 case Assembler::LessThan:
741 case Assembler::LessThanOrEqual:
742 load8SignExtend(lhs, scratch);
743 cmp32Set(cond, scratch, Imm32(int8_t(rhs.value)), dest);
744 break;
745
746 default:
747 MOZ_CRASH("unexpected condition");
748 }
749 }
750
cmp16Set(Condition cond,Address lhs,Imm32 rhs,Register dest)751 void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
752 Register dest) {
753 vixl::UseScratchRegisterScope temps(this);
754 Register scratch = temps.AcquireX().asUnsized();
755 MOZ_ASSERT(scratch != lhs.base);
756
757 switch (cond) {
758 case Assembler::Equal:
759 case Assembler::NotEqual:
760 case Assembler::Above:
761 case Assembler::AboveOrEqual:
762 case Assembler::Below:
763 case Assembler::BelowOrEqual:
764 load16ZeroExtend(lhs, scratch);
765 cmp32Set(cond, scratch, Imm32(uint16_t(rhs.value)), dest);
766 break;
767
768 case Assembler::GreaterThan:
769 case Assembler::GreaterThanOrEqual:
770 case Assembler::LessThan:
771 case Assembler::LessThanOrEqual:
772 load16SignExtend(lhs, scratch);
773 cmp32Set(cond, scratch, Imm32(int16_t(rhs.value)), dest);
774 break;
775
776 default:
777 MOZ_CRASH("unexpected condition");
778 }
779 }
780
781 template <typename T1, typename T2>
cmp32Set(Condition cond,T1 lhs,T2 rhs,Register dest)782 void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
783 cmp32(lhs, rhs);
784 emitSet(cond, dest);
785 }
786
cmp64Set(Condition cond,Address lhs,Imm64 rhs,Register dest)787 void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
788 Register dest) {
789 cmpPtrSet(cond, lhs, ImmWord(static_cast<uintptr_t>(rhs.value)), dest);
790 }
791
792 template <typename T1, typename T2>
cmpPtrSet(Condition cond,T1 lhs,T2 rhs,Register dest)793 void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
794 cmpPtr(lhs, rhs);
795 emitSet(cond, dest);
796 }
797
798 // ===============================================================
799 // Rotation functions
800
rotateLeft(Imm32 count,Register input,Register dest)801 void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
802 Ror(ARMRegister(dest, 32), ARMRegister(input, 32), (32 - count.value) & 31);
803 }
804
rotateLeft(Register count,Register input,Register dest)805 void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
806 vixl::UseScratchRegisterScope temps(this);
807 const ARMRegister scratch = temps.AcquireW();
808 // Really 32 - count, but the upper bits of the result are ignored.
809 Neg(scratch, ARMRegister(count, 32));
810 Ror(ARMRegister(dest, 32), ARMRegister(input, 32), scratch);
811 }
812
rotateRight(Imm32 count,Register input,Register dest)813 void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
814 Ror(ARMRegister(dest, 32), ARMRegister(input, 32), count.value & 31);
815 }
816
rotateRight(Register count,Register input,Register dest)817 void MacroAssembler::rotateRight(Register count, Register input,
818 Register dest) {
819 Ror(ARMRegister(dest, 32), ARMRegister(input, 32), ARMRegister(count, 32));
820 }
821
rotateLeft64(Register count,Register64 input,Register64 dest,Register temp)822 void MacroAssembler::rotateLeft64(Register count, Register64 input,
823 Register64 dest, Register temp) {
824 MOZ_ASSERT(temp == Register::Invalid());
825
826 vixl::UseScratchRegisterScope temps(this);
827 const ARMRegister scratch = temps.AcquireX();
828 // Really 64 - count, but the upper bits of the result are ignored.
829 Neg(scratch, ARMRegister(count, 64));
830 Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64), scratch);
831 }
832
rotateLeft64(Imm32 count,Register64 input,Register64 dest,Register temp)833 void MacroAssembler::rotateLeft64(Imm32 count, Register64 input,
834 Register64 dest, Register temp) {
835 MOZ_ASSERT(temp == Register::Invalid());
836
837 Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64),
838 (64 - count.value) & 63);
839 }
840
rotateRight64(Register count,Register64 input,Register64 dest,Register temp)841 void MacroAssembler::rotateRight64(Register count, Register64 input,
842 Register64 dest, Register temp) {
843 MOZ_ASSERT(temp == Register::Invalid());
844
845 Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64),
846 ARMRegister(count, 64));
847 }
848
rotateRight64(Imm32 count,Register64 input,Register64 dest,Register temp)849 void MacroAssembler::rotateRight64(Imm32 count, Register64 input,
850 Register64 dest, Register temp) {
851 MOZ_ASSERT(temp == Register::Invalid());
852
853 Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64), count.value & 63);
854 }
855
856 // ===============================================================
857 // Bit counting functions
858
clz32(Register src,Register dest,bool knownNotZero)859 void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
860 Clz(ARMRegister(dest, 32), ARMRegister(src, 32));
861 }
862
ctz32(Register src,Register dest,bool knownNotZero)863 void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
864 Rbit(ARMRegister(dest, 32), ARMRegister(src, 32));
865 Clz(ARMRegister(dest, 32), ARMRegister(dest, 32));
866 }
867
clz64(Register64 src,Register dest)868 void MacroAssembler::clz64(Register64 src, Register dest) {
869 Clz(ARMRegister(dest, 64), ARMRegister(src.reg, 64));
870 }
871
ctz64(Register64 src,Register dest)872 void MacroAssembler::ctz64(Register64 src, Register dest) {
873 Rbit(ARMRegister(dest, 64), ARMRegister(src.reg, 64));
874 Clz(ARMRegister(dest, 64), ARMRegister(dest, 64));
875 }
876
popcnt32(Register src_,Register dest_,Register tmp_)877 void MacroAssembler::popcnt32(Register src_, Register dest_, Register tmp_) {
878 MOZ_ASSERT(tmp_ != Register::Invalid());
879
880 // Equivalent to mozilla::CountPopulation32().
881
882 ARMRegister src(src_, 32);
883 ARMRegister dest(dest_, 32);
884 ARMRegister tmp(tmp_, 32);
885
886 Mov(tmp, src);
887 if (src_ != dest_) {
888 Mov(dest, src);
889 }
890 Lsr(dest, dest, 1);
891 And(dest, dest, 0x55555555);
892 Sub(dest, tmp, dest);
893 Lsr(tmp, dest, 2);
894 And(tmp, tmp, 0x33333333);
895 And(dest, dest, 0x33333333);
896 Add(dest, tmp, dest);
897 Add(dest, dest, Operand(dest, vixl::LSR, 4));
898 And(dest, dest, 0x0F0F0F0F);
899 Add(dest, dest, Operand(dest, vixl::LSL, 8));
900 Add(dest, dest, Operand(dest, vixl::LSL, 16));
901 Lsr(dest, dest, 24);
902 }
903
popcnt64(Register64 src_,Register64 dest_,Register tmp_)904 void MacroAssembler::popcnt64(Register64 src_, Register64 dest_,
905 Register tmp_) {
906 MOZ_ASSERT(tmp_ != Register::Invalid());
907
908 // Equivalent to mozilla::CountPopulation64(), though likely more efficient.
909
910 ARMRegister src(src_.reg, 64);
911 ARMRegister dest(dest_.reg, 64);
912 ARMRegister tmp(tmp_, 64);
913
914 Mov(tmp, src);
915 if (src_ != dest_) {
916 Mov(dest, src);
917 }
918 Lsr(dest, dest, 1);
919 And(dest, dest, 0x5555555555555555);
920 Sub(dest, tmp, dest);
921 Lsr(tmp, dest, 2);
922 And(tmp, tmp, 0x3333333333333333);
923 And(dest, dest, 0x3333333333333333);
924 Add(dest, tmp, dest);
925 Add(dest, dest, Operand(dest, vixl::LSR, 4));
926 And(dest, dest, 0x0F0F0F0F0F0F0F0F);
927 Add(dest, dest, Operand(dest, vixl::LSL, 8));
928 Add(dest, dest, Operand(dest, vixl::LSL, 16));
929 Add(dest, dest, Operand(dest, vixl::LSL, 32));
930 Lsr(dest, dest, 56);
931 }
932
933 // ===============================================================
934 // Branch functions
935
branch8(Condition cond,const Address & lhs,Imm32 rhs,Label * label)936 void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
937 Label* label) {
938 vixl::UseScratchRegisterScope temps(this);
939 Register scratch = temps.AcquireX().asUnsized();
940 MOZ_ASSERT(scratch != lhs.base);
941
942 switch (cond) {
943 case Assembler::Equal:
944 case Assembler::NotEqual:
945 case Assembler::Above:
946 case Assembler::AboveOrEqual:
947 case Assembler::Below:
948 case Assembler::BelowOrEqual:
949 load8ZeroExtend(lhs, scratch);
950 branch32(cond, scratch, Imm32(uint8_t(rhs.value)), label);
951 break;
952
953 case Assembler::GreaterThan:
954 case Assembler::GreaterThanOrEqual:
955 case Assembler::LessThan:
956 case Assembler::LessThanOrEqual:
957 load8SignExtend(lhs, scratch);
958 branch32(cond, scratch, Imm32(int8_t(rhs.value)), label);
959 break;
960
961 default:
962 MOZ_CRASH("unexpected condition");
963 }
964 }
965
branch16(Condition cond,const Address & lhs,Imm32 rhs,Label * label)966 void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
967 Label* label) {
968 vixl::UseScratchRegisterScope temps(this);
969 Register scratch = temps.AcquireX().asUnsized();
970 MOZ_ASSERT(scratch != lhs.base);
971
972 switch (cond) {
973 case Assembler::Equal:
974 case Assembler::NotEqual:
975 case Assembler::Above:
976 case Assembler::AboveOrEqual:
977 case Assembler::Below:
978 case Assembler::BelowOrEqual:
979 load16ZeroExtend(lhs, scratch);
980 branch32(cond, scratch, Imm32(uint16_t(rhs.value)), label);
981 break;
982
983 case Assembler::GreaterThan:
984 case Assembler::GreaterThanOrEqual:
985 case Assembler::LessThan:
986 case Assembler::LessThanOrEqual:
987 load16SignExtend(lhs, scratch);
988 branch32(cond, scratch, Imm32(int16_t(rhs.value)), label);
989 break;
990
991 default:
992 MOZ_CRASH("unexpected condition");
993 }
994 }
995
996 template <class L>
branch32(Condition cond,Register lhs,Register rhs,L label)997 void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
998 L label) {
999 cmp32(lhs, rhs);
1000 B(label, cond);
1001 }
1002
1003 template <class L>
branch32(Condition cond,Register lhs,Imm32 imm,L label)1004 void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm,
1005 L label) {
1006 if (imm.value == 0 && cond == Assembler::Equal) {
1007 Cbz(ARMRegister(lhs, 32), label);
1008 } else if (imm.value == 0 && cond == Assembler::NotEqual) {
1009 Cbnz(ARMRegister(lhs, 32), label);
1010 } else {
1011 cmp32(lhs, imm);
1012 B(label, cond);
1013 }
1014 }
1015
branch32(Condition cond,Register lhs,const Address & rhs,Label * label)1016 void MacroAssembler::branch32(Condition cond, Register lhs, const Address& rhs,
1017 Label* label) {
1018 vixl::UseScratchRegisterScope temps(this);
1019 const Register scratch = temps.AcquireX().asUnsized();
1020 MOZ_ASSERT(scratch != lhs);
1021 MOZ_ASSERT(scratch != rhs.base);
1022 load32(rhs, scratch);
1023 branch32(cond, lhs, scratch, label);
1024 }
1025
branch32(Condition cond,const Address & lhs,Register rhs,Label * label)1026 void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
1027 Label* label) {
1028 vixl::UseScratchRegisterScope temps(this);
1029 const Register scratch = temps.AcquireX().asUnsized();
1030 MOZ_ASSERT(scratch != lhs.base);
1031 MOZ_ASSERT(scratch != rhs);
1032 load32(lhs, scratch);
1033 branch32(cond, scratch, rhs, label);
1034 }
1035
branch32(Condition cond,const Address & lhs,Imm32 imm,Label * label)1036 void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 imm,
1037 Label* label) {
1038 vixl::UseScratchRegisterScope temps(this);
1039 const Register scratch = temps.AcquireX().asUnsized();
1040 MOZ_ASSERT(scratch != lhs.base);
1041 load32(lhs, scratch);
1042 branch32(cond, scratch, imm, label);
1043 }
1044
branch32(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)1045 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
1046 Register rhs, Label* label) {
1047 vixl::UseScratchRegisterScope temps(this);
1048 const Register scratch = temps.AcquireX().asUnsized();
1049 movePtr(ImmPtr(lhs.addr), scratch);
1050 branch32(cond, Address(scratch, 0), rhs, label);
1051 }
1052
branch32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)1053 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
1054 Imm32 rhs, Label* label) {
1055 vixl::UseScratchRegisterScope temps(this);
1056 const Register scratch = temps.AcquireX().asUnsized();
1057 load32(lhs, scratch);
1058 branch32(cond, scratch, rhs, label);
1059 }
1060
branch32(Condition cond,const BaseIndex & lhs,Imm32 rhs,Label * label)1061 void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
1062 Label* label) {
1063 vixl::UseScratchRegisterScope temps(this);
1064 const ARMRegister scratch32 = temps.AcquireW();
1065 MOZ_ASSERT(scratch32.asUnsized() != lhs.base);
1066 MOZ_ASSERT(scratch32.asUnsized() != lhs.index);
1067 doBaseIndex(scratch32, lhs, vixl::LDR_w);
1068 branch32(cond, scratch32.asUnsized(), rhs, label);
1069 }
1070
branch32(Condition cond,wasm::SymbolicAddress lhs,Imm32 rhs,Label * label)1071 void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
1072 Imm32 rhs, Label* label) {
1073 vixl::UseScratchRegisterScope temps(this);
1074 const Register scratch = temps.AcquireX().asUnsized();
1075 movePtr(lhs, scratch);
1076 branch32(cond, Address(scratch, 0), rhs, label);
1077 }
1078
branch64(Condition cond,Register64 lhs,Imm64 val,Label * success,Label * fail)1079 void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
1080 Label* success, Label* fail) {
1081 if (val.value == 0 && cond == Assembler::Equal) {
1082 Cbz(ARMRegister(lhs.reg, 64), success);
1083 } else if (val.value == 0 && cond == Assembler::NotEqual) {
1084 Cbnz(ARMRegister(lhs.reg, 64), success);
1085 } else {
1086 Cmp(ARMRegister(lhs.reg, 64), val.value);
1087 B(success, cond);
1088 }
1089 if (fail) {
1090 B(fail);
1091 }
1092 }
1093
branch64(Condition cond,Register64 lhs,Register64 rhs,Label * success,Label * fail)1094 void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
1095 Label* success, Label* fail) {
1096 Cmp(ARMRegister(lhs.reg, 64), ARMRegister(rhs.reg, 64));
1097 B(success, cond);
1098 if (fail) {
1099 B(fail);
1100 }
1101 }
1102
branch64(Condition cond,const Address & lhs,Imm64 val,Label * label)1103 void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
1104 Label* label) {
1105 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
1106 "other condition codes not supported");
1107
1108 branchPtr(cond, lhs, ImmWord(val.value), label);
1109 }
1110
branch64(Condition cond,const Address & lhs,Register64 rhs,Label * label)1111 void MacroAssembler::branch64(Condition cond, const Address& lhs,
1112 Register64 rhs, Label* label) {
1113 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
1114 "other condition codes not supported");
1115
1116 branchPtr(cond, lhs, rhs.reg, label);
1117 }
1118
branch64(Condition cond,const Address & lhs,const Address & rhs,Register scratch,Label * label)1119 void MacroAssembler::branch64(Condition cond, const Address& lhs,
1120 const Address& rhs, Register scratch,
1121 Label* label) {
1122 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
1123 "other condition codes not supported");
1124 MOZ_ASSERT(lhs.base != scratch);
1125 MOZ_ASSERT(rhs.base != scratch);
1126
1127 loadPtr(rhs, scratch);
1128 branchPtr(cond, lhs, scratch, label);
1129 }
1130
1131 template <class L>
branchPtr(Condition cond,Register lhs,Register rhs,L label)1132 void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
1133 L label) {
1134 Cmp(ARMRegister(lhs, 64), ARMRegister(rhs, 64));
1135 B(label, cond);
1136 }
1137
branchPtr(Condition cond,Register lhs,Imm32 rhs,Label * label)1138 void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
1139 Label* label) {
1140 if (rhs.value == 0 && cond == Assembler::Equal) {
1141 Cbz(ARMRegister(lhs, 64), label);
1142 } else if (rhs.value == 0 && cond == Assembler::NotEqual) {
1143 Cbnz(ARMRegister(lhs, 64), label);
1144 } else {
1145 cmpPtr(lhs, rhs);
1146 B(label, cond);
1147 }
1148 }
1149
branchPtr(Condition cond,Register lhs,ImmPtr rhs,Label * label)1150 void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
1151 Label* label) {
1152 if (rhs.value == 0 && cond == Assembler::Equal) {
1153 Cbz(ARMRegister(lhs, 64), label);
1154 } else if (rhs.value == 0 && cond == Assembler::NotEqual) {
1155 Cbnz(ARMRegister(lhs, 64), label);
1156 } else {
1157 cmpPtr(lhs, rhs);
1158 B(label, cond);
1159 }
1160 }
1161
branchPtr(Condition cond,Register lhs,ImmGCPtr rhs,Label * label)1162 void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
1163 Label* label) {
1164 vixl::UseScratchRegisterScope temps(this);
1165 const Register scratch = temps.AcquireX().asUnsized();
1166 MOZ_ASSERT(scratch != lhs);
1167 movePtr(rhs, scratch);
1168 branchPtr(cond, lhs, scratch, label);
1169 }
1170
branchPtr(Condition cond,Register lhs,ImmWord rhs,Label * label)1171 void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
1172 Label* label) {
1173 if (rhs.value == 0 && cond == Assembler::Equal) {
1174 Cbz(ARMRegister(lhs, 64), label);
1175 } else if (rhs.value == 0 && cond == Assembler::NotEqual) {
1176 Cbnz(ARMRegister(lhs, 64), label);
1177 } else {
1178 cmpPtr(lhs, rhs);
1179 B(label, cond);
1180 }
1181 }
1182
1183 template <class L>
branchPtr(Condition cond,const Address & lhs,Register rhs,L label)1184 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
1185 L label) {
1186 vixl::UseScratchRegisterScope temps(this);
1187 const Register scratch = temps.AcquireX().asUnsized();
1188 MOZ_ASSERT(scratch != lhs.base);
1189 MOZ_ASSERT(scratch != rhs);
1190 loadPtr(lhs, scratch);
1191 branchPtr(cond, scratch, rhs, label);
1192 }
1193
branchPtr(Condition cond,const Address & lhs,ImmPtr rhs,Label * label)1194 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
1195 Label* label) {
1196 vixl::UseScratchRegisterScope temps(this);
1197 const Register scratch = temps.AcquireX().asUnsized();
1198 MOZ_ASSERT(scratch != lhs.base);
1199 loadPtr(lhs, scratch);
1200 branchPtr(cond, scratch, rhs, label);
1201 }
1202
branchPtr(Condition cond,const Address & lhs,ImmGCPtr rhs,Label * label)1203 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
1204 Label* label) {
1205 vixl::UseScratchRegisterScope temps(this);
1206 const ARMRegister scratch1_64 = temps.AcquireX();
1207 const ARMRegister scratch2_64 = temps.AcquireX();
1208 MOZ_ASSERT(scratch1_64.asUnsized() != lhs.base);
1209 MOZ_ASSERT(scratch2_64.asUnsized() != lhs.base);
1210
1211 movePtr(rhs, scratch1_64.asUnsized());
1212 loadPtr(lhs, scratch2_64.asUnsized());
1213 branchPtr(cond, scratch2_64.asUnsized(), scratch1_64.asUnsized(), label);
1214 }
1215
branchPtr(Condition cond,const Address & lhs,ImmWord rhs,Label * label)1216 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
1217 Label* label) {
1218 vixl::UseScratchRegisterScope temps(this);
1219 const Register scratch = temps.AcquireX().asUnsized();
1220 MOZ_ASSERT(scratch != lhs.base);
1221 loadPtr(lhs, scratch);
1222 branchPtr(cond, scratch, rhs, label);
1223 }
1224
branchPtr(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)1225 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
1226 Register rhs, Label* label) {
1227 vixl::UseScratchRegisterScope temps(this);
1228 const Register scratch = temps.AcquireX().asUnsized();
1229 MOZ_ASSERT(scratch != rhs);
1230 loadPtr(lhs, scratch);
1231 branchPtr(cond, scratch, rhs, label);
1232 }
1233
branchPtr(Condition cond,const AbsoluteAddress & lhs,ImmWord rhs,Label * label)1234 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
1235 ImmWord rhs, Label* label) {
1236 vixl::UseScratchRegisterScope temps(this);
1237 const Register scratch = temps.AcquireX().asUnsized();
1238 loadPtr(lhs, scratch);
1239 branchPtr(cond, scratch, rhs, label);
1240 }
1241
branchPtr(Condition cond,wasm::SymbolicAddress lhs,Register rhs,Label * label)1242 void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
1243 Register rhs, Label* label) {
1244 vixl::UseScratchRegisterScope temps(this);
1245 const Register scratch = temps.AcquireX().asUnsized();
1246 MOZ_ASSERT(scratch != rhs);
1247 loadPtr(lhs, scratch);
1248 branchPtr(cond, scratch, rhs, label);
1249 }
1250
branchPtr(Condition cond,const BaseIndex & lhs,ImmWord rhs,Label * label)1251 void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
1252 ImmWord rhs, Label* label) {
1253 vixl::UseScratchRegisterScope temps(this);
1254 const Register scratch = temps.AcquireX().asUnsized();
1255 MOZ_ASSERT(scratch != lhs.base);
1256 MOZ_ASSERT(scratch != lhs.index);
1257 loadPtr(lhs, scratch);
1258 branchPtr(cond, scratch, rhs, label);
1259 }
1260
branchPtr(Condition cond,const BaseIndex & lhs,Register rhs,Label * label)1261 void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
1262 Register rhs, Label* label) {
1263 vixl::UseScratchRegisterScope temps(this);
1264 const Register scratch = temps.AcquireX().asUnsized();
1265 MOZ_ASSERT(scratch != lhs.base);
1266 MOZ_ASSERT(scratch != lhs.index);
1267 loadPtr(lhs, scratch);
1268 branchPtr(cond, scratch, rhs, label);
1269 }
1270
branchPrivatePtr(Condition cond,const Address & lhs,Register rhs,Label * label)1271 void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
1272 Register rhs, Label* label) {
1273 branchPtr(cond, lhs, rhs, label);
1274 }
1275
branchFloat(DoubleCondition cond,FloatRegister lhs,FloatRegister rhs,Label * label)1276 void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
1277 FloatRegister rhs, Label* label) {
1278 compareFloat(cond, lhs, rhs);
1279 switch (cond) {
1280 case DoubleNotEqual: {
1281 Label unordered;
1282 // not equal *and* ordered
1283 branch(Overflow, &unordered);
1284 branch(NotEqual, label);
1285 bind(&unordered);
1286 break;
1287 }
1288 case DoubleEqualOrUnordered:
1289 branch(Overflow, label);
1290 branch(Equal, label);
1291 break;
1292 default:
1293 branch(Condition(cond), label);
1294 }
1295 }
1296
branchTruncateFloat32MaybeModUint32(FloatRegister src,Register dest,Label * fail)1297 void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
1298 Register dest,
1299 Label* fail) {
1300 vixl::UseScratchRegisterScope temps(this);
1301 const ARMRegister scratch64 = temps.AcquireX();
1302
1303 ARMFPRegister src32(src, 32);
1304 ARMRegister dest64(dest, 64);
1305
1306 MOZ_ASSERT(!scratch64.Is(dest64));
1307
1308 Fcvtzs(dest64, src32);
1309 Add(scratch64, dest64, Operand(0x7fffffffffffffff));
1310 Cmn(scratch64, 3);
1311 B(fail, Assembler::Above);
1312 And(dest64, dest64, Operand(0xffffffff));
1313 }
1314
branchTruncateFloat32ToInt32(FloatRegister src,Register dest,Label * fail)1315 void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
1316 Register dest, Label* fail) {
1317 convertFloat32ToInt32(src, dest, fail, false);
1318 }
1319
branchDouble(DoubleCondition cond,FloatRegister lhs,FloatRegister rhs,Label * label)1320 void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
1321 FloatRegister rhs, Label* label) {
1322 compareDouble(cond, lhs, rhs);
1323 switch (cond) {
1324 case DoubleNotEqual: {
1325 Label unordered;
1326 // not equal *and* ordered
1327 branch(Overflow, &unordered);
1328 branch(NotEqual, label);
1329 bind(&unordered);
1330 break;
1331 }
1332 case DoubleEqualOrUnordered:
1333 branch(Overflow, label);
1334 branch(Equal, label);
1335 break;
1336 default:
1337 branch(Condition(cond), label);
1338 }
1339 }
1340
branchTruncateDoubleMaybeModUint32(FloatRegister src,Register dest,Label * fail)1341 void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
1342 Register dest,
1343 Label* fail) {
1344 vixl::UseScratchRegisterScope temps(this);
1345 const ARMRegister scratch64 = temps.AcquireX();
1346
1347 // An out of range integer will be saturated to the destination size.
1348 ARMFPRegister src64(src, 64);
1349 ARMRegister dest64(dest, 64);
1350
1351 MOZ_ASSERT(!scratch64.Is(dest64));
1352
1353 Fcvtzs(dest64, src64);
1354 Add(scratch64, dest64, Operand(0x7fffffffffffffff));
1355 Cmn(scratch64, 3);
1356 B(fail, Assembler::Above);
1357 And(dest64, dest64, Operand(0xffffffff));
1358 }
1359
branchTruncateDoubleToInt32(FloatRegister src,Register dest,Label * fail)1360 void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
1361 Register dest, Label* fail) {
1362 convertDoubleToInt32(src, dest, fail, false);
1363 }
1364
1365 template <typename T>
branchAdd32(Condition cond,T src,Register dest,Label * label)1366 void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
1367 Label* label) {
1368 adds32(src, dest);
1369 B(label, cond);
1370 }
1371
1372 template <typename T>
branchSub32(Condition cond,T src,Register dest,Label * label)1373 void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
1374 Label* label) {
1375 subs32(src, dest);
1376 branch(cond, label);
1377 }
1378
1379 template <typename T>
branchMul32(Condition cond,T src,Register dest,Label * label)1380 void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
1381 Label* label) {
1382 MOZ_ASSERT(cond == Assembler::Overflow);
1383 vixl::UseScratchRegisterScope temps(this);
1384 mul32(src, dest, dest, label);
1385 }
1386
1387 template <typename T>
branchRshift32(Condition cond,T src,Register dest,Label * label)1388 void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
1389 Label* label) {
1390 MOZ_ASSERT(cond == Zero || cond == NonZero);
1391 rshift32(src, dest);
1392 branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
1393 }
1394
branchNeg32(Condition cond,Register reg,Label * label)1395 void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
1396 MOZ_ASSERT(cond == Overflow);
1397 neg32(reg);
1398 B(label, cond);
1399 }
1400
1401 template <typename T>
branchAddPtr(Condition cond,T src,Register dest,Label * label)1402 void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
1403 Label* label) {
1404 adds64(src, dest);
1405 B(label, cond);
1406 }
1407
1408 template <typename T>
branchSubPtr(Condition cond,T src,Register dest,Label * label)1409 void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
1410 Label* label) {
1411 subs64(src, dest);
1412 B(label, cond);
1413 }
1414
branchMulPtr(Condition cond,Register src,Register dest,Label * label)1415 void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
1416 Label* label) {
1417 MOZ_ASSERT(cond == Assembler::Overflow);
1418
1419 vixl::UseScratchRegisterScope temps(this);
1420 const ARMRegister scratch64 = temps.AcquireX();
1421 const ARMRegister src64(src, 64);
1422 const ARMRegister dest64(dest, 64);
1423
1424 Smulh(scratch64, dest64, src64);
1425 Mul(dest64, dest64, src64);
1426 Cmp(scratch64, Operand(dest64, vixl::ASR, 63));
1427 B(label, NotEqual);
1428 }
1429
decBranchPtr(Condition cond,Register lhs,Imm32 rhs,Label * label)1430 void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
1431 Label* label) {
1432 Subs(ARMRegister(lhs, 64), ARMRegister(lhs, 64), Operand(rhs.value));
1433 B(cond, label);
1434 }
1435
1436 template <class L>
branchTest32(Condition cond,Register lhs,Register rhs,L label)1437 void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
1438 L label) {
1439 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
1440 cond == NotSigned);
1441 // The x86-biased front end prefers |test foo, foo| to |cmp foo, #0|. We look
1442 // for the former pattern and expand as Cbz/Cbnz when possible.
1443 if (lhs == rhs && cond == Zero) {
1444 Cbz(ARMRegister(lhs, 32), label);
1445 } else if (lhs == rhs && cond == NonZero) {
1446 Cbnz(ARMRegister(lhs, 32), label);
1447 } else {
1448 test32(lhs, rhs);
1449 B(label, cond);
1450 }
1451 }
1452
1453 template <class L>
branchTest32(Condition cond,Register lhs,Imm32 rhs,L label)1454 void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
1455 L label) {
1456 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
1457 cond == NotSigned);
1458 test32(lhs, rhs);
1459 B(label, cond);
1460 }
1461
branchTest32(Condition cond,const Address & lhs,Imm32 rhs,Label * label)1462 void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
1463 Label* label) {
1464 vixl::UseScratchRegisterScope temps(this);
1465 const Register scratch = temps.AcquireX().asUnsized();
1466 MOZ_ASSERT(scratch != lhs.base);
1467 load32(lhs, scratch);
1468 branchTest32(cond, scratch, rhs, label);
1469 }
1470
branchTest32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)1471 void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
1472 Imm32 rhs, Label* label) {
1473 vixl::UseScratchRegisterScope temps(this);
1474 const Register scratch = temps.AcquireX().asUnsized();
1475 load32(lhs, scratch);
1476 branchTest32(cond, scratch, rhs, label);
1477 }
1478
1479 template <class L>
branchTestPtr(Condition cond,Register lhs,Register rhs,L label)1480 void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
1481 L label) {
1482 // See branchTest32.
1483 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
1484 cond == NotSigned);
1485 if (lhs == rhs && cond == Zero) {
1486 Cbz(ARMRegister(lhs, 64), label);
1487 } else if (lhs == rhs && cond == NonZero) {
1488 Cbnz(ARMRegister(lhs, 64), label);
1489 } else {
1490 Tst(ARMRegister(lhs, 64), Operand(ARMRegister(rhs, 64)));
1491 B(label, cond);
1492 }
1493 }
1494
branchTestPtr(Condition cond,Register lhs,Imm32 rhs,Label * label)1495 void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
1496 Label* label) {
1497 Tst(ARMRegister(lhs, 64), Operand(rhs.value));
1498 B(label, cond);
1499 }
1500
branchTestPtr(Condition cond,const Address & lhs,Imm32 rhs,Label * label)1501 void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
1502 Imm32 rhs, Label* label) {
1503 vixl::UseScratchRegisterScope temps(this);
1504 const Register scratch = temps.AcquireX().asUnsized();
1505 MOZ_ASSERT(scratch != lhs.base);
1506 loadPtr(lhs, scratch);
1507 branchTestPtr(cond, scratch, rhs, label);
1508 }
1509
1510 template <class L>
branchTest64(Condition cond,Register64 lhs,Register64 rhs,Register temp,L label)1511 void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
1512 Register64 rhs, Register temp, L label) {
1513 branchTestPtr(cond, lhs.reg, rhs.reg, label);
1514 }
1515
branchTestUndefined(Condition cond,Register tag,Label * label)1516 void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
1517 Label* label) {
1518 branchTestUndefinedImpl(cond, tag, label);
1519 }
1520
branchTestUndefined(Condition cond,const Address & address,Label * label)1521 void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
1522 Label* label) {
1523 branchTestUndefinedImpl(cond, address, label);
1524 }
1525
branchTestUndefined(Condition cond,const BaseIndex & address,Label * label)1526 void MacroAssembler::branchTestUndefined(Condition cond,
1527 const BaseIndex& address,
1528 Label* label) {
1529 branchTestUndefinedImpl(cond, address, label);
1530 }
1531
branchTestUndefined(Condition cond,const ValueOperand & value,Label * label)1532 void MacroAssembler::branchTestUndefined(Condition cond,
1533 const ValueOperand& value,
1534 Label* label) {
1535 branchTestUndefinedImpl(cond, value, label);
1536 }
1537
1538 template <typename T>
branchTestUndefinedImpl(Condition cond,const T & t,Label * label)1539 void MacroAssembler::branchTestUndefinedImpl(Condition cond, const T& t,
1540 Label* label) {
1541 Condition c = testUndefined(cond, t);
1542 B(label, c);
1543 }
1544
branchTestInt32(Condition cond,Register tag,Label * label)1545 void MacroAssembler::branchTestInt32(Condition cond, Register tag,
1546 Label* label) {
1547 branchTestInt32Impl(cond, tag, label);
1548 }
1549
branchTestInt32(Condition cond,const Address & address,Label * label)1550 void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
1551 Label* label) {
1552 branchTestInt32Impl(cond, address, label);
1553 }
1554
branchTestInt32(Condition cond,const BaseIndex & address,Label * label)1555 void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
1556 Label* label) {
1557 branchTestInt32Impl(cond, address, label);
1558 }
1559
branchTestInt32(Condition cond,const ValueOperand & value,Label * label)1560 void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
1561 Label* label) {
1562 branchTestInt32Impl(cond, value, label);
1563 }
1564
1565 template <typename T>
branchTestInt32Impl(Condition cond,const T & t,Label * label)1566 void MacroAssembler::branchTestInt32Impl(Condition cond, const T& t,
1567 Label* label) {
1568 Condition c = testInt32(cond, t);
1569 B(label, c);
1570 }
1571
branchTestInt32Truthy(bool truthy,const ValueOperand & value,Label * label)1572 void MacroAssembler::branchTestInt32Truthy(bool truthy,
1573 const ValueOperand& value,
1574 Label* label) {
1575 Condition c = testInt32Truthy(truthy, value);
1576 B(label, c);
1577 }
1578
branchTestDouble(Condition cond,Register tag,Label * label)1579 void MacroAssembler::branchTestDouble(Condition cond, Register tag,
1580 Label* label) {
1581 branchTestDoubleImpl(cond, tag, label);
1582 }
1583
branchTestDouble(Condition cond,const Address & address,Label * label)1584 void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
1585 Label* label) {
1586 branchTestDoubleImpl(cond, address, label);
1587 }
1588
branchTestDouble(Condition cond,const BaseIndex & address,Label * label)1589 void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
1590 Label* label) {
1591 branchTestDoubleImpl(cond, address, label);
1592 }
1593
branchTestDouble(Condition cond,const ValueOperand & value,Label * label)1594 void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
1595 Label* label) {
1596 branchTestDoubleImpl(cond, value, label);
1597 }
1598
1599 template <typename T>
branchTestDoubleImpl(Condition cond,const T & t,Label * label)1600 void MacroAssembler::branchTestDoubleImpl(Condition cond, const T& t,
1601 Label* label) {
1602 Condition c = testDouble(cond, t);
1603 B(label, c);
1604 }
1605
branchTestDoubleTruthy(bool truthy,FloatRegister reg,Label * label)1606 void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
1607 Label* label) {
1608 Fcmp(ARMFPRegister(reg, 64), 0.0);
1609 if (!truthy) {
1610 // falsy values are zero, and NaN.
1611 branch(Zero, label);
1612 branch(Overflow, label);
1613 } else {
1614 // truthy values are non-zero and not nan.
1615 // If it is overflow
1616 Label onFalse;
1617 branch(Zero, &onFalse);
1618 branch(Overflow, &onFalse);
1619 B(label);
1620 bind(&onFalse);
1621 }
1622 }
1623
branchTestNumber(Condition cond,Register tag,Label * label)1624 void MacroAssembler::branchTestNumber(Condition cond, Register tag,
1625 Label* label) {
1626 branchTestNumberImpl(cond, tag, label);
1627 }
1628
branchTestNumber(Condition cond,const ValueOperand & value,Label * label)1629 void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
1630 Label* label) {
1631 branchTestNumberImpl(cond, value, label);
1632 }
1633
1634 template <typename T>
branchTestNumberImpl(Condition cond,const T & t,Label * label)1635 void MacroAssembler::branchTestNumberImpl(Condition cond, const T& t,
1636 Label* label) {
1637 Condition c = testNumber(cond, t);
1638 B(label, c);
1639 }
1640
branchTestBoolean(Condition cond,Register tag,Label * label)1641 void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
1642 Label* label) {
1643 branchTestBooleanImpl(cond, tag, label);
1644 }
1645
branchTestBoolean(Condition cond,const Address & address,Label * label)1646 void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
1647 Label* label) {
1648 branchTestBooleanImpl(cond, address, label);
1649 }
1650
branchTestBoolean(Condition cond,const BaseIndex & address,Label * label)1651 void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
1652 Label* label) {
1653 branchTestBooleanImpl(cond, address, label);
1654 }
1655
branchTestBoolean(Condition cond,const ValueOperand & value,Label * label)1656 void MacroAssembler::branchTestBoolean(Condition cond,
1657 const ValueOperand& value,
1658 Label* label) {
1659 branchTestBooleanImpl(cond, value, label);
1660 }
1661
1662 template <typename T>
branchTestBooleanImpl(Condition cond,const T & tag,Label * label)1663 void MacroAssembler::branchTestBooleanImpl(Condition cond, const T& tag,
1664 Label* label) {
1665 Condition c = testBoolean(cond, tag);
1666 B(label, c);
1667 }
1668
branchTestBooleanTruthy(bool truthy,const ValueOperand & value,Label * label)1669 void MacroAssembler::branchTestBooleanTruthy(bool truthy,
1670 const ValueOperand& value,
1671 Label* label) {
1672 Condition c = testBooleanTruthy(truthy, value);
1673 B(label, c);
1674 }
1675
branchTestString(Condition cond,Register tag,Label * label)1676 void MacroAssembler::branchTestString(Condition cond, Register tag,
1677 Label* label) {
1678 branchTestStringImpl(cond, tag, label);
1679 }
1680
branchTestString(Condition cond,const Address & address,Label * label)1681 void MacroAssembler::branchTestString(Condition cond, const Address& address,
1682 Label* label) {
1683 branchTestStringImpl(cond, address, label);
1684 }
1685
branchTestString(Condition cond,const BaseIndex & address,Label * label)1686 void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
1687 Label* label) {
1688 branchTestStringImpl(cond, address, label);
1689 }
1690
branchTestString(Condition cond,const ValueOperand & value,Label * label)1691 void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
1692 Label* label) {
1693 branchTestStringImpl(cond, value, label);
1694 }
1695
1696 template <typename T>
branchTestStringImpl(Condition cond,const T & t,Label * label)1697 void MacroAssembler::branchTestStringImpl(Condition cond, const T& t,
1698 Label* label) {
1699 Condition c = testString(cond, t);
1700 B(label, c);
1701 }
1702
branchTestStringTruthy(bool truthy,const ValueOperand & value,Label * label)1703 void MacroAssembler::branchTestStringTruthy(bool truthy,
1704 const ValueOperand& value,
1705 Label* label) {
1706 Condition c = testStringTruthy(truthy, value);
1707 B(label, c);
1708 }
1709
branchTestSymbol(Condition cond,Register tag,Label * label)1710 void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
1711 Label* label) {
1712 branchTestSymbolImpl(cond, tag, label);
1713 }
1714
branchTestSymbol(Condition cond,const Address & address,Label * label)1715 void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
1716 Label* label) {
1717 branchTestSymbolImpl(cond, address, label);
1718 }
1719
branchTestSymbol(Condition cond,const BaseIndex & address,Label * label)1720 void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
1721 Label* label) {
1722 branchTestSymbolImpl(cond, address, label);
1723 }
1724
branchTestSymbol(Condition cond,const ValueOperand & value,Label * label)1725 void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
1726 Label* label) {
1727 branchTestSymbolImpl(cond, value, label);
1728 }
1729
1730 template <typename T>
branchTestSymbolImpl(Condition cond,const T & t,Label * label)1731 void MacroAssembler::branchTestSymbolImpl(Condition cond, const T& t,
1732 Label* label) {
1733 Condition c = testSymbol(cond, t);
1734 B(label, c);
1735 }
1736
branchTestBigInt(Condition cond,Register tag,Label * label)1737 void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
1738 Label* label) {
1739 branchTestBigIntImpl(cond, tag, label);
1740 }
1741
branchTestBigInt(Condition cond,const Address & address,Label * label)1742 void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
1743 Label* label) {
1744 branchTestBigIntImpl(cond, address, label);
1745 }
1746
branchTestBigInt(Condition cond,const BaseIndex & address,Label * label)1747 void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
1748 Label* label) {
1749 branchTestBigIntImpl(cond, address, label);
1750 }
1751
branchTestBigInt(Condition cond,const ValueOperand & value,Label * label)1752 void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
1753 Label* label) {
1754 branchTestBigIntImpl(cond, value, label);
1755 }
1756
1757 template <typename T>
branchTestBigIntImpl(Condition cond,const T & t,Label * label)1758 void MacroAssembler::branchTestBigIntImpl(Condition cond, const T& t,
1759 Label* label) {
1760 Condition c = testBigInt(cond, t);
1761 B(label, c);
1762 }
1763
branchTestBigIntTruthy(bool truthy,const ValueOperand & value,Label * label)1764 void MacroAssembler::branchTestBigIntTruthy(bool truthy,
1765 const ValueOperand& value,
1766 Label* label) {
1767 Condition c = testBigIntTruthy(truthy, value);
1768 B(label, c);
1769 }
1770
branchTestNull(Condition cond,Register tag,Label * label)1771 void MacroAssembler::branchTestNull(Condition cond, Register tag,
1772 Label* label) {
1773 branchTestNullImpl(cond, tag, label);
1774 }
1775
branchTestNull(Condition cond,const Address & address,Label * label)1776 void MacroAssembler::branchTestNull(Condition cond, const Address& address,
1777 Label* label) {
1778 branchTestNullImpl(cond, address, label);
1779 }
1780
branchTestNull(Condition cond,const BaseIndex & address,Label * label)1781 void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
1782 Label* label) {
1783 branchTestNullImpl(cond, address, label);
1784 }
1785
branchTestNull(Condition cond,const ValueOperand & value,Label * label)1786 void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
1787 Label* label) {
1788 branchTestNullImpl(cond, value, label);
1789 }
1790
1791 template <typename T>
branchTestNullImpl(Condition cond,const T & t,Label * label)1792 void MacroAssembler::branchTestNullImpl(Condition cond, const T& t,
1793 Label* label) {
1794 Condition c = testNull(cond, t);
1795 B(label, c);
1796 }
1797
branchTestObject(Condition cond,Register tag,Label * label)1798 void MacroAssembler::branchTestObject(Condition cond, Register tag,
1799 Label* label) {
1800 branchTestObjectImpl(cond, tag, label);
1801 }
1802
branchTestObject(Condition cond,const Address & address,Label * label)1803 void MacroAssembler::branchTestObject(Condition cond, const Address& address,
1804 Label* label) {
1805 branchTestObjectImpl(cond, address, label);
1806 }
1807
branchTestObject(Condition cond,const BaseIndex & address,Label * label)1808 void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
1809 Label* label) {
1810 branchTestObjectImpl(cond, address, label);
1811 }
1812
branchTestObject(Condition cond,const ValueOperand & value,Label * label)1813 void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
1814 Label* label) {
1815 branchTestObjectImpl(cond, value, label);
1816 }
1817
1818 template <typename T>
branchTestObjectImpl(Condition cond,const T & t,Label * label)1819 void MacroAssembler::branchTestObjectImpl(Condition cond, const T& t,
1820 Label* label) {
1821 Condition c = testObject(cond, t);
1822 B(label, c);
1823 }
1824
branchTestGCThing(Condition cond,const Address & address,Label * label)1825 void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
1826 Label* label) {
1827 branchTestGCThingImpl(cond, address, label);
1828 }
1829
branchTestGCThing(Condition cond,const BaseIndex & address,Label * label)1830 void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
1831 Label* label) {
1832 branchTestGCThingImpl(cond, address, label);
1833 }
1834
branchTestGCThing(Condition cond,const ValueOperand & value,Label * label)1835 void MacroAssembler::branchTestGCThing(Condition cond,
1836 const ValueOperand& value,
1837 Label* label) {
1838 branchTestGCThingImpl(cond, value, label);
1839 }
1840
1841 template <typename T>
branchTestGCThingImpl(Condition cond,const T & src,Label * label)1842 void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& src,
1843 Label* label) {
1844 Condition c = testGCThing(cond, src);
1845 B(label, c);
1846 }
1847
branchTestPrimitive(Condition cond,Register tag,Label * label)1848 void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
1849 Label* label) {
1850 branchTestPrimitiveImpl(cond, tag, label);
1851 }
1852
branchTestPrimitive(Condition cond,const ValueOperand & value,Label * label)1853 void MacroAssembler::branchTestPrimitive(Condition cond,
1854 const ValueOperand& value,
1855 Label* label) {
1856 branchTestPrimitiveImpl(cond, value, label);
1857 }
1858
1859 template <typename T>
branchTestPrimitiveImpl(Condition cond,const T & t,Label * label)1860 void MacroAssembler::branchTestPrimitiveImpl(Condition cond, const T& t,
1861 Label* label) {
1862 Condition c = testPrimitive(cond, t);
1863 B(label, c);
1864 }
1865
branchTestMagic(Condition cond,Register tag,Label * label)1866 void MacroAssembler::branchTestMagic(Condition cond, Register tag,
1867 Label* label) {
1868 branchTestMagicImpl(cond, tag, label);
1869 }
1870
branchTestMagic(Condition cond,const Address & address,Label * label)1871 void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
1872 Label* label) {
1873 branchTestMagicImpl(cond, address, label);
1874 }
1875
branchTestMagic(Condition cond,const BaseIndex & address,Label * label)1876 void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
1877 Label* label) {
1878 branchTestMagicImpl(cond, address, label);
1879 }
1880
1881 template <class L>
branchTestMagic(Condition cond,const ValueOperand & value,L label)1882 void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
1883 L label) {
1884 branchTestMagicImpl(cond, value, label);
1885 }
1886
1887 template <typename T, class L>
branchTestMagicImpl(Condition cond,const T & t,L label)1888 void MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) {
1889 Condition c = testMagic(cond, t);
1890 B(label, c);
1891 }
1892
branchTestMagic(Condition cond,const Address & valaddr,JSWhyMagic why,Label * label)1893 void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
1894 JSWhyMagic why, Label* label) {
1895 uint64_t magic = MagicValue(why).asRawBits();
1896 cmpPtr(valaddr, ImmWord(magic));
1897 B(label, cond);
1898 }
1899
branchTestValue(Condition cond,const BaseIndex & lhs,const ValueOperand & rhs,Label * label)1900 void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
1901 const ValueOperand& rhs, Label* label) {
1902 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
1903 branchPtr(cond, lhs, rhs.valueReg(), label);
1904 }
1905
1906 template <typename T>
testNumberSet(Condition cond,const T & src,Register dest)1907 void MacroAssembler::testNumberSet(Condition cond, const T& src,
1908 Register dest) {
1909 cond = testNumber(cond, src);
1910 emitSet(cond, dest);
1911 }
1912
1913 template <typename T>
testBooleanSet(Condition cond,const T & src,Register dest)1914 void MacroAssembler::testBooleanSet(Condition cond, const T& src,
1915 Register dest) {
1916 cond = testBoolean(cond, src);
1917 emitSet(cond, dest);
1918 }
1919
1920 template <typename T>
testStringSet(Condition cond,const T & src,Register dest)1921 void MacroAssembler::testStringSet(Condition cond, const T& src,
1922 Register dest) {
1923 cond = testString(cond, src);
1924 emitSet(cond, dest);
1925 }
1926
1927 template <typename T>
testSymbolSet(Condition cond,const T & src,Register dest)1928 void MacroAssembler::testSymbolSet(Condition cond, const T& src,
1929 Register dest) {
1930 cond = testSymbol(cond, src);
1931 emitSet(cond, dest);
1932 }
1933
1934 template <typename T>
testBigIntSet(Condition cond,const T & src,Register dest)1935 void MacroAssembler::testBigIntSet(Condition cond, const T& src,
1936 Register dest) {
1937 cond = testBigInt(cond, src);
1938 emitSet(cond, dest);
1939 }
1940
branchToComputedAddress(const BaseIndex & addr)1941 void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
1942 vixl::UseScratchRegisterScope temps(&this->asVIXL());
1943 const ARMRegister scratch64 = temps.AcquireX();
1944 loadPtr(addr, scratch64.asUnsized());
1945 Br(scratch64);
1946 }
1947
cmp32Move32(Condition cond,Register lhs,Register rhs,Register src,Register dest)1948 void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
1949 Register src, Register dest) {
1950 cmp32(lhs, rhs);
1951 Csel(ARMRegister(dest, 32), ARMRegister(src, 32), ARMRegister(dest, 32),
1952 cond);
1953 }
1954
cmp32Move32(Condition cond,Register lhs,const Address & rhs,Register src,Register dest)1955 void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
1956 const Address& rhs, Register src,
1957 Register dest) {
1958 MOZ_CRASH("NYI");
1959 }
1960
cmpPtrMovePtr(Condition cond,Register lhs,Register rhs,Register src,Register dest)1961 void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
1962 Register src, Register dest) {
1963 cmpPtr(lhs, rhs);
1964 Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
1965 cond);
1966 }
1967
cmpPtrMovePtr(Condition cond,Register lhs,const Address & rhs,Register src,Register dest)1968 void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
1969 const Address& rhs, Register src,
1970 Register dest) {
1971 MOZ_CRASH("NYI");
1972 }
1973
cmp32Load32(Condition cond,Register lhs,const Address & rhs,const Address & src,Register dest)1974 void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
1975 const Address& rhs, const Address& src,
1976 Register dest) {
1977 MOZ_CRASH("NYI");
1978 }
1979
cmp32Load32(Condition cond,Register lhs,Register rhs,const Address & src,Register dest)1980 void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
1981 const Address& src, Register dest) {
1982 MOZ_CRASH("NYI");
1983 }
1984
cmp32MovePtr(Condition cond,Register lhs,Imm32 rhs,Register src,Register dest)1985 void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
1986 Register src, Register dest) {
1987 cmp32(lhs, rhs);
1988 Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
1989 cond);
1990 }
1991
cmp32LoadPtr(Condition cond,const Address & lhs,Imm32 rhs,const Address & src,Register dest)1992 void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
1993 const Address& src, Register dest) {
1994 // ARM64 does not support conditional loads, so we use a branch with a CSel
1995 // (to prevent Spectre attacks).
1996 vixl::UseScratchRegisterScope temps(this);
1997 const ARMRegister scratch64 = temps.AcquireX();
1998
1999 // Can't use branch32() here, because it may select Cbz/Cbnz which don't
2000 // affect condition flags.
2001 Label done;
2002 cmp32(lhs, rhs);
2003 B(&done, Assembler::InvertCondition(cond));
2004
2005 loadPtr(src, scratch64.asUnsized());
2006 Csel(ARMRegister(dest, 64), scratch64, ARMRegister(dest, 64), cond);
2007 bind(&done);
2008 }
2009
test32LoadPtr(Condition cond,const Address & addr,Imm32 mask,const Address & src,Register dest)2010 void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
2011 Imm32 mask, const Address& src,
2012 Register dest) {
2013 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
2014
2015 // ARM64 does not support conditional loads, so we use a branch with a CSel
2016 // (to prevent Spectre attacks).
2017 vixl::UseScratchRegisterScope temps(this);
2018 const ARMRegister scratch64 = temps.AcquireX();
2019 Label done;
2020 branchTest32(Assembler::InvertCondition(cond), addr, mask, &done);
2021 loadPtr(src, scratch64.asUnsized());
2022 Csel(ARMRegister(dest, 64), scratch64, ARMRegister(dest, 64), cond);
2023 bind(&done);
2024 }
2025
test32MovePtr(Condition cond,const Address & addr,Imm32 mask,Register src,Register dest)2026 void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
2027 Imm32 mask, Register src, Register dest) {
2028 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
2029 test32(addr, mask);
2030 Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
2031 cond);
2032 }
2033
spectreMovePtr(Condition cond,Register src,Register dest)2034 void MacroAssembler::spectreMovePtr(Condition cond, Register src,
2035 Register dest) {
2036 Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
2037 cond);
2038 }
2039
spectreZeroRegister(Condition cond,Register,Register dest)2040 void MacroAssembler::spectreZeroRegister(Condition cond, Register,
2041 Register dest) {
2042 Csel(ARMRegister(dest, 64), ARMRegister(dest, 64), vixl::xzr,
2043 Assembler::InvertCondition(cond));
2044 }
2045
spectreBoundsCheck32(Register index,Register length,Register maybeScratch,Label * failure)2046 void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
2047 Register maybeScratch,
2048 Label* failure) {
2049 MOZ_ASSERT(length != maybeScratch);
2050 MOZ_ASSERT(index != maybeScratch);
2051
2052 branch32(Assembler::BelowOrEqual, length, index, failure);
2053
2054 if (JitOptions.spectreIndexMasking) {
2055 Csel(ARMRegister(index, 32), ARMRegister(index, 32), vixl::wzr,
2056 Assembler::Above);
2057 }
2058 }
2059
spectreBoundsCheck32(Register index,const Address & length,Register maybeScratch,Label * failure)2060 void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
2061 Register maybeScratch,
2062 Label* failure) {
2063 MOZ_ASSERT(index != length.base);
2064 MOZ_ASSERT(length.base != maybeScratch);
2065 MOZ_ASSERT(index != maybeScratch);
2066
2067 branch32(Assembler::BelowOrEqual, length, index, failure);
2068
2069 if (JitOptions.spectreIndexMasking) {
2070 Csel(ARMRegister(index, 32), ARMRegister(index, 32), vixl::wzr,
2071 Assembler::Above);
2072 }
2073 }
2074
spectreBoundsCheckPtr(Register index,Register length,Register maybeScratch,Label * failure)2075 void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
2076 Register maybeScratch,
2077 Label* failure) {
2078 MOZ_ASSERT(length != maybeScratch);
2079 MOZ_ASSERT(index != maybeScratch);
2080
2081 branchPtr(Assembler::BelowOrEqual, length, index, failure);
2082
2083 if (JitOptions.spectreIndexMasking) {
2084 Csel(ARMRegister(index, 64), ARMRegister(index, 64), vixl::xzr,
2085 Assembler::Above);
2086 }
2087 }
2088
spectreBoundsCheckPtr(Register index,const Address & length,Register maybeScratch,Label * failure)2089 void MacroAssembler::spectreBoundsCheckPtr(Register index,
2090 const Address& length,
2091 Register maybeScratch,
2092 Label* failure) {
2093 MOZ_ASSERT(index != length.base);
2094 MOZ_ASSERT(length.base != maybeScratch);
2095 MOZ_ASSERT(index != maybeScratch);
2096
2097 branchPtr(Assembler::BelowOrEqual, length, index, failure);
2098
2099 if (JitOptions.spectreIndexMasking) {
2100 Csel(ARMRegister(index, 64), ARMRegister(index, 64), vixl::xzr,
2101 Assembler::Above);
2102 }
2103 }
2104
2105 // ========================================================================
2106 // Memory access primitives.
storeUncanonicalizedDouble(FloatRegister src,const Address & dest)2107 void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
2108 const Address& dest) {
2109 Str(ARMFPRegister(src, 64), toMemOperand(dest));
2110 }
storeUncanonicalizedDouble(FloatRegister src,const BaseIndex & dest)2111 void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
2112 const BaseIndex& dest) {
2113 doBaseIndex(ARMFPRegister(src, 64), dest, vixl::STR_d);
2114 }
2115
storeUncanonicalizedFloat32(FloatRegister src,const Address & addr)2116 void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
2117 const Address& addr) {
2118 Str(ARMFPRegister(src, 32), toMemOperand(addr));
2119 }
storeUncanonicalizedFloat32(FloatRegister src,const BaseIndex & addr)2120 void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
2121 const BaseIndex& addr) {
2122 doBaseIndex(ARMFPRegister(src, 32), addr, vixl::STR_s);
2123 }
2124
memoryBarrier(MemoryBarrierBits barrier)2125 void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
2126 // Bug 1715494: Discriminating barriers such as StoreStore are hard to reason
2127 // about. Execute the full barrier for everything that requires a barrier.
2128 if (barrier) {
2129 Dmb(vixl::InnerShareable, vixl::BarrierAll);
2130 }
2131 }
2132
2133 // ===============================================================
2134 // Clamping functions.
2135
clampIntToUint8(Register reg)2136 void MacroAssembler::clampIntToUint8(Register reg) {
2137 vixl::UseScratchRegisterScope temps(this);
2138 const ARMRegister scratch32 = temps.AcquireW();
2139 const ARMRegister reg32(reg, 32);
2140 MOZ_ASSERT(!scratch32.Is(reg32));
2141
2142 Cmp(reg32, Operand(reg32, vixl::UXTB));
2143 Csel(reg32, reg32, vixl::wzr, Assembler::GreaterThanOrEqual);
2144 Mov(scratch32, Operand(0xff));
2145 Csel(reg32, reg32, scratch32, Assembler::LessThanOrEqual);
2146 }
2147
fallibleUnboxPtr(const ValueOperand & src,Register dest,JSValueType type,Label * fail)2148 void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
2149 JSValueType type, Label* fail) {
2150 MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
2151 type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
2152 // dest := src XOR mask
2153 // fail if dest >> JSVAL_TAG_SHIFT != 0
2154 const ARMRegister src64(src.valueReg(), 64);
2155 const ARMRegister dest64(dest, 64);
2156 Eor(dest64, src64, Operand(JSVAL_TYPE_TO_SHIFTED_TAG(type)));
2157 Cmp(vixl::xzr, Operand(dest64, vixl::LSR, JSVAL_TAG_SHIFT));
2158 j(Assembler::NotEqual, fail);
2159 }
2160
fallibleUnboxPtr(const Address & src,Register dest,JSValueType type,Label * fail)2161 void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
2162 JSValueType type, Label* fail) {
2163 loadValue(src, ValueOperand(dest));
2164 fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
2165 }
2166
fallibleUnboxPtr(const BaseIndex & src,Register dest,JSValueType type,Label * fail)2167 void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
2168 JSValueType type, Label* fail) {
2169 loadValue(src, ValueOperand(dest));
2170 fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
2171 }
2172
2173 //}}} check_macroassembler_style
2174
2175 // Wasm SIMD
2176
SimdReg(FloatRegister r)2177 static inline ARMFPRegister SimdReg(FloatRegister r) {
2178 MOZ_ASSERT(r.isSimd128());
2179 return ARMFPRegister(r, 128);
2180 }
2181
Simd16B(FloatRegister r)2182 static inline ARMFPRegister Simd16B(FloatRegister r) {
2183 return SimdReg(r).V16B();
2184 }
2185
Simd8B(FloatRegister r)2186 static inline ARMFPRegister Simd8B(FloatRegister r) { return SimdReg(r).V8B(); }
2187
Simd8H(FloatRegister r)2188 static inline ARMFPRegister Simd8H(FloatRegister r) { return SimdReg(r).V8H(); }
2189
Simd4H(FloatRegister r)2190 static inline ARMFPRegister Simd4H(FloatRegister r) { return SimdReg(r).V4H(); }
2191
Simd4S(FloatRegister r)2192 static inline ARMFPRegister Simd4S(FloatRegister r) { return SimdReg(r).V4S(); }
2193
Simd2S(FloatRegister r)2194 static inline ARMFPRegister Simd2S(FloatRegister r) { return SimdReg(r).V2S(); }
2195
Simd2D(FloatRegister r)2196 static inline ARMFPRegister Simd2D(FloatRegister r) { return SimdReg(r).V2D(); }
2197
Simd1D(FloatRegister r)2198 static inline ARMFPRegister Simd1D(FloatRegister r) { return SimdReg(r).V1D(); }
2199
SimdQ(FloatRegister r)2200 static inline ARMFPRegister SimdQ(FloatRegister r) { return SimdReg(r).Q(); }
2201
2202 //{{{ check_macroassembler_style
2203
2204 // Moves
2205
moveSimd128(FloatRegister src,FloatRegister dest)2206 void MacroAssembler::moveSimd128(FloatRegister src, FloatRegister dest) {
2207 if (src != dest) {
2208 Mov(SimdReg(dest), SimdReg(src));
2209 }
2210 }
2211
zeroSimd128(FloatRegister dest)2212 void MacroAssembler::zeroSimd128(FloatRegister dest) {
2213 // Unclear what the best code is here, xor is just what we do on x86.
2214 // Alternatives would be `FMOV dest.4s, #0` and `FMOV dest, xzr`.
2215 Eor(Simd16B(dest), Simd16B(dest), Simd16B(dest));
2216 }
2217
loadConstantSimd128(const SimdConstant & v,FloatRegister dest)2218 void MacroAssembler::loadConstantSimd128(const SimdConstant& v,
2219 FloatRegister dest) {
2220 // Movi does not yet generate good code for many cases, bug 1664397.
2221 SimdConstant c = SimdConstant::CreateX2((const int64_t*)v.bytes());
2222 Movi(SimdReg(dest), c.asInt64x2()[1], c.asInt64x2()[0]);
2223 }
2224
2225 // Splat
2226
splatX16(Register src,FloatRegister dest)2227 void MacroAssembler::splatX16(Register src, FloatRegister dest) {
2228 Dup(Simd16B(dest), ARMRegister(src, 32));
2229 }
2230
splatX16(uint32_t srcLane,FloatRegister src,FloatRegister dest)2231 void MacroAssembler::splatX16(uint32_t srcLane, FloatRegister src,
2232 FloatRegister dest) {
2233 Dup(Simd16B(dest), Simd16B(src), srcLane);
2234 }
2235
splatX8(Register src,FloatRegister dest)2236 void MacroAssembler::splatX8(Register src, FloatRegister dest) {
2237 Dup(Simd8H(dest), ARMRegister(src, 32));
2238 }
2239
splatX8(uint32_t srcLane,FloatRegister src,FloatRegister dest)2240 void MacroAssembler::splatX8(uint32_t srcLane, FloatRegister src,
2241 FloatRegister dest) {
2242 Dup(Simd8H(dest), Simd8H(src), srcLane);
2243 }
2244
splatX4(Register src,FloatRegister dest)2245 void MacroAssembler::splatX4(Register src, FloatRegister dest) {
2246 Dup(Simd4S(dest), ARMRegister(src, 32));
2247 }
2248
splatX4(FloatRegister src,FloatRegister dest)2249 void MacroAssembler::splatX4(FloatRegister src, FloatRegister dest) {
2250 Dup(Simd4S(dest), ARMFPRegister(src), 0);
2251 }
2252
splatX2(Register64 src,FloatRegister dest)2253 void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
2254 Dup(Simd2D(dest), ARMRegister(src.reg, 64));
2255 }
2256
splatX2(FloatRegister src,FloatRegister dest)2257 void MacroAssembler::splatX2(FloatRegister src, FloatRegister dest) {
2258 Dup(Simd2D(dest), ARMFPRegister(src), 0);
2259 }
2260
2261 // Extract lane as scalar. Float extraction does not canonicalize the value.
2262
extractLaneInt8x16(uint32_t lane,FloatRegister src,Register dest_)2263 void MacroAssembler::extractLaneInt8x16(uint32_t lane, FloatRegister src,
2264 Register dest_) {
2265 MOZ_ASSERT(lane < 16);
2266 ARMRegister dest(dest_, 32);
2267 Umov(dest, Simd4S(src), lane / 4);
2268 Sbfx(dest, dest, (lane % 4) * 8, 8);
2269 }
2270
unsignedExtractLaneInt8x16(uint32_t lane,FloatRegister src,Register dest_)2271 void MacroAssembler::unsignedExtractLaneInt8x16(uint32_t lane,
2272 FloatRegister src,
2273 Register dest_) {
2274 MOZ_ASSERT(lane < 16);
2275 ARMRegister dest(dest_, 32);
2276 Umov(dest, Simd4S(src), lane / 4);
2277 Ubfx(dest, dest, (lane % 4) * 8, 8);
2278 }
2279
extractLaneInt16x8(uint32_t lane,FloatRegister src,Register dest_)2280 void MacroAssembler::extractLaneInt16x8(uint32_t lane, FloatRegister src,
2281 Register dest_) {
2282 MOZ_ASSERT(lane < 8);
2283 ARMRegister dest(dest_, 32);
2284 Umov(dest, Simd4S(src), lane / 2);
2285 Sbfx(dest, dest, (lane % 2) * 16, 16);
2286 }
2287
unsignedExtractLaneInt16x8(uint32_t lane,FloatRegister src,Register dest_)2288 void MacroAssembler::unsignedExtractLaneInt16x8(uint32_t lane,
2289 FloatRegister src,
2290 Register dest_) {
2291 MOZ_ASSERT(lane < 8);
2292 ARMRegister dest(dest_, 32);
2293 Umov(dest, Simd4S(src), lane / 2);
2294 Ubfx(dest, dest, (lane % 2) * 16, 16);
2295 }
2296
extractLaneInt32x4(uint32_t lane,FloatRegister src,Register dest_)2297 void MacroAssembler::extractLaneInt32x4(uint32_t lane, FloatRegister src,
2298 Register dest_) {
2299 MOZ_ASSERT(lane < 4);
2300 ARMRegister dest(dest_, 32);
2301 Umov(dest, Simd4S(src), lane);
2302 }
2303
extractLaneInt64x2(uint32_t lane,FloatRegister src,Register64 dest_)2304 void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
2305 Register64 dest_) {
2306 MOZ_ASSERT(lane < 2);
2307 ARMRegister dest(dest_.reg, 64);
2308 Umov(dest, Simd2D(src), lane);
2309 }
2310
extractLaneFloat32x4(uint32_t lane,FloatRegister src,FloatRegister dest)2311 void MacroAssembler::extractLaneFloat32x4(uint32_t lane, FloatRegister src,
2312 FloatRegister dest) {
2313 MOZ_ASSERT(lane < 4);
2314 Mov(ARMFPRegister(dest).V4S(), 0, Simd4S(src), lane);
2315 }
2316
extractLaneFloat64x2(uint32_t lane,FloatRegister src,FloatRegister dest)2317 void MacroAssembler::extractLaneFloat64x2(uint32_t lane, FloatRegister src,
2318 FloatRegister dest) {
2319 MOZ_ASSERT(lane < 2);
2320 Mov(ARMFPRegister(dest).V2D(), 0, Simd2D(src), lane);
2321 }
2322
2323 // Replace lane value
2324
replaceLaneInt8x16(unsigned lane,Register rhs,FloatRegister lhsDest)2325 void MacroAssembler::replaceLaneInt8x16(unsigned lane, Register rhs,
2326 FloatRegister lhsDest) {
2327 MOZ_ASSERT(lane < 16);
2328 Mov(Simd16B(lhsDest), lane, ARMRegister(rhs, 32));
2329 }
2330
replaceLaneInt16x8(unsigned lane,Register rhs,FloatRegister lhsDest)2331 void MacroAssembler::replaceLaneInt16x8(unsigned lane, Register rhs,
2332 FloatRegister lhsDest) {
2333 MOZ_ASSERT(lane < 8);
2334 Mov(Simd8H(lhsDest), lane, ARMRegister(rhs, 32));
2335 }
2336
replaceLaneInt32x4(unsigned lane,Register rhs,FloatRegister lhsDest)2337 void MacroAssembler::replaceLaneInt32x4(unsigned lane, Register rhs,
2338 FloatRegister lhsDest) {
2339 MOZ_ASSERT(lane < 4);
2340 Mov(Simd4S(lhsDest), lane, ARMRegister(rhs, 32));
2341 }
2342
replaceLaneInt64x2(unsigned lane,Register64 rhs,FloatRegister lhsDest)2343 void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
2344 FloatRegister lhsDest) {
2345 MOZ_ASSERT(lane < 2);
2346 Mov(Simd2D(lhsDest), lane, ARMRegister(rhs.reg, 64));
2347 }
2348
replaceLaneFloat32x4(unsigned lane,FloatRegister rhs,FloatRegister lhsDest)2349 void MacroAssembler::replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
2350 FloatRegister lhsDest) {
2351 MOZ_ASSERT(lane < 4);
2352 Mov(Simd4S(lhsDest), lane, ARMFPRegister(rhs).V4S(), 0);
2353 }
2354
replaceLaneFloat64x2(unsigned lane,FloatRegister rhs,FloatRegister lhsDest)2355 void MacroAssembler::replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
2356 FloatRegister lhsDest) {
2357 MOZ_ASSERT(lane < 2);
2358 Mov(Simd2D(lhsDest), lane, ARMFPRegister(rhs).V2D(), 0);
2359 }
2360
2361 // Shuffle - blend and permute with immediate indices, and its many
2362 // specializations. Lane values other than those mentioned are illegal.
2363
2364 // lane values 0..31
shuffleInt8x16(const uint8_t lanes[16],FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2365 void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
2366 FloatRegister rhs, FloatRegister dest) {
2367 // The general solution generates ho-hum code. Realistic programs will use
2368 // patterns that can be specialized, and this will be much better. That will
2369 // be handled by bug 1656834, so don't worry about it here.
2370
2371 // Set scratch to the lanevalue when it selects from lhs or ~lanevalue when it
2372 // selects from rhs.
2373 ScratchSimd128Scope scratch(*this);
2374 int8_t idx[16];
2375
2376 if (lhs == rhs) {
2377 for (unsigned i = 0; i < 16; i++) {
2378 idx[i] = lanes[i] < 16 ? lanes[i] : (lanes[i] - 16);
2379 }
2380 loadConstantSimd128(SimdConstant::CreateX16(idx), scratch);
2381 Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(scratch));
2382 return;
2383 }
2384
2385 if (rhs != dest) {
2386 for (unsigned i = 0; i < 16; i++) {
2387 idx[i] = lanes[i] < 16 ? lanes[i] : ~(lanes[i] - 16);
2388 }
2389 } else {
2390 MOZ_ASSERT(lhs != dest);
2391 for (unsigned i = 0; i < 16; i++) {
2392 idx[i] = lanes[i] < 16 ? ~lanes[i] : (lanes[i] - 16);
2393 }
2394 std::swap(lhs, rhs);
2395 }
2396 loadConstantSimd128(SimdConstant::CreateX16(idx), scratch);
2397 Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(scratch));
2398 Not(Simd16B(scratch), Simd16B(scratch));
2399 Tbx(Simd16B(dest), Simd16B(rhs), Simd16B(scratch));
2400 }
2401
shuffleInt8x16(const uint8_t lanes[16],FloatRegister rhs,FloatRegister lhsDest)2402 void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
2403 FloatRegister lhsDest) {
2404 shuffleInt8x16(lanes, lhsDest, rhs, lhsDest);
2405 }
2406
blendInt8x16(const uint8_t lanes[16],FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2407 void MacroAssembler::blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
2408 FloatRegister rhs, FloatRegister dest) {
2409 ScratchSimd128Scope scratch(*this);
2410 int8_t lanes_[16];
2411
2412 if (rhs == dest) {
2413 for (unsigned i = 0; i < 16; i++) {
2414 lanes_[i] = lanes[i] == 0 ? i : 16 + i;
2415 }
2416 loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
2417 Tbx(Simd16B(dest), Simd16B(lhs), Simd16B(scratch));
2418 return;
2419 }
2420
2421 moveSimd128(lhs, dest);
2422 for (unsigned i = 0; i < 16; i++) {
2423 lanes_[i] = lanes[i] != 0 ? i : 16 + i;
2424 }
2425 loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
2426 Tbx(Simd16B(dest), Simd16B(rhs), Simd16B(scratch));
2427 }
2428
blendInt16x8(const uint16_t lanes[8],FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2429 void MacroAssembler::blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
2430 FloatRegister rhs, FloatRegister dest) {
2431 static_assert(sizeof(const uint16_t /*lanes*/[8]) == sizeof(uint8_t[16]));
2432 blendInt8x16(reinterpret_cast<const uint8_t*>(lanes), lhs, rhs, dest);
2433 }
2434
laneSelectSimd128(FloatRegister mask,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2435 void MacroAssembler::laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
2436 FloatRegister rhs, FloatRegister dest) {
2437 MOZ_ASSERT(mask == dest);
2438 Bsl(Simd16B(mask), Simd16B(lhs), Simd16B(rhs));
2439 }
2440
interleaveHighInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2441 void MacroAssembler::interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
2442 FloatRegister dest) {
2443 Zip2(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2444 }
2445
interleaveHighInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2446 void MacroAssembler::interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
2447 FloatRegister dest) {
2448 Zip2(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2449 }
2450
interleaveHighInt64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2451 void MacroAssembler::interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
2452 FloatRegister dest) {
2453 Zip2(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
2454 }
2455
interleaveHighInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2456 void MacroAssembler::interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
2457 FloatRegister dest) {
2458 Zip2(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2459 }
2460
interleaveLowInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2461 void MacroAssembler::interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
2462 FloatRegister dest) {
2463 Zip1(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2464 }
2465
interleaveLowInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2466 void MacroAssembler::interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
2467 FloatRegister dest) {
2468 Zip1(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2469 }
2470
interleaveLowInt64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2471 void MacroAssembler::interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
2472 FloatRegister dest) {
2473 Zip1(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
2474 }
2475
interleaveLowInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2476 void MacroAssembler::interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
2477 FloatRegister dest) {
2478 Zip1(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2479 }
2480
permuteInt8x16(const uint8_t lanes[16],FloatRegister src,FloatRegister dest)2481 void MacroAssembler::permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
2482 FloatRegister dest) {
2483 ScratchSimd128Scope scratch(*this);
2484 loadConstantSimd128(SimdConstant::CreateX16((const int8_t*)lanes), scratch);
2485 Tbl(Simd16B(dest), Simd16B(src), Simd16B(scratch));
2486 }
2487
permuteInt16x8(const uint16_t lanes[8],FloatRegister src,FloatRegister dest)2488 void MacroAssembler::permuteInt16x8(const uint16_t lanes[8], FloatRegister src,
2489 FloatRegister dest) {
2490 MOZ_ASSERT(lanes[0] < 8 && lanes[1] < 8 && lanes[2] < 8 && lanes[3] < 8 &&
2491 lanes[4] < 8 && lanes[5] < 8 && lanes[6] < 8 && lanes[7] < 8);
2492 const int8_t lanes_[16] = {
2493 (int8_t)(lanes[0] << 1), (int8_t)((lanes[0] << 1) + 1),
2494 (int8_t)(lanes[1] << 1), (int8_t)((lanes[1] << 1) + 1),
2495 (int8_t)(lanes[2] << 1), (int8_t)((lanes[2] << 1) + 1),
2496 (int8_t)(lanes[3] << 1), (int8_t)((lanes[3] << 1) + 1),
2497 (int8_t)(lanes[4] << 1), (int8_t)((lanes[4] << 1) + 1),
2498 (int8_t)(lanes[5] << 1), (int8_t)((lanes[5] << 1) + 1),
2499 (int8_t)(lanes[6] << 1), (int8_t)((lanes[6] << 1) + 1),
2500 (int8_t)(lanes[7] << 1), (int8_t)((lanes[7] << 1) + 1),
2501 };
2502 ScratchSimd128Scope scratch(*this);
2503 loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
2504 Tbl(Simd16B(dest), Simd16B(src), Simd16B(scratch));
2505 }
2506
permuteInt32x4(const uint32_t lanes[4],FloatRegister src,FloatRegister dest)2507 void MacroAssembler::permuteInt32x4(const uint32_t lanes[4], FloatRegister src,
2508 FloatRegister dest) {
2509 ScratchSimd128Scope scratch(*this);
2510 const int8_t lanes_[16] = {
2511 (int8_t)(lanes[0] << 2), (int8_t)((lanes[0] << 2) + 1),
2512 (int8_t)((lanes[0] << 2) + 2), (int8_t)((lanes[0] << 2) + 3),
2513 (int8_t)(lanes[1] << 2), (int8_t)((lanes[1] << 2) + 1),
2514 (int8_t)((lanes[1] << 2) + 2), (int8_t)((lanes[1] << 2) + 3),
2515 (int8_t)(lanes[2] << 2), (int8_t)((lanes[2] << 2) + 1),
2516 (int8_t)((lanes[2] << 2) + 2), (int8_t)((lanes[2] << 2) + 3),
2517 (int8_t)(lanes[3] << 2), (int8_t)((lanes[3] << 2) + 1),
2518 (int8_t)((lanes[3] << 2) + 2), (int8_t)((lanes[3] << 2) + 3),
2519 };
2520 loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
2521 Tbl(Simd16B(dest), Simd16B(src), Simd16B(scratch));
2522 }
2523
rotateRightSimd128(FloatRegister src,FloatRegister dest,uint32_t shift)2524 void MacroAssembler::rotateRightSimd128(FloatRegister src, FloatRegister dest,
2525 uint32_t shift) {
2526 Ext(Simd16B(dest), Simd16B(src), Simd16B(src), shift);
2527 }
2528
leftShiftSimd128(Imm32 count,FloatRegister src,FloatRegister dest)2529 void MacroAssembler::leftShiftSimd128(Imm32 count, FloatRegister src,
2530 FloatRegister dest) {
2531 MOZ_ASSERT(count.value < 16);
2532 ScratchSimd128Scope scratch(*this);
2533 Movi(Simd16B(scratch), 0);
2534 Ext(Simd16B(dest), Simd16B(scratch), Simd16B(src), 16 - count.value);
2535 }
2536
rightShiftSimd128(Imm32 count,FloatRegister src,FloatRegister dest)2537 void MacroAssembler::rightShiftSimd128(Imm32 count, FloatRegister src,
2538 FloatRegister dest) {
2539 MOZ_ASSERT(count.value < 16);
2540 ScratchSimd128Scope scratch(*this);
2541 Movi(Simd16B(scratch), 0);
2542 Ext(Simd16B(dest), Simd16B(src), Simd16B(scratch), count.value);
2543 }
2544
concatAndRightShiftSimd128(FloatRegister lhs,FloatRegister rhs,FloatRegister dest,uint32_t shift)2545 void MacroAssembler::concatAndRightShiftSimd128(FloatRegister lhs,
2546 FloatRegister rhs,
2547 FloatRegister dest,
2548 uint32_t shift) {
2549 MOZ_ASSERT(shift < 16);
2550 Ext(Simd16B(dest), Simd16B(rhs), Simd16B(lhs), shift);
2551 }
2552
2553 // Reverse bytes in lanes.
2554
reverseInt16x8(FloatRegister src,FloatRegister dest)2555 void MacroAssembler::reverseInt16x8(FloatRegister src, FloatRegister dest) {
2556 Rev16(Simd16B(dest), Simd16B(src));
2557 }
2558
reverseInt32x4(FloatRegister src,FloatRegister dest)2559 void MacroAssembler::reverseInt32x4(FloatRegister src, FloatRegister dest) {
2560 Rev32(Simd16B(dest), Simd16B(src));
2561 }
2562
reverseInt64x2(FloatRegister src,FloatRegister dest)2563 void MacroAssembler::reverseInt64x2(FloatRegister src, FloatRegister dest) {
2564 Rev64(Simd16B(dest), Simd16B(src));
2565 }
2566
2567 // Swizzle - permute with variable indices. `rhs` holds the lanes parameter.
2568
swizzleInt8x16(FloatRegister rhs,FloatRegister lhsDest)2569 void MacroAssembler::swizzleInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2570 Tbl(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2571 }
2572
swizzleInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2573 void MacroAssembler::swizzleInt8x16(FloatRegister lhs, FloatRegister rhs,
2574 FloatRegister dest) {
2575 Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2576 }
2577
swizzleInt8x16Relaxed(FloatRegister rhs,FloatRegister lhsDest)2578 void MacroAssembler::swizzleInt8x16Relaxed(FloatRegister rhs,
2579 FloatRegister lhsDest) {
2580 Tbl(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2581 }
2582
swizzleInt8x16Relaxed(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2583 void MacroAssembler::swizzleInt8x16Relaxed(FloatRegister lhs, FloatRegister rhs,
2584 FloatRegister dest) {
2585 Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2586 }
2587
2588 // Integer Add
2589
addInt8x16(FloatRegister rhs,FloatRegister lhsDest)2590 void MacroAssembler::addInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2591 Add(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2592 }
2593
addInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2594 void MacroAssembler::addInt8x16(FloatRegister lhs, FloatRegister rhs,
2595 FloatRegister dest) {
2596 Add(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2597 }
2598
addInt16x8(FloatRegister rhs,FloatRegister lhsDest)2599 void MacroAssembler::addInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
2600 Add(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2601 }
2602
addInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2603 void MacroAssembler::addInt16x8(FloatRegister lhs, FloatRegister rhs,
2604 FloatRegister dest) {
2605 Add(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2606 }
2607
addInt32x4(FloatRegister rhs,FloatRegister lhsDest)2608 void MacroAssembler::addInt32x4(FloatRegister rhs, FloatRegister lhsDest) {
2609 Add(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2610 }
2611
addInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2612 void MacroAssembler::addInt32x4(FloatRegister lhs, FloatRegister rhs,
2613 FloatRegister dest) {
2614 Add(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2615 }
2616
addInt64x2(FloatRegister rhs,FloatRegister lhsDest)2617 void MacroAssembler::addInt64x2(FloatRegister rhs, FloatRegister lhsDest) {
2618 Add(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
2619 }
2620
addInt64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2621 void MacroAssembler::addInt64x2(FloatRegister lhs, FloatRegister rhs,
2622 FloatRegister dest) {
2623 Add(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
2624 }
2625
2626 // Integer Subtract
2627
subInt8x16(FloatRegister rhs,FloatRegister lhsDest)2628 void MacroAssembler::subInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2629 Sub(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2630 }
2631
subInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2632 void MacroAssembler::subInt8x16(FloatRegister lhs, FloatRegister rhs,
2633 FloatRegister dest) {
2634 Sub(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2635 }
2636
subInt16x8(FloatRegister rhs,FloatRegister lhsDest)2637 void MacroAssembler::subInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
2638 Sub(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2639 }
2640
subInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2641 void MacroAssembler::subInt16x8(FloatRegister lhs, FloatRegister rhs,
2642 FloatRegister dest) {
2643 Sub(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2644 }
2645
subInt32x4(FloatRegister rhs,FloatRegister lhsDest)2646 void MacroAssembler::subInt32x4(FloatRegister rhs, FloatRegister lhsDest) {
2647 Sub(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2648 }
2649
subInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2650 void MacroAssembler::subInt32x4(FloatRegister lhs, FloatRegister rhs,
2651 FloatRegister dest) {
2652 Sub(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2653 }
2654
subInt64x2(FloatRegister rhs,FloatRegister lhsDest)2655 void MacroAssembler::subInt64x2(FloatRegister rhs, FloatRegister lhsDest) {
2656 Sub(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
2657 }
2658
subInt64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2659 void MacroAssembler::subInt64x2(FloatRegister lhs, FloatRegister rhs,
2660 FloatRegister dest) {
2661 Sub(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
2662 }
2663
2664 // Integer Multiply
2665
mulInt16x8(FloatRegister rhs,FloatRegister lhsDest)2666 void MacroAssembler::mulInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
2667 Mul(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2668 }
2669
mulInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2670 void MacroAssembler::mulInt16x8(FloatRegister lhs, FloatRegister rhs,
2671 FloatRegister dest) {
2672 Mul(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2673 }
2674
mulInt32x4(FloatRegister rhs,FloatRegister lhsDest)2675 void MacroAssembler::mulInt32x4(FloatRegister rhs, FloatRegister lhsDest) {
2676 Mul(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2677 }
2678
mulInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2679 void MacroAssembler::mulInt32x4(FloatRegister lhs, FloatRegister rhs,
2680 FloatRegister dest) {
2681 Mul(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2682 }
2683
mulInt64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest,FloatRegister temp1,FloatRegister temp2)2684 void MacroAssembler::mulInt64x2(FloatRegister lhs, FloatRegister rhs,
2685 FloatRegister dest, FloatRegister temp1,
2686 FloatRegister temp2) {
2687 // As documented at https://chromium-review.googlesource.com/c/v8/v8/+/1781696
2688 // lhs = <D C> <B A>
2689 // rhs = <H G> <F E>
2690 // result = <(DG+CH)_low+CG_high CG_low> <(BE+AF)_low+AE_high AE_low>
2691 ScratchSimd128Scope scratch(*this);
2692 Rev64(Simd4S(temp2), Simd4S(lhs)); // temp2 = <C D> <A B>
2693 Mul(Simd4S(temp2), Simd4S(temp2), Simd4S(rhs)); // temp2 = <CH DG> <AF BE>
2694 Xtn(Simd2S(temp1), Simd2D(rhs)); // temp1 = <0 0> <G E>
2695 Addp(Simd4S(temp2), Simd4S(temp2), Simd4S(temp2)); // temp2 = <CH+DG AF+BE>..
2696 Xtn(Simd2S(scratch), Simd2D(lhs)); // scratch = <0 0> <C A>
2697 Shll(Simd2D(dest), Simd2S(temp2), 32); // dest = <(DG+CH)_low 0>
2698 // <(BE+AF)_low 0>
2699 Umlal(Simd2D(dest), Simd2S(scratch), Simd2S(temp1));
2700 }
2701
extMulLowInt8x16(FloatRegister rhs,FloatRegister lhsDest)2702 void MacroAssembler::extMulLowInt8x16(FloatRegister rhs,
2703 FloatRegister lhsDest) {
2704 Smull(Simd8H(lhsDest), Simd8B(lhsDest), Simd8B(rhs));
2705 }
2706
extMulLowInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2707 void MacroAssembler::extMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
2708 FloatRegister dest) {
2709 Smull(Simd8H(dest), Simd8B(lhs), Simd8B(rhs));
2710 }
2711
extMulHighInt8x16(FloatRegister rhs,FloatRegister lhsDest)2712 void MacroAssembler::extMulHighInt8x16(FloatRegister rhs,
2713 FloatRegister lhsDest) {
2714 Smull2(Simd8H(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2715 }
2716
extMulHighInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2717 void MacroAssembler::extMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
2718 FloatRegister dest) {
2719 Smull2(Simd8H(dest), Simd16B(lhs), Simd16B(rhs));
2720 }
2721
unsignedExtMulLowInt8x16(FloatRegister rhs,FloatRegister lhsDest)2722 void MacroAssembler::unsignedExtMulLowInt8x16(FloatRegister rhs,
2723 FloatRegister lhsDest) {
2724 Umull(Simd8H(lhsDest), Simd8B(lhsDest), Simd8B(rhs));
2725 }
2726
unsignedExtMulLowInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2727 void MacroAssembler::unsignedExtMulLowInt8x16(FloatRegister lhs,
2728 FloatRegister rhs,
2729 FloatRegister dest) {
2730 Umull(Simd8H(dest), Simd8B(lhs), Simd8B(rhs));
2731 }
2732
unsignedExtMulHighInt8x16(FloatRegister rhs,FloatRegister lhsDest)2733 void MacroAssembler::unsignedExtMulHighInt8x16(FloatRegister rhs,
2734 FloatRegister lhsDest) {
2735 Umull2(Simd8H(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2736 }
2737
unsignedExtMulHighInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2738 void MacroAssembler::unsignedExtMulHighInt8x16(FloatRegister lhs,
2739 FloatRegister rhs,
2740 FloatRegister dest) {
2741 Umull2(Simd8H(dest), Simd16B(lhs), Simd16B(rhs));
2742 }
2743
extMulLowInt16x8(FloatRegister rhs,FloatRegister lhsDest)2744 void MacroAssembler::extMulLowInt16x8(FloatRegister rhs,
2745 FloatRegister lhsDest) {
2746 Smull(Simd4S(lhsDest), Simd4H(lhsDest), Simd4H(rhs));
2747 }
2748
extMulLowInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2749 void MacroAssembler::extMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
2750 FloatRegister dest) {
2751 Smull(Simd4S(dest), Simd4H(lhs), Simd4H(rhs));
2752 }
2753
extMulHighInt16x8(FloatRegister rhs,FloatRegister lhsDest)2754 void MacroAssembler::extMulHighInt16x8(FloatRegister rhs,
2755 FloatRegister lhsDest) {
2756 Smull2(Simd4S(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2757 }
2758
extMulHighInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2759 void MacroAssembler::extMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
2760 FloatRegister dest) {
2761 Smull2(Simd4S(dest), Simd8H(lhs), Simd8H(rhs));
2762 }
2763
unsignedExtMulLowInt16x8(FloatRegister rhs,FloatRegister lhsDest)2764 void MacroAssembler::unsignedExtMulLowInt16x8(FloatRegister rhs,
2765 FloatRegister lhsDest) {
2766 Umull(Simd4S(lhsDest), Simd4H(lhsDest), Simd4H(rhs));
2767 }
2768
unsignedExtMulLowInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2769 void MacroAssembler::unsignedExtMulLowInt16x8(FloatRegister lhs,
2770 FloatRegister rhs,
2771 FloatRegister dest) {
2772 Umull(Simd4S(dest), Simd4H(lhs), Simd4H(rhs));
2773 }
2774
unsignedExtMulHighInt16x8(FloatRegister rhs,FloatRegister lhsDest)2775 void MacroAssembler::unsignedExtMulHighInt16x8(FloatRegister rhs,
2776 FloatRegister lhsDest) {
2777 Umull2(Simd4S(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2778 }
2779
unsignedExtMulHighInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2780 void MacroAssembler::unsignedExtMulHighInt16x8(FloatRegister lhs,
2781 FloatRegister rhs,
2782 FloatRegister dest) {
2783 Umull2(Simd4S(dest), Simd8H(lhs), Simd8H(rhs));
2784 }
2785
extMulLowInt32x4(FloatRegister rhs,FloatRegister lhsDest)2786 void MacroAssembler::extMulLowInt32x4(FloatRegister rhs,
2787 FloatRegister lhsDest) {
2788 Smull(Simd2D(lhsDest), Simd2S(lhsDest), Simd2S(rhs));
2789 }
2790
extMulLowInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2791 void MacroAssembler::extMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
2792 FloatRegister dest) {
2793 Smull(Simd2D(dest), Simd2S(lhs), Simd2S(rhs));
2794 }
2795
extMulHighInt32x4(FloatRegister rhs,FloatRegister lhsDest)2796 void MacroAssembler::extMulHighInt32x4(FloatRegister rhs,
2797 FloatRegister lhsDest) {
2798 Smull2(Simd2D(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2799 }
2800
extMulHighInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2801 void MacroAssembler::extMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
2802 FloatRegister dest) {
2803 Smull2(Simd2D(dest), Simd4S(lhs), Simd4S(rhs));
2804 }
2805
unsignedExtMulLowInt32x4(FloatRegister rhs,FloatRegister lhsDest)2806 void MacroAssembler::unsignedExtMulLowInt32x4(FloatRegister rhs,
2807 FloatRegister lhsDest) {
2808 Umull(Simd2D(lhsDest), Simd2S(lhsDest), Simd2S(rhs));
2809 }
2810
unsignedExtMulLowInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2811 void MacroAssembler::unsignedExtMulLowInt32x4(FloatRegister lhs,
2812 FloatRegister rhs,
2813 FloatRegister dest) {
2814 Umull(Simd2D(dest), Simd2S(lhs), Simd2S(rhs));
2815 }
2816
unsignedExtMulHighInt32x4(FloatRegister rhs,FloatRegister lhsDest)2817 void MacroAssembler::unsignedExtMulHighInt32x4(FloatRegister rhs,
2818 FloatRegister lhsDest) {
2819 Umull2(Simd2D(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2820 }
2821
unsignedExtMulHighInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2822 void MacroAssembler::unsignedExtMulHighInt32x4(FloatRegister lhs,
2823 FloatRegister rhs,
2824 FloatRegister dest) {
2825 Umull2(Simd2D(dest), Simd4S(lhs), Simd4S(rhs));
2826 }
2827
q15MulrSatInt16x8(FloatRegister rhs,FloatRegister lhsDest)2828 void MacroAssembler::q15MulrSatInt16x8(FloatRegister rhs,
2829 FloatRegister lhsDest) {
2830 Sqrdmulh(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2831 }
2832
q15MulrSatInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2833 void MacroAssembler::q15MulrSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2834 FloatRegister dest) {
2835 Sqrdmulh(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2836 }
2837
2838 // Integer Negate
2839
negInt8x16(FloatRegister src,FloatRegister dest)2840 void MacroAssembler::negInt8x16(FloatRegister src, FloatRegister dest) {
2841 Neg(Simd16B(dest), Simd16B(src));
2842 }
2843
negInt16x8(FloatRegister src,FloatRegister dest)2844 void MacroAssembler::negInt16x8(FloatRegister src, FloatRegister dest) {
2845 Neg(Simd8H(dest), Simd8H(src));
2846 }
2847
negInt32x4(FloatRegister src,FloatRegister dest)2848 void MacroAssembler::negInt32x4(FloatRegister src, FloatRegister dest) {
2849 Neg(Simd4S(dest), Simd4S(src));
2850 }
2851
negInt64x2(FloatRegister src,FloatRegister dest)2852 void MacroAssembler::negInt64x2(FloatRegister src, FloatRegister dest) {
2853 Neg(Simd2D(dest), Simd2D(src));
2854 }
2855
2856 // Saturating integer add
2857
addSatInt8x16(FloatRegister rhs,FloatRegister lhsDest)2858 void MacroAssembler::addSatInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2859 Sqadd(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2860 }
2861
addSatInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2862 void MacroAssembler::addSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2863 FloatRegister dest) {
2864 Sqadd(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2865 }
2866
unsignedAddSatInt8x16(FloatRegister rhs,FloatRegister lhsDest)2867 void MacroAssembler::unsignedAddSatInt8x16(FloatRegister rhs,
2868 FloatRegister lhsDest) {
2869 Uqadd(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2870 }
2871
unsignedAddSatInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2872 void MacroAssembler::unsignedAddSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2873 FloatRegister dest) {
2874 Uqadd(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2875 }
2876
addSatInt16x8(FloatRegister rhs,FloatRegister lhsDest)2877 void MacroAssembler::addSatInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
2878 Sqadd(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2879 }
2880
addSatInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2881 void MacroAssembler::addSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2882 FloatRegister dest) {
2883 Sqadd(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2884 }
2885
unsignedAddSatInt16x8(FloatRegister rhs,FloatRegister lhsDest)2886 void MacroAssembler::unsignedAddSatInt16x8(FloatRegister rhs,
2887 FloatRegister lhsDest) {
2888 Uqadd(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2889 }
2890
unsignedAddSatInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2891 void MacroAssembler::unsignedAddSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2892 FloatRegister dest) {
2893 Uqadd(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2894 }
2895
2896 // Saturating integer subtract
2897
subSatInt8x16(FloatRegister rhs,FloatRegister lhsDest)2898 void MacroAssembler::subSatInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2899 Sqsub(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2900 }
2901
subSatInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2902 void MacroAssembler::subSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2903 FloatRegister dest) {
2904 Sqsub(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2905 }
2906
unsignedSubSatInt8x16(FloatRegister rhs,FloatRegister lhsDest)2907 void MacroAssembler::unsignedSubSatInt8x16(FloatRegister rhs,
2908 FloatRegister lhsDest) {
2909 Uqsub(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2910 }
2911
unsignedSubSatInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2912 void MacroAssembler::unsignedSubSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2913 FloatRegister dest) {
2914 Uqsub(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2915 }
2916
subSatInt16x8(FloatRegister rhs,FloatRegister lhsDest)2917 void MacroAssembler::subSatInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
2918 Sqsub(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2919 }
2920
subSatInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2921 void MacroAssembler::subSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2922 FloatRegister dest) {
2923 Sqsub(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2924 }
2925
unsignedSubSatInt16x8(FloatRegister rhs,FloatRegister lhsDest)2926 void MacroAssembler::unsignedSubSatInt16x8(FloatRegister rhs,
2927 FloatRegister lhsDest) {
2928 Uqsub(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2929 }
2930
unsignedSubSatInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2931 void MacroAssembler::unsignedSubSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2932 FloatRegister dest) {
2933 Uqsub(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2934 }
2935
2936 // Lane-wise integer minimum
2937
minInt8x16(FloatRegister rhs,FloatRegister lhsDest)2938 void MacroAssembler::minInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2939 Smin(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2940 }
2941
minInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2942 void MacroAssembler::minInt8x16(FloatRegister lhs, FloatRegister rhs,
2943 FloatRegister dest) {
2944 Smin(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2945 }
2946
unsignedMinInt8x16(FloatRegister rhs,FloatRegister lhsDest)2947 void MacroAssembler::unsignedMinInt8x16(FloatRegister rhs,
2948 FloatRegister lhsDest) {
2949 Umin(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2950 }
2951
unsignedMinInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2952 void MacroAssembler::unsignedMinInt8x16(FloatRegister lhs, FloatRegister rhs,
2953 FloatRegister dest) {
2954 Umin(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
2955 }
2956
minInt16x8(FloatRegister rhs,FloatRegister lhsDest)2957 void MacroAssembler::minInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
2958 Smin(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2959 }
2960
minInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2961 void MacroAssembler::minInt16x8(FloatRegister lhs, FloatRegister rhs,
2962 FloatRegister dest) {
2963 Smin(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2964 }
2965
unsignedMinInt16x8(FloatRegister rhs,FloatRegister lhsDest)2966 void MacroAssembler::unsignedMinInt16x8(FloatRegister rhs,
2967 FloatRegister lhsDest) {
2968 Umin(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
2969 }
2970
unsignedMinInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2971 void MacroAssembler::unsignedMinInt16x8(FloatRegister lhs, FloatRegister rhs,
2972 FloatRegister dest) {
2973 Umin(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
2974 }
2975
minInt32x4(FloatRegister rhs,FloatRegister lhsDest)2976 void MacroAssembler::minInt32x4(FloatRegister rhs, FloatRegister lhsDest) {
2977 Smin(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2978 }
2979
minInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2980 void MacroAssembler::minInt32x4(FloatRegister lhs, FloatRegister rhs,
2981 FloatRegister dest) {
2982 Smin(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2983 }
2984
unsignedMinInt32x4(FloatRegister rhs,FloatRegister lhsDest)2985 void MacroAssembler::unsignedMinInt32x4(FloatRegister rhs,
2986 FloatRegister lhsDest) {
2987 Umin(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
2988 }
2989
unsignedMinInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)2990 void MacroAssembler::unsignedMinInt32x4(FloatRegister lhs, FloatRegister rhs,
2991 FloatRegister dest) {
2992 Umin(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
2993 }
2994
2995 // Lane-wise integer maximum
2996
maxInt8x16(FloatRegister rhs,FloatRegister lhsDest)2997 void MacroAssembler::maxInt8x16(FloatRegister rhs, FloatRegister lhsDest) {
2998 Smax(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
2999 }
3000
maxInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3001 void MacroAssembler::maxInt8x16(FloatRegister lhs, FloatRegister rhs,
3002 FloatRegister dest) {
3003 Smax(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3004 }
3005
unsignedMaxInt8x16(FloatRegister rhs,FloatRegister lhsDest)3006 void MacroAssembler::unsignedMaxInt8x16(FloatRegister rhs,
3007 FloatRegister lhsDest) {
3008 Umax(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
3009 }
3010
unsignedMaxInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3011 void MacroAssembler::unsignedMaxInt8x16(FloatRegister lhs, FloatRegister rhs,
3012 FloatRegister dest) {
3013 Umax(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3014 }
3015
maxInt16x8(FloatRegister rhs,FloatRegister lhsDest)3016 void MacroAssembler::maxInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
3017 Smax(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
3018 }
3019
maxInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3020 void MacroAssembler::maxInt16x8(FloatRegister lhs, FloatRegister rhs,
3021 FloatRegister dest) {
3022 Smax(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
3023 }
3024
unsignedMaxInt16x8(FloatRegister rhs,FloatRegister lhsDest)3025 void MacroAssembler::unsignedMaxInt16x8(FloatRegister rhs,
3026 FloatRegister lhsDest) {
3027 Umax(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
3028 }
3029
unsignedMaxInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3030 void MacroAssembler::unsignedMaxInt16x8(FloatRegister lhs, FloatRegister rhs,
3031 FloatRegister dest) {
3032 Umax(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
3033 }
3034
maxInt32x4(FloatRegister rhs,FloatRegister lhsDest)3035 void MacroAssembler::maxInt32x4(FloatRegister rhs, FloatRegister lhsDest) {
3036 Smax(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3037 }
3038
maxInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3039 void MacroAssembler::maxInt32x4(FloatRegister lhs, FloatRegister rhs,
3040 FloatRegister dest) {
3041 Smax(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3042 }
3043
unsignedMaxInt32x4(FloatRegister rhs,FloatRegister lhsDest)3044 void MacroAssembler::unsignedMaxInt32x4(FloatRegister rhs,
3045 FloatRegister lhsDest) {
3046 Umax(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3047 }
3048
unsignedMaxInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3049 void MacroAssembler::unsignedMaxInt32x4(FloatRegister lhs, FloatRegister rhs,
3050 FloatRegister dest) {
3051 Umax(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3052 }
3053
3054 // Lane-wise integer rounding average
3055
unsignedAverageInt8x16(FloatRegister rhs,FloatRegister lhsDest)3056 void MacroAssembler::unsignedAverageInt8x16(FloatRegister rhs,
3057 FloatRegister lhsDest) {
3058 Urhadd(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
3059 }
3060
unsignedAverageInt8x16(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3061 void MacroAssembler::unsignedAverageInt8x16(FloatRegister lhs,
3062 FloatRegister rhs,
3063 FloatRegister dest) {
3064 Urhadd(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3065 }
3066
unsignedAverageInt16x8(FloatRegister rhs,FloatRegister lhsDest)3067 void MacroAssembler::unsignedAverageInt16x8(FloatRegister rhs,
3068 FloatRegister lhsDest) {
3069 Urhadd(Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
3070 }
3071
unsignedAverageInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3072 void MacroAssembler::unsignedAverageInt16x8(FloatRegister lhs,
3073 FloatRegister rhs,
3074 FloatRegister dest) {
3075 Urhadd(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
3076 }
3077
3078 // Lane-wise integer absolute value
3079
absInt8x16(FloatRegister src,FloatRegister dest)3080 void MacroAssembler::absInt8x16(FloatRegister src, FloatRegister dest) {
3081 Abs(Simd16B(dest), Simd16B(src));
3082 }
3083
absInt16x8(FloatRegister src,FloatRegister dest)3084 void MacroAssembler::absInt16x8(FloatRegister src, FloatRegister dest) {
3085 Abs(Simd8H(dest), Simd8H(src));
3086 }
3087
absInt32x4(FloatRegister src,FloatRegister dest)3088 void MacroAssembler::absInt32x4(FloatRegister src, FloatRegister dest) {
3089 Abs(Simd4S(dest), Simd4S(src));
3090 }
3091
absInt64x2(FloatRegister src,FloatRegister dest)3092 void MacroAssembler::absInt64x2(FloatRegister src, FloatRegister dest) {
3093 Abs(Simd2D(dest), Simd2D(src));
3094 }
3095
3096 // Left shift by variable scalar
3097
leftShiftInt8x16(FloatRegister lhs,Register rhs,FloatRegister dest)3098 void MacroAssembler::leftShiftInt8x16(FloatRegister lhs, Register rhs,
3099 FloatRegister dest) {
3100 ScratchSimd128Scope vscratch(*this);
3101 Dup(Simd16B(vscratch), ARMRegister(rhs, 32));
3102 Sshl(Simd16B(dest), Simd16B(lhs), Simd16B(vscratch));
3103 }
3104
leftShiftInt8x16(Imm32 count,FloatRegister src,FloatRegister dest)3105 void MacroAssembler::leftShiftInt8x16(Imm32 count, FloatRegister src,
3106 FloatRegister dest) {
3107 Shl(Simd16B(dest), Simd16B(src), count.value);
3108 }
3109
leftShiftInt16x8(FloatRegister lhs,Register rhs,FloatRegister dest)3110 void MacroAssembler::leftShiftInt16x8(FloatRegister lhs, Register rhs,
3111 FloatRegister dest) {
3112 ScratchSimd128Scope vscratch(*this);
3113 Dup(Simd8H(vscratch), ARMRegister(rhs, 32));
3114 Sshl(Simd8H(dest), Simd8H(lhs), Simd8H(vscratch));
3115 }
3116
leftShiftInt16x8(Imm32 count,FloatRegister src,FloatRegister dest)3117 void MacroAssembler::leftShiftInt16x8(Imm32 count, FloatRegister src,
3118 FloatRegister dest) {
3119 Shl(Simd8H(dest), Simd8H(src), count.value);
3120 }
3121
leftShiftInt32x4(FloatRegister lhs,Register rhs,FloatRegister dest)3122 void MacroAssembler::leftShiftInt32x4(FloatRegister lhs, Register rhs,
3123 FloatRegister dest) {
3124 ScratchSimd128Scope vscratch(*this);
3125 Dup(Simd4S(vscratch), ARMRegister(rhs, 32));
3126 Sshl(Simd4S(dest), Simd4S(lhs), Simd4S(vscratch));
3127 }
3128
leftShiftInt32x4(Imm32 count,FloatRegister src,FloatRegister dest)3129 void MacroAssembler::leftShiftInt32x4(Imm32 count, FloatRegister src,
3130 FloatRegister dest) {
3131 Shl(Simd4S(dest), Simd4S(src), count.value);
3132 }
3133
leftShiftInt64x2(FloatRegister lhs,Register rhs,FloatRegister dest)3134 void MacroAssembler::leftShiftInt64x2(FloatRegister lhs, Register rhs,
3135 FloatRegister dest) {
3136 ScratchSimd128Scope vscratch(*this);
3137 Dup(Simd2D(vscratch), ARMRegister(rhs, 64));
3138 Sshl(Simd2D(dest), Simd2D(lhs), Simd2D(vscratch));
3139 }
3140
leftShiftInt64x2(Imm32 count,FloatRegister src,FloatRegister dest)3141 void MacroAssembler::leftShiftInt64x2(Imm32 count, FloatRegister src,
3142 FloatRegister dest) {
3143 Shl(Simd2D(dest), Simd2D(src), count.value);
3144 }
3145
3146 // Right shift by variable scalar
3147
rightShiftInt8x16(FloatRegister lhs,Register rhs,FloatRegister dest)3148 void MacroAssembler::rightShiftInt8x16(FloatRegister lhs, Register rhs,
3149 FloatRegister dest) {
3150 MacroAssemblerCompat::rightShiftInt8x16(lhs, rhs, dest,
3151 /* isUnsigned */ false);
3152 }
3153
rightShiftInt8x16(Imm32 count,FloatRegister src,FloatRegister dest)3154 void MacroAssembler::rightShiftInt8x16(Imm32 count, FloatRegister src,
3155 FloatRegister dest) {
3156 Sshr(Simd16B(dest), Simd16B(src), count.value);
3157 }
3158
unsignedRightShiftInt8x16(FloatRegister lhs,Register rhs,FloatRegister dest)3159 void MacroAssembler::unsignedRightShiftInt8x16(FloatRegister lhs, Register rhs,
3160 FloatRegister dest) {
3161 MacroAssemblerCompat::rightShiftInt8x16(lhs, rhs, dest,
3162 /* isUnsigned */ true);
3163 }
3164
unsignedRightShiftInt8x16(Imm32 count,FloatRegister src,FloatRegister dest)3165 void MacroAssembler::unsignedRightShiftInt8x16(Imm32 count, FloatRegister src,
3166 FloatRegister dest) {
3167 Ushr(Simd16B(dest), Simd16B(src), count.value);
3168 }
3169
rightShiftInt16x8(FloatRegister lhs,Register rhs,FloatRegister dest)3170 void MacroAssembler::rightShiftInt16x8(FloatRegister lhs, Register rhs,
3171 FloatRegister dest) {
3172 MacroAssemblerCompat::rightShiftInt16x8(lhs, rhs, dest,
3173 /* isUnsigned */ false);
3174 }
3175
rightShiftInt16x8(Imm32 count,FloatRegister src,FloatRegister dest)3176 void MacroAssembler::rightShiftInt16x8(Imm32 count, FloatRegister src,
3177 FloatRegister dest) {
3178 Sshr(Simd8H(dest), Simd8H(src), count.value);
3179 }
3180
unsignedRightShiftInt16x8(FloatRegister lhs,Register rhs,FloatRegister dest)3181 void MacroAssembler::unsignedRightShiftInt16x8(FloatRegister lhs, Register rhs,
3182 FloatRegister dest) {
3183 MacroAssemblerCompat::rightShiftInt16x8(lhs, rhs, dest,
3184 /* isUnsigned */ true);
3185 }
3186
unsignedRightShiftInt16x8(Imm32 count,FloatRegister src,FloatRegister dest)3187 void MacroAssembler::unsignedRightShiftInt16x8(Imm32 count, FloatRegister src,
3188 FloatRegister dest) {
3189 Ushr(Simd8H(dest), Simd8H(src), count.value);
3190 }
3191
rightShiftInt32x4(FloatRegister lhs,Register rhs,FloatRegister dest)3192 void MacroAssembler::rightShiftInt32x4(FloatRegister lhs, Register rhs,
3193 FloatRegister dest) {
3194 MacroAssemblerCompat::rightShiftInt32x4(lhs, rhs, dest,
3195 /* isUnsigned */ false);
3196 }
3197
rightShiftInt32x4(Imm32 count,FloatRegister src,FloatRegister dest)3198 void MacroAssembler::rightShiftInt32x4(Imm32 count, FloatRegister src,
3199 FloatRegister dest) {
3200 Sshr(Simd4S(dest), Simd4S(src), count.value);
3201 }
3202
unsignedRightShiftInt32x4(FloatRegister lhs,Register rhs,FloatRegister dest)3203 void MacroAssembler::unsignedRightShiftInt32x4(FloatRegister lhs, Register rhs,
3204 FloatRegister dest) {
3205 MacroAssemblerCompat::rightShiftInt32x4(lhs, rhs, dest,
3206 /* isUnsigned */ true);
3207 }
3208
unsignedRightShiftInt32x4(Imm32 count,FloatRegister src,FloatRegister dest)3209 void MacroAssembler::unsignedRightShiftInt32x4(Imm32 count, FloatRegister src,
3210 FloatRegister dest) {
3211 Ushr(Simd4S(dest), Simd4S(src), count.value);
3212 }
3213
rightShiftInt64x2(FloatRegister lhs,Register rhs,FloatRegister dest)3214 void MacroAssembler::rightShiftInt64x2(FloatRegister lhs, Register rhs,
3215 FloatRegister dest) {
3216 MacroAssemblerCompat::rightShiftInt64x2(lhs, rhs, dest,
3217 /* isUnsigned */ false);
3218 }
3219
rightShiftInt64x2(Imm32 count,FloatRegister src,FloatRegister dest)3220 void MacroAssembler::rightShiftInt64x2(Imm32 count, FloatRegister src,
3221 FloatRegister dest) {
3222 Sshr(Simd2D(dest), Simd2D(src), count.value);
3223 }
3224
unsignedRightShiftInt64x2(FloatRegister lhs,Register rhs,FloatRegister dest)3225 void MacroAssembler::unsignedRightShiftInt64x2(FloatRegister lhs, Register rhs,
3226 FloatRegister dest) {
3227 MacroAssemblerCompat::rightShiftInt64x2(lhs, rhs, dest,
3228 /* isUnsigned */ true);
3229 }
3230
unsignedRightShiftInt64x2(Imm32 count,FloatRegister src,FloatRegister dest)3231 void MacroAssembler::unsignedRightShiftInt64x2(Imm32 count, FloatRegister src,
3232 FloatRegister dest) {
3233 Ushr(Simd2D(dest), Simd2D(src), count.value);
3234 }
3235
3236 // Bitwise and, or, xor, not
3237
bitwiseAndSimd128(FloatRegister rhs,FloatRegister lhsDest)3238 void MacroAssembler::bitwiseAndSimd128(FloatRegister rhs,
3239 FloatRegister lhsDest) {
3240 And(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
3241 }
3242
bitwiseAndSimd128(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3243 void MacroAssembler::bitwiseAndSimd128(FloatRegister lhs, FloatRegister rhs,
3244 FloatRegister dest) {
3245 And(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3246 }
3247
bitwiseOrSimd128(FloatRegister rhs,FloatRegister lhsDest)3248 void MacroAssembler::bitwiseOrSimd128(FloatRegister rhs,
3249 FloatRegister lhsDest) {
3250 Orr(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
3251 }
3252
bitwiseOrSimd128(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3253 void MacroAssembler::bitwiseOrSimd128(FloatRegister lhs, FloatRegister rhs,
3254 FloatRegister dest) {
3255 Orr(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3256 }
3257
bitwiseXorSimd128(FloatRegister rhs,FloatRegister lhsDest)3258 void MacroAssembler::bitwiseXorSimd128(FloatRegister rhs,
3259 FloatRegister lhsDest) {
3260 Eor(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
3261 }
3262
bitwiseXorSimd128(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3263 void MacroAssembler::bitwiseXorSimd128(FloatRegister lhs, FloatRegister rhs,
3264 FloatRegister dest) {
3265 Eor(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3266 }
3267
bitwiseNotSimd128(FloatRegister src,FloatRegister dest)3268 void MacroAssembler::bitwiseNotSimd128(FloatRegister src, FloatRegister dest) {
3269 Not(Simd16B(dest), Simd16B(src));
3270 }
3271
bitwiseAndNotSimd128(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3272 void MacroAssembler::bitwiseAndNotSimd128(FloatRegister lhs, FloatRegister rhs,
3273 FloatRegister dest) {
3274 Bic(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3275 }
3276
3277 // Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
3278 // wants but what the x86 hardware offers. Hence the name. Since arm64 has
3279 // dest = lhs & ~rhs we just swap operands.
3280
bitwiseNotAndSimd128(FloatRegister rhs,FloatRegister lhsDest)3281 void MacroAssembler::bitwiseNotAndSimd128(FloatRegister rhs,
3282 FloatRegister lhsDest) {
3283 Bic(Simd16B(lhsDest), Simd16B(rhs), Simd16B(lhsDest));
3284 }
3285
3286 // Bitwise select
3287
bitwiseSelectSimd128(FloatRegister onTrue,FloatRegister onFalse,FloatRegister maskDest)3288 void MacroAssembler::bitwiseSelectSimd128(FloatRegister onTrue,
3289 FloatRegister onFalse,
3290 FloatRegister maskDest) {
3291 Bsl(Simd16B(maskDest), Simd16B(onTrue), Simd16B(onFalse));
3292 }
3293
3294 // Population count
3295
popcntInt8x16(FloatRegister src,FloatRegister dest)3296 void MacroAssembler::popcntInt8x16(FloatRegister src, FloatRegister dest) {
3297 Cnt(Simd16B(dest), Simd16B(src));
3298 }
3299
3300 // Any lane true, ie, any bit set
3301
anyTrueSimd128(FloatRegister src,Register dest_)3302 void MacroAssembler::anyTrueSimd128(FloatRegister src, Register dest_) {
3303 ScratchSimd128Scope scratch_(*this);
3304 ARMFPRegister scratch(Simd1D(scratch_));
3305 ARMRegister dest(dest_, 64);
3306 Addp(scratch, Simd2D(src));
3307 Umov(dest, scratch, 0);
3308 Cmp(dest, Operand(0));
3309 Cset(dest, Assembler::NonZero);
3310 }
3311
3312 // All lanes true
3313
allTrueInt8x16(FloatRegister src,Register dest_)3314 void MacroAssembler::allTrueInt8x16(FloatRegister src, Register dest_) {
3315 ScratchSimd128Scope scratch(*this);
3316 ARMRegister dest(dest_, 64);
3317 Cmeq(Simd16B(scratch), Simd16B(src), 0);
3318 Addp(Simd1D(scratch), Simd2D(scratch));
3319 Umov(dest, Simd1D(scratch), 0);
3320 Cmp(dest, Operand(0));
3321 Cset(dest, Assembler::Zero);
3322 }
3323
allTrueInt16x8(FloatRegister src,Register dest_)3324 void MacroAssembler::allTrueInt16x8(FloatRegister src, Register dest_) {
3325 ScratchSimd128Scope scratch(*this);
3326 ARMRegister dest(dest_, 64);
3327 Cmeq(Simd8H(scratch), Simd8H(src), 0);
3328 Addp(Simd1D(scratch), Simd2D(scratch));
3329 Umov(dest, Simd1D(scratch), 0);
3330 Cmp(dest, Operand(0));
3331 Cset(dest, Assembler::Zero);
3332 }
3333
allTrueInt32x4(FloatRegister src,Register dest_)3334 void MacroAssembler::allTrueInt32x4(FloatRegister src, Register dest_) {
3335 ScratchSimd128Scope scratch(*this);
3336 ARMRegister dest(dest_, 64);
3337 Cmeq(Simd4S(scratch), Simd4S(src), 0);
3338 Addp(Simd1D(scratch), Simd2D(scratch));
3339 Umov(dest, Simd1D(scratch), 0);
3340 Cmp(dest, Operand(0));
3341 Cset(dest, Assembler::Zero);
3342 }
3343
allTrueInt64x2(FloatRegister src,Register dest_)3344 void MacroAssembler::allTrueInt64x2(FloatRegister src, Register dest_) {
3345 ScratchSimd128Scope scratch(*this);
3346 ARMRegister dest(dest_, 64);
3347 Cmeq(Simd2D(scratch), Simd2D(src), 0);
3348 Addp(Simd1D(scratch), Simd2D(scratch));
3349 Umov(dest, Simd1D(scratch), 0);
3350 Cmp(dest, Operand(0));
3351 Cset(dest, Assembler::Zero);
3352 }
3353
3354 // Bitmask, ie extract and compress high bits of all lanes
3355 //
3356 // There's no direct support for this on the chip. These implementations come
3357 // from the writeup that added the instruction to the SIMD instruction set.
3358 // Generally, shifting and masking is used to isolate the sign bit of each
3359 // element in the right position, then a horizontal add creates the result. For
3360 // 8-bit elements an intermediate step is needed to assemble the bits of the
3361 // upper and lower 8 bytes into 8 halfwords.
3362
bitmaskInt8x16(FloatRegister src,Register dest,FloatRegister temp)3363 void MacroAssembler::bitmaskInt8x16(FloatRegister src, Register dest,
3364 FloatRegister temp) {
3365 ScratchSimd128Scope scratch(*this);
3366 int8_t values[] = {1, 2, 4, 8, 16, 32, 64, -128,
3367 1, 2, 4, 8, 16, 32, 64, -128};
3368 loadConstantSimd128(SimdConstant::CreateX16(values), temp);
3369 Sshr(Simd16B(scratch), Simd16B(src), 7);
3370 And(Simd16B(scratch), Simd16B(scratch), Simd16B(temp));
3371 Ext(Simd16B(temp), Simd16B(scratch), Simd16B(scratch), 8);
3372 Zip1(Simd16B(temp), Simd16B(scratch), Simd16B(temp));
3373 Addv(ARMFPRegister(temp, 16), Simd8H(temp));
3374 Mov(ARMRegister(dest, 32), Simd8H(temp), 0);
3375 }
3376
bitmaskInt16x8(FloatRegister src,Register dest,FloatRegister temp)3377 void MacroAssembler::bitmaskInt16x8(FloatRegister src, Register dest,
3378 FloatRegister temp) {
3379 ScratchSimd128Scope scratch(*this);
3380 int16_t values[] = {1, 2, 4, 8, 16, 32, 64, 128};
3381 loadConstantSimd128(SimdConstant::CreateX8(values), temp);
3382 Sshr(Simd8H(scratch), Simd8H(src), 15);
3383 And(Simd16B(scratch), Simd16B(scratch), Simd16B(temp));
3384 Addv(ARMFPRegister(scratch, 16), Simd8H(scratch));
3385 Mov(ARMRegister(dest, 32), Simd8H(scratch), 0);
3386 }
3387
bitmaskInt32x4(FloatRegister src,Register dest,FloatRegister temp)3388 void MacroAssembler::bitmaskInt32x4(FloatRegister src, Register dest,
3389 FloatRegister temp) {
3390 ScratchSimd128Scope scratch(*this);
3391 int32_t values[] = {1, 2, 4, 8};
3392 loadConstantSimd128(SimdConstant::CreateX4(values), temp);
3393 Sshr(Simd4S(scratch), Simd4S(src), 31);
3394 And(Simd16B(scratch), Simd16B(scratch), Simd16B(temp));
3395 Addv(ARMFPRegister(scratch, 32), Simd4S(scratch));
3396 Mov(ARMRegister(dest, 32), Simd4S(scratch), 0);
3397 }
3398
bitmaskInt64x2(FloatRegister src,Register dest,FloatRegister temp)3399 void MacroAssembler::bitmaskInt64x2(FloatRegister src, Register dest,
3400 FloatRegister temp) {
3401 Sqxtn(Simd2S(temp), Simd2D(src));
3402 Ushr(Simd2S(temp), Simd2S(temp), 31);
3403 Usra(ARMFPRegister(temp, 64), ARMFPRegister(temp, 64), 31);
3404 Fmov(ARMRegister(dest, 32), ARMFPRegister(temp, 32));
3405 }
3406
3407 // Comparisons (integer and floating-point)
3408
compareInt8x16(Assembler::Condition cond,FloatRegister rhs,FloatRegister lhsDest)3409 void MacroAssembler::compareInt8x16(Assembler::Condition cond,
3410 FloatRegister rhs, FloatRegister lhsDest) {
3411 compareSimd128Int(cond, Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
3412 }
3413
compareInt8x16(Assembler::Condition cond,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3414 void MacroAssembler::compareInt8x16(Assembler::Condition cond,
3415 FloatRegister lhs, FloatRegister rhs,
3416 FloatRegister dest) {
3417 compareSimd128Int(cond, Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
3418 }
3419
compareInt16x8(Assembler::Condition cond,FloatRegister rhs,FloatRegister lhsDest)3420 void MacroAssembler::compareInt16x8(Assembler::Condition cond,
3421 FloatRegister rhs, FloatRegister lhsDest) {
3422 compareSimd128Int(cond, Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
3423 }
3424
compareInt16x8(Assembler::Condition cond,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3425 void MacroAssembler::compareInt16x8(Assembler::Condition cond,
3426 FloatRegister lhs, FloatRegister rhs,
3427 FloatRegister dest) {
3428 compareSimd128Int(cond, Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
3429 }
3430
compareInt32x4(Assembler::Condition cond,FloatRegister rhs,FloatRegister lhsDest)3431 void MacroAssembler::compareInt32x4(Assembler::Condition cond,
3432 FloatRegister rhs, FloatRegister lhsDest) {
3433 compareSimd128Int(cond, Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3434 }
3435
compareInt32x4(Assembler::Condition cond,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3436 void MacroAssembler::compareInt32x4(Assembler::Condition cond,
3437 FloatRegister lhs, FloatRegister rhs,
3438 FloatRegister dest) {
3439 compareSimd128Int(cond, Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3440 }
3441
compareInt64x2(Assembler::Condition cond,FloatRegister rhs,FloatRegister lhsDest)3442 void MacroAssembler::compareInt64x2(Assembler::Condition cond,
3443 FloatRegister rhs, FloatRegister lhsDest) {
3444 compareSimd128Int(cond, Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3445 }
3446
compareInt64x2(Assembler::Condition cond,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3447 void MacroAssembler::compareInt64x2(Assembler::Condition cond,
3448 FloatRegister lhs, FloatRegister rhs,
3449 FloatRegister dest) {
3450 compareSimd128Int(cond, Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3451 }
3452
compareFloat32x4(Assembler::Condition cond,FloatRegister rhs,FloatRegister lhsDest)3453 void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
3454 FloatRegister rhs,
3455 FloatRegister lhsDest) {
3456 compareSimd128Float(cond, Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3457 }
3458
compareFloat32x4(Assembler::Condition cond,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3459 void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
3460 FloatRegister lhs, FloatRegister rhs,
3461 FloatRegister dest) {
3462 compareSimd128Float(cond, Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3463 }
3464
compareFloat64x2(Assembler::Condition cond,FloatRegister rhs,FloatRegister lhsDest)3465 void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
3466 FloatRegister rhs,
3467 FloatRegister lhsDest) {
3468 compareSimd128Float(cond, Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3469 }
3470
compareFloat64x2(Assembler::Condition cond,FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3471 void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
3472 FloatRegister lhs, FloatRegister rhs,
3473 FloatRegister dest) {
3474 compareSimd128Float(cond, Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3475 }
3476
3477 // Load
3478
loadUnalignedSimd128(const Address & src,FloatRegister dest)3479 void MacroAssembler::loadUnalignedSimd128(const Address& src,
3480 FloatRegister dest) {
3481 Ldr(ARMFPRegister(dest, 128), toMemOperand(src));
3482 }
3483
loadUnalignedSimd128(const BaseIndex & address,FloatRegister dest)3484 void MacroAssembler::loadUnalignedSimd128(const BaseIndex& address,
3485 FloatRegister dest) {
3486 doBaseIndex(ARMFPRegister(dest, 128), address, vixl::LDR_q);
3487 }
3488
3489 // Store
3490
storeUnalignedSimd128(FloatRegister src,const Address & dest)3491 void MacroAssembler::storeUnalignedSimd128(FloatRegister src,
3492 const Address& dest) {
3493 Str(ARMFPRegister(src, 128), toMemOperand(dest));
3494 }
3495
storeUnalignedSimd128(FloatRegister src,const BaseIndex & dest)3496 void MacroAssembler::storeUnalignedSimd128(FloatRegister src,
3497 const BaseIndex& dest) {
3498 doBaseIndex(ARMFPRegister(src, 128), dest, vixl::STR_q);
3499 }
3500
3501 // Floating point negation
3502
negFloat32x4(FloatRegister src,FloatRegister dest)3503 void MacroAssembler::negFloat32x4(FloatRegister src, FloatRegister dest) {
3504 Fneg(Simd4S(dest), Simd4S(src));
3505 }
3506
negFloat64x2(FloatRegister src,FloatRegister dest)3507 void MacroAssembler::negFloat64x2(FloatRegister src, FloatRegister dest) {
3508 Fneg(Simd2D(dest), Simd2D(src));
3509 }
3510
3511 // Floating point absolute value
3512
absFloat32x4(FloatRegister src,FloatRegister dest)3513 void MacroAssembler::absFloat32x4(FloatRegister src, FloatRegister dest) {
3514 Fabs(Simd4S(dest), Simd4S(src));
3515 }
3516
absFloat64x2(FloatRegister src,FloatRegister dest)3517 void MacroAssembler::absFloat64x2(FloatRegister src, FloatRegister dest) {
3518 Fabs(Simd2D(dest), Simd2D(src));
3519 }
3520
3521 // NaN-propagating minimum
3522
minFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3523 void MacroAssembler::minFloat32x4(FloatRegister lhs, FloatRegister rhs,
3524 FloatRegister dest) {
3525 Fmin(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3526 }
3527
minFloat32x4(FloatRegister rhs,FloatRegister lhsDest)3528 void MacroAssembler::minFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
3529 Fmin(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3530 }
3531
minFloat64x2(FloatRegister rhs,FloatRegister lhsDest)3532 void MacroAssembler::minFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
3533 Fmin(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3534 }
3535
minFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3536 void MacroAssembler::minFloat64x2(FloatRegister lhs, FloatRegister rhs,
3537 FloatRegister dest) {
3538 Fmin(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3539 }
3540
3541 // NaN-propagating maximum
3542
maxFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3543 void MacroAssembler::maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
3544 FloatRegister dest) {
3545 Fmax(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3546 }
3547
maxFloat32x4(FloatRegister rhs,FloatRegister lhsDest)3548 void MacroAssembler::maxFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
3549 Fmax(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3550 }
3551
maxFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3552 void MacroAssembler::maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
3553 FloatRegister dest) {
3554 Fmax(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3555 }
3556
maxFloat64x2(FloatRegister rhs,FloatRegister lhsDest)3557 void MacroAssembler::maxFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
3558 Fmax(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3559 }
3560
3561 // Floating add
3562
addFloat32x4(FloatRegister rhs,FloatRegister lhsDest)3563 void MacroAssembler::addFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
3564 Fadd(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3565 }
3566
addFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3567 void MacroAssembler::addFloat32x4(FloatRegister lhs, FloatRegister rhs,
3568 FloatRegister dest) {
3569 Fadd(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3570 }
3571
addFloat64x2(FloatRegister rhs,FloatRegister lhsDest)3572 void MacroAssembler::addFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
3573 Fadd(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3574 }
3575
addFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3576 void MacroAssembler::addFloat64x2(FloatRegister lhs, FloatRegister rhs,
3577 FloatRegister dest) {
3578 Fadd(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3579 }
3580
3581 // Floating subtract
3582
subFloat32x4(FloatRegister rhs,FloatRegister lhsDest)3583 void MacroAssembler::subFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
3584 Fsub(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3585 }
3586
subFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3587 void MacroAssembler::subFloat32x4(FloatRegister lhs, FloatRegister rhs,
3588 FloatRegister dest) {
3589 Fsub(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3590 }
3591
subFloat64x2(FloatRegister rhs,FloatRegister lhsDest)3592 void MacroAssembler::subFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
3593 Fsub(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3594 }
3595
subFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3596 void MacroAssembler::subFloat64x2(FloatRegister lhs, FloatRegister rhs,
3597 FloatRegister dest) {
3598 Fsub(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3599 }
3600
3601 // Floating division
3602
divFloat32x4(FloatRegister rhs,FloatRegister lhsDest)3603 void MacroAssembler::divFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
3604 Fdiv(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3605 }
3606
divFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3607 void MacroAssembler::divFloat32x4(FloatRegister lhs, FloatRegister rhs,
3608 FloatRegister dest) {
3609 Fdiv(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3610 }
3611
divFloat64x2(FloatRegister rhs,FloatRegister lhsDest)3612 void MacroAssembler::divFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
3613 Fdiv(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3614 }
3615
divFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3616 void MacroAssembler::divFloat64x2(FloatRegister lhs, FloatRegister rhs,
3617 FloatRegister dest) {
3618 Fdiv(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3619 }
3620
3621 // Floating Multiply
3622
mulFloat32x4(FloatRegister rhs,FloatRegister lhsDest)3623 void MacroAssembler::mulFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
3624 Fmul(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
3625 }
3626
mulFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3627 void MacroAssembler::mulFloat32x4(FloatRegister lhs, FloatRegister rhs,
3628 FloatRegister dest) {
3629 Fmul(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
3630 }
3631
mulFloat64x2(FloatRegister rhs,FloatRegister lhsDest)3632 void MacroAssembler::mulFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
3633 Fmul(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
3634 }
3635
mulFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3636 void MacroAssembler::mulFloat64x2(FloatRegister lhs, FloatRegister rhs,
3637 FloatRegister dest) {
3638 Fmul(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
3639 }
3640
3641 // Pairwise add
3642
extAddPairwiseInt8x16(FloatRegister src,FloatRegister dest)3643 void MacroAssembler::extAddPairwiseInt8x16(FloatRegister src,
3644 FloatRegister dest) {
3645 Saddlp(Simd8H(dest), Simd16B(src));
3646 }
3647
unsignedExtAddPairwiseInt8x16(FloatRegister src,FloatRegister dest)3648 void MacroAssembler::unsignedExtAddPairwiseInt8x16(FloatRegister src,
3649 FloatRegister dest) {
3650 Uaddlp(Simd8H(dest), Simd16B(src));
3651 }
3652
extAddPairwiseInt16x8(FloatRegister src,FloatRegister dest)3653 void MacroAssembler::extAddPairwiseInt16x8(FloatRegister src,
3654 FloatRegister dest) {
3655 Saddlp(Simd4S(dest), Simd8H(src));
3656 }
3657
unsignedExtAddPairwiseInt16x8(FloatRegister src,FloatRegister dest)3658 void MacroAssembler::unsignedExtAddPairwiseInt16x8(FloatRegister src,
3659 FloatRegister dest) {
3660 Uaddlp(Simd4S(dest), Simd8H(src));
3661 }
3662
3663 // Floating square root
3664
sqrtFloat32x4(FloatRegister src,FloatRegister dest)3665 void MacroAssembler::sqrtFloat32x4(FloatRegister src, FloatRegister dest) {
3666 Fsqrt(Simd4S(dest), Simd4S(src));
3667 }
3668
sqrtFloat64x2(FloatRegister src,FloatRegister dest)3669 void MacroAssembler::sqrtFloat64x2(FloatRegister src, FloatRegister dest) {
3670 Fsqrt(Simd2D(dest), Simd2D(src));
3671 }
3672
3673 // Integer to floating point with rounding
3674
convertInt32x4ToFloat32x4(FloatRegister src,FloatRegister dest)3675 void MacroAssembler::convertInt32x4ToFloat32x4(FloatRegister src,
3676 FloatRegister dest) {
3677 Scvtf(Simd4S(dest), Simd4S(src));
3678 }
3679
unsignedConvertInt32x4ToFloat32x4(FloatRegister src,FloatRegister dest)3680 void MacroAssembler::unsignedConvertInt32x4ToFloat32x4(FloatRegister src,
3681 FloatRegister dest) {
3682 Ucvtf(Simd4S(dest), Simd4S(src));
3683 }
3684
convertInt32x4ToFloat64x2(FloatRegister src,FloatRegister dest)3685 void MacroAssembler::convertInt32x4ToFloat64x2(FloatRegister src,
3686 FloatRegister dest) {
3687 Sshll(Simd2D(dest), Simd2S(src), 0);
3688 Scvtf(Simd2D(dest), Simd2D(dest));
3689 }
3690
unsignedConvertInt32x4ToFloat64x2(FloatRegister src,FloatRegister dest)3691 void MacroAssembler::unsignedConvertInt32x4ToFloat64x2(FloatRegister src,
3692 FloatRegister dest) {
3693 Ushll(Simd2D(dest), Simd2S(src), 0);
3694 Ucvtf(Simd2D(dest), Simd2D(dest));
3695 }
3696
3697 // Floating point to integer with saturation
3698
truncSatFloat32x4ToInt32x4(FloatRegister src,FloatRegister dest)3699 void MacroAssembler::truncSatFloat32x4ToInt32x4(FloatRegister src,
3700 FloatRegister dest) {
3701 Fcvtzs(Simd4S(dest), Simd4S(src));
3702 }
3703
unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,FloatRegister dest)3704 void MacroAssembler::unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
3705 FloatRegister dest) {
3706 Fcvtzu(Simd4S(dest), Simd4S(src));
3707 }
3708
truncSatFloat64x2ToInt32x4(FloatRegister src,FloatRegister dest,FloatRegister temp)3709 void MacroAssembler::truncSatFloat64x2ToInt32x4(FloatRegister src,
3710 FloatRegister dest,
3711 FloatRegister temp) {
3712 Fcvtzs(Simd2D(dest), Simd2D(src));
3713 Sqxtn(Simd2S(dest), Simd2D(dest));
3714 }
3715
unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,FloatRegister dest,FloatRegister temp)3716 void MacroAssembler::unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,
3717 FloatRegister dest,
3718 FloatRegister temp) {
3719 Fcvtzu(Simd2D(dest), Simd2D(src));
3720 Uqxtn(Simd2S(dest), Simd2D(dest));
3721 }
3722
truncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,FloatRegister dest)3723 void MacroAssembler::truncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
3724 FloatRegister dest) {
3725 Fcvtzs(Simd4S(dest), Simd4S(src));
3726 }
3727
unsignedTruncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,FloatRegister dest)3728 void MacroAssembler::unsignedTruncSatFloat32x4ToInt32x4Relaxed(
3729 FloatRegister src, FloatRegister dest) {
3730 Fcvtzu(Simd4S(dest), Simd4S(src));
3731 }
3732
truncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,FloatRegister dest)3733 void MacroAssembler::truncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
3734 FloatRegister dest) {
3735 Fcvtzs(Simd2D(dest), Simd2D(src));
3736 Sqxtn(Simd2S(dest), Simd2D(dest));
3737 }
3738
unsignedTruncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,FloatRegister dest)3739 void MacroAssembler::unsignedTruncSatFloat64x2ToInt32x4Relaxed(
3740 FloatRegister src, FloatRegister dest) {
3741 Fcvtzu(Simd2D(dest), Simd2D(src));
3742 Uqxtn(Simd2S(dest), Simd2D(dest));
3743 }
3744
3745 // Floating point narrowing
3746
convertFloat64x2ToFloat32x4(FloatRegister src,FloatRegister dest)3747 void MacroAssembler::convertFloat64x2ToFloat32x4(FloatRegister src,
3748 FloatRegister dest) {
3749 Fcvtn(Simd2S(dest), Simd2D(src));
3750 }
3751
3752 // Floating point widening
3753
convertFloat32x4ToFloat64x2(FloatRegister src,FloatRegister dest)3754 void MacroAssembler::convertFloat32x4ToFloat64x2(FloatRegister src,
3755 FloatRegister dest) {
3756 Fcvtl(Simd2D(dest), Simd2S(src));
3757 }
3758
3759 // Integer to integer narrowing
3760
narrowInt16x8(FloatRegister rhs,FloatRegister lhsDest)3761 void MacroAssembler::narrowInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
3762 ScratchSimd128Scope scratch(*this);
3763 if (rhs == lhsDest) {
3764 Mov(scratch, SimdReg(rhs));
3765 rhs = scratch;
3766 }
3767 Sqxtn(Simd8B(lhsDest), Simd8H(lhsDest));
3768 Sqxtn2(Simd16B(lhsDest), Simd8H(rhs));
3769 }
3770
narrowInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3771 void MacroAssembler::narrowInt16x8(FloatRegister lhs, FloatRegister rhs,
3772 FloatRegister dest) {
3773 ScratchSimd128Scope scratch(*this);
3774 if (rhs == dest) {
3775 Mov(scratch, SimdReg(rhs));
3776 rhs = scratch;
3777 }
3778 Sqxtn(Simd8B(dest), Simd8H(lhs));
3779 Sqxtn2(Simd16B(dest), Simd8H(rhs));
3780 }
3781
unsignedNarrowInt16x8(FloatRegister rhs,FloatRegister lhsDest)3782 void MacroAssembler::unsignedNarrowInt16x8(FloatRegister rhs,
3783 FloatRegister lhsDest) {
3784 ScratchSimd128Scope scratch(*this);
3785 if (rhs == lhsDest) {
3786 Mov(scratch, SimdReg(rhs));
3787 rhs = scratch;
3788 }
3789 Sqxtun(Simd8B(lhsDest), Simd8H(lhsDest));
3790 Sqxtun2(Simd16B(lhsDest), Simd8H(rhs));
3791 }
3792
unsignedNarrowInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3793 void MacroAssembler::unsignedNarrowInt16x8(FloatRegister lhs, FloatRegister rhs,
3794 FloatRegister dest) {
3795 ScratchSimd128Scope scratch(*this);
3796 if (rhs == dest) {
3797 Mov(scratch, SimdReg(rhs));
3798 rhs = scratch;
3799 }
3800 Sqxtun(Simd8B(dest), Simd8H(lhs));
3801 Sqxtun2(Simd16B(dest), Simd8H(rhs));
3802 }
3803
narrowInt32x4(FloatRegister rhs,FloatRegister lhsDest)3804 void MacroAssembler::narrowInt32x4(FloatRegister rhs, FloatRegister lhsDest) {
3805 ScratchSimd128Scope scratch(*this);
3806 if (rhs == lhsDest) {
3807 Mov(scratch, SimdReg(rhs));
3808 rhs = scratch;
3809 }
3810 Sqxtn(Simd4H(lhsDest), Simd4S(lhsDest));
3811 Sqxtn2(Simd8H(lhsDest), Simd4S(rhs));
3812 }
3813
narrowInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3814 void MacroAssembler::narrowInt32x4(FloatRegister lhs, FloatRegister rhs,
3815 FloatRegister dest) {
3816 ScratchSimd128Scope scratch(*this);
3817 if (rhs == dest) {
3818 Mov(scratch, SimdReg(rhs));
3819 rhs = scratch;
3820 }
3821 Sqxtn(Simd4H(dest), Simd4S(lhs));
3822 Sqxtn2(Simd8H(dest), Simd4S(rhs));
3823 }
3824
unsignedNarrowInt32x4(FloatRegister rhs,FloatRegister lhsDest)3825 void MacroAssembler::unsignedNarrowInt32x4(FloatRegister rhs,
3826 FloatRegister lhsDest) {
3827 ScratchSimd128Scope scratch(*this);
3828 if (rhs == lhsDest) {
3829 Mov(scratch, SimdReg(rhs));
3830 rhs = scratch;
3831 }
3832 Sqxtun(Simd4H(lhsDest), Simd4S(lhsDest));
3833 Sqxtun2(Simd8H(lhsDest), Simd4S(rhs));
3834 }
3835
unsignedNarrowInt32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3836 void MacroAssembler::unsignedNarrowInt32x4(FloatRegister lhs, FloatRegister rhs,
3837 FloatRegister dest) {
3838 ScratchSimd128Scope scratch(*this);
3839 if (rhs == dest) {
3840 Mov(scratch, SimdReg(rhs));
3841 rhs = scratch;
3842 }
3843 Sqxtun(Simd4H(dest), Simd4S(lhs));
3844 Sqxtun2(Simd8H(dest), Simd4S(rhs));
3845 }
3846
3847 // Integer to integer widening
3848
widenLowInt8x16(FloatRegister src,FloatRegister dest)3849 void MacroAssembler::widenLowInt8x16(FloatRegister src, FloatRegister dest) {
3850 Sshll(Simd8H(dest), Simd8B(src), 0);
3851 }
3852
widenHighInt8x16(FloatRegister src,FloatRegister dest)3853 void MacroAssembler::widenHighInt8x16(FloatRegister src, FloatRegister dest) {
3854 Sshll2(Simd8H(dest), Simd16B(src), 0);
3855 }
3856
unsignedWidenLowInt8x16(FloatRegister src,FloatRegister dest)3857 void MacroAssembler::unsignedWidenLowInt8x16(FloatRegister src,
3858 FloatRegister dest) {
3859 Ushll(Simd8H(dest), Simd8B(src), 0);
3860 }
3861
unsignedWidenHighInt8x16(FloatRegister src,FloatRegister dest)3862 void MacroAssembler::unsignedWidenHighInt8x16(FloatRegister src,
3863 FloatRegister dest) {
3864 Ushll2(Simd8H(dest), Simd16B(src), 0);
3865 }
3866
widenLowInt16x8(FloatRegister src,FloatRegister dest)3867 void MacroAssembler::widenLowInt16x8(FloatRegister src, FloatRegister dest) {
3868 Sshll(Simd4S(dest), Simd4H(src), 0);
3869 }
3870
widenHighInt16x8(FloatRegister src,FloatRegister dest)3871 void MacroAssembler::widenHighInt16x8(FloatRegister src, FloatRegister dest) {
3872 Sshll2(Simd4S(dest), Simd8H(src), 0);
3873 }
3874
unsignedWidenLowInt16x8(FloatRegister src,FloatRegister dest)3875 void MacroAssembler::unsignedWidenLowInt16x8(FloatRegister src,
3876 FloatRegister dest) {
3877 Ushll(Simd4S(dest), Simd4H(src), 0);
3878 }
3879
unsignedWidenHighInt16x8(FloatRegister src,FloatRegister dest)3880 void MacroAssembler::unsignedWidenHighInt16x8(FloatRegister src,
3881 FloatRegister dest) {
3882 Ushll2(Simd4S(dest), Simd8H(src), 0);
3883 }
3884
widenLowInt32x4(FloatRegister src,FloatRegister dest)3885 void MacroAssembler::widenLowInt32x4(FloatRegister src, FloatRegister dest) {
3886 Sshll(Simd2D(dest), Simd2S(src), 0);
3887 }
3888
unsignedWidenLowInt32x4(FloatRegister src,FloatRegister dest)3889 void MacroAssembler::unsignedWidenLowInt32x4(FloatRegister src,
3890 FloatRegister dest) {
3891 Ushll(Simd2D(dest), Simd2S(src), 0);
3892 }
3893
widenHighInt32x4(FloatRegister src,FloatRegister dest)3894 void MacroAssembler::widenHighInt32x4(FloatRegister src, FloatRegister dest) {
3895 Sshll2(Simd2D(dest), Simd4S(src), 0);
3896 }
3897
unsignedWidenHighInt32x4(FloatRegister src,FloatRegister dest)3898 void MacroAssembler::unsignedWidenHighInt32x4(FloatRegister src,
3899 FloatRegister dest) {
3900 Ushll2(Simd2D(dest), Simd4S(src), 0);
3901 }
3902
3903 // Compare-based minimum/maximum (experimental as of August, 2020)
3904 // https://github.com/WebAssembly/simd/pull/122
3905
pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,FloatRegister lhsOrLhsDest)3906 void MacroAssembler::pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,
3907 FloatRegister lhsOrLhsDest) {
3908 // Shut up the linter by using the same names as in the declaration, then
3909 // aliasing here.
3910 FloatRegister rhs = rhsOrRhsDest;
3911 FloatRegister lhsDest = lhsOrLhsDest;
3912 ScratchSimd128Scope scratch(*this);
3913 Fcmgt(Simd4S(scratch), Simd4S(lhsDest), Simd4S(rhs));
3914 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
3915 Mov(SimdReg(lhsDest), scratch);
3916 }
3917
pseudoMinFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3918 void MacroAssembler::pseudoMinFloat32x4(FloatRegister lhs, FloatRegister rhs,
3919 FloatRegister dest) {
3920 ScratchSimd128Scope scratch(*this);
3921 Fcmgt(Simd4S(scratch), Simd4S(lhs), Simd4S(rhs));
3922 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
3923 Mov(SimdReg(dest), scratch);
3924 }
3925
pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,FloatRegister lhsOrLhsDest)3926 void MacroAssembler::pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,
3927 FloatRegister lhsOrLhsDest) {
3928 FloatRegister rhs = rhsOrRhsDest;
3929 FloatRegister lhsDest = lhsOrLhsDest;
3930 ScratchSimd128Scope scratch(*this);
3931 Fcmgt(Simd2D(scratch), Simd2D(lhsDest), Simd2D(rhs));
3932 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
3933 Mov(SimdReg(lhsDest), scratch);
3934 }
3935
pseudoMinFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3936 void MacroAssembler::pseudoMinFloat64x2(FloatRegister lhs, FloatRegister rhs,
3937 FloatRegister dest) {
3938 ScratchSimd128Scope scratch(*this);
3939 Fcmgt(Simd2D(scratch), Simd2D(lhs), Simd2D(rhs));
3940 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
3941 Mov(SimdReg(dest), scratch);
3942 }
3943
pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,FloatRegister lhsOrLhsDest)3944 void MacroAssembler::pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,
3945 FloatRegister lhsOrLhsDest) {
3946 FloatRegister rhs = rhsOrRhsDest;
3947 FloatRegister lhsDest = lhsOrLhsDest;
3948 ScratchSimd128Scope scratch(*this);
3949 Fcmgt(Simd4S(scratch), Simd4S(rhs), Simd4S(lhsDest));
3950 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
3951 Mov(SimdReg(lhsDest), scratch);
3952 }
3953
pseudoMaxFloat32x4(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3954 void MacroAssembler::pseudoMaxFloat32x4(FloatRegister lhs, FloatRegister rhs,
3955 FloatRegister dest) {
3956 ScratchSimd128Scope scratch(*this);
3957 Fcmgt(Simd4S(scratch), Simd4S(rhs), Simd4S(lhs));
3958 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
3959 Mov(SimdReg(dest), scratch);
3960 }
3961
pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,FloatRegister lhsOrLhsDest)3962 void MacroAssembler::pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,
3963 FloatRegister lhsOrLhsDest) {
3964 FloatRegister rhs = rhsOrRhsDest;
3965 FloatRegister lhsDest = lhsOrLhsDest;
3966 ScratchSimd128Scope scratch(*this);
3967 Fcmgt(Simd2D(scratch), Simd2D(rhs), Simd2D(lhsDest));
3968 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
3969 Mov(SimdReg(lhsDest), scratch);
3970 }
3971
pseudoMaxFloat64x2(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3972 void MacroAssembler::pseudoMaxFloat64x2(FloatRegister lhs, FloatRegister rhs,
3973 FloatRegister dest) {
3974 ScratchSimd128Scope scratch(*this);
3975 Fcmgt(Simd2D(scratch), Simd2D(rhs), Simd2D(lhs));
3976 Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
3977 Mov(SimdReg(dest), scratch);
3978 }
3979
3980 // Widening/pairwise integer dot product (experimental as of August, 2020)
3981 // https://github.com/WebAssembly/simd/pull/127
3982
widenDotInt16x8(FloatRegister rhs,FloatRegister lhsDest)3983 void MacroAssembler::widenDotInt16x8(FloatRegister rhs, FloatRegister lhsDest) {
3984 widenDotInt16x8(lhsDest, rhs, lhsDest);
3985 }
3986
widenDotInt16x8(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)3987 void MacroAssembler::widenDotInt16x8(FloatRegister lhs, FloatRegister rhs,
3988 FloatRegister dest) {
3989 ScratchSimd128Scope scratch(*this);
3990 Smull(Simd4S(scratch), Simd4H(lhs), Simd4H(rhs));
3991 Smull2(Simd4S(dest), Simd8H(lhs), Simd8H(rhs));
3992 Addp(Simd4S(dest), Simd4S(scratch), Simd4S(dest));
3993 }
3994
3995 // Floating point rounding (experimental as of August, 2020)
3996 // https://github.com/WebAssembly/simd/pull/232
3997
ceilFloat32x4(FloatRegister src,FloatRegister dest)3998 void MacroAssembler::ceilFloat32x4(FloatRegister src, FloatRegister dest) {
3999 Frintp(Simd4S(dest), Simd4S(src));
4000 }
4001
ceilFloat64x2(FloatRegister src,FloatRegister dest)4002 void MacroAssembler::ceilFloat64x2(FloatRegister src, FloatRegister dest) {
4003 Frintp(Simd2D(dest), Simd2D(src));
4004 }
4005
floorFloat32x4(FloatRegister src,FloatRegister dest)4006 void MacroAssembler::floorFloat32x4(FloatRegister src, FloatRegister dest) {
4007 Frintm(Simd4S(dest), Simd4S(src));
4008 }
4009
floorFloat64x2(FloatRegister src,FloatRegister dest)4010 void MacroAssembler::floorFloat64x2(FloatRegister src, FloatRegister dest) {
4011 Frintm(Simd2D(dest), Simd2D(src));
4012 }
4013
truncFloat32x4(FloatRegister src,FloatRegister dest)4014 void MacroAssembler::truncFloat32x4(FloatRegister src, FloatRegister dest) {
4015 Frintz(Simd4S(dest), Simd4S(src));
4016 }
4017
truncFloat64x2(FloatRegister src,FloatRegister dest)4018 void MacroAssembler::truncFloat64x2(FloatRegister src, FloatRegister dest) {
4019 Frintz(Simd2D(dest), Simd2D(src));
4020 }
4021
nearestFloat32x4(FloatRegister src,FloatRegister dest)4022 void MacroAssembler::nearestFloat32x4(FloatRegister src, FloatRegister dest) {
4023 Frintn(Simd4S(dest), Simd4S(src));
4024 }
4025
nearestFloat64x2(FloatRegister src,FloatRegister dest)4026 void MacroAssembler::nearestFloat64x2(FloatRegister src, FloatRegister dest) {
4027 Frintn(Simd2D(dest), Simd2D(src));
4028 }
4029
4030 // Floating multiply-accumulate: srcDest [+-]= src1 * src2
4031
fmaFloat32x4(FloatRegister src1,FloatRegister src2,FloatRegister srcDest)4032 void MacroAssembler::fmaFloat32x4(FloatRegister src1, FloatRegister src2,
4033 FloatRegister srcDest) {
4034 Fmla(Simd4S(srcDest), Simd4S(src1), Simd4S(src2));
4035 }
4036
fmsFloat32x4(FloatRegister src1,FloatRegister src2,FloatRegister srcDest)4037 void MacroAssembler::fmsFloat32x4(FloatRegister src1, FloatRegister src2,
4038 FloatRegister srcDest) {
4039 Fmls(Simd4S(srcDest), Simd4S(src1), Simd4S(src2));
4040 }
4041
fmaFloat64x2(FloatRegister src1,FloatRegister src2,FloatRegister srcDest)4042 void MacroAssembler::fmaFloat64x2(FloatRegister src1, FloatRegister src2,
4043 FloatRegister srcDest) {
4044 Fmla(Simd2D(srcDest), Simd2D(src1), Simd2D(src2));
4045 }
4046
fmsFloat64x2(FloatRegister src1,FloatRegister src2,FloatRegister srcDest)4047 void MacroAssembler::fmsFloat64x2(FloatRegister src1, FloatRegister src2,
4048 FloatRegister srcDest) {
4049 Fmls(Simd2D(srcDest), Simd2D(src1), Simd2D(src2));
4050 }
4051
minFloat32x4Relaxed(FloatRegister src,FloatRegister srcDest)4052 void MacroAssembler::minFloat32x4Relaxed(FloatRegister src,
4053 FloatRegister srcDest) {
4054 Fmin(Simd4S(srcDest), Simd4S(src), Simd4S(srcDest));
4055 }
4056
minFloat32x4Relaxed(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)4057 void MacroAssembler::minFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
4058 FloatRegister dest) {
4059 Fmin(Simd4S(dest), Simd4S(rhs), Simd4S(lhs));
4060 }
4061
maxFloat32x4Relaxed(FloatRegister src,FloatRegister srcDest)4062 void MacroAssembler::maxFloat32x4Relaxed(FloatRegister src,
4063 FloatRegister srcDest) {
4064 Fmax(Simd4S(srcDest), Simd4S(src), Simd4S(srcDest));
4065 }
4066
maxFloat32x4Relaxed(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)4067 void MacroAssembler::maxFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
4068 FloatRegister dest) {
4069 Fmax(Simd4S(dest), Simd4S(rhs), Simd4S(lhs));
4070 }
4071
minFloat64x2Relaxed(FloatRegister src,FloatRegister srcDest)4072 void MacroAssembler::minFloat64x2Relaxed(FloatRegister src,
4073 FloatRegister srcDest) {
4074 Fmin(Simd2D(srcDest), Simd2D(src), Simd2D(srcDest));
4075 }
4076
minFloat64x2Relaxed(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)4077 void MacroAssembler::minFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
4078 FloatRegister dest) {
4079 Fmin(Simd2D(dest), Simd2D(rhs), Simd2D(lhs));
4080 }
4081
maxFloat64x2Relaxed(FloatRegister src,FloatRegister srcDest)4082 void MacroAssembler::maxFloat64x2Relaxed(FloatRegister src,
4083 FloatRegister srcDest) {
4084 Fmax(Simd2D(srcDest), Simd2D(src), Simd2D(srcDest));
4085 }
4086
maxFloat64x2Relaxed(FloatRegister lhs,FloatRegister rhs,FloatRegister dest)4087 void MacroAssembler::maxFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
4088 FloatRegister dest) {
4089 Fmax(Simd2D(dest), Simd2D(rhs), Simd2D(lhs));
4090 }
4091
4092 //}}} check_macroassembler_style
4093 // ===============================================================
4094
addToStackPtr(Register src)4095 void MacroAssemblerCompat::addToStackPtr(Register src) {
4096 Add(GetStackPointer64(), GetStackPointer64(), ARMRegister(src, 64));
4097 // Given that required invariant SP <= PSP, this is probably pointless,
4098 // since it gives PSP a larger value.
4099 syncStackPtr();
4100 }
4101
addToStackPtr(Imm32 imm)4102 void MacroAssemblerCompat::addToStackPtr(Imm32 imm) {
4103 Add(GetStackPointer64(), GetStackPointer64(), Operand(imm.value));
4104 // As above, probably pointless.
4105 syncStackPtr();
4106 }
4107
addToStackPtr(const Address & src)4108 void MacroAssemblerCompat::addToStackPtr(const Address& src) {
4109 vixl::UseScratchRegisterScope temps(this);
4110 const ARMRegister scratch = temps.AcquireX();
4111 Ldr(scratch, toMemOperand(src));
4112 Add(GetStackPointer64(), GetStackPointer64(), scratch);
4113 // As above, probably pointless.
4114 syncStackPtr();
4115 }
4116
addStackPtrTo(Register dest)4117 void MacroAssemblerCompat::addStackPtrTo(Register dest) {
4118 Add(ARMRegister(dest, 64), ARMRegister(dest, 64), GetStackPointer64());
4119 }
4120
subFromStackPtr(Register src)4121 void MacroAssemblerCompat::subFromStackPtr(Register src) {
4122 Sub(GetStackPointer64(), GetStackPointer64(), ARMRegister(src, 64));
4123 syncStackPtr();
4124 }
4125
subFromStackPtr(Imm32 imm)4126 void MacroAssemblerCompat::subFromStackPtr(Imm32 imm) {
4127 Sub(GetStackPointer64(), GetStackPointer64(), Operand(imm.value));
4128 syncStackPtr();
4129 }
4130
subStackPtrFrom(Register dest)4131 void MacroAssemblerCompat::subStackPtrFrom(Register dest) {
4132 Sub(ARMRegister(dest, 64), ARMRegister(dest, 64), GetStackPointer64());
4133 }
4134
andToStackPtr(Imm32 imm)4135 void MacroAssemblerCompat::andToStackPtr(Imm32 imm) {
4136 if (sp.Is(GetStackPointer64())) {
4137 vixl::UseScratchRegisterScope temps(this);
4138 const ARMRegister scratch = temps.AcquireX();
4139 Mov(scratch, sp);
4140 And(sp, scratch, Operand(imm.value));
4141 // syncStackPtr() not needed since our SP is the real SP.
4142 } else {
4143 And(GetStackPointer64(), GetStackPointer64(), Operand(imm.value));
4144 syncStackPtr();
4145 }
4146 }
4147
moveToStackPtr(Register src)4148 void MacroAssemblerCompat::moveToStackPtr(Register src) {
4149 Mov(GetStackPointer64(), ARMRegister(src, 64));
4150 syncStackPtr();
4151 }
4152
moveStackPtrTo(Register dest)4153 void MacroAssemblerCompat::moveStackPtrTo(Register dest) {
4154 Mov(ARMRegister(dest, 64), GetStackPointer64());
4155 }
4156
loadStackPtr(const Address & src)4157 void MacroAssemblerCompat::loadStackPtr(const Address& src) {
4158 if (sp.Is(GetStackPointer64())) {
4159 vixl::UseScratchRegisterScope temps(this);
4160 const ARMRegister scratch = temps.AcquireX();
4161 Ldr(scratch, toMemOperand(src));
4162 Mov(sp, scratch);
4163 // syncStackPtr() not needed since our SP is the real SP.
4164 } else {
4165 Ldr(GetStackPointer64(), toMemOperand(src));
4166 syncStackPtr();
4167 }
4168 }
4169
storeStackPtr(const Address & dest)4170 void MacroAssemblerCompat::storeStackPtr(const Address& dest) {
4171 if (sp.Is(GetStackPointer64())) {
4172 vixl::UseScratchRegisterScope temps(this);
4173 const ARMRegister scratch = temps.AcquireX();
4174 Mov(scratch, sp);
4175 Str(scratch, toMemOperand(dest));
4176 } else {
4177 Str(GetStackPointer64(), toMemOperand(dest));
4178 }
4179 }
4180
branchTestStackPtr(Condition cond,Imm32 rhs,Label * label)4181 void MacroAssemblerCompat::branchTestStackPtr(Condition cond, Imm32 rhs,
4182 Label* label) {
4183 if (sp.Is(GetStackPointer64())) {
4184 vixl::UseScratchRegisterScope temps(this);
4185 const ARMRegister scratch = temps.AcquireX();
4186 Mov(scratch, sp);
4187 Tst(scratch, Operand(rhs.value));
4188 } else {
4189 Tst(GetStackPointer64(), Operand(rhs.value));
4190 }
4191 B(label, cond);
4192 }
4193
branchStackPtr(Condition cond,Register rhs_,Label * label)4194 void MacroAssemblerCompat::branchStackPtr(Condition cond, Register rhs_,
4195 Label* label) {
4196 ARMRegister rhs(rhs_, 64);
4197 if (sp.Is(GetStackPointer64())) {
4198 vixl::UseScratchRegisterScope temps(this);
4199 const ARMRegister scratch = temps.AcquireX();
4200 Mov(scratch, sp);
4201 Cmp(scratch, rhs);
4202 } else {
4203 Cmp(GetStackPointer64(), rhs);
4204 }
4205 B(label, cond);
4206 }
4207
branchStackPtrRhs(Condition cond,Address lhs,Label * label)4208 void MacroAssemblerCompat::branchStackPtrRhs(Condition cond, Address lhs,
4209 Label* label) {
4210 vixl::UseScratchRegisterScope temps(this);
4211 const ARMRegister scratch = temps.AcquireX();
4212 Ldr(scratch, toMemOperand(lhs));
4213 // Cmp disallows SP as the rhs, so flip the operands and invert the
4214 // condition.
4215 Cmp(GetStackPointer64(), scratch);
4216 B(label, Assembler::InvertCondition(cond));
4217 }
4218
branchStackPtrRhs(Condition cond,AbsoluteAddress lhs,Label * label)4219 void MacroAssemblerCompat::branchStackPtrRhs(Condition cond,
4220 AbsoluteAddress lhs,
4221 Label* label) {
4222 vixl::UseScratchRegisterScope temps(this);
4223 const ARMRegister scratch = temps.AcquireX();
4224 loadPtr(lhs, scratch.asUnsized());
4225 // Cmp disallows SP as the rhs, so flip the operands and invert the
4226 // condition.
4227 Cmp(GetStackPointer64(), scratch);
4228 B(label, Assembler::InvertCondition(cond));
4229 }
4230
4231 // If source is a double, load into dest.
4232 // If source is int32, convert to double and store in dest.
4233 // Else, branch to failure.
ensureDouble(const ValueOperand & source,FloatRegister dest,Label * failure)4234 void MacroAssemblerCompat::ensureDouble(const ValueOperand& source,
4235 FloatRegister dest, Label* failure) {
4236 Label isDouble, done;
4237
4238 {
4239 ScratchTagScope tag(asMasm(), source);
4240 splitTagForTest(source, tag);
4241 asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
4242 asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
4243 }
4244
4245 convertInt32ToDouble(source.valueReg(), dest);
4246 jump(&done);
4247
4248 bind(&isDouble);
4249 unboxDouble(source, dest);
4250
4251 bind(&done);
4252 }
4253
unboxValue(const ValueOperand & src,AnyRegister dest,JSValueType type)4254 void MacroAssemblerCompat::unboxValue(const ValueOperand& src, AnyRegister dest,
4255 JSValueType type) {
4256 if (dest.isFloat()) {
4257 Label notInt32, end;
4258 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
4259 convertInt32ToDouble(src.valueReg(), dest.fpu());
4260 jump(&end);
4261 bind(¬Int32);
4262 unboxDouble(src, dest.fpu());
4263 bind(&end);
4264 } else {
4265 unboxNonDouble(src, dest.gpr(), type);
4266 }
4267 }
4268
4269 } // namespace jit
4270 } // namespace js
4271
4272 #endif /* jit_arm64_MacroAssembler_arm64_inl_h */
4273