1 // Copyright 2015, ARM Limited
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #include "jit/arm64/vixl/Assembler-vixl.h"
28
29 #include <cmath>
30
31 #include "jit/arm64/vixl/MacroAssembler-vixl.h"
32
33 namespace vixl {
34
35 // CPURegList utilities.
PopLowestIndex()36 CPURegister CPURegList::PopLowestIndex() {
37 if (IsEmpty()) {
38 return NoCPUReg;
39 }
40 int index = CountTrailingZeros(list_);
41 VIXL_ASSERT((1ULL << index) & list_);
42 Remove(index);
43 return CPURegister(index, size_, type_);
44 }
45
46
PopHighestIndex()47 CPURegister CPURegList::PopHighestIndex() {
48 VIXL_ASSERT(IsValid());
49 if (IsEmpty()) {
50 return NoCPUReg;
51 }
52 int index = CountLeadingZeros(list_);
53 index = kRegListSizeInBits - 1 - index;
54 VIXL_ASSERT((1ULL << index) & list_);
55 Remove(index);
56 return CPURegister(index, size_, type_);
57 }
58
59
IsValid() const60 bool CPURegList::IsValid() const {
61 if ((type_ == CPURegister::kRegister) ||
62 (type_ == CPURegister::kVRegister)) {
63 bool is_valid = true;
64 // Try to create a CPURegister for each element in the list.
65 for (int i = 0; i < kRegListSizeInBits; i++) {
66 if (((list_ >> i) & 1) != 0) {
67 is_valid &= CPURegister(i, size_, type_).IsValid();
68 }
69 }
70 return is_valid;
71 } else if (type_ == CPURegister::kNoRegister) {
72 // We can't use IsEmpty here because that asserts IsValid().
73 return list_ == 0;
74 } else {
75 return false;
76 }
77 }
78
79
RemoveCalleeSaved()80 void CPURegList::RemoveCalleeSaved() {
81 if (type() == CPURegister::kRegister) {
82 Remove(GetCalleeSaved(RegisterSizeInBits()));
83 } else if (type() == CPURegister::kVRegister) {
84 Remove(GetCalleeSavedV(RegisterSizeInBits()));
85 } else {
86 VIXL_ASSERT(type() == CPURegister::kNoRegister);
87 VIXL_ASSERT(IsEmpty());
88 // The list must already be empty, so do nothing.
89 }
90 }
91
92
Union(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3)93 CPURegList CPURegList::Union(const CPURegList& list_1,
94 const CPURegList& list_2,
95 const CPURegList& list_3) {
96 return Union(list_1, Union(list_2, list_3));
97 }
98
99
Union(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3,const CPURegList & list_4)100 CPURegList CPURegList::Union(const CPURegList& list_1,
101 const CPURegList& list_2,
102 const CPURegList& list_3,
103 const CPURegList& list_4) {
104 return Union(Union(list_1, list_2), Union(list_3, list_4));
105 }
106
107
Intersection(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3)108 CPURegList CPURegList::Intersection(const CPURegList& list_1,
109 const CPURegList& list_2,
110 const CPURegList& list_3) {
111 return Intersection(list_1, Intersection(list_2, list_3));
112 }
113
114
Intersection(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3,const CPURegList & list_4)115 CPURegList CPURegList::Intersection(const CPURegList& list_1,
116 const CPURegList& list_2,
117 const CPURegList& list_3,
118 const CPURegList& list_4) {
119 return Intersection(Intersection(list_1, list_2),
120 Intersection(list_3, list_4));
121 }
122
123
GetCalleeSaved(unsigned size)124 CPURegList CPURegList::GetCalleeSaved(unsigned size) {
125 return CPURegList(CPURegister::kRegister, size, 19, 29);
126 }
127
128
GetCalleeSavedV(unsigned size)129 CPURegList CPURegList::GetCalleeSavedV(unsigned size) {
130 return CPURegList(CPURegister::kVRegister, size, 8, 15);
131 }
132
133
GetCallerSaved(unsigned size)134 CPURegList CPURegList::GetCallerSaved(unsigned size) {
135 // Registers x0-x18 and lr (x30) are caller-saved.
136 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18);
137 // Do not use lr directly to avoid initialisation order fiasco bugs for users.
138 list.Combine(Register(30, kXRegSize));
139 return list;
140 }
141
142
GetCallerSavedV(unsigned size)143 CPURegList CPURegList::GetCallerSavedV(unsigned size) {
144 // Registers d0-d7 and d16-d31 are caller-saved.
145 CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7);
146 list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31));
147 return list;
148 }
149
150
151 const CPURegList kCalleeSaved = CPURegList::GetCalleeSaved();
152 const CPURegList kCalleeSavedV = CPURegList::GetCalleeSavedV();
153 const CPURegList kCallerSaved = CPURegList::GetCallerSaved();
154 const CPURegList kCallerSavedV = CPURegList::GetCallerSavedV();
155
156
157 // Registers.
158 #define WREG(n) w##n,
159 const Register Register::wregisters[] = {
160 REGISTER_CODE_LIST(WREG)
161 };
162 #undef WREG
163
164 #define XREG(n) x##n,
165 const Register Register::xregisters[] = {
166 REGISTER_CODE_LIST(XREG)
167 };
168 #undef XREG
169
170 #define BREG(n) b##n,
171 const VRegister VRegister::bregisters[] = {
172 REGISTER_CODE_LIST(BREG)
173 };
174 #undef BREG
175
176 #define HREG(n) h##n,
177 const VRegister VRegister::hregisters[] = {
178 REGISTER_CODE_LIST(HREG)
179 };
180 #undef HREG
181
182 #define SREG(n) s##n,
183 const VRegister VRegister::sregisters[] = {
184 REGISTER_CODE_LIST(SREG)
185 };
186 #undef SREG
187
188 #define DREG(n) d##n,
189 const VRegister VRegister::dregisters[] = {
190 REGISTER_CODE_LIST(DREG)
191 };
192 #undef DREG
193
194 #define QREG(n) q##n,
195 const VRegister VRegister::qregisters[] = {
196 REGISTER_CODE_LIST(QREG)
197 };
198 #undef QREG
199
200 #define VREG(n) v##n,
201 const VRegister VRegister::vregisters[] = {
202 REGISTER_CODE_LIST(VREG)
203 };
204 #undef VREG
205
206
WRegFromCode(unsigned code)207 const Register& Register::WRegFromCode(unsigned code) {
208 if (code == kSPRegInternalCode) {
209 return wsp;
210 } else {
211 VIXL_ASSERT(code < kNumberOfRegisters);
212 return wregisters[code];
213 }
214 }
215
216
XRegFromCode(unsigned code)217 const Register& Register::XRegFromCode(unsigned code) {
218 if (code == kSPRegInternalCode) {
219 return sp;
220 } else {
221 VIXL_ASSERT(code < kNumberOfRegisters);
222 return xregisters[code];
223 }
224 }
225
226
BRegFromCode(unsigned code)227 const VRegister& VRegister::BRegFromCode(unsigned code) {
228 VIXL_ASSERT(code < kNumberOfVRegisters);
229 return bregisters[code];
230 }
231
232
HRegFromCode(unsigned code)233 const VRegister& VRegister::HRegFromCode(unsigned code) {
234 VIXL_ASSERT(code < kNumberOfVRegisters);
235 return hregisters[code];
236 }
237
238
SRegFromCode(unsigned code)239 const VRegister& VRegister::SRegFromCode(unsigned code) {
240 VIXL_ASSERT(code < kNumberOfVRegisters);
241 return sregisters[code];
242 }
243
244
DRegFromCode(unsigned code)245 const VRegister& VRegister::DRegFromCode(unsigned code) {
246 VIXL_ASSERT(code < kNumberOfVRegisters);
247 return dregisters[code];
248 }
249
250
QRegFromCode(unsigned code)251 const VRegister& VRegister::QRegFromCode(unsigned code) {
252 VIXL_ASSERT(code < kNumberOfVRegisters);
253 return qregisters[code];
254 }
255
256
VRegFromCode(unsigned code)257 const VRegister& VRegister::VRegFromCode(unsigned code) {
258 VIXL_ASSERT(code < kNumberOfVRegisters);
259 return vregisters[code];
260 }
261
262
W() const263 const Register& CPURegister::W() const {
264 VIXL_ASSERT(IsValidRegister());
265 return Register::WRegFromCode(code_);
266 }
267
268
X() const269 const Register& CPURegister::X() const {
270 VIXL_ASSERT(IsValidRegister());
271 return Register::XRegFromCode(code_);
272 }
273
274
B() const275 const VRegister& CPURegister::B() const {
276 VIXL_ASSERT(IsValidVRegister());
277 return VRegister::BRegFromCode(code_);
278 }
279
280
H() const281 const VRegister& CPURegister::H() const {
282 VIXL_ASSERT(IsValidVRegister());
283 return VRegister::HRegFromCode(code_);
284 }
285
286
S() const287 const VRegister& CPURegister::S() const {
288 VIXL_ASSERT(IsValidVRegister());
289 return VRegister::SRegFromCode(code_);
290 }
291
292
D() const293 const VRegister& CPURegister::D() const {
294 VIXL_ASSERT(IsValidVRegister());
295 return VRegister::DRegFromCode(code_);
296 }
297
298
Q() const299 const VRegister& CPURegister::Q() const {
300 VIXL_ASSERT(IsValidVRegister());
301 return VRegister::QRegFromCode(code_);
302 }
303
304
V() const305 const VRegister& CPURegister::V() const {
306 VIXL_ASSERT(IsValidVRegister());
307 return VRegister::VRegFromCode(code_);
308 }
309
310
311 // Operand.
Operand(int64_t immediate)312 Operand::Operand(int64_t immediate)
313 : immediate_(immediate),
314 reg_(NoReg),
315 shift_(NO_SHIFT),
316 extend_(NO_EXTEND),
317 shift_amount_(0) {}
318
319
Operand(Register reg,Shift shift,unsigned shift_amount)320 Operand::Operand(Register reg, Shift shift, unsigned shift_amount)
321 : reg_(reg),
322 shift_(shift),
323 extend_(NO_EXTEND),
324 shift_amount_(shift_amount) {
325 VIXL_ASSERT(shift != MSL);
326 VIXL_ASSERT(reg.Is64Bits() || (shift_amount < kWRegSize));
327 VIXL_ASSERT(reg.Is32Bits() || (shift_amount < kXRegSize));
328 VIXL_ASSERT(!reg.IsSP());
329 }
330
331
Operand(Register reg,Extend extend,unsigned shift_amount)332 Operand::Operand(Register reg, Extend extend, unsigned shift_amount)
333 : reg_(reg),
334 shift_(NO_SHIFT),
335 extend_(extend),
336 shift_amount_(shift_amount) {
337 VIXL_ASSERT(reg.IsValid());
338 VIXL_ASSERT(shift_amount <= 4);
339 VIXL_ASSERT(!reg.IsSP());
340
341 // Extend modes SXTX and UXTX require a 64-bit register.
342 VIXL_ASSERT(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX)));
343 }
344
345
IsImmediate() const346 bool Operand::IsImmediate() const {
347 return reg_.Is(NoReg);
348 }
349
350
IsShiftedRegister() const351 bool Operand::IsShiftedRegister() const {
352 return reg_.IsValid() && (shift_ != NO_SHIFT);
353 }
354
355
IsExtendedRegister() const356 bool Operand::IsExtendedRegister() const {
357 return reg_.IsValid() && (extend_ != NO_EXTEND);
358 }
359
360
IsZero() const361 bool Operand::IsZero() const {
362 if (IsImmediate()) {
363 return immediate() == 0;
364 } else {
365 return reg().IsZero();
366 }
367 }
368
369
ToExtendedRegister() const370 Operand Operand::ToExtendedRegister() const {
371 VIXL_ASSERT(IsShiftedRegister());
372 VIXL_ASSERT((shift_ == LSL) && (shift_amount_ <= 4));
373 return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_);
374 }
375
376
377 // MemOperand
MemOperand(Register base,int64_t offset,AddrMode addrmode)378 MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode)
379 : base_(base), regoffset_(NoReg), offset_(offset), addrmode_(addrmode) {
380 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
381 }
382
383
MemOperand(Register base,Register regoffset,Extend extend,unsigned shift_amount)384 MemOperand::MemOperand(Register base,
385 Register regoffset,
386 Extend extend,
387 unsigned shift_amount)
388 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
389 shift_(NO_SHIFT), extend_(extend), shift_amount_(shift_amount) {
390 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
391 VIXL_ASSERT(!regoffset.IsSP());
392 VIXL_ASSERT((extend == UXTW) || (extend == SXTW) || (extend == SXTX));
393
394 // SXTX extend mode requires a 64-bit offset register.
395 VIXL_ASSERT(regoffset.Is64Bits() || (extend != SXTX));
396 }
397
398
MemOperand(Register base,Register regoffset,Shift shift,unsigned shift_amount)399 MemOperand::MemOperand(Register base,
400 Register regoffset,
401 Shift shift,
402 unsigned shift_amount)
403 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
404 shift_(shift), extend_(NO_EXTEND), shift_amount_(shift_amount) {
405 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
406 VIXL_ASSERT(regoffset.Is64Bits() && !regoffset.IsSP());
407 VIXL_ASSERT(shift == LSL);
408 }
409
410
MemOperand(Register base,const Operand & offset,AddrMode addrmode)411 MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode)
412 : base_(base), regoffset_(NoReg), addrmode_(addrmode) {
413 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
414
415 if (offset.IsImmediate()) {
416 offset_ = offset.immediate();
417 } else if (offset.IsShiftedRegister()) {
418 VIXL_ASSERT((addrmode == Offset) || (addrmode == PostIndex));
419
420 regoffset_ = offset.reg();
421 shift_ = offset.shift();
422 shift_amount_ = offset.shift_amount();
423
424 extend_ = NO_EXTEND;
425 offset_ = 0;
426
427 // These assertions match those in the shifted-register constructor.
428 VIXL_ASSERT(regoffset_.Is64Bits() && !regoffset_.IsSP());
429 VIXL_ASSERT(shift_ == LSL);
430 } else {
431 VIXL_ASSERT(offset.IsExtendedRegister());
432 VIXL_ASSERT(addrmode == Offset);
433
434 regoffset_ = offset.reg();
435 extend_ = offset.extend();
436 shift_amount_ = offset.shift_amount();
437
438 shift_ = NO_SHIFT;
439 offset_ = 0;
440
441 // These assertions match those in the extended-register constructor.
442 VIXL_ASSERT(!regoffset_.IsSP());
443 VIXL_ASSERT((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX));
444 VIXL_ASSERT((regoffset_.Is64Bits() || (extend_ != SXTX)));
445 }
446 }
447
448
IsImmediateOffset() const449 bool MemOperand::IsImmediateOffset() const {
450 return (addrmode_ == Offset) && regoffset_.Is(NoReg);
451 }
452
453
IsRegisterOffset() const454 bool MemOperand::IsRegisterOffset() const {
455 return (addrmode_ == Offset) && !regoffset_.Is(NoReg);
456 }
457
458
IsPreIndex() const459 bool MemOperand::IsPreIndex() const {
460 return addrmode_ == PreIndex;
461 }
462
463
IsPostIndex() const464 bool MemOperand::IsPostIndex() const {
465 return addrmode_ == PostIndex;
466 }
467
468
AddOffset(int64_t offset)469 void MemOperand::AddOffset(int64_t offset) {
470 VIXL_ASSERT(IsImmediateOffset());
471 offset_ += offset;
472 }
473
474
475 // Assembler
Assembler(PositionIndependentCodeOption pic)476 Assembler::Assembler(PositionIndependentCodeOption pic)
477 : pic_(pic),
478 cpu_features_(CPUFeatures::AArch64LegacyBaseline())
479 {
480 // Mozilla change: always use maximally-present features.
481 cpu_features_.Combine(CPUFeatures::InferFromOS());
482
483 // Mozilla change: Compile time hard-coded value from js-config.mozbuild.
484 #ifndef MOZ_AARCH64_JSCVT
485 # error "MOZ_AARCH64_JSCVT must be defined."
486 #elif MOZ_AARCH64_JSCVT >= 1
487 // Note, vixl backend implements the JSCVT flag as a boolean despite having 3
488 // extra bits reserved for forward compatibility in the ARMv8 documentation.
489 cpu_features_.Combine(CPUFeatures::kJSCVT);
490 #endif
491 }
492
493
494 // Code generation.
br(const Register & xn)495 void Assembler::br(const Register& xn) {
496 VIXL_ASSERT(xn.Is64Bits());
497 Emit(BR | Rn(xn));
498 }
499
500
blr(const Register & xn)501 void Assembler::blr(const Register& xn) {
502 VIXL_ASSERT(xn.Is64Bits());
503 Emit(BLR | Rn(xn));
504 }
505
506
ret(const Register & xn)507 void Assembler::ret(const Register& xn) {
508 VIXL_ASSERT(xn.Is64Bits());
509 Emit(RET | Rn(xn));
510 }
511
512
NEONTable(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEONTableOp op)513 void Assembler::NEONTable(const VRegister& vd,
514 const VRegister& vn,
515 const VRegister& vm,
516 NEONTableOp op) {
517 VIXL_ASSERT(vd.Is16B() || vd.Is8B());
518 VIXL_ASSERT(vn.Is16B());
519 VIXL_ASSERT(AreSameFormat(vd, vm));
520 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
521 }
522
523
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vm)524 void Assembler::tbl(const VRegister& vd,
525 const VRegister& vn,
526 const VRegister& vm) {
527 NEONTable(vd, vn, vm, NEON_TBL_1v);
528 }
529
530
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)531 void Assembler::tbl(const VRegister& vd,
532 const VRegister& vn,
533 const VRegister& vn2,
534 const VRegister& vm) {
535 USE(vn2);
536 VIXL_ASSERT(AreSameFormat(vn, vn2));
537 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
538
539 NEONTable(vd, vn, vm, NEON_TBL_2v);
540 }
541
542
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)543 void Assembler::tbl(const VRegister& vd,
544 const VRegister& vn,
545 const VRegister& vn2,
546 const VRegister& vn3,
547 const VRegister& vm) {
548 USE(vn2, vn3);
549 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
550 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
551 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
552
553 NEONTable(vd, vn, vm, NEON_TBL_3v);
554 }
555
556
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)557 void Assembler::tbl(const VRegister& vd,
558 const VRegister& vn,
559 const VRegister& vn2,
560 const VRegister& vn3,
561 const VRegister& vn4,
562 const VRegister& vm) {
563 USE(vn2, vn3, vn4);
564 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
565 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
566 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
567 VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
568
569 NEONTable(vd, vn, vm, NEON_TBL_4v);
570 }
571
572
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vm)573 void Assembler::tbx(const VRegister& vd,
574 const VRegister& vn,
575 const VRegister& vm) {
576 NEONTable(vd, vn, vm, NEON_TBX_1v);
577 }
578
579
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)580 void Assembler::tbx(const VRegister& vd,
581 const VRegister& vn,
582 const VRegister& vn2,
583 const VRegister& vm) {
584 USE(vn2);
585 VIXL_ASSERT(AreSameFormat(vn, vn2));
586 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
587
588 NEONTable(vd, vn, vm, NEON_TBX_2v);
589 }
590
591
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)592 void Assembler::tbx(const VRegister& vd,
593 const VRegister& vn,
594 const VRegister& vn2,
595 const VRegister& vn3,
596 const VRegister& vm) {
597 USE(vn2, vn3);
598 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
599 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
600 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
601
602 NEONTable(vd, vn, vm, NEON_TBX_3v);
603 }
604
605
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)606 void Assembler::tbx(const VRegister& vd,
607 const VRegister& vn,
608 const VRegister& vn2,
609 const VRegister& vn3,
610 const VRegister& vn4,
611 const VRegister& vm) {
612 USE(vn2, vn3, vn4);
613 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
614 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
615 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
616 VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
617
618 NEONTable(vd, vn, vm, NEON_TBX_4v);
619 }
620
621
add(const Register & rd,const Register & rn,const Operand & operand)622 void Assembler::add(const Register& rd,
623 const Register& rn,
624 const Operand& operand) {
625 AddSub(rd, rn, operand, LeaveFlags, ADD);
626 }
627
628
adds(const Register & rd,const Register & rn,const Operand & operand)629 void Assembler::adds(const Register& rd,
630 const Register& rn,
631 const Operand& operand) {
632 AddSub(rd, rn, operand, SetFlags, ADD);
633 }
634
635
cmn(const Register & rn,const Operand & operand)636 void Assembler::cmn(const Register& rn,
637 const Operand& operand) {
638 Register zr = AppropriateZeroRegFor(rn);
639 adds(zr, rn, operand);
640 }
641
642
sub(const Register & rd,const Register & rn,const Operand & operand)643 void Assembler::sub(const Register& rd,
644 const Register& rn,
645 const Operand& operand) {
646 AddSub(rd, rn, operand, LeaveFlags, SUB);
647 }
648
649
subs(const Register & rd,const Register & rn,const Operand & operand)650 void Assembler::subs(const Register& rd,
651 const Register& rn,
652 const Operand& operand) {
653 AddSub(rd, rn, operand, SetFlags, SUB);
654 }
655
656
cmp(const Register & rn,const Operand & operand)657 void Assembler::cmp(const Register& rn, const Operand& operand) {
658 Register zr = AppropriateZeroRegFor(rn);
659 subs(zr, rn, operand);
660 }
661
662
neg(const Register & rd,const Operand & operand)663 void Assembler::neg(const Register& rd, const Operand& operand) {
664 Register zr = AppropriateZeroRegFor(rd);
665 sub(rd, zr, operand);
666 }
667
668
negs(const Register & rd,const Operand & operand)669 void Assembler::negs(const Register& rd, const Operand& operand) {
670 Register zr = AppropriateZeroRegFor(rd);
671 subs(rd, zr, operand);
672 }
673
674
adc(const Register & rd,const Register & rn,const Operand & operand)675 void Assembler::adc(const Register& rd,
676 const Register& rn,
677 const Operand& operand) {
678 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
679 }
680
681
adcs(const Register & rd,const Register & rn,const Operand & operand)682 void Assembler::adcs(const Register& rd,
683 const Register& rn,
684 const Operand& operand) {
685 AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
686 }
687
688
sbc(const Register & rd,const Register & rn,const Operand & operand)689 void Assembler::sbc(const Register& rd,
690 const Register& rn,
691 const Operand& operand) {
692 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
693 }
694
695
sbcs(const Register & rd,const Register & rn,const Operand & operand)696 void Assembler::sbcs(const Register& rd,
697 const Register& rn,
698 const Operand& operand) {
699 AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
700 }
701
702
ngc(const Register & rd,const Operand & operand)703 void Assembler::ngc(const Register& rd, const Operand& operand) {
704 Register zr = AppropriateZeroRegFor(rd);
705 sbc(rd, zr, operand);
706 }
707
708
ngcs(const Register & rd,const Operand & operand)709 void Assembler::ngcs(const Register& rd, const Operand& operand) {
710 Register zr = AppropriateZeroRegFor(rd);
711 sbcs(rd, zr, operand);
712 }
713
714
715 // Logical instructions.
and_(const Register & rd,const Register & rn,const Operand & operand)716 void Assembler::and_(const Register& rd,
717 const Register& rn,
718 const Operand& operand) {
719 Logical(rd, rn, operand, AND);
720 }
721
722
bic(const Register & rd,const Register & rn,const Operand & operand)723 void Assembler::bic(const Register& rd,
724 const Register& rn,
725 const Operand& operand) {
726 Logical(rd, rn, operand, BIC);
727 }
728
729
bics(const Register & rd,const Register & rn,const Operand & operand)730 void Assembler::bics(const Register& rd,
731 const Register& rn,
732 const Operand& operand) {
733 Logical(rd, rn, operand, BICS);
734 }
735
736
orr(const Register & rd,const Register & rn,const Operand & operand)737 void Assembler::orr(const Register& rd,
738 const Register& rn,
739 const Operand& operand) {
740 Logical(rd, rn, operand, ORR);
741 }
742
743
orn(const Register & rd,const Register & rn,const Operand & operand)744 void Assembler::orn(const Register& rd,
745 const Register& rn,
746 const Operand& operand) {
747 Logical(rd, rn, operand, ORN);
748 }
749
750
eor(const Register & rd,const Register & rn,const Operand & operand)751 void Assembler::eor(const Register& rd,
752 const Register& rn,
753 const Operand& operand) {
754 Logical(rd, rn, operand, EOR);
755 }
756
757
eon(const Register & rd,const Register & rn,const Operand & operand)758 void Assembler::eon(const Register& rd,
759 const Register& rn,
760 const Operand& operand) {
761 Logical(rd, rn, operand, EON);
762 }
763
764
lslv(const Register & rd,const Register & rn,const Register & rm)765 void Assembler::lslv(const Register& rd,
766 const Register& rn,
767 const Register& rm) {
768 VIXL_ASSERT(rd.size() == rn.size());
769 VIXL_ASSERT(rd.size() == rm.size());
770 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
771 }
772
773
lsrv(const Register & rd,const Register & rn,const Register & rm)774 void Assembler::lsrv(const Register& rd,
775 const Register& rn,
776 const Register& rm) {
777 VIXL_ASSERT(rd.size() == rn.size());
778 VIXL_ASSERT(rd.size() == rm.size());
779 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
780 }
781
782
asrv(const Register & rd,const Register & rn,const Register & rm)783 void Assembler::asrv(const Register& rd,
784 const Register& rn,
785 const Register& rm) {
786 VIXL_ASSERT(rd.size() == rn.size());
787 VIXL_ASSERT(rd.size() == rm.size());
788 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
789 }
790
791
rorv(const Register & rd,const Register & rn,const Register & rm)792 void Assembler::rorv(const Register& rd,
793 const Register& rn,
794 const Register& rm) {
795 VIXL_ASSERT(rd.size() == rn.size());
796 VIXL_ASSERT(rd.size() == rm.size());
797 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
798 }
799
800
801 // Bitfield operations.
bfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)802 void Assembler::bfm(const Register& rd,
803 const Register& rn,
804 unsigned immr,
805 unsigned imms) {
806 VIXL_ASSERT(rd.size() == rn.size());
807 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
808 Emit(SF(rd) | BFM | N |
809 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
810 }
811
812
sbfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)813 void Assembler::sbfm(const Register& rd,
814 const Register& rn,
815 unsigned immr,
816 unsigned imms) {
817 VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
818 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
819 Emit(SF(rd) | SBFM | N |
820 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
821 }
822
823
ubfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)824 void Assembler::ubfm(const Register& rd,
825 const Register& rn,
826 unsigned immr,
827 unsigned imms) {
828 VIXL_ASSERT(rd.size() == rn.size());
829 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
830 Emit(SF(rd) | UBFM | N |
831 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
832 }
833
834
extr(const Register & rd,const Register & rn,const Register & rm,unsigned lsb)835 void Assembler::extr(const Register& rd,
836 const Register& rn,
837 const Register& rm,
838 unsigned lsb) {
839 VIXL_ASSERT(rd.size() == rn.size());
840 VIXL_ASSERT(rd.size() == rm.size());
841 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
842 Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.size()) | Rn(rn) | Rd(rd));
843 }
844
845
csel(const Register & rd,const Register & rn,const Register & rm,Condition cond)846 void Assembler::csel(const Register& rd,
847 const Register& rn,
848 const Register& rm,
849 Condition cond) {
850 ConditionalSelect(rd, rn, rm, cond, CSEL);
851 }
852
853
csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)854 void Assembler::csinc(const Register& rd,
855 const Register& rn,
856 const Register& rm,
857 Condition cond) {
858 ConditionalSelect(rd, rn, rm, cond, CSINC);
859 }
860
861
csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)862 void Assembler::csinv(const Register& rd,
863 const Register& rn,
864 const Register& rm,
865 Condition cond) {
866 ConditionalSelect(rd, rn, rm, cond, CSINV);
867 }
868
869
csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)870 void Assembler::csneg(const Register& rd,
871 const Register& rn,
872 const Register& rm,
873 Condition cond) {
874 ConditionalSelect(rd, rn, rm, cond, CSNEG);
875 }
876
877
cset(const Register & rd,Condition cond)878 void Assembler::cset(const Register &rd, Condition cond) {
879 VIXL_ASSERT((cond != al) && (cond != nv));
880 Register zr = AppropriateZeroRegFor(rd);
881 csinc(rd, zr, zr, InvertCondition(cond));
882 }
883
884
csetm(const Register & rd,Condition cond)885 void Assembler::csetm(const Register &rd, Condition cond) {
886 VIXL_ASSERT((cond != al) && (cond != nv));
887 Register zr = AppropriateZeroRegFor(rd);
888 csinv(rd, zr, zr, InvertCondition(cond));
889 }
890
891
cinc(const Register & rd,const Register & rn,Condition cond)892 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) {
893 VIXL_ASSERT((cond != al) && (cond != nv));
894 csinc(rd, rn, rn, InvertCondition(cond));
895 }
896
897
cinv(const Register & rd,const Register & rn,Condition cond)898 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) {
899 VIXL_ASSERT((cond != al) && (cond != nv));
900 csinv(rd, rn, rn, InvertCondition(cond));
901 }
902
903
cneg(const Register & rd,const Register & rn,Condition cond)904 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) {
905 VIXL_ASSERT((cond != al) && (cond != nv));
906 csneg(rd, rn, rn, InvertCondition(cond));
907 }
908
909
ConditionalSelect(const Register & rd,const Register & rn,const Register & rm,Condition cond,ConditionalSelectOp op)910 void Assembler::ConditionalSelect(const Register& rd,
911 const Register& rn,
912 const Register& rm,
913 Condition cond,
914 ConditionalSelectOp op) {
915 VIXL_ASSERT(rd.size() == rn.size());
916 VIXL_ASSERT(rd.size() == rm.size());
917 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
918 }
919
920
ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)921 void Assembler::ccmn(const Register& rn,
922 const Operand& operand,
923 StatusFlags nzcv,
924 Condition cond) {
925 ConditionalCompare(rn, operand, nzcv, cond, CCMN);
926 }
927
928
ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)929 void Assembler::ccmp(const Register& rn,
930 const Operand& operand,
931 StatusFlags nzcv,
932 Condition cond) {
933 ConditionalCompare(rn, operand, nzcv, cond, CCMP);
934 }
935
936
DataProcessing3Source(const Register & rd,const Register & rn,const Register & rm,const Register & ra,DataProcessing3SourceOp op)937 void Assembler::DataProcessing3Source(const Register& rd,
938 const Register& rn,
939 const Register& rm,
940 const Register& ra,
941 DataProcessing3SourceOp op) {
942 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
943 }
944
945
crc32b(const Register & rd,const Register & rn,const Register & rm)946 void Assembler::crc32b(const Register& rd,
947 const Register& rn,
948 const Register& rm) {
949 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
950 Emit(SF(rm) | Rm(rm) | CRC32B | Rn(rn) | Rd(rd));
951 }
952
953
crc32h(const Register & rd,const Register & rn,const Register & rm)954 void Assembler::crc32h(const Register& rd,
955 const Register& rn,
956 const Register& rm) {
957 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
958 Emit(SF(rm) | Rm(rm) | CRC32H | Rn(rn) | Rd(rd));
959 }
960
961
crc32w(const Register & rd,const Register & rn,const Register & rm)962 void Assembler::crc32w(const Register& rd,
963 const Register& rn,
964 const Register& rm) {
965 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
966 Emit(SF(rm) | Rm(rm) | CRC32W | Rn(rn) | Rd(rd));
967 }
968
969
crc32x(const Register & rd,const Register & rn,const Register & rm)970 void Assembler::crc32x(const Register& rd,
971 const Register& rn,
972 const Register& rm) {
973 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
974 Emit(SF(rm) | Rm(rm) | CRC32X | Rn(rn) | Rd(rd));
975 }
976
977
crc32cb(const Register & rd,const Register & rn,const Register & rm)978 void Assembler::crc32cb(const Register& rd,
979 const Register& rn,
980 const Register& rm) {
981 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
982 Emit(SF(rm) | Rm(rm) | CRC32CB | Rn(rn) | Rd(rd));
983 }
984
985
crc32ch(const Register & rd,const Register & rn,const Register & rm)986 void Assembler::crc32ch(const Register& rd,
987 const Register& rn,
988 const Register& rm) {
989 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
990 Emit(SF(rm) | Rm(rm) | CRC32CH | Rn(rn) | Rd(rd));
991 }
992
993
crc32cw(const Register & rd,const Register & rn,const Register & rm)994 void Assembler::crc32cw(const Register& rd,
995 const Register& rn,
996 const Register& rm) {
997 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
998 Emit(SF(rm) | Rm(rm) | CRC32CW | Rn(rn) | Rd(rd));
999 }
1000
1001
crc32cx(const Register & rd,const Register & rn,const Register & rm)1002 void Assembler::crc32cx(const Register& rd,
1003 const Register& rn,
1004 const Register& rm) {
1005 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
1006 Emit(SF(rm) | Rm(rm) | CRC32CX | Rn(rn) | Rd(rd));
1007 }
1008
1009
mul(const Register & rd,const Register & rn,const Register & rm)1010 void Assembler::mul(const Register& rd,
1011 const Register& rn,
1012 const Register& rm) {
1013 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
1014 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
1015 }
1016
1017
madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1018 void Assembler::madd(const Register& rd,
1019 const Register& rn,
1020 const Register& rm,
1021 const Register& ra) {
1022 DataProcessing3Source(rd, rn, rm, ra, MADD);
1023 }
1024
1025
mneg(const Register & rd,const Register & rn,const Register & rm)1026 void Assembler::mneg(const Register& rd,
1027 const Register& rn,
1028 const Register& rm) {
1029 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
1030 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
1031 }
1032
1033
msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1034 void Assembler::msub(const Register& rd,
1035 const Register& rn,
1036 const Register& rm,
1037 const Register& ra) {
1038 DataProcessing3Source(rd, rn, rm, ra, MSUB);
1039 }
1040
1041
umaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1042 void Assembler::umaddl(const Register& rd,
1043 const Register& rn,
1044 const Register& rm,
1045 const Register& ra) {
1046 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1047 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1048 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
1049 }
1050
1051
smaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1052 void Assembler::smaddl(const Register& rd,
1053 const Register& rn,
1054 const Register& rm,
1055 const Register& ra) {
1056 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1057 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1058 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
1059 }
1060
1061
umsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1062 void Assembler::umsubl(const Register& rd,
1063 const Register& rn,
1064 const Register& rm,
1065 const Register& ra) {
1066 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1067 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1068 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
1069 }
1070
1071
smsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1072 void Assembler::smsubl(const Register& rd,
1073 const Register& rn,
1074 const Register& rm,
1075 const Register& ra) {
1076 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1077 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1078 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
1079 }
1080
1081
smull(const Register & rd,const Register & rn,const Register & rm)1082 void Assembler::smull(const Register& rd,
1083 const Register& rn,
1084 const Register& rm) {
1085 VIXL_ASSERT(rd.Is64Bits());
1086 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1087 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
1088 }
1089
1090
sdiv(const Register & rd,const Register & rn,const Register & rm)1091 void Assembler::sdiv(const Register& rd,
1092 const Register& rn,
1093 const Register& rm) {
1094 VIXL_ASSERT(rd.size() == rn.size());
1095 VIXL_ASSERT(rd.size() == rm.size());
1096 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
1097 }
1098
1099
smulh(const Register & xd,const Register & xn,const Register & xm)1100 void Assembler::smulh(const Register& xd,
1101 const Register& xn,
1102 const Register& xm) {
1103 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1104 DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
1105 }
1106
1107
umulh(const Register & xd,const Register & xn,const Register & xm)1108 void Assembler::umulh(const Register& xd,
1109 const Register& xn,
1110 const Register& xm) {
1111 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1112 DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
1113 }
1114
1115
udiv(const Register & rd,const Register & rn,const Register & rm)1116 void Assembler::udiv(const Register& rd,
1117 const Register& rn,
1118 const Register& rm) {
1119 VIXL_ASSERT(rd.size() == rn.size());
1120 VIXL_ASSERT(rd.size() == rm.size());
1121 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1122 }
1123
1124
rbit(const Register & rd,const Register & rn)1125 void Assembler::rbit(const Register& rd,
1126 const Register& rn) {
1127 DataProcessing1Source(rd, rn, RBIT);
1128 }
1129
1130
rev16(const Register & rd,const Register & rn)1131 void Assembler::rev16(const Register& rd,
1132 const Register& rn) {
1133 DataProcessing1Source(rd, rn, REV16);
1134 }
1135
1136
rev32(const Register & rd,const Register & rn)1137 void Assembler::rev32(const Register& rd,
1138 const Register& rn) {
1139 VIXL_ASSERT(rd.Is64Bits());
1140 DataProcessing1Source(rd, rn, REV);
1141 }
1142
1143
rev(const Register & rd,const Register & rn)1144 void Assembler::rev(const Register& rd,
1145 const Register& rn) {
1146 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1147 }
1148
1149
clz(const Register & rd,const Register & rn)1150 void Assembler::clz(const Register& rd,
1151 const Register& rn) {
1152 DataProcessing1Source(rd, rn, CLZ);
1153 }
1154
1155
cls(const Register & rd,const Register & rn)1156 void Assembler::cls(const Register& rd,
1157 const Register& rn) {
1158 DataProcessing1Source(rd, rn, CLS);
1159 }
1160
1161
ldp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1162 void Assembler::ldp(const CPURegister& rt,
1163 const CPURegister& rt2,
1164 const MemOperand& src) {
1165 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1166 }
1167
1168
stp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1169 void Assembler::stp(const CPURegister& rt,
1170 const CPURegister& rt2,
1171 const MemOperand& dst) {
1172 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1173 }
1174
1175
ldpsw(const Register & rt,const Register & rt2,const MemOperand & src)1176 void Assembler::ldpsw(const Register& rt,
1177 const Register& rt2,
1178 const MemOperand& src) {
1179 VIXL_ASSERT(rt.Is64Bits());
1180 LoadStorePair(rt, rt2, src, LDPSW_x);
1181 }
1182
1183
LoadStorePair(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)1184 void Assembler::LoadStorePair(const CPURegister& rt,
1185 const CPURegister& rt2,
1186 const MemOperand& addr,
1187 LoadStorePairOp op) {
1188 // 'rt' and 'rt2' can only be aliased for stores.
1189 VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1190 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1191 VIXL_ASSERT(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
1192
1193 int offset = static_cast<int>(addr.offset());
1194 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
1195 ImmLSPair(offset, CalcLSPairDataSize(op));
1196
1197 Instr addrmodeop;
1198 if (addr.IsImmediateOffset()) {
1199 addrmodeop = LoadStorePairOffsetFixed;
1200 } else {
1201 VIXL_ASSERT(addr.offset() != 0);
1202 if (addr.IsPreIndex()) {
1203 addrmodeop = LoadStorePairPreIndexFixed;
1204 } else {
1205 VIXL_ASSERT(addr.IsPostIndex());
1206 addrmodeop = LoadStorePairPostIndexFixed;
1207 }
1208 }
1209 Emit(addrmodeop | memop);
1210 }
1211
1212
ldnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1213 void Assembler::ldnp(const CPURegister& rt,
1214 const CPURegister& rt2,
1215 const MemOperand& src) {
1216 LoadStorePairNonTemporal(rt, rt2, src,
1217 LoadPairNonTemporalOpFor(rt, rt2));
1218 }
1219
1220
stnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1221 void Assembler::stnp(const CPURegister& rt,
1222 const CPURegister& rt2,
1223 const MemOperand& dst) {
1224 LoadStorePairNonTemporal(rt, rt2, dst,
1225 StorePairNonTemporalOpFor(rt, rt2));
1226 }
1227
1228
LoadStorePairNonTemporal(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairNonTemporalOp op)1229 void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1230 const CPURegister& rt2,
1231 const MemOperand& addr,
1232 LoadStorePairNonTemporalOp op) {
1233 VIXL_ASSERT(!rt.Is(rt2));
1234 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1235 VIXL_ASSERT(addr.IsImmediateOffset());
1236
1237 unsigned size = CalcLSPairDataSize(
1238 static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1239 VIXL_ASSERT(IsImmLSPair(addr.offset(), size));
1240 int offset = static_cast<int>(addr.offset());
1241 Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | ImmLSPair(offset, size));
1242 }
1243
1244
1245 // Memory instructions.
ldrb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1246 void Assembler::ldrb(const Register& rt, const MemOperand& src,
1247 LoadStoreScalingOption option) {
1248 VIXL_ASSERT(option != RequireUnscaledOffset);
1249 VIXL_ASSERT(option != PreferUnscaledOffset);
1250 LoadStore(rt, src, LDRB_w, option);
1251 }
1252
1253
strb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1254 void Assembler::strb(const Register& rt, const MemOperand& dst,
1255 LoadStoreScalingOption option) {
1256 VIXL_ASSERT(option != RequireUnscaledOffset);
1257 VIXL_ASSERT(option != PreferUnscaledOffset);
1258 LoadStore(rt, dst, STRB_w, option);
1259 }
1260
1261
ldrsb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1262 void Assembler::ldrsb(const Register& rt, const MemOperand& src,
1263 LoadStoreScalingOption option) {
1264 VIXL_ASSERT(option != RequireUnscaledOffset);
1265 VIXL_ASSERT(option != PreferUnscaledOffset);
1266 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1267 }
1268
1269
ldrh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1270 void Assembler::ldrh(const Register& rt, const MemOperand& src,
1271 LoadStoreScalingOption option) {
1272 VIXL_ASSERT(option != RequireUnscaledOffset);
1273 VIXL_ASSERT(option != PreferUnscaledOffset);
1274 LoadStore(rt, src, LDRH_w, option);
1275 }
1276
1277
strh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1278 void Assembler::strh(const Register& rt, const MemOperand& dst,
1279 LoadStoreScalingOption option) {
1280 VIXL_ASSERT(option != RequireUnscaledOffset);
1281 VIXL_ASSERT(option != PreferUnscaledOffset);
1282 LoadStore(rt, dst, STRH_w, option);
1283 }
1284
1285
ldrsh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1286 void Assembler::ldrsh(const Register& rt, const MemOperand& src,
1287 LoadStoreScalingOption option) {
1288 VIXL_ASSERT(option != RequireUnscaledOffset);
1289 VIXL_ASSERT(option != PreferUnscaledOffset);
1290 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1291 }
1292
1293
ldr(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1294 void Assembler::ldr(const CPURegister& rt, const MemOperand& src,
1295 LoadStoreScalingOption option) {
1296 VIXL_ASSERT(option != RequireUnscaledOffset);
1297 VIXL_ASSERT(option != PreferUnscaledOffset);
1298 LoadStore(rt, src, LoadOpFor(rt), option);
1299 }
1300
1301
str(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1302 void Assembler::str(const CPURegister& rt, const MemOperand& dst,
1303 LoadStoreScalingOption option) {
1304 VIXL_ASSERT(option != RequireUnscaledOffset);
1305 VIXL_ASSERT(option != PreferUnscaledOffset);
1306 LoadStore(rt, dst, StoreOpFor(rt), option);
1307 }
1308
1309
ldrsw(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1310 void Assembler::ldrsw(const Register& rt, const MemOperand& src,
1311 LoadStoreScalingOption option) {
1312 VIXL_ASSERT(rt.Is64Bits());
1313 VIXL_ASSERT(option != RequireUnscaledOffset);
1314 VIXL_ASSERT(option != PreferUnscaledOffset);
1315 LoadStore(rt, src, LDRSW_x, option);
1316 }
1317
1318
ldurb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1319 void Assembler::ldurb(const Register& rt, const MemOperand& src,
1320 LoadStoreScalingOption option) {
1321 VIXL_ASSERT(option != RequireScaledOffset);
1322 VIXL_ASSERT(option != PreferScaledOffset);
1323 LoadStore(rt, src, LDRB_w, option);
1324 }
1325
1326
sturb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1327 void Assembler::sturb(const Register& rt, const MemOperand& dst,
1328 LoadStoreScalingOption option) {
1329 VIXL_ASSERT(option != RequireScaledOffset);
1330 VIXL_ASSERT(option != PreferScaledOffset);
1331 LoadStore(rt, dst, STRB_w, option);
1332 }
1333
1334
ldursb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1335 void Assembler::ldursb(const Register& rt, const MemOperand& src,
1336 LoadStoreScalingOption option) {
1337 VIXL_ASSERT(option != RequireScaledOffset);
1338 VIXL_ASSERT(option != PreferScaledOffset);
1339 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1340 }
1341
1342
ldurh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1343 void Assembler::ldurh(const Register& rt, const MemOperand& src,
1344 LoadStoreScalingOption option) {
1345 VIXL_ASSERT(option != RequireScaledOffset);
1346 VIXL_ASSERT(option != PreferScaledOffset);
1347 LoadStore(rt, src, LDRH_w, option);
1348 }
1349
1350
sturh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1351 void Assembler::sturh(const Register& rt, const MemOperand& dst,
1352 LoadStoreScalingOption option) {
1353 VIXL_ASSERT(option != RequireScaledOffset);
1354 VIXL_ASSERT(option != PreferScaledOffset);
1355 LoadStore(rt, dst, STRH_w, option);
1356 }
1357
1358
ldursh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1359 void Assembler::ldursh(const Register& rt, const MemOperand& src,
1360 LoadStoreScalingOption option) {
1361 VIXL_ASSERT(option != RequireScaledOffset);
1362 VIXL_ASSERT(option != PreferScaledOffset);
1363 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1364 }
1365
1366
ldur(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1367 void Assembler::ldur(const CPURegister& rt, const MemOperand& src,
1368 LoadStoreScalingOption option) {
1369 VIXL_ASSERT(option != RequireScaledOffset);
1370 VIXL_ASSERT(option != PreferScaledOffset);
1371 LoadStore(rt, src, LoadOpFor(rt), option);
1372 }
1373
1374
stur(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1375 void Assembler::stur(const CPURegister& rt, const MemOperand& dst,
1376 LoadStoreScalingOption option) {
1377 VIXL_ASSERT(option != RequireScaledOffset);
1378 VIXL_ASSERT(option != PreferScaledOffset);
1379 LoadStore(rt, dst, StoreOpFor(rt), option);
1380 }
1381
1382
ldursw(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1383 void Assembler::ldursw(const Register& rt, const MemOperand& src,
1384 LoadStoreScalingOption option) {
1385 VIXL_ASSERT(rt.Is64Bits());
1386 VIXL_ASSERT(option != RequireScaledOffset);
1387 VIXL_ASSERT(option != PreferScaledOffset);
1388 LoadStore(rt, src, LDRSW_x, option);
1389 }
1390
1391
ldrsw(const Register & rt,int imm19)1392 void Assembler::ldrsw(const Register& rt, int imm19) {
1393 Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1394 }
1395
1396
ldr(const CPURegister & rt,int imm19)1397 void Assembler::ldr(const CPURegister& rt, int imm19) {
1398 LoadLiteralOp op = LoadLiteralOpFor(rt);
1399 Emit(op | ImmLLiteral(imm19) | Rt(rt));
1400 }
1401
1402
prfm(PrefetchOperation op,int imm19)1403 void Assembler::prfm(PrefetchOperation op, int imm19) {
1404 Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1405 }
1406
1407
1408 // Exclusive-access instructions.
stxrb(const Register & rs,const Register & rt,const MemOperand & dst)1409 void Assembler::stxrb(const Register& rs,
1410 const Register& rt,
1411 const MemOperand& dst) {
1412 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1413 Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1414 }
1415
1416
stxrh(const Register & rs,const Register & rt,const MemOperand & dst)1417 void Assembler::stxrh(const Register& rs,
1418 const Register& rt,
1419 const MemOperand& dst) {
1420 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1421 Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1422 }
1423
1424
stxr(const Register & rs,const Register & rt,const MemOperand & dst)1425 void Assembler::stxr(const Register& rs,
1426 const Register& rt,
1427 const MemOperand& dst) {
1428 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1429 LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1430 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1431 }
1432
1433
ldxrb(const Register & rt,const MemOperand & src)1434 void Assembler::ldxrb(const Register& rt,
1435 const MemOperand& src) {
1436 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1437 Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1438 }
1439
1440
ldxrh(const Register & rt,const MemOperand & src)1441 void Assembler::ldxrh(const Register& rt,
1442 const MemOperand& src) {
1443 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1444 Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1445 }
1446
1447
ldxr(const Register & rt,const MemOperand & src)1448 void Assembler::ldxr(const Register& rt,
1449 const MemOperand& src) {
1450 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1451 LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1452 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1453 }
1454
1455
stxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1456 void Assembler::stxp(const Register& rs,
1457 const Register& rt,
1458 const Register& rt2,
1459 const MemOperand& dst) {
1460 VIXL_ASSERT(rt.size() == rt2.size());
1461 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1462 LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1463 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
1464 }
1465
1466
ldxp(const Register & rt,const Register & rt2,const MemOperand & src)1467 void Assembler::ldxp(const Register& rt,
1468 const Register& rt2,
1469 const MemOperand& src) {
1470 VIXL_ASSERT(rt.size() == rt2.size());
1471 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1472 LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1473 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
1474 }
1475
1476
stlxrb(const Register & rs,const Register & rt,const MemOperand & dst)1477 void Assembler::stlxrb(const Register& rs,
1478 const Register& rt,
1479 const MemOperand& dst) {
1480 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1481 Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1482 }
1483
1484
stlxrh(const Register & rs,const Register & rt,const MemOperand & dst)1485 void Assembler::stlxrh(const Register& rs,
1486 const Register& rt,
1487 const MemOperand& dst) {
1488 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1489 Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1490 }
1491
1492
stlxr(const Register & rs,const Register & rt,const MemOperand & dst)1493 void Assembler::stlxr(const Register& rs,
1494 const Register& rt,
1495 const MemOperand& dst) {
1496 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1497 LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1498 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1499 }
1500
1501
ldaxrb(const Register & rt,const MemOperand & src)1502 void Assembler::ldaxrb(const Register& rt,
1503 const MemOperand& src) {
1504 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1505 Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1506 }
1507
1508
ldaxrh(const Register & rt,const MemOperand & src)1509 void Assembler::ldaxrh(const Register& rt,
1510 const MemOperand& src) {
1511 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1512 Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1513 }
1514
1515
ldaxr(const Register & rt,const MemOperand & src)1516 void Assembler::ldaxr(const Register& rt,
1517 const MemOperand& src) {
1518 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1519 LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1520 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1521 }
1522
1523
stlxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1524 void Assembler::stlxp(const Register& rs,
1525 const Register& rt,
1526 const Register& rt2,
1527 const MemOperand& dst) {
1528 VIXL_ASSERT(rt.size() == rt2.size());
1529 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1530 LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1531 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
1532 }
1533
1534
ldaxp(const Register & rt,const Register & rt2,const MemOperand & src)1535 void Assembler::ldaxp(const Register& rt,
1536 const Register& rt2,
1537 const MemOperand& src) {
1538 VIXL_ASSERT(rt.size() == rt2.size());
1539 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1540 LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1541 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
1542 }
1543
1544
stlrb(const Register & rt,const MemOperand & dst)1545 void Assembler::stlrb(const Register& rt,
1546 const MemOperand& dst) {
1547 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1548 Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1549 }
1550
1551
stlrh(const Register & rt,const MemOperand & dst)1552 void Assembler::stlrh(const Register& rt,
1553 const MemOperand& dst) {
1554 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1555 Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1556 }
1557
1558
stlr(const Register & rt,const MemOperand & dst)1559 void Assembler::stlr(const Register& rt,
1560 const MemOperand& dst) {
1561 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1562 LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1563 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1564 }
1565
1566
ldarb(const Register & rt,const MemOperand & src)1567 void Assembler::ldarb(const Register& rt,
1568 const MemOperand& src) {
1569 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1570 Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1571 }
1572
1573
ldarh(const Register & rt,const MemOperand & src)1574 void Assembler::ldarh(const Register& rt,
1575 const MemOperand& src) {
1576 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1577 Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1578 }
1579
1580
ldar(const Register & rt,const MemOperand & src)1581 void Assembler::ldar(const Register& rt,
1582 const MemOperand& src) {
1583 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1584 LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1585 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1586 }
1587
1588
prfm(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1589 void Assembler::prfm(PrefetchOperation op, const MemOperand& address,
1590 LoadStoreScalingOption option) {
1591 VIXL_ASSERT(option != RequireUnscaledOffset);
1592 VIXL_ASSERT(option != PreferUnscaledOffset);
1593 Prefetch(op, address, option);
1594 }
1595
1596
prfum(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1597 void Assembler::prfum(PrefetchOperation op, const MemOperand& address,
1598 LoadStoreScalingOption option) {
1599 VIXL_ASSERT(option != RequireScaledOffset);
1600 VIXL_ASSERT(option != PreferScaledOffset);
1601 Prefetch(op, address, option);
1602 }
1603
1604
sys(int op1,int crn,int crm,int op2,const Register & rt)1605 void Assembler::sys(int op1, int crn, int crm, int op2, const Register& rt) {
1606 Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(rt));
1607 }
1608
1609
sys(int op,const Register & rt)1610 void Assembler::sys(int op, const Register& rt) {
1611 Emit(SYS | SysOp(op) | Rt(rt));
1612 }
1613
1614
dc(DataCacheOp op,const Register & rt)1615 void Assembler::dc(DataCacheOp op, const Register& rt) {
1616 VIXL_ASSERT((op == CVAC) || (op == CVAU) || (op == CIVAC) || (op == ZVA));
1617 sys(op, rt);
1618 }
1619
1620
ic(InstructionCacheOp op,const Register & rt)1621 void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1622 VIXL_ASSERT(op == IVAU);
1623 sys(op, rt);
1624 }
1625
1626
1627 // NEON structure loads and stores.
LoadStoreStructAddrModeField(const MemOperand & addr)1628 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
1629 Instr addr_field = RnSP(addr.base());
1630
1631 if (addr.IsPostIndex()) {
1632 VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
1633 static_cast<NEONLoadStoreMultiStructPostIndexOp>(
1634 NEONLoadStoreSingleStructPostIndex));
1635
1636 addr_field |= NEONLoadStoreMultiStructPostIndex;
1637 if (addr.offset() == 0) {
1638 addr_field |= RmNot31(addr.regoffset());
1639 } else {
1640 // The immediate post index addressing mode is indicated by rm = 31.
1641 // The immediate is implied by the number of vector registers used.
1642 addr_field |= (0x1f << Rm_offset);
1643 }
1644 } else {
1645 VIXL_ASSERT(addr.IsImmediateOffset() && (addr.offset() == 0));
1646 }
1647 return addr_field;
1648 }
1649
LoadStoreStructVerify(const VRegister & vt,const MemOperand & addr,Instr op)1650 void Assembler::LoadStoreStructVerify(const VRegister& vt,
1651 const MemOperand& addr,
1652 Instr op) {
1653 #ifdef DEBUG
1654 // Assert that addressing mode is either offset (with immediate 0), post
1655 // index by immediate of the size of the register list, or post index by a
1656 // value in a core register.
1657 if (addr.IsImmediateOffset()) {
1658 VIXL_ASSERT(addr.offset() == 0);
1659 } else {
1660 int offset = vt.SizeInBytes();
1661 switch (op) {
1662 case NEON_LD1_1v:
1663 case NEON_ST1_1v:
1664 offset *= 1; break;
1665 case NEONLoadStoreSingleStructLoad1:
1666 case NEONLoadStoreSingleStructStore1:
1667 case NEON_LD1R:
1668 offset = (offset / vt.lanes()) * 1; break;
1669
1670 case NEON_LD1_2v:
1671 case NEON_ST1_2v:
1672 case NEON_LD2:
1673 case NEON_ST2:
1674 offset *= 2;
1675 break;
1676 case NEONLoadStoreSingleStructLoad2:
1677 case NEONLoadStoreSingleStructStore2:
1678 case NEON_LD2R:
1679 offset = (offset / vt.lanes()) * 2; break;
1680
1681 case NEON_LD1_3v:
1682 case NEON_ST1_3v:
1683 case NEON_LD3:
1684 case NEON_ST3:
1685 offset *= 3; break;
1686 case NEONLoadStoreSingleStructLoad3:
1687 case NEONLoadStoreSingleStructStore3:
1688 case NEON_LD3R:
1689 offset = (offset / vt.lanes()) * 3; break;
1690
1691 case NEON_LD1_4v:
1692 case NEON_ST1_4v:
1693 case NEON_LD4:
1694 case NEON_ST4:
1695 offset *= 4; break;
1696 case NEONLoadStoreSingleStructLoad4:
1697 case NEONLoadStoreSingleStructStore4:
1698 case NEON_LD4R:
1699 offset = (offset / vt.lanes()) * 4; break;
1700 default:
1701 VIXL_UNREACHABLE();
1702 }
1703 VIXL_ASSERT(!addr.regoffset().Is(NoReg) ||
1704 addr.offset() == offset);
1705 }
1706 #else
1707 USE(vt, addr, op);
1708 #endif
1709 }
1710
LoadStoreStruct(const VRegister & vt,const MemOperand & addr,NEONLoadStoreMultiStructOp op)1711 void Assembler::LoadStoreStruct(const VRegister& vt,
1712 const MemOperand& addr,
1713 NEONLoadStoreMultiStructOp op) {
1714 LoadStoreStructVerify(vt, addr, op);
1715 VIXL_ASSERT(vt.IsVector() || vt.Is1D());
1716 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
1717 }
1718
1719
LoadStoreStructSingleAllLanes(const VRegister & vt,const MemOperand & addr,NEONLoadStoreSingleStructOp op)1720 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
1721 const MemOperand& addr,
1722 NEONLoadStoreSingleStructOp op) {
1723 LoadStoreStructVerify(vt, addr, op);
1724 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
1725 }
1726
1727
ld1(const VRegister & vt,const MemOperand & src)1728 void Assembler::ld1(const VRegister& vt,
1729 const MemOperand& src) {
1730 LoadStoreStruct(vt, src, NEON_LD1_1v);
1731 }
1732
1733
ld1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1734 void Assembler::ld1(const VRegister& vt,
1735 const VRegister& vt2,
1736 const MemOperand& src) {
1737 USE(vt2);
1738 VIXL_ASSERT(AreSameFormat(vt, vt2));
1739 VIXL_ASSERT(AreConsecutive(vt, vt2));
1740 LoadStoreStruct(vt, src, NEON_LD1_2v);
1741 }
1742
1743
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1744 void Assembler::ld1(const VRegister& vt,
1745 const VRegister& vt2,
1746 const VRegister& vt3,
1747 const MemOperand& src) {
1748 USE(vt2, vt3);
1749 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1750 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1751 LoadStoreStruct(vt, src, NEON_LD1_3v);
1752 }
1753
1754
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1755 void Assembler::ld1(const VRegister& vt,
1756 const VRegister& vt2,
1757 const VRegister& vt3,
1758 const VRegister& vt4,
1759 const MemOperand& src) {
1760 USE(vt2, vt3, vt4);
1761 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1762 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1763 LoadStoreStruct(vt, src, NEON_LD1_4v);
1764 }
1765
1766
ld2(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1767 void Assembler::ld2(const VRegister& vt,
1768 const VRegister& vt2,
1769 const MemOperand& src) {
1770 USE(vt2);
1771 VIXL_ASSERT(AreSameFormat(vt, vt2));
1772 VIXL_ASSERT(AreConsecutive(vt, vt2));
1773 LoadStoreStruct(vt, src, NEON_LD2);
1774 }
1775
1776
ld2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & src)1777 void Assembler::ld2(const VRegister& vt,
1778 const VRegister& vt2,
1779 int lane,
1780 const MemOperand& src) {
1781 USE(vt2);
1782 VIXL_ASSERT(AreSameFormat(vt, vt2));
1783 VIXL_ASSERT(AreConsecutive(vt, vt2));
1784 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
1785 }
1786
1787
ld2r(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1788 void Assembler::ld2r(const VRegister& vt,
1789 const VRegister& vt2,
1790 const MemOperand& src) {
1791 USE(vt2);
1792 VIXL_ASSERT(AreSameFormat(vt, vt2));
1793 VIXL_ASSERT(AreConsecutive(vt, vt2));
1794 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
1795 }
1796
1797
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1798 void Assembler::ld3(const VRegister& vt,
1799 const VRegister& vt2,
1800 const VRegister& vt3,
1801 const MemOperand& src) {
1802 USE(vt2, vt3);
1803 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1804 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1805 LoadStoreStruct(vt, src, NEON_LD3);
1806 }
1807
1808
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & src)1809 void Assembler::ld3(const VRegister& vt,
1810 const VRegister& vt2,
1811 const VRegister& vt3,
1812 int lane,
1813 const MemOperand& src) {
1814 USE(vt2, vt3);
1815 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1816 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1817 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
1818 }
1819
1820
ld3r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1821 void Assembler::ld3r(const VRegister& vt,
1822 const VRegister& vt2,
1823 const VRegister& vt3,
1824 const MemOperand& src) {
1825 USE(vt2, vt3);
1826 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1827 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1828 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
1829 }
1830
1831
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1832 void Assembler::ld4(const VRegister& vt,
1833 const VRegister& vt2,
1834 const VRegister& vt3,
1835 const VRegister& vt4,
1836 const MemOperand& src) {
1837 USE(vt2, vt3, vt4);
1838 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1839 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1840 LoadStoreStruct(vt, src, NEON_LD4);
1841 }
1842
1843
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & src)1844 void Assembler::ld4(const VRegister& vt,
1845 const VRegister& vt2,
1846 const VRegister& vt3,
1847 const VRegister& vt4,
1848 int lane,
1849 const MemOperand& src) {
1850 USE(vt2, vt3, vt4);
1851 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1852 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1853 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
1854 }
1855
1856
ld4r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1857 void Assembler::ld4r(const VRegister& vt,
1858 const VRegister& vt2,
1859 const VRegister& vt3,
1860 const VRegister& vt4,
1861 const MemOperand& src) {
1862 USE(vt2, vt3, vt4);
1863 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1864 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1865 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
1866 }
1867
1868
st1(const VRegister & vt,const MemOperand & src)1869 void Assembler::st1(const VRegister& vt,
1870 const MemOperand& src) {
1871 LoadStoreStruct(vt, src, NEON_ST1_1v);
1872 }
1873
1874
st1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1875 void Assembler::st1(const VRegister& vt,
1876 const VRegister& vt2,
1877 const MemOperand& src) {
1878 USE(vt2);
1879 VIXL_ASSERT(AreSameFormat(vt, vt2));
1880 VIXL_ASSERT(AreConsecutive(vt, vt2));
1881 LoadStoreStruct(vt, src, NEON_ST1_2v);
1882 }
1883
1884
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1885 void Assembler::st1(const VRegister& vt,
1886 const VRegister& vt2,
1887 const VRegister& vt3,
1888 const MemOperand& src) {
1889 USE(vt2, vt3);
1890 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1891 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1892 LoadStoreStruct(vt, src, NEON_ST1_3v);
1893 }
1894
1895
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1896 void Assembler::st1(const VRegister& vt,
1897 const VRegister& vt2,
1898 const VRegister& vt3,
1899 const VRegister& vt4,
1900 const MemOperand& src) {
1901 USE(vt2, vt3, vt4);
1902 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1903 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1904 LoadStoreStruct(vt, src, NEON_ST1_4v);
1905 }
1906
1907
st2(const VRegister & vt,const VRegister & vt2,const MemOperand & dst)1908 void Assembler::st2(const VRegister& vt,
1909 const VRegister& vt2,
1910 const MemOperand& dst) {
1911 USE(vt2);
1912 VIXL_ASSERT(AreSameFormat(vt, vt2));
1913 VIXL_ASSERT(AreConsecutive(vt, vt2));
1914 LoadStoreStruct(vt, dst, NEON_ST2);
1915 }
1916
1917
st2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & dst)1918 void Assembler::st2(const VRegister& vt,
1919 const VRegister& vt2,
1920 int lane,
1921 const MemOperand& dst) {
1922 USE(vt2);
1923 VIXL_ASSERT(AreSameFormat(vt, vt2));
1924 VIXL_ASSERT(AreConsecutive(vt, vt2));
1925 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
1926 }
1927
1928
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & dst)1929 void Assembler::st3(const VRegister& vt,
1930 const VRegister& vt2,
1931 const VRegister& vt3,
1932 const MemOperand& dst) {
1933 USE(vt2, vt3);
1934 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1935 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1936 LoadStoreStruct(vt, dst, NEON_ST3);
1937 }
1938
1939
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & dst)1940 void Assembler::st3(const VRegister& vt,
1941 const VRegister& vt2,
1942 const VRegister& vt3,
1943 int lane,
1944 const MemOperand& dst) {
1945 USE(vt2, vt3);
1946 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1947 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1948 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
1949 }
1950
1951
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & dst)1952 void Assembler::st4(const VRegister& vt,
1953 const VRegister& vt2,
1954 const VRegister& vt3,
1955 const VRegister& vt4,
1956 const MemOperand& dst) {
1957 USE(vt2, vt3, vt4);
1958 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1959 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1960 LoadStoreStruct(vt, dst, NEON_ST4);
1961 }
1962
1963
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & dst)1964 void Assembler::st4(const VRegister& vt,
1965 const VRegister& vt2,
1966 const VRegister& vt3,
1967 const VRegister& vt4,
1968 int lane,
1969 const MemOperand& dst) {
1970 USE(vt2, vt3, vt4);
1971 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1972 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1973 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
1974 }
1975
1976
LoadStoreStructSingle(const VRegister & vt,uint32_t lane,const MemOperand & addr,NEONLoadStoreSingleStructOp op)1977 void Assembler::LoadStoreStructSingle(const VRegister& vt,
1978 uint32_t lane,
1979 const MemOperand& addr,
1980 NEONLoadStoreSingleStructOp op) {
1981 LoadStoreStructVerify(vt, addr, op);
1982
1983 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
1984 // number of lanes, and T is b, h, s or d.
1985 unsigned lane_size = vt.LaneSizeInBytes();
1986 VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
1987
1988 // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
1989 // S and size fields.
1990 lane *= lane_size;
1991 if (lane_size == 8) lane++;
1992
1993 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
1994 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
1995 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
1996
1997 Instr instr = op;
1998 switch (lane_size) {
1999 case 1: instr |= NEONLoadStoreSingle_b; break;
2000 case 2: instr |= NEONLoadStoreSingle_h; break;
2001 case 4: instr |= NEONLoadStoreSingle_s; break;
2002 default:
2003 VIXL_ASSERT(lane_size == 8);
2004 instr |= NEONLoadStoreSingle_d;
2005 }
2006
2007 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
2008 }
2009
2010
ld1(const VRegister & vt,int lane,const MemOperand & src)2011 void Assembler::ld1(const VRegister& vt,
2012 int lane,
2013 const MemOperand& src) {
2014 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2015 }
2016
2017
ld1r(const VRegister & vt,const MemOperand & src)2018 void Assembler::ld1r(const VRegister& vt,
2019 const MemOperand& src) {
2020 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2021 }
2022
2023
st1(const VRegister & vt,int lane,const MemOperand & dst)2024 void Assembler::st1(const VRegister& vt,
2025 int lane,
2026 const MemOperand& dst) {
2027 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2028 }
2029
2030
NEON3DifferentL(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2031 void Assembler::NEON3DifferentL(const VRegister& vd,
2032 const VRegister& vn,
2033 const VRegister& vm,
2034 NEON3DifferentOp vop) {
2035 VIXL_ASSERT(AreSameFormat(vn, vm));
2036 VIXL_ASSERT((vn.Is1H() && vd.Is1S()) ||
2037 (vn.Is1S() && vd.Is1D()) ||
2038 (vn.Is8B() && vd.Is8H()) ||
2039 (vn.Is4H() && vd.Is4S()) ||
2040 (vn.Is2S() && vd.Is2D()) ||
2041 (vn.Is16B() && vd.Is8H())||
2042 (vn.Is8H() && vd.Is4S()) ||
2043 (vn.Is4S() && vd.Is2D()));
2044 Instr format, op = vop;
2045 if (vd.IsScalar()) {
2046 op |= NEON_Q | NEONScalar;
2047 format = SFormat(vn);
2048 } else {
2049 format = VFormat(vn);
2050 }
2051 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2052 }
2053
2054
NEON3DifferentW(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2055 void Assembler::NEON3DifferentW(const VRegister& vd,
2056 const VRegister& vn,
2057 const VRegister& vm,
2058 NEON3DifferentOp vop) {
2059 VIXL_ASSERT(AreSameFormat(vd, vn));
2060 VIXL_ASSERT((vm.Is8B() && vd.Is8H()) ||
2061 (vm.Is4H() && vd.Is4S()) ||
2062 (vm.Is2S() && vd.Is2D()) ||
2063 (vm.Is16B() && vd.Is8H())||
2064 (vm.Is8H() && vd.Is4S()) ||
2065 (vm.Is4S() && vd.Is2D()));
2066 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2067 }
2068
2069
NEON3DifferentHN(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2070 void Assembler::NEON3DifferentHN(const VRegister& vd,
2071 const VRegister& vn,
2072 const VRegister& vm,
2073 NEON3DifferentOp vop) {
2074 VIXL_ASSERT(AreSameFormat(vm, vn));
2075 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
2076 (vd.Is4H() && vn.Is4S()) ||
2077 (vd.Is2S() && vn.Is2D()) ||
2078 (vd.Is16B() && vn.Is8H())||
2079 (vd.Is8H() && vn.Is4S()) ||
2080 (vd.Is4S() && vn.Is2D()));
2081 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2082 }
2083
2084
2085 #define NEON_3DIFF_LONG_LIST(V) \
2086 V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \
2087 V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \
2088 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
2089 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
2090 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
2091 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
2092 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
2093 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
2094 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
2095 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
2096 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
2097 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
2098 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
2099 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
2100 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
2101 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
2102 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
2103 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
2104 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
2105 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
2106 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
2107 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
2108 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
2109 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
2110 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
2111 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
2112 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
2113 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
2114 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
2115 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
2116 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2117 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2118 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2119 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2120 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2121 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2122
2123
2124 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2125 void Assembler::FN(const VRegister& vd, \
2126 const VRegister& vn, \
2127 const VRegister& vm) { \
2128 VIXL_ASSERT(AS); \
2129 NEON3DifferentL(vd, vn, vm, OP); \
2130 }
2131 NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
2132 #undef DEFINE_ASM_FUNC
2133
2134 #define NEON_3DIFF_HN_LIST(V) \
2135 V(addhn, NEON_ADDHN, vd.IsD()) \
2136 V(addhn2, NEON_ADDHN2, vd.IsQ()) \
2137 V(raddhn, NEON_RADDHN, vd.IsD()) \
2138 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
2139 V(subhn, NEON_SUBHN, vd.IsD()) \
2140 V(subhn2, NEON_SUBHN2, vd.IsQ()) \
2141 V(rsubhn, NEON_RSUBHN, vd.IsD()) \
2142 V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
2143
2144 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2145 void Assembler::FN(const VRegister& vd, \
2146 const VRegister& vn, \
2147 const VRegister& vm) { \
2148 VIXL_ASSERT(AS); \
2149 NEON3DifferentHN(vd, vn, vm, OP); \
2150 }
NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)2151 NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
2152 #undef DEFINE_ASM_FUNC
2153
2154 void Assembler::uaddw(const VRegister& vd,
2155 const VRegister& vn,
2156 const VRegister& vm) {
2157 VIXL_ASSERT(vm.IsD());
2158 NEON3DifferentW(vd, vn, vm, NEON_UADDW);
2159 }
2160
2161
uaddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2162 void Assembler::uaddw2(const VRegister& vd,
2163 const VRegister& vn,
2164 const VRegister& vm) {
2165 VIXL_ASSERT(vm.IsQ());
2166 NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
2167 }
2168
2169
saddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2170 void Assembler::saddw(const VRegister& vd,
2171 const VRegister& vn,
2172 const VRegister& vm) {
2173 VIXL_ASSERT(vm.IsD());
2174 NEON3DifferentW(vd, vn, vm, NEON_SADDW);
2175 }
2176
2177
saddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2178 void Assembler::saddw2(const VRegister& vd,
2179 const VRegister& vn,
2180 const VRegister& vm) {
2181 VIXL_ASSERT(vm.IsQ());
2182 NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
2183 }
2184
2185
usubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2186 void Assembler::usubw(const VRegister& vd,
2187 const VRegister& vn,
2188 const VRegister& vm) {
2189 VIXL_ASSERT(vm.IsD());
2190 NEON3DifferentW(vd, vn, vm, NEON_USUBW);
2191 }
2192
2193
usubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2194 void Assembler::usubw2(const VRegister& vd,
2195 const VRegister& vn,
2196 const VRegister& vm) {
2197 VIXL_ASSERT(vm.IsQ());
2198 NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
2199 }
2200
2201
ssubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2202 void Assembler::ssubw(const VRegister& vd,
2203 const VRegister& vn,
2204 const VRegister& vm) {
2205 VIXL_ASSERT(vm.IsD());
2206 NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
2207 }
2208
2209
ssubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2210 void Assembler::ssubw2(const VRegister& vd,
2211 const VRegister& vn,
2212 const VRegister& vm) {
2213 VIXL_ASSERT(vm.IsQ());
2214 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
2215 }
2216
2217
mov(const Register & rd,const Register & rm)2218 void Assembler::mov(const Register& rd, const Register& rm) {
2219 // Moves involving the stack pointer are encoded as add immediate with
2220 // second operand of zero. Otherwise, orr with first operand zr is
2221 // used.
2222 if (rd.IsSP() || rm.IsSP()) {
2223 add(rd, rm, 0);
2224 } else {
2225 orr(rd, AppropriateZeroRegFor(rd), rm);
2226 }
2227 }
2228
2229
mvn(const Register & rd,const Operand & operand)2230 void Assembler::mvn(const Register& rd, const Operand& operand) {
2231 orn(rd, AppropriateZeroRegFor(rd), operand);
2232 }
2233
2234
mrs(const Register & rt,SystemRegister sysreg)2235 void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
2236 VIXL_ASSERT(rt.Is64Bits());
2237 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
2238 }
2239
2240
msr(SystemRegister sysreg,const Register & rt)2241 void Assembler::msr(SystemRegister sysreg, const Register& rt) {
2242 VIXL_ASSERT(rt.Is64Bits());
2243 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
2244 }
2245
2246
clrex(int imm4)2247 void Assembler::clrex(int imm4) {
2248 Emit(CLREX | CRm(imm4));
2249 }
2250
2251
dmb(BarrierDomain domain,BarrierType type)2252 void Assembler::dmb(BarrierDomain domain, BarrierType type) {
2253 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2254 }
2255
2256
dsb(BarrierDomain domain,BarrierType type)2257 void Assembler::dsb(BarrierDomain domain, BarrierType type) {
2258 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2259 }
2260
2261
isb()2262 void Assembler::isb() {
2263 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
2264 }
2265
2266
fmov(const VRegister & vd,double imm)2267 void Assembler::fmov(const VRegister& vd, double imm) {
2268 if (vd.IsScalar()) {
2269 VIXL_ASSERT(vd.Is1D());
2270 Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
2271 } else {
2272 VIXL_ASSERT(vd.Is2D());
2273 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
2274 Instr q = NEON_Q;
2275 uint32_t encoded_imm = FP64ToImm8(imm);
2276 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2277 }
2278 }
2279
2280
fmov(const VRegister & vd,float imm)2281 void Assembler::fmov(const VRegister& vd, float imm) {
2282 if (vd.IsScalar()) {
2283 VIXL_ASSERT(vd.Is1S());
2284 Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
2285 } else {
2286 VIXL_ASSERT(vd.Is2S() | vd.Is4S());
2287 Instr op = NEONModifiedImmediate_MOVI;
2288 Instr q = vd.Is4S() ? NEON_Q : 0;
2289 uint32_t encoded_imm = FP32ToImm8(imm);
2290 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2291 }
2292 }
2293
2294
fmov(const Register & rd,const VRegister & vn)2295 void Assembler::fmov(const Register& rd, const VRegister& vn) {
2296 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2297 VIXL_ASSERT(rd.size() == vn.size());
2298 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
2299 Emit(op | Rd(rd) | Rn(vn));
2300 }
2301
2302
fmov(const VRegister & vd,const Register & rn)2303 void Assembler::fmov(const VRegister& vd, const Register& rn) {
2304 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2305 VIXL_ASSERT(vd.size() == rn.size());
2306 FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
2307 Emit(op | Rd(vd) | Rn(rn));
2308 }
2309
2310
fmov(const VRegister & vd,const VRegister & vn)2311 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
2312 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2313 VIXL_ASSERT(vd.IsSameFormat(vn));
2314 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
2315 }
2316
2317
fmov(const VRegister & vd,int index,const Register & rn)2318 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
2319 VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
2320 USE(index);
2321 Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
2322 }
2323
2324
fmov(const Register & rd,const VRegister & vn,int index)2325 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
2326 VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
2327 USE(index);
2328 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
2329 }
2330
2331
fmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2332 void Assembler::fmadd(const VRegister& vd,
2333 const VRegister& vn,
2334 const VRegister& vm,
2335 const VRegister& va) {
2336 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMADD_s : FMADD_d);
2337 }
2338
2339
fmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2340 void Assembler::fmsub(const VRegister& vd,
2341 const VRegister& vn,
2342 const VRegister& vm,
2343 const VRegister& va) {
2344 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMSUB_s : FMSUB_d);
2345 }
2346
2347
fnmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2348 void Assembler::fnmadd(const VRegister& vd,
2349 const VRegister& vn,
2350 const VRegister& vm,
2351 const VRegister& va) {
2352 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMADD_s : FNMADD_d);
2353 }
2354
2355
fnmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2356 void Assembler::fnmsub(const VRegister& vd,
2357 const VRegister& vn,
2358 const VRegister& vm,
2359 const VRegister& va) {
2360 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMSUB_s : FNMSUB_d);
2361 }
2362
2363
fnmul(const VRegister & vd,const VRegister & vn,const VRegister & vm)2364 void Assembler::fnmul(const VRegister& vd,
2365 const VRegister& vn,
2366 const VRegister& vm) {
2367 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
2368 Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
2369 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2370 }
2371
2372
FPCompareMacro(const VRegister & vn,double value,FPTrapFlags trap)2373 void Assembler::FPCompareMacro(const VRegister& vn,
2374 double value,
2375 FPTrapFlags trap) {
2376 USE(value);
2377 // Although the fcmp{e} instructions can strictly only take an immediate
2378 // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
2379 // doesn't affect the result of the comparison.
2380 VIXL_ASSERT(value == 0.0);
2381 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2382 Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
2383 Emit(FPType(vn) | op | Rn(vn));
2384 }
2385
2386
FPCompareMacro(const VRegister & vn,const VRegister & vm,FPTrapFlags trap)2387 void Assembler::FPCompareMacro(const VRegister& vn,
2388 const VRegister& vm,
2389 FPTrapFlags trap) {
2390 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2391 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2392 Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
2393 Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
2394 }
2395
2396
fcmp(const VRegister & vn,const VRegister & vm)2397 void Assembler::fcmp(const VRegister& vn,
2398 const VRegister& vm) {
2399 FPCompareMacro(vn, vm, DisableTrap);
2400 }
2401
2402
fcmpe(const VRegister & vn,const VRegister & vm)2403 void Assembler::fcmpe(const VRegister& vn,
2404 const VRegister& vm) {
2405 FPCompareMacro(vn, vm, EnableTrap);
2406 }
2407
2408
fcmp(const VRegister & vn,double value)2409 void Assembler::fcmp(const VRegister& vn,
2410 double value) {
2411 FPCompareMacro(vn, value, DisableTrap);
2412 }
2413
2414
fcmpe(const VRegister & vn,double value)2415 void Assembler::fcmpe(const VRegister& vn,
2416 double value) {
2417 FPCompareMacro(vn, value, EnableTrap);
2418 }
2419
2420
FPCCompareMacro(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond,FPTrapFlags trap)2421 void Assembler::FPCCompareMacro(const VRegister& vn,
2422 const VRegister& vm,
2423 StatusFlags nzcv,
2424 Condition cond,
2425 FPTrapFlags trap) {
2426 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2427 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2428 Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
2429 Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
2430 }
2431
fccmp(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)2432 void Assembler::fccmp(const VRegister& vn,
2433 const VRegister& vm,
2434 StatusFlags nzcv,
2435 Condition cond) {
2436 FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
2437 }
2438
2439
fccmpe(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)2440 void Assembler::fccmpe(const VRegister& vn,
2441 const VRegister& vm,
2442 StatusFlags nzcv,
2443 Condition cond) {
2444 FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
2445 }
2446
2447
fcsel(const VRegister & vd,const VRegister & vn,const VRegister & vm,Condition cond)2448 void Assembler::fcsel(const VRegister& vd,
2449 const VRegister& vn,
2450 const VRegister& vm,
2451 Condition cond) {
2452 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2453 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2454 Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
2455 }
2456
fjcvtzs(const Register & rd,const VRegister & vn)2457 void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
2458 VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kJSCVT));
2459 VIXL_ASSERT(rd.IsW() && vn.Is1D());
2460 Emit(FJCVTZS | Rn(vn) | Rd(rd));
2461 }
2462
2463
NEONFPConvertToInt(const Register & rd,const VRegister & vn,Instr op)2464 void Assembler::NEONFPConvertToInt(const Register& rd,
2465 const VRegister& vn,
2466 Instr op) {
2467 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
2468 }
2469
2470
NEONFPConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)2471 void Assembler::NEONFPConvertToInt(const VRegister& vd,
2472 const VRegister& vn,
2473 Instr op) {
2474 if (vn.IsScalar()) {
2475 VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
2476 op |= NEON_Q | NEONScalar;
2477 }
2478 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
2479 }
2480
2481
fcvt(const VRegister & vd,const VRegister & vn)2482 void Assembler::fcvt(const VRegister& vd,
2483 const VRegister& vn) {
2484 FPDataProcessing1SourceOp op;
2485 if (vd.Is1D()) {
2486 VIXL_ASSERT(vn.Is1S() || vn.Is1H());
2487 op = vn.Is1S() ? FCVT_ds : FCVT_dh;
2488 } else if (vd.Is1S()) {
2489 VIXL_ASSERT(vn.Is1D() || vn.Is1H());
2490 op = vn.Is1D() ? FCVT_sd : FCVT_sh;
2491 } else {
2492 VIXL_ASSERT(vd.Is1H());
2493 VIXL_ASSERT(vn.Is1D() || vn.Is1S());
2494 op = vn.Is1D() ? FCVT_hd : FCVT_hs;
2495 }
2496 FPDataProcessing1Source(vd, vn, op);
2497 }
2498
2499
fcvtl(const VRegister & vd,const VRegister & vn)2500 void Assembler::fcvtl(const VRegister& vd,
2501 const VRegister& vn) {
2502 VIXL_ASSERT((vd.Is4S() && vn.Is4H()) ||
2503 (vd.Is2D() && vn.Is2S()));
2504 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2505 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
2506 }
2507
2508
fcvtl2(const VRegister & vd,const VRegister & vn)2509 void Assembler::fcvtl2(const VRegister& vd,
2510 const VRegister& vn) {
2511 VIXL_ASSERT((vd.Is4S() && vn.Is8H()) ||
2512 (vd.Is2D() && vn.Is4S()));
2513 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2514 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
2515 }
2516
2517
fcvtn(const VRegister & vd,const VRegister & vn)2518 void Assembler::fcvtn(const VRegister& vd,
2519 const VRegister& vn) {
2520 VIXL_ASSERT((vn.Is4S() && vd.Is4H()) ||
2521 (vn.Is2D() && vd.Is2S()));
2522 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2523 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
2524 }
2525
2526
fcvtn2(const VRegister & vd,const VRegister & vn)2527 void Assembler::fcvtn2(const VRegister& vd,
2528 const VRegister& vn) {
2529 VIXL_ASSERT((vn.Is4S() && vd.Is8H()) ||
2530 (vn.Is2D() && vd.Is4S()));
2531 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2532 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
2533 }
2534
2535
fcvtxn(const VRegister & vd,const VRegister & vn)2536 void Assembler::fcvtxn(const VRegister& vd,
2537 const VRegister& vn) {
2538 Instr format = 1 << NEONSize_offset;
2539 if (vd.IsScalar()) {
2540 VIXL_ASSERT(vd.Is1S() && vn.Is1D());
2541 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
2542 } else {
2543 VIXL_ASSERT(vd.Is2S() && vn.Is2D());
2544 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2545 }
2546 }
2547
2548
fcvtxn2(const VRegister & vd,const VRegister & vn)2549 void Assembler::fcvtxn2(const VRegister& vd,
2550 const VRegister& vn) {
2551 VIXL_ASSERT(vd.Is4S() && vn.Is2D());
2552 Instr format = 1 << NEONSize_offset;
2553 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2554 }
2555
2556
2557 #define NEON_FP2REGMISC_FCVT_LIST(V) \
2558 V(fcvtnu, NEON_FCVTNU, FCVTNU) \
2559 V(fcvtns, NEON_FCVTNS, FCVTNS) \
2560 V(fcvtpu, NEON_FCVTPU, FCVTPU) \
2561 V(fcvtps, NEON_FCVTPS, FCVTPS) \
2562 V(fcvtmu, NEON_FCVTMU, FCVTMU) \
2563 V(fcvtms, NEON_FCVTMS, FCVTMS) \
2564 V(fcvtau, NEON_FCVTAU, FCVTAU) \
2565 V(fcvtas, NEON_FCVTAS, FCVTAS)
2566
2567 #define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \
2568 void Assembler::FN(const Register& rd, \
2569 const VRegister& vn) { \
2570 NEONFPConvertToInt(rd, vn, SCA_OP); \
2571 } \
2572 void Assembler::FN(const VRegister& vd, \
2573 const VRegister& vn) { \
2574 NEONFPConvertToInt(vd, vn, VEC_OP); \
2575 }
NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)2576 NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
2577 #undef DEFINE_ASM_FUNCS
2578
2579
2580 void Assembler::fcvtzs(const Register& rd,
2581 const VRegister& vn,
2582 int fbits) {
2583 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2584 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
2585 if (fbits == 0) {
2586 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
2587 } else {
2588 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
2589 Rd(rd));
2590 }
2591 }
2592
2593
fcvtzs(const VRegister & vd,const VRegister & vn,int fbits)2594 void Assembler::fcvtzs(const VRegister& vd,
2595 const VRegister& vn,
2596 int fbits) {
2597 VIXL_ASSERT(fbits >= 0);
2598 if (fbits == 0) {
2599 NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
2600 } else {
2601 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2602 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
2603 }
2604 }
2605
2606
fcvtzu(const Register & rd,const VRegister & vn,int fbits)2607 void Assembler::fcvtzu(const Register& rd,
2608 const VRegister& vn,
2609 int fbits) {
2610 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2611 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
2612 if (fbits == 0) {
2613 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
2614 } else {
2615 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
2616 Rd(rd));
2617 }
2618 }
2619
2620
fcvtzu(const VRegister & vd,const VRegister & vn,int fbits)2621 void Assembler::fcvtzu(const VRegister& vd,
2622 const VRegister& vn,
2623 int fbits) {
2624 VIXL_ASSERT(fbits >= 0);
2625 if (fbits == 0) {
2626 NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
2627 } else {
2628 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2629 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
2630 }
2631 }
2632
ucvtf(const VRegister & vd,const VRegister & vn,int fbits)2633 void Assembler::ucvtf(const VRegister& vd,
2634 const VRegister& vn,
2635 int fbits) {
2636 VIXL_ASSERT(fbits >= 0);
2637 if (fbits == 0) {
2638 NEONFP2RegMisc(vd, vn, NEON_UCVTF);
2639 } else {
2640 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2641 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
2642 }
2643 }
2644
scvtf(const VRegister & vd,const VRegister & vn,int fbits)2645 void Assembler::scvtf(const VRegister& vd,
2646 const VRegister& vn,
2647 int fbits) {
2648 VIXL_ASSERT(fbits >= 0);
2649 if (fbits == 0) {
2650 NEONFP2RegMisc(vd, vn, NEON_SCVTF);
2651 } else {
2652 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2653 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
2654 }
2655 }
2656
2657
scvtf(const VRegister & vd,const Register & rn,int fbits)2658 void Assembler::scvtf(const VRegister& vd,
2659 const Register& rn,
2660 int fbits) {
2661 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2662 VIXL_ASSERT(fbits >= 0);
2663 if (fbits == 0) {
2664 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
2665 } else {
2666 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2667 Rd(vd));
2668 }
2669 }
2670
2671
ucvtf(const VRegister & vd,const Register & rn,int fbits)2672 void Assembler::ucvtf(const VRegister& vd,
2673 const Register& rn,
2674 int fbits) {
2675 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2676 VIXL_ASSERT(fbits >= 0);
2677 if (fbits == 0) {
2678 Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
2679 } else {
2680 Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2681 Rd(vd));
2682 }
2683 }
2684
2685
NEON3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3SameOp vop)2686 void Assembler::NEON3Same(const VRegister& vd,
2687 const VRegister& vn,
2688 const VRegister& vm,
2689 NEON3SameOp vop) {
2690 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2691 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
2692
2693 Instr format, op = vop;
2694 if (vd.IsScalar()) {
2695 op |= NEON_Q | NEONScalar;
2696 format = SFormat(vd);
2697 } else {
2698 format = VFormat(vd);
2699 }
2700
2701 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2702 }
2703
2704
NEONFP3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)2705 void Assembler::NEONFP3Same(const VRegister& vd,
2706 const VRegister& vn,
2707 const VRegister& vm,
2708 Instr op) {
2709 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2710 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2711 }
2712
2713
2714 #define NEON_FP2REGMISC_LIST(V) \
2715 V(fabs, NEON_FABS, FABS) \
2716 V(fneg, NEON_FNEG, FNEG) \
2717 V(fsqrt, NEON_FSQRT, FSQRT) \
2718 V(frintn, NEON_FRINTN, FRINTN) \
2719 V(frinta, NEON_FRINTA, FRINTA) \
2720 V(frintp, NEON_FRINTP, FRINTP) \
2721 V(frintm, NEON_FRINTM, FRINTM) \
2722 V(frintx, NEON_FRINTX, FRINTX) \
2723 V(frintz, NEON_FRINTZ, FRINTZ) \
2724 V(frinti, NEON_FRINTI, FRINTI) \
2725 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
2726 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar )
2727
2728
2729 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
2730 void Assembler::FN(const VRegister& vd, \
2731 const VRegister& vn) { \
2732 Instr op; \
2733 if (vd.IsScalar()) { \
2734 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
2735 op = SCA_OP; \
2736 } else { \
2737 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
2738 op = VEC_OP; \
2739 } \
2740 NEONFP2RegMisc(vd, vn, op); \
2741 }
NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)2742 NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
2743 #undef DEFINE_ASM_FUNC
2744
2745
2746 void Assembler::NEONFP2RegMisc(const VRegister& vd,
2747 const VRegister& vn,
2748 Instr op) {
2749 VIXL_ASSERT(AreSameFormat(vd, vn));
2750 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
2751 }
2752
2753
NEON2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,int value)2754 void Assembler::NEON2RegMisc(const VRegister& vd,
2755 const VRegister& vn,
2756 NEON2RegMiscOp vop,
2757 int value) {
2758 VIXL_ASSERT(AreSameFormat(vd, vn));
2759 VIXL_ASSERT(value == 0);
2760 USE(value);
2761
2762 Instr format, op = vop;
2763 if (vd.IsScalar()) {
2764 op |= NEON_Q | NEONScalar;
2765 format = SFormat(vd);
2766 } else {
2767 format = VFormat(vd);
2768 }
2769
2770 Emit(format | op | Rn(vn) | Rd(vd));
2771 }
2772
2773
cmeq(const VRegister & vd,const VRegister & vn,int value)2774 void Assembler::cmeq(const VRegister& vd,
2775 const VRegister& vn,
2776 int value) {
2777 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2778 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
2779 }
2780
2781
cmge(const VRegister & vd,const VRegister & vn,int value)2782 void Assembler::cmge(const VRegister& vd,
2783 const VRegister& vn,
2784 int value) {
2785 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2786 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
2787 }
2788
2789
cmgt(const VRegister & vd,const VRegister & vn,int value)2790 void Assembler::cmgt(const VRegister& vd,
2791 const VRegister& vn,
2792 int value) {
2793 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2794 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
2795 }
2796
2797
cmle(const VRegister & vd,const VRegister & vn,int value)2798 void Assembler::cmle(const VRegister& vd,
2799 const VRegister& vn,
2800 int value) {
2801 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2802 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
2803 }
2804
2805
cmlt(const VRegister & vd,const VRegister & vn,int value)2806 void Assembler::cmlt(const VRegister& vd,
2807 const VRegister& vn,
2808 int value) {
2809 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2810 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
2811 }
2812
2813
shll(const VRegister & vd,const VRegister & vn,int shift)2814 void Assembler::shll(const VRegister& vd,
2815 const VRegister& vn,
2816 int shift) {
2817 VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
2818 (vd.Is4S() && vn.Is4H() && shift == 16) ||
2819 (vd.Is2D() && vn.Is2S() && shift == 32));
2820 USE(shift);
2821 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
2822 }
2823
2824
shll2(const VRegister & vd,const VRegister & vn,int shift)2825 void Assembler::shll2(const VRegister& vd,
2826 const VRegister& vn,
2827 int shift) {
2828 USE(shift);
2829 VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
2830 (vd.Is4S() && vn.Is8H() && shift == 16) ||
2831 (vd.Is2D() && vn.Is4S() && shift == 32));
2832 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
2833 }
2834
2835
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,double value)2836 void Assembler::NEONFP2RegMisc(const VRegister& vd,
2837 const VRegister& vn,
2838 NEON2RegMiscOp vop,
2839 double value) {
2840 VIXL_ASSERT(AreSameFormat(vd, vn));
2841 VIXL_ASSERT(value == 0.0);
2842 USE(value);
2843
2844 Instr op = vop;
2845 if (vd.IsScalar()) {
2846 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2847 op |= NEON_Q | NEONScalar;
2848 } else {
2849 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
2850 }
2851
2852 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
2853 }
2854
2855
fcmeq(const VRegister & vd,const VRegister & vn,double value)2856 void Assembler::fcmeq(const VRegister& vd,
2857 const VRegister& vn,
2858 double value) {
2859 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
2860 }
2861
2862
fcmge(const VRegister & vd,const VRegister & vn,double value)2863 void Assembler::fcmge(const VRegister& vd,
2864 const VRegister& vn,
2865 double value) {
2866 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
2867 }
2868
2869
fcmgt(const VRegister & vd,const VRegister & vn,double value)2870 void Assembler::fcmgt(const VRegister& vd,
2871 const VRegister& vn,
2872 double value) {
2873 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
2874 }
2875
2876
fcmle(const VRegister & vd,const VRegister & vn,double value)2877 void Assembler::fcmle(const VRegister& vd,
2878 const VRegister& vn,
2879 double value) {
2880 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
2881 }
2882
2883
fcmlt(const VRegister & vd,const VRegister & vn,double value)2884 void Assembler::fcmlt(const VRegister& vd,
2885 const VRegister& vn,
2886 double value) {
2887 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
2888 }
2889
2890
frecpx(const VRegister & vd,const VRegister & vn)2891 void Assembler::frecpx(const VRegister& vd,
2892 const VRegister& vn) {
2893 VIXL_ASSERT(vd.IsScalar());
2894 VIXL_ASSERT(AreSameFormat(vd, vn));
2895 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2896 Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
2897 }
2898
2899
2900 #define NEON_3SAME_LIST(V) \
2901 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
2902 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
2903 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
2904 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
2905 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
2906 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
2907 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
2908 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
2909 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
2910 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
2911 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
2912 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
2913 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
2914 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
2915 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
2916 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2917 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2918 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2919 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2920 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
2921 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
2922 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
2923 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
2924 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
2925 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
2926 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
2927 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
2928 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
2929 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
2930 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
2931 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
2932 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
2933 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
2934 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
2935 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
2936 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
2937 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
2938 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
2939 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
2940 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
2941 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
2942 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
2943 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
2944 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
2945 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
2946 V(uqadd, NEON_UQADD, true) \
2947 V(sqadd, NEON_SQADD, true) \
2948 V(uqsub, NEON_UQSUB, true) \
2949 V(sqsub, NEON_SQSUB, true) \
2950 V(sqshl, NEON_SQSHL, true) \
2951 V(uqshl, NEON_UQSHL, true) \
2952 V(sqrshl, NEON_SQRSHL, true) \
2953 V(uqrshl, NEON_UQRSHL, true)
2954
2955 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2956 void Assembler::FN(const VRegister& vd, \
2957 const VRegister& vn, \
2958 const VRegister& vm) { \
2959 VIXL_ASSERT(AS); \
2960 NEON3Same(vd, vn, vm, OP); \
2961 }
2962 NEON_3SAME_LIST(DEFINE_ASM_FUNC)
2963 #undef DEFINE_ASM_FUNC
2964
2965
2966 #define NEON_FP3SAME_OP_LIST(V) \
2967 V(fadd, NEON_FADD, FADD) \
2968 V(fsub, NEON_FSUB, FSUB) \
2969 V(fmul, NEON_FMUL, FMUL) \
2970 V(fdiv, NEON_FDIV, FDIV) \
2971 V(fmax, NEON_FMAX, FMAX) \
2972 V(fmaxnm, NEON_FMAXNM, FMAXNM) \
2973 V(fmin, NEON_FMIN, FMIN) \
2974 V(fminnm, NEON_FMINNM, FMINNM) \
2975 V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \
2976 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \
2977 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \
2978 V(fabd, NEON_FABD, NEON_FABD_scalar) \
2979 V(fmla, NEON_FMLA, 0) \
2980 V(fmls, NEON_FMLS, 0) \
2981 V(facge, NEON_FACGE, NEON_FACGE_scalar) \
2982 V(facgt, NEON_FACGT, NEON_FACGT_scalar) \
2983 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \
2984 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \
2985 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \
2986 V(faddp, NEON_FADDP, 0) \
2987 V(fmaxp, NEON_FMAXP, 0) \
2988 V(fminp, NEON_FMINP, 0) \
2989 V(fmaxnmp, NEON_FMAXNMP, 0) \
2990 V(fminnmp, NEON_FMINNMP, 0)
2991
2992 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
2993 void Assembler::FN(const VRegister& vd, \
2994 const VRegister& vn, \
2995 const VRegister& vm) { \
2996 Instr op; \
2997 if ((SCA_OP != 0) && vd.IsScalar()) { \
2998 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
2999 op = SCA_OP; \
3000 } else { \
3001 VIXL_ASSERT(vd.IsVector()); \
3002 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
3003 op = VEC_OP; \
3004 } \
3005 NEONFP3Same(vd, vn, vm, op); \
3006 }
NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)3007 NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)
3008 #undef DEFINE_ASM_FUNC
3009
3010
3011 void Assembler::addp(const VRegister& vd,
3012 const VRegister& vn) {
3013 VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
3014 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
3015 }
3016
3017
faddp(const VRegister & vd,const VRegister & vn)3018 void Assembler::faddp(const VRegister& vd,
3019 const VRegister& vn) {
3020 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3021 (vd.Is1D() && vn.Is2D()));
3022 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
3023 }
3024
3025
fmaxp(const VRegister & vd,const VRegister & vn)3026 void Assembler::fmaxp(const VRegister& vd,
3027 const VRegister& vn) {
3028 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3029 (vd.Is1D() && vn.Is2D()));
3030 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
3031 }
3032
3033
fminp(const VRegister & vd,const VRegister & vn)3034 void Assembler::fminp(const VRegister& vd,
3035 const VRegister& vn) {
3036 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3037 (vd.Is1D() && vn.Is2D()));
3038 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
3039 }
3040
3041
fmaxnmp(const VRegister & vd,const VRegister & vn)3042 void Assembler::fmaxnmp(const VRegister& vd,
3043 const VRegister& vn) {
3044 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3045 (vd.Is1D() && vn.Is2D()));
3046 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
3047 }
3048
3049
fminnmp(const VRegister & vd,const VRegister & vn)3050 void Assembler::fminnmp(const VRegister& vd,
3051 const VRegister& vn) {
3052 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3053 (vd.Is1D() && vn.Is2D()));
3054 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
3055 }
3056
3057
orr(const VRegister & vd,const int imm8,const int left_shift)3058 void Assembler::orr(const VRegister& vd,
3059 const int imm8,
3060 const int left_shift) {
3061 NEONModifiedImmShiftLsl(vd, imm8, left_shift,
3062 NEONModifiedImmediate_ORR);
3063 }
3064
3065
mov(const VRegister & vd,const VRegister & vn)3066 void Assembler::mov(const VRegister& vd,
3067 const VRegister& vn) {
3068 VIXL_ASSERT(AreSameFormat(vd, vn));
3069 if (vd.IsD()) {
3070 orr(vd.V8B(), vn.V8B(), vn.V8B());
3071 } else {
3072 VIXL_ASSERT(vd.IsQ());
3073 orr(vd.V16B(), vn.V16B(), vn.V16B());
3074 }
3075 }
3076
3077
bic(const VRegister & vd,const int imm8,const int left_shift)3078 void Assembler::bic(const VRegister& vd,
3079 const int imm8,
3080 const int left_shift) {
3081 NEONModifiedImmShiftLsl(vd, imm8, left_shift,
3082 NEONModifiedImmediate_BIC);
3083 }
3084
3085
movi(const VRegister & vd,const uint64_t imm,Shift shift,const int shift_amount)3086 void Assembler::movi(const VRegister& vd,
3087 const uint64_t imm,
3088 Shift shift,
3089 const int shift_amount) {
3090 VIXL_ASSERT((shift == LSL) || (shift == MSL));
3091 if (vd.Is2D() || vd.Is1D()) {
3092 VIXL_ASSERT(shift_amount == 0);
3093 int imm8 = 0;
3094 for (int i = 0; i < 8; ++i) {
3095 int byte = (imm >> (i * 8)) & 0xff;
3096 VIXL_ASSERT((byte == 0) || (byte == 0xff));
3097 if (byte == 0xff) {
3098 imm8 |= (1 << i);
3099 }
3100 }
3101 int q = vd.Is2D() ? NEON_Q : 0;
3102 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
3103 ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
3104 } else if (shift == LSL) {
3105 VIXL_ASSERT(IsUint8(imm));
3106 NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount,
3107 NEONModifiedImmediate_MOVI);
3108 } else {
3109 VIXL_ASSERT(IsUint8(imm));
3110 NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount,
3111 NEONModifiedImmediate_MOVI);
3112 }
3113 }
3114
3115
mvn(const VRegister & vd,const VRegister & vn)3116 void Assembler::mvn(const VRegister& vd,
3117 const VRegister& vn) {
3118 VIXL_ASSERT(AreSameFormat(vd, vn));
3119 if (vd.IsD()) {
3120 not_(vd.V8B(), vn.V8B());
3121 } else {
3122 VIXL_ASSERT(vd.IsQ());
3123 not_(vd.V16B(), vn.V16B());
3124 }
3125 }
3126
3127
mvni(const VRegister & vd,const int imm8,Shift shift,const int shift_amount)3128 void Assembler::mvni(const VRegister& vd,
3129 const int imm8,
3130 Shift shift,
3131 const int shift_amount) {
3132 VIXL_ASSERT((shift == LSL) || (shift == MSL));
3133 if (shift == LSL) {
3134 NEONModifiedImmShiftLsl(vd, imm8, shift_amount,
3135 NEONModifiedImmediate_MVNI);
3136 } else {
3137 NEONModifiedImmShiftMsl(vd, imm8, shift_amount,
3138 NEONModifiedImmediate_MVNI);
3139 }
3140 }
3141
3142
NEONFPByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3143 void Assembler::NEONFPByElement(const VRegister& vd,
3144 const VRegister& vn,
3145 const VRegister& vm,
3146 int vm_index,
3147 NEONByIndexedElementOp vop) {
3148 VIXL_ASSERT(AreSameFormat(vd, vn));
3149 VIXL_ASSERT((vd.Is2S() && vm.Is1S()) ||
3150 (vd.Is4S() && vm.Is1S()) ||
3151 (vd.Is1S() && vm.Is1S()) ||
3152 (vd.Is2D() && vm.Is1D()) ||
3153 (vd.Is1D() && vm.Is1D()));
3154 VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) ||
3155 (vm.Is1D() && (vm_index < 2)));
3156
3157 Instr op = vop;
3158 int index_num_bits = vm.Is1S() ? 2 : 1;
3159 if (vd.IsScalar()) {
3160 op |= NEON_Q | NEONScalar;
3161 }
3162
3163 Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) |
3164 Rm(vm) | Rn(vn) | Rd(vd));
3165 }
3166
3167
NEONByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3168 void Assembler::NEONByElement(const VRegister& vd,
3169 const VRegister& vn,
3170 const VRegister& vm,
3171 int vm_index,
3172 NEONByIndexedElementOp vop) {
3173 VIXL_ASSERT(AreSameFormat(vd, vn));
3174 VIXL_ASSERT((vd.Is4H() && vm.Is1H()) ||
3175 (vd.Is8H() && vm.Is1H()) ||
3176 (vd.Is1H() && vm.Is1H()) ||
3177 (vd.Is2S() && vm.Is1S()) ||
3178 (vd.Is4S() && vm.Is1S()) ||
3179 (vd.Is1S() && vm.Is1S()));
3180 VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3181 (vm.Is1S() && (vm_index < 4)));
3182
3183 Instr format, op = vop;
3184 int index_num_bits = vm.Is1H() ? 3 : 2;
3185 if (vd.IsScalar()) {
3186 op |= NEONScalar | NEON_Q;
3187 format = SFormat(vn);
3188 } else {
3189 format = VFormat(vn);
3190 }
3191 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
3192 Rm(vm) | Rn(vn) | Rd(vd));
3193 }
3194
3195
NEONByElementL(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3196 void Assembler::NEONByElementL(const VRegister& vd,
3197 const VRegister& vn,
3198 const VRegister& vm,
3199 int vm_index,
3200 NEONByIndexedElementOp vop) {
3201 VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
3202 (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
3203 (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
3204 (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
3205 (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
3206 (vd.Is1D() && vn.Is1S() && vm.Is1S()));
3207
3208 VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3209 (vm.Is1S() && (vm_index < 4)));
3210
3211 Instr format, op = vop;
3212 int index_num_bits = vm.Is1H() ? 3 : 2;
3213 if (vd.IsScalar()) {
3214 op |= NEONScalar | NEON_Q;
3215 format = SFormat(vn);
3216 } else {
3217 format = VFormat(vn);
3218 }
3219 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
3220 Rm(vm) | Rn(vn) | Rd(vd));
3221 }
3222
3223
3224 #define NEON_BYELEMENT_LIST(V) \
3225 V(mul, NEON_MUL_byelement, vn.IsVector()) \
3226 V(mla, NEON_MLA_byelement, vn.IsVector()) \
3227 V(mls, NEON_MLS_byelement, vn.IsVector()) \
3228 V(sqdmulh, NEON_SQDMULH_byelement, true) \
3229 V(sqrdmulh, NEON_SQRDMULH_byelement, true)
3230
3231
3232 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3233 void Assembler::FN(const VRegister& vd, \
3234 const VRegister& vn, \
3235 const VRegister& vm, \
3236 int vm_index) { \
3237 VIXL_ASSERT(AS); \
3238 NEONByElement(vd, vn, vm, vm_index, OP); \
3239 }
3240 NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
3241 #undef DEFINE_ASM_FUNC
3242
3243
3244 #define NEON_FPBYELEMENT_LIST(V) \
3245 V(fmul, NEON_FMUL_byelement) \
3246 V(fmla, NEON_FMLA_byelement) \
3247 V(fmls, NEON_FMLS_byelement) \
3248 V(fmulx, NEON_FMULX_byelement)
3249
3250
3251 #define DEFINE_ASM_FUNC(FN, OP) \
3252 void Assembler::FN(const VRegister& vd, \
3253 const VRegister& vn, \
3254 const VRegister& vm, \
3255 int vm_index) { \
3256 NEONFPByElement(vd, vn, vm, vm_index, OP); \
3257 }
NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)3258 NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
3259 #undef DEFINE_ASM_FUNC
3260
3261
3262 #define NEON_BYELEMENT_LONG_LIST(V) \
3263 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
3264 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
3265 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
3266 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3267 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
3268 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
3269 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
3270 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
3271 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
3272 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
3273 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
3274 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3275 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
3276 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3277 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
3278 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
3279 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
3280 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
3281
3282
3283 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3284 void Assembler::FN(const VRegister& vd, \
3285 const VRegister& vn, \
3286 const VRegister& vm, \
3287 int vm_index) { \
3288 VIXL_ASSERT(AS); \
3289 NEONByElementL(vd, vn, vm, vm_index, OP); \
3290 }
3291 NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
3292 #undef DEFINE_ASM_FUNC
3293
3294
3295 void Assembler::suqadd(const VRegister& vd,
3296 const VRegister& vn) {
3297 NEON2RegMisc(vd, vn, NEON_SUQADD);
3298 }
3299
3300
usqadd(const VRegister & vd,const VRegister & vn)3301 void Assembler::usqadd(const VRegister& vd,
3302 const VRegister& vn) {
3303 NEON2RegMisc(vd, vn, NEON_USQADD);
3304 }
3305
3306
abs(const VRegister & vd,const VRegister & vn)3307 void Assembler::abs(const VRegister& vd,
3308 const VRegister& vn) {
3309 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3310 NEON2RegMisc(vd, vn, NEON_ABS);
3311 }
3312
3313
sqabs(const VRegister & vd,const VRegister & vn)3314 void Assembler::sqabs(const VRegister& vd,
3315 const VRegister& vn) {
3316 NEON2RegMisc(vd, vn, NEON_SQABS);
3317 }
3318
3319
neg(const VRegister & vd,const VRegister & vn)3320 void Assembler::neg(const VRegister& vd,
3321 const VRegister& vn) {
3322 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3323 NEON2RegMisc(vd, vn, NEON_NEG);
3324 }
3325
3326
sqneg(const VRegister & vd,const VRegister & vn)3327 void Assembler::sqneg(const VRegister& vd,
3328 const VRegister& vn) {
3329 NEON2RegMisc(vd, vn, NEON_SQNEG);
3330 }
3331
3332
NEONXtn(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop)3333 void Assembler::NEONXtn(const VRegister& vd,
3334 const VRegister& vn,
3335 NEON2RegMiscOp vop) {
3336 Instr format, op = vop;
3337 if (vd.IsScalar()) {
3338 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
3339 (vd.Is1H() && vn.Is1S()) ||
3340 (vd.Is1S() && vn.Is1D()));
3341 op |= NEON_Q | NEONScalar;
3342 format = SFormat(vd);
3343 } else {
3344 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
3345 (vd.Is4H() && vn.Is4S()) ||
3346 (vd.Is2S() && vn.Is2D()) ||
3347 (vd.Is16B() && vn.Is8H()) ||
3348 (vd.Is8H() && vn.Is4S()) ||
3349 (vd.Is4S() && vn.Is2D()));
3350 format = VFormat(vd);
3351 }
3352 Emit(format | op | Rn(vn) | Rd(vd));
3353 }
3354
3355
xtn(const VRegister & vd,const VRegister & vn)3356 void Assembler::xtn(const VRegister& vd,
3357 const VRegister& vn) {
3358 VIXL_ASSERT(vd.IsVector() && vd.IsD());
3359 NEONXtn(vd, vn, NEON_XTN);
3360 }
3361
3362
xtn2(const VRegister & vd,const VRegister & vn)3363 void Assembler::xtn2(const VRegister& vd,
3364 const VRegister& vn) {
3365 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3366 NEONXtn(vd, vn, NEON_XTN);
3367 }
3368
3369
sqxtn(const VRegister & vd,const VRegister & vn)3370 void Assembler::sqxtn(const VRegister& vd,
3371 const VRegister& vn) {
3372 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3373 NEONXtn(vd, vn, NEON_SQXTN);
3374 }
3375
3376
sqxtn2(const VRegister & vd,const VRegister & vn)3377 void Assembler::sqxtn2(const VRegister& vd,
3378 const VRegister& vn) {
3379 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3380 NEONXtn(vd, vn, NEON_SQXTN);
3381 }
3382
3383
sqxtun(const VRegister & vd,const VRegister & vn)3384 void Assembler::sqxtun(const VRegister& vd,
3385 const VRegister& vn) {
3386 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3387 NEONXtn(vd, vn, NEON_SQXTUN);
3388 }
3389
3390
sqxtun2(const VRegister & vd,const VRegister & vn)3391 void Assembler::sqxtun2(const VRegister& vd,
3392 const VRegister& vn) {
3393 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3394 NEONXtn(vd, vn, NEON_SQXTUN);
3395 }
3396
3397
uqxtn(const VRegister & vd,const VRegister & vn)3398 void Assembler::uqxtn(const VRegister& vd,
3399 const VRegister& vn) {
3400 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3401 NEONXtn(vd, vn, NEON_UQXTN);
3402 }
3403
3404
uqxtn2(const VRegister & vd,const VRegister & vn)3405 void Assembler::uqxtn2(const VRegister& vd,
3406 const VRegister& vn) {
3407 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3408 NEONXtn(vd, vn, NEON_UQXTN);
3409 }
3410
3411
3412 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
not_(const VRegister & vd,const VRegister & vn)3413 void Assembler::not_(const VRegister& vd,
3414 const VRegister& vn) {
3415 VIXL_ASSERT(AreSameFormat(vd, vn));
3416 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3417 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3418 }
3419
3420
rbit(const VRegister & vd,const VRegister & vn)3421 void Assembler::rbit(const VRegister& vd,
3422 const VRegister& vn) {
3423 VIXL_ASSERT(AreSameFormat(vd, vn));
3424 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3425 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3426 }
3427
3428
ext(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)3429 void Assembler::ext(const VRegister& vd,
3430 const VRegister& vn,
3431 const VRegister& vm,
3432 int index) {
3433 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3434 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3435 VIXL_ASSERT((0 <= index) && (index < vd.lanes()));
3436 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
3437 }
3438
3439
dup(const VRegister & vd,const VRegister & vn,int vn_index)3440 void Assembler::dup(const VRegister& vd,
3441 const VRegister& vn,
3442 int vn_index) {
3443 Instr q, scalar;
3444
3445 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3446 // number of lanes, and T is b, h, s or d.
3447 int lane_size = vn.LaneSizeInBytes();
3448 NEONFormatField format;
3449 switch (lane_size) {
3450 case 1: format = NEON_16B; break;
3451 case 2: format = NEON_8H; break;
3452 case 4: format = NEON_4S; break;
3453 default:
3454 VIXL_ASSERT(lane_size == 8);
3455 format = NEON_2D;
3456 break;
3457 }
3458
3459 if (vd.IsScalar()) {
3460 q = NEON_Q;
3461 scalar = NEONScalar;
3462 } else {
3463 VIXL_ASSERT(!vd.Is1D());
3464 q = vd.IsD() ? 0 : NEON_Q;
3465 scalar = 0;
3466 }
3467 Emit(q | scalar | NEON_DUP_ELEMENT |
3468 ImmNEON5(format, vn_index) | Rn(vn) | Rd(vd));
3469 }
3470
3471
mov(const VRegister & vd,const VRegister & vn,int vn_index)3472 void Assembler::mov(const VRegister& vd,
3473 const VRegister& vn,
3474 int vn_index) {
3475 VIXL_ASSERT(vn.IsScalar());
3476 dup(vd, vn, vn_index);
3477 }
3478
3479
dup(const VRegister & vd,const Register & rn)3480 void Assembler::dup(const VRegister& vd, const Register& rn) {
3481 VIXL_ASSERT(!vd.Is1D());
3482 VIXL_ASSERT(vd.Is2D() == rn.IsX());
3483 int q = vd.IsD() ? 0 : NEON_Q;
3484 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
3485 }
3486
3487
ins(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)3488 void Assembler::ins(const VRegister& vd,
3489 int vd_index,
3490 const VRegister& vn,
3491 int vn_index) {
3492 VIXL_ASSERT(AreSameFormat(vd, vn));
3493 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3494 // number of lanes, and T is b, h, s or d.
3495 int lane_size = vd.LaneSizeInBytes();
3496 NEONFormatField format;
3497 switch (lane_size) {
3498 case 1: format = NEON_16B; break;
3499 case 2: format = NEON_8H; break;
3500 case 4: format = NEON_4S; break;
3501 default:
3502 VIXL_ASSERT(lane_size == 8);
3503 format = NEON_2D;
3504 break;
3505 }
3506
3507 VIXL_ASSERT((0 <= vd_index) &&
3508 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3509 VIXL_ASSERT((0 <= vn_index) &&
3510 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3511 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
3512 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
3513 }
3514
3515
mov(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)3516 void Assembler::mov(const VRegister& vd,
3517 int vd_index,
3518 const VRegister& vn,
3519 int vn_index) {
3520 ins(vd, vd_index, vn, vn_index);
3521 }
3522
3523
ins(const VRegister & vd,int vd_index,const Register & rn)3524 void Assembler::ins(const VRegister& vd,
3525 int vd_index,
3526 const Register& rn) {
3527 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3528 // number of lanes, and T is b, h, s or d.
3529 int lane_size = vd.LaneSizeInBytes();
3530 NEONFormatField format;
3531 switch (lane_size) {
3532 case 1: format = NEON_16B; VIXL_ASSERT(rn.IsW()); break;
3533 case 2: format = NEON_8H; VIXL_ASSERT(rn.IsW()); break;
3534 case 4: format = NEON_4S; VIXL_ASSERT(rn.IsW()); break;
3535 default:
3536 VIXL_ASSERT(lane_size == 8);
3537 VIXL_ASSERT(rn.IsX());
3538 format = NEON_2D;
3539 break;
3540 }
3541
3542 VIXL_ASSERT((0 <= vd_index) &&
3543 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3544 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
3545 }
3546
3547
mov(const VRegister & vd,int vd_index,const Register & rn)3548 void Assembler::mov(const VRegister& vd,
3549 int vd_index,
3550 const Register& rn) {
3551 ins(vd, vd_index, rn);
3552 }
3553
3554
umov(const Register & rd,const VRegister & vn,int vn_index)3555 void Assembler::umov(const Register& rd,
3556 const VRegister& vn,
3557 int vn_index) {
3558 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3559 // number of lanes, and T is b, h, s or d.
3560 int lane_size = vn.LaneSizeInBytes();
3561 NEONFormatField format;
3562 Instr q = 0;
3563 switch (lane_size) {
3564 case 1: format = NEON_16B; VIXL_ASSERT(rd.IsW()); break;
3565 case 2: format = NEON_8H; VIXL_ASSERT(rd.IsW()); break;
3566 case 4: format = NEON_4S; VIXL_ASSERT(rd.IsW()); break;
3567 default:
3568 VIXL_ASSERT(lane_size == 8);
3569 VIXL_ASSERT(rd.IsX());
3570 format = NEON_2D;
3571 q = NEON_Q;
3572 break;
3573 }
3574
3575 VIXL_ASSERT((0 <= vn_index) &&
3576 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3577 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3578 }
3579
3580
mov(const Register & rd,const VRegister & vn,int vn_index)3581 void Assembler::mov(const Register& rd,
3582 const VRegister& vn,
3583 int vn_index) {
3584 VIXL_ASSERT(vn.SizeInBytes() >= 4);
3585 umov(rd, vn, vn_index);
3586 }
3587
3588
smov(const Register & rd,const VRegister & vn,int vn_index)3589 void Assembler::smov(const Register& rd,
3590 const VRegister& vn,
3591 int vn_index) {
3592 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3593 // number of lanes, and T is b, h, s.
3594 int lane_size = vn.LaneSizeInBytes();
3595 NEONFormatField format;
3596 Instr q = 0;
3597 VIXL_ASSERT(lane_size != 8);
3598 switch (lane_size) {
3599 case 1: format = NEON_16B; break;
3600 case 2: format = NEON_8H; break;
3601 default:
3602 VIXL_ASSERT(lane_size == 4);
3603 VIXL_ASSERT(rd.IsX());
3604 format = NEON_4S;
3605 break;
3606 }
3607 q = rd.IsW() ? 0 : NEON_Q;
3608 VIXL_ASSERT((0 <= vn_index) &&
3609 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3610 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3611 }
3612
3613
cls(const VRegister & vd,const VRegister & vn)3614 void Assembler::cls(const VRegister& vd,
3615 const VRegister& vn) {
3616 VIXL_ASSERT(AreSameFormat(vd, vn));
3617 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3618 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
3619 }
3620
3621
clz(const VRegister & vd,const VRegister & vn)3622 void Assembler::clz(const VRegister& vd,
3623 const VRegister& vn) {
3624 VIXL_ASSERT(AreSameFormat(vd, vn));
3625 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3626 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
3627 }
3628
3629
cnt(const VRegister & vd,const VRegister & vn)3630 void Assembler::cnt(const VRegister& vd,
3631 const VRegister& vn) {
3632 VIXL_ASSERT(AreSameFormat(vd, vn));
3633 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3634 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
3635 }
3636
3637
rev16(const VRegister & vd,const VRegister & vn)3638 void Assembler::rev16(const VRegister& vd,
3639 const VRegister& vn) {
3640 VIXL_ASSERT(AreSameFormat(vd, vn));
3641 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3642 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
3643 }
3644
3645
rev32(const VRegister & vd,const VRegister & vn)3646 void Assembler::rev32(const VRegister& vd,
3647 const VRegister& vn) {
3648 VIXL_ASSERT(AreSameFormat(vd, vn));
3649 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
3650 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
3651 }
3652
3653
rev64(const VRegister & vd,const VRegister & vn)3654 void Assembler::rev64(const VRegister& vd,
3655 const VRegister& vn) {
3656 VIXL_ASSERT(AreSameFormat(vd, vn));
3657 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3658 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
3659 }
3660
3661
ursqrte(const VRegister & vd,const VRegister & vn)3662 void Assembler::ursqrte(const VRegister& vd,
3663 const VRegister& vn) {
3664 VIXL_ASSERT(AreSameFormat(vd, vn));
3665 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3666 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
3667 }
3668
3669
urecpe(const VRegister & vd,const VRegister & vn)3670 void Assembler::urecpe(const VRegister& vd,
3671 const VRegister& vn) {
3672 VIXL_ASSERT(AreSameFormat(vd, vn));
3673 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3674 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
3675 }
3676
3677
NEONAddlp(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp op)3678 void Assembler::NEONAddlp(const VRegister& vd,
3679 const VRegister& vn,
3680 NEON2RegMiscOp op) {
3681 VIXL_ASSERT((op == NEON_SADDLP) ||
3682 (op == NEON_UADDLP) ||
3683 (op == NEON_SADALP) ||
3684 (op == NEON_UADALP));
3685
3686 VIXL_ASSERT((vn.Is8B() && vd.Is4H()) ||
3687 (vn.Is4H() && vd.Is2S()) ||
3688 (vn.Is2S() && vd.Is1D()) ||
3689 (vn.Is16B() && vd.Is8H())||
3690 (vn.Is8H() && vd.Is4S()) ||
3691 (vn.Is4S() && vd.Is2D()));
3692 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3693 }
3694
3695
saddlp(const VRegister & vd,const VRegister & vn)3696 void Assembler::saddlp(const VRegister& vd,
3697 const VRegister& vn) {
3698 NEONAddlp(vd, vn, NEON_SADDLP);
3699 }
3700
3701
uaddlp(const VRegister & vd,const VRegister & vn)3702 void Assembler::uaddlp(const VRegister& vd,
3703 const VRegister& vn) {
3704 NEONAddlp(vd, vn, NEON_UADDLP);
3705 }
3706
3707
sadalp(const VRegister & vd,const VRegister & vn)3708 void Assembler::sadalp(const VRegister& vd,
3709 const VRegister& vn) {
3710 NEONAddlp(vd, vn, NEON_SADALP);
3711 }
3712
3713
uadalp(const VRegister & vd,const VRegister & vn)3714 void Assembler::uadalp(const VRegister& vd,
3715 const VRegister& vn) {
3716 NEONAddlp(vd, vn, NEON_UADALP);
3717 }
3718
3719
NEONAcrossLanesL(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)3720 void Assembler::NEONAcrossLanesL(const VRegister& vd,
3721 const VRegister& vn,
3722 NEONAcrossLanesOp op) {
3723 VIXL_ASSERT((vn.Is8B() && vd.Is1H()) ||
3724 (vn.Is16B() && vd.Is1H()) ||
3725 (vn.Is4H() && vd.Is1S()) ||
3726 (vn.Is8H() && vd.Is1S()) ||
3727 (vn.Is4S() && vd.Is1D()));
3728 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3729 }
3730
3731
saddlv(const VRegister & vd,const VRegister & vn)3732 void Assembler::saddlv(const VRegister& vd,
3733 const VRegister& vn) {
3734 NEONAcrossLanesL(vd, vn, NEON_SADDLV);
3735 }
3736
3737
uaddlv(const VRegister & vd,const VRegister & vn)3738 void Assembler::uaddlv(const VRegister& vd,
3739 const VRegister& vn) {
3740 NEONAcrossLanesL(vd, vn, NEON_UADDLV);
3741 }
3742
3743
NEONAcrossLanes(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)3744 void Assembler::NEONAcrossLanes(const VRegister& vd,
3745 const VRegister& vn,
3746 NEONAcrossLanesOp op) {
3747 VIXL_ASSERT((vn.Is8B() && vd.Is1B()) ||
3748 (vn.Is16B() && vd.Is1B()) ||
3749 (vn.Is4H() && vd.Is1H()) ||
3750 (vn.Is8H() && vd.Is1H()) ||
3751 (vn.Is4S() && vd.Is1S()));
3752 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
3753 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3754 } else {
3755 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3756 }
3757 }
3758
3759
3760 #define NEON_ACROSSLANES_LIST(V) \
3761 V(fmaxv, NEON_FMAXV, vd.Is1S()) \
3762 V(fminv, NEON_FMINV, vd.Is1S()) \
3763 V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
3764 V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
3765 V(addv, NEON_ADDV, true) \
3766 V(smaxv, NEON_SMAXV, true) \
3767 V(sminv, NEON_SMINV, true) \
3768 V(umaxv, NEON_UMAXV, true) \
3769 V(uminv, NEON_UMINV, true)
3770
3771
3772 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3773 void Assembler::FN(const VRegister& vd, \
3774 const VRegister& vn) { \
3775 VIXL_ASSERT(AS); \
3776 NEONAcrossLanes(vd, vn, OP); \
3777 }
NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)3778 NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
3779 #undef DEFINE_ASM_FUNC
3780
3781
3782 void Assembler::NEONPerm(const VRegister& vd,
3783 const VRegister& vn,
3784 const VRegister& vm,
3785 NEONPermOp op) {
3786 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3787 VIXL_ASSERT(!vd.Is1D());
3788 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3789 }
3790
3791
trn1(const VRegister & vd,const VRegister & vn,const VRegister & vm)3792 void Assembler::trn1(const VRegister& vd,
3793 const VRegister& vn,
3794 const VRegister& vm) {
3795 NEONPerm(vd, vn, vm, NEON_TRN1);
3796 }
3797
3798
trn2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3799 void Assembler::trn2(const VRegister& vd,
3800 const VRegister& vn,
3801 const VRegister& vm) {
3802 NEONPerm(vd, vn, vm, NEON_TRN2);
3803 }
3804
3805
uzp1(const VRegister & vd,const VRegister & vn,const VRegister & vm)3806 void Assembler::uzp1(const VRegister& vd,
3807 const VRegister& vn,
3808 const VRegister& vm) {
3809 NEONPerm(vd, vn, vm, NEON_UZP1);
3810 }
3811
3812
uzp2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3813 void Assembler::uzp2(const VRegister& vd,
3814 const VRegister& vn,
3815 const VRegister& vm) {
3816 NEONPerm(vd, vn, vm, NEON_UZP2);
3817 }
3818
3819
zip1(const VRegister & vd,const VRegister & vn,const VRegister & vm)3820 void Assembler::zip1(const VRegister& vd,
3821 const VRegister& vn,
3822 const VRegister& vm) {
3823 NEONPerm(vd, vn, vm, NEON_ZIP1);
3824 }
3825
3826
zip2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3827 void Assembler::zip2(const VRegister& vd,
3828 const VRegister& vn,
3829 const VRegister& vm) {
3830 NEONPerm(vd, vn, vm, NEON_ZIP2);
3831 }
3832
3833
NEONShiftImmediate(const VRegister & vd,const VRegister & vn,NEONShiftImmediateOp op,int immh_immb)3834 void Assembler::NEONShiftImmediate(const VRegister& vd,
3835 const VRegister& vn,
3836 NEONShiftImmediateOp op,
3837 int immh_immb) {
3838 VIXL_ASSERT(AreSameFormat(vd, vn));
3839 Instr q, scalar;
3840 if (vn.IsScalar()) {
3841 q = NEON_Q;
3842 scalar = NEONScalar;
3843 } else {
3844 q = vd.IsD() ? 0 : NEON_Q;
3845 scalar = 0;
3846 }
3847 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
3848 }
3849
3850
NEONShiftLeftImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3851 void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
3852 const VRegister& vn,
3853 int shift,
3854 NEONShiftImmediateOp op) {
3855 int laneSizeInBits = vn.LaneSizeInBits();
3856 VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
3857 NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
3858 }
3859
3860
NEONShiftRightImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3861 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
3862 const VRegister& vn,
3863 int shift,
3864 NEONShiftImmediateOp op) {
3865 int laneSizeInBits = vn.LaneSizeInBits();
3866 VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
3867 NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
3868 }
3869
3870
NEONShiftImmediateL(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3871 void Assembler::NEONShiftImmediateL(const VRegister& vd,
3872 const VRegister& vn,
3873 int shift,
3874 NEONShiftImmediateOp op) {
3875 int laneSizeInBits = vn.LaneSizeInBits();
3876 VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
3877 int immh_immb = (laneSizeInBits + shift) << 16;
3878
3879 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) ||
3880 (vn.Is4H() && vd.Is4S()) ||
3881 (vn.Is2S() && vd.Is2D()) ||
3882 (vn.Is16B() && vd.Is8H())||
3883 (vn.Is8H() && vd.Is4S()) ||
3884 (vn.Is4S() && vd.Is2D()));
3885 Instr q;
3886 q = vn.IsD() ? 0 : NEON_Q;
3887 Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
3888 }
3889
3890
NEONShiftImmediateN(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3891 void Assembler::NEONShiftImmediateN(const VRegister& vd,
3892 const VRegister& vn,
3893 int shift,
3894 NEONShiftImmediateOp op) {
3895 Instr q, scalar;
3896 int laneSizeInBits = vd.LaneSizeInBits();
3897 VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
3898 int immh_immb = (2 * laneSizeInBits - shift) << 16;
3899
3900 if (vn.IsScalar()) {
3901 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
3902 (vd.Is1H() && vn.Is1S()) ||
3903 (vd.Is1S() && vn.Is1D()));
3904 q = NEON_Q;
3905 scalar = NEONScalar;
3906 } else {
3907 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
3908 (vd.Is4H() && vn.Is4S()) ||
3909 (vd.Is2S() && vn.Is2D()) ||
3910 (vd.Is16B() && vn.Is8H())||
3911 (vd.Is8H() && vn.Is4S()) ||
3912 (vd.Is4S() && vn.Is2D()));
3913 scalar = 0;
3914 q = vd.IsD() ? 0 : NEON_Q;
3915 }
3916 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
3917 }
3918
3919
shl(const VRegister & vd,const VRegister & vn,int shift)3920 void Assembler::shl(const VRegister& vd,
3921 const VRegister& vn,
3922 int shift) {
3923 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3924 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
3925 }
3926
3927
sli(const VRegister & vd,const VRegister & vn,int shift)3928 void Assembler::sli(const VRegister& vd,
3929 const VRegister& vn,
3930 int shift) {
3931 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3932 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
3933 }
3934
3935
sqshl(const VRegister & vd,const VRegister & vn,int shift)3936 void Assembler::sqshl(const VRegister& vd,
3937 const VRegister& vn,
3938 int shift) {
3939 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
3940 }
3941
3942
sqshlu(const VRegister & vd,const VRegister & vn,int shift)3943 void Assembler::sqshlu(const VRegister& vd,
3944 const VRegister& vn,
3945 int shift) {
3946 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
3947 }
3948
3949
uqshl(const VRegister & vd,const VRegister & vn,int shift)3950 void Assembler::uqshl(const VRegister& vd,
3951 const VRegister& vn,
3952 int shift) {
3953 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
3954 }
3955
3956
sshll(const VRegister & vd,const VRegister & vn,int shift)3957 void Assembler::sshll(const VRegister& vd,
3958 const VRegister& vn,
3959 int shift) {
3960 VIXL_ASSERT(vn.IsD());
3961 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
3962 }
3963
3964
sshll2(const VRegister & vd,const VRegister & vn,int shift)3965 void Assembler::sshll2(const VRegister& vd,
3966 const VRegister& vn,
3967 int shift) {
3968 VIXL_ASSERT(vn.IsQ());
3969 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
3970 }
3971
3972
sxtl(const VRegister & vd,const VRegister & vn)3973 void Assembler::sxtl(const VRegister& vd,
3974 const VRegister& vn) {
3975 sshll(vd, vn, 0);
3976 }
3977
3978
sxtl2(const VRegister & vd,const VRegister & vn)3979 void Assembler::sxtl2(const VRegister& vd,
3980 const VRegister& vn) {
3981 sshll2(vd, vn, 0);
3982 }
3983
3984
ushll(const VRegister & vd,const VRegister & vn,int shift)3985 void Assembler::ushll(const VRegister& vd,
3986 const VRegister& vn,
3987 int shift) {
3988 VIXL_ASSERT(vn.IsD());
3989 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
3990 }
3991
3992
ushll2(const VRegister & vd,const VRegister & vn,int shift)3993 void Assembler::ushll2(const VRegister& vd,
3994 const VRegister& vn,
3995 int shift) {
3996 VIXL_ASSERT(vn.IsQ());
3997 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
3998 }
3999
4000
uxtl(const VRegister & vd,const VRegister & vn)4001 void Assembler::uxtl(const VRegister& vd,
4002 const VRegister& vn) {
4003 ushll(vd, vn, 0);
4004 }
4005
4006
uxtl2(const VRegister & vd,const VRegister & vn)4007 void Assembler::uxtl2(const VRegister& vd,
4008 const VRegister& vn) {
4009 ushll2(vd, vn, 0);
4010 }
4011
4012
sri(const VRegister & vd,const VRegister & vn,int shift)4013 void Assembler::sri(const VRegister& vd,
4014 const VRegister& vn,
4015 int shift) {
4016 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4017 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
4018 }
4019
4020
sshr(const VRegister & vd,const VRegister & vn,int shift)4021 void Assembler::sshr(const VRegister& vd,
4022 const VRegister& vn,
4023 int shift) {
4024 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4025 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
4026 }
4027
4028
ushr(const VRegister & vd,const VRegister & vn,int shift)4029 void Assembler::ushr(const VRegister& vd,
4030 const VRegister& vn,
4031 int shift) {
4032 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4033 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
4034 }
4035
4036
srshr(const VRegister & vd,const VRegister & vn,int shift)4037 void Assembler::srshr(const VRegister& vd,
4038 const VRegister& vn,
4039 int shift) {
4040 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4041 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
4042 }
4043
4044
urshr(const VRegister & vd,const VRegister & vn,int shift)4045 void Assembler::urshr(const VRegister& vd,
4046 const VRegister& vn,
4047 int shift) {
4048 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4049 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
4050 }
4051
4052
ssra(const VRegister & vd,const VRegister & vn,int shift)4053 void Assembler::ssra(const VRegister& vd,
4054 const VRegister& vn,
4055 int shift) {
4056 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4057 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
4058 }
4059
4060
usra(const VRegister & vd,const VRegister & vn,int shift)4061 void Assembler::usra(const VRegister& vd,
4062 const VRegister& vn,
4063 int shift) {
4064 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4065 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
4066 }
4067
4068
srsra(const VRegister & vd,const VRegister & vn,int shift)4069 void Assembler::srsra(const VRegister& vd,
4070 const VRegister& vn,
4071 int shift) {
4072 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4073 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
4074 }
4075
4076
ursra(const VRegister & vd,const VRegister & vn,int shift)4077 void Assembler::ursra(const VRegister& vd,
4078 const VRegister& vn,
4079 int shift) {
4080 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4081 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
4082 }
4083
4084
shrn(const VRegister & vd,const VRegister & vn,int shift)4085 void Assembler::shrn(const VRegister& vd,
4086 const VRegister& vn,
4087 int shift) {
4088 VIXL_ASSERT(vn.IsVector() && vd.IsD());
4089 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
4090 }
4091
4092
shrn2(const VRegister & vd,const VRegister & vn,int shift)4093 void Assembler::shrn2(const VRegister& vd,
4094 const VRegister& vn,
4095 int shift) {
4096 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4097 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
4098 }
4099
4100
rshrn(const VRegister & vd,const VRegister & vn,int shift)4101 void Assembler::rshrn(const VRegister& vd,
4102 const VRegister& vn,
4103 int shift) {
4104 VIXL_ASSERT(vn.IsVector() && vd.IsD());
4105 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
4106 }
4107
4108
rshrn2(const VRegister & vd,const VRegister & vn,int shift)4109 void Assembler::rshrn2(const VRegister& vd,
4110 const VRegister& vn,
4111 int shift) {
4112 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4113 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
4114 }
4115
4116
sqshrn(const VRegister & vd,const VRegister & vn,int shift)4117 void Assembler::sqshrn(const VRegister& vd,
4118 const VRegister& vn,
4119 int shift) {
4120 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4121 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
4122 }
4123
4124
sqshrn2(const VRegister & vd,const VRegister & vn,int shift)4125 void Assembler::sqshrn2(const VRegister& vd,
4126 const VRegister& vn,
4127 int shift) {
4128 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4129 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
4130 }
4131
4132
sqrshrn(const VRegister & vd,const VRegister & vn,int shift)4133 void Assembler::sqrshrn(const VRegister& vd,
4134 const VRegister& vn,
4135 int shift) {
4136 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4137 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
4138 }
4139
4140
sqrshrn2(const VRegister & vd,const VRegister & vn,int shift)4141 void Assembler::sqrshrn2(const VRegister& vd,
4142 const VRegister& vn,
4143 int shift) {
4144 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4145 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
4146 }
4147
4148
sqshrun(const VRegister & vd,const VRegister & vn,int shift)4149 void Assembler::sqshrun(const VRegister& vd,
4150 const VRegister& vn,
4151 int shift) {
4152 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4153 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
4154 }
4155
4156
sqshrun2(const VRegister & vd,const VRegister & vn,int shift)4157 void Assembler::sqshrun2(const VRegister& vd,
4158 const VRegister& vn,
4159 int shift) {
4160 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4161 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
4162 }
4163
4164
sqrshrun(const VRegister & vd,const VRegister & vn,int shift)4165 void Assembler::sqrshrun(const VRegister& vd,
4166 const VRegister& vn,
4167 int shift) {
4168 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4169 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
4170 }
4171
4172
sqrshrun2(const VRegister & vd,const VRegister & vn,int shift)4173 void Assembler::sqrshrun2(const VRegister& vd,
4174 const VRegister& vn,
4175 int shift) {
4176 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4177 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
4178 }
4179
4180
uqshrn(const VRegister & vd,const VRegister & vn,int shift)4181 void Assembler::uqshrn(const VRegister& vd,
4182 const VRegister& vn,
4183 int shift) {
4184 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4185 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
4186 }
4187
4188
uqshrn2(const VRegister & vd,const VRegister & vn,int shift)4189 void Assembler::uqshrn2(const VRegister& vd,
4190 const VRegister& vn,
4191 int shift) {
4192 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4193 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
4194 }
4195
4196
uqrshrn(const VRegister & vd,const VRegister & vn,int shift)4197 void Assembler::uqrshrn(const VRegister& vd,
4198 const VRegister& vn,
4199 int shift) {
4200 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4201 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
4202 }
4203
4204
uqrshrn2(const VRegister & vd,const VRegister & vn,int shift)4205 void Assembler::uqrshrn2(const VRegister& vd,
4206 const VRegister& vn,
4207 int shift) {
4208 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4209 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
4210 }
4211
4212
4213 // Note:
4214 // Below, a difference in case for the same letter indicates a
4215 // negated bit.
4216 // If b is 1, then B is 0.
FP32ToImm8(float imm)4217 uint32_t Assembler::FP32ToImm8(float imm) {
4218 VIXL_ASSERT(IsImmFP32(imm));
4219 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
4220 uint32_t bits = FloatToRawbits(imm);
4221 // bit7: a000.0000
4222 uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
4223 // bit6: 0b00.0000
4224 uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
4225 // bit5_to_0: 00cd.efgh
4226 uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
4227
4228 return bit7 | bit6 | bit5_to_0;
4229 }
4230
4231
ImmFP32(float imm)4232 Instr Assembler::ImmFP32(float imm) {
4233 return FP32ToImm8(imm) << ImmFP_offset;
4234 }
4235
4236
FP64ToImm8(double imm)4237 uint32_t Assembler::FP64ToImm8(double imm) {
4238 VIXL_ASSERT(IsImmFP64(imm));
4239 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4240 // 0000.0000.0000.0000.0000.0000.0000.0000
4241 uint64_t bits = DoubleToRawbits(imm);
4242 // bit7: a000.0000
4243 uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
4244 // bit6: 0b00.0000
4245 uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
4246 // bit5_to_0: 00cd.efgh
4247 uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
4248
4249 return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
4250 }
4251
4252
ImmFP64(double imm)4253 Instr Assembler::ImmFP64(double imm) {
4254 return FP64ToImm8(imm) << ImmFP_offset;
4255 }
4256
4257
4258 // Code generation helpers.
MoveWide(const Register & rd,uint64_t imm,int shift,MoveWideImmediateOp mov_op)4259 void Assembler::MoveWide(const Register& rd,
4260 uint64_t imm,
4261 int shift,
4262 MoveWideImmediateOp mov_op) {
4263 // Ignore the top 32 bits of an immediate if we're moving to a W register.
4264 if (rd.Is32Bits()) {
4265 // Check that the top 32 bits are zero (a positive 32-bit number) or top
4266 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
4267 VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
4268 ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
4269 imm &= kWRegMask;
4270 }
4271
4272 if (shift >= 0) {
4273 // Explicit shift specified.
4274 VIXL_ASSERT((shift == 0) || (shift == 16) ||
4275 (shift == 32) || (shift == 48));
4276 VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
4277 shift /= 16;
4278 } else {
4279 // Calculate a new immediate and shift combination to encode the immediate
4280 // argument.
4281 shift = 0;
4282 if ((imm & 0xffffffffffff0000) == 0) {
4283 // Nothing to do.
4284 } else if ((imm & 0xffffffff0000ffff) == 0) {
4285 imm >>= 16;
4286 shift = 1;
4287 } else if ((imm & 0xffff0000ffffffff) == 0) {
4288 VIXL_ASSERT(rd.Is64Bits());
4289 imm >>= 32;
4290 shift = 2;
4291 } else if ((imm & 0x0000ffffffffffff) == 0) {
4292 VIXL_ASSERT(rd.Is64Bits());
4293 imm >>= 48;
4294 shift = 3;
4295 }
4296 }
4297
4298 VIXL_ASSERT(IsUint16(imm));
4299
4300 Emit(SF(rd) | MoveWideImmediateFixed | mov_op |
4301 Rd(rd) | ImmMoveWide(imm) | ShiftMoveWide(shift));
4302 }
4303
4304
AddSub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)4305 void Assembler::AddSub(const Register& rd,
4306 const Register& rn,
4307 const Operand& operand,
4308 FlagsUpdate S,
4309 AddSubOp op) {
4310 VIXL_ASSERT(rd.size() == rn.size());
4311 if (operand.IsImmediate()) {
4312 int64_t immediate = operand.immediate();
4313 VIXL_ASSERT(IsImmAddSub(immediate));
4314 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4315 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
4316 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
4317 } else if (operand.IsShiftedRegister()) {
4318 VIXL_ASSERT(operand.reg().size() == rd.size());
4319 VIXL_ASSERT(operand.shift() != ROR);
4320
4321 // For instructions of the form:
4322 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
4323 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
4324 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
4325 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
4326 // or their 64-bit register equivalents, convert the operand from shifted to
4327 // extended register mode, and emit an add/sub extended instruction.
4328 if (rn.IsSP() || rd.IsSP()) {
4329 VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
4330 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
4331 AddSubExtendedFixed | op);
4332 } else {
4333 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
4334 }
4335 } else {
4336 VIXL_ASSERT(operand.IsExtendedRegister());
4337 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
4338 }
4339 }
4340
4341
AddSubWithCarry(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)4342 void Assembler::AddSubWithCarry(const Register& rd,
4343 const Register& rn,
4344 const Operand& operand,
4345 FlagsUpdate S,
4346 AddSubWithCarryOp op) {
4347 VIXL_ASSERT(rd.size() == rn.size());
4348 VIXL_ASSERT(rd.size() == operand.reg().size());
4349 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4350 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
4351 }
4352
4353
hlt(int code)4354 void Assembler::hlt(int code) {
4355 VIXL_ASSERT(IsUint16(code));
4356 Emit(HLT | ImmException(code));
4357 }
4358
4359
brk(int code)4360 void Assembler::brk(int code) {
4361 VIXL_ASSERT(IsUint16(code));
4362 Emit(BRK | ImmException(code));
4363 }
4364
4365
svc(int code)4366 void Assembler::svc(int code) {
4367 Emit(SVC | ImmException(code));
4368 }
4369
4370
ConditionalCompare(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)4371 void Assembler::ConditionalCompare(const Register& rn,
4372 const Operand& operand,
4373 StatusFlags nzcv,
4374 Condition cond,
4375 ConditionalCompareOp op) {
4376 Instr ccmpop;
4377 if (operand.IsImmediate()) {
4378 int64_t immediate = operand.immediate();
4379 VIXL_ASSERT(IsImmConditionalCompare(immediate));
4380 ccmpop = ConditionalCompareImmediateFixed | op |
4381 ImmCondCmp(static_cast<unsigned>(immediate));
4382 } else {
4383 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4384 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
4385 }
4386 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
4387 }
4388
4389
DataProcessing1Source(const Register & rd,const Register & rn,DataProcessing1SourceOp op)4390 void Assembler::DataProcessing1Source(const Register& rd,
4391 const Register& rn,
4392 DataProcessing1SourceOp op) {
4393 VIXL_ASSERT(rd.size() == rn.size());
4394 Emit(SF(rn) | op | Rn(rn) | Rd(rd));
4395 }
4396
4397
FPDataProcessing1Source(const VRegister & vd,const VRegister & vn,FPDataProcessing1SourceOp op)4398 void Assembler::FPDataProcessing1Source(const VRegister& vd,
4399 const VRegister& vn,
4400 FPDataProcessing1SourceOp op) {
4401 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
4402 Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
4403 }
4404
4405
FPDataProcessing3Source(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va,FPDataProcessing3SourceOp op)4406 void Assembler::FPDataProcessing3Source(const VRegister& vd,
4407 const VRegister& vn,
4408 const VRegister& vm,
4409 const VRegister& va,
4410 FPDataProcessing3SourceOp op) {
4411 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4412 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
4413 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
4414 }
4415
4416
NEONModifiedImmShiftLsl(const VRegister & vd,const int imm8,const int left_shift,NEONModifiedImmediateOp op)4417 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
4418 const int imm8,
4419 const int left_shift,
4420 NEONModifiedImmediateOp op) {
4421 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() ||
4422 vd.Is2S() || vd.Is4S());
4423 VIXL_ASSERT((left_shift == 0) || (left_shift == 8) ||
4424 (left_shift == 16) || (left_shift == 24));
4425 VIXL_ASSERT(IsUint8(imm8));
4426
4427 int cmode_1, cmode_2, cmode_3;
4428 if (vd.Is8B() || vd.Is16B()) {
4429 VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
4430 cmode_1 = 1;
4431 cmode_2 = 1;
4432 cmode_3 = 1;
4433 } else {
4434 cmode_1 = (left_shift >> 3) & 1;
4435 cmode_2 = left_shift >> 4;
4436 cmode_3 = 0;
4437 if (vd.Is4H() || vd.Is8H()) {
4438 VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
4439 cmode_3 = 1;
4440 }
4441 }
4442 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
4443
4444 int q = vd.IsQ() ? NEON_Q : 0;
4445
4446 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4447 }
4448
4449
NEONModifiedImmShiftMsl(const VRegister & vd,const int imm8,const int shift_amount,NEONModifiedImmediateOp op)4450 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
4451 const int imm8,
4452 const int shift_amount,
4453 NEONModifiedImmediateOp op) {
4454 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
4455 VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
4456 VIXL_ASSERT(IsUint8(imm8));
4457
4458 int cmode_0 = (shift_amount >> 4) & 1;
4459 int cmode = 0xc | cmode_0;
4460
4461 int q = vd.IsQ() ? NEON_Q : 0;
4462
4463 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4464 }
4465
4466
EmitShift(const Register & rd,const Register & rn,Shift shift,unsigned shift_amount)4467 void Assembler::EmitShift(const Register& rd,
4468 const Register& rn,
4469 Shift shift,
4470 unsigned shift_amount) {
4471 switch (shift) {
4472 case LSL:
4473 lsl(rd, rn, shift_amount);
4474 break;
4475 case LSR:
4476 lsr(rd, rn, shift_amount);
4477 break;
4478 case ASR:
4479 asr(rd, rn, shift_amount);
4480 break;
4481 case ROR:
4482 ror(rd, rn, shift_amount);
4483 break;
4484 default:
4485 VIXL_UNREACHABLE();
4486 }
4487 }
4488
4489
EmitExtendShift(const Register & rd,const Register & rn,Extend extend,unsigned left_shift)4490 void Assembler::EmitExtendShift(const Register& rd,
4491 const Register& rn,
4492 Extend extend,
4493 unsigned left_shift) {
4494 VIXL_ASSERT(rd.size() >= rn.size());
4495 unsigned reg_size = rd.size();
4496 // Use the correct size of register.
4497 Register rn_ = Register(rn.code(), rd.size());
4498 // Bits extracted are high_bit:0.
4499 unsigned high_bit = (8 << (extend & 0x3)) - 1;
4500 // Number of bits left in the result that are not introduced by the shift.
4501 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
4502
4503 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
4504 switch (extend) {
4505 case UXTB:
4506 case UXTH:
4507 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break;
4508 case SXTB:
4509 case SXTH:
4510 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break;
4511 case UXTX:
4512 case SXTX: {
4513 VIXL_ASSERT(rn.size() == kXRegSize);
4514 // Nothing to extend. Just shift.
4515 lsl(rd, rn_, left_shift);
4516 break;
4517 }
4518 default: VIXL_UNREACHABLE();
4519 }
4520 } else {
4521 // No need to extend as the extended bits would be shifted away.
4522 lsl(rd, rn_, left_shift);
4523 }
4524 }
4525
4526
DataProcExtendedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)4527 void Assembler::DataProcExtendedRegister(const Register& rd,
4528 const Register& rn,
4529 const Operand& operand,
4530 FlagsUpdate S,
4531 Instr op) {
4532 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4533 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
4534 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
4535 dest_reg | RnSP(rn));
4536 }
4537
4538
LoadStoreMemOperand(const MemOperand & addr,unsigned access_size,LoadStoreScalingOption option)4539 Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
4540 unsigned access_size,
4541 LoadStoreScalingOption option) {
4542 Instr base = RnSP(addr.base());
4543 int64_t offset = addr.offset();
4544
4545 if (addr.IsImmediateOffset()) {
4546 bool prefer_unscaled = (option == PreferUnscaledOffset) ||
4547 (option == RequireUnscaledOffset);
4548 if (prefer_unscaled && IsImmLSUnscaled(offset)) {
4549 // Use the unscaled addressing mode.
4550 return base | LoadStoreUnscaledOffsetFixed |
4551 ImmLS(static_cast<int>(offset));
4552 }
4553
4554 if ((option != RequireUnscaledOffset) &&
4555 IsImmLSScaled(offset, access_size)) {
4556 // Use the scaled addressing mode.
4557 return base | LoadStoreUnsignedOffsetFixed |
4558 ImmLSUnsigned(static_cast<int>(offset) >> access_size);
4559 }
4560
4561 if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
4562 // Use the unscaled addressing mode.
4563 return base | LoadStoreUnscaledOffsetFixed |
4564 ImmLS(static_cast<int>(offset));
4565 }
4566 }
4567
4568 // All remaining addressing modes are register-offset, pre-indexed or
4569 // post-indexed modes.
4570 VIXL_ASSERT((option != RequireUnscaledOffset) &&
4571 (option != RequireScaledOffset));
4572
4573 if (addr.IsRegisterOffset()) {
4574 Extend ext = addr.extend();
4575 Shift shift = addr.shift();
4576 unsigned shift_amount = addr.shift_amount();
4577
4578 // LSL is encoded in the option field as UXTX.
4579 if (shift == LSL) {
4580 ext = UXTX;
4581 }
4582
4583 // Shifts are encoded in one bit, indicating a left shift by the memory
4584 // access size.
4585 VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size));
4586 return base | LoadStoreRegisterOffsetFixed | Rm(addr.regoffset()) |
4587 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
4588 }
4589
4590 if (addr.IsPreIndex() && IsImmLSUnscaled(offset)) {
4591 return base | LoadStorePreIndexFixed | ImmLS(static_cast<int>(offset));
4592 }
4593
4594 if (addr.IsPostIndex() && IsImmLSUnscaled(offset)) {
4595 return base | LoadStorePostIndexFixed | ImmLS(static_cast<int>(offset));
4596 }
4597
4598 // If this point is reached, the MemOperand (addr) cannot be encoded.
4599 VIXL_UNREACHABLE();
4600 return 0;
4601 }
4602
4603
LoadStore(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op,LoadStoreScalingOption option)4604 void Assembler::LoadStore(const CPURegister& rt,
4605 const MemOperand& addr,
4606 LoadStoreOp op,
4607 LoadStoreScalingOption option) {
4608 Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
4609 }
4610
4611
Prefetch(PrefetchOperation op,const MemOperand & addr,LoadStoreScalingOption option)4612 void Assembler::Prefetch(PrefetchOperation op,
4613 const MemOperand& addr,
4614 LoadStoreScalingOption option) {
4615 VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
4616
4617 Instr prfop = ImmPrefetchOperation(op);
4618 Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
4619 }
4620
4621
IsImmAddSub(int64_t immediate)4622 bool Assembler::IsImmAddSub(int64_t immediate) {
4623 return IsUint12(immediate) ||
4624 (IsUint12(immediate >> 12) && ((immediate & 0xfff) == 0));
4625 }
4626
4627
IsImmConditionalCompare(int64_t immediate)4628 bool Assembler::IsImmConditionalCompare(int64_t immediate) {
4629 return IsUint5(immediate);
4630 }
4631
4632
IsImmFP32(float imm)4633 bool Assembler::IsImmFP32(float imm) {
4634 // Valid values will have the form:
4635 // aBbb.bbbc.defg.h000.0000.0000.0000.0000
4636 uint32_t bits = FloatToRawbits(imm);
4637 // bits[19..0] are cleared.
4638 if ((bits & 0x7ffff) != 0) {
4639 return false;
4640 }
4641
4642 // bits[29..25] are all set or all cleared.
4643 uint32_t b_pattern = (bits >> 16) & 0x3e00;
4644 if (b_pattern != 0 && b_pattern != 0x3e00) {
4645 return false;
4646 }
4647
4648 // bit[30] and bit[29] are opposite.
4649 if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
4650 return false;
4651 }
4652
4653 return true;
4654 }
4655
4656
IsImmFP64(double imm)4657 bool Assembler::IsImmFP64(double imm) {
4658 // Valid values will have the form:
4659 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4660 // 0000.0000.0000.0000.0000.0000.0000.0000
4661 uint64_t bits = DoubleToRawbits(imm);
4662 // bits[47..0] are cleared.
4663 if ((bits & 0x0000ffffffffffff) != 0) {
4664 return false;
4665 }
4666
4667 // bits[61..54] are all set or all cleared.
4668 uint32_t b_pattern = (bits >> 48) & 0x3fc0;
4669 if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
4670 return false;
4671 }
4672
4673 // bit[62] and bit[61] are opposite.
4674 if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
4675 return false;
4676 }
4677
4678 return true;
4679 }
4680
4681
IsImmLSPair(int64_t offset,unsigned access_size)4682 bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size) {
4683 VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
4684 bool offset_is_size_multiple =
4685 (((offset >> access_size) << access_size) == offset);
4686 return offset_is_size_multiple && IsInt7(offset >> access_size);
4687 }
4688
4689
IsImmLSScaled(int64_t offset,unsigned access_size)4690 bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size) {
4691 VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
4692 bool offset_is_size_multiple =
4693 (((offset >> access_size) << access_size) == offset);
4694 return offset_is_size_multiple && IsUint12(offset >> access_size);
4695 }
4696
4697
IsImmLSUnscaled(int64_t offset)4698 bool Assembler::IsImmLSUnscaled(int64_t offset) {
4699 return IsInt9(offset);
4700 }
4701
4702
4703 // The movn instruction can generate immediates containing an arbitrary 16-bit
4704 // value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
IsImmMovn(uint64_t imm,unsigned reg_size)4705 bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
4706 return IsImmMovz(~imm, reg_size);
4707 }
4708
4709
4710 // The movz instruction can generate immediates containing an arbitrary 16-bit
4711 // value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)4712 bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
4713 VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
4714 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
4715 }
4716
4717
4718 // Test if a given value can be encoded in the immediate field of a logical
4719 // instruction.
4720 // If it can be encoded, the function returns true, and values pointed to by n,
4721 // imm_s and imm_r are updated with immediates encoded in the format required
4722 // by the corresponding fields in the logical instruction.
4723 // If it can not be encoded, the function returns false, and the values pointed
4724 // to by n, imm_s and imm_r are undefined.
IsImmLogical(uint64_t value,unsigned width,unsigned * n,unsigned * imm_s,unsigned * imm_r)4725 bool Assembler::IsImmLogical(uint64_t value,
4726 unsigned width,
4727 unsigned* n,
4728 unsigned* imm_s,
4729 unsigned* imm_r) {
4730 VIXL_ASSERT((width == kWRegSize) || (width == kXRegSize));
4731
4732 bool negate = false;
4733
4734 // Logical immediates are encoded using parameters n, imm_s and imm_r using
4735 // the following table:
4736 //
4737 // N imms immr size S R
4738 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
4739 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
4740 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
4741 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
4742 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
4743 // 0 11110s xxxxxr 2 UInt(s) UInt(r)
4744 // (s bits must not be all set)
4745 //
4746 // A pattern is constructed of size bits, where the least significant S+1 bits
4747 // are set. The pattern is rotated right by R, and repeated across a 32 or
4748 // 64-bit value, depending on destination register width.
4749 //
4750 // Put another way: the basic format of a logical immediate is a single
4751 // contiguous stretch of 1 bits, repeated across the whole word at intervals
4752 // given by a power of 2. To identify them quickly, we first locate the
4753 // lowest stretch of 1 bits, then the next 1 bit above that; that combination
4754 // is different for every logical immediate, so it gives us all the
4755 // information we need to identify the only logical immediate that our input
4756 // could be, and then we simply check if that's the value we actually have.
4757 //
4758 // (The rotation parameter does give the possibility of the stretch of 1 bits
4759 // going 'round the end' of the word. To deal with that, we observe that in
4760 // any situation where that happens the bitwise NOT of the value is also a
4761 // valid logical immediate. So we simply invert the input whenever its low bit
4762 // is set, and then we know that the rotated case can't arise.)
4763
4764 if (value & 1) {
4765 // If the low bit is 1, negate the value, and set a flag to remember that we
4766 // did (so that we can adjust the return values appropriately).
4767 negate = true;
4768 value = ~value;
4769 }
4770
4771 if (width == kWRegSize) {
4772 // To handle 32-bit logical immediates, the very easiest thing is to repeat
4773 // the input value twice to make a 64-bit word. The correct encoding of that
4774 // as a logical immediate will also be the correct encoding of the 32-bit
4775 // value.
4776
4777 // Avoid making the assumption that the most-significant 32 bits are zero by
4778 // shifting the value left and duplicating it.
4779 value <<= kWRegSize;
4780 value |= value >> kWRegSize;
4781 }
4782
4783 // The basic analysis idea: imagine our input word looks like this.
4784 //
4785 // 0011111000111110001111100011111000111110001111100011111000111110
4786 // c b a
4787 // |<--d-->|
4788 //
4789 // We find the lowest set bit (as an actual power-of-2 value, not its index)
4790 // and call it a. Then we add a to our original number, which wipes out the
4791 // bottommost stretch of set bits and replaces it with a 1 carried into the
4792 // next zero bit. Then we look for the new lowest set bit, which is in
4793 // position b, and subtract it, so now our number is just like the original
4794 // but with the lowest stretch of set bits completely gone. Now we find the
4795 // lowest set bit again, which is position c in the diagram above. Then we'll
4796 // measure the distance d between bit positions a and c (using CLZ), and that
4797 // tells us that the only valid logical immediate that could possibly be equal
4798 // to this number is the one in which a stretch of bits running from a to just
4799 // below b is replicated every d bits.
4800 uint64_t a = LowestSetBit(value);
4801 uint64_t value_plus_a = value + a;
4802 uint64_t b = LowestSetBit(value_plus_a);
4803 uint64_t value_plus_a_minus_b = value_plus_a - b;
4804 uint64_t c = LowestSetBit(value_plus_a_minus_b);
4805
4806 int d, clz_a, out_n;
4807 uint64_t mask;
4808
4809 if (c != 0) {
4810 // The general case, in which there is more than one stretch of set bits.
4811 // Compute the repeat distance d, and set up a bitmask covering the basic
4812 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
4813 // of these cases the N bit of the output will be zero.
4814 clz_a = CountLeadingZeros(a, kXRegSize);
4815 int clz_c = CountLeadingZeros(c, kXRegSize);
4816 d = clz_a - clz_c;
4817 mask = ((UINT64_C(1) << d) - 1);
4818 out_n = 0;
4819 } else {
4820 // Handle degenerate cases.
4821 //
4822 // If any of those 'find lowest set bit' operations didn't find a set bit at
4823 // all, then the word will have been zero thereafter, so in particular the
4824 // last lowest_set_bit operation will have returned zero. So we can test for
4825 // all the special case conditions in one go by seeing if c is zero.
4826 if (a == 0) {
4827 // The input was zero (or all 1 bits, which will come to here too after we
4828 // inverted it at the start of the function), for which we just return
4829 // false.
4830 return false;
4831 } else {
4832 // Otherwise, if c was zero but a was not, then there's just one stretch
4833 // of set bits in our word, meaning that we have the trivial case of
4834 // d == 64 and only one 'repetition'. Set up all the same variables as in
4835 // the general case above, and set the N bit in the output.
4836 clz_a = CountLeadingZeros(a, kXRegSize);
4837 d = 64;
4838 mask = ~UINT64_C(0);
4839 out_n = 1;
4840 }
4841 }
4842
4843 // If the repeat period d is not a power of two, it can't be encoded.
4844 if (!IsPowerOf2(d)) {
4845 return false;
4846 }
4847
4848 if (((b - a) & ~mask) != 0) {
4849 // If the bit stretch (b - a) does not fit within the mask derived from the
4850 // repeat period, then fail.
4851 return false;
4852 }
4853
4854 // The only possible option is b - a repeated every d bits. Now we're going to
4855 // actually construct the valid logical immediate derived from that
4856 // specification, and see if it equals our original input.
4857 //
4858 // To repeat a value every d bits, we multiply it by a number of the form
4859 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
4860 // be derived using a table lookup on CLZ(d).
4861 static const uint64_t multipliers[] = {
4862 0x0000000000000001UL,
4863 0x0000000100000001UL,
4864 0x0001000100010001UL,
4865 0x0101010101010101UL,
4866 0x1111111111111111UL,
4867 0x5555555555555555UL,
4868 };
4869 uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
4870 uint64_t candidate = (b - a) * multiplier;
4871
4872 if (value != candidate) {
4873 // The candidate pattern doesn't match our input value, so fail.
4874 return false;
4875 }
4876
4877 // We have a match! This is a valid logical immediate, so now we have to
4878 // construct the bits and pieces of the instruction encoding that generates
4879 // it.
4880
4881 // Count the set bits in our basic stretch. The special case of clz(0) == -1
4882 // makes the answer come out right for stretches that reach the very top of
4883 // the word (e.g. numbers like 0xffffc00000000000).
4884 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
4885 int s = clz_a - clz_b;
4886
4887 // Decide how many bits to rotate right by, to put the low bit of that basic
4888 // stretch in position a.
4889 int r;
4890 if (negate) {
4891 // If we inverted the input right at the start of this function, here's
4892 // where we compensate: the number of set bits becomes the number of clear
4893 // bits, and the rotation count is based on position b rather than position
4894 // a (since b is the location of the 'lowest' 1 bit after inversion).
4895 s = d - s;
4896 r = (clz_b + 1) & (d - 1);
4897 } else {
4898 r = (clz_a + 1) & (d - 1);
4899 }
4900
4901 // Now we're done, except for having to encode the S output in such a way that
4902 // it gives both the number of set bits and the length of the repeated
4903 // segment. The s field is encoded like this:
4904 //
4905 // imms size S
4906 // ssssss 64 UInt(ssssss)
4907 // 0sssss 32 UInt(sssss)
4908 // 10ssss 16 UInt(ssss)
4909 // 110sss 8 UInt(sss)
4910 // 1110ss 4 UInt(ss)
4911 // 11110s 2 UInt(s)
4912 //
4913 // So we 'or' (-d << 1) with our computed s to form imms.
4914 if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
4915 *n = out_n;
4916 *imm_s = ((-d << 1) | (s - 1)) & 0x3f;
4917 *imm_r = r;
4918 }
4919
4920 return true;
4921 }
4922
4923
LoadOpFor(const CPURegister & rt)4924 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
4925 VIXL_ASSERT(rt.IsValid());
4926 if (rt.IsRegister()) {
4927 return rt.Is64Bits() ? LDR_x : LDR_w;
4928 } else {
4929 VIXL_ASSERT(rt.IsVRegister());
4930 switch (rt.SizeInBits()) {
4931 case kBRegSize: return LDR_b;
4932 case kHRegSize: return LDR_h;
4933 case kSRegSize: return LDR_s;
4934 case kDRegSize: return LDR_d;
4935 default:
4936 VIXL_ASSERT(rt.IsQ());
4937 return LDR_q;
4938 }
4939 }
4940 }
4941
4942
StoreOpFor(const CPURegister & rt)4943 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
4944 VIXL_ASSERT(rt.IsValid());
4945 if (rt.IsRegister()) {
4946 return rt.Is64Bits() ? STR_x : STR_w;
4947 } else {
4948 VIXL_ASSERT(rt.IsVRegister());
4949 switch (rt.SizeInBits()) {
4950 case kBRegSize: return STR_b;
4951 case kHRegSize: return STR_h;
4952 case kSRegSize: return STR_s;
4953 case kDRegSize: return STR_d;
4954 default:
4955 VIXL_ASSERT(rt.IsQ());
4956 return STR_q;
4957 }
4958 }
4959 }
4960
4961
StorePairOpFor(const CPURegister & rt,const CPURegister & rt2)4962 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
4963 const CPURegister& rt2) {
4964 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
4965 USE(rt2);
4966 if (rt.IsRegister()) {
4967 return rt.Is64Bits() ? STP_x : STP_w;
4968 } else {
4969 VIXL_ASSERT(rt.IsVRegister());
4970 switch (rt.SizeInBytes()) {
4971 case kSRegSizeInBytes: return STP_s;
4972 case kDRegSizeInBytes: return STP_d;
4973 default:
4974 VIXL_ASSERT(rt.IsQ());
4975 return STP_q;
4976 }
4977 }
4978 }
4979
4980
LoadPairOpFor(const CPURegister & rt,const CPURegister & rt2)4981 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
4982 const CPURegister& rt2) {
4983 VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
4984 return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
4985 LoadStorePairLBit);
4986 }
4987
4988
StorePairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)4989 LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
4990 const CPURegister& rt, const CPURegister& rt2) {
4991 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
4992 USE(rt2);
4993 if (rt.IsRegister()) {
4994 return rt.Is64Bits() ? STNP_x : STNP_w;
4995 } else {
4996 VIXL_ASSERT(rt.IsVRegister());
4997 switch (rt.SizeInBytes()) {
4998 case kSRegSizeInBytes: return STNP_s;
4999 case kDRegSizeInBytes: return STNP_d;
5000 default:
5001 VIXL_ASSERT(rt.IsQ());
5002 return STNP_q;
5003 }
5004 }
5005 }
5006
5007
LoadPairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)5008 LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
5009 const CPURegister& rt, const CPURegister& rt2) {
5010 VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
5011 return static_cast<LoadStorePairNonTemporalOp>(
5012 StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
5013 }
5014
5015
LoadLiteralOpFor(const CPURegister & rt)5016 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
5017 if (rt.IsRegister()) {
5018 return rt.IsX() ? LDR_x_lit : LDR_w_lit;
5019 } else {
5020 VIXL_ASSERT(rt.IsVRegister());
5021 switch (rt.SizeInBytes()) {
5022 case kSRegSizeInBytes: return LDR_s_lit;
5023 case kDRegSizeInBytes: return LDR_d_lit;
5024 default:
5025 VIXL_ASSERT(rt.IsQ());
5026 return LDR_q_lit;
5027 }
5028 }
5029 }
5030
5031
CPUHas(const CPURegister & rt) const5032 bool Assembler::CPUHas(const CPURegister& rt) const {
5033 // Core registers are available without any particular CPU features.
5034 if (rt.IsRegister()) return true;
5035 VIXL_ASSERT(rt.IsVRegister());
5036 // The architecture does not allow FP and NEON to be implemented separately,
5037 // but we can crudely categorise them based on register size, since FP only
5038 // uses D, S and (occasionally) H registers.
5039 if (rt.IsH() || rt.IsS() || rt.IsD()) {
5040 return CPUHas(CPUFeatures::kFP) || CPUHas(CPUFeatures::kNEON);
5041 }
5042 VIXL_ASSERT(rt.IsB() || rt.IsQ());
5043 return CPUHas(CPUFeatures::kNEON);
5044 }
5045
5046
CPUHas(const CPURegister & rt,const CPURegister & rt2) const5047 bool Assembler::CPUHas(const CPURegister& rt, const CPURegister& rt2) const {
5048 // This is currently only used for loads and stores, where rt and rt2 must
5049 // have the same size and type. We could extend this to cover other cases if
5050 // necessary, but for now we can avoid checking both registers.
5051 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
5052 USE(rt2);
5053 return CPUHas(rt);
5054 }
5055
5056
CPUHas(SystemRegister sysreg) const5057 bool Assembler::CPUHas(SystemRegister sysreg) const {
5058 switch (sysreg) {
5059 case RNDR:
5060 case RNDRRS:
5061 return CPUHas(CPUFeatures::kRNG);
5062 case FPCR:
5063 case NZCV:
5064 break;
5065 }
5066 return true;
5067 }
5068
5069
AreAliased(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4,const CPURegister & reg5,const CPURegister & reg6,const CPURegister & reg7,const CPURegister & reg8)5070 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
5071 const CPURegister& reg3, const CPURegister& reg4,
5072 const CPURegister& reg5, const CPURegister& reg6,
5073 const CPURegister& reg7, const CPURegister& reg8) {
5074 int number_of_valid_regs = 0;
5075 int number_of_valid_fpregs = 0;
5076
5077 RegList unique_regs = 0;
5078 RegList unique_fpregs = 0;
5079
5080 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
5081
5082 for (unsigned i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
5083 if (regs[i].IsRegister()) {
5084 number_of_valid_regs++;
5085 unique_regs |= regs[i].Bit();
5086 } else if (regs[i].IsVRegister()) {
5087 number_of_valid_fpregs++;
5088 unique_fpregs |= regs[i].Bit();
5089 } else {
5090 VIXL_ASSERT(!regs[i].IsValid());
5091 }
5092 }
5093
5094 int number_of_unique_regs = CountSetBits(unique_regs);
5095 int number_of_unique_fpregs = CountSetBits(unique_fpregs);
5096
5097 VIXL_ASSERT(number_of_valid_regs >= number_of_unique_regs);
5098 VIXL_ASSERT(number_of_valid_fpregs >= number_of_unique_fpregs);
5099
5100 return (number_of_valid_regs != number_of_unique_regs) ||
5101 (number_of_valid_fpregs != number_of_unique_fpregs);
5102 }
5103
5104
AreSameSizeAndType(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4,const CPURegister & reg5,const CPURegister & reg6,const CPURegister & reg7,const CPURegister & reg8)5105 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
5106 const CPURegister& reg3, const CPURegister& reg4,
5107 const CPURegister& reg5, const CPURegister& reg6,
5108 const CPURegister& reg7, const CPURegister& reg8) {
5109 VIXL_ASSERT(reg1.IsValid());
5110 bool match = true;
5111 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
5112 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
5113 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
5114 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
5115 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
5116 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
5117 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
5118 return match;
5119 }
5120
5121
AreSameFormat(const VRegister & reg1,const VRegister & reg2,const VRegister & reg3,const VRegister & reg4)5122 bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
5123 const VRegister& reg3, const VRegister& reg4) {
5124 VIXL_ASSERT(reg1.IsValid());
5125 bool match = true;
5126 match &= !reg2.IsValid() || reg2.IsSameFormat(reg1);
5127 match &= !reg3.IsValid() || reg3.IsSameFormat(reg1);
5128 match &= !reg4.IsValid() || reg4.IsSameFormat(reg1);
5129 return match;
5130 }
5131
5132
AreConsecutive(const VRegister & reg1,const VRegister & reg2,const VRegister & reg3,const VRegister & reg4)5133 bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
5134 const VRegister& reg3, const VRegister& reg4) {
5135 VIXL_ASSERT(reg1.IsValid());
5136 bool match = true;
5137 match &= !reg2.IsValid() ||
5138 (reg2.code() == ((reg1.code() + 1) % kNumberOfVRegisters));
5139 match &= !reg3.IsValid() ||
5140 (reg3.code() == ((reg1.code() + 2) % kNumberOfVRegisters));
5141 match &= !reg4.IsValid() ||
5142 (reg4.code() == ((reg1.code() + 3) % kNumberOfVRegisters));
5143 return match;
5144 }
5145 } // namespace vixl
5146