1 // Copyright 2015, ARM Limited
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #include "jit/arm64/vixl/Assembler-vixl.h"
28
29 #include <cmath>
30
31 #include "jit/arm64/vixl/MacroAssembler-vixl.h"
32
33 namespace vixl {
34
35 // CPURegList utilities.
PopLowestIndex()36 CPURegister CPURegList::PopLowestIndex() {
37 if (IsEmpty()) {
38 return NoCPUReg;
39 }
40 int index = CountTrailingZeros(list_);
41 VIXL_ASSERT((1 << index) & list_);
42 Remove(index);
43 return CPURegister(index, size_, type_);
44 }
45
46
PopHighestIndex()47 CPURegister CPURegList::PopHighestIndex() {
48 VIXL_ASSERT(IsValid());
49 if (IsEmpty()) {
50 return NoCPUReg;
51 }
52 int index = CountLeadingZeros(list_);
53 index = kRegListSizeInBits - 1 - index;
54 VIXL_ASSERT((1 << index) & list_);
55 Remove(index);
56 return CPURegister(index, size_, type_);
57 }
58
59
IsValid() const60 bool CPURegList::IsValid() const {
61 if ((type_ == CPURegister::kRegister) ||
62 (type_ == CPURegister::kVRegister)) {
63 bool is_valid = true;
64 // Try to create a CPURegister for each element in the list.
65 for (int i = 0; i < kRegListSizeInBits; i++) {
66 if (((list_ >> i) & 1) != 0) {
67 is_valid &= CPURegister(i, size_, type_).IsValid();
68 }
69 }
70 return is_valid;
71 } else if (type_ == CPURegister::kNoRegister) {
72 // We can't use IsEmpty here because that asserts IsValid().
73 return list_ == 0;
74 } else {
75 return false;
76 }
77 }
78
79
RemoveCalleeSaved()80 void CPURegList::RemoveCalleeSaved() {
81 if (type() == CPURegister::kRegister) {
82 Remove(GetCalleeSaved(RegisterSizeInBits()));
83 } else if (type() == CPURegister::kVRegister) {
84 Remove(GetCalleeSavedV(RegisterSizeInBits()));
85 } else {
86 VIXL_ASSERT(type() == CPURegister::kNoRegister);
87 VIXL_ASSERT(IsEmpty());
88 // The list must already be empty, so do nothing.
89 }
90 }
91
92
Union(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3)93 CPURegList CPURegList::Union(const CPURegList& list_1,
94 const CPURegList& list_2,
95 const CPURegList& list_3) {
96 return Union(list_1, Union(list_2, list_3));
97 }
98
99
Union(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3,const CPURegList & list_4)100 CPURegList CPURegList::Union(const CPURegList& list_1,
101 const CPURegList& list_2,
102 const CPURegList& list_3,
103 const CPURegList& list_4) {
104 return Union(Union(list_1, list_2), Union(list_3, list_4));
105 }
106
107
Intersection(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3)108 CPURegList CPURegList::Intersection(const CPURegList& list_1,
109 const CPURegList& list_2,
110 const CPURegList& list_3) {
111 return Intersection(list_1, Intersection(list_2, list_3));
112 }
113
114
Intersection(const CPURegList & list_1,const CPURegList & list_2,const CPURegList & list_3,const CPURegList & list_4)115 CPURegList CPURegList::Intersection(const CPURegList& list_1,
116 const CPURegList& list_2,
117 const CPURegList& list_3,
118 const CPURegList& list_4) {
119 return Intersection(Intersection(list_1, list_2),
120 Intersection(list_3, list_4));
121 }
122
123
GetCalleeSaved(unsigned size)124 CPURegList CPURegList::GetCalleeSaved(unsigned size) {
125 return CPURegList(CPURegister::kRegister, size, 19, 29);
126 }
127
128
GetCalleeSavedV(unsigned size)129 CPURegList CPURegList::GetCalleeSavedV(unsigned size) {
130 return CPURegList(CPURegister::kVRegister, size, 8, 15);
131 }
132
133
GetCallerSaved(unsigned size)134 CPURegList CPURegList::GetCallerSaved(unsigned size) {
135 // Registers x0-x18 and lr (x30) are caller-saved.
136 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18);
137 // Do not use lr directly to avoid initialisation order fiasco bugs for users.
138 list.Combine(Register(30, kXRegSize));
139 return list;
140 }
141
142
GetCallerSavedV(unsigned size)143 CPURegList CPURegList::GetCallerSavedV(unsigned size) {
144 // Registers d0-d7 and d16-d31 are caller-saved.
145 CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7);
146 list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31));
147 return list;
148 }
149
150
151 const CPURegList kCalleeSaved = CPURegList::GetCalleeSaved();
152 const CPURegList kCalleeSavedV = CPURegList::GetCalleeSavedV();
153 const CPURegList kCallerSaved = CPURegList::GetCallerSaved();
154 const CPURegList kCallerSavedV = CPURegList::GetCallerSavedV();
155
156
157 // Registers.
158 #define WREG(n) w##n,
159 const Register Register::wregisters[] = {
160 REGISTER_CODE_LIST(WREG)
161 };
162 #undef WREG
163
164 #define XREG(n) x##n,
165 const Register Register::xregisters[] = {
166 REGISTER_CODE_LIST(XREG)
167 };
168 #undef XREG
169
170 #define BREG(n) b##n,
171 const VRegister VRegister::bregisters[] = {
172 REGISTER_CODE_LIST(BREG)
173 };
174 #undef BREG
175
176 #define HREG(n) h##n,
177 const VRegister VRegister::hregisters[] = {
178 REGISTER_CODE_LIST(HREG)
179 };
180 #undef HREG
181
182 #define SREG(n) s##n,
183 const VRegister VRegister::sregisters[] = {
184 REGISTER_CODE_LIST(SREG)
185 };
186 #undef SREG
187
188 #define DREG(n) d##n,
189 const VRegister VRegister::dregisters[] = {
190 REGISTER_CODE_LIST(DREG)
191 };
192 #undef DREG
193
194 #define QREG(n) q##n,
195 const VRegister VRegister::qregisters[] = {
196 REGISTER_CODE_LIST(QREG)
197 };
198 #undef QREG
199
200 #define VREG(n) v##n,
201 const VRegister VRegister::vregisters[] = {
202 REGISTER_CODE_LIST(VREG)
203 };
204 #undef VREG
205
206
WRegFromCode(unsigned code)207 const Register& Register::WRegFromCode(unsigned code) {
208 if (code == kSPRegInternalCode) {
209 return wsp;
210 } else {
211 VIXL_ASSERT(code < kNumberOfRegisters);
212 return wregisters[code];
213 }
214 }
215
216
XRegFromCode(unsigned code)217 const Register& Register::XRegFromCode(unsigned code) {
218 if (code == kSPRegInternalCode) {
219 return sp;
220 } else {
221 VIXL_ASSERT(code < kNumberOfRegisters);
222 return xregisters[code];
223 }
224 }
225
226
BRegFromCode(unsigned code)227 const VRegister& VRegister::BRegFromCode(unsigned code) {
228 VIXL_ASSERT(code < kNumberOfVRegisters);
229 return bregisters[code];
230 }
231
232
HRegFromCode(unsigned code)233 const VRegister& VRegister::HRegFromCode(unsigned code) {
234 VIXL_ASSERT(code < kNumberOfVRegisters);
235 return hregisters[code];
236 }
237
238
SRegFromCode(unsigned code)239 const VRegister& VRegister::SRegFromCode(unsigned code) {
240 VIXL_ASSERT(code < kNumberOfVRegisters);
241 return sregisters[code];
242 }
243
244
DRegFromCode(unsigned code)245 const VRegister& VRegister::DRegFromCode(unsigned code) {
246 VIXL_ASSERT(code < kNumberOfVRegisters);
247 return dregisters[code];
248 }
249
250
QRegFromCode(unsigned code)251 const VRegister& VRegister::QRegFromCode(unsigned code) {
252 VIXL_ASSERT(code < kNumberOfVRegisters);
253 return qregisters[code];
254 }
255
256
VRegFromCode(unsigned code)257 const VRegister& VRegister::VRegFromCode(unsigned code) {
258 VIXL_ASSERT(code < kNumberOfVRegisters);
259 return vregisters[code];
260 }
261
262
W() const263 const Register& CPURegister::W() const {
264 VIXL_ASSERT(IsValidRegister());
265 return Register::WRegFromCode(code_);
266 }
267
268
X() const269 const Register& CPURegister::X() const {
270 VIXL_ASSERT(IsValidRegister());
271 return Register::XRegFromCode(code_);
272 }
273
274
B() const275 const VRegister& CPURegister::B() const {
276 VIXL_ASSERT(IsValidVRegister());
277 return VRegister::BRegFromCode(code_);
278 }
279
280
H() const281 const VRegister& CPURegister::H() const {
282 VIXL_ASSERT(IsValidVRegister());
283 return VRegister::HRegFromCode(code_);
284 }
285
286
S() const287 const VRegister& CPURegister::S() const {
288 VIXL_ASSERT(IsValidVRegister());
289 return VRegister::SRegFromCode(code_);
290 }
291
292
D() const293 const VRegister& CPURegister::D() const {
294 VIXL_ASSERT(IsValidVRegister());
295 return VRegister::DRegFromCode(code_);
296 }
297
298
Q() const299 const VRegister& CPURegister::Q() const {
300 VIXL_ASSERT(IsValidVRegister());
301 return VRegister::QRegFromCode(code_);
302 }
303
304
V() const305 const VRegister& CPURegister::V() const {
306 VIXL_ASSERT(IsValidVRegister());
307 return VRegister::VRegFromCode(code_);
308 }
309
310
311 // Operand.
Operand(int64_t immediate)312 Operand::Operand(int64_t immediate)
313 : immediate_(immediate),
314 reg_(NoReg),
315 shift_(NO_SHIFT),
316 extend_(NO_EXTEND),
317 shift_amount_(0) {}
318
319
Operand(Register reg,Shift shift,unsigned shift_amount)320 Operand::Operand(Register reg, Shift shift, unsigned shift_amount)
321 : reg_(reg),
322 shift_(shift),
323 extend_(NO_EXTEND),
324 shift_amount_(shift_amount) {
325 VIXL_ASSERT(shift != MSL);
326 VIXL_ASSERT(reg.Is64Bits() || (shift_amount < kWRegSize));
327 VIXL_ASSERT(reg.Is32Bits() || (shift_amount < kXRegSize));
328 VIXL_ASSERT(!reg.IsSP());
329 }
330
331
Operand(Register reg,Extend extend,unsigned shift_amount)332 Operand::Operand(Register reg, Extend extend, unsigned shift_amount)
333 : reg_(reg),
334 shift_(NO_SHIFT),
335 extend_(extend),
336 shift_amount_(shift_amount) {
337 VIXL_ASSERT(reg.IsValid());
338 VIXL_ASSERT(shift_amount <= 4);
339 VIXL_ASSERT(!reg.IsSP());
340
341 // Extend modes SXTX and UXTX require a 64-bit register.
342 VIXL_ASSERT(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX)));
343 }
344
345
IsImmediate() const346 bool Operand::IsImmediate() const {
347 return reg_.Is(NoReg);
348 }
349
350
IsShiftedRegister() const351 bool Operand::IsShiftedRegister() const {
352 return reg_.IsValid() && (shift_ != NO_SHIFT);
353 }
354
355
IsExtendedRegister() const356 bool Operand::IsExtendedRegister() const {
357 return reg_.IsValid() && (extend_ != NO_EXTEND);
358 }
359
360
IsZero() const361 bool Operand::IsZero() const {
362 if (IsImmediate()) {
363 return immediate() == 0;
364 } else {
365 return reg().IsZero();
366 }
367 }
368
369
ToExtendedRegister() const370 Operand Operand::ToExtendedRegister() const {
371 VIXL_ASSERT(IsShiftedRegister());
372 VIXL_ASSERT((shift_ == LSL) && (shift_amount_ <= 4));
373 return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_);
374 }
375
376
377 // MemOperand
MemOperand(Register base,int64_t offset,AddrMode addrmode)378 MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode)
379 : base_(base), regoffset_(NoReg), offset_(offset), addrmode_(addrmode) {
380 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
381 }
382
383
MemOperand(Register base,Register regoffset,Extend extend,unsigned shift_amount)384 MemOperand::MemOperand(Register base,
385 Register regoffset,
386 Extend extend,
387 unsigned shift_amount)
388 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
389 shift_(NO_SHIFT), extend_(extend), shift_amount_(shift_amount) {
390 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
391 VIXL_ASSERT(!regoffset.IsSP());
392 VIXL_ASSERT((extend == UXTW) || (extend == SXTW) || (extend == SXTX));
393
394 // SXTX extend mode requires a 64-bit offset register.
395 VIXL_ASSERT(regoffset.Is64Bits() || (extend != SXTX));
396 }
397
398
MemOperand(Register base,Register regoffset,Shift shift,unsigned shift_amount)399 MemOperand::MemOperand(Register base,
400 Register regoffset,
401 Shift shift,
402 unsigned shift_amount)
403 : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
404 shift_(shift), extend_(NO_EXTEND), shift_amount_(shift_amount) {
405 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
406 VIXL_ASSERT(regoffset.Is64Bits() && !regoffset.IsSP());
407 VIXL_ASSERT(shift == LSL);
408 }
409
410
MemOperand(Register base,const Operand & offset,AddrMode addrmode)411 MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode)
412 : base_(base), regoffset_(NoReg), addrmode_(addrmode) {
413 VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
414
415 if (offset.IsImmediate()) {
416 offset_ = offset.immediate();
417 } else if (offset.IsShiftedRegister()) {
418 VIXL_ASSERT((addrmode == Offset) || (addrmode == PostIndex));
419
420 regoffset_ = offset.reg();
421 shift_ = offset.shift();
422 shift_amount_ = offset.shift_amount();
423
424 extend_ = NO_EXTEND;
425 offset_ = 0;
426
427 // These assertions match those in the shifted-register constructor.
428 VIXL_ASSERT(regoffset_.Is64Bits() && !regoffset_.IsSP());
429 VIXL_ASSERT(shift_ == LSL);
430 } else {
431 VIXL_ASSERT(offset.IsExtendedRegister());
432 VIXL_ASSERT(addrmode == Offset);
433
434 regoffset_ = offset.reg();
435 extend_ = offset.extend();
436 shift_amount_ = offset.shift_amount();
437
438 shift_ = NO_SHIFT;
439 offset_ = 0;
440
441 // These assertions match those in the extended-register constructor.
442 VIXL_ASSERT(!regoffset_.IsSP());
443 VIXL_ASSERT((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX));
444 VIXL_ASSERT((regoffset_.Is64Bits() || (extend_ != SXTX)));
445 }
446 }
447
448
IsImmediateOffset() const449 bool MemOperand::IsImmediateOffset() const {
450 return (addrmode_ == Offset) && regoffset_.Is(NoReg);
451 }
452
453
IsRegisterOffset() const454 bool MemOperand::IsRegisterOffset() const {
455 return (addrmode_ == Offset) && !regoffset_.Is(NoReg);
456 }
457
458
IsPreIndex() const459 bool MemOperand::IsPreIndex() const {
460 return addrmode_ == PreIndex;
461 }
462
463
IsPostIndex() const464 bool MemOperand::IsPostIndex() const {
465 return addrmode_ == PostIndex;
466 }
467
468
AddOffset(int64_t offset)469 void MemOperand::AddOffset(int64_t offset) {
470 VIXL_ASSERT(IsImmediateOffset());
471 offset_ += offset;
472 }
473
474
475 // Assembler
Assembler(PositionIndependentCodeOption pic)476 Assembler::Assembler(PositionIndependentCodeOption pic)
477 : pic_(pic) {
478 }
479
480
481 // Code generation.
br(const Register & xn)482 void Assembler::br(const Register& xn) {
483 VIXL_ASSERT(xn.Is64Bits());
484 Emit(BR | Rn(xn));
485 }
486
487
blr(const Register & xn)488 void Assembler::blr(const Register& xn) {
489 VIXL_ASSERT(xn.Is64Bits());
490 Emit(BLR | Rn(xn));
491 }
492
493
ret(const Register & xn)494 void Assembler::ret(const Register& xn) {
495 VIXL_ASSERT(xn.Is64Bits());
496 Emit(RET | Rn(xn));
497 }
498
499
NEONTable(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEONTableOp op)500 void Assembler::NEONTable(const VRegister& vd,
501 const VRegister& vn,
502 const VRegister& vm,
503 NEONTableOp op) {
504 VIXL_ASSERT(vd.Is16B() || vd.Is8B());
505 VIXL_ASSERT(vn.Is16B());
506 VIXL_ASSERT(AreSameFormat(vd, vm));
507 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
508 }
509
510
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vm)511 void Assembler::tbl(const VRegister& vd,
512 const VRegister& vn,
513 const VRegister& vm) {
514 NEONTable(vd, vn, vm, NEON_TBL_1v);
515 }
516
517
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)518 void Assembler::tbl(const VRegister& vd,
519 const VRegister& vn,
520 const VRegister& vn2,
521 const VRegister& vm) {
522 USE(vn2);
523 VIXL_ASSERT(AreSameFormat(vn, vn2));
524 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
525
526 NEONTable(vd, vn, vm, NEON_TBL_2v);
527 }
528
529
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)530 void Assembler::tbl(const VRegister& vd,
531 const VRegister& vn,
532 const VRegister& vn2,
533 const VRegister& vn3,
534 const VRegister& vm) {
535 USE(vn2, vn3);
536 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
537 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
538 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
539
540 NEONTable(vd, vn, vm, NEON_TBL_3v);
541 }
542
543
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)544 void Assembler::tbl(const VRegister& vd,
545 const VRegister& vn,
546 const VRegister& vn2,
547 const VRegister& vn3,
548 const VRegister& vn4,
549 const VRegister& vm) {
550 USE(vn2, vn3, vn4);
551 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
552 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
553 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
554 VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
555
556 NEONTable(vd, vn, vm, NEON_TBL_4v);
557 }
558
559
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vm)560 void Assembler::tbx(const VRegister& vd,
561 const VRegister& vn,
562 const VRegister& vm) {
563 NEONTable(vd, vn, vm, NEON_TBX_1v);
564 }
565
566
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)567 void Assembler::tbx(const VRegister& vd,
568 const VRegister& vn,
569 const VRegister& vn2,
570 const VRegister& vm) {
571 USE(vn2);
572 VIXL_ASSERT(AreSameFormat(vn, vn2));
573 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
574
575 NEONTable(vd, vn, vm, NEON_TBX_2v);
576 }
577
578
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)579 void Assembler::tbx(const VRegister& vd,
580 const VRegister& vn,
581 const VRegister& vn2,
582 const VRegister& vn3,
583 const VRegister& vm) {
584 USE(vn2, vn3);
585 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
586 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
587 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
588
589 NEONTable(vd, vn, vm, NEON_TBX_3v);
590 }
591
592
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)593 void Assembler::tbx(const VRegister& vd,
594 const VRegister& vn,
595 const VRegister& vn2,
596 const VRegister& vn3,
597 const VRegister& vn4,
598 const VRegister& vm) {
599 USE(vn2, vn3, vn4);
600 VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
601 VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
602 VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
603 VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
604
605 NEONTable(vd, vn, vm, NEON_TBX_4v);
606 }
607
608
add(const Register & rd,const Register & rn,const Operand & operand)609 void Assembler::add(const Register& rd,
610 const Register& rn,
611 const Operand& operand) {
612 AddSub(rd, rn, operand, LeaveFlags, ADD);
613 }
614
615
adds(const Register & rd,const Register & rn,const Operand & operand)616 void Assembler::adds(const Register& rd,
617 const Register& rn,
618 const Operand& operand) {
619 AddSub(rd, rn, operand, SetFlags, ADD);
620 }
621
622
cmn(const Register & rn,const Operand & operand)623 void Assembler::cmn(const Register& rn,
624 const Operand& operand) {
625 Register zr = AppropriateZeroRegFor(rn);
626 adds(zr, rn, operand);
627 }
628
629
sub(const Register & rd,const Register & rn,const Operand & operand)630 void Assembler::sub(const Register& rd,
631 const Register& rn,
632 const Operand& operand) {
633 AddSub(rd, rn, operand, LeaveFlags, SUB);
634 }
635
636
subs(const Register & rd,const Register & rn,const Operand & operand)637 void Assembler::subs(const Register& rd,
638 const Register& rn,
639 const Operand& operand) {
640 AddSub(rd, rn, operand, SetFlags, SUB);
641 }
642
643
cmp(const Register & rn,const Operand & operand)644 void Assembler::cmp(const Register& rn, const Operand& operand) {
645 Register zr = AppropriateZeroRegFor(rn);
646 subs(zr, rn, operand);
647 }
648
649
neg(const Register & rd,const Operand & operand)650 void Assembler::neg(const Register& rd, const Operand& operand) {
651 Register zr = AppropriateZeroRegFor(rd);
652 sub(rd, zr, operand);
653 }
654
655
negs(const Register & rd,const Operand & operand)656 void Assembler::negs(const Register& rd, const Operand& operand) {
657 Register zr = AppropriateZeroRegFor(rd);
658 subs(rd, zr, operand);
659 }
660
661
adc(const Register & rd,const Register & rn,const Operand & operand)662 void Assembler::adc(const Register& rd,
663 const Register& rn,
664 const Operand& operand) {
665 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
666 }
667
668
adcs(const Register & rd,const Register & rn,const Operand & operand)669 void Assembler::adcs(const Register& rd,
670 const Register& rn,
671 const Operand& operand) {
672 AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
673 }
674
675
sbc(const Register & rd,const Register & rn,const Operand & operand)676 void Assembler::sbc(const Register& rd,
677 const Register& rn,
678 const Operand& operand) {
679 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
680 }
681
682
sbcs(const Register & rd,const Register & rn,const Operand & operand)683 void Assembler::sbcs(const Register& rd,
684 const Register& rn,
685 const Operand& operand) {
686 AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
687 }
688
689
ngc(const Register & rd,const Operand & operand)690 void Assembler::ngc(const Register& rd, const Operand& operand) {
691 Register zr = AppropriateZeroRegFor(rd);
692 sbc(rd, zr, operand);
693 }
694
695
ngcs(const Register & rd,const Operand & operand)696 void Assembler::ngcs(const Register& rd, const Operand& operand) {
697 Register zr = AppropriateZeroRegFor(rd);
698 sbcs(rd, zr, operand);
699 }
700
701
702 // Logical instructions.
and_(const Register & rd,const Register & rn,const Operand & operand)703 void Assembler::and_(const Register& rd,
704 const Register& rn,
705 const Operand& operand) {
706 Logical(rd, rn, operand, AND);
707 }
708
709
bic(const Register & rd,const Register & rn,const Operand & operand)710 void Assembler::bic(const Register& rd,
711 const Register& rn,
712 const Operand& operand) {
713 Logical(rd, rn, operand, BIC);
714 }
715
716
bics(const Register & rd,const Register & rn,const Operand & operand)717 void Assembler::bics(const Register& rd,
718 const Register& rn,
719 const Operand& operand) {
720 Logical(rd, rn, operand, BICS);
721 }
722
723
orr(const Register & rd,const Register & rn,const Operand & operand)724 void Assembler::orr(const Register& rd,
725 const Register& rn,
726 const Operand& operand) {
727 Logical(rd, rn, operand, ORR);
728 }
729
730
orn(const Register & rd,const Register & rn,const Operand & operand)731 void Assembler::orn(const Register& rd,
732 const Register& rn,
733 const Operand& operand) {
734 Logical(rd, rn, operand, ORN);
735 }
736
737
eor(const Register & rd,const Register & rn,const Operand & operand)738 void Assembler::eor(const Register& rd,
739 const Register& rn,
740 const Operand& operand) {
741 Logical(rd, rn, operand, EOR);
742 }
743
744
eon(const Register & rd,const Register & rn,const Operand & operand)745 void Assembler::eon(const Register& rd,
746 const Register& rn,
747 const Operand& operand) {
748 Logical(rd, rn, operand, EON);
749 }
750
751
lslv(const Register & rd,const Register & rn,const Register & rm)752 void Assembler::lslv(const Register& rd,
753 const Register& rn,
754 const Register& rm) {
755 VIXL_ASSERT(rd.size() == rn.size());
756 VIXL_ASSERT(rd.size() == rm.size());
757 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
758 }
759
760
lsrv(const Register & rd,const Register & rn,const Register & rm)761 void Assembler::lsrv(const Register& rd,
762 const Register& rn,
763 const Register& rm) {
764 VIXL_ASSERT(rd.size() == rn.size());
765 VIXL_ASSERT(rd.size() == rm.size());
766 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
767 }
768
769
asrv(const Register & rd,const Register & rn,const Register & rm)770 void Assembler::asrv(const Register& rd,
771 const Register& rn,
772 const Register& rm) {
773 VIXL_ASSERT(rd.size() == rn.size());
774 VIXL_ASSERT(rd.size() == rm.size());
775 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
776 }
777
778
rorv(const Register & rd,const Register & rn,const Register & rm)779 void Assembler::rorv(const Register& rd,
780 const Register& rn,
781 const Register& rm) {
782 VIXL_ASSERT(rd.size() == rn.size());
783 VIXL_ASSERT(rd.size() == rm.size());
784 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
785 }
786
787
788 // Bitfield operations.
bfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)789 void Assembler::bfm(const Register& rd,
790 const Register& rn,
791 unsigned immr,
792 unsigned imms) {
793 VIXL_ASSERT(rd.size() == rn.size());
794 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
795 Emit(SF(rd) | BFM | N |
796 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
797 }
798
799
sbfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)800 void Assembler::sbfm(const Register& rd,
801 const Register& rn,
802 unsigned immr,
803 unsigned imms) {
804 VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
805 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
806 Emit(SF(rd) | SBFM | N |
807 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
808 }
809
810
ubfm(const Register & rd,const Register & rn,unsigned immr,unsigned imms)811 void Assembler::ubfm(const Register& rd,
812 const Register& rn,
813 unsigned immr,
814 unsigned imms) {
815 VIXL_ASSERT(rd.size() == rn.size());
816 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
817 Emit(SF(rd) | UBFM | N |
818 ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
819 }
820
821
extr(const Register & rd,const Register & rn,const Register & rm,unsigned lsb)822 void Assembler::extr(const Register& rd,
823 const Register& rn,
824 const Register& rm,
825 unsigned lsb) {
826 VIXL_ASSERT(rd.size() == rn.size());
827 VIXL_ASSERT(rd.size() == rm.size());
828 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
829 Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.size()) | Rn(rn) | Rd(rd));
830 }
831
832
csel(const Register & rd,const Register & rn,const Register & rm,Condition cond)833 void Assembler::csel(const Register& rd,
834 const Register& rn,
835 const Register& rm,
836 Condition cond) {
837 ConditionalSelect(rd, rn, rm, cond, CSEL);
838 }
839
840
csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)841 void Assembler::csinc(const Register& rd,
842 const Register& rn,
843 const Register& rm,
844 Condition cond) {
845 ConditionalSelect(rd, rn, rm, cond, CSINC);
846 }
847
848
csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)849 void Assembler::csinv(const Register& rd,
850 const Register& rn,
851 const Register& rm,
852 Condition cond) {
853 ConditionalSelect(rd, rn, rm, cond, CSINV);
854 }
855
856
csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)857 void Assembler::csneg(const Register& rd,
858 const Register& rn,
859 const Register& rm,
860 Condition cond) {
861 ConditionalSelect(rd, rn, rm, cond, CSNEG);
862 }
863
864
cset(const Register & rd,Condition cond)865 void Assembler::cset(const Register &rd, Condition cond) {
866 VIXL_ASSERT((cond != al) && (cond != nv));
867 Register zr = AppropriateZeroRegFor(rd);
868 csinc(rd, zr, zr, InvertCondition(cond));
869 }
870
871
csetm(const Register & rd,Condition cond)872 void Assembler::csetm(const Register &rd, Condition cond) {
873 VIXL_ASSERT((cond != al) && (cond != nv));
874 Register zr = AppropriateZeroRegFor(rd);
875 csinv(rd, zr, zr, InvertCondition(cond));
876 }
877
878
cinc(const Register & rd,const Register & rn,Condition cond)879 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) {
880 VIXL_ASSERT((cond != al) && (cond != nv));
881 csinc(rd, rn, rn, InvertCondition(cond));
882 }
883
884
cinv(const Register & rd,const Register & rn,Condition cond)885 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) {
886 VIXL_ASSERT((cond != al) && (cond != nv));
887 csinv(rd, rn, rn, InvertCondition(cond));
888 }
889
890
cneg(const Register & rd,const Register & rn,Condition cond)891 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) {
892 VIXL_ASSERT((cond != al) && (cond != nv));
893 csneg(rd, rn, rn, InvertCondition(cond));
894 }
895
896
ConditionalSelect(const Register & rd,const Register & rn,const Register & rm,Condition cond,ConditionalSelectOp op)897 void Assembler::ConditionalSelect(const Register& rd,
898 const Register& rn,
899 const Register& rm,
900 Condition cond,
901 ConditionalSelectOp op) {
902 VIXL_ASSERT(rd.size() == rn.size());
903 VIXL_ASSERT(rd.size() == rm.size());
904 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
905 }
906
907
ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)908 void Assembler::ccmn(const Register& rn,
909 const Operand& operand,
910 StatusFlags nzcv,
911 Condition cond) {
912 ConditionalCompare(rn, operand, nzcv, cond, CCMN);
913 }
914
915
ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)916 void Assembler::ccmp(const Register& rn,
917 const Operand& operand,
918 StatusFlags nzcv,
919 Condition cond) {
920 ConditionalCompare(rn, operand, nzcv, cond, CCMP);
921 }
922
923
DataProcessing3Source(const Register & rd,const Register & rn,const Register & rm,const Register & ra,DataProcessing3SourceOp op)924 void Assembler::DataProcessing3Source(const Register& rd,
925 const Register& rn,
926 const Register& rm,
927 const Register& ra,
928 DataProcessing3SourceOp op) {
929 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
930 }
931
932
crc32b(const Register & rd,const Register & rn,const Register & rm)933 void Assembler::crc32b(const Register& rd,
934 const Register& rn,
935 const Register& rm) {
936 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
937 Emit(SF(rm) | Rm(rm) | CRC32B | Rn(rn) | Rd(rd));
938 }
939
940
crc32h(const Register & rd,const Register & rn,const Register & rm)941 void Assembler::crc32h(const Register& rd,
942 const Register& rn,
943 const Register& rm) {
944 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
945 Emit(SF(rm) | Rm(rm) | CRC32H | Rn(rn) | Rd(rd));
946 }
947
948
crc32w(const Register & rd,const Register & rn,const Register & rm)949 void Assembler::crc32w(const Register& rd,
950 const Register& rn,
951 const Register& rm) {
952 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
953 Emit(SF(rm) | Rm(rm) | CRC32W | Rn(rn) | Rd(rd));
954 }
955
956
crc32x(const Register & rd,const Register & rn,const Register & rm)957 void Assembler::crc32x(const Register& rd,
958 const Register& rn,
959 const Register& rm) {
960 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
961 Emit(SF(rm) | Rm(rm) | CRC32X | Rn(rn) | Rd(rd));
962 }
963
964
crc32cb(const Register & rd,const Register & rn,const Register & rm)965 void Assembler::crc32cb(const Register& rd,
966 const Register& rn,
967 const Register& rm) {
968 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
969 Emit(SF(rm) | Rm(rm) | CRC32CB | Rn(rn) | Rd(rd));
970 }
971
972
crc32ch(const Register & rd,const Register & rn,const Register & rm)973 void Assembler::crc32ch(const Register& rd,
974 const Register& rn,
975 const Register& rm) {
976 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
977 Emit(SF(rm) | Rm(rm) | CRC32CH | Rn(rn) | Rd(rd));
978 }
979
980
crc32cw(const Register & rd,const Register & rn,const Register & rm)981 void Assembler::crc32cw(const Register& rd,
982 const Register& rn,
983 const Register& rm) {
984 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
985 Emit(SF(rm) | Rm(rm) | CRC32CW | Rn(rn) | Rd(rd));
986 }
987
988
crc32cx(const Register & rd,const Register & rn,const Register & rm)989 void Assembler::crc32cx(const Register& rd,
990 const Register& rn,
991 const Register& rm) {
992 VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
993 Emit(SF(rm) | Rm(rm) | CRC32CX | Rn(rn) | Rd(rd));
994 }
995
996
mul(const Register & rd,const Register & rn,const Register & rm)997 void Assembler::mul(const Register& rd,
998 const Register& rn,
999 const Register& rm) {
1000 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
1001 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
1002 }
1003
1004
madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1005 void Assembler::madd(const Register& rd,
1006 const Register& rn,
1007 const Register& rm,
1008 const Register& ra) {
1009 DataProcessing3Source(rd, rn, rm, ra, MADD);
1010 }
1011
1012
mneg(const Register & rd,const Register & rn,const Register & rm)1013 void Assembler::mneg(const Register& rd,
1014 const Register& rn,
1015 const Register& rm) {
1016 VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
1017 DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
1018 }
1019
1020
msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1021 void Assembler::msub(const Register& rd,
1022 const Register& rn,
1023 const Register& rm,
1024 const Register& ra) {
1025 DataProcessing3Source(rd, rn, rm, ra, MSUB);
1026 }
1027
1028
umaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1029 void Assembler::umaddl(const Register& rd,
1030 const Register& rn,
1031 const Register& rm,
1032 const Register& ra) {
1033 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1034 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1035 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
1036 }
1037
1038
smaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1039 void Assembler::smaddl(const Register& rd,
1040 const Register& rn,
1041 const Register& rm,
1042 const Register& ra) {
1043 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1044 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1045 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
1046 }
1047
1048
umsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1049 void Assembler::umsubl(const Register& rd,
1050 const Register& rn,
1051 const Register& rm,
1052 const Register& ra) {
1053 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1054 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1055 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
1056 }
1057
1058
smsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1059 void Assembler::smsubl(const Register& rd,
1060 const Register& rn,
1061 const Register& rm,
1062 const Register& ra) {
1063 VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
1064 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1065 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
1066 }
1067
1068
smull(const Register & rd,const Register & rn,const Register & rm)1069 void Assembler::smull(const Register& rd,
1070 const Register& rn,
1071 const Register& rm) {
1072 VIXL_ASSERT(rd.Is64Bits());
1073 VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
1074 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
1075 }
1076
1077
sdiv(const Register & rd,const Register & rn,const Register & rm)1078 void Assembler::sdiv(const Register& rd,
1079 const Register& rn,
1080 const Register& rm) {
1081 VIXL_ASSERT(rd.size() == rn.size());
1082 VIXL_ASSERT(rd.size() == rm.size());
1083 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
1084 }
1085
1086
smulh(const Register & xd,const Register & xn,const Register & xm)1087 void Assembler::smulh(const Register& xd,
1088 const Register& xn,
1089 const Register& xm) {
1090 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1091 DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
1092 }
1093
1094
umulh(const Register & xd,const Register & xn,const Register & xm)1095 void Assembler::umulh(const Register& xd,
1096 const Register& xn,
1097 const Register& xm) {
1098 VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
1099 DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
1100 }
1101
1102
udiv(const Register & rd,const Register & rn,const Register & rm)1103 void Assembler::udiv(const Register& rd,
1104 const Register& rn,
1105 const Register& rm) {
1106 VIXL_ASSERT(rd.size() == rn.size());
1107 VIXL_ASSERT(rd.size() == rm.size());
1108 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1109 }
1110
1111
rbit(const Register & rd,const Register & rn)1112 void Assembler::rbit(const Register& rd,
1113 const Register& rn) {
1114 DataProcessing1Source(rd, rn, RBIT);
1115 }
1116
1117
rev16(const Register & rd,const Register & rn)1118 void Assembler::rev16(const Register& rd,
1119 const Register& rn) {
1120 DataProcessing1Source(rd, rn, REV16);
1121 }
1122
1123
rev32(const Register & rd,const Register & rn)1124 void Assembler::rev32(const Register& rd,
1125 const Register& rn) {
1126 VIXL_ASSERT(rd.Is64Bits());
1127 DataProcessing1Source(rd, rn, REV);
1128 }
1129
1130
rev(const Register & rd,const Register & rn)1131 void Assembler::rev(const Register& rd,
1132 const Register& rn) {
1133 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1134 }
1135
1136
clz(const Register & rd,const Register & rn)1137 void Assembler::clz(const Register& rd,
1138 const Register& rn) {
1139 DataProcessing1Source(rd, rn, CLZ);
1140 }
1141
1142
cls(const Register & rd,const Register & rn)1143 void Assembler::cls(const Register& rd,
1144 const Register& rn) {
1145 DataProcessing1Source(rd, rn, CLS);
1146 }
1147
1148
ldp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1149 void Assembler::ldp(const CPURegister& rt,
1150 const CPURegister& rt2,
1151 const MemOperand& src) {
1152 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1153 }
1154
1155
stp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1156 void Assembler::stp(const CPURegister& rt,
1157 const CPURegister& rt2,
1158 const MemOperand& dst) {
1159 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1160 }
1161
1162
ldpsw(const Register & rt,const Register & rt2,const MemOperand & src)1163 void Assembler::ldpsw(const Register& rt,
1164 const Register& rt2,
1165 const MemOperand& src) {
1166 VIXL_ASSERT(rt.Is64Bits());
1167 LoadStorePair(rt, rt2, src, LDPSW_x);
1168 }
1169
1170
LoadStorePair(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)1171 void Assembler::LoadStorePair(const CPURegister& rt,
1172 const CPURegister& rt2,
1173 const MemOperand& addr,
1174 LoadStorePairOp op) {
1175 // 'rt' and 'rt2' can only be aliased for stores.
1176 VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1177 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1178 VIXL_ASSERT(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
1179
1180 int offset = static_cast<int>(addr.offset());
1181 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
1182 ImmLSPair(offset, CalcLSPairDataSize(op));
1183
1184 Instr addrmodeop;
1185 if (addr.IsImmediateOffset()) {
1186 addrmodeop = LoadStorePairOffsetFixed;
1187 } else {
1188 VIXL_ASSERT(addr.offset() != 0);
1189 if (addr.IsPreIndex()) {
1190 addrmodeop = LoadStorePairPreIndexFixed;
1191 } else {
1192 VIXL_ASSERT(addr.IsPostIndex());
1193 addrmodeop = LoadStorePairPostIndexFixed;
1194 }
1195 }
1196 Emit(addrmodeop | memop);
1197 }
1198
1199
ldnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1200 void Assembler::ldnp(const CPURegister& rt,
1201 const CPURegister& rt2,
1202 const MemOperand& src) {
1203 LoadStorePairNonTemporal(rt, rt2, src,
1204 LoadPairNonTemporalOpFor(rt, rt2));
1205 }
1206
1207
stnp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1208 void Assembler::stnp(const CPURegister& rt,
1209 const CPURegister& rt2,
1210 const MemOperand& dst) {
1211 LoadStorePairNonTemporal(rt, rt2, dst,
1212 StorePairNonTemporalOpFor(rt, rt2));
1213 }
1214
1215
LoadStorePairNonTemporal(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairNonTemporalOp op)1216 void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
1217 const CPURegister& rt2,
1218 const MemOperand& addr,
1219 LoadStorePairNonTemporalOp op) {
1220 VIXL_ASSERT(!rt.Is(rt2));
1221 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
1222 VIXL_ASSERT(addr.IsImmediateOffset());
1223
1224 unsigned size = CalcLSPairDataSize(
1225 static_cast<LoadStorePairOp>(op & LoadStorePairMask));
1226 VIXL_ASSERT(IsImmLSPair(addr.offset(), size));
1227 int offset = static_cast<int>(addr.offset());
1228 Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | ImmLSPair(offset, size));
1229 }
1230
1231
1232 // Memory instructions.
ldrb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1233 void Assembler::ldrb(const Register& rt, const MemOperand& src,
1234 LoadStoreScalingOption option) {
1235 VIXL_ASSERT(option != RequireUnscaledOffset);
1236 VIXL_ASSERT(option != PreferUnscaledOffset);
1237 LoadStore(rt, src, LDRB_w, option);
1238 }
1239
1240
strb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1241 void Assembler::strb(const Register& rt, const MemOperand& dst,
1242 LoadStoreScalingOption option) {
1243 VIXL_ASSERT(option != RequireUnscaledOffset);
1244 VIXL_ASSERT(option != PreferUnscaledOffset);
1245 LoadStore(rt, dst, STRB_w, option);
1246 }
1247
1248
ldrsb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1249 void Assembler::ldrsb(const Register& rt, const MemOperand& src,
1250 LoadStoreScalingOption option) {
1251 VIXL_ASSERT(option != RequireUnscaledOffset);
1252 VIXL_ASSERT(option != PreferUnscaledOffset);
1253 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1254 }
1255
1256
ldrh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1257 void Assembler::ldrh(const Register& rt, const MemOperand& src,
1258 LoadStoreScalingOption option) {
1259 VIXL_ASSERT(option != RequireUnscaledOffset);
1260 VIXL_ASSERT(option != PreferUnscaledOffset);
1261 LoadStore(rt, src, LDRH_w, option);
1262 }
1263
1264
strh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1265 void Assembler::strh(const Register& rt, const MemOperand& dst,
1266 LoadStoreScalingOption option) {
1267 VIXL_ASSERT(option != RequireUnscaledOffset);
1268 VIXL_ASSERT(option != PreferUnscaledOffset);
1269 LoadStore(rt, dst, STRH_w, option);
1270 }
1271
1272
ldrsh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1273 void Assembler::ldrsh(const Register& rt, const MemOperand& src,
1274 LoadStoreScalingOption option) {
1275 VIXL_ASSERT(option != RequireUnscaledOffset);
1276 VIXL_ASSERT(option != PreferUnscaledOffset);
1277 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1278 }
1279
1280
ldr(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1281 void Assembler::ldr(const CPURegister& rt, const MemOperand& src,
1282 LoadStoreScalingOption option) {
1283 VIXL_ASSERT(option != RequireUnscaledOffset);
1284 VIXL_ASSERT(option != PreferUnscaledOffset);
1285 LoadStore(rt, src, LoadOpFor(rt), option);
1286 }
1287
1288
str(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1289 void Assembler::str(const CPURegister& rt, const MemOperand& dst,
1290 LoadStoreScalingOption option) {
1291 VIXL_ASSERT(option != RequireUnscaledOffset);
1292 VIXL_ASSERT(option != PreferUnscaledOffset);
1293 LoadStore(rt, dst, StoreOpFor(rt), option);
1294 }
1295
1296
ldrsw(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1297 void Assembler::ldrsw(const Register& rt, const MemOperand& src,
1298 LoadStoreScalingOption option) {
1299 VIXL_ASSERT(rt.Is64Bits());
1300 VIXL_ASSERT(option != RequireUnscaledOffset);
1301 VIXL_ASSERT(option != PreferUnscaledOffset);
1302 LoadStore(rt, src, LDRSW_x, option);
1303 }
1304
1305
ldurb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1306 void Assembler::ldurb(const Register& rt, const MemOperand& src,
1307 LoadStoreScalingOption option) {
1308 VIXL_ASSERT(option != RequireScaledOffset);
1309 VIXL_ASSERT(option != PreferScaledOffset);
1310 LoadStore(rt, src, LDRB_w, option);
1311 }
1312
1313
sturb(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1314 void Assembler::sturb(const Register& rt, const MemOperand& dst,
1315 LoadStoreScalingOption option) {
1316 VIXL_ASSERT(option != RequireScaledOffset);
1317 VIXL_ASSERT(option != PreferScaledOffset);
1318 LoadStore(rt, dst, STRB_w, option);
1319 }
1320
1321
ldursb(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1322 void Assembler::ldursb(const Register& rt, const MemOperand& src,
1323 LoadStoreScalingOption option) {
1324 VIXL_ASSERT(option != RequireScaledOffset);
1325 VIXL_ASSERT(option != PreferScaledOffset);
1326 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
1327 }
1328
1329
ldurh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1330 void Assembler::ldurh(const Register& rt, const MemOperand& src,
1331 LoadStoreScalingOption option) {
1332 VIXL_ASSERT(option != RequireScaledOffset);
1333 VIXL_ASSERT(option != PreferScaledOffset);
1334 LoadStore(rt, src, LDRH_w, option);
1335 }
1336
1337
sturh(const Register & rt,const MemOperand & dst,LoadStoreScalingOption option)1338 void Assembler::sturh(const Register& rt, const MemOperand& dst,
1339 LoadStoreScalingOption option) {
1340 VIXL_ASSERT(option != RequireScaledOffset);
1341 VIXL_ASSERT(option != PreferScaledOffset);
1342 LoadStore(rt, dst, STRH_w, option);
1343 }
1344
1345
ldursh(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1346 void Assembler::ldursh(const Register& rt, const MemOperand& src,
1347 LoadStoreScalingOption option) {
1348 VIXL_ASSERT(option != RequireScaledOffset);
1349 VIXL_ASSERT(option != PreferScaledOffset);
1350 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
1351 }
1352
1353
ldur(const CPURegister & rt,const MemOperand & src,LoadStoreScalingOption option)1354 void Assembler::ldur(const CPURegister& rt, const MemOperand& src,
1355 LoadStoreScalingOption option) {
1356 VIXL_ASSERT(option != RequireScaledOffset);
1357 VIXL_ASSERT(option != PreferScaledOffset);
1358 LoadStore(rt, src, LoadOpFor(rt), option);
1359 }
1360
1361
stur(const CPURegister & rt,const MemOperand & dst,LoadStoreScalingOption option)1362 void Assembler::stur(const CPURegister& rt, const MemOperand& dst,
1363 LoadStoreScalingOption option) {
1364 VIXL_ASSERT(option != RequireScaledOffset);
1365 VIXL_ASSERT(option != PreferScaledOffset);
1366 LoadStore(rt, dst, StoreOpFor(rt), option);
1367 }
1368
1369
ldursw(const Register & rt,const MemOperand & src,LoadStoreScalingOption option)1370 void Assembler::ldursw(const Register& rt, const MemOperand& src,
1371 LoadStoreScalingOption option) {
1372 VIXL_ASSERT(rt.Is64Bits());
1373 VIXL_ASSERT(option != RequireScaledOffset);
1374 VIXL_ASSERT(option != PreferScaledOffset);
1375 LoadStore(rt, src, LDRSW_x, option);
1376 }
1377
1378
ldrsw(const Register & rt,int imm19)1379 void Assembler::ldrsw(const Register& rt, int imm19) {
1380 Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
1381 }
1382
1383
ldr(const CPURegister & rt,int imm19)1384 void Assembler::ldr(const CPURegister& rt, int imm19) {
1385 LoadLiteralOp op = LoadLiteralOpFor(rt);
1386 Emit(op | ImmLLiteral(imm19) | Rt(rt));
1387 }
1388
1389
prfm(PrefetchOperation op,int imm19)1390 void Assembler::prfm(PrefetchOperation op, int imm19) {
1391 Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
1392 }
1393
1394
1395 // Exclusive-access instructions.
stxrb(const Register & rs,const Register & rt,const MemOperand & dst)1396 void Assembler::stxrb(const Register& rs,
1397 const Register& rt,
1398 const MemOperand& dst) {
1399 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1400 Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1401 }
1402
1403
stxrh(const Register & rs,const Register & rt,const MemOperand & dst)1404 void Assembler::stxrh(const Register& rs,
1405 const Register& rt,
1406 const MemOperand& dst) {
1407 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1408 Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1409 }
1410
1411
stxr(const Register & rs,const Register & rt,const MemOperand & dst)1412 void Assembler::stxr(const Register& rs,
1413 const Register& rt,
1414 const MemOperand& dst) {
1415 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1416 LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
1417 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1418 }
1419
1420
ldxrb(const Register & rt,const MemOperand & src)1421 void Assembler::ldxrb(const Register& rt,
1422 const MemOperand& src) {
1423 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1424 Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1425 }
1426
1427
ldxrh(const Register & rt,const MemOperand & src)1428 void Assembler::ldxrh(const Register& rt,
1429 const MemOperand& src) {
1430 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1431 Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1432 }
1433
1434
ldxr(const Register & rt,const MemOperand & src)1435 void Assembler::ldxr(const Register& rt,
1436 const MemOperand& src) {
1437 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1438 LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
1439 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1440 }
1441
1442
stxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1443 void Assembler::stxp(const Register& rs,
1444 const Register& rt,
1445 const Register& rt2,
1446 const MemOperand& dst) {
1447 VIXL_ASSERT(rt.size() == rt2.size());
1448 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1449 LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
1450 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
1451 }
1452
1453
ldxp(const Register & rt,const Register & rt2,const MemOperand & src)1454 void Assembler::ldxp(const Register& rt,
1455 const Register& rt2,
1456 const MemOperand& src) {
1457 VIXL_ASSERT(rt.size() == rt2.size());
1458 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1459 LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
1460 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
1461 }
1462
1463
stlxrb(const Register & rs,const Register & rt,const MemOperand & dst)1464 void Assembler::stlxrb(const Register& rs,
1465 const Register& rt,
1466 const MemOperand& dst) {
1467 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1468 Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1469 }
1470
1471
stlxrh(const Register & rs,const Register & rt,const MemOperand & dst)1472 void Assembler::stlxrh(const Register& rs,
1473 const Register& rt,
1474 const MemOperand& dst) {
1475 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1476 Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1477 }
1478
1479
stlxr(const Register & rs,const Register & rt,const MemOperand & dst)1480 void Assembler::stlxr(const Register& rs,
1481 const Register& rt,
1482 const MemOperand& dst) {
1483 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1484 LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
1485 Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1486 }
1487
1488
ldaxrb(const Register & rt,const MemOperand & src)1489 void Assembler::ldaxrb(const Register& rt,
1490 const MemOperand& src) {
1491 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1492 Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1493 }
1494
1495
ldaxrh(const Register & rt,const MemOperand & src)1496 void Assembler::ldaxrh(const Register& rt,
1497 const MemOperand& src) {
1498 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1499 Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1500 }
1501
1502
ldaxr(const Register & rt,const MemOperand & src)1503 void Assembler::ldaxr(const Register& rt,
1504 const MemOperand& src) {
1505 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1506 LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
1507 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1508 }
1509
1510
stlxp(const Register & rs,const Register & rt,const Register & rt2,const MemOperand & dst)1511 void Assembler::stlxp(const Register& rs,
1512 const Register& rt,
1513 const Register& rt2,
1514 const MemOperand& dst) {
1515 VIXL_ASSERT(rt.size() == rt2.size());
1516 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1517 LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
1518 Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
1519 }
1520
1521
ldaxp(const Register & rt,const Register & rt2,const MemOperand & src)1522 void Assembler::ldaxp(const Register& rt,
1523 const Register& rt2,
1524 const MemOperand& src) {
1525 VIXL_ASSERT(rt.size() == rt2.size());
1526 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1527 LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
1528 Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
1529 }
1530
1531
stlrb(const Register & rt,const MemOperand & dst)1532 void Assembler::stlrb(const Register& rt,
1533 const MemOperand& dst) {
1534 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1535 Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1536 }
1537
1538
stlrh(const Register & rt,const MemOperand & dst)1539 void Assembler::stlrh(const Register& rt,
1540 const MemOperand& dst) {
1541 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1542 Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1543 }
1544
1545
stlr(const Register & rt,const MemOperand & dst)1546 void Assembler::stlr(const Register& rt,
1547 const MemOperand& dst) {
1548 VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
1549 LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
1550 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
1551 }
1552
1553
ldarb(const Register & rt,const MemOperand & src)1554 void Assembler::ldarb(const Register& rt,
1555 const MemOperand& src) {
1556 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1557 Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1558 }
1559
1560
ldarh(const Register & rt,const MemOperand & src)1561 void Assembler::ldarh(const Register& rt,
1562 const MemOperand& src) {
1563 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1564 Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1565 }
1566
1567
ldar(const Register & rt,const MemOperand & src)1568 void Assembler::ldar(const Register& rt,
1569 const MemOperand& src) {
1570 VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
1571 LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
1572 Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
1573 }
1574
1575
prfm(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1576 void Assembler::prfm(PrefetchOperation op, const MemOperand& address,
1577 LoadStoreScalingOption option) {
1578 VIXL_ASSERT(option != RequireUnscaledOffset);
1579 VIXL_ASSERT(option != PreferUnscaledOffset);
1580 Prefetch(op, address, option);
1581 }
1582
1583
prfum(PrefetchOperation op,const MemOperand & address,LoadStoreScalingOption option)1584 void Assembler::prfum(PrefetchOperation op, const MemOperand& address,
1585 LoadStoreScalingOption option) {
1586 VIXL_ASSERT(option != RequireScaledOffset);
1587 VIXL_ASSERT(option != PreferScaledOffset);
1588 Prefetch(op, address, option);
1589 }
1590
1591
sys(int op1,int crn,int crm,int op2,const Register & rt)1592 void Assembler::sys(int op1, int crn, int crm, int op2, const Register& rt) {
1593 Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(rt));
1594 }
1595
1596
sys(int op,const Register & rt)1597 void Assembler::sys(int op, const Register& rt) {
1598 Emit(SYS | SysOp(op) | Rt(rt));
1599 }
1600
1601
dc(DataCacheOp op,const Register & rt)1602 void Assembler::dc(DataCacheOp op, const Register& rt) {
1603 VIXL_ASSERT((op == CVAC) || (op == CVAU) || (op == CIVAC) || (op == ZVA));
1604 sys(op, rt);
1605 }
1606
1607
ic(InstructionCacheOp op,const Register & rt)1608 void Assembler::ic(InstructionCacheOp op, const Register& rt) {
1609 VIXL_ASSERT(op == IVAU);
1610 sys(op, rt);
1611 }
1612
1613
1614 // NEON structure loads and stores.
LoadStoreStructAddrModeField(const MemOperand & addr)1615 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
1616 Instr addr_field = RnSP(addr.base());
1617
1618 if (addr.IsPostIndex()) {
1619 VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
1620 static_cast<NEONLoadStoreMultiStructPostIndexOp>(
1621 NEONLoadStoreSingleStructPostIndex));
1622
1623 addr_field |= NEONLoadStoreMultiStructPostIndex;
1624 if (addr.offset() == 0) {
1625 addr_field |= RmNot31(addr.regoffset());
1626 } else {
1627 // The immediate post index addressing mode is indicated by rm = 31.
1628 // The immediate is implied by the number of vector registers used.
1629 addr_field |= (0x1f << Rm_offset);
1630 }
1631 } else {
1632 VIXL_ASSERT(addr.IsImmediateOffset() && (addr.offset() == 0));
1633 }
1634 return addr_field;
1635 }
1636
LoadStoreStructVerify(const VRegister & vt,const MemOperand & addr,Instr op)1637 void Assembler::LoadStoreStructVerify(const VRegister& vt,
1638 const MemOperand& addr,
1639 Instr op) {
1640 #ifdef DEBUG
1641 // Assert that addressing mode is either offset (with immediate 0), post
1642 // index by immediate of the size of the register list, or post index by a
1643 // value in a core register.
1644 if (addr.IsImmediateOffset()) {
1645 VIXL_ASSERT(addr.offset() == 0);
1646 } else {
1647 int offset = vt.SizeInBytes();
1648 switch (op) {
1649 case NEON_LD1_1v:
1650 case NEON_ST1_1v:
1651 offset *= 1; break;
1652 case NEONLoadStoreSingleStructLoad1:
1653 case NEONLoadStoreSingleStructStore1:
1654 case NEON_LD1R:
1655 offset = (offset / vt.lanes()) * 1; break;
1656
1657 case NEON_LD1_2v:
1658 case NEON_ST1_2v:
1659 case NEON_LD2:
1660 case NEON_ST2:
1661 offset *= 2;
1662 break;
1663 case NEONLoadStoreSingleStructLoad2:
1664 case NEONLoadStoreSingleStructStore2:
1665 case NEON_LD2R:
1666 offset = (offset / vt.lanes()) * 2; break;
1667
1668 case NEON_LD1_3v:
1669 case NEON_ST1_3v:
1670 case NEON_LD3:
1671 case NEON_ST3:
1672 offset *= 3; break;
1673 case NEONLoadStoreSingleStructLoad3:
1674 case NEONLoadStoreSingleStructStore3:
1675 case NEON_LD3R:
1676 offset = (offset / vt.lanes()) * 3; break;
1677
1678 case NEON_LD1_4v:
1679 case NEON_ST1_4v:
1680 case NEON_LD4:
1681 case NEON_ST4:
1682 offset *= 4; break;
1683 case NEONLoadStoreSingleStructLoad4:
1684 case NEONLoadStoreSingleStructStore4:
1685 case NEON_LD4R:
1686 offset = (offset / vt.lanes()) * 4; break;
1687 default:
1688 VIXL_UNREACHABLE();
1689 }
1690 VIXL_ASSERT(!addr.regoffset().Is(NoReg) ||
1691 addr.offset() == offset);
1692 }
1693 #else
1694 USE(vt, addr, op);
1695 #endif
1696 }
1697
LoadStoreStruct(const VRegister & vt,const MemOperand & addr,NEONLoadStoreMultiStructOp op)1698 void Assembler::LoadStoreStruct(const VRegister& vt,
1699 const MemOperand& addr,
1700 NEONLoadStoreMultiStructOp op) {
1701 LoadStoreStructVerify(vt, addr, op);
1702 VIXL_ASSERT(vt.IsVector() || vt.Is1D());
1703 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
1704 }
1705
1706
LoadStoreStructSingleAllLanes(const VRegister & vt,const MemOperand & addr,NEONLoadStoreSingleStructOp op)1707 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
1708 const MemOperand& addr,
1709 NEONLoadStoreSingleStructOp op) {
1710 LoadStoreStructVerify(vt, addr, op);
1711 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
1712 }
1713
1714
ld1(const VRegister & vt,const MemOperand & src)1715 void Assembler::ld1(const VRegister& vt,
1716 const MemOperand& src) {
1717 LoadStoreStruct(vt, src, NEON_LD1_1v);
1718 }
1719
1720
ld1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1721 void Assembler::ld1(const VRegister& vt,
1722 const VRegister& vt2,
1723 const MemOperand& src) {
1724 USE(vt2);
1725 VIXL_ASSERT(AreSameFormat(vt, vt2));
1726 VIXL_ASSERT(AreConsecutive(vt, vt2));
1727 LoadStoreStruct(vt, src, NEON_LD1_2v);
1728 }
1729
1730
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1731 void Assembler::ld1(const VRegister& vt,
1732 const VRegister& vt2,
1733 const VRegister& vt3,
1734 const MemOperand& src) {
1735 USE(vt2, vt3);
1736 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1737 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1738 LoadStoreStruct(vt, src, NEON_LD1_3v);
1739 }
1740
1741
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1742 void Assembler::ld1(const VRegister& vt,
1743 const VRegister& vt2,
1744 const VRegister& vt3,
1745 const VRegister& vt4,
1746 const MemOperand& src) {
1747 USE(vt2, vt3, vt4);
1748 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1749 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1750 LoadStoreStruct(vt, src, NEON_LD1_4v);
1751 }
1752
1753
ld2(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1754 void Assembler::ld2(const VRegister& vt,
1755 const VRegister& vt2,
1756 const MemOperand& src) {
1757 USE(vt2);
1758 VIXL_ASSERT(AreSameFormat(vt, vt2));
1759 VIXL_ASSERT(AreConsecutive(vt, vt2));
1760 LoadStoreStruct(vt, src, NEON_LD2);
1761 }
1762
1763
ld2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & src)1764 void Assembler::ld2(const VRegister& vt,
1765 const VRegister& vt2,
1766 int lane,
1767 const MemOperand& src) {
1768 USE(vt2);
1769 VIXL_ASSERT(AreSameFormat(vt, vt2));
1770 VIXL_ASSERT(AreConsecutive(vt, vt2));
1771 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
1772 }
1773
1774
ld2r(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1775 void Assembler::ld2r(const VRegister& vt,
1776 const VRegister& vt2,
1777 const MemOperand& src) {
1778 USE(vt2);
1779 VIXL_ASSERT(AreSameFormat(vt, vt2));
1780 VIXL_ASSERT(AreConsecutive(vt, vt2));
1781 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
1782 }
1783
1784
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1785 void Assembler::ld3(const VRegister& vt,
1786 const VRegister& vt2,
1787 const VRegister& vt3,
1788 const MemOperand& src) {
1789 USE(vt2, vt3);
1790 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1791 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1792 LoadStoreStruct(vt, src, NEON_LD3);
1793 }
1794
1795
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & src)1796 void Assembler::ld3(const VRegister& vt,
1797 const VRegister& vt2,
1798 const VRegister& vt3,
1799 int lane,
1800 const MemOperand& src) {
1801 USE(vt2, vt3);
1802 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1803 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1804 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
1805 }
1806
1807
ld3r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1808 void Assembler::ld3r(const VRegister& vt,
1809 const VRegister& vt2,
1810 const VRegister& vt3,
1811 const MemOperand& src) {
1812 USE(vt2, vt3);
1813 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1814 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1815 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
1816 }
1817
1818
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1819 void Assembler::ld4(const VRegister& vt,
1820 const VRegister& vt2,
1821 const VRegister& vt3,
1822 const VRegister& vt4,
1823 const MemOperand& src) {
1824 USE(vt2, vt3, vt4);
1825 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1826 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1827 LoadStoreStruct(vt, src, NEON_LD4);
1828 }
1829
1830
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & src)1831 void Assembler::ld4(const VRegister& vt,
1832 const VRegister& vt2,
1833 const VRegister& vt3,
1834 const VRegister& vt4,
1835 int lane,
1836 const MemOperand& src) {
1837 USE(vt2, vt3, vt4);
1838 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1839 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1840 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
1841 }
1842
1843
ld4r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1844 void Assembler::ld4r(const VRegister& vt,
1845 const VRegister& vt2,
1846 const VRegister& vt3,
1847 const VRegister& vt4,
1848 const MemOperand& src) {
1849 USE(vt2, vt3, vt4);
1850 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1851 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1852 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
1853 }
1854
1855
st1(const VRegister & vt,const MemOperand & src)1856 void Assembler::st1(const VRegister& vt,
1857 const MemOperand& src) {
1858 LoadStoreStruct(vt, src, NEON_ST1_1v);
1859 }
1860
1861
st1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)1862 void Assembler::st1(const VRegister& vt,
1863 const VRegister& vt2,
1864 const MemOperand& src) {
1865 USE(vt2);
1866 VIXL_ASSERT(AreSameFormat(vt, vt2));
1867 VIXL_ASSERT(AreConsecutive(vt, vt2));
1868 LoadStoreStruct(vt, src, NEON_ST1_2v);
1869 }
1870
1871
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)1872 void Assembler::st1(const VRegister& vt,
1873 const VRegister& vt2,
1874 const VRegister& vt3,
1875 const MemOperand& src) {
1876 USE(vt2, vt3);
1877 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1878 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1879 LoadStoreStruct(vt, src, NEON_ST1_3v);
1880 }
1881
1882
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)1883 void Assembler::st1(const VRegister& vt,
1884 const VRegister& vt2,
1885 const VRegister& vt3,
1886 const VRegister& vt4,
1887 const MemOperand& src) {
1888 USE(vt2, vt3, vt4);
1889 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1890 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1891 LoadStoreStruct(vt, src, NEON_ST1_4v);
1892 }
1893
1894
st2(const VRegister & vt,const VRegister & vt2,const MemOperand & dst)1895 void Assembler::st2(const VRegister& vt,
1896 const VRegister& vt2,
1897 const MemOperand& dst) {
1898 USE(vt2);
1899 VIXL_ASSERT(AreSameFormat(vt, vt2));
1900 VIXL_ASSERT(AreConsecutive(vt, vt2));
1901 LoadStoreStruct(vt, dst, NEON_ST2);
1902 }
1903
1904
st2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & dst)1905 void Assembler::st2(const VRegister& vt,
1906 const VRegister& vt2,
1907 int lane,
1908 const MemOperand& dst) {
1909 USE(vt2);
1910 VIXL_ASSERT(AreSameFormat(vt, vt2));
1911 VIXL_ASSERT(AreConsecutive(vt, vt2));
1912 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
1913 }
1914
1915
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & dst)1916 void Assembler::st3(const VRegister& vt,
1917 const VRegister& vt2,
1918 const VRegister& vt3,
1919 const MemOperand& dst) {
1920 USE(vt2, vt3);
1921 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1922 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1923 LoadStoreStruct(vt, dst, NEON_ST3);
1924 }
1925
1926
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & dst)1927 void Assembler::st3(const VRegister& vt,
1928 const VRegister& vt2,
1929 const VRegister& vt3,
1930 int lane,
1931 const MemOperand& dst) {
1932 USE(vt2, vt3);
1933 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
1934 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
1935 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
1936 }
1937
1938
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & dst)1939 void Assembler::st4(const VRegister& vt,
1940 const VRegister& vt2,
1941 const VRegister& vt3,
1942 const VRegister& vt4,
1943 const MemOperand& dst) {
1944 USE(vt2, vt3, vt4);
1945 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1946 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1947 LoadStoreStruct(vt, dst, NEON_ST4);
1948 }
1949
1950
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & dst)1951 void Assembler::st4(const VRegister& vt,
1952 const VRegister& vt2,
1953 const VRegister& vt3,
1954 const VRegister& vt4,
1955 int lane,
1956 const MemOperand& dst) {
1957 USE(vt2, vt3, vt4);
1958 VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
1959 VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
1960 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
1961 }
1962
1963
LoadStoreStructSingle(const VRegister & vt,uint32_t lane,const MemOperand & addr,NEONLoadStoreSingleStructOp op)1964 void Assembler::LoadStoreStructSingle(const VRegister& vt,
1965 uint32_t lane,
1966 const MemOperand& addr,
1967 NEONLoadStoreSingleStructOp op) {
1968 LoadStoreStructVerify(vt, addr, op);
1969
1970 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
1971 // number of lanes, and T is b, h, s or d.
1972 unsigned lane_size = vt.LaneSizeInBytes();
1973 VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
1974
1975 // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
1976 // S and size fields.
1977 lane *= lane_size;
1978 if (lane_size == 8) lane++;
1979
1980 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
1981 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
1982 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
1983
1984 Instr instr = op;
1985 switch (lane_size) {
1986 case 1: instr |= NEONLoadStoreSingle_b; break;
1987 case 2: instr |= NEONLoadStoreSingle_h; break;
1988 case 4: instr |= NEONLoadStoreSingle_s; break;
1989 default:
1990 VIXL_ASSERT(lane_size == 8);
1991 instr |= NEONLoadStoreSingle_d;
1992 }
1993
1994 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
1995 }
1996
1997
ld1(const VRegister & vt,int lane,const MemOperand & src)1998 void Assembler::ld1(const VRegister& vt,
1999 int lane,
2000 const MemOperand& src) {
2001 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2002 }
2003
2004
ld1r(const VRegister & vt,const MemOperand & src)2005 void Assembler::ld1r(const VRegister& vt,
2006 const MemOperand& src) {
2007 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2008 }
2009
2010
st1(const VRegister & vt,int lane,const MemOperand & dst)2011 void Assembler::st1(const VRegister& vt,
2012 int lane,
2013 const MemOperand& dst) {
2014 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2015 }
2016
2017
NEON3DifferentL(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2018 void Assembler::NEON3DifferentL(const VRegister& vd,
2019 const VRegister& vn,
2020 const VRegister& vm,
2021 NEON3DifferentOp vop) {
2022 VIXL_ASSERT(AreSameFormat(vn, vm));
2023 VIXL_ASSERT((vn.Is1H() && vd.Is1S()) ||
2024 (vn.Is1S() && vd.Is1D()) ||
2025 (vn.Is8B() && vd.Is8H()) ||
2026 (vn.Is4H() && vd.Is4S()) ||
2027 (vn.Is2S() && vd.Is2D()) ||
2028 (vn.Is16B() && vd.Is8H())||
2029 (vn.Is8H() && vd.Is4S()) ||
2030 (vn.Is4S() && vd.Is2D()));
2031 Instr format, op = vop;
2032 if (vd.IsScalar()) {
2033 op |= NEON_Q | NEONScalar;
2034 format = SFormat(vn);
2035 } else {
2036 format = VFormat(vn);
2037 }
2038 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2039 }
2040
2041
NEON3DifferentW(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2042 void Assembler::NEON3DifferentW(const VRegister& vd,
2043 const VRegister& vn,
2044 const VRegister& vm,
2045 NEON3DifferentOp vop) {
2046 VIXL_ASSERT(AreSameFormat(vd, vn));
2047 VIXL_ASSERT((vm.Is8B() && vd.Is8H()) ||
2048 (vm.Is4H() && vd.Is4S()) ||
2049 (vm.Is2S() && vd.Is2D()) ||
2050 (vm.Is16B() && vd.Is8H())||
2051 (vm.Is8H() && vd.Is4S()) ||
2052 (vm.Is4S() && vd.Is2D()));
2053 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2054 }
2055
2056
NEON3DifferentHN(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)2057 void Assembler::NEON3DifferentHN(const VRegister& vd,
2058 const VRegister& vn,
2059 const VRegister& vm,
2060 NEON3DifferentOp vop) {
2061 VIXL_ASSERT(AreSameFormat(vm, vn));
2062 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
2063 (vd.Is4H() && vn.Is4S()) ||
2064 (vd.Is2S() && vn.Is2D()) ||
2065 (vd.Is16B() && vn.Is8H())||
2066 (vd.Is8H() && vn.Is4S()) ||
2067 (vd.Is4S() && vn.Is2D()));
2068 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2069 }
2070
2071
2072 #define NEON_3DIFF_LONG_LIST(V) \
2073 V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \
2074 V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \
2075 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
2076 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
2077 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
2078 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
2079 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
2080 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
2081 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
2082 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
2083 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
2084 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
2085 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
2086 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
2087 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
2088 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
2089 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
2090 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
2091 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
2092 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
2093 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
2094 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
2095 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
2096 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
2097 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
2098 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
2099 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
2100 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
2101 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
2102 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
2103 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2104 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2105 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2106 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2107 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
2108 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
2109
2110
2111 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2112 void Assembler::FN(const VRegister& vd, \
2113 const VRegister& vn, \
2114 const VRegister& vm) { \
2115 VIXL_ASSERT(AS); \
2116 NEON3DifferentL(vd, vn, vm, OP); \
2117 }
2118 NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
2119 #undef DEFINE_ASM_FUNC
2120
2121 #define NEON_3DIFF_HN_LIST(V) \
2122 V(addhn, NEON_ADDHN, vd.IsD()) \
2123 V(addhn2, NEON_ADDHN2, vd.IsQ()) \
2124 V(raddhn, NEON_RADDHN, vd.IsD()) \
2125 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
2126 V(subhn, NEON_SUBHN, vd.IsD()) \
2127 V(subhn2, NEON_SUBHN2, vd.IsQ()) \
2128 V(rsubhn, NEON_RSUBHN, vd.IsD()) \
2129 V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
2130
2131 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2132 void Assembler::FN(const VRegister& vd, \
2133 const VRegister& vn, \
2134 const VRegister& vm) { \
2135 VIXL_ASSERT(AS); \
2136 NEON3DifferentHN(vd, vn, vm, OP); \
2137 }
NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)2138 NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
2139 #undef DEFINE_ASM_FUNC
2140
2141 void Assembler::uaddw(const VRegister& vd,
2142 const VRegister& vn,
2143 const VRegister& vm) {
2144 VIXL_ASSERT(vm.IsD());
2145 NEON3DifferentW(vd, vn, vm, NEON_UADDW);
2146 }
2147
2148
uaddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2149 void Assembler::uaddw2(const VRegister& vd,
2150 const VRegister& vn,
2151 const VRegister& vm) {
2152 VIXL_ASSERT(vm.IsQ());
2153 NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
2154 }
2155
2156
saddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2157 void Assembler::saddw(const VRegister& vd,
2158 const VRegister& vn,
2159 const VRegister& vm) {
2160 VIXL_ASSERT(vm.IsD());
2161 NEON3DifferentW(vd, vn, vm, NEON_SADDW);
2162 }
2163
2164
saddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2165 void Assembler::saddw2(const VRegister& vd,
2166 const VRegister& vn,
2167 const VRegister& vm) {
2168 VIXL_ASSERT(vm.IsQ());
2169 NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
2170 }
2171
2172
usubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2173 void Assembler::usubw(const VRegister& vd,
2174 const VRegister& vn,
2175 const VRegister& vm) {
2176 VIXL_ASSERT(vm.IsD());
2177 NEON3DifferentW(vd, vn, vm, NEON_USUBW);
2178 }
2179
2180
usubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2181 void Assembler::usubw2(const VRegister& vd,
2182 const VRegister& vn,
2183 const VRegister& vm) {
2184 VIXL_ASSERT(vm.IsQ());
2185 NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
2186 }
2187
2188
ssubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2189 void Assembler::ssubw(const VRegister& vd,
2190 const VRegister& vn,
2191 const VRegister& vm) {
2192 VIXL_ASSERT(vm.IsD());
2193 NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
2194 }
2195
2196
ssubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2197 void Assembler::ssubw2(const VRegister& vd,
2198 const VRegister& vn,
2199 const VRegister& vm) {
2200 VIXL_ASSERT(vm.IsQ());
2201 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
2202 }
2203
2204
mov(const Register & rd,const Register & rm)2205 void Assembler::mov(const Register& rd, const Register& rm) {
2206 // Moves involving the stack pointer are encoded as add immediate with
2207 // second operand of zero. Otherwise, orr with first operand zr is
2208 // used.
2209 if (rd.IsSP() || rm.IsSP()) {
2210 add(rd, rm, 0);
2211 } else {
2212 orr(rd, AppropriateZeroRegFor(rd), rm);
2213 }
2214 }
2215
2216
mvn(const Register & rd,const Operand & operand)2217 void Assembler::mvn(const Register& rd, const Operand& operand) {
2218 orn(rd, AppropriateZeroRegFor(rd), operand);
2219 }
2220
2221
mrs(const Register & rt,SystemRegister sysreg)2222 void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
2223 VIXL_ASSERT(rt.Is64Bits());
2224 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
2225 }
2226
2227
msr(SystemRegister sysreg,const Register & rt)2228 void Assembler::msr(SystemRegister sysreg, const Register& rt) {
2229 VIXL_ASSERT(rt.Is64Bits());
2230 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
2231 }
2232
2233
clrex(int imm4)2234 void Assembler::clrex(int imm4) {
2235 Emit(CLREX | CRm(imm4));
2236 }
2237
2238
dmb(BarrierDomain domain,BarrierType type)2239 void Assembler::dmb(BarrierDomain domain, BarrierType type) {
2240 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2241 }
2242
2243
dsb(BarrierDomain domain,BarrierType type)2244 void Assembler::dsb(BarrierDomain domain, BarrierType type) {
2245 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2246 }
2247
2248
isb()2249 void Assembler::isb() {
2250 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
2251 }
2252
2253
fmov(const VRegister & vd,double imm)2254 void Assembler::fmov(const VRegister& vd, double imm) {
2255 if (vd.IsScalar()) {
2256 VIXL_ASSERT(vd.Is1D());
2257 Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
2258 } else {
2259 VIXL_ASSERT(vd.Is2D());
2260 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
2261 Instr q = NEON_Q;
2262 uint32_t encoded_imm = FP64ToImm8(imm);
2263 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2264 }
2265 }
2266
2267
fmov(const VRegister & vd,float imm)2268 void Assembler::fmov(const VRegister& vd, float imm) {
2269 if (vd.IsScalar()) {
2270 VIXL_ASSERT(vd.Is1S());
2271 Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
2272 } else {
2273 VIXL_ASSERT(vd.Is2S() | vd.Is4S());
2274 Instr op = NEONModifiedImmediate_MOVI;
2275 Instr q = vd.Is4S() ? NEON_Q : 0;
2276 uint32_t encoded_imm = FP32ToImm8(imm);
2277 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
2278 }
2279 }
2280
2281
fmov(const Register & rd,const VRegister & vn)2282 void Assembler::fmov(const Register& rd, const VRegister& vn) {
2283 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2284 VIXL_ASSERT(rd.size() == vn.size());
2285 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
2286 Emit(op | Rd(rd) | Rn(vn));
2287 }
2288
2289
fmov(const VRegister & vd,const Register & rn)2290 void Assembler::fmov(const VRegister& vd, const Register& rn) {
2291 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2292 VIXL_ASSERT(vd.size() == rn.size());
2293 FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
2294 Emit(op | Rd(vd) | Rn(rn));
2295 }
2296
2297
fmov(const VRegister & vd,const VRegister & vn)2298 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
2299 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2300 VIXL_ASSERT(vd.IsSameFormat(vn));
2301 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
2302 }
2303
2304
fmov(const VRegister & vd,int index,const Register & rn)2305 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
2306 VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
2307 USE(index);
2308 Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
2309 }
2310
2311
fmov(const Register & rd,const VRegister & vn,int index)2312 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
2313 VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
2314 USE(index);
2315 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
2316 }
2317
2318
fmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2319 void Assembler::fmadd(const VRegister& vd,
2320 const VRegister& vn,
2321 const VRegister& vm,
2322 const VRegister& va) {
2323 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMADD_s : FMADD_d);
2324 }
2325
2326
fmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2327 void Assembler::fmsub(const VRegister& vd,
2328 const VRegister& vn,
2329 const VRegister& vm,
2330 const VRegister& va) {
2331 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMSUB_s : FMSUB_d);
2332 }
2333
2334
fnmadd(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2335 void Assembler::fnmadd(const VRegister& vd,
2336 const VRegister& vn,
2337 const VRegister& vm,
2338 const VRegister& va) {
2339 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMADD_s : FNMADD_d);
2340 }
2341
2342
fnmsub(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va)2343 void Assembler::fnmsub(const VRegister& vd,
2344 const VRegister& vn,
2345 const VRegister& vm,
2346 const VRegister& va) {
2347 FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMSUB_s : FNMSUB_d);
2348 }
2349
2350
fnmul(const VRegister & vd,const VRegister & vn,const VRegister & vm)2351 void Assembler::fnmul(const VRegister& vd,
2352 const VRegister& vn,
2353 const VRegister& vm) {
2354 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
2355 Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
2356 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2357 }
2358
2359
FPCompareMacro(const VRegister & vn,double value,FPTrapFlags trap)2360 void Assembler::FPCompareMacro(const VRegister& vn,
2361 double value,
2362 FPTrapFlags trap) {
2363 USE(value);
2364 // Although the fcmp{e} instructions can strictly only take an immediate
2365 // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
2366 // doesn't affect the result of the comparison.
2367 VIXL_ASSERT(value == 0.0);
2368 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2369 Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
2370 Emit(FPType(vn) | op | Rn(vn));
2371 }
2372
2373
FPCompareMacro(const VRegister & vn,const VRegister & vm,FPTrapFlags trap)2374 void Assembler::FPCompareMacro(const VRegister& vn,
2375 const VRegister& vm,
2376 FPTrapFlags trap) {
2377 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2378 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2379 Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
2380 Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
2381 }
2382
2383
fcmp(const VRegister & vn,const VRegister & vm)2384 void Assembler::fcmp(const VRegister& vn,
2385 const VRegister& vm) {
2386 FPCompareMacro(vn, vm, DisableTrap);
2387 }
2388
2389
fcmpe(const VRegister & vn,const VRegister & vm)2390 void Assembler::fcmpe(const VRegister& vn,
2391 const VRegister& vm) {
2392 FPCompareMacro(vn, vm, EnableTrap);
2393 }
2394
2395
fcmp(const VRegister & vn,double value)2396 void Assembler::fcmp(const VRegister& vn,
2397 double value) {
2398 FPCompareMacro(vn, value, DisableTrap);
2399 }
2400
2401
fcmpe(const VRegister & vn,double value)2402 void Assembler::fcmpe(const VRegister& vn,
2403 double value) {
2404 FPCompareMacro(vn, value, EnableTrap);
2405 }
2406
2407
FPCCompareMacro(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond,FPTrapFlags trap)2408 void Assembler::FPCCompareMacro(const VRegister& vn,
2409 const VRegister& vm,
2410 StatusFlags nzcv,
2411 Condition cond,
2412 FPTrapFlags trap) {
2413 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2414 VIXL_ASSERT(vn.IsSameSizeAndType(vm));
2415 Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
2416 Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
2417 }
2418
fccmp(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)2419 void Assembler::fccmp(const VRegister& vn,
2420 const VRegister& vm,
2421 StatusFlags nzcv,
2422 Condition cond) {
2423 FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
2424 }
2425
2426
fccmpe(const VRegister & vn,const VRegister & vm,StatusFlags nzcv,Condition cond)2427 void Assembler::fccmpe(const VRegister& vn,
2428 const VRegister& vm,
2429 StatusFlags nzcv,
2430 Condition cond) {
2431 FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
2432 }
2433
2434
fcsel(const VRegister & vd,const VRegister & vn,const VRegister & vm,Condition cond)2435 void Assembler::fcsel(const VRegister& vd,
2436 const VRegister& vn,
2437 const VRegister& vm,
2438 Condition cond) {
2439 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2440 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2441 Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
2442 }
2443
2444
NEONFPConvertToInt(const Register & rd,const VRegister & vn,Instr op)2445 void Assembler::NEONFPConvertToInt(const Register& rd,
2446 const VRegister& vn,
2447 Instr op) {
2448 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
2449 }
2450
2451
NEONFPConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)2452 void Assembler::NEONFPConvertToInt(const VRegister& vd,
2453 const VRegister& vn,
2454 Instr op) {
2455 if (vn.IsScalar()) {
2456 VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
2457 op |= NEON_Q | NEONScalar;
2458 }
2459 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
2460 }
2461
2462
fcvt(const VRegister & vd,const VRegister & vn)2463 void Assembler::fcvt(const VRegister& vd,
2464 const VRegister& vn) {
2465 FPDataProcessing1SourceOp op;
2466 if (vd.Is1D()) {
2467 VIXL_ASSERT(vn.Is1S() || vn.Is1H());
2468 op = vn.Is1S() ? FCVT_ds : FCVT_dh;
2469 } else if (vd.Is1S()) {
2470 VIXL_ASSERT(vn.Is1D() || vn.Is1H());
2471 op = vn.Is1D() ? FCVT_sd : FCVT_sh;
2472 } else {
2473 VIXL_ASSERT(vd.Is1H());
2474 VIXL_ASSERT(vn.Is1D() || vn.Is1S());
2475 op = vn.Is1D() ? FCVT_hd : FCVT_hs;
2476 }
2477 FPDataProcessing1Source(vd, vn, op);
2478 }
2479
2480
fcvtl(const VRegister & vd,const VRegister & vn)2481 void Assembler::fcvtl(const VRegister& vd,
2482 const VRegister& vn) {
2483 VIXL_ASSERT((vd.Is4S() && vn.Is4H()) ||
2484 (vd.Is2D() && vn.Is2S()));
2485 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2486 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
2487 }
2488
2489
fcvtl2(const VRegister & vd,const VRegister & vn)2490 void Assembler::fcvtl2(const VRegister& vd,
2491 const VRegister& vn) {
2492 VIXL_ASSERT((vd.Is4S() && vn.Is8H()) ||
2493 (vd.Is2D() && vn.Is4S()));
2494 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
2495 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
2496 }
2497
2498
fcvtn(const VRegister & vd,const VRegister & vn)2499 void Assembler::fcvtn(const VRegister& vd,
2500 const VRegister& vn) {
2501 VIXL_ASSERT((vn.Is4S() && vd.Is4H()) ||
2502 (vn.Is2D() && vd.Is2S()));
2503 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2504 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
2505 }
2506
2507
fcvtn2(const VRegister & vd,const VRegister & vn)2508 void Assembler::fcvtn2(const VRegister& vd,
2509 const VRegister& vn) {
2510 VIXL_ASSERT((vn.Is4S() && vd.Is8H()) ||
2511 (vn.Is2D() && vd.Is4S()));
2512 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
2513 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
2514 }
2515
2516
fcvtxn(const VRegister & vd,const VRegister & vn)2517 void Assembler::fcvtxn(const VRegister& vd,
2518 const VRegister& vn) {
2519 Instr format = 1 << NEONSize_offset;
2520 if (vd.IsScalar()) {
2521 VIXL_ASSERT(vd.Is1S() && vn.Is1D());
2522 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
2523 } else {
2524 VIXL_ASSERT(vd.Is2S() && vn.Is2D());
2525 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2526 }
2527 }
2528
2529
fcvtxn2(const VRegister & vd,const VRegister & vn)2530 void Assembler::fcvtxn2(const VRegister& vd,
2531 const VRegister& vn) {
2532 VIXL_ASSERT(vd.Is4S() && vn.Is2D());
2533 Instr format = 1 << NEONSize_offset;
2534 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
2535 }
2536
2537
2538 #define NEON_FP2REGMISC_FCVT_LIST(V) \
2539 V(fcvtnu, NEON_FCVTNU, FCVTNU) \
2540 V(fcvtns, NEON_FCVTNS, FCVTNS) \
2541 V(fcvtpu, NEON_FCVTPU, FCVTPU) \
2542 V(fcvtps, NEON_FCVTPS, FCVTPS) \
2543 V(fcvtmu, NEON_FCVTMU, FCVTMU) \
2544 V(fcvtms, NEON_FCVTMS, FCVTMS) \
2545 V(fcvtau, NEON_FCVTAU, FCVTAU) \
2546 V(fcvtas, NEON_FCVTAS, FCVTAS)
2547
2548 #define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \
2549 void Assembler::FN(const Register& rd, \
2550 const VRegister& vn) { \
2551 NEONFPConvertToInt(rd, vn, SCA_OP); \
2552 } \
2553 void Assembler::FN(const VRegister& vd, \
2554 const VRegister& vn) { \
2555 NEONFPConvertToInt(vd, vn, VEC_OP); \
2556 }
NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)2557 NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
2558 #undef DEFINE_ASM_FUNCS
2559
2560
2561 void Assembler::fcvtzs(const Register& rd,
2562 const VRegister& vn,
2563 int fbits) {
2564 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2565 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
2566 if (fbits == 0) {
2567 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
2568 } else {
2569 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
2570 Rd(rd));
2571 }
2572 }
2573
2574
fcvtzs(const VRegister & vd,const VRegister & vn,int fbits)2575 void Assembler::fcvtzs(const VRegister& vd,
2576 const VRegister& vn,
2577 int fbits) {
2578 VIXL_ASSERT(fbits >= 0);
2579 if (fbits == 0) {
2580 NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
2581 } else {
2582 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2583 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
2584 }
2585 }
2586
2587
fcvtzu(const Register & rd,const VRegister & vn,int fbits)2588 void Assembler::fcvtzu(const Register& rd,
2589 const VRegister& vn,
2590 int fbits) {
2591 VIXL_ASSERT(vn.Is1S() || vn.Is1D());
2592 VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
2593 if (fbits == 0) {
2594 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
2595 } else {
2596 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
2597 Rd(rd));
2598 }
2599 }
2600
2601
fcvtzu(const VRegister & vd,const VRegister & vn,int fbits)2602 void Assembler::fcvtzu(const VRegister& vd,
2603 const VRegister& vn,
2604 int fbits) {
2605 VIXL_ASSERT(fbits >= 0);
2606 if (fbits == 0) {
2607 NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
2608 } else {
2609 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2610 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
2611 }
2612 }
2613
ucvtf(const VRegister & vd,const VRegister & vn,int fbits)2614 void Assembler::ucvtf(const VRegister& vd,
2615 const VRegister& vn,
2616 int fbits) {
2617 VIXL_ASSERT(fbits >= 0);
2618 if (fbits == 0) {
2619 NEONFP2RegMisc(vd, vn, NEON_UCVTF);
2620 } else {
2621 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2622 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
2623 }
2624 }
2625
scvtf(const VRegister & vd,const VRegister & vn,int fbits)2626 void Assembler::scvtf(const VRegister& vd,
2627 const VRegister& vn,
2628 int fbits) {
2629 VIXL_ASSERT(fbits >= 0);
2630 if (fbits == 0) {
2631 NEONFP2RegMisc(vd, vn, NEON_SCVTF);
2632 } else {
2633 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
2634 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
2635 }
2636 }
2637
2638
scvtf(const VRegister & vd,const Register & rn,int fbits)2639 void Assembler::scvtf(const VRegister& vd,
2640 const Register& rn,
2641 int fbits) {
2642 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2643 VIXL_ASSERT(fbits >= 0);
2644 if (fbits == 0) {
2645 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
2646 } else {
2647 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2648 Rd(vd));
2649 }
2650 }
2651
2652
ucvtf(const VRegister & vd,const Register & rn,int fbits)2653 void Assembler::ucvtf(const VRegister& vd,
2654 const Register& rn,
2655 int fbits) {
2656 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2657 VIXL_ASSERT(fbits >= 0);
2658 if (fbits == 0) {
2659 Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
2660 } else {
2661 Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
2662 Rd(vd));
2663 }
2664 }
2665
2666
NEON3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3SameOp vop)2667 void Assembler::NEON3Same(const VRegister& vd,
2668 const VRegister& vn,
2669 const VRegister& vm,
2670 NEON3SameOp vop) {
2671 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2672 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
2673
2674 Instr format, op = vop;
2675 if (vd.IsScalar()) {
2676 op |= NEON_Q | NEONScalar;
2677 format = SFormat(vd);
2678 } else {
2679 format = VFormat(vd);
2680 }
2681
2682 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2683 }
2684
2685
NEONFP3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)2686 void Assembler::NEONFP3Same(const VRegister& vd,
2687 const VRegister& vn,
2688 const VRegister& vm,
2689 Instr op) {
2690 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
2691 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
2692 }
2693
2694
2695 #define NEON_FP2REGMISC_LIST(V) \
2696 V(fabs, NEON_FABS, FABS) \
2697 V(fneg, NEON_FNEG, FNEG) \
2698 V(fsqrt, NEON_FSQRT, FSQRT) \
2699 V(frintn, NEON_FRINTN, FRINTN) \
2700 V(frinta, NEON_FRINTA, FRINTA) \
2701 V(frintp, NEON_FRINTP, FRINTP) \
2702 V(frintm, NEON_FRINTM, FRINTM) \
2703 V(frintx, NEON_FRINTX, FRINTX) \
2704 V(frintz, NEON_FRINTZ, FRINTZ) \
2705 V(frinti, NEON_FRINTI, FRINTI) \
2706 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
2707 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar )
2708
2709
2710 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
2711 void Assembler::FN(const VRegister& vd, \
2712 const VRegister& vn) { \
2713 Instr op; \
2714 if (vd.IsScalar()) { \
2715 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
2716 op = SCA_OP; \
2717 } else { \
2718 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
2719 op = VEC_OP; \
2720 } \
2721 NEONFP2RegMisc(vd, vn, op); \
2722 }
NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)2723 NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
2724 #undef DEFINE_ASM_FUNC
2725
2726
2727 void Assembler::NEONFP2RegMisc(const VRegister& vd,
2728 const VRegister& vn,
2729 Instr op) {
2730 VIXL_ASSERT(AreSameFormat(vd, vn));
2731 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
2732 }
2733
2734
NEON2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,int value)2735 void Assembler::NEON2RegMisc(const VRegister& vd,
2736 const VRegister& vn,
2737 NEON2RegMiscOp vop,
2738 int value) {
2739 VIXL_ASSERT(AreSameFormat(vd, vn));
2740 VIXL_ASSERT(value == 0);
2741 USE(value);
2742
2743 Instr format, op = vop;
2744 if (vd.IsScalar()) {
2745 op |= NEON_Q | NEONScalar;
2746 format = SFormat(vd);
2747 } else {
2748 format = VFormat(vd);
2749 }
2750
2751 Emit(format | op | Rn(vn) | Rd(vd));
2752 }
2753
2754
cmeq(const VRegister & vd,const VRegister & vn,int value)2755 void Assembler::cmeq(const VRegister& vd,
2756 const VRegister& vn,
2757 int value) {
2758 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2759 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
2760 }
2761
2762
cmge(const VRegister & vd,const VRegister & vn,int value)2763 void Assembler::cmge(const VRegister& vd,
2764 const VRegister& vn,
2765 int value) {
2766 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2767 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
2768 }
2769
2770
cmgt(const VRegister & vd,const VRegister & vn,int value)2771 void Assembler::cmgt(const VRegister& vd,
2772 const VRegister& vn,
2773 int value) {
2774 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2775 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
2776 }
2777
2778
cmle(const VRegister & vd,const VRegister & vn,int value)2779 void Assembler::cmle(const VRegister& vd,
2780 const VRegister& vn,
2781 int value) {
2782 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2783 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
2784 }
2785
2786
cmlt(const VRegister & vd,const VRegister & vn,int value)2787 void Assembler::cmlt(const VRegister& vd,
2788 const VRegister& vn,
2789 int value) {
2790 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
2791 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
2792 }
2793
2794
shll(const VRegister & vd,const VRegister & vn,int shift)2795 void Assembler::shll(const VRegister& vd,
2796 const VRegister& vn,
2797 int shift) {
2798 VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
2799 (vd.Is4S() && vn.Is4H() && shift == 16) ||
2800 (vd.Is2D() && vn.Is2S() && shift == 32));
2801 USE(shift);
2802 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
2803 }
2804
2805
shll2(const VRegister & vd,const VRegister & vn,int shift)2806 void Assembler::shll2(const VRegister& vd,
2807 const VRegister& vn,
2808 int shift) {
2809 USE(shift);
2810 VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
2811 (vd.Is4S() && vn.Is8H() && shift == 16) ||
2812 (vd.Is2D() && vn.Is4S() && shift == 32));
2813 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
2814 }
2815
2816
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,double value)2817 void Assembler::NEONFP2RegMisc(const VRegister& vd,
2818 const VRegister& vn,
2819 NEON2RegMiscOp vop,
2820 double value) {
2821 VIXL_ASSERT(AreSameFormat(vd, vn));
2822 VIXL_ASSERT(value == 0.0);
2823 USE(value);
2824
2825 Instr op = vop;
2826 if (vd.IsScalar()) {
2827 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2828 op |= NEON_Q | NEONScalar;
2829 } else {
2830 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
2831 }
2832
2833 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
2834 }
2835
2836
fcmeq(const VRegister & vd,const VRegister & vn,double value)2837 void Assembler::fcmeq(const VRegister& vd,
2838 const VRegister& vn,
2839 double value) {
2840 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
2841 }
2842
2843
fcmge(const VRegister & vd,const VRegister & vn,double value)2844 void Assembler::fcmge(const VRegister& vd,
2845 const VRegister& vn,
2846 double value) {
2847 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
2848 }
2849
2850
fcmgt(const VRegister & vd,const VRegister & vn,double value)2851 void Assembler::fcmgt(const VRegister& vd,
2852 const VRegister& vn,
2853 double value) {
2854 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
2855 }
2856
2857
fcmle(const VRegister & vd,const VRegister & vn,double value)2858 void Assembler::fcmle(const VRegister& vd,
2859 const VRegister& vn,
2860 double value) {
2861 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
2862 }
2863
2864
fcmlt(const VRegister & vd,const VRegister & vn,double value)2865 void Assembler::fcmlt(const VRegister& vd,
2866 const VRegister& vn,
2867 double value) {
2868 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
2869 }
2870
2871
frecpx(const VRegister & vd,const VRegister & vn)2872 void Assembler::frecpx(const VRegister& vd,
2873 const VRegister& vn) {
2874 VIXL_ASSERT(vd.IsScalar());
2875 VIXL_ASSERT(AreSameFormat(vd, vn));
2876 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
2877 Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
2878 }
2879
2880
2881 #define NEON_3SAME_LIST(V) \
2882 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
2883 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
2884 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
2885 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
2886 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
2887 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
2888 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
2889 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
2890 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
2891 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
2892 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
2893 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
2894 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
2895 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
2896 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
2897 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2898 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2899 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2900 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
2901 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
2902 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
2903 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
2904 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
2905 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
2906 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
2907 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
2908 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
2909 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
2910 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
2911 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
2912 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
2913 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
2914 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
2915 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
2916 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
2917 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
2918 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
2919 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
2920 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
2921 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
2922 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
2923 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
2924 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
2925 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
2926 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
2927 V(uqadd, NEON_UQADD, true) \
2928 V(sqadd, NEON_SQADD, true) \
2929 V(uqsub, NEON_UQSUB, true) \
2930 V(sqsub, NEON_SQSUB, true) \
2931 V(sqshl, NEON_SQSHL, true) \
2932 V(uqshl, NEON_UQSHL, true) \
2933 V(sqrshl, NEON_SQRSHL, true) \
2934 V(uqrshl, NEON_UQRSHL, true)
2935
2936 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2937 void Assembler::FN(const VRegister& vd, \
2938 const VRegister& vn, \
2939 const VRegister& vm) { \
2940 VIXL_ASSERT(AS); \
2941 NEON3Same(vd, vn, vm, OP); \
2942 }
2943 NEON_3SAME_LIST(DEFINE_ASM_FUNC)
2944 #undef DEFINE_ASM_FUNC
2945
2946
2947 #define NEON_FP3SAME_OP_LIST(V) \
2948 V(fadd, NEON_FADD, FADD) \
2949 V(fsub, NEON_FSUB, FSUB) \
2950 V(fmul, NEON_FMUL, FMUL) \
2951 V(fdiv, NEON_FDIV, FDIV) \
2952 V(fmax, NEON_FMAX, FMAX) \
2953 V(fmaxnm, NEON_FMAXNM, FMAXNM) \
2954 V(fmin, NEON_FMIN, FMIN) \
2955 V(fminnm, NEON_FMINNM, FMINNM) \
2956 V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \
2957 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \
2958 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \
2959 V(fabd, NEON_FABD, NEON_FABD_scalar) \
2960 V(fmla, NEON_FMLA, 0) \
2961 V(fmls, NEON_FMLS, 0) \
2962 V(facge, NEON_FACGE, NEON_FACGE_scalar) \
2963 V(facgt, NEON_FACGT, NEON_FACGT_scalar) \
2964 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \
2965 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \
2966 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \
2967 V(faddp, NEON_FADDP, 0) \
2968 V(fmaxp, NEON_FMAXP, 0) \
2969 V(fminp, NEON_FMINP, 0) \
2970 V(fmaxnmp, NEON_FMAXNMP, 0) \
2971 V(fminnmp, NEON_FMINNMP, 0)
2972
2973 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
2974 void Assembler::FN(const VRegister& vd, \
2975 const VRegister& vn, \
2976 const VRegister& vm) { \
2977 Instr op; \
2978 if ((SCA_OP != 0) && vd.IsScalar()) { \
2979 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
2980 op = SCA_OP; \
2981 } else { \
2982 VIXL_ASSERT(vd.IsVector()); \
2983 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
2984 op = VEC_OP; \
2985 } \
2986 NEONFP3Same(vd, vn, vm, op); \
2987 }
NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)2988 NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)
2989 #undef DEFINE_ASM_FUNC
2990
2991
2992 void Assembler::addp(const VRegister& vd,
2993 const VRegister& vn) {
2994 VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
2995 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
2996 }
2997
2998
faddp(const VRegister & vd,const VRegister & vn)2999 void Assembler::faddp(const VRegister& vd,
3000 const VRegister& vn) {
3001 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3002 (vd.Is1D() && vn.Is2D()));
3003 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
3004 }
3005
3006
fmaxp(const VRegister & vd,const VRegister & vn)3007 void Assembler::fmaxp(const VRegister& vd,
3008 const VRegister& vn) {
3009 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3010 (vd.Is1D() && vn.Is2D()));
3011 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
3012 }
3013
3014
fminp(const VRegister & vd,const VRegister & vn)3015 void Assembler::fminp(const VRegister& vd,
3016 const VRegister& vn) {
3017 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3018 (vd.Is1D() && vn.Is2D()));
3019 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
3020 }
3021
3022
fmaxnmp(const VRegister & vd,const VRegister & vn)3023 void Assembler::fmaxnmp(const VRegister& vd,
3024 const VRegister& vn) {
3025 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3026 (vd.Is1D() && vn.Is2D()));
3027 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
3028 }
3029
3030
fminnmp(const VRegister & vd,const VRegister & vn)3031 void Assembler::fminnmp(const VRegister& vd,
3032 const VRegister& vn) {
3033 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
3034 (vd.Is1D() && vn.Is2D()));
3035 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
3036 }
3037
3038
orr(const VRegister & vd,const int imm8,const int left_shift)3039 void Assembler::orr(const VRegister& vd,
3040 const int imm8,
3041 const int left_shift) {
3042 NEONModifiedImmShiftLsl(vd, imm8, left_shift,
3043 NEONModifiedImmediate_ORR);
3044 }
3045
3046
mov(const VRegister & vd,const VRegister & vn)3047 void Assembler::mov(const VRegister& vd,
3048 const VRegister& vn) {
3049 VIXL_ASSERT(AreSameFormat(vd, vn));
3050 if (vd.IsD()) {
3051 orr(vd.V8B(), vn.V8B(), vn.V8B());
3052 } else {
3053 VIXL_ASSERT(vd.IsQ());
3054 orr(vd.V16B(), vn.V16B(), vn.V16B());
3055 }
3056 }
3057
3058
bic(const VRegister & vd,const int imm8,const int left_shift)3059 void Assembler::bic(const VRegister& vd,
3060 const int imm8,
3061 const int left_shift) {
3062 NEONModifiedImmShiftLsl(vd, imm8, left_shift,
3063 NEONModifiedImmediate_BIC);
3064 }
3065
3066
movi(const VRegister & vd,const uint64_t imm,Shift shift,const int shift_amount)3067 void Assembler::movi(const VRegister& vd,
3068 const uint64_t imm,
3069 Shift shift,
3070 const int shift_amount) {
3071 VIXL_ASSERT((shift == LSL) || (shift == MSL));
3072 if (vd.Is2D() || vd.Is1D()) {
3073 VIXL_ASSERT(shift_amount == 0);
3074 int imm8 = 0;
3075 for (int i = 0; i < 8; ++i) {
3076 int byte = (imm >> (i * 8)) & 0xff;
3077 VIXL_ASSERT((byte == 0) || (byte == 0xff));
3078 if (byte == 0xff) {
3079 imm8 |= (1 << i);
3080 }
3081 }
3082 int q = vd.Is2D() ? NEON_Q : 0;
3083 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
3084 ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
3085 } else if (shift == LSL) {
3086 VIXL_ASSERT(is_uint8(imm));
3087 NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount,
3088 NEONModifiedImmediate_MOVI);
3089 } else {
3090 VIXL_ASSERT(is_uint8(imm));
3091 NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount,
3092 NEONModifiedImmediate_MOVI);
3093 }
3094 }
3095
3096
mvn(const VRegister & vd,const VRegister & vn)3097 void Assembler::mvn(const VRegister& vd,
3098 const VRegister& vn) {
3099 VIXL_ASSERT(AreSameFormat(vd, vn));
3100 if (vd.IsD()) {
3101 not_(vd.V8B(), vn.V8B());
3102 } else {
3103 VIXL_ASSERT(vd.IsQ());
3104 not_(vd.V16B(), vn.V16B());
3105 }
3106 }
3107
3108
mvni(const VRegister & vd,const int imm8,Shift shift,const int shift_amount)3109 void Assembler::mvni(const VRegister& vd,
3110 const int imm8,
3111 Shift shift,
3112 const int shift_amount) {
3113 VIXL_ASSERT((shift == LSL) || (shift == MSL));
3114 if (shift == LSL) {
3115 NEONModifiedImmShiftLsl(vd, imm8, shift_amount,
3116 NEONModifiedImmediate_MVNI);
3117 } else {
3118 NEONModifiedImmShiftMsl(vd, imm8, shift_amount,
3119 NEONModifiedImmediate_MVNI);
3120 }
3121 }
3122
3123
NEONFPByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3124 void Assembler::NEONFPByElement(const VRegister& vd,
3125 const VRegister& vn,
3126 const VRegister& vm,
3127 int vm_index,
3128 NEONByIndexedElementOp vop) {
3129 VIXL_ASSERT(AreSameFormat(vd, vn));
3130 VIXL_ASSERT((vd.Is2S() && vm.Is1S()) ||
3131 (vd.Is4S() && vm.Is1S()) ||
3132 (vd.Is1S() && vm.Is1S()) ||
3133 (vd.Is2D() && vm.Is1D()) ||
3134 (vd.Is1D() && vm.Is1D()));
3135 VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) ||
3136 (vm.Is1D() && (vm_index < 2)));
3137
3138 Instr op = vop;
3139 int index_num_bits = vm.Is1S() ? 2 : 1;
3140 if (vd.IsScalar()) {
3141 op |= NEON_Q | NEONScalar;
3142 }
3143
3144 Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) |
3145 Rm(vm) | Rn(vn) | Rd(vd));
3146 }
3147
3148
NEONByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3149 void Assembler::NEONByElement(const VRegister& vd,
3150 const VRegister& vn,
3151 const VRegister& vm,
3152 int vm_index,
3153 NEONByIndexedElementOp vop) {
3154 VIXL_ASSERT(AreSameFormat(vd, vn));
3155 VIXL_ASSERT((vd.Is4H() && vm.Is1H()) ||
3156 (vd.Is8H() && vm.Is1H()) ||
3157 (vd.Is1H() && vm.Is1H()) ||
3158 (vd.Is2S() && vm.Is1S()) ||
3159 (vd.Is4S() && vm.Is1S()) ||
3160 (vd.Is1S() && vm.Is1S()));
3161 VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3162 (vm.Is1S() && (vm_index < 4)));
3163
3164 Instr format, op = vop;
3165 int index_num_bits = vm.Is1H() ? 3 : 2;
3166 if (vd.IsScalar()) {
3167 op |= NEONScalar | NEON_Q;
3168 format = SFormat(vn);
3169 } else {
3170 format = VFormat(vn);
3171 }
3172 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
3173 Rm(vm) | Rn(vn) | Rd(vd));
3174 }
3175
3176
NEONByElementL(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3177 void Assembler::NEONByElementL(const VRegister& vd,
3178 const VRegister& vn,
3179 const VRegister& vm,
3180 int vm_index,
3181 NEONByIndexedElementOp vop) {
3182 VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
3183 (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
3184 (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
3185 (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
3186 (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
3187 (vd.Is1D() && vn.Is1S() && vm.Is1S()));
3188
3189 VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3190 (vm.Is1S() && (vm_index < 4)));
3191
3192 Instr format, op = vop;
3193 int index_num_bits = vm.Is1H() ? 3 : 2;
3194 if (vd.IsScalar()) {
3195 op |= NEONScalar | NEON_Q;
3196 format = SFormat(vn);
3197 } else {
3198 format = VFormat(vn);
3199 }
3200 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
3201 Rm(vm) | Rn(vn) | Rd(vd));
3202 }
3203
3204
3205 #define NEON_BYELEMENT_LIST(V) \
3206 V(mul, NEON_MUL_byelement, vn.IsVector()) \
3207 V(mla, NEON_MLA_byelement, vn.IsVector()) \
3208 V(mls, NEON_MLS_byelement, vn.IsVector()) \
3209 V(sqdmulh, NEON_SQDMULH_byelement, true) \
3210 V(sqrdmulh, NEON_SQRDMULH_byelement, true)
3211
3212
3213 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3214 void Assembler::FN(const VRegister& vd, \
3215 const VRegister& vn, \
3216 const VRegister& vm, \
3217 int vm_index) { \
3218 VIXL_ASSERT(AS); \
3219 NEONByElement(vd, vn, vm, vm_index, OP); \
3220 }
3221 NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
3222 #undef DEFINE_ASM_FUNC
3223
3224
3225 #define NEON_FPBYELEMENT_LIST(V) \
3226 V(fmul, NEON_FMUL_byelement) \
3227 V(fmla, NEON_FMLA_byelement) \
3228 V(fmls, NEON_FMLS_byelement) \
3229 V(fmulx, NEON_FMULX_byelement)
3230
3231
3232 #define DEFINE_ASM_FUNC(FN, OP) \
3233 void Assembler::FN(const VRegister& vd, \
3234 const VRegister& vn, \
3235 const VRegister& vm, \
3236 int vm_index) { \
3237 NEONFPByElement(vd, vn, vm, vm_index, OP); \
3238 }
NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)3239 NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
3240 #undef DEFINE_ASM_FUNC
3241
3242
3243 #define NEON_BYELEMENT_LONG_LIST(V) \
3244 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
3245 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
3246 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
3247 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3248 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
3249 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
3250 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
3251 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
3252 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
3253 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
3254 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
3255 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3256 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
3257 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3258 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
3259 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
3260 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
3261 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
3262
3263
3264 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3265 void Assembler::FN(const VRegister& vd, \
3266 const VRegister& vn, \
3267 const VRegister& vm, \
3268 int vm_index) { \
3269 VIXL_ASSERT(AS); \
3270 NEONByElementL(vd, vn, vm, vm_index, OP); \
3271 }
3272 NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
3273 #undef DEFINE_ASM_FUNC
3274
3275
3276 void Assembler::suqadd(const VRegister& vd,
3277 const VRegister& vn) {
3278 NEON2RegMisc(vd, vn, NEON_SUQADD);
3279 }
3280
3281
usqadd(const VRegister & vd,const VRegister & vn)3282 void Assembler::usqadd(const VRegister& vd,
3283 const VRegister& vn) {
3284 NEON2RegMisc(vd, vn, NEON_USQADD);
3285 }
3286
3287
abs(const VRegister & vd,const VRegister & vn)3288 void Assembler::abs(const VRegister& vd,
3289 const VRegister& vn) {
3290 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3291 NEON2RegMisc(vd, vn, NEON_ABS);
3292 }
3293
3294
sqabs(const VRegister & vd,const VRegister & vn)3295 void Assembler::sqabs(const VRegister& vd,
3296 const VRegister& vn) {
3297 NEON2RegMisc(vd, vn, NEON_SQABS);
3298 }
3299
3300
neg(const VRegister & vd,const VRegister & vn)3301 void Assembler::neg(const VRegister& vd,
3302 const VRegister& vn) {
3303 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3304 NEON2RegMisc(vd, vn, NEON_NEG);
3305 }
3306
3307
sqneg(const VRegister & vd,const VRegister & vn)3308 void Assembler::sqneg(const VRegister& vd,
3309 const VRegister& vn) {
3310 NEON2RegMisc(vd, vn, NEON_SQNEG);
3311 }
3312
3313
NEONXtn(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop)3314 void Assembler::NEONXtn(const VRegister& vd,
3315 const VRegister& vn,
3316 NEON2RegMiscOp vop) {
3317 Instr format, op = vop;
3318 if (vd.IsScalar()) {
3319 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
3320 (vd.Is1H() && vn.Is1S()) ||
3321 (vd.Is1S() && vn.Is1D()));
3322 op |= NEON_Q | NEONScalar;
3323 format = SFormat(vd);
3324 } else {
3325 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
3326 (vd.Is4H() && vn.Is4S()) ||
3327 (vd.Is2S() && vn.Is2D()) ||
3328 (vd.Is16B() && vn.Is8H()) ||
3329 (vd.Is8H() && vn.Is4S()) ||
3330 (vd.Is4S() && vn.Is2D()));
3331 format = VFormat(vd);
3332 }
3333 Emit(format | op | Rn(vn) | Rd(vd));
3334 }
3335
3336
xtn(const VRegister & vd,const VRegister & vn)3337 void Assembler::xtn(const VRegister& vd,
3338 const VRegister& vn) {
3339 VIXL_ASSERT(vd.IsVector() && vd.IsD());
3340 NEONXtn(vd, vn, NEON_XTN);
3341 }
3342
3343
xtn2(const VRegister & vd,const VRegister & vn)3344 void Assembler::xtn2(const VRegister& vd,
3345 const VRegister& vn) {
3346 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3347 NEONXtn(vd, vn, NEON_XTN);
3348 }
3349
3350
sqxtn(const VRegister & vd,const VRegister & vn)3351 void Assembler::sqxtn(const VRegister& vd,
3352 const VRegister& vn) {
3353 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3354 NEONXtn(vd, vn, NEON_SQXTN);
3355 }
3356
3357
sqxtn2(const VRegister & vd,const VRegister & vn)3358 void Assembler::sqxtn2(const VRegister& vd,
3359 const VRegister& vn) {
3360 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3361 NEONXtn(vd, vn, NEON_SQXTN);
3362 }
3363
3364
sqxtun(const VRegister & vd,const VRegister & vn)3365 void Assembler::sqxtun(const VRegister& vd,
3366 const VRegister& vn) {
3367 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3368 NEONXtn(vd, vn, NEON_SQXTUN);
3369 }
3370
3371
sqxtun2(const VRegister & vd,const VRegister & vn)3372 void Assembler::sqxtun2(const VRegister& vd,
3373 const VRegister& vn) {
3374 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3375 NEONXtn(vd, vn, NEON_SQXTUN);
3376 }
3377
3378
uqxtn(const VRegister & vd,const VRegister & vn)3379 void Assembler::uqxtn(const VRegister& vd,
3380 const VRegister& vn) {
3381 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
3382 NEONXtn(vd, vn, NEON_UQXTN);
3383 }
3384
3385
uqxtn2(const VRegister & vd,const VRegister & vn)3386 void Assembler::uqxtn2(const VRegister& vd,
3387 const VRegister& vn) {
3388 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
3389 NEONXtn(vd, vn, NEON_UQXTN);
3390 }
3391
3392
3393 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
not_(const VRegister & vd,const VRegister & vn)3394 void Assembler::not_(const VRegister& vd,
3395 const VRegister& vn) {
3396 VIXL_ASSERT(AreSameFormat(vd, vn));
3397 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3398 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3399 }
3400
3401
rbit(const VRegister & vd,const VRegister & vn)3402 void Assembler::rbit(const VRegister& vd,
3403 const VRegister& vn) {
3404 VIXL_ASSERT(AreSameFormat(vd, vn));
3405 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3406 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3407 }
3408
3409
ext(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)3410 void Assembler::ext(const VRegister& vd,
3411 const VRegister& vn,
3412 const VRegister& vm,
3413 int index) {
3414 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3415 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3416 VIXL_ASSERT((0 <= index) && (index < vd.lanes()));
3417 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
3418 }
3419
3420
dup(const VRegister & vd,const VRegister & vn,int vn_index)3421 void Assembler::dup(const VRegister& vd,
3422 const VRegister& vn,
3423 int vn_index) {
3424 Instr q, scalar;
3425
3426 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3427 // number of lanes, and T is b, h, s or d.
3428 int lane_size = vn.LaneSizeInBytes();
3429 NEONFormatField format;
3430 switch (lane_size) {
3431 case 1: format = NEON_16B; break;
3432 case 2: format = NEON_8H; break;
3433 case 4: format = NEON_4S; break;
3434 default:
3435 VIXL_ASSERT(lane_size == 8);
3436 format = NEON_2D;
3437 break;
3438 }
3439
3440 if (vd.IsScalar()) {
3441 q = NEON_Q;
3442 scalar = NEONScalar;
3443 } else {
3444 VIXL_ASSERT(!vd.Is1D());
3445 q = vd.IsD() ? 0 : NEON_Q;
3446 scalar = 0;
3447 }
3448 Emit(q | scalar | NEON_DUP_ELEMENT |
3449 ImmNEON5(format, vn_index) | Rn(vn) | Rd(vd));
3450 }
3451
3452
mov(const VRegister & vd,const VRegister & vn,int vn_index)3453 void Assembler::mov(const VRegister& vd,
3454 const VRegister& vn,
3455 int vn_index) {
3456 VIXL_ASSERT(vn.IsScalar());
3457 dup(vd, vn, vn_index);
3458 }
3459
3460
dup(const VRegister & vd,const Register & rn)3461 void Assembler::dup(const VRegister& vd, const Register& rn) {
3462 VIXL_ASSERT(!vd.Is1D());
3463 VIXL_ASSERT(vd.Is2D() == rn.IsX());
3464 int q = vd.IsD() ? 0 : NEON_Q;
3465 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
3466 }
3467
3468
ins(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)3469 void Assembler::ins(const VRegister& vd,
3470 int vd_index,
3471 const VRegister& vn,
3472 int vn_index) {
3473 VIXL_ASSERT(AreSameFormat(vd, vn));
3474 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3475 // number of lanes, and T is b, h, s or d.
3476 int lane_size = vd.LaneSizeInBytes();
3477 NEONFormatField format;
3478 switch (lane_size) {
3479 case 1: format = NEON_16B; break;
3480 case 2: format = NEON_8H; break;
3481 case 4: format = NEON_4S; break;
3482 default:
3483 VIXL_ASSERT(lane_size == 8);
3484 format = NEON_2D;
3485 break;
3486 }
3487
3488 VIXL_ASSERT((0 <= vd_index) &&
3489 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3490 VIXL_ASSERT((0 <= vn_index) &&
3491 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3492 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
3493 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
3494 }
3495
3496
mov(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)3497 void Assembler::mov(const VRegister& vd,
3498 int vd_index,
3499 const VRegister& vn,
3500 int vn_index) {
3501 ins(vd, vd_index, vn, vn_index);
3502 }
3503
3504
ins(const VRegister & vd,int vd_index,const Register & rn)3505 void Assembler::ins(const VRegister& vd,
3506 int vd_index,
3507 const Register& rn) {
3508 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3509 // number of lanes, and T is b, h, s or d.
3510 int lane_size = vd.LaneSizeInBytes();
3511 NEONFormatField format;
3512 switch (lane_size) {
3513 case 1: format = NEON_16B; VIXL_ASSERT(rn.IsW()); break;
3514 case 2: format = NEON_8H; VIXL_ASSERT(rn.IsW()); break;
3515 case 4: format = NEON_4S; VIXL_ASSERT(rn.IsW()); break;
3516 default:
3517 VIXL_ASSERT(lane_size == 8);
3518 VIXL_ASSERT(rn.IsX());
3519 format = NEON_2D;
3520 break;
3521 }
3522
3523 VIXL_ASSERT((0 <= vd_index) &&
3524 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3525 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
3526 }
3527
3528
mov(const VRegister & vd,int vd_index,const Register & rn)3529 void Assembler::mov(const VRegister& vd,
3530 int vd_index,
3531 const Register& rn) {
3532 ins(vd, vd_index, rn);
3533 }
3534
3535
umov(const Register & rd,const VRegister & vn,int vn_index)3536 void Assembler::umov(const Register& rd,
3537 const VRegister& vn,
3538 int vn_index) {
3539 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3540 // number of lanes, and T is b, h, s or d.
3541 int lane_size = vn.LaneSizeInBytes();
3542 NEONFormatField format;
3543 Instr q = 0;
3544 switch (lane_size) {
3545 case 1: format = NEON_16B; VIXL_ASSERT(rd.IsW()); break;
3546 case 2: format = NEON_8H; VIXL_ASSERT(rd.IsW()); break;
3547 case 4: format = NEON_4S; VIXL_ASSERT(rd.IsW()); break;
3548 default:
3549 VIXL_ASSERT(lane_size == 8);
3550 VIXL_ASSERT(rd.IsX());
3551 format = NEON_2D;
3552 q = NEON_Q;
3553 break;
3554 }
3555
3556 VIXL_ASSERT((0 <= vn_index) &&
3557 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3558 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3559 }
3560
3561
mov(const Register & rd,const VRegister & vn,int vn_index)3562 void Assembler::mov(const Register& rd,
3563 const VRegister& vn,
3564 int vn_index) {
3565 VIXL_ASSERT(vn.SizeInBytes() >= 4);
3566 umov(rd, vn, vn_index);
3567 }
3568
3569
smov(const Register & rd,const VRegister & vn,int vn_index)3570 void Assembler::smov(const Register& rd,
3571 const VRegister& vn,
3572 int vn_index) {
3573 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
3574 // number of lanes, and T is b, h, s.
3575 int lane_size = vn.LaneSizeInBytes();
3576 NEONFormatField format;
3577 Instr q = 0;
3578 VIXL_ASSERT(lane_size != 8);
3579 switch (lane_size) {
3580 case 1: format = NEON_16B; break;
3581 case 2: format = NEON_8H; break;
3582 default:
3583 VIXL_ASSERT(lane_size == 4);
3584 VIXL_ASSERT(rd.IsX());
3585 format = NEON_4S;
3586 break;
3587 }
3588 q = rd.IsW() ? 0 : NEON_Q;
3589 VIXL_ASSERT((0 <= vn_index) &&
3590 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
3591 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
3592 }
3593
3594
cls(const VRegister & vd,const VRegister & vn)3595 void Assembler::cls(const VRegister& vd,
3596 const VRegister& vn) {
3597 VIXL_ASSERT(AreSameFormat(vd, vn));
3598 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3599 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
3600 }
3601
3602
clz(const VRegister & vd,const VRegister & vn)3603 void Assembler::clz(const VRegister& vd,
3604 const VRegister& vn) {
3605 VIXL_ASSERT(AreSameFormat(vd, vn));
3606 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3607 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
3608 }
3609
3610
cnt(const VRegister & vd,const VRegister & vn)3611 void Assembler::cnt(const VRegister& vd,
3612 const VRegister& vn) {
3613 VIXL_ASSERT(AreSameFormat(vd, vn));
3614 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3615 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
3616 }
3617
3618
rev16(const VRegister & vd,const VRegister & vn)3619 void Assembler::rev16(const VRegister& vd,
3620 const VRegister& vn) {
3621 VIXL_ASSERT(AreSameFormat(vd, vn));
3622 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
3623 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
3624 }
3625
3626
rev32(const VRegister & vd,const VRegister & vn)3627 void Assembler::rev32(const VRegister& vd,
3628 const VRegister& vn) {
3629 VIXL_ASSERT(AreSameFormat(vd, vn));
3630 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
3631 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
3632 }
3633
3634
rev64(const VRegister & vd,const VRegister & vn)3635 void Assembler::rev64(const VRegister& vd,
3636 const VRegister& vn) {
3637 VIXL_ASSERT(AreSameFormat(vd, vn));
3638 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
3639 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
3640 }
3641
3642
ursqrte(const VRegister & vd,const VRegister & vn)3643 void Assembler::ursqrte(const VRegister& vd,
3644 const VRegister& vn) {
3645 VIXL_ASSERT(AreSameFormat(vd, vn));
3646 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3647 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
3648 }
3649
3650
urecpe(const VRegister & vd,const VRegister & vn)3651 void Assembler::urecpe(const VRegister& vd,
3652 const VRegister& vn) {
3653 VIXL_ASSERT(AreSameFormat(vd, vn));
3654 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3655 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
3656 }
3657
3658
NEONAddlp(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp op)3659 void Assembler::NEONAddlp(const VRegister& vd,
3660 const VRegister& vn,
3661 NEON2RegMiscOp op) {
3662 VIXL_ASSERT((op == NEON_SADDLP) ||
3663 (op == NEON_UADDLP) ||
3664 (op == NEON_SADALP) ||
3665 (op == NEON_UADALP));
3666
3667 VIXL_ASSERT((vn.Is8B() && vd.Is4H()) ||
3668 (vn.Is4H() && vd.Is2S()) ||
3669 (vn.Is2S() && vd.Is1D()) ||
3670 (vn.Is16B() && vd.Is8H())||
3671 (vn.Is8H() && vd.Is4S()) ||
3672 (vn.Is4S() && vd.Is2D()));
3673 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3674 }
3675
3676
saddlp(const VRegister & vd,const VRegister & vn)3677 void Assembler::saddlp(const VRegister& vd,
3678 const VRegister& vn) {
3679 NEONAddlp(vd, vn, NEON_SADDLP);
3680 }
3681
3682
uaddlp(const VRegister & vd,const VRegister & vn)3683 void Assembler::uaddlp(const VRegister& vd,
3684 const VRegister& vn) {
3685 NEONAddlp(vd, vn, NEON_UADDLP);
3686 }
3687
3688
sadalp(const VRegister & vd,const VRegister & vn)3689 void Assembler::sadalp(const VRegister& vd,
3690 const VRegister& vn) {
3691 NEONAddlp(vd, vn, NEON_SADALP);
3692 }
3693
3694
uadalp(const VRegister & vd,const VRegister & vn)3695 void Assembler::uadalp(const VRegister& vd,
3696 const VRegister& vn) {
3697 NEONAddlp(vd, vn, NEON_UADALP);
3698 }
3699
3700
NEONAcrossLanesL(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)3701 void Assembler::NEONAcrossLanesL(const VRegister& vd,
3702 const VRegister& vn,
3703 NEONAcrossLanesOp op) {
3704 VIXL_ASSERT((vn.Is8B() && vd.Is1H()) ||
3705 (vn.Is16B() && vd.Is1H()) ||
3706 (vn.Is4H() && vd.Is1S()) ||
3707 (vn.Is8H() && vd.Is1S()) ||
3708 (vn.Is4S() && vd.Is1D()));
3709 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3710 }
3711
3712
saddlv(const VRegister & vd,const VRegister & vn)3713 void Assembler::saddlv(const VRegister& vd,
3714 const VRegister& vn) {
3715 NEONAcrossLanesL(vd, vn, NEON_SADDLV);
3716 }
3717
3718
uaddlv(const VRegister & vd,const VRegister & vn)3719 void Assembler::uaddlv(const VRegister& vd,
3720 const VRegister& vn) {
3721 NEONAcrossLanesL(vd, vn, NEON_UADDLV);
3722 }
3723
3724
NEONAcrossLanes(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)3725 void Assembler::NEONAcrossLanes(const VRegister& vd,
3726 const VRegister& vn,
3727 NEONAcrossLanesOp op) {
3728 VIXL_ASSERT((vn.Is8B() && vd.Is1B()) ||
3729 (vn.Is16B() && vd.Is1B()) ||
3730 (vn.Is4H() && vd.Is1H()) ||
3731 (vn.Is8H() && vd.Is1H()) ||
3732 (vn.Is4S() && vd.Is1S()));
3733 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
3734 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3735 } else {
3736 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
3737 }
3738 }
3739
3740
3741 #define NEON_ACROSSLANES_LIST(V) \
3742 V(fmaxv, NEON_FMAXV, vd.Is1S()) \
3743 V(fminv, NEON_FMINV, vd.Is1S()) \
3744 V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
3745 V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
3746 V(addv, NEON_ADDV, true) \
3747 V(smaxv, NEON_SMAXV, true) \
3748 V(sminv, NEON_SMINV, true) \
3749 V(umaxv, NEON_UMAXV, true) \
3750 V(uminv, NEON_UMINV, true)
3751
3752
3753 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3754 void Assembler::FN(const VRegister& vd, \
3755 const VRegister& vn) { \
3756 VIXL_ASSERT(AS); \
3757 NEONAcrossLanes(vd, vn, OP); \
3758 }
NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)3759 NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
3760 #undef DEFINE_ASM_FUNC
3761
3762
3763 void Assembler::NEONPerm(const VRegister& vd,
3764 const VRegister& vn,
3765 const VRegister& vm,
3766 NEONPermOp op) {
3767 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3768 VIXL_ASSERT(!vd.Is1D());
3769 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3770 }
3771
3772
trn1(const VRegister & vd,const VRegister & vn,const VRegister & vm)3773 void Assembler::trn1(const VRegister& vd,
3774 const VRegister& vn,
3775 const VRegister& vm) {
3776 NEONPerm(vd, vn, vm, NEON_TRN1);
3777 }
3778
3779
trn2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3780 void Assembler::trn2(const VRegister& vd,
3781 const VRegister& vn,
3782 const VRegister& vm) {
3783 NEONPerm(vd, vn, vm, NEON_TRN2);
3784 }
3785
3786
uzp1(const VRegister & vd,const VRegister & vn,const VRegister & vm)3787 void Assembler::uzp1(const VRegister& vd,
3788 const VRegister& vn,
3789 const VRegister& vm) {
3790 NEONPerm(vd, vn, vm, NEON_UZP1);
3791 }
3792
3793
uzp2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3794 void Assembler::uzp2(const VRegister& vd,
3795 const VRegister& vn,
3796 const VRegister& vm) {
3797 NEONPerm(vd, vn, vm, NEON_UZP2);
3798 }
3799
3800
zip1(const VRegister & vd,const VRegister & vn,const VRegister & vm)3801 void Assembler::zip1(const VRegister& vd,
3802 const VRegister& vn,
3803 const VRegister& vm) {
3804 NEONPerm(vd, vn, vm, NEON_ZIP1);
3805 }
3806
3807
zip2(const VRegister & vd,const VRegister & vn,const VRegister & vm)3808 void Assembler::zip2(const VRegister& vd,
3809 const VRegister& vn,
3810 const VRegister& vm) {
3811 NEONPerm(vd, vn, vm, NEON_ZIP2);
3812 }
3813
3814
NEONShiftImmediate(const VRegister & vd,const VRegister & vn,NEONShiftImmediateOp op,int immh_immb)3815 void Assembler::NEONShiftImmediate(const VRegister& vd,
3816 const VRegister& vn,
3817 NEONShiftImmediateOp op,
3818 int immh_immb) {
3819 VIXL_ASSERT(AreSameFormat(vd, vn));
3820 Instr q, scalar;
3821 if (vn.IsScalar()) {
3822 q = NEON_Q;
3823 scalar = NEONScalar;
3824 } else {
3825 q = vd.IsD() ? 0 : NEON_Q;
3826 scalar = 0;
3827 }
3828 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
3829 }
3830
3831
NEONShiftLeftImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3832 void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
3833 const VRegister& vn,
3834 int shift,
3835 NEONShiftImmediateOp op) {
3836 int laneSizeInBits = vn.LaneSizeInBits();
3837 VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
3838 NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
3839 }
3840
3841
NEONShiftRightImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3842 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
3843 const VRegister& vn,
3844 int shift,
3845 NEONShiftImmediateOp op) {
3846 int laneSizeInBits = vn.LaneSizeInBits();
3847 VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
3848 NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
3849 }
3850
3851
NEONShiftImmediateL(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3852 void Assembler::NEONShiftImmediateL(const VRegister& vd,
3853 const VRegister& vn,
3854 int shift,
3855 NEONShiftImmediateOp op) {
3856 int laneSizeInBits = vn.LaneSizeInBits();
3857 VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
3858 int immh_immb = (laneSizeInBits + shift) << 16;
3859
3860 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) ||
3861 (vn.Is4H() && vd.Is4S()) ||
3862 (vn.Is2S() && vd.Is2D()) ||
3863 (vn.Is16B() && vd.Is8H())||
3864 (vn.Is8H() && vd.Is4S()) ||
3865 (vn.Is4S() && vd.Is2D()));
3866 Instr q;
3867 q = vn.IsD() ? 0 : NEON_Q;
3868 Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
3869 }
3870
3871
NEONShiftImmediateN(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)3872 void Assembler::NEONShiftImmediateN(const VRegister& vd,
3873 const VRegister& vn,
3874 int shift,
3875 NEONShiftImmediateOp op) {
3876 Instr q, scalar;
3877 int laneSizeInBits = vd.LaneSizeInBits();
3878 VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
3879 int immh_immb = (2 * laneSizeInBits - shift) << 16;
3880
3881 if (vn.IsScalar()) {
3882 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
3883 (vd.Is1H() && vn.Is1S()) ||
3884 (vd.Is1S() && vn.Is1D()));
3885 q = NEON_Q;
3886 scalar = NEONScalar;
3887 } else {
3888 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
3889 (vd.Is4H() && vn.Is4S()) ||
3890 (vd.Is2S() && vn.Is2D()) ||
3891 (vd.Is16B() && vn.Is8H())||
3892 (vd.Is8H() && vn.Is4S()) ||
3893 (vd.Is4S() && vn.Is2D()));
3894 scalar = 0;
3895 q = vd.IsD() ? 0 : NEON_Q;
3896 }
3897 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
3898 }
3899
3900
shl(const VRegister & vd,const VRegister & vn,int shift)3901 void Assembler::shl(const VRegister& vd,
3902 const VRegister& vn,
3903 int shift) {
3904 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3905 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
3906 }
3907
3908
sli(const VRegister & vd,const VRegister & vn,int shift)3909 void Assembler::sli(const VRegister& vd,
3910 const VRegister& vn,
3911 int shift) {
3912 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3913 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
3914 }
3915
3916
sqshl(const VRegister & vd,const VRegister & vn,int shift)3917 void Assembler::sqshl(const VRegister& vd,
3918 const VRegister& vn,
3919 int shift) {
3920 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
3921 }
3922
3923
sqshlu(const VRegister & vd,const VRegister & vn,int shift)3924 void Assembler::sqshlu(const VRegister& vd,
3925 const VRegister& vn,
3926 int shift) {
3927 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
3928 }
3929
3930
uqshl(const VRegister & vd,const VRegister & vn,int shift)3931 void Assembler::uqshl(const VRegister& vd,
3932 const VRegister& vn,
3933 int shift) {
3934 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
3935 }
3936
3937
sshll(const VRegister & vd,const VRegister & vn,int shift)3938 void Assembler::sshll(const VRegister& vd,
3939 const VRegister& vn,
3940 int shift) {
3941 VIXL_ASSERT(vn.IsD());
3942 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
3943 }
3944
3945
sshll2(const VRegister & vd,const VRegister & vn,int shift)3946 void Assembler::sshll2(const VRegister& vd,
3947 const VRegister& vn,
3948 int shift) {
3949 VIXL_ASSERT(vn.IsQ());
3950 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
3951 }
3952
3953
sxtl(const VRegister & vd,const VRegister & vn)3954 void Assembler::sxtl(const VRegister& vd,
3955 const VRegister& vn) {
3956 sshll(vd, vn, 0);
3957 }
3958
3959
sxtl2(const VRegister & vd,const VRegister & vn)3960 void Assembler::sxtl2(const VRegister& vd,
3961 const VRegister& vn) {
3962 sshll2(vd, vn, 0);
3963 }
3964
3965
ushll(const VRegister & vd,const VRegister & vn,int shift)3966 void Assembler::ushll(const VRegister& vd,
3967 const VRegister& vn,
3968 int shift) {
3969 VIXL_ASSERT(vn.IsD());
3970 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
3971 }
3972
3973
ushll2(const VRegister & vd,const VRegister & vn,int shift)3974 void Assembler::ushll2(const VRegister& vd,
3975 const VRegister& vn,
3976 int shift) {
3977 VIXL_ASSERT(vn.IsQ());
3978 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
3979 }
3980
3981
uxtl(const VRegister & vd,const VRegister & vn)3982 void Assembler::uxtl(const VRegister& vd,
3983 const VRegister& vn) {
3984 ushll(vd, vn, 0);
3985 }
3986
3987
uxtl2(const VRegister & vd,const VRegister & vn)3988 void Assembler::uxtl2(const VRegister& vd,
3989 const VRegister& vn) {
3990 ushll2(vd, vn, 0);
3991 }
3992
3993
sri(const VRegister & vd,const VRegister & vn,int shift)3994 void Assembler::sri(const VRegister& vd,
3995 const VRegister& vn,
3996 int shift) {
3997 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3998 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
3999 }
4000
4001
sshr(const VRegister & vd,const VRegister & vn,int shift)4002 void Assembler::sshr(const VRegister& vd,
4003 const VRegister& vn,
4004 int shift) {
4005 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4006 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
4007 }
4008
4009
ushr(const VRegister & vd,const VRegister & vn,int shift)4010 void Assembler::ushr(const VRegister& vd,
4011 const VRegister& vn,
4012 int shift) {
4013 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4014 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
4015 }
4016
4017
srshr(const VRegister & vd,const VRegister & vn,int shift)4018 void Assembler::srshr(const VRegister& vd,
4019 const VRegister& vn,
4020 int shift) {
4021 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4022 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
4023 }
4024
4025
urshr(const VRegister & vd,const VRegister & vn,int shift)4026 void Assembler::urshr(const VRegister& vd,
4027 const VRegister& vn,
4028 int shift) {
4029 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4030 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
4031 }
4032
4033
ssra(const VRegister & vd,const VRegister & vn,int shift)4034 void Assembler::ssra(const VRegister& vd,
4035 const VRegister& vn,
4036 int shift) {
4037 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4038 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
4039 }
4040
4041
usra(const VRegister & vd,const VRegister & vn,int shift)4042 void Assembler::usra(const VRegister& vd,
4043 const VRegister& vn,
4044 int shift) {
4045 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4046 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
4047 }
4048
4049
srsra(const VRegister & vd,const VRegister & vn,int shift)4050 void Assembler::srsra(const VRegister& vd,
4051 const VRegister& vn,
4052 int shift) {
4053 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4054 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
4055 }
4056
4057
ursra(const VRegister & vd,const VRegister & vn,int shift)4058 void Assembler::ursra(const VRegister& vd,
4059 const VRegister& vn,
4060 int shift) {
4061 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4062 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
4063 }
4064
4065
shrn(const VRegister & vd,const VRegister & vn,int shift)4066 void Assembler::shrn(const VRegister& vd,
4067 const VRegister& vn,
4068 int shift) {
4069 VIXL_ASSERT(vn.IsVector() && vd.IsD());
4070 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
4071 }
4072
4073
shrn2(const VRegister & vd,const VRegister & vn,int shift)4074 void Assembler::shrn2(const VRegister& vd,
4075 const VRegister& vn,
4076 int shift) {
4077 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4078 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
4079 }
4080
4081
rshrn(const VRegister & vd,const VRegister & vn,int shift)4082 void Assembler::rshrn(const VRegister& vd,
4083 const VRegister& vn,
4084 int shift) {
4085 VIXL_ASSERT(vn.IsVector() && vd.IsD());
4086 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
4087 }
4088
4089
rshrn2(const VRegister & vd,const VRegister & vn,int shift)4090 void Assembler::rshrn2(const VRegister& vd,
4091 const VRegister& vn,
4092 int shift) {
4093 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4094 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
4095 }
4096
4097
sqshrn(const VRegister & vd,const VRegister & vn,int shift)4098 void Assembler::sqshrn(const VRegister& vd,
4099 const VRegister& vn,
4100 int shift) {
4101 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4102 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
4103 }
4104
4105
sqshrn2(const VRegister & vd,const VRegister & vn,int shift)4106 void Assembler::sqshrn2(const VRegister& vd,
4107 const VRegister& vn,
4108 int shift) {
4109 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4110 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
4111 }
4112
4113
sqrshrn(const VRegister & vd,const VRegister & vn,int shift)4114 void Assembler::sqrshrn(const VRegister& vd,
4115 const VRegister& vn,
4116 int shift) {
4117 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4118 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
4119 }
4120
4121
sqrshrn2(const VRegister & vd,const VRegister & vn,int shift)4122 void Assembler::sqrshrn2(const VRegister& vd,
4123 const VRegister& vn,
4124 int shift) {
4125 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4126 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
4127 }
4128
4129
sqshrun(const VRegister & vd,const VRegister & vn,int shift)4130 void Assembler::sqshrun(const VRegister& vd,
4131 const VRegister& vn,
4132 int shift) {
4133 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4134 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
4135 }
4136
4137
sqshrun2(const VRegister & vd,const VRegister & vn,int shift)4138 void Assembler::sqshrun2(const VRegister& vd,
4139 const VRegister& vn,
4140 int shift) {
4141 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4142 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
4143 }
4144
4145
sqrshrun(const VRegister & vd,const VRegister & vn,int shift)4146 void Assembler::sqrshrun(const VRegister& vd,
4147 const VRegister& vn,
4148 int shift) {
4149 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4150 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
4151 }
4152
4153
sqrshrun2(const VRegister & vd,const VRegister & vn,int shift)4154 void Assembler::sqrshrun2(const VRegister& vd,
4155 const VRegister& vn,
4156 int shift) {
4157 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4158 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
4159 }
4160
4161
uqshrn(const VRegister & vd,const VRegister & vn,int shift)4162 void Assembler::uqshrn(const VRegister& vd,
4163 const VRegister& vn,
4164 int shift) {
4165 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4166 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
4167 }
4168
4169
uqshrn2(const VRegister & vd,const VRegister & vn,int shift)4170 void Assembler::uqshrn2(const VRegister& vd,
4171 const VRegister& vn,
4172 int shift) {
4173 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4174 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
4175 }
4176
4177
uqrshrn(const VRegister & vd,const VRegister & vn,int shift)4178 void Assembler::uqrshrn(const VRegister& vd,
4179 const VRegister& vn,
4180 int shift) {
4181 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
4182 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
4183 }
4184
4185
uqrshrn2(const VRegister & vd,const VRegister & vn,int shift)4186 void Assembler::uqrshrn2(const VRegister& vd,
4187 const VRegister& vn,
4188 int shift) {
4189 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
4190 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
4191 }
4192
4193
4194 // Note:
4195 // Below, a difference in case for the same letter indicates a
4196 // negated bit.
4197 // If b is 1, then B is 0.
FP32ToImm8(float imm)4198 uint32_t Assembler::FP32ToImm8(float imm) {
4199 VIXL_ASSERT(IsImmFP32(imm));
4200 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
4201 uint32_t bits = float_to_rawbits(imm);
4202 // bit7: a000.0000
4203 uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
4204 // bit6: 0b00.0000
4205 uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
4206 // bit5_to_0: 00cd.efgh
4207 uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
4208
4209 return bit7 | bit6 | bit5_to_0;
4210 }
4211
4212
ImmFP32(float imm)4213 Instr Assembler::ImmFP32(float imm) {
4214 return FP32ToImm8(imm) << ImmFP_offset;
4215 }
4216
4217
FP64ToImm8(double imm)4218 uint32_t Assembler::FP64ToImm8(double imm) {
4219 VIXL_ASSERT(IsImmFP64(imm));
4220 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4221 // 0000.0000.0000.0000.0000.0000.0000.0000
4222 uint64_t bits = double_to_rawbits(imm);
4223 // bit7: a000.0000
4224 uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
4225 // bit6: 0b00.0000
4226 uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
4227 // bit5_to_0: 00cd.efgh
4228 uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
4229
4230 return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
4231 }
4232
4233
ImmFP64(double imm)4234 Instr Assembler::ImmFP64(double imm) {
4235 return FP64ToImm8(imm) << ImmFP_offset;
4236 }
4237
4238
4239 // Code generation helpers.
MoveWide(const Register & rd,uint64_t imm,int shift,MoveWideImmediateOp mov_op)4240 void Assembler::MoveWide(const Register& rd,
4241 uint64_t imm,
4242 int shift,
4243 MoveWideImmediateOp mov_op) {
4244 // Ignore the top 32 bits of an immediate if we're moving to a W register.
4245 if (rd.Is32Bits()) {
4246 // Check that the top 32 bits are zero (a positive 32-bit number) or top
4247 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
4248 VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
4249 ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
4250 imm &= kWRegMask;
4251 }
4252
4253 if (shift >= 0) {
4254 // Explicit shift specified.
4255 VIXL_ASSERT((shift == 0) || (shift == 16) ||
4256 (shift == 32) || (shift == 48));
4257 VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
4258 shift /= 16;
4259 } else {
4260 // Calculate a new immediate and shift combination to encode the immediate
4261 // argument.
4262 shift = 0;
4263 if ((imm & 0xffffffffffff0000) == 0) {
4264 // Nothing to do.
4265 } else if ((imm & 0xffffffff0000ffff) == 0) {
4266 imm >>= 16;
4267 shift = 1;
4268 } else if ((imm & 0xffff0000ffffffff) == 0) {
4269 VIXL_ASSERT(rd.Is64Bits());
4270 imm >>= 32;
4271 shift = 2;
4272 } else if ((imm & 0x0000ffffffffffff) == 0) {
4273 VIXL_ASSERT(rd.Is64Bits());
4274 imm >>= 48;
4275 shift = 3;
4276 }
4277 }
4278
4279 VIXL_ASSERT(is_uint16(imm));
4280
4281 Emit(SF(rd) | MoveWideImmediateFixed | mov_op |
4282 Rd(rd) | ImmMoveWide(imm) | ShiftMoveWide(shift));
4283 }
4284
4285
AddSub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)4286 void Assembler::AddSub(const Register& rd,
4287 const Register& rn,
4288 const Operand& operand,
4289 FlagsUpdate S,
4290 AddSubOp op) {
4291 VIXL_ASSERT(rd.size() == rn.size());
4292 if (operand.IsImmediate()) {
4293 int64_t immediate = operand.immediate();
4294 VIXL_ASSERT(IsImmAddSub(immediate));
4295 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4296 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
4297 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
4298 } else if (operand.IsShiftedRegister()) {
4299 VIXL_ASSERT(operand.reg().size() == rd.size());
4300 VIXL_ASSERT(operand.shift() != ROR);
4301
4302 // For instructions of the form:
4303 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
4304 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
4305 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
4306 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
4307 // or their 64-bit register equivalents, convert the operand from shifted to
4308 // extended register mode, and emit an add/sub extended instruction.
4309 if (rn.IsSP() || rd.IsSP()) {
4310 VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
4311 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
4312 AddSubExtendedFixed | op);
4313 } else {
4314 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
4315 }
4316 } else {
4317 VIXL_ASSERT(operand.IsExtendedRegister());
4318 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
4319 }
4320 }
4321
4322
AddSubWithCarry(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)4323 void Assembler::AddSubWithCarry(const Register& rd,
4324 const Register& rn,
4325 const Operand& operand,
4326 FlagsUpdate S,
4327 AddSubWithCarryOp op) {
4328 VIXL_ASSERT(rd.size() == rn.size());
4329 VIXL_ASSERT(rd.size() == operand.reg().size());
4330 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4331 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
4332 }
4333
4334
hlt(int code)4335 void Assembler::hlt(int code) {
4336 VIXL_ASSERT(is_uint16(code));
4337 Emit(HLT | ImmException(code));
4338 }
4339
4340
brk(int code)4341 void Assembler::brk(int code) {
4342 VIXL_ASSERT(is_uint16(code));
4343 Emit(BRK | ImmException(code));
4344 }
4345
4346
svc(int code)4347 void Assembler::svc(int code) {
4348 Emit(SVC | ImmException(code));
4349 }
4350
4351
ConditionalCompare(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)4352 void Assembler::ConditionalCompare(const Register& rn,
4353 const Operand& operand,
4354 StatusFlags nzcv,
4355 Condition cond,
4356 ConditionalCompareOp op) {
4357 Instr ccmpop;
4358 if (operand.IsImmediate()) {
4359 int64_t immediate = operand.immediate();
4360 VIXL_ASSERT(IsImmConditionalCompare(immediate));
4361 ccmpop = ConditionalCompareImmediateFixed | op |
4362 ImmCondCmp(static_cast<unsigned>(immediate));
4363 } else {
4364 VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4365 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
4366 }
4367 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
4368 }
4369
4370
DataProcessing1Source(const Register & rd,const Register & rn,DataProcessing1SourceOp op)4371 void Assembler::DataProcessing1Source(const Register& rd,
4372 const Register& rn,
4373 DataProcessing1SourceOp op) {
4374 VIXL_ASSERT(rd.size() == rn.size());
4375 Emit(SF(rn) | op | Rn(rn) | Rd(rd));
4376 }
4377
4378
FPDataProcessing1Source(const VRegister & vd,const VRegister & vn,FPDataProcessing1SourceOp op)4379 void Assembler::FPDataProcessing1Source(const VRegister& vd,
4380 const VRegister& vn,
4381 FPDataProcessing1SourceOp op) {
4382 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
4383 Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
4384 }
4385
4386
FPDataProcessing3Source(const VRegister & vd,const VRegister & vn,const VRegister & vm,const VRegister & va,FPDataProcessing3SourceOp op)4387 void Assembler::FPDataProcessing3Source(const VRegister& vd,
4388 const VRegister& vn,
4389 const VRegister& vm,
4390 const VRegister& va,
4391 FPDataProcessing3SourceOp op) {
4392 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4393 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
4394 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
4395 }
4396
4397
NEONModifiedImmShiftLsl(const VRegister & vd,const int imm8,const int left_shift,NEONModifiedImmediateOp op)4398 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
4399 const int imm8,
4400 const int left_shift,
4401 NEONModifiedImmediateOp op) {
4402 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() ||
4403 vd.Is2S() || vd.Is4S());
4404 VIXL_ASSERT((left_shift == 0) || (left_shift == 8) ||
4405 (left_shift == 16) || (left_shift == 24));
4406 VIXL_ASSERT(is_uint8(imm8));
4407
4408 int cmode_1, cmode_2, cmode_3;
4409 if (vd.Is8B() || vd.Is16B()) {
4410 VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
4411 cmode_1 = 1;
4412 cmode_2 = 1;
4413 cmode_3 = 1;
4414 } else {
4415 cmode_1 = (left_shift >> 3) & 1;
4416 cmode_2 = left_shift >> 4;
4417 cmode_3 = 0;
4418 if (vd.Is4H() || vd.Is8H()) {
4419 VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
4420 cmode_3 = 1;
4421 }
4422 }
4423 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
4424
4425 int q = vd.IsQ() ? NEON_Q : 0;
4426
4427 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4428 }
4429
4430
NEONModifiedImmShiftMsl(const VRegister & vd,const int imm8,const int shift_amount,NEONModifiedImmediateOp op)4431 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
4432 const int imm8,
4433 const int shift_amount,
4434 NEONModifiedImmediateOp op) {
4435 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
4436 VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
4437 VIXL_ASSERT(is_uint8(imm8));
4438
4439 int cmode_0 = (shift_amount >> 4) & 1;
4440 int cmode = 0xc | cmode_0;
4441
4442 int q = vd.IsQ() ? NEON_Q : 0;
4443
4444 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4445 }
4446
4447
EmitShift(const Register & rd,const Register & rn,Shift shift,unsigned shift_amount)4448 void Assembler::EmitShift(const Register& rd,
4449 const Register& rn,
4450 Shift shift,
4451 unsigned shift_amount) {
4452 switch (shift) {
4453 case LSL:
4454 lsl(rd, rn, shift_amount);
4455 break;
4456 case LSR:
4457 lsr(rd, rn, shift_amount);
4458 break;
4459 case ASR:
4460 asr(rd, rn, shift_amount);
4461 break;
4462 case ROR:
4463 ror(rd, rn, shift_amount);
4464 break;
4465 default:
4466 VIXL_UNREACHABLE();
4467 }
4468 }
4469
4470
EmitExtendShift(const Register & rd,const Register & rn,Extend extend,unsigned left_shift)4471 void Assembler::EmitExtendShift(const Register& rd,
4472 const Register& rn,
4473 Extend extend,
4474 unsigned left_shift) {
4475 VIXL_ASSERT(rd.size() >= rn.size());
4476 unsigned reg_size = rd.size();
4477 // Use the correct size of register.
4478 Register rn_ = Register(rn.code(), rd.size());
4479 // Bits extracted are high_bit:0.
4480 unsigned high_bit = (8 << (extend & 0x3)) - 1;
4481 // Number of bits left in the result that are not introduced by the shift.
4482 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
4483
4484 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
4485 switch (extend) {
4486 case UXTB:
4487 case UXTH:
4488 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break;
4489 case SXTB:
4490 case SXTH:
4491 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break;
4492 case UXTX:
4493 case SXTX: {
4494 VIXL_ASSERT(rn.size() == kXRegSize);
4495 // Nothing to extend. Just shift.
4496 lsl(rd, rn_, left_shift);
4497 break;
4498 }
4499 default: VIXL_UNREACHABLE();
4500 }
4501 } else {
4502 // No need to extend as the extended bits would be shifted away.
4503 lsl(rd, rn_, left_shift);
4504 }
4505 }
4506
4507
DataProcExtendedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)4508 void Assembler::DataProcExtendedRegister(const Register& rd,
4509 const Register& rn,
4510 const Operand& operand,
4511 FlagsUpdate S,
4512 Instr op) {
4513 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4514 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
4515 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
4516 dest_reg | RnSP(rn));
4517 }
4518
4519
LoadStoreMemOperand(const MemOperand & addr,unsigned access_size,LoadStoreScalingOption option)4520 Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
4521 unsigned access_size,
4522 LoadStoreScalingOption option) {
4523 Instr base = RnSP(addr.base());
4524 int64_t offset = addr.offset();
4525
4526 if (addr.IsImmediateOffset()) {
4527 bool prefer_unscaled = (option == PreferUnscaledOffset) ||
4528 (option == RequireUnscaledOffset);
4529 if (prefer_unscaled && IsImmLSUnscaled(offset)) {
4530 // Use the unscaled addressing mode.
4531 return base | LoadStoreUnscaledOffsetFixed |
4532 ImmLS(static_cast<int>(offset));
4533 }
4534
4535 if ((option != RequireUnscaledOffset) &&
4536 IsImmLSScaled(offset, access_size)) {
4537 // Use the scaled addressing mode.
4538 return base | LoadStoreUnsignedOffsetFixed |
4539 ImmLSUnsigned(static_cast<int>(offset) >> access_size);
4540 }
4541
4542 if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
4543 // Use the unscaled addressing mode.
4544 return base | LoadStoreUnscaledOffsetFixed |
4545 ImmLS(static_cast<int>(offset));
4546 }
4547 }
4548
4549 // All remaining addressing modes are register-offset, pre-indexed or
4550 // post-indexed modes.
4551 VIXL_ASSERT((option != RequireUnscaledOffset) &&
4552 (option != RequireScaledOffset));
4553
4554 if (addr.IsRegisterOffset()) {
4555 Extend ext = addr.extend();
4556 Shift shift = addr.shift();
4557 unsigned shift_amount = addr.shift_amount();
4558
4559 // LSL is encoded in the option field as UXTX.
4560 if (shift == LSL) {
4561 ext = UXTX;
4562 }
4563
4564 // Shifts are encoded in one bit, indicating a left shift by the memory
4565 // access size.
4566 VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size));
4567 return base | LoadStoreRegisterOffsetFixed | Rm(addr.regoffset()) |
4568 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
4569 }
4570
4571 if (addr.IsPreIndex() && IsImmLSUnscaled(offset)) {
4572 return base | LoadStorePreIndexFixed | ImmLS(static_cast<int>(offset));
4573 }
4574
4575 if (addr.IsPostIndex() && IsImmLSUnscaled(offset)) {
4576 return base | LoadStorePostIndexFixed | ImmLS(static_cast<int>(offset));
4577 }
4578
4579 // If this point is reached, the MemOperand (addr) cannot be encoded.
4580 VIXL_UNREACHABLE();
4581 return 0;
4582 }
4583
4584
LoadStore(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op,LoadStoreScalingOption option)4585 void Assembler::LoadStore(const CPURegister& rt,
4586 const MemOperand& addr,
4587 LoadStoreOp op,
4588 LoadStoreScalingOption option) {
4589 Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
4590 }
4591
4592
Prefetch(PrefetchOperation op,const MemOperand & addr,LoadStoreScalingOption option)4593 void Assembler::Prefetch(PrefetchOperation op,
4594 const MemOperand& addr,
4595 LoadStoreScalingOption option) {
4596 VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
4597
4598 Instr prfop = ImmPrefetchOperation(op);
4599 Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
4600 }
4601
4602
IsImmAddSub(int64_t immediate)4603 bool Assembler::IsImmAddSub(int64_t immediate) {
4604 return is_uint12(immediate) ||
4605 (is_uint12(immediate >> 12) && ((immediate & 0xfff) == 0));
4606 }
4607
4608
IsImmConditionalCompare(int64_t immediate)4609 bool Assembler::IsImmConditionalCompare(int64_t immediate) {
4610 return is_uint5(immediate);
4611 }
4612
4613
IsImmFP32(float imm)4614 bool Assembler::IsImmFP32(float imm) {
4615 // Valid values will have the form:
4616 // aBbb.bbbc.defg.h000.0000.0000.0000.0000
4617 uint32_t bits = float_to_rawbits(imm);
4618 // bits[19..0] are cleared.
4619 if ((bits & 0x7ffff) != 0) {
4620 return false;
4621 }
4622
4623 // bits[29..25] are all set or all cleared.
4624 uint32_t b_pattern = (bits >> 16) & 0x3e00;
4625 if (b_pattern != 0 && b_pattern != 0x3e00) {
4626 return false;
4627 }
4628
4629 // bit[30] and bit[29] are opposite.
4630 if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
4631 return false;
4632 }
4633
4634 return true;
4635 }
4636
4637
IsImmFP64(double imm)4638 bool Assembler::IsImmFP64(double imm) {
4639 // Valid values will have the form:
4640 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4641 // 0000.0000.0000.0000.0000.0000.0000.0000
4642 uint64_t bits = double_to_rawbits(imm);
4643 // bits[47..0] are cleared.
4644 if ((bits & 0x0000ffffffffffff) != 0) {
4645 return false;
4646 }
4647
4648 // bits[61..54] are all set or all cleared.
4649 uint32_t b_pattern = (bits >> 48) & 0x3fc0;
4650 if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
4651 return false;
4652 }
4653
4654 // bit[62] and bit[61] are opposite.
4655 if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
4656 return false;
4657 }
4658
4659 return true;
4660 }
4661
4662
IsImmLSPair(int64_t offset,unsigned access_size)4663 bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size) {
4664 VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
4665 bool offset_is_size_multiple =
4666 (((offset >> access_size) << access_size) == offset);
4667 return offset_is_size_multiple && is_int7(offset >> access_size);
4668 }
4669
4670
IsImmLSScaled(int64_t offset,unsigned access_size)4671 bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size) {
4672 VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
4673 bool offset_is_size_multiple =
4674 (((offset >> access_size) << access_size) == offset);
4675 return offset_is_size_multiple && is_uint12(offset >> access_size);
4676 }
4677
4678
IsImmLSUnscaled(int64_t offset)4679 bool Assembler::IsImmLSUnscaled(int64_t offset) {
4680 return is_int9(offset);
4681 }
4682
4683
4684 // The movn instruction can generate immediates containing an arbitrary 16-bit
4685 // value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
IsImmMovn(uint64_t imm,unsigned reg_size)4686 bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
4687 return IsImmMovz(~imm, reg_size);
4688 }
4689
4690
4691 // The movz instruction can generate immediates containing an arbitrary 16-bit
4692 // value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)4693 bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
4694 VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
4695 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
4696 }
4697
4698
4699 // Test if a given value can be encoded in the immediate field of a logical
4700 // instruction.
4701 // If it can be encoded, the function returns true, and values pointed to by n,
4702 // imm_s and imm_r are updated with immediates encoded in the format required
4703 // by the corresponding fields in the logical instruction.
4704 // If it can not be encoded, the function returns false, and the values pointed
4705 // to by n, imm_s and imm_r are undefined.
IsImmLogical(uint64_t value,unsigned width,unsigned * n,unsigned * imm_s,unsigned * imm_r)4706 bool Assembler::IsImmLogical(uint64_t value,
4707 unsigned width,
4708 unsigned* n,
4709 unsigned* imm_s,
4710 unsigned* imm_r) {
4711 VIXL_ASSERT((width == kWRegSize) || (width == kXRegSize));
4712
4713 bool negate = false;
4714
4715 // Logical immediates are encoded using parameters n, imm_s and imm_r using
4716 // the following table:
4717 //
4718 // N imms immr size S R
4719 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
4720 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
4721 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
4722 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
4723 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
4724 // 0 11110s xxxxxr 2 UInt(s) UInt(r)
4725 // (s bits must not be all set)
4726 //
4727 // A pattern is constructed of size bits, where the least significant S+1 bits
4728 // are set. The pattern is rotated right by R, and repeated across a 32 or
4729 // 64-bit value, depending on destination register width.
4730 //
4731 // Put another way: the basic format of a logical immediate is a single
4732 // contiguous stretch of 1 bits, repeated across the whole word at intervals
4733 // given by a power of 2. To identify them quickly, we first locate the
4734 // lowest stretch of 1 bits, then the next 1 bit above that; that combination
4735 // is different for every logical immediate, so it gives us all the
4736 // information we need to identify the only logical immediate that our input
4737 // could be, and then we simply check if that's the value we actually have.
4738 //
4739 // (The rotation parameter does give the possibility of the stretch of 1 bits
4740 // going 'round the end' of the word. To deal with that, we observe that in
4741 // any situation where that happens the bitwise NOT of the value is also a
4742 // valid logical immediate. So we simply invert the input whenever its low bit
4743 // is set, and then we know that the rotated case can't arise.)
4744
4745 if (value & 1) {
4746 // If the low bit is 1, negate the value, and set a flag to remember that we
4747 // did (so that we can adjust the return values appropriately).
4748 negate = true;
4749 value = ~value;
4750 }
4751
4752 if (width == kWRegSize) {
4753 // To handle 32-bit logical immediates, the very easiest thing is to repeat
4754 // the input value twice to make a 64-bit word. The correct encoding of that
4755 // as a logical immediate will also be the correct encoding of the 32-bit
4756 // value.
4757
4758 // Avoid making the assumption that the most-significant 32 bits are zero by
4759 // shifting the value left and duplicating it.
4760 value <<= kWRegSize;
4761 value |= value >> kWRegSize;
4762 }
4763
4764 // The basic analysis idea: imagine our input word looks like this.
4765 //
4766 // 0011111000111110001111100011111000111110001111100011111000111110
4767 // c b a
4768 // |<--d-->|
4769 //
4770 // We find the lowest set bit (as an actual power-of-2 value, not its index)
4771 // and call it a. Then we add a to our original number, which wipes out the
4772 // bottommost stretch of set bits and replaces it with a 1 carried into the
4773 // next zero bit. Then we look for the new lowest set bit, which is in
4774 // position b, and subtract it, so now our number is just like the original
4775 // but with the lowest stretch of set bits completely gone. Now we find the
4776 // lowest set bit again, which is position c in the diagram above. Then we'll
4777 // measure the distance d between bit positions a and c (using CLZ), and that
4778 // tells us that the only valid logical immediate that could possibly be equal
4779 // to this number is the one in which a stretch of bits running from a to just
4780 // below b is replicated every d bits.
4781 uint64_t a = LowestSetBit(value);
4782 uint64_t value_plus_a = value + a;
4783 uint64_t b = LowestSetBit(value_plus_a);
4784 uint64_t value_plus_a_minus_b = value_plus_a - b;
4785 uint64_t c = LowestSetBit(value_plus_a_minus_b);
4786
4787 int d, clz_a, out_n;
4788 uint64_t mask;
4789
4790 if (c != 0) {
4791 // The general case, in which there is more than one stretch of set bits.
4792 // Compute the repeat distance d, and set up a bitmask covering the basic
4793 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
4794 // of these cases the N bit of the output will be zero.
4795 clz_a = CountLeadingZeros(a, kXRegSize);
4796 int clz_c = CountLeadingZeros(c, kXRegSize);
4797 d = clz_a - clz_c;
4798 mask = ((UINT64_C(1) << d) - 1);
4799 out_n = 0;
4800 } else {
4801 // Handle degenerate cases.
4802 //
4803 // If any of those 'find lowest set bit' operations didn't find a set bit at
4804 // all, then the word will have been zero thereafter, so in particular the
4805 // last lowest_set_bit operation will have returned zero. So we can test for
4806 // all the special case conditions in one go by seeing if c is zero.
4807 if (a == 0) {
4808 // The input was zero (or all 1 bits, which will come to here too after we
4809 // inverted it at the start of the function), for which we just return
4810 // false.
4811 return false;
4812 } else {
4813 // Otherwise, if c was zero but a was not, then there's just one stretch
4814 // of set bits in our word, meaning that we have the trivial case of
4815 // d == 64 and only one 'repetition'. Set up all the same variables as in
4816 // the general case above, and set the N bit in the output.
4817 clz_a = CountLeadingZeros(a, kXRegSize);
4818 d = 64;
4819 mask = ~UINT64_C(0);
4820 out_n = 1;
4821 }
4822 }
4823
4824 // If the repeat period d is not a power of two, it can't be encoded.
4825 if (!IsPowerOf2(d)) {
4826 return false;
4827 }
4828
4829 if (((b - a) & ~mask) != 0) {
4830 // If the bit stretch (b - a) does not fit within the mask derived from the
4831 // repeat period, then fail.
4832 return false;
4833 }
4834
4835 // The only possible option is b - a repeated every d bits. Now we're going to
4836 // actually construct the valid logical immediate derived from that
4837 // specification, and see if it equals our original input.
4838 //
4839 // To repeat a value every d bits, we multiply it by a number of the form
4840 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
4841 // be derived using a table lookup on CLZ(d).
4842 static const uint64_t multipliers[] = {
4843 0x0000000000000001UL,
4844 0x0000000100000001UL,
4845 0x0001000100010001UL,
4846 0x0101010101010101UL,
4847 0x1111111111111111UL,
4848 0x5555555555555555UL,
4849 };
4850 uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
4851 uint64_t candidate = (b - a) * multiplier;
4852
4853 if (value != candidate) {
4854 // The candidate pattern doesn't match our input value, so fail.
4855 return false;
4856 }
4857
4858 // We have a match! This is a valid logical immediate, so now we have to
4859 // construct the bits and pieces of the instruction encoding that generates
4860 // it.
4861
4862 // Count the set bits in our basic stretch. The special case of clz(0) == -1
4863 // makes the answer come out right for stretches that reach the very top of
4864 // the word (e.g. numbers like 0xffffc00000000000).
4865 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
4866 int s = clz_a - clz_b;
4867
4868 // Decide how many bits to rotate right by, to put the low bit of that basic
4869 // stretch in position a.
4870 int r;
4871 if (negate) {
4872 // If we inverted the input right at the start of this function, here's
4873 // where we compensate: the number of set bits becomes the number of clear
4874 // bits, and the rotation count is based on position b rather than position
4875 // a (since b is the location of the 'lowest' 1 bit after inversion).
4876 s = d - s;
4877 r = (clz_b + 1) & (d - 1);
4878 } else {
4879 r = (clz_a + 1) & (d - 1);
4880 }
4881
4882 // Now we're done, except for having to encode the S output in such a way that
4883 // it gives both the number of set bits and the length of the repeated
4884 // segment. The s field is encoded like this:
4885 //
4886 // imms size S
4887 // ssssss 64 UInt(ssssss)
4888 // 0sssss 32 UInt(sssss)
4889 // 10ssss 16 UInt(ssss)
4890 // 110sss 8 UInt(sss)
4891 // 1110ss 4 UInt(ss)
4892 // 11110s 2 UInt(s)
4893 //
4894 // So we 'or' (-d << 1) with our computed s to form imms.
4895 if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
4896 *n = out_n;
4897 *imm_s = ((-d << 1) | (s - 1)) & 0x3f;
4898 *imm_r = r;
4899 }
4900
4901 return true;
4902 }
4903
4904
LoadOpFor(const CPURegister & rt)4905 LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
4906 VIXL_ASSERT(rt.IsValid());
4907 if (rt.IsRegister()) {
4908 return rt.Is64Bits() ? LDR_x : LDR_w;
4909 } else {
4910 VIXL_ASSERT(rt.IsVRegister());
4911 switch (rt.SizeInBits()) {
4912 case kBRegSize: return LDR_b;
4913 case kHRegSize: return LDR_h;
4914 case kSRegSize: return LDR_s;
4915 case kDRegSize: return LDR_d;
4916 default:
4917 VIXL_ASSERT(rt.IsQ());
4918 return LDR_q;
4919 }
4920 }
4921 }
4922
4923
StoreOpFor(const CPURegister & rt)4924 LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
4925 VIXL_ASSERT(rt.IsValid());
4926 if (rt.IsRegister()) {
4927 return rt.Is64Bits() ? STR_x : STR_w;
4928 } else {
4929 VIXL_ASSERT(rt.IsVRegister());
4930 switch (rt.SizeInBits()) {
4931 case kBRegSize: return STR_b;
4932 case kHRegSize: return STR_h;
4933 case kSRegSize: return STR_s;
4934 case kDRegSize: return STR_d;
4935 default:
4936 VIXL_ASSERT(rt.IsQ());
4937 return STR_q;
4938 }
4939 }
4940 }
4941
4942
StorePairOpFor(const CPURegister & rt,const CPURegister & rt2)4943 LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
4944 const CPURegister& rt2) {
4945 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
4946 USE(rt2);
4947 if (rt.IsRegister()) {
4948 return rt.Is64Bits() ? STP_x : STP_w;
4949 } else {
4950 VIXL_ASSERT(rt.IsVRegister());
4951 switch (rt.SizeInBytes()) {
4952 case kSRegSizeInBytes: return STP_s;
4953 case kDRegSizeInBytes: return STP_d;
4954 default:
4955 VIXL_ASSERT(rt.IsQ());
4956 return STP_q;
4957 }
4958 }
4959 }
4960
4961
LoadPairOpFor(const CPURegister & rt,const CPURegister & rt2)4962 LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
4963 const CPURegister& rt2) {
4964 VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
4965 return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
4966 LoadStorePairLBit);
4967 }
4968
4969
StorePairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)4970 LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
4971 const CPURegister& rt, const CPURegister& rt2) {
4972 VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
4973 USE(rt2);
4974 if (rt.IsRegister()) {
4975 return rt.Is64Bits() ? STNP_x : STNP_w;
4976 } else {
4977 VIXL_ASSERT(rt.IsVRegister());
4978 switch (rt.SizeInBytes()) {
4979 case kSRegSizeInBytes: return STNP_s;
4980 case kDRegSizeInBytes: return STNP_d;
4981 default:
4982 VIXL_ASSERT(rt.IsQ());
4983 return STNP_q;
4984 }
4985 }
4986 }
4987
4988
LoadPairNonTemporalOpFor(const CPURegister & rt,const CPURegister & rt2)4989 LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
4990 const CPURegister& rt, const CPURegister& rt2) {
4991 VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
4992 return static_cast<LoadStorePairNonTemporalOp>(
4993 StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
4994 }
4995
4996
LoadLiteralOpFor(const CPURegister & rt)4997 LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
4998 if (rt.IsRegister()) {
4999 return rt.IsX() ? LDR_x_lit : LDR_w_lit;
5000 } else {
5001 VIXL_ASSERT(rt.IsVRegister());
5002 switch (rt.SizeInBytes()) {
5003 case kSRegSizeInBytes: return LDR_s_lit;
5004 case kDRegSizeInBytes: return LDR_d_lit;
5005 default:
5006 VIXL_ASSERT(rt.IsQ());
5007 return LDR_q_lit;
5008 }
5009 }
5010 }
5011
5012
AreAliased(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4,const CPURegister & reg5,const CPURegister & reg6,const CPURegister & reg7,const CPURegister & reg8)5013 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
5014 const CPURegister& reg3, const CPURegister& reg4,
5015 const CPURegister& reg5, const CPURegister& reg6,
5016 const CPURegister& reg7, const CPURegister& reg8) {
5017 int number_of_valid_regs = 0;
5018 int number_of_valid_fpregs = 0;
5019
5020 RegList unique_regs = 0;
5021 RegList unique_fpregs = 0;
5022
5023 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
5024
5025 for (unsigned i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
5026 if (regs[i].IsRegister()) {
5027 number_of_valid_regs++;
5028 unique_regs |= regs[i].Bit();
5029 } else if (regs[i].IsVRegister()) {
5030 number_of_valid_fpregs++;
5031 unique_fpregs |= regs[i].Bit();
5032 } else {
5033 VIXL_ASSERT(!regs[i].IsValid());
5034 }
5035 }
5036
5037 int number_of_unique_regs = CountSetBits(unique_regs);
5038 int number_of_unique_fpregs = CountSetBits(unique_fpregs);
5039
5040 VIXL_ASSERT(number_of_valid_regs >= number_of_unique_regs);
5041 VIXL_ASSERT(number_of_valid_fpregs >= number_of_unique_fpregs);
5042
5043 return (number_of_valid_regs != number_of_unique_regs) ||
5044 (number_of_valid_fpregs != number_of_unique_fpregs);
5045 }
5046
5047
AreSameSizeAndType(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4,const CPURegister & reg5,const CPURegister & reg6,const CPURegister & reg7,const CPURegister & reg8)5048 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
5049 const CPURegister& reg3, const CPURegister& reg4,
5050 const CPURegister& reg5, const CPURegister& reg6,
5051 const CPURegister& reg7, const CPURegister& reg8) {
5052 VIXL_ASSERT(reg1.IsValid());
5053 bool match = true;
5054 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
5055 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
5056 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
5057 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
5058 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
5059 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
5060 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
5061 return match;
5062 }
5063
5064
AreSameFormat(const VRegister & reg1,const VRegister & reg2,const VRegister & reg3,const VRegister & reg4)5065 bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
5066 const VRegister& reg3, const VRegister& reg4) {
5067 VIXL_ASSERT(reg1.IsValid());
5068 bool match = true;
5069 match &= !reg2.IsValid() || reg2.IsSameFormat(reg1);
5070 match &= !reg3.IsValid() || reg3.IsSameFormat(reg1);
5071 match &= !reg4.IsValid() || reg4.IsSameFormat(reg1);
5072 return match;
5073 }
5074
5075
AreConsecutive(const VRegister & reg1,const VRegister & reg2,const VRegister & reg3,const VRegister & reg4)5076 bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
5077 const VRegister& reg3, const VRegister& reg4) {
5078 VIXL_ASSERT(reg1.IsValid());
5079 bool match = true;
5080 match &= !reg2.IsValid() ||
5081 (reg2.code() == ((reg1.code() + 1) % kNumberOfVRegisters));
5082 match &= !reg3.IsValid() ||
5083 (reg3.code() == ((reg1.code() + 2) % kNumberOfVRegisters));
5084 match &= !reg4.IsValid() ||
5085 (reg4.code() == ((reg1.code() + 3) % kNumberOfVRegisters));
5086 return match;
5087 }
5088 } // namespace vixl
5089