1 // Copyright 2013 the V8 project authors. All rights reserved.
2 //
3 // Redistribution and use in source and binary forms, with or without
4 // modification, are permitted provided that the following conditions are
5 // met:
6 //
7 // * Redistributions of source code must retain the above copyright
8 // notice, this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above
10 // copyright notice, this list of conditions and the following
11 // disclaimer in the documentation and/or other materials provided
12 // with the distribution.
13 // * Neither the name of Google Inc. nor the names of its
14 // contributors may be used to endorse or promote products derived
15 // from this software without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #if V8_TARGET_ARCH_ARM64
30
31 #include "src/arm64/assembler-arm64.h"
32
33 #include "src/arm64/assembler-arm64-inl.h"
34 #include "src/base/bits.h"
35 #include "src/base/cpu.h"
36 #include "src/code-stubs.h"
37 #include "src/frame-constants.h"
38 #include "src/register-configuration.h"
39
40 namespace v8 {
41 namespace internal {
42
43 // -----------------------------------------------------------------------------
44 // CpuFeatures implementation.
45
ProbeImpl(bool cross_compile)46 void CpuFeatures::ProbeImpl(bool cross_compile) {
47 // AArch64 has no configuration options, no further probing is required.
48 supported_ = 0;
49
50 // Only use statically determined features for cross compile (snapshot).
51 if (cross_compile) return;
52
53 // We used to probe for coherent cache support, but on older CPUs it
54 // causes crashes (crbug.com/524337), and newer CPUs don't even have
55 // the feature any more.
56 }
57
PrintTarget()58 void CpuFeatures::PrintTarget() { }
PrintFeatures()59 void CpuFeatures::PrintFeatures() {}
60
61 // -----------------------------------------------------------------------------
62 // CPURegList utilities.
63
PopLowestIndex()64 CPURegister CPURegList::PopLowestIndex() {
65 DCHECK(IsValid());
66 if (IsEmpty()) {
67 return NoCPUReg;
68 }
69 int index = CountTrailingZeros(list_, kRegListSizeInBits);
70 DCHECK((1 << index) & list_);
71 Remove(index);
72 return CPURegister::Create(index, size_, type_);
73 }
74
75
PopHighestIndex()76 CPURegister CPURegList::PopHighestIndex() {
77 DCHECK(IsValid());
78 if (IsEmpty()) {
79 return NoCPUReg;
80 }
81 int index = CountLeadingZeros(list_, kRegListSizeInBits);
82 index = kRegListSizeInBits - 1 - index;
83 DCHECK((1 << index) & list_);
84 Remove(index);
85 return CPURegister::Create(index, size_, type_);
86 }
87
88
RemoveCalleeSaved()89 void CPURegList::RemoveCalleeSaved() {
90 if (type() == CPURegister::kRegister) {
91 Remove(GetCalleeSaved(RegisterSizeInBits()));
92 } else if (type() == CPURegister::kVRegister) {
93 Remove(GetCalleeSavedV(RegisterSizeInBits()));
94 } else {
95 DCHECK_EQ(type(), CPURegister::kNoRegister);
96 DCHECK(IsEmpty());
97 // The list must already be empty, so do nothing.
98 }
99 }
100
101
GetCalleeSaved(int size)102 CPURegList CPURegList::GetCalleeSaved(int size) {
103 return CPURegList(CPURegister::kRegister, size, 19, 29);
104 }
105
GetCalleeSavedV(int size)106 CPURegList CPURegList::GetCalleeSavedV(int size) {
107 return CPURegList(CPURegister::kVRegister, size, 8, 15);
108 }
109
110
GetCallerSaved(int size)111 CPURegList CPURegList::GetCallerSaved(int size) {
112 // Registers x0-x18 and lr (x30) are caller-saved.
113 CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18);
114 list.Combine(lr);
115 return list;
116 }
117
GetCallerSavedV(int size)118 CPURegList CPURegList::GetCallerSavedV(int size) {
119 // Registers d0-d7 and d16-d31 are caller-saved.
120 CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7);
121 list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31));
122 return list;
123 }
124
125
126 // This function defines the list of registers which are associated with a
127 // safepoint slot. Safepoint register slots are saved contiguously on the stack.
128 // MacroAssembler::SafepointRegisterStackIndex handles mapping from register
129 // code to index in the safepoint register slots. Any change here can affect
130 // this mapping.
GetSafepointSavedRegisters()131 CPURegList CPURegList::GetSafepointSavedRegisters() {
132 CPURegList list = CPURegList::GetCalleeSaved();
133 list.Combine(
134 CPURegList(CPURegister::kRegister, kXRegSizeInBits, kJSCallerSaved));
135
136 // Note that unfortunately we can't use symbolic names for registers and have
137 // to directly use register codes. This is because this function is used to
138 // initialize some static variables and we can't rely on register variables
139 // to be initialized due to static initialization order issues in C++.
140
141 // Drop ip0 and ip1 (i.e. x16 and x17), as they should not be expected to be
142 // preserved outside of the macro assembler.
143 list.Remove(16);
144 list.Remove(17);
145
146 // Add x18 to the safepoint list, as although it's not in kJSCallerSaved, it
147 // is a caller-saved register according to the procedure call standard.
148 list.Combine(18);
149
150 // Add the link register (x30) to the safepoint list.
151 list.Combine(30);
152
153 return list;
154 }
155
156
157 // -----------------------------------------------------------------------------
158 // Implementation of RelocInfo
159
160 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
161 1 << RelocInfo::RUNTIME_ENTRY |
162 1 << RelocInfo::INTERNAL_REFERENCE;
163
IsCodedSpecially()164 bool RelocInfo::IsCodedSpecially() {
165 // The deserializer needs to know whether a pointer is specially coded. Being
166 // specially coded on ARM64 means that it is an immediate branch.
167 Instruction* instr = reinterpret_cast<Instruction*>(pc_);
168 if (instr->IsLdrLiteralX()) {
169 return false;
170 } else {
171 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
172 return true;
173 }
174 }
175
176
IsInConstantPool()177 bool RelocInfo::IsInConstantPool() {
178 Instruction* instr = reinterpret_cast<Instruction*>(pc_);
179 return instr->IsLdrLiteralX();
180 }
181
embedded_address() const182 Address RelocInfo::embedded_address() const {
183 return Assembler::target_address_at(pc_, constant_pool_);
184 }
185
embedded_size() const186 uint32_t RelocInfo::embedded_size() const {
187 return Memory::uint32_at(Assembler::target_pointer_address_at(pc_));
188 }
189
set_embedded_address(Address address,ICacheFlushMode flush_mode)190 void RelocInfo::set_embedded_address(Address address,
191 ICacheFlushMode flush_mode) {
192 Assembler::set_target_address_at(pc_, constant_pool_, address, flush_mode);
193 }
194
set_embedded_size(uint32_t size,ICacheFlushMode flush_mode)195 void RelocInfo::set_embedded_size(uint32_t size, ICacheFlushMode flush_mode) {
196 Memory::uint32_at(Assembler::target_pointer_address_at(pc_)) = size;
197 // No icache flushing needed, see comment in set_target_address_at.
198 }
199
set_js_to_wasm_address(Address address,ICacheFlushMode icache_flush_mode)200 void RelocInfo::set_js_to_wasm_address(Address address,
201 ICacheFlushMode icache_flush_mode) {
202 DCHECK_EQ(rmode_, JS_TO_WASM_CALL);
203 set_embedded_address(address, icache_flush_mode);
204 }
205
js_to_wasm_address() const206 Address RelocInfo::js_to_wasm_address() const {
207 DCHECK_EQ(rmode_, JS_TO_WASM_CALL);
208 return embedded_address();
209 }
210
AreAliased(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4,const CPURegister & reg5,const CPURegister & reg6,const CPURegister & reg7,const CPURegister & reg8)211 bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
212 const CPURegister& reg3, const CPURegister& reg4,
213 const CPURegister& reg5, const CPURegister& reg6,
214 const CPURegister& reg7, const CPURegister& reg8) {
215 int number_of_valid_regs = 0;
216 int number_of_valid_fpregs = 0;
217
218 RegList unique_regs = 0;
219 RegList unique_fpregs = 0;
220
221 const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
222
223 for (unsigned i = 0; i < arraysize(regs); i++) {
224 if (regs[i].IsRegister()) {
225 number_of_valid_regs++;
226 unique_regs |= regs[i].bit();
227 } else if (regs[i].IsVRegister()) {
228 number_of_valid_fpregs++;
229 unique_fpregs |= regs[i].bit();
230 } else {
231 DCHECK(!regs[i].IsValid());
232 }
233 }
234
235 int number_of_unique_regs =
236 CountSetBits(unique_regs, sizeof(unique_regs) * kBitsPerByte);
237 int number_of_unique_fpregs =
238 CountSetBits(unique_fpregs, sizeof(unique_fpregs) * kBitsPerByte);
239
240 DCHECK(number_of_valid_regs >= number_of_unique_regs);
241 DCHECK(number_of_valid_fpregs >= number_of_unique_fpregs);
242
243 return (number_of_valid_regs != number_of_unique_regs) ||
244 (number_of_valid_fpregs != number_of_unique_fpregs);
245 }
246
247
AreSameSizeAndType(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4,const CPURegister & reg5,const CPURegister & reg6,const CPURegister & reg7,const CPURegister & reg8)248 bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
249 const CPURegister& reg3, const CPURegister& reg4,
250 const CPURegister& reg5, const CPURegister& reg6,
251 const CPURegister& reg7, const CPURegister& reg8) {
252 DCHECK(reg1.IsValid());
253 bool match = true;
254 match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
255 match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
256 match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
257 match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
258 match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
259 match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
260 match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
261 return match;
262 }
263
AreSameFormat(const VRegister & reg1,const VRegister & reg2,const VRegister & reg3,const VRegister & reg4)264 bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
265 const VRegister& reg3, const VRegister& reg4) {
266 DCHECK(reg1.IsValid());
267 return (!reg2.IsValid() || reg2.IsSameFormat(reg1)) &&
268 (!reg3.IsValid() || reg3.IsSameFormat(reg1)) &&
269 (!reg4.IsValid() || reg4.IsSameFormat(reg1));
270 }
271
AreConsecutive(const VRegister & reg1,const VRegister & reg2,const VRegister & reg3,const VRegister & reg4)272 bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
273 const VRegister& reg3, const VRegister& reg4) {
274 DCHECK(reg1.IsValid());
275 if (!reg2.IsValid()) {
276 DCHECK(!reg3.IsValid() && !reg4.IsValid());
277 return true;
278 } else if (reg2.code() != ((reg1.code() + 1) % kNumberOfVRegisters)) {
279 return false;
280 }
281
282 if (!reg3.IsValid()) {
283 DCHECK(!reg4.IsValid());
284 return true;
285 } else if (reg3.code() != ((reg2.code() + 1) % kNumberOfVRegisters)) {
286 return false;
287 }
288
289 if (!reg4.IsValid()) {
290 return true;
291 } else if (reg4.code() != ((reg3.code() + 1) % kNumberOfVRegisters)) {
292 return false;
293 }
294
295 return true;
296 }
297
InitializeHandle(Handle<HeapObject> handle)298 void Immediate::InitializeHandle(Handle<HeapObject> handle) {
299 value_ = static_cast<intptr_t>(handle.address());
300 rmode_ = RelocInfo::EMBEDDED_OBJECT;
301 }
302
303
NeedsRelocation(const Assembler * assembler) const304 bool Operand::NeedsRelocation(const Assembler* assembler) const {
305 RelocInfo::Mode rmode = immediate_.rmode();
306
307 if (rmode == RelocInfo::EXTERNAL_REFERENCE) {
308 return assembler->serializer_enabled();
309 }
310
311 return !RelocInfo::IsNone(rmode);
312 }
313
AddSharedEntry(SharedEntryMap & entry_map,uint64_t data,int offset)314 bool ConstPool::AddSharedEntry(SharedEntryMap& entry_map, uint64_t data,
315 int offset) {
316 auto existing = entry_map.find(data);
317 if (existing == entry_map.end()) {
318 entry_map[data] = static_cast<int>(entries_.size());
319 entries_.push_back(std::make_pair(data, std::vector<int>(1, offset)));
320 return true;
321 }
322 int index = existing->second;
323 entries_[index].second.push_back(offset);
324 return false;
325 }
326
327 // Constant Pool.
RecordEntry(intptr_t data,RelocInfo::Mode mode)328 bool ConstPool::RecordEntry(intptr_t data, RelocInfo::Mode mode) {
329 DCHECK(mode != RelocInfo::COMMENT && mode != RelocInfo::CONST_POOL &&
330 mode != RelocInfo::VENEER_POOL &&
331 mode != RelocInfo::DEOPT_SCRIPT_OFFSET &&
332 mode != RelocInfo::DEOPT_INLINING_ID &&
333 mode != RelocInfo::DEOPT_REASON && mode != RelocInfo::DEOPT_ID);
334
335 bool write_reloc_info = true;
336
337 uint64_t raw_data = static_cast<uint64_t>(data);
338 int offset = assm_->pc_offset();
339 if (IsEmpty()) {
340 first_use_ = offset;
341 }
342
343 if (CanBeShared(mode)) {
344 write_reloc_info = AddSharedEntry(shared_entries_, raw_data, offset);
345 } else if (mode == RelocInfo::CODE_TARGET &&
346 assm_->IsCodeTargetSharingAllowed() && raw_data != 0) {
347 // A zero data value is a placeholder and must not be shared.
348 write_reloc_info = AddSharedEntry(handle_to_index_map_, raw_data, offset);
349 } else {
350 entries_.push_back(std::make_pair(raw_data, std::vector<int>(1, offset)));
351 }
352
353 if (EntryCount() > Assembler::kApproxMaxPoolEntryCount) {
354 // Request constant pool emission after the next instruction.
355 assm_->SetNextConstPoolCheckIn(1);
356 }
357
358 return write_reloc_info;
359 }
360
361
DistanceToFirstUse()362 int ConstPool::DistanceToFirstUse() {
363 DCHECK_GE(first_use_, 0);
364 return assm_->pc_offset() - first_use_;
365 }
366
367
MaxPcOffset()368 int ConstPool::MaxPcOffset() {
369 // There are no pending entries in the pool so we can never get out of
370 // range.
371 if (IsEmpty()) return kMaxInt;
372
373 // Entries are not necessarily emitted in the order they are added so in the
374 // worst case the first constant pool use will be accessing the last entry.
375 return first_use_ + kMaxLoadLiteralRange - WorstCaseSize();
376 }
377
378
WorstCaseSize()379 int ConstPool::WorstCaseSize() {
380 if (IsEmpty()) return 0;
381
382 // Max size prologue:
383 // b over
384 // ldr xzr, #pool_size
385 // blr xzr
386 // nop
387 // All entries are 64-bit for now.
388 return 4 * kInstructionSize + EntryCount() * kPointerSize;
389 }
390
391
SizeIfEmittedAtCurrentPc(bool require_jump)392 int ConstPool::SizeIfEmittedAtCurrentPc(bool require_jump) {
393 if (IsEmpty()) return 0;
394
395 // Prologue is:
396 // b over ;; if require_jump
397 // ldr xzr, #pool_size
398 // blr xzr
399 // nop ;; if not 64-bit aligned
400 int prologue_size = require_jump ? kInstructionSize : 0;
401 prologue_size += 2 * kInstructionSize;
402 prologue_size += IsAligned(assm_->pc_offset() + prologue_size, 8) ?
403 0 : kInstructionSize;
404
405 // All entries are 64-bit for now.
406 return prologue_size + EntryCount() * kPointerSize;
407 }
408
409
Emit(bool require_jump)410 void ConstPool::Emit(bool require_jump) {
411 DCHECK(!assm_->is_const_pool_blocked());
412 // Prevent recursive pool emission and protect from veneer pools.
413 Assembler::BlockPoolsScope block_pools(assm_);
414
415 int size = SizeIfEmittedAtCurrentPc(require_jump);
416 Label size_check;
417 assm_->bind(&size_check);
418
419 assm_->RecordConstPool(size);
420 // Emit the constant pool. It is preceded by an optional branch if
421 // require_jump and a header which will:
422 // 1) Encode the size of the constant pool, for use by the disassembler.
423 // 2) Terminate the program, to try to prevent execution from accidentally
424 // flowing into the constant pool.
425 // 3) align the pool entries to 64-bit.
426 // The header is therefore made of up to three arm64 instructions:
427 // ldr xzr, #<size of the constant pool in 32-bit words>
428 // blr xzr
429 // nop
430 //
431 // If executed, the header will likely segfault and lr will point to the
432 // instruction following the offending blr.
433 // TODO(all): Make the alignment part less fragile. Currently code is
434 // allocated as a byte array so there are no guarantees the alignment will
435 // be preserved on compaction. Currently it works as allocation seems to be
436 // 64-bit aligned.
437
438 // Emit branch if required
439 Label after_pool;
440 if (require_jump) {
441 assm_->b(&after_pool);
442 }
443
444 // Emit the header.
445 assm_->RecordComment("[ Constant Pool");
446 EmitMarker();
447 EmitGuard();
448 assm_->Align(8);
449
450 // Emit constant pool entries.
451 // TODO(all): currently each relocated constant is 64 bits, consider adding
452 // support for 32-bit entries.
453 EmitEntries();
454 assm_->RecordComment("]");
455
456 if (after_pool.is_linked()) {
457 assm_->bind(&after_pool);
458 }
459
460 DCHECK(assm_->SizeOfCodeGeneratedSince(&size_check) ==
461 static_cast<unsigned>(size));
462 }
463
464
Clear()465 void ConstPool::Clear() {
466 shared_entries_.clear();
467 handle_to_index_map_.clear();
468 entries_.clear();
469 first_use_ = -1;
470 }
471
472
CanBeShared(RelocInfo::Mode mode)473 bool ConstPool::CanBeShared(RelocInfo::Mode mode) {
474 return RelocInfo::IsNone(mode) ||
475 (mode >= RelocInfo::FIRST_SHAREABLE_RELOC_MODE);
476 }
477
478
EmitMarker()479 void ConstPool::EmitMarker() {
480 // A constant pool size is expressed in number of 32-bits words.
481 // Currently all entries are 64-bit.
482 // + 1 is for the crash guard.
483 // + 0/1 for alignment.
484 int word_count = EntryCount() * 2 + 1 +
485 (IsAligned(assm_->pc_offset(), 8) ? 0 : 1);
486 assm_->Emit(LDR_x_lit |
487 Assembler::ImmLLiteral(word_count) |
488 Assembler::Rt(xzr));
489 }
490
491
AreConsistentForPair(const MemOperand & operandA,const MemOperand & operandB,int access_size_log2)492 MemOperand::PairResult MemOperand::AreConsistentForPair(
493 const MemOperand& operandA,
494 const MemOperand& operandB,
495 int access_size_log2) {
496 DCHECK_GE(access_size_log2, 0);
497 DCHECK_LE(access_size_log2, 3);
498 // Step one: check that they share the same base, that the mode is Offset
499 // and that the offset is a multiple of access size.
500 if (!operandA.base().Is(operandB.base()) ||
501 (operandA.addrmode() != Offset) ||
502 (operandB.addrmode() != Offset) ||
503 ((operandA.offset() & ((1 << access_size_log2) - 1)) != 0)) {
504 return kNotPair;
505 }
506 // Step two: check that the offsets are contiguous and that the range
507 // is OK for ldp/stp.
508 if ((operandB.offset() == operandA.offset() + (1 << access_size_log2)) &&
509 is_int7(operandA.offset() >> access_size_log2)) {
510 return kPairAB;
511 }
512 if ((operandA.offset() == operandB.offset() + (1 << access_size_log2)) &&
513 is_int7(operandB.offset() >> access_size_log2)) {
514 return kPairBA;
515 }
516 return kNotPair;
517 }
518
519
EmitGuard()520 void ConstPool::EmitGuard() {
521 #ifdef DEBUG
522 Instruction* instr = reinterpret_cast<Instruction*>(assm_->pc());
523 DCHECK(instr->preceding()->IsLdrLiteralX() &&
524 instr->preceding()->Rt() == xzr.code());
525 #endif
526 assm_->EmitPoolGuard();
527 }
528
529
EmitEntries()530 void ConstPool::EmitEntries() {
531 DCHECK(IsAligned(assm_->pc_offset(), 8));
532
533 // Emit entries.
534 for (const auto& entry : entries_) {
535 for (const auto& pc : entry.second) {
536 Instruction* instr = assm_->InstructionAt(pc);
537
538 // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0.
539 DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0);
540 instr->SetImmPCOffsetTarget(assm_->isolate_data(), assm_->pc());
541 }
542
543 assm_->dc64(entry.first);
544 }
545 Clear();
546 }
547
548
549 // Assembler
Assembler(IsolateData isolate_data,void * buffer,int buffer_size)550 Assembler::Assembler(IsolateData isolate_data, void* buffer, int buffer_size)
551 : AssemblerBase(isolate_data, buffer, buffer_size),
552 constpool_(this),
553 unresolved_branches_() {
554 const_pool_blocked_nesting_ = 0;
555 veneer_pool_blocked_nesting_ = 0;
556 code_target_sharing_blocked_nesting_ = 0;
557 Reset();
558 }
559
560
~Assembler()561 Assembler::~Assembler() {
562 DCHECK(constpool_.IsEmpty());
563 DCHECK_EQ(const_pool_blocked_nesting_, 0);
564 DCHECK_EQ(veneer_pool_blocked_nesting_, 0);
565 DCHECK_EQ(code_target_sharing_blocked_nesting_, 0);
566 }
567
568
Reset()569 void Assembler::Reset() {
570 #ifdef DEBUG
571 DCHECK((pc_ >= buffer_) && (pc_ < buffer_ + buffer_size_));
572 DCHECK_EQ(const_pool_blocked_nesting_, 0);
573 DCHECK_EQ(veneer_pool_blocked_nesting_, 0);
574 DCHECK_EQ(code_target_sharing_blocked_nesting_, 0);
575 DCHECK(unresolved_branches_.empty());
576 memset(buffer_, 0, pc_ - buffer_);
577 #endif
578 pc_ = buffer_;
579 code_targets_.reserve(64);
580 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
581 constpool_.Clear();
582 next_constant_pool_check_ = 0;
583 next_veneer_pool_check_ = kMaxInt;
584 no_const_pool_before_ = 0;
585 }
586
AllocateAndInstallRequestedHeapObjects(Isolate * isolate)587 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
588 for (auto& request : heap_object_requests_) {
589 Address pc = reinterpret_cast<Address>(buffer_) + request.offset();
590 switch (request.kind()) {
591 case HeapObjectRequest::kHeapNumber: {
592 Handle<HeapObject> object = isolate->factory()->NewHeapNumber(
593 request.heap_number(), IMMUTABLE, TENURED);
594 set_target_address_at(pc, 0 /* unused */, object.address());
595 break;
596 }
597 case HeapObjectRequest::kCodeStub: {
598 request.code_stub()->set_isolate(isolate);
599 Instruction* instr = reinterpret_cast<Instruction*>(pc);
600 DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
601 DCHECK_GE(instr->ImmPCOffset(), 0);
602 DCHECK_EQ(instr->ImmPCOffset() % kInstructionSize, 0);
603 DCHECK_LT(instr->ImmPCOffset() >> kInstructionSizeLog2,
604 code_targets_.size());
605 code_targets_[instr->ImmPCOffset() >> kInstructionSizeLog2] =
606 request.code_stub()->GetCode();
607 break;
608 }
609 }
610 }
611 }
612
GetCode(Isolate * isolate,CodeDesc * desc)613 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc) {
614 // Emit constant pool if necessary.
615 CheckConstPool(true, false);
616 DCHECK(constpool_.IsEmpty());
617
618 AllocateAndInstallRequestedHeapObjects(isolate);
619
620 // Set up code descriptor.
621 if (desc) {
622 desc->buffer = reinterpret_cast<byte*>(buffer_);
623 desc->buffer_size = buffer_size_;
624 desc->instr_size = pc_offset();
625 desc->reloc_size =
626 static_cast<int>((reinterpret_cast<byte*>(buffer_) + buffer_size_) -
627 reloc_info_writer.pos());
628 desc->origin = this;
629 desc->constant_pool_size = 0;
630 desc->unwinding_info_size = 0;
631 desc->unwinding_info = nullptr;
632 }
633 }
634
635
Align(int m)636 void Assembler::Align(int m) {
637 DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m));
638 while ((pc_offset() & (m - 1)) != 0) {
639 nop();
640 }
641 }
642
643
CheckLabelLinkChain(Label const * label)644 void Assembler::CheckLabelLinkChain(Label const * label) {
645 #ifdef DEBUG
646 if (label->is_linked()) {
647 static const int kMaxLinksToCheck = 64; // Avoid O(n2) behaviour.
648 int links_checked = 0;
649 int64_t linkoffset = label->pos();
650 bool end_of_chain = false;
651 while (!end_of_chain) {
652 if (++links_checked > kMaxLinksToCheck) break;
653 Instruction * link = InstructionAt(linkoffset);
654 int64_t linkpcoffset = link->ImmPCOffset();
655 int64_t prevlinkoffset = linkoffset + linkpcoffset;
656
657 end_of_chain = (linkoffset == prevlinkoffset);
658 linkoffset = linkoffset + linkpcoffset;
659 }
660 }
661 #endif
662 }
663
664
RemoveBranchFromLabelLinkChain(Instruction * branch,Label * label,Instruction * label_veneer)665 void Assembler::RemoveBranchFromLabelLinkChain(Instruction* branch,
666 Label* label,
667 Instruction* label_veneer) {
668 DCHECK(label->is_linked());
669
670 CheckLabelLinkChain(label);
671
672 Instruction* link = InstructionAt(label->pos());
673 Instruction* prev_link = link;
674 Instruction* next_link;
675 bool end_of_chain = false;
676
677 while (link != branch && !end_of_chain) {
678 next_link = link->ImmPCOffsetTarget();
679 end_of_chain = (link == next_link);
680 prev_link = link;
681 link = next_link;
682 }
683
684 DCHECK(branch == link);
685 next_link = branch->ImmPCOffsetTarget();
686
687 if (branch == prev_link) {
688 // The branch is the first instruction in the chain.
689 if (branch == next_link) {
690 // It is also the last instruction in the chain, so it is the only branch
691 // currently referring to this label.
692 label->Unuse();
693 } else {
694 label->link_to(
695 static_cast<int>(reinterpret_cast<byte*>(next_link) - buffer_));
696 }
697
698 } else if (branch == next_link) {
699 // The branch is the last (but not also the first) instruction in the chain.
700 prev_link->SetImmPCOffsetTarget(isolate_data(), prev_link);
701
702 } else {
703 // The branch is in the middle of the chain.
704 if (prev_link->IsTargetInImmPCOffsetRange(next_link)) {
705 prev_link->SetImmPCOffsetTarget(isolate_data(), next_link);
706 } else if (label_veneer != nullptr) {
707 // Use the veneer for all previous links in the chain.
708 prev_link->SetImmPCOffsetTarget(isolate_data(), prev_link);
709
710 end_of_chain = false;
711 link = next_link;
712 while (!end_of_chain) {
713 next_link = link->ImmPCOffsetTarget();
714 end_of_chain = (link == next_link);
715 link->SetImmPCOffsetTarget(isolate_data(), label_veneer);
716 link = next_link;
717 }
718 } else {
719 // The assert below will fire.
720 // Some other work could be attempted to fix up the chain, but it would be
721 // rather complicated. If we crash here, we may want to consider using an
722 // other mechanism than a chain of branches.
723 //
724 // Note that this situation currently should not happen, as we always call
725 // this function with a veneer to the target label.
726 // However this could happen with a MacroAssembler in the following state:
727 // [previous code]
728 // B(label);
729 // [20KB code]
730 // Tbz(label); // First tbz. Pointing to unconditional branch.
731 // [20KB code]
732 // Tbz(label); // Second tbz. Pointing to the first tbz.
733 // [more code]
734 // and this function is called to remove the first tbz from the label link
735 // chain. Since tbz has a range of +-32KB, the second tbz cannot point to
736 // the unconditional branch.
737 CHECK(prev_link->IsTargetInImmPCOffsetRange(next_link));
738 UNREACHABLE();
739 }
740 }
741
742 CheckLabelLinkChain(label);
743 }
744
745
bind(Label * label)746 void Assembler::bind(Label* label) {
747 // Bind label to the address at pc_. All instructions (most likely branches)
748 // that are linked to this label will be updated to point to the newly-bound
749 // label.
750
751 DCHECK(!label->is_near_linked());
752 DCHECK(!label->is_bound());
753
754 DeleteUnresolvedBranchInfoForLabel(label);
755
756 // If the label is linked, the link chain looks something like this:
757 //
758 // |--I----I-------I-------L
759 // |---------------------->| pc_offset
760 // |-------------->| linkoffset = label->pos()
761 // |<------| link->ImmPCOffset()
762 // |------>| prevlinkoffset = linkoffset + link->ImmPCOffset()
763 //
764 // On each iteration, the last link is updated and then removed from the
765 // chain until only one remains. At that point, the label is bound.
766 //
767 // If the label is not linked, no preparation is required before binding.
768 while (label->is_linked()) {
769 int linkoffset = label->pos();
770 Instruction* link = InstructionAt(linkoffset);
771 int prevlinkoffset = linkoffset + static_cast<int>(link->ImmPCOffset());
772
773 CheckLabelLinkChain(label);
774
775 DCHECK_GE(linkoffset, 0);
776 DCHECK(linkoffset < pc_offset());
777 DCHECK((linkoffset > prevlinkoffset) ||
778 (linkoffset - prevlinkoffset == kStartOfLabelLinkChain));
779 DCHECK_GE(prevlinkoffset, 0);
780
781 // Update the link to point to the label.
782 if (link->IsUnresolvedInternalReference()) {
783 // Internal references do not get patched to an instruction but directly
784 // to an address.
785 internal_reference_positions_.push_back(linkoffset);
786 PatchingAssembler patcher(isolate_data(), reinterpret_cast<byte*>(link),
787 2);
788 patcher.dc64(reinterpret_cast<uintptr_t>(pc_));
789 } else {
790 link->SetImmPCOffsetTarget(isolate_data(),
791 reinterpret_cast<Instruction*>(pc_));
792 }
793
794 // Link the label to the previous link in the chain.
795 if (linkoffset - prevlinkoffset == kStartOfLabelLinkChain) {
796 // We hit kStartOfLabelLinkChain, so the chain is fully processed.
797 label->Unuse();
798 } else {
799 // Update the label for the next iteration.
800 label->link_to(prevlinkoffset);
801 }
802 }
803 label->bind_to(pc_offset());
804
805 DCHECK(label->is_bound());
806 DCHECK(!label->is_linked());
807 }
808
809
LinkAndGetByteOffsetTo(Label * label)810 int Assembler::LinkAndGetByteOffsetTo(Label* label) {
811 DCHECK_EQ(sizeof(*pc_), 1);
812 CheckLabelLinkChain(label);
813
814 int offset;
815 if (label->is_bound()) {
816 // The label is bound, so it does not need to be updated. Referring
817 // instructions must link directly to the label as they will not be
818 // updated.
819 //
820 // In this case, label->pos() returns the offset of the label from the
821 // start of the buffer.
822 //
823 // Note that offset can be zero for self-referential instructions. (This
824 // could be useful for ADR, for example.)
825 offset = label->pos() - pc_offset();
826 DCHECK_LE(offset, 0);
827 } else {
828 if (label->is_linked()) {
829 // The label is linked, so the referring instruction should be added onto
830 // the end of the label's link chain.
831 //
832 // In this case, label->pos() returns the offset of the last linked
833 // instruction from the start of the buffer.
834 offset = label->pos() - pc_offset();
835 DCHECK_NE(offset, kStartOfLabelLinkChain);
836 // Note that the offset here needs to be PC-relative only so that the
837 // first instruction in a buffer can link to an unbound label. Otherwise,
838 // the offset would be 0 for this case, and 0 is reserved for
839 // kStartOfLabelLinkChain.
840 } else {
841 // The label is unused, so it now becomes linked and the referring
842 // instruction is at the start of the new link chain.
843 offset = kStartOfLabelLinkChain;
844 }
845 // The instruction at pc is now the last link in the label's chain.
846 label->link_to(pc_offset());
847 }
848
849 return offset;
850 }
851
852
DeleteUnresolvedBranchInfoForLabelTraverse(Label * label)853 void Assembler::DeleteUnresolvedBranchInfoForLabelTraverse(Label* label) {
854 DCHECK(label->is_linked());
855 CheckLabelLinkChain(label);
856
857 int link_offset = label->pos();
858 int link_pcoffset;
859 bool end_of_chain = false;
860
861 while (!end_of_chain) {
862 Instruction * link = InstructionAt(link_offset);
863 link_pcoffset = static_cast<int>(link->ImmPCOffset());
864
865 // ADR instructions are not handled by veneers.
866 if (link->IsImmBranch()) {
867 int max_reachable_pc =
868 static_cast<int>(InstructionOffset(link) +
869 Instruction::ImmBranchRange(link->BranchType()));
870 typedef std::multimap<int, FarBranchInfo>::iterator unresolved_info_it;
871 std::pair<unresolved_info_it, unresolved_info_it> range;
872 range = unresolved_branches_.equal_range(max_reachable_pc);
873 unresolved_info_it it;
874 for (it = range.first; it != range.second; ++it) {
875 if (it->second.pc_offset_ == link_offset) {
876 unresolved_branches_.erase(it);
877 break;
878 }
879 }
880 }
881
882 end_of_chain = (link_pcoffset == 0);
883 link_offset = link_offset + link_pcoffset;
884 }
885 }
886
887
DeleteUnresolvedBranchInfoForLabel(Label * label)888 void Assembler::DeleteUnresolvedBranchInfoForLabel(Label* label) {
889 if (unresolved_branches_.empty()) {
890 DCHECK_EQ(next_veneer_pool_check_, kMaxInt);
891 return;
892 }
893
894 if (label->is_linked()) {
895 // Branches to this label will be resolved when the label is bound, normally
896 // just after all the associated info has been deleted.
897 DeleteUnresolvedBranchInfoForLabelTraverse(label);
898 }
899 if (unresolved_branches_.empty()) {
900 next_veneer_pool_check_ = kMaxInt;
901 } else {
902 next_veneer_pool_check_ =
903 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
904 }
905 }
906
907
StartBlockConstPool()908 void Assembler::StartBlockConstPool() {
909 if (const_pool_blocked_nesting_++ == 0) {
910 // Prevent constant pool checks happening by setting the next check to
911 // the biggest possible offset.
912 next_constant_pool_check_ = kMaxInt;
913 }
914 }
915
916
EndBlockConstPool()917 void Assembler::EndBlockConstPool() {
918 if (--const_pool_blocked_nesting_ == 0) {
919 // Check the constant pool hasn't been blocked for too long.
920 DCHECK(pc_offset() < constpool_.MaxPcOffset());
921 // Two cases:
922 // * no_const_pool_before_ >= next_constant_pool_check_ and the emission is
923 // still blocked
924 // * no_const_pool_before_ < next_constant_pool_check_ and the next emit
925 // will trigger a check.
926 next_constant_pool_check_ = no_const_pool_before_;
927 }
928 }
929
930
is_const_pool_blocked() const931 bool Assembler::is_const_pool_blocked() const {
932 return (const_pool_blocked_nesting_ > 0) ||
933 (pc_offset() < no_const_pool_before_);
934 }
935
936
IsConstantPoolAt(Instruction * instr)937 bool Assembler::IsConstantPoolAt(Instruction* instr) {
938 // The constant pool marker is made of two instructions. These instructions
939 // will never be emitted by the JIT, so checking for the first one is enough:
940 // 0: ldr xzr, #<size of pool>
941 bool result = instr->IsLdrLiteralX() && (instr->Rt() == kZeroRegCode);
942
943 // It is still worth asserting the marker is complete.
944 // 4: blr xzr
945 DCHECK(!result || (instr->following()->IsBranchAndLinkToRegister() &&
946 instr->following()->Rn() == kZeroRegCode));
947
948 return result;
949 }
950
951
ConstantPoolSizeAt(Instruction * instr)952 int Assembler::ConstantPoolSizeAt(Instruction* instr) {
953 #ifdef USE_SIMULATOR
954 // Assembler::debug() embeds constants directly into the instruction stream.
955 // Although this is not a genuine constant pool, treat it like one to avoid
956 // disassembling the constants.
957 if ((instr->Mask(ExceptionMask) == HLT) &&
958 (instr->ImmException() == kImmExceptionIsDebug)) {
959 const char* message =
960 reinterpret_cast<const char*>(
961 instr->InstructionAtOffset(kDebugMessageOffset));
962 int size = static_cast<int>(kDebugMessageOffset + strlen(message) + 1);
963 return RoundUp(size, kInstructionSize) / kInstructionSize;
964 }
965 // Same for printf support, see MacroAssembler::CallPrintf().
966 if ((instr->Mask(ExceptionMask) == HLT) &&
967 (instr->ImmException() == kImmExceptionIsPrintf)) {
968 return kPrintfLength / kInstructionSize;
969 }
970 #endif
971 if (IsConstantPoolAt(instr)) {
972 return instr->ImmLLiteral();
973 } else {
974 return -1;
975 }
976 }
977
978
EmitPoolGuard()979 void Assembler::EmitPoolGuard() {
980 // We must generate only one instruction as this is used in scopes that
981 // control the size of the code generated.
982 Emit(BLR | Rn(xzr));
983 }
984
985
StartBlockVeneerPool()986 void Assembler::StartBlockVeneerPool() {
987 ++veneer_pool_blocked_nesting_;
988 }
989
990
EndBlockVeneerPool()991 void Assembler::EndBlockVeneerPool() {
992 if (--veneer_pool_blocked_nesting_ == 0) {
993 // Check the veneer pool hasn't been blocked for too long.
994 DCHECK(unresolved_branches_.empty() ||
995 (pc_offset() < unresolved_branches_first_limit()));
996 }
997 }
998
999
br(const Register & xn)1000 void Assembler::br(const Register& xn) {
1001 DCHECK(xn.Is64Bits());
1002 Emit(BR | Rn(xn));
1003 }
1004
1005
blr(const Register & xn)1006 void Assembler::blr(const Register& xn) {
1007 DCHECK(xn.Is64Bits());
1008 // The pattern 'blr xzr' is used as a guard to detect when execution falls
1009 // through the constant pool. It should not be emitted.
1010 DCHECK(!xn.Is(xzr));
1011 Emit(BLR | Rn(xn));
1012 }
1013
1014
ret(const Register & xn)1015 void Assembler::ret(const Register& xn) {
1016 DCHECK(xn.Is64Bits());
1017 Emit(RET | Rn(xn));
1018 }
1019
1020
b(int imm26)1021 void Assembler::b(int imm26) {
1022 Emit(B | ImmUncondBranch(imm26));
1023 }
1024
1025
b(Label * label)1026 void Assembler::b(Label* label) {
1027 b(LinkAndGetInstructionOffsetTo(label));
1028 }
1029
1030
b(int imm19,Condition cond)1031 void Assembler::b(int imm19, Condition cond) {
1032 Emit(B_cond | ImmCondBranch(imm19) | cond);
1033 }
1034
1035
b(Label * label,Condition cond)1036 void Assembler::b(Label* label, Condition cond) {
1037 b(LinkAndGetInstructionOffsetTo(label), cond);
1038 }
1039
1040
bl(int imm26)1041 void Assembler::bl(int imm26) {
1042 Emit(BL | ImmUncondBranch(imm26));
1043 }
1044
1045
bl(Label * label)1046 void Assembler::bl(Label* label) {
1047 bl(LinkAndGetInstructionOffsetTo(label));
1048 }
1049
1050
cbz(const Register & rt,int imm19)1051 void Assembler::cbz(const Register& rt,
1052 int imm19) {
1053 Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
1054 }
1055
1056
cbz(const Register & rt,Label * label)1057 void Assembler::cbz(const Register& rt,
1058 Label* label) {
1059 cbz(rt, LinkAndGetInstructionOffsetTo(label));
1060 }
1061
1062
cbnz(const Register & rt,int imm19)1063 void Assembler::cbnz(const Register& rt,
1064 int imm19) {
1065 Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
1066 }
1067
1068
cbnz(const Register & rt,Label * label)1069 void Assembler::cbnz(const Register& rt,
1070 Label* label) {
1071 cbnz(rt, LinkAndGetInstructionOffsetTo(label));
1072 }
1073
1074
tbz(const Register & rt,unsigned bit_pos,int imm14)1075 void Assembler::tbz(const Register& rt,
1076 unsigned bit_pos,
1077 int imm14) {
1078 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits)));
1079 Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
1080 }
1081
1082
tbz(const Register & rt,unsigned bit_pos,Label * label)1083 void Assembler::tbz(const Register& rt,
1084 unsigned bit_pos,
1085 Label* label) {
1086 tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label));
1087 }
1088
1089
tbnz(const Register & rt,unsigned bit_pos,int imm14)1090 void Assembler::tbnz(const Register& rt,
1091 unsigned bit_pos,
1092 int imm14) {
1093 DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits)));
1094 Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
1095 }
1096
1097
tbnz(const Register & rt,unsigned bit_pos,Label * label)1098 void Assembler::tbnz(const Register& rt,
1099 unsigned bit_pos,
1100 Label* label) {
1101 tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label));
1102 }
1103
1104
adr(const Register & rd,int imm21)1105 void Assembler::adr(const Register& rd, int imm21) {
1106 DCHECK(rd.Is64Bits());
1107 Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd));
1108 }
1109
1110
adr(const Register & rd,Label * label)1111 void Assembler::adr(const Register& rd, Label* label) {
1112 adr(rd, LinkAndGetByteOffsetTo(label));
1113 }
1114
1115
add(const Register & rd,const Register & rn,const Operand & operand)1116 void Assembler::add(const Register& rd,
1117 const Register& rn,
1118 const Operand& operand) {
1119 AddSub(rd, rn, operand, LeaveFlags, ADD);
1120 }
1121
1122
adds(const Register & rd,const Register & rn,const Operand & operand)1123 void Assembler::adds(const Register& rd,
1124 const Register& rn,
1125 const Operand& operand) {
1126 AddSub(rd, rn, operand, SetFlags, ADD);
1127 }
1128
1129
cmn(const Register & rn,const Operand & operand)1130 void Assembler::cmn(const Register& rn,
1131 const Operand& operand) {
1132 Register zr = AppropriateZeroRegFor(rn);
1133 adds(zr, rn, operand);
1134 }
1135
1136
sub(const Register & rd,const Register & rn,const Operand & operand)1137 void Assembler::sub(const Register& rd,
1138 const Register& rn,
1139 const Operand& operand) {
1140 AddSub(rd, rn, operand, LeaveFlags, SUB);
1141 }
1142
1143
subs(const Register & rd,const Register & rn,const Operand & operand)1144 void Assembler::subs(const Register& rd,
1145 const Register& rn,
1146 const Operand& operand) {
1147 AddSub(rd, rn, operand, SetFlags, SUB);
1148 }
1149
1150
cmp(const Register & rn,const Operand & operand)1151 void Assembler::cmp(const Register& rn, const Operand& operand) {
1152 Register zr = AppropriateZeroRegFor(rn);
1153 subs(zr, rn, operand);
1154 }
1155
1156
neg(const Register & rd,const Operand & operand)1157 void Assembler::neg(const Register& rd, const Operand& operand) {
1158 Register zr = AppropriateZeroRegFor(rd);
1159 sub(rd, zr, operand);
1160 }
1161
1162
negs(const Register & rd,const Operand & operand)1163 void Assembler::negs(const Register& rd, const Operand& operand) {
1164 Register zr = AppropriateZeroRegFor(rd);
1165 subs(rd, zr, operand);
1166 }
1167
1168
adc(const Register & rd,const Register & rn,const Operand & operand)1169 void Assembler::adc(const Register& rd,
1170 const Register& rn,
1171 const Operand& operand) {
1172 AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
1173 }
1174
1175
adcs(const Register & rd,const Register & rn,const Operand & operand)1176 void Assembler::adcs(const Register& rd,
1177 const Register& rn,
1178 const Operand& operand) {
1179 AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
1180 }
1181
1182
sbc(const Register & rd,const Register & rn,const Operand & operand)1183 void Assembler::sbc(const Register& rd,
1184 const Register& rn,
1185 const Operand& operand) {
1186 AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
1187 }
1188
1189
sbcs(const Register & rd,const Register & rn,const Operand & operand)1190 void Assembler::sbcs(const Register& rd,
1191 const Register& rn,
1192 const Operand& operand) {
1193 AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
1194 }
1195
1196
ngc(const Register & rd,const Operand & operand)1197 void Assembler::ngc(const Register& rd, const Operand& operand) {
1198 Register zr = AppropriateZeroRegFor(rd);
1199 sbc(rd, zr, operand);
1200 }
1201
1202
ngcs(const Register & rd,const Operand & operand)1203 void Assembler::ngcs(const Register& rd, const Operand& operand) {
1204 Register zr = AppropriateZeroRegFor(rd);
1205 sbcs(rd, zr, operand);
1206 }
1207
1208
1209 // Logical instructions.
and_(const Register & rd,const Register & rn,const Operand & operand)1210 void Assembler::and_(const Register& rd,
1211 const Register& rn,
1212 const Operand& operand) {
1213 Logical(rd, rn, operand, AND);
1214 }
1215
1216
ands(const Register & rd,const Register & rn,const Operand & operand)1217 void Assembler::ands(const Register& rd,
1218 const Register& rn,
1219 const Operand& operand) {
1220 Logical(rd, rn, operand, ANDS);
1221 }
1222
1223
tst(const Register & rn,const Operand & operand)1224 void Assembler::tst(const Register& rn,
1225 const Operand& operand) {
1226 ands(AppropriateZeroRegFor(rn), rn, operand);
1227 }
1228
1229
bic(const Register & rd,const Register & rn,const Operand & operand)1230 void Assembler::bic(const Register& rd,
1231 const Register& rn,
1232 const Operand& operand) {
1233 Logical(rd, rn, operand, BIC);
1234 }
1235
1236
bics(const Register & rd,const Register & rn,const Operand & operand)1237 void Assembler::bics(const Register& rd,
1238 const Register& rn,
1239 const Operand& operand) {
1240 Logical(rd, rn, operand, BICS);
1241 }
1242
1243
orr(const Register & rd,const Register & rn,const Operand & operand)1244 void Assembler::orr(const Register& rd,
1245 const Register& rn,
1246 const Operand& operand) {
1247 Logical(rd, rn, operand, ORR);
1248 }
1249
1250
orn(const Register & rd,const Register & rn,const Operand & operand)1251 void Assembler::orn(const Register& rd,
1252 const Register& rn,
1253 const Operand& operand) {
1254 Logical(rd, rn, operand, ORN);
1255 }
1256
1257
eor(const Register & rd,const Register & rn,const Operand & operand)1258 void Assembler::eor(const Register& rd,
1259 const Register& rn,
1260 const Operand& operand) {
1261 Logical(rd, rn, operand, EOR);
1262 }
1263
1264
eon(const Register & rd,const Register & rn,const Operand & operand)1265 void Assembler::eon(const Register& rd,
1266 const Register& rn,
1267 const Operand& operand) {
1268 Logical(rd, rn, operand, EON);
1269 }
1270
1271
lslv(const Register & rd,const Register & rn,const Register & rm)1272 void Assembler::lslv(const Register& rd,
1273 const Register& rn,
1274 const Register& rm) {
1275 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1276 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1277 Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
1278 }
1279
1280
lsrv(const Register & rd,const Register & rn,const Register & rm)1281 void Assembler::lsrv(const Register& rd,
1282 const Register& rn,
1283 const Register& rm) {
1284 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1285 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1286 Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
1287 }
1288
1289
asrv(const Register & rd,const Register & rn,const Register & rm)1290 void Assembler::asrv(const Register& rd,
1291 const Register& rn,
1292 const Register& rm) {
1293 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1294 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1295 Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
1296 }
1297
1298
rorv(const Register & rd,const Register & rn,const Register & rm)1299 void Assembler::rorv(const Register& rd,
1300 const Register& rn,
1301 const Register& rm) {
1302 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1303 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1304 Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
1305 }
1306
1307
1308 // Bitfield operations.
bfm(const Register & rd,const Register & rn,int immr,int imms)1309 void Assembler::bfm(const Register& rd, const Register& rn, int immr,
1310 int imms) {
1311 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1312 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1313 Emit(SF(rd) | BFM | N |
1314 ImmR(immr, rd.SizeInBits()) |
1315 ImmS(imms, rn.SizeInBits()) |
1316 Rn(rn) | Rd(rd));
1317 }
1318
1319
sbfm(const Register & rd,const Register & rn,int immr,int imms)1320 void Assembler::sbfm(const Register& rd, const Register& rn, int immr,
1321 int imms) {
1322 DCHECK(rd.Is64Bits() || rn.Is32Bits());
1323 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1324 Emit(SF(rd) | SBFM | N |
1325 ImmR(immr, rd.SizeInBits()) |
1326 ImmS(imms, rn.SizeInBits()) |
1327 Rn(rn) | Rd(rd));
1328 }
1329
1330
ubfm(const Register & rd,const Register & rn,int immr,int imms)1331 void Assembler::ubfm(const Register& rd, const Register& rn, int immr,
1332 int imms) {
1333 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1334 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1335 Emit(SF(rd) | UBFM | N |
1336 ImmR(immr, rd.SizeInBits()) |
1337 ImmS(imms, rn.SizeInBits()) |
1338 Rn(rn) | Rd(rd));
1339 }
1340
1341
extr(const Register & rd,const Register & rn,const Register & rm,int lsb)1342 void Assembler::extr(const Register& rd, const Register& rn, const Register& rm,
1343 int lsb) {
1344 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1345 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1346 Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
1347 Emit(SF(rd) | EXTR | N | Rm(rm) |
1348 ImmS(lsb, rn.SizeInBits()) | Rn(rn) | Rd(rd));
1349 }
1350
1351
csel(const Register & rd,const Register & rn,const Register & rm,Condition cond)1352 void Assembler::csel(const Register& rd,
1353 const Register& rn,
1354 const Register& rm,
1355 Condition cond) {
1356 ConditionalSelect(rd, rn, rm, cond, CSEL);
1357 }
1358
1359
csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)1360 void Assembler::csinc(const Register& rd,
1361 const Register& rn,
1362 const Register& rm,
1363 Condition cond) {
1364 ConditionalSelect(rd, rn, rm, cond, CSINC);
1365 }
1366
1367
csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)1368 void Assembler::csinv(const Register& rd,
1369 const Register& rn,
1370 const Register& rm,
1371 Condition cond) {
1372 ConditionalSelect(rd, rn, rm, cond, CSINV);
1373 }
1374
1375
csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)1376 void Assembler::csneg(const Register& rd,
1377 const Register& rn,
1378 const Register& rm,
1379 Condition cond) {
1380 ConditionalSelect(rd, rn, rm, cond, CSNEG);
1381 }
1382
1383
cset(const Register & rd,Condition cond)1384 void Assembler::cset(const Register &rd, Condition cond) {
1385 DCHECK((cond != al) && (cond != nv));
1386 Register zr = AppropriateZeroRegFor(rd);
1387 csinc(rd, zr, zr, NegateCondition(cond));
1388 }
1389
1390
csetm(const Register & rd,Condition cond)1391 void Assembler::csetm(const Register &rd, Condition cond) {
1392 DCHECK((cond != al) && (cond != nv));
1393 Register zr = AppropriateZeroRegFor(rd);
1394 csinv(rd, zr, zr, NegateCondition(cond));
1395 }
1396
1397
cinc(const Register & rd,const Register & rn,Condition cond)1398 void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) {
1399 DCHECK((cond != al) && (cond != nv));
1400 csinc(rd, rn, rn, NegateCondition(cond));
1401 }
1402
1403
cinv(const Register & rd,const Register & rn,Condition cond)1404 void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) {
1405 DCHECK((cond != al) && (cond != nv));
1406 csinv(rd, rn, rn, NegateCondition(cond));
1407 }
1408
1409
cneg(const Register & rd,const Register & rn,Condition cond)1410 void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) {
1411 DCHECK((cond != al) && (cond != nv));
1412 csneg(rd, rn, rn, NegateCondition(cond));
1413 }
1414
1415
ConditionalSelect(const Register & rd,const Register & rn,const Register & rm,Condition cond,ConditionalSelectOp op)1416 void Assembler::ConditionalSelect(const Register& rd,
1417 const Register& rn,
1418 const Register& rm,
1419 Condition cond,
1420 ConditionalSelectOp op) {
1421 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1422 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1423 Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
1424 }
1425
1426
ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)1427 void Assembler::ccmn(const Register& rn,
1428 const Operand& operand,
1429 StatusFlags nzcv,
1430 Condition cond) {
1431 ConditionalCompare(rn, operand, nzcv, cond, CCMN);
1432 }
1433
1434
ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)1435 void Assembler::ccmp(const Register& rn,
1436 const Operand& operand,
1437 StatusFlags nzcv,
1438 Condition cond) {
1439 ConditionalCompare(rn, operand, nzcv, cond, CCMP);
1440 }
1441
1442
DataProcessing3Source(const Register & rd,const Register & rn,const Register & rm,const Register & ra,DataProcessing3SourceOp op)1443 void Assembler::DataProcessing3Source(const Register& rd,
1444 const Register& rn,
1445 const Register& rm,
1446 const Register& ra,
1447 DataProcessing3SourceOp op) {
1448 Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
1449 }
1450
1451
mul(const Register & rd,const Register & rn,const Register & rm)1452 void Assembler::mul(const Register& rd,
1453 const Register& rn,
1454 const Register& rm) {
1455 DCHECK(AreSameSizeAndType(rd, rn, rm));
1456 Register zr = AppropriateZeroRegFor(rn);
1457 DataProcessing3Source(rd, rn, rm, zr, MADD);
1458 }
1459
1460
madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1461 void Assembler::madd(const Register& rd,
1462 const Register& rn,
1463 const Register& rm,
1464 const Register& ra) {
1465 DCHECK(AreSameSizeAndType(rd, rn, rm, ra));
1466 DataProcessing3Source(rd, rn, rm, ra, MADD);
1467 }
1468
1469
mneg(const Register & rd,const Register & rn,const Register & rm)1470 void Assembler::mneg(const Register& rd,
1471 const Register& rn,
1472 const Register& rm) {
1473 DCHECK(AreSameSizeAndType(rd, rn, rm));
1474 Register zr = AppropriateZeroRegFor(rn);
1475 DataProcessing3Source(rd, rn, rm, zr, MSUB);
1476 }
1477
1478
msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1479 void Assembler::msub(const Register& rd,
1480 const Register& rn,
1481 const Register& rm,
1482 const Register& ra) {
1483 DCHECK(AreSameSizeAndType(rd, rn, rm, ra));
1484 DataProcessing3Source(rd, rn, rm, ra, MSUB);
1485 }
1486
1487
smaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1488 void Assembler::smaddl(const Register& rd,
1489 const Register& rn,
1490 const Register& rm,
1491 const Register& ra) {
1492 DCHECK(rd.Is64Bits() && ra.Is64Bits());
1493 DCHECK(rn.Is32Bits() && rm.Is32Bits());
1494 DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
1495 }
1496
1497
smsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1498 void Assembler::smsubl(const Register& rd,
1499 const Register& rn,
1500 const Register& rm,
1501 const Register& ra) {
1502 DCHECK(rd.Is64Bits() && ra.Is64Bits());
1503 DCHECK(rn.Is32Bits() && rm.Is32Bits());
1504 DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
1505 }
1506
1507
umaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1508 void Assembler::umaddl(const Register& rd,
1509 const Register& rn,
1510 const Register& rm,
1511 const Register& ra) {
1512 DCHECK(rd.Is64Bits() && ra.Is64Bits());
1513 DCHECK(rn.Is32Bits() && rm.Is32Bits());
1514 DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
1515 }
1516
1517
umsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1518 void Assembler::umsubl(const Register& rd,
1519 const Register& rn,
1520 const Register& rm,
1521 const Register& ra) {
1522 DCHECK(rd.Is64Bits() && ra.Is64Bits());
1523 DCHECK(rn.Is32Bits() && rm.Is32Bits());
1524 DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
1525 }
1526
1527
smull(const Register & rd,const Register & rn,const Register & rm)1528 void Assembler::smull(const Register& rd,
1529 const Register& rn,
1530 const Register& rm) {
1531 DCHECK(rd.Is64Bits());
1532 DCHECK(rn.Is32Bits() && rm.Is32Bits());
1533 DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
1534 }
1535
1536
smulh(const Register & rd,const Register & rn,const Register & rm)1537 void Assembler::smulh(const Register& rd,
1538 const Register& rn,
1539 const Register& rm) {
1540 DCHECK(AreSameSizeAndType(rd, rn, rm));
1541 DataProcessing3Source(rd, rn, rm, xzr, SMULH_x);
1542 }
1543
1544
sdiv(const Register & rd,const Register & rn,const Register & rm)1545 void Assembler::sdiv(const Register& rd,
1546 const Register& rn,
1547 const Register& rm) {
1548 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1549 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1550 Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
1551 }
1552
1553
udiv(const Register & rd,const Register & rn,const Register & rm)1554 void Assembler::udiv(const Register& rd,
1555 const Register& rn,
1556 const Register& rm) {
1557 DCHECK(rd.SizeInBits() == rn.SizeInBits());
1558 DCHECK(rd.SizeInBits() == rm.SizeInBits());
1559 Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
1560 }
1561
1562
rbit(const Register & rd,const Register & rn)1563 void Assembler::rbit(const Register& rd,
1564 const Register& rn) {
1565 DataProcessing1Source(rd, rn, RBIT);
1566 }
1567
1568
rev16(const Register & rd,const Register & rn)1569 void Assembler::rev16(const Register& rd,
1570 const Register& rn) {
1571 DataProcessing1Source(rd, rn, REV16);
1572 }
1573
1574
rev32(const Register & rd,const Register & rn)1575 void Assembler::rev32(const Register& rd,
1576 const Register& rn) {
1577 DCHECK(rd.Is64Bits());
1578 DataProcessing1Source(rd, rn, REV);
1579 }
1580
1581
rev(const Register & rd,const Register & rn)1582 void Assembler::rev(const Register& rd,
1583 const Register& rn) {
1584 DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
1585 }
1586
1587
clz(const Register & rd,const Register & rn)1588 void Assembler::clz(const Register& rd,
1589 const Register& rn) {
1590 DataProcessing1Source(rd, rn, CLZ);
1591 }
1592
1593
cls(const Register & rd,const Register & rn)1594 void Assembler::cls(const Register& rd,
1595 const Register& rn) {
1596 DataProcessing1Source(rd, rn, CLS);
1597 }
1598
1599
ldp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & src)1600 void Assembler::ldp(const CPURegister& rt,
1601 const CPURegister& rt2,
1602 const MemOperand& src) {
1603 LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
1604 }
1605
1606
stp(const CPURegister & rt,const CPURegister & rt2,const MemOperand & dst)1607 void Assembler::stp(const CPURegister& rt,
1608 const CPURegister& rt2,
1609 const MemOperand& dst) {
1610 LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
1611 }
1612
1613
ldpsw(const Register & rt,const Register & rt2,const MemOperand & src)1614 void Assembler::ldpsw(const Register& rt,
1615 const Register& rt2,
1616 const MemOperand& src) {
1617 DCHECK(rt.Is64Bits());
1618 LoadStorePair(rt, rt2, src, LDPSW_x);
1619 }
1620
1621
LoadStorePair(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)1622 void Assembler::LoadStorePair(const CPURegister& rt,
1623 const CPURegister& rt2,
1624 const MemOperand& addr,
1625 LoadStorePairOp op) {
1626 // 'rt' and 'rt2' can only be aliased for stores.
1627 DCHECK(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
1628 DCHECK(AreSameSizeAndType(rt, rt2));
1629 DCHECK(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
1630 int offset = static_cast<int>(addr.offset());
1631
1632 Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
1633 ImmLSPair(offset, CalcLSPairDataSize(op));
1634
1635 Instr addrmodeop;
1636 if (addr.IsImmediateOffset()) {
1637 addrmodeop = LoadStorePairOffsetFixed;
1638 } else {
1639 // Pre-index and post-index modes.
1640 DCHECK(!rt.Is(addr.base()));
1641 DCHECK(!rt2.Is(addr.base()));
1642 DCHECK_NE(addr.offset(), 0);
1643 if (addr.IsPreIndex()) {
1644 addrmodeop = LoadStorePairPreIndexFixed;
1645 } else {
1646 DCHECK(addr.IsPostIndex());
1647 addrmodeop = LoadStorePairPostIndexFixed;
1648 }
1649 }
1650 Emit(addrmodeop | memop);
1651 }
1652
1653
1654 // Memory instructions.
ldrb(const Register & rt,const MemOperand & src)1655 void Assembler::ldrb(const Register& rt, const MemOperand& src) {
1656 LoadStore(rt, src, LDRB_w);
1657 }
1658
1659
strb(const Register & rt,const MemOperand & dst)1660 void Assembler::strb(const Register& rt, const MemOperand& dst) {
1661 LoadStore(rt, dst, STRB_w);
1662 }
1663
1664
ldrsb(const Register & rt,const MemOperand & src)1665 void Assembler::ldrsb(const Register& rt, const MemOperand& src) {
1666 LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w);
1667 }
1668
1669
ldrh(const Register & rt,const MemOperand & src)1670 void Assembler::ldrh(const Register& rt, const MemOperand& src) {
1671 LoadStore(rt, src, LDRH_w);
1672 }
1673
1674
strh(const Register & rt,const MemOperand & dst)1675 void Assembler::strh(const Register& rt, const MemOperand& dst) {
1676 LoadStore(rt, dst, STRH_w);
1677 }
1678
1679
ldrsh(const Register & rt,const MemOperand & src)1680 void Assembler::ldrsh(const Register& rt, const MemOperand& src) {
1681 LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w);
1682 }
1683
1684
ldr(const CPURegister & rt,const MemOperand & src)1685 void Assembler::ldr(const CPURegister& rt, const MemOperand& src) {
1686 LoadStore(rt, src, LoadOpFor(rt));
1687 }
1688
1689
str(const CPURegister & rt,const MemOperand & src)1690 void Assembler::str(const CPURegister& rt, const MemOperand& src) {
1691 LoadStore(rt, src, StoreOpFor(rt));
1692 }
1693
1694
ldrsw(const Register & rt,const MemOperand & src)1695 void Assembler::ldrsw(const Register& rt, const MemOperand& src) {
1696 DCHECK(rt.Is64Bits());
1697 LoadStore(rt, src, LDRSW_x);
1698 }
1699
1700
ldr_pcrel(const CPURegister & rt,int imm19)1701 void Assembler::ldr_pcrel(const CPURegister& rt, int imm19) {
1702 // The pattern 'ldr xzr, #offset' is used to indicate the beginning of a
1703 // constant pool. It should not be emitted.
1704 DCHECK(!rt.IsZero());
1705 Emit(LoadLiteralOpFor(rt) | ImmLLiteral(imm19) | Rt(rt));
1706 }
1707
EmbeddedNumber(double number)1708 Operand Operand::EmbeddedNumber(double number) {
1709 int32_t smi;
1710 if (DoubleToSmiInteger(number, &smi)) {
1711 return Operand(Immediate(Smi::FromInt(smi)));
1712 }
1713 Operand result(0, RelocInfo::EMBEDDED_OBJECT);
1714 result.heap_object_request_.emplace(number);
1715 DCHECK(result.IsHeapObjectRequest());
1716 return result;
1717 }
1718
EmbeddedCode(CodeStub * stub)1719 Operand Operand::EmbeddedCode(CodeStub* stub) {
1720 Operand result(0, RelocInfo::CODE_TARGET);
1721 result.heap_object_request_.emplace(stub);
1722 DCHECK(result.IsHeapObjectRequest());
1723 return result;
1724 }
1725
ldr(const CPURegister & rt,const Operand & operand)1726 void Assembler::ldr(const CPURegister& rt, const Operand& operand) {
1727 if (operand.IsHeapObjectRequest()) {
1728 RequestHeapObject(operand.heap_object_request());
1729 ldr(rt, operand.immediate_for_heap_object_request());
1730 } else {
1731 ldr(rt, operand.immediate());
1732 }
1733 }
1734
ldr(const CPURegister & rt,const Immediate & imm)1735 void Assembler::ldr(const CPURegister& rt, const Immediate& imm) {
1736 // Currently we only support 64-bit literals.
1737 DCHECK(rt.Is64Bits());
1738
1739 RecordRelocInfo(imm.rmode(), imm.value());
1740 BlockConstPoolFor(1);
1741 // The load will be patched when the constpool is emitted, patching code
1742 // expect a load literal with offset 0.
1743 ldr_pcrel(rt, 0);
1744 }
1745
ldar(const Register & rt,const Register & rn)1746 void Assembler::ldar(const Register& rt, const Register& rn) {
1747 DCHECK(rn.Is64Bits());
1748 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x;
1749 Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1750 }
1751
ldaxr(const Register & rt,const Register & rn)1752 void Assembler::ldaxr(const Register& rt, const Register& rn) {
1753 DCHECK(rn.Is64Bits());
1754 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x;
1755 Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1756 }
1757
stlr(const Register & rt,const Register & rn)1758 void Assembler::stlr(const Register& rt, const Register& rn) {
1759 DCHECK(rn.Is64Bits());
1760 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x;
1761 Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1762 }
1763
stlxr(const Register & rs,const Register & rt,const Register & rn)1764 void Assembler::stlxr(const Register& rs, const Register& rt,
1765 const Register& rn) {
1766 DCHECK(rn.Is64Bits());
1767 DCHECK(!rs.Is(rt) && !rs.Is(rn));
1768 LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x;
1769 Emit(op | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
1770 }
1771
ldarb(const Register & rt,const Register & rn)1772 void Assembler::ldarb(const Register& rt, const Register& rn) {
1773 DCHECK(rt.Is32Bits());
1774 DCHECK(rn.Is64Bits());
1775 Emit(LDAR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1776 }
1777
ldaxrb(const Register & rt,const Register & rn)1778 void Assembler::ldaxrb(const Register& rt, const Register& rn) {
1779 DCHECK(rt.Is32Bits());
1780 DCHECK(rn.Is64Bits());
1781 Emit(LDAXR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1782 }
1783
stlrb(const Register & rt,const Register & rn)1784 void Assembler::stlrb(const Register& rt, const Register& rn) {
1785 DCHECK(rt.Is32Bits());
1786 DCHECK(rn.Is64Bits());
1787 Emit(STLR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1788 }
1789
stlxrb(const Register & rs,const Register & rt,const Register & rn)1790 void Assembler::stlxrb(const Register& rs, const Register& rt,
1791 const Register& rn) {
1792 DCHECK(rs.Is32Bits());
1793 DCHECK(rt.Is32Bits());
1794 DCHECK(rn.Is64Bits());
1795 DCHECK(!rs.Is(rt) && !rs.Is(rn));
1796 Emit(STLXR_b | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
1797 }
1798
ldarh(const Register & rt,const Register & rn)1799 void Assembler::ldarh(const Register& rt, const Register& rn) {
1800 DCHECK(rt.Is32Bits());
1801 DCHECK(rn.Is64Bits());
1802 Emit(LDAR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1803 }
1804
ldaxrh(const Register & rt,const Register & rn)1805 void Assembler::ldaxrh(const Register& rt, const Register& rn) {
1806 DCHECK(rt.Is32Bits());
1807 DCHECK(rn.Is64Bits());
1808 Emit(LDAXR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1809 }
1810
stlrh(const Register & rt,const Register & rn)1811 void Assembler::stlrh(const Register& rt, const Register& rn) {
1812 DCHECK(rt.Is32Bits());
1813 DCHECK(rn.Is64Bits());
1814 Emit(STLR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
1815 }
1816
stlxrh(const Register & rs,const Register & rt,const Register & rn)1817 void Assembler::stlxrh(const Register& rs, const Register& rt,
1818 const Register& rn) {
1819 DCHECK(rs.Is32Bits());
1820 DCHECK(rt.Is32Bits());
1821 DCHECK(rn.Is64Bits());
1822 DCHECK(!rs.Is(rt) && !rs.Is(rn));
1823 Emit(STLXR_h | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
1824 }
1825
NEON3DifferentL(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)1826 void Assembler::NEON3DifferentL(const VRegister& vd, const VRegister& vn,
1827 const VRegister& vm, NEON3DifferentOp vop) {
1828 DCHECK(AreSameFormat(vn, vm));
1829 DCHECK((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
1830 (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
1831 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
1832 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
1833 Instr format, op = vop;
1834 if (vd.IsScalar()) {
1835 op |= NEON_Q | NEONScalar;
1836 format = SFormat(vn);
1837 } else {
1838 format = VFormat(vn);
1839 }
1840 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
1841 }
1842
NEON3DifferentW(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)1843 void Assembler::NEON3DifferentW(const VRegister& vd, const VRegister& vn,
1844 const VRegister& vm, NEON3DifferentOp vop) {
1845 DCHECK(AreSameFormat(vd, vn));
1846 DCHECK((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
1847 (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
1848 (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
1849 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
1850 }
1851
NEON3DifferentHN(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3DifferentOp vop)1852 void Assembler::NEON3DifferentHN(const VRegister& vd, const VRegister& vn,
1853 const VRegister& vm, NEON3DifferentOp vop) {
1854 DCHECK(AreSameFormat(vm, vn));
1855 DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
1856 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
1857 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
1858 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
1859 }
1860
1861 #define NEON_3DIFF_LONG_LIST(V) \
1862 V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \
1863 V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \
1864 V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
1865 V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
1866 V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
1867 V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
1868 V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
1869 V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
1870 V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
1871 V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
1872 V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
1873 V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
1874 V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
1875 V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
1876 V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
1877 V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
1878 V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
1879 V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
1880 V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
1881 V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
1882 V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
1883 V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
1884 V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
1885 V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
1886 V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
1887 V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
1888 V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
1889 V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
1890 V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
1891 V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
1892 V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
1893 V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
1894 V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
1895 V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
1896 V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
1897 V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S())
1898
1899 #define DEFINE_ASM_FUNC(FN, OP, AS) \
1900 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
1901 const VRegister& vm) { \
1902 DCHECK(AS); \
1903 NEON3DifferentL(vd, vn, vm, OP); \
1904 }
1905 NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
1906 #undef DEFINE_ASM_FUNC
1907
1908 #define NEON_3DIFF_HN_LIST(V) \
1909 V(addhn, NEON_ADDHN, vd.IsD()) \
1910 V(addhn2, NEON_ADDHN2, vd.IsQ()) \
1911 V(raddhn, NEON_RADDHN, vd.IsD()) \
1912 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
1913 V(subhn, NEON_SUBHN, vd.IsD()) \
1914 V(subhn2, NEON_SUBHN2, vd.IsQ()) \
1915 V(rsubhn, NEON_RSUBHN, vd.IsD()) \
1916 V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
1917
1918 #define DEFINE_ASM_FUNC(FN, OP, AS) \
1919 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
1920 const VRegister& vm) { \
1921 DCHECK(AS); \
1922 NEON3DifferentHN(vd, vn, vm, OP); \
1923 }
NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)1924 NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
1925 #undef DEFINE_ASM_FUNC
1926
1927 void Assembler::NEONPerm(const VRegister& vd, const VRegister& vn,
1928 const VRegister& vm, NEONPermOp op) {
1929 DCHECK(AreSameFormat(vd, vn, vm));
1930 DCHECK(!vd.Is1D());
1931 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
1932 }
1933
trn1(const VRegister & vd,const VRegister & vn,const VRegister & vm)1934 void Assembler::trn1(const VRegister& vd, const VRegister& vn,
1935 const VRegister& vm) {
1936 NEONPerm(vd, vn, vm, NEON_TRN1);
1937 }
1938
trn2(const VRegister & vd,const VRegister & vn,const VRegister & vm)1939 void Assembler::trn2(const VRegister& vd, const VRegister& vn,
1940 const VRegister& vm) {
1941 NEONPerm(vd, vn, vm, NEON_TRN2);
1942 }
1943
uzp1(const VRegister & vd,const VRegister & vn,const VRegister & vm)1944 void Assembler::uzp1(const VRegister& vd, const VRegister& vn,
1945 const VRegister& vm) {
1946 NEONPerm(vd, vn, vm, NEON_UZP1);
1947 }
1948
uzp2(const VRegister & vd,const VRegister & vn,const VRegister & vm)1949 void Assembler::uzp2(const VRegister& vd, const VRegister& vn,
1950 const VRegister& vm) {
1951 NEONPerm(vd, vn, vm, NEON_UZP2);
1952 }
1953
zip1(const VRegister & vd,const VRegister & vn,const VRegister & vm)1954 void Assembler::zip1(const VRegister& vd, const VRegister& vn,
1955 const VRegister& vm) {
1956 NEONPerm(vd, vn, vm, NEON_ZIP1);
1957 }
1958
zip2(const VRegister & vd,const VRegister & vn,const VRegister & vm)1959 void Assembler::zip2(const VRegister& vd, const VRegister& vn,
1960 const VRegister& vm) {
1961 NEONPerm(vd, vn, vm, NEON_ZIP2);
1962 }
1963
NEONShiftImmediate(const VRegister & vd,const VRegister & vn,NEONShiftImmediateOp op,int immh_immb)1964 void Assembler::NEONShiftImmediate(const VRegister& vd, const VRegister& vn,
1965 NEONShiftImmediateOp op, int immh_immb) {
1966 DCHECK(AreSameFormat(vd, vn));
1967 Instr q, scalar;
1968 if (vn.IsScalar()) {
1969 q = NEON_Q;
1970 scalar = NEONScalar;
1971 } else {
1972 q = vd.IsD() ? 0 : NEON_Q;
1973 scalar = 0;
1974 }
1975 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
1976 }
1977
NEONShiftLeftImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)1978 void Assembler::NEONShiftLeftImmediate(const VRegister& vd, const VRegister& vn,
1979 int shift, NEONShiftImmediateOp op) {
1980 int laneSizeInBits = vn.LaneSizeInBits();
1981 DCHECK((shift >= 0) && (shift < laneSizeInBits));
1982 NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
1983 }
1984
NEONShiftRightImmediate(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)1985 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
1986 const VRegister& vn, int shift,
1987 NEONShiftImmediateOp op) {
1988 int laneSizeInBits = vn.LaneSizeInBits();
1989 DCHECK((shift >= 1) && (shift <= laneSizeInBits));
1990 NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
1991 }
1992
NEONShiftImmediateL(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)1993 void Assembler::NEONShiftImmediateL(const VRegister& vd, const VRegister& vn,
1994 int shift, NEONShiftImmediateOp op) {
1995 int laneSizeInBits = vn.LaneSizeInBits();
1996 DCHECK((shift >= 0) && (shift < laneSizeInBits));
1997 int immh_immb = (laneSizeInBits + shift) << 16;
1998
1999 DCHECK((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
2000 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
2001 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
2002 Instr q;
2003 q = vn.IsD() ? 0 : NEON_Q;
2004 Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
2005 }
2006
NEONShiftImmediateN(const VRegister & vd,const VRegister & vn,int shift,NEONShiftImmediateOp op)2007 void Assembler::NEONShiftImmediateN(const VRegister& vd, const VRegister& vn,
2008 int shift, NEONShiftImmediateOp op) {
2009 Instr q, scalar;
2010 int laneSizeInBits = vd.LaneSizeInBits();
2011 DCHECK((shift >= 1) && (shift <= laneSizeInBits));
2012 int immh_immb = (2 * laneSizeInBits - shift) << 16;
2013
2014 if (vn.IsScalar()) {
2015 DCHECK((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
2016 (vd.Is1S() && vn.Is1D()));
2017 q = NEON_Q;
2018 scalar = NEONScalar;
2019 } else {
2020 DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
2021 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
2022 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
2023 scalar = 0;
2024 q = vd.IsD() ? 0 : NEON_Q;
2025 }
2026 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
2027 }
2028
shl(const VRegister & vd,const VRegister & vn,int shift)2029 void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
2030 DCHECK(vd.IsVector() || vd.Is1D());
2031 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
2032 }
2033
sli(const VRegister & vd,const VRegister & vn,int shift)2034 void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
2035 DCHECK(vd.IsVector() || vd.Is1D());
2036 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
2037 }
2038
sqshl(const VRegister & vd,const VRegister & vn,int shift)2039 void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
2040 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
2041 }
2042
sqshlu(const VRegister & vd,const VRegister & vn,int shift)2043 void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
2044 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
2045 }
2046
uqshl(const VRegister & vd,const VRegister & vn,int shift)2047 void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
2048 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
2049 }
2050
sshll(const VRegister & vd,const VRegister & vn,int shift)2051 void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
2052 DCHECK(vn.IsD());
2053 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
2054 }
2055
sshll2(const VRegister & vd,const VRegister & vn,int shift)2056 void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
2057 DCHECK(vn.IsQ());
2058 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
2059 }
2060
sxtl(const VRegister & vd,const VRegister & vn)2061 void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
2062 sshll(vd, vn, 0);
2063 }
2064
sxtl2(const VRegister & vd,const VRegister & vn)2065 void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
2066 sshll2(vd, vn, 0);
2067 }
2068
ushll(const VRegister & vd,const VRegister & vn,int shift)2069 void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
2070 DCHECK(vn.IsD());
2071 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
2072 }
2073
ushll2(const VRegister & vd,const VRegister & vn,int shift)2074 void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
2075 DCHECK(vn.IsQ());
2076 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
2077 }
2078
uxtl(const VRegister & vd,const VRegister & vn)2079 void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
2080 ushll(vd, vn, 0);
2081 }
2082
uxtl2(const VRegister & vd,const VRegister & vn)2083 void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
2084 ushll2(vd, vn, 0);
2085 }
2086
sri(const VRegister & vd,const VRegister & vn,int shift)2087 void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
2088 DCHECK(vd.IsVector() || vd.Is1D());
2089 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
2090 }
2091
sshr(const VRegister & vd,const VRegister & vn,int shift)2092 void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
2093 DCHECK(vd.IsVector() || vd.Is1D());
2094 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
2095 }
2096
ushr(const VRegister & vd,const VRegister & vn,int shift)2097 void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
2098 DCHECK(vd.IsVector() || vd.Is1D());
2099 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
2100 }
2101
srshr(const VRegister & vd,const VRegister & vn,int shift)2102 void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
2103 DCHECK(vd.IsVector() || vd.Is1D());
2104 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
2105 }
2106
urshr(const VRegister & vd,const VRegister & vn,int shift)2107 void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
2108 DCHECK(vd.IsVector() || vd.Is1D());
2109 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
2110 }
2111
ssra(const VRegister & vd,const VRegister & vn,int shift)2112 void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
2113 DCHECK(vd.IsVector() || vd.Is1D());
2114 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
2115 }
2116
usra(const VRegister & vd,const VRegister & vn,int shift)2117 void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
2118 DCHECK(vd.IsVector() || vd.Is1D());
2119 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
2120 }
2121
srsra(const VRegister & vd,const VRegister & vn,int shift)2122 void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
2123 DCHECK(vd.IsVector() || vd.Is1D());
2124 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
2125 }
2126
ursra(const VRegister & vd,const VRegister & vn,int shift)2127 void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
2128 DCHECK(vd.IsVector() || vd.Is1D());
2129 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
2130 }
2131
shrn(const VRegister & vd,const VRegister & vn,int shift)2132 void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
2133 DCHECK(vn.IsVector() && vd.IsD());
2134 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
2135 }
2136
shrn2(const VRegister & vd,const VRegister & vn,int shift)2137 void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
2138 DCHECK(vn.IsVector() && vd.IsQ());
2139 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
2140 }
2141
rshrn(const VRegister & vd,const VRegister & vn,int shift)2142 void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
2143 DCHECK(vn.IsVector() && vd.IsD());
2144 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
2145 }
2146
rshrn2(const VRegister & vd,const VRegister & vn,int shift)2147 void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
2148 DCHECK(vn.IsVector() && vd.IsQ());
2149 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
2150 }
2151
sqshrn(const VRegister & vd,const VRegister & vn,int shift)2152 void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
2153 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
2154 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
2155 }
2156
sqshrn2(const VRegister & vd,const VRegister & vn,int shift)2157 void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
2158 DCHECK(vn.IsVector() && vd.IsQ());
2159 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
2160 }
2161
sqrshrn(const VRegister & vd,const VRegister & vn,int shift)2162 void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
2163 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
2164 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
2165 }
2166
sqrshrn2(const VRegister & vd,const VRegister & vn,int shift)2167 void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
2168 DCHECK(vn.IsVector() && vd.IsQ());
2169 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
2170 }
2171
sqshrun(const VRegister & vd,const VRegister & vn,int shift)2172 void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
2173 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
2174 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
2175 }
2176
sqshrun2(const VRegister & vd,const VRegister & vn,int shift)2177 void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
2178 DCHECK(vn.IsVector() && vd.IsQ());
2179 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
2180 }
2181
sqrshrun(const VRegister & vd,const VRegister & vn,int shift)2182 void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
2183 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
2184 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
2185 }
2186
sqrshrun2(const VRegister & vd,const VRegister & vn,int shift)2187 void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
2188 DCHECK(vn.IsVector() && vd.IsQ());
2189 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
2190 }
2191
uqshrn(const VRegister & vd,const VRegister & vn,int shift)2192 void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
2193 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
2194 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
2195 }
2196
uqshrn2(const VRegister & vd,const VRegister & vn,int shift)2197 void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
2198 DCHECK(vn.IsVector() && vd.IsQ());
2199 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
2200 }
2201
uqrshrn(const VRegister & vd,const VRegister & vn,int shift)2202 void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
2203 DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
2204 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
2205 }
2206
uqrshrn2(const VRegister & vd,const VRegister & vn,int shift)2207 void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
2208 DCHECK(vn.IsVector() && vd.IsQ());
2209 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
2210 }
2211
uaddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2212 void Assembler::uaddw(const VRegister& vd, const VRegister& vn,
2213 const VRegister& vm) {
2214 DCHECK(vm.IsD());
2215 NEON3DifferentW(vd, vn, vm, NEON_UADDW);
2216 }
2217
uaddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2218 void Assembler::uaddw2(const VRegister& vd, const VRegister& vn,
2219 const VRegister& vm) {
2220 DCHECK(vm.IsQ());
2221 NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
2222 }
2223
saddw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2224 void Assembler::saddw(const VRegister& vd, const VRegister& vn,
2225 const VRegister& vm) {
2226 DCHECK(vm.IsD());
2227 NEON3DifferentW(vd, vn, vm, NEON_SADDW);
2228 }
2229
saddw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2230 void Assembler::saddw2(const VRegister& vd, const VRegister& vn,
2231 const VRegister& vm) {
2232 DCHECK(vm.IsQ());
2233 NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
2234 }
2235
usubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2236 void Assembler::usubw(const VRegister& vd, const VRegister& vn,
2237 const VRegister& vm) {
2238 DCHECK(vm.IsD());
2239 NEON3DifferentW(vd, vn, vm, NEON_USUBW);
2240 }
2241
usubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2242 void Assembler::usubw2(const VRegister& vd, const VRegister& vn,
2243 const VRegister& vm) {
2244 DCHECK(vm.IsQ());
2245 NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
2246 }
2247
ssubw(const VRegister & vd,const VRegister & vn,const VRegister & vm)2248 void Assembler::ssubw(const VRegister& vd, const VRegister& vn,
2249 const VRegister& vm) {
2250 DCHECK(vm.IsD());
2251 NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
2252 }
2253
ssubw2(const VRegister & vd,const VRegister & vn,const VRegister & vm)2254 void Assembler::ssubw2(const VRegister& vd, const VRegister& vn,
2255 const VRegister& vm) {
2256 DCHECK(vm.IsQ());
2257 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
2258 }
2259
mov(const Register & rd,const Register & rm)2260 void Assembler::mov(const Register& rd, const Register& rm) {
2261 // Moves involving the stack pointer are encoded as add immediate with
2262 // second operand of zero. Otherwise, orr with first operand zr is
2263 // used.
2264 if (rd.IsSP() || rm.IsSP()) {
2265 add(rd, rm, 0);
2266 } else {
2267 orr(rd, AppropriateZeroRegFor(rd), rm);
2268 }
2269 }
2270
ins(const VRegister & vd,int vd_index,const Register & rn)2271 void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
2272 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
2273 // number of lanes, and T is b, h, s or d.
2274 int lane_size = vd.LaneSizeInBytes();
2275 NEONFormatField format;
2276 switch (lane_size) {
2277 case 1:
2278 format = NEON_16B;
2279 DCHECK(rn.IsW());
2280 break;
2281 case 2:
2282 format = NEON_8H;
2283 DCHECK(rn.IsW());
2284 break;
2285 case 4:
2286 format = NEON_4S;
2287 DCHECK(rn.IsW());
2288 break;
2289 default:
2290 DCHECK_EQ(lane_size, 8);
2291 DCHECK(rn.IsX());
2292 format = NEON_2D;
2293 break;
2294 }
2295
2296 DCHECK((0 <= vd_index) &&
2297 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
2298 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
2299 }
2300
mov(const Register & rd,const VRegister & vn,int vn_index)2301 void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) {
2302 DCHECK_GE(vn.SizeInBytes(), 4);
2303 umov(rd, vn, vn_index);
2304 }
2305
smov(const Register & rd,const VRegister & vn,int vn_index)2306 void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) {
2307 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
2308 // number of lanes, and T is b, h, s.
2309 int lane_size = vn.LaneSizeInBytes();
2310 NEONFormatField format;
2311 Instr q = 0;
2312 switch (lane_size) {
2313 case 1:
2314 format = NEON_16B;
2315 break;
2316 case 2:
2317 format = NEON_8H;
2318 break;
2319 default:
2320 DCHECK_EQ(lane_size, 4);
2321 DCHECK(rd.IsX());
2322 format = NEON_4S;
2323 break;
2324 }
2325 q = rd.IsW() ? 0 : NEON_Q;
2326 DCHECK((0 <= vn_index) &&
2327 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
2328 Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
2329 }
2330
cls(const VRegister & vd,const VRegister & vn)2331 void Assembler::cls(const VRegister& vd, const VRegister& vn) {
2332 DCHECK(AreSameFormat(vd, vn));
2333 DCHECK(!vd.Is1D() && !vd.Is2D());
2334 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
2335 }
2336
clz(const VRegister & vd,const VRegister & vn)2337 void Assembler::clz(const VRegister& vd, const VRegister& vn) {
2338 DCHECK(AreSameFormat(vd, vn));
2339 DCHECK(!vd.Is1D() && !vd.Is2D());
2340 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
2341 }
2342
cnt(const VRegister & vd,const VRegister & vn)2343 void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
2344 DCHECK(AreSameFormat(vd, vn));
2345 DCHECK(vd.Is8B() || vd.Is16B());
2346 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
2347 }
2348
rev16(const VRegister & vd,const VRegister & vn)2349 void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
2350 DCHECK(AreSameFormat(vd, vn));
2351 DCHECK(vd.Is8B() || vd.Is16B());
2352 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
2353 }
2354
rev32(const VRegister & vd,const VRegister & vn)2355 void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
2356 DCHECK(AreSameFormat(vd, vn));
2357 DCHECK(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
2358 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
2359 }
2360
rev64(const VRegister & vd,const VRegister & vn)2361 void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
2362 DCHECK(AreSameFormat(vd, vn));
2363 DCHECK(!vd.Is1D() && !vd.Is2D());
2364 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
2365 }
2366
ursqrte(const VRegister & vd,const VRegister & vn)2367 void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
2368 DCHECK(AreSameFormat(vd, vn));
2369 DCHECK(vd.Is2S() || vd.Is4S());
2370 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
2371 }
2372
urecpe(const VRegister & vd,const VRegister & vn)2373 void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
2374 DCHECK(AreSameFormat(vd, vn));
2375 DCHECK(vd.Is2S() || vd.Is4S());
2376 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
2377 }
2378
NEONAddlp(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp op)2379 void Assembler::NEONAddlp(const VRegister& vd, const VRegister& vn,
2380 NEON2RegMiscOp op) {
2381 DCHECK((op == NEON_SADDLP) || (op == NEON_UADDLP) || (op == NEON_SADALP) ||
2382 (op == NEON_UADALP));
2383
2384 DCHECK((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
2385 (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
2386 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
2387 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
2388 }
2389
saddlp(const VRegister & vd,const VRegister & vn)2390 void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
2391 NEONAddlp(vd, vn, NEON_SADDLP);
2392 }
2393
uaddlp(const VRegister & vd,const VRegister & vn)2394 void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
2395 NEONAddlp(vd, vn, NEON_UADDLP);
2396 }
2397
sadalp(const VRegister & vd,const VRegister & vn)2398 void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
2399 NEONAddlp(vd, vn, NEON_SADALP);
2400 }
2401
uadalp(const VRegister & vd,const VRegister & vn)2402 void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
2403 NEONAddlp(vd, vn, NEON_UADALP);
2404 }
2405
NEONAcrossLanesL(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)2406 void Assembler::NEONAcrossLanesL(const VRegister& vd, const VRegister& vn,
2407 NEONAcrossLanesOp op) {
2408 DCHECK((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
2409 (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
2410 (vn.Is4S() && vd.Is1D()));
2411 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
2412 }
2413
saddlv(const VRegister & vd,const VRegister & vn)2414 void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
2415 NEONAcrossLanesL(vd, vn, NEON_SADDLV);
2416 }
2417
uaddlv(const VRegister & vd,const VRegister & vn)2418 void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
2419 NEONAcrossLanesL(vd, vn, NEON_UADDLV);
2420 }
2421
NEONAcrossLanes(const VRegister & vd,const VRegister & vn,NEONAcrossLanesOp op)2422 void Assembler::NEONAcrossLanes(const VRegister& vd, const VRegister& vn,
2423 NEONAcrossLanesOp op) {
2424 DCHECK((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
2425 (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
2426 (vn.Is4S() && vd.Is1S()));
2427 if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
2428 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
2429 } else {
2430 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
2431 }
2432 }
2433
2434 #define NEON_ACROSSLANES_LIST(V) \
2435 V(fmaxv, NEON_FMAXV, vd.Is1S()) \
2436 V(fminv, NEON_FMINV, vd.Is1S()) \
2437 V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
2438 V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
2439 V(addv, NEON_ADDV, true) \
2440 V(smaxv, NEON_SMAXV, true) \
2441 V(sminv, NEON_SMINV, true) \
2442 V(umaxv, NEON_UMAXV, true) \
2443 V(uminv, NEON_UMINV, true)
2444
2445 #define DEFINE_ASM_FUNC(FN, OP, AS) \
2446 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
2447 DCHECK(AS); \
2448 NEONAcrossLanes(vd, vn, OP); \
2449 }
NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)2450 NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
2451 #undef DEFINE_ASM_FUNC
2452
2453 void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
2454 ins(vd, vd_index, rn);
2455 }
2456
umov(const Register & rd,const VRegister & vn,int vn_index)2457 void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) {
2458 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
2459 // number of lanes, and T is b, h, s or d.
2460 int lane_size = vn.LaneSizeInBytes();
2461 NEONFormatField format;
2462 Instr q = 0;
2463 switch (lane_size) {
2464 case 1:
2465 format = NEON_16B;
2466 DCHECK(rd.IsW());
2467 break;
2468 case 2:
2469 format = NEON_8H;
2470 DCHECK(rd.IsW());
2471 break;
2472 case 4:
2473 format = NEON_4S;
2474 DCHECK(rd.IsW());
2475 break;
2476 default:
2477 DCHECK_EQ(lane_size, 8);
2478 DCHECK(rd.IsX());
2479 format = NEON_2D;
2480 q = NEON_Q;
2481 break;
2482 }
2483
2484 DCHECK((0 <= vn_index) &&
2485 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
2486 Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
2487 }
2488
mov(const VRegister & vd,const VRegister & vn,int vn_index)2489 void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
2490 DCHECK(vd.IsScalar());
2491 dup(vd, vn, vn_index);
2492 }
2493
dup(const VRegister & vd,const Register & rn)2494 void Assembler::dup(const VRegister& vd, const Register& rn) {
2495 DCHECK(!vd.Is1D());
2496 DCHECK_EQ(vd.Is2D(), rn.IsX());
2497 Instr q = vd.IsD() ? 0 : NEON_Q;
2498 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
2499 }
2500
ins(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)2501 void Assembler::ins(const VRegister& vd, int vd_index, const VRegister& vn,
2502 int vn_index) {
2503 DCHECK(AreSameFormat(vd, vn));
2504 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
2505 // number of lanes, and T is b, h, s or d.
2506 int lane_size = vd.LaneSizeInBytes();
2507 NEONFormatField format;
2508 switch (lane_size) {
2509 case 1:
2510 format = NEON_16B;
2511 break;
2512 case 2:
2513 format = NEON_8H;
2514 break;
2515 case 4:
2516 format = NEON_4S;
2517 break;
2518 default:
2519 DCHECK_EQ(lane_size, 8);
2520 format = NEON_2D;
2521 break;
2522 }
2523
2524 DCHECK((0 <= vd_index) &&
2525 (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
2526 DCHECK((0 <= vn_index) &&
2527 (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
2528 Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
2529 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
2530 }
2531
NEONTable(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEONTableOp op)2532 void Assembler::NEONTable(const VRegister& vd, const VRegister& vn,
2533 const VRegister& vm, NEONTableOp op) {
2534 DCHECK(vd.Is16B() || vd.Is8B());
2535 DCHECK(vn.Is16B());
2536 DCHECK(AreSameFormat(vd, vm));
2537 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
2538 }
2539
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vm)2540 void Assembler::tbl(const VRegister& vd, const VRegister& vn,
2541 const VRegister& vm) {
2542 NEONTable(vd, vn, vm, NEON_TBL_1v);
2543 }
2544
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)2545 void Assembler::tbl(const VRegister& vd, const VRegister& vn,
2546 const VRegister& vn2, const VRegister& vm) {
2547 USE(vn2);
2548 DCHECK(AreSameFormat(vn, vn2));
2549 DCHECK(AreConsecutive(vn, vn2));
2550 NEONTable(vd, vn, vm, NEON_TBL_2v);
2551 }
2552
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)2553 void Assembler::tbl(const VRegister& vd, const VRegister& vn,
2554 const VRegister& vn2, const VRegister& vn3,
2555 const VRegister& vm) {
2556 USE(vn2);
2557 USE(vn3);
2558 DCHECK(AreSameFormat(vn, vn2, vn3));
2559 DCHECK(AreConsecutive(vn, vn2, vn3));
2560 NEONTable(vd, vn, vm, NEON_TBL_3v);
2561 }
2562
tbl(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)2563 void Assembler::tbl(const VRegister& vd, const VRegister& vn,
2564 const VRegister& vn2, const VRegister& vn3,
2565 const VRegister& vn4, const VRegister& vm) {
2566 USE(vn2);
2567 USE(vn3);
2568 USE(vn4);
2569 DCHECK(AreSameFormat(vn, vn2, vn3, vn4));
2570 DCHECK(AreConsecutive(vn, vn2, vn3, vn4));
2571 NEONTable(vd, vn, vm, NEON_TBL_4v);
2572 }
2573
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vm)2574 void Assembler::tbx(const VRegister& vd, const VRegister& vn,
2575 const VRegister& vm) {
2576 NEONTable(vd, vn, vm, NEON_TBX_1v);
2577 }
2578
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vm)2579 void Assembler::tbx(const VRegister& vd, const VRegister& vn,
2580 const VRegister& vn2, const VRegister& vm) {
2581 USE(vn2);
2582 DCHECK(AreSameFormat(vn, vn2));
2583 DCHECK(AreConsecutive(vn, vn2));
2584 NEONTable(vd, vn, vm, NEON_TBX_2v);
2585 }
2586
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vm)2587 void Assembler::tbx(const VRegister& vd, const VRegister& vn,
2588 const VRegister& vn2, const VRegister& vn3,
2589 const VRegister& vm) {
2590 USE(vn2);
2591 USE(vn3);
2592 DCHECK(AreSameFormat(vn, vn2, vn3));
2593 DCHECK(AreConsecutive(vn, vn2, vn3));
2594 NEONTable(vd, vn, vm, NEON_TBX_3v);
2595 }
2596
tbx(const VRegister & vd,const VRegister & vn,const VRegister & vn2,const VRegister & vn3,const VRegister & vn4,const VRegister & vm)2597 void Assembler::tbx(const VRegister& vd, const VRegister& vn,
2598 const VRegister& vn2, const VRegister& vn3,
2599 const VRegister& vn4, const VRegister& vm) {
2600 USE(vn2);
2601 USE(vn3);
2602 USE(vn4);
2603 DCHECK(AreSameFormat(vn, vn2, vn3, vn4));
2604 DCHECK(AreConsecutive(vn, vn2, vn3, vn4));
2605 NEONTable(vd, vn, vm, NEON_TBX_4v);
2606 }
2607
mov(const VRegister & vd,int vd_index,const VRegister & vn,int vn_index)2608 void Assembler::mov(const VRegister& vd, int vd_index, const VRegister& vn,
2609 int vn_index) {
2610 ins(vd, vd_index, vn, vn_index);
2611 }
2612
mvn(const Register & rd,const Operand & operand)2613 void Assembler::mvn(const Register& rd, const Operand& operand) {
2614 orn(rd, AppropriateZeroRegFor(rd), operand);
2615 }
2616
mrs(const Register & rt,SystemRegister sysreg)2617 void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
2618 DCHECK(rt.Is64Bits());
2619 Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
2620 }
2621
msr(SystemRegister sysreg,const Register & rt)2622 void Assembler::msr(SystemRegister sysreg, const Register& rt) {
2623 DCHECK(rt.Is64Bits());
2624 Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
2625 }
2626
hint(SystemHint code)2627 void Assembler::hint(SystemHint code) { Emit(HINT | ImmHint(code) | Rt(xzr)); }
2628
2629 // NEON structure loads and stores.
LoadStoreStructAddrModeField(const MemOperand & addr)2630 Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
2631 Instr addr_field = RnSP(addr.base());
2632
2633 if (addr.IsPostIndex()) {
2634 static_assert(NEONLoadStoreMultiStructPostIndex ==
2635 static_cast<NEONLoadStoreMultiStructPostIndexOp>(
2636 NEONLoadStoreSingleStructPostIndex),
2637 "Opcodes must match for NEON post index memop.");
2638
2639 addr_field |= NEONLoadStoreMultiStructPostIndex;
2640 if (addr.offset() == 0) {
2641 addr_field |= RmNot31(addr.regoffset());
2642 } else {
2643 // The immediate post index addressing mode is indicated by rm = 31.
2644 // The immediate is implied by the number of vector registers used.
2645 addr_field |= (0x1F << Rm_offset);
2646 }
2647 } else {
2648 DCHECK(addr.IsImmediateOffset() && (addr.offset() == 0));
2649 }
2650 return addr_field;
2651 }
2652
LoadStoreStructVerify(const VRegister & vt,const MemOperand & addr,Instr op)2653 void Assembler::LoadStoreStructVerify(const VRegister& vt,
2654 const MemOperand& addr, Instr op) {
2655 #ifdef DEBUG
2656 // Assert that addressing mode is either offset (with immediate 0), post
2657 // index by immediate of the size of the register list, or post index by a
2658 // value in a core register.
2659 if (addr.IsImmediateOffset()) {
2660 DCHECK_EQ(addr.offset(), 0);
2661 } else {
2662 int offset = vt.SizeInBytes();
2663 switch (op) {
2664 case NEON_LD1_1v:
2665 case NEON_ST1_1v:
2666 offset *= 1;
2667 break;
2668 case NEONLoadStoreSingleStructLoad1:
2669 case NEONLoadStoreSingleStructStore1:
2670 case NEON_LD1R:
2671 offset = (offset / vt.LaneCount()) * 1;
2672 break;
2673
2674 case NEON_LD1_2v:
2675 case NEON_ST1_2v:
2676 case NEON_LD2:
2677 case NEON_ST2:
2678 offset *= 2;
2679 break;
2680 case NEONLoadStoreSingleStructLoad2:
2681 case NEONLoadStoreSingleStructStore2:
2682 case NEON_LD2R:
2683 offset = (offset / vt.LaneCount()) * 2;
2684 break;
2685
2686 case NEON_LD1_3v:
2687 case NEON_ST1_3v:
2688 case NEON_LD3:
2689 case NEON_ST3:
2690 offset *= 3;
2691 break;
2692 case NEONLoadStoreSingleStructLoad3:
2693 case NEONLoadStoreSingleStructStore3:
2694 case NEON_LD3R:
2695 offset = (offset / vt.LaneCount()) * 3;
2696 break;
2697
2698 case NEON_LD1_4v:
2699 case NEON_ST1_4v:
2700 case NEON_LD4:
2701 case NEON_ST4:
2702 offset *= 4;
2703 break;
2704 case NEONLoadStoreSingleStructLoad4:
2705 case NEONLoadStoreSingleStructStore4:
2706 case NEON_LD4R:
2707 offset = (offset / vt.LaneCount()) * 4;
2708 break;
2709 default:
2710 UNREACHABLE();
2711 }
2712 DCHECK(!addr.regoffset().Is(NoReg) || addr.offset() == offset);
2713 }
2714 #else
2715 USE(vt);
2716 USE(addr);
2717 USE(op);
2718 #endif
2719 }
2720
LoadStoreStruct(const VRegister & vt,const MemOperand & addr,NEONLoadStoreMultiStructOp op)2721 void Assembler::LoadStoreStruct(const VRegister& vt, const MemOperand& addr,
2722 NEONLoadStoreMultiStructOp op) {
2723 LoadStoreStructVerify(vt, addr, op);
2724 DCHECK(vt.IsVector() || vt.Is1D());
2725 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2726 }
2727
LoadStoreStructSingleAllLanes(const VRegister & vt,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2728 void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
2729 const MemOperand& addr,
2730 NEONLoadStoreSingleStructOp op) {
2731 LoadStoreStructVerify(vt, addr, op);
2732 Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
2733 }
2734
ld1(const VRegister & vt,const MemOperand & src)2735 void Assembler::ld1(const VRegister& vt, const MemOperand& src) {
2736 LoadStoreStruct(vt, src, NEON_LD1_1v);
2737 }
2738
ld1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2739 void Assembler::ld1(const VRegister& vt, const VRegister& vt2,
2740 const MemOperand& src) {
2741 USE(vt2);
2742 DCHECK(AreSameFormat(vt, vt2));
2743 DCHECK(AreConsecutive(vt, vt2));
2744 LoadStoreStruct(vt, src, NEON_LD1_2v);
2745 }
2746
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2747 void Assembler::ld1(const VRegister& vt, const VRegister& vt2,
2748 const VRegister& vt3, const MemOperand& src) {
2749 USE(vt2);
2750 USE(vt3);
2751 DCHECK(AreSameFormat(vt, vt2, vt3));
2752 DCHECK(AreConsecutive(vt, vt2, vt3));
2753 LoadStoreStruct(vt, src, NEON_LD1_3v);
2754 }
2755
ld1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2756 void Assembler::ld1(const VRegister& vt, const VRegister& vt2,
2757 const VRegister& vt3, const VRegister& vt4,
2758 const MemOperand& src) {
2759 USE(vt2);
2760 USE(vt3);
2761 USE(vt4);
2762 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2763 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2764 LoadStoreStruct(vt, src, NEON_LD1_4v);
2765 }
2766
ld2(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2767 void Assembler::ld2(const VRegister& vt, const VRegister& vt2,
2768 const MemOperand& src) {
2769 USE(vt2);
2770 DCHECK(AreSameFormat(vt, vt2));
2771 DCHECK(AreConsecutive(vt, vt2));
2772 LoadStoreStruct(vt, src, NEON_LD2);
2773 }
2774
ld2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & src)2775 void Assembler::ld2(const VRegister& vt, const VRegister& vt2, int lane,
2776 const MemOperand& src) {
2777 USE(vt2);
2778 DCHECK(AreSameFormat(vt, vt2));
2779 DCHECK(AreConsecutive(vt, vt2));
2780 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
2781 }
2782
ld2r(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2783 void Assembler::ld2r(const VRegister& vt, const VRegister& vt2,
2784 const MemOperand& src) {
2785 USE(vt2);
2786 DCHECK(AreSameFormat(vt, vt2));
2787 DCHECK(AreConsecutive(vt, vt2));
2788 LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
2789 }
2790
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2791 void Assembler::ld3(const VRegister& vt, const VRegister& vt2,
2792 const VRegister& vt3, const MemOperand& src) {
2793 USE(vt2);
2794 USE(vt3);
2795 DCHECK(AreSameFormat(vt, vt2, vt3));
2796 DCHECK(AreConsecutive(vt, vt2, vt3));
2797 LoadStoreStruct(vt, src, NEON_LD3);
2798 }
2799
ld3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & src)2800 void Assembler::ld3(const VRegister& vt, const VRegister& vt2,
2801 const VRegister& vt3, int lane, const MemOperand& src) {
2802 USE(vt2);
2803 USE(vt3);
2804 DCHECK(AreSameFormat(vt, vt2, vt3));
2805 DCHECK(AreConsecutive(vt, vt2, vt3));
2806 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
2807 }
2808
ld3r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2809 void Assembler::ld3r(const VRegister& vt, const VRegister& vt2,
2810 const VRegister& vt3, const MemOperand& src) {
2811 USE(vt2);
2812 USE(vt3);
2813 DCHECK(AreSameFormat(vt, vt2, vt3));
2814 DCHECK(AreConsecutive(vt, vt2, vt3));
2815 LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
2816 }
2817
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2818 void Assembler::ld4(const VRegister& vt, const VRegister& vt2,
2819 const VRegister& vt3, const VRegister& vt4,
2820 const MemOperand& src) {
2821 USE(vt2);
2822 USE(vt3);
2823 USE(vt4);
2824 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2825 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2826 LoadStoreStruct(vt, src, NEON_LD4);
2827 }
2828
ld4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & src)2829 void Assembler::ld4(const VRegister& vt, const VRegister& vt2,
2830 const VRegister& vt3, const VRegister& vt4, int lane,
2831 const MemOperand& src) {
2832 USE(vt2);
2833 USE(vt3);
2834 USE(vt4);
2835 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2836 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2837 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
2838 }
2839
ld4r(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2840 void Assembler::ld4r(const VRegister& vt, const VRegister& vt2,
2841 const VRegister& vt3, const VRegister& vt4,
2842 const MemOperand& src) {
2843 USE(vt2);
2844 USE(vt3);
2845 USE(vt4);
2846 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2847 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2848 LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
2849 }
2850
st1(const VRegister & vt,const MemOperand & src)2851 void Assembler::st1(const VRegister& vt, const MemOperand& src) {
2852 LoadStoreStruct(vt, src, NEON_ST1_1v);
2853 }
2854
st1(const VRegister & vt,const VRegister & vt2,const MemOperand & src)2855 void Assembler::st1(const VRegister& vt, const VRegister& vt2,
2856 const MemOperand& src) {
2857 USE(vt2);
2858 DCHECK(AreSameFormat(vt, vt2));
2859 DCHECK(AreConsecutive(vt, vt2));
2860 LoadStoreStruct(vt, src, NEON_ST1_2v);
2861 }
2862
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & src)2863 void Assembler::st1(const VRegister& vt, const VRegister& vt2,
2864 const VRegister& vt3, const MemOperand& src) {
2865 USE(vt2);
2866 USE(vt3);
2867 DCHECK(AreSameFormat(vt, vt2, vt3));
2868 DCHECK(AreConsecutive(vt, vt2, vt3));
2869 LoadStoreStruct(vt, src, NEON_ST1_3v);
2870 }
2871
st1(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & src)2872 void Assembler::st1(const VRegister& vt, const VRegister& vt2,
2873 const VRegister& vt3, const VRegister& vt4,
2874 const MemOperand& src) {
2875 USE(vt2);
2876 USE(vt3);
2877 USE(vt4);
2878 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2879 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2880 LoadStoreStruct(vt, src, NEON_ST1_4v);
2881 }
2882
st2(const VRegister & vt,const VRegister & vt2,const MemOperand & dst)2883 void Assembler::st2(const VRegister& vt, const VRegister& vt2,
2884 const MemOperand& dst) {
2885 USE(vt2);
2886 DCHECK(AreSameFormat(vt, vt2));
2887 DCHECK(AreConsecutive(vt, vt2));
2888 LoadStoreStruct(vt, dst, NEON_ST2);
2889 }
2890
st2(const VRegister & vt,const VRegister & vt2,int lane,const MemOperand & dst)2891 void Assembler::st2(const VRegister& vt, const VRegister& vt2, int lane,
2892 const MemOperand& dst) {
2893 USE(vt2);
2894 DCHECK(AreSameFormat(vt, vt2));
2895 DCHECK(AreConsecutive(vt, vt2));
2896 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
2897 }
2898
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const MemOperand & dst)2899 void Assembler::st3(const VRegister& vt, const VRegister& vt2,
2900 const VRegister& vt3, const MemOperand& dst) {
2901 USE(vt2);
2902 USE(vt3);
2903 DCHECK(AreSameFormat(vt, vt2, vt3));
2904 DCHECK(AreConsecutive(vt, vt2, vt3));
2905 LoadStoreStruct(vt, dst, NEON_ST3);
2906 }
2907
st3(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,int lane,const MemOperand & dst)2908 void Assembler::st3(const VRegister& vt, const VRegister& vt2,
2909 const VRegister& vt3, int lane, const MemOperand& dst) {
2910 USE(vt2);
2911 USE(vt3);
2912 DCHECK(AreSameFormat(vt, vt2, vt3));
2913 DCHECK(AreConsecutive(vt, vt2, vt3));
2914 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
2915 }
2916
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,const MemOperand & dst)2917 void Assembler::st4(const VRegister& vt, const VRegister& vt2,
2918 const VRegister& vt3, const VRegister& vt4,
2919 const MemOperand& dst) {
2920 USE(vt2);
2921 USE(vt3);
2922 USE(vt4);
2923 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2924 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2925 LoadStoreStruct(vt, dst, NEON_ST4);
2926 }
2927
st4(const VRegister & vt,const VRegister & vt2,const VRegister & vt3,const VRegister & vt4,int lane,const MemOperand & dst)2928 void Assembler::st4(const VRegister& vt, const VRegister& vt2,
2929 const VRegister& vt3, const VRegister& vt4, int lane,
2930 const MemOperand& dst) {
2931 USE(vt2);
2932 USE(vt3);
2933 USE(vt4);
2934 DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
2935 DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
2936 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
2937 }
2938
LoadStoreStructSingle(const VRegister & vt,uint32_t lane,const MemOperand & addr,NEONLoadStoreSingleStructOp op)2939 void Assembler::LoadStoreStructSingle(const VRegister& vt, uint32_t lane,
2940 const MemOperand& addr,
2941 NEONLoadStoreSingleStructOp op) {
2942 LoadStoreStructVerify(vt, addr, op);
2943
2944 // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
2945 // number of lanes, and T is b, h, s or d.
2946 unsigned lane_size = vt.LaneSizeInBytes();
2947 DCHECK_LT(lane, kQRegSize / lane_size);
2948
2949 // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
2950 // S and size fields.
2951 lane *= lane_size;
2952
2953 // Encodings for S[0]/D[0] and S[2]/D[1] are distinguished using the least-
2954 // significant bit of the size field, so we increment lane here to account for
2955 // that.
2956 if (lane_size == 8) lane++;
2957
2958 Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
2959 Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
2960 Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
2961
2962 Instr instr = op;
2963 switch (lane_size) {
2964 case 1:
2965 instr |= NEONLoadStoreSingle_b;
2966 break;
2967 case 2:
2968 instr |= NEONLoadStoreSingle_h;
2969 break;
2970 case 4:
2971 instr |= NEONLoadStoreSingle_s;
2972 break;
2973 default:
2974 DCHECK_EQ(lane_size, 8U);
2975 instr |= NEONLoadStoreSingle_d;
2976 }
2977
2978 Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
2979 }
2980
ld1(const VRegister & vt,int lane,const MemOperand & src)2981 void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) {
2982 LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
2983 }
2984
ld1r(const VRegister & vt,const MemOperand & src)2985 void Assembler::ld1r(const VRegister& vt, const MemOperand& src) {
2986 LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
2987 }
2988
st1(const VRegister & vt,int lane,const MemOperand & dst)2989 void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) {
2990 LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
2991 }
2992
dmb(BarrierDomain domain,BarrierType type)2993 void Assembler::dmb(BarrierDomain domain, BarrierType type) {
2994 Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2995 }
2996
dsb(BarrierDomain domain,BarrierType type)2997 void Assembler::dsb(BarrierDomain domain, BarrierType type) {
2998 Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
2999 }
3000
isb()3001 void Assembler::isb() {
3002 Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
3003 }
3004
csdb()3005 void Assembler::csdb() { hint(CSDB); }
3006
fmov(const VRegister & vd,double imm)3007 void Assembler::fmov(const VRegister& vd, double imm) {
3008 if (vd.IsScalar()) {
3009 DCHECK(vd.Is1D());
3010 Emit(FMOV_d_imm | Rd(vd) | ImmFP(imm));
3011 } else {
3012 DCHECK(vd.Is2D());
3013 Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
3014 Emit(NEON_Q | op | ImmNEONFP(imm) | NEONCmode(0xF) | Rd(vd));
3015 }
3016 }
3017
fmov(const VRegister & vd,float imm)3018 void Assembler::fmov(const VRegister& vd, float imm) {
3019 if (vd.IsScalar()) {
3020 DCHECK(vd.Is1S());
3021 Emit(FMOV_s_imm | Rd(vd) | ImmFP(imm));
3022 } else {
3023 DCHECK(vd.Is2S() | vd.Is4S());
3024 Instr op = NEONModifiedImmediate_MOVI;
3025 Instr q = vd.Is4S() ? NEON_Q : 0;
3026 Emit(q | op | ImmNEONFP(imm) | NEONCmode(0xF) | Rd(vd));
3027 }
3028 }
3029
fmov(const Register & rd,const VRegister & fn)3030 void Assembler::fmov(const Register& rd, const VRegister& fn) {
3031 DCHECK_EQ(rd.SizeInBits(), fn.SizeInBits());
3032 FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
3033 Emit(op | Rd(rd) | Rn(fn));
3034 }
3035
fmov(const VRegister & vd,const Register & rn)3036 void Assembler::fmov(const VRegister& vd, const Register& rn) {
3037 DCHECK_EQ(vd.SizeInBits(), rn.SizeInBits());
3038 FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
3039 Emit(op | Rd(vd) | Rn(rn));
3040 }
3041
fmov(const VRegister & vd,const VRegister & vn)3042 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
3043 DCHECK_EQ(vd.SizeInBits(), vn.SizeInBits());
3044 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
3045 }
3046
fmov(const VRegister & vd,int index,const Register & rn)3047 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
3048 DCHECK((index == 1) && vd.Is1D() && rn.IsX());
3049 USE(index);
3050 Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
3051 }
3052
fmov(const Register & rd,const VRegister & vn,int index)3053 void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
3054 DCHECK((index == 1) && vn.Is1D() && rd.IsX());
3055 USE(index);
3056 Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
3057 }
3058
fmadd(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)3059 void Assembler::fmadd(const VRegister& fd, const VRegister& fn,
3060 const VRegister& fm, const VRegister& fa) {
3061 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMADD_s : FMADD_d);
3062 }
3063
fmsub(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)3064 void Assembler::fmsub(const VRegister& fd, const VRegister& fn,
3065 const VRegister& fm, const VRegister& fa) {
3066 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMSUB_s : FMSUB_d);
3067 }
3068
fnmadd(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)3069 void Assembler::fnmadd(const VRegister& fd, const VRegister& fn,
3070 const VRegister& fm, const VRegister& fa) {
3071 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMADD_s : FNMADD_d);
3072 }
3073
fnmsub(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)3074 void Assembler::fnmsub(const VRegister& fd, const VRegister& fn,
3075 const VRegister& fm, const VRegister& fa) {
3076 FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMSUB_s : FNMSUB_d);
3077 }
3078
fnmul(const VRegister & vd,const VRegister & vn,const VRegister & vm)3079 void Assembler::fnmul(const VRegister& vd, const VRegister& vn,
3080 const VRegister& vm) {
3081 DCHECK(AreSameSizeAndType(vd, vn, vm));
3082 Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
3083 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3084 }
3085
fcmp(const VRegister & fn,const VRegister & fm)3086 void Assembler::fcmp(const VRegister& fn, const VRegister& fm) {
3087 DCHECK_EQ(fn.SizeInBits(), fm.SizeInBits());
3088 Emit(FPType(fn) | FCMP | Rm(fm) | Rn(fn));
3089 }
3090
fcmp(const VRegister & fn,double value)3091 void Assembler::fcmp(const VRegister& fn, double value) {
3092 USE(value);
3093 // Although the fcmp instruction can strictly only take an immediate value of
3094 // +0.0, we don't need to check for -0.0 because the sign of 0.0 doesn't
3095 // affect the result of the comparison.
3096 DCHECK_EQ(value, 0.0);
3097 Emit(FPType(fn) | FCMP_zero | Rn(fn));
3098 }
3099
fccmp(const VRegister & fn,const VRegister & fm,StatusFlags nzcv,Condition cond)3100 void Assembler::fccmp(const VRegister& fn, const VRegister& fm,
3101 StatusFlags nzcv, Condition cond) {
3102 DCHECK_EQ(fn.SizeInBits(), fm.SizeInBits());
3103 Emit(FPType(fn) | FCCMP | Rm(fm) | Cond(cond) | Rn(fn) | Nzcv(nzcv));
3104 }
3105
fcsel(const VRegister & fd,const VRegister & fn,const VRegister & fm,Condition cond)3106 void Assembler::fcsel(const VRegister& fd, const VRegister& fn,
3107 const VRegister& fm, Condition cond) {
3108 DCHECK_EQ(fd.SizeInBits(), fn.SizeInBits());
3109 DCHECK_EQ(fd.SizeInBits(), fm.SizeInBits());
3110 Emit(FPType(fd) | FCSEL | Rm(fm) | Cond(cond) | Rn(fn) | Rd(fd));
3111 }
3112
NEONFPConvertToInt(const Register & rd,const VRegister & vn,Instr op)3113 void Assembler::NEONFPConvertToInt(const Register& rd, const VRegister& vn,
3114 Instr op) {
3115 Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
3116 }
3117
NEONFPConvertToInt(const VRegister & vd,const VRegister & vn,Instr op)3118 void Assembler::NEONFPConvertToInt(const VRegister& vd, const VRegister& vn,
3119 Instr op) {
3120 if (vn.IsScalar()) {
3121 DCHECK((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
3122 op |= NEON_Q | NEONScalar;
3123 }
3124 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3125 }
3126
fcvt(const VRegister & vd,const VRegister & vn)3127 void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
3128 FPDataProcessing1SourceOp op;
3129 if (vd.Is1D()) {
3130 DCHECK(vn.Is1S() || vn.Is1H());
3131 op = vn.Is1S() ? FCVT_ds : FCVT_dh;
3132 } else if (vd.Is1S()) {
3133 DCHECK(vn.Is1D() || vn.Is1H());
3134 op = vn.Is1D() ? FCVT_sd : FCVT_sh;
3135 } else {
3136 DCHECK(vd.Is1H());
3137 DCHECK(vn.Is1D() || vn.Is1S());
3138 op = vn.Is1D() ? FCVT_hd : FCVT_hs;
3139 }
3140 FPDataProcessing1Source(vd, vn, op);
3141 }
3142
fcvtl(const VRegister & vd,const VRegister & vn)3143 void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
3144 DCHECK((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
3145 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3146 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
3147 }
3148
fcvtl2(const VRegister & vd,const VRegister & vn)3149 void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
3150 DCHECK((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
3151 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3152 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
3153 }
3154
fcvtn(const VRegister & vd,const VRegister & vn)3155 void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
3156 DCHECK((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
3157 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3158 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
3159 }
3160
fcvtn2(const VRegister & vd,const VRegister & vn)3161 void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
3162 DCHECK((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
3163 Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
3164 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
3165 }
3166
fcvtxn(const VRegister & vd,const VRegister & vn)3167 void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
3168 Instr format = 1 << NEONSize_offset;
3169 if (vd.IsScalar()) {
3170 DCHECK(vd.Is1S() && vn.Is1D());
3171 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
3172 } else {
3173 DCHECK(vd.Is2S() && vn.Is2D());
3174 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3175 }
3176 }
3177
fcvtxn2(const VRegister & vd,const VRegister & vn)3178 void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
3179 DCHECK(vd.Is4S() && vn.Is2D());
3180 Instr format = 1 << NEONSize_offset;
3181 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3182 }
3183
3184 #define NEON_FP2REGMISC_FCVT_LIST(V) \
3185 V(fcvtnu, NEON_FCVTNU, FCVTNU) \
3186 V(fcvtns, NEON_FCVTNS, FCVTNS) \
3187 V(fcvtpu, NEON_FCVTPU, FCVTPU) \
3188 V(fcvtps, NEON_FCVTPS, FCVTPS) \
3189 V(fcvtmu, NEON_FCVTMU, FCVTMU) \
3190 V(fcvtms, NEON_FCVTMS, FCVTMS) \
3191 V(fcvtau, NEON_FCVTAU, FCVTAU) \
3192 V(fcvtas, NEON_FCVTAS, FCVTAS)
3193
3194 #define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \
3195 void Assembler::FN(const Register& rd, const VRegister& vn) { \
3196 NEONFPConvertToInt(rd, vn, SCA_OP); \
3197 } \
3198 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3199 NEONFPConvertToInt(vd, vn, VEC_OP); \
3200 }
NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)3201 NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
3202 #undef DEFINE_ASM_FUNCS
3203
3204 void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3205 DCHECK_GE(fbits, 0);
3206 if (fbits == 0) {
3207 NEONFP2RegMisc(vd, vn, NEON_SCVTF);
3208 } else {
3209 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
3210 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
3211 }
3212 }
3213
ucvtf(const VRegister & vd,const VRegister & vn,int fbits)3214 void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3215 DCHECK_GE(fbits, 0);
3216 if (fbits == 0) {
3217 NEONFP2RegMisc(vd, vn, NEON_UCVTF);
3218 } else {
3219 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
3220 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
3221 }
3222 }
3223
scvtf(const VRegister & vd,const Register & rn,int fbits)3224 void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
3225 DCHECK_GE(fbits, 0);
3226 if (fbits == 0) {
3227 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
3228 } else {
3229 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3230 Rd(vd));
3231 }
3232 }
3233
ucvtf(const VRegister & fd,const Register & rn,int fbits)3234 void Assembler::ucvtf(const VRegister& fd, const Register& rn, int fbits) {
3235 DCHECK_GE(fbits, 0);
3236 if (fbits == 0) {
3237 Emit(SF(rn) | FPType(fd) | UCVTF | Rn(rn) | Rd(fd));
3238 } else {
3239 Emit(SF(rn) | FPType(fd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3240 Rd(fd));
3241 }
3242 }
3243
NEON3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,NEON3SameOp vop)3244 void Assembler::NEON3Same(const VRegister& vd, const VRegister& vn,
3245 const VRegister& vm, NEON3SameOp vop) {
3246 DCHECK(AreSameFormat(vd, vn, vm));
3247 DCHECK(vd.IsVector() || !vd.IsQ());
3248
3249 Instr format, op = vop;
3250 if (vd.IsScalar()) {
3251 op |= NEON_Q | NEONScalar;
3252 format = SFormat(vd);
3253 } else {
3254 format = VFormat(vd);
3255 }
3256
3257 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3258 }
3259
NEONFP3Same(const VRegister & vd,const VRegister & vn,const VRegister & vm,Instr op)3260 void Assembler::NEONFP3Same(const VRegister& vd, const VRegister& vn,
3261 const VRegister& vm, Instr op) {
3262 DCHECK(AreSameFormat(vd, vn, vm));
3263 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3264 }
3265
3266 #define NEON_FP2REGMISC_LIST(V) \
3267 V(fabs, NEON_FABS, FABS) \
3268 V(fneg, NEON_FNEG, FNEG) \
3269 V(fsqrt, NEON_FSQRT, FSQRT) \
3270 V(frintn, NEON_FRINTN, FRINTN) \
3271 V(frinta, NEON_FRINTA, FRINTA) \
3272 V(frintp, NEON_FRINTP, FRINTP) \
3273 V(frintm, NEON_FRINTM, FRINTM) \
3274 V(frintx, NEON_FRINTX, FRINTX) \
3275 V(frintz, NEON_FRINTZ, FRINTZ) \
3276 V(frinti, NEON_FRINTI, FRINTI) \
3277 V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
3278 V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar)
3279
3280 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
3281 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3282 Instr op; \
3283 if (vd.IsScalar()) { \
3284 DCHECK(vd.Is1S() || vd.Is1D()); \
3285 op = SCA_OP; \
3286 } else { \
3287 DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
3288 op = VEC_OP; \
3289 } \
3290 NEONFP2RegMisc(vd, vn, op); \
3291 }
NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)3292 NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
3293 #undef DEFINE_ASM_FUNC
3294
3295 void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
3296 DCHECK((vd.Is8H() && vn.Is8B() && shift == 8) ||
3297 (vd.Is4S() && vn.Is4H() && shift == 16) ||
3298 (vd.Is2D() && vn.Is2S() && shift == 32));
3299 USE(shift);
3300 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
3301 }
3302
shll2(const VRegister & vd,const VRegister & vn,int shift)3303 void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
3304 USE(shift);
3305 DCHECK((vd.Is8H() && vn.Is16B() && shift == 8) ||
3306 (vd.Is4S() && vn.Is8H() && shift == 16) ||
3307 (vd.Is2D() && vn.Is4S() && shift == 32));
3308 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
3309 }
3310
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,double value)3311 void Assembler::NEONFP2RegMisc(const VRegister& vd, const VRegister& vn,
3312 NEON2RegMiscOp vop, double value) {
3313 DCHECK(AreSameFormat(vd, vn));
3314 DCHECK_EQ(value, 0.0);
3315 USE(value);
3316
3317 Instr op = vop;
3318 if (vd.IsScalar()) {
3319 DCHECK(vd.Is1S() || vd.Is1D());
3320 op |= NEON_Q | NEONScalar;
3321 } else {
3322 DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S());
3323 }
3324
3325 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3326 }
3327
fcmeq(const VRegister & vd,const VRegister & vn,double value)3328 void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
3329 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
3330 }
3331
fcmge(const VRegister & vd,const VRegister & vn,double value)3332 void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
3333 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
3334 }
3335
fcmgt(const VRegister & vd,const VRegister & vn,double value)3336 void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
3337 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
3338 }
3339
fcmle(const VRegister & vd,const VRegister & vn,double value)3340 void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
3341 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
3342 }
3343
fcmlt(const VRegister & vd,const VRegister & vn,double value)3344 void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
3345 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
3346 }
3347
frecpx(const VRegister & vd,const VRegister & vn)3348 void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
3349 DCHECK(vd.IsScalar());
3350 DCHECK(AreSameFormat(vd, vn));
3351 DCHECK(vd.Is1S() || vd.Is1D());
3352 Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
3353 }
3354
fcvtzs(const Register & rd,const VRegister & vn,int fbits)3355 void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) {
3356 DCHECK(vn.Is1S() || vn.Is1D());
3357 DCHECK((fbits >= 0) && (fbits <= rd.SizeInBits()));
3358 if (fbits == 0) {
3359 Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
3360 } else {
3361 Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
3362 Rd(rd));
3363 }
3364 }
3365
fcvtzs(const VRegister & vd,const VRegister & vn,int fbits)3366 void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
3367 DCHECK_GE(fbits, 0);
3368 if (fbits == 0) {
3369 NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
3370 } else {
3371 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
3372 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
3373 }
3374 }
3375
fcvtzu(const Register & rd,const VRegister & vn,int fbits)3376 void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) {
3377 DCHECK(vn.Is1S() || vn.Is1D());
3378 DCHECK((fbits >= 0) && (fbits <= rd.SizeInBits()));
3379 if (fbits == 0) {
3380 Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
3381 } else {
3382 Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
3383 Rd(rd));
3384 }
3385 }
3386
fcvtzu(const VRegister & vd,const VRegister & vn,int fbits)3387 void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
3388 DCHECK_GE(fbits, 0);
3389 if (fbits == 0) {
3390 NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
3391 } else {
3392 DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
3393 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
3394 }
3395 }
3396
NEONFP2RegMisc(const VRegister & vd,const VRegister & vn,Instr op)3397 void Assembler::NEONFP2RegMisc(const VRegister& vd, const VRegister& vn,
3398 Instr op) {
3399 DCHECK(AreSameFormat(vd, vn));
3400 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3401 }
3402
NEON2RegMisc(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop,int value)3403 void Assembler::NEON2RegMisc(const VRegister& vd, const VRegister& vn,
3404 NEON2RegMiscOp vop, int value) {
3405 DCHECK(AreSameFormat(vd, vn));
3406 DCHECK_EQ(value, 0);
3407 USE(value);
3408
3409 Instr format, op = vop;
3410 if (vd.IsScalar()) {
3411 op |= NEON_Q | NEONScalar;
3412 format = SFormat(vd);
3413 } else {
3414 format = VFormat(vd);
3415 }
3416
3417 Emit(format | op | Rn(vn) | Rd(vd));
3418 }
3419
cmeq(const VRegister & vd,const VRegister & vn,int value)3420 void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
3421 DCHECK(vd.IsVector() || vd.Is1D());
3422 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
3423 }
3424
cmge(const VRegister & vd,const VRegister & vn,int value)3425 void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
3426 DCHECK(vd.IsVector() || vd.Is1D());
3427 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
3428 }
3429
cmgt(const VRegister & vd,const VRegister & vn,int value)3430 void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
3431 DCHECK(vd.IsVector() || vd.Is1D());
3432 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
3433 }
3434
cmle(const VRegister & vd,const VRegister & vn,int value)3435 void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
3436 DCHECK(vd.IsVector() || vd.Is1D());
3437 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
3438 }
3439
cmlt(const VRegister & vd,const VRegister & vn,int value)3440 void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
3441 DCHECK(vd.IsVector() || vd.Is1D());
3442 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
3443 }
3444
3445 #define NEON_3SAME_LIST(V) \
3446 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
3447 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
3448 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
3449 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
3450 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
3451 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
3452 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
3453 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
3454 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
3455 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
3456 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
3457 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
3458 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
3459 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
3460 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
3461 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
3462 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
3463 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
3464 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
3465 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
3466 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
3467 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
3468 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
3469 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
3470 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
3471 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
3472 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
3473 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
3474 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
3475 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
3476 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
3477 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
3478 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
3479 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
3480 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
3481 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
3482 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
3483 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
3484 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
3485 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
3486 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
3487 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
3488 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
3489 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
3490 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
3491 V(uqadd, NEON_UQADD, true) \
3492 V(sqadd, NEON_SQADD, true) \
3493 V(uqsub, NEON_UQSUB, true) \
3494 V(sqsub, NEON_SQSUB, true) \
3495 V(sqshl, NEON_SQSHL, true) \
3496 V(uqshl, NEON_UQSHL, true) \
3497 V(sqrshl, NEON_SQRSHL, true) \
3498 V(uqrshl, NEON_UQRSHL, true)
3499
3500 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3501 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
3502 const VRegister& vm) { \
3503 DCHECK(AS); \
3504 NEON3Same(vd, vn, vm, OP); \
3505 }
3506 NEON_3SAME_LIST(DEFINE_ASM_FUNC)
3507 #undef DEFINE_ASM_FUNC
3508
3509 #define NEON_FP3SAME_LIST_V2(V) \
3510 V(fadd, NEON_FADD, FADD) \
3511 V(fsub, NEON_FSUB, FSUB) \
3512 V(fmul, NEON_FMUL, FMUL) \
3513 V(fdiv, NEON_FDIV, FDIV) \
3514 V(fmax, NEON_FMAX, FMAX) \
3515 V(fmaxnm, NEON_FMAXNM, FMAXNM) \
3516 V(fmin, NEON_FMIN, FMIN) \
3517 V(fminnm, NEON_FMINNM, FMINNM) \
3518 V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \
3519 V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \
3520 V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \
3521 V(fabd, NEON_FABD, NEON_FABD_scalar) \
3522 V(fmla, NEON_FMLA, 0) \
3523 V(fmls, NEON_FMLS, 0) \
3524 V(facge, NEON_FACGE, NEON_FACGE_scalar) \
3525 V(facgt, NEON_FACGT, NEON_FACGT_scalar) \
3526 V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \
3527 V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \
3528 V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \
3529 V(faddp, NEON_FADDP, 0) \
3530 V(fmaxp, NEON_FMAXP, 0) \
3531 V(fminp, NEON_FMINP, 0) \
3532 V(fmaxnmp, NEON_FMAXNMP, 0) \
3533 V(fminnmp, NEON_FMINNMP, 0)
3534
3535 #define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
3536 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
3537 const VRegister& vm) { \
3538 Instr op; \
3539 if ((SCA_OP != 0) && vd.IsScalar()) { \
3540 DCHECK(vd.Is1S() || vd.Is1D()); \
3541 op = SCA_OP; \
3542 } else { \
3543 DCHECK(vd.IsVector()); \
3544 DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
3545 op = VEC_OP; \
3546 } \
3547 NEONFP3Same(vd, vn, vm, op); \
3548 }
NEON_FP3SAME_LIST_V2(DEFINE_ASM_FUNC)3549 NEON_FP3SAME_LIST_V2(DEFINE_ASM_FUNC)
3550 #undef DEFINE_ASM_FUNC
3551
3552 void Assembler::addp(const VRegister& vd, const VRegister& vn) {
3553 DCHECK((vd.Is1D() && vn.Is2D()));
3554 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
3555 }
3556
faddp(const VRegister & vd,const VRegister & vn)3557 void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
3558 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
3559 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
3560 }
3561
fmaxp(const VRegister & vd,const VRegister & vn)3562 void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
3563 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
3564 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
3565 }
3566
fminp(const VRegister & vd,const VRegister & vn)3567 void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
3568 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
3569 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
3570 }
3571
fmaxnmp(const VRegister & vd,const VRegister & vn)3572 void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
3573 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
3574 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
3575 }
3576
fminnmp(const VRegister & vd,const VRegister & vn)3577 void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
3578 DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
3579 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
3580 }
3581
orr(const VRegister & vd,const int imm8,const int left_shift)3582 void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
3583 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
3584 }
3585
mov(const VRegister & vd,const VRegister & vn)3586 void Assembler::mov(const VRegister& vd, const VRegister& vn) {
3587 DCHECK(AreSameFormat(vd, vn));
3588 if (vd.IsD()) {
3589 orr(vd.V8B(), vn.V8B(), vn.V8B());
3590 } else {
3591 DCHECK(vd.IsQ());
3592 orr(vd.V16B(), vn.V16B(), vn.V16B());
3593 }
3594 }
3595
bic(const VRegister & vd,const int imm8,const int left_shift)3596 void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
3597 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
3598 }
3599
movi(const VRegister & vd,const uint64_t imm,Shift shift,const int shift_amount)3600 void Assembler::movi(const VRegister& vd, const uint64_t imm, Shift shift,
3601 const int shift_amount) {
3602 DCHECK((shift == LSL) || (shift == MSL));
3603 if (vd.Is2D() || vd.Is1D()) {
3604 DCHECK_EQ(shift_amount, 0);
3605 int imm8 = 0;
3606 for (int i = 0; i < 8; ++i) {
3607 int byte = (imm >> (i * 8)) & 0xFF;
3608 DCHECK((byte == 0) || (byte == 0xFF));
3609 if (byte == 0xFF) {
3610 imm8 |= (1 << i);
3611 }
3612 }
3613 Instr q = vd.Is2D() ? NEON_Q : 0;
3614 Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
3615 ImmNEONabcdefgh(imm8) | NEONCmode(0xE) | Rd(vd));
3616 } else if (shift == LSL) {
3617 NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount,
3618 NEONModifiedImmediate_MOVI);
3619 } else {
3620 NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount,
3621 NEONModifiedImmediate_MOVI);
3622 }
3623 }
3624
mvn(const VRegister & vd,const VRegister & vn)3625 void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
3626 DCHECK(AreSameFormat(vd, vn));
3627 if (vd.IsD()) {
3628 not_(vd.V8B(), vn.V8B());
3629 } else {
3630 DCHECK(vd.IsQ());
3631 not_(vd.V16B(), vn.V16B());
3632 }
3633 }
3634
mvni(const VRegister & vd,const int imm8,Shift shift,const int shift_amount)3635 void Assembler::mvni(const VRegister& vd, const int imm8, Shift shift,
3636 const int shift_amount) {
3637 DCHECK((shift == LSL) || (shift == MSL));
3638 if (shift == LSL) {
3639 NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
3640 } else {
3641 NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
3642 }
3643 }
3644
NEONFPByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3645 void Assembler::NEONFPByElement(const VRegister& vd, const VRegister& vn,
3646 const VRegister& vm, int vm_index,
3647 NEONByIndexedElementOp vop) {
3648 DCHECK(AreSameFormat(vd, vn));
3649 DCHECK((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
3650 (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
3651 (vd.Is1D() && vm.Is1D()));
3652 DCHECK((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2)));
3653
3654 Instr op = vop;
3655 int index_num_bits = vm.Is1S() ? 2 : 1;
3656 if (vd.IsScalar()) {
3657 op |= NEON_Q | NEONScalar;
3658 }
3659
3660 Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) |
3661 Rn(vn) | Rd(vd));
3662 }
3663
NEONByElement(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3664 void Assembler::NEONByElement(const VRegister& vd, const VRegister& vn,
3665 const VRegister& vm, int vm_index,
3666 NEONByIndexedElementOp vop) {
3667 DCHECK(AreSameFormat(vd, vn));
3668 DCHECK((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
3669 (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
3670 (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
3671 DCHECK((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3672 (vm.Is1S() && (vm_index < 4)));
3673
3674 Instr format, op = vop;
3675 int index_num_bits = vm.Is1H() ? 3 : 2;
3676 if (vd.IsScalar()) {
3677 op |= NEONScalar | NEON_Q;
3678 format = SFormat(vn);
3679 } else {
3680 format = VFormat(vn);
3681 }
3682 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
3683 Rd(vd));
3684 }
3685
NEONByElementL(const VRegister & vd,const VRegister & vn,const VRegister & vm,int vm_index,NEONByIndexedElementOp vop)3686 void Assembler::NEONByElementL(const VRegister& vd, const VRegister& vn,
3687 const VRegister& vm, int vm_index,
3688 NEONByIndexedElementOp vop) {
3689 DCHECK((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
3690 (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
3691 (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
3692 (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
3693 (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
3694 (vd.Is1D() && vn.Is1S() && vm.Is1S()));
3695
3696 DCHECK((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
3697 (vm.Is1S() && (vm_index < 4)));
3698
3699 Instr format, op = vop;
3700 int index_num_bits = vm.Is1H() ? 3 : 2;
3701 if (vd.IsScalar()) {
3702 op |= NEONScalar | NEON_Q;
3703 format = SFormat(vn);
3704 } else {
3705 format = VFormat(vn);
3706 }
3707 Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
3708 Rd(vd));
3709 }
3710
3711 #define NEON_BYELEMENT_LIST(V) \
3712 V(mul, NEON_MUL_byelement, vn.IsVector()) \
3713 V(mla, NEON_MLA_byelement, vn.IsVector()) \
3714 V(mls, NEON_MLS_byelement, vn.IsVector()) \
3715 V(sqdmulh, NEON_SQDMULH_byelement, true) \
3716 V(sqrdmulh, NEON_SQRDMULH_byelement, true)
3717
3718 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3719 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
3720 const VRegister& vm, int vm_index) { \
3721 DCHECK(AS); \
3722 NEONByElement(vd, vn, vm, vm_index, OP); \
3723 }
3724 NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
3725 #undef DEFINE_ASM_FUNC
3726
3727 #define NEON_FPBYELEMENT_LIST(V) \
3728 V(fmul, NEON_FMUL_byelement) \
3729 V(fmla, NEON_FMLA_byelement) \
3730 V(fmls, NEON_FMLS_byelement) \
3731 V(fmulx, NEON_FMULX_byelement)
3732
3733 #define DEFINE_ASM_FUNC(FN, OP) \
3734 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
3735 const VRegister& vm, int vm_index) { \
3736 NEONFPByElement(vd, vn, vm, vm_index, OP); \
3737 }
NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)3738 NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
3739 #undef DEFINE_ASM_FUNC
3740
3741 #define NEON_BYELEMENT_LONG_LIST(V) \
3742 V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
3743 V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
3744 V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
3745 V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3746 V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
3747 V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
3748 V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
3749 V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
3750 V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
3751 V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
3752 V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
3753 V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3754 V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
3755 V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
3756 V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
3757 V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
3758 V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
3759 V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
3760
3761 #define DEFINE_ASM_FUNC(FN, OP, AS) \
3762 void Assembler::FN(const VRegister& vd, const VRegister& vn, \
3763 const VRegister& vm, int vm_index) { \
3764 DCHECK(AS); \
3765 NEONByElementL(vd, vn, vm, vm_index, OP); \
3766 }
3767 NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
3768 #undef DEFINE_ASM_FUNC
3769
3770 void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
3771 NEON2RegMisc(vd, vn, NEON_SUQADD);
3772 }
3773
usqadd(const VRegister & vd,const VRegister & vn)3774 void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
3775 NEON2RegMisc(vd, vn, NEON_USQADD);
3776 }
3777
abs(const VRegister & vd,const VRegister & vn)3778 void Assembler::abs(const VRegister& vd, const VRegister& vn) {
3779 DCHECK(vd.IsVector() || vd.Is1D());
3780 NEON2RegMisc(vd, vn, NEON_ABS);
3781 }
3782
sqabs(const VRegister & vd,const VRegister & vn)3783 void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
3784 NEON2RegMisc(vd, vn, NEON_SQABS);
3785 }
3786
neg(const VRegister & vd,const VRegister & vn)3787 void Assembler::neg(const VRegister& vd, const VRegister& vn) {
3788 DCHECK(vd.IsVector() || vd.Is1D());
3789 NEON2RegMisc(vd, vn, NEON_NEG);
3790 }
3791
sqneg(const VRegister & vd,const VRegister & vn)3792 void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
3793 NEON2RegMisc(vd, vn, NEON_SQNEG);
3794 }
3795
NEONXtn(const VRegister & vd,const VRegister & vn,NEON2RegMiscOp vop)3796 void Assembler::NEONXtn(const VRegister& vd, const VRegister& vn,
3797 NEON2RegMiscOp vop) {
3798 Instr format, op = vop;
3799 if (vd.IsScalar()) {
3800 DCHECK((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
3801 (vd.Is1S() && vn.Is1D()));
3802 op |= NEON_Q | NEONScalar;
3803 format = SFormat(vd);
3804 } else {
3805 DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
3806 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
3807 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
3808 format = VFormat(vd);
3809 }
3810 Emit(format | op | Rn(vn) | Rd(vd));
3811 }
3812
xtn(const VRegister & vd,const VRegister & vn)3813 void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
3814 DCHECK(vd.IsVector() && vd.IsD());
3815 NEONXtn(vd, vn, NEON_XTN);
3816 }
3817
xtn2(const VRegister & vd,const VRegister & vn)3818 void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
3819 DCHECK(vd.IsVector() && vd.IsQ());
3820 NEONXtn(vd, vn, NEON_XTN);
3821 }
3822
sqxtn(const VRegister & vd,const VRegister & vn)3823 void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
3824 DCHECK(vd.IsScalar() || vd.IsD());
3825 NEONXtn(vd, vn, NEON_SQXTN);
3826 }
3827
sqxtn2(const VRegister & vd,const VRegister & vn)3828 void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
3829 DCHECK(vd.IsVector() && vd.IsQ());
3830 NEONXtn(vd, vn, NEON_SQXTN);
3831 }
3832
sqxtun(const VRegister & vd,const VRegister & vn)3833 void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
3834 DCHECK(vd.IsScalar() || vd.IsD());
3835 NEONXtn(vd, vn, NEON_SQXTUN);
3836 }
3837
sqxtun2(const VRegister & vd,const VRegister & vn)3838 void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
3839 DCHECK(vd.IsVector() && vd.IsQ());
3840 NEONXtn(vd, vn, NEON_SQXTUN);
3841 }
3842
uqxtn(const VRegister & vd,const VRegister & vn)3843 void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
3844 DCHECK(vd.IsScalar() || vd.IsD());
3845 NEONXtn(vd, vn, NEON_UQXTN);
3846 }
3847
uqxtn2(const VRegister & vd,const VRegister & vn)3848 void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
3849 DCHECK(vd.IsVector() && vd.IsQ());
3850 NEONXtn(vd, vn, NEON_UQXTN);
3851 }
3852
3853 // NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
not_(const VRegister & vd,const VRegister & vn)3854 void Assembler::not_(const VRegister& vd, const VRegister& vn) {
3855 DCHECK(AreSameFormat(vd, vn));
3856 DCHECK(vd.Is8B() || vd.Is16B());
3857 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3858 }
3859
rbit(const VRegister & vd,const VRegister & vn)3860 void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
3861 DCHECK(AreSameFormat(vd, vn));
3862 DCHECK(vd.Is8B() || vd.Is16B());
3863 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
3864 }
3865
ext(const VRegister & vd,const VRegister & vn,const VRegister & vm,int index)3866 void Assembler::ext(const VRegister& vd, const VRegister& vn,
3867 const VRegister& vm, int index) {
3868 DCHECK(AreSameFormat(vd, vn, vm));
3869 DCHECK(vd.Is8B() || vd.Is16B());
3870 DCHECK((0 <= index) && (index < vd.LaneCount()));
3871 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
3872 }
3873
dup(const VRegister & vd,const VRegister & vn,int vn_index)3874 void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
3875 Instr q, scalar;
3876
3877 // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
3878 // number of lanes, and T is b, h, s or d.
3879 int lane_size = vn.LaneSizeInBytes();
3880 NEONFormatField format;
3881 switch (lane_size) {
3882 case 1:
3883 format = NEON_16B;
3884 break;
3885 case 2:
3886 format = NEON_8H;
3887 break;
3888 case 4:
3889 format = NEON_4S;
3890 break;
3891 default:
3892 DCHECK_EQ(lane_size, 8);
3893 format = NEON_2D;
3894 break;
3895 }
3896
3897 if (vd.IsScalar()) {
3898 q = NEON_Q;
3899 scalar = NEONScalar;
3900 } else {
3901 DCHECK(!vd.Is1D());
3902 q = vd.IsD() ? 0 : NEON_Q;
3903 scalar = 0;
3904 }
3905 Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) |
3906 Rd(vd));
3907 }
3908
dcptr(Label * label)3909 void Assembler::dcptr(Label* label) {
3910 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
3911 if (label->is_bound()) {
3912 // The label is bound, so it does not need to be updated and the internal
3913 // reference should be emitted.
3914 //
3915 // In this case, label->pos() returns the offset of the label from the
3916 // start of the buffer.
3917 internal_reference_positions_.push_back(pc_offset());
3918 dc64(reinterpret_cast<uintptr_t>(buffer_ + label->pos()));
3919 } else {
3920 int32_t offset;
3921 if (label->is_linked()) {
3922 // The label is linked, so the internal reference should be added
3923 // onto the end of the label's link chain.
3924 //
3925 // In this case, label->pos() returns the offset of the last linked
3926 // instruction from the start of the buffer.
3927 offset = label->pos() - pc_offset();
3928 DCHECK_NE(offset, kStartOfLabelLinkChain);
3929 } else {
3930 // The label is unused, so it now becomes linked and the internal
3931 // reference is at the start of the new link chain.
3932 offset = kStartOfLabelLinkChain;
3933 }
3934 // The instruction at pc is now the last link in the label's chain.
3935 label->link_to(pc_offset());
3936
3937 // Traditionally the offset to the previous instruction in the chain is
3938 // encoded in the instruction payload (e.g. branch range) but internal
3939 // references are not instructions so while unbound they are encoded as
3940 // two consecutive brk instructions. The two 16-bit immediates are used
3941 // to encode the offset.
3942 offset >>= kInstructionSizeLog2;
3943 DCHECK(is_int32(offset));
3944 uint32_t high16 = unsigned_bitextract_32(31, 16, offset);
3945 uint32_t low16 = unsigned_bitextract_32(15, 0, offset);
3946
3947 brk(high16);
3948 brk(low16);
3949 }
3950 }
3951
3952 // Below, a difference in case for the same letter indicates a
3953 // negated bit. If b is 1, then B is 0.
FPToImm8(double imm)3954 uint32_t Assembler::FPToImm8(double imm) {
3955 DCHECK(IsImmFP64(imm));
3956 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
3957 // 0000.0000.0000.0000.0000.0000.0000.0000
3958 uint64_t bits = bit_cast<uint64_t>(imm);
3959 // bit7: a000.0000
3960 uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
3961 // bit6: 0b00.0000
3962 uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
3963 // bit5_to_0: 00cd.efgh
3964 uint64_t bit5_to_0 = (bits >> 48) & 0x3F;
3965
3966 return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
3967 }
3968
ImmFP(double imm)3969 Instr Assembler::ImmFP(double imm) { return FPToImm8(imm) << ImmFP_offset; }
ImmNEONFP(double imm)3970 Instr Assembler::ImmNEONFP(double imm) {
3971 return ImmNEONabcdefgh(FPToImm8(imm));
3972 }
3973
3974 // Code generation helpers.
MoveWide(const Register & rd,uint64_t imm,int shift,MoveWideImmediateOp mov_op)3975 void Assembler::MoveWide(const Register& rd, uint64_t imm, int shift,
3976 MoveWideImmediateOp mov_op) {
3977 // Ignore the top 32 bits of an immediate if we're moving to a W register.
3978 if (rd.Is32Bits()) {
3979 // Check that the top 32 bits are zero (a positive 32-bit number) or top
3980 // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
3981 DCHECK(((imm >> kWRegSizeInBits) == 0) ||
3982 ((imm >> (kWRegSizeInBits - 1)) == 0x1FFFFFFFF));
3983 imm &= kWRegMask;
3984 }
3985
3986 if (shift >= 0) {
3987 // Explicit shift specified.
3988 DCHECK((shift == 0) || (shift == 16) || (shift == 32) || (shift == 48));
3989 DCHECK(rd.Is64Bits() || (shift == 0) || (shift == 16));
3990 shift /= 16;
3991 } else {
3992 // Calculate a new immediate and shift combination to encode the immediate
3993 // argument.
3994 shift = 0;
3995 if ((imm & ~0xFFFFUL) == 0) {
3996 // Nothing to do.
3997 } else if ((imm & ~(0xFFFFUL << 16)) == 0) {
3998 imm >>= 16;
3999 shift = 1;
4000 } else if ((imm & ~(0xFFFFUL << 32)) == 0) {
4001 DCHECK(rd.Is64Bits());
4002 imm >>= 32;
4003 shift = 2;
4004 } else if ((imm & ~(0xFFFFUL << 48)) == 0) {
4005 DCHECK(rd.Is64Bits());
4006 imm >>= 48;
4007 shift = 3;
4008 }
4009 }
4010
4011 DCHECK(is_uint16(imm));
4012
4013 Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) |
4014 ImmMoveWide(static_cast<int>(imm)) | ShiftMoveWide(shift));
4015 }
4016
AddSub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)4017 void Assembler::AddSub(const Register& rd, const Register& rn,
4018 const Operand& operand, FlagsUpdate S, AddSubOp op) {
4019 DCHECK_EQ(rd.SizeInBits(), rn.SizeInBits());
4020 DCHECK(!operand.NeedsRelocation(this));
4021 if (operand.IsImmediate()) {
4022 int64_t immediate = operand.ImmediateValue();
4023 DCHECK(IsImmAddSub(immediate));
4024 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4025 Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
4026 ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
4027 } else if (operand.IsShiftedRegister()) {
4028 DCHECK_EQ(operand.reg().SizeInBits(), rd.SizeInBits());
4029 DCHECK_NE(operand.shift(), ROR);
4030
4031 // For instructions of the form:
4032 // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
4033 // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
4034 // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
4035 // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
4036 // or their 64-bit register equivalents, convert the operand from shifted to
4037 // extended register mode, and emit an add/sub extended instruction.
4038 if (rn.IsSP() || rd.IsSP()) {
4039 DCHECK(!(rd.IsSP() && (S == SetFlags)));
4040 DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
4041 AddSubExtendedFixed | op);
4042 } else {
4043 DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
4044 }
4045 } else {
4046 DCHECK(operand.IsExtendedRegister());
4047 DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
4048 }
4049 }
4050
AddSubWithCarry(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)4051 void Assembler::AddSubWithCarry(const Register& rd, const Register& rn,
4052 const Operand& operand, FlagsUpdate S,
4053 AddSubWithCarryOp op) {
4054 DCHECK_EQ(rd.SizeInBits(), rn.SizeInBits());
4055 DCHECK_EQ(rd.SizeInBits(), operand.reg().SizeInBits());
4056 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4057 DCHECK(!operand.NeedsRelocation(this));
4058 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
4059 }
4060
hlt(int code)4061 void Assembler::hlt(int code) {
4062 DCHECK(is_uint16(code));
4063 Emit(HLT | ImmException(code));
4064 }
4065
brk(int code)4066 void Assembler::brk(int code) {
4067 DCHECK(is_uint16(code));
4068 Emit(BRK | ImmException(code));
4069 }
4070
EmitStringData(const char * string)4071 void Assembler::EmitStringData(const char* string) {
4072 size_t len = strlen(string) + 1;
4073 DCHECK_LE(RoundUp(len, kInstructionSize), static_cast<size_t>(kGap));
4074 EmitData(string, static_cast<int>(len));
4075 // Pad with nullptr characters until pc_ is aligned.
4076 const char pad[] = {'\0', '\0', '\0', '\0'};
4077 static_assert(sizeof(pad) == kInstructionSize,
4078 "Size of padding must match instruction size.");
4079 EmitData(pad, RoundUp(pc_offset(), kInstructionSize) - pc_offset());
4080 }
4081
4082
debug(const char * message,uint32_t code,Instr params)4083 void Assembler::debug(const char* message, uint32_t code, Instr params) {
4084 #ifdef USE_SIMULATOR
4085 // Don't generate simulator specific code if we are building a snapshot, which
4086 // might be run on real hardware.
4087 if (!serializer_enabled()) {
4088 // The arguments to the debug marker need to be contiguous in memory, so
4089 // make sure we don't try to emit pools.
4090 BlockPoolsScope scope(this);
4091
4092 Label start;
4093 bind(&start);
4094
4095 // Refer to instructions-arm64.h for a description of the marker and its
4096 // arguments.
4097 hlt(kImmExceptionIsDebug);
4098 DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugCodeOffset);
4099 dc32(code);
4100 DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugParamsOffset);
4101 dc32(params);
4102 DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugMessageOffset);
4103 EmitStringData(message);
4104 hlt(kImmExceptionIsUnreachable);
4105
4106 return;
4107 }
4108 // Fall through if Serializer is enabled.
4109 #endif
4110
4111 if (params & BREAK) {
4112 brk(0);
4113 }
4114 }
4115
4116
Logical(const Register & rd,const Register & rn,const Operand & operand,LogicalOp op)4117 void Assembler::Logical(const Register& rd,
4118 const Register& rn,
4119 const Operand& operand,
4120 LogicalOp op) {
4121 DCHECK(rd.SizeInBits() == rn.SizeInBits());
4122 DCHECK(!operand.NeedsRelocation(this));
4123 if (operand.IsImmediate()) {
4124 int64_t immediate = operand.ImmediateValue();
4125 unsigned reg_size = rd.SizeInBits();
4126
4127 DCHECK_NE(immediate, 0);
4128 DCHECK_NE(immediate, -1);
4129 DCHECK(rd.Is64Bits() || is_uint32(immediate));
4130
4131 // If the operation is NOT, invert the operation and immediate.
4132 if ((op & NOT) == NOT) {
4133 op = static_cast<LogicalOp>(op & ~NOT);
4134 immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
4135 }
4136
4137 unsigned n, imm_s, imm_r;
4138 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
4139 // Immediate can be encoded in the instruction.
4140 LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
4141 } else {
4142 // This case is handled in the macro assembler.
4143 UNREACHABLE();
4144 }
4145 } else {
4146 DCHECK(operand.IsShiftedRegister());
4147 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
4148 Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
4149 DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
4150 }
4151 }
4152
4153
LogicalImmediate(const Register & rd,const Register & rn,unsigned n,unsigned imm_s,unsigned imm_r,LogicalOp op)4154 void Assembler::LogicalImmediate(const Register& rd,
4155 const Register& rn,
4156 unsigned n,
4157 unsigned imm_s,
4158 unsigned imm_r,
4159 LogicalOp op) {
4160 unsigned reg_size = rd.SizeInBits();
4161 Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
4162 Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
4163 ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
4164 Rn(rn));
4165 }
4166
4167
ConditionalCompare(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)4168 void Assembler::ConditionalCompare(const Register& rn,
4169 const Operand& operand,
4170 StatusFlags nzcv,
4171 Condition cond,
4172 ConditionalCompareOp op) {
4173 Instr ccmpop;
4174 DCHECK(!operand.NeedsRelocation(this));
4175 if (operand.IsImmediate()) {
4176 int64_t immediate = operand.ImmediateValue();
4177 DCHECK(IsImmConditionalCompare(immediate));
4178 ccmpop = ConditionalCompareImmediateFixed | op |
4179 ImmCondCmp(static_cast<unsigned>(immediate));
4180 } else {
4181 DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
4182 ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
4183 }
4184 Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
4185 }
4186
4187
DataProcessing1Source(const Register & rd,const Register & rn,DataProcessing1SourceOp op)4188 void Assembler::DataProcessing1Source(const Register& rd,
4189 const Register& rn,
4190 DataProcessing1SourceOp op) {
4191 DCHECK(rd.SizeInBits() == rn.SizeInBits());
4192 Emit(SF(rn) | op | Rn(rn) | Rd(rd));
4193 }
4194
FPDataProcessing1Source(const VRegister & vd,const VRegister & vn,FPDataProcessing1SourceOp op)4195 void Assembler::FPDataProcessing1Source(const VRegister& vd,
4196 const VRegister& vn,
4197 FPDataProcessing1SourceOp op) {
4198 Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
4199 }
4200
FPDataProcessing2Source(const VRegister & fd,const VRegister & fn,const VRegister & fm,FPDataProcessing2SourceOp op)4201 void Assembler::FPDataProcessing2Source(const VRegister& fd,
4202 const VRegister& fn,
4203 const VRegister& fm,
4204 FPDataProcessing2SourceOp op) {
4205 DCHECK(fd.SizeInBits() == fn.SizeInBits());
4206 DCHECK(fd.SizeInBits() == fm.SizeInBits());
4207 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd));
4208 }
4209
FPDataProcessing3Source(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa,FPDataProcessing3SourceOp op)4210 void Assembler::FPDataProcessing3Source(const VRegister& fd,
4211 const VRegister& fn,
4212 const VRegister& fm,
4213 const VRegister& fa,
4214 FPDataProcessing3SourceOp op) {
4215 DCHECK(AreSameSizeAndType(fd, fn, fm, fa));
4216 Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd) | Ra(fa));
4217 }
4218
NEONModifiedImmShiftLsl(const VRegister & vd,const int imm8,const int left_shift,NEONModifiedImmediateOp op)4219 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd, const int imm8,
4220 const int left_shift,
4221 NEONModifiedImmediateOp op) {
4222 DCHECK(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
4223 vd.Is4S());
4224 DCHECK((left_shift == 0) || (left_shift == 8) || (left_shift == 16) ||
4225 (left_shift == 24));
4226 DCHECK(is_uint8(imm8));
4227
4228 int cmode_1, cmode_2, cmode_3;
4229 if (vd.Is8B() || vd.Is16B()) {
4230 DCHECK_EQ(op, NEONModifiedImmediate_MOVI);
4231 cmode_1 = 1;
4232 cmode_2 = 1;
4233 cmode_3 = 1;
4234 } else {
4235 cmode_1 = (left_shift >> 3) & 1;
4236 cmode_2 = left_shift >> 4;
4237 cmode_3 = 0;
4238 if (vd.Is4H() || vd.Is8H()) {
4239 DCHECK((left_shift == 0) || (left_shift == 8));
4240 cmode_3 = 1;
4241 }
4242 }
4243 int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
4244
4245 Instr q = vd.IsQ() ? NEON_Q : 0;
4246
4247 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4248 }
4249
NEONModifiedImmShiftMsl(const VRegister & vd,const int imm8,const int shift_amount,NEONModifiedImmediateOp op)4250 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd, const int imm8,
4251 const int shift_amount,
4252 NEONModifiedImmediateOp op) {
4253 DCHECK(vd.Is2S() || vd.Is4S());
4254 DCHECK((shift_amount == 8) || (shift_amount == 16));
4255 DCHECK(is_uint8(imm8));
4256
4257 int cmode_0 = (shift_amount >> 4) & 1;
4258 int cmode = 0xC | cmode_0;
4259
4260 Instr q = vd.IsQ() ? NEON_Q : 0;
4261
4262 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
4263 }
4264
EmitShift(const Register & rd,const Register & rn,Shift shift,unsigned shift_amount)4265 void Assembler::EmitShift(const Register& rd,
4266 const Register& rn,
4267 Shift shift,
4268 unsigned shift_amount) {
4269 switch (shift) {
4270 case LSL:
4271 lsl(rd, rn, shift_amount);
4272 break;
4273 case LSR:
4274 lsr(rd, rn, shift_amount);
4275 break;
4276 case ASR:
4277 asr(rd, rn, shift_amount);
4278 break;
4279 case ROR:
4280 ror(rd, rn, shift_amount);
4281 break;
4282 default:
4283 UNREACHABLE();
4284 }
4285 }
4286
4287
EmitExtendShift(const Register & rd,const Register & rn,Extend extend,unsigned left_shift)4288 void Assembler::EmitExtendShift(const Register& rd,
4289 const Register& rn,
4290 Extend extend,
4291 unsigned left_shift) {
4292 DCHECK(rd.SizeInBits() >= rn.SizeInBits());
4293 unsigned reg_size = rd.SizeInBits();
4294 // Use the correct size of register.
4295 Register rn_ = Register::Create(rn.code(), rd.SizeInBits());
4296 // Bits extracted are high_bit:0.
4297 unsigned high_bit = (8 << (extend & 0x3)) - 1;
4298 // Number of bits left in the result that are not introduced by the shift.
4299 unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
4300
4301 if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
4302 switch (extend) {
4303 case UXTB:
4304 case UXTH:
4305 case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break;
4306 case SXTB:
4307 case SXTH:
4308 case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break;
4309 case UXTX:
4310 case SXTX: {
4311 DCHECK_EQ(rn.SizeInBits(), kXRegSizeInBits);
4312 // Nothing to extend. Just shift.
4313 lsl(rd, rn_, left_shift);
4314 break;
4315 }
4316 default: UNREACHABLE();
4317 }
4318 } else {
4319 // No need to extend as the extended bits would be shifted away.
4320 lsl(rd, rn_, left_shift);
4321 }
4322 }
4323
4324
DataProcShiftedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)4325 void Assembler::DataProcShiftedRegister(const Register& rd,
4326 const Register& rn,
4327 const Operand& operand,
4328 FlagsUpdate S,
4329 Instr op) {
4330 DCHECK(operand.IsShiftedRegister());
4331 DCHECK(rn.Is64Bits() || (rn.Is32Bits() && is_uint5(operand.shift_amount())));
4332 DCHECK(!operand.NeedsRelocation(this));
4333 Emit(SF(rd) | op | Flags(S) |
4334 ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) |
4335 Rm(operand.reg()) | Rn(rn) | Rd(rd));
4336 }
4337
4338
DataProcExtendedRegister(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,Instr op)4339 void Assembler::DataProcExtendedRegister(const Register& rd,
4340 const Register& rn,
4341 const Operand& operand,
4342 FlagsUpdate S,
4343 Instr op) {
4344 DCHECK(!operand.NeedsRelocation(this));
4345 Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
4346 Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
4347 ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
4348 dest_reg | RnSP(rn));
4349 }
4350
4351
IsImmAddSub(int64_t immediate)4352 bool Assembler::IsImmAddSub(int64_t immediate) {
4353 return is_uint12(immediate) ||
4354 (is_uint12(immediate >> 12) && ((immediate & 0xFFF) == 0));
4355 }
4356
LoadStore(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op)4357 void Assembler::LoadStore(const CPURegister& rt,
4358 const MemOperand& addr,
4359 LoadStoreOp op) {
4360 Instr memop = op | Rt(rt) | RnSP(addr.base());
4361
4362 if (addr.IsImmediateOffset()) {
4363 unsigned size = CalcLSDataSize(op);
4364 if (IsImmLSScaled(addr.offset(), size)) {
4365 int offset = static_cast<int>(addr.offset());
4366 // Use the scaled addressing mode.
4367 Emit(LoadStoreUnsignedOffsetFixed | memop |
4368 ImmLSUnsigned(offset >> size));
4369 } else if (IsImmLSUnscaled(addr.offset())) {
4370 int offset = static_cast<int>(addr.offset());
4371 // Use the unscaled addressing mode.
4372 Emit(LoadStoreUnscaledOffsetFixed | memop | ImmLS(offset));
4373 } else {
4374 // This case is handled in the macro assembler.
4375 UNREACHABLE();
4376 }
4377 } else if (addr.IsRegisterOffset()) {
4378 Extend ext = addr.extend();
4379 Shift shift = addr.shift();
4380 unsigned shift_amount = addr.shift_amount();
4381
4382 // LSL is encoded in the option field as UXTX.
4383 if (shift == LSL) {
4384 ext = UXTX;
4385 }
4386
4387 // Shifts are encoded in one bit, indicating a left shift by the memory
4388 // access size.
4389 DCHECK((shift_amount == 0) ||
4390 (shift_amount == static_cast<unsigned>(CalcLSDataSize(op))));
4391 Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) |
4392 ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0));
4393 } else {
4394 // Pre-index and post-index modes.
4395 DCHECK(!rt.Is(addr.base()));
4396 if (IsImmLSUnscaled(addr.offset())) {
4397 int offset = static_cast<int>(addr.offset());
4398 if (addr.IsPreIndex()) {
4399 Emit(LoadStorePreIndexFixed | memop | ImmLS(offset));
4400 } else {
4401 DCHECK(addr.IsPostIndex());
4402 Emit(LoadStorePostIndexFixed | memop | ImmLS(offset));
4403 }
4404 } else {
4405 // This case is handled in the macro assembler.
4406 UNREACHABLE();
4407 }
4408 }
4409 }
4410
4411
IsImmLSUnscaled(int64_t offset)4412 bool Assembler::IsImmLSUnscaled(int64_t offset) {
4413 return is_int9(offset);
4414 }
4415
IsImmLSScaled(int64_t offset,unsigned size)4416 bool Assembler::IsImmLSScaled(int64_t offset, unsigned size) {
4417 bool offset_is_size_multiple = (((offset >> size) << size) == offset);
4418 return offset_is_size_multiple && is_uint12(offset >> size);
4419 }
4420
IsImmLSPair(int64_t offset,unsigned size)4421 bool Assembler::IsImmLSPair(int64_t offset, unsigned size) {
4422 bool offset_is_size_multiple = (((offset >> size) << size) == offset);
4423 return offset_is_size_multiple && is_int7(offset >> size);
4424 }
4425
4426
IsImmLLiteral(int64_t offset)4427 bool Assembler::IsImmLLiteral(int64_t offset) {
4428 int inst_size = static_cast<int>(kInstructionSizeLog2);
4429 bool offset_is_inst_multiple =
4430 (((offset >> inst_size) << inst_size) == offset);
4431 DCHECK_GT(offset, 0);
4432 offset >>= kLoadLiteralScaleLog2;
4433 return offset_is_inst_multiple && is_intn(offset, ImmLLiteral_width);
4434 }
4435
4436
4437 // Test if a given value can be encoded in the immediate field of a logical
4438 // instruction.
4439 // If it can be encoded, the function returns true, and values pointed to by n,
4440 // imm_s and imm_r are updated with immediates encoded in the format required
4441 // by the corresponding fields in the logical instruction.
4442 // If it can not be encoded, the function returns false, and the values pointed
4443 // to by n, imm_s and imm_r are undefined.
IsImmLogical(uint64_t value,unsigned width,unsigned * n,unsigned * imm_s,unsigned * imm_r)4444 bool Assembler::IsImmLogical(uint64_t value,
4445 unsigned width,
4446 unsigned* n,
4447 unsigned* imm_s,
4448 unsigned* imm_r) {
4449 DCHECK((n != nullptr) && (imm_s != nullptr) && (imm_r != nullptr));
4450 DCHECK((width == kWRegSizeInBits) || (width == kXRegSizeInBits));
4451
4452 bool negate = false;
4453
4454 // Logical immediates are encoded using parameters n, imm_s and imm_r using
4455 // the following table:
4456 //
4457 // N imms immr size S R
4458 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
4459 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
4460 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
4461 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
4462 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
4463 // 0 11110s xxxxxr 2 UInt(s) UInt(r)
4464 // (s bits must not be all set)
4465 //
4466 // A pattern is constructed of size bits, where the least significant S+1 bits
4467 // are set. The pattern is rotated right by R, and repeated across a 32 or
4468 // 64-bit value, depending on destination register width.
4469 //
4470 // Put another way: the basic format of a logical immediate is a single
4471 // contiguous stretch of 1 bits, repeated across the whole word at intervals
4472 // given by a power of 2. To identify them quickly, we first locate the
4473 // lowest stretch of 1 bits, then the next 1 bit above that; that combination
4474 // is different for every logical immediate, so it gives us all the
4475 // information we need to identify the only logical immediate that our input
4476 // could be, and then we simply check if that's the value we actually have.
4477 //
4478 // (The rotation parameter does give the possibility of the stretch of 1 bits
4479 // going 'round the end' of the word. To deal with that, we observe that in
4480 // any situation where that happens the bitwise NOT of the value is also a
4481 // valid logical immediate. So we simply invert the input whenever its low bit
4482 // is set, and then we know that the rotated case can't arise.)
4483
4484 if (value & 1) {
4485 // If the low bit is 1, negate the value, and set a flag to remember that we
4486 // did (so that we can adjust the return values appropriately).
4487 negate = true;
4488 value = ~value;
4489 }
4490
4491 if (width == kWRegSizeInBits) {
4492 // To handle 32-bit logical immediates, the very easiest thing is to repeat
4493 // the input value twice to make a 64-bit word. The correct encoding of that
4494 // as a logical immediate will also be the correct encoding of the 32-bit
4495 // value.
4496
4497 // The most-significant 32 bits may not be zero (ie. negate is true) so
4498 // shift the value left before duplicating it.
4499 value <<= kWRegSizeInBits;
4500 value |= value >> kWRegSizeInBits;
4501 }
4502
4503 // The basic analysis idea: imagine our input word looks like this.
4504 //
4505 // 0011111000111110001111100011111000111110001111100011111000111110
4506 // c b a
4507 // |<--d-->|
4508 //
4509 // We find the lowest set bit (as an actual power-of-2 value, not its index)
4510 // and call it a. Then we add a to our original number, which wipes out the
4511 // bottommost stretch of set bits and replaces it with a 1 carried into the
4512 // next zero bit. Then we look for the new lowest set bit, which is in
4513 // position b, and subtract it, so now our number is just like the original
4514 // but with the lowest stretch of set bits completely gone. Now we find the
4515 // lowest set bit again, which is position c in the diagram above. Then we'll
4516 // measure the distance d between bit positions a and c (using CLZ), and that
4517 // tells us that the only valid logical immediate that could possibly be equal
4518 // to this number is the one in which a stretch of bits running from a to just
4519 // below b is replicated every d bits.
4520 uint64_t a = LargestPowerOf2Divisor(value);
4521 uint64_t value_plus_a = value + a;
4522 uint64_t b = LargestPowerOf2Divisor(value_plus_a);
4523 uint64_t value_plus_a_minus_b = value_plus_a - b;
4524 uint64_t c = LargestPowerOf2Divisor(value_plus_a_minus_b);
4525
4526 int d, clz_a, out_n;
4527 uint64_t mask;
4528
4529 if (c != 0) {
4530 // The general case, in which there is more than one stretch of set bits.
4531 // Compute the repeat distance d, and set up a bitmask covering the basic
4532 // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
4533 // of these cases the N bit of the output will be zero.
4534 clz_a = CountLeadingZeros(a, kXRegSizeInBits);
4535 int clz_c = CountLeadingZeros(c, kXRegSizeInBits);
4536 d = clz_a - clz_c;
4537 mask = ((uint64_t{1} << d) - 1);
4538 out_n = 0;
4539 } else {
4540 // Handle degenerate cases.
4541 //
4542 // If any of those 'find lowest set bit' operations didn't find a set bit at
4543 // all, then the word will have been zero thereafter, so in particular the
4544 // last lowest_set_bit operation will have returned zero. So we can test for
4545 // all the special case conditions in one go by seeing if c is zero.
4546 if (a == 0) {
4547 // The input was zero (or all 1 bits, which will come to here too after we
4548 // inverted it at the start of the function), for which we just return
4549 // false.
4550 return false;
4551 } else {
4552 // Otherwise, if c was zero but a was not, then there's just one stretch
4553 // of set bits in our word, meaning that we have the trivial case of
4554 // d == 64 and only one 'repetition'. Set up all the same variables as in
4555 // the general case above, and set the N bit in the output.
4556 clz_a = CountLeadingZeros(a, kXRegSizeInBits);
4557 d = 64;
4558 mask = ~uint64_t{0};
4559 out_n = 1;
4560 }
4561 }
4562
4563 // If the repeat period d is not a power of two, it can't be encoded.
4564 if (!base::bits::IsPowerOfTwo(d)) {
4565 return false;
4566 }
4567
4568 if (((b - a) & ~mask) != 0) {
4569 // If the bit stretch (b - a) does not fit within the mask derived from the
4570 // repeat period, then fail.
4571 return false;
4572 }
4573
4574 // The only possible option is b - a repeated every d bits. Now we're going to
4575 // actually construct the valid logical immediate derived from that
4576 // specification, and see if it equals our original input.
4577 //
4578 // To repeat a value every d bits, we multiply it by a number of the form
4579 // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
4580 // be derived using a table lookup on CLZ(d).
4581 static const uint64_t multipliers[] = {
4582 0x0000000000000001UL,
4583 0x0000000100000001UL,
4584 0x0001000100010001UL,
4585 0x0101010101010101UL,
4586 0x1111111111111111UL,
4587 0x5555555555555555UL,
4588 };
4589 int multiplier_idx = CountLeadingZeros(d, kXRegSizeInBits) - 57;
4590 // Ensure that the index to the multipliers array is within bounds.
4591 DCHECK((multiplier_idx >= 0) &&
4592 (static_cast<size_t>(multiplier_idx) < arraysize(multipliers)));
4593 uint64_t multiplier = multipliers[multiplier_idx];
4594 uint64_t candidate = (b - a) * multiplier;
4595
4596 if (value != candidate) {
4597 // The candidate pattern doesn't match our input value, so fail.
4598 return false;
4599 }
4600
4601 // We have a match! This is a valid logical immediate, so now we have to
4602 // construct the bits and pieces of the instruction encoding that generates
4603 // it.
4604
4605 // Count the set bits in our basic stretch. The special case of clz(0) == -1
4606 // makes the answer come out right for stretches that reach the very top of
4607 // the word (e.g. numbers like 0xFFFFC00000000000).
4608 int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSizeInBits);
4609 int s = clz_a - clz_b;
4610
4611 // Decide how many bits to rotate right by, to put the low bit of that basic
4612 // stretch in position a.
4613 int r;
4614 if (negate) {
4615 // If we inverted the input right at the start of this function, here's
4616 // where we compensate: the number of set bits becomes the number of clear
4617 // bits, and the rotation count is based on position b rather than position
4618 // a (since b is the location of the 'lowest' 1 bit after inversion).
4619 s = d - s;
4620 r = (clz_b + 1) & (d - 1);
4621 } else {
4622 r = (clz_a + 1) & (d - 1);
4623 }
4624
4625 // Now we're done, except for having to encode the S output in such a way that
4626 // it gives both the number of set bits and the length of the repeated
4627 // segment. The s field is encoded like this:
4628 //
4629 // imms size S
4630 // ssssss 64 UInt(ssssss)
4631 // 0sssss 32 UInt(sssss)
4632 // 10ssss 16 UInt(ssss)
4633 // 110sss 8 UInt(sss)
4634 // 1110ss 4 UInt(ss)
4635 // 11110s 2 UInt(s)
4636 //
4637 // So we 'or' (-d << 1) with our computed s to form imms.
4638 *n = out_n;
4639 *imm_s = ((-d << 1) | (s - 1)) & 0x3F;
4640 *imm_r = r;
4641
4642 return true;
4643 }
4644
4645
IsImmConditionalCompare(int64_t immediate)4646 bool Assembler::IsImmConditionalCompare(int64_t immediate) {
4647 return is_uint5(immediate);
4648 }
4649
4650
IsImmFP32(float imm)4651 bool Assembler::IsImmFP32(float imm) {
4652 // Valid values will have the form:
4653 // aBbb.bbbc.defg.h000.0000.0000.0000.0000
4654 uint32_t bits = bit_cast<uint32_t>(imm);
4655 // bits[19..0] are cleared.
4656 if ((bits & 0x7FFFF) != 0) {
4657 return false;
4658 }
4659
4660 // bits[29..25] are all set or all cleared.
4661 uint32_t b_pattern = (bits >> 16) & 0x3E00;
4662 if (b_pattern != 0 && b_pattern != 0x3E00) {
4663 return false;
4664 }
4665
4666 // bit[30] and bit[29] are opposite.
4667 if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
4668 return false;
4669 }
4670
4671 return true;
4672 }
4673
4674
IsImmFP64(double imm)4675 bool Assembler::IsImmFP64(double imm) {
4676 // Valid values will have the form:
4677 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
4678 // 0000.0000.0000.0000.0000.0000.0000.0000
4679 uint64_t bits = bit_cast<uint64_t>(imm);
4680 // bits[47..0] are cleared.
4681 if ((bits & 0xFFFFFFFFFFFFL) != 0) {
4682 return false;
4683 }
4684
4685 // bits[61..54] are all set or all cleared.
4686 uint32_t b_pattern = (bits >> 48) & 0x3FC0;
4687 if (b_pattern != 0 && b_pattern != 0x3FC0) {
4688 return false;
4689 }
4690
4691 // bit[62] and bit[61] are opposite.
4692 if (((bits ^ (bits << 1)) & 0x4000000000000000L) == 0) {
4693 return false;
4694 }
4695
4696 return true;
4697 }
4698
4699
GrowBuffer()4700 void Assembler::GrowBuffer() {
4701 if (!own_buffer_) FATAL("external code buffer is too small");
4702
4703 // Compute new buffer size.
4704 CodeDesc desc; // the new buffer
4705 if (buffer_size_ < 1 * MB) {
4706 desc.buffer_size = 2 * buffer_size_;
4707 } else {
4708 desc.buffer_size = buffer_size_ + 1 * MB;
4709 }
4710
4711 // Some internal data structures overflow for very large buffers,
4712 // they must ensure that kMaximalBufferSize is not too large.
4713 if (desc.buffer_size > kMaximalBufferSize) {
4714 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
4715 }
4716
4717 byte* buffer = reinterpret_cast<byte*>(buffer_);
4718
4719 // Set up new buffer.
4720 desc.buffer = NewArray<byte>(desc.buffer_size);
4721 desc.origin = this;
4722
4723 desc.instr_size = pc_offset();
4724 desc.reloc_size =
4725 static_cast<int>((buffer + buffer_size_) - reloc_info_writer.pos());
4726
4727 // Copy the data.
4728 intptr_t pc_delta = desc.buffer - buffer;
4729 intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
4730 (buffer + buffer_size_);
4731 memmove(desc.buffer, buffer, desc.instr_size);
4732 memmove(reloc_info_writer.pos() + rc_delta,
4733 reloc_info_writer.pos(), desc.reloc_size);
4734
4735 // Switch buffers.
4736 DeleteArray(buffer_);
4737 buffer_ = desc.buffer;
4738 buffer_size_ = desc.buffer_size;
4739 pc_ = pc_ + pc_delta;
4740 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
4741 reloc_info_writer.last_pc() + pc_delta);
4742
4743 // None of our relocation types are pc relative pointing outside the code
4744 // buffer nor pc absolute pointing inside the code buffer, so there is no need
4745 // to relocate any emitted relocation entries.
4746
4747 // Relocate internal references.
4748 for (auto pos : internal_reference_positions_) {
4749 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
4750 *p += pc_delta;
4751 }
4752
4753 // Pending relocation entries are also relative, no need to relocate.
4754 }
4755
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data,ConstantPoolMode constant_pool_mode)4756 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data,
4757 ConstantPoolMode constant_pool_mode) {
4758 // Non-relocatable constants should not end up in the literal pool.
4759 DCHECK(!RelocInfo::IsNone(rmode));
4760
4761 // We do not try to reuse pool constants.
4762 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, nullptr);
4763 bool write_reloc_info = true;
4764
4765 if ((rmode == RelocInfo::COMMENT) ||
4766 (rmode == RelocInfo::INTERNAL_REFERENCE) ||
4767 (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) ||
4768 (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) ||
4769 (rmode == RelocInfo::DEOPT_INLINING_ID) ||
4770 (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID)) {
4771 // Adjust code for new modes.
4772 DCHECK(RelocInfo::IsComment(rmode) || RelocInfo::IsDeoptReason(rmode) ||
4773 RelocInfo::IsDeoptId(rmode) || RelocInfo::IsDeoptPosition(rmode) ||
4774 RelocInfo::IsInternalReference(rmode) ||
4775 RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode));
4776 // These modes do not need an entry in the constant pool.
4777 } else if (constant_pool_mode == NEEDS_POOL_ENTRY) {
4778 write_reloc_info = constpool_.RecordEntry(data, rmode);
4779 // Make sure the constant pool is not emitted in place of the next
4780 // instruction for which we just recorded relocation info.
4781 BlockConstPoolFor(1);
4782 }
4783 // For modes that cannot use the constant pool, a different sequence of
4784 // instructions will be emitted by this function's caller.
4785
4786 if (!RelocInfo::IsNone(rmode) && write_reloc_info) {
4787 // Don't record external references unless the heap will be serialized.
4788 if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
4789 !serializer_enabled() && !emit_debug_code()) {
4790 return;
4791 }
4792 DCHECK_GE(buffer_space(), kMaxRelocSize); // too late to grow buffer here
4793 reloc_info_writer.Write(&rinfo);
4794 }
4795 }
4796
GetCodeTargetIndex(Handle<Code> target)4797 int Assembler::GetCodeTargetIndex(Handle<Code> target) {
4798 int current = static_cast<int>(code_targets_.size());
4799 if (current > 0 && !target.is_null() &&
4800 code_targets_.back().address() == target.address()) {
4801 // Optimization if we keep jumping to the same code target.
4802 return (current - 1);
4803 } else {
4804 code_targets_.push_back(target);
4805 return current;
4806 }
4807 }
4808
near_jump(int offset,RelocInfo::Mode rmode)4809 void Assembler::near_jump(int offset, RelocInfo::Mode rmode) {
4810 if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode, offset, NO_POOL_ENTRY);
4811 b(offset);
4812 }
4813
near_call(int offset,RelocInfo::Mode rmode)4814 void Assembler::near_call(int offset, RelocInfo::Mode rmode) {
4815 if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode, offset, NO_POOL_ENTRY);
4816 bl(offset);
4817 }
4818
near_call(HeapObjectRequest request)4819 void Assembler::near_call(HeapObjectRequest request) {
4820 RequestHeapObject(request);
4821 int index = GetCodeTargetIndex(Handle<Code>());
4822 RecordRelocInfo(RelocInfo::CODE_TARGET, index, NO_POOL_ENTRY);
4823 bl(index);
4824 }
4825
BlockConstPoolFor(int instructions)4826 void Assembler::BlockConstPoolFor(int instructions) {
4827 int pc_limit = pc_offset() + instructions * kInstructionSize;
4828 if (no_const_pool_before_ < pc_limit) {
4829 no_const_pool_before_ = pc_limit;
4830 // Make sure the pool won't be blocked for too long.
4831 DCHECK(pc_limit < constpool_.MaxPcOffset());
4832 }
4833
4834 if (next_constant_pool_check_ < no_const_pool_before_) {
4835 next_constant_pool_check_ = no_const_pool_before_;
4836 }
4837 }
4838
4839
CheckConstPool(bool force_emit,bool require_jump)4840 void Assembler::CheckConstPool(bool force_emit, bool require_jump) {
4841 // Some short sequence of instruction mustn't be broken up by constant pool
4842 // emission, such sequences are protected by calls to BlockConstPoolFor and
4843 // BlockConstPoolScope.
4844 if (is_const_pool_blocked()) {
4845 // Something is wrong if emission is forced and blocked at the same time.
4846 DCHECK(!force_emit);
4847 return;
4848 }
4849
4850 // There is nothing to do if there are no pending constant pool entries.
4851 if (constpool_.IsEmpty()) {
4852 // Calculate the offset of the next check.
4853 SetNextConstPoolCheckIn(kCheckConstPoolInterval);
4854 return;
4855 }
4856
4857 // We emit a constant pool when:
4858 // * requested to do so by parameter force_emit (e.g. after each function).
4859 // * the distance to the first instruction accessing the constant pool is
4860 // kApproxMaxDistToConstPool or more.
4861 // * the number of entries in the pool is kApproxMaxPoolEntryCount or more.
4862 int dist = constpool_.DistanceToFirstUse();
4863 int count = constpool_.EntryCount();
4864 if (!force_emit &&
4865 (dist < kApproxMaxDistToConstPool) &&
4866 (count < kApproxMaxPoolEntryCount)) {
4867 return;
4868 }
4869
4870
4871 // Emit veneers for branches that would go out of range during emission of the
4872 // constant pool.
4873 int worst_case_size = constpool_.WorstCaseSize();
4874 CheckVeneerPool(false, require_jump,
4875 kVeneerDistanceMargin + worst_case_size);
4876
4877 // Check that the code buffer is large enough before emitting the constant
4878 // pool (this includes the gap to the relocation information).
4879 int needed_space = worst_case_size + kGap + 1 * kInstructionSize;
4880 while (buffer_space() <= needed_space) {
4881 GrowBuffer();
4882 }
4883
4884 Label size_check;
4885 bind(&size_check);
4886 constpool_.Emit(require_jump);
4887 DCHECK(SizeOfCodeGeneratedSince(&size_check) <=
4888 static_cast<unsigned>(worst_case_size));
4889
4890 // Since a constant pool was just emitted, move the check offset forward by
4891 // the standard interval.
4892 SetNextConstPoolCheckIn(kCheckConstPoolInterval);
4893 }
4894
4895
ShouldEmitVeneer(int max_reachable_pc,int margin)4896 bool Assembler::ShouldEmitVeneer(int max_reachable_pc, int margin) {
4897 // Account for the branch around the veneers and the guard.
4898 int protection_offset = 2 * kInstructionSize;
4899 return pc_offset() > max_reachable_pc - margin - protection_offset -
4900 static_cast<int>(unresolved_branches_.size() * kMaxVeneerCodeSize);
4901 }
4902
4903
RecordVeneerPool(int location_offset,int size)4904 void Assembler::RecordVeneerPool(int location_offset, int size) {
4905 RelocInfo rinfo(reinterpret_cast<Address>(buffer_) + location_offset,
4906 RelocInfo::VENEER_POOL, static_cast<intptr_t>(size), nullptr);
4907 reloc_info_writer.Write(&rinfo);
4908 }
4909
4910
EmitVeneers(bool force_emit,bool need_protection,int margin)4911 void Assembler::EmitVeneers(bool force_emit, bool need_protection, int margin) {
4912 BlockPoolsScope scope(this);
4913 RecordComment("[ Veneers");
4914
4915 // The exact size of the veneer pool must be recorded (see the comment at the
4916 // declaration site of RecordConstPool()), but computing the number of
4917 // veneers that will be generated is not obvious. So instead we remember the
4918 // current position and will record the size after the pool has been
4919 // generated.
4920 Label size_check;
4921 bind(&size_check);
4922 int veneer_pool_relocinfo_loc = pc_offset();
4923
4924 Label end;
4925 if (need_protection) {
4926 b(&end);
4927 }
4928
4929 EmitVeneersGuard();
4930
4931 Label veneer_size_check;
4932
4933 std::multimap<int, FarBranchInfo>::iterator it, it_to_delete;
4934
4935 it = unresolved_branches_.begin();
4936 while (it != unresolved_branches_.end()) {
4937 if (force_emit || ShouldEmitVeneer(it->first, margin)) {
4938 Instruction* branch = InstructionAt(it->second.pc_offset_);
4939 Label* label = it->second.label_;
4940
4941 #ifdef DEBUG
4942 bind(&veneer_size_check);
4943 #endif
4944 // Patch the branch to point to the current position, and emit a branch
4945 // to the label.
4946 Instruction* veneer = reinterpret_cast<Instruction*>(pc_);
4947 RemoveBranchFromLabelLinkChain(branch, label, veneer);
4948 branch->SetImmPCOffsetTarget(isolate_data(), veneer);
4949 b(label);
4950 #ifdef DEBUG
4951 DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <=
4952 static_cast<uint64_t>(kMaxVeneerCodeSize));
4953 veneer_size_check.Unuse();
4954 #endif
4955
4956 it_to_delete = it++;
4957 unresolved_branches_.erase(it_to_delete);
4958 } else {
4959 ++it;
4960 }
4961 }
4962
4963 // Record the veneer pool size.
4964 int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check));
4965 RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size);
4966
4967 if (unresolved_branches_.empty()) {
4968 next_veneer_pool_check_ = kMaxInt;
4969 } else {
4970 next_veneer_pool_check_ =
4971 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
4972 }
4973
4974 bind(&end);
4975
4976 RecordComment("]");
4977 }
4978
4979
CheckVeneerPool(bool force_emit,bool require_jump,int margin)4980 void Assembler::CheckVeneerPool(bool force_emit, bool require_jump,
4981 int margin) {
4982 // There is nothing to do if there are no pending veneer pool entries.
4983 if (unresolved_branches_.empty()) {
4984 DCHECK_EQ(next_veneer_pool_check_, kMaxInt);
4985 return;
4986 }
4987
4988 DCHECK(pc_offset() < unresolved_branches_first_limit());
4989
4990 // Some short sequence of instruction mustn't be broken up by veneer pool
4991 // emission, such sequences are protected by calls to BlockVeneerPoolFor and
4992 // BlockVeneerPoolScope.
4993 if (is_veneer_pool_blocked()) {
4994 DCHECK(!force_emit);
4995 return;
4996 }
4997
4998 if (!require_jump) {
4999 // Prefer emitting veneers protected by an existing instruction.
5000 margin *= kVeneerNoProtectionFactor;
5001 }
5002 if (force_emit || ShouldEmitVeneers(margin)) {
5003 EmitVeneers(force_emit, require_jump, margin);
5004 } else {
5005 next_veneer_pool_check_ =
5006 unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
5007 }
5008 }
5009
5010
buffer_space() const5011 int Assembler::buffer_space() const {
5012 return static_cast<int>(reloc_info_writer.pos() - pc_);
5013 }
5014
5015
RecordConstPool(int size)5016 void Assembler::RecordConstPool(int size) {
5017 // We only need this for debugger support, to correctly compute offsets in the
5018 // code.
5019 RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size));
5020 }
5021
5022
PatchAdrFar(int64_t target_offset)5023 void PatchingAssembler::PatchAdrFar(int64_t target_offset) {
5024 // The code at the current instruction should be:
5025 // adr rd, 0
5026 // nop (adr_far)
5027 // nop (adr_far)
5028 // movz scratch, 0
5029
5030 // Verify the expected code.
5031 Instruction* expected_adr = InstructionAt(0);
5032 CHECK(expected_adr->IsAdr() && (expected_adr->ImmPCRel() == 0));
5033 int rd_code = expected_adr->Rd();
5034 for (int i = 0; i < kAdrFarPatchableNNops; ++i) {
5035 CHECK(InstructionAt((i + 1) * kInstructionSize)->IsNop(ADR_FAR_NOP));
5036 }
5037 Instruction* expected_movz =
5038 InstructionAt((kAdrFarPatchableNInstrs - 1) * kInstructionSize);
5039 CHECK(expected_movz->IsMovz() &&
5040 (expected_movz->ImmMoveWide() == 0) &&
5041 (expected_movz->ShiftMoveWide() == 0));
5042 int scratch_code = expected_movz->Rd();
5043
5044 // Patch to load the correct address.
5045 Register rd = Register::XRegFromCode(rd_code);
5046 Register scratch = Register::XRegFromCode(scratch_code);
5047 // Addresses are only 48 bits.
5048 adr(rd, target_offset & 0xFFFF);
5049 movz(scratch, (target_offset >> 16) & 0xFFFF, 16);
5050 movk(scratch, (target_offset >> 32) & 0xFFFF, 32);
5051 DCHECK_EQ(target_offset >> 48, 0);
5052 add(rd, rd, scratch);
5053 }
5054
PatchSubSp(uint32_t immediate)5055 void PatchingAssembler::PatchSubSp(uint32_t immediate) {
5056 // The code at the current instruction should be:
5057 // sub sp, sp, #0
5058
5059 // Verify the expected code.
5060 Instruction* expected_adr = InstructionAt(0);
5061 CHECK(expected_adr->IsAddSubImmediate());
5062 sub(sp, sp, immediate);
5063 }
5064
5065 } // namespace internal
5066 } // namespace v8
5067
5068 #endif // V8_TARGET_ARCH_ARM64
5069