1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_INL_H_
6 #define V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_INL_H_
7
8 #include <ctype.h>
9
10 #include "src/common/globals.h"
11
12 #include "src/base/bits.h"
13 #include "src/codegen/arm64/assembler-arm64-inl.h"
14 #include "src/codegen/arm64/assembler-arm64.h"
15 #include "src/codegen/macro-assembler.h"
16
17 namespace v8 {
18 namespace internal {
19
FieldMemOperand(Register object,int offset)20 MemOperand FieldMemOperand(Register object, int offset) {
21 return MemOperand(object, offset - kHeapObjectTag);
22 }
23
And(const Register & rd,const Register & rn,const Operand & operand)24 void TurboAssembler::And(const Register& rd, const Register& rn,
25 const Operand& operand) {
26 DCHECK(allow_macro_instructions());
27 DCHECK(!rd.IsZero());
28 LogicalMacro(rd, rn, operand, AND);
29 }
30
Ands(const Register & rd,const Register & rn,const Operand & operand)31 void TurboAssembler::Ands(const Register& rd, const Register& rn,
32 const Operand& operand) {
33 DCHECK(allow_macro_instructions());
34 DCHECK(!rd.IsZero());
35 LogicalMacro(rd, rn, operand, ANDS);
36 }
37
Tst(const Register & rn,const Operand & operand)38 void TurboAssembler::Tst(const Register& rn, const Operand& operand) {
39 DCHECK(allow_macro_instructions());
40 LogicalMacro(AppropriateZeroRegFor(rn), rn, operand, ANDS);
41 }
42
Bic(const Register & rd,const Register & rn,const Operand & operand)43 void TurboAssembler::Bic(const Register& rd, const Register& rn,
44 const Operand& operand) {
45 DCHECK(allow_macro_instructions());
46 DCHECK(!rd.IsZero());
47 LogicalMacro(rd, rn, operand, BIC);
48 }
49
Bics(const Register & rd,const Register & rn,const Operand & operand)50 void MacroAssembler::Bics(const Register& rd, const Register& rn,
51 const Operand& operand) {
52 DCHECK(allow_macro_instructions());
53 DCHECK(!rd.IsZero());
54 LogicalMacro(rd, rn, operand, BICS);
55 }
56
Orr(const Register & rd,const Register & rn,const Operand & operand)57 void TurboAssembler::Orr(const Register& rd, const Register& rn,
58 const Operand& operand) {
59 DCHECK(allow_macro_instructions());
60 DCHECK(!rd.IsZero());
61 LogicalMacro(rd, rn, operand, ORR);
62 }
63
Orn(const Register & rd,const Register & rn,const Operand & operand)64 void TurboAssembler::Orn(const Register& rd, const Register& rn,
65 const Operand& operand) {
66 DCHECK(allow_macro_instructions());
67 DCHECK(!rd.IsZero());
68 LogicalMacro(rd, rn, operand, ORN);
69 }
70
Eor(const Register & rd,const Register & rn,const Operand & operand)71 void TurboAssembler::Eor(const Register& rd, const Register& rn,
72 const Operand& operand) {
73 DCHECK(allow_macro_instructions());
74 DCHECK(!rd.IsZero());
75 LogicalMacro(rd, rn, operand, EOR);
76 }
77
Eon(const Register & rd,const Register & rn,const Operand & operand)78 void TurboAssembler::Eon(const Register& rd, const Register& rn,
79 const Operand& operand) {
80 DCHECK(allow_macro_instructions());
81 DCHECK(!rd.IsZero());
82 LogicalMacro(rd, rn, operand, EON);
83 }
84
Ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)85 void TurboAssembler::Ccmp(const Register& rn, const Operand& operand,
86 StatusFlags nzcv, Condition cond) {
87 DCHECK(allow_macro_instructions());
88 if (operand.IsImmediate() && (operand.ImmediateValue() < 0)) {
89 ConditionalCompareMacro(rn, -operand.ImmediateValue(), nzcv, cond, CCMN);
90 } else {
91 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP);
92 }
93 }
94
CcmpTagged(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)95 void TurboAssembler::CcmpTagged(const Register& rn, const Operand& operand,
96 StatusFlags nzcv, Condition cond) {
97 if (COMPRESS_POINTERS_BOOL) {
98 Ccmp(rn.W(), operand.ToW(), nzcv, cond);
99 } else {
100 Ccmp(rn, operand, nzcv, cond);
101 }
102 }
103
Ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)104 void MacroAssembler::Ccmn(const Register& rn, const Operand& operand,
105 StatusFlags nzcv, Condition cond) {
106 DCHECK(allow_macro_instructions());
107 if (operand.IsImmediate() && (operand.ImmediateValue() < 0)) {
108 ConditionalCompareMacro(rn, -operand.ImmediateValue(), nzcv, cond, CCMP);
109 } else {
110 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN);
111 }
112 }
113
Add(const Register & rd,const Register & rn,const Operand & operand)114 void TurboAssembler::Add(const Register& rd, const Register& rn,
115 const Operand& operand) {
116 DCHECK(allow_macro_instructions());
117 if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
118 IsImmAddSub(-operand.ImmediateValue())) {
119 AddSubMacro(rd, rn, -operand.ImmediateValue(), LeaveFlags, SUB);
120 } else {
121 AddSubMacro(rd, rn, operand, LeaveFlags, ADD);
122 }
123 }
124
Adds(const Register & rd,const Register & rn,const Operand & operand)125 void TurboAssembler::Adds(const Register& rd, const Register& rn,
126 const Operand& operand) {
127 DCHECK(allow_macro_instructions());
128 if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
129 IsImmAddSub(-operand.ImmediateValue())) {
130 AddSubMacro(rd, rn, -operand.ImmediateValue(), SetFlags, SUB);
131 } else {
132 AddSubMacro(rd, rn, operand, SetFlags, ADD);
133 }
134 }
135
Sub(const Register & rd,const Register & rn,const Operand & operand)136 void TurboAssembler::Sub(const Register& rd, const Register& rn,
137 const Operand& operand) {
138 DCHECK(allow_macro_instructions());
139 if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
140 IsImmAddSub(-operand.ImmediateValue())) {
141 AddSubMacro(rd, rn, -operand.ImmediateValue(), LeaveFlags, ADD);
142 } else {
143 AddSubMacro(rd, rn, operand, LeaveFlags, SUB);
144 }
145 }
146
Subs(const Register & rd,const Register & rn,const Operand & operand)147 void TurboAssembler::Subs(const Register& rd, const Register& rn,
148 const Operand& operand) {
149 DCHECK(allow_macro_instructions());
150 if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
151 IsImmAddSub(-operand.ImmediateValue())) {
152 AddSubMacro(rd, rn, -operand.ImmediateValue(), SetFlags, ADD);
153 } else {
154 AddSubMacro(rd, rn, operand, SetFlags, SUB);
155 }
156 }
157
Cmn(const Register & rn,const Operand & operand)158 void TurboAssembler::Cmn(const Register& rn, const Operand& operand) {
159 DCHECK(allow_macro_instructions());
160 Adds(AppropriateZeroRegFor(rn), rn, operand);
161 }
162
Cmp(const Register & rn,const Operand & operand)163 void TurboAssembler::Cmp(const Register& rn, const Operand& operand) {
164 DCHECK(allow_macro_instructions());
165 Subs(AppropriateZeroRegFor(rn), rn, operand);
166 }
167
CmpTagged(const Register & rn,const Operand & operand)168 void TurboAssembler::CmpTagged(const Register& rn, const Operand& operand) {
169 if (COMPRESS_POINTERS_BOOL) {
170 Cmp(rn.W(), operand.ToW());
171 } else {
172 Cmp(rn, operand);
173 }
174 }
175
Neg(const Register & rd,const Operand & operand)176 void TurboAssembler::Neg(const Register& rd, const Operand& operand) {
177 DCHECK(allow_macro_instructions());
178 DCHECK(!rd.IsZero());
179 if (operand.IsImmediate()) {
180 Mov(rd, -operand.ImmediateValue());
181 } else {
182 Sub(rd, AppropriateZeroRegFor(rd), operand);
183 }
184 }
185
Negs(const Register & rd,const Operand & operand)186 void TurboAssembler::Negs(const Register& rd, const Operand& operand) {
187 DCHECK(allow_macro_instructions());
188 Subs(rd, AppropriateZeroRegFor(rd), operand);
189 }
190
Adc(const Register & rd,const Register & rn,const Operand & operand)191 void TurboAssembler::Adc(const Register& rd, const Register& rn,
192 const Operand& operand) {
193 DCHECK(allow_macro_instructions());
194 DCHECK(!rd.IsZero());
195 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC);
196 }
197
Adcs(const Register & rd,const Register & rn,const Operand & operand)198 void MacroAssembler::Adcs(const Register& rd, const Register& rn,
199 const Operand& operand) {
200 DCHECK(allow_macro_instructions());
201 DCHECK(!rd.IsZero());
202 AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC);
203 }
204
Sbc(const Register & rd,const Register & rn,const Operand & operand)205 void MacroAssembler::Sbc(const Register& rd, const Register& rn,
206 const Operand& operand) {
207 DCHECK(allow_macro_instructions());
208 DCHECK(!rd.IsZero());
209 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC);
210 }
211
Sbcs(const Register & rd,const Register & rn,const Operand & operand)212 void MacroAssembler::Sbcs(const Register& rd, const Register& rn,
213 const Operand& operand) {
214 DCHECK(allow_macro_instructions());
215 DCHECK(!rd.IsZero());
216 AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC);
217 }
218
Ngc(const Register & rd,const Operand & operand)219 void MacroAssembler::Ngc(const Register& rd, const Operand& operand) {
220 DCHECK(allow_macro_instructions());
221 DCHECK(!rd.IsZero());
222 Register zr = AppropriateZeroRegFor(rd);
223 Sbc(rd, zr, operand);
224 }
225
Ngcs(const Register & rd,const Operand & operand)226 void MacroAssembler::Ngcs(const Register& rd, const Operand& operand) {
227 DCHECK(allow_macro_instructions());
228 DCHECK(!rd.IsZero());
229 Register zr = AppropriateZeroRegFor(rd);
230 Sbcs(rd, zr, operand);
231 }
232
Mvn(const Register & rd,uint64_t imm)233 void TurboAssembler::Mvn(const Register& rd, uint64_t imm) {
234 DCHECK(allow_macro_instructions());
235 DCHECK(!rd.IsZero());
236 Mov(rd, ~imm);
237 }
238
239 #define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \
240 void TurboAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \
241 DCHECK(allow_macro_instructions()); \
242 LoadStoreMacro(REG, addr, OP); \
243 }
244 LS_MACRO_LIST(DEFINE_FUNCTION)
245 #undef DEFINE_FUNCTION
246
247 #define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
248 void TurboAssembler::FN(const REGTYPE REG, const REGTYPE REG2, \
249 const MemOperand& addr) { \
250 DCHECK(allow_macro_instructions()); \
251 LoadStorePairMacro(REG, REG2, addr, OP); \
252 }
LSPAIR_MACRO_LIST(DEFINE_FUNCTION)253 LSPAIR_MACRO_LIST(DEFINE_FUNCTION)
254 #undef DEFINE_FUNCTION
255
256 #define DECLARE_FUNCTION(FN, OP) \
257 void TurboAssembler::FN(const Register& rt, const Register& rn) { \
258 DCHECK(allow_macro_instructions()); \
259 OP(rt, rn); \
260 }
261 LDA_STL_MACRO_LIST(DECLARE_FUNCTION)
262 #undef DECLARE_FUNCTION
263
264 #define DECLARE_FUNCTION(FN, OP) \
265 void MacroAssembler::FN(const Register& rs, const Register& rt, \
266 const Register& rn) { \
267 DCHECK(allow_macro_instructions()); \
268 OP(rs, rt, rn); \
269 }
270 STLX_MACRO_LIST(DECLARE_FUNCTION)
271 #undef DECLARE_FUNCTION
272
273 void TurboAssembler::Asr(const Register& rd, const Register& rn,
274 unsigned shift) {
275 DCHECK(allow_macro_instructions());
276 DCHECK(!rd.IsZero());
277 asr(rd, rn, shift);
278 }
279
Asr(const Register & rd,const Register & rn,const Register & rm)280 void TurboAssembler::Asr(const Register& rd, const Register& rn,
281 const Register& rm) {
282 DCHECK(allow_macro_instructions());
283 DCHECK(!rd.IsZero());
284 asrv(rd, rn, rm);
285 }
286
B(Label * label)287 void TurboAssembler::B(Label* label) {
288 DCHECK(allow_macro_instructions());
289 b(label);
290 CheckVeneerPool(false, false);
291 }
292
B(Condition cond,Label * label)293 void TurboAssembler::B(Condition cond, Label* label) {
294 DCHECK(allow_macro_instructions());
295 B(label, cond);
296 }
297
Bfi(const Register & rd,const Register & rn,unsigned lsb,unsigned width)298 void TurboAssembler::Bfi(const Register& rd, const Register& rn, unsigned lsb,
299 unsigned width) {
300 DCHECK(allow_macro_instructions());
301 DCHECK(!rd.IsZero());
302 bfi(rd, rn, lsb, width);
303 }
304
Bfxil(const Register & rd,const Register & rn,unsigned lsb,unsigned width)305 void MacroAssembler::Bfxil(const Register& rd, const Register& rn, unsigned lsb,
306 unsigned width) {
307 DCHECK(allow_macro_instructions());
308 DCHECK(!rd.IsZero());
309 bfxil(rd, rn, lsb, width);
310 }
311
Bind(Label * label,BranchTargetIdentifier id)312 void TurboAssembler::Bind(Label* label, BranchTargetIdentifier id) {
313 DCHECK(allow_macro_instructions());
314 if (id == BranchTargetIdentifier::kNone) {
315 bind(label);
316 } else {
317 // Emit this inside an InstructionAccurateScope to ensure there are no extra
318 // instructions between the bind and the target identifier instruction.
319 InstructionAccurateScope scope(this, 1);
320 bind(label);
321 if (id == BranchTargetIdentifier::kPaciasp) {
322 paciasp();
323 } else {
324 bti(id);
325 }
326 }
327 }
328
CodeEntry()329 void TurboAssembler::CodeEntry() {
330 // Since `kJavaScriptCallCodeStartRegister` is the target register for tail
331 // calls, we have to allow for jumps too, with "BTI jc". We also allow the
332 // register allocator to pick the target register for calls made from
333 // WebAssembly.
334 // TODO(v8:10026): Consider changing this so that we can use CallTarget(),
335 // which maps to "BTI c", here instead.
336 JumpOrCallTarget();
337 }
338
ExceptionHandler()339 void TurboAssembler::ExceptionHandler() { JumpTarget(); }
340
BindExceptionHandler(Label * label)341 void TurboAssembler::BindExceptionHandler(Label* label) {
342 BindJumpTarget(label);
343 }
344
JumpTarget()345 void TurboAssembler::JumpTarget() {
346 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
347 bti(BranchTargetIdentifier::kBtiJump);
348 #endif
349 }
350
BindJumpTarget(Label * label)351 void TurboAssembler::BindJumpTarget(Label* label) {
352 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
353 Bind(label, BranchTargetIdentifier::kBtiJump);
354 #else
355 Bind(label);
356 #endif
357 }
358
CallTarget()359 void TurboAssembler::CallTarget() {
360 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
361 bti(BranchTargetIdentifier::kBtiCall);
362 #endif
363 }
364
JumpOrCallTarget()365 void TurboAssembler::JumpOrCallTarget() {
366 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
367 bti(BranchTargetIdentifier::kBtiJumpCall);
368 #endif
369 }
370
BindJumpOrCallTarget(Label * label)371 void TurboAssembler::BindJumpOrCallTarget(Label* label) {
372 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
373 Bind(label, BranchTargetIdentifier::kBtiJumpCall);
374 #else
375 Bind(label);
376 #endif
377 }
378
Bl(Label * label)379 void TurboAssembler::Bl(Label* label) {
380 DCHECK(allow_macro_instructions());
381 bl(label);
382 }
383
Blr(const Register & xn)384 void TurboAssembler::Blr(const Register& xn) {
385 DCHECK(allow_macro_instructions());
386 DCHECK(!xn.IsZero());
387 blr(xn);
388 }
389
Br(const Register & xn)390 void TurboAssembler::Br(const Register& xn) {
391 DCHECK(allow_macro_instructions());
392 DCHECK(!xn.IsZero());
393 br(xn);
394 }
395
Brk(int code)396 void TurboAssembler::Brk(int code) {
397 DCHECK(allow_macro_instructions());
398 brk(code);
399 }
400
Cinc(const Register & rd,const Register & rn,Condition cond)401 void MacroAssembler::Cinc(const Register& rd, const Register& rn,
402 Condition cond) {
403 DCHECK(allow_macro_instructions());
404 DCHECK(!rd.IsZero());
405 DCHECK((cond != al) && (cond != nv));
406 cinc(rd, rn, cond);
407 }
408
Cinv(const Register & rd,const Register & rn,Condition cond)409 void MacroAssembler::Cinv(const Register& rd, const Register& rn,
410 Condition cond) {
411 DCHECK(allow_macro_instructions());
412 DCHECK(!rd.IsZero());
413 DCHECK((cond != al) && (cond != nv));
414 cinv(rd, rn, cond);
415 }
416
Cls(const Register & rd,const Register & rn)417 void TurboAssembler::Cls(const Register& rd, const Register& rn) {
418 DCHECK(allow_macro_instructions());
419 DCHECK(!rd.IsZero());
420 cls(rd, rn);
421 }
422
Clz(const Register & rd,const Register & rn)423 void TurboAssembler::Clz(const Register& rd, const Register& rn) {
424 DCHECK(allow_macro_instructions());
425 DCHECK(!rd.IsZero());
426 clz(rd, rn);
427 }
428
Cneg(const Register & rd,const Register & rn,Condition cond)429 void TurboAssembler::Cneg(const Register& rd, const Register& rn,
430 Condition cond) {
431 DCHECK(allow_macro_instructions());
432 DCHECK(!rd.IsZero());
433 DCHECK((cond != al) && (cond != nv));
434 cneg(rd, rn, cond);
435 }
436
437 // Conditionally zero the destination register. Only X registers are supported
438 // due to the truncation side-effect when used on W registers.
CzeroX(const Register & rd,Condition cond)439 void MacroAssembler::CzeroX(const Register& rd, Condition cond) {
440 DCHECK(allow_macro_instructions());
441 DCHECK(!rd.IsSP() && rd.Is64Bits());
442 DCHECK((cond != al) && (cond != nv));
443 csel(rd, xzr, rd, cond);
444 }
445
446 // Conditionally move a value into the destination register. Only X registers
447 // are supported due to the truncation side-effect when used on W registers.
CmovX(const Register & rd,const Register & rn,Condition cond)448 void TurboAssembler::CmovX(const Register& rd, const Register& rn,
449 Condition cond) {
450 DCHECK(allow_macro_instructions());
451 DCHECK(!rd.IsSP());
452 DCHECK(rd.Is64Bits() && rn.Is64Bits());
453 DCHECK((cond != al) && (cond != nv));
454 if (rd != rn) {
455 csel(rd, rn, rd, cond);
456 }
457 }
458
Csdb()459 void TurboAssembler::Csdb() {
460 DCHECK(allow_macro_instructions());
461 csdb();
462 }
463
Cset(const Register & rd,Condition cond)464 void TurboAssembler::Cset(const Register& rd, Condition cond) {
465 DCHECK(allow_macro_instructions());
466 DCHECK(!rd.IsZero());
467 DCHECK((cond != al) && (cond != nv));
468 cset(rd, cond);
469 }
470
Csetm(const Register & rd,Condition cond)471 void TurboAssembler::Csetm(const Register& rd, Condition cond) {
472 DCHECK(allow_macro_instructions());
473 DCHECK(!rd.IsZero());
474 DCHECK((cond != al) && (cond != nv));
475 csetm(rd, cond);
476 }
477
Csinc(const Register & rd,const Register & rn,const Register & rm,Condition cond)478 void TurboAssembler::Csinc(const Register& rd, const Register& rn,
479 const Register& rm, Condition cond) {
480 DCHECK(allow_macro_instructions());
481 DCHECK(!rd.IsZero());
482 DCHECK((cond != al) && (cond != nv));
483 csinc(rd, rn, rm, cond);
484 }
485
Csinv(const Register & rd,const Register & rn,const Register & rm,Condition cond)486 void MacroAssembler::Csinv(const Register& rd, const Register& rn,
487 const Register& rm, Condition cond) {
488 DCHECK(allow_macro_instructions());
489 DCHECK(!rd.IsZero());
490 DCHECK((cond != al) && (cond != nv));
491 csinv(rd, rn, rm, cond);
492 }
493
Csneg(const Register & rd,const Register & rn,const Register & rm,Condition cond)494 void MacroAssembler::Csneg(const Register& rd, const Register& rn,
495 const Register& rm, Condition cond) {
496 DCHECK(allow_macro_instructions());
497 DCHECK(!rd.IsZero());
498 DCHECK((cond != al) && (cond != nv));
499 csneg(rd, rn, rm, cond);
500 }
501
Dmb(BarrierDomain domain,BarrierType type)502 void TurboAssembler::Dmb(BarrierDomain domain, BarrierType type) {
503 DCHECK(allow_macro_instructions());
504 dmb(domain, type);
505 }
506
Dsb(BarrierDomain domain,BarrierType type)507 void TurboAssembler::Dsb(BarrierDomain domain, BarrierType type) {
508 DCHECK(allow_macro_instructions());
509 dsb(domain, type);
510 }
511
Debug(const char * message,uint32_t code,Instr params)512 void TurboAssembler::Debug(const char* message, uint32_t code, Instr params) {
513 DCHECK(allow_macro_instructions());
514 debug(message, code, params);
515 }
516
Extr(const Register & rd,const Register & rn,const Register & rm,unsigned lsb)517 void MacroAssembler::Extr(const Register& rd, const Register& rn,
518 const Register& rm, unsigned lsb) {
519 DCHECK(allow_macro_instructions());
520 DCHECK(!rd.IsZero());
521 extr(rd, rn, rm, lsb);
522 }
523
Fabs(const VRegister & fd,const VRegister & fn)524 void TurboAssembler::Fabs(const VRegister& fd, const VRegister& fn) {
525 DCHECK(allow_macro_instructions());
526 fabs(fd, fn);
527 }
528
Fadd(const VRegister & fd,const VRegister & fn,const VRegister & fm)529 void TurboAssembler::Fadd(const VRegister& fd, const VRegister& fn,
530 const VRegister& fm) {
531 DCHECK(allow_macro_instructions());
532 fadd(fd, fn, fm);
533 }
534
Fccmp(const VRegister & fn,const VRegister & fm,StatusFlags nzcv,Condition cond)535 void TurboAssembler::Fccmp(const VRegister& fn, const VRegister& fm,
536 StatusFlags nzcv, Condition cond) {
537 DCHECK(allow_macro_instructions());
538 DCHECK((cond != al) && (cond != nv));
539 fccmp(fn, fm, nzcv, cond);
540 }
541
Fcmp(const VRegister & fn,const VRegister & fm)542 void TurboAssembler::Fcmp(const VRegister& fn, const VRegister& fm) {
543 DCHECK(allow_macro_instructions());
544 fcmp(fn, fm);
545 }
546
Fcmp(const VRegister & fn,double value)547 void TurboAssembler::Fcmp(const VRegister& fn, double value) {
548 DCHECK(allow_macro_instructions());
549 if (value != 0.0) {
550 UseScratchRegisterScope temps(this);
551 VRegister tmp = temps.AcquireSameSizeAs(fn);
552 Fmov(tmp, value);
553 fcmp(fn, tmp);
554 } else {
555 fcmp(fn, value);
556 }
557 }
558
Fcsel(const VRegister & fd,const VRegister & fn,const VRegister & fm,Condition cond)559 void MacroAssembler::Fcsel(const VRegister& fd, const VRegister& fn,
560 const VRegister& fm, Condition cond) {
561 DCHECK(allow_macro_instructions());
562 DCHECK((cond != al) && (cond != nv));
563 fcsel(fd, fn, fm, cond);
564 }
565
Fcvt(const VRegister & fd,const VRegister & fn)566 void TurboAssembler::Fcvt(const VRegister& fd, const VRegister& fn) {
567 DCHECK(allow_macro_instructions());
568 fcvt(fd, fn);
569 }
570
Fcvtas(const Register & rd,const VRegister & fn)571 void TurboAssembler::Fcvtas(const Register& rd, const VRegister& fn) {
572 DCHECK(allow_macro_instructions());
573 DCHECK(!rd.IsZero());
574 fcvtas(rd, fn);
575 }
576
Fcvtau(const Register & rd,const VRegister & fn)577 void TurboAssembler::Fcvtau(const Register& rd, const VRegister& fn) {
578 DCHECK(allow_macro_instructions());
579 DCHECK(!rd.IsZero());
580 fcvtau(rd, fn);
581 }
582
Fcvtms(const Register & rd,const VRegister & fn)583 void TurboAssembler::Fcvtms(const Register& rd, const VRegister& fn) {
584 DCHECK(allow_macro_instructions());
585 DCHECK(!rd.IsZero());
586 fcvtms(rd, fn);
587 }
588
Fcvtmu(const Register & rd,const VRegister & fn)589 void TurboAssembler::Fcvtmu(const Register& rd, const VRegister& fn) {
590 DCHECK(allow_macro_instructions());
591 DCHECK(!rd.IsZero());
592 fcvtmu(rd, fn);
593 }
594
Fcvtns(const Register & rd,const VRegister & fn)595 void TurboAssembler::Fcvtns(const Register& rd, const VRegister& fn) {
596 DCHECK(allow_macro_instructions());
597 DCHECK(!rd.IsZero());
598 fcvtns(rd, fn);
599 }
600
Fcvtnu(const Register & rd,const VRegister & fn)601 void TurboAssembler::Fcvtnu(const Register& rd, const VRegister& fn) {
602 DCHECK(allow_macro_instructions());
603 DCHECK(!rd.IsZero());
604 fcvtnu(rd, fn);
605 }
606
Fcvtzs(const Register & rd,const VRegister & fn)607 void TurboAssembler::Fcvtzs(const Register& rd, const VRegister& fn) {
608 DCHECK(allow_macro_instructions());
609 DCHECK(!rd.IsZero());
610 fcvtzs(rd, fn);
611 }
Fcvtzu(const Register & rd,const VRegister & fn)612 void TurboAssembler::Fcvtzu(const Register& rd, const VRegister& fn) {
613 DCHECK(allow_macro_instructions());
614 DCHECK(!rd.IsZero());
615 fcvtzu(rd, fn);
616 }
617
Fdiv(const VRegister & fd,const VRegister & fn,const VRegister & fm)618 void TurboAssembler::Fdiv(const VRegister& fd, const VRegister& fn,
619 const VRegister& fm) {
620 DCHECK(allow_macro_instructions());
621 fdiv(fd, fn, fm);
622 }
623
Fmadd(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)624 void MacroAssembler::Fmadd(const VRegister& fd, const VRegister& fn,
625 const VRegister& fm, const VRegister& fa) {
626 DCHECK(allow_macro_instructions());
627 fmadd(fd, fn, fm, fa);
628 }
629
Fmax(const VRegister & fd,const VRegister & fn,const VRegister & fm)630 void TurboAssembler::Fmax(const VRegister& fd, const VRegister& fn,
631 const VRegister& fm) {
632 DCHECK(allow_macro_instructions());
633 fmax(fd, fn, fm);
634 }
635
Fmaxnm(const VRegister & fd,const VRegister & fn,const VRegister & fm)636 void MacroAssembler::Fmaxnm(const VRegister& fd, const VRegister& fn,
637 const VRegister& fm) {
638 DCHECK(allow_macro_instructions());
639 fmaxnm(fd, fn, fm);
640 }
641
Fmin(const VRegister & fd,const VRegister & fn,const VRegister & fm)642 void TurboAssembler::Fmin(const VRegister& fd, const VRegister& fn,
643 const VRegister& fm) {
644 DCHECK(allow_macro_instructions());
645 fmin(fd, fn, fm);
646 }
647
Fminnm(const VRegister & fd,const VRegister & fn,const VRegister & fm)648 void MacroAssembler::Fminnm(const VRegister& fd, const VRegister& fn,
649 const VRegister& fm) {
650 DCHECK(allow_macro_instructions());
651 fminnm(fd, fn, fm);
652 }
653
Fmov(VRegister fd,VRegister fn)654 void TurboAssembler::Fmov(VRegister fd, VRegister fn) {
655 DCHECK(allow_macro_instructions());
656 // Only emit an instruction if fd and fn are different, and they are both D
657 // registers. fmov(s0, s0) is not a no-op because it clears the top word of
658 // d0. Technically, fmov(d0, d0) is not a no-op either because it clears the
659 // top of q0, but VRegister does not currently support Q registers.
660 if (fd != fn || !fd.Is64Bits()) {
661 fmov(fd, fn);
662 }
663 }
664
Fmov(VRegister fd,Register rn)665 void TurboAssembler::Fmov(VRegister fd, Register rn) {
666 DCHECK(allow_macro_instructions());
667 fmov(fd, rn);
668 }
669
Fmov(VRegister vd,double imm)670 void TurboAssembler::Fmov(VRegister vd, double imm) {
671 DCHECK(allow_macro_instructions());
672
673 if (vd.Is1S() || vd.Is2S() || vd.Is4S()) {
674 Fmov(vd, static_cast<float>(imm));
675 return;
676 }
677
678 DCHECK(vd.Is1D() || vd.Is2D());
679 if (IsImmFP64(imm)) {
680 fmov(vd, imm);
681 } else {
682 uint64_t bits = bit_cast<uint64_t>(imm);
683 if (vd.IsScalar()) {
684 if (bits == 0) {
685 fmov(vd, xzr);
686 } else {
687 UseScratchRegisterScope temps(this);
688 Register tmp = temps.AcquireX();
689 Mov(tmp, bits);
690 fmov(vd, tmp);
691 }
692 } else {
693 Movi(vd, bits);
694 }
695 }
696 }
697
Fmov(VRegister vd,float imm)698 void TurboAssembler::Fmov(VRegister vd, float imm) {
699 DCHECK(allow_macro_instructions());
700 if (vd.Is1D() || vd.Is2D()) {
701 Fmov(vd, static_cast<double>(imm));
702 return;
703 }
704
705 DCHECK(vd.Is1S() || vd.Is2S() || vd.Is4S());
706 if (IsImmFP32(imm)) {
707 fmov(vd, imm);
708 } else {
709 uint32_t bits = bit_cast<uint32_t>(imm);
710 if (vd.IsScalar()) {
711 if (bits == 0) {
712 fmov(vd, wzr);
713 } else {
714 UseScratchRegisterScope temps(this);
715 Register tmp = temps.AcquireW();
716 Mov(tmp, bit_cast<uint32_t>(imm));
717 Fmov(vd, tmp);
718 }
719 } else {
720 Movi(vd, bits);
721 }
722 }
723 }
724
Fmov(Register rd,VRegister fn)725 void TurboAssembler::Fmov(Register rd, VRegister fn) {
726 DCHECK(allow_macro_instructions());
727 DCHECK(!rd.IsZero());
728 fmov(rd, fn);
729 }
730
Fmsub(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)731 void MacroAssembler::Fmsub(const VRegister& fd, const VRegister& fn,
732 const VRegister& fm, const VRegister& fa) {
733 DCHECK(allow_macro_instructions());
734 fmsub(fd, fn, fm, fa);
735 }
736
Fmul(const VRegister & fd,const VRegister & fn,const VRegister & fm)737 void TurboAssembler::Fmul(const VRegister& fd, const VRegister& fn,
738 const VRegister& fm) {
739 DCHECK(allow_macro_instructions());
740 fmul(fd, fn, fm);
741 }
742
Fnmadd(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)743 void MacroAssembler::Fnmadd(const VRegister& fd, const VRegister& fn,
744 const VRegister& fm, const VRegister& fa) {
745 DCHECK(allow_macro_instructions());
746 fnmadd(fd, fn, fm, fa);
747 }
748
Fnmsub(const VRegister & fd,const VRegister & fn,const VRegister & fm,const VRegister & fa)749 void MacroAssembler::Fnmsub(const VRegister& fd, const VRegister& fn,
750 const VRegister& fm, const VRegister& fa) {
751 DCHECK(allow_macro_instructions());
752 fnmsub(fd, fn, fm, fa);
753 }
754
Fsub(const VRegister & fd,const VRegister & fn,const VRegister & fm)755 void TurboAssembler::Fsub(const VRegister& fd, const VRegister& fn,
756 const VRegister& fm) {
757 DCHECK(allow_macro_instructions());
758 fsub(fd, fn, fm);
759 }
760
Hint(SystemHint code)761 void MacroAssembler::Hint(SystemHint code) {
762 DCHECK(allow_macro_instructions());
763 hint(code);
764 }
765
Hlt(int code)766 void MacroAssembler::Hlt(int code) {
767 DCHECK(allow_macro_instructions());
768 hlt(code);
769 }
770
Isb()771 void TurboAssembler::Isb() {
772 DCHECK(allow_macro_instructions());
773 isb();
774 }
775
Ldr(const CPURegister & rt,const Operand & operand)776 void TurboAssembler::Ldr(const CPURegister& rt, const Operand& operand) {
777 DCHECK(allow_macro_instructions());
778 ldr(rt, operand);
779 }
780
Lsl(const Register & rd,const Register & rn,unsigned shift)781 void TurboAssembler::Lsl(const Register& rd, const Register& rn,
782 unsigned shift) {
783 DCHECK(allow_macro_instructions());
784 DCHECK(!rd.IsZero());
785 lsl(rd, rn, shift);
786 }
787
Lsl(const Register & rd,const Register & rn,const Register & rm)788 void TurboAssembler::Lsl(const Register& rd, const Register& rn,
789 const Register& rm) {
790 DCHECK(allow_macro_instructions());
791 DCHECK(!rd.IsZero());
792 lslv(rd, rn, rm);
793 }
794
Lsr(const Register & rd,const Register & rn,unsigned shift)795 void TurboAssembler::Lsr(const Register& rd, const Register& rn,
796 unsigned shift) {
797 DCHECK(allow_macro_instructions());
798 DCHECK(!rd.IsZero());
799 lsr(rd, rn, shift);
800 }
801
Lsr(const Register & rd,const Register & rn,const Register & rm)802 void TurboAssembler::Lsr(const Register& rd, const Register& rn,
803 const Register& rm) {
804 DCHECK(allow_macro_instructions());
805 DCHECK(!rd.IsZero());
806 lsrv(rd, rn, rm);
807 }
808
Madd(const Register & rd,const Register & rn,const Register & rm,const Register & ra)809 void TurboAssembler::Madd(const Register& rd, const Register& rn,
810 const Register& rm, const Register& ra) {
811 DCHECK(allow_macro_instructions());
812 DCHECK(!rd.IsZero());
813 madd(rd, rn, rm, ra);
814 }
815
Mneg(const Register & rd,const Register & rn,const Register & rm)816 void TurboAssembler::Mneg(const Register& rd, const Register& rn,
817 const Register& rm) {
818 DCHECK(allow_macro_instructions());
819 DCHECK(!rd.IsZero());
820 mneg(rd, rn, rm);
821 }
822
Movk(const Register & rd,uint64_t imm,int shift)823 void MacroAssembler::Movk(const Register& rd, uint64_t imm, int shift) {
824 DCHECK(allow_macro_instructions());
825 DCHECK(!rd.IsZero());
826 movk(rd, imm, shift);
827 }
828
Mrs(const Register & rt,SystemRegister sysreg)829 void TurboAssembler::Mrs(const Register& rt, SystemRegister sysreg) {
830 DCHECK(allow_macro_instructions());
831 DCHECK(!rt.IsZero());
832 mrs(rt, sysreg);
833 }
834
Msr(SystemRegister sysreg,const Register & rt)835 void TurboAssembler::Msr(SystemRegister sysreg, const Register& rt) {
836 DCHECK(allow_macro_instructions());
837 msr(sysreg, rt);
838 }
839
Msub(const Register & rd,const Register & rn,const Register & rm,const Register & ra)840 void TurboAssembler::Msub(const Register& rd, const Register& rn,
841 const Register& rm, const Register& ra) {
842 DCHECK(allow_macro_instructions());
843 DCHECK(!rd.IsZero());
844 msub(rd, rn, rm, ra);
845 }
846
Mul(const Register & rd,const Register & rn,const Register & rm)847 void TurboAssembler::Mul(const Register& rd, const Register& rn,
848 const Register& rm) {
849 DCHECK(allow_macro_instructions());
850 DCHECK(!rd.IsZero());
851 mul(rd, rn, rm);
852 }
853
Rbit(const Register & rd,const Register & rn)854 void TurboAssembler::Rbit(const Register& rd, const Register& rn) {
855 DCHECK(allow_macro_instructions());
856 DCHECK(!rd.IsZero());
857 rbit(rd, rn);
858 }
859
Rev(const Register & rd,const Register & rn)860 void TurboAssembler::Rev(const Register& rd, const Register& rn) {
861 DCHECK(allow_macro_instructions());
862 DCHECK(!rd.IsZero());
863 rev(rd, rn);
864 }
865
Ret(const Register & xn)866 void TurboAssembler::Ret(const Register& xn) {
867 DCHECK(allow_macro_instructions());
868 DCHECK(!xn.IsZero());
869 ret(xn);
870 CheckVeneerPool(false, false);
871 }
872
Rev(const Register & rd,const Register & rn)873 void MacroAssembler::Rev(const Register& rd, const Register& rn) {
874 DCHECK(allow_macro_instructions());
875 DCHECK(!rd.IsZero());
876 rev(rd, rn);
877 }
878
Rev16(const Register & rd,const Register & rn)879 void TurboAssembler::Rev16(const Register& rd, const Register& rn) {
880 DCHECK(allow_macro_instructions());
881 DCHECK(!rd.IsZero());
882 rev16(rd, rn);
883 }
884
Rev32(const Register & rd,const Register & rn)885 void TurboAssembler::Rev32(const Register& rd, const Register& rn) {
886 DCHECK(allow_macro_instructions());
887 DCHECK(!rd.IsZero());
888 rev32(rd, rn);
889 }
890
Ror(const Register & rd,const Register & rs,unsigned shift)891 void TurboAssembler::Ror(const Register& rd, const Register& rs,
892 unsigned shift) {
893 DCHECK(allow_macro_instructions());
894 DCHECK(!rd.IsZero());
895 ror(rd, rs, shift);
896 }
897
Ror(const Register & rd,const Register & rn,const Register & rm)898 void TurboAssembler::Ror(const Register& rd, const Register& rn,
899 const Register& rm) {
900 DCHECK(allow_macro_instructions());
901 DCHECK(!rd.IsZero());
902 rorv(rd, rn, rm);
903 }
904
Sbfiz(const Register & rd,const Register & rn,unsigned lsb,unsigned width)905 void MacroAssembler::Sbfiz(const Register& rd, const Register& rn, unsigned lsb,
906 unsigned width) {
907 DCHECK(allow_macro_instructions());
908 DCHECK(!rd.IsZero());
909 sbfiz(rd, rn, lsb, width);
910 }
911
Sbfx(const Register & rd,const Register & rn,unsigned lsb,unsigned width)912 void TurboAssembler::Sbfx(const Register& rd, const Register& rn, unsigned lsb,
913 unsigned width) {
914 DCHECK(allow_macro_instructions());
915 DCHECK(!rd.IsZero());
916 sbfx(rd, rn, lsb, width);
917 }
918
Scvtf(const VRegister & fd,const Register & rn,unsigned fbits)919 void TurboAssembler::Scvtf(const VRegister& fd, const Register& rn,
920 unsigned fbits) {
921 DCHECK(allow_macro_instructions());
922 scvtf(fd, rn, fbits);
923 }
924
Sdiv(const Register & rd,const Register & rn,const Register & rm)925 void TurboAssembler::Sdiv(const Register& rd, const Register& rn,
926 const Register& rm) {
927 DCHECK(allow_macro_instructions());
928 DCHECK(!rd.IsZero());
929 sdiv(rd, rn, rm);
930 }
931
Smaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)932 void MacroAssembler::Smaddl(const Register& rd, const Register& rn,
933 const Register& rm, const Register& ra) {
934 DCHECK(allow_macro_instructions());
935 DCHECK(!rd.IsZero());
936 smaddl(rd, rn, rm, ra);
937 }
938
Smsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)939 void MacroAssembler::Smsubl(const Register& rd, const Register& rn,
940 const Register& rm, const Register& ra) {
941 DCHECK(allow_macro_instructions());
942 DCHECK(!rd.IsZero());
943 smsubl(rd, rn, rm, ra);
944 }
945
Smull(const Register & rd,const Register & rn,const Register & rm)946 void TurboAssembler::Smull(const Register& rd, const Register& rn,
947 const Register& rm) {
948 DCHECK(allow_macro_instructions());
949 DCHECK(!rd.IsZero());
950 smull(rd, rn, rm);
951 }
952
Smulh(const Register & rd,const Register & rn,const Register & rm)953 void MacroAssembler::Smulh(const Register& rd, const Register& rn,
954 const Register& rm) {
955 DCHECK(allow_macro_instructions());
956 DCHECK(!rd.IsZero());
957 smulh(rd, rn, rm);
958 }
959
Umull(const Register & rd,const Register & rn,const Register & rm)960 void TurboAssembler::Umull(const Register& rd, const Register& rn,
961 const Register& rm) {
962 DCHECK(allow_macro_instructions());
963 DCHECK(!rd.IsZero());
964 umaddl(rd, rn, rm, xzr);
965 }
966
Sxtb(const Register & rd,const Register & rn)967 void TurboAssembler::Sxtb(const Register& rd, const Register& rn) {
968 DCHECK(allow_macro_instructions());
969 DCHECK(!rd.IsZero());
970 sxtb(rd, rn);
971 }
972
Sxth(const Register & rd,const Register & rn)973 void TurboAssembler::Sxth(const Register& rd, const Register& rn) {
974 DCHECK(allow_macro_instructions());
975 DCHECK(!rd.IsZero());
976 sxth(rd, rn);
977 }
978
Sxtw(const Register & rd,const Register & rn)979 void TurboAssembler::Sxtw(const Register& rd, const Register& rn) {
980 DCHECK(allow_macro_instructions());
981 DCHECK(!rd.IsZero());
982 sxtw(rd, rn);
983 }
984
Ubfiz(const Register & rd,const Register & rn,unsigned lsb,unsigned width)985 void TurboAssembler::Ubfiz(const Register& rd, const Register& rn, unsigned lsb,
986 unsigned width) {
987 DCHECK(allow_macro_instructions());
988 DCHECK(!rd.IsZero());
989 ubfiz(rd, rn, lsb, width);
990 }
991
Ubfx(const Register & rd,const Register & rn,unsigned lsb,unsigned width)992 void TurboAssembler::Ubfx(const Register& rd, const Register& rn, unsigned lsb,
993 unsigned width) {
994 DCHECK(allow_macro_instructions());
995 DCHECK(!rd.IsZero());
996 ubfx(rd, rn, lsb, width);
997 }
998
Ucvtf(const VRegister & fd,const Register & rn,unsigned fbits)999 void TurboAssembler::Ucvtf(const VRegister& fd, const Register& rn,
1000 unsigned fbits) {
1001 DCHECK(allow_macro_instructions());
1002 ucvtf(fd, rn, fbits);
1003 }
1004
Udiv(const Register & rd,const Register & rn,const Register & rm)1005 void TurboAssembler::Udiv(const Register& rd, const Register& rn,
1006 const Register& rm) {
1007 DCHECK(allow_macro_instructions());
1008 DCHECK(!rd.IsZero());
1009 udiv(rd, rn, rm);
1010 }
1011
Umaddl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1012 void MacroAssembler::Umaddl(const Register& rd, const Register& rn,
1013 const Register& rm, const Register& ra) {
1014 DCHECK(allow_macro_instructions());
1015 DCHECK(!rd.IsZero());
1016 umaddl(rd, rn, rm, ra);
1017 }
1018
Umsubl(const Register & rd,const Register & rn,const Register & rm,const Register & ra)1019 void MacroAssembler::Umsubl(const Register& rd, const Register& rn,
1020 const Register& rm, const Register& ra) {
1021 DCHECK(allow_macro_instructions());
1022 DCHECK(!rd.IsZero());
1023 umsubl(rd, rn, rm, ra);
1024 }
1025
Uxtb(const Register & rd,const Register & rn)1026 void TurboAssembler::Uxtb(const Register& rd, const Register& rn) {
1027 DCHECK(allow_macro_instructions());
1028 DCHECK(!rd.IsZero());
1029 uxtb(rd, rn);
1030 }
1031
Uxth(const Register & rd,const Register & rn)1032 void TurboAssembler::Uxth(const Register& rd, const Register& rn) {
1033 DCHECK(allow_macro_instructions());
1034 DCHECK(!rd.IsZero());
1035 uxth(rd, rn);
1036 }
1037
Uxtw(const Register & rd,const Register & rn)1038 void TurboAssembler::Uxtw(const Register& rd, const Register& rn) {
1039 DCHECK(allow_macro_instructions());
1040 DCHECK(!rd.IsZero());
1041 uxtw(rd, rn);
1042 }
1043
InitializeRootRegister()1044 void TurboAssembler::InitializeRootRegister() {
1045 ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
1046 Mov(kRootRegister, Operand(isolate_root));
1047 }
1048
SmiTag(Register dst,Register src)1049 void MacroAssembler::SmiTag(Register dst, Register src) {
1050 DCHECK(dst.Is64Bits() && src.Is64Bits());
1051 DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
1052 Lsl(dst, src, kSmiShift);
1053 }
1054
SmiTag(Register smi)1055 void MacroAssembler::SmiTag(Register smi) { SmiTag(smi, smi); }
1056
SmiUntag(Register dst,Register src)1057 void TurboAssembler::SmiUntag(Register dst, Register src) {
1058 DCHECK(dst.Is64Bits() && src.Is64Bits());
1059 if (FLAG_enable_slow_asserts) {
1060 AssertSmi(src);
1061 }
1062 DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
1063 if (COMPRESS_POINTERS_BOOL) {
1064 Asr(dst.W(), src.W(), kSmiShift);
1065 Sxtw(dst, dst);
1066 } else {
1067 Asr(dst, src, kSmiShift);
1068 }
1069 }
1070
SmiUntag(Register dst,const MemOperand & src)1071 void TurboAssembler::SmiUntag(Register dst, const MemOperand& src) {
1072 DCHECK(dst.Is64Bits());
1073 if (SmiValuesAre32Bits()) {
1074 if (src.IsImmediateOffset() && src.shift_amount() == 0) {
1075 // Load value directly from the upper half-word.
1076 // Assumes that Smis are shifted by 32 bits and little endianness.
1077 DCHECK_EQ(kSmiShift, 32);
1078 Ldrsw(dst,
1079 MemOperand(src.base(), src.offset() + (kSmiShift / kBitsPerByte),
1080 src.addrmode()));
1081
1082 } else {
1083 Ldr(dst, src);
1084 SmiUntag(dst);
1085 }
1086 } else {
1087 DCHECK(SmiValuesAre31Bits());
1088 if (COMPRESS_POINTERS_BOOL) {
1089 Ldr(dst.W(), src);
1090 } else {
1091 Ldr(dst, src);
1092 }
1093 SmiUntag(dst);
1094 }
1095 }
1096
SmiUntag(Register smi)1097 void TurboAssembler::SmiUntag(Register smi) { SmiUntag(smi, smi); }
1098
JumpIfSmi(Register value,Label * smi_label,Label * not_smi_label)1099 void TurboAssembler::JumpIfSmi(Register value, Label* smi_label,
1100 Label* not_smi_label) {
1101 STATIC_ASSERT((kSmiTagSize == 1) && (kSmiTag == 0));
1102 // Check if the tag bit is set.
1103 if (smi_label) {
1104 Tbz(value, 0, smi_label);
1105 if (not_smi_label) {
1106 B(not_smi_label);
1107 }
1108 } else {
1109 DCHECK(not_smi_label);
1110 Tbnz(value, 0, not_smi_label);
1111 }
1112 }
1113
JumpIfEqual(Register x,int32_t y,Label * dest)1114 void TurboAssembler::JumpIfEqual(Register x, int32_t y, Label* dest) {
1115 CompareAndBranch(x, y, eq, dest);
1116 }
1117
JumpIfLessThan(Register x,int32_t y,Label * dest)1118 void TurboAssembler::JumpIfLessThan(Register x, int32_t y, Label* dest) {
1119 CompareAndBranch(x, y, lt, dest);
1120 }
1121
JumpIfNotSmi(Register value,Label * not_smi_label)1122 void MacroAssembler::JumpIfNotSmi(Register value, Label* not_smi_label) {
1123 JumpIfSmi(value, nullptr, not_smi_label);
1124 }
1125
jmp(Label * L)1126 void TurboAssembler::jmp(Label* L) { B(L); }
1127
1128 template <TurboAssembler::StoreLRMode lr_mode>
Push(const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3)1129 void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1130 const CPURegister& src2, const CPURegister& src3) {
1131 DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
1132 DCHECK_IMPLIES((lr_mode == kSignLR), ((src0 == lr) || (src1 == lr) ||
1133 (src2 == lr) || (src3 == lr)));
1134 DCHECK_IMPLIES((lr_mode == kDontStoreLR), ((src0 != lr) && (src1 != lr) &&
1135 (src2 != lr) && (src3 != lr)));
1136
1137 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1138 if (lr_mode == kSignLR) {
1139 Paciasp();
1140 }
1141 #endif
1142
1143 int count = 1 + src1.is_valid() + src2.is_valid() + src3.is_valid();
1144 int size = src0.SizeInBytes();
1145 DCHECK_EQ(0, (size * count) % 16);
1146
1147 PushHelper(count, size, src0, src1, src2, src3);
1148 }
1149
1150 template <TurboAssembler::StoreLRMode lr_mode>
Push(const Register & src0,const VRegister & src1)1151 void TurboAssembler::Push(const Register& src0, const VRegister& src1) {
1152 DCHECK_IMPLIES((lr_mode == kSignLR), ((src0 == lr) || (src1 == lr)));
1153 DCHECK_IMPLIES((lr_mode == kDontStoreLR), ((src0 != lr) && (src1 != lr)));
1154 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1155 if (lr_mode == kSignLR) {
1156 Paciasp();
1157 }
1158 #endif
1159
1160 int size = src0.SizeInBytes() + src1.SizeInBytes();
1161 DCHECK_EQ(0, size % 16);
1162
1163 // Reserve room for src0 and push src1.
1164 str(src1, MemOperand(sp, -size, PreIndex));
1165 // Fill the gap with src0.
1166 str(src0, MemOperand(sp, src1.SizeInBytes()));
1167 }
1168
1169 template <TurboAssembler::LoadLRMode lr_mode>
Pop(const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3)1170 void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1171 const CPURegister& dst2, const CPURegister& dst3) {
1172 // It is not valid to pop into the same register more than once in one
1173 // instruction, not even into the zero register.
1174 DCHECK(!AreAliased(dst0, dst1, dst2, dst3));
1175 DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1176 DCHECK(dst0.is_valid());
1177
1178 int count = 1 + dst1.is_valid() + dst2.is_valid() + dst3.is_valid();
1179 int size = dst0.SizeInBytes();
1180 DCHECK_EQ(0, (size * count) % 16);
1181
1182 PopHelper(count, size, dst0, dst1, dst2, dst3);
1183
1184 DCHECK_IMPLIES((lr_mode == kAuthLR), ((dst0 == lr) || (dst1 == lr) ||
1185 (dst2 == lr) || (dst3 == lr)));
1186 DCHECK_IMPLIES((lr_mode == kDontLoadLR), ((dst0 != lr) && (dst1 != lr)) &&
1187 (dst2 != lr) && (dst3 != lr));
1188
1189 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1190 if (lr_mode == kAuthLR) {
1191 Autiasp();
1192 }
1193 #endif
1194 }
1195
1196 template <TurboAssembler::StoreLRMode lr_mode>
Poke(const CPURegister & src,const Operand & offset)1197 void TurboAssembler::Poke(const CPURegister& src, const Operand& offset) {
1198 DCHECK_IMPLIES((lr_mode == kSignLR), (src == lr));
1199 DCHECK_IMPLIES((lr_mode == kDontStoreLR), (src != lr));
1200 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1201 if (lr_mode == kSignLR) {
1202 Paciasp();
1203 }
1204 #endif
1205
1206 if (offset.IsImmediate()) {
1207 DCHECK_GE(offset.ImmediateValue(), 0);
1208 } else if (emit_debug_code()) {
1209 Cmp(xzr, offset);
1210 Check(le, AbortReason::kStackAccessBelowStackPointer);
1211 }
1212
1213 Str(src, MemOperand(sp, offset));
1214 }
1215
1216 template <TurboAssembler::LoadLRMode lr_mode>
Peek(const CPURegister & dst,const Operand & offset)1217 void TurboAssembler::Peek(const CPURegister& dst, const Operand& offset) {
1218 if (offset.IsImmediate()) {
1219 DCHECK_GE(offset.ImmediateValue(), 0);
1220 } else if (emit_debug_code()) {
1221 Cmp(xzr, offset);
1222 Check(le, AbortReason::kStackAccessBelowStackPointer);
1223 }
1224
1225 Ldr(dst, MemOperand(sp, offset));
1226
1227 DCHECK_IMPLIES((lr_mode == kAuthLR), (dst == lr));
1228 DCHECK_IMPLIES((lr_mode == kDontLoadLR), (dst != lr));
1229 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1230 if (lr_mode == kAuthLR) {
1231 Autiasp();
1232 }
1233 #endif
1234 }
1235
1236 template <TurboAssembler::StoreLRMode lr_mode>
PushCPURegList(CPURegList registers)1237 void TurboAssembler::PushCPURegList(CPURegList registers) {
1238 DCHECK_IMPLIES((lr_mode == kDontStoreLR), !registers.IncludesAliasOf(lr));
1239 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1240 if (lr_mode == kSignLR && registers.IncludesAliasOf(lr)) {
1241 Paciasp();
1242 }
1243 #endif
1244
1245 int size = registers.RegisterSizeInBytes();
1246 DCHECK_EQ(0, (size * registers.Count()) % 16);
1247
1248 // Push up to four registers at a time.
1249 while (!registers.IsEmpty()) {
1250 int count_before = registers.Count();
1251 const CPURegister& src0 = registers.PopHighestIndex();
1252 const CPURegister& src1 = registers.PopHighestIndex();
1253 const CPURegister& src2 = registers.PopHighestIndex();
1254 const CPURegister& src3 = registers.PopHighestIndex();
1255 int count = count_before - registers.Count();
1256 PushHelper(count, size, src0, src1, src2, src3);
1257 }
1258 }
1259
1260 template <TurboAssembler::LoadLRMode lr_mode>
PopCPURegList(CPURegList registers)1261 void TurboAssembler::PopCPURegList(CPURegList registers) {
1262 int size = registers.RegisterSizeInBytes();
1263 DCHECK_EQ(0, (size * registers.Count()) % 16);
1264
1265 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1266 bool contains_lr = registers.IncludesAliasOf(lr);
1267 DCHECK_IMPLIES((lr_mode == kDontLoadLR), !contains_lr);
1268 #endif
1269
1270 // Pop up to four registers at a time.
1271 while (!registers.IsEmpty()) {
1272 int count_before = registers.Count();
1273 const CPURegister& dst0 = registers.PopLowestIndex();
1274 const CPURegister& dst1 = registers.PopLowestIndex();
1275 const CPURegister& dst2 = registers.PopLowestIndex();
1276 const CPURegister& dst3 = registers.PopLowestIndex();
1277 int count = count_before - registers.Count();
1278 PopHelper(count, size, dst0, dst1, dst2, dst3);
1279 }
1280
1281 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1282 if (lr_mode == kAuthLR && contains_lr) {
1283 Autiasp();
1284 }
1285 #endif
1286 }
1287
Push(Handle<HeapObject> handle)1288 void TurboAssembler::Push(Handle<HeapObject> handle) {
1289 UseScratchRegisterScope temps(this);
1290 Register tmp = temps.AcquireX();
1291 Mov(tmp, Operand(handle));
1292 // This is only used in test-heap.cc, for generating code that is not
1293 // executed. Push a padding slot together with the handle here, to
1294 // satisfy the alignment requirement.
1295 Push(padreg, tmp);
1296 }
1297
Push(Smi smi)1298 void TurboAssembler::Push(Smi smi) {
1299 UseScratchRegisterScope temps(this);
1300 Register tmp = temps.AcquireX();
1301 Mov(tmp, Operand(smi));
1302 Push(tmp);
1303 }
1304
Claim(int64_t count,uint64_t unit_size)1305 void TurboAssembler::Claim(int64_t count, uint64_t unit_size) {
1306 DCHECK_GE(count, 0);
1307 uint64_t size = count * unit_size;
1308
1309 if (size == 0) {
1310 return;
1311 }
1312 DCHECK_EQ(size % 16, 0);
1313 #if V8_OS_WIN
1314 while (size > kStackPageSize) {
1315 Sub(sp, sp, kStackPageSize);
1316 Str(xzr, MemOperand(sp));
1317 size -= kStackPageSize;
1318 }
1319 #endif
1320 Sub(sp, sp, size);
1321 }
1322
Claim(const Register & count,uint64_t unit_size)1323 void TurboAssembler::Claim(const Register& count, uint64_t unit_size) {
1324 if (unit_size == 0) return;
1325 DCHECK(base::bits::IsPowerOfTwo(unit_size));
1326
1327 const int shift = base::bits::CountTrailingZeros(unit_size);
1328 const Operand size(count, LSL, shift);
1329
1330 if (size.IsZero()) {
1331 return;
1332 }
1333 AssertPositiveOrZero(count);
1334
1335 #if V8_OS_WIN
1336 // "Functions that allocate 4k or more worth of stack must ensure that each
1337 // page prior to the final page is touched in order." Source:
1338 // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=vs-2019#stack
1339
1340 // Callers expect count register to not be clobbered, so copy it.
1341 UseScratchRegisterScope temps(this);
1342 Register bytes_scratch = temps.AcquireX();
1343 Mov(bytes_scratch, size);
1344
1345 Label check_offset;
1346 Label touch_next_page;
1347 B(&check_offset);
1348 Bind(&touch_next_page);
1349 Sub(sp, sp, kStackPageSize);
1350 // Just to touch the page, before we increment further.
1351 Str(xzr, MemOperand(sp));
1352 Sub(bytes_scratch, bytes_scratch, kStackPageSize);
1353
1354 Bind(&check_offset);
1355 Cmp(bytes_scratch, kStackPageSize);
1356 B(gt, &touch_next_page);
1357
1358 Sub(sp, sp, bytes_scratch);
1359 #else
1360 Sub(sp, sp, size);
1361 #endif
1362 }
1363
Drop(int64_t count,uint64_t unit_size)1364 void TurboAssembler::Drop(int64_t count, uint64_t unit_size) {
1365 DCHECK_GE(count, 0);
1366 uint64_t size = count * unit_size;
1367
1368 if (size == 0) {
1369 return;
1370 }
1371
1372 Add(sp, sp, size);
1373 DCHECK_EQ(size % 16, 0);
1374 }
1375
Drop(const Register & count,uint64_t unit_size)1376 void TurboAssembler::Drop(const Register& count, uint64_t unit_size) {
1377 if (unit_size == 0) return;
1378 DCHECK(base::bits::IsPowerOfTwo(unit_size));
1379
1380 const int shift = base::bits::CountTrailingZeros(unit_size);
1381 const Operand size(count, LSL, shift);
1382
1383 if (size.IsZero()) {
1384 return;
1385 }
1386
1387 AssertPositiveOrZero(count);
1388 Add(sp, sp, size);
1389 }
1390
DropArguments(const Register & count,ArgumentsCountMode mode)1391 void TurboAssembler::DropArguments(const Register& count,
1392 ArgumentsCountMode mode) {
1393 int extra_slots = 1; // Padding slot.
1394 if (mode == kCountExcludesReceiver) {
1395 // Add a slot for the receiver.
1396 ++extra_slots;
1397 }
1398 UseScratchRegisterScope temps(this);
1399 Register tmp = temps.AcquireX();
1400 Add(tmp, count, extra_slots);
1401 Bic(tmp, tmp, 1);
1402 Drop(tmp, kXRegSize);
1403 }
1404
DropArguments(int64_t count,ArgumentsCountMode mode)1405 void TurboAssembler::DropArguments(int64_t count, ArgumentsCountMode mode) {
1406 if (mode == kCountExcludesReceiver) {
1407 // Add a slot for the receiver.
1408 ++count;
1409 }
1410 Drop(RoundUp(count, 2), kXRegSize);
1411 }
1412
DropSlots(int64_t count)1413 void TurboAssembler::DropSlots(int64_t count) {
1414 Drop(RoundUp(count, 2), kXRegSize);
1415 }
1416
PushArgument(const Register & arg)1417 void TurboAssembler::PushArgument(const Register& arg) { Push(padreg, arg); }
1418
CompareAndBranch(const Register & lhs,const Operand & rhs,Condition cond,Label * label)1419 void TurboAssembler::CompareAndBranch(const Register& lhs, const Operand& rhs,
1420 Condition cond, Label* label) {
1421 if (rhs.IsImmediate() && (rhs.ImmediateValue() == 0) &&
1422 ((cond == eq) || (cond == ne))) {
1423 if (cond == eq) {
1424 Cbz(lhs, label);
1425 } else {
1426 Cbnz(lhs, label);
1427 }
1428 } else {
1429 Cmp(lhs, rhs);
1430 B(cond, label);
1431 }
1432 }
1433
CompareTaggedAndBranch(const Register & lhs,const Operand & rhs,Condition cond,Label * label)1434 void TurboAssembler::CompareTaggedAndBranch(const Register& lhs,
1435 const Operand& rhs, Condition cond,
1436 Label* label) {
1437 if (COMPRESS_POINTERS_BOOL) {
1438 CompareAndBranch(lhs.W(), rhs.ToW(), cond, label);
1439 } else {
1440 CompareAndBranch(lhs, rhs, cond, label);
1441 }
1442 }
1443
TestAndBranchIfAnySet(const Register & reg,const uint64_t bit_pattern,Label * label)1444 void TurboAssembler::TestAndBranchIfAnySet(const Register& reg,
1445 const uint64_t bit_pattern,
1446 Label* label) {
1447 int bits = reg.SizeInBits();
1448 DCHECK_GT(CountSetBits(bit_pattern, bits), 0);
1449 if (CountSetBits(bit_pattern, bits) == 1) {
1450 Tbnz(reg, MaskToBit(bit_pattern), label);
1451 } else {
1452 Tst(reg, bit_pattern);
1453 B(ne, label);
1454 }
1455 }
1456
TestAndBranchIfAllClear(const Register & reg,const uint64_t bit_pattern,Label * label)1457 void TurboAssembler::TestAndBranchIfAllClear(const Register& reg,
1458 const uint64_t bit_pattern,
1459 Label* label) {
1460 int bits = reg.SizeInBits();
1461 DCHECK_GT(CountSetBits(bit_pattern, bits), 0);
1462 if (CountSetBits(bit_pattern, bits) == 1) {
1463 Tbz(reg, MaskToBit(bit_pattern), label);
1464 } else {
1465 Tst(reg, bit_pattern);
1466 B(eq, label);
1467 }
1468 }
1469
1470 } // namespace internal
1471 } // namespace v8
1472
1473 #endif // V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_INL_H_
1474