1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/assembler-inl.h"
6 #include "src/compiler/instruction-selector-impl.h"
7 #include "src/compiler/node-matchers.h"
8 #include "src/compiler/node-properties.h"
9
10 namespace v8 {
11 namespace internal {
12 namespace compiler {
13
14 enum ImmediateMode {
15 kArithmeticImm, // 12 bit unsigned immediate shifted left 0 or 12 bits
16 kShift32Imm, // 0 - 31
17 kShift64Imm, // 0 - 63
18 kLogical32Imm,
19 kLogical64Imm,
20 kLoadStoreImm8, // signed 8 bit or 12 bit unsigned scaled by access size
21 kLoadStoreImm16,
22 kLoadStoreImm32,
23 kLoadStoreImm64,
24 kNoImmediate
25 };
26
27
28 // Adds Arm64-specific methods for generating operands.
29 class Arm64OperandGenerator final : public OperandGenerator {
30 public:
Arm64OperandGenerator(InstructionSelector * selector)31 explicit Arm64OperandGenerator(InstructionSelector* selector)
32 : OperandGenerator(selector) {}
33
UseOperand(Node * node,ImmediateMode mode)34 InstructionOperand UseOperand(Node* node, ImmediateMode mode) {
35 if (CanBeImmediate(node, mode)) {
36 return UseImmediate(node);
37 }
38 return UseRegister(node);
39 }
40
41 // Use the zero register if the node has the immediate value zero, otherwise
42 // assign a register.
UseRegisterOrImmediateZero(Node * node)43 InstructionOperand UseRegisterOrImmediateZero(Node* node) {
44 if ((IsIntegerConstant(node) && (GetIntegerConstantValue(node) == 0)) ||
45 (IsFloatConstant(node) &&
46 (bit_cast<int64_t>(GetFloatConstantValue(node)) == 0))) {
47 return UseImmediate(node);
48 }
49 return UseRegister(node);
50 }
51
52 // Use the provided node if it has the required value, or create a
53 // TempImmediate otherwise.
UseImmediateOrTemp(Node * node,int32_t value)54 InstructionOperand UseImmediateOrTemp(Node* node, int32_t value) {
55 if (GetIntegerConstantValue(node) == value) {
56 return UseImmediate(node);
57 }
58 return TempImmediate(value);
59 }
60
IsIntegerConstant(Node * node)61 bool IsIntegerConstant(Node* node) {
62 return (node->opcode() == IrOpcode::kInt32Constant) ||
63 (node->opcode() == IrOpcode::kInt64Constant);
64 }
65
GetIntegerConstantValue(Node * node)66 int64_t GetIntegerConstantValue(Node* node) {
67 if (node->opcode() == IrOpcode::kInt32Constant) {
68 return OpParameter<int32_t>(node->op());
69 }
70 DCHECK_EQ(IrOpcode::kInt64Constant, node->opcode());
71 return OpParameter<int64_t>(node->op());
72 }
73
IsFloatConstant(Node * node)74 bool IsFloatConstant(Node* node) {
75 return (node->opcode() == IrOpcode::kFloat32Constant) ||
76 (node->opcode() == IrOpcode::kFloat64Constant);
77 }
78
GetFloatConstantValue(Node * node)79 double GetFloatConstantValue(Node* node) {
80 if (node->opcode() == IrOpcode::kFloat32Constant) {
81 return OpParameter<float>(node->op());
82 }
83 DCHECK_EQ(IrOpcode::kFloat64Constant, node->opcode());
84 return OpParameter<double>(node->op());
85 }
86
CanBeImmediate(Node * node,ImmediateMode mode)87 bool CanBeImmediate(Node* node, ImmediateMode mode) {
88 return IsIntegerConstant(node) &&
89 CanBeImmediate(GetIntegerConstantValue(node), mode);
90 }
91
CanBeImmediate(int64_t value,ImmediateMode mode)92 bool CanBeImmediate(int64_t value, ImmediateMode mode) {
93 unsigned ignored;
94 switch (mode) {
95 case kLogical32Imm:
96 // TODO(dcarney): some unencodable values can be handled by
97 // switching instructions.
98 return Assembler::IsImmLogical(static_cast<uint64_t>(value), 32,
99 &ignored, &ignored, &ignored);
100 case kLogical64Imm:
101 return Assembler::IsImmLogical(static_cast<uint64_t>(value), 64,
102 &ignored, &ignored, &ignored);
103 case kArithmeticImm:
104 return Assembler::IsImmAddSub(value);
105 case kLoadStoreImm8:
106 return IsLoadStoreImmediate(value, 0);
107 case kLoadStoreImm16:
108 return IsLoadStoreImmediate(value, 1);
109 case kLoadStoreImm32:
110 return IsLoadStoreImmediate(value, 2);
111 case kLoadStoreImm64:
112 return IsLoadStoreImmediate(value, 3);
113 case kNoImmediate:
114 return false;
115 case kShift32Imm: // Fall through.
116 case kShift64Imm:
117 // Shift operations only observe the bottom 5 or 6 bits of the value.
118 // All possible shifts can be encoded by discarding bits which have no
119 // effect.
120 return true;
121 }
122 return false;
123 }
124
CanBeLoadStoreShiftImmediate(Node * node,MachineRepresentation rep)125 bool CanBeLoadStoreShiftImmediate(Node* node, MachineRepresentation rep) {
126 // TODO(arm64): Load and Store on 128 bit Q registers is not supported yet.
127 DCHECK_GT(MachineRepresentation::kSimd128, rep);
128 return IsIntegerConstant(node) &&
129 (GetIntegerConstantValue(node) == ElementSizeLog2Of(rep));
130 }
131
132 private:
IsLoadStoreImmediate(int64_t value,unsigned size)133 bool IsLoadStoreImmediate(int64_t value, unsigned size) {
134 return Assembler::IsImmLSScaled(value, size) ||
135 Assembler::IsImmLSUnscaled(value);
136 }
137 };
138
139
140 namespace {
141
VisitRR(InstructionSelector * selector,ArchOpcode opcode,Node * node)142 void VisitRR(InstructionSelector* selector, ArchOpcode opcode, Node* node) {
143 Arm64OperandGenerator g(selector);
144 selector->Emit(opcode, g.DefineAsRegister(node),
145 g.UseRegister(node->InputAt(0)));
146 }
147
148
VisitRRR(InstructionSelector * selector,ArchOpcode opcode,Node * node)149 void VisitRRR(InstructionSelector* selector, ArchOpcode opcode, Node* node) {
150 Arm64OperandGenerator g(selector);
151 selector->Emit(opcode, g.DefineAsRegister(node),
152 g.UseRegister(node->InputAt(0)),
153 g.UseRegister(node->InputAt(1)));
154 }
155
VisitRRI(InstructionSelector * selector,ArchOpcode opcode,Node * node)156 void VisitRRI(InstructionSelector* selector, ArchOpcode opcode, Node* node) {
157 Arm64OperandGenerator g(selector);
158 int32_t imm = OpParameter<int32_t>(node->op());
159 selector->Emit(opcode, g.DefineAsRegister(node),
160 g.UseRegister(node->InputAt(0)), g.UseImmediate(imm));
161 }
162
VisitRRO(InstructionSelector * selector,ArchOpcode opcode,Node * node,ImmediateMode operand_mode)163 void VisitRRO(InstructionSelector* selector, ArchOpcode opcode, Node* node,
164 ImmediateMode operand_mode) {
165 Arm64OperandGenerator g(selector);
166 selector->Emit(opcode, g.DefineAsRegister(node),
167 g.UseRegister(node->InputAt(0)),
168 g.UseOperand(node->InputAt(1), operand_mode));
169 }
170
VisitRRIR(InstructionSelector * selector,ArchOpcode opcode,Node * node)171 void VisitRRIR(InstructionSelector* selector, ArchOpcode opcode, Node* node) {
172 Arm64OperandGenerator g(selector);
173 int32_t imm = OpParameter<int32_t>(node->op());
174 selector->Emit(opcode, g.DefineAsRegister(node),
175 g.UseRegister(node->InputAt(0)), g.UseImmediate(imm),
176 g.UseRegister(node->InputAt(1)));
177 }
178
179 struct ExtendingLoadMatcher {
ExtendingLoadMatcherv8::internal::compiler::__anon5d604d4d0111::ExtendingLoadMatcher180 ExtendingLoadMatcher(Node* node, InstructionSelector* selector)
181 : matches_(false), selector_(selector), base_(nullptr), immediate_(0) {
182 Initialize(node);
183 }
184
Matchesv8::internal::compiler::__anon5d604d4d0111::ExtendingLoadMatcher185 bool Matches() const { return matches_; }
186
basev8::internal::compiler::__anon5d604d4d0111::ExtendingLoadMatcher187 Node* base() const {
188 DCHECK(Matches());
189 return base_;
190 }
immediatev8::internal::compiler::__anon5d604d4d0111::ExtendingLoadMatcher191 int64_t immediate() const {
192 DCHECK(Matches());
193 return immediate_;
194 }
opcodev8::internal::compiler::__anon5d604d4d0111::ExtendingLoadMatcher195 ArchOpcode opcode() const {
196 DCHECK(Matches());
197 return opcode_;
198 }
199
200 private:
201 bool matches_;
202 InstructionSelector* selector_;
203 Node* base_;
204 int64_t immediate_;
205 ArchOpcode opcode_;
206
Initializev8::internal::compiler::__anon5d604d4d0111::ExtendingLoadMatcher207 void Initialize(Node* node) {
208 Int64BinopMatcher m(node);
209 // When loading a 64-bit value and shifting by 32, we should
210 // just load and sign-extend the interesting 4 bytes instead.
211 // This happens, for example, when we're loading and untagging SMIs.
212 DCHECK(m.IsWord64Sar());
213 if (m.left().IsLoad() && m.right().Is(32) &&
214 selector_->CanCover(m.node(), m.left().node())) {
215 Arm64OperandGenerator g(selector_);
216 Node* load = m.left().node();
217 Node* offset = load->InputAt(1);
218 base_ = load->InputAt(0);
219 opcode_ = kArm64Ldrsw;
220 if (g.IsIntegerConstant(offset)) {
221 immediate_ = g.GetIntegerConstantValue(offset) + 4;
222 matches_ = g.CanBeImmediate(immediate_, kLoadStoreImm32);
223 }
224 }
225 }
226 };
227
TryMatchExtendingLoad(InstructionSelector * selector,Node * node)228 bool TryMatchExtendingLoad(InstructionSelector* selector, Node* node) {
229 ExtendingLoadMatcher m(node, selector);
230 return m.Matches();
231 }
232
TryEmitExtendingLoad(InstructionSelector * selector,Node * node)233 bool TryEmitExtendingLoad(InstructionSelector* selector, Node* node) {
234 ExtendingLoadMatcher m(node, selector);
235 Arm64OperandGenerator g(selector);
236 if (m.Matches()) {
237 InstructionOperand inputs[2];
238 inputs[0] = g.UseRegister(m.base());
239 InstructionCode opcode =
240 m.opcode() | AddressingModeField::encode(kMode_MRI);
241 DCHECK(is_int32(m.immediate()));
242 inputs[1] = g.TempImmediate(static_cast<int32_t>(m.immediate()));
243 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
244 selector->Emit(opcode, arraysize(outputs), outputs, arraysize(inputs),
245 inputs);
246 return true;
247 }
248 return false;
249 }
250
TryMatchAnyShift(InstructionSelector * selector,Node * node,Node * input_node,InstructionCode * opcode,bool try_ror)251 bool TryMatchAnyShift(InstructionSelector* selector, Node* node,
252 Node* input_node, InstructionCode* opcode, bool try_ror) {
253 Arm64OperandGenerator g(selector);
254
255 if (!selector->CanCover(node, input_node)) return false;
256 if (input_node->InputCount() != 2) return false;
257 if (!g.IsIntegerConstant(input_node->InputAt(1))) return false;
258
259 switch (input_node->opcode()) {
260 case IrOpcode::kWord32Shl:
261 case IrOpcode::kWord64Shl:
262 *opcode |= AddressingModeField::encode(kMode_Operand2_R_LSL_I);
263 return true;
264 case IrOpcode::kWord32Shr:
265 case IrOpcode::kWord64Shr:
266 *opcode |= AddressingModeField::encode(kMode_Operand2_R_LSR_I);
267 return true;
268 case IrOpcode::kWord32Sar:
269 *opcode |= AddressingModeField::encode(kMode_Operand2_R_ASR_I);
270 return true;
271 case IrOpcode::kWord64Sar:
272 if (TryMatchExtendingLoad(selector, input_node)) return false;
273 *opcode |= AddressingModeField::encode(kMode_Operand2_R_ASR_I);
274 return true;
275 case IrOpcode::kWord32Ror:
276 case IrOpcode::kWord64Ror:
277 if (try_ror) {
278 *opcode |= AddressingModeField::encode(kMode_Operand2_R_ROR_I);
279 return true;
280 }
281 return false;
282 default:
283 return false;
284 }
285 }
286
287
TryMatchAnyExtend(Arm64OperandGenerator * g,InstructionSelector * selector,Node * node,Node * left_node,Node * right_node,InstructionOperand * left_op,InstructionOperand * right_op,InstructionCode * opcode)288 bool TryMatchAnyExtend(Arm64OperandGenerator* g, InstructionSelector* selector,
289 Node* node, Node* left_node, Node* right_node,
290 InstructionOperand* left_op,
291 InstructionOperand* right_op, InstructionCode* opcode) {
292 if (!selector->CanCover(node, right_node)) return false;
293
294 NodeMatcher nm(right_node);
295
296 if (nm.IsWord32And()) {
297 Int32BinopMatcher mright(right_node);
298 if (mright.right().Is(0xFF) || mright.right().Is(0xFFFF)) {
299 int32_t mask = mright.right().Value();
300 *left_op = g->UseRegister(left_node);
301 *right_op = g->UseRegister(mright.left().node());
302 *opcode |= AddressingModeField::encode(
303 (mask == 0xFF) ? kMode_Operand2_R_UXTB : kMode_Operand2_R_UXTH);
304 return true;
305 }
306 } else if (nm.IsWord32Sar()) {
307 Int32BinopMatcher mright(right_node);
308 if (selector->CanCover(mright.node(), mright.left().node()) &&
309 mright.left().IsWord32Shl()) {
310 Int32BinopMatcher mleft_of_right(mright.left().node());
311 if ((mright.right().Is(16) && mleft_of_right.right().Is(16)) ||
312 (mright.right().Is(24) && mleft_of_right.right().Is(24))) {
313 int32_t shift = mright.right().Value();
314 *left_op = g->UseRegister(left_node);
315 *right_op = g->UseRegister(mleft_of_right.left().node());
316 *opcode |= AddressingModeField::encode(
317 (shift == 24) ? kMode_Operand2_R_SXTB : kMode_Operand2_R_SXTH);
318 return true;
319 }
320 }
321 }
322 return false;
323 }
324
TryMatchLoadStoreShift(Arm64OperandGenerator * g,InstructionSelector * selector,MachineRepresentation rep,Node * node,Node * index,InstructionOperand * index_op,InstructionOperand * shift_immediate_op)325 bool TryMatchLoadStoreShift(Arm64OperandGenerator* g,
326 InstructionSelector* selector,
327 MachineRepresentation rep, Node* node, Node* index,
328 InstructionOperand* index_op,
329 InstructionOperand* shift_immediate_op) {
330 if (!selector->CanCover(node, index)) return false;
331 if (index->InputCount() != 2) return false;
332 Node* left = index->InputAt(0);
333 Node* right = index->InputAt(1);
334 switch (index->opcode()) {
335 case IrOpcode::kWord32Shl:
336 case IrOpcode::kWord64Shl:
337 if (!g->CanBeLoadStoreShiftImmediate(right, rep)) {
338 return false;
339 }
340 *index_op = g->UseRegister(left);
341 *shift_immediate_op = g->UseImmediate(right);
342 return true;
343 default:
344 return false;
345 }
346 }
347
348 // Bitfields describing binary operator properties:
349 // CanCommuteField is true if we can switch the two operands, potentially
350 // requiring commuting the flags continuation condition.
351 typedef BitField8<bool, 1, 1> CanCommuteField;
352 // MustCommuteCondField is true when we need to commute the flags continuation
353 // condition in order to switch the operands.
354 typedef BitField8<bool, 2, 1> MustCommuteCondField;
355 // IsComparisonField is true when the operation is a comparison and has no other
356 // result other than the condition.
357 typedef BitField8<bool, 3, 1> IsComparisonField;
358 // IsAddSubField is true when an instruction is encoded as ADD or SUB.
359 typedef BitField8<bool, 4, 1> IsAddSubField;
360
361 // Get properties of a binary operator.
GetBinopProperties(InstructionCode opcode)362 uint8_t GetBinopProperties(InstructionCode opcode) {
363 uint8_t result = 0;
364 switch (opcode) {
365 case kArm64Cmp32:
366 case kArm64Cmp:
367 // We can commute CMP by switching the inputs and commuting
368 // the flags continuation.
369 result = CanCommuteField::update(result, true);
370 result = MustCommuteCondField::update(result, true);
371 result = IsComparisonField::update(result, true);
372 // The CMP and CMN instructions are encoded as SUB or ADD
373 // with zero output register, and therefore support the same
374 // operand modes.
375 result = IsAddSubField::update(result, true);
376 break;
377 case kArm64Cmn32:
378 case kArm64Cmn:
379 result = CanCommuteField::update(result, true);
380 result = IsComparisonField::update(result, true);
381 result = IsAddSubField::update(result, true);
382 break;
383 case kArm64Add32:
384 case kArm64Add:
385 result = CanCommuteField::update(result, true);
386 result = IsAddSubField::update(result, true);
387 break;
388 case kArm64Sub32:
389 case kArm64Sub:
390 result = IsAddSubField::update(result, true);
391 break;
392 case kArm64Tst32:
393 case kArm64Tst:
394 result = CanCommuteField::update(result, true);
395 result = IsComparisonField::update(result, true);
396 break;
397 case kArm64And32:
398 case kArm64And:
399 case kArm64Or32:
400 case kArm64Or:
401 case kArm64Eor32:
402 case kArm64Eor:
403 result = CanCommuteField::update(result, true);
404 break;
405 default:
406 UNREACHABLE();
407 }
408 DCHECK_IMPLIES(MustCommuteCondField::decode(result),
409 CanCommuteField::decode(result));
410 return result;
411 }
412
413 // Shared routine for multiple binary operations.
414 template <typename Matcher>
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode,ImmediateMode operand_mode,FlagsContinuation * cont)415 void VisitBinop(InstructionSelector* selector, Node* node,
416 InstructionCode opcode, ImmediateMode operand_mode,
417 FlagsContinuation* cont) {
418 Arm64OperandGenerator g(selector);
419 InstructionOperand inputs[3];
420 size_t input_count = 0;
421 InstructionOperand outputs[1];
422 size_t output_count = 0;
423
424 Node* left_node = node->InputAt(0);
425 Node* right_node = node->InputAt(1);
426
427 uint8_t properties = GetBinopProperties(opcode);
428 bool can_commute = CanCommuteField::decode(properties);
429 bool must_commute_cond = MustCommuteCondField::decode(properties);
430 bool is_add_sub = IsAddSubField::decode(properties);
431
432 if (g.CanBeImmediate(right_node, operand_mode)) {
433 inputs[input_count++] = g.UseRegister(left_node);
434 inputs[input_count++] = g.UseImmediate(right_node);
435 } else if (can_commute && g.CanBeImmediate(left_node, operand_mode)) {
436 if (must_commute_cond) cont->Commute();
437 inputs[input_count++] = g.UseRegister(right_node);
438 inputs[input_count++] = g.UseImmediate(left_node);
439 } else if (is_add_sub &&
440 TryMatchAnyExtend(&g, selector, node, left_node, right_node,
441 &inputs[0], &inputs[1], &opcode)) {
442 input_count += 2;
443 } else if (is_add_sub && can_commute &&
444 TryMatchAnyExtend(&g, selector, node, right_node, left_node,
445 &inputs[0], &inputs[1], &opcode)) {
446 if (must_commute_cond) cont->Commute();
447 input_count += 2;
448 } else if (TryMatchAnyShift(selector, node, right_node, &opcode,
449 !is_add_sub)) {
450 Matcher m_shift(right_node);
451 inputs[input_count++] = g.UseRegisterOrImmediateZero(left_node);
452 inputs[input_count++] = g.UseRegister(m_shift.left().node());
453 // We only need at most the last 6 bits of the shift.
454 inputs[input_count++] =
455 g.UseImmediate(static_cast<int>(m_shift.right().Value() & 0x3F));
456 } else if (can_commute && TryMatchAnyShift(selector, node, left_node, &opcode,
457 !is_add_sub)) {
458 if (must_commute_cond) cont->Commute();
459 Matcher m_shift(left_node);
460 inputs[input_count++] = g.UseRegisterOrImmediateZero(right_node);
461 inputs[input_count++] = g.UseRegister(m_shift.left().node());
462 // We only need at most the last 6 bits of the shift.
463 inputs[input_count++] =
464 g.UseImmediate(static_cast<int>(m_shift.right().Value() & 0x3F));
465 } else {
466 inputs[input_count++] = g.UseRegisterOrImmediateZero(left_node);
467 inputs[input_count++] = g.UseRegister(right_node);
468 }
469
470 if (!IsComparisonField::decode(properties)) {
471 outputs[output_count++] = g.DefineAsRegister(node);
472 }
473
474 DCHECK_NE(0u, input_count);
475 DCHECK((output_count != 0) || IsComparisonField::decode(properties));
476 DCHECK_GE(arraysize(inputs), input_count);
477 DCHECK_GE(arraysize(outputs), output_count);
478
479 selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
480 inputs, cont);
481 }
482
483
484 // Shared routine for multiple binary operations.
485 template <typename Matcher>
VisitBinop(InstructionSelector * selector,Node * node,ArchOpcode opcode,ImmediateMode operand_mode)486 void VisitBinop(InstructionSelector* selector, Node* node, ArchOpcode opcode,
487 ImmediateMode operand_mode) {
488 FlagsContinuation cont;
489 VisitBinop<Matcher>(selector, node, opcode, operand_mode, &cont);
490 }
491
492
493 template <typename Matcher>
VisitAddSub(InstructionSelector * selector,Node * node,ArchOpcode opcode,ArchOpcode negate_opcode)494 void VisitAddSub(InstructionSelector* selector, Node* node, ArchOpcode opcode,
495 ArchOpcode negate_opcode) {
496 Arm64OperandGenerator g(selector);
497 Matcher m(node);
498 if (m.right().HasValue() && (m.right().Value() < 0) &&
499 g.CanBeImmediate(-m.right().Value(), kArithmeticImm)) {
500 selector->Emit(negate_opcode, g.DefineAsRegister(node),
501 g.UseRegister(m.left().node()),
502 g.TempImmediate(static_cast<int32_t>(-m.right().Value())));
503 } else {
504 VisitBinop<Matcher>(selector, node, opcode, kArithmeticImm);
505 }
506 }
507
508
509 // For multiplications by immediate of the form x * (2^k + 1), where k > 0,
510 // return the value of k, otherwise return zero. This is used to reduce the
511 // multiplication to addition with left shift: x + (x << k).
512 template <typename Matcher>
LeftShiftForReducedMultiply(Matcher * m)513 int32_t LeftShiftForReducedMultiply(Matcher* m) {
514 DCHECK(m->IsInt32Mul() || m->IsInt64Mul());
515 if (m->right().HasValue() && m->right().Value() >= 3) {
516 uint64_t value_minus_one = m->right().Value() - 1;
517 if (base::bits::IsPowerOfTwo(value_minus_one)) {
518 return WhichPowerOf2(value_minus_one);
519 }
520 }
521 return 0;
522 }
523
524 } // namespace
525
VisitStackSlot(Node * node)526 void InstructionSelector::VisitStackSlot(Node* node) {
527 StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
528 int slot = frame_->AllocateSpillSlot(rep.size());
529 OperandGenerator g(this);
530
531 Emit(kArchStackSlot, g.DefineAsRegister(node),
532 sequence()->AddImmediate(Constant(slot)), 0, nullptr);
533 }
534
VisitDebugAbort(Node * node)535 void InstructionSelector::VisitDebugAbort(Node* node) {
536 Arm64OperandGenerator g(this);
537 Emit(kArchDebugAbort, g.NoOutput(), g.UseFixed(node->InputAt(0), x1));
538 }
539
EmitLoad(InstructionSelector * selector,Node * node,InstructionCode opcode,ImmediateMode immediate_mode,MachineRepresentation rep,Node * output=nullptr)540 void EmitLoad(InstructionSelector* selector, Node* node, InstructionCode opcode,
541 ImmediateMode immediate_mode, MachineRepresentation rep,
542 Node* output = nullptr) {
543 Arm64OperandGenerator g(selector);
544 Node* base = node->InputAt(0);
545 Node* index = node->InputAt(1);
546 InstructionOperand inputs[3];
547 size_t input_count = 0;
548 InstructionOperand outputs[1];
549
550 // If output is not nullptr, use that as the output register. This
551 // is used when we merge a conversion into the load.
552 outputs[0] = g.DefineAsRegister(output == nullptr ? node : output);
553 inputs[0] = g.UseRegister(base);
554
555 if (g.CanBeImmediate(index, immediate_mode)) {
556 input_count = 2;
557 inputs[1] = g.UseImmediate(index);
558 opcode |= AddressingModeField::encode(kMode_MRI);
559 } else if (TryMatchLoadStoreShift(&g, selector, rep, node, index, &inputs[1],
560 &inputs[2])) {
561 input_count = 3;
562 opcode |= AddressingModeField::encode(kMode_Operand2_R_LSL_I);
563 } else {
564 input_count = 2;
565 inputs[1] = g.UseRegister(index);
566 opcode |= AddressingModeField::encode(kMode_MRR);
567 }
568
569 selector->Emit(opcode, arraysize(outputs), outputs, input_count, inputs);
570 }
571
VisitLoad(Node * node)572 void InstructionSelector::VisitLoad(Node* node) {
573 InstructionCode opcode = kArchNop;
574 ImmediateMode immediate_mode = kNoImmediate;
575 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
576 MachineRepresentation rep = load_rep.representation();
577 switch (rep) {
578 case MachineRepresentation::kFloat32:
579 opcode = kArm64LdrS;
580 immediate_mode = kLoadStoreImm32;
581 break;
582 case MachineRepresentation::kFloat64:
583 opcode = kArm64LdrD;
584 immediate_mode = kLoadStoreImm64;
585 break;
586 case MachineRepresentation::kBit: // Fall through.
587 case MachineRepresentation::kWord8:
588 opcode = load_rep.IsSigned() ? kArm64Ldrsb : kArm64Ldrb;
589 immediate_mode = kLoadStoreImm8;
590 break;
591 case MachineRepresentation::kWord16:
592 opcode = load_rep.IsSigned() ? kArm64Ldrsh : kArm64Ldrh;
593 immediate_mode = kLoadStoreImm16;
594 break;
595 case MachineRepresentation::kWord32:
596 opcode = kArm64LdrW;
597 immediate_mode = kLoadStoreImm32;
598 break;
599 case MachineRepresentation::kTaggedSigned: // Fall through.
600 case MachineRepresentation::kTaggedPointer: // Fall through.
601 case MachineRepresentation::kTagged: // Fall through.
602 case MachineRepresentation::kWord64:
603 opcode = kArm64Ldr;
604 immediate_mode = kLoadStoreImm64;
605 break;
606 case MachineRepresentation::kSimd128:
607 opcode = kArm64LdrQ;
608 immediate_mode = kNoImmediate;
609 break;
610 case MachineRepresentation::kNone:
611 UNREACHABLE();
612 return;
613 }
614 if (node->opcode() == IrOpcode::kPoisonedLoad) {
615 CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
616 opcode |= MiscField::encode(kMemoryAccessPoisoned);
617 }
618
619 EmitLoad(this, node, opcode, immediate_mode, rep);
620 }
621
VisitPoisonedLoad(Node * node)622 void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }
623
VisitProtectedLoad(Node * node)624 void InstructionSelector::VisitProtectedLoad(Node* node) {
625 // TODO(eholk)
626 UNIMPLEMENTED();
627 }
628
VisitStore(Node * node)629 void InstructionSelector::VisitStore(Node* node) {
630 Arm64OperandGenerator g(this);
631 Node* base = node->InputAt(0);
632 Node* index = node->InputAt(1);
633 Node* value = node->InputAt(2);
634
635 StoreRepresentation store_rep = StoreRepresentationOf(node->op());
636 WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
637 MachineRepresentation rep = store_rep.representation();
638
639 // TODO(arm64): I guess this could be done in a better way.
640 if (write_barrier_kind != kNoWriteBarrier) {
641 DCHECK(CanBeTaggedPointer(rep));
642 AddressingMode addressing_mode;
643 InstructionOperand inputs[3];
644 size_t input_count = 0;
645 inputs[input_count++] = g.UseUniqueRegister(base);
646 // OutOfLineRecordWrite uses the index in an arithmetic instruction, so we
647 // must check kArithmeticImm as well as kLoadStoreImm64.
648 if (g.CanBeImmediate(index, kArithmeticImm) &&
649 g.CanBeImmediate(index, kLoadStoreImm64)) {
650 inputs[input_count++] = g.UseImmediate(index);
651 addressing_mode = kMode_MRI;
652 } else {
653 inputs[input_count++] = g.UseUniqueRegister(index);
654 addressing_mode = kMode_MRR;
655 }
656 inputs[input_count++] = g.UseUniqueRegister(value);
657 RecordWriteMode record_write_mode = RecordWriteMode::kValueIsAny;
658 switch (write_barrier_kind) {
659 case kNoWriteBarrier:
660 UNREACHABLE();
661 break;
662 case kMapWriteBarrier:
663 record_write_mode = RecordWriteMode::kValueIsMap;
664 break;
665 case kPointerWriteBarrier:
666 record_write_mode = RecordWriteMode::kValueIsPointer;
667 break;
668 case kFullWriteBarrier:
669 record_write_mode = RecordWriteMode::kValueIsAny;
670 break;
671 }
672 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
673 size_t const temp_count = arraysize(temps);
674 InstructionCode code = kArchStoreWithWriteBarrier;
675 code |= AddressingModeField::encode(addressing_mode);
676 code |= MiscField::encode(static_cast<int>(record_write_mode));
677 Emit(code, 0, nullptr, input_count, inputs, temp_count, temps);
678 } else {
679 InstructionOperand inputs[4];
680 size_t input_count = 0;
681 InstructionCode opcode = kArchNop;
682 ImmediateMode immediate_mode = kNoImmediate;
683 switch (rep) {
684 case MachineRepresentation::kFloat32:
685 opcode = kArm64StrS;
686 immediate_mode = kLoadStoreImm32;
687 break;
688 case MachineRepresentation::kFloat64:
689 opcode = kArm64StrD;
690 immediate_mode = kLoadStoreImm64;
691 break;
692 case MachineRepresentation::kBit: // Fall through.
693 case MachineRepresentation::kWord8:
694 opcode = kArm64Strb;
695 immediate_mode = kLoadStoreImm8;
696 break;
697 case MachineRepresentation::kWord16:
698 opcode = kArm64Strh;
699 immediate_mode = kLoadStoreImm16;
700 break;
701 case MachineRepresentation::kWord32:
702 opcode = kArm64StrW;
703 immediate_mode = kLoadStoreImm32;
704 break;
705 case MachineRepresentation::kTaggedSigned: // Fall through.
706 case MachineRepresentation::kTaggedPointer: // Fall through.
707 case MachineRepresentation::kTagged: // Fall through.
708 case MachineRepresentation::kWord64:
709 opcode = kArm64Str;
710 immediate_mode = kLoadStoreImm64;
711 break;
712 case MachineRepresentation::kSimd128:
713 opcode = kArm64StrQ;
714 immediate_mode = kNoImmediate;
715 break;
716 case MachineRepresentation::kNone:
717 UNREACHABLE();
718 return;
719 }
720
721 inputs[0] = g.UseRegisterOrImmediateZero(value);
722 inputs[1] = g.UseRegister(base);
723
724 if (g.CanBeImmediate(index, immediate_mode)) {
725 input_count = 3;
726 inputs[2] = g.UseImmediate(index);
727 opcode |= AddressingModeField::encode(kMode_MRI);
728 } else if (TryMatchLoadStoreShift(&g, this, rep, node, index, &inputs[2],
729 &inputs[3])) {
730 input_count = 4;
731 opcode |= AddressingModeField::encode(kMode_Operand2_R_LSL_I);
732 } else {
733 input_count = 3;
734 inputs[2] = g.UseRegister(index);
735 opcode |= AddressingModeField::encode(kMode_MRR);
736 }
737
738 Emit(opcode, 0, nullptr, input_count, inputs);
739 }
740 }
741
VisitProtectedStore(Node * node)742 void InstructionSelector::VisitProtectedStore(Node* node) {
743 // TODO(eholk)
744 UNIMPLEMENTED();
745 }
746
747 // Architecture supports unaligned access, therefore VisitLoad is used instead
VisitUnalignedLoad(Node * node)748 void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
749
750 // Architecture supports unaligned access, therefore VisitStore is used instead
VisitUnalignedStore(Node * node)751 void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
752
753 template <typename Matcher>
VisitLogical(InstructionSelector * selector,Node * node,Matcher * m,ArchOpcode opcode,bool left_can_cover,bool right_can_cover,ImmediateMode imm_mode)754 static void VisitLogical(InstructionSelector* selector, Node* node, Matcher* m,
755 ArchOpcode opcode, bool left_can_cover,
756 bool right_can_cover, ImmediateMode imm_mode) {
757 Arm64OperandGenerator g(selector);
758
759 // Map instruction to equivalent operation with inverted right input.
760 ArchOpcode inv_opcode = opcode;
761 switch (opcode) {
762 case kArm64And32:
763 inv_opcode = kArm64Bic32;
764 break;
765 case kArm64And:
766 inv_opcode = kArm64Bic;
767 break;
768 case kArm64Or32:
769 inv_opcode = kArm64Orn32;
770 break;
771 case kArm64Or:
772 inv_opcode = kArm64Orn;
773 break;
774 case kArm64Eor32:
775 inv_opcode = kArm64Eon32;
776 break;
777 case kArm64Eor:
778 inv_opcode = kArm64Eon;
779 break;
780 default:
781 UNREACHABLE();
782 }
783
784 // Select Logical(y, ~x) for Logical(Xor(x, -1), y).
785 if ((m->left().IsWord32Xor() || m->left().IsWord64Xor()) && left_can_cover) {
786 Matcher mleft(m->left().node());
787 if (mleft.right().Is(-1)) {
788 // TODO(all): support shifted operand on right.
789 selector->Emit(inv_opcode, g.DefineAsRegister(node),
790 g.UseRegister(m->right().node()),
791 g.UseRegister(mleft.left().node()));
792 return;
793 }
794 }
795
796 // Select Logical(x, ~y) for Logical(x, Xor(y, -1)).
797 if ((m->right().IsWord32Xor() || m->right().IsWord64Xor()) &&
798 right_can_cover) {
799 Matcher mright(m->right().node());
800 if (mright.right().Is(-1)) {
801 // TODO(all): support shifted operand on right.
802 selector->Emit(inv_opcode, g.DefineAsRegister(node),
803 g.UseRegister(m->left().node()),
804 g.UseRegister(mright.left().node()));
805 return;
806 }
807 }
808
809 if (m->IsWord32Xor() && m->right().Is(-1)) {
810 selector->Emit(kArm64Not32, g.DefineAsRegister(node),
811 g.UseRegister(m->left().node()));
812 } else if (m->IsWord64Xor() && m->right().Is(-1)) {
813 selector->Emit(kArm64Not, g.DefineAsRegister(node),
814 g.UseRegister(m->left().node()));
815 } else {
816 VisitBinop<Matcher>(selector, node, opcode, imm_mode);
817 }
818 }
819
820
VisitWord32And(Node * node)821 void InstructionSelector::VisitWord32And(Node* node) {
822 Arm64OperandGenerator g(this);
823 Int32BinopMatcher m(node);
824 if (m.left().IsWord32Shr() && CanCover(node, m.left().node()) &&
825 m.right().HasValue()) {
826 uint32_t mask = m.right().Value();
827 uint32_t mask_width = base::bits::CountPopulation(mask);
828 uint32_t mask_msb = base::bits::CountLeadingZeros32(mask);
829 if ((mask_width != 0) && (mask_width != 32) &&
830 (mask_msb + mask_width == 32)) {
831 // The mask must be contiguous, and occupy the least-significant bits.
832 DCHECK_EQ(0u, base::bits::CountTrailingZeros32(mask));
833
834 // Select Ubfx for And(Shr(x, imm), mask) where the mask is in the least
835 // significant bits.
836 Int32BinopMatcher mleft(m.left().node());
837 if (mleft.right().HasValue()) {
838 // Any shift value can match; int32 shifts use `value % 32`.
839 uint32_t lsb = mleft.right().Value() & 0x1F;
840
841 // Ubfx cannot extract bits past the register size, however since
842 // shifting the original value would have introduced some zeros we can
843 // still use ubfx with a smaller mask and the remaining bits will be
844 // zeros.
845 if (lsb + mask_width > 32) mask_width = 32 - lsb;
846
847 Emit(kArm64Ubfx32, g.DefineAsRegister(node),
848 g.UseRegister(mleft.left().node()),
849 g.UseImmediateOrTemp(mleft.right().node(), lsb),
850 g.TempImmediate(mask_width));
851 return;
852 }
853 // Other cases fall through to the normal And operation.
854 }
855 }
856 VisitLogical<Int32BinopMatcher>(
857 this, node, &m, kArm64And32, CanCover(node, m.left().node()),
858 CanCover(node, m.right().node()), kLogical32Imm);
859 }
860
861
VisitWord64And(Node * node)862 void InstructionSelector::VisitWord64And(Node* node) {
863 Arm64OperandGenerator g(this);
864 Int64BinopMatcher m(node);
865 if (m.left().IsWord64Shr() && CanCover(node, m.left().node()) &&
866 m.right().HasValue()) {
867 uint64_t mask = m.right().Value();
868 uint64_t mask_width = base::bits::CountPopulation(mask);
869 uint64_t mask_msb = base::bits::CountLeadingZeros64(mask);
870 if ((mask_width != 0) && (mask_width != 64) &&
871 (mask_msb + mask_width == 64)) {
872 // The mask must be contiguous, and occupy the least-significant bits.
873 DCHECK_EQ(0u, base::bits::CountTrailingZeros64(mask));
874
875 // Select Ubfx for And(Shr(x, imm), mask) where the mask is in the least
876 // significant bits.
877 Int64BinopMatcher mleft(m.left().node());
878 if (mleft.right().HasValue()) {
879 // Any shift value can match; int64 shifts use `value % 64`.
880 uint32_t lsb = static_cast<uint32_t>(mleft.right().Value() & 0x3F);
881
882 // Ubfx cannot extract bits past the register size, however since
883 // shifting the original value would have introduced some zeros we can
884 // still use ubfx with a smaller mask and the remaining bits will be
885 // zeros.
886 if (lsb + mask_width > 64) mask_width = 64 - lsb;
887
888 Emit(kArm64Ubfx, g.DefineAsRegister(node),
889 g.UseRegister(mleft.left().node()),
890 g.UseImmediateOrTemp(mleft.right().node(), lsb),
891 g.TempImmediate(static_cast<int32_t>(mask_width)));
892 return;
893 }
894 // Other cases fall through to the normal And operation.
895 }
896 }
897 VisitLogical<Int64BinopMatcher>(
898 this, node, &m, kArm64And, CanCover(node, m.left().node()),
899 CanCover(node, m.right().node()), kLogical64Imm);
900 }
901
902
VisitWord32Or(Node * node)903 void InstructionSelector::VisitWord32Or(Node* node) {
904 Int32BinopMatcher m(node);
905 VisitLogical<Int32BinopMatcher>(
906 this, node, &m, kArm64Or32, CanCover(node, m.left().node()),
907 CanCover(node, m.right().node()), kLogical32Imm);
908 }
909
910
VisitWord64Or(Node * node)911 void InstructionSelector::VisitWord64Or(Node* node) {
912 Int64BinopMatcher m(node);
913 VisitLogical<Int64BinopMatcher>(
914 this, node, &m, kArm64Or, CanCover(node, m.left().node()),
915 CanCover(node, m.right().node()), kLogical64Imm);
916 }
917
918
VisitWord32Xor(Node * node)919 void InstructionSelector::VisitWord32Xor(Node* node) {
920 Int32BinopMatcher m(node);
921 VisitLogical<Int32BinopMatcher>(
922 this, node, &m, kArm64Eor32, CanCover(node, m.left().node()),
923 CanCover(node, m.right().node()), kLogical32Imm);
924 }
925
926
VisitWord64Xor(Node * node)927 void InstructionSelector::VisitWord64Xor(Node* node) {
928 Int64BinopMatcher m(node);
929 VisitLogical<Int64BinopMatcher>(
930 this, node, &m, kArm64Eor, CanCover(node, m.left().node()),
931 CanCover(node, m.right().node()), kLogical64Imm);
932 }
933
934
VisitWord32Shl(Node * node)935 void InstructionSelector::VisitWord32Shl(Node* node) {
936 Int32BinopMatcher m(node);
937 if (m.left().IsWord32And() && CanCover(node, m.left().node()) &&
938 m.right().IsInRange(1, 31)) {
939 Arm64OperandGenerator g(this);
940 Int32BinopMatcher mleft(m.left().node());
941 if (mleft.right().HasValue()) {
942 uint32_t mask = mleft.right().Value();
943 uint32_t mask_width = base::bits::CountPopulation(mask);
944 uint32_t mask_msb = base::bits::CountLeadingZeros32(mask);
945 if ((mask_width != 0) && (mask_msb + mask_width == 32)) {
946 uint32_t shift = m.right().Value();
947 DCHECK_EQ(0u, base::bits::CountTrailingZeros32(mask));
948 DCHECK_NE(0u, shift);
949
950 if ((shift + mask_width) >= 32) {
951 // If the mask is contiguous and reaches or extends beyond the top
952 // bit, only the shift is needed.
953 Emit(kArm64Lsl32, g.DefineAsRegister(node),
954 g.UseRegister(mleft.left().node()),
955 g.UseImmediate(m.right().node()));
956 return;
957 } else {
958 // Select Ubfiz for Shl(And(x, mask), imm) where the mask is
959 // contiguous, and the shift immediate non-zero.
960 Emit(kArm64Ubfiz32, g.DefineAsRegister(node),
961 g.UseRegister(mleft.left().node()),
962 g.UseImmediate(m.right().node()), g.TempImmediate(mask_width));
963 return;
964 }
965 }
966 }
967 }
968 VisitRRO(this, kArm64Lsl32, node, kShift32Imm);
969 }
970
971
VisitWord64Shl(Node * node)972 void InstructionSelector::VisitWord64Shl(Node* node) {
973 Arm64OperandGenerator g(this);
974 Int64BinopMatcher m(node);
975 if ((m.left().IsChangeInt32ToInt64() || m.left().IsChangeUint32ToUint64()) &&
976 m.right().IsInRange(32, 63) && CanCover(node, m.left().node())) {
977 // There's no need to sign/zero-extend to 64-bit if we shift out the upper
978 // 32 bits anyway.
979 Emit(kArm64Lsl, g.DefineAsRegister(node),
980 g.UseRegister(m.left().node()->InputAt(0)),
981 g.UseImmediate(m.right().node()));
982 return;
983 }
984 VisitRRO(this, kArm64Lsl, node, kShift64Imm);
985 }
986
987
988 namespace {
989
TryEmitBitfieldExtract32(InstructionSelector * selector,Node * node)990 bool TryEmitBitfieldExtract32(InstructionSelector* selector, Node* node) {
991 Arm64OperandGenerator g(selector);
992 Int32BinopMatcher m(node);
993 if (selector->CanCover(node, m.left().node()) && m.left().IsWord32Shl()) {
994 // Select Ubfx or Sbfx for (x << (K & 0x1F)) OP (K & 0x1F), where
995 // OP is >>> or >> and (K & 0x1F) != 0.
996 Int32BinopMatcher mleft(m.left().node());
997 if (mleft.right().HasValue() && m.right().HasValue() &&
998 (mleft.right().Value() & 0x1F) != 0 &&
999 (mleft.right().Value() & 0x1F) == (m.right().Value() & 0x1F)) {
1000 DCHECK(m.IsWord32Shr() || m.IsWord32Sar());
1001 ArchOpcode opcode = m.IsWord32Sar() ? kArm64Sbfx32 : kArm64Ubfx32;
1002
1003 int right_val = m.right().Value() & 0x1F;
1004 DCHECK_NE(right_val, 0);
1005
1006 selector->Emit(opcode, g.DefineAsRegister(node),
1007 g.UseRegister(mleft.left().node()), g.TempImmediate(0),
1008 g.TempImmediate(32 - right_val));
1009 return true;
1010 }
1011 }
1012 return false;
1013 }
1014
1015 } // namespace
1016
1017
VisitWord32Shr(Node * node)1018 void InstructionSelector::VisitWord32Shr(Node* node) {
1019 Int32BinopMatcher m(node);
1020 if (m.left().IsWord32And() && m.right().HasValue()) {
1021 uint32_t lsb = m.right().Value() & 0x1F;
1022 Int32BinopMatcher mleft(m.left().node());
1023 if (mleft.right().HasValue() && mleft.right().Value() != 0) {
1024 // Select Ubfx for Shr(And(x, mask), imm) where the result of the mask is
1025 // shifted into the least-significant bits.
1026 uint32_t mask = (mleft.right().Value() >> lsb) << lsb;
1027 unsigned mask_width = base::bits::CountPopulation(mask);
1028 unsigned mask_msb = base::bits::CountLeadingZeros32(mask);
1029 if ((mask_msb + mask_width + lsb) == 32) {
1030 Arm64OperandGenerator g(this);
1031 DCHECK_EQ(lsb, base::bits::CountTrailingZeros32(mask));
1032 Emit(kArm64Ubfx32, g.DefineAsRegister(node),
1033 g.UseRegister(mleft.left().node()),
1034 g.UseImmediateOrTemp(m.right().node(), lsb),
1035 g.TempImmediate(mask_width));
1036 return;
1037 }
1038 }
1039 } else if (TryEmitBitfieldExtract32(this, node)) {
1040 return;
1041 }
1042
1043 if (m.left().IsUint32MulHigh() && m.right().HasValue() &&
1044 CanCover(node, node->InputAt(0))) {
1045 // Combine this shift with the multiply and shift that would be generated
1046 // by Uint32MulHigh.
1047 Arm64OperandGenerator g(this);
1048 Node* left = m.left().node();
1049 int shift = m.right().Value() & 0x1F;
1050 InstructionOperand const smull_operand = g.TempRegister();
1051 Emit(kArm64Umull, smull_operand, g.UseRegister(left->InputAt(0)),
1052 g.UseRegister(left->InputAt(1)));
1053 Emit(kArm64Lsr, g.DefineAsRegister(node), smull_operand,
1054 g.TempImmediate(32 + shift));
1055 return;
1056 }
1057
1058 VisitRRO(this, kArm64Lsr32, node, kShift32Imm);
1059 }
1060
1061
VisitWord64Shr(Node * node)1062 void InstructionSelector::VisitWord64Shr(Node* node) {
1063 Int64BinopMatcher m(node);
1064 if (m.left().IsWord64And() && m.right().HasValue()) {
1065 uint32_t lsb = m.right().Value() & 0x3F;
1066 Int64BinopMatcher mleft(m.left().node());
1067 if (mleft.right().HasValue() && mleft.right().Value() != 0) {
1068 // Select Ubfx for Shr(And(x, mask), imm) where the result of the mask is
1069 // shifted into the least-significant bits.
1070 uint64_t mask = (mleft.right().Value() >> lsb) << lsb;
1071 unsigned mask_width = base::bits::CountPopulation(mask);
1072 unsigned mask_msb = base::bits::CountLeadingZeros64(mask);
1073 if ((mask_msb + mask_width + lsb) == 64) {
1074 Arm64OperandGenerator g(this);
1075 DCHECK_EQ(lsb, base::bits::CountTrailingZeros64(mask));
1076 Emit(kArm64Ubfx, g.DefineAsRegister(node),
1077 g.UseRegister(mleft.left().node()),
1078 g.UseImmediateOrTemp(m.right().node(), lsb),
1079 g.TempImmediate(mask_width));
1080 return;
1081 }
1082 }
1083 }
1084 VisitRRO(this, kArm64Lsr, node, kShift64Imm);
1085 }
1086
1087
VisitWord32Sar(Node * node)1088 void InstructionSelector::VisitWord32Sar(Node* node) {
1089 if (TryEmitBitfieldExtract32(this, node)) {
1090 return;
1091 }
1092
1093 Int32BinopMatcher m(node);
1094 if (m.left().IsInt32MulHigh() && m.right().HasValue() &&
1095 CanCover(node, node->InputAt(0))) {
1096 // Combine this shift with the multiply and shift that would be generated
1097 // by Int32MulHigh.
1098 Arm64OperandGenerator g(this);
1099 Node* left = m.left().node();
1100 int shift = m.right().Value() & 0x1F;
1101 InstructionOperand const smull_operand = g.TempRegister();
1102 Emit(kArm64Smull, smull_operand, g.UseRegister(left->InputAt(0)),
1103 g.UseRegister(left->InputAt(1)));
1104 Emit(kArm64Asr, g.DefineAsRegister(node), smull_operand,
1105 g.TempImmediate(32 + shift));
1106 return;
1107 }
1108
1109 if (m.left().IsInt32Add() && m.right().HasValue() &&
1110 CanCover(node, node->InputAt(0))) {
1111 Node* add_node = m.left().node();
1112 Int32BinopMatcher madd_node(add_node);
1113 if (madd_node.left().IsInt32MulHigh() &&
1114 CanCover(add_node, madd_node.left().node())) {
1115 // Combine the shift that would be generated by Int32MulHigh with the add
1116 // on the left of this Sar operation. We do it here, as the result of the
1117 // add potentially has 33 bits, so we have to ensure the result is
1118 // truncated by being the input to this 32-bit Sar operation.
1119 Arm64OperandGenerator g(this);
1120 Node* mul_node = madd_node.left().node();
1121
1122 InstructionOperand const smull_operand = g.TempRegister();
1123 Emit(kArm64Smull, smull_operand, g.UseRegister(mul_node->InputAt(0)),
1124 g.UseRegister(mul_node->InputAt(1)));
1125
1126 InstructionOperand const add_operand = g.TempRegister();
1127 Emit(kArm64Add | AddressingModeField::encode(kMode_Operand2_R_ASR_I),
1128 add_operand, g.UseRegister(add_node->InputAt(1)), smull_operand,
1129 g.TempImmediate(32));
1130
1131 Emit(kArm64Asr32, g.DefineAsRegister(node), add_operand,
1132 g.UseImmediate(node->InputAt(1)));
1133 return;
1134 }
1135 }
1136
1137 VisitRRO(this, kArm64Asr32, node, kShift32Imm);
1138 }
1139
1140
VisitWord64Sar(Node * node)1141 void InstructionSelector::VisitWord64Sar(Node* node) {
1142 if (TryEmitExtendingLoad(this, node)) return;
1143 VisitRRO(this, kArm64Asr, node, kShift64Imm);
1144 }
1145
1146
VisitWord32Ror(Node * node)1147 void InstructionSelector::VisitWord32Ror(Node* node) {
1148 VisitRRO(this, kArm64Ror32, node, kShift32Imm);
1149 }
1150
1151
VisitWord64Ror(Node * node)1152 void InstructionSelector::VisitWord64Ror(Node* node) {
1153 VisitRRO(this, kArm64Ror, node, kShift64Imm);
1154 }
1155
1156 #define RR_OP_LIST(V) \
1157 V(Word64Clz, kArm64Clz) \
1158 V(Word32Clz, kArm64Clz32) \
1159 V(Word32ReverseBits, kArm64Rbit32) \
1160 V(Word64ReverseBits, kArm64Rbit) \
1161 V(ChangeFloat32ToFloat64, kArm64Float32ToFloat64) \
1162 V(RoundInt32ToFloat32, kArm64Int32ToFloat32) \
1163 V(RoundUint32ToFloat32, kArm64Uint32ToFloat32) \
1164 V(ChangeInt32ToFloat64, kArm64Int32ToFloat64) \
1165 V(ChangeUint32ToFloat64, kArm64Uint32ToFloat64) \
1166 V(TruncateFloat32ToInt32, kArm64Float32ToInt32) \
1167 V(ChangeFloat64ToInt32, kArm64Float64ToInt32) \
1168 V(TruncateFloat32ToUint32, kArm64Float32ToUint32) \
1169 V(ChangeFloat64ToUint32, kArm64Float64ToUint32) \
1170 V(ChangeFloat64ToUint64, kArm64Float64ToUint64) \
1171 V(TruncateFloat64ToUint32, kArm64Float64ToUint32) \
1172 V(TruncateFloat64ToFloat32, kArm64Float64ToFloat32) \
1173 V(TruncateFloat64ToWord32, kArchTruncateDoubleToI) \
1174 V(RoundFloat64ToInt32, kArm64Float64ToInt32) \
1175 V(RoundInt64ToFloat32, kArm64Int64ToFloat32) \
1176 V(RoundInt64ToFloat64, kArm64Int64ToFloat64) \
1177 V(RoundUint64ToFloat32, kArm64Uint64ToFloat32) \
1178 V(RoundUint64ToFloat64, kArm64Uint64ToFloat64) \
1179 V(BitcastFloat32ToInt32, kArm64Float64ExtractLowWord32) \
1180 V(BitcastFloat64ToInt64, kArm64U64MoveFloat64) \
1181 V(BitcastInt32ToFloat32, kArm64Float64MoveU64) \
1182 V(BitcastInt64ToFloat64, kArm64Float64MoveU64) \
1183 V(Float32Abs, kArm64Float32Abs) \
1184 V(Float64Abs, kArm64Float64Abs) \
1185 V(Float32Sqrt, kArm64Float32Sqrt) \
1186 V(Float64Sqrt, kArm64Float64Sqrt) \
1187 V(Float32RoundDown, kArm64Float32RoundDown) \
1188 V(Float64RoundDown, kArm64Float64RoundDown) \
1189 V(Float32RoundUp, kArm64Float32RoundUp) \
1190 V(Float64RoundUp, kArm64Float64RoundUp) \
1191 V(Float32RoundTruncate, kArm64Float32RoundTruncate) \
1192 V(Float64RoundTruncate, kArm64Float64RoundTruncate) \
1193 V(Float64RoundTiesAway, kArm64Float64RoundTiesAway) \
1194 V(Float32RoundTiesEven, kArm64Float32RoundTiesEven) \
1195 V(Float64RoundTiesEven, kArm64Float64RoundTiesEven) \
1196 V(Float32Neg, kArm64Float32Neg) \
1197 V(Float64Neg, kArm64Float64Neg) \
1198 V(Float64ExtractLowWord32, kArm64Float64ExtractLowWord32) \
1199 V(Float64ExtractHighWord32, kArm64Float64ExtractHighWord32) \
1200 V(Float64SilenceNaN, kArm64Float64SilenceNaN)
1201
1202 #define RRR_OP_LIST(V) \
1203 V(Int32Div, kArm64Idiv32) \
1204 V(Int64Div, kArm64Idiv) \
1205 V(Uint32Div, kArm64Udiv32) \
1206 V(Uint64Div, kArm64Udiv) \
1207 V(Int32Mod, kArm64Imod32) \
1208 V(Int64Mod, kArm64Imod) \
1209 V(Uint32Mod, kArm64Umod32) \
1210 V(Uint64Mod, kArm64Umod) \
1211 V(Float32Add, kArm64Float32Add) \
1212 V(Float64Add, kArm64Float64Add) \
1213 V(Float32Sub, kArm64Float32Sub) \
1214 V(Float64Sub, kArm64Float64Sub) \
1215 V(Float32Mul, kArm64Float32Mul) \
1216 V(Float64Mul, kArm64Float64Mul) \
1217 V(Float32Div, kArm64Float32Div) \
1218 V(Float64Div, kArm64Float64Div) \
1219 V(Float32Max, kArm64Float32Max) \
1220 V(Float64Max, kArm64Float64Max) \
1221 V(Float32Min, kArm64Float32Min) \
1222 V(Float64Min, kArm64Float64Min)
1223
1224 #define RR_VISITOR(Name, opcode) \
1225 void InstructionSelector::Visit##Name(Node* node) { \
1226 VisitRR(this, opcode, node); \
1227 }
1228 RR_OP_LIST(RR_VISITOR)
1229 #undef RR_VISITOR
1230 #undef RR_OP_LIST
1231
1232 #define RRR_VISITOR(Name, opcode) \
1233 void InstructionSelector::Visit##Name(Node* node) { \
1234 VisitRRR(this, opcode, node); \
1235 }
RRR_OP_LIST(RRR_VISITOR)1236 RRR_OP_LIST(RRR_VISITOR)
1237 #undef RRR_VISITOR
1238 #undef RRR_OP_LIST
1239
1240 void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
1241
VisitWord64Ctz(Node * node)1242 void InstructionSelector::VisitWord64Ctz(Node* node) { UNREACHABLE(); }
1243
VisitWord64ReverseBytes(Node * node)1244 void InstructionSelector::VisitWord64ReverseBytes(Node* node) { UNREACHABLE(); }
1245
VisitWord32ReverseBytes(Node * node)1246 void InstructionSelector::VisitWord32ReverseBytes(Node* node) { UNREACHABLE(); }
1247
VisitWord32Popcnt(Node * node)1248 void InstructionSelector::VisitWord32Popcnt(Node* node) { UNREACHABLE(); }
1249
VisitWord64Popcnt(Node * node)1250 void InstructionSelector::VisitWord64Popcnt(Node* node) { UNREACHABLE(); }
1251
VisitSpeculationFence(Node * node)1252 void InstructionSelector::VisitSpeculationFence(Node* node) {
1253 Arm64OperandGenerator g(this);
1254 Emit(kArm64DsbIsb, g.NoOutput());
1255 }
1256
VisitInt32Add(Node * node)1257 void InstructionSelector::VisitInt32Add(Node* node) {
1258 Arm64OperandGenerator g(this);
1259 Int32BinopMatcher m(node);
1260 // Select Madd(x, y, z) for Add(Mul(x, y), z).
1261 if (m.left().IsInt32Mul() && CanCover(node, m.left().node())) {
1262 Int32BinopMatcher mleft(m.left().node());
1263 // Check multiply can't be later reduced to addition with shift.
1264 if (LeftShiftForReducedMultiply(&mleft) == 0) {
1265 Emit(kArm64Madd32, g.DefineAsRegister(node),
1266 g.UseRegister(mleft.left().node()),
1267 g.UseRegister(mleft.right().node()),
1268 g.UseRegister(m.right().node()));
1269 return;
1270 }
1271 }
1272 // Select Madd(x, y, z) for Add(z, Mul(x, y)).
1273 if (m.right().IsInt32Mul() && CanCover(node, m.right().node())) {
1274 Int32BinopMatcher mright(m.right().node());
1275 // Check multiply can't be later reduced to addition with shift.
1276 if (LeftShiftForReducedMultiply(&mright) == 0) {
1277 Emit(kArm64Madd32, g.DefineAsRegister(node),
1278 g.UseRegister(mright.left().node()),
1279 g.UseRegister(mright.right().node()),
1280 g.UseRegister(m.left().node()));
1281 return;
1282 }
1283 }
1284 VisitAddSub<Int32BinopMatcher>(this, node, kArm64Add32, kArm64Sub32);
1285 }
1286
1287
VisitInt64Add(Node * node)1288 void InstructionSelector::VisitInt64Add(Node* node) {
1289 Arm64OperandGenerator g(this);
1290 Int64BinopMatcher m(node);
1291 // Select Madd(x, y, z) for Add(Mul(x, y), z).
1292 if (m.left().IsInt64Mul() && CanCover(node, m.left().node())) {
1293 Int64BinopMatcher mleft(m.left().node());
1294 // Check multiply can't be later reduced to addition with shift.
1295 if (LeftShiftForReducedMultiply(&mleft) == 0) {
1296 Emit(kArm64Madd, g.DefineAsRegister(node),
1297 g.UseRegister(mleft.left().node()),
1298 g.UseRegister(mleft.right().node()),
1299 g.UseRegister(m.right().node()));
1300 return;
1301 }
1302 }
1303 // Select Madd(x, y, z) for Add(z, Mul(x, y)).
1304 if (m.right().IsInt64Mul() && CanCover(node, m.right().node())) {
1305 Int64BinopMatcher mright(m.right().node());
1306 // Check multiply can't be later reduced to addition with shift.
1307 if (LeftShiftForReducedMultiply(&mright) == 0) {
1308 Emit(kArm64Madd, g.DefineAsRegister(node),
1309 g.UseRegister(mright.left().node()),
1310 g.UseRegister(mright.right().node()),
1311 g.UseRegister(m.left().node()));
1312 return;
1313 }
1314 }
1315 VisitAddSub<Int64BinopMatcher>(this, node, kArm64Add, kArm64Sub);
1316 }
1317
1318
VisitInt32Sub(Node * node)1319 void InstructionSelector::VisitInt32Sub(Node* node) {
1320 Arm64OperandGenerator g(this);
1321 Int32BinopMatcher m(node);
1322
1323 // Select Msub(x, y, a) for Sub(a, Mul(x, y)).
1324 if (m.right().IsInt32Mul() && CanCover(node, m.right().node())) {
1325 Int32BinopMatcher mright(m.right().node());
1326 // Check multiply can't be later reduced to addition with shift.
1327 if (LeftShiftForReducedMultiply(&mright) == 0) {
1328 Emit(kArm64Msub32, g.DefineAsRegister(node),
1329 g.UseRegister(mright.left().node()),
1330 g.UseRegister(mright.right().node()),
1331 g.UseRegister(m.left().node()));
1332 return;
1333 }
1334 }
1335
1336 VisitAddSub<Int32BinopMatcher>(this, node, kArm64Sub32, kArm64Add32);
1337 }
1338
1339
VisitInt64Sub(Node * node)1340 void InstructionSelector::VisitInt64Sub(Node* node) {
1341 Arm64OperandGenerator g(this);
1342 Int64BinopMatcher m(node);
1343
1344 // Select Msub(x, y, a) for Sub(a, Mul(x, y)).
1345 if (m.right().IsInt64Mul() && CanCover(node, m.right().node())) {
1346 Int64BinopMatcher mright(m.right().node());
1347 // Check multiply can't be later reduced to addition with shift.
1348 if (LeftShiftForReducedMultiply(&mright) == 0) {
1349 Emit(kArm64Msub, g.DefineAsRegister(node),
1350 g.UseRegister(mright.left().node()),
1351 g.UseRegister(mright.right().node()),
1352 g.UseRegister(m.left().node()));
1353 return;
1354 }
1355 }
1356
1357 VisitAddSub<Int64BinopMatcher>(this, node, kArm64Sub, kArm64Add);
1358 }
1359
1360 namespace {
1361
EmitInt32MulWithOverflow(InstructionSelector * selector,Node * node,FlagsContinuation * cont)1362 void EmitInt32MulWithOverflow(InstructionSelector* selector, Node* node,
1363 FlagsContinuation* cont) {
1364 Arm64OperandGenerator g(selector);
1365 Int32BinopMatcher m(node);
1366 InstructionOperand result = g.DefineAsRegister(node);
1367 InstructionOperand left = g.UseRegister(m.left().node());
1368 InstructionOperand right = g.UseRegister(m.right().node());
1369 selector->Emit(kArm64Smull, result, left, right);
1370
1371 InstructionCode opcode =
1372 kArm64Cmp | AddressingModeField::encode(kMode_Operand2_R_SXTW);
1373 selector->EmitWithContinuation(opcode, result, result, cont);
1374 }
1375
1376 } // namespace
1377
VisitInt32Mul(Node * node)1378 void InstructionSelector::VisitInt32Mul(Node* node) {
1379 Arm64OperandGenerator g(this);
1380 Int32BinopMatcher m(node);
1381
1382 // First, try to reduce the multiplication to addition with left shift.
1383 // x * (2^k + 1) -> x + (x << k)
1384 int32_t shift = LeftShiftForReducedMultiply(&m);
1385 if (shift > 0) {
1386 Emit(kArm64Add32 | AddressingModeField::encode(kMode_Operand2_R_LSL_I),
1387 g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1388 g.UseRegister(m.left().node()), g.TempImmediate(shift));
1389 return;
1390 }
1391
1392 if (m.left().IsInt32Sub() && CanCover(node, m.left().node())) {
1393 Int32BinopMatcher mleft(m.left().node());
1394
1395 // Select Mneg(x, y) for Mul(Sub(0, x), y).
1396 if (mleft.left().Is(0)) {
1397 Emit(kArm64Mneg32, g.DefineAsRegister(node),
1398 g.UseRegister(mleft.right().node()),
1399 g.UseRegister(m.right().node()));
1400 return;
1401 }
1402 }
1403
1404 if (m.right().IsInt32Sub() && CanCover(node, m.right().node())) {
1405 Int32BinopMatcher mright(m.right().node());
1406
1407 // Select Mneg(x, y) for Mul(x, Sub(0, y)).
1408 if (mright.left().Is(0)) {
1409 Emit(kArm64Mneg32, g.DefineAsRegister(node),
1410 g.UseRegister(m.left().node()),
1411 g.UseRegister(mright.right().node()));
1412 return;
1413 }
1414 }
1415
1416 VisitRRR(this, kArm64Mul32, node);
1417 }
1418
1419
VisitInt64Mul(Node * node)1420 void InstructionSelector::VisitInt64Mul(Node* node) {
1421 Arm64OperandGenerator g(this);
1422 Int64BinopMatcher m(node);
1423
1424 // First, try to reduce the multiplication to addition with left shift.
1425 // x * (2^k + 1) -> x + (x << k)
1426 int32_t shift = LeftShiftForReducedMultiply(&m);
1427 if (shift > 0) {
1428 Emit(kArm64Add | AddressingModeField::encode(kMode_Operand2_R_LSL_I),
1429 g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1430 g.UseRegister(m.left().node()), g.TempImmediate(shift));
1431 return;
1432 }
1433
1434 if (m.left().IsInt64Sub() && CanCover(node, m.left().node())) {
1435 Int64BinopMatcher mleft(m.left().node());
1436
1437 // Select Mneg(x, y) for Mul(Sub(0, x), y).
1438 if (mleft.left().Is(0)) {
1439 Emit(kArm64Mneg, g.DefineAsRegister(node),
1440 g.UseRegister(mleft.right().node()),
1441 g.UseRegister(m.right().node()));
1442 return;
1443 }
1444 }
1445
1446 if (m.right().IsInt64Sub() && CanCover(node, m.right().node())) {
1447 Int64BinopMatcher mright(m.right().node());
1448
1449 // Select Mneg(x, y) for Mul(x, Sub(0, y)).
1450 if (mright.left().Is(0)) {
1451 Emit(kArm64Mneg, g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1452 g.UseRegister(mright.right().node()));
1453 return;
1454 }
1455 }
1456
1457 VisitRRR(this, kArm64Mul, node);
1458 }
1459
VisitInt32MulHigh(Node * node)1460 void InstructionSelector::VisitInt32MulHigh(Node* node) {
1461 Arm64OperandGenerator g(this);
1462 InstructionOperand const smull_operand = g.TempRegister();
1463 Emit(kArm64Smull, smull_operand, g.UseRegister(node->InputAt(0)),
1464 g.UseRegister(node->InputAt(1)));
1465 Emit(kArm64Asr, g.DefineAsRegister(node), smull_operand, g.TempImmediate(32));
1466 }
1467
1468
VisitUint32MulHigh(Node * node)1469 void InstructionSelector::VisitUint32MulHigh(Node* node) {
1470 Arm64OperandGenerator g(this);
1471 InstructionOperand const smull_operand = g.TempRegister();
1472 Emit(kArm64Umull, smull_operand, g.UseRegister(node->InputAt(0)),
1473 g.UseRegister(node->InputAt(1)));
1474 Emit(kArm64Lsr, g.DefineAsRegister(node), smull_operand, g.TempImmediate(32));
1475 }
1476
1477
VisitTryTruncateFloat32ToInt64(Node * node)1478 void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
1479 Arm64OperandGenerator g(this);
1480
1481 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1482 InstructionOperand outputs[2];
1483 size_t output_count = 0;
1484 outputs[output_count++] = g.DefineAsRegister(node);
1485
1486 Node* success_output = NodeProperties::FindProjection(node, 1);
1487 if (success_output) {
1488 outputs[output_count++] = g.DefineAsRegister(success_output);
1489 }
1490
1491 Emit(kArm64Float32ToInt64, output_count, outputs, 1, inputs);
1492 }
1493
1494
VisitTryTruncateFloat64ToInt64(Node * node)1495 void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
1496 Arm64OperandGenerator g(this);
1497
1498 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1499 InstructionOperand outputs[2];
1500 size_t output_count = 0;
1501 outputs[output_count++] = g.DefineAsRegister(node);
1502
1503 Node* success_output = NodeProperties::FindProjection(node, 1);
1504 if (success_output) {
1505 outputs[output_count++] = g.DefineAsRegister(success_output);
1506 }
1507
1508 Emit(kArm64Float64ToInt64, output_count, outputs, 1, inputs);
1509 }
1510
1511
VisitTryTruncateFloat32ToUint64(Node * node)1512 void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
1513 Arm64OperandGenerator g(this);
1514
1515 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1516 InstructionOperand outputs[2];
1517 size_t output_count = 0;
1518 outputs[output_count++] = g.DefineAsRegister(node);
1519
1520 Node* success_output = NodeProperties::FindProjection(node, 1);
1521 if (success_output) {
1522 outputs[output_count++] = g.DefineAsRegister(success_output);
1523 }
1524
1525 Emit(kArm64Float32ToUint64, output_count, outputs, 1, inputs);
1526 }
1527
1528
VisitTryTruncateFloat64ToUint64(Node * node)1529 void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
1530 Arm64OperandGenerator g(this);
1531
1532 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1533 InstructionOperand outputs[2];
1534 size_t output_count = 0;
1535 outputs[output_count++] = g.DefineAsRegister(node);
1536
1537 Node* success_output = NodeProperties::FindProjection(node, 1);
1538 if (success_output) {
1539 outputs[output_count++] = g.DefineAsRegister(success_output);
1540 }
1541
1542 Emit(kArm64Float64ToUint64, output_count, outputs, 1, inputs);
1543 }
1544
1545
VisitChangeInt32ToInt64(Node * node)1546 void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
1547 Node* value = node->InputAt(0);
1548 if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
1549 // Generate sign-extending load.
1550 LoadRepresentation load_rep = LoadRepresentationOf(value->op());
1551 MachineRepresentation rep = load_rep.representation();
1552 InstructionCode opcode = kArchNop;
1553 ImmediateMode immediate_mode = kNoImmediate;
1554 switch (rep) {
1555 case MachineRepresentation::kBit: // Fall through.
1556 case MachineRepresentation::kWord8:
1557 opcode = load_rep.IsSigned() ? kArm64Ldrsb : kArm64Ldrb;
1558 immediate_mode = kLoadStoreImm8;
1559 break;
1560 case MachineRepresentation::kWord16:
1561 opcode = load_rep.IsSigned() ? kArm64Ldrsh : kArm64Ldrh;
1562 immediate_mode = kLoadStoreImm16;
1563 break;
1564 case MachineRepresentation::kWord32:
1565 opcode = kArm64Ldrsw;
1566 immediate_mode = kLoadStoreImm32;
1567 break;
1568 default:
1569 UNREACHABLE();
1570 return;
1571 }
1572 EmitLoad(this, value, opcode, immediate_mode, rep, node);
1573 } else {
1574 VisitRR(this, kArm64Sxtw, node);
1575 }
1576 }
1577
1578
VisitChangeUint32ToUint64(Node * node)1579 void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
1580 Arm64OperandGenerator g(this);
1581 Node* value = node->InputAt(0);
1582 switch (value->opcode()) {
1583 case IrOpcode::kWord32And:
1584 case IrOpcode::kWord32Or:
1585 case IrOpcode::kWord32Xor:
1586 case IrOpcode::kWord32Shl:
1587 case IrOpcode::kWord32Shr:
1588 case IrOpcode::kWord32Sar:
1589 case IrOpcode::kWord32Ror:
1590 case IrOpcode::kWord32Equal:
1591 case IrOpcode::kInt32Add:
1592 case IrOpcode::kInt32AddWithOverflow:
1593 case IrOpcode::kInt32Sub:
1594 case IrOpcode::kInt32SubWithOverflow:
1595 case IrOpcode::kInt32Mul:
1596 case IrOpcode::kInt32MulHigh:
1597 case IrOpcode::kInt32Div:
1598 case IrOpcode::kInt32Mod:
1599 case IrOpcode::kInt32LessThan:
1600 case IrOpcode::kInt32LessThanOrEqual:
1601 case IrOpcode::kUint32Div:
1602 case IrOpcode::kUint32LessThan:
1603 case IrOpcode::kUint32LessThanOrEqual:
1604 case IrOpcode::kUint32Mod:
1605 case IrOpcode::kUint32MulHigh: {
1606 // 32-bit operations will write their result in a W register (implicitly
1607 // clearing the top 32-bit of the corresponding X register) so the
1608 // zero-extension is a no-op.
1609 Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(value));
1610 return;
1611 }
1612 case IrOpcode::kLoad: {
1613 // As for the operations above, a 32-bit load will implicitly clear the
1614 // top 32 bits of the destination register.
1615 LoadRepresentation load_rep = LoadRepresentationOf(value->op());
1616 switch (load_rep.representation()) {
1617 case MachineRepresentation::kWord8:
1618 case MachineRepresentation::kWord16:
1619 case MachineRepresentation::kWord32:
1620 Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(value));
1621 return;
1622 default:
1623 break;
1624 }
1625 break;
1626 }
1627 default:
1628 break;
1629 }
1630 Emit(kArm64Mov32, g.DefineAsRegister(node), g.UseRegister(value));
1631 }
1632
VisitTruncateInt64ToInt32(Node * node)1633 void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
1634 Arm64OperandGenerator g(this);
1635 Node* value = node->InputAt(0);
1636 // The top 32 bits in the 64-bit register will be undefined, and
1637 // must not be used by a dependent node.
1638 Emit(kArchNop, g.DefineSameAsFirst(node), g.UseRegister(value));
1639 }
1640
VisitFloat64Mod(Node * node)1641 void InstructionSelector::VisitFloat64Mod(Node* node) {
1642 Arm64OperandGenerator g(this);
1643 Emit(kArm64Float64Mod, g.DefineAsFixed(node, d0),
1644 g.UseFixed(node->InputAt(0), d0),
1645 g.UseFixed(node->InputAt(1), d1))->MarkAsCall();
1646 }
1647
VisitFloat64Ieee754Binop(Node * node,InstructionCode opcode)1648 void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
1649 InstructionCode opcode) {
1650 Arm64OperandGenerator g(this);
1651 Emit(opcode, g.DefineAsFixed(node, d0), g.UseFixed(node->InputAt(0), d0),
1652 g.UseFixed(node->InputAt(1), d1))
1653 ->MarkAsCall();
1654 }
1655
VisitFloat64Ieee754Unop(Node * node,InstructionCode opcode)1656 void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
1657 InstructionCode opcode) {
1658 Arm64OperandGenerator g(this);
1659 Emit(opcode, g.DefineAsFixed(node, d0), g.UseFixed(node->InputAt(0), d0))
1660 ->MarkAsCall();
1661 }
1662
EmitPrepareArguments(ZoneVector<PushParameter> * arguments,const CallDescriptor * call_descriptor,Node * node)1663 void InstructionSelector::EmitPrepareArguments(
1664 ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
1665 Node* node) {
1666 Arm64OperandGenerator g(this);
1667
1668 // `arguments` includes alignment "holes". This means that slots bigger than
1669 // kPointerSize, e.g. Simd128, will span across multiple arguments.
1670 int claim_count = static_cast<int>(arguments->size());
1671 int slot = claim_count - 1;
1672 claim_count = RoundUp(claim_count, 2);
1673 // Bump the stack pointer(s).
1674 if (claim_count > 0) {
1675 // TODO(titzer): claim and poke probably take small immediates.
1676 // TODO(titzer): it would be better to bump the sp here only
1677 // and emit paired stores with increment for non c frames.
1678 Emit(kArm64Claim, g.NoOutput(), g.TempImmediate(claim_count));
1679 }
1680
1681 if (claim_count > 0) {
1682 // Store padding, which might be overwritten.
1683 Emit(kArm64Poke, g.NoOutput(), g.UseImmediate(0),
1684 g.TempImmediate(claim_count - 1));
1685 }
1686
1687 // Poke the arguments into the stack.
1688 while (slot >= 0) {
1689 Node* input_node = (*arguments)[slot].node;
1690 // Skip any alignment holes in pushed nodes.
1691 if (input_node != nullptr) {
1692 Emit(kArm64Poke, g.NoOutput(), g.UseRegister(input_node),
1693 g.TempImmediate(slot));
1694 }
1695 slot--;
1696 // TODO(ahaas): Poke arguments in pairs if two subsequent arguments have the
1697 // same type.
1698 // Emit(kArm64PokePair, g.NoOutput(), g.UseRegister((*arguments)[slot]),
1699 // g.UseRegister((*arguments)[slot - 1]), g.TempImmediate(slot));
1700 // slot -= 2;
1701 }
1702 }
1703
EmitPrepareResults(ZoneVector<PushParameter> * results,const CallDescriptor * call_descriptor,Node * node)1704 void InstructionSelector::EmitPrepareResults(
1705 ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
1706 Node* node) {
1707 Arm64OperandGenerator g(this);
1708
1709 int reverse_slot = 0;
1710 for (PushParameter output : *results) {
1711 if (!output.location.IsCallerFrameSlot()) continue;
1712 reverse_slot += output.location.GetSizeInPointers();
1713 // Skip any alignment holes in nodes.
1714 if (output.node == nullptr) continue;
1715 DCHECK(!call_descriptor->IsCFunctionCall());
1716
1717 if (output.location.GetType() == MachineType::Float32()) {
1718 MarkAsFloat32(output.node);
1719 } else if (output.location.GetType() == MachineType::Float64()) {
1720 MarkAsFloat64(output.node);
1721 }
1722
1723 Emit(kArm64Peek, g.DefineAsRegister(output.node),
1724 g.UseImmediate(reverse_slot));
1725 }
1726 }
1727
IsTailCallAddressImmediate()1728 bool InstructionSelector::IsTailCallAddressImmediate() { return false; }
1729
GetTempsCountForTailCallFromJSFunction()1730 int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 3; }
1731
1732 namespace {
1733
1734 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,InstructionOperand left,InstructionOperand right,FlagsContinuation * cont)1735 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1736 InstructionOperand left, InstructionOperand right,
1737 FlagsContinuation* cont) {
1738 selector->EmitWithContinuation(opcode, left, right, cont);
1739 }
1740
1741
1742 // Shared routine for multiple word compare operations.
VisitWordCompare(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont,bool commutative,ImmediateMode immediate_mode)1743 void VisitWordCompare(InstructionSelector* selector, Node* node,
1744 InstructionCode opcode, FlagsContinuation* cont,
1745 bool commutative, ImmediateMode immediate_mode) {
1746 Arm64OperandGenerator g(selector);
1747 Node* left = node->InputAt(0);
1748 Node* right = node->InputAt(1);
1749
1750 // Match immediates on left or right side of comparison.
1751 if (g.CanBeImmediate(right, immediate_mode)) {
1752 VisitCompare(selector, opcode, g.UseRegister(left), g.UseImmediate(right),
1753 cont);
1754 } else if (g.CanBeImmediate(left, immediate_mode)) {
1755 if (!commutative) cont->Commute();
1756 VisitCompare(selector, opcode, g.UseRegister(right), g.UseImmediate(left),
1757 cont);
1758 } else {
1759 VisitCompare(selector, opcode, g.UseRegister(left), g.UseRegister(right),
1760 cont);
1761 }
1762 }
1763
1764 // This function checks whether we can convert:
1765 // ((a <op> b) cmp 0), b.<cond>
1766 // to:
1767 // (a <ops> b), b.<cond'>
1768 // where <ops> is the flag setting version of <op>.
1769 // We only generate conditions <cond'> that are a combination of the N
1770 // and Z flags. This avoids the need to make this function dependent on
1771 // the flag-setting operation.
CanUseFlagSettingBinop(FlagsCondition cond)1772 bool CanUseFlagSettingBinop(FlagsCondition cond) {
1773 switch (cond) {
1774 case kEqual:
1775 case kNotEqual:
1776 case kSignedLessThan:
1777 case kSignedGreaterThanOrEqual:
1778 case kUnsignedLessThanOrEqual: // x <= 0 -> x == 0
1779 case kUnsignedGreaterThan: // x > 0 -> x != 0
1780 return true;
1781 default:
1782 return false;
1783 }
1784 }
1785
1786 // Map <cond> to <cond'> so that the following transformation is possible:
1787 // ((a <op> b) cmp 0), b.<cond>
1788 // to:
1789 // (a <ops> b), b.<cond'>
1790 // where <ops> is the flag setting version of <op>.
MapForFlagSettingBinop(FlagsCondition cond)1791 FlagsCondition MapForFlagSettingBinop(FlagsCondition cond) {
1792 DCHECK(CanUseFlagSettingBinop(cond));
1793 switch (cond) {
1794 case kEqual:
1795 case kNotEqual:
1796 return cond;
1797 case kSignedLessThan:
1798 return kNegative;
1799 case kSignedGreaterThanOrEqual:
1800 return kPositiveOrZero;
1801 case kUnsignedLessThanOrEqual: // x <= 0 -> x == 0
1802 return kEqual;
1803 case kUnsignedGreaterThan: // x > 0 -> x != 0
1804 return kNotEqual;
1805 default:
1806 UNREACHABLE();
1807 }
1808 }
1809
1810 // This function checks if we can perform the transformation:
1811 // ((a <op> b) cmp 0), b.<cond>
1812 // to:
1813 // (a <ops> b), b.<cond'>
1814 // where <ops> is the flag setting version of <op>, and if so,
1815 // updates {node}, {opcode} and {cont} accordingly.
MaybeReplaceCmpZeroWithFlagSettingBinop(InstructionSelector * selector,Node ** node,Node * binop,ArchOpcode * opcode,FlagsCondition cond,FlagsContinuation * cont,ImmediateMode * immediate_mode)1816 void MaybeReplaceCmpZeroWithFlagSettingBinop(InstructionSelector* selector,
1817 Node** node, Node* binop,
1818 ArchOpcode* opcode,
1819 FlagsCondition cond,
1820 FlagsContinuation* cont,
1821 ImmediateMode* immediate_mode) {
1822 ArchOpcode binop_opcode;
1823 ArchOpcode no_output_opcode;
1824 ImmediateMode binop_immediate_mode;
1825 switch (binop->opcode()) {
1826 case IrOpcode::kInt32Add:
1827 binop_opcode = kArm64Add32;
1828 no_output_opcode = kArm64Cmn32;
1829 binop_immediate_mode = kArithmeticImm;
1830 break;
1831 case IrOpcode::kWord32And:
1832 binop_opcode = kArm64And32;
1833 no_output_opcode = kArm64Tst32;
1834 binop_immediate_mode = kLogical32Imm;
1835 break;
1836 default:
1837 UNREACHABLE();
1838 return;
1839 }
1840 if (selector->CanCover(*node, binop)) {
1841 // The comparison is the only user of the add or and, so we can generate
1842 // a cmn or tst instead.
1843 cont->Overwrite(MapForFlagSettingBinop(cond));
1844 *opcode = no_output_opcode;
1845 *node = binop;
1846 *immediate_mode = binop_immediate_mode;
1847 } else if (selector->IsOnlyUserOfNodeInSameBlock(*node, binop)) {
1848 // We can also handle the case where the add and the compare are in the
1849 // same basic block, and the compare is the only use of add in this basic
1850 // block (the add has users in other basic blocks).
1851 cont->Overwrite(MapForFlagSettingBinop(cond));
1852 *opcode = binop_opcode;
1853 *node = binop;
1854 *immediate_mode = binop_immediate_mode;
1855 }
1856 }
1857
1858 // Map {cond} to kEqual or kNotEqual, so that we can select
1859 // either TBZ or TBNZ when generating code for:
1860 // (x cmp 0), b.{cond}
MapForTbz(FlagsCondition cond)1861 FlagsCondition MapForTbz(FlagsCondition cond) {
1862 switch (cond) {
1863 case kSignedLessThan: // generate TBNZ
1864 return kNotEqual;
1865 case kSignedGreaterThanOrEqual: // generate TBZ
1866 return kEqual;
1867 default:
1868 UNREACHABLE();
1869 }
1870 }
1871
1872 // Map {cond} to kEqual or kNotEqual, so that we can select
1873 // either CBZ or CBNZ when generating code for:
1874 // (x cmp 0), b.{cond}
MapForCbz(FlagsCondition cond)1875 FlagsCondition MapForCbz(FlagsCondition cond) {
1876 switch (cond) {
1877 case kEqual: // generate CBZ
1878 case kNotEqual: // generate CBNZ
1879 return cond;
1880 case kUnsignedLessThanOrEqual: // generate CBZ
1881 return kEqual;
1882 case kUnsignedGreaterThan: // generate CBNZ
1883 return kNotEqual;
1884 default:
1885 UNREACHABLE();
1886 }
1887 }
1888
EmitBranchOrDeoptimize(InstructionSelector * selector,InstructionCode opcode,InstructionOperand value,FlagsContinuation * cont)1889 void EmitBranchOrDeoptimize(InstructionSelector* selector,
1890 InstructionCode opcode, InstructionOperand value,
1891 FlagsContinuation* cont) {
1892 DCHECK(cont->IsBranch() || cont->IsDeoptimize());
1893 selector->EmitWithContinuation(opcode, value, cont);
1894 }
1895
1896 // Try to emit TBZ, TBNZ, CBZ or CBNZ for certain comparisons of {node}
1897 // against {value}, depending on the condition.
TryEmitCbzOrTbz(InstructionSelector * selector,Node * node,uint32_t value,Node * user,FlagsCondition cond,FlagsContinuation * cont)1898 bool TryEmitCbzOrTbz(InstructionSelector* selector, Node* node, uint32_t value,
1899 Node* user, FlagsCondition cond, FlagsContinuation* cont) {
1900 // Branch poisoning requires flags to be set, so when it's enabled for
1901 // a particular branch, we shouldn't be applying the cbz/tbz optimization.
1902 DCHECK(!cont->IsPoisoned());
1903 // Only handle branches and deoptimisations.
1904 if (!cont->IsBranch() && !cont->IsDeoptimize()) return false;
1905
1906 switch (cond) {
1907 case kSignedLessThan:
1908 case kSignedGreaterThanOrEqual: {
1909 // Here we handle sign tests, aka. comparisons with zero.
1910 if (value != 0) return false;
1911 // We don't generate TBZ/TBNZ for deoptimisations, as they have a
1912 // shorter range than conditional branches and generating them for
1913 // deoptimisations results in more veneers.
1914 if (cont->IsDeoptimize()) return false;
1915 Arm64OperandGenerator g(selector);
1916 cont->Overwrite(MapForTbz(cond));
1917 Int32Matcher m(node);
1918 if (m.IsFloat64ExtractHighWord32() && selector->CanCover(user, node)) {
1919 // SignedLessThan(Float64ExtractHighWord32(x), 0) and
1920 // SignedGreaterThanOrEqual(Float64ExtractHighWord32(x), 0) essentially
1921 // check the sign bit of a 64-bit floating point value.
1922 InstructionOperand temp = g.TempRegister();
1923 selector->Emit(kArm64U64MoveFloat64, temp,
1924 g.UseRegister(node->InputAt(0)));
1925 selector->EmitWithContinuation(kArm64TestAndBranch, temp,
1926 g.TempImmediate(63), cont);
1927 return true;
1928 }
1929 selector->EmitWithContinuation(kArm64TestAndBranch32, g.UseRegister(node),
1930 g.TempImmediate(31), cont);
1931 return true;
1932 }
1933 case kEqual:
1934 case kNotEqual: {
1935 if (node->opcode() == IrOpcode::kWord32And) {
1936 // Emit a tbz/tbnz if we are comparing with a single-bit mask:
1937 // Branch(Word32Equal(Word32And(x, 1 << N), 1 << N), true, false)
1938 Int32BinopMatcher m_and(node);
1939 if (cont->IsBranch() && base::bits::IsPowerOfTwo(value) &&
1940 m_and.right().Is(value) && selector->CanCover(user, node)) {
1941 Arm64OperandGenerator g(selector);
1942 // In the code generator, Equal refers to a bit being cleared. We want
1943 // the opposite here so negate the condition.
1944 cont->Negate();
1945 selector->EmitWithContinuation(
1946 kArm64TestAndBranch32, g.UseRegister(m_and.left().node()),
1947 g.TempImmediate(base::bits::CountTrailingZeros(value)), cont);
1948 return true;
1949 }
1950 }
1951 V8_FALLTHROUGH;
1952 }
1953 case kUnsignedLessThanOrEqual:
1954 case kUnsignedGreaterThan: {
1955 if (value != 0) return false;
1956 Arm64OperandGenerator g(selector);
1957 cont->Overwrite(MapForCbz(cond));
1958 EmitBranchOrDeoptimize(selector, kArm64CompareAndBranch32,
1959 g.UseRegister(node), cont);
1960 return true;
1961 }
1962 default:
1963 return false;
1964 }
1965 }
1966
VisitWord32Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)1967 void VisitWord32Compare(InstructionSelector* selector, Node* node,
1968 FlagsContinuation* cont) {
1969 Int32BinopMatcher m(node);
1970 FlagsCondition cond = cont->condition();
1971 if (!cont->IsPoisoned()) {
1972 if (m.right().HasValue()) {
1973 if (TryEmitCbzOrTbz(selector, m.left().node(), m.right().Value(), node,
1974 cond, cont)) {
1975 return;
1976 }
1977 } else if (m.left().HasValue()) {
1978 FlagsCondition commuted_cond = CommuteFlagsCondition(cond);
1979 if (TryEmitCbzOrTbz(selector, m.right().node(), m.left().Value(), node,
1980 commuted_cond, cont)) {
1981 return;
1982 }
1983 }
1984 }
1985 ArchOpcode opcode = kArm64Cmp32;
1986 ImmediateMode immediate_mode = kArithmeticImm;
1987 if (m.right().Is(0) && (m.left().IsInt32Add() || m.left().IsWord32And())) {
1988 // Emit flag setting add/and instructions for comparisons against zero.
1989 if (CanUseFlagSettingBinop(cond)) {
1990 Node* binop = m.left().node();
1991 MaybeReplaceCmpZeroWithFlagSettingBinop(selector, &node, binop, &opcode,
1992 cond, cont, &immediate_mode);
1993 }
1994 } else if (m.left().Is(0) &&
1995 (m.right().IsInt32Add() || m.right().IsWord32And())) {
1996 // Same as above, but we need to commute the condition before we
1997 // continue with the rest of the checks.
1998 FlagsCondition commuted_cond = CommuteFlagsCondition(cond);
1999 if (CanUseFlagSettingBinop(commuted_cond)) {
2000 Node* binop = m.right().node();
2001 MaybeReplaceCmpZeroWithFlagSettingBinop(selector, &node, binop, &opcode,
2002 commuted_cond, cont,
2003 &immediate_mode);
2004 }
2005 } else if (m.right().IsInt32Sub() && (cond == kEqual || cond == kNotEqual)) {
2006 // Select negated compare for comparisons with negated right input.
2007 // Only do this for kEqual and kNotEqual, which do not depend on the
2008 // C and V flags, as those flags will be different with CMN when the
2009 // right-hand side of the original subtraction is INT_MIN.
2010 Node* sub = m.right().node();
2011 Int32BinopMatcher msub(sub);
2012 if (msub.left().Is(0)) {
2013 bool can_cover = selector->CanCover(node, sub);
2014 node->ReplaceInput(1, msub.right().node());
2015 // Even if the comparison node covers the subtraction, after the input
2016 // replacement above, the node still won't cover the input to the
2017 // subtraction; the subtraction still uses it.
2018 // In order to get shifted operations to work, we must remove the rhs
2019 // input to the subtraction, as TryMatchAnyShift requires this node to
2020 // cover the input shift. We do this by setting it to the lhs input,
2021 // as we know it's zero, and the result of the subtraction isn't used by
2022 // any other node.
2023 if (can_cover) sub->ReplaceInput(1, msub.left().node());
2024 opcode = kArm64Cmn32;
2025 }
2026 }
2027 VisitBinop<Int32BinopMatcher>(selector, node, opcode, immediate_mode, cont);
2028 }
2029
2030
VisitWordTest(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)2031 void VisitWordTest(InstructionSelector* selector, Node* node,
2032 InstructionCode opcode, FlagsContinuation* cont) {
2033 Arm64OperandGenerator g(selector);
2034 VisitCompare(selector, opcode, g.UseRegister(node), g.UseRegister(node),
2035 cont);
2036 }
2037
2038
VisitWord32Test(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2039 void VisitWord32Test(InstructionSelector* selector, Node* node,
2040 FlagsContinuation* cont) {
2041 VisitWordTest(selector, node, kArm64Tst32, cont);
2042 }
2043
2044
VisitWord64Test(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2045 void VisitWord64Test(InstructionSelector* selector, Node* node,
2046 FlagsContinuation* cont) {
2047 VisitWordTest(selector, node, kArm64Tst, cont);
2048 }
2049
2050 template <typename Matcher, ArchOpcode kOpcode>
TryEmitTestAndBranch(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2051 bool TryEmitTestAndBranch(InstructionSelector* selector, Node* node,
2052 FlagsContinuation* cont) {
2053 Arm64OperandGenerator g(selector);
2054 Matcher m(node);
2055 if (cont->IsBranch() && !cont->IsPoisoned() && m.right().HasValue() &&
2056 base::bits::IsPowerOfTwo(m.right().Value())) {
2057 // If the mask has only one bit set, we can use tbz/tbnz.
2058 DCHECK((cont->condition() == kEqual) || (cont->condition() == kNotEqual));
2059 selector->EmitWithContinuation(
2060 kOpcode, g.UseRegister(m.left().node()),
2061 g.TempImmediate(base::bits::CountTrailingZeros(m.right().Value())),
2062 cont);
2063 return true;
2064 }
2065 return false;
2066 }
2067
2068 // Shared routine for multiple float32 compare operations.
VisitFloat32Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2069 void VisitFloat32Compare(InstructionSelector* selector, Node* node,
2070 FlagsContinuation* cont) {
2071 Arm64OperandGenerator g(selector);
2072 Float32BinopMatcher m(node);
2073 if (m.right().Is(0.0f)) {
2074 VisitCompare(selector, kArm64Float32Cmp, g.UseRegister(m.left().node()),
2075 g.UseImmediate(m.right().node()), cont);
2076 } else if (m.left().Is(0.0f)) {
2077 cont->Commute();
2078 VisitCompare(selector, kArm64Float32Cmp, g.UseRegister(m.right().node()),
2079 g.UseImmediate(m.left().node()), cont);
2080 } else {
2081 VisitCompare(selector, kArm64Float32Cmp, g.UseRegister(m.left().node()),
2082 g.UseRegister(m.right().node()), cont);
2083 }
2084 }
2085
2086
2087 // Shared routine for multiple float64 compare operations.
VisitFloat64Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2088 void VisitFloat64Compare(InstructionSelector* selector, Node* node,
2089 FlagsContinuation* cont) {
2090 Arm64OperandGenerator g(selector);
2091 Float64BinopMatcher m(node);
2092 if (m.right().Is(0.0)) {
2093 VisitCompare(selector, kArm64Float64Cmp, g.UseRegister(m.left().node()),
2094 g.UseImmediate(m.right().node()), cont);
2095 } else if (m.left().Is(0.0)) {
2096 cont->Commute();
2097 VisitCompare(selector, kArm64Float64Cmp, g.UseRegister(m.right().node()),
2098 g.UseImmediate(m.left().node()), cont);
2099 } else {
2100 VisitCompare(selector, kArm64Float64Cmp, g.UseRegister(m.left().node()),
2101 g.UseRegister(m.right().node()), cont);
2102 }
2103 }
2104
VisitAtomicExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode)2105 void VisitAtomicExchange(InstructionSelector* selector, Node* node,
2106 ArchOpcode opcode) {
2107 Arm64OperandGenerator g(selector);
2108 Node* base = node->InputAt(0);
2109 Node* index = node->InputAt(1);
2110 Node* value = node->InputAt(2);
2111 InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index),
2112 g.UseUniqueRegister(value)};
2113 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
2114 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
2115 InstructionCode code = opcode | AddressingModeField::encode(kMode_MRR);
2116 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
2117 arraysize(temps), temps);
2118 }
2119
VisitAtomicCompareExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode)2120 void VisitAtomicCompareExchange(InstructionSelector* selector, Node* node,
2121 ArchOpcode opcode) {
2122 Arm64OperandGenerator g(selector);
2123 Node* base = node->InputAt(0);
2124 Node* index = node->InputAt(1);
2125 Node* old_value = node->InputAt(2);
2126 Node* new_value = node->InputAt(3);
2127 InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index),
2128 g.UseUniqueRegister(old_value),
2129 g.UseUniqueRegister(new_value)};
2130 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
2131 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
2132 InstructionCode code = opcode | AddressingModeField::encode(kMode_MRR);
2133 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
2134 arraysize(temps), temps);
2135 }
2136
VisitAtomicLoad(InstructionSelector * selector,Node * node,ArchOpcode opcode)2137 void VisitAtomicLoad(InstructionSelector* selector, Node* node,
2138 ArchOpcode opcode) {
2139 Arm64OperandGenerator g(selector);
2140 Node* base = node->InputAt(0);
2141 Node* index = node->InputAt(1);
2142 InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index)};
2143 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
2144 InstructionOperand temps[] = {g.TempRegister()};
2145 InstructionCode code = opcode | AddressingModeField::encode(kMode_MRR);
2146 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
2147 arraysize(temps), temps);
2148 }
2149
VisitAtomicStore(InstructionSelector * selector,Node * node,ArchOpcode opcode)2150 void VisitAtomicStore(InstructionSelector* selector, Node* node,
2151 ArchOpcode opcode) {
2152 Arm64OperandGenerator g(selector);
2153 Node* base = node->InputAt(0);
2154 Node* index = node->InputAt(1);
2155 Node* value = node->InputAt(2);
2156 InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index),
2157 g.UseUniqueRegister(value)};
2158 InstructionOperand temps[] = {g.TempRegister()};
2159 InstructionCode code = opcode | AddressingModeField::encode(kMode_MRR);
2160 selector->Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps),
2161 temps);
2162 }
2163
VisitAtomicBinop(InstructionSelector * selector,Node * node,ArchOpcode opcode)2164 void VisitAtomicBinop(InstructionSelector* selector, Node* node,
2165 ArchOpcode opcode) {
2166 Arm64OperandGenerator g(selector);
2167 Node* base = node->InputAt(0);
2168 Node* index = node->InputAt(1);
2169 Node* value = node->InputAt(2);
2170 AddressingMode addressing_mode = kMode_MRR;
2171 InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index),
2172 g.UseUniqueRegister(value)};
2173 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
2174 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
2175 g.TempRegister()};
2176 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
2177 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
2178 arraysize(temps), temps);
2179 }
2180
2181 } // namespace
2182
VisitWordCompareZero(Node * user,Node * value,FlagsContinuation * cont)2183 void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
2184 FlagsContinuation* cont) {
2185 Arm64OperandGenerator g(this);
2186 // Try to combine with comparisons against 0 by simply inverting the branch.
2187 while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
2188 Int32BinopMatcher m(value);
2189 if (!m.right().Is(0)) break;
2190
2191 user = value;
2192 value = m.left().node();
2193 cont->Negate();
2194 }
2195
2196 if (CanCover(user, value)) {
2197 switch (value->opcode()) {
2198 case IrOpcode::kWord32Equal:
2199 cont->OverwriteAndNegateIfEqual(kEqual);
2200 return VisitWord32Compare(this, value, cont);
2201 case IrOpcode::kInt32LessThan:
2202 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2203 return VisitWord32Compare(this, value, cont);
2204 case IrOpcode::kInt32LessThanOrEqual:
2205 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2206 return VisitWord32Compare(this, value, cont);
2207 case IrOpcode::kUint32LessThan:
2208 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2209 return VisitWord32Compare(this, value, cont);
2210 case IrOpcode::kUint32LessThanOrEqual:
2211 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2212 return VisitWord32Compare(this, value, cont);
2213 case IrOpcode::kWord64Equal: {
2214 cont->OverwriteAndNegateIfEqual(kEqual);
2215 Int64BinopMatcher m(value);
2216 if (m.right().Is(0)) {
2217 Node* const left = m.left().node();
2218 if (CanCover(value, left) && left->opcode() == IrOpcode::kWord64And) {
2219 // Attempt to merge the Word64Equal(Word64And(x, y), 0) comparison
2220 // into a tbz/tbnz instruction.
2221 if (TryEmitTestAndBranch<Uint64BinopMatcher, kArm64TestAndBranch>(
2222 this, left, cont)) {
2223 return;
2224 }
2225 return VisitWordCompare(this, left, kArm64Tst, cont, true,
2226 kLogical64Imm);
2227 }
2228 // Merge the Word64Equal(x, 0) comparison into a cbz instruction.
2229 if ((cont->IsBranch() || cont->IsDeoptimize()) &&
2230 !cont->IsPoisoned()) {
2231 EmitBranchOrDeoptimize(this, kArm64CompareAndBranch,
2232 g.UseRegister(left), cont);
2233 return;
2234 }
2235 }
2236 return VisitWordCompare(this, value, kArm64Cmp, cont, false,
2237 kArithmeticImm);
2238 }
2239 case IrOpcode::kInt64LessThan:
2240 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2241 return VisitWordCompare(this, value, kArm64Cmp, cont, false,
2242 kArithmeticImm);
2243 case IrOpcode::kInt64LessThanOrEqual:
2244 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2245 return VisitWordCompare(this, value, kArm64Cmp, cont, false,
2246 kArithmeticImm);
2247 case IrOpcode::kUint64LessThan:
2248 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2249 return VisitWordCompare(this, value, kArm64Cmp, cont, false,
2250 kArithmeticImm);
2251 case IrOpcode::kUint64LessThanOrEqual:
2252 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2253 return VisitWordCompare(this, value, kArm64Cmp, cont, false,
2254 kArithmeticImm);
2255 case IrOpcode::kFloat32Equal:
2256 cont->OverwriteAndNegateIfEqual(kEqual);
2257 return VisitFloat32Compare(this, value, cont);
2258 case IrOpcode::kFloat32LessThan:
2259 cont->OverwriteAndNegateIfEqual(kFloatLessThan);
2260 return VisitFloat32Compare(this, value, cont);
2261 case IrOpcode::kFloat32LessThanOrEqual:
2262 cont->OverwriteAndNegateIfEqual(kFloatLessThanOrEqual);
2263 return VisitFloat32Compare(this, value, cont);
2264 case IrOpcode::kFloat64Equal:
2265 cont->OverwriteAndNegateIfEqual(kEqual);
2266 return VisitFloat64Compare(this, value, cont);
2267 case IrOpcode::kFloat64LessThan:
2268 cont->OverwriteAndNegateIfEqual(kFloatLessThan);
2269 return VisitFloat64Compare(this, value, cont);
2270 case IrOpcode::kFloat64LessThanOrEqual:
2271 cont->OverwriteAndNegateIfEqual(kFloatLessThanOrEqual);
2272 return VisitFloat64Compare(this, value, cont);
2273 case IrOpcode::kProjection:
2274 // Check if this is the overflow output projection of an
2275 // <Operation>WithOverflow node.
2276 if (ProjectionIndexOf(value->op()) == 1u) {
2277 // We cannot combine the <Operation>WithOverflow with this branch
2278 // unless the 0th projection (the use of the actual value of the
2279 // <Operation> is either nullptr, which means there's no use of the
2280 // actual value, or was already defined, which means it is scheduled
2281 // *AFTER* this branch).
2282 Node* const node = value->InputAt(0);
2283 Node* const result = NodeProperties::FindProjection(node, 0);
2284 if (result == nullptr || IsDefined(result)) {
2285 switch (node->opcode()) {
2286 case IrOpcode::kInt32AddWithOverflow:
2287 cont->OverwriteAndNegateIfEqual(kOverflow);
2288 return VisitBinop<Int32BinopMatcher>(this, node, kArm64Add32,
2289 kArithmeticImm, cont);
2290 case IrOpcode::kInt32SubWithOverflow:
2291 cont->OverwriteAndNegateIfEqual(kOverflow);
2292 return VisitBinop<Int32BinopMatcher>(this, node, kArm64Sub32,
2293 kArithmeticImm, cont);
2294 case IrOpcode::kInt32MulWithOverflow:
2295 // ARM64 doesn't set the overflow flag for multiplication, so we
2296 // need to test on kNotEqual. Here is the code sequence used:
2297 // smull result, left, right
2298 // cmp result.X(), Operand(result, SXTW)
2299 cont->OverwriteAndNegateIfEqual(kNotEqual);
2300 return EmitInt32MulWithOverflow(this, node, cont);
2301 case IrOpcode::kInt64AddWithOverflow:
2302 cont->OverwriteAndNegateIfEqual(kOverflow);
2303 return VisitBinop<Int64BinopMatcher>(this, node, kArm64Add,
2304 kArithmeticImm, cont);
2305 case IrOpcode::kInt64SubWithOverflow:
2306 cont->OverwriteAndNegateIfEqual(kOverflow);
2307 return VisitBinop<Int64BinopMatcher>(this, node, kArm64Sub,
2308 kArithmeticImm, cont);
2309 default:
2310 break;
2311 }
2312 }
2313 }
2314 break;
2315 case IrOpcode::kInt32Add:
2316 return VisitWordCompare(this, value, kArm64Cmn32, cont, true,
2317 kArithmeticImm);
2318 case IrOpcode::kInt32Sub:
2319 return VisitWord32Compare(this, value, cont);
2320 case IrOpcode::kWord32And:
2321 if (TryEmitTestAndBranch<Uint32BinopMatcher, kArm64TestAndBranch32>(
2322 this, value, cont)) {
2323 return;
2324 }
2325 return VisitWordCompare(this, value, kArm64Tst32, cont, true,
2326 kLogical32Imm);
2327 case IrOpcode::kWord64And:
2328 if (TryEmitTestAndBranch<Uint64BinopMatcher, kArm64TestAndBranch>(
2329 this, value, cont)) {
2330 return;
2331 }
2332 return VisitWordCompare(this, value, kArm64Tst, cont, true,
2333 kLogical64Imm);
2334 default:
2335 break;
2336 }
2337 }
2338
2339 // Branch could not be combined with a compare, compare against 0 and branch.
2340 if (!cont->IsPoisoned() && cont->IsBranch()) {
2341 Emit(cont->Encode(kArm64CompareAndBranch32), g.NoOutput(),
2342 g.UseRegister(value), g.Label(cont->true_block()),
2343 g.Label(cont->false_block()));
2344 } else {
2345 EmitWithContinuation(cont->Encode(kArm64Tst32), g.UseRegister(value),
2346 g.UseRegister(value), cont);
2347 }
2348 }
2349
VisitSwitch(Node * node,const SwitchInfo & sw)2350 void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
2351 Arm64OperandGenerator g(this);
2352 InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
2353
2354 // Emit either ArchTableSwitch or ArchLookupSwitch.
2355 if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
2356 static const size_t kMaxTableSwitchValueRange = 2 << 16;
2357 size_t table_space_cost = 4 + sw.value_range();
2358 size_t table_time_cost = 3;
2359 size_t lookup_space_cost = 3 + 2 * sw.case_count();
2360 size_t lookup_time_cost = sw.case_count();
2361 if (sw.case_count() > 0 &&
2362 table_space_cost + 3 * table_time_cost <=
2363 lookup_space_cost + 3 * lookup_time_cost &&
2364 sw.min_value() > std::numeric_limits<int32_t>::min() &&
2365 sw.value_range() <= kMaxTableSwitchValueRange) {
2366 InstructionOperand index_operand = value_operand;
2367 if (sw.min_value()) {
2368 index_operand = g.TempRegister();
2369 Emit(kArm64Sub32, index_operand, value_operand,
2370 g.TempImmediate(sw.min_value()));
2371 }
2372 // Generate a table lookup.
2373 return EmitTableSwitch(sw, index_operand);
2374 }
2375 }
2376
2377 // Generate a sequence of conditional jumps.
2378 return EmitLookupSwitch(sw, value_operand);
2379 }
2380
2381
VisitWord32Equal(Node * const node)2382 void InstructionSelector::VisitWord32Equal(Node* const node) {
2383 Node* const user = node;
2384 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2385 Int32BinopMatcher m(user);
2386 if (m.right().Is(0)) {
2387 Node* const value = m.left().node();
2388 if (CanCover(user, value)) {
2389 switch (value->opcode()) {
2390 case IrOpcode::kInt32Add:
2391 case IrOpcode::kWord32And:
2392 return VisitWord32Compare(this, node, &cont);
2393 case IrOpcode::kInt32Sub:
2394 return VisitWordCompare(this, value, kArm64Cmp32, &cont, false,
2395 kArithmeticImm);
2396 case IrOpcode::kWord32Equal: {
2397 // Word32Equal(Word32Equal(x, y), 0) => Word32Compare(x, y, ne).
2398 Int32BinopMatcher mequal(value);
2399 node->ReplaceInput(0, mequal.left().node());
2400 node->ReplaceInput(1, mequal.right().node());
2401 cont.Negate();
2402 // {node} still does not cover its new operands, because {mequal} is
2403 // still using them.
2404 // Since we won't generate any more code for {mequal}, set its
2405 // operands to zero to make sure {node} can cover them.
2406 // This improves pattern matching in VisitWord32Compare.
2407 mequal.node()->ReplaceInput(0, m.right().node());
2408 mequal.node()->ReplaceInput(1, m.right().node());
2409 return VisitWord32Compare(this, node, &cont);
2410 }
2411 default:
2412 break;
2413 }
2414 return VisitWord32Test(this, value, &cont);
2415 }
2416 }
2417 VisitWord32Compare(this, node, &cont);
2418 }
2419
2420
VisitInt32LessThan(Node * node)2421 void InstructionSelector::VisitInt32LessThan(Node* node) {
2422 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2423 VisitWord32Compare(this, node, &cont);
2424 }
2425
2426
VisitInt32LessThanOrEqual(Node * node)2427 void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
2428 FlagsContinuation cont =
2429 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2430 VisitWord32Compare(this, node, &cont);
2431 }
2432
2433
VisitUint32LessThan(Node * node)2434 void InstructionSelector::VisitUint32LessThan(Node* node) {
2435 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2436 VisitWord32Compare(this, node, &cont);
2437 }
2438
2439
VisitUint32LessThanOrEqual(Node * node)2440 void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
2441 FlagsContinuation cont =
2442 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2443 VisitWord32Compare(this, node, &cont);
2444 }
2445
2446
VisitWord64Equal(Node * const node)2447 void InstructionSelector::VisitWord64Equal(Node* const node) {
2448 Node* const user = node;
2449 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2450 Int64BinopMatcher m(user);
2451 if (m.right().Is(0)) {
2452 Node* const value = m.left().node();
2453 if (CanCover(user, value)) {
2454 switch (value->opcode()) {
2455 case IrOpcode::kWord64And:
2456 return VisitWordCompare(this, value, kArm64Tst, &cont, true,
2457 kLogical64Imm);
2458 default:
2459 break;
2460 }
2461 return VisitWord64Test(this, value, &cont);
2462 }
2463 }
2464 VisitWordCompare(this, node, kArm64Cmp, &cont, false, kArithmeticImm);
2465 }
2466
2467
VisitInt32AddWithOverflow(Node * node)2468 void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
2469 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2470 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2471 return VisitBinop<Int32BinopMatcher>(this, node, kArm64Add32,
2472 kArithmeticImm, &cont);
2473 }
2474 FlagsContinuation cont;
2475 VisitBinop<Int32BinopMatcher>(this, node, kArm64Add32, kArithmeticImm, &cont);
2476 }
2477
2478
VisitInt32SubWithOverflow(Node * node)2479 void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
2480 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2481 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2482 return VisitBinop<Int32BinopMatcher>(this, node, kArm64Sub32,
2483 kArithmeticImm, &cont);
2484 }
2485 FlagsContinuation cont;
2486 VisitBinop<Int32BinopMatcher>(this, node, kArm64Sub32, kArithmeticImm, &cont);
2487 }
2488
VisitInt32MulWithOverflow(Node * node)2489 void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
2490 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2491 // ARM64 doesn't set the overflow flag for multiplication, so we need to
2492 // test on kNotEqual. Here is the code sequence used:
2493 // smull result, left, right
2494 // cmp result.X(), Operand(result, SXTW)
2495 FlagsContinuation cont = FlagsContinuation::ForSet(kNotEqual, ovf);
2496 return EmitInt32MulWithOverflow(this, node, &cont);
2497 }
2498 FlagsContinuation cont;
2499 EmitInt32MulWithOverflow(this, node, &cont);
2500 }
2501
VisitInt64AddWithOverflow(Node * node)2502 void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
2503 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2504 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2505 return VisitBinop<Int64BinopMatcher>(this, node, kArm64Add, kArithmeticImm,
2506 &cont);
2507 }
2508 FlagsContinuation cont;
2509 VisitBinop<Int64BinopMatcher>(this, node, kArm64Add, kArithmeticImm, &cont);
2510 }
2511
2512
VisitInt64SubWithOverflow(Node * node)2513 void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
2514 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2515 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2516 return VisitBinop<Int64BinopMatcher>(this, node, kArm64Sub, kArithmeticImm,
2517 &cont);
2518 }
2519 FlagsContinuation cont;
2520 VisitBinop<Int64BinopMatcher>(this, node, kArm64Sub, kArithmeticImm, &cont);
2521 }
2522
2523
VisitInt64LessThan(Node * node)2524 void InstructionSelector::VisitInt64LessThan(Node* node) {
2525 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2526 VisitWordCompare(this, node, kArm64Cmp, &cont, false, kArithmeticImm);
2527 }
2528
2529
VisitInt64LessThanOrEqual(Node * node)2530 void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2531 FlagsContinuation cont =
2532 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2533 VisitWordCompare(this, node, kArm64Cmp, &cont, false, kArithmeticImm);
2534 }
2535
2536
VisitUint64LessThan(Node * node)2537 void InstructionSelector::VisitUint64LessThan(Node* node) {
2538 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2539 VisitWordCompare(this, node, kArm64Cmp, &cont, false, kArithmeticImm);
2540 }
2541
2542
VisitUint64LessThanOrEqual(Node * node)2543 void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2544 FlagsContinuation cont =
2545 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2546 VisitWordCompare(this, node, kArm64Cmp, &cont, false, kArithmeticImm);
2547 }
2548
2549
VisitFloat32Equal(Node * node)2550 void InstructionSelector::VisitFloat32Equal(Node* node) {
2551 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2552 VisitFloat32Compare(this, node, &cont);
2553 }
2554
2555
VisitFloat32LessThan(Node * node)2556 void InstructionSelector::VisitFloat32LessThan(Node* node) {
2557 FlagsContinuation cont = FlagsContinuation::ForSet(kFloatLessThan, node);
2558 VisitFloat32Compare(this, node, &cont);
2559 }
2560
2561
VisitFloat32LessThanOrEqual(Node * node)2562 void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
2563 FlagsContinuation cont =
2564 FlagsContinuation::ForSet(kFloatLessThanOrEqual, node);
2565 VisitFloat32Compare(this, node, &cont);
2566 }
2567
2568
VisitFloat64Equal(Node * node)2569 void InstructionSelector::VisitFloat64Equal(Node* node) {
2570 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2571 VisitFloat64Compare(this, node, &cont);
2572 }
2573
2574
VisitFloat64LessThan(Node * node)2575 void InstructionSelector::VisitFloat64LessThan(Node* node) {
2576 FlagsContinuation cont = FlagsContinuation::ForSet(kFloatLessThan, node);
2577 VisitFloat64Compare(this, node, &cont);
2578 }
2579
2580
VisitFloat64LessThanOrEqual(Node * node)2581 void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
2582 FlagsContinuation cont =
2583 FlagsContinuation::ForSet(kFloatLessThanOrEqual, node);
2584 VisitFloat64Compare(this, node, &cont);
2585 }
2586
VisitFloat64InsertLowWord32(Node * node)2587 void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
2588 Arm64OperandGenerator g(this);
2589 Node* left = node->InputAt(0);
2590 Node* right = node->InputAt(1);
2591 if (left->opcode() == IrOpcode::kFloat64InsertHighWord32 &&
2592 CanCover(node, left)) {
2593 Node* right_of_left = left->InputAt(1);
2594 Emit(kArm64Bfi, g.DefineSameAsFirst(right), g.UseRegister(right),
2595 g.UseRegister(right_of_left), g.TempImmediate(32),
2596 g.TempImmediate(32));
2597 Emit(kArm64Float64MoveU64, g.DefineAsRegister(node), g.UseRegister(right));
2598 return;
2599 }
2600 Emit(kArm64Float64InsertLowWord32, g.DefineSameAsFirst(node),
2601 g.UseRegister(left), g.UseRegister(right));
2602 }
2603
2604
VisitFloat64InsertHighWord32(Node * node)2605 void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
2606 Arm64OperandGenerator g(this);
2607 Node* left = node->InputAt(0);
2608 Node* right = node->InputAt(1);
2609 if (left->opcode() == IrOpcode::kFloat64InsertLowWord32 &&
2610 CanCover(node, left)) {
2611 Node* right_of_left = left->InputAt(1);
2612 Emit(kArm64Bfi, g.DefineSameAsFirst(left), g.UseRegister(right_of_left),
2613 g.UseRegister(right), g.TempImmediate(32), g.TempImmediate(32));
2614 Emit(kArm64Float64MoveU64, g.DefineAsRegister(node), g.UseRegister(left));
2615 return;
2616 }
2617 Emit(kArm64Float64InsertHighWord32, g.DefineSameAsFirst(node),
2618 g.UseRegister(left), g.UseRegister(right));
2619 }
2620
VisitWord32AtomicLoad(Node * node)2621 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
2622 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2623 ArchOpcode opcode = kArchNop;
2624 switch (load_rep.representation()) {
2625 case MachineRepresentation::kWord8:
2626 opcode =
2627 load_rep.IsSigned() ? kWord32AtomicLoadInt8 : kWord32AtomicLoadUint8;
2628 break;
2629 case MachineRepresentation::kWord16:
2630 opcode = load_rep.IsSigned() ? kWord32AtomicLoadInt16
2631 : kWord32AtomicLoadUint16;
2632 break;
2633 case MachineRepresentation::kWord32:
2634 opcode = kWord32AtomicLoadWord32;
2635 break;
2636 default:
2637 UNREACHABLE();
2638 return;
2639 }
2640 VisitAtomicLoad(this, node, opcode);
2641 }
2642
VisitWord64AtomicLoad(Node * node)2643 void InstructionSelector::VisitWord64AtomicLoad(Node* node) {
2644 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2645 ArchOpcode opcode = kArchNop;
2646 switch (load_rep.representation()) {
2647 case MachineRepresentation::kWord8:
2648 opcode = kArm64Word64AtomicLoadUint8;
2649 break;
2650 case MachineRepresentation::kWord16:
2651 opcode = kArm64Word64AtomicLoadUint16;
2652 break;
2653 case MachineRepresentation::kWord32:
2654 opcode = kArm64Word64AtomicLoadUint32;
2655 break;
2656 case MachineRepresentation::kWord64:
2657 opcode = kArm64Word64AtomicLoadUint64;
2658 break;
2659 default:
2660 UNREACHABLE();
2661 return;
2662 }
2663 VisitAtomicLoad(this, node, opcode);
2664 }
2665
VisitWord32AtomicStore(Node * node)2666 void InstructionSelector::VisitWord32AtomicStore(Node* node) {
2667 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2668 ArchOpcode opcode = kArchNop;
2669 switch (rep) {
2670 case MachineRepresentation::kWord8:
2671 opcode = kWord32AtomicStoreWord8;
2672 break;
2673 case MachineRepresentation::kWord16:
2674 opcode = kWord32AtomicStoreWord16;
2675 break;
2676 case MachineRepresentation::kWord32:
2677 opcode = kWord32AtomicStoreWord32;
2678 break;
2679 default:
2680 UNREACHABLE();
2681 return;
2682 }
2683 VisitAtomicStore(this, node, opcode);
2684 }
2685
VisitWord64AtomicStore(Node * node)2686 void InstructionSelector::VisitWord64AtomicStore(Node* node) {
2687 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2688 ArchOpcode opcode = kArchNop;
2689 switch (rep) {
2690 case MachineRepresentation::kWord8:
2691 opcode = kArm64Word64AtomicStoreWord8;
2692 break;
2693 case MachineRepresentation::kWord16:
2694 opcode = kArm64Word64AtomicStoreWord16;
2695 break;
2696 case MachineRepresentation::kWord32:
2697 opcode = kArm64Word64AtomicStoreWord32;
2698 break;
2699 case MachineRepresentation::kWord64:
2700 opcode = kArm64Word64AtomicStoreWord64;
2701 break;
2702 default:
2703 UNREACHABLE();
2704 return;
2705 }
2706 VisitAtomicStore(this, node, opcode);
2707 }
2708
VisitWord32AtomicExchange(Node * node)2709 void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
2710 ArchOpcode opcode = kArchNop;
2711 MachineType type = AtomicOpRepresentationOf(node->op());
2712 if (type == MachineType::Int8()) {
2713 opcode = kWord32AtomicExchangeInt8;
2714 } else if (type == MachineType::Uint8()) {
2715 opcode = kWord32AtomicExchangeUint8;
2716 } else if (type == MachineType::Int16()) {
2717 opcode = kWord32AtomicExchangeInt16;
2718 } else if (type == MachineType::Uint16()) {
2719 opcode = kWord32AtomicExchangeUint16;
2720 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2721 opcode = kWord32AtomicExchangeWord32;
2722 } else {
2723 UNREACHABLE();
2724 return;
2725 }
2726 VisitAtomicExchange(this, node, opcode);
2727 }
2728
VisitWord64AtomicExchange(Node * node)2729 void InstructionSelector::VisitWord64AtomicExchange(Node* node) {
2730 ArchOpcode opcode = kArchNop;
2731 MachineType type = AtomicOpRepresentationOf(node->op());
2732 if (type == MachineType::Uint8()) {
2733 opcode = kArm64Word64AtomicExchangeUint8;
2734 } else if (type == MachineType::Uint16()) {
2735 opcode = kArm64Word64AtomicExchangeUint16;
2736 } else if (type == MachineType::Uint32()) {
2737 opcode = kArm64Word64AtomicExchangeUint32;
2738 } else if (type == MachineType::Uint64()) {
2739 opcode = kArm64Word64AtomicExchangeUint64;
2740 } else {
2741 UNREACHABLE();
2742 return;
2743 }
2744 VisitAtomicExchange(this, node, opcode);
2745 }
2746
VisitWord32AtomicCompareExchange(Node * node)2747 void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
2748 ArchOpcode opcode = kArchNop;
2749 MachineType type = AtomicOpRepresentationOf(node->op());
2750 if (type == MachineType::Int8()) {
2751 opcode = kWord32AtomicCompareExchangeInt8;
2752 } else if (type == MachineType::Uint8()) {
2753 opcode = kWord32AtomicCompareExchangeUint8;
2754 } else if (type == MachineType::Int16()) {
2755 opcode = kWord32AtomicCompareExchangeInt16;
2756 } else if (type == MachineType::Uint16()) {
2757 opcode = kWord32AtomicCompareExchangeUint16;
2758 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2759 opcode = kWord32AtomicCompareExchangeWord32;
2760 } else {
2761 UNREACHABLE();
2762 return;
2763 }
2764 VisitAtomicCompareExchange(this, node, opcode);
2765 }
2766
VisitWord64AtomicCompareExchange(Node * node)2767 void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) {
2768 ArchOpcode opcode = kArchNop;
2769 MachineType type = AtomicOpRepresentationOf(node->op());
2770 if (type == MachineType::Uint8()) {
2771 opcode = kArm64Word64AtomicCompareExchangeUint8;
2772 } else if (type == MachineType::Uint16()) {
2773 opcode = kArm64Word64AtomicCompareExchangeUint16;
2774 } else if (type == MachineType::Uint32()) {
2775 opcode = kArm64Word64AtomicCompareExchangeUint32;
2776 } else if (type == MachineType::Uint64()) {
2777 opcode = kArm64Word64AtomicCompareExchangeUint64;
2778 } else {
2779 UNREACHABLE();
2780 return;
2781 }
2782 VisitAtomicCompareExchange(this, node, opcode);
2783 }
2784
VisitWord32AtomicBinaryOperation(Node * node,ArchOpcode int8_op,ArchOpcode uint8_op,ArchOpcode int16_op,ArchOpcode uint16_op,ArchOpcode word32_op)2785 void InstructionSelector::VisitWord32AtomicBinaryOperation(
2786 Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
2787 ArchOpcode uint16_op, ArchOpcode word32_op) {
2788 ArchOpcode opcode = kArchNop;
2789 MachineType type = AtomicOpRepresentationOf(node->op());
2790 if (type == MachineType::Int8()) {
2791 opcode = int8_op;
2792 } else if (type == MachineType::Uint8()) {
2793 opcode = uint8_op;
2794 } else if (type == MachineType::Int16()) {
2795 opcode = int16_op;
2796 } else if (type == MachineType::Uint16()) {
2797 opcode = uint16_op;
2798 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2799 opcode = word32_op;
2800 } else {
2801 UNREACHABLE();
2802 return;
2803 }
2804 VisitAtomicBinop(this, node, opcode);
2805 }
2806
2807 #define VISIT_ATOMIC_BINOP(op) \
2808 void InstructionSelector::VisitWord32Atomic##op(Node* node) { \
2809 VisitWord32AtomicBinaryOperation( \
2810 node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \
2811 kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \
2812 kWord32Atomic##op##Word32); \
2813 }
2814 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)2815 VISIT_ATOMIC_BINOP(Sub)
2816 VISIT_ATOMIC_BINOP(And)
2817 VISIT_ATOMIC_BINOP(Or)
2818 VISIT_ATOMIC_BINOP(Xor)
2819 #undef VISIT_ATOMIC_BINOP
2820
2821 void InstructionSelector::VisitWord64AtomicBinaryOperation(
2822 Node* node, ArchOpcode uint8_op, ArchOpcode uint16_op, ArchOpcode uint32_op,
2823 ArchOpcode uint64_op) {
2824 ArchOpcode opcode = kArchNop;
2825 MachineType type = AtomicOpRepresentationOf(node->op());
2826 if (type == MachineType::Uint8()) {
2827 opcode = uint8_op;
2828 } else if (type == MachineType::Uint16()) {
2829 opcode = uint16_op;
2830 } else if (type == MachineType::Uint32()) {
2831 opcode = uint32_op;
2832 } else if (type == MachineType::Uint64()) {
2833 opcode = uint64_op;
2834 } else {
2835 UNREACHABLE();
2836 return;
2837 }
2838 VisitAtomicBinop(this, node, opcode);
2839 }
2840
2841 #define VISIT_ATOMIC_BINOP(op) \
2842 void InstructionSelector::VisitWord64Atomic##op(Node* node) { \
2843 VisitWord64AtomicBinaryOperation( \
2844 node, kArm64Word64Atomic##op##Uint8, kArm64Word64Atomic##op##Uint16, \
2845 kArm64Word64Atomic##op##Uint32, kArm64Word64Atomic##op##Uint64); \
2846 }
2847 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)2848 VISIT_ATOMIC_BINOP(Sub)
2849 VISIT_ATOMIC_BINOP(And)
2850 VISIT_ATOMIC_BINOP(Or)
2851 VISIT_ATOMIC_BINOP(Xor)
2852 #undef VISIT_ATOMIC_BINOP
2853
2854 void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
2855 UNREACHABLE();
2856 }
2857
VisitInt64AbsWithOverflow(Node * node)2858 void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
2859 UNREACHABLE();
2860 }
2861
2862 #define SIMD_TYPE_LIST(V) \
2863 V(F32x4) \
2864 V(I32x4) \
2865 V(I16x8) \
2866 V(I8x16)
2867
2868 #define SIMD_UNOP_LIST(V) \
2869 V(F32x4SConvertI32x4, kArm64F32x4SConvertI32x4) \
2870 V(F32x4UConvertI32x4, kArm64F32x4UConvertI32x4) \
2871 V(F32x4Abs, kArm64F32x4Abs) \
2872 V(F32x4Neg, kArm64F32x4Neg) \
2873 V(F32x4RecipApprox, kArm64F32x4RecipApprox) \
2874 V(F32x4RecipSqrtApprox, kArm64F32x4RecipSqrtApprox) \
2875 V(I32x4SConvertF32x4, kArm64I32x4SConvertF32x4) \
2876 V(I32x4SConvertI16x8Low, kArm64I32x4SConvertI16x8Low) \
2877 V(I32x4SConvertI16x8High, kArm64I32x4SConvertI16x8High) \
2878 V(I32x4Neg, kArm64I32x4Neg) \
2879 V(I32x4UConvertF32x4, kArm64I32x4UConvertF32x4) \
2880 V(I32x4UConvertI16x8Low, kArm64I32x4UConvertI16x8Low) \
2881 V(I32x4UConvertI16x8High, kArm64I32x4UConvertI16x8High) \
2882 V(I16x8SConvertI8x16Low, kArm64I16x8SConvertI8x16Low) \
2883 V(I16x8SConvertI8x16High, kArm64I16x8SConvertI8x16High) \
2884 V(I16x8Neg, kArm64I16x8Neg) \
2885 V(I16x8UConvertI8x16Low, kArm64I16x8UConvertI8x16Low) \
2886 V(I16x8UConvertI8x16High, kArm64I16x8UConvertI8x16High) \
2887 V(I8x16Neg, kArm64I8x16Neg) \
2888 V(S128Not, kArm64S128Not) \
2889 V(S1x4AnyTrue, kArm64S1x4AnyTrue) \
2890 V(S1x4AllTrue, kArm64S1x4AllTrue) \
2891 V(S1x8AnyTrue, kArm64S1x8AnyTrue) \
2892 V(S1x8AllTrue, kArm64S1x8AllTrue) \
2893 V(S1x16AnyTrue, kArm64S1x16AnyTrue) \
2894 V(S1x16AllTrue, kArm64S1x16AllTrue)
2895
2896 #define SIMD_SHIFT_OP_LIST(V) \
2897 V(I32x4Shl) \
2898 V(I32x4ShrS) \
2899 V(I32x4ShrU) \
2900 V(I16x8Shl) \
2901 V(I16x8ShrS) \
2902 V(I16x8ShrU) \
2903 V(I8x16Shl) \
2904 V(I8x16ShrS) \
2905 V(I8x16ShrU)
2906
2907 #define SIMD_BINOP_LIST(V) \
2908 V(F32x4Add, kArm64F32x4Add) \
2909 V(F32x4AddHoriz, kArm64F32x4AddHoriz) \
2910 V(F32x4Sub, kArm64F32x4Sub) \
2911 V(F32x4Mul, kArm64F32x4Mul) \
2912 V(F32x4Min, kArm64F32x4Min) \
2913 V(F32x4Max, kArm64F32x4Max) \
2914 V(F32x4Eq, kArm64F32x4Eq) \
2915 V(F32x4Ne, kArm64F32x4Ne) \
2916 V(F32x4Lt, kArm64F32x4Lt) \
2917 V(F32x4Le, kArm64F32x4Le) \
2918 V(I32x4Add, kArm64I32x4Add) \
2919 V(I32x4AddHoriz, kArm64I32x4AddHoriz) \
2920 V(I32x4Sub, kArm64I32x4Sub) \
2921 V(I32x4Mul, kArm64I32x4Mul) \
2922 V(I32x4MinS, kArm64I32x4MinS) \
2923 V(I32x4MaxS, kArm64I32x4MaxS) \
2924 V(I32x4Eq, kArm64I32x4Eq) \
2925 V(I32x4Ne, kArm64I32x4Ne) \
2926 V(I32x4GtS, kArm64I32x4GtS) \
2927 V(I32x4GeS, kArm64I32x4GeS) \
2928 V(I32x4MinU, kArm64I32x4MinU) \
2929 V(I32x4MaxU, kArm64I32x4MaxU) \
2930 V(I32x4GtU, kArm64I32x4GtU) \
2931 V(I32x4GeU, kArm64I32x4GeU) \
2932 V(I16x8SConvertI32x4, kArm64I16x8SConvertI32x4) \
2933 V(I16x8Add, kArm64I16x8Add) \
2934 V(I16x8AddSaturateS, kArm64I16x8AddSaturateS) \
2935 V(I16x8AddHoriz, kArm64I16x8AddHoriz) \
2936 V(I16x8Sub, kArm64I16x8Sub) \
2937 V(I16x8SubSaturateS, kArm64I16x8SubSaturateS) \
2938 V(I16x8Mul, kArm64I16x8Mul) \
2939 V(I16x8MinS, kArm64I16x8MinS) \
2940 V(I16x8MaxS, kArm64I16x8MaxS) \
2941 V(I16x8Eq, kArm64I16x8Eq) \
2942 V(I16x8Ne, kArm64I16x8Ne) \
2943 V(I16x8GtS, kArm64I16x8GtS) \
2944 V(I16x8GeS, kArm64I16x8GeS) \
2945 V(I16x8UConvertI32x4, kArm64I16x8UConvertI32x4) \
2946 V(I16x8AddSaturateU, kArm64I16x8AddSaturateU) \
2947 V(I16x8SubSaturateU, kArm64I16x8SubSaturateU) \
2948 V(I16x8MinU, kArm64I16x8MinU) \
2949 V(I16x8MaxU, kArm64I16x8MaxU) \
2950 V(I16x8GtU, kArm64I16x8GtU) \
2951 V(I16x8GeU, kArm64I16x8GeU) \
2952 V(I8x16SConvertI16x8, kArm64I8x16SConvertI16x8) \
2953 V(I8x16Add, kArm64I8x16Add) \
2954 V(I8x16AddSaturateS, kArm64I8x16AddSaturateS) \
2955 V(I8x16Sub, kArm64I8x16Sub) \
2956 V(I8x16SubSaturateS, kArm64I8x16SubSaturateS) \
2957 V(I8x16Mul, kArm64I8x16Mul) \
2958 V(I8x16MinS, kArm64I8x16MinS) \
2959 V(I8x16MaxS, kArm64I8x16MaxS) \
2960 V(I8x16Eq, kArm64I8x16Eq) \
2961 V(I8x16Ne, kArm64I8x16Ne) \
2962 V(I8x16GtS, kArm64I8x16GtS) \
2963 V(I8x16GeS, kArm64I8x16GeS) \
2964 V(I8x16UConvertI16x8, kArm64I8x16UConvertI16x8) \
2965 V(I8x16AddSaturateU, kArm64I8x16AddSaturateU) \
2966 V(I8x16SubSaturateU, kArm64I8x16SubSaturateU) \
2967 V(I8x16MinU, kArm64I8x16MinU) \
2968 V(I8x16MaxU, kArm64I8x16MaxU) \
2969 V(I8x16GtU, kArm64I8x16GtU) \
2970 V(I8x16GeU, kArm64I8x16GeU) \
2971 V(S128And, kArm64S128And) \
2972 V(S128Or, kArm64S128Or) \
2973 V(S128Xor, kArm64S128Xor)
2974
VisitS128Zero(Node * node)2975 void InstructionSelector::VisitS128Zero(Node* node) {
2976 Arm64OperandGenerator g(this);
2977 Emit(kArm64S128Zero, g.DefineAsRegister(node), g.DefineAsRegister(node));
2978 }
2979
2980 #define SIMD_VISIT_SPLAT(Type) \
2981 void InstructionSelector::Visit##Type##Splat(Node* node) { \
2982 VisitRR(this, kArm64##Type##Splat, node); \
2983 }
2984 SIMD_TYPE_LIST(SIMD_VISIT_SPLAT)
2985 #undef SIMD_VISIT_SPLAT
2986
2987 #define SIMD_VISIT_EXTRACT_LANE(Type) \
2988 void InstructionSelector::Visit##Type##ExtractLane(Node* node) { \
2989 VisitRRI(this, kArm64##Type##ExtractLane, node); \
2990 }
SIMD_TYPE_LIST(SIMD_VISIT_EXTRACT_LANE)2991 SIMD_TYPE_LIST(SIMD_VISIT_EXTRACT_LANE)
2992 #undef SIMD_VISIT_EXTRACT_LANE
2993
2994 #define SIMD_VISIT_REPLACE_LANE(Type) \
2995 void InstructionSelector::Visit##Type##ReplaceLane(Node* node) { \
2996 VisitRRIR(this, kArm64##Type##ReplaceLane, node); \
2997 }
2998 SIMD_TYPE_LIST(SIMD_VISIT_REPLACE_LANE)
2999 #undef SIMD_VISIT_REPLACE_LANE
3000 #undef SIMD_TYPE_LIST
3001
3002 #define SIMD_VISIT_UNOP(Name, instruction) \
3003 void InstructionSelector::Visit##Name(Node* node) { \
3004 VisitRR(this, instruction, node); \
3005 }
3006 SIMD_UNOP_LIST(SIMD_VISIT_UNOP)
3007 #undef SIMD_VISIT_UNOP
3008 #undef SIMD_UNOP_LIST
3009
3010 #define SIMD_VISIT_SHIFT_OP(Name) \
3011 void InstructionSelector::Visit##Name(Node* node) { \
3012 VisitRRI(this, kArm64##Name, node); \
3013 }
3014 SIMD_SHIFT_OP_LIST(SIMD_VISIT_SHIFT_OP)
3015 #undef SIMD_VISIT_SHIFT_OP
3016 #undef SIMD_SHIFT_OP_LIST
3017
3018 #define SIMD_VISIT_BINOP(Name, instruction) \
3019 void InstructionSelector::Visit##Name(Node* node) { \
3020 VisitRRR(this, instruction, node); \
3021 }
3022 SIMD_BINOP_LIST(SIMD_VISIT_BINOP)
3023 #undef SIMD_VISIT_BINOP
3024 #undef SIMD_BINOP_LIST
3025
3026 void InstructionSelector::VisitS128Select(Node* node) {
3027 Arm64OperandGenerator g(this);
3028 Emit(kArm64S128Select, g.DefineSameAsFirst(node),
3029 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
3030 g.UseRegister(node->InputAt(2)));
3031 }
3032
3033 namespace {
3034
3035 struct ShuffleEntry {
3036 uint8_t shuffle[kSimd128Size];
3037 ArchOpcode opcode;
3038 };
3039
3040 static const ShuffleEntry arch_shuffles[] = {
3041 {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
3042 kArm64S32x4ZipLeft},
3043 {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
3044 kArm64S32x4ZipRight},
3045 {{0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27},
3046 kArm64S32x4UnzipLeft},
3047 {{4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31},
3048 kArm64S32x4UnzipRight},
3049 {{0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27},
3050 kArm64S32x4TransposeLeft},
3051 {{4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 21, 22, 23, 24},
3052 kArm64S32x4TransposeRight},
3053 {{4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11},
3054 kArm64S32x2Reverse},
3055
3056 {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
3057 kArm64S16x8ZipLeft},
3058 {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
3059 kArm64S16x8ZipRight},
3060 {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
3061 kArm64S16x8UnzipLeft},
3062 {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
3063 kArm64S16x8UnzipRight},
3064 {{0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29},
3065 kArm64S16x8TransposeLeft},
3066 {{2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31},
3067 kArm64S16x8TransposeRight},
3068 {{6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9},
3069 kArm64S16x4Reverse},
3070 {{2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13},
3071 kArm64S16x2Reverse},
3072
3073 {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
3074 kArm64S8x16ZipLeft},
3075 {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
3076 kArm64S8x16ZipRight},
3077 {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
3078 kArm64S8x16UnzipLeft},
3079 {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
3080 kArm64S8x16UnzipRight},
3081 {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
3082 kArm64S8x16TransposeLeft},
3083 {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
3084 kArm64S8x16TransposeRight},
3085 {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8}, kArm64S8x8Reverse},
3086 {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12}, kArm64S8x4Reverse},
3087 {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
3088 kArm64S8x2Reverse}};
3089
TryMatchArchShuffle(const uint8_t * shuffle,const ShuffleEntry * table,size_t num_entries,uint8_t mask,ArchOpcode * opcode)3090 bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
3091 size_t num_entries, uint8_t mask, ArchOpcode* opcode) {
3092 for (size_t i = 0; i < num_entries; i++) {
3093 const ShuffleEntry& entry = table[i];
3094 int j = 0;
3095 for (; j < kSimd128Size; j++) {
3096 if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
3097 break;
3098 }
3099 }
3100 if (j == kSimd128Size) {
3101 *opcode = entry.opcode;
3102 return true;
3103 }
3104 }
3105 return false;
3106 }
3107
ArrangeShuffleTable(Arm64OperandGenerator * g,Node * input0,Node * input1,InstructionOperand * src0,InstructionOperand * src1)3108 void ArrangeShuffleTable(Arm64OperandGenerator* g, Node* input0, Node* input1,
3109 InstructionOperand* src0, InstructionOperand* src1) {
3110 if (input0 == input1) {
3111 // Unary, any q-register can be the table.
3112 *src0 = *src1 = g->UseRegister(input0);
3113 } else {
3114 // Binary, table registers must be consecutive.
3115 *src0 = g->UseFixed(input0, fp_fixed1);
3116 *src1 = g->UseFixed(input1, fp_fixed2);
3117 }
3118 }
3119
3120 } // namespace
3121
VisitS8x16Shuffle(Node * node)3122 void InstructionSelector::VisitS8x16Shuffle(Node* node) {
3123 const uint8_t* shuffle = OpParameter<uint8_t*>(node->op());
3124 uint8_t mask = CanonicalizeShuffle(node);
3125 uint8_t shuffle32x4[4];
3126 Arm64OperandGenerator g(this);
3127 ArchOpcode opcode;
3128 if (TryMatchArchShuffle(shuffle, arch_shuffles, arraysize(arch_shuffles),
3129 mask, &opcode)) {
3130 VisitRRR(this, opcode, node);
3131 return;
3132 }
3133 Node* input0 = node->InputAt(0);
3134 Node* input1 = node->InputAt(1);
3135 uint8_t bias;
3136 if (TryMatchConcat(shuffle, mask, &bias)) {
3137 Emit(kArm64S8x16Concat, g.DefineAsRegister(node), g.UseRegister(input0),
3138 g.UseRegister(input1), g.UseImmediate(bias));
3139 return;
3140 }
3141 int index = 0;
3142 if (TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
3143 if (TryMatchDup<4>(shuffle, &index)) {
3144 InstructionOperand src = index < 4 ? g.UseRegister(node->InputAt(0))
3145 : g.UseRegister(node->InputAt(1));
3146 Emit(kArm64S128Dup, g.DefineAsRegister(node), src, g.UseImmediate(4),
3147 g.UseImmediate(index % 4));
3148 } else {
3149 Emit(kArm64S32x4Shuffle, g.DefineAsRegister(node),
3150 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
3151 g.UseImmediate(Pack4Lanes(shuffle32x4, mask)));
3152 }
3153 return;
3154 }
3155 if (TryMatchDup<8>(shuffle, &index)) {
3156 InstructionOperand src = index < 8 ? g.UseRegister(node->InputAt(0))
3157 : g.UseRegister(node->InputAt(1));
3158 Emit(kArm64S128Dup, g.DefineAsRegister(node), src, g.UseImmediate(8),
3159 g.UseImmediate(index % 8));
3160 return;
3161 }
3162 if (TryMatchDup<16>(shuffle, &index)) {
3163 InstructionOperand src = index < 16 ? g.UseRegister(node->InputAt(0))
3164 : g.UseRegister(node->InputAt(1));
3165 Emit(kArm64S128Dup, g.DefineAsRegister(node), src, g.UseImmediate(16),
3166 g.UseImmediate(index % 16));
3167 return;
3168 }
3169 // Code generator uses vtbl, arrange sources to form a valid lookup table.
3170 InstructionOperand src0, src1;
3171 ArrangeShuffleTable(&g, input0, input1, &src0, &src1);
3172 Emit(kArm64S8x16Shuffle, g.DefineAsRegister(node), src0, src1,
3173 g.UseImmediate(Pack4Lanes(shuffle, mask)),
3174 g.UseImmediate(Pack4Lanes(shuffle + 4, mask)),
3175 g.UseImmediate(Pack4Lanes(shuffle + 8, mask)),
3176 g.UseImmediate(Pack4Lanes(shuffle + 12, mask)));
3177 }
3178
VisitSignExtendWord8ToInt32(Node * node)3179 void InstructionSelector::VisitSignExtendWord8ToInt32(Node* node) {
3180 VisitRR(this, kArm64Sxtb32, node);
3181 }
3182
VisitSignExtendWord16ToInt32(Node * node)3183 void InstructionSelector::VisitSignExtendWord16ToInt32(Node* node) {
3184 VisitRR(this, kArm64Sxth32, node);
3185 }
3186
VisitSignExtendWord8ToInt64(Node * node)3187 void InstructionSelector::VisitSignExtendWord8ToInt64(Node* node) {
3188 VisitRR(this, kArm64Sxtb, node);
3189 }
3190
VisitSignExtendWord16ToInt64(Node * node)3191 void InstructionSelector::VisitSignExtendWord16ToInt64(Node* node) {
3192 VisitRR(this, kArm64Sxth, node);
3193 }
3194
VisitSignExtendWord32ToInt64(Node * node)3195 void InstructionSelector::VisitSignExtendWord32ToInt64(Node* node) {
3196 VisitRR(this, kArm64Sxtw, node);
3197 }
3198
3199 // static
3200 MachineOperatorBuilder::Flags
SupportedMachineOperatorFlags()3201 InstructionSelector::SupportedMachineOperatorFlags() {
3202 return MachineOperatorBuilder::kFloat32RoundDown |
3203 MachineOperatorBuilder::kFloat64RoundDown |
3204 MachineOperatorBuilder::kFloat32RoundUp |
3205 MachineOperatorBuilder::kFloat64RoundUp |
3206 MachineOperatorBuilder::kFloat32RoundTruncate |
3207 MachineOperatorBuilder::kFloat64RoundTruncate |
3208 MachineOperatorBuilder::kFloat64RoundTiesAway |
3209 MachineOperatorBuilder::kFloat32RoundTiesEven |
3210 MachineOperatorBuilder::kFloat64RoundTiesEven |
3211 MachineOperatorBuilder::kWord32ShiftIsSafe |
3212 MachineOperatorBuilder::kInt32DivIsSafe |
3213 MachineOperatorBuilder::kUint32DivIsSafe |
3214 MachineOperatorBuilder::kWord32ReverseBits |
3215 MachineOperatorBuilder::kWord64ReverseBits |
3216 MachineOperatorBuilder::kSpeculationFence;
3217 }
3218
3219 // static
3220 MachineOperatorBuilder::AlignmentRequirements
AlignmentRequirements()3221 InstructionSelector::AlignmentRequirements() {
3222 return MachineOperatorBuilder::AlignmentRequirements::
3223 FullUnalignedAccessSupport();
3224 }
3225
3226 } // namespace compiler
3227 } // namespace internal
3228 } // namespace v8
3229