1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <algorithm>
6
7 #include "src/base/iterator.h"
8 #include "src/base/logging.h"
9 #include "src/base/overflowing-math.h"
10 #include "src/codegen/machine-type.h"
11 #include "src/compiler/backend/instruction-selector-impl.h"
12 #include "src/compiler/machine-operator.h"
13 #include "src/compiler/node-matchers.h"
14 #include "src/compiler/node-properties.h"
15 #include "src/roots/roots-inl.h"
16 #include "src/wasm/simd-shuffle.h"
17
18 namespace v8 {
19 namespace internal {
20 namespace compiler {
21
22 // Adds X64-specific methods for generating operands.
23 class X64OperandGenerator final : public OperandGenerator {
24 public:
X64OperandGenerator(InstructionSelector * selector)25 explicit X64OperandGenerator(InstructionSelector* selector)
26 : OperandGenerator(selector) {}
27
CanBeImmediate(Node * node)28 bool CanBeImmediate(Node* node) {
29 switch (node->opcode()) {
30 case IrOpcode::kInt32Constant:
31 case IrOpcode::kRelocatableInt32Constant: {
32 const int32_t value = OpParameter<int32_t>(node->op());
33 // int32_t min will overflow if displacement mode is
34 // kNegativeDisplacement.
35 return value != std::numeric_limits<int32_t>::min();
36 }
37 case IrOpcode::kInt64Constant: {
38 const int64_t value = OpParameter<int64_t>(node->op());
39 return std::numeric_limits<int32_t>::min() < value &&
40 value <= std::numeric_limits<int32_t>::max();
41 }
42 case IrOpcode::kNumberConstant: {
43 const double value = OpParameter<double>(node->op());
44 return bit_cast<int64_t>(value) == 0;
45 }
46 default:
47 return false;
48 }
49 }
50
GetImmediateIntegerValue(Node * node)51 int32_t GetImmediateIntegerValue(Node* node) {
52 DCHECK(CanBeImmediate(node));
53 if (node->opcode() == IrOpcode::kInt32Constant) {
54 return OpParameter<int32_t>(node->op());
55 }
56 DCHECK_EQ(IrOpcode::kInt64Constant, node->opcode());
57 return static_cast<int32_t>(OpParameter<int64_t>(node->op()));
58 }
59
CanBeMemoryOperand(InstructionCode opcode,Node * node,Node * input,int effect_level)60 bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
61 int effect_level) {
62 if (input->opcode() != IrOpcode::kLoad ||
63 !selector()->CanCover(node, input)) {
64 return false;
65 }
66 if (effect_level != selector()->GetEffectLevel(input)) {
67 return false;
68 }
69 MachineRepresentation rep =
70 LoadRepresentationOf(input->op()).representation();
71 switch (opcode) {
72 case kX64And:
73 case kX64Or:
74 case kX64Xor:
75 case kX64Add:
76 case kX64Sub:
77 case kX64Push:
78 case kX64Cmp:
79 case kX64Test:
80 // When pointer compression is enabled 64-bit memory operands can't be
81 // used for tagged values.
82 return rep == MachineRepresentation::kWord64 ||
83 (!COMPRESS_POINTERS_BOOL && IsAnyTagged(rep));
84 case kX64And32:
85 case kX64Or32:
86 case kX64Xor32:
87 case kX64Add32:
88 case kX64Sub32:
89 case kX64Cmp32:
90 case kX64Test32:
91 // When pointer compression is enabled 32-bit memory operands can be
92 // used for tagged values.
93 return rep == MachineRepresentation::kWord32 ||
94 (COMPRESS_POINTERS_BOOL &&
95 (IsAnyTagged(rep) || IsAnyCompressed(rep)));
96 case kAVXFloat64Add:
97 case kAVXFloat64Sub:
98 case kAVXFloat64Mul:
99 DCHECK_EQ(MachineRepresentation::kFloat64, rep);
100 return true;
101 case kAVXFloat32Add:
102 case kAVXFloat32Sub:
103 case kAVXFloat32Mul:
104 DCHECK_EQ(MachineRepresentation::kFloat32, rep);
105 return true;
106 case kX64Cmp16:
107 case kX64Test16:
108 return rep == MachineRepresentation::kWord16;
109 case kX64Cmp8:
110 case kX64Test8:
111 return rep == MachineRepresentation::kWord8;
112 default:
113 break;
114 }
115 return false;
116 }
117
GenerateMemoryOperandInputs(Node * index,int scale_exponent,Node * base,Node * displacement,DisplacementMode displacement_mode,InstructionOperand inputs[],size_t * input_count)118 AddressingMode GenerateMemoryOperandInputs(Node* index, int scale_exponent,
119 Node* base, Node* displacement,
120 DisplacementMode displacement_mode,
121 InstructionOperand inputs[],
122 size_t* input_count) {
123 AddressingMode mode = kMode_MRI;
124 if (base != nullptr && (index != nullptr || displacement != nullptr)) {
125 if (base->opcode() == IrOpcode::kInt32Constant &&
126 OpParameter<int32_t>(base->op()) == 0) {
127 base = nullptr;
128 } else if (base->opcode() == IrOpcode::kInt64Constant &&
129 OpParameter<int64_t>(base->op()) == 0) {
130 base = nullptr;
131 }
132 }
133 if (base != nullptr) {
134 inputs[(*input_count)++] = UseRegister(base);
135 if (index != nullptr) {
136 DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
137 inputs[(*input_count)++] = UseRegister(index);
138 if (displacement != nullptr) {
139 inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
140 ? UseNegatedImmediate(displacement)
141 : UseImmediate(displacement);
142 static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
143 kMode_MR4I, kMode_MR8I};
144 mode = kMRnI_modes[scale_exponent];
145 } else {
146 static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
147 kMode_MR4, kMode_MR8};
148 mode = kMRn_modes[scale_exponent];
149 }
150 } else {
151 if (displacement == nullptr) {
152 mode = kMode_MR;
153 } else {
154 inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
155 ? UseNegatedImmediate(displacement)
156 : UseImmediate(displacement);
157 mode = kMode_MRI;
158 }
159 }
160 } else {
161 DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
162 if (displacement != nullptr) {
163 if (index == nullptr) {
164 inputs[(*input_count)++] = UseRegister(displacement);
165 mode = kMode_MR;
166 } else {
167 inputs[(*input_count)++] = UseRegister(index);
168 inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
169 ? UseNegatedImmediate(displacement)
170 : UseImmediate(displacement);
171 static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
172 kMode_M4I, kMode_M8I};
173 mode = kMnI_modes[scale_exponent];
174 }
175 } else {
176 inputs[(*input_count)++] = UseRegister(index);
177 static const AddressingMode kMn_modes[] = {kMode_MR, kMode_MR1,
178 kMode_M4, kMode_M8};
179 mode = kMn_modes[scale_exponent];
180 if (mode == kMode_MR1) {
181 // [%r1 + %r1*1] has a smaller encoding than [%r1*2+0]
182 inputs[(*input_count)++] = UseRegister(index);
183 }
184 }
185 }
186 return mode;
187 }
188
GetEffectiveAddressMemoryOperand(Node * operand,InstructionOperand inputs[],size_t * input_count)189 AddressingMode GetEffectiveAddressMemoryOperand(Node* operand,
190 InstructionOperand inputs[],
191 size_t* input_count) {
192 {
193 LoadMatcher<ExternalReferenceMatcher> m(operand);
194 if (m.index().HasResolvedValue() && m.object().HasResolvedValue() &&
195 selector()->CanAddressRelativeToRootsRegister(
196 m.object().ResolvedValue())) {
197 ptrdiff_t const delta =
198 m.index().ResolvedValue() +
199 TurboAssemblerBase::RootRegisterOffsetForExternalReference(
200 selector()->isolate(), m.object().ResolvedValue());
201 if (is_int32(delta)) {
202 inputs[(*input_count)++] = TempImmediate(static_cast<int32_t>(delta));
203 return kMode_Root;
204 }
205 }
206 }
207 BaseWithIndexAndDisplacement64Matcher m(operand, AddressOption::kAllowAll);
208 DCHECK(m.matches());
209 if (m.displacement() == nullptr || CanBeImmediate(m.displacement())) {
210 return GenerateMemoryOperandInputs(
211 m.index(), m.scale(), m.base(), m.displacement(),
212 m.displacement_mode(), inputs, input_count);
213 } else if (m.base() == nullptr &&
214 m.displacement_mode() == kPositiveDisplacement) {
215 // The displacement cannot be an immediate, but we can use the
216 // displacement as base instead and still benefit from addressing
217 // modes for the scale.
218 return GenerateMemoryOperandInputs(m.index(), m.scale(), m.displacement(),
219 nullptr, m.displacement_mode(), inputs,
220 input_count);
221 } else {
222 inputs[(*input_count)++] = UseRegister(operand->InputAt(0));
223 inputs[(*input_count)++] = UseRegister(operand->InputAt(1));
224 return kMode_MR1;
225 }
226 }
227
GetEffectiveIndexOperand(Node * index,AddressingMode * mode)228 InstructionOperand GetEffectiveIndexOperand(Node* index,
229 AddressingMode* mode) {
230 if (CanBeImmediate(index)) {
231 *mode = kMode_MRI;
232 return UseImmediate(index);
233 } else {
234 *mode = kMode_MR1;
235 return UseUniqueRegister(index);
236 }
237 }
238
CanBeBetterLeftOperand(Node * node) const239 bool CanBeBetterLeftOperand(Node* node) const {
240 return !selector()->IsLive(node);
241 }
242 };
243
244 namespace {
GetLoadOpcode(LoadRepresentation load_rep)245 ArchOpcode GetLoadOpcode(LoadRepresentation load_rep) {
246 ArchOpcode opcode;
247 switch (load_rep.representation()) {
248 case MachineRepresentation::kFloat32:
249 opcode = kX64Movss;
250 break;
251 case MachineRepresentation::kFloat64:
252 opcode = kX64Movsd;
253 break;
254 case MachineRepresentation::kBit: // Fall through.
255 case MachineRepresentation::kWord8:
256 opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
257 break;
258 case MachineRepresentation::kWord16:
259 opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
260 break;
261 case MachineRepresentation::kWord32:
262 opcode = kX64Movl;
263 break;
264 case MachineRepresentation::kCompressedPointer: // Fall through.
265 case MachineRepresentation::kCompressed:
266 #ifdef V8_COMPRESS_POINTERS
267 opcode = kX64Movl;
268 break;
269 #else
270 UNREACHABLE();
271 #endif
272 #ifdef V8_COMPRESS_POINTERS
273 case MachineRepresentation::kTaggedSigned:
274 opcode = kX64MovqDecompressTaggedSigned;
275 break;
276 case MachineRepresentation::kTaggedPointer:
277 opcode = kX64MovqDecompressTaggedPointer;
278 break;
279 case MachineRepresentation::kTagged:
280 opcode = kX64MovqDecompressAnyTagged;
281 break;
282 #else
283 case MachineRepresentation::kTaggedSigned: // Fall through.
284 case MachineRepresentation::kTaggedPointer: // Fall through.
285 case MachineRepresentation::kTagged: // Fall through.
286 #endif
287 case MachineRepresentation::kWord64:
288 opcode = kX64Movq;
289 break;
290 case MachineRepresentation::kSimd128: // Fall through.
291 opcode = kX64Movdqu;
292 break;
293 case MachineRepresentation::kNone:
294 UNREACHABLE();
295 }
296 return opcode;
297 }
298
GetStoreOpcode(StoreRepresentation store_rep)299 ArchOpcode GetStoreOpcode(StoreRepresentation store_rep) {
300 switch (store_rep.representation()) {
301 case MachineRepresentation::kFloat32:
302 return kX64Movss;
303 case MachineRepresentation::kFloat64:
304 return kX64Movsd;
305 case MachineRepresentation::kBit: // Fall through.
306 case MachineRepresentation::kWord8:
307 return kX64Movb;
308 case MachineRepresentation::kWord16:
309 return kX64Movw;
310 case MachineRepresentation::kWord32:
311 return kX64Movl;
312 case MachineRepresentation::kCompressedPointer: // Fall through.
313 case MachineRepresentation::kCompressed:
314 #ifdef V8_COMPRESS_POINTERS
315 return kX64MovqCompressTagged;
316 #else
317 UNREACHABLE();
318 #endif
319 case MachineRepresentation::kTaggedSigned: // Fall through.
320 case MachineRepresentation::kTaggedPointer: // Fall through.
321 case MachineRepresentation::kTagged:
322 return kX64MovqCompressTagged;
323 case MachineRepresentation::kWord64:
324 return kX64Movq;
325 case MachineRepresentation::kSimd128: // Fall through.
326 return kX64Movdqu;
327 case MachineRepresentation::kNone:
328 UNREACHABLE();
329 }
330 UNREACHABLE();
331 }
332
333 } // namespace
334
VisitStackSlot(Node * node)335 void InstructionSelector::VisitStackSlot(Node* node) {
336 StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
337 int slot = frame_->AllocateSpillSlot(rep.size());
338 OperandGenerator g(this);
339
340 Emit(kArchStackSlot, g.DefineAsRegister(node),
341 sequence()->AddImmediate(Constant(slot)), 0, nullptr);
342 }
343
VisitAbortCSAAssert(Node * node)344 void InstructionSelector::VisitAbortCSAAssert(Node* node) {
345 X64OperandGenerator g(this);
346 Emit(kArchAbortCSAAssert, g.NoOutput(), g.UseFixed(node->InputAt(0), rdx));
347 }
348
VisitLoadLane(Node * node)349 void InstructionSelector::VisitLoadLane(Node* node) {
350 LoadLaneParameters params = LoadLaneParametersOf(node->op());
351 InstructionCode opcode = kArchNop;
352 if (params.rep == MachineType::Int8()) {
353 opcode = kX64Pinsrb;
354 } else if (params.rep == MachineType::Int16()) {
355 opcode = kX64Pinsrw;
356 } else if (params.rep == MachineType::Int32()) {
357 opcode = kX64Pinsrd;
358 } else if (params.rep == MachineType::Int64()) {
359 opcode = kX64Pinsrq;
360 } else {
361 UNREACHABLE();
362 }
363
364 X64OperandGenerator g(this);
365 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
366 // Input 0 is value node, 1 is lane idx, and GetEffectiveAddressMemoryOperand
367 // uses up to 3 inputs. This ordering is consistent with other operations that
368 // use the same opcode.
369 InstructionOperand inputs[5];
370 size_t input_count = 0;
371
372 inputs[input_count++] = g.UseRegister(node->InputAt(2));
373 inputs[input_count++] = g.UseImmediate(params.laneidx);
374
375 AddressingMode mode =
376 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
377 opcode |= AddressingModeField::encode(mode);
378
379 DCHECK_GE(5, input_count);
380
381 // x64 supports unaligned loads.
382 DCHECK_NE(params.kind, MemoryAccessKind::kUnaligned);
383 if (params.kind == MemoryAccessKind::kProtected) {
384 opcode |= MiscField::encode(kMemoryAccessProtected);
385 }
386 Emit(opcode, 1, outputs, input_count, inputs);
387 }
388
VisitLoadTransform(Node * node)389 void InstructionSelector::VisitLoadTransform(Node* node) {
390 LoadTransformParameters params = LoadTransformParametersOf(node->op());
391 ArchOpcode opcode;
392 switch (params.transformation) {
393 case LoadTransformation::kS128Load8Splat:
394 opcode = kX64S128Load8Splat;
395 break;
396 case LoadTransformation::kS128Load16Splat:
397 opcode = kX64S128Load16Splat;
398 break;
399 case LoadTransformation::kS128Load32Splat:
400 opcode = kX64S128Load32Splat;
401 break;
402 case LoadTransformation::kS128Load64Splat:
403 opcode = kX64S128Load64Splat;
404 break;
405 case LoadTransformation::kS128Load8x8S:
406 opcode = kX64S128Load8x8S;
407 break;
408 case LoadTransformation::kS128Load8x8U:
409 opcode = kX64S128Load8x8U;
410 break;
411 case LoadTransformation::kS128Load16x4S:
412 opcode = kX64S128Load16x4S;
413 break;
414 case LoadTransformation::kS128Load16x4U:
415 opcode = kX64S128Load16x4U;
416 break;
417 case LoadTransformation::kS128Load32x2S:
418 opcode = kX64S128Load32x2S;
419 break;
420 case LoadTransformation::kS128Load32x2U:
421 opcode = kX64S128Load32x2U;
422 break;
423 case LoadTransformation::kS128Load32Zero:
424 opcode = kX64Movss;
425 break;
426 case LoadTransformation::kS128Load64Zero:
427 opcode = kX64Movsd;
428 break;
429 default:
430 UNREACHABLE();
431 }
432 // x64 supports unaligned loads
433 DCHECK_NE(params.kind, MemoryAccessKind::kUnaligned);
434 InstructionCode code = opcode;
435 if (params.kind == MemoryAccessKind::kProtected) {
436 code |= MiscField::encode(kMemoryAccessProtected);
437 }
438 VisitLoad(node, node, code);
439 }
440
VisitLoad(Node * node,Node * value,InstructionCode opcode)441 void InstructionSelector::VisitLoad(Node* node, Node* value,
442 InstructionCode opcode) {
443 X64OperandGenerator g(this);
444 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
445 InstructionOperand inputs[3];
446 size_t input_count = 0;
447 AddressingMode mode =
448 g.GetEffectiveAddressMemoryOperand(value, inputs, &input_count);
449 InstructionCode code = opcode | AddressingModeField::encode(mode);
450 if (node->opcode() == IrOpcode::kProtectedLoad) {
451 code |= MiscField::encode(kMemoryAccessProtected);
452 } else if (node->opcode() == IrOpcode::kPoisonedLoad) {
453 CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
454 code |= MiscField::encode(kMemoryAccessPoisoned);
455 }
456 Emit(code, 1, outputs, input_count, inputs);
457 }
458
VisitLoad(Node * node)459 void InstructionSelector::VisitLoad(Node* node) {
460 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
461 VisitLoad(node, node, GetLoadOpcode(load_rep));
462 }
463
VisitPoisonedLoad(Node * node)464 void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }
465
VisitProtectedLoad(Node * node)466 void InstructionSelector::VisitProtectedLoad(Node* node) { VisitLoad(node); }
467
VisitStore(Node * node)468 void InstructionSelector::VisitStore(Node* node) {
469 X64OperandGenerator g(this);
470 Node* base = node->InputAt(0);
471 Node* index = node->InputAt(1);
472 Node* value = node->InputAt(2);
473
474 StoreRepresentation store_rep = StoreRepresentationOf(node->op());
475 WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
476
477 if (FLAG_enable_unconditional_write_barriers &&
478 CanBeTaggedOrCompressedPointer(store_rep.representation())) {
479 write_barrier_kind = kFullWriteBarrier;
480 }
481
482 if (write_barrier_kind != kNoWriteBarrier &&
483 V8_LIKELY(!FLAG_disable_write_barriers)) {
484 DCHECK(CanBeTaggedOrCompressedPointer(store_rep.representation()));
485 AddressingMode addressing_mode;
486 InstructionOperand inputs[] = {
487 g.UseUniqueRegister(base),
488 g.GetEffectiveIndexOperand(index, &addressing_mode),
489 g.UseUniqueRegister(value)};
490 RecordWriteMode record_write_mode =
491 WriteBarrierKindToRecordWriteMode(write_barrier_kind);
492 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
493 InstructionCode code = kArchStoreWithWriteBarrier;
494 code |= AddressingModeField::encode(addressing_mode);
495 code |= MiscField::encode(static_cast<int>(record_write_mode));
496 Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
497 } else {
498 ArchOpcode opcode = GetStoreOpcode(store_rep);
499 InstructionOperand inputs[4];
500 size_t input_count = 0;
501 AddressingMode addressing_mode =
502 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
503 InstructionCode code =
504 opcode | AddressingModeField::encode(addressing_mode);
505 if ((ElementSizeLog2Of(store_rep.representation()) <
506 kSystemPointerSizeLog2) &&
507 value->opcode() == IrOpcode::kTruncateInt64ToInt32) {
508 value = value->InputAt(0);
509 }
510 InstructionOperand value_operand =
511 g.CanBeImmediate(value) ? g.UseImmediate(value) : g.UseRegister(value);
512 inputs[input_count++] = value_operand;
513 Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count,
514 inputs);
515 }
516 }
517
VisitProtectedStore(Node * node)518 void InstructionSelector::VisitProtectedStore(Node* node) {
519 X64OperandGenerator g(this);
520 Node* value = node->InputAt(2);
521
522 StoreRepresentation store_rep = StoreRepresentationOf(node->op());
523
524 ArchOpcode opcode = GetStoreOpcode(store_rep);
525 InstructionOperand inputs[4];
526 size_t input_count = 0;
527 AddressingMode addressing_mode =
528 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
529 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) |
530 MiscField::encode(kMemoryAccessProtected);
531 InstructionOperand value_operand =
532 g.CanBeImmediate(value) ? g.UseImmediate(value) : g.UseRegister(value);
533 inputs[input_count++] = value_operand;
534 Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count, inputs);
535 }
536
537 // Architecture supports unaligned access, therefore VisitLoad is used instead
VisitUnalignedLoad(Node * node)538 void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
539
540 // Architecture supports unaligned access, therefore VisitStore is used instead
VisitUnalignedStore(Node * node)541 void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
542
VisitStoreLane(Node * node)543 void InstructionSelector::VisitStoreLane(Node* node) {
544 X64OperandGenerator g(this);
545
546 StoreLaneParameters params = StoreLaneParametersOf(node->op());
547 InstructionCode opcode = kArchNop;
548 if (params.rep == MachineRepresentation::kWord8) {
549 opcode = kX64Pextrb;
550 } else if (params.rep == MachineRepresentation::kWord16) {
551 opcode = kX64Pextrw;
552 } else if (params.rep == MachineRepresentation::kWord32) {
553 opcode = kX64S128Store32Lane;
554 } else if (params.rep == MachineRepresentation::kWord64) {
555 opcode = kX64S128Store64Lane;
556 } else {
557 UNREACHABLE();
558 }
559
560 InstructionOperand inputs[4];
561 size_t input_count = 0;
562 AddressingMode addressing_mode =
563 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
564 opcode |= AddressingModeField::encode(addressing_mode);
565
566 if (params.kind == MemoryAccessKind::kProtected) {
567 opcode |= MiscField::encode(kMemoryAccessProtected);
568 }
569
570 InstructionOperand value_operand = g.UseRegister(node->InputAt(2));
571 inputs[input_count++] = value_operand;
572 inputs[input_count++] = g.UseImmediate(params.laneidx);
573 DCHECK_GE(4, input_count);
574 Emit(opcode, 0, nullptr, input_count, inputs);
575 }
576
577 // Shared routine for multiple binary operations.
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)578 static void VisitBinop(InstructionSelector* selector, Node* node,
579 InstructionCode opcode, FlagsContinuation* cont) {
580 X64OperandGenerator g(selector);
581 Int32BinopMatcher m(node);
582 Node* left = m.left().node();
583 Node* right = m.right().node();
584 InstructionOperand inputs[8];
585 size_t input_count = 0;
586 InstructionOperand outputs[1];
587 size_t output_count = 0;
588
589 // TODO(turbofan): match complex addressing modes.
590 if (left == right) {
591 // If both inputs refer to the same operand, enforce allocating a register
592 // for both of them to ensure that we don't end up generating code like
593 // this:
594 //
595 // mov rax, [rbp-0x10]
596 // add rax, [rbp-0x10]
597 // jo label
598 InstructionOperand const input = g.UseRegister(left);
599 inputs[input_count++] = input;
600 inputs[input_count++] = input;
601 } else if (g.CanBeImmediate(right)) {
602 inputs[input_count++] = g.UseRegister(left);
603 inputs[input_count++] = g.UseImmediate(right);
604 } else {
605 int effect_level = selector->GetEffectLevel(node, cont);
606 if (node->op()->HasProperty(Operator::kCommutative) &&
607 g.CanBeBetterLeftOperand(right) &&
608 (!g.CanBeBetterLeftOperand(left) ||
609 !g.CanBeMemoryOperand(opcode, node, right, effect_level))) {
610 std::swap(left, right);
611 }
612 if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) {
613 inputs[input_count++] = g.UseRegister(left);
614 AddressingMode addressing_mode =
615 g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
616 opcode |= AddressingModeField::encode(addressing_mode);
617 } else {
618 inputs[input_count++] = g.UseRegister(left);
619 inputs[input_count++] = g.Use(right);
620 }
621 }
622
623 if (cont->IsBranch()) {
624 inputs[input_count++] = g.Label(cont->true_block());
625 inputs[input_count++] = g.Label(cont->false_block());
626 }
627
628 outputs[output_count++] = g.DefineSameAsFirst(node);
629
630 DCHECK_NE(0u, input_count);
631 DCHECK_EQ(1u, output_count);
632 DCHECK_GE(arraysize(inputs), input_count);
633 DCHECK_GE(arraysize(outputs), output_count);
634
635 selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
636 inputs, cont);
637 }
638
639 // Shared routine for multiple binary operations.
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode)640 static void VisitBinop(InstructionSelector* selector, Node* node,
641 InstructionCode opcode) {
642 FlagsContinuation cont;
643 VisitBinop(selector, node, opcode, &cont);
644 }
645
VisitWord32And(Node * node)646 void InstructionSelector::VisitWord32And(Node* node) {
647 X64OperandGenerator g(this);
648 Uint32BinopMatcher m(node);
649 if (m.right().Is(0xFF)) {
650 Emit(kX64Movzxbl, g.DefineAsRegister(node), g.Use(m.left().node()));
651 } else if (m.right().Is(0xFFFF)) {
652 Emit(kX64Movzxwl, g.DefineAsRegister(node), g.Use(m.left().node()));
653 } else {
654 VisitBinop(this, node, kX64And32);
655 }
656 }
657
VisitWord64And(Node * node)658 void InstructionSelector::VisitWord64And(Node* node) {
659 VisitBinop(this, node, kX64And);
660 }
661
VisitWord32Or(Node * node)662 void InstructionSelector::VisitWord32Or(Node* node) {
663 VisitBinop(this, node, kX64Or32);
664 }
665
VisitWord64Or(Node * node)666 void InstructionSelector::VisitWord64Or(Node* node) {
667 VisitBinop(this, node, kX64Or);
668 }
669
VisitWord32Xor(Node * node)670 void InstructionSelector::VisitWord32Xor(Node* node) {
671 X64OperandGenerator g(this);
672 Uint32BinopMatcher m(node);
673 if (m.right().Is(-1)) {
674 Emit(kX64Not32, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
675 } else {
676 VisitBinop(this, node, kX64Xor32);
677 }
678 }
679
VisitWord64Xor(Node * node)680 void InstructionSelector::VisitWord64Xor(Node* node) {
681 X64OperandGenerator g(this);
682 Uint64BinopMatcher m(node);
683 if (m.right().Is(-1)) {
684 Emit(kX64Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
685 } else {
686 VisitBinop(this, node, kX64Xor);
687 }
688 }
689
VisitStackPointerGreaterThan(Node * node,FlagsContinuation * cont)690 void InstructionSelector::VisitStackPointerGreaterThan(
691 Node* node, FlagsContinuation* cont) {
692 StackCheckKind kind = StackCheckKindOf(node->op());
693 InstructionCode opcode =
694 kArchStackPointerGreaterThan | MiscField::encode(static_cast<int>(kind));
695
696 int effect_level = GetEffectLevel(node, cont);
697
698 X64OperandGenerator g(this);
699 Node* const value = node->InputAt(0);
700 if (g.CanBeMemoryOperand(kX64Cmp, node, value, effect_level)) {
701 DCHECK_EQ(IrOpcode::kLoad, value->opcode());
702
703 // GetEffectiveAddressMemoryOperand can create at most 3 inputs.
704 static constexpr int kMaxInputCount = 3;
705
706 size_t input_count = 0;
707 InstructionOperand inputs[kMaxInputCount];
708 AddressingMode addressing_mode =
709 g.GetEffectiveAddressMemoryOperand(value, inputs, &input_count);
710 opcode |= AddressingModeField::encode(addressing_mode);
711 DCHECK_LE(input_count, kMaxInputCount);
712
713 EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
714 } else {
715 EmitWithContinuation(opcode, g.UseRegister(value), cont);
716 }
717 }
718
719 namespace {
720
TryMergeTruncateInt64ToInt32IntoLoad(InstructionSelector * selector,Node * node,Node * load)721 bool TryMergeTruncateInt64ToInt32IntoLoad(InstructionSelector* selector,
722 Node* node, Node* load) {
723 if (load->opcode() == IrOpcode::kLoad && selector->CanCover(node, load)) {
724 LoadRepresentation load_rep = LoadRepresentationOf(load->op());
725 MachineRepresentation rep = load_rep.representation();
726 InstructionCode opcode;
727 switch (rep) {
728 case MachineRepresentation::kBit: // Fall through.
729 case MachineRepresentation::kWord8:
730 opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
731 break;
732 case MachineRepresentation::kWord16:
733 opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
734 break;
735 case MachineRepresentation::kWord32:
736 case MachineRepresentation::kWord64:
737 case MachineRepresentation::kTaggedSigned:
738 case MachineRepresentation::kTagged:
739 case MachineRepresentation::kCompressed: // Fall through.
740 opcode = kX64Movl;
741 break;
742 default:
743 UNREACHABLE();
744 return false;
745 }
746 X64OperandGenerator g(selector);
747 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
748 size_t input_count = 0;
749 InstructionOperand inputs[3];
750 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
751 node->InputAt(0), inputs, &input_count);
752 opcode |= AddressingModeField::encode(mode);
753 selector->Emit(opcode, 1, outputs, input_count, inputs);
754 return true;
755 }
756 return false;
757 }
758
759 // Shared routine for multiple 32-bit shift operations.
760 // TODO(bmeurer): Merge this with VisitWord64Shift using template magic?
VisitWord32Shift(InstructionSelector * selector,Node * node,ArchOpcode opcode)761 void VisitWord32Shift(InstructionSelector* selector, Node* node,
762 ArchOpcode opcode) {
763 X64OperandGenerator g(selector);
764 Int32BinopMatcher m(node);
765 Node* left = m.left().node();
766 Node* right = m.right().node();
767
768 if (left->opcode() == IrOpcode::kTruncateInt64ToInt32) {
769 left = left->InputAt(0);
770 }
771
772 if (g.CanBeImmediate(right)) {
773 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
774 g.UseImmediate(right));
775 } else {
776 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
777 g.UseFixed(right, rcx));
778 }
779 }
780
781 // Shared routine for multiple 64-bit shift operations.
782 // TODO(bmeurer): Merge this with VisitWord32Shift using template magic?
VisitWord64Shift(InstructionSelector * selector,Node * node,ArchOpcode opcode)783 void VisitWord64Shift(InstructionSelector* selector, Node* node,
784 ArchOpcode opcode) {
785 X64OperandGenerator g(selector);
786 Int64BinopMatcher m(node);
787 Node* left = m.left().node();
788 Node* right = m.right().node();
789
790 if (g.CanBeImmediate(right)) {
791 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
792 g.UseImmediate(right));
793 } else {
794 if (m.right().IsWord64And()) {
795 Int64BinopMatcher mright(right);
796 if (mright.right().Is(0x3F)) {
797 right = mright.left().node();
798 }
799 }
800 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
801 g.UseFixed(right, rcx));
802 }
803 }
804
805 // Shared routine for multiple shift operations with continuation.
806 template <typename BinopMatcher, int Bits>
TryVisitWordShift(InstructionSelector * selector,Node * node,ArchOpcode opcode,FlagsContinuation * cont)807 bool TryVisitWordShift(InstructionSelector* selector, Node* node,
808 ArchOpcode opcode, FlagsContinuation* cont) {
809 X64OperandGenerator g(selector);
810 BinopMatcher m(node);
811 Node* left = m.left().node();
812 Node* right = m.right().node();
813
814 // If the shift count is 0, the flags are not affected.
815 if (!g.CanBeImmediate(right) ||
816 (g.GetImmediateIntegerValue(right) & (Bits - 1)) == 0) {
817 return false;
818 }
819 InstructionOperand output = g.DefineSameAsFirst(node);
820 InstructionOperand inputs[2];
821 inputs[0] = g.UseRegister(left);
822 inputs[1] = g.UseImmediate(right);
823 selector->EmitWithContinuation(opcode, 1, &output, 2, inputs, cont);
824 return true;
825 }
826
EmitLea(InstructionSelector * selector,InstructionCode opcode,Node * result,Node * index,int scale,Node * base,Node * displacement,DisplacementMode displacement_mode)827 void EmitLea(InstructionSelector* selector, InstructionCode opcode,
828 Node* result, Node* index, int scale, Node* base,
829 Node* displacement, DisplacementMode displacement_mode) {
830 X64OperandGenerator g(selector);
831
832 InstructionOperand inputs[4];
833 size_t input_count = 0;
834 AddressingMode mode =
835 g.GenerateMemoryOperandInputs(index, scale, base, displacement,
836 displacement_mode, inputs, &input_count);
837
838 DCHECK_NE(0u, input_count);
839 DCHECK_GE(arraysize(inputs), input_count);
840
841 InstructionOperand outputs[1];
842 outputs[0] = g.DefineAsRegister(result);
843
844 opcode = AddressingModeField::encode(mode) | opcode;
845
846 selector->Emit(opcode, 1, outputs, input_count, inputs);
847 }
848
849 } // namespace
850
VisitWord32Shl(Node * node)851 void InstructionSelector::VisitWord32Shl(Node* node) {
852 Int32ScaleMatcher m(node, true);
853 if (m.matches()) {
854 Node* index = node->InputAt(0);
855 Node* base = m.power_of_two_plus_one() ? index : nullptr;
856 EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
857 kPositiveDisplacement);
858 return;
859 }
860 VisitWord32Shift(this, node, kX64Shl32);
861 }
862
VisitWord64Shl(Node * node)863 void InstructionSelector::VisitWord64Shl(Node* node) {
864 X64OperandGenerator g(this);
865 Int64ScaleMatcher m(node, true);
866 if (m.matches()) {
867 Node* index = node->InputAt(0);
868 Node* base = m.power_of_two_plus_one() ? index : nullptr;
869 EmitLea(this, kX64Lea, node, index, m.scale(), base, nullptr,
870 kPositiveDisplacement);
871 return;
872 } else {
873 Int64BinopMatcher m(node);
874 if ((m.left().IsChangeInt32ToInt64() ||
875 m.left().IsChangeUint32ToUint64()) &&
876 m.right().IsInRange(32, 63)) {
877 // There's no need to sign/zero-extend to 64-bit if we shift out the upper
878 // 32 bits anyway.
879 Emit(kX64Shl, g.DefineSameAsFirst(node),
880 g.UseRegister(m.left().node()->InputAt(0)),
881 g.UseImmediate(m.right().node()));
882 return;
883 }
884 }
885 VisitWord64Shift(this, node, kX64Shl);
886 }
887
VisitWord32Shr(Node * node)888 void InstructionSelector::VisitWord32Shr(Node* node) {
889 VisitWord32Shift(this, node, kX64Shr32);
890 }
891
892 namespace {
893
AddDisplacementToAddressingMode(AddressingMode mode)894 inline AddressingMode AddDisplacementToAddressingMode(AddressingMode mode) {
895 switch (mode) {
896 case kMode_MR:
897 return kMode_MRI;
898 break;
899 case kMode_MR1:
900 return kMode_MR1I;
901 break;
902 case kMode_MR2:
903 return kMode_MR2I;
904 break;
905 case kMode_MR4:
906 return kMode_MR4I;
907 break;
908 case kMode_MR8:
909 return kMode_MR8I;
910 break;
911 case kMode_M1:
912 return kMode_M1I;
913 break;
914 case kMode_M2:
915 return kMode_M2I;
916 break;
917 case kMode_M4:
918 return kMode_M4I;
919 break;
920 case kMode_M8:
921 return kMode_M8I;
922 break;
923 case kMode_None:
924 case kMode_MRI:
925 case kMode_MR1I:
926 case kMode_MR2I:
927 case kMode_MR4I:
928 case kMode_MR8I:
929 case kMode_M1I:
930 case kMode_M2I:
931 case kMode_M4I:
932 case kMode_M8I:
933 case kMode_Root:
934 UNREACHABLE();
935 }
936 UNREACHABLE();
937 }
938
TryMatchLoadWord64AndShiftRight(InstructionSelector * selector,Node * node,InstructionCode opcode)939 bool TryMatchLoadWord64AndShiftRight(InstructionSelector* selector, Node* node,
940 InstructionCode opcode) {
941 DCHECK(IrOpcode::kWord64Sar == node->opcode() ||
942 IrOpcode::kWord64Shr == node->opcode());
943 X64OperandGenerator g(selector);
944 Int64BinopMatcher m(node);
945 if (selector->CanCover(m.node(), m.left().node()) && m.left().IsLoad() &&
946 m.right().Is(32)) {
947 DCHECK_EQ(selector->GetEffectLevel(node),
948 selector->GetEffectLevel(m.left().node()));
949 // Just load and sign-extend the interesting 4 bytes instead. This happens,
950 // for example, when we're loading and untagging SMIs.
951 BaseWithIndexAndDisplacement64Matcher mleft(m.left().node(),
952 AddressOption::kAllowAll);
953 if (mleft.matches() && (mleft.displacement() == nullptr ||
954 g.CanBeImmediate(mleft.displacement()))) {
955 size_t input_count = 0;
956 InstructionOperand inputs[3];
957 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
958 m.left().node(), inputs, &input_count);
959 if (mleft.displacement() == nullptr) {
960 // Make sure that the addressing mode indicates the presence of an
961 // immediate displacement. It seems that we never use M1 and M2, but we
962 // handle them here anyways.
963 mode = AddDisplacementToAddressingMode(mode);
964 inputs[input_count++] = ImmediateOperand(ImmediateOperand::INLINE, 4);
965 } else {
966 // In the case that the base address was zero, the displacement will be
967 // in a register and replacing it with an immediate is not allowed. This
968 // usually only happens in dead code anyway.
969 if (!inputs[input_count - 1].IsImmediate()) return false;
970 int32_t displacement = g.GetImmediateIntegerValue(mleft.displacement());
971 inputs[input_count - 1] =
972 ImmediateOperand(ImmediateOperand::INLINE, displacement + 4);
973 }
974 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
975 InstructionCode code = opcode | AddressingModeField::encode(mode);
976 selector->Emit(code, 1, outputs, input_count, inputs);
977 return true;
978 }
979 }
980 return false;
981 }
982
983 } // namespace
984
VisitWord64Shr(Node * node)985 void InstructionSelector::VisitWord64Shr(Node* node) {
986 if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movl)) return;
987 VisitWord64Shift(this, node, kX64Shr);
988 }
989
VisitWord32Sar(Node * node)990 void InstructionSelector::VisitWord32Sar(Node* node) {
991 X64OperandGenerator g(this);
992 Int32BinopMatcher m(node);
993 if (CanCover(m.node(), m.left().node()) && m.left().IsWord32Shl()) {
994 Int32BinopMatcher mleft(m.left().node());
995 if (mleft.right().Is(16) && m.right().Is(16)) {
996 Emit(kX64Movsxwl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
997 return;
998 } else if (mleft.right().Is(24) && m.right().Is(24)) {
999 Emit(kX64Movsxbl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
1000 return;
1001 }
1002 }
1003 VisitWord32Shift(this, node, kX64Sar32);
1004 }
1005
VisitWord64Sar(Node * node)1006 void InstructionSelector::VisitWord64Sar(Node* node) {
1007 if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movsxlq)) return;
1008 VisitWord64Shift(this, node, kX64Sar);
1009 }
1010
VisitWord32Rol(Node * node)1011 void InstructionSelector::VisitWord32Rol(Node* node) {
1012 VisitWord32Shift(this, node, kX64Rol32);
1013 }
1014
VisitWord64Rol(Node * node)1015 void InstructionSelector::VisitWord64Rol(Node* node) {
1016 VisitWord64Shift(this, node, kX64Rol);
1017 }
1018
VisitWord32Ror(Node * node)1019 void InstructionSelector::VisitWord32Ror(Node* node) {
1020 VisitWord32Shift(this, node, kX64Ror32);
1021 }
1022
VisitWord64Ror(Node * node)1023 void InstructionSelector::VisitWord64Ror(Node* node) {
1024 VisitWord64Shift(this, node, kX64Ror);
1025 }
1026
VisitWord32ReverseBits(Node * node)1027 void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
1028
VisitWord64ReverseBits(Node * node)1029 void InstructionSelector::VisitWord64ReverseBits(Node* node) { UNREACHABLE(); }
1030
VisitWord64ReverseBytes(Node * node)1031 void InstructionSelector::VisitWord64ReverseBytes(Node* node) {
1032 X64OperandGenerator g(this);
1033 Emit(kX64Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
1034 }
1035
VisitWord32ReverseBytes(Node * node)1036 void InstructionSelector::VisitWord32ReverseBytes(Node* node) {
1037 X64OperandGenerator g(this);
1038 Emit(kX64Bswap32, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
1039 }
1040
VisitSimd128ReverseBytes(Node * node)1041 void InstructionSelector::VisitSimd128ReverseBytes(Node* node) {
1042 UNREACHABLE();
1043 }
1044
VisitInt32Add(Node * node)1045 void InstructionSelector::VisitInt32Add(Node* node) {
1046 X64OperandGenerator g(this);
1047
1048 // No need to truncate the values before Int32Add.
1049 DCHECK_EQ(node->InputCount(), 2);
1050 Node* left = node->InputAt(0);
1051 Node* right = node->InputAt(1);
1052 if (left->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1053 node->ReplaceInput(0, left->InputAt(0));
1054 }
1055 if (right->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1056 node->ReplaceInput(1, right->InputAt(0));
1057 }
1058
1059 // Try to match the Add to a leal pattern
1060 BaseWithIndexAndDisplacement32Matcher m(node);
1061 if (m.matches() &&
1062 (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
1063 EmitLea(this, kX64Lea32, node, m.index(), m.scale(), m.base(),
1064 m.displacement(), m.displacement_mode());
1065 return;
1066 }
1067
1068 // No leal pattern match, use addl
1069 VisitBinop(this, node, kX64Add32);
1070 }
1071
VisitInt64Add(Node * node)1072 void InstructionSelector::VisitInt64Add(Node* node) {
1073 X64OperandGenerator g(this);
1074
1075 // Try to match the Add to a leaq pattern
1076 BaseWithIndexAndDisplacement64Matcher m(node);
1077 if (m.matches() &&
1078 (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
1079 EmitLea(this, kX64Lea, node, m.index(), m.scale(), m.base(),
1080 m.displacement(), m.displacement_mode());
1081 return;
1082 }
1083
1084 // No leal pattern match, use addq
1085 VisitBinop(this, node, kX64Add);
1086 }
1087
VisitInt64AddWithOverflow(Node * node)1088 void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
1089 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1090 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1091 return VisitBinop(this, node, kX64Add, &cont);
1092 }
1093 FlagsContinuation cont;
1094 VisitBinop(this, node, kX64Add, &cont);
1095 }
1096
VisitInt32Sub(Node * node)1097 void InstructionSelector::VisitInt32Sub(Node* node) {
1098 X64OperandGenerator g(this);
1099 DCHECK_EQ(node->InputCount(), 2);
1100 Node* input1 = node->InputAt(0);
1101 Node* input2 = node->InputAt(1);
1102 if (input1->opcode() == IrOpcode::kTruncateInt64ToInt32 &&
1103 g.CanBeImmediate(input2)) {
1104 int32_t imm = g.GetImmediateIntegerValue(input2);
1105 InstructionOperand int64_input = g.UseRegister(input1->InputAt(0));
1106 if (imm == 0) {
1107 // Emit "movl" for subtraction of 0.
1108 Emit(kX64Movl, g.DefineAsRegister(node), int64_input);
1109 } else {
1110 // Omit truncation and turn subtractions of constant values into immediate
1111 // "leal" instructions by negating the value.
1112 Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI),
1113 g.DefineAsRegister(node), int64_input,
1114 g.TempImmediate(base::NegateWithWraparound(imm)));
1115 }
1116 return;
1117 }
1118
1119 Int32BinopMatcher m(node);
1120 if (m.left().Is(0)) {
1121 Emit(kX64Neg32, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
1122 } else if (m.right().Is(0)) {
1123 // {EmitIdentity} reuses the virtual register of the first input
1124 // for the output. This is exactly what we want here.
1125 EmitIdentity(node);
1126 } else if (m.right().HasResolvedValue() &&
1127 g.CanBeImmediate(m.right().node())) {
1128 // Turn subtractions of constant values into immediate "leal" instructions
1129 // by negating the value.
1130 Emit(
1131 kX64Lea32 | AddressingModeField::encode(kMode_MRI),
1132 g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1133 g.TempImmediate(base::NegateWithWraparound(m.right().ResolvedValue())));
1134 } else {
1135 VisitBinop(this, node, kX64Sub32);
1136 }
1137 }
1138
VisitInt64Sub(Node * node)1139 void InstructionSelector::VisitInt64Sub(Node* node) {
1140 X64OperandGenerator g(this);
1141 Int64BinopMatcher m(node);
1142 if (m.left().Is(0)) {
1143 Emit(kX64Neg, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
1144 } else {
1145 if (m.right().HasResolvedValue() && g.CanBeImmediate(m.right().node())) {
1146 // Turn subtractions of constant values into immediate "leaq" instructions
1147 // by negating the value.
1148 Emit(kX64Lea | AddressingModeField::encode(kMode_MRI),
1149 g.DefineAsRegister(node), g.UseRegister(m.left().node()),
1150 g.TempImmediate(-static_cast<int32_t>(m.right().ResolvedValue())));
1151 return;
1152 }
1153 VisitBinop(this, node, kX64Sub);
1154 }
1155 }
1156
VisitInt64SubWithOverflow(Node * node)1157 void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
1158 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1159 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1160 return VisitBinop(this, node, kX64Sub, &cont);
1161 }
1162 FlagsContinuation cont;
1163 VisitBinop(this, node, kX64Sub, &cont);
1164 }
1165
1166 namespace {
1167
VisitMul(InstructionSelector * selector,Node * node,ArchOpcode opcode)1168 void VisitMul(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1169 X64OperandGenerator g(selector);
1170 Int32BinopMatcher m(node);
1171 Node* left = m.left().node();
1172 Node* right = m.right().node();
1173 if (g.CanBeImmediate(right)) {
1174 selector->Emit(opcode, g.DefineAsRegister(node), g.Use(left),
1175 g.UseImmediate(right));
1176 } else {
1177 if (g.CanBeBetterLeftOperand(right)) {
1178 std::swap(left, right);
1179 }
1180 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
1181 g.Use(right));
1182 }
1183 }
1184
VisitMulHigh(InstructionSelector * selector,Node * node,ArchOpcode opcode)1185 void VisitMulHigh(InstructionSelector* selector, Node* node,
1186 ArchOpcode opcode) {
1187 X64OperandGenerator g(selector);
1188 Node* left = node->InputAt(0);
1189 Node* right = node->InputAt(1);
1190 if (selector->IsLive(left) && !selector->IsLive(right)) {
1191 std::swap(left, right);
1192 }
1193 InstructionOperand temps[] = {g.TempRegister(rax)};
1194 // TODO(turbofan): We use UseUniqueRegister here to improve register
1195 // allocation.
1196 selector->Emit(opcode, g.DefineAsFixed(node, rdx), g.UseFixed(left, rax),
1197 g.UseUniqueRegister(right), arraysize(temps), temps);
1198 }
1199
VisitDiv(InstructionSelector * selector,Node * node,ArchOpcode opcode)1200 void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1201 X64OperandGenerator g(selector);
1202 InstructionOperand temps[] = {g.TempRegister(rdx)};
1203 selector->Emit(
1204 opcode, g.DefineAsFixed(node, rax), g.UseFixed(node->InputAt(0), rax),
1205 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
1206 }
1207
VisitMod(InstructionSelector * selector,Node * node,ArchOpcode opcode)1208 void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1209 X64OperandGenerator g(selector);
1210 InstructionOperand temps[] = {g.TempRegister(rax)};
1211 selector->Emit(
1212 opcode, g.DefineAsFixed(node, rdx), g.UseFixed(node->InputAt(0), rax),
1213 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
1214 }
1215
1216 } // namespace
1217
VisitInt32Mul(Node * node)1218 void InstructionSelector::VisitInt32Mul(Node* node) {
1219 Int32ScaleMatcher m(node, true);
1220 if (m.matches()) {
1221 Node* index = node->InputAt(0);
1222 Node* base = m.power_of_two_plus_one() ? index : nullptr;
1223 EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
1224 kPositiveDisplacement);
1225 return;
1226 }
1227 VisitMul(this, node, kX64Imul32);
1228 }
1229
VisitInt32MulWithOverflow(Node * node)1230 void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
1231 // TODO(mvstanton): Use Int32ScaleMatcher somehow.
1232 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1233 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1234 return VisitBinop(this, node, kX64Imul32, &cont);
1235 }
1236 FlagsContinuation cont;
1237 VisitBinop(this, node, kX64Imul32, &cont);
1238 }
1239
VisitInt64Mul(Node * node)1240 void InstructionSelector::VisitInt64Mul(Node* node) {
1241 VisitMul(this, node, kX64Imul);
1242 }
1243
VisitInt32MulHigh(Node * node)1244 void InstructionSelector::VisitInt32MulHigh(Node* node) {
1245 VisitMulHigh(this, node, kX64ImulHigh32);
1246 }
1247
VisitInt32Div(Node * node)1248 void InstructionSelector::VisitInt32Div(Node* node) {
1249 VisitDiv(this, node, kX64Idiv32);
1250 }
1251
VisitInt64Div(Node * node)1252 void InstructionSelector::VisitInt64Div(Node* node) {
1253 VisitDiv(this, node, kX64Idiv);
1254 }
1255
VisitUint32Div(Node * node)1256 void InstructionSelector::VisitUint32Div(Node* node) {
1257 VisitDiv(this, node, kX64Udiv32);
1258 }
1259
VisitUint64Div(Node * node)1260 void InstructionSelector::VisitUint64Div(Node* node) {
1261 VisitDiv(this, node, kX64Udiv);
1262 }
1263
VisitInt32Mod(Node * node)1264 void InstructionSelector::VisitInt32Mod(Node* node) {
1265 VisitMod(this, node, kX64Idiv32);
1266 }
1267
VisitInt64Mod(Node * node)1268 void InstructionSelector::VisitInt64Mod(Node* node) {
1269 VisitMod(this, node, kX64Idiv);
1270 }
1271
VisitUint32Mod(Node * node)1272 void InstructionSelector::VisitUint32Mod(Node* node) {
1273 VisitMod(this, node, kX64Udiv32);
1274 }
1275
VisitUint64Mod(Node * node)1276 void InstructionSelector::VisitUint64Mod(Node* node) {
1277 VisitMod(this, node, kX64Udiv);
1278 }
1279
VisitUint32MulHigh(Node * node)1280 void InstructionSelector::VisitUint32MulHigh(Node* node) {
1281 VisitMulHigh(this, node, kX64UmulHigh32);
1282 }
1283
VisitTryTruncateFloat32ToInt64(Node * node)1284 void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
1285 X64OperandGenerator g(this);
1286 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1287 InstructionOperand outputs[2];
1288 size_t output_count = 0;
1289 outputs[output_count++] = g.DefineAsRegister(node);
1290
1291 Node* success_output = NodeProperties::FindProjection(node, 1);
1292 if (success_output) {
1293 outputs[output_count++] = g.DefineAsRegister(success_output);
1294 }
1295
1296 Emit(kSSEFloat32ToInt64, output_count, outputs, 1, inputs);
1297 }
1298
VisitTryTruncateFloat64ToInt64(Node * node)1299 void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
1300 X64OperandGenerator g(this);
1301 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1302 InstructionOperand outputs[2];
1303 size_t output_count = 0;
1304 outputs[output_count++] = g.DefineAsRegister(node);
1305
1306 Node* success_output = NodeProperties::FindProjection(node, 1);
1307 if (success_output) {
1308 outputs[output_count++] = g.DefineAsRegister(success_output);
1309 }
1310
1311 Emit(kSSEFloat64ToInt64, output_count, outputs, 1, inputs);
1312 }
1313
VisitTryTruncateFloat32ToUint64(Node * node)1314 void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
1315 X64OperandGenerator g(this);
1316 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1317 InstructionOperand outputs[2];
1318 size_t output_count = 0;
1319 outputs[output_count++] = g.DefineAsRegister(node);
1320
1321 Node* success_output = NodeProperties::FindProjection(node, 1);
1322 if (success_output) {
1323 outputs[output_count++] = g.DefineAsRegister(success_output);
1324 }
1325
1326 Emit(kSSEFloat32ToUint64, output_count, outputs, 1, inputs);
1327 }
1328
VisitTryTruncateFloat64ToUint64(Node * node)1329 void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
1330 X64OperandGenerator g(this);
1331 InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1332 InstructionOperand outputs[2];
1333 size_t output_count = 0;
1334 outputs[output_count++] = g.DefineAsRegister(node);
1335
1336 Node* success_output = NodeProperties::FindProjection(node, 1);
1337 if (success_output) {
1338 outputs[output_count++] = g.DefineAsRegister(success_output);
1339 }
1340
1341 Emit(kSSEFloat64ToUint64, output_count, outputs, 1, inputs);
1342 }
1343
VisitBitcastWord32ToWord64(Node * node)1344 void InstructionSelector::VisitBitcastWord32ToWord64(Node* node) {
1345 DCHECK(SmiValuesAre31Bits());
1346 DCHECK(COMPRESS_POINTERS_BOOL);
1347 EmitIdentity(node);
1348 }
1349
VisitChangeInt32ToInt64(Node * node)1350 void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
1351 DCHECK_EQ(node->InputCount(), 1);
1352 Node* input = node->InputAt(0);
1353 if (input->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1354 node->ReplaceInput(0, input->InputAt(0));
1355 }
1356
1357 X64OperandGenerator g(this);
1358 Node* const value = node->InputAt(0);
1359 if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
1360 LoadRepresentation load_rep = LoadRepresentationOf(value->op());
1361 MachineRepresentation rep = load_rep.representation();
1362 InstructionCode opcode;
1363 switch (rep) {
1364 case MachineRepresentation::kBit: // Fall through.
1365 case MachineRepresentation::kWord8:
1366 opcode = load_rep.IsSigned() ? kX64Movsxbq : kX64Movzxbq;
1367 break;
1368 case MachineRepresentation::kWord16:
1369 opcode = load_rep.IsSigned() ? kX64Movsxwq : kX64Movzxwq;
1370 break;
1371 case MachineRepresentation::kWord32:
1372 opcode = load_rep.IsSigned() ? kX64Movsxlq : kX64Movl;
1373 break;
1374 default:
1375 UNREACHABLE();
1376 }
1377 InstructionOperand outputs[] = {g.DefineAsRegister(node)};
1378 size_t input_count = 0;
1379 InstructionOperand inputs[3];
1380 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1381 node->InputAt(0), inputs, &input_count);
1382 opcode |= AddressingModeField::encode(mode);
1383 Emit(opcode, 1, outputs, input_count, inputs);
1384 } else {
1385 Emit(kX64Movsxlq, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1386 }
1387 }
1388
ZeroExtendsWord32ToWord64NoPhis(Node * node)1389 bool InstructionSelector::ZeroExtendsWord32ToWord64NoPhis(Node* node) {
1390 X64OperandGenerator g(this);
1391 DCHECK_NE(node->opcode(), IrOpcode::kPhi);
1392 switch (node->opcode()) {
1393 case IrOpcode::kWord32And:
1394 case IrOpcode::kWord32Or:
1395 case IrOpcode::kWord32Xor:
1396 case IrOpcode::kWord32Shl:
1397 case IrOpcode::kWord32Shr:
1398 case IrOpcode::kWord32Sar:
1399 case IrOpcode::kWord32Rol:
1400 case IrOpcode::kWord32Ror:
1401 case IrOpcode::kWord32Equal:
1402 case IrOpcode::kInt32Add:
1403 case IrOpcode::kInt32Sub:
1404 case IrOpcode::kInt32Mul:
1405 case IrOpcode::kInt32MulHigh:
1406 case IrOpcode::kInt32Div:
1407 case IrOpcode::kInt32LessThan:
1408 case IrOpcode::kInt32LessThanOrEqual:
1409 case IrOpcode::kInt32Mod:
1410 case IrOpcode::kUint32Div:
1411 case IrOpcode::kUint32LessThan:
1412 case IrOpcode::kUint32LessThanOrEqual:
1413 case IrOpcode::kUint32Mod:
1414 case IrOpcode::kUint32MulHigh:
1415 case IrOpcode::kTruncateInt64ToInt32:
1416 // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1417 // zero-extension is a no-op.
1418 return true;
1419 case IrOpcode::kProjection: {
1420 Node* const value = node->InputAt(0);
1421 switch (value->opcode()) {
1422 case IrOpcode::kInt32AddWithOverflow:
1423 case IrOpcode::kInt32SubWithOverflow:
1424 case IrOpcode::kInt32MulWithOverflow:
1425 return true;
1426 default:
1427 return false;
1428 }
1429 }
1430 case IrOpcode::kLoad:
1431 case IrOpcode::kProtectedLoad:
1432 case IrOpcode::kPoisonedLoad: {
1433 // The movzxbl/movsxbl/movzxwl/movsxwl/movl operations implicitly
1434 // zero-extend to 64-bit on x64, so the zero-extension is a no-op.
1435 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
1436 switch (load_rep.representation()) {
1437 case MachineRepresentation::kWord8:
1438 case MachineRepresentation::kWord16:
1439 case MachineRepresentation::kWord32:
1440 return true;
1441 default:
1442 return false;
1443 }
1444 }
1445 case IrOpcode::kInt32Constant:
1446 case IrOpcode::kInt64Constant:
1447 // Constants are loaded with movl or movq, or xorl for zero; see
1448 // CodeGenerator::AssembleMove. So any non-negative constant that fits
1449 // in a 32-bit signed integer is zero-extended to 64 bits.
1450 if (g.CanBeImmediate(node)) {
1451 return g.GetImmediateIntegerValue(node) >= 0;
1452 }
1453 return false;
1454 default:
1455 return false;
1456 }
1457 }
1458
VisitChangeUint32ToUint64(Node * node)1459 void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
1460 X64OperandGenerator g(this);
1461 Node* value = node->InputAt(0);
1462 if (ZeroExtendsWord32ToWord64(value)) {
1463 // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1464 // zero-extension is a no-op.
1465 return EmitIdentity(node);
1466 }
1467 Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1468 }
1469
1470 namespace {
1471
VisitRO(InstructionSelector * selector,Node * node,InstructionCode opcode)1472 void VisitRO(InstructionSelector* selector, Node* node,
1473 InstructionCode opcode) {
1474 X64OperandGenerator g(selector);
1475 selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1476 }
1477
VisitRR(InstructionSelector * selector,Node * node,InstructionCode opcode)1478 void VisitRR(InstructionSelector* selector, Node* node,
1479 InstructionCode opcode) {
1480 X64OperandGenerator g(selector);
1481 selector->Emit(opcode, g.DefineAsRegister(node),
1482 g.UseRegister(node->InputAt(0)));
1483 }
1484
VisitRRO(InstructionSelector * selector,Node * node,InstructionCode opcode)1485 void VisitRRO(InstructionSelector* selector, Node* node,
1486 InstructionCode opcode) {
1487 X64OperandGenerator g(selector);
1488 selector->Emit(opcode, g.DefineSameAsFirst(node),
1489 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)));
1490 }
1491
VisitFloatBinop(InstructionSelector * selector,Node * node,InstructionCode avx_opcode,InstructionCode sse_opcode)1492 void VisitFloatBinop(InstructionSelector* selector, Node* node,
1493 InstructionCode avx_opcode, InstructionCode sse_opcode) {
1494 X64OperandGenerator g(selector);
1495 Node* left = node->InputAt(0);
1496 Node* right = node->InputAt(1);
1497 InstructionOperand inputs[8];
1498 size_t input_count = 0;
1499 InstructionOperand outputs[1];
1500 size_t output_count = 0;
1501
1502 if (left == right) {
1503 // If both inputs refer to the same operand, enforce allocating a register
1504 // for both of them to ensure that we don't end up generating code like
1505 // this:
1506 //
1507 // movss rax, [rbp-0x10]
1508 // addss rax, [rbp-0x10]
1509 // jo label
1510 InstructionOperand const input = g.UseRegister(left);
1511 inputs[input_count++] = input;
1512 inputs[input_count++] = input;
1513 } else {
1514 int effect_level = selector->GetEffectLevel(node);
1515 if (node->op()->HasProperty(Operator::kCommutative) &&
1516 (g.CanBeBetterLeftOperand(right) ||
1517 g.CanBeMemoryOperand(avx_opcode, node, left, effect_level)) &&
1518 (!g.CanBeBetterLeftOperand(left) ||
1519 !g.CanBeMemoryOperand(avx_opcode, node, right, effect_level))) {
1520 std::swap(left, right);
1521 }
1522 if (g.CanBeMemoryOperand(avx_opcode, node, right, effect_level)) {
1523 inputs[input_count++] = g.UseRegister(left);
1524 AddressingMode addressing_mode =
1525 g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
1526 avx_opcode |= AddressingModeField::encode(addressing_mode);
1527 sse_opcode |= AddressingModeField::encode(addressing_mode);
1528 } else {
1529 inputs[input_count++] = g.UseRegister(left);
1530 inputs[input_count++] = g.Use(right);
1531 }
1532 }
1533
1534 DCHECK_NE(0u, input_count);
1535 DCHECK_GE(arraysize(inputs), input_count);
1536
1537 if (selector->IsSupported(AVX)) {
1538 outputs[output_count++] = g.DefineAsRegister(node);
1539 DCHECK_EQ(1u, output_count);
1540 DCHECK_GE(arraysize(outputs), output_count);
1541 selector->Emit(avx_opcode, output_count, outputs, input_count, inputs);
1542 } else {
1543 outputs[output_count++] = g.DefineSameAsFirst(node);
1544 DCHECK_EQ(1u, output_count);
1545 DCHECK_GE(arraysize(outputs), output_count);
1546 selector->Emit(sse_opcode, output_count, outputs, input_count, inputs);
1547 }
1548 }
1549
VisitFloatUnop(InstructionSelector * selector,Node * node,Node * input,ArchOpcode avx_opcode,ArchOpcode sse_opcode)1550 void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
1551 ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
1552 X64OperandGenerator g(selector);
1553 InstructionOperand temps[] = {g.TempDoubleRegister()};
1554 if (selector->IsSupported(AVX)) {
1555 selector->Emit(avx_opcode, g.DefineAsRegister(node), g.UseUnique(input),
1556 arraysize(temps), temps);
1557 } else {
1558 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), g.UseRegister(input),
1559 arraysize(temps), temps);
1560 }
1561 }
1562
1563 } // namespace
1564
1565 #define RO_OP_LIST(V) \
1566 V(Word64Clz, kX64Lzcnt) \
1567 V(Word32Clz, kX64Lzcnt32) \
1568 V(Word64Ctz, kX64Tzcnt) \
1569 V(Word32Ctz, kX64Tzcnt32) \
1570 V(Word64Popcnt, kX64Popcnt) \
1571 V(Word32Popcnt, kX64Popcnt32) \
1572 V(Float64Sqrt, kSSEFloat64Sqrt) \
1573 V(Float32Sqrt, kSSEFloat32Sqrt) \
1574 V(ChangeFloat64ToInt32, kSSEFloat64ToInt32) \
1575 V(ChangeFloat64ToInt64, kSSEFloat64ToInt64) \
1576 V(ChangeFloat64ToUint32, kSSEFloat64ToUint32 | MiscField::encode(1)) \
1577 V(TruncateFloat64ToInt64, kSSEFloat64ToInt64) \
1578 V(TruncateFloat64ToUint32, kSSEFloat64ToUint32 | MiscField::encode(0)) \
1579 V(ChangeFloat64ToUint64, kSSEFloat64ToUint64) \
1580 V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32) \
1581 V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64) \
1582 V(TruncateFloat32ToInt32, kSSEFloat32ToInt32) \
1583 V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \
1584 V(ChangeInt32ToFloat64, kSSEInt32ToFloat64) \
1585 V(ChangeInt64ToFloat64, kSSEInt64ToFloat64) \
1586 V(ChangeUint32ToFloat64, kSSEUint32ToFloat64) \
1587 V(RoundFloat64ToInt32, kSSEFloat64ToInt32) \
1588 V(RoundInt32ToFloat32, kSSEInt32ToFloat32) \
1589 V(RoundInt64ToFloat32, kSSEInt64ToFloat32) \
1590 V(RoundUint64ToFloat32, kSSEUint64ToFloat32) \
1591 V(RoundInt64ToFloat64, kSSEInt64ToFloat64) \
1592 V(RoundUint64ToFloat64, kSSEUint64ToFloat64) \
1593 V(RoundUint32ToFloat32, kSSEUint32ToFloat32) \
1594 V(BitcastFloat32ToInt32, kX64BitcastFI) \
1595 V(BitcastFloat64ToInt64, kX64BitcastDL) \
1596 V(BitcastInt32ToFloat32, kX64BitcastIF) \
1597 V(BitcastInt64ToFloat64, kX64BitcastLD) \
1598 V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32) \
1599 V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \
1600 V(SignExtendWord8ToInt32, kX64Movsxbl) \
1601 V(SignExtendWord16ToInt32, kX64Movsxwl) \
1602 V(SignExtendWord8ToInt64, kX64Movsxbq) \
1603 V(SignExtendWord16ToInt64, kX64Movsxwq) \
1604 V(SignExtendWord32ToInt64, kX64Movsxlq)
1605
1606 #define RR_OP_LIST(V) \
1607 V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown)) \
1608 V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown)) \
1609 V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp)) \
1610 V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp)) \
1611 V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \
1612 V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \
1613 V(Float32RoundTiesEven, \
1614 kSSEFloat32Round | MiscField::encode(kRoundToNearest)) \
1615 V(Float64RoundTiesEven, \
1616 kSSEFloat64Round | MiscField::encode(kRoundToNearest)) \
1617 V(F32x4Ceil, kX64F32x4Round | MiscField::encode(kRoundUp)) \
1618 V(F32x4Floor, kX64F32x4Round | MiscField::encode(kRoundDown)) \
1619 V(F32x4Trunc, kX64F32x4Round | MiscField::encode(kRoundToZero)) \
1620 V(F32x4NearestInt, kX64F32x4Round | MiscField::encode(kRoundToNearest)) \
1621 V(F64x2Ceil, kX64F64x2Round | MiscField::encode(kRoundUp)) \
1622 V(F64x2Floor, kX64F64x2Round | MiscField::encode(kRoundDown)) \
1623 V(F64x2Trunc, kX64F64x2Round | MiscField::encode(kRoundToZero)) \
1624 V(F64x2NearestInt, kX64F64x2Round | MiscField::encode(kRoundToNearest))
1625
1626 #define RO_VISITOR(Name, opcode) \
1627 void InstructionSelector::Visit##Name(Node* node) { \
1628 VisitRO(this, node, opcode); \
1629 }
1630 RO_OP_LIST(RO_VISITOR)
1631 #undef RO_VISITOR
1632 #undef RO_OP_LIST
1633
1634 #define RR_VISITOR(Name, opcode) \
1635 void InstructionSelector::Visit##Name(Node* node) { \
1636 VisitRR(this, node, opcode); \
1637 }
RR_OP_LIST(RR_VISITOR)1638 RR_OP_LIST(RR_VISITOR)
1639 #undef RR_VISITOR
1640 #undef RR_OP_LIST
1641
1642 void InstructionSelector::VisitTruncateFloat64ToWord32(Node* node) {
1643 VisitRR(this, node, kArchTruncateDoubleToI);
1644 }
1645
VisitTruncateInt64ToInt32(Node * node)1646 void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
1647 // We rely on the fact that TruncateInt64ToInt32 zero extends the
1648 // value (see ZeroExtendsWord32ToWord64). So all code paths here
1649 // have to satisfy that condition.
1650 X64OperandGenerator g(this);
1651 Node* value = node->InputAt(0);
1652 if (CanCover(node, value)) {
1653 switch (value->opcode()) {
1654 case IrOpcode::kWord64Sar:
1655 case IrOpcode::kWord64Shr: {
1656 Int64BinopMatcher m(value);
1657 if (m.right().Is(32)) {
1658 if (CanCoverTransitively(node, value, value->InputAt(0)) &&
1659 TryMatchLoadWord64AndShiftRight(this, value, kX64Movl)) {
1660 return EmitIdentity(node);
1661 }
1662 Emit(kX64Shr, g.DefineSameAsFirst(node),
1663 g.UseRegister(m.left().node()), g.TempImmediate(32));
1664 return;
1665 }
1666 break;
1667 }
1668 case IrOpcode::kLoad: {
1669 if (TryMergeTruncateInt64ToInt32IntoLoad(this, node, value)) {
1670 return;
1671 }
1672 break;
1673 }
1674 default:
1675 break;
1676 }
1677 }
1678 Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1679 }
1680
VisitFloat32Add(Node * node)1681 void InstructionSelector::VisitFloat32Add(Node* node) {
1682 VisitFloatBinop(this, node, kAVXFloat32Add, kSSEFloat32Add);
1683 }
1684
VisitFloat32Sub(Node * node)1685 void InstructionSelector::VisitFloat32Sub(Node* node) {
1686 VisitFloatBinop(this, node, kAVXFloat32Sub, kSSEFloat32Sub);
1687 }
1688
VisitFloat32Mul(Node * node)1689 void InstructionSelector::VisitFloat32Mul(Node* node) {
1690 VisitFloatBinop(this, node, kAVXFloat32Mul, kSSEFloat32Mul);
1691 }
1692
VisitFloat32Div(Node * node)1693 void InstructionSelector::VisitFloat32Div(Node* node) {
1694 VisitFloatBinop(this, node, kAVXFloat32Div, kSSEFloat32Div);
1695 }
1696
VisitFloat32Abs(Node * node)1697 void InstructionSelector::VisitFloat32Abs(Node* node) {
1698 VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat32Abs, kSSEFloat32Abs);
1699 }
1700
VisitFloat32Max(Node * node)1701 void InstructionSelector::VisitFloat32Max(Node* node) {
1702 VisitRRO(this, node, kSSEFloat32Max);
1703 }
1704
VisitFloat32Min(Node * node)1705 void InstructionSelector::VisitFloat32Min(Node* node) {
1706 VisitRRO(this, node, kSSEFloat32Min);
1707 }
1708
VisitFloat64Add(Node * node)1709 void InstructionSelector::VisitFloat64Add(Node* node) {
1710 VisitFloatBinop(this, node, kAVXFloat64Add, kSSEFloat64Add);
1711 }
1712
VisitFloat64Sub(Node * node)1713 void InstructionSelector::VisitFloat64Sub(Node* node) {
1714 VisitFloatBinop(this, node, kAVXFloat64Sub, kSSEFloat64Sub);
1715 }
1716
VisitFloat64Mul(Node * node)1717 void InstructionSelector::VisitFloat64Mul(Node* node) {
1718 VisitFloatBinop(this, node, kAVXFloat64Mul, kSSEFloat64Mul);
1719 }
1720
VisitFloat64Div(Node * node)1721 void InstructionSelector::VisitFloat64Div(Node* node) {
1722 VisitFloatBinop(this, node, kAVXFloat64Div, kSSEFloat64Div);
1723 }
1724
VisitFloat64Mod(Node * node)1725 void InstructionSelector::VisitFloat64Mod(Node* node) {
1726 X64OperandGenerator g(this);
1727 InstructionOperand temps[] = {g.TempRegister(rax)};
1728 Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
1729 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), 1,
1730 temps);
1731 }
1732
VisitFloat64Max(Node * node)1733 void InstructionSelector::VisitFloat64Max(Node* node) {
1734 VisitRRO(this, node, kSSEFloat64Max);
1735 }
1736
VisitFloat64Min(Node * node)1737 void InstructionSelector::VisitFloat64Min(Node* node) {
1738 VisitRRO(this, node, kSSEFloat64Min);
1739 }
1740
VisitFloat64Abs(Node * node)1741 void InstructionSelector::VisitFloat64Abs(Node* node) {
1742 VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat64Abs, kSSEFloat64Abs);
1743 }
1744
VisitFloat64RoundTiesAway(Node * node)1745 void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
1746 UNREACHABLE();
1747 }
1748
VisitFloat32Neg(Node * node)1749 void InstructionSelector::VisitFloat32Neg(Node* node) {
1750 VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat32Neg, kSSEFloat32Neg);
1751 }
1752
VisitFloat64Neg(Node * node)1753 void InstructionSelector::VisitFloat64Neg(Node* node) {
1754 VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat64Neg, kSSEFloat64Neg);
1755 }
1756
VisitFloat64Ieee754Binop(Node * node,InstructionCode opcode)1757 void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
1758 InstructionCode opcode) {
1759 X64OperandGenerator g(this);
1760 Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0),
1761 g.UseFixed(node->InputAt(1), xmm1))
1762 ->MarkAsCall();
1763 }
1764
VisitFloat64Ieee754Unop(Node * node,InstructionCode opcode)1765 void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
1766 InstructionCode opcode) {
1767 X64OperandGenerator g(this);
1768 Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0))
1769 ->MarkAsCall();
1770 }
1771
EmitPrepareArguments(ZoneVector<PushParameter> * arguments,const CallDescriptor * call_descriptor,Node * node)1772 void InstructionSelector::EmitPrepareArguments(
1773 ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
1774 Node* node) {
1775 X64OperandGenerator g(this);
1776
1777 // Prepare for C function call.
1778 if (call_descriptor->IsCFunctionCall()) {
1779 Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>(
1780 call_descriptor->ParameterCount())),
1781 0, nullptr, 0, nullptr);
1782
1783 // Poke any stack arguments.
1784 for (size_t n = 0; n < arguments->size(); ++n) {
1785 PushParameter input = (*arguments)[n];
1786 if (input.node) {
1787 int slot = static_cast<int>(n);
1788 InstructionOperand value = g.CanBeImmediate(input.node)
1789 ? g.UseImmediate(input.node)
1790 : g.UseRegister(input.node);
1791 Emit(kX64Poke | MiscField::encode(slot), g.NoOutput(), value);
1792 }
1793 }
1794 } else {
1795 // Push any stack arguments.
1796 int effect_level = GetEffectLevel(node);
1797 for (PushParameter input : base::Reversed(*arguments)) {
1798 // Skip any alignment holes in pushed nodes. We may have one in case of a
1799 // Simd128 stack argument.
1800 if (input.node == nullptr) continue;
1801 if (g.CanBeImmediate(input.node)) {
1802 Emit(kX64Push, g.NoOutput(), g.UseImmediate(input.node));
1803 } else if (IsSupported(ATOM) ||
1804 sequence()->IsFP(GetVirtualRegister(input.node))) {
1805 // TODO(titzer): X64Push cannot handle stack->stack double moves
1806 // because there is no way to encode fixed double slots.
1807 Emit(kX64Push, g.NoOutput(), g.UseRegister(input.node));
1808 } else if (g.CanBeMemoryOperand(kX64Push, node, input.node,
1809 effect_level)) {
1810 InstructionOperand outputs[1];
1811 InstructionOperand inputs[4];
1812 size_t input_count = 0;
1813 InstructionCode opcode = kX64Push;
1814 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1815 input.node, inputs, &input_count);
1816 opcode |= AddressingModeField::encode(mode);
1817 Emit(opcode, 0, outputs, input_count, inputs);
1818 } else {
1819 Emit(kX64Push, g.NoOutput(), g.UseAny(input.node));
1820 }
1821 }
1822 }
1823 }
1824
EmitPrepareResults(ZoneVector<PushParameter> * results,const CallDescriptor * call_descriptor,Node * node)1825 void InstructionSelector::EmitPrepareResults(
1826 ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
1827 Node* node) {
1828 X64OperandGenerator g(this);
1829
1830 int reverse_slot = 1;
1831 for (PushParameter output : *results) {
1832 if (!output.location.IsCallerFrameSlot()) continue;
1833 // Skip any alignment holes in nodes.
1834 if (output.node != nullptr) {
1835 DCHECK(!call_descriptor->IsCFunctionCall());
1836 if (output.location.GetType() == MachineType::Float32()) {
1837 MarkAsFloat32(output.node);
1838 } else if (output.location.GetType() == MachineType::Float64()) {
1839 MarkAsFloat64(output.node);
1840 } else if (output.location.GetType() == MachineType::Simd128()) {
1841 MarkAsSimd128(output.node);
1842 }
1843 InstructionOperand result = g.DefineAsRegister(output.node);
1844 InstructionOperand slot = g.UseImmediate(reverse_slot);
1845 Emit(kX64Peek, 1, &result, 1, &slot);
1846 }
1847 reverse_slot += output.location.GetSizeInPointers();
1848 }
1849 }
1850
IsTailCallAddressImmediate()1851 bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
1852
GetTempsCountForTailCallFromJSFunction()1853 int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 3; }
1854
1855 namespace {
1856
VisitCompareWithMemoryOperand(InstructionSelector * selector,InstructionCode opcode,Node * left,InstructionOperand right,FlagsContinuation * cont)1857 void VisitCompareWithMemoryOperand(InstructionSelector* selector,
1858 InstructionCode opcode, Node* left,
1859 InstructionOperand right,
1860 FlagsContinuation* cont) {
1861 DCHECK_EQ(IrOpcode::kLoad, left->opcode());
1862 X64OperandGenerator g(selector);
1863 size_t input_count = 0;
1864 InstructionOperand inputs[4];
1865 AddressingMode addressing_mode =
1866 g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
1867 opcode |= AddressingModeField::encode(addressing_mode);
1868 inputs[input_count++] = right;
1869
1870 selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
1871 }
1872
1873 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,InstructionOperand left,InstructionOperand right,FlagsContinuation * cont)1874 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1875 InstructionOperand left, InstructionOperand right,
1876 FlagsContinuation* cont) {
1877 selector->EmitWithContinuation(opcode, left, right, cont);
1878 }
1879
1880 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,Node * left,Node * right,FlagsContinuation * cont,bool commutative)1881 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1882 Node* left, Node* right, FlagsContinuation* cont,
1883 bool commutative) {
1884 X64OperandGenerator g(selector);
1885 if (commutative && g.CanBeBetterLeftOperand(right)) {
1886 std::swap(left, right);
1887 }
1888 VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
1889 }
1890
MachineTypeForNarrow(Node * node,Node * hint_node)1891 MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
1892 if (hint_node->opcode() == IrOpcode::kLoad) {
1893 MachineType hint = LoadRepresentationOf(hint_node->op());
1894 if (node->opcode() == IrOpcode::kInt32Constant ||
1895 node->opcode() == IrOpcode::kInt64Constant) {
1896 int64_t constant = node->opcode() == IrOpcode::kInt32Constant
1897 ? OpParameter<int32_t>(node->op())
1898 : OpParameter<int64_t>(node->op());
1899 if (hint == MachineType::Int8()) {
1900 if (constant >= std::numeric_limits<int8_t>::min() &&
1901 constant <= std::numeric_limits<int8_t>::max()) {
1902 return hint;
1903 }
1904 } else if (hint == MachineType::Uint8()) {
1905 if (constant >= std::numeric_limits<uint8_t>::min() &&
1906 constant <= std::numeric_limits<uint8_t>::max()) {
1907 return hint;
1908 }
1909 } else if (hint == MachineType::Int16()) {
1910 if (constant >= std::numeric_limits<int16_t>::min() &&
1911 constant <= std::numeric_limits<int16_t>::max()) {
1912 return hint;
1913 }
1914 } else if (hint == MachineType::Uint16()) {
1915 if (constant >= std::numeric_limits<uint16_t>::min() &&
1916 constant <= std::numeric_limits<uint16_t>::max()) {
1917 return hint;
1918 }
1919 } else if (hint == MachineType::Int32()) {
1920 return hint;
1921 } else if (hint == MachineType::Uint32()) {
1922 if (constant >= 0) return hint;
1923 }
1924 }
1925 }
1926 return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op())
1927 : MachineType::None();
1928 }
1929
1930 // Tries to match the size of the given opcode to that of the operands, if
1931 // possible.
TryNarrowOpcodeSize(InstructionCode opcode,Node * left,Node * right,FlagsContinuation * cont)1932 InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
1933 Node* right, FlagsContinuation* cont) {
1934 // TODO(epertoso): we can probably get some size information out phi nodes.
1935 // If the load representations don't match, both operands will be
1936 // zero/sign-extended to 32bit.
1937 MachineType left_type = MachineTypeForNarrow(left, right);
1938 MachineType right_type = MachineTypeForNarrow(right, left);
1939 if (left_type == right_type) {
1940 switch (left_type.representation()) {
1941 case MachineRepresentation::kBit:
1942 case MachineRepresentation::kWord8: {
1943 if (opcode == kX64Test32) return kX64Test8;
1944 if (opcode == kX64Cmp32) {
1945 if (left_type.semantic() == MachineSemantic::kUint32) {
1946 cont->OverwriteUnsignedIfSigned();
1947 } else {
1948 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1949 }
1950 return kX64Cmp8;
1951 }
1952 break;
1953 }
1954 case MachineRepresentation::kWord16:
1955 if (opcode == kX64Test32) return kX64Test16;
1956 if (opcode == kX64Cmp32) {
1957 if (left_type.semantic() == MachineSemantic::kUint32) {
1958 cont->OverwriteUnsignedIfSigned();
1959 } else {
1960 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1961 }
1962 return kX64Cmp16;
1963 }
1964 break;
1965 #ifdef V8_COMPRESS_POINTERS
1966 case MachineRepresentation::kTaggedSigned:
1967 case MachineRepresentation::kTaggedPointer:
1968 case MachineRepresentation::kTagged:
1969 // When pointer compression is enabled the lower 32-bits uniquely
1970 // identify tagged value.
1971 if (opcode == kX64Cmp) return kX64Cmp32;
1972 break;
1973 #endif
1974 default:
1975 break;
1976 }
1977 }
1978 return opcode;
1979 }
1980
1981 // Shared routine for multiple word compare operations.
VisitWordCompare(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)1982 void VisitWordCompare(InstructionSelector* selector, Node* node,
1983 InstructionCode opcode, FlagsContinuation* cont) {
1984 X64OperandGenerator g(selector);
1985 Node* left = node->InputAt(0);
1986 Node* right = node->InputAt(1);
1987
1988 // The 32-bit comparisons automatically truncate Word64
1989 // values to Word32 range, no need to do that explicitly.
1990 if (opcode == kX64Cmp32 || opcode == kX64Test32) {
1991 if (left->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1992 left = left->InputAt(0);
1993 }
1994
1995 if (right->opcode() == IrOpcode::kTruncateInt64ToInt32) {
1996 right = right->InputAt(0);
1997 }
1998 }
1999
2000 opcode = TryNarrowOpcodeSize(opcode, left, right, cont);
2001
2002 // If one of the two inputs is an immediate, make sure it's on the right, or
2003 // if one of the two inputs is a memory operand, make sure it's on the left.
2004 int effect_level = selector->GetEffectLevel(node, cont);
2005
2006 if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
2007 (g.CanBeMemoryOperand(opcode, node, right, effect_level) &&
2008 !g.CanBeMemoryOperand(opcode, node, left, effect_level))) {
2009 if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
2010 std::swap(left, right);
2011 }
2012
2013 // Match immediates on right side of comparison.
2014 if (g.CanBeImmediate(right)) {
2015 if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
2016 return VisitCompareWithMemoryOperand(selector, opcode, left,
2017 g.UseImmediate(right), cont);
2018 }
2019 return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
2020 cont);
2021 }
2022
2023 // Match memory operands on left side of comparison.
2024 if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
2025 return VisitCompareWithMemoryOperand(selector, opcode, left,
2026 g.UseRegister(right), cont);
2027 }
2028
2029 return VisitCompare(selector, opcode, left, right, cont,
2030 node->op()->HasProperty(Operator::kCommutative));
2031 }
2032
VisitWord64EqualImpl(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2033 void VisitWord64EqualImpl(InstructionSelector* selector, Node* node,
2034 FlagsContinuation* cont) {
2035 if (selector->CanUseRootsRegister()) {
2036 X64OperandGenerator g(selector);
2037 const RootsTable& roots_table = selector->isolate()->roots_table();
2038 RootIndex root_index;
2039 HeapObjectBinopMatcher m(node);
2040 if (m.right().HasResolvedValue() &&
2041 roots_table.IsRootHandle(m.right().ResolvedValue(), &root_index)) {
2042 InstructionCode opcode =
2043 kX64Cmp | AddressingModeField::encode(kMode_Root);
2044 return VisitCompare(
2045 selector, opcode,
2046 g.TempImmediate(
2047 TurboAssemblerBase::RootRegisterOffsetForRootIndex(root_index)),
2048 g.UseRegister(m.left().node()), cont);
2049 }
2050 }
2051 VisitWordCompare(selector, node, kX64Cmp, cont);
2052 }
2053
VisitWord32EqualImpl(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2054 void VisitWord32EqualImpl(InstructionSelector* selector, Node* node,
2055 FlagsContinuation* cont) {
2056 if (COMPRESS_POINTERS_BOOL && selector->CanUseRootsRegister()) {
2057 X64OperandGenerator g(selector);
2058 const RootsTable& roots_table = selector->isolate()->roots_table();
2059 RootIndex root_index;
2060 Node* left = nullptr;
2061 Handle<HeapObject> right;
2062 // HeapConstants and CompressedHeapConstants can be treated the same when
2063 // using them as an input to a 32-bit comparison. Check whether either is
2064 // present.
2065 {
2066 CompressedHeapObjectBinopMatcher m(node);
2067 if (m.right().HasResolvedValue()) {
2068 left = m.left().node();
2069 right = m.right().ResolvedValue();
2070 } else {
2071 HeapObjectBinopMatcher m2(node);
2072 if (m2.right().HasResolvedValue()) {
2073 left = m2.left().node();
2074 right = m2.right().ResolvedValue();
2075 }
2076 }
2077 }
2078 if (!right.is_null() && roots_table.IsRootHandle(right, &root_index)) {
2079 DCHECK_NE(left, nullptr);
2080 InstructionCode opcode =
2081 kX64Cmp32 | AddressingModeField::encode(kMode_Root);
2082 return VisitCompare(
2083 selector, opcode,
2084 g.TempImmediate(
2085 TurboAssemblerBase::RootRegisterOffsetForRootIndex(root_index)),
2086 g.UseRegister(left), cont);
2087 }
2088 }
2089 VisitWordCompare(selector, node, kX64Cmp32, cont);
2090 }
2091
2092 // Shared routine for comparison with zero.
VisitCompareZero(InstructionSelector * selector,Node * user,Node * node,InstructionCode opcode,FlagsContinuation * cont)2093 void VisitCompareZero(InstructionSelector* selector, Node* user, Node* node,
2094 InstructionCode opcode, FlagsContinuation* cont) {
2095 X64OperandGenerator g(selector);
2096 if (cont->IsBranch() &&
2097 (cont->condition() == kNotEqual || cont->condition() == kEqual)) {
2098 switch (node->opcode()) {
2099 #define FLAGS_SET_BINOP_LIST(V) \
2100 V(kInt32Add, VisitBinop, kX64Add32) \
2101 V(kInt32Sub, VisitBinop, kX64Sub32) \
2102 V(kWord32And, VisitBinop, kX64And32) \
2103 V(kWord32Or, VisitBinop, kX64Or32) \
2104 V(kInt64Add, VisitBinop, kX64Add) \
2105 V(kInt64Sub, VisitBinop, kX64Sub) \
2106 V(kWord64And, VisitBinop, kX64And) \
2107 V(kWord64Or, VisitBinop, kX64Or)
2108 #define FLAGS_SET_BINOP(opcode, Visit, archOpcode) \
2109 case IrOpcode::opcode: \
2110 if (selector->IsOnlyUserOfNodeInSameBlock(user, node)) { \
2111 return Visit(selector, node, archOpcode, cont); \
2112 } \
2113 break;
2114 FLAGS_SET_BINOP_LIST(FLAGS_SET_BINOP)
2115 #undef FLAGS_SET_BINOP_LIST
2116 #undef FLAGS_SET_BINOP
2117
2118 #define TRY_VISIT_WORD32_SHIFT TryVisitWordShift<Int32BinopMatcher, 32>
2119 #define TRY_VISIT_WORD64_SHIFT TryVisitWordShift<Int64BinopMatcher, 64>
2120 // Skip Word64Sar/Word32Sar since no instruction reduction in most cases.
2121 #define FLAGS_SET_SHIFT_LIST(V) \
2122 V(kWord32Shl, TRY_VISIT_WORD32_SHIFT, kX64Shl32) \
2123 V(kWord32Shr, TRY_VISIT_WORD32_SHIFT, kX64Shr32) \
2124 V(kWord64Shl, TRY_VISIT_WORD64_SHIFT, kX64Shl) \
2125 V(kWord64Shr, TRY_VISIT_WORD64_SHIFT, kX64Shr)
2126 #define FLAGS_SET_SHIFT(opcode, TryVisit, archOpcode) \
2127 case IrOpcode::opcode: \
2128 if (selector->IsOnlyUserOfNodeInSameBlock(user, node)) { \
2129 if (TryVisit(selector, node, archOpcode, cont)) return; \
2130 } \
2131 break;
2132 FLAGS_SET_SHIFT_LIST(FLAGS_SET_SHIFT)
2133 #undef TRY_VISIT_WORD32_SHIFT
2134 #undef TRY_VISIT_WORD64_SHIFT
2135 #undef FLAGS_SET_SHIFT_LIST
2136 #undef FLAGS_SET_SHIFT
2137 default:
2138 break;
2139 }
2140 }
2141 int effect_level = selector->GetEffectLevel(node, cont);
2142 if (node->opcode() == IrOpcode::kLoad) {
2143 switch (LoadRepresentationOf(node->op()).representation()) {
2144 case MachineRepresentation::kWord8:
2145 if (opcode == kX64Cmp32) {
2146 opcode = kX64Cmp8;
2147 } else if (opcode == kX64Test32) {
2148 opcode = kX64Test8;
2149 }
2150 break;
2151 case MachineRepresentation::kWord16:
2152 if (opcode == kX64Cmp32) {
2153 opcode = kX64Cmp16;
2154 } else if (opcode == kX64Test32) {
2155 opcode = kX64Test16;
2156 }
2157 break;
2158 default:
2159 break;
2160 }
2161 }
2162 if (g.CanBeMemoryOperand(opcode, user, node, effect_level)) {
2163 VisitCompareWithMemoryOperand(selector, opcode, node, g.TempImmediate(0),
2164 cont);
2165 } else {
2166 VisitCompare(selector, opcode, g.Use(node), g.TempImmediate(0), cont);
2167 }
2168 }
2169
2170 // Shared routine for multiple float32 compare operations (inputs commuted).
VisitFloat32Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2171 void VisitFloat32Compare(InstructionSelector* selector, Node* node,
2172 FlagsContinuation* cont) {
2173 Node* const left = node->InputAt(0);
2174 Node* const right = node->InputAt(1);
2175 InstructionCode const opcode =
2176 selector->IsSupported(AVX) ? kAVXFloat32Cmp : kSSEFloat32Cmp;
2177 VisitCompare(selector, opcode, right, left, cont, false);
2178 }
2179
2180 // Shared routine for multiple float64 compare operations (inputs commuted).
VisitFloat64Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)2181 void VisitFloat64Compare(InstructionSelector* selector, Node* node,
2182 FlagsContinuation* cont) {
2183 Node* const left = node->InputAt(0);
2184 Node* const right = node->InputAt(1);
2185 InstructionCode const opcode =
2186 selector->IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2187 VisitCompare(selector, opcode, right, left, cont, false);
2188 }
2189
2190 // Shared routine for Word32/Word64 Atomic Binops
VisitAtomicBinop(InstructionSelector * selector,Node * node,ArchOpcode opcode)2191 void VisitAtomicBinop(InstructionSelector* selector, Node* node,
2192 ArchOpcode opcode) {
2193 X64OperandGenerator g(selector);
2194 Node* base = node->InputAt(0);
2195 Node* index = node->InputAt(1);
2196 Node* value = node->InputAt(2);
2197 AddressingMode addressing_mode;
2198 InstructionOperand inputs[] = {
2199 g.UseUniqueRegister(value), g.UseUniqueRegister(base),
2200 g.GetEffectiveIndexOperand(index, &addressing_mode)};
2201 InstructionOperand outputs[] = {g.DefineAsFixed(node, rax)};
2202 InstructionOperand temps[] = {g.TempRegister()};
2203 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
2204 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
2205 arraysize(temps), temps);
2206 }
2207
2208 // Shared routine for Word32/Word64 Atomic CmpExchg
VisitAtomicCompareExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode)2209 void VisitAtomicCompareExchange(InstructionSelector* selector, Node* node,
2210 ArchOpcode opcode) {
2211 X64OperandGenerator g(selector);
2212 Node* base = node->InputAt(0);
2213 Node* index = node->InputAt(1);
2214 Node* old_value = node->InputAt(2);
2215 Node* new_value = node->InputAt(3);
2216 AddressingMode addressing_mode;
2217 InstructionOperand inputs[] = {
2218 g.UseFixed(old_value, rax), g.UseUniqueRegister(new_value),
2219 g.UseUniqueRegister(base),
2220 g.GetEffectiveIndexOperand(index, &addressing_mode)};
2221 InstructionOperand outputs[] = {g.DefineAsFixed(node, rax)};
2222 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
2223 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
2224 }
2225
2226 // Shared routine for Word32/Word64 Atomic Exchange
VisitAtomicExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode)2227 void VisitAtomicExchange(InstructionSelector* selector, Node* node,
2228 ArchOpcode opcode) {
2229 X64OperandGenerator g(selector);
2230 Node* base = node->InputAt(0);
2231 Node* index = node->InputAt(1);
2232 Node* value = node->InputAt(2);
2233 AddressingMode addressing_mode;
2234 InstructionOperand inputs[] = {
2235 g.UseUniqueRegister(value), g.UseUniqueRegister(base),
2236 g.GetEffectiveIndexOperand(index, &addressing_mode)};
2237 InstructionOperand outputs[] = {g.DefineSameAsFirst(node)};
2238 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
2239 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
2240 }
2241
2242 } // namespace
2243
2244 // Shared routine for word comparison against zero.
VisitWordCompareZero(Node * user,Node * value,FlagsContinuation * cont)2245 void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
2246 FlagsContinuation* cont) {
2247 // Try to combine with comparisons against 0 by simply inverting the branch.
2248 while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
2249 Int32BinopMatcher m(value);
2250 if (!m.right().Is(0)) break;
2251
2252 user = value;
2253 value = m.left().node();
2254 cont->Negate();
2255 }
2256
2257 if (CanCover(user, value)) {
2258 switch (value->opcode()) {
2259 case IrOpcode::kWord32Equal:
2260 cont->OverwriteAndNegateIfEqual(kEqual);
2261 return VisitWord32EqualImpl(this, value, cont);
2262 case IrOpcode::kInt32LessThan:
2263 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2264 return VisitWordCompare(this, value, kX64Cmp32, cont);
2265 case IrOpcode::kInt32LessThanOrEqual:
2266 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2267 return VisitWordCompare(this, value, kX64Cmp32, cont);
2268 case IrOpcode::kUint32LessThan:
2269 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2270 return VisitWordCompare(this, value, kX64Cmp32, cont);
2271 case IrOpcode::kUint32LessThanOrEqual:
2272 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2273 return VisitWordCompare(this, value, kX64Cmp32, cont);
2274 case IrOpcode::kWord64Equal: {
2275 cont->OverwriteAndNegateIfEqual(kEqual);
2276 Int64BinopMatcher m(value);
2277 if (m.right().Is(0)) {
2278 // Try to combine the branch with a comparison.
2279 Node* const user = m.node();
2280 Node* const value = m.left().node();
2281 if (CanCover(user, value)) {
2282 switch (value->opcode()) {
2283 case IrOpcode::kInt64Sub:
2284 return VisitWordCompare(this, value, kX64Cmp, cont);
2285 case IrOpcode::kWord64And:
2286 return VisitWordCompare(this, value, kX64Test, cont);
2287 default:
2288 break;
2289 }
2290 }
2291 return VisitCompareZero(this, user, value, kX64Cmp, cont);
2292 }
2293 return VisitWord64EqualImpl(this, value, cont);
2294 }
2295 case IrOpcode::kInt64LessThan:
2296 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2297 return VisitWordCompare(this, value, kX64Cmp, cont);
2298 case IrOpcode::kInt64LessThanOrEqual:
2299 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2300 return VisitWordCompare(this, value, kX64Cmp, cont);
2301 case IrOpcode::kUint64LessThan:
2302 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2303 return VisitWordCompare(this, value, kX64Cmp, cont);
2304 case IrOpcode::kUint64LessThanOrEqual:
2305 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2306 return VisitWordCompare(this, value, kX64Cmp, cont);
2307 case IrOpcode::kFloat32Equal:
2308 cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
2309 return VisitFloat32Compare(this, value, cont);
2310 case IrOpcode::kFloat32LessThan:
2311 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
2312 return VisitFloat32Compare(this, value, cont);
2313 case IrOpcode::kFloat32LessThanOrEqual:
2314 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
2315 return VisitFloat32Compare(this, value, cont);
2316 case IrOpcode::kFloat64Equal:
2317 cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
2318 return VisitFloat64Compare(this, value, cont);
2319 case IrOpcode::kFloat64LessThan: {
2320 Float64BinopMatcher m(value);
2321 if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2322 // This matches the pattern
2323 //
2324 // Float64LessThan(#0.0, Float64Abs(x))
2325 //
2326 // which TurboFan generates for NumberToBoolean in the general case,
2327 // and which evaluates to false if x is 0, -0 or NaN. We can compile
2328 // this to a simple (v)ucomisd using not_equal flags condition, which
2329 // avoids the costly Float64Abs.
2330 cont->OverwriteAndNegateIfEqual(kNotEqual);
2331 InstructionCode const opcode =
2332 IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2333 return VisitCompare(this, opcode, m.left().node(),
2334 m.right().InputAt(0), cont, false);
2335 }
2336 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
2337 return VisitFloat64Compare(this, value, cont);
2338 }
2339 case IrOpcode::kFloat64LessThanOrEqual:
2340 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
2341 return VisitFloat64Compare(this, value, cont);
2342 case IrOpcode::kProjection:
2343 // Check if this is the overflow output projection of an
2344 // <Operation>WithOverflow node.
2345 if (ProjectionIndexOf(value->op()) == 1u) {
2346 // We cannot combine the <Operation>WithOverflow with this branch
2347 // unless the 0th projection (the use of the actual value of the
2348 // <Operation> is either nullptr, which means there's no use of the
2349 // actual value, or was already defined, which means it is scheduled
2350 // *AFTER* this branch).
2351 Node* const node = value->InputAt(0);
2352 Node* const result = NodeProperties::FindProjection(node, 0);
2353 if (result == nullptr || IsDefined(result)) {
2354 switch (node->opcode()) {
2355 case IrOpcode::kInt32AddWithOverflow:
2356 cont->OverwriteAndNegateIfEqual(kOverflow);
2357 return VisitBinop(this, node, kX64Add32, cont);
2358 case IrOpcode::kInt32SubWithOverflow:
2359 cont->OverwriteAndNegateIfEqual(kOverflow);
2360 return VisitBinop(this, node, kX64Sub32, cont);
2361 case IrOpcode::kInt32MulWithOverflow:
2362 cont->OverwriteAndNegateIfEqual(kOverflow);
2363 return VisitBinop(this, node, kX64Imul32, cont);
2364 case IrOpcode::kInt64AddWithOverflow:
2365 cont->OverwriteAndNegateIfEqual(kOverflow);
2366 return VisitBinop(this, node, kX64Add, cont);
2367 case IrOpcode::kInt64SubWithOverflow:
2368 cont->OverwriteAndNegateIfEqual(kOverflow);
2369 return VisitBinop(this, node, kX64Sub, cont);
2370 default:
2371 break;
2372 }
2373 }
2374 }
2375 break;
2376 case IrOpcode::kInt32Sub:
2377 return VisitWordCompare(this, value, kX64Cmp32, cont);
2378 case IrOpcode::kWord32And:
2379 return VisitWordCompare(this, value, kX64Test32, cont);
2380 case IrOpcode::kStackPointerGreaterThan:
2381 cont->OverwriteAndNegateIfEqual(kStackPointerGreaterThanCondition);
2382 return VisitStackPointerGreaterThan(value, cont);
2383 default:
2384 break;
2385 }
2386 }
2387
2388 // Branch could not be combined with a compare, emit compare against 0.
2389 VisitCompareZero(this, user, value, kX64Cmp32, cont);
2390 }
2391
VisitSwitch(Node * node,const SwitchInfo & sw)2392 void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
2393 X64OperandGenerator g(this);
2394 InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
2395
2396 // Emit either ArchTableSwitch or ArchBinarySearchSwitch.
2397 if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
2398 static const size_t kMaxTableSwitchValueRange = 2 << 16;
2399 size_t table_space_cost = 4 + sw.value_range();
2400 size_t table_time_cost = 3;
2401 size_t lookup_space_cost = 3 + 2 * sw.case_count();
2402 size_t lookup_time_cost = sw.case_count();
2403 if (sw.case_count() > 4 &&
2404 table_space_cost + 3 * table_time_cost <=
2405 lookup_space_cost + 3 * lookup_time_cost &&
2406 sw.min_value() > std::numeric_limits<int32_t>::min() &&
2407 sw.value_range() <= kMaxTableSwitchValueRange) {
2408 InstructionOperand index_operand = g.TempRegister();
2409 if (sw.min_value()) {
2410 // The leal automatically zero extends, so result is a valid 64-bit
2411 // index.
2412 Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI), index_operand,
2413 value_operand, g.TempImmediate(-sw.min_value()));
2414 } else {
2415 // Zero extend, because we use it as 64-bit index into the jump table.
2416 if (ZeroExtendsWord32ToWord64(node->InputAt(0))) {
2417 // Input value has already been zero-extended.
2418 index_operand = value_operand;
2419 } else {
2420 Emit(kX64Movl, index_operand, value_operand);
2421 }
2422 }
2423 // Generate a table lookup.
2424 return EmitTableSwitch(sw, index_operand);
2425 }
2426 }
2427
2428 // Generate a tree of conditional jumps.
2429 return EmitBinarySearchSwitch(sw, value_operand);
2430 }
2431
VisitWord32Equal(Node * const node)2432 void InstructionSelector::VisitWord32Equal(Node* const node) {
2433 Node* user = node;
2434 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2435 Int32BinopMatcher m(user);
2436 if (m.right().Is(0)) {
2437 return VisitWordCompareZero(m.node(), m.left().node(), &cont);
2438 }
2439 VisitWord32EqualImpl(this, node, &cont);
2440 }
2441
VisitInt32LessThan(Node * node)2442 void InstructionSelector::VisitInt32LessThan(Node* node) {
2443 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2444 VisitWordCompare(this, node, kX64Cmp32, &cont);
2445 }
2446
VisitInt32LessThanOrEqual(Node * node)2447 void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
2448 FlagsContinuation cont =
2449 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2450 VisitWordCompare(this, node, kX64Cmp32, &cont);
2451 }
2452
VisitUint32LessThan(Node * node)2453 void InstructionSelector::VisitUint32LessThan(Node* node) {
2454 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2455 VisitWordCompare(this, node, kX64Cmp32, &cont);
2456 }
2457
VisitUint32LessThanOrEqual(Node * node)2458 void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
2459 FlagsContinuation cont =
2460 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2461 VisitWordCompare(this, node, kX64Cmp32, &cont);
2462 }
2463
VisitWord64Equal(Node * node)2464 void InstructionSelector::VisitWord64Equal(Node* node) {
2465 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2466 Int64BinopMatcher m(node);
2467 if (m.right().Is(0)) {
2468 // Try to combine the equality check with a comparison.
2469 Node* const user = m.node();
2470 Node* const value = m.left().node();
2471 if (CanCover(user, value)) {
2472 switch (value->opcode()) {
2473 case IrOpcode::kInt64Sub:
2474 return VisitWordCompare(this, value, kX64Cmp, &cont);
2475 case IrOpcode::kWord64And:
2476 return VisitWordCompare(this, value, kX64Test, &cont);
2477 default:
2478 break;
2479 }
2480 }
2481 }
2482 VisitWord64EqualImpl(this, node, &cont);
2483 }
2484
VisitInt32AddWithOverflow(Node * node)2485 void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
2486 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2487 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2488 return VisitBinop(this, node, kX64Add32, &cont);
2489 }
2490 FlagsContinuation cont;
2491 VisitBinop(this, node, kX64Add32, &cont);
2492 }
2493
VisitInt32SubWithOverflow(Node * node)2494 void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
2495 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2496 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2497 return VisitBinop(this, node, kX64Sub32, &cont);
2498 }
2499 FlagsContinuation cont;
2500 VisitBinop(this, node, kX64Sub32, &cont);
2501 }
2502
VisitInt64LessThan(Node * node)2503 void InstructionSelector::VisitInt64LessThan(Node* node) {
2504 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2505 VisitWordCompare(this, node, kX64Cmp, &cont);
2506 }
2507
VisitInt64LessThanOrEqual(Node * node)2508 void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2509 FlagsContinuation cont =
2510 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2511 VisitWordCompare(this, node, kX64Cmp, &cont);
2512 }
2513
VisitUint64LessThan(Node * node)2514 void InstructionSelector::VisitUint64LessThan(Node* node) {
2515 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2516 VisitWordCompare(this, node, kX64Cmp, &cont);
2517 }
2518
VisitUint64LessThanOrEqual(Node * node)2519 void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2520 FlagsContinuation cont =
2521 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2522 VisitWordCompare(this, node, kX64Cmp, &cont);
2523 }
2524
VisitFloat32Equal(Node * node)2525 void InstructionSelector::VisitFloat32Equal(Node* node) {
2526 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2527 VisitFloat32Compare(this, node, &cont);
2528 }
2529
VisitFloat32LessThan(Node * node)2530 void InstructionSelector::VisitFloat32LessThan(Node* node) {
2531 FlagsContinuation cont =
2532 FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2533 VisitFloat32Compare(this, node, &cont);
2534 }
2535
VisitFloat32LessThanOrEqual(Node * node)2536 void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
2537 FlagsContinuation cont =
2538 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2539 VisitFloat32Compare(this, node, &cont);
2540 }
2541
VisitFloat64Equal(Node * node)2542 void InstructionSelector::VisitFloat64Equal(Node* node) {
2543 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2544 VisitFloat64Compare(this, node, &cont);
2545 }
2546
VisitFloat64LessThan(Node * node)2547 void InstructionSelector::VisitFloat64LessThan(Node* node) {
2548 Float64BinopMatcher m(node);
2549 if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2550 // This matches the pattern
2551 //
2552 // Float64LessThan(#0.0, Float64Abs(x))
2553 //
2554 // which TurboFan generates for NumberToBoolean in the general case,
2555 // and which evaluates to false if x is 0, -0 or NaN. We can compile
2556 // this to a simple (v)ucomisd using not_equal flags condition, which
2557 // avoids the costly Float64Abs.
2558 FlagsContinuation cont = FlagsContinuation::ForSet(kNotEqual, node);
2559 InstructionCode const opcode =
2560 IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2561 return VisitCompare(this, opcode, m.left().node(), m.right().InputAt(0),
2562 &cont, false);
2563 }
2564 FlagsContinuation cont =
2565 FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2566 VisitFloat64Compare(this, node, &cont);
2567 }
2568
VisitFloat64LessThanOrEqual(Node * node)2569 void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
2570 FlagsContinuation cont =
2571 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2572 VisitFloat64Compare(this, node, &cont);
2573 }
2574
VisitFloat64InsertLowWord32(Node * node)2575 void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
2576 X64OperandGenerator g(this);
2577 Node* left = node->InputAt(0);
2578 Node* right = node->InputAt(1);
2579 Float64Matcher mleft(left);
2580 if (mleft.HasResolvedValue() &&
2581 (bit_cast<uint64_t>(mleft.ResolvedValue()) >> 32) == 0u) {
2582 Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
2583 return;
2584 }
2585 Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
2586 g.UseRegister(left), g.Use(right));
2587 }
2588
VisitFloat64InsertHighWord32(Node * node)2589 void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
2590 X64OperandGenerator g(this);
2591 Node* left = node->InputAt(0);
2592 Node* right = node->InputAt(1);
2593 Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
2594 g.UseRegister(left), g.Use(right));
2595 }
2596
VisitFloat64SilenceNaN(Node * node)2597 void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
2598 X64OperandGenerator g(this);
2599 Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
2600 g.UseRegister(node->InputAt(0)));
2601 }
2602
VisitMemoryBarrier(Node * node)2603 void InstructionSelector::VisitMemoryBarrier(Node* node) {
2604 X64OperandGenerator g(this);
2605 Emit(kX64MFence, g.NoOutput());
2606 }
2607
VisitWord32AtomicLoad(Node * node)2608 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
2609 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2610 DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
2611 load_rep.representation() == MachineRepresentation::kWord16 ||
2612 load_rep.representation() == MachineRepresentation::kWord32);
2613 USE(load_rep);
2614 VisitLoad(node);
2615 }
2616
VisitWord64AtomicLoad(Node * node)2617 void InstructionSelector::VisitWord64AtomicLoad(Node* node) {
2618 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2619 USE(load_rep);
2620 VisitLoad(node);
2621 }
2622
VisitWord32AtomicStore(Node * node)2623 void InstructionSelector::VisitWord32AtomicStore(Node* node) {
2624 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2625 ArchOpcode opcode;
2626 switch (rep) {
2627 case MachineRepresentation::kWord8:
2628 opcode = kWord32AtomicExchangeInt8;
2629 break;
2630 case MachineRepresentation::kWord16:
2631 opcode = kWord32AtomicExchangeInt16;
2632 break;
2633 case MachineRepresentation::kWord32:
2634 opcode = kWord32AtomicExchangeWord32;
2635 break;
2636 default:
2637 UNREACHABLE();
2638 }
2639 VisitAtomicExchange(this, node, opcode);
2640 }
2641
VisitWord64AtomicStore(Node * node)2642 void InstructionSelector::VisitWord64AtomicStore(Node* node) {
2643 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2644 ArchOpcode opcode;
2645 switch (rep) {
2646 case MachineRepresentation::kWord8:
2647 opcode = kX64Word64AtomicExchangeUint8;
2648 break;
2649 case MachineRepresentation::kWord16:
2650 opcode = kX64Word64AtomicExchangeUint16;
2651 break;
2652 case MachineRepresentation::kWord32:
2653 opcode = kX64Word64AtomicExchangeUint32;
2654 break;
2655 case MachineRepresentation::kWord64:
2656 opcode = kX64Word64AtomicExchangeUint64;
2657 break;
2658 default:
2659 UNREACHABLE();
2660 }
2661 VisitAtomicExchange(this, node, opcode);
2662 }
2663
VisitWord32AtomicExchange(Node * node)2664 void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
2665 MachineType type = AtomicOpType(node->op());
2666 ArchOpcode opcode;
2667 if (type == MachineType::Int8()) {
2668 opcode = kWord32AtomicExchangeInt8;
2669 } else if (type == MachineType::Uint8()) {
2670 opcode = kWord32AtomicExchangeUint8;
2671 } else if (type == MachineType::Int16()) {
2672 opcode = kWord32AtomicExchangeInt16;
2673 } else if (type == MachineType::Uint16()) {
2674 opcode = kWord32AtomicExchangeUint16;
2675 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2676 opcode = kWord32AtomicExchangeWord32;
2677 } else {
2678 UNREACHABLE();
2679 }
2680 VisitAtomicExchange(this, node, opcode);
2681 }
2682
VisitWord64AtomicExchange(Node * node)2683 void InstructionSelector::VisitWord64AtomicExchange(Node* node) {
2684 MachineType type = AtomicOpType(node->op());
2685 ArchOpcode opcode;
2686 if (type == MachineType::Uint8()) {
2687 opcode = kX64Word64AtomicExchangeUint8;
2688 } else if (type == MachineType::Uint16()) {
2689 opcode = kX64Word64AtomicExchangeUint16;
2690 } else if (type == MachineType::Uint32()) {
2691 opcode = kX64Word64AtomicExchangeUint32;
2692 } else if (type == MachineType::Uint64()) {
2693 opcode = kX64Word64AtomicExchangeUint64;
2694 } else {
2695 UNREACHABLE();
2696 }
2697 VisitAtomicExchange(this, node, opcode);
2698 }
2699
VisitWord32AtomicCompareExchange(Node * node)2700 void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
2701 MachineType type = AtomicOpType(node->op());
2702 ArchOpcode opcode;
2703 if (type == MachineType::Int8()) {
2704 opcode = kWord32AtomicCompareExchangeInt8;
2705 } else if (type == MachineType::Uint8()) {
2706 opcode = kWord32AtomicCompareExchangeUint8;
2707 } else if (type == MachineType::Int16()) {
2708 opcode = kWord32AtomicCompareExchangeInt16;
2709 } else if (type == MachineType::Uint16()) {
2710 opcode = kWord32AtomicCompareExchangeUint16;
2711 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2712 opcode = kWord32AtomicCompareExchangeWord32;
2713 } else {
2714 UNREACHABLE();
2715 }
2716 VisitAtomicCompareExchange(this, node, opcode);
2717 }
2718
VisitWord64AtomicCompareExchange(Node * node)2719 void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) {
2720 MachineType type = AtomicOpType(node->op());
2721 ArchOpcode opcode;
2722 if (type == MachineType::Uint8()) {
2723 opcode = kX64Word64AtomicCompareExchangeUint8;
2724 } else if (type == MachineType::Uint16()) {
2725 opcode = kX64Word64AtomicCompareExchangeUint16;
2726 } else if (type == MachineType::Uint32()) {
2727 opcode = kX64Word64AtomicCompareExchangeUint32;
2728 } else if (type == MachineType::Uint64()) {
2729 opcode = kX64Word64AtomicCompareExchangeUint64;
2730 } else {
2731 UNREACHABLE();
2732 }
2733 VisitAtomicCompareExchange(this, node, opcode);
2734 }
2735
VisitWord32AtomicBinaryOperation(Node * node,ArchOpcode int8_op,ArchOpcode uint8_op,ArchOpcode int16_op,ArchOpcode uint16_op,ArchOpcode word32_op)2736 void InstructionSelector::VisitWord32AtomicBinaryOperation(
2737 Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
2738 ArchOpcode uint16_op, ArchOpcode word32_op) {
2739 MachineType type = AtomicOpType(node->op());
2740 ArchOpcode opcode;
2741 if (type == MachineType::Int8()) {
2742 opcode = int8_op;
2743 } else if (type == MachineType::Uint8()) {
2744 opcode = uint8_op;
2745 } else if (type == MachineType::Int16()) {
2746 opcode = int16_op;
2747 } else if (type == MachineType::Uint16()) {
2748 opcode = uint16_op;
2749 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2750 opcode = word32_op;
2751 } else {
2752 UNREACHABLE();
2753 }
2754 VisitAtomicBinop(this, node, opcode);
2755 }
2756
2757 #define VISIT_ATOMIC_BINOP(op) \
2758 void InstructionSelector::VisitWord32Atomic##op(Node* node) { \
2759 VisitWord32AtomicBinaryOperation( \
2760 node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \
2761 kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \
2762 kWord32Atomic##op##Word32); \
2763 }
2764 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)2765 VISIT_ATOMIC_BINOP(Sub)
2766 VISIT_ATOMIC_BINOP(And)
2767 VISIT_ATOMIC_BINOP(Or)
2768 VISIT_ATOMIC_BINOP(Xor)
2769 #undef VISIT_ATOMIC_BINOP
2770
2771 void InstructionSelector::VisitWord64AtomicBinaryOperation(
2772 Node* node, ArchOpcode uint8_op, ArchOpcode uint16_op, ArchOpcode uint32_op,
2773 ArchOpcode word64_op) {
2774 MachineType type = AtomicOpType(node->op());
2775 ArchOpcode opcode;
2776 if (type == MachineType::Uint8()) {
2777 opcode = uint8_op;
2778 } else if (type == MachineType::Uint16()) {
2779 opcode = uint16_op;
2780 } else if (type == MachineType::Uint32()) {
2781 opcode = uint32_op;
2782 } else if (type == MachineType::Uint64()) {
2783 opcode = word64_op;
2784 } else {
2785 UNREACHABLE();
2786 }
2787 VisitAtomicBinop(this, node, opcode);
2788 }
2789
2790 #define VISIT_ATOMIC_BINOP(op) \
2791 void InstructionSelector::VisitWord64Atomic##op(Node* node) { \
2792 VisitWord64AtomicBinaryOperation( \
2793 node, kX64Word64Atomic##op##Uint8, kX64Word64Atomic##op##Uint16, \
2794 kX64Word64Atomic##op##Uint32, kX64Word64Atomic##op##Uint64); \
2795 }
2796 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)2797 VISIT_ATOMIC_BINOP(Sub)
2798 VISIT_ATOMIC_BINOP(And)
2799 VISIT_ATOMIC_BINOP(Or)
2800 VISIT_ATOMIC_BINOP(Xor)
2801 #undef VISIT_ATOMIC_BINOP
2802
2803 #define SIMD_BINOP_SSE_AVX_LIST(V) \
2804 V(F64x2Add) \
2805 V(F64x2Sub) \
2806 V(F64x2Mul) \
2807 V(F64x2Div) \
2808 V(F64x2Eq) \
2809 V(F64x2Ne) \
2810 V(F64x2Lt) \
2811 V(F64x2Le) \
2812 V(F32x4Add) \
2813 V(F32x4Sub) \
2814 V(F32x4Mul) \
2815 V(F32x4Div) \
2816 V(F32x4Eq) \
2817 V(F32x4Ne) \
2818 V(F32x4Lt) \
2819 V(F32x4Le) \
2820 V(I64x2Add) \
2821 V(I64x2Sub) \
2822 V(I64x2Eq) \
2823 V(I32x4Add) \
2824 V(I32x4AddHoriz) \
2825 V(I32x4Sub) \
2826 V(I32x4Mul) \
2827 V(I32x4MinS) \
2828 V(I32x4MaxS) \
2829 V(I32x4Eq) \
2830 V(I32x4GtS) \
2831 V(I32x4MinU) \
2832 V(I32x4MaxU) \
2833 V(I32x4DotI16x8S) \
2834 V(I16x8SConvertI32x4) \
2835 V(I16x8UConvertI32x4) \
2836 V(I16x8Add) \
2837 V(I16x8AddSatS) \
2838 V(I16x8AddHoriz) \
2839 V(I16x8Sub) \
2840 V(I16x8SubSatS) \
2841 V(I16x8Mul) \
2842 V(I16x8MinS) \
2843 V(I16x8MaxS) \
2844 V(I16x8Eq) \
2845 V(I16x8GtS) \
2846 V(I16x8AddSatU) \
2847 V(I16x8SubSatU) \
2848 V(I16x8MinU) \
2849 V(I16x8MaxU) \
2850 V(I16x8RoundingAverageU) \
2851 V(I8x16SConvertI16x8) \
2852 V(I8x16UConvertI16x8) \
2853 V(I8x16Add) \
2854 V(I8x16AddSatS) \
2855 V(I8x16Sub) \
2856 V(I8x16SubSatS) \
2857 V(I8x16MinS) \
2858 V(I8x16MaxS) \
2859 V(I8x16Eq) \
2860 V(I8x16GtS) \
2861 V(I8x16AddSatU) \
2862 V(I8x16SubSatU) \
2863 V(I8x16MinU) \
2864 V(I8x16MaxU) \
2865 V(I8x16RoundingAverageU) \
2866 V(S128And) \
2867 V(S128Or) \
2868 V(S128Xor)
2869
2870 #define SIMD_BINOP_LIST(V) \
2871 V(F64x2Min) \
2872 V(F64x2Max) \
2873 V(F32x4AddHoriz) \
2874 V(F32x4Min) \
2875 V(F32x4Max) \
2876 V(I32x4GeS) \
2877 V(I32x4GeU) \
2878 V(I16x8GeS) \
2879 V(I16x8GeU) \
2880 V(I8x16GeS) \
2881 V(I8x16GeU)
2882
2883 #define SIMD_BINOP_ONE_TEMP_LIST(V) \
2884 V(I32x4Ne) \
2885 V(I32x4GtU) \
2886 V(I16x8Ne) \
2887 V(I16x8GtU) \
2888 V(I8x16Ne) \
2889 V(I8x16GtU)
2890
2891 #define SIMD_UNOP_LIST(V) \
2892 V(F64x2Sqrt) \
2893 V(F32x4SConvertI32x4) \
2894 V(F32x4Abs) \
2895 V(F32x4Neg) \
2896 V(F32x4Sqrt) \
2897 V(F32x4RecipApprox) \
2898 V(F32x4RecipSqrtApprox) \
2899 V(I64x2Neg) \
2900 V(I64x2BitMask) \
2901 V(I32x4SConvertI16x8Low) \
2902 V(I32x4SConvertI16x8High) \
2903 V(I32x4Neg) \
2904 V(I32x4UConvertI16x8Low) \
2905 V(I32x4UConvertI16x8High) \
2906 V(I32x4Abs) \
2907 V(I32x4BitMask) \
2908 V(I16x8SConvertI8x16Low) \
2909 V(I16x8SConvertI8x16High) \
2910 V(I16x8Neg) \
2911 V(I16x8UConvertI8x16Low) \
2912 V(I16x8UConvertI8x16High) \
2913 V(I16x8Abs) \
2914 V(I8x16Neg) \
2915 V(I8x16Abs) \
2916 V(I8x16BitMask) \
2917 V(S128Not)
2918
2919 #define SIMD_SHIFT_OPCODES(V) \
2920 V(I64x2Shl) \
2921 V(I64x2ShrU) \
2922 V(I32x4Shl) \
2923 V(I32x4ShrS) \
2924 V(I32x4ShrU) \
2925 V(I16x8Shl) \
2926 V(I16x8ShrS) \
2927 V(I16x8ShrU)
2928
2929 #define SIMD_NARROW_SHIFT_OPCODES(V) \
2930 V(I8x16Shl) \
2931 V(I8x16ShrU)
2932
2933 #define SIMD_ANYTRUE_LIST(V) \
2934 V(V32x4AnyTrue) \
2935 V(V16x8AnyTrue) \
2936 V(V8x16AnyTrue)
2937
2938 #define SIMD_ALLTRUE_LIST(V) \
2939 V(V32x4AllTrue) \
2940 V(V16x8AllTrue) \
2941 V(V8x16AllTrue)
2942
2943 void InstructionSelector::VisitS128Const(Node* node) {
2944 X64OperandGenerator g(this);
2945 static const int kUint32Immediates = kSimd128Size / sizeof(uint32_t);
2946 uint32_t val[kUint32Immediates];
2947 memcpy(val, S128ImmediateParameterOf(node->op()).data(), kSimd128Size);
2948 // If all bytes are zeros or ones, avoid emitting code for generic constants
2949 bool all_zeros = !(val[0] || val[1] || val[2] || val[3]);
2950 bool all_ones = val[0] == UINT32_MAX && val[1] == UINT32_MAX &&
2951 val[2] == UINT32_MAX && val[3] == UINT32_MAX;
2952 InstructionOperand dst = g.DefineAsRegister(node);
2953 if (all_zeros) {
2954 Emit(kX64S128Zero, dst);
2955 } else if (all_ones) {
2956 Emit(kX64S128AllOnes, dst);
2957 } else {
2958 Emit(kX64S128Const, dst, g.UseImmediate(val[0]), g.UseImmediate(val[1]),
2959 g.UseImmediate(val[2]), g.UseImmediate(val[3]));
2960 }
2961 }
2962
VisitS128Zero(Node * node)2963 void InstructionSelector::VisitS128Zero(Node* node) {
2964 X64OperandGenerator g(this);
2965 Emit(kX64S128Zero, g.DefineAsRegister(node));
2966 }
2967
2968 #define SIMD_TYPES_FOR_SPLAT(V) \
2969 V(I64x2) \
2970 V(I32x4) \
2971 V(I16x8) \
2972 V(I8x16)
2973
2974 // Splat with an optimization for const 0.
2975 #define VISIT_SIMD_SPLAT(Type) \
2976 void InstructionSelector::Visit##Type##Splat(Node* node) { \
2977 X64OperandGenerator g(this); \
2978 Node* input = node->InputAt(0); \
2979 if (g.CanBeImmediate(input) && g.GetImmediateIntegerValue(input) == 0) { \
2980 Emit(kX64S128Zero, g.DefineAsRegister(node)); \
2981 } else { \
2982 Emit(kX64##Type##Splat, g.DefineAsRegister(node), g.Use(input)); \
2983 } \
2984 }
SIMD_TYPES_FOR_SPLAT(VISIT_SIMD_SPLAT)2985 SIMD_TYPES_FOR_SPLAT(VISIT_SIMD_SPLAT)
2986 #undef VISIT_SIMD_SPLAT
2987 #undef SIMD_TYPES_FOR_SPLAT
2988
2989 void InstructionSelector::VisitF64x2Splat(Node* node) {
2990 X64OperandGenerator g(this);
2991 Emit(kX64F64x2Splat, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
2992 }
2993
VisitF32x4Splat(Node * node)2994 void InstructionSelector::VisitF32x4Splat(Node* node) {
2995 X64OperandGenerator g(this);
2996 InstructionOperand dst =
2997 IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
2998 Emit(kX64F32x4Splat, dst, g.UseRegister(node->InputAt(0)));
2999 }
3000
3001 #define SIMD_VISIT_EXTRACT_LANE(Type, Sign, Op) \
3002 void InstructionSelector::Visit##Type##ExtractLane##Sign(Node* node) { \
3003 X64OperandGenerator g(this); \
3004 int32_t lane = OpParameter<int32_t>(node->op()); \
3005 Emit(kX64##Op, g.DefineAsRegister(node), g.UseRegister(node->InputAt(0)), \
3006 g.UseImmediate(lane)); \
3007 }
3008 SIMD_VISIT_EXTRACT_LANE(F64x2, , F64x2ExtractLane)
3009 SIMD_VISIT_EXTRACT_LANE(F32x4, , F32x4ExtractLane)
3010 SIMD_VISIT_EXTRACT_LANE(I64x2, , I64x2ExtractLane)
3011 SIMD_VISIT_EXTRACT_LANE(I32x4, , I32x4ExtractLane)
SIMD_VISIT_EXTRACT_LANE(I16x8,S,I16x8ExtractLaneS)3012 SIMD_VISIT_EXTRACT_LANE(I16x8, S, I16x8ExtractLaneS)
3013 SIMD_VISIT_EXTRACT_LANE(I16x8, U, Pextrw)
3014 SIMD_VISIT_EXTRACT_LANE(I8x16, S, I8x16ExtractLaneS)
3015 SIMD_VISIT_EXTRACT_LANE(I8x16, U, Pextrb)
3016 #undef SIMD_VISIT_EXTRACT_LANE
3017
3018 void InstructionSelector::VisitF32x4ReplaceLane(Node* node) {
3019 X64OperandGenerator g(this);
3020 int32_t lane = OpParameter<int32_t>(node->op());
3021 Emit(kX64F32x4ReplaceLane, g.DefineSameAsFirst(node),
3022 g.UseRegister(node->InputAt(0)), g.UseImmediate(lane),
3023 g.Use(node->InputAt(1)));
3024 }
3025
3026 #define VISIT_SIMD_REPLACE_LANE(TYPE, OPCODE) \
3027 void InstructionSelector::Visit##TYPE##ReplaceLane(Node* node) { \
3028 X64OperandGenerator g(this); \
3029 int32_t lane = OpParameter<int32_t>(node->op()); \
3030 Emit(OPCODE, g.DefineAsRegister(node), g.UseRegister(node->InputAt(0)), \
3031 g.UseImmediate(lane), g.Use(node->InputAt(1))); \
3032 }
3033
3034 #define SIMD_TYPES_FOR_REPLACE_LANE(V) \
3035 V(F64x2, kX64Pinsrq) \
3036 V(I64x2, kX64Pinsrq) \
3037 V(I32x4, kX64Pinsrd) \
3038 V(I16x8, kX64Pinsrw) \
3039 V(I8x16, kX64Pinsrb)
3040
3041 SIMD_TYPES_FOR_REPLACE_LANE(VISIT_SIMD_REPLACE_LANE)
3042 #undef SIMD_TYPES_FOR_REPLACE_LANE
3043 #undef VISIT_SIMD_REPLACE_LANE
3044
3045 #define VISIT_SIMD_SHIFT(Opcode) \
3046 void InstructionSelector::Visit##Opcode(Node* node) { \
3047 X64OperandGenerator g(this); \
3048 InstructionOperand dst = IsSupported(AVX) ? g.DefineAsRegister(node) \
3049 : g.DefineSameAsFirst(node); \
3050 if (g.CanBeImmediate(node->InputAt(1))) { \
3051 Emit(kX64##Opcode, dst, g.UseRegister(node->InputAt(0)), \
3052 g.UseImmediate(node->InputAt(1))); \
3053 } else { \
3054 InstructionOperand temps[] = {g.TempSimd128Register(), \
3055 g.TempRegister()}; \
3056 Emit(kX64##Opcode, dst, g.UseUniqueRegister(node->InputAt(0)), \
3057 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps); \
3058 } \
3059 }
SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT)3060 SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT)
3061 #undef VISIT_SIMD_SHIFT
3062 #undef SIMD_SHIFT_OPCODES
3063
3064 #define VISIT_SIMD_NARROW_SHIFT(Opcode) \
3065 void InstructionSelector::Visit##Opcode(Node* node) { \
3066 X64OperandGenerator g(this); \
3067 InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()}; \
3068 if (g.CanBeImmediate(node->InputAt(1))) { \
3069 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3070 g.UseRegister(node->InputAt(0)), g.UseImmediate(node->InputAt(1)), \
3071 arraysize(temps), temps); \
3072 } else { \
3073 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3074 g.UseUniqueRegister(node->InputAt(0)), \
3075 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps); \
3076 } \
3077 }
3078 SIMD_NARROW_SHIFT_OPCODES(VISIT_SIMD_NARROW_SHIFT)
3079 #undef VISIT_SIMD_NARROW_SHIFT
3080 #undef SIMD_NARROW_SHIFT_OPCODES
3081
3082 #define VISIT_SIMD_UNOP(Opcode) \
3083 void InstructionSelector::Visit##Opcode(Node* node) { \
3084 X64OperandGenerator g(this); \
3085 Emit(kX64##Opcode, g.DefineAsRegister(node), \
3086 g.UseRegister(node->InputAt(0))); \
3087 }
3088 SIMD_UNOP_LIST(VISIT_SIMD_UNOP)
3089 #undef VISIT_SIMD_UNOP
3090 #undef SIMD_UNOP_LIST
3091
3092 #define VISIT_SIMD_BINOP(Opcode) \
3093 void InstructionSelector::Visit##Opcode(Node* node) { \
3094 X64OperandGenerator g(this); \
3095 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3096 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
3097 }
3098 SIMD_BINOP_LIST(VISIT_SIMD_BINOP)
3099 #undef VISIT_SIMD_BINOP
3100 #undef SIMD_BINOP_LIST
3101
3102 #define VISIT_SIMD_BINOP(Opcode) \
3103 void InstructionSelector::Visit##Opcode(Node* node) { \
3104 X64OperandGenerator g(this); \
3105 if (IsSupported(AVX)) { \
3106 Emit(kX64##Opcode, g.DefineAsRegister(node), \
3107 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
3108 } else { \
3109 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3110 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
3111 } \
3112 }
3113 SIMD_BINOP_SSE_AVX_LIST(VISIT_SIMD_BINOP)
3114 #undef VISIT_SIMD_BINOP
3115 #undef SIMD_BINOP_SSE_AVX_LIST
3116
3117 #define VISIT_SIMD_BINOP_ONE_TEMP(Opcode) \
3118 void InstructionSelector::Visit##Opcode(Node* node) { \
3119 X64OperandGenerator g(this); \
3120 InstructionOperand temps[] = {g.TempSimd128Register()}; \
3121 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3122 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), \
3123 arraysize(temps), temps); \
3124 }
3125 SIMD_BINOP_ONE_TEMP_LIST(VISIT_SIMD_BINOP_ONE_TEMP)
3126 #undef VISIT_SIMD_BINOP_ONE_TEMP
3127 #undef SIMD_BINOP_ONE_TEMP_LIST
3128
3129 #define VISIT_SIMD_ANYTRUE(Opcode) \
3130 void InstructionSelector::Visit##Opcode(Node* node) { \
3131 X64OperandGenerator g(this); \
3132 Emit(kX64##Opcode, g.DefineAsRegister(node), \
3133 g.UseUniqueRegister(node->InputAt(0))); \
3134 }
3135 SIMD_ANYTRUE_LIST(VISIT_SIMD_ANYTRUE)
3136 #undef VISIT_SIMD_ANYTRUE
3137 #undef SIMD_ANYTRUE_LIST
3138
3139 #define VISIT_SIMD_ALLTRUE(Opcode) \
3140 void InstructionSelector::Visit##Opcode(Node* node) { \
3141 X64OperandGenerator g(this); \
3142 InstructionOperand temps[] = {g.TempSimd128Register()}; \
3143 Emit(kX64##Opcode, g.DefineAsRegister(node), \
3144 g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps); \
3145 }
3146 SIMD_ALLTRUE_LIST(VISIT_SIMD_ALLTRUE)
3147 #undef VISIT_SIMD_ALLTRUE
3148 #undef SIMD_ALLTRUE_LIST
3149
3150 void InstructionSelector::VisitS128Select(Node* node) {
3151 X64OperandGenerator g(this);
3152 Emit(kX64S128Select, g.DefineSameAsFirst(node),
3153 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
3154 g.UseRegister(node->InputAt(2)));
3155 }
3156
3157 namespace {
VisitSignSelect(InstructionSelector * selector,Node * node,ArchOpcode opcode)3158 void VisitSignSelect(InstructionSelector* selector, Node* node,
3159 ArchOpcode opcode) {
3160 X64OperandGenerator g(selector);
3161 // signselect(x, y, -1) = x
3162 // pblendvb(dst, x, y, -1) = dst <- y, so we need to swap x and y.
3163 if (selector->IsSupported(AVX)) {
3164 selector->Emit(
3165 opcode, g.DefineAsRegister(node), g.UseRegister(node->InputAt(1)),
3166 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(2)));
3167 } else {
3168 selector->Emit(
3169 opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(1)),
3170 g.UseRegister(node->InputAt(0)), g.UseFixed(node->InputAt(2), xmm0));
3171 }
3172 }
3173 } // namespace
3174
VisitI8x16SignSelect(Node * node)3175 void InstructionSelector::VisitI8x16SignSelect(Node* node) {
3176 VisitSignSelect(this, node, kX64I8x16SignSelect);
3177 }
3178
VisitI16x8SignSelect(Node * node)3179 void InstructionSelector::VisitI16x8SignSelect(Node* node) {
3180 VisitSignSelect(this, node, kX64I16x8SignSelect);
3181 }
3182
VisitI32x4SignSelect(Node * node)3183 void InstructionSelector::VisitI32x4SignSelect(Node* node) {
3184 VisitSignSelect(this, node, kX64I32x4SignSelect);
3185 }
3186
VisitI64x2SignSelect(Node * node)3187 void InstructionSelector::VisitI64x2SignSelect(Node* node) {
3188 VisitSignSelect(this, node, kX64I64x2SignSelect);
3189 }
3190
VisitS128AndNot(Node * node)3191 void InstructionSelector::VisitS128AndNot(Node* node) {
3192 X64OperandGenerator g(this);
3193 // andnps a b does ~a & b, but we want a & !b, so flip the input.
3194 Emit(kX64S128AndNot, g.DefineSameAsFirst(node),
3195 g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(0)));
3196 }
3197
VisitF64x2Abs(Node * node)3198 void InstructionSelector::VisitF64x2Abs(Node* node) {
3199 X64OperandGenerator g(this);
3200 InstructionOperand temps[] = {g.TempDoubleRegister()};
3201 Emit(kX64F64x2Abs, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
3202 arraysize(temps), temps);
3203 }
3204
VisitF64x2Neg(Node * node)3205 void InstructionSelector::VisitF64x2Neg(Node* node) {
3206 X64OperandGenerator g(this);
3207 InstructionOperand temps[] = {g.TempDoubleRegister()};
3208 Emit(kX64F64x2Neg, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
3209 arraysize(temps), temps);
3210 }
3211
VisitF32x4UConvertI32x4(Node * node)3212 void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
3213 X64OperandGenerator g(this);
3214 Emit(kX64F32x4UConvertI32x4, g.DefineSameAsFirst(node),
3215 g.UseRegister(node->InputAt(0)));
3216 }
3217
3218 #define VISIT_SIMD_QFMOP(Opcode) \
3219 void InstructionSelector::Visit##Opcode(Node* node) { \
3220 X64OperandGenerator g(this); \
3221 if (CpuFeatures::IsSupported(FMA3)) { \
3222 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3223 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), \
3224 g.UseRegister(node->InputAt(2))); \
3225 } else { \
3226 InstructionOperand temps[] = {g.TempSimd128Register()}; \
3227 Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
3228 g.UseUniqueRegister(node->InputAt(0)), \
3229 g.UseUniqueRegister(node->InputAt(1)), \
3230 g.UseRegister(node->InputAt(2)), arraysize(temps), temps); \
3231 } \
3232 }
3233 VISIT_SIMD_QFMOP(F64x2Qfma)
VISIT_SIMD_QFMOP(F64x2Qfms)3234 VISIT_SIMD_QFMOP(F64x2Qfms)
3235 VISIT_SIMD_QFMOP(F32x4Qfma)
3236 VISIT_SIMD_QFMOP(F32x4Qfms)
3237 #undef VISIT_SIMD_QFMOP
3238
3239 void InstructionSelector::VisitI64x2ShrS(Node* node) {
3240 X64OperandGenerator g(this);
3241 InstructionOperand temps[] = {g.TempRegister()};
3242 // Use fixed to rcx, to use sarq_cl in codegen.
3243 Emit(kX64I64x2ShrS, g.DefineSameAsFirst(node),
3244 g.UseUniqueRegister(node->InputAt(0)), g.UseFixed(node->InputAt(1), rcx),
3245 arraysize(temps), temps);
3246 }
3247
VisitI64x2Mul(Node * node)3248 void InstructionSelector::VisitI64x2Mul(Node* node) {
3249 X64OperandGenerator g(this);
3250 InstructionOperand temps[] = {g.TempSimd128Register(),
3251 g.TempSimd128Register()};
3252 Emit(kX64I64x2Mul, g.DefineSameAsFirst(node),
3253 g.UseUniqueRegister(node->InputAt(0)),
3254 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
3255 }
3256
VisitI32x4SConvertF32x4(Node * node)3257 void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
3258 X64OperandGenerator g(this);
3259 InstructionOperand temps[] = {g.TempSimd128Register()};
3260 Emit(kX64I32x4SConvertF32x4, g.DefineSameAsFirst(node),
3261 g.UseRegister(node->InputAt(0)), arraysize(temps), temps);
3262 }
3263
VisitI32x4UConvertF32x4(Node * node)3264 void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
3265 X64OperandGenerator g(this);
3266 InstructionOperand temps[] = {g.TempSimd128Register(),
3267 g.TempSimd128Register()};
3268 Emit(kX64I32x4UConvertF32x4, g.DefineSameAsFirst(node),
3269 g.UseRegister(node->InputAt(0)), arraysize(temps), temps);
3270 }
3271
VisitI16x8BitMask(Node * node)3272 void InstructionSelector::VisitI16x8BitMask(Node* node) {
3273 X64OperandGenerator g(this);
3274 InstructionOperand temps[] = {g.TempSimd128Register()};
3275 Emit(kX64I16x8BitMask, g.DefineAsRegister(node),
3276 g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps);
3277 }
3278
VisitI8x16Mul(Node * node)3279 void InstructionSelector::VisitI8x16Mul(Node* node) {
3280 X64OperandGenerator g(this);
3281 InstructionOperand temps[] = {g.TempSimd128Register()};
3282 Emit(kX64I8x16Mul, g.DefineSameAsFirst(node),
3283 g.UseUniqueRegister(node->InputAt(0)),
3284 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
3285 }
3286
VisitI8x16ShrS(Node * node)3287 void InstructionSelector::VisitI8x16ShrS(Node* node) {
3288 X64OperandGenerator g(this);
3289 if (g.CanBeImmediate(node->InputAt(1))) {
3290 Emit(kX64I8x16ShrS, g.DefineSameAsFirst(node),
3291 g.UseRegister(node->InputAt(0)), g.UseImmediate(node->InputAt(1)));
3292 } else {
3293 InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
3294 Emit(kX64I8x16ShrS, g.DefineSameAsFirst(node),
3295 g.UseUniqueRegister(node->InputAt(0)),
3296 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
3297 }
3298 }
3299
VisitInt32AbsWithOverflow(Node * node)3300 void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
3301 UNREACHABLE();
3302 }
3303
VisitInt64AbsWithOverflow(Node * node)3304 void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
3305 UNREACHABLE();
3306 }
3307
3308 namespace {
3309
3310 // Returns true if shuffle can be decomposed into two 16x4 half shuffles
3311 // followed by a 16x8 blend.
3312 // E.g. [3 2 1 0 15 14 13 12].
TryMatch16x8HalfShuffle(uint8_t * shuffle16x8,uint8_t * blend_mask)3313 bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) {
3314 *blend_mask = 0;
3315 for (int i = 0; i < 8; i++) {
3316 if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false;
3317 *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i;
3318 }
3319 return true;
3320 }
3321
3322 struct ShuffleEntry {
3323 uint8_t shuffle[kSimd128Size];
3324 ArchOpcode opcode;
3325 bool src0_needs_reg;
3326 bool src1_needs_reg;
3327 };
3328
3329 // Shuffles that map to architecture-specific instruction sequences. These are
3330 // matched very early, so we shouldn't include shuffles that match better in
3331 // later tests, like 32x4 and 16x8 shuffles. In general, these patterns should
3332 // map to either a single instruction, or be finer grained, such as zip/unzip or
3333 // transpose patterns.
3334 static const ShuffleEntry arch_shuffles[] = {
3335 {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23},
3336 kX64S64x2UnpackLow,
3337 true,
3338 true},
3339 {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31},
3340 kX64S64x2UnpackHigh,
3341 true,
3342 true},
3343 {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
3344 kX64S32x4UnpackLow,
3345 true,
3346 true},
3347 {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
3348 kX64S32x4UnpackHigh,
3349 true,
3350 true},
3351 {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
3352 kX64S16x8UnpackLow,
3353 true,
3354 true},
3355 {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
3356 kX64S16x8UnpackHigh,
3357 true,
3358 true},
3359 {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
3360 kX64S8x16UnpackLow,
3361 true,
3362 true},
3363 {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
3364 kX64S8x16UnpackHigh,
3365 true,
3366 true},
3367
3368 {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
3369 kX64S16x8UnzipLow,
3370 true,
3371 true},
3372 {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
3373 kX64S16x8UnzipHigh,
3374 true,
3375 true},
3376 {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
3377 kX64S8x16UnzipLow,
3378 true,
3379 true},
3380 {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
3381 kX64S8x16UnzipHigh,
3382 true,
3383 true},
3384 {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
3385 kX64S8x16TransposeLow,
3386 true,
3387 true},
3388 {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
3389 kX64S8x16TransposeHigh,
3390 true,
3391 true},
3392 {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8},
3393 kX64S8x8Reverse,
3394 true,
3395 true},
3396 {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12},
3397 kX64S8x4Reverse,
3398 true,
3399 true},
3400 {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
3401 kX64S8x2Reverse,
3402 true,
3403 true}};
3404
TryMatchArchShuffle(const uint8_t * shuffle,const ShuffleEntry * table,size_t num_entries,bool is_swizzle,const ShuffleEntry ** arch_shuffle)3405 bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
3406 size_t num_entries, bool is_swizzle,
3407 const ShuffleEntry** arch_shuffle) {
3408 uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1;
3409 for (size_t i = 0; i < num_entries; ++i) {
3410 const ShuffleEntry& entry = table[i];
3411 int j = 0;
3412 for (; j < kSimd128Size; ++j) {
3413 if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
3414 break;
3415 }
3416 }
3417 if (j == kSimd128Size) {
3418 *arch_shuffle = &entry;
3419 return true;
3420 }
3421 }
3422 return false;
3423 }
3424
3425 } // namespace
3426
VisitI8x16Shuffle(Node * node)3427 void InstructionSelector::VisitI8x16Shuffle(Node* node) {
3428 uint8_t shuffle[kSimd128Size];
3429 bool is_swizzle;
3430 CanonicalizeShuffle(node, shuffle, &is_swizzle);
3431
3432 int imm_count = 0;
3433 static const int kMaxImms = 6;
3434 uint32_t imms[kMaxImms];
3435 int temp_count = 0;
3436 static const int kMaxTemps = 2;
3437 InstructionOperand temps[kMaxTemps];
3438
3439 X64OperandGenerator g(this);
3440 // Swizzles don't generally need DefineSameAsFirst to avoid a move.
3441 bool no_same_as_first = is_swizzle;
3442 // We generally need UseRegister for input0, Use for input1.
3443 // TODO(v8:9198): We don't have 16-byte alignment for SIMD operands yet, but
3444 // we retain this logic (continue setting these in the various shuffle match
3445 // clauses), but ignore it when selecting registers or slots.
3446 bool src0_needs_reg = true;
3447 bool src1_needs_reg = false;
3448 ArchOpcode opcode = kX64I8x16Shuffle; // general shuffle is the default
3449
3450 uint8_t offset;
3451 uint8_t shuffle32x4[4];
3452 uint8_t shuffle16x8[8];
3453 int index;
3454 const ShuffleEntry* arch_shuffle;
3455 if (wasm::SimdShuffle::TryMatchConcat(shuffle, &offset)) {
3456 // Swap inputs from the normal order for (v)palignr.
3457 SwapShuffleInputs(node);
3458 is_swizzle = false; // It's simpler to just handle the general case.
3459 no_same_as_first = false; // SSE requires same-as-first.
3460 // TODO(v8:9608): also see v8:9083
3461 src1_needs_reg = true;
3462 opcode = kX64S8x16Alignr;
3463 // palignr takes a single imm8 offset.
3464 imms[imm_count++] = offset;
3465 } else if (TryMatchArchShuffle(shuffle, arch_shuffles,
3466 arraysize(arch_shuffles), is_swizzle,
3467 &arch_shuffle)) {
3468 opcode = arch_shuffle->opcode;
3469 src0_needs_reg = arch_shuffle->src0_needs_reg;
3470 // SSE can't take advantage of both operands in registers and needs
3471 // same-as-first.
3472 src1_needs_reg = arch_shuffle->src1_needs_reg;
3473 no_same_as_first = false;
3474 } else if (wasm::SimdShuffle::TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
3475 uint8_t shuffle_mask = wasm::SimdShuffle::PackShuffle4(shuffle32x4);
3476 if (is_swizzle) {
3477 if (wasm::SimdShuffle::TryMatchIdentity(shuffle)) {
3478 // Bypass normal shuffle code generation in this case.
3479 EmitIdentity(node);
3480 return;
3481 } else {
3482 // pshufd takes a single imm8 shuffle mask.
3483 opcode = kX64S32x4Swizzle;
3484 no_same_as_first = true;
3485 // TODO(v8:9083): This doesn't strictly require a register, forcing the
3486 // swizzles to always use registers until generation of incorrect memory
3487 // operands can be fixed.
3488 src0_needs_reg = true;
3489 imms[imm_count++] = shuffle_mask;
3490 }
3491 } else {
3492 // 2 operand shuffle
3493 // A blend is more efficient than a general 32x4 shuffle; try it first.
3494 if (wasm::SimdShuffle::TryMatchBlend(shuffle)) {
3495 opcode = kX64S16x8Blend;
3496 uint8_t blend_mask = wasm::SimdShuffle::PackBlend4(shuffle32x4);
3497 imms[imm_count++] = blend_mask;
3498 } else {
3499 opcode = kX64S32x4Shuffle;
3500 no_same_as_first = true;
3501 // TODO(v8:9083): src0 and src1 is used by pshufd in codegen, which
3502 // requires memory to be 16-byte aligned, since we cannot guarantee that
3503 // yet, force using a register here.
3504 src0_needs_reg = true;
3505 src1_needs_reg = true;
3506 imms[imm_count++] = shuffle_mask;
3507 uint8_t blend_mask = wasm::SimdShuffle::PackBlend4(shuffle32x4);
3508 imms[imm_count++] = blend_mask;
3509 }
3510 }
3511 } else if (wasm::SimdShuffle::TryMatch16x8Shuffle(shuffle, shuffle16x8)) {
3512 uint8_t blend_mask;
3513 if (wasm::SimdShuffle::TryMatchBlend(shuffle)) {
3514 opcode = kX64S16x8Blend;
3515 blend_mask = wasm::SimdShuffle::PackBlend8(shuffle16x8);
3516 imms[imm_count++] = blend_mask;
3517 } else if (wasm::SimdShuffle::TryMatchSplat<8>(shuffle, &index)) {
3518 opcode = kX64S16x8Dup;
3519 src0_needs_reg = false;
3520 imms[imm_count++] = index;
3521 } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) {
3522 opcode = is_swizzle ? kX64S16x8HalfShuffle1 : kX64S16x8HalfShuffle2;
3523 // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0).
3524 no_same_as_first = true;
3525 src0_needs_reg = false;
3526 uint8_t mask_lo = wasm::SimdShuffle::PackShuffle4(shuffle16x8);
3527 uint8_t mask_hi = wasm::SimdShuffle::PackShuffle4(shuffle16x8 + 4);
3528 imms[imm_count++] = mask_lo;
3529 imms[imm_count++] = mask_hi;
3530 if (!is_swizzle) imms[imm_count++] = blend_mask;
3531 }
3532 } else if (wasm::SimdShuffle::TryMatchSplat<16>(shuffle, &index)) {
3533 opcode = kX64S8x16Dup;
3534 no_same_as_first = false;
3535 src0_needs_reg = true;
3536 imms[imm_count++] = index;
3537 }
3538 if (opcode == kX64I8x16Shuffle) {
3539 // Use same-as-first for general swizzle, but not shuffle.
3540 no_same_as_first = !is_swizzle;
3541 src0_needs_reg = !no_same_as_first;
3542 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle);
3543 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 4);
3544 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 8);
3545 imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 12);
3546 temps[temp_count++] = g.TempSimd128Register();
3547 }
3548
3549 // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra
3550 // move instruction in the CodeGenerator.
3551 Node* input0 = node->InputAt(0);
3552 InstructionOperand dst =
3553 no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3554 // TODO(v8:9198): Use src0_needs_reg when we have memory alignment for SIMD.
3555 InstructionOperand src0 = g.UseUniqueRegister(input0);
3556 USE(src0_needs_reg);
3557
3558 int input_count = 0;
3559 InstructionOperand inputs[2 + kMaxImms + kMaxTemps];
3560 inputs[input_count++] = src0;
3561 if (!is_swizzle) {
3562 Node* input1 = node->InputAt(1);
3563 // TODO(v8:9198): Use src1_needs_reg when we have memory alignment for SIMD.
3564 inputs[input_count++] = g.UseUniqueRegister(input1);
3565 USE(src1_needs_reg);
3566 }
3567 for (int i = 0; i < imm_count; ++i) {
3568 inputs[input_count++] = g.UseImmediate(imms[i]);
3569 }
3570 Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps);
3571 }
3572
VisitI8x16Swizzle(Node * node)3573 void InstructionSelector::VisitI8x16Swizzle(Node* node) {
3574 X64OperandGenerator g(this);
3575 InstructionOperand temps[] = {g.TempSimd128Register()};
3576 Emit(kX64I8x16Swizzle, g.DefineSameAsFirst(node),
3577 g.UseRegister(node->InputAt(0)), g.UseUniqueRegister(node->InputAt(1)),
3578 arraysize(temps), temps);
3579 }
3580
3581 namespace {
VisitPminOrPmax(InstructionSelector * selector,Node * node,ArchOpcode opcode)3582 void VisitPminOrPmax(InstructionSelector* selector, Node* node,
3583 ArchOpcode opcode) {
3584 // Due to the way minps/minpd work, we want the dst to be same as the second
3585 // input: b = pmin(a, b) directly maps to minps b a.
3586 X64OperandGenerator g(selector);
3587 selector->Emit(opcode, g.DefineSameAsFirst(node),
3588 g.UseRegister(node->InputAt(1)),
3589 g.UseRegister(node->InputAt(0)));
3590 }
3591 } // namespace
3592
VisitF32x4Pmin(Node * node)3593 void InstructionSelector::VisitF32x4Pmin(Node* node) {
3594 VisitPminOrPmax(this, node, kX64F32x4Pmin);
3595 }
3596
VisitF32x4Pmax(Node * node)3597 void InstructionSelector::VisitF32x4Pmax(Node* node) {
3598 VisitPminOrPmax(this, node, kX64F32x4Pmax);
3599 }
3600
VisitF64x2Pmin(Node * node)3601 void InstructionSelector::VisitF64x2Pmin(Node* node) {
3602 VisitPminOrPmax(this, node, kX64F64x2Pmin);
3603 }
3604
VisitF64x2Pmax(Node * node)3605 void InstructionSelector::VisitF64x2Pmax(Node* node) {
3606 VisitPminOrPmax(this, node, kX64F64x2Pmax);
3607 }
3608
3609 // static
3610 MachineOperatorBuilder::Flags
SupportedMachineOperatorFlags()3611 InstructionSelector::SupportedMachineOperatorFlags() {
3612 MachineOperatorBuilder::Flags flags =
3613 MachineOperatorBuilder::kWord32ShiftIsSafe |
3614 MachineOperatorBuilder::kWord32Ctz | MachineOperatorBuilder::kWord64Ctz |
3615 MachineOperatorBuilder::kWord32Rol | MachineOperatorBuilder::kWord64Rol;
3616 if (CpuFeatures::IsSupported(POPCNT)) {
3617 flags |= MachineOperatorBuilder::kWord32Popcnt |
3618 MachineOperatorBuilder::kWord64Popcnt;
3619 }
3620 if (CpuFeatures::IsSupported(SSE4_1)) {
3621 flags |= MachineOperatorBuilder::kFloat32RoundDown |
3622 MachineOperatorBuilder::kFloat64RoundDown |
3623 MachineOperatorBuilder::kFloat32RoundUp |
3624 MachineOperatorBuilder::kFloat64RoundUp |
3625 MachineOperatorBuilder::kFloat32RoundTruncate |
3626 MachineOperatorBuilder::kFloat64RoundTruncate |
3627 MachineOperatorBuilder::kFloat32RoundTiesEven |
3628 MachineOperatorBuilder::kFloat64RoundTiesEven;
3629 }
3630 return flags;
3631 }
3632
3633 // static
3634 MachineOperatorBuilder::AlignmentRequirements
AlignmentRequirements()3635 InstructionSelector::AlignmentRequirements() {
3636 return MachineOperatorBuilder::AlignmentRequirements::
3637 FullUnalignedAccessSupport();
3638 }
3639
3640 } // namespace compiler
3641 } // namespace internal
3642 } // namespace v8
3643