1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/base/iterator.h"
6 #include "src/compiler/backend/instruction-selector-impl.h"
7 #include "src/compiler/node-matchers.h"
8 #include "src/compiler/node-properties.h"
9
10 namespace v8 {
11 namespace internal {
12 namespace compiler {
13
14 // Adds IA32-specific methods for generating operands.
15 class IA32OperandGenerator final : public OperandGenerator {
16 public:
IA32OperandGenerator(InstructionSelector * selector)17 explicit IA32OperandGenerator(InstructionSelector* selector)
18 : OperandGenerator(selector) {}
19
UseByteRegister(Node * node)20 InstructionOperand UseByteRegister(Node* node) {
21 // TODO(titzer): encode byte register use constraints.
22 return UseFixed(node, edx);
23 }
24
DefineAsByteRegister(Node * node)25 InstructionOperand DefineAsByteRegister(Node* node) {
26 // TODO(titzer): encode byte register def constraints.
27 return DefineAsRegister(node);
28 }
29
CanBeMemoryOperand(InstructionCode opcode,Node * node,Node * input,int effect_level)30 bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
31 int effect_level) {
32 if (input->opcode() != IrOpcode::kLoad ||
33 !selector()->CanCover(node, input)) {
34 return false;
35 }
36 if (effect_level != selector()->GetEffectLevel(input)) {
37 return false;
38 }
39 MachineRepresentation rep =
40 LoadRepresentationOf(input->op()).representation();
41 switch (opcode) {
42 case kIA32And:
43 case kIA32Or:
44 case kIA32Xor:
45 case kIA32Add:
46 case kIA32Sub:
47 case kIA32Cmp:
48 case kIA32Test:
49 return rep == MachineRepresentation::kWord32 || IsAnyTagged(rep);
50 case kIA32Cmp16:
51 case kIA32Test16:
52 return rep == MachineRepresentation::kWord16;
53 case kIA32Cmp8:
54 case kIA32Test8:
55 return rep == MachineRepresentation::kWord8;
56 default:
57 break;
58 }
59 return false;
60 }
61
CanBeImmediate(Node * node)62 bool CanBeImmediate(Node* node) {
63 switch (node->opcode()) {
64 case IrOpcode::kInt32Constant:
65 case IrOpcode::kNumberConstant:
66 case IrOpcode::kExternalConstant:
67 case IrOpcode::kRelocatableInt32Constant:
68 case IrOpcode::kRelocatableInt64Constant:
69 return true;
70 case IrOpcode::kHeapConstant: {
71 // TODO(bmeurer): We must not dereference handles concurrently. If we
72 // really have to this here, then we need to find a way to put this
73 // information on the HeapConstant node already.
74 #if 0
75 // Constants in young generation cannot be used as immediates in V8
76 // because the GC does not scan code objects when collecting the young
77 // generation.
78 Handle<HeapObject> value = HeapConstantOf(node->op());
79 return !Heap::InYoungGeneration(*value);
80 #else
81 return false;
82 #endif
83 }
84 default:
85 return false;
86 }
87 }
88
GenerateMemoryOperandInputs(Node * index,int scale,Node * base,int32_t displacement,DisplacementMode displacement_mode,InstructionOperand inputs[],size_t * input_count,RegisterMode register_mode=kRegister)89 AddressingMode GenerateMemoryOperandInputs(
90 Node* index, int scale, Node* base, int32_t displacement,
91 DisplacementMode displacement_mode, InstructionOperand inputs[],
92 size_t* input_count, RegisterMode register_mode = kRegister) {
93 AddressingMode mode = kMode_MRI;
94 if (displacement_mode == kNegativeDisplacement) {
95 displacement = -displacement;
96 }
97 if (base != nullptr) {
98 if (base->opcode() == IrOpcode::kInt32Constant) {
99 displacement += OpParameter<int32_t>(base->op());
100 base = nullptr;
101 }
102 }
103 if (base != nullptr) {
104 inputs[(*input_count)++] = UseRegisterWithMode(base, register_mode);
105 if (index != nullptr) {
106 DCHECK(scale >= 0 && scale <= 3);
107 inputs[(*input_count)++] = UseRegisterWithMode(index, register_mode);
108 if (displacement != 0) {
109 inputs[(*input_count)++] = TempImmediate(displacement);
110 static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
111 kMode_MR4I, kMode_MR8I};
112 mode = kMRnI_modes[scale];
113 } else {
114 static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
115 kMode_MR4, kMode_MR8};
116 mode = kMRn_modes[scale];
117 }
118 } else {
119 if (displacement == 0) {
120 mode = kMode_MR;
121 } else {
122 inputs[(*input_count)++] = TempImmediate(displacement);
123 mode = kMode_MRI;
124 }
125 }
126 } else {
127 DCHECK(scale >= 0 && scale <= 3);
128 if (index != nullptr) {
129 inputs[(*input_count)++] = UseRegisterWithMode(index, register_mode);
130 if (displacement != 0) {
131 inputs[(*input_count)++] = TempImmediate(displacement);
132 static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
133 kMode_M4I, kMode_M8I};
134 mode = kMnI_modes[scale];
135 } else {
136 static const AddressingMode kMn_modes[] = {kMode_MR, kMode_M2,
137 kMode_M4, kMode_M8};
138 mode = kMn_modes[scale];
139 }
140 } else {
141 inputs[(*input_count)++] = TempImmediate(displacement);
142 return kMode_MI;
143 }
144 }
145 return mode;
146 }
147
GenerateMemoryOperandInputs(Node * index,int scale,Node * base,Node * displacement_node,DisplacementMode displacement_mode,InstructionOperand inputs[],size_t * input_count,RegisterMode register_mode=kRegister)148 AddressingMode GenerateMemoryOperandInputs(
149 Node* index, int scale, Node* base, Node* displacement_node,
150 DisplacementMode displacement_mode, InstructionOperand inputs[],
151 size_t* input_count, RegisterMode register_mode = kRegister) {
152 int32_t displacement = (displacement_node == nullptr)
153 ? 0
154 : OpParameter<int32_t>(displacement_node->op());
155 return GenerateMemoryOperandInputs(index, scale, base, displacement,
156 displacement_mode, inputs, input_count,
157 register_mode);
158 }
159
GetEffectiveAddressMemoryOperand(Node * node,InstructionOperand inputs[],size_t * input_count,RegisterMode register_mode=kRegister)160 AddressingMode GetEffectiveAddressMemoryOperand(
161 Node* node, InstructionOperand inputs[], size_t* input_count,
162 RegisterMode register_mode = kRegister) {
163 {
164 LoadMatcher<ExternalReferenceMatcher> m(node);
165 if (m.index().HasValue() && m.object().HasValue() &&
166 selector()->CanAddressRelativeToRootsRegister(m.object().Value())) {
167 ptrdiff_t const delta =
168 m.index().Value() +
169 TurboAssemblerBase::RootRegisterOffsetForExternalReference(
170 selector()->isolate(), m.object().Value());
171 if (is_int32(delta)) {
172 inputs[(*input_count)++] = TempImmediate(static_cast<int32_t>(delta));
173 return kMode_Root;
174 }
175 }
176 }
177
178 BaseWithIndexAndDisplacement32Matcher m(node, AddressOption::kAllowAll);
179 DCHECK(m.matches());
180 if ((m.displacement() == nullptr || CanBeImmediate(m.displacement()))) {
181 return GenerateMemoryOperandInputs(
182 m.index(), m.scale(), m.base(), m.displacement(),
183 m.displacement_mode(), inputs, input_count, register_mode);
184 } else {
185 inputs[(*input_count)++] =
186 UseRegisterWithMode(node->InputAt(0), register_mode);
187 inputs[(*input_count)++] =
188 UseRegisterWithMode(node->InputAt(1), register_mode);
189 return kMode_MR1;
190 }
191 }
192
GetEffectiveIndexOperand(Node * index,AddressingMode * mode)193 InstructionOperand GetEffectiveIndexOperand(Node* index,
194 AddressingMode* mode) {
195 if (CanBeImmediate(index)) {
196 *mode = kMode_MRI;
197 return UseImmediate(index);
198 } else {
199 *mode = kMode_MR1;
200 return UseUniqueRegister(index);
201 }
202 }
203
CanBeBetterLeftOperand(Node * node) const204 bool CanBeBetterLeftOperand(Node* node) const {
205 return !selector()->IsLive(node);
206 }
207 };
208
209 namespace {
210
VisitRO(InstructionSelector * selector,Node * node,ArchOpcode opcode)211 void VisitRO(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
212 IA32OperandGenerator g(selector);
213 Node* input = node->InputAt(0);
214 // We have to use a byte register as input to movsxb.
215 InstructionOperand input_op =
216 opcode == kIA32Movsxbl ? g.UseFixed(input, eax) : g.Use(input);
217 selector->Emit(opcode, g.DefineAsRegister(node), input_op);
218 }
219
VisitROWithTemp(InstructionSelector * selector,Node * node,ArchOpcode opcode)220 void VisitROWithTemp(InstructionSelector* selector, Node* node,
221 ArchOpcode opcode) {
222 IA32OperandGenerator g(selector);
223 InstructionOperand temps[] = {g.TempRegister()};
224 selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
225 arraysize(temps), temps);
226 }
227
VisitROWithTempSimd(InstructionSelector * selector,Node * node,ArchOpcode opcode)228 void VisitROWithTempSimd(InstructionSelector* selector, Node* node,
229 ArchOpcode opcode) {
230 IA32OperandGenerator g(selector);
231 InstructionOperand temps[] = {g.TempSimd128Register()};
232 selector->Emit(opcode, g.DefineAsRegister(node),
233 g.UseUniqueRegister(node->InputAt(0)), arraysize(temps),
234 temps);
235 }
236
VisitRR(InstructionSelector * selector,Node * node,InstructionCode opcode)237 void VisitRR(InstructionSelector* selector, Node* node,
238 InstructionCode opcode) {
239 IA32OperandGenerator g(selector);
240 selector->Emit(opcode, g.DefineAsRegister(node),
241 g.UseRegister(node->InputAt(0)));
242 }
243
VisitRROFloat(InstructionSelector * selector,Node * node,ArchOpcode avx_opcode,ArchOpcode sse_opcode)244 void VisitRROFloat(InstructionSelector* selector, Node* node,
245 ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
246 IA32OperandGenerator g(selector);
247 InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
248 InstructionOperand operand1 = g.Use(node->InputAt(1));
249 if (selector->IsSupported(AVX)) {
250 selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
251 } else {
252 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
253 }
254 }
255
VisitFloatUnop(InstructionSelector * selector,Node * node,Node * input,ArchOpcode avx_opcode,ArchOpcode sse_opcode)256 void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
257 ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
258 IA32OperandGenerator g(selector);
259 InstructionOperand temps[] = {g.TempSimd128Register()};
260 if (selector->IsSupported(AVX)) {
261 selector->Emit(avx_opcode, g.DefineAsRegister(node), g.UseUnique(input),
262 arraysize(temps), temps);
263 } else {
264 selector->Emit(sse_opcode, g.DefineSameAsFirst(node),
265 g.UseUniqueRegister(input), arraysize(temps), temps);
266 }
267 }
268
VisitRRSimd(InstructionSelector * selector,Node * node,ArchOpcode avx_opcode,ArchOpcode sse_opcode)269 void VisitRRSimd(InstructionSelector* selector, Node* node,
270 ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
271 IA32OperandGenerator g(selector);
272 InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
273 if (selector->IsSupported(AVX)) {
274 selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0);
275 } else {
276 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0);
277 }
278 }
279
VisitRRISimd(InstructionSelector * selector,Node * node,ArchOpcode opcode)280 void VisitRRISimd(InstructionSelector* selector, Node* node,
281 ArchOpcode opcode) {
282 IA32OperandGenerator g(selector);
283 InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
284 InstructionOperand operand1 =
285 g.UseImmediate(OpParameter<int32_t>(node->op()));
286 // 8x16 uses movsx_b on dest to extract a byte, which only works
287 // if dest is a byte register.
288 InstructionOperand dest = opcode == kIA32I8x16ExtractLaneS
289 ? g.DefineAsFixed(node, eax)
290 : g.DefineAsRegister(node);
291 selector->Emit(opcode, dest, operand0, operand1);
292 }
293
VisitRRISimd(InstructionSelector * selector,Node * node,ArchOpcode avx_opcode,ArchOpcode sse_opcode)294 void VisitRRISimd(InstructionSelector* selector, Node* node,
295 ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
296 IA32OperandGenerator g(selector);
297 InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
298 InstructionOperand operand1 =
299 g.UseImmediate(OpParameter<int32_t>(node->op()));
300 if (selector->IsSupported(AVX)) {
301 selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
302 } else {
303 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
304 }
305 }
306
VisitRROSimdShift(InstructionSelector * selector,Node * node,ArchOpcode opcode)307 void VisitRROSimdShift(InstructionSelector* selector, Node* node,
308 ArchOpcode opcode) {
309 IA32OperandGenerator g(selector);
310 if (g.CanBeImmediate(node->InputAt(1))) {
311 selector->Emit(opcode, g.DefineSameAsFirst(node),
312 g.UseRegister(node->InputAt(0)),
313 g.UseImmediate(node->InputAt(1)));
314 } else {
315 InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
316 InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1));
317 InstructionOperand temps[] = {g.TempSimd128Register()};
318 selector->Emit(opcode, g.DefineSameAsFirst(node), operand0, operand1,
319 arraysize(temps), temps);
320 }
321 }
322
VisitRROI8x16SimdShift(InstructionSelector * selector,Node * node,ArchOpcode opcode)323 void VisitRROI8x16SimdShift(InstructionSelector* selector, Node* node,
324 ArchOpcode opcode) {
325 IA32OperandGenerator g(selector);
326 InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
327 InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1));
328 InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
329 selector->Emit(opcode, g.DefineSameAsFirst(node), operand0, operand1,
330 arraysize(temps), temps);
331 }
332 } // namespace
333
VisitStackSlot(Node * node)334 void InstructionSelector::VisitStackSlot(Node* node) {
335 StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
336 int slot = frame_->AllocateSpillSlot(rep.size());
337 OperandGenerator g(this);
338
339 Emit(kArchStackSlot, g.DefineAsRegister(node),
340 sequence()->AddImmediate(Constant(slot)), 0, nullptr);
341 }
342
VisitAbortCSAAssert(Node * node)343 void InstructionSelector::VisitAbortCSAAssert(Node* node) {
344 IA32OperandGenerator g(this);
345 Emit(kArchAbortCSAAssert, g.NoOutput(), g.UseFixed(node->InputAt(0), edx));
346 }
347
VisitLoadTransform(Node * node)348 void InstructionSelector::VisitLoadTransform(Node* node) {
349 LoadTransformParameters params = LoadTransformParametersOf(node->op());
350 InstructionCode opcode = kArchNop;
351 switch (params.transformation) {
352 case LoadTransformation::kS8x16LoadSplat:
353 opcode = kIA32S8x16LoadSplat;
354 break;
355 case LoadTransformation::kS16x8LoadSplat:
356 opcode = kIA32S16x8LoadSplat;
357 break;
358 case LoadTransformation::kS32x4LoadSplat:
359 opcode = kIA32S32x4LoadSplat;
360 break;
361 case LoadTransformation::kS64x2LoadSplat:
362 opcode = kIA32S64x2LoadSplat;
363 break;
364 case LoadTransformation::kI16x8Load8x8S:
365 opcode = kIA32I16x8Load8x8S;
366 break;
367 case LoadTransformation::kI16x8Load8x8U:
368 opcode = kIA32I16x8Load8x8U;
369 break;
370 case LoadTransformation::kI32x4Load16x4S:
371 opcode = kIA32I32x4Load16x4S;
372 break;
373 case LoadTransformation::kI32x4Load16x4U:
374 opcode = kIA32I32x4Load16x4U;
375 break;
376 case LoadTransformation::kI64x2Load32x2S:
377 opcode = kIA32I64x2Load32x2S;
378 break;
379 case LoadTransformation::kI64x2Load32x2U:
380 opcode = kIA32I64x2Load32x2U;
381 break;
382 default:
383 UNREACHABLE();
384 }
385
386 // IA32 supports unaligned loads.
387 DCHECK_NE(params.kind, LoadKind::kUnaligned);
388 // Trap handler is not supported on IA32.
389 DCHECK_NE(params.kind, LoadKind::kProtected);
390
391 IA32OperandGenerator g(this);
392 InstructionOperand outputs[1];
393 outputs[0] = g.DefineAsRegister(node);
394 InstructionOperand inputs[3];
395 size_t input_count = 0;
396 AddressingMode mode =
397 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
398 InstructionCode code = opcode | AddressingModeField::encode(mode);
399 Emit(code, 1, outputs, input_count, inputs);
400 }
401
VisitLoad(Node * node)402 void InstructionSelector::VisitLoad(Node* node) {
403 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
404
405 ArchOpcode opcode = kArchNop;
406 switch (load_rep.representation()) {
407 case MachineRepresentation::kFloat32:
408 opcode = kIA32Movss;
409 break;
410 case MachineRepresentation::kFloat64:
411 opcode = kIA32Movsd;
412 break;
413 case MachineRepresentation::kBit: // Fall through.
414 case MachineRepresentation::kWord8:
415 opcode = load_rep.IsSigned() ? kIA32Movsxbl : kIA32Movzxbl;
416 break;
417 case MachineRepresentation::kWord16:
418 opcode = load_rep.IsSigned() ? kIA32Movsxwl : kIA32Movzxwl;
419 break;
420 case MachineRepresentation::kTaggedSigned: // Fall through.
421 case MachineRepresentation::kTaggedPointer: // Fall through.
422 case MachineRepresentation::kTagged: // Fall through.
423 case MachineRepresentation::kWord32:
424 opcode = kIA32Movl;
425 break;
426 case MachineRepresentation::kSimd128:
427 opcode = kIA32Movdqu;
428 break;
429 case MachineRepresentation::kCompressedPointer: // Fall through.
430 case MachineRepresentation::kCompressed: // Fall through.
431 case MachineRepresentation::kWord64: // Fall through.
432 case MachineRepresentation::kNone:
433 UNREACHABLE();
434 }
435
436 IA32OperandGenerator g(this);
437 InstructionOperand outputs[1];
438 outputs[0] = g.DefineAsRegister(node);
439 InstructionOperand inputs[3];
440 size_t input_count = 0;
441 AddressingMode mode =
442 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
443 InstructionCode code = opcode | AddressingModeField::encode(mode);
444 if (node->opcode() == IrOpcode::kPoisonedLoad) {
445 CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
446 code |= MiscField::encode(kMemoryAccessPoisoned);
447 }
448 Emit(code, 1, outputs, input_count, inputs);
449 }
450
VisitPoisonedLoad(Node * node)451 void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }
452
VisitProtectedLoad(Node * node)453 void InstructionSelector::VisitProtectedLoad(Node* node) {
454 // TODO(eholk)
455 UNIMPLEMENTED();
456 }
457
VisitStore(Node * node)458 void InstructionSelector::VisitStore(Node* node) {
459 IA32OperandGenerator g(this);
460 Node* base = node->InputAt(0);
461 Node* index = node->InputAt(1);
462 Node* value = node->InputAt(2);
463
464 StoreRepresentation store_rep = StoreRepresentationOf(node->op());
465 WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
466 MachineRepresentation rep = store_rep.representation();
467
468 if (write_barrier_kind != kNoWriteBarrier &&
469 V8_LIKELY(!FLAG_disable_write_barriers)) {
470 DCHECK(CanBeTaggedPointer(rep));
471 AddressingMode addressing_mode;
472 InstructionOperand inputs[] = {
473 g.UseUniqueRegister(base),
474 g.GetEffectiveIndexOperand(index, &addressing_mode),
475 g.UseUniqueRegister(value)};
476 RecordWriteMode record_write_mode =
477 WriteBarrierKindToRecordWriteMode(write_barrier_kind);
478 InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
479 size_t const temp_count = arraysize(temps);
480 InstructionCode code = kArchStoreWithWriteBarrier;
481 code |= AddressingModeField::encode(addressing_mode);
482 code |= MiscField::encode(static_cast<int>(record_write_mode));
483 Emit(code, 0, nullptr, arraysize(inputs), inputs, temp_count, temps);
484 } else {
485 ArchOpcode opcode = kArchNop;
486 switch (rep) {
487 case MachineRepresentation::kFloat32:
488 opcode = kIA32Movss;
489 break;
490 case MachineRepresentation::kFloat64:
491 opcode = kIA32Movsd;
492 break;
493 case MachineRepresentation::kBit: // Fall through.
494 case MachineRepresentation::kWord8:
495 opcode = kIA32Movb;
496 break;
497 case MachineRepresentation::kWord16:
498 opcode = kIA32Movw;
499 break;
500 case MachineRepresentation::kTaggedSigned: // Fall through.
501 case MachineRepresentation::kTaggedPointer: // Fall through.
502 case MachineRepresentation::kTagged: // Fall through.
503 case MachineRepresentation::kWord32:
504 opcode = kIA32Movl;
505 break;
506 case MachineRepresentation::kSimd128:
507 opcode = kIA32Movdqu;
508 break;
509 case MachineRepresentation::kCompressedPointer: // Fall through.
510 case MachineRepresentation::kCompressed: // Fall through.
511 case MachineRepresentation::kWord64: // Fall through.
512 case MachineRepresentation::kNone:
513 UNREACHABLE();
514 return;
515 }
516
517 InstructionOperand val;
518 if (g.CanBeImmediate(value)) {
519 val = g.UseImmediate(value);
520 } else if (rep == MachineRepresentation::kWord8 ||
521 rep == MachineRepresentation::kBit) {
522 val = g.UseByteRegister(value);
523 } else {
524 val = g.UseRegister(value);
525 }
526
527 InstructionOperand inputs[4];
528 size_t input_count = 0;
529 AddressingMode addressing_mode =
530 g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
531 InstructionCode code =
532 opcode | AddressingModeField::encode(addressing_mode);
533 inputs[input_count++] = val;
534 Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count,
535 inputs);
536 }
537 }
538
VisitProtectedStore(Node * node)539 void InstructionSelector::VisitProtectedStore(Node* node) {
540 // TODO(eholk)
541 UNIMPLEMENTED();
542 }
543
544 // Architecture supports unaligned access, therefore VisitLoad is used instead
VisitUnalignedLoad(Node * node)545 void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
546
547 // Architecture supports unaligned access, therefore VisitStore is used instead
VisitUnalignedStore(Node * node)548 void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
549
550 namespace {
551
552 // Shared routine for multiple binary operations.
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)553 void VisitBinop(InstructionSelector* selector, Node* node,
554 InstructionCode opcode, FlagsContinuation* cont) {
555 IA32OperandGenerator g(selector);
556 Int32BinopMatcher m(node);
557 Node* left = m.left().node();
558 Node* right = m.right().node();
559 InstructionOperand inputs[6];
560 size_t input_count = 0;
561 InstructionOperand outputs[1];
562 size_t output_count = 0;
563
564 // TODO(turbofan): match complex addressing modes.
565 if (left == right) {
566 // If both inputs refer to the same operand, enforce allocating a register
567 // for both of them to ensure that we don't end up generating code like
568 // this:
569 //
570 // mov eax, [ebp-0x10]
571 // add eax, [ebp-0x10]
572 // jo label
573 InstructionOperand const input = g.UseRegister(left);
574 inputs[input_count++] = input;
575 inputs[input_count++] = input;
576 } else if (g.CanBeImmediate(right)) {
577 inputs[input_count++] = g.UseRegister(left);
578 inputs[input_count++] = g.UseImmediate(right);
579 } else {
580 int effect_level = selector->GetEffectLevel(node);
581 if (cont->IsBranch()) {
582 effect_level = selector->GetEffectLevel(
583 cont->true_block()->PredecessorAt(0)->control_input());
584 }
585 if (node->op()->HasProperty(Operator::kCommutative) &&
586 g.CanBeBetterLeftOperand(right) &&
587 (!g.CanBeBetterLeftOperand(left) ||
588 !g.CanBeMemoryOperand(opcode, node, right, effect_level))) {
589 std::swap(left, right);
590 }
591 if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) {
592 inputs[input_count++] = g.UseRegister(left);
593 AddressingMode addressing_mode =
594 g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
595 opcode |= AddressingModeField::encode(addressing_mode);
596 } else {
597 inputs[input_count++] = g.UseRegister(left);
598 inputs[input_count++] = g.Use(right);
599 }
600 }
601
602 outputs[output_count++] = g.DefineSameAsFirst(node);
603
604 DCHECK_NE(0u, input_count);
605 DCHECK_EQ(1u, output_count);
606 DCHECK_GE(arraysize(inputs), input_count);
607 DCHECK_GE(arraysize(outputs), output_count);
608
609 selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
610 inputs, cont);
611 }
612
613 // Shared routine for multiple binary operations.
VisitBinop(InstructionSelector * selector,Node * node,InstructionCode opcode)614 void VisitBinop(InstructionSelector* selector, Node* node,
615 InstructionCode opcode) {
616 FlagsContinuation cont;
617 VisitBinop(selector, node, opcode, &cont);
618 }
619
620 } // namespace
621
VisitWord32And(Node * node)622 void InstructionSelector::VisitWord32And(Node* node) {
623 VisitBinop(this, node, kIA32And);
624 }
625
VisitWord32Or(Node * node)626 void InstructionSelector::VisitWord32Or(Node* node) {
627 VisitBinop(this, node, kIA32Or);
628 }
629
VisitWord32Xor(Node * node)630 void InstructionSelector::VisitWord32Xor(Node* node) {
631 IA32OperandGenerator g(this);
632 Int32BinopMatcher m(node);
633 if (m.right().Is(-1)) {
634 Emit(kIA32Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
635 } else {
636 VisitBinop(this, node, kIA32Xor);
637 }
638 }
639
VisitStackPointerGreaterThan(Node * node,FlagsContinuation * cont)640 void InstructionSelector::VisitStackPointerGreaterThan(
641 Node* node, FlagsContinuation* cont) {
642 StackCheckKind kind = StackCheckKindOf(node->op());
643 InstructionCode opcode =
644 kArchStackPointerGreaterThan | MiscField::encode(static_cast<int>(kind));
645
646 int effect_level = GetEffectLevel(node);
647 if (cont->IsBranch()) {
648 effect_level =
649 GetEffectLevel(cont->true_block()->PredecessorAt(0)->control_input());
650 }
651
652 IA32OperandGenerator g(this);
653
654 // No outputs.
655 InstructionOperand* const outputs = nullptr;
656 const int output_count = 0;
657
658 // Applying an offset to this stack check requires a temp register. Offsets
659 // are only applied to the first stack check. If applying an offset, we must
660 // ensure the input and temp registers do not alias, thus kUniqueRegister.
661 InstructionOperand temps[] = {g.TempRegister()};
662 const int temp_count = (kind == StackCheckKind::kJSFunctionEntry) ? 1 : 0;
663 const auto register_mode = (kind == StackCheckKind::kJSFunctionEntry)
664 ? OperandGenerator::kUniqueRegister
665 : OperandGenerator::kRegister;
666
667 Node* const value = node->InputAt(0);
668 if (g.CanBeMemoryOperand(kIA32Cmp, node, value, effect_level)) {
669 DCHECK_EQ(IrOpcode::kLoad, value->opcode());
670
671 // GetEffectiveAddressMemoryOperand can create at most 3 inputs.
672 static constexpr int kMaxInputCount = 3;
673
674 size_t input_count = 0;
675 InstructionOperand inputs[kMaxInputCount];
676 AddressingMode addressing_mode = g.GetEffectiveAddressMemoryOperand(
677 value, inputs, &input_count, register_mode);
678 opcode |= AddressingModeField::encode(addressing_mode);
679 DCHECK_LE(input_count, kMaxInputCount);
680
681 EmitWithContinuation(opcode, output_count, outputs, input_count, inputs,
682 temp_count, temps, cont);
683 } else {
684 InstructionOperand inputs[] = {g.UseRegisterWithMode(value, register_mode)};
685 static constexpr int input_count = arraysize(inputs);
686 EmitWithContinuation(opcode, output_count, outputs, input_count, inputs,
687 temp_count, temps, cont);
688 }
689 }
690
691 // Shared routine for multiple shift operations.
VisitShift(InstructionSelector * selector,Node * node,ArchOpcode opcode)692 static inline void VisitShift(InstructionSelector* selector, Node* node,
693 ArchOpcode opcode) {
694 IA32OperandGenerator g(selector);
695 Node* left = node->InputAt(0);
696 Node* right = node->InputAt(1);
697
698 if (g.CanBeImmediate(right)) {
699 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
700 g.UseImmediate(right));
701 } else {
702 selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
703 g.UseFixed(right, ecx));
704 }
705 }
706
707 namespace {
708
VisitMulHigh(InstructionSelector * selector,Node * node,ArchOpcode opcode)709 void VisitMulHigh(InstructionSelector* selector, Node* node,
710 ArchOpcode opcode) {
711 IA32OperandGenerator g(selector);
712 InstructionOperand temps[] = {g.TempRegister(eax)};
713 selector->Emit(
714 opcode, g.DefineAsFixed(node, edx), g.UseFixed(node->InputAt(0), eax),
715 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
716 }
717
VisitDiv(InstructionSelector * selector,Node * node,ArchOpcode opcode)718 void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
719 IA32OperandGenerator g(selector);
720 InstructionOperand temps[] = {g.TempRegister(edx)};
721 selector->Emit(opcode, g.DefineAsFixed(node, eax),
722 g.UseFixed(node->InputAt(0), eax),
723 g.UseUnique(node->InputAt(1)), arraysize(temps), temps);
724 }
725
VisitMod(InstructionSelector * selector,Node * node,ArchOpcode opcode)726 void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
727 IA32OperandGenerator g(selector);
728 InstructionOperand temps[] = {g.TempRegister(eax)};
729 selector->Emit(opcode, g.DefineAsFixed(node, edx),
730 g.UseFixed(node->InputAt(0), eax),
731 g.UseUnique(node->InputAt(1)), arraysize(temps), temps);
732 }
733
EmitLea(InstructionSelector * selector,Node * result,Node * index,int scale,Node * base,Node * displacement,DisplacementMode displacement_mode)734 void EmitLea(InstructionSelector* selector, Node* result, Node* index,
735 int scale, Node* base, Node* displacement,
736 DisplacementMode displacement_mode) {
737 IA32OperandGenerator g(selector);
738 InstructionOperand inputs[4];
739 size_t input_count = 0;
740 AddressingMode mode =
741 g.GenerateMemoryOperandInputs(index, scale, base, displacement,
742 displacement_mode, inputs, &input_count);
743
744 DCHECK_NE(0u, input_count);
745 DCHECK_GE(arraysize(inputs), input_count);
746
747 InstructionOperand outputs[1];
748 outputs[0] = g.DefineAsRegister(result);
749
750 InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea;
751
752 selector->Emit(opcode, 1, outputs, input_count, inputs);
753 }
754
755 } // namespace
756
VisitWord32Shl(Node * node)757 void InstructionSelector::VisitWord32Shl(Node* node) {
758 Int32ScaleMatcher m(node, true);
759 if (m.matches()) {
760 Node* index = node->InputAt(0);
761 Node* base = m.power_of_two_plus_one() ? index : nullptr;
762 EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement);
763 return;
764 }
765 VisitShift(this, node, kIA32Shl);
766 }
767
VisitWord32Shr(Node * node)768 void InstructionSelector::VisitWord32Shr(Node* node) {
769 VisitShift(this, node, kIA32Shr);
770 }
771
VisitWord32Sar(Node * node)772 void InstructionSelector::VisitWord32Sar(Node* node) {
773 VisitShift(this, node, kIA32Sar);
774 }
775
VisitInt32PairAdd(Node * node)776 void InstructionSelector::VisitInt32PairAdd(Node* node) {
777 IA32OperandGenerator g(this);
778
779 Node* projection1 = NodeProperties::FindProjection(node, 1);
780 if (projection1) {
781 // We use UseUniqueRegister here to avoid register sharing with the temp
782 // register.
783 InstructionOperand inputs[] = {
784 g.UseRegister(node->InputAt(0)),
785 g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
786 g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))};
787
788 InstructionOperand outputs[] = {g.DefineSameAsFirst(node),
789 g.DefineAsRegister(projection1)};
790
791 InstructionOperand temps[] = {g.TempRegister()};
792
793 Emit(kIA32AddPair, 2, outputs, 4, inputs, 1, temps);
794 } else {
795 // The high word of the result is not used, so we emit the standard 32 bit
796 // instruction.
797 Emit(kIA32Add, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
798 g.Use(node->InputAt(2)));
799 }
800 }
801
VisitInt32PairSub(Node * node)802 void InstructionSelector::VisitInt32PairSub(Node* node) {
803 IA32OperandGenerator g(this);
804
805 Node* projection1 = NodeProperties::FindProjection(node, 1);
806 if (projection1) {
807 // We use UseUniqueRegister here to avoid register sharing with the temp
808 // register.
809 InstructionOperand inputs[] = {
810 g.UseRegister(node->InputAt(0)),
811 g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
812 g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))};
813
814 InstructionOperand outputs[] = {g.DefineSameAsFirst(node),
815 g.DefineAsRegister(projection1)};
816
817 InstructionOperand temps[] = {g.TempRegister()};
818
819 Emit(kIA32SubPair, 2, outputs, 4, inputs, 1, temps);
820 } else {
821 // The high word of the result is not used, so we emit the standard 32 bit
822 // instruction.
823 Emit(kIA32Sub, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
824 g.Use(node->InputAt(2)));
825 }
826 }
827
VisitInt32PairMul(Node * node)828 void InstructionSelector::VisitInt32PairMul(Node* node) {
829 IA32OperandGenerator g(this);
830
831 Node* projection1 = NodeProperties::FindProjection(node, 1);
832 if (projection1) {
833 // InputAt(3) explicitly shares ecx with OutputRegister(1) to save one
834 // register and one mov instruction.
835 InstructionOperand inputs[] = {
836 g.UseUnique(node->InputAt(0)),
837 g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
838 g.UseUniqueRegister(node->InputAt(2)),
839 g.UseFixed(node->InputAt(3), ecx)};
840
841 InstructionOperand outputs[] = {
842 g.DefineAsFixed(node, eax),
843 g.DefineAsFixed(NodeProperties::FindProjection(node, 1), ecx)};
844
845 InstructionOperand temps[] = {g.TempRegister(edx)};
846
847 Emit(kIA32MulPair, 2, outputs, 4, inputs, 1, temps);
848 } else {
849 // The high word of the result is not used, so we emit the standard 32 bit
850 // instruction.
851 Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
852 g.Use(node->InputAt(2)));
853 }
854 }
855
VisitWord32PairShift(InstructionSelector * selector,InstructionCode opcode,Node * node)856 void VisitWord32PairShift(InstructionSelector* selector, InstructionCode opcode,
857 Node* node) {
858 IA32OperandGenerator g(selector);
859
860 Node* shift = node->InputAt(2);
861 InstructionOperand shift_operand;
862 if (g.CanBeImmediate(shift)) {
863 shift_operand = g.UseImmediate(shift);
864 } else {
865 shift_operand = g.UseFixed(shift, ecx);
866 }
867 InstructionOperand inputs[] = {g.UseFixed(node->InputAt(0), eax),
868 g.UseFixed(node->InputAt(1), edx),
869 shift_operand};
870
871 InstructionOperand outputs[2];
872 InstructionOperand temps[1];
873 int32_t output_count = 0;
874 int32_t temp_count = 0;
875 outputs[output_count++] = g.DefineAsFixed(node, eax);
876 Node* projection1 = NodeProperties::FindProjection(node, 1);
877 if (projection1) {
878 outputs[output_count++] = g.DefineAsFixed(projection1, edx);
879 } else {
880 temps[temp_count++] = g.TempRegister(edx);
881 }
882
883 selector->Emit(opcode, output_count, outputs, 3, inputs, temp_count, temps);
884 }
885
VisitWord32PairShl(Node * node)886 void InstructionSelector::VisitWord32PairShl(Node* node) {
887 VisitWord32PairShift(this, kIA32ShlPair, node);
888 }
889
VisitWord32PairShr(Node * node)890 void InstructionSelector::VisitWord32PairShr(Node* node) {
891 VisitWord32PairShift(this, kIA32ShrPair, node);
892 }
893
VisitWord32PairSar(Node * node)894 void InstructionSelector::VisitWord32PairSar(Node* node) {
895 VisitWord32PairShift(this, kIA32SarPair, node);
896 }
897
VisitWord32Ror(Node * node)898 void InstructionSelector::VisitWord32Ror(Node* node) {
899 VisitShift(this, node, kIA32Ror);
900 }
901
902 #define RO_OP_LIST(V) \
903 V(Word32Clz, kIA32Lzcnt) \
904 V(Word32Ctz, kIA32Tzcnt) \
905 V(Word32Popcnt, kIA32Popcnt) \
906 V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64) \
907 V(RoundInt32ToFloat32, kSSEInt32ToFloat32) \
908 V(ChangeInt32ToFloat64, kSSEInt32ToFloat64) \
909 V(TruncateFloat32ToInt32, kSSEFloat32ToInt32) \
910 V(ChangeFloat64ToInt32, kSSEFloat64ToInt32) \
911 V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32) \
912 V(RoundFloat64ToInt32, kSSEFloat64ToInt32) \
913 V(BitcastFloat32ToInt32, kIA32BitcastFI) \
914 V(BitcastInt32ToFloat32, kIA32BitcastIF) \
915 V(Float32Sqrt, kSSEFloat32Sqrt) \
916 V(Float64Sqrt, kSSEFloat64Sqrt) \
917 V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32) \
918 V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \
919 V(SignExtendWord8ToInt32, kIA32Movsxbl) \
920 V(SignExtendWord16ToInt32, kIA32Movsxwl) \
921 V(F64x2Sqrt, kIA32F64x2Sqrt)
922
923 #define RO_WITH_TEMP_OP_LIST(V) V(ChangeUint32ToFloat64, kSSEUint32ToFloat64)
924
925 #define RO_WITH_TEMP_SIMD_OP_LIST(V) \
926 V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \
927 V(ChangeFloat64ToUint32, kSSEFloat64ToUint32) \
928 V(TruncateFloat64ToUint32, kSSEFloat64ToUint32)
929
930 #define RR_OP_LIST(V) \
931 V(TruncateFloat64ToWord32, kArchTruncateDoubleToI) \
932 V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown)) \
933 V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown)) \
934 V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp)) \
935 V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp)) \
936 V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \
937 V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \
938 V(Float32RoundTiesEven, \
939 kSSEFloat32Round | MiscField::encode(kRoundToNearest)) \
940 V(Float64RoundTiesEven, kSSEFloat64Round | MiscField::encode(kRoundToNearest))
941
942 #define RRO_FLOAT_OP_LIST(V) \
943 V(Float32Add, kAVXFloat32Add, kSSEFloat32Add) \
944 V(Float64Add, kAVXFloat64Add, kSSEFloat64Add) \
945 V(Float32Sub, kAVXFloat32Sub, kSSEFloat32Sub) \
946 V(Float64Sub, kAVXFloat64Sub, kSSEFloat64Sub) \
947 V(Float32Mul, kAVXFloat32Mul, kSSEFloat32Mul) \
948 V(Float64Mul, kAVXFloat64Mul, kSSEFloat64Mul) \
949 V(Float32Div, kAVXFloat32Div, kSSEFloat32Div) \
950 V(Float64Div, kAVXFloat64Div, kSSEFloat64Div) \
951 V(F64x2Add, kIA32F64x2Add, kIA32F64x2Add) \
952 V(F64x2Sub, kIA32F64x2Sub, kIA32F64x2Sub) \
953 V(F64x2Mul, kIA32F64x2Mul, kIA32F64x2Mul) \
954 V(F64x2Div, kIA32F64x2Div, kIA32F64x2Div) \
955 V(F64x2Eq, kIA32F64x2Eq, kIA32F64x2Eq) \
956 V(F64x2Ne, kIA32F64x2Ne, kIA32F64x2Ne) \
957 V(F64x2Lt, kIA32F64x2Lt, kIA32F64x2Lt) \
958 V(F64x2Le, kIA32F64x2Le, kIA32F64x2Le)
959
960 #define FLOAT_UNOP_LIST(V) \
961 V(Float32Abs, kAVXFloat32Abs, kSSEFloat32Abs) \
962 V(Float64Abs, kAVXFloat64Abs, kSSEFloat64Abs) \
963 V(Float32Neg, kAVXFloat32Neg, kSSEFloat32Neg) \
964 V(Float64Neg, kAVXFloat64Neg, kSSEFloat64Neg) \
965 V(F64x2Abs, kAVXFloat64Abs, kSSEFloat64Abs) \
966 V(F64x2Neg, kAVXFloat64Neg, kSSEFloat64Neg)
967
968 #define RO_VISITOR(Name, opcode) \
969 void InstructionSelector::Visit##Name(Node* node) { \
970 VisitRO(this, node, opcode); \
971 }
972 RO_OP_LIST(RO_VISITOR)
973 #undef RO_VISITOR
974 #undef RO_OP_LIST
975
976 #define RO_WITH_TEMP_VISITOR(Name, opcode) \
977 void InstructionSelector::Visit##Name(Node* node) { \
978 VisitROWithTemp(this, node, opcode); \
979 }
RO_WITH_TEMP_OP_LIST(RO_WITH_TEMP_VISITOR)980 RO_WITH_TEMP_OP_LIST(RO_WITH_TEMP_VISITOR)
981 #undef RO_WITH_TEMP_VISITOR
982 #undef RO_WITH_TEMP_OP_LIST
983
984 #define RO_WITH_TEMP_SIMD_VISITOR(Name, opcode) \
985 void InstructionSelector::Visit##Name(Node* node) { \
986 VisitROWithTempSimd(this, node, opcode); \
987 }
988 RO_WITH_TEMP_SIMD_OP_LIST(RO_WITH_TEMP_SIMD_VISITOR)
989 #undef RO_WITH_TEMP_SIMD_VISITOR
990 #undef RO_WITH_TEMP_SIMD_OP_LIST
991
992 #define RR_VISITOR(Name, opcode) \
993 void InstructionSelector::Visit##Name(Node* node) { \
994 VisitRR(this, node, opcode); \
995 }
996 RR_OP_LIST(RR_VISITOR)
997 #undef RR_VISITOR
998 #undef RR_OP_LIST
999
1000 #define RRO_FLOAT_VISITOR(Name, avx, sse) \
1001 void InstructionSelector::Visit##Name(Node* node) { \
1002 VisitRROFloat(this, node, avx, sse); \
1003 }
1004 RRO_FLOAT_OP_LIST(RRO_FLOAT_VISITOR)
1005 #undef RRO_FLOAT_VISITOR
1006 #undef RRO_FLOAT_OP_LIST
1007
1008 #define FLOAT_UNOP_VISITOR(Name, avx, sse) \
1009 void InstructionSelector::Visit##Name(Node* node) { \
1010 VisitFloatUnop(this, node, node->InputAt(0), avx, sse); \
1011 }
1012 FLOAT_UNOP_LIST(FLOAT_UNOP_VISITOR)
1013 #undef FLOAT_UNOP_VISITOR
1014 #undef FLOAT_UNOP_LIST
1015
1016 void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
1017
VisitWord64ReverseBytes(Node * node)1018 void InstructionSelector::VisitWord64ReverseBytes(Node* node) { UNREACHABLE(); }
1019
VisitWord32ReverseBytes(Node * node)1020 void InstructionSelector::VisitWord32ReverseBytes(Node* node) {
1021 IA32OperandGenerator g(this);
1022 Emit(kIA32Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
1023 }
1024
VisitSimd128ReverseBytes(Node * node)1025 void InstructionSelector::VisitSimd128ReverseBytes(Node* node) {
1026 UNREACHABLE();
1027 }
1028
VisitInt32Add(Node * node)1029 void InstructionSelector::VisitInt32Add(Node* node) {
1030 IA32OperandGenerator g(this);
1031
1032 // Try to match the Add to a lea pattern
1033 BaseWithIndexAndDisplacement32Matcher m(node);
1034 if (m.matches() &&
1035 (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
1036 InstructionOperand inputs[4];
1037 size_t input_count = 0;
1038 AddressingMode mode = g.GenerateMemoryOperandInputs(
1039 m.index(), m.scale(), m.base(), m.displacement(), m.displacement_mode(),
1040 inputs, &input_count);
1041
1042 DCHECK_NE(0u, input_count);
1043 DCHECK_GE(arraysize(inputs), input_count);
1044
1045 InstructionOperand outputs[1];
1046 outputs[0] = g.DefineAsRegister(node);
1047
1048 InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea;
1049 Emit(opcode, 1, outputs, input_count, inputs);
1050 return;
1051 }
1052
1053 // No lea pattern match, use add
1054 VisitBinop(this, node, kIA32Add);
1055 }
1056
VisitInt32Sub(Node * node)1057 void InstructionSelector::VisitInt32Sub(Node* node) {
1058 IA32OperandGenerator g(this);
1059 Int32BinopMatcher m(node);
1060 if (m.left().Is(0)) {
1061 Emit(kIA32Neg, g.DefineSameAsFirst(node), g.Use(m.right().node()));
1062 } else {
1063 VisitBinop(this, node, kIA32Sub);
1064 }
1065 }
1066
VisitInt32Mul(Node * node)1067 void InstructionSelector::VisitInt32Mul(Node* node) {
1068 Int32ScaleMatcher m(node, true);
1069 if (m.matches()) {
1070 Node* index = node->InputAt(0);
1071 Node* base = m.power_of_two_plus_one() ? index : nullptr;
1072 EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement);
1073 return;
1074 }
1075 IA32OperandGenerator g(this);
1076 Node* left = node->InputAt(0);
1077 Node* right = node->InputAt(1);
1078 if (g.CanBeImmediate(right)) {
1079 Emit(kIA32Imul, g.DefineAsRegister(node), g.Use(left),
1080 g.UseImmediate(right));
1081 } else {
1082 if (g.CanBeBetterLeftOperand(right)) {
1083 std::swap(left, right);
1084 }
1085 Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(left),
1086 g.Use(right));
1087 }
1088 }
1089
VisitInt32MulHigh(Node * node)1090 void InstructionSelector::VisitInt32MulHigh(Node* node) {
1091 VisitMulHigh(this, node, kIA32ImulHigh);
1092 }
1093
VisitUint32MulHigh(Node * node)1094 void InstructionSelector::VisitUint32MulHigh(Node* node) {
1095 VisitMulHigh(this, node, kIA32UmulHigh);
1096 }
1097
VisitInt32Div(Node * node)1098 void InstructionSelector::VisitInt32Div(Node* node) {
1099 VisitDiv(this, node, kIA32Idiv);
1100 }
1101
VisitUint32Div(Node * node)1102 void InstructionSelector::VisitUint32Div(Node* node) {
1103 VisitDiv(this, node, kIA32Udiv);
1104 }
1105
VisitInt32Mod(Node * node)1106 void InstructionSelector::VisitInt32Mod(Node* node) {
1107 VisitMod(this, node, kIA32Idiv);
1108 }
1109
VisitUint32Mod(Node * node)1110 void InstructionSelector::VisitUint32Mod(Node* node) {
1111 VisitMod(this, node, kIA32Udiv);
1112 }
1113
VisitRoundUint32ToFloat32(Node * node)1114 void InstructionSelector::VisitRoundUint32ToFloat32(Node* node) {
1115 IA32OperandGenerator g(this);
1116 InstructionOperand temps[] = {g.TempRegister()};
1117 Emit(kSSEUint32ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
1118 arraysize(temps), temps);
1119 }
1120
VisitFloat64Mod(Node * node)1121 void InstructionSelector::VisitFloat64Mod(Node* node) {
1122 IA32OperandGenerator g(this);
1123 InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister()};
1124 Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
1125 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
1126 arraysize(temps), temps);
1127 }
1128
VisitFloat32Max(Node * node)1129 void InstructionSelector::VisitFloat32Max(Node* node) {
1130 IA32OperandGenerator g(this);
1131 InstructionOperand temps[] = {g.TempRegister()};
1132 Emit(kSSEFloat32Max, g.DefineSameAsFirst(node),
1133 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
1134 arraysize(temps), temps);
1135 }
1136
VisitFloat64Max(Node * node)1137 void InstructionSelector::VisitFloat64Max(Node* node) {
1138 IA32OperandGenerator g(this);
1139 InstructionOperand temps[] = {g.TempRegister()};
1140 Emit(kSSEFloat64Max, g.DefineSameAsFirst(node),
1141 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
1142 arraysize(temps), temps);
1143 }
1144
VisitFloat32Min(Node * node)1145 void InstructionSelector::VisitFloat32Min(Node* node) {
1146 IA32OperandGenerator g(this);
1147 InstructionOperand temps[] = {g.TempRegister()};
1148 Emit(kSSEFloat32Min, g.DefineSameAsFirst(node),
1149 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
1150 arraysize(temps), temps);
1151 }
1152
VisitFloat64Min(Node * node)1153 void InstructionSelector::VisitFloat64Min(Node* node) {
1154 IA32OperandGenerator g(this);
1155 InstructionOperand temps[] = {g.TempRegister()};
1156 Emit(kSSEFloat64Min, g.DefineSameAsFirst(node),
1157 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
1158 arraysize(temps), temps);
1159 }
1160
VisitFloat64RoundTiesAway(Node * node)1161 void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
1162 UNREACHABLE();
1163 }
1164
VisitFloat64Ieee754Binop(Node * node,InstructionCode opcode)1165 void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
1166 InstructionCode opcode) {
1167 IA32OperandGenerator g(this);
1168 Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
1169 g.UseRegister(node->InputAt(1)))
1170 ->MarkAsCall();
1171 }
1172
VisitFloat64Ieee754Unop(Node * node,InstructionCode opcode)1173 void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
1174 InstructionCode opcode) {
1175 IA32OperandGenerator g(this);
1176 Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)))
1177 ->MarkAsCall();
1178 }
1179
EmitPrepareArguments(ZoneVector<PushParameter> * arguments,const CallDescriptor * call_descriptor,Node * node)1180 void InstructionSelector::EmitPrepareArguments(
1181 ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
1182 Node* node) {
1183 IA32OperandGenerator g(this);
1184
1185 // Prepare for C function call.
1186 if (call_descriptor->IsCFunctionCall()) {
1187 InstructionOperand temps[] = {g.TempRegister()};
1188 size_t const temp_count = arraysize(temps);
1189 Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>(
1190 call_descriptor->ParameterCount())),
1191 0, nullptr, 0, nullptr, temp_count, temps);
1192
1193 // Poke any stack arguments.
1194 for (size_t n = 0; n < arguments->size(); ++n) {
1195 PushParameter input = (*arguments)[n];
1196 if (input.node) {
1197 int const slot = static_cast<int>(n);
1198 InstructionOperand value = g.CanBeImmediate(node)
1199 ? g.UseImmediate(input.node)
1200 : g.UseRegister(input.node);
1201 Emit(kIA32Poke | MiscField::encode(slot), g.NoOutput(), value);
1202 }
1203 }
1204 } else {
1205 // Push any stack arguments.
1206 int effect_level = GetEffectLevel(node);
1207 for (PushParameter input : base::Reversed(*arguments)) {
1208 // Skip any alignment holes in pushed nodes.
1209 if (input.node == nullptr) continue;
1210 if (g.CanBeMemoryOperand(kIA32Push, node, input.node, effect_level)) {
1211 InstructionOperand outputs[1];
1212 InstructionOperand inputs[4];
1213 size_t input_count = 0;
1214 InstructionCode opcode = kIA32Push;
1215 AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1216 input.node, inputs, &input_count);
1217 opcode |= AddressingModeField::encode(mode);
1218 Emit(opcode, 0, outputs, input_count, inputs);
1219 } else {
1220 InstructionOperand value =
1221 g.CanBeImmediate(input.node)
1222 ? g.UseImmediate(input.node)
1223 : IsSupported(ATOM) ||
1224 sequence()->IsFP(GetVirtualRegister(input.node))
1225 ? g.UseRegister(input.node)
1226 : g.Use(input.node);
1227 if (input.location.GetType() == MachineType::Float32()) {
1228 Emit(kIA32PushFloat32, g.NoOutput(), value);
1229 } else if (input.location.GetType() == MachineType::Float64()) {
1230 Emit(kIA32PushFloat64, g.NoOutput(), value);
1231 } else if (input.location.GetType() == MachineType::Simd128()) {
1232 Emit(kIA32PushSimd128, g.NoOutput(), value);
1233 } else {
1234 Emit(kIA32Push, g.NoOutput(), value);
1235 }
1236 }
1237 }
1238 }
1239 }
1240
EmitPrepareResults(ZoneVector<PushParameter> * results,const CallDescriptor * call_descriptor,Node * node)1241 void InstructionSelector::EmitPrepareResults(
1242 ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
1243 Node* node) {
1244 IA32OperandGenerator g(this);
1245
1246 int reverse_slot = 0;
1247 for (PushParameter output : *results) {
1248 if (!output.location.IsCallerFrameSlot()) continue;
1249 // Skip any alignment holes in nodes.
1250 if (output.node != nullptr) {
1251 DCHECK(!call_descriptor->IsCFunctionCall());
1252 if (output.location.GetType() == MachineType::Float32()) {
1253 MarkAsFloat32(output.node);
1254 } else if (output.location.GetType() == MachineType::Float64()) {
1255 MarkAsFloat64(output.node);
1256 }
1257 Emit(kIA32Peek, g.DefineAsRegister(output.node),
1258 g.UseImmediate(reverse_slot));
1259 }
1260 reverse_slot += output.location.GetSizeInPointers();
1261 }
1262 }
1263
IsTailCallAddressImmediate()1264 bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
1265
GetTempsCountForTailCallFromJSFunction()1266 int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 0; }
1267
1268 namespace {
1269
VisitCompareWithMemoryOperand(InstructionSelector * selector,InstructionCode opcode,Node * left,InstructionOperand right,FlagsContinuation * cont)1270 void VisitCompareWithMemoryOperand(InstructionSelector* selector,
1271 InstructionCode opcode, Node* left,
1272 InstructionOperand right,
1273 FlagsContinuation* cont) {
1274 DCHECK_EQ(IrOpcode::kLoad, left->opcode());
1275 IA32OperandGenerator g(selector);
1276 size_t input_count = 0;
1277 InstructionOperand inputs[4];
1278 AddressingMode addressing_mode =
1279 g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
1280 opcode |= AddressingModeField::encode(addressing_mode);
1281 inputs[input_count++] = right;
1282
1283 selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
1284 }
1285
1286 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,InstructionOperand left,InstructionOperand right,FlagsContinuation * cont)1287 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1288 InstructionOperand left, InstructionOperand right,
1289 FlagsContinuation* cont) {
1290 selector->EmitWithContinuation(opcode, left, right, cont);
1291 }
1292
1293 // Shared routine for multiple compare operations.
VisitCompare(InstructionSelector * selector,InstructionCode opcode,Node * left,Node * right,FlagsContinuation * cont,bool commutative)1294 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1295 Node* left, Node* right, FlagsContinuation* cont,
1296 bool commutative) {
1297 IA32OperandGenerator g(selector);
1298 if (commutative && g.CanBeBetterLeftOperand(right)) {
1299 std::swap(left, right);
1300 }
1301 VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
1302 }
1303
MachineTypeForNarrow(Node * node,Node * hint_node)1304 MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
1305 if (hint_node->opcode() == IrOpcode::kLoad) {
1306 MachineType hint = LoadRepresentationOf(hint_node->op());
1307 if (node->opcode() == IrOpcode::kInt32Constant ||
1308 node->opcode() == IrOpcode::kInt64Constant) {
1309 int64_t constant = node->opcode() == IrOpcode::kInt32Constant
1310 ? OpParameter<int32_t>(node->op())
1311 : OpParameter<int64_t>(node->op());
1312 if (hint == MachineType::Int8()) {
1313 if (constant >= std::numeric_limits<int8_t>::min() &&
1314 constant <= std::numeric_limits<int8_t>::max()) {
1315 return hint;
1316 }
1317 } else if (hint == MachineType::Uint8()) {
1318 if (constant >= std::numeric_limits<uint8_t>::min() &&
1319 constant <= std::numeric_limits<uint8_t>::max()) {
1320 return hint;
1321 }
1322 } else if (hint == MachineType::Int16()) {
1323 if (constant >= std::numeric_limits<int16_t>::min() &&
1324 constant <= std::numeric_limits<int16_t>::max()) {
1325 return hint;
1326 }
1327 } else if (hint == MachineType::Uint16()) {
1328 if (constant >= std::numeric_limits<uint16_t>::min() &&
1329 constant <= std::numeric_limits<uint16_t>::max()) {
1330 return hint;
1331 }
1332 } else if (hint == MachineType::Int32()) {
1333 return hint;
1334 } else if (hint == MachineType::Uint32()) {
1335 if (constant >= 0) return hint;
1336 }
1337 }
1338 }
1339 return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op())
1340 : MachineType::None();
1341 }
1342
1343 // Tries to match the size of the given opcode to that of the operands, if
1344 // possible.
TryNarrowOpcodeSize(InstructionCode opcode,Node * left,Node * right,FlagsContinuation * cont)1345 InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
1346 Node* right, FlagsContinuation* cont) {
1347 // TODO(epertoso): we can probably get some size information out of phi nodes.
1348 // If the load representations don't match, both operands will be
1349 // zero/sign-extended to 32bit.
1350 MachineType left_type = MachineTypeForNarrow(left, right);
1351 MachineType right_type = MachineTypeForNarrow(right, left);
1352 if (left_type == right_type) {
1353 switch (left_type.representation()) {
1354 case MachineRepresentation::kBit:
1355 case MachineRepresentation::kWord8: {
1356 if (opcode == kIA32Test) return kIA32Test8;
1357 if (opcode == kIA32Cmp) {
1358 if (left_type.semantic() == MachineSemantic::kUint32) {
1359 cont->OverwriteUnsignedIfSigned();
1360 } else {
1361 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1362 }
1363 return kIA32Cmp8;
1364 }
1365 break;
1366 }
1367 case MachineRepresentation::kWord16:
1368 if (opcode == kIA32Test) return kIA32Test16;
1369 if (opcode == kIA32Cmp) {
1370 if (left_type.semantic() == MachineSemantic::kUint32) {
1371 cont->OverwriteUnsignedIfSigned();
1372 } else {
1373 CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1374 }
1375 return kIA32Cmp16;
1376 }
1377 break;
1378 default:
1379 break;
1380 }
1381 }
1382 return opcode;
1383 }
1384
1385 // Shared routine for multiple float32 compare operations (inputs commuted).
VisitFloat32Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)1386 void VisitFloat32Compare(InstructionSelector* selector, Node* node,
1387 FlagsContinuation* cont) {
1388 Node* const left = node->InputAt(0);
1389 Node* const right = node->InputAt(1);
1390 VisitCompare(selector, kSSEFloat32Cmp, right, left, cont, false);
1391 }
1392
1393 // Shared routine for multiple float64 compare operations (inputs commuted).
VisitFloat64Compare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)1394 void VisitFloat64Compare(InstructionSelector* selector, Node* node,
1395 FlagsContinuation* cont) {
1396 Node* const left = node->InputAt(0);
1397 Node* const right = node->InputAt(1);
1398 VisitCompare(selector, kSSEFloat64Cmp, right, left, cont, false);
1399 }
1400
1401 // Shared routine for multiple word compare operations.
VisitWordCompare(InstructionSelector * selector,Node * node,InstructionCode opcode,FlagsContinuation * cont)1402 void VisitWordCompare(InstructionSelector* selector, Node* node,
1403 InstructionCode opcode, FlagsContinuation* cont) {
1404 IA32OperandGenerator g(selector);
1405 Node* left = node->InputAt(0);
1406 Node* right = node->InputAt(1);
1407
1408 InstructionCode narrowed_opcode =
1409 TryNarrowOpcodeSize(opcode, left, right, cont);
1410
1411 int effect_level = selector->GetEffectLevel(node);
1412 if (cont->IsBranch()) {
1413 effect_level = selector->GetEffectLevel(
1414 cont->true_block()->PredecessorAt(0)->control_input());
1415 }
1416
1417 // If one of the two inputs is an immediate, make sure it's on the right, or
1418 // if one of the two inputs is a memory operand, make sure it's on the left.
1419 if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
1420 (g.CanBeMemoryOperand(narrowed_opcode, node, right, effect_level) &&
1421 !g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level))) {
1422 if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1423 std::swap(left, right);
1424 }
1425
1426 // Match immediates on right side of comparison.
1427 if (g.CanBeImmediate(right)) {
1428 if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
1429 return VisitCompareWithMemoryOperand(selector, narrowed_opcode, left,
1430 g.UseImmediate(right), cont);
1431 }
1432 return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
1433 cont);
1434 }
1435
1436 // Match memory operands on left side of comparison.
1437 if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
1438 bool needs_byte_register =
1439 narrowed_opcode == kIA32Test8 || narrowed_opcode == kIA32Cmp8;
1440 return VisitCompareWithMemoryOperand(
1441 selector, narrowed_opcode, left,
1442 needs_byte_register ? g.UseByteRegister(right) : g.UseRegister(right),
1443 cont);
1444 }
1445
1446 return VisitCompare(selector, opcode, left, right, cont,
1447 node->op()->HasProperty(Operator::kCommutative));
1448 }
1449
VisitWordCompare(InstructionSelector * selector,Node * node,FlagsContinuation * cont)1450 void VisitWordCompare(InstructionSelector* selector, Node* node,
1451 FlagsContinuation* cont) {
1452 VisitWordCompare(selector, node, kIA32Cmp, cont);
1453 }
1454
VisitAtomicExchange(InstructionSelector * selector,Node * node,ArchOpcode opcode,MachineRepresentation rep)1455 void VisitAtomicExchange(InstructionSelector* selector, Node* node,
1456 ArchOpcode opcode, MachineRepresentation rep) {
1457 IA32OperandGenerator g(selector);
1458 Node* base = node->InputAt(0);
1459 Node* index = node->InputAt(1);
1460 Node* value = node->InputAt(2);
1461
1462 AddressingMode addressing_mode;
1463 InstructionOperand value_operand = (rep == MachineRepresentation::kWord8)
1464 ? g.UseFixed(value, edx)
1465 : g.UseUniqueRegister(value);
1466 InstructionOperand inputs[] = {
1467 value_operand, g.UseUniqueRegister(base),
1468 g.GetEffectiveIndexOperand(index, &addressing_mode)};
1469 InstructionOperand outputs[] = {
1470 (rep == MachineRepresentation::kWord8)
1471 // Using DefineSameAsFirst requires the register to be unallocated.
1472 ? g.DefineAsFixed(node, edx)
1473 : g.DefineSameAsFirst(node)};
1474 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1475 selector->Emit(code, 1, outputs, arraysize(inputs), inputs);
1476 }
1477
VisitAtomicBinOp(InstructionSelector * selector,Node * node,ArchOpcode opcode,MachineRepresentation rep)1478 void VisitAtomicBinOp(InstructionSelector* selector, Node* node,
1479 ArchOpcode opcode, MachineRepresentation rep) {
1480 AddressingMode addressing_mode;
1481 IA32OperandGenerator g(selector);
1482 Node* base = node->InputAt(0);
1483 Node* index = node->InputAt(1);
1484 Node* value = node->InputAt(2);
1485 InstructionOperand inputs[] = {
1486 g.UseUniqueRegister(value), g.UseUniqueRegister(base),
1487 g.GetEffectiveIndexOperand(index, &addressing_mode)};
1488 InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)};
1489 InstructionOperand temp[] = {(rep == MachineRepresentation::kWord8)
1490 ? g.UseByteRegister(node)
1491 : g.TempRegister()};
1492 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1493 selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1494 arraysize(temp), temp);
1495 }
1496
VisitPairAtomicBinOp(InstructionSelector * selector,Node * node,ArchOpcode opcode)1497 void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node,
1498 ArchOpcode opcode) {
1499 IA32OperandGenerator g(selector);
1500 Node* base = node->InputAt(0);
1501 Node* index = node->InputAt(1);
1502 Node* value = node->InputAt(2);
1503 // For Word64 operations, the value input is split into the a high node,
1504 // and a low node in the int64-lowering phase.
1505 Node* value_high = node->InputAt(3);
1506
1507 // Wasm lives in 32-bit address space, so we do not need to worry about
1508 // base/index lowering. This will need to be fixed for Wasm64.
1509 AddressingMode addressing_mode;
1510 InstructionOperand inputs[] = {
1511 g.UseUniqueRegisterOrSlotOrConstant(value), g.UseFixed(value_high, ecx),
1512 g.UseUniqueRegister(base),
1513 g.GetEffectiveIndexOperand(index, &addressing_mode)};
1514 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1515 Node* projection0 = NodeProperties::FindProjection(node, 0);
1516 Node* projection1 = NodeProperties::FindProjection(node, 1);
1517 InstructionOperand outputs[2];
1518 size_t output_count = 0;
1519 InstructionOperand temps[2];
1520 size_t temp_count = 0;
1521 if (projection0) {
1522 outputs[output_count++] = g.DefineAsFixed(projection0, eax);
1523 } else {
1524 temps[temp_count++] = g.TempRegister(eax);
1525 }
1526 if (projection1) {
1527 outputs[output_count++] = g.DefineAsFixed(projection1, edx);
1528 } else {
1529 temps[temp_count++] = g.TempRegister(edx);
1530 }
1531 selector->Emit(code, output_count, outputs, arraysize(inputs), inputs,
1532 temp_count, temps);
1533 }
1534
1535 } // namespace
1536
1537 // Shared routine for word comparison with zero.
VisitWordCompareZero(Node * user,Node * value,FlagsContinuation * cont)1538 void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
1539 FlagsContinuation* cont) {
1540 // Try to combine with comparisons against 0 by simply inverting the branch.
1541 while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
1542 Int32BinopMatcher m(value);
1543 if (!m.right().Is(0)) break;
1544
1545 user = value;
1546 value = m.left().node();
1547 cont->Negate();
1548 }
1549
1550 if (CanCover(user, value)) {
1551 switch (value->opcode()) {
1552 case IrOpcode::kWord32Equal:
1553 cont->OverwriteAndNegateIfEqual(kEqual);
1554 return VisitWordCompare(this, value, cont);
1555 case IrOpcode::kInt32LessThan:
1556 cont->OverwriteAndNegateIfEqual(kSignedLessThan);
1557 return VisitWordCompare(this, value, cont);
1558 case IrOpcode::kInt32LessThanOrEqual:
1559 cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
1560 return VisitWordCompare(this, value, cont);
1561 case IrOpcode::kUint32LessThan:
1562 cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
1563 return VisitWordCompare(this, value, cont);
1564 case IrOpcode::kUint32LessThanOrEqual:
1565 cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
1566 return VisitWordCompare(this, value, cont);
1567 case IrOpcode::kFloat32Equal:
1568 cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1569 return VisitFloat32Compare(this, value, cont);
1570 case IrOpcode::kFloat32LessThan:
1571 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1572 return VisitFloat32Compare(this, value, cont);
1573 case IrOpcode::kFloat32LessThanOrEqual:
1574 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1575 return VisitFloat32Compare(this, value, cont);
1576 case IrOpcode::kFloat64Equal:
1577 cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1578 return VisitFloat64Compare(this, value, cont);
1579 case IrOpcode::kFloat64LessThan:
1580 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1581 return VisitFloat64Compare(this, value, cont);
1582 case IrOpcode::kFloat64LessThanOrEqual:
1583 cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1584 return VisitFloat64Compare(this, value, cont);
1585 case IrOpcode::kProjection:
1586 // Check if this is the overflow output projection of an
1587 // <Operation>WithOverflow node.
1588 if (ProjectionIndexOf(value->op()) == 1u) {
1589 // We cannot combine the <Operation>WithOverflow with this branch
1590 // unless the 0th projection (the use of the actual value of the
1591 // <Operation> is either nullptr, which means there's no use of the
1592 // actual value, or was already defined, which means it is scheduled
1593 // *AFTER* this branch).
1594 Node* const node = value->InputAt(0);
1595 Node* const result = NodeProperties::FindProjection(node, 0);
1596 if (result == nullptr || IsDefined(result)) {
1597 switch (node->opcode()) {
1598 case IrOpcode::kInt32AddWithOverflow:
1599 cont->OverwriteAndNegateIfEqual(kOverflow);
1600 return VisitBinop(this, node, kIA32Add, cont);
1601 case IrOpcode::kInt32SubWithOverflow:
1602 cont->OverwriteAndNegateIfEqual(kOverflow);
1603 return VisitBinop(this, node, kIA32Sub, cont);
1604 case IrOpcode::kInt32MulWithOverflow:
1605 cont->OverwriteAndNegateIfEqual(kOverflow);
1606 return VisitBinop(this, node, kIA32Imul, cont);
1607 default:
1608 break;
1609 }
1610 }
1611 }
1612 break;
1613 case IrOpcode::kInt32Sub:
1614 return VisitWordCompare(this, value, cont);
1615 case IrOpcode::kWord32And:
1616 return VisitWordCompare(this, value, kIA32Test, cont);
1617 case IrOpcode::kStackPointerGreaterThan:
1618 cont->OverwriteAndNegateIfEqual(kStackPointerGreaterThanCondition);
1619 return VisitStackPointerGreaterThan(value, cont);
1620 default:
1621 break;
1622 }
1623 }
1624
1625 // Continuation could not be combined with a compare, emit compare against 0.
1626 IA32OperandGenerator g(this);
1627 VisitCompare(this, kIA32Cmp, g.Use(value), g.TempImmediate(0), cont);
1628 }
1629
VisitSwitch(Node * node,const SwitchInfo & sw)1630 void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
1631 IA32OperandGenerator g(this);
1632 InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
1633
1634 // Emit either ArchTableSwitch or ArchBinarySearchSwitch.
1635 if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
1636 static const size_t kMaxTableSwitchValueRange = 2 << 16;
1637 size_t table_space_cost = 4 + sw.value_range();
1638 size_t table_time_cost = 3;
1639 size_t lookup_space_cost = 3 + 2 * sw.case_count();
1640 size_t lookup_time_cost = sw.case_count();
1641 if (sw.case_count() > 4 &&
1642 table_space_cost + 3 * table_time_cost <=
1643 lookup_space_cost + 3 * lookup_time_cost &&
1644 sw.min_value() > std::numeric_limits<int32_t>::min() &&
1645 sw.value_range() <= kMaxTableSwitchValueRange) {
1646 InstructionOperand index_operand = value_operand;
1647 if (sw.min_value()) {
1648 index_operand = g.TempRegister();
1649 Emit(kIA32Lea | AddressingModeField::encode(kMode_MRI), index_operand,
1650 value_operand, g.TempImmediate(-sw.min_value()));
1651 }
1652 // Generate a table lookup.
1653 return EmitTableSwitch(sw, index_operand);
1654 }
1655 }
1656
1657 // Generate a tree of conditional jumps.
1658 return EmitBinarySearchSwitch(sw, value_operand);
1659 }
1660
VisitWord32Equal(Node * const node)1661 void InstructionSelector::VisitWord32Equal(Node* const node) {
1662 FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
1663 Int32BinopMatcher m(node);
1664 if (m.right().Is(0)) {
1665 return VisitWordCompareZero(m.node(), m.left().node(), &cont);
1666 }
1667 VisitWordCompare(this, node, &cont);
1668 }
1669
VisitInt32LessThan(Node * node)1670 void InstructionSelector::VisitInt32LessThan(Node* node) {
1671 FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
1672 VisitWordCompare(this, node, &cont);
1673 }
1674
VisitInt32LessThanOrEqual(Node * node)1675 void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
1676 FlagsContinuation cont =
1677 FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
1678 VisitWordCompare(this, node, &cont);
1679 }
1680
VisitUint32LessThan(Node * node)1681 void InstructionSelector::VisitUint32LessThan(Node* node) {
1682 FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
1683 VisitWordCompare(this, node, &cont);
1684 }
1685
VisitUint32LessThanOrEqual(Node * node)1686 void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
1687 FlagsContinuation cont =
1688 FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
1689 VisitWordCompare(this, node, &cont);
1690 }
1691
VisitInt32AddWithOverflow(Node * node)1692 void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
1693 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1694 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1695 return VisitBinop(this, node, kIA32Add, &cont);
1696 }
1697 FlagsContinuation cont;
1698 VisitBinop(this, node, kIA32Add, &cont);
1699 }
1700
VisitInt32SubWithOverflow(Node * node)1701 void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
1702 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1703 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1704 return VisitBinop(this, node, kIA32Sub, &cont);
1705 }
1706 FlagsContinuation cont;
1707 VisitBinop(this, node, kIA32Sub, &cont);
1708 }
1709
VisitInt32MulWithOverflow(Node * node)1710 void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
1711 if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1712 FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1713 return VisitBinop(this, node, kIA32Imul, &cont);
1714 }
1715 FlagsContinuation cont;
1716 VisitBinop(this, node, kIA32Imul, &cont);
1717 }
1718
VisitFloat32Equal(Node * node)1719 void InstructionSelector::VisitFloat32Equal(Node* node) {
1720 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
1721 VisitFloat32Compare(this, node, &cont);
1722 }
1723
VisitFloat32LessThan(Node * node)1724 void InstructionSelector::VisitFloat32LessThan(Node* node) {
1725 FlagsContinuation cont =
1726 FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
1727 VisitFloat32Compare(this, node, &cont);
1728 }
1729
VisitFloat32LessThanOrEqual(Node * node)1730 void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
1731 FlagsContinuation cont =
1732 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
1733 VisitFloat32Compare(this, node, &cont);
1734 }
1735
VisitFloat64Equal(Node * node)1736 void InstructionSelector::VisitFloat64Equal(Node* node) {
1737 FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
1738 VisitFloat64Compare(this, node, &cont);
1739 }
1740
VisitFloat64LessThan(Node * node)1741 void InstructionSelector::VisitFloat64LessThan(Node* node) {
1742 FlagsContinuation cont =
1743 FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
1744 VisitFloat64Compare(this, node, &cont);
1745 }
1746
VisitFloat64LessThanOrEqual(Node * node)1747 void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
1748 FlagsContinuation cont =
1749 FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
1750 VisitFloat64Compare(this, node, &cont);
1751 }
1752
VisitFloat64InsertLowWord32(Node * node)1753 void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
1754 IA32OperandGenerator g(this);
1755 Node* left = node->InputAt(0);
1756 Node* right = node->InputAt(1);
1757 Float64Matcher mleft(left);
1758 if (mleft.HasValue() && (bit_cast<uint64_t>(mleft.Value()) >> 32) == 0u) {
1759 Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
1760 return;
1761 }
1762 Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
1763 g.UseRegister(left), g.Use(right));
1764 }
1765
VisitFloat64InsertHighWord32(Node * node)1766 void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
1767 IA32OperandGenerator g(this);
1768 Node* left = node->InputAt(0);
1769 Node* right = node->InputAt(1);
1770 Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
1771 g.UseRegister(left), g.Use(right));
1772 }
1773
VisitFloat64SilenceNaN(Node * node)1774 void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
1775 IA32OperandGenerator g(this);
1776 Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
1777 g.UseRegister(node->InputAt(0)));
1778 }
1779
VisitMemoryBarrier(Node * node)1780 void InstructionSelector::VisitMemoryBarrier(Node* node) {
1781 IA32OperandGenerator g(this);
1782 Emit(kIA32MFence, g.NoOutput());
1783 }
1784
VisitWord32AtomicLoad(Node * node)1785 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
1786 LoadRepresentation load_rep = LoadRepresentationOf(node->op());
1787 DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
1788 load_rep.representation() == MachineRepresentation::kWord16 ||
1789 load_rep.representation() == MachineRepresentation::kWord32);
1790 USE(load_rep);
1791 VisitLoad(node);
1792 }
1793
VisitWord32AtomicStore(Node * node)1794 void InstructionSelector::VisitWord32AtomicStore(Node* node) {
1795 IA32OperandGenerator g(this);
1796 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
1797 ArchOpcode opcode = kArchNop;
1798 switch (rep) {
1799 case MachineRepresentation::kWord8:
1800 opcode = kWord32AtomicExchangeInt8;
1801 break;
1802 case MachineRepresentation::kWord16:
1803 opcode = kWord32AtomicExchangeInt16;
1804 break;
1805 case MachineRepresentation::kWord32:
1806 opcode = kWord32AtomicExchangeWord32;
1807 break;
1808 default:
1809 UNREACHABLE();
1810 }
1811 VisitAtomicExchange(this, node, opcode, rep);
1812 }
1813
VisitWord32AtomicExchange(Node * node)1814 void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
1815 IA32OperandGenerator g(this);
1816 MachineType type = AtomicOpType(node->op());
1817 ArchOpcode opcode = kArchNop;
1818 if (type == MachineType::Int8()) {
1819 opcode = kWord32AtomicExchangeInt8;
1820 } else if (type == MachineType::Uint8()) {
1821 opcode = kWord32AtomicExchangeUint8;
1822 } else if (type == MachineType::Int16()) {
1823 opcode = kWord32AtomicExchangeInt16;
1824 } else if (type == MachineType::Uint16()) {
1825 opcode = kWord32AtomicExchangeUint16;
1826 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1827 opcode = kWord32AtomicExchangeWord32;
1828 } else {
1829 UNREACHABLE();
1830 return;
1831 }
1832 VisitAtomicExchange(this, node, opcode, type.representation());
1833 }
1834
VisitWord32AtomicCompareExchange(Node * node)1835 void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
1836 IA32OperandGenerator g(this);
1837 Node* base = node->InputAt(0);
1838 Node* index = node->InputAt(1);
1839 Node* old_value = node->InputAt(2);
1840 Node* new_value = node->InputAt(3);
1841
1842 MachineType type = AtomicOpType(node->op());
1843 ArchOpcode opcode = kArchNop;
1844 if (type == MachineType::Int8()) {
1845 opcode = kWord32AtomicCompareExchangeInt8;
1846 } else if (type == MachineType::Uint8()) {
1847 opcode = kWord32AtomicCompareExchangeUint8;
1848 } else if (type == MachineType::Int16()) {
1849 opcode = kWord32AtomicCompareExchangeInt16;
1850 } else if (type == MachineType::Uint16()) {
1851 opcode = kWord32AtomicCompareExchangeUint16;
1852 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1853 opcode = kWord32AtomicCompareExchangeWord32;
1854 } else {
1855 UNREACHABLE();
1856 return;
1857 }
1858 AddressingMode addressing_mode;
1859 InstructionOperand new_val_operand =
1860 (type.representation() == MachineRepresentation::kWord8)
1861 ? g.UseByteRegister(new_value)
1862 : g.UseUniqueRegister(new_value);
1863 InstructionOperand inputs[] = {
1864 g.UseFixed(old_value, eax), new_val_operand, g.UseUniqueRegister(base),
1865 g.GetEffectiveIndexOperand(index, &addressing_mode)};
1866 InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)};
1867 InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1868 Emit(code, 1, outputs, arraysize(inputs), inputs);
1869 }
1870
VisitWord32AtomicBinaryOperation(Node * node,ArchOpcode int8_op,ArchOpcode uint8_op,ArchOpcode int16_op,ArchOpcode uint16_op,ArchOpcode word32_op)1871 void InstructionSelector::VisitWord32AtomicBinaryOperation(
1872 Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
1873 ArchOpcode uint16_op, ArchOpcode word32_op) {
1874 MachineType type = AtomicOpType(node->op());
1875 ArchOpcode opcode = kArchNop;
1876 if (type == MachineType::Int8()) {
1877 opcode = int8_op;
1878 } else if (type == MachineType::Uint8()) {
1879 opcode = uint8_op;
1880 } else if (type == MachineType::Int16()) {
1881 opcode = int16_op;
1882 } else if (type == MachineType::Uint16()) {
1883 opcode = uint16_op;
1884 } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1885 opcode = word32_op;
1886 } else {
1887 UNREACHABLE();
1888 return;
1889 }
1890 VisitAtomicBinOp(this, node, opcode, type.representation());
1891 }
1892
1893 #define VISIT_ATOMIC_BINOP(op) \
1894 void InstructionSelector::VisitWord32Atomic##op(Node* node) { \
1895 VisitWord32AtomicBinaryOperation( \
1896 node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \
1897 kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \
1898 kWord32Atomic##op##Word32); \
1899 }
1900 VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)1901 VISIT_ATOMIC_BINOP(Sub)
1902 VISIT_ATOMIC_BINOP(And)
1903 VISIT_ATOMIC_BINOP(Or)
1904 VISIT_ATOMIC_BINOP(Xor)
1905 #undef VISIT_ATOMIC_BINOP
1906
1907 void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
1908 IA32OperandGenerator g(this);
1909 AddressingMode mode;
1910 Node* base = node->InputAt(0);
1911 Node* index = node->InputAt(1);
1912 Node* projection0 = NodeProperties::FindProjection(node, 0);
1913 Node* projection1 = NodeProperties::FindProjection(node, 1);
1914 if (projection0 && projection1) {
1915 InstructionOperand inputs[] = {g.UseUniqueRegister(base),
1916 g.GetEffectiveIndexOperand(index, &mode)};
1917 InstructionCode code =
1918 kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode);
1919 InstructionOperand temps[] = {g.TempDoubleRegister()};
1920 InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
1921 g.DefineAsRegister(projection1)};
1922 Emit(code, 2, outputs, 2, inputs, 1, temps);
1923 } else if (projection0 || projection1) {
1924 // Only one word is needed, so it's enough to load just that.
1925 ArchOpcode opcode = kIA32Movl;
1926
1927 InstructionOperand outputs[] = {
1928 g.DefineAsRegister(projection0 ? projection0 : projection1)};
1929 InstructionOperand inputs[3];
1930 size_t input_count = 0;
1931 // TODO(ahaas): Introduce an enum for {scale} instead of an integer.
1932 // {scale = 0} means *1 in the generated code.
1933 int scale = 0;
1934 AddressingMode mode = g.GenerateMemoryOperandInputs(
1935 index, scale, base, projection0 ? 0 : 4, kPositiveDisplacement, inputs,
1936 &input_count);
1937 InstructionCode code = opcode | AddressingModeField::encode(mode);
1938 Emit(code, 1, outputs, input_count, inputs);
1939 }
1940 }
1941
VisitWord32AtomicPairStore(Node * node)1942 void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
1943 IA32OperandGenerator g(this);
1944 Node* base = node->InputAt(0);
1945 Node* index = node->InputAt(1);
1946 Node* value = node->InputAt(2);
1947 Node* value_high = node->InputAt(3);
1948
1949 AddressingMode addressing_mode;
1950 InstructionOperand inputs[] = {
1951 g.UseUniqueRegisterOrSlotOrConstant(value), g.UseFixed(value_high, ecx),
1952 g.UseUniqueRegister(base),
1953 g.GetEffectiveIndexOperand(index, &addressing_mode)};
1954 // Allocating temp registers here as stores are performed using an atomic
1955 // exchange, the output of which is stored in edx:eax, which should be saved
1956 // and restored at the end of the instruction.
1957 InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
1958 const int num_temps = arraysize(temps);
1959 InstructionCode code =
1960 kIA32Word32AtomicPairStore | AddressingModeField::encode(addressing_mode);
1961 Emit(code, 0, nullptr, arraysize(inputs), inputs, num_temps, temps);
1962 }
1963
VisitWord32AtomicPairAdd(Node * node)1964 void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
1965 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAdd);
1966 }
1967
VisitWord32AtomicPairSub(Node * node)1968 void InstructionSelector::VisitWord32AtomicPairSub(Node* node) {
1969 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairSub);
1970 }
1971
VisitWord32AtomicPairAnd(Node * node)1972 void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) {
1973 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAnd);
1974 }
1975
VisitWord32AtomicPairOr(Node * node)1976 void InstructionSelector::VisitWord32AtomicPairOr(Node* node) {
1977 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairOr);
1978 }
1979
VisitWord32AtomicPairXor(Node * node)1980 void InstructionSelector::VisitWord32AtomicPairXor(Node* node) {
1981 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairXor);
1982 }
1983
VisitWord32AtomicPairExchange(Node * node)1984 void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
1985 VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairExchange);
1986 }
1987
VisitWord32AtomicPairCompareExchange(Node * node)1988 void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
1989 IA32OperandGenerator g(this);
1990 Node* index = node->InputAt(1);
1991 AddressingMode addressing_mode;
1992
1993 InstructionOperand inputs[] = {
1994 // High, Low values of old value
1995 g.UseFixed(node->InputAt(2), eax), g.UseFixed(node->InputAt(3), edx),
1996 // High, Low values of new value
1997 g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(4)),
1998 g.UseFixed(node->InputAt(5), ecx),
1999 // InputAt(0) => base
2000 g.UseUniqueRegister(node->InputAt(0)),
2001 g.GetEffectiveIndexOperand(index, &addressing_mode)};
2002 Node* projection0 = NodeProperties::FindProjection(node, 0);
2003 Node* projection1 = NodeProperties::FindProjection(node, 1);
2004 InstructionCode code = kIA32Word32AtomicPairCompareExchange |
2005 AddressingModeField::encode(addressing_mode);
2006
2007 InstructionOperand outputs[2];
2008 size_t output_count = 0;
2009 InstructionOperand temps[2];
2010 size_t temp_count = 0;
2011 if (projection0) {
2012 outputs[output_count++] = g.DefineAsFixed(projection0, eax);
2013 } else {
2014 temps[temp_count++] = g.TempRegister(eax);
2015 }
2016 if (projection1) {
2017 outputs[output_count++] = g.DefineAsFixed(projection1, edx);
2018 } else {
2019 temps[temp_count++] = g.TempRegister(edx);
2020 }
2021 Emit(code, output_count, outputs, arraysize(inputs), inputs, temp_count,
2022 temps);
2023 }
2024
2025 #define SIMD_INT_TYPES(V) \
2026 V(I32x4) \
2027 V(I16x8) \
2028 V(I8x16)
2029
2030 #define SIMD_BINOP_LIST(V) \
2031 V(F32x4Add) \
2032 V(F32x4AddHoriz) \
2033 V(F32x4Sub) \
2034 V(F32x4Mul) \
2035 V(F32x4Div) \
2036 V(F32x4Min) \
2037 V(F32x4Max) \
2038 V(F32x4Eq) \
2039 V(F32x4Ne) \
2040 V(F32x4Lt) \
2041 V(F32x4Le) \
2042 V(I32x4Add) \
2043 V(I32x4AddHoriz) \
2044 V(I32x4Sub) \
2045 V(I32x4Mul) \
2046 V(I32x4MinS) \
2047 V(I32x4MaxS) \
2048 V(I32x4Eq) \
2049 V(I32x4Ne) \
2050 V(I32x4GtS) \
2051 V(I32x4GeS) \
2052 V(I32x4MinU) \
2053 V(I32x4MaxU) \
2054 V(I32x4GtU) \
2055 V(I32x4GeU) \
2056 V(I16x8SConvertI32x4) \
2057 V(I16x8Add) \
2058 V(I16x8AddSaturateS) \
2059 V(I16x8AddHoriz) \
2060 V(I16x8Sub) \
2061 V(I16x8SubSaturateS) \
2062 V(I16x8Mul) \
2063 V(I16x8MinS) \
2064 V(I16x8MaxS) \
2065 V(I16x8Eq) \
2066 V(I16x8Ne) \
2067 V(I16x8GtS) \
2068 V(I16x8GeS) \
2069 V(I16x8AddSaturateU) \
2070 V(I16x8SubSaturateU) \
2071 V(I16x8MinU) \
2072 V(I16x8MaxU) \
2073 V(I16x8GtU) \
2074 V(I16x8GeU) \
2075 V(I8x16SConvertI16x8) \
2076 V(I8x16Add) \
2077 V(I8x16AddSaturateS) \
2078 V(I8x16Sub) \
2079 V(I8x16SubSaturateS) \
2080 V(I8x16MinS) \
2081 V(I8x16MaxS) \
2082 V(I8x16Eq) \
2083 V(I8x16Ne) \
2084 V(I8x16GtS) \
2085 V(I8x16GeS) \
2086 V(I8x16AddSaturateU) \
2087 V(I8x16SubSaturateU) \
2088 V(I8x16MinU) \
2089 V(I8x16MaxU) \
2090 V(I8x16GtU) \
2091 V(I8x16GeU) \
2092 V(S128And) \
2093 V(S128Or) \
2094 V(S128Xor)
2095
2096 #define SIMD_BINOP_UNIFIED_SSE_AVX_LIST(V) \
2097 V(I64x2Add) \
2098 V(I64x2Sub) \
2099 V(I16x8RoundingAverageU) \
2100 V(I8x16RoundingAverageU)
2101
2102 #define SIMD_UNOP_LIST(V) \
2103 V(F32x4SConvertI32x4) \
2104 V(F32x4RecipApprox) \
2105 V(F32x4RecipSqrtApprox) \
2106 V(I32x4SConvertI16x8Low) \
2107 V(I32x4SConvertI16x8High) \
2108 V(I32x4Neg) \
2109 V(I32x4UConvertI16x8Low) \
2110 V(I32x4UConvertI16x8High) \
2111 V(I32x4Abs) \
2112 V(I16x8SConvertI8x16Low) \
2113 V(I16x8SConvertI8x16High) \
2114 V(I16x8Neg) \
2115 V(I16x8UConvertI8x16Low) \
2116 V(I16x8UConvertI8x16High) \
2117 V(I16x8Abs) \
2118 V(I8x16Neg) \
2119 V(I8x16Abs)
2120
2121 #define SIMD_UNOP_PREFIX_LIST(V) \
2122 V(F32x4Abs) \
2123 V(F32x4Neg) \
2124 V(F32x4Sqrt) \
2125 V(S128Not)
2126
2127 #define SIMD_ANYTRUE_LIST(V) \
2128 V(S1x4AnyTrue) \
2129 V(S1x8AnyTrue) \
2130 V(S1x16AnyTrue)
2131
2132 #define SIMD_ALLTRUE_LIST(V) \
2133 V(S1x4AllTrue) \
2134 V(S1x8AllTrue) \
2135 V(S1x16AllTrue)
2136
2137 #define SIMD_SHIFT_OPCODES_UNIFED_SSE_AVX(V) \
2138 V(I64x2Shl) \
2139 V(I64x2ShrU) \
2140 V(I32x4Shl) \
2141 V(I32x4ShrS) \
2142 V(I32x4ShrU) \
2143 V(I16x8Shl) \
2144 V(I16x8ShrS) \
2145 V(I16x8ShrU)
2146
VisitF64x2Min(Node * node)2147 void InstructionSelector::VisitF64x2Min(Node* node) {
2148 IA32OperandGenerator g(this);
2149 InstructionOperand temps[] = {g.TempSimd128Register()};
2150 InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
2151 InstructionOperand operand1 = g.UseUnique(node->InputAt(1));
2152
2153 if (IsSupported(AVX)) {
2154 Emit(kIA32F64x2Min, g.DefineAsRegister(node), operand0, operand1,
2155 arraysize(temps), temps);
2156 } else {
2157 Emit(kIA32F64x2Min, g.DefineSameAsFirst(node), operand0, operand1,
2158 arraysize(temps), temps);
2159 }
2160 }
2161
VisitF64x2Max(Node * node)2162 void InstructionSelector::VisitF64x2Max(Node* node) {
2163 IA32OperandGenerator g(this);
2164 InstructionOperand temps[] = {g.TempSimd128Register()};
2165 InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
2166 InstructionOperand operand1 = g.UseUnique(node->InputAt(1));
2167 if (IsSupported(AVX)) {
2168 Emit(kIA32F64x2Max, g.DefineAsRegister(node), operand0, operand1,
2169 arraysize(temps), temps);
2170 } else {
2171 Emit(kIA32F64x2Max, g.DefineSameAsFirst(node), operand0, operand1,
2172 arraysize(temps), temps);
2173 }
2174 }
2175
VisitF64x2Splat(Node * node)2176 void InstructionSelector::VisitF64x2Splat(Node* node) {
2177 VisitRRSimd(this, node, kAVXF64x2Splat, kSSEF64x2Splat);
2178 }
2179
VisitF64x2ExtractLane(Node * node)2180 void InstructionSelector::VisitF64x2ExtractLane(Node* node) {
2181 VisitRRISimd(this, node, kAVXF64x2ExtractLane, kSSEF64x2ExtractLane);
2182 }
2183
VisitI64x2SplatI32Pair(Node * node)2184 void InstructionSelector::VisitI64x2SplatI32Pair(Node* node) {
2185 IA32OperandGenerator g(this);
2186 InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
2187 InstructionOperand operand1 = g.Use(node->InputAt(1));
2188 Emit(kIA32I64x2SplatI32Pair, g.DefineAsRegister(node), operand0, operand1);
2189 }
2190
VisitI64x2ReplaceLaneI32Pair(Node * node)2191 void InstructionSelector::VisitI64x2ReplaceLaneI32Pair(Node* node) {
2192 IA32OperandGenerator g(this);
2193 InstructionOperand operand = g.UseRegister(node->InputAt(0));
2194 InstructionOperand lane = g.UseImmediate(OpParameter<int32_t>(node->op()));
2195 InstructionOperand low = g.Use(node->InputAt(1));
2196 InstructionOperand high = g.Use(node->InputAt(2));
2197 Emit(kIA32I64x2ReplaceLaneI32Pair, g.DefineSameAsFirst(node), operand, lane,
2198 low, high);
2199 }
2200
VisitI64x2Neg(Node * node)2201 void InstructionSelector::VisitI64x2Neg(Node* node) {
2202 IA32OperandGenerator g(this);
2203 InstructionOperand operand0 = g.UseUnique(node->InputAt(0));
2204 Emit(kIA32I64x2Neg, g.DefineAsRegister(node), operand0);
2205 }
2206
VisitI64x2ShrS(Node * node)2207 void InstructionSelector::VisitI64x2ShrS(Node* node) {
2208 IA32OperandGenerator g(this);
2209 InstructionOperand temps[] = {g.TempSimd128Register(),
2210 g.TempSimd128Register()};
2211 if (IsSupported(AVX)) {
2212 Emit(kIA32I64x2ShrS, g.DefineAsRegister(node),
2213 g.UseUniqueRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
2214 arraysize(temps), temps);
2215 } else {
2216 Emit(kIA32I64x2ShrS, g.DefineSameAsFirst(node),
2217 g.UseUniqueRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
2218 arraysize(temps), temps);
2219 }
2220 }
2221
VisitI64x2Mul(Node * node)2222 void InstructionSelector::VisitI64x2Mul(Node* node) {
2223 IA32OperandGenerator g(this);
2224 InstructionOperand temps[] = {g.TempSimd128Register(),
2225 g.TempSimd128Register()};
2226 if (IsSupported(AVX)) {
2227 Emit(kIA32I64x2Mul, g.DefineAsRegister(node),
2228 g.UseUniqueRegister(node->InputAt(0)),
2229 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
2230 } else {
2231 Emit(kIA32I64x2Mul, g.DefineSameAsFirst(node),
2232 g.UseUniqueRegister(node->InputAt(0)),
2233 g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
2234 }
2235 }
2236
VisitF32x4Splat(Node * node)2237 void InstructionSelector::VisitF32x4Splat(Node* node) {
2238 VisitRRSimd(this, node, kAVXF32x4Splat, kSSEF32x4Splat);
2239 }
2240
VisitF32x4ExtractLane(Node * node)2241 void InstructionSelector::VisitF32x4ExtractLane(Node* node) {
2242 VisitRRISimd(this, node, kAVXF32x4ExtractLane, kSSEF32x4ExtractLane);
2243 }
2244
VisitF32x4UConvertI32x4(Node * node)2245 void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
2246 VisitRRSimd(this, node, kAVXF32x4UConvertI32x4, kSSEF32x4UConvertI32x4);
2247 }
2248
VisitI32x4SConvertF32x4(Node * node)2249 void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
2250 VisitRRSimd(this, node, kAVXI32x4SConvertF32x4, kSSEI32x4SConvertF32x4);
2251 }
2252
VisitI32x4UConvertF32x4(Node * node)2253 void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
2254 IA32OperandGenerator g(this);
2255 InstructionOperand temps[] = {g.TempSimd128Register()};
2256 InstructionCode opcode =
2257 IsSupported(AVX) ? kAVXI32x4UConvertF32x4 : kSSEI32x4UConvertF32x4;
2258 Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
2259 arraysize(temps), temps);
2260 }
2261
VisitI8x16Mul(Node * node)2262 void InstructionSelector::VisitI8x16Mul(Node* node) {
2263 IA32OperandGenerator g(this);
2264 InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
2265 InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1));
2266 InstructionOperand temps[] = {g.TempSimd128Register()};
2267 if (IsSupported(AVX)) {
2268 Emit(kAVXI8x16Mul, g.DefineAsRegister(node), operand0, operand1,
2269 arraysize(temps), temps);
2270 } else {
2271 Emit(kSSEI8x16Mul, g.DefineSameAsFirst(node), operand0, operand1,
2272 arraysize(temps), temps);
2273 }
2274 }
2275
VisitS128Zero(Node * node)2276 void InstructionSelector::VisitS128Zero(Node* node) {
2277 IA32OperandGenerator g(this);
2278 Emit(kIA32S128Zero, g.DefineAsRegister(node));
2279 }
2280
VisitS128Select(Node * node)2281 void InstructionSelector::VisitS128Select(Node* node) {
2282 IA32OperandGenerator g(this);
2283 InstructionOperand operand2 = g.UseRegister(node->InputAt(2));
2284 if (IsSupported(AVX)) {
2285 Emit(kAVXS128Select, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
2286 g.Use(node->InputAt(1)), operand2);
2287 } else {
2288 Emit(kSSES128Select, g.DefineSameAsFirst(node),
2289 g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
2290 operand2);
2291 }
2292 }
2293
VisitS128AndNot(Node * node)2294 void InstructionSelector::VisitS128AndNot(Node* node) {
2295 IA32OperandGenerator g(this);
2296 // andnps a b does ~a & b, but we want a & !b, so flip the input.
2297 Emit(kIA32S128AndNot, g.DefineSameAsFirst(node),
2298 g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(0)));
2299 }
2300
2301 #define VISIT_SIMD_SPLAT(Type) \
2302 void InstructionSelector::Visit##Type##Splat(Node* node) { \
2303 VisitRO(this, node, kIA32##Type##Splat); \
2304 }
2305 SIMD_INT_TYPES(VISIT_SIMD_SPLAT)
2306 #undef VISIT_SIMD_SPLAT
2307
2308 #define SIMD_VISIT_EXTRACT_LANE(Type, Sign) \
2309 void InstructionSelector::Visit##Type##ExtractLane##Sign(Node* node) { \
2310 VisitRRISimd(this, node, kIA32##Type##ExtractLane##Sign); \
2311 }
2312 SIMD_VISIT_EXTRACT_LANE(I32x4, )
SIMD_VISIT_EXTRACT_LANE(I16x8,U)2313 SIMD_VISIT_EXTRACT_LANE(I16x8, U)
2314 SIMD_VISIT_EXTRACT_LANE(I16x8, S)
2315 SIMD_VISIT_EXTRACT_LANE(I8x16, U)
2316 SIMD_VISIT_EXTRACT_LANE(I8x16, S)
2317 #undef SIMD_VISIT_EXTRACT_LANE
2318
2319 #define VISIT_SIMD_REPLACE_LANE(Type) \
2320 void InstructionSelector::Visit##Type##ReplaceLane(Node* node) { \
2321 IA32OperandGenerator g(this); \
2322 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); \
2323 InstructionOperand operand1 = \
2324 g.UseImmediate(OpParameter<int32_t>(node->op())); \
2325 InstructionOperand operand2 = g.Use(node->InputAt(1)); \
2326 if (IsSupported(AVX)) { \
2327 Emit(kAVX##Type##ReplaceLane, g.DefineAsRegister(node), operand0, \
2328 operand1, operand2); \
2329 } else { \
2330 Emit(kSSE##Type##ReplaceLane, g.DefineSameAsFirst(node), operand0, \
2331 operand1, operand2); \
2332 } \
2333 }
2334 SIMD_INT_TYPES(VISIT_SIMD_REPLACE_LANE)
2335 VISIT_SIMD_REPLACE_LANE(F32x4)
2336 #undef VISIT_SIMD_REPLACE_LANE
2337 #undef SIMD_INT_TYPES
2338
2339 // The difference between this and VISIT_SIMD_REPLACE_LANE is that this forces
2340 // operand2 to be UseRegister, because the codegen relies on insertps using
2341 // registers.
2342 // TODO(v8:9764) Remove this UseRegister requirement
2343 #define VISIT_SIMD_REPLACE_LANE_USE_REG(Type) \
2344 void InstructionSelector::Visit##Type##ReplaceLane(Node* node) { \
2345 IA32OperandGenerator g(this); \
2346 InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); \
2347 InstructionOperand operand1 = \
2348 g.UseImmediate(OpParameter<int32_t>(node->op())); \
2349 InstructionOperand operand2 = g.UseUniqueRegister(node->InputAt(1)); \
2350 if (IsSupported(AVX)) { \
2351 Emit(kAVX##Type##ReplaceLane, g.DefineAsRegister(node), operand0, \
2352 operand1, operand2); \
2353 } else { \
2354 Emit(kSSE##Type##ReplaceLane, g.DefineSameAsFirst(node), operand0, \
2355 operand1, operand2); \
2356 } \
2357 }
2358 VISIT_SIMD_REPLACE_LANE_USE_REG(F64x2)
2359 #undef VISIT_SIMD_REPLACE_LANE_USE_REG
2360
2361 #define VISIT_SIMD_SHIFT_UNIFIED_SSE_AVX(Opcode) \
2362 void InstructionSelector::Visit##Opcode(Node* node) { \
2363 VisitRROSimdShift(this, node, kIA32##Opcode); \
2364 }
2365 SIMD_SHIFT_OPCODES_UNIFED_SSE_AVX(VISIT_SIMD_SHIFT_UNIFIED_SSE_AVX)
2366 #undef VISIT_SIMD_SHIFT_UNIFIED_SSE_AVX
2367 #undef SIMD_SHIFT_OPCODES_UNIFED_SSE_AVX
2368
2369 #define VISIT_SIMD_UNOP(Opcode) \
2370 void InstructionSelector::Visit##Opcode(Node* node) { \
2371 IA32OperandGenerator g(this); \
2372 Emit(kIA32##Opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); \
2373 }
2374 SIMD_UNOP_LIST(VISIT_SIMD_UNOP)
2375 #undef VISIT_SIMD_UNOP
2376 #undef SIMD_UNOP_LIST
2377
2378 #define VISIT_SIMD_UNOP_PREFIX(Opcode) \
2379 void InstructionSelector::Visit##Opcode(Node* node) { \
2380 IA32OperandGenerator g(this); \
2381 InstructionCode opcode = IsSupported(AVX) ? kAVX##Opcode : kSSE##Opcode; \
2382 Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); \
2383 }
2384 SIMD_UNOP_PREFIX_LIST(VISIT_SIMD_UNOP_PREFIX)
2385 #undef VISIT_SIMD_UNOP_PREFIX
2386 #undef SIMD_UNOP_PREFIX_LIST
2387
2388 #define VISIT_SIMD_ANYTRUE(Opcode) \
2389 void InstructionSelector::Visit##Opcode(Node* node) { \
2390 IA32OperandGenerator g(this); \
2391 InstructionOperand temps[] = {g.TempRegister()}; \
2392 Emit(kIA32##Opcode, g.DefineAsRegister(node), \
2393 g.UseRegister(node->InputAt(0)), arraysize(temps), temps); \
2394 }
2395 SIMD_ANYTRUE_LIST(VISIT_SIMD_ANYTRUE)
2396 #undef VISIT_SIMD_ANYTRUE
2397 #undef SIMD_ANYTRUE_LIST
2398
2399 #define VISIT_SIMD_ALLTRUE(Opcode) \
2400 void InstructionSelector::Visit##Opcode(Node* node) { \
2401 IA32OperandGenerator g(this); \
2402 InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()}; \
2403 Emit(kIA32##Opcode, g.DefineAsRegister(node), \
2404 g.UseUnique(node->InputAt(0)), arraysize(temps), temps); \
2405 }
2406 SIMD_ALLTRUE_LIST(VISIT_SIMD_ALLTRUE)
2407 #undef VISIT_SIMD_ALLTRUE
2408 #undef SIMD_ALLTRUE_LIST
2409
2410 #define VISIT_SIMD_BINOP(Opcode) \
2411 void InstructionSelector::Visit##Opcode(Node* node) { \
2412 VisitRROFloat(this, node, kAVX##Opcode, kSSE##Opcode); \
2413 }
2414 SIMD_BINOP_LIST(VISIT_SIMD_BINOP)
2415 #undef VISIT_SIMD_BINOP
2416 #undef SIMD_BINOP_LIST
2417
2418 #define VISIT_SIMD_BINOP_UNIFIED_SSE_AVX(Opcode) \
2419 void InstructionSelector::Visit##Opcode(Node* node) { \
2420 VisitRROFloat(this, node, kIA32##Opcode, kIA32##Opcode); \
2421 }
2422 SIMD_BINOP_UNIFIED_SSE_AVX_LIST(VISIT_SIMD_BINOP_UNIFIED_SSE_AVX)
2423 #undef VISIT_SIMD_BINOP_UNIFIED_SSE_AVX
2424 #undef SIMD_BINOP_UNIFIED_SSE_AVX_LIST
2425
2426 void VisitPack(InstructionSelector* selector, Node* node, ArchOpcode avx_opcode,
2427 ArchOpcode sse_opcode) {
2428 IA32OperandGenerator g(selector);
2429 InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
2430 InstructionOperand operand1 = g.Use(node->InputAt(1));
2431 if (selector->IsSupported(AVX)) {
2432 selector->Emit(avx_opcode, g.DefineSameAsFirst(node), operand0, operand1);
2433 } else {
2434 selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
2435 }
2436 }
2437
VisitI16x8UConvertI32x4(Node * node)2438 void InstructionSelector::VisitI16x8UConvertI32x4(Node* node) {
2439 VisitPack(this, node, kAVXI16x8UConvertI32x4, kSSEI16x8UConvertI32x4);
2440 }
2441
VisitI8x16UConvertI16x8(Node * node)2442 void InstructionSelector::VisitI8x16UConvertI16x8(Node* node) {
2443 VisitPack(this, node, kAVXI8x16UConvertI16x8, kSSEI8x16UConvertI16x8);
2444 }
2445
VisitI8x16Shl(Node * node)2446 void InstructionSelector::VisitI8x16Shl(Node* node) {
2447 IA32OperandGenerator g(this);
2448 if (g.CanBeImmediate(node->InputAt(1))) {
2449 InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
2450 this->Emit(kIA32I8x16Shl, g.DefineSameAsFirst(node),
2451 g.UseRegister(node->InputAt(0)),
2452 g.UseImmediate(node->InputAt(1)), arraysize(temps), temps);
2453 } else {
2454 VisitRROI8x16SimdShift(this, node, kIA32I8x16Shl);
2455 }
2456 }
2457
VisitI8x16ShrS(Node * node)2458 void InstructionSelector::VisitI8x16ShrS(Node* node) {
2459 IA32OperandGenerator g(this);
2460 if (g.CanBeImmediate(node->InputAt(1))) {
2461 this->Emit(kIA32I8x16ShrS, g.DefineSameAsFirst(node),
2462 g.UseRegister(node->InputAt(0)),
2463 g.UseImmediate(node->InputAt(1)));
2464 } else {
2465 VisitRROI8x16SimdShift(this, node, kIA32I8x16ShrS);
2466 }
2467 }
2468
VisitI8x16ShrU(Node * node)2469 void InstructionSelector::VisitI8x16ShrU(Node* node) {
2470 IA32OperandGenerator g(this);
2471 if (g.CanBeImmediate(node->InputAt(1))) {
2472 InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
2473 this->Emit(kIA32I8x16ShrU, g.DefineSameAsFirst(node),
2474 g.UseRegister(node->InputAt(0)),
2475 g.UseImmediate(node->InputAt(1)), arraysize(temps), temps);
2476 } else {
2477 VisitRROI8x16SimdShift(this, node, kIA32I8x16ShrU);
2478 }
2479 }
2480
VisitInt32AbsWithOverflow(Node * node)2481 void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
2482 UNREACHABLE();
2483 }
2484
VisitInt64AbsWithOverflow(Node * node)2485 void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
2486 UNREACHABLE();
2487 }
2488
2489 namespace {
2490
2491 // Packs a 4 lane shuffle into a single imm8 suitable for use by pshufd,
2492 // pshuflw, and pshufhw.
PackShuffle4(uint8_t * shuffle)2493 uint8_t PackShuffle4(uint8_t* shuffle) {
2494 return (shuffle[0] & 3) | ((shuffle[1] & 3) << 2) | ((shuffle[2] & 3) << 4) |
2495 ((shuffle[3] & 3) << 6);
2496 }
2497
2498 // Gets an 8 bit lane mask suitable for 16x8 pblendw.
PackBlend8(const uint8_t * shuffle16x8)2499 uint8_t PackBlend8(const uint8_t* shuffle16x8) {
2500 int8_t result = 0;
2501 for (int i = 0; i < 8; ++i) {
2502 result |= (shuffle16x8[i] >= 8 ? 1 : 0) << i;
2503 }
2504 return result;
2505 }
2506
2507 // Gets an 8 bit lane mask suitable for 32x4 pblendw.
PackBlend4(const uint8_t * shuffle32x4)2508 uint8_t PackBlend4(const uint8_t* shuffle32x4) {
2509 int8_t result = 0;
2510 for (int i = 0; i < 4; ++i) {
2511 result |= (shuffle32x4[i] >= 4 ? 0x3 : 0) << (i * 2);
2512 }
2513 return result;
2514 }
2515
2516 // Returns true if shuffle can be decomposed into two 16x4 half shuffles
2517 // followed by a 16x8 blend.
2518 // E.g. [3 2 1 0 15 14 13 12].
TryMatch16x8HalfShuffle(uint8_t * shuffle16x8,uint8_t * blend_mask)2519 bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) {
2520 *blend_mask = 0;
2521 for (int i = 0; i < 8; i++) {
2522 if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false;
2523 *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i;
2524 }
2525 return true;
2526 }
2527
2528 struct ShuffleEntry {
2529 uint8_t shuffle[kSimd128Size];
2530 ArchOpcode opcode;
2531 ArchOpcode avx_opcode;
2532 bool src0_needs_reg;
2533 bool src1_needs_reg;
2534 };
2535
2536 // Shuffles that map to architecture-specific instruction sequences. These are
2537 // matched very early, so we shouldn't include shuffles that match better in
2538 // later tests, like 32x4 and 16x8 shuffles. In general, these patterns should
2539 // map to either a single instruction, or be finer grained, such as zip/unzip or
2540 // transpose patterns.
2541 static const ShuffleEntry arch_shuffles[] = {
2542 {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23},
2543 kIA32S64x2UnpackLow,
2544 kIA32S64x2UnpackLow,
2545 true,
2546 false},
2547 {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31},
2548 kIA32S64x2UnpackHigh,
2549 kIA32S64x2UnpackHigh,
2550 true,
2551 false},
2552 {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
2553 kIA32S32x4UnpackLow,
2554 kIA32S32x4UnpackLow,
2555 true,
2556 false},
2557 {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
2558 kIA32S32x4UnpackHigh,
2559 kIA32S32x4UnpackHigh,
2560 true,
2561 false},
2562 {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
2563 kIA32S16x8UnpackLow,
2564 kIA32S16x8UnpackLow,
2565 true,
2566 false},
2567 {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
2568 kIA32S16x8UnpackHigh,
2569 kIA32S16x8UnpackHigh,
2570 true,
2571 false},
2572 {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
2573 kIA32S8x16UnpackLow,
2574 kIA32S8x16UnpackLow,
2575 true,
2576 false},
2577 {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
2578 kIA32S8x16UnpackHigh,
2579 kIA32S8x16UnpackHigh,
2580 true,
2581 false},
2582
2583 {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
2584 kSSES16x8UnzipLow,
2585 kAVXS16x8UnzipLow,
2586 true,
2587 false},
2588 {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
2589 kSSES16x8UnzipHigh,
2590 kAVXS16x8UnzipHigh,
2591 true,
2592 true},
2593 {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
2594 kSSES8x16UnzipLow,
2595 kAVXS8x16UnzipLow,
2596 true,
2597 true},
2598 {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
2599 kSSES8x16UnzipHigh,
2600 kAVXS8x16UnzipHigh,
2601 true,
2602 true},
2603
2604 {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
2605 kSSES8x16TransposeLow,
2606 kAVXS8x16TransposeLow,
2607 true,
2608 true},
2609 {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
2610 kSSES8x16TransposeHigh,
2611 kAVXS8x16TransposeHigh,
2612 true,
2613 true},
2614 {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8},
2615 kSSES8x8Reverse,
2616 kAVXS8x8Reverse,
2617 true,
2618 true},
2619 {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12},
2620 kSSES8x4Reverse,
2621 kAVXS8x4Reverse,
2622 true,
2623 true},
2624 {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
2625 kSSES8x2Reverse,
2626 kAVXS8x2Reverse,
2627 true,
2628 true}};
2629
TryMatchArchShuffle(const uint8_t * shuffle,const ShuffleEntry * table,size_t num_entries,bool is_swizzle,const ShuffleEntry ** arch_shuffle)2630 bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
2631 size_t num_entries, bool is_swizzle,
2632 const ShuffleEntry** arch_shuffle) {
2633 uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1;
2634 for (size_t i = 0; i < num_entries; ++i) {
2635 const ShuffleEntry& entry = table[i];
2636 int j = 0;
2637 for (; j < kSimd128Size; ++j) {
2638 if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
2639 break;
2640 }
2641 }
2642 if (j == kSimd128Size) {
2643 *arch_shuffle = &entry;
2644 return true;
2645 }
2646 }
2647 return false;
2648 }
2649
2650 } // namespace
2651
VisitS8x16Shuffle(Node * node)2652 void InstructionSelector::VisitS8x16Shuffle(Node* node) {
2653 uint8_t shuffle[kSimd128Size];
2654 bool is_swizzle;
2655 CanonicalizeShuffle(node, shuffle, &is_swizzle);
2656
2657 int imm_count = 0;
2658 static const int kMaxImms = 6;
2659 uint32_t imms[kMaxImms];
2660 int temp_count = 0;
2661 static const int kMaxTemps = 2;
2662 InstructionOperand temps[kMaxTemps];
2663
2664 IA32OperandGenerator g(this);
2665 bool use_avx = CpuFeatures::IsSupported(AVX);
2666 // AVX and swizzles don't generally need DefineSameAsFirst to avoid a move.
2667 bool no_same_as_first = use_avx || is_swizzle;
2668 // We generally need UseRegister for input0, Use for input1.
2669 bool src0_needs_reg = true;
2670 bool src1_needs_reg = false;
2671 ArchOpcode opcode = kIA32S8x16Shuffle; // general shuffle is the default
2672
2673 uint8_t offset;
2674 uint8_t shuffle32x4[4];
2675 uint8_t shuffle16x8[8];
2676 int index;
2677 const ShuffleEntry* arch_shuffle;
2678 if (TryMatchConcat(shuffle, &offset)) {
2679 // Swap inputs from the normal order for (v)palignr.
2680 SwapShuffleInputs(node);
2681 is_swizzle = false; // It's simpler to just handle the general case.
2682 no_same_as_first = use_avx; // SSE requires same-as-first.
2683 opcode = kIA32S8x16Alignr;
2684 // palignr takes a single imm8 offset.
2685 imms[imm_count++] = offset;
2686 } else if (TryMatchArchShuffle(shuffle, arch_shuffles,
2687 arraysize(arch_shuffles), is_swizzle,
2688 &arch_shuffle)) {
2689 opcode = use_avx ? arch_shuffle->avx_opcode : arch_shuffle->opcode;
2690 src0_needs_reg = !use_avx || arch_shuffle->src0_needs_reg;
2691 // SSE can't take advantage of both operands in registers and needs
2692 // same-as-first.
2693 src1_needs_reg = use_avx && arch_shuffle->src1_needs_reg;
2694 no_same_as_first = use_avx;
2695 } else if (TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
2696 uint8_t shuffle_mask = PackShuffle4(shuffle32x4);
2697 if (is_swizzle) {
2698 if (TryMatchIdentity(shuffle)) {
2699 // Bypass normal shuffle code generation in this case.
2700 EmitIdentity(node);
2701 return;
2702 } else {
2703 // pshufd takes a single imm8 shuffle mask.
2704 opcode = kIA32S32x4Swizzle;
2705 no_same_as_first = true;
2706 // TODO(v8:9198): This doesn't strictly require a register, forcing the
2707 // swizzles to always use registers until generation of incorrect memory
2708 // operands can be fixed.
2709 src0_needs_reg = true;
2710 imms[imm_count++] = shuffle_mask;
2711 }
2712 } else {
2713 // 2 operand shuffle
2714 // A blend is more efficient than a general 32x4 shuffle; try it first.
2715 if (TryMatchBlend(shuffle)) {
2716 opcode = kIA32S16x8Blend;
2717 uint8_t blend_mask = PackBlend4(shuffle32x4);
2718 imms[imm_count++] = blend_mask;
2719 } else {
2720 opcode = kIA32S32x4Shuffle;
2721 no_same_as_first = true;
2722 // TODO(v8:9198): src0 and src1 is used by pshufd in codegen, which
2723 // requires memory to be 16-byte aligned, since we cannot guarantee that
2724 // yet, force using a register here.
2725 src0_needs_reg = true;
2726 src1_needs_reg = true;
2727 imms[imm_count++] = shuffle_mask;
2728 int8_t blend_mask = PackBlend4(shuffle32x4);
2729 imms[imm_count++] = blend_mask;
2730 }
2731 }
2732 } else if (TryMatch16x8Shuffle(shuffle, shuffle16x8)) {
2733 uint8_t blend_mask;
2734 if (TryMatchBlend(shuffle)) {
2735 opcode = kIA32S16x8Blend;
2736 blend_mask = PackBlend8(shuffle16x8);
2737 imms[imm_count++] = blend_mask;
2738 } else if (TryMatchDup<8>(shuffle, &index)) {
2739 opcode = kIA32S16x8Dup;
2740 src0_needs_reg = false;
2741 imms[imm_count++] = index;
2742 } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) {
2743 opcode = is_swizzle ? kIA32S16x8HalfShuffle1 : kIA32S16x8HalfShuffle2;
2744 // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0).
2745 no_same_as_first = true;
2746 src0_needs_reg = false;
2747 uint8_t mask_lo = PackShuffle4(shuffle16x8);
2748 uint8_t mask_hi = PackShuffle4(shuffle16x8 + 4);
2749 imms[imm_count++] = mask_lo;
2750 imms[imm_count++] = mask_hi;
2751 if (!is_swizzle) imms[imm_count++] = blend_mask;
2752 }
2753 } else if (TryMatchDup<16>(shuffle, &index)) {
2754 opcode = kIA32S8x16Dup;
2755 no_same_as_first = use_avx;
2756 src0_needs_reg = true;
2757 imms[imm_count++] = index;
2758 }
2759 if (opcode == kIA32S8x16Shuffle) {
2760 // Use same-as-first for general swizzle, but not shuffle.
2761 no_same_as_first = !is_swizzle;
2762 src0_needs_reg = !no_same_as_first;
2763 imms[imm_count++] = Pack4Lanes(shuffle);
2764 imms[imm_count++] = Pack4Lanes(shuffle + 4);
2765 imms[imm_count++] = Pack4Lanes(shuffle + 8);
2766 imms[imm_count++] = Pack4Lanes(shuffle + 12);
2767 temps[temp_count++] = g.TempRegister();
2768 }
2769
2770 // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra
2771 // move instruction in the CodeGenerator.
2772 Node* input0 = node->InputAt(0);
2773 InstructionOperand dst =
2774 no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
2775 InstructionOperand src0 =
2776 src0_needs_reg ? g.UseRegister(input0) : g.Use(input0);
2777
2778 int input_count = 0;
2779 InstructionOperand inputs[2 + kMaxImms + kMaxTemps];
2780 inputs[input_count++] = src0;
2781 if (!is_swizzle) {
2782 Node* input1 = node->InputAt(1);
2783 inputs[input_count++] =
2784 src1_needs_reg ? g.UseRegister(input1) : g.Use(input1);
2785 }
2786 for (int i = 0; i < imm_count; ++i) {
2787 inputs[input_count++] = g.UseImmediate(imms[i]);
2788 }
2789 Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps);
2790 }
2791
VisitS8x16Swizzle(Node * node)2792 void InstructionSelector::VisitS8x16Swizzle(Node* node) {
2793 IA32OperandGenerator g(this);
2794 InstructionOperand temps[] = {g.TempSimd128Register()};
2795 Emit(kIA32S8x16Swizzle, g.DefineSameAsFirst(node),
2796 g.UseRegister(node->InputAt(0)), g.UseUniqueRegister(node->InputAt(1)),
2797 arraysize(temps), temps);
2798 }
2799
2800 // static
2801 MachineOperatorBuilder::Flags
SupportedMachineOperatorFlags()2802 InstructionSelector::SupportedMachineOperatorFlags() {
2803 MachineOperatorBuilder::Flags flags =
2804 MachineOperatorBuilder::kWord32ShiftIsSafe |
2805 MachineOperatorBuilder::kWord32Ctz;
2806 if (CpuFeatures::IsSupported(POPCNT)) {
2807 flags |= MachineOperatorBuilder::kWord32Popcnt;
2808 }
2809 if (CpuFeatures::IsSupported(SSE4_1)) {
2810 flags |= MachineOperatorBuilder::kFloat32RoundDown |
2811 MachineOperatorBuilder::kFloat64RoundDown |
2812 MachineOperatorBuilder::kFloat32RoundUp |
2813 MachineOperatorBuilder::kFloat64RoundUp |
2814 MachineOperatorBuilder::kFloat32RoundTruncate |
2815 MachineOperatorBuilder::kFloat64RoundTruncate |
2816 MachineOperatorBuilder::kFloat32RoundTiesEven |
2817 MachineOperatorBuilder::kFloat64RoundTiesEven;
2818 }
2819 return flags;
2820 }
2821
2822 // static
2823 MachineOperatorBuilder::AlignmentRequirements
AlignmentRequirements()2824 InstructionSelector::AlignmentRequirements() {
2825 return MachineOperatorBuilder::AlignmentRequirements::
2826 FullUnalignedAccessSupport();
2827 }
2828
2829 } // namespace compiler
2830 } // namespace internal
2831 } // namespace v8
2832