1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/int64-lowering.h"
6
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature,std::unique_ptr<Int64LoweringSpecialCase> special_case)24 Int64Lowering::Int64Lowering(
25 Graph* graph, MachineOperatorBuilder* machine,
26 CommonOperatorBuilder* common, Zone* zone,
27 Signature<MachineRepresentation>* signature,
28 std::unique_ptr<Int64LoweringSpecialCase> special_case)
29 : zone_(zone),
30 graph_(graph),
31 machine_(machine),
32 common_(common),
33 state_(graph, 3),
34 stack_(zone),
35 replacements_(nullptr),
36 signature_(signature),
37 placeholder_(
38 graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
39 special_case_(std::move(special_case)) {
40 DCHECK_NOT_NULL(graph);
41 DCHECK_NOT_NULL(graph->end());
42 replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
43 memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
44 }
45
LowerGraph()46 void Int64Lowering::LowerGraph() {
47 if (!machine()->Is32()) {
48 return;
49 }
50 stack_.push_back({graph()->end(), 0});
51 state_.Set(graph()->end(), State::kOnStack);
52
53 while (!stack_.empty()) {
54 NodeState& top = stack_.back();
55 if (top.input_index == top.node->InputCount()) {
56 // All inputs of top have already been lowered, now lower top.
57 stack_.pop_back();
58 state_.Set(top.node, State::kVisited);
59 LowerNode(top.node);
60 } else {
61 // Push the next input onto the stack.
62 Node* input = top.node->InputAt(top.input_index++);
63 if (state_.Get(input) == State::kUnvisited) {
64 if (input->opcode() == IrOpcode::kPhi) {
65 // To break cycles with phi nodes we push phis on a separate stack so
66 // that they are processed after all other nodes.
67 PreparePhiReplacement(input);
68 stack_.push_front({input, 0});
69 } else if (input->opcode() == IrOpcode::kEffectPhi ||
70 input->opcode() == IrOpcode::kLoop) {
71 stack_.push_front({input, 0});
72 } else {
73 stack_.push_back({input, 0});
74 }
75 state_.Set(input, State::kOnStack);
76 }
77 }
78 }
79 }
80
81 namespace {
82
GetReturnIndexAfterLowering(const CallDescriptor * call_descriptor,int old_index)83 int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
84 int old_index) {
85 int result = old_index;
86 for (int i = 0; i < old_index; i++) {
87 if (call_descriptor->GetReturnType(i).representation() ==
88 MachineRepresentation::kWord64) {
89 result++;
90 }
91 }
92 return result;
93 }
94
GetReturnCountAfterLowering(const CallDescriptor * call_descriptor)95 int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
96 return GetReturnIndexAfterLowering(
97 call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
98 }
99
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)100 int GetParameterIndexAfterLowering(
101 Signature<MachineRepresentation>* signature, int old_index) {
102 int result = old_index;
103 // Be robust towards special indexes (>= param count).
104 int max_to_check =
105 std::min(old_index, static_cast<int>(signature->parameter_count()));
106 for (int i = 0; i < max_to_check; i++) {
107 if (signature->GetParam(i) == MachineRepresentation::kWord64) {
108 result++;
109 }
110 }
111 return result;
112 }
113
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)114 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
115 int result = static_cast<int>(signature->return_count());
116 for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
117 if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
118 result++;
119 }
120 }
121 return result;
122 }
123
124 } // namespace
125
LowerWord64AtomicBinop(Node * node,const Operator * op)126 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
127 DCHECK_EQ(5, node->InputCount());
128 LowerMemoryBaseAndIndex(node);
129 Node* value = node->InputAt(2);
130 node->ReplaceInput(2, GetReplacementLow(value));
131 node->InsertInput(zone(), 3, GetReplacementHigh(value));
132 NodeProperties::ChangeOp(node, op);
133 ReplaceNodeWithProjections(node);
134 }
135
LowerWord64AtomicNarrowOp(Node * node,const Operator * op)136 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
137 DefaultLowering(node, true);
138 NodeProperties::ChangeOp(node, op);
139 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
140 }
141
142 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)143 int Int64Lowering::GetParameterCountAfterLowering(
144 Signature<MachineRepresentation>* signature) {
145 // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
146 // after lowering.
147 return GetParameterIndexAfterLowering(
148 signature, static_cast<int>(signature->parameter_count()));
149 }
150
GetIndexNodes(Node * index,Node ** index_low,Node ** index_high)151 void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
152 Node** index_high) {
153 #if defined(V8_TARGET_LITTLE_ENDIAN)
154 *index_low = index;
155 *index_high = graph()->NewNode(machine()->Int32Add(), index,
156 graph()->NewNode(common()->Int32Constant(4)));
157 #elif defined(V8_TARGET_BIG_ENDIAN)
158 *index_low = graph()->NewNode(machine()->Int32Add(), index,
159 graph()->NewNode(common()->Int32Constant(4)));
160 *index_high = index;
161 #endif
162 }
163
LowerNode(Node * node)164 void Int64Lowering::LowerNode(Node* node) {
165 switch (node->opcode()) {
166 case IrOpcode::kInt64Constant: {
167 int64_t value = OpParameter<int64_t>(node->op());
168 Node* low_node = graph()->NewNode(
169 common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
170 Node* high_node = graph()->NewNode(
171 common()->Int32Constant(static_cast<int32_t>(value >> 32)));
172 ReplaceNode(node, low_node, high_node);
173 break;
174 }
175 case IrOpcode::kLoad:
176 case IrOpcode::kUnalignedLoad: {
177 MachineRepresentation rep;
178 if (node->opcode() == IrOpcode::kLoad) {
179 rep = LoadRepresentationOf(node->op()).representation();
180 } else {
181 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
182 rep = LoadRepresentationOf(node->op()).representation();
183 }
184
185 if (rep == MachineRepresentation::kWord64) {
186 LowerMemoryBaseAndIndex(node);
187 Node* base = node->InputAt(0);
188 Node* index = node->InputAt(1);
189 Node* index_low;
190 Node* index_high;
191 GetIndexNodes(index, &index_low, &index_high);
192 const Operator* load_op;
193
194 if (node->opcode() == IrOpcode::kLoad) {
195 load_op = machine()->Load(MachineType::Int32());
196 } else {
197 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
198 load_op = machine()->UnalignedLoad(MachineType::Int32());
199 }
200
201 Node* high_node;
202 if (node->InputCount() > 2) {
203 Node* effect_high = node->InputAt(2);
204 Node* control_high = node->InputAt(3);
205 high_node = graph()->NewNode(load_op, base, index_high, effect_high,
206 control_high);
207 // change the effect change from old_node --> old_effect to
208 // old_node --> high_node --> old_effect.
209 node->ReplaceInput(2, high_node);
210 } else {
211 high_node = graph()->NewNode(load_op, base, index_high);
212 }
213 node->ReplaceInput(1, index_low);
214 NodeProperties::ChangeOp(node, load_op);
215 ReplaceNode(node, node, high_node);
216 } else {
217 DefaultLowering(node);
218 }
219 break;
220 }
221 case IrOpcode::kStore:
222 case IrOpcode::kUnalignedStore: {
223 MachineRepresentation rep;
224 if (node->opcode() == IrOpcode::kStore) {
225 rep = StoreRepresentationOf(node->op()).representation();
226 } else {
227 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
228 rep = UnalignedStoreRepresentationOf(node->op());
229 }
230
231 if (rep == MachineRepresentation::kWord64) {
232 // We change the original store node to store the low word, and create
233 // a new store node to store the high word. The effect and control edges
234 // are copied from the original store to the new store node, the effect
235 // edge of the original store is redirected to the new store.
236 LowerMemoryBaseAndIndex(node);
237 Node* base = node->InputAt(0);
238 Node* index = node->InputAt(1);
239 Node* index_low;
240 Node* index_high;
241 GetIndexNodes(index, &index_low, &index_high);
242 Node* value = node->InputAt(2);
243 DCHECK(HasReplacementLow(value));
244 DCHECK(HasReplacementHigh(value));
245
246 const Operator* store_op;
247 if (node->opcode() == IrOpcode::kStore) {
248 WriteBarrierKind write_barrier_kind =
249 StoreRepresentationOf(node->op()).write_barrier_kind();
250 store_op = machine()->Store(StoreRepresentation(
251 MachineRepresentation::kWord32, write_barrier_kind));
252 } else {
253 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
254 store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
255 }
256
257 Node* high_node;
258 if (node->InputCount() > 3) {
259 Node* effect_high = node->InputAt(3);
260 Node* control_high = node->InputAt(4);
261 high_node = graph()->NewNode(store_op, base, index_high,
262 GetReplacementHigh(value), effect_high,
263 control_high);
264 node->ReplaceInput(3, high_node);
265
266 } else {
267 high_node = graph()->NewNode(store_op, base, index_high,
268 GetReplacementHigh(value));
269 }
270
271 node->ReplaceInput(1, index_low);
272 node->ReplaceInput(2, GetReplacementLow(value));
273 NodeProperties::ChangeOp(node, store_op);
274 ReplaceNode(node, node, high_node);
275 } else {
276 DefaultLowering(node, true);
277 }
278 break;
279 }
280 case IrOpcode::kStart: {
281 int parameter_count = GetParameterCountAfterLowering(signature());
282 // Only exchange the node if the parameter count actually changed.
283 if (parameter_count != static_cast<int>(signature()->parameter_count())) {
284 int delta =
285 parameter_count - static_cast<int>(signature()->parameter_count());
286 int new_output_count = node->op()->ValueOutputCount() + delta;
287 NodeProperties::ChangeOp(node, common()->Start(new_output_count));
288 }
289 break;
290 }
291 case IrOpcode::kParameter: {
292 DCHECK_EQ(1, node->InputCount());
293 int param_count = static_cast<int>(signature()->parameter_count());
294 // Only exchange the node if the parameter count actually changed. We do
295 // not even have to do the default lowering because the the start node,
296 // the only input of a parameter node, only changes if the parameter count
297 // changes.
298 if (GetParameterCountAfterLowering(signature()) != param_count) {
299 int old_index = ParameterIndexOf(node->op());
300 // Adjust old_index to be compliant with the signature.
301 --old_index;
302 int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303 // Adjust new_index to consider the instance parameter.
304 ++new_index;
305 NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306
307 if (old_index < 0 || old_index >= param_count) {
308 // Special parameters (JS closure/context) don't have kWord64
309 // representation anyway.
310 break;
311 }
312
313 if (signature()->GetParam(old_index) ==
314 MachineRepresentation::kWord64) {
315 Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
316 graph()->start());
317 ReplaceNode(node, node, high_node);
318 }
319 }
320 break;
321 }
322 case IrOpcode::kReturn: {
323 int input_count = node->InputCount();
324 DefaultLowering(node);
325 if (input_count != node->InputCount()) {
326 int new_return_count = GetReturnCountAfterLowering(signature());
327 if (static_cast<int>(signature()->return_count()) != new_return_count) {
328 NodeProperties::ChangeOp(node, common()->Return(new_return_count));
329 }
330 }
331 break;
332 }
333 case IrOpcode::kTailCall: {
334 auto call_descriptor =
335 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
336 bool returns_require_lowering =
337 GetReturnCountAfterLowering(call_descriptor) !=
338 static_cast<int>(call_descriptor->ReturnCount());
339 if (DefaultLowering(node) || returns_require_lowering) {
340 // Tail calls do not have return values, so adjusting the call
341 // descriptor is enough.
342 NodeProperties::ChangeOp(
343 node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
344 }
345 break;
346 }
347 case IrOpcode::kCall: {
348 auto call_descriptor = CallDescriptorOf(node->op());
349
350 bool returns_require_lowering =
351 GetReturnCountAfterLowering(call_descriptor) !=
352 static_cast<int>(call_descriptor->ReturnCount());
353 if (DefaultLowering(node) || returns_require_lowering) {
354 // We have to adjust the call descriptor.
355 NodeProperties::ChangeOp(
356 node, common()->Call(LowerCallDescriptor(call_descriptor)));
357 }
358 if (returns_require_lowering) {
359 size_t return_arity = call_descriptor->ReturnCount();
360 if (return_arity == 1) {
361 // We access the additional return values through projections.
362 ReplaceNodeWithProjections(node);
363 } else {
364 ZoneVector<Node*> projections(return_arity, zone());
365 NodeProperties::CollectValueProjections(node, projections.data(),
366 return_arity);
367 for (size_t old_index = 0, new_index = 0; old_index < return_arity;
368 ++old_index, ++new_index) {
369 Node* use_node = projections[old_index];
370 DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
371 DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
372 static_cast<int>(old_index)),
373 static_cast<int>(new_index));
374 if (new_index != old_index) {
375 NodeProperties::ChangeOp(
376 use_node, common()->Projection(new_index));
377 }
378 if (call_descriptor->GetReturnType(old_index).representation() ==
379 MachineRepresentation::kWord64) {
380 Node* high_node = graph()->NewNode(
381 common()->Projection(new_index + 1), node,
382 graph()->start());
383 ReplaceNode(use_node, use_node, high_node);
384 ++new_index;
385 }
386 }
387 }
388 }
389 break;
390 }
391 case IrOpcode::kWord64And: {
392 DCHECK_EQ(2, node->InputCount());
393 Node* left = node->InputAt(0);
394 Node* right = node->InputAt(1);
395
396 Node* low_node =
397 graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
398 GetReplacementLow(right));
399 Node* high_node =
400 graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
401 GetReplacementHigh(right));
402 ReplaceNode(node, low_node, high_node);
403 break;
404 }
405 case IrOpcode::kTruncateInt64ToInt32: {
406 DCHECK_EQ(1, node->InputCount());
407 Node* input = node->InputAt(0);
408 ReplaceNode(node, GetReplacementLow(input), nullptr);
409 node->NullAllInputs();
410 break;
411 }
412 case IrOpcode::kInt64Add: {
413 DCHECK_EQ(2, node->InputCount());
414
415 Node* right = node->InputAt(1);
416 node->ReplaceInput(1, GetReplacementLow(right));
417 node->AppendInput(zone(), GetReplacementHigh(right));
418
419 Node* left = node->InputAt(0);
420 node->ReplaceInput(0, GetReplacementLow(left));
421 node->InsertInput(zone(), 1, GetReplacementHigh(left));
422
423 NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
424 // We access the additional return values through projections.
425 ReplaceNodeWithProjections(node);
426 break;
427 }
428 case IrOpcode::kInt64Sub: {
429 DCHECK_EQ(2, node->InputCount());
430
431 Node* right = node->InputAt(1);
432 node->ReplaceInput(1, GetReplacementLow(right));
433 node->AppendInput(zone(), GetReplacementHigh(right));
434
435 Node* left = node->InputAt(0);
436 node->ReplaceInput(0, GetReplacementLow(left));
437 node->InsertInput(zone(), 1, GetReplacementHigh(left));
438
439 NodeProperties::ChangeOp(node, machine()->Int32PairSub());
440 // We access the additional return values through projections.
441 ReplaceNodeWithProjections(node);
442 break;
443 }
444 case IrOpcode::kInt64Mul: {
445 DCHECK_EQ(2, node->InputCount());
446
447 Node* right = node->InputAt(1);
448 node->ReplaceInput(1, GetReplacementLow(right));
449 node->AppendInput(zone(), GetReplacementHigh(right));
450
451 Node* left = node->InputAt(0);
452 node->ReplaceInput(0, GetReplacementLow(left));
453 node->InsertInput(zone(), 1, GetReplacementHigh(left));
454
455 NodeProperties::ChangeOp(node, machine()->Int32PairMul());
456 // We access the additional return values through projections.
457 ReplaceNodeWithProjections(node);
458 break;
459 }
460 case IrOpcode::kWord64Or: {
461 DCHECK_EQ(2, node->InputCount());
462 Node* left = node->InputAt(0);
463 Node* right = node->InputAt(1);
464
465 Node* low_node =
466 graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
467 GetReplacementLow(right));
468 Node* high_node =
469 graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
470 GetReplacementHigh(right));
471 ReplaceNode(node, low_node, high_node);
472 break;
473 }
474 case IrOpcode::kWord64Xor: {
475 DCHECK_EQ(2, node->InputCount());
476 Node* left = node->InputAt(0);
477 Node* right = node->InputAt(1);
478
479 Node* low_node =
480 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
481 GetReplacementLow(right));
482 Node* high_node =
483 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
484 GetReplacementHigh(right));
485 ReplaceNode(node, low_node, high_node);
486 break;
487 }
488 case IrOpcode::kWord64Shl: {
489 // TODO(turbofan): if the shift count >= 32, then we can set the low word
490 // of the output to 0 and just calculate the high word.
491 DCHECK_EQ(2, node->InputCount());
492 Node* shift = node->InputAt(1);
493 if (HasReplacementLow(shift)) {
494 // We do not have to care about the high word replacement, because
495 // the shift can only be between 0 and 63 anyways.
496 node->ReplaceInput(1, GetReplacementLow(shift));
497 }
498
499 Node* value = node->InputAt(0);
500 node->ReplaceInput(0, GetReplacementLow(value));
501 node->InsertInput(zone(), 1, GetReplacementHigh(value));
502
503 NodeProperties::ChangeOp(node, machine()->Word32PairShl());
504 // We access the additional return values through projections.
505 ReplaceNodeWithProjections(node);
506 break;
507 }
508 case IrOpcode::kWord64Shr: {
509 // TODO(turbofan): if the shift count >= 32, then we can set the low word
510 // of the output to 0 and just calculate the high word.
511 DCHECK_EQ(2, node->InputCount());
512 Node* shift = node->InputAt(1);
513 if (HasReplacementLow(shift)) {
514 // We do not have to care about the high word replacement, because
515 // the shift can only be between 0 and 63 anyways.
516 node->ReplaceInput(1, GetReplacementLow(shift));
517 }
518
519 Node* value = node->InputAt(0);
520 node->ReplaceInput(0, GetReplacementLow(value));
521 node->InsertInput(zone(), 1, GetReplacementHigh(value));
522
523 NodeProperties::ChangeOp(node, machine()->Word32PairShr());
524 // We access the additional return values through projections.
525 ReplaceNodeWithProjections(node);
526 break;
527 }
528 case IrOpcode::kWord64Sar: {
529 // TODO(turbofan): if the shift count >= 32, then we can set the low word
530 // of the output to 0 and just calculate the high word.
531 DCHECK_EQ(2, node->InputCount());
532 Node* shift = node->InputAt(1);
533 if (HasReplacementLow(shift)) {
534 // We do not have to care about the high word replacement, because
535 // the shift can only be between 0 and 63 anyways.
536 node->ReplaceInput(1, GetReplacementLow(shift));
537 }
538
539 Node* value = node->InputAt(0);
540 node->ReplaceInput(0, GetReplacementLow(value));
541 node->InsertInput(zone(), 1, GetReplacementHigh(value));
542
543 NodeProperties::ChangeOp(node, machine()->Word32PairSar());
544 // We access the additional return values through projections.
545 ReplaceNodeWithProjections(node);
546 break;
547 }
548 case IrOpcode::kWord64Equal: {
549 DCHECK_EQ(2, node->InputCount());
550 Node* left = node->InputAt(0);
551 Node* right = node->InputAt(1);
552
553 // TODO(wasm): Use explicit comparisons and && here?
554 Node* replacement = graph()->NewNode(
555 machine()->Word32Equal(),
556 graph()->NewNode(
557 machine()->Word32Or(),
558 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
559 GetReplacementLow(right)),
560 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
561 GetReplacementHigh(right))),
562 graph()->NewNode(common()->Int32Constant(0)));
563
564 ReplaceNode(node, replacement, nullptr);
565 break;
566 }
567 case IrOpcode::kInt64LessThan: {
568 LowerComparison(node, machine()->Int32LessThan(),
569 machine()->Uint32LessThan());
570 break;
571 }
572 case IrOpcode::kInt64LessThanOrEqual: {
573 LowerComparison(node, machine()->Int32LessThan(),
574 machine()->Uint32LessThanOrEqual());
575 break;
576 }
577 case IrOpcode::kUint64LessThan: {
578 LowerComparison(node, machine()->Uint32LessThan(),
579 machine()->Uint32LessThan());
580 break;
581 }
582 case IrOpcode::kUint64LessThanOrEqual: {
583 LowerComparison(node, machine()->Uint32LessThan(),
584 machine()->Uint32LessThanOrEqual());
585 break;
586 }
587 case IrOpcode::kSignExtendWord32ToInt64:
588 case IrOpcode::kChangeInt32ToInt64: {
589 DCHECK_EQ(1, node->InputCount());
590 Node* input = node->InputAt(0);
591 if (HasReplacementLow(input)) {
592 input = GetReplacementLow(input);
593 }
594 // We use SAR to preserve the sign in the high word.
595 ReplaceNode(
596 node, input,
597 graph()->NewNode(machine()->Word32Sar(), input,
598 graph()->NewNode(common()->Int32Constant(31))));
599 node->NullAllInputs();
600 break;
601 }
602 case IrOpcode::kChangeUint32ToUint64: {
603 DCHECK_EQ(1, node->InputCount());
604 Node* input = node->InputAt(0);
605 if (HasReplacementLow(input)) {
606 input = GetReplacementLow(input);
607 }
608 ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
609 node->NullAllInputs();
610 break;
611 }
612 case IrOpcode::kBitcastInt64ToFloat64: {
613 DCHECK_EQ(1, node->InputCount());
614 Node* input = node->InputAt(0);
615 Node* stack_slot = graph()->NewNode(
616 machine()->StackSlot(MachineRepresentation::kWord64));
617
618 Node* store_high_word = graph()->NewNode(
619 machine()->Store(
620 StoreRepresentation(MachineRepresentation::kWord32,
621 WriteBarrierKind::kNoWriteBarrier)),
622 stack_slot,
623 graph()->NewNode(
624 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
625 GetReplacementHigh(input), graph()->start(), graph()->start());
626
627 Node* store_low_word = graph()->NewNode(
628 machine()->Store(
629 StoreRepresentation(MachineRepresentation::kWord32,
630 WriteBarrierKind::kNoWriteBarrier)),
631 stack_slot,
632 graph()->NewNode(
633 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
634 GetReplacementLow(input), store_high_word, graph()->start());
635
636 Node* load =
637 graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
638 graph()->NewNode(common()->Int32Constant(0)),
639 store_low_word, graph()->start());
640
641 ReplaceNode(node, load, nullptr);
642 break;
643 }
644 case IrOpcode::kBitcastFloat64ToInt64: {
645 DCHECK_EQ(1, node->InputCount());
646 Node* input = node->InputAt(0);
647 if (HasReplacementLow(input)) {
648 input = GetReplacementLow(input);
649 }
650 Node* stack_slot = graph()->NewNode(
651 machine()->StackSlot(MachineRepresentation::kWord64));
652 Node* store = graph()->NewNode(
653 machine()->Store(
654 StoreRepresentation(MachineRepresentation::kFloat64,
655 WriteBarrierKind::kNoWriteBarrier)),
656 stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
657 graph()->start(), graph()->start());
658
659 Node* high_node = graph()->NewNode(
660 machine()->Load(MachineType::Int32()), stack_slot,
661 graph()->NewNode(
662 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
663 store, graph()->start());
664
665 Node* low_node = graph()->NewNode(
666 machine()->Load(MachineType::Int32()), stack_slot,
667 graph()->NewNode(
668 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
669 store, graph()->start());
670 ReplaceNode(node, low_node, high_node);
671 break;
672 }
673 case IrOpcode::kWord64Ror: {
674 DCHECK_EQ(2, node->InputCount());
675 Node* input = node->InputAt(0);
676 Node* shift = HasReplacementLow(node->InputAt(1))
677 ? GetReplacementLow(node->InputAt(1))
678 : node->InputAt(1);
679 Int32Matcher m(shift);
680 if (m.HasValue()) {
681 // Precondition: 0 <= shift < 64.
682 int32_t shift_value = m.Value() & 0x3F;
683 if (shift_value == 0) {
684 ReplaceNode(node, GetReplacementLow(input),
685 GetReplacementHigh(input));
686 } else if (shift_value == 32) {
687 ReplaceNode(node, GetReplacementHigh(input),
688 GetReplacementLow(input));
689 } else {
690 Node* low_input;
691 Node* high_input;
692 if (shift_value < 32) {
693 low_input = GetReplacementLow(input);
694 high_input = GetReplacementHigh(input);
695 } else {
696 low_input = GetReplacementHigh(input);
697 high_input = GetReplacementLow(input);
698 }
699 int32_t masked_shift_value = shift_value & 0x1F;
700 Node* masked_shift =
701 graph()->NewNode(common()->Int32Constant(masked_shift_value));
702 Node* inv_shift = graph()->NewNode(
703 common()->Int32Constant(32 - masked_shift_value));
704
705 Node* low_node = graph()->NewNode(
706 machine()->Word32Or(),
707 graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
708 graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
709 Node* high_node = graph()->NewNode(
710 machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
711 high_input, masked_shift),
712 graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
713 ReplaceNode(node, low_node, high_node);
714 }
715 } else {
716 Node* safe_shift = shift;
717 if (!machine()->Word32ShiftIsSafe()) {
718 safe_shift =
719 graph()->NewNode(machine()->Word32And(), shift,
720 graph()->NewNode(common()->Int32Constant(0x1F)));
721 }
722
723 // By creating this bit-mask with SAR and SHL we do not have to deal
724 // with shift == 0 as a special case.
725 Node* inv_mask = graph()->NewNode(
726 machine()->Word32Shl(),
727 graph()->NewNode(machine()->Word32Sar(),
728 graph()->NewNode(common()->Int32Constant(
729 std::numeric_limits<int32_t>::min())),
730 safe_shift),
731 graph()->NewNode(common()->Int32Constant(1)));
732
733 Node* bit_mask =
734 graph()->NewNode(machine()->Word32Xor(), inv_mask,
735 graph()->NewNode(common()->Int32Constant(-1)));
736
737 // We have to mask the shift value for this comparison. If
738 // !machine()->Word32ShiftIsSafe() then the masking should already be
739 // part of the graph.
740 Node* masked_shift6 = shift;
741 if (machine()->Word32ShiftIsSafe()) {
742 masked_shift6 =
743 graph()->NewNode(machine()->Word32And(), shift,
744 graph()->NewNode(common()->Int32Constant(0x3F)));
745 }
746
747 Diamond lt32(
748 graph(), common(),
749 graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
750 graph()->NewNode(common()->Int32Constant(32))));
751
752 // The low word and the high word can be swapped either at the input or
753 // at the output. We swap the inputs so that shift does not have to be
754 // kept for so long in a register.
755 Node* input_low =
756 lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
757 GetReplacementHigh(input));
758 Node* input_high =
759 lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
760 GetReplacementLow(input));
761
762 Node* rotate_low =
763 graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
764 Node* rotate_high =
765 graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
766
767 Node* low_node = graph()->NewNode(
768 machine()->Word32Or(),
769 graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
770 graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
771
772 Node* high_node = graph()->NewNode(
773 machine()->Word32Or(),
774 graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
775 graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
776
777 ReplaceNode(node, low_node, high_node);
778 }
779 break;
780 }
781 case IrOpcode::kWord64Clz: {
782 DCHECK_EQ(1, node->InputCount());
783 Node* input = node->InputAt(0);
784 Diamond d(
785 graph(), common(),
786 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
787 graph()->NewNode(common()->Int32Constant(0))));
788
789 Node* low_node = d.Phi(
790 MachineRepresentation::kWord32,
791 graph()->NewNode(machine()->Int32Add(),
792 graph()->NewNode(machine()->Word32Clz(),
793 GetReplacementLow(input)),
794 graph()->NewNode(common()->Int32Constant(32))),
795 graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
796 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
797 break;
798 }
799 case IrOpcode::kWord64Ctz: {
800 DCHECK_EQ(1, node->InputCount());
801 DCHECK(machine()->Word32Ctz().IsSupported());
802 Node* input = node->InputAt(0);
803 Diamond d(
804 graph(), common(),
805 graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
806 graph()->NewNode(common()->Int32Constant(0))));
807 Node* low_node =
808 d.Phi(MachineRepresentation::kWord32,
809 graph()->NewNode(machine()->Int32Add(),
810 graph()->NewNode(machine()->Word32Ctz().op(),
811 GetReplacementHigh(input)),
812 graph()->NewNode(common()->Int32Constant(32))),
813 graph()->NewNode(machine()->Word32Ctz().op(),
814 GetReplacementLow(input)));
815 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
816 break;
817 }
818 case IrOpcode::kWord64Popcnt: {
819 DCHECK_EQ(1, node->InputCount());
820 Node* input = node->InputAt(0);
821 // We assume that a Word64Popcnt node only has been created if
822 // Word32Popcnt is actually supported.
823 DCHECK(machine()->Word32Popcnt().IsSupported());
824 ReplaceNode(node, graph()->NewNode(
825 machine()->Int32Add(),
826 graph()->NewNode(machine()->Word32Popcnt().op(),
827 GetReplacementLow(input)),
828 graph()->NewNode(machine()->Word32Popcnt().op(),
829 GetReplacementHigh(input))),
830 graph()->NewNode(common()->Int32Constant(0)));
831 break;
832 }
833 case IrOpcode::kPhi: {
834 MachineRepresentation rep = PhiRepresentationOf(node->op());
835 if (rep == MachineRepresentation::kWord64) {
836 // The replacement nodes have already been created, we only have to
837 // replace placeholder nodes.
838 Node* low_node = GetReplacementLow(node);
839 Node* high_node = GetReplacementHigh(node);
840 for (int i = 0; i < node->op()->ValueInputCount(); i++) {
841 low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
842 high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
843 }
844 } else {
845 DefaultLowering(node);
846 }
847 break;
848 }
849 case IrOpcode::kWord64ReverseBytes: {
850 Node* input = node->InputAt(0);
851 ReplaceNode(node,
852 graph()->NewNode(machine()->Word32ReverseBytes(),
853 GetReplacementHigh(input)),
854 graph()->NewNode(machine()->Word32ReverseBytes(),
855 GetReplacementLow(input)));
856 break;
857 }
858 case IrOpcode::kSignExtendWord8ToInt64: {
859 DCHECK_EQ(1, node->InputCount());
860 Node* input = node->InputAt(0);
861 if (HasReplacementLow(input)) {
862 input = GetReplacementLow(input);
863 }
864 // Sign extend low node to Int32
865 input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
866
867 // We use SAR to preserve the sign in the high word.
868 ReplaceNode(
869 node, input,
870 graph()->NewNode(machine()->Word32Sar(), input,
871 graph()->NewNode(common()->Int32Constant(31))));
872 node->NullAllInputs();
873 break;
874 }
875 case IrOpcode::kSignExtendWord16ToInt64: {
876 DCHECK_EQ(1, node->InputCount());
877 Node* input = node->InputAt(0);
878 if (HasReplacementLow(input)) {
879 input = GetReplacementLow(input);
880 }
881 // Sign extend low node to Int32
882 input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
883
884 // We use SAR to preserve the sign in the high word.
885 ReplaceNode(
886 node, input,
887 graph()->NewNode(machine()->Word32Sar(), input,
888 graph()->NewNode(common()->Int32Constant(31))));
889 node->NullAllInputs();
890 break;
891 }
892 case IrOpcode::kWord64AtomicLoad: {
893 DCHECK_EQ(4, node->InputCount());
894 MachineType type = AtomicOpType(node->op());
895 DefaultLowering(node, true);
896 if (type == MachineType::Uint64()) {
897 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
898 ReplaceNodeWithProjections(node);
899 } else {
900 NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
901 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
902 }
903 break;
904 }
905 case IrOpcode::kWord64AtomicStore: {
906 DCHECK_EQ(5, node->InputCount());
907 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
908 if (rep == MachineRepresentation::kWord64) {
909 LowerMemoryBaseAndIndex(node);
910 Node* value = node->InputAt(2);
911 node->ReplaceInput(2, GetReplacementLow(value));
912 node->InsertInput(zone(), 3, GetReplacementHigh(value));
913 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
914 } else {
915 DefaultLowering(node, true);
916 NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
917 }
918 break;
919 }
920 #define ATOMIC_CASE(name) \
921 case IrOpcode::kWord64Atomic##name: { \
922 MachineType type = AtomicOpType(node->op()); \
923 if (type == MachineType::Uint64()) { \
924 LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \
925 } else { \
926 LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
927 } \
928 break; \
929 }
930 ATOMIC_CASE(Add)
931 ATOMIC_CASE(Sub)
932 ATOMIC_CASE(And)
933 ATOMIC_CASE(Or)
934 ATOMIC_CASE(Xor)
935 ATOMIC_CASE(Exchange)
936 #undef ATOMIC_CASE
937 case IrOpcode::kWord64AtomicCompareExchange: {
938 MachineType type = AtomicOpType(node->op());
939 if (type == MachineType::Uint64()) {
940 LowerMemoryBaseAndIndex(node);
941 Node* old_value = node->InputAt(2);
942 Node* new_value = node->InputAt(3);
943 node->ReplaceInput(2, GetReplacementLow(old_value));
944 node->ReplaceInput(3, GetReplacementHigh(old_value));
945 node->InsertInput(zone(), 4, GetReplacementLow(new_value));
946 node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
947 NodeProperties::ChangeOp(node,
948 machine()->Word32AtomicPairCompareExchange());
949 ReplaceNodeWithProjections(node);
950 } else {
951 DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
952 type == MachineType::Uint8());
953 DefaultLowering(node, true);
954 NodeProperties::ChangeOp(node,
955 machine()->Word32AtomicCompareExchange(type));
956 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
957 }
958 break;
959 }
960 case IrOpcode::kI64x2Splat: {
961 DCHECK_EQ(1, node->InputCount());
962 Node* input = node->InputAt(0);
963 node->ReplaceInput(0, GetReplacementLow(input));
964 node->AppendInput(zone(), GetReplacementHigh(input));
965 NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
966 break;
967 }
968 case IrOpcode::kI64x2ExtractLane: {
969 DCHECK_EQ(1, node->InputCount());
970 Node* input = node->InputAt(0);
971 int32_t lane = OpParameter<int32_t>(node->op());
972 ReplaceNode(
973 node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
974 graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
975 break;
976 }
977 case IrOpcode::kI64x2ReplaceLane: {
978 DCHECK_EQ(2, node->InputCount());
979 int32_t lane = OpParameter<int32_t>(node->op());
980 Node* input = node->InputAt(1);
981 node->ReplaceInput(1, GetReplacementLow(input));
982 node->AppendInput(zone(), GetReplacementHigh(input));
983 NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
984 break;
985 }
986
987 default: { DefaultLowering(node); }
988 }
989 } // NOLINT(readability/fn_size)
990
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)991 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
992 const Operator* low_word_op) {
993 DCHECK_EQ(2, node->InputCount());
994 Node* left = node->InputAt(0);
995 Node* right = node->InputAt(1);
996 Node* replacement = graph()->NewNode(
997 machine()->Word32Or(),
998 graph()->NewNode(high_word_op, GetReplacementHigh(left),
999 GetReplacementHigh(right)),
1000 graph()->NewNode(
1001 machine()->Word32And(),
1002 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
1003 GetReplacementHigh(right)),
1004 graph()->NewNode(low_word_op, GetReplacementLow(left),
1005 GetReplacementLow(right))));
1006
1007 ReplaceNode(node, replacement, nullptr);
1008 }
1009
DefaultLowering(Node * node,bool low_word_only)1010 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
1011 bool something_changed = false;
1012 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1013 Node* input = node->InputAt(i);
1014 if (HasReplacementLow(input)) {
1015 something_changed = true;
1016 node->ReplaceInput(i, GetReplacementLow(input));
1017 }
1018 if (!low_word_only && HasReplacementHigh(input)) {
1019 something_changed = true;
1020 node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
1021 }
1022 }
1023 return something_changed;
1024 }
1025
LowerCallDescriptor(const CallDescriptor * call_descriptor)1026 const CallDescriptor* Int64Lowering::LowerCallDescriptor(
1027 const CallDescriptor* call_descriptor) {
1028 if (special_case_) {
1029 auto replacement = special_case_->replacements.find(call_descriptor);
1030 if (replacement != special_case_->replacements.end()) {
1031 return replacement->second;
1032 }
1033 }
1034 return GetI32WasmCallDescriptor(zone(), call_descriptor);
1035 }
1036
ReplaceNode(Node * old,Node * new_low,Node * new_high)1037 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
1038 // if new_low == nullptr, then also new_high == nullptr.
1039 DCHECK(new_low != nullptr || new_high == nullptr);
1040 replacements_[old->id()].low = new_low;
1041 replacements_[old->id()].high = new_high;
1042 }
1043
HasReplacementLow(Node * node)1044 bool Int64Lowering::HasReplacementLow(Node* node) {
1045 return replacements_[node->id()].low != nullptr;
1046 }
1047
GetReplacementLow(Node * node)1048 Node* Int64Lowering::GetReplacementLow(Node* node) {
1049 Node* result = replacements_[node->id()].low;
1050 DCHECK(result);
1051 return result;
1052 }
1053
HasReplacementHigh(Node * node)1054 bool Int64Lowering::HasReplacementHigh(Node* node) {
1055 return replacements_[node->id()].high != nullptr;
1056 }
1057
GetReplacementHigh(Node * node)1058 Node* Int64Lowering::GetReplacementHigh(Node* node) {
1059 Node* result = replacements_[node->id()].high;
1060 DCHECK(result);
1061 return result;
1062 }
1063
PreparePhiReplacement(Node * phi)1064 void Int64Lowering::PreparePhiReplacement(Node* phi) {
1065 MachineRepresentation rep = PhiRepresentationOf(phi->op());
1066 if (rep == MachineRepresentation::kWord64) {
1067 // We have to create the replacements for a phi node before we actually
1068 // lower the phi to break potential cycles in the graph. The replacements of
1069 // input nodes do not exist yet, so we use a placeholder node to pass the
1070 // graph verifier.
1071 int value_count = phi->op()->ValueInputCount();
1072 Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1073 Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1074 for (int i = 0; i < value_count; i++) {
1075 inputs_low[i] = placeholder_;
1076 inputs_high[i] = placeholder_;
1077 }
1078 inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1079 inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1080 ReplaceNode(phi,
1081 graph()->NewNode(
1082 common()->Phi(MachineRepresentation::kWord32, value_count),
1083 value_count + 1, inputs_low, false),
1084 graph()->NewNode(
1085 common()->Phi(MachineRepresentation::kWord32, value_count),
1086 value_count + 1, inputs_high, false));
1087 }
1088 }
1089
ReplaceNodeWithProjections(Node * node)1090 void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1091 DCHECK(node != nullptr);
1092 Node* low_node =
1093 graph()->NewNode(common()->Projection(0), node, graph()->start());
1094 Node* high_node =
1095 graph()->NewNode(common()->Projection(1), node, graph()->start());
1096 ReplaceNode(node, low_node, high_node);
1097 }
1098
LowerMemoryBaseAndIndex(Node * node)1099 void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
1100 DCHECK(node != nullptr);
1101 // Low word only replacements for memory operands for 32-bit address space.
1102 Node* base = node->InputAt(0);
1103 Node* index = node->InputAt(1);
1104 if (HasReplacementLow(base)) {
1105 node->ReplaceInput(0, GetReplacementLow(base));
1106 }
1107 if (HasReplacementLow(index)) {
1108 node->ReplaceInput(1, GetReplacementLow(index));
1109 }
1110 }
1111
1112 } // namespace compiler
1113 } // namespace internal
1114 } // namespace v8
1115