1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/int64-lowering.h"
6 
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19 
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23 
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature,std::unique_ptr<Int64LoweringSpecialCase> special_case)24 Int64Lowering::Int64Lowering(
25     Graph* graph, MachineOperatorBuilder* machine,
26     CommonOperatorBuilder* common, Zone* zone,
27     Signature<MachineRepresentation>* signature,
28     std::unique_ptr<Int64LoweringSpecialCase> special_case)
29     : zone_(zone),
30       graph_(graph),
31       machine_(machine),
32       common_(common),
33       state_(graph, 3),
34       stack_(zone),
35       replacements_(nullptr),
36       signature_(signature),
37       placeholder_(
38           graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
39       special_case_(std::move(special_case)) {
40   DCHECK_NOT_NULL(graph);
41   DCHECK_NOT_NULL(graph->end());
42   replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
43   memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
44 }
45 
LowerGraph()46 void Int64Lowering::LowerGraph() {
47   if (!machine()->Is32()) {
48     return;
49   }
50   stack_.push_back({graph()->end(), 0});
51   state_.Set(graph()->end(), State::kOnStack);
52 
53   while (!stack_.empty()) {
54     NodeState& top = stack_.back();
55     if (top.input_index == top.node->InputCount()) {
56       // All inputs of top have already been lowered, now lower top.
57       stack_.pop_back();
58       state_.Set(top.node, State::kVisited);
59       LowerNode(top.node);
60     } else {
61       // Push the next input onto the stack.
62       Node* input = top.node->InputAt(top.input_index++);
63       if (state_.Get(input) == State::kUnvisited) {
64         if (input->opcode() == IrOpcode::kPhi) {
65           // To break cycles with phi nodes we push phis on a separate stack so
66           // that they are processed after all other nodes.
67           PreparePhiReplacement(input);
68           stack_.push_front({input, 0});
69         } else if (input->opcode() == IrOpcode::kEffectPhi ||
70                    input->opcode() == IrOpcode::kLoop) {
71           stack_.push_front({input, 0});
72         } else {
73           stack_.push_back({input, 0});
74         }
75         state_.Set(input, State::kOnStack);
76       }
77     }
78   }
79 }
80 
81 namespace {
82 
GetReturnIndexAfterLowering(const CallDescriptor * call_descriptor,int old_index)83 int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
84                                 int old_index) {
85   int result = old_index;
86   for (int i = 0; i < old_index; i++) {
87     if (call_descriptor->GetReturnType(i).representation() ==
88         MachineRepresentation::kWord64) {
89       result++;
90     }
91   }
92   return result;
93 }
94 
GetReturnCountAfterLowering(const CallDescriptor * call_descriptor)95 int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
96   return GetReturnIndexAfterLowering(
97       call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
98 }
99 
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)100 int GetParameterIndexAfterLowering(
101     Signature<MachineRepresentation>* signature, int old_index) {
102   int result = old_index;
103   // Be robust towards special indexes (>= param count).
104   int max_to_check =
105       std::min(old_index, static_cast<int>(signature->parameter_count()));
106   for (int i = 0; i < max_to_check; i++) {
107     if (signature->GetParam(i) == MachineRepresentation::kWord64) {
108       result++;
109     }
110   }
111   return result;
112 }
113 
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)114 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
115   int result = static_cast<int>(signature->return_count());
116   for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
117     if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
118       result++;
119     }
120   }
121   return result;
122 }
123 
124 }  // namespace
125 
LowerWord64AtomicBinop(Node * node,const Operator * op)126 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
127   DCHECK_EQ(5, node->InputCount());
128   LowerMemoryBaseAndIndex(node);
129   Node* value = node->InputAt(2);
130   node->ReplaceInput(2, GetReplacementLow(value));
131   node->InsertInput(zone(), 3, GetReplacementHigh(value));
132   NodeProperties::ChangeOp(node, op);
133   ReplaceNodeWithProjections(node);
134 }
135 
LowerWord64AtomicNarrowOp(Node * node,const Operator * op)136 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
137   DefaultLowering(node, true);
138   NodeProperties::ChangeOp(node, op);
139   ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
140 }
141 
142 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)143 int Int64Lowering::GetParameterCountAfterLowering(
144     Signature<MachineRepresentation>* signature) {
145   // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
146   // after lowering.
147   return GetParameterIndexAfterLowering(
148       signature, static_cast<int>(signature->parameter_count()));
149 }
150 
GetIndexNodes(Node * index,Node ** index_low,Node ** index_high)151 void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
152                                   Node** index_high) {
153 #if defined(V8_TARGET_LITTLE_ENDIAN)
154   *index_low = index;
155   *index_high = graph()->NewNode(machine()->Int32Add(), index,
156                                  graph()->NewNode(common()->Int32Constant(4)));
157 #elif defined(V8_TARGET_BIG_ENDIAN)
158   *index_low = graph()->NewNode(machine()->Int32Add(), index,
159                                 graph()->NewNode(common()->Int32Constant(4)));
160   *index_high = index;
161 #endif
162 }
163 
LowerNode(Node * node)164 void Int64Lowering::LowerNode(Node* node) {
165   switch (node->opcode()) {
166     case IrOpcode::kInt64Constant: {
167       int64_t value = OpParameter<int64_t>(node->op());
168       Node* low_node = graph()->NewNode(
169           common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
170       Node* high_node = graph()->NewNode(
171           common()->Int32Constant(static_cast<int32_t>(value >> 32)));
172       ReplaceNode(node, low_node, high_node);
173       break;
174     }
175     case IrOpcode::kLoad:
176     case IrOpcode::kUnalignedLoad: {
177       MachineRepresentation rep;
178       if (node->opcode() == IrOpcode::kLoad) {
179         rep = LoadRepresentationOf(node->op()).representation();
180       } else {
181         DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
182         rep = LoadRepresentationOf(node->op()).representation();
183       }
184 
185       if (rep == MachineRepresentation::kWord64) {
186         LowerMemoryBaseAndIndex(node);
187         Node* base = node->InputAt(0);
188         Node* index = node->InputAt(1);
189         Node* index_low;
190         Node* index_high;
191         GetIndexNodes(index, &index_low, &index_high);
192         const Operator* load_op;
193 
194         if (node->opcode() == IrOpcode::kLoad) {
195           load_op = machine()->Load(MachineType::Int32());
196         } else {
197           DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
198           load_op = machine()->UnalignedLoad(MachineType::Int32());
199         }
200 
201         Node* high_node;
202         if (node->InputCount() > 2) {
203           Node* effect_high = node->InputAt(2);
204           Node* control_high = node->InputAt(3);
205           high_node = graph()->NewNode(load_op, base, index_high, effect_high,
206                                        control_high);
207           // change the effect change from old_node --> old_effect to
208           // old_node --> high_node --> old_effect.
209           node->ReplaceInput(2, high_node);
210         } else {
211           high_node = graph()->NewNode(load_op, base, index_high);
212         }
213         node->ReplaceInput(1, index_low);
214         NodeProperties::ChangeOp(node, load_op);
215         ReplaceNode(node, node, high_node);
216       } else {
217         DefaultLowering(node);
218       }
219       break;
220     }
221     case IrOpcode::kStore:
222     case IrOpcode::kUnalignedStore: {
223       MachineRepresentation rep;
224       if (node->opcode() == IrOpcode::kStore) {
225         rep = StoreRepresentationOf(node->op()).representation();
226       } else {
227         DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
228         rep = UnalignedStoreRepresentationOf(node->op());
229       }
230 
231       if (rep == MachineRepresentation::kWord64) {
232         // We change the original store node to store the low word, and create
233         // a new store node to store the high word. The effect and control edges
234         // are copied from the original store to the new store node, the effect
235         // edge of the original store is redirected to the new store.
236         LowerMemoryBaseAndIndex(node);
237         Node* base = node->InputAt(0);
238         Node* index = node->InputAt(1);
239         Node* index_low;
240         Node* index_high;
241         GetIndexNodes(index, &index_low, &index_high);
242         Node* value = node->InputAt(2);
243         DCHECK(HasReplacementLow(value));
244         DCHECK(HasReplacementHigh(value));
245 
246         const Operator* store_op;
247         if (node->opcode() == IrOpcode::kStore) {
248           WriteBarrierKind write_barrier_kind =
249               StoreRepresentationOf(node->op()).write_barrier_kind();
250           store_op = machine()->Store(StoreRepresentation(
251               MachineRepresentation::kWord32, write_barrier_kind));
252         } else {
253           DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
254           store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
255         }
256 
257         Node* high_node;
258         if (node->InputCount() > 3) {
259           Node* effect_high = node->InputAt(3);
260           Node* control_high = node->InputAt(4);
261           high_node = graph()->NewNode(store_op, base, index_high,
262                                        GetReplacementHigh(value), effect_high,
263                                        control_high);
264           node->ReplaceInput(3, high_node);
265 
266         } else {
267           high_node = graph()->NewNode(store_op, base, index_high,
268                                        GetReplacementHigh(value));
269         }
270 
271         node->ReplaceInput(1, index_low);
272         node->ReplaceInput(2, GetReplacementLow(value));
273         NodeProperties::ChangeOp(node, store_op);
274         ReplaceNode(node, node, high_node);
275       } else {
276         DefaultLowering(node, true);
277       }
278       break;
279     }
280     case IrOpcode::kStart: {
281       int parameter_count = GetParameterCountAfterLowering(signature());
282       // Only exchange the node if the parameter count actually changed.
283       if (parameter_count != static_cast<int>(signature()->parameter_count())) {
284         int delta =
285             parameter_count - static_cast<int>(signature()->parameter_count());
286         int new_output_count = node->op()->ValueOutputCount() + delta;
287         NodeProperties::ChangeOp(node, common()->Start(new_output_count));
288       }
289       break;
290     }
291     case IrOpcode::kParameter: {
292       DCHECK_EQ(1, node->InputCount());
293       int param_count = static_cast<int>(signature()->parameter_count());
294       // Only exchange the node if the parameter count actually changed. We do
295       // not even have to do the default lowering because the the start node,
296       // the only input of a parameter node, only changes if the parameter count
297       // changes.
298       if (GetParameterCountAfterLowering(signature()) != param_count) {
299         int old_index = ParameterIndexOf(node->op());
300         // Adjust old_index to be compliant with the signature.
301         --old_index;
302         int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303         // Adjust new_index to consider the instance parameter.
304         ++new_index;
305         NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306 
307         if (old_index < 0 || old_index >= param_count) {
308           // Special parameters (JS closure/context) don't have kWord64
309           // representation anyway.
310           break;
311         }
312 
313         if (signature()->GetParam(old_index) ==
314             MachineRepresentation::kWord64) {
315           Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
316                                              graph()->start());
317           ReplaceNode(node, node, high_node);
318         }
319       }
320       break;
321     }
322     case IrOpcode::kReturn: {
323       int input_count = node->InputCount();
324       DefaultLowering(node);
325       if (input_count != node->InputCount()) {
326         int new_return_count = GetReturnCountAfterLowering(signature());
327         if (static_cast<int>(signature()->return_count()) != new_return_count) {
328           NodeProperties::ChangeOp(node, common()->Return(new_return_count));
329         }
330       }
331       break;
332     }
333     case IrOpcode::kTailCall: {
334       auto call_descriptor =
335           const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
336       bool returns_require_lowering =
337           GetReturnCountAfterLowering(call_descriptor) !=
338           static_cast<int>(call_descriptor->ReturnCount());
339       if (DefaultLowering(node) || returns_require_lowering) {
340         // Tail calls do not have return values, so adjusting the call
341         // descriptor is enough.
342         NodeProperties::ChangeOp(
343             node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
344       }
345       break;
346     }
347     case IrOpcode::kCall: {
348       auto call_descriptor = CallDescriptorOf(node->op());
349 
350       bool returns_require_lowering =
351           GetReturnCountAfterLowering(call_descriptor) !=
352           static_cast<int>(call_descriptor->ReturnCount());
353       if (DefaultLowering(node) || returns_require_lowering) {
354         // We have to adjust the call descriptor.
355         NodeProperties::ChangeOp(
356             node, common()->Call(LowerCallDescriptor(call_descriptor)));
357       }
358       if (returns_require_lowering) {
359         size_t return_arity = call_descriptor->ReturnCount();
360         if (return_arity == 1) {
361           // We access the additional return values through projections.
362           ReplaceNodeWithProjections(node);
363         } else {
364           ZoneVector<Node*> projections(return_arity, zone());
365           NodeProperties::CollectValueProjections(node, projections.data(),
366                                                   return_arity);
367           for (size_t old_index = 0, new_index = 0; old_index < return_arity;
368                ++old_index, ++new_index) {
369             Node* use_node = projections[old_index];
370             DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
371             DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
372                                                   static_cast<int>(old_index)),
373                       static_cast<int>(new_index));
374             if (new_index != old_index) {
375               NodeProperties::ChangeOp(
376                   use_node, common()->Projection(new_index));
377             }
378             if (call_descriptor->GetReturnType(old_index).representation() ==
379                 MachineRepresentation::kWord64) {
380               Node* high_node = graph()->NewNode(
381                   common()->Projection(new_index + 1), node,
382                   graph()->start());
383               ReplaceNode(use_node, use_node, high_node);
384               ++new_index;
385             }
386           }
387         }
388       }
389       break;
390     }
391     case IrOpcode::kWord64And: {
392       DCHECK_EQ(2, node->InputCount());
393       Node* left = node->InputAt(0);
394       Node* right = node->InputAt(1);
395 
396       Node* low_node =
397           graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
398                            GetReplacementLow(right));
399       Node* high_node =
400           graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
401                            GetReplacementHigh(right));
402       ReplaceNode(node, low_node, high_node);
403       break;
404     }
405     case IrOpcode::kTruncateInt64ToInt32: {
406       DCHECK_EQ(1, node->InputCount());
407       Node* input = node->InputAt(0);
408       ReplaceNode(node, GetReplacementLow(input), nullptr);
409       node->NullAllInputs();
410       break;
411     }
412     case IrOpcode::kInt64Add: {
413       DCHECK_EQ(2, node->InputCount());
414 
415       Node* right = node->InputAt(1);
416       node->ReplaceInput(1, GetReplacementLow(right));
417       node->AppendInput(zone(), GetReplacementHigh(right));
418 
419       Node* left = node->InputAt(0);
420       node->ReplaceInput(0, GetReplacementLow(left));
421       node->InsertInput(zone(), 1, GetReplacementHigh(left));
422 
423       NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
424       // We access the additional return values through projections.
425       ReplaceNodeWithProjections(node);
426       break;
427     }
428     case IrOpcode::kInt64Sub: {
429       DCHECK_EQ(2, node->InputCount());
430 
431       Node* right = node->InputAt(1);
432       node->ReplaceInput(1, GetReplacementLow(right));
433       node->AppendInput(zone(), GetReplacementHigh(right));
434 
435       Node* left = node->InputAt(0);
436       node->ReplaceInput(0, GetReplacementLow(left));
437       node->InsertInput(zone(), 1, GetReplacementHigh(left));
438 
439       NodeProperties::ChangeOp(node, machine()->Int32PairSub());
440       // We access the additional return values through projections.
441       ReplaceNodeWithProjections(node);
442       break;
443     }
444     case IrOpcode::kInt64Mul: {
445       DCHECK_EQ(2, node->InputCount());
446 
447       Node* right = node->InputAt(1);
448       node->ReplaceInput(1, GetReplacementLow(right));
449       node->AppendInput(zone(), GetReplacementHigh(right));
450 
451       Node* left = node->InputAt(0);
452       node->ReplaceInput(0, GetReplacementLow(left));
453       node->InsertInput(zone(), 1, GetReplacementHigh(left));
454 
455       NodeProperties::ChangeOp(node, machine()->Int32PairMul());
456       // We access the additional return values through projections.
457       ReplaceNodeWithProjections(node);
458       break;
459     }
460     case IrOpcode::kWord64Or: {
461       DCHECK_EQ(2, node->InputCount());
462       Node* left = node->InputAt(0);
463       Node* right = node->InputAt(1);
464 
465       Node* low_node =
466           graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
467                            GetReplacementLow(right));
468       Node* high_node =
469           graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
470                            GetReplacementHigh(right));
471       ReplaceNode(node, low_node, high_node);
472       break;
473     }
474     case IrOpcode::kWord64Xor: {
475       DCHECK_EQ(2, node->InputCount());
476       Node* left = node->InputAt(0);
477       Node* right = node->InputAt(1);
478 
479       Node* low_node =
480           graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
481                            GetReplacementLow(right));
482       Node* high_node =
483           graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
484                            GetReplacementHigh(right));
485       ReplaceNode(node, low_node, high_node);
486       break;
487     }
488     case IrOpcode::kWord64Shl: {
489       // TODO(turbofan): if the shift count >= 32, then we can set the low word
490       // of the output to 0 and just calculate the high word.
491       DCHECK_EQ(2, node->InputCount());
492       Node* shift = node->InputAt(1);
493       if (HasReplacementLow(shift)) {
494         // We do not have to care about the high word replacement, because
495         // the shift can only be between 0 and 63 anyways.
496         node->ReplaceInput(1, GetReplacementLow(shift));
497       }
498 
499       Node* value = node->InputAt(0);
500       node->ReplaceInput(0, GetReplacementLow(value));
501       node->InsertInput(zone(), 1, GetReplacementHigh(value));
502 
503       NodeProperties::ChangeOp(node, machine()->Word32PairShl());
504       // We access the additional return values through projections.
505       ReplaceNodeWithProjections(node);
506       break;
507     }
508     case IrOpcode::kWord64Shr: {
509       // TODO(turbofan): if the shift count >= 32, then we can set the low word
510       // of the output to 0 and just calculate the high word.
511       DCHECK_EQ(2, node->InputCount());
512       Node* shift = node->InputAt(1);
513       if (HasReplacementLow(shift)) {
514         // We do not have to care about the high word replacement, because
515         // the shift can only be between 0 and 63 anyways.
516         node->ReplaceInput(1, GetReplacementLow(shift));
517       }
518 
519       Node* value = node->InputAt(0);
520       node->ReplaceInput(0, GetReplacementLow(value));
521       node->InsertInput(zone(), 1, GetReplacementHigh(value));
522 
523       NodeProperties::ChangeOp(node, machine()->Word32PairShr());
524       // We access the additional return values through projections.
525       ReplaceNodeWithProjections(node);
526       break;
527     }
528     case IrOpcode::kWord64Sar: {
529       // TODO(turbofan): if the shift count >= 32, then we can set the low word
530       // of the output to 0 and just calculate the high word.
531       DCHECK_EQ(2, node->InputCount());
532       Node* shift = node->InputAt(1);
533       if (HasReplacementLow(shift)) {
534         // We do not have to care about the high word replacement, because
535         // the shift can only be between 0 and 63 anyways.
536         node->ReplaceInput(1, GetReplacementLow(shift));
537       }
538 
539       Node* value = node->InputAt(0);
540       node->ReplaceInput(0, GetReplacementLow(value));
541       node->InsertInput(zone(), 1, GetReplacementHigh(value));
542 
543       NodeProperties::ChangeOp(node, machine()->Word32PairSar());
544       // We access the additional return values through projections.
545       ReplaceNodeWithProjections(node);
546       break;
547     }
548     case IrOpcode::kWord64Equal: {
549       DCHECK_EQ(2, node->InputCount());
550       Node* left = node->InputAt(0);
551       Node* right = node->InputAt(1);
552 
553       // TODO(wasm): Use explicit comparisons and && here?
554       Node* replacement = graph()->NewNode(
555           machine()->Word32Equal(),
556           graph()->NewNode(
557               machine()->Word32Or(),
558               graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
559                                GetReplacementLow(right)),
560               graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
561                                GetReplacementHigh(right))),
562           graph()->NewNode(common()->Int32Constant(0)));
563 
564       ReplaceNode(node, replacement, nullptr);
565       break;
566     }
567     case IrOpcode::kInt64LessThan: {
568       LowerComparison(node, machine()->Int32LessThan(),
569                       machine()->Uint32LessThan());
570       break;
571     }
572     case IrOpcode::kInt64LessThanOrEqual: {
573       LowerComparison(node, machine()->Int32LessThan(),
574                       machine()->Uint32LessThanOrEqual());
575       break;
576     }
577     case IrOpcode::kUint64LessThan: {
578       LowerComparison(node, machine()->Uint32LessThan(),
579                       machine()->Uint32LessThan());
580       break;
581     }
582     case IrOpcode::kUint64LessThanOrEqual: {
583       LowerComparison(node, machine()->Uint32LessThan(),
584                       machine()->Uint32LessThanOrEqual());
585       break;
586     }
587     case IrOpcode::kSignExtendWord32ToInt64:
588     case IrOpcode::kChangeInt32ToInt64: {
589       DCHECK_EQ(1, node->InputCount());
590       Node* input = node->InputAt(0);
591       if (HasReplacementLow(input)) {
592         input = GetReplacementLow(input);
593       }
594       // We use SAR to preserve the sign in the high word.
595       ReplaceNode(
596           node, input,
597           graph()->NewNode(machine()->Word32Sar(), input,
598                            graph()->NewNode(common()->Int32Constant(31))));
599       node->NullAllInputs();
600       break;
601     }
602     case IrOpcode::kChangeUint32ToUint64: {
603       DCHECK_EQ(1, node->InputCount());
604       Node* input = node->InputAt(0);
605       if (HasReplacementLow(input)) {
606         input = GetReplacementLow(input);
607       }
608       ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
609       node->NullAllInputs();
610       break;
611     }
612     case IrOpcode::kBitcastInt64ToFloat64: {
613       DCHECK_EQ(1, node->InputCount());
614       Node* input = node->InputAt(0);
615       Node* stack_slot = graph()->NewNode(
616           machine()->StackSlot(MachineRepresentation::kWord64));
617 
618       Node* store_high_word = graph()->NewNode(
619           machine()->Store(
620               StoreRepresentation(MachineRepresentation::kWord32,
621                                   WriteBarrierKind::kNoWriteBarrier)),
622           stack_slot,
623           graph()->NewNode(
624               common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
625           GetReplacementHigh(input), graph()->start(), graph()->start());
626 
627       Node* store_low_word = graph()->NewNode(
628           machine()->Store(
629               StoreRepresentation(MachineRepresentation::kWord32,
630                                   WriteBarrierKind::kNoWriteBarrier)),
631           stack_slot,
632           graph()->NewNode(
633               common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
634           GetReplacementLow(input), store_high_word, graph()->start());
635 
636       Node* load =
637           graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
638                            graph()->NewNode(common()->Int32Constant(0)),
639                            store_low_word, graph()->start());
640 
641       ReplaceNode(node, load, nullptr);
642       break;
643     }
644     case IrOpcode::kBitcastFloat64ToInt64: {
645       DCHECK_EQ(1, node->InputCount());
646       Node* input = node->InputAt(0);
647       if (HasReplacementLow(input)) {
648         input = GetReplacementLow(input);
649       }
650       Node* stack_slot = graph()->NewNode(
651           machine()->StackSlot(MachineRepresentation::kWord64));
652       Node* store = graph()->NewNode(
653           machine()->Store(
654               StoreRepresentation(MachineRepresentation::kFloat64,
655                                   WriteBarrierKind::kNoWriteBarrier)),
656           stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
657           graph()->start(), graph()->start());
658 
659       Node* high_node = graph()->NewNode(
660           machine()->Load(MachineType::Int32()), stack_slot,
661           graph()->NewNode(
662               common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
663           store, graph()->start());
664 
665       Node* low_node = graph()->NewNode(
666           machine()->Load(MachineType::Int32()), stack_slot,
667           graph()->NewNode(
668               common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
669           store, graph()->start());
670       ReplaceNode(node, low_node, high_node);
671       break;
672     }
673     case IrOpcode::kWord64Ror: {
674       DCHECK_EQ(2, node->InputCount());
675       Node* input = node->InputAt(0);
676       Node* shift = HasReplacementLow(node->InputAt(1))
677                         ? GetReplacementLow(node->InputAt(1))
678                         : node->InputAt(1);
679       Int32Matcher m(shift);
680       if (m.HasValue()) {
681         // Precondition: 0 <= shift < 64.
682         int32_t shift_value = m.Value() & 0x3F;
683         if (shift_value == 0) {
684           ReplaceNode(node, GetReplacementLow(input),
685                       GetReplacementHigh(input));
686         } else if (shift_value == 32) {
687           ReplaceNode(node, GetReplacementHigh(input),
688                       GetReplacementLow(input));
689         } else {
690           Node* low_input;
691           Node* high_input;
692           if (shift_value < 32) {
693             low_input = GetReplacementLow(input);
694             high_input = GetReplacementHigh(input);
695           } else {
696             low_input = GetReplacementHigh(input);
697             high_input = GetReplacementLow(input);
698           }
699           int32_t masked_shift_value = shift_value & 0x1F;
700           Node* masked_shift =
701               graph()->NewNode(common()->Int32Constant(masked_shift_value));
702           Node* inv_shift = graph()->NewNode(
703               common()->Int32Constant(32 - masked_shift_value));
704 
705           Node* low_node = graph()->NewNode(
706               machine()->Word32Or(),
707               graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
708               graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
709           Node* high_node = graph()->NewNode(
710               machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
711                                                       high_input, masked_shift),
712               graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
713           ReplaceNode(node, low_node, high_node);
714         }
715       } else {
716         Node* safe_shift = shift;
717         if (!machine()->Word32ShiftIsSafe()) {
718           safe_shift =
719               graph()->NewNode(machine()->Word32And(), shift,
720                                graph()->NewNode(common()->Int32Constant(0x1F)));
721         }
722 
723         // By creating this bit-mask with SAR and SHL we do not have to deal
724         // with shift == 0 as a special case.
725         Node* inv_mask = graph()->NewNode(
726             machine()->Word32Shl(),
727             graph()->NewNode(machine()->Word32Sar(),
728                              graph()->NewNode(common()->Int32Constant(
729                                  std::numeric_limits<int32_t>::min())),
730                              safe_shift),
731             graph()->NewNode(common()->Int32Constant(1)));
732 
733         Node* bit_mask =
734             graph()->NewNode(machine()->Word32Xor(), inv_mask,
735                              graph()->NewNode(common()->Int32Constant(-1)));
736 
737         // We have to mask the shift value for this comparison. If
738         // !machine()->Word32ShiftIsSafe() then the masking should already be
739         // part of the graph.
740         Node* masked_shift6 = shift;
741         if (machine()->Word32ShiftIsSafe()) {
742           masked_shift6 =
743               graph()->NewNode(machine()->Word32And(), shift,
744                                graph()->NewNode(common()->Int32Constant(0x3F)));
745         }
746 
747         Diamond lt32(
748             graph(), common(),
749             graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
750                              graph()->NewNode(common()->Int32Constant(32))));
751 
752         // The low word and the high word can be swapped either at the input or
753         // at the output. We swap the inputs so that shift does not have to be
754         // kept for so long in a register.
755         Node* input_low =
756             lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
757                      GetReplacementHigh(input));
758         Node* input_high =
759             lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
760                      GetReplacementLow(input));
761 
762         Node* rotate_low =
763             graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
764         Node* rotate_high =
765             graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
766 
767         Node* low_node = graph()->NewNode(
768             machine()->Word32Or(),
769             graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
770             graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
771 
772         Node* high_node = graph()->NewNode(
773             machine()->Word32Or(),
774             graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
775             graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
776 
777         ReplaceNode(node, low_node, high_node);
778       }
779       break;
780     }
781     case IrOpcode::kWord64Clz: {
782       DCHECK_EQ(1, node->InputCount());
783       Node* input = node->InputAt(0);
784       Diamond d(
785           graph(), common(),
786           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
787                            graph()->NewNode(common()->Int32Constant(0))));
788 
789       Node* low_node = d.Phi(
790           MachineRepresentation::kWord32,
791           graph()->NewNode(machine()->Int32Add(),
792                            graph()->NewNode(machine()->Word32Clz(),
793                                             GetReplacementLow(input)),
794                            graph()->NewNode(common()->Int32Constant(32))),
795           graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
796       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
797       break;
798     }
799     case IrOpcode::kWord64Ctz: {
800       DCHECK_EQ(1, node->InputCount());
801       DCHECK(machine()->Word32Ctz().IsSupported());
802       Node* input = node->InputAt(0);
803       Diamond d(
804           graph(), common(),
805           graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
806                            graph()->NewNode(common()->Int32Constant(0))));
807       Node* low_node =
808           d.Phi(MachineRepresentation::kWord32,
809                 graph()->NewNode(machine()->Int32Add(),
810                                  graph()->NewNode(machine()->Word32Ctz().op(),
811                                                   GetReplacementHigh(input)),
812                                  graph()->NewNode(common()->Int32Constant(32))),
813                 graph()->NewNode(machine()->Word32Ctz().op(),
814                                  GetReplacementLow(input)));
815       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
816       break;
817     }
818     case IrOpcode::kWord64Popcnt: {
819       DCHECK_EQ(1, node->InputCount());
820       Node* input = node->InputAt(0);
821       // We assume that a Word64Popcnt node only has been created if
822       // Word32Popcnt is actually supported.
823       DCHECK(machine()->Word32Popcnt().IsSupported());
824       ReplaceNode(node, graph()->NewNode(
825                             machine()->Int32Add(),
826                             graph()->NewNode(machine()->Word32Popcnt().op(),
827                                              GetReplacementLow(input)),
828                             graph()->NewNode(machine()->Word32Popcnt().op(),
829                                              GetReplacementHigh(input))),
830                   graph()->NewNode(common()->Int32Constant(0)));
831       break;
832     }
833     case IrOpcode::kPhi: {
834       MachineRepresentation rep = PhiRepresentationOf(node->op());
835       if (rep == MachineRepresentation::kWord64) {
836         // The replacement nodes have already been created, we only have to
837         // replace placeholder nodes.
838         Node* low_node = GetReplacementLow(node);
839         Node* high_node = GetReplacementHigh(node);
840         for (int i = 0; i < node->op()->ValueInputCount(); i++) {
841           low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
842           high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
843         }
844       } else {
845         DefaultLowering(node);
846       }
847       break;
848     }
849     case IrOpcode::kWord64ReverseBytes: {
850       Node* input = node->InputAt(0);
851       ReplaceNode(node,
852                   graph()->NewNode(machine()->Word32ReverseBytes(),
853                                    GetReplacementHigh(input)),
854                   graph()->NewNode(machine()->Word32ReverseBytes(),
855                                    GetReplacementLow(input)));
856       break;
857     }
858     case IrOpcode::kSignExtendWord8ToInt64: {
859       DCHECK_EQ(1, node->InputCount());
860       Node* input = node->InputAt(0);
861       if (HasReplacementLow(input)) {
862         input = GetReplacementLow(input);
863       }
864       // Sign extend low node to Int32
865       input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
866 
867       // We use SAR to preserve the sign in the high word.
868       ReplaceNode(
869           node, input,
870           graph()->NewNode(machine()->Word32Sar(), input,
871                            graph()->NewNode(common()->Int32Constant(31))));
872       node->NullAllInputs();
873       break;
874     }
875     case IrOpcode::kSignExtendWord16ToInt64: {
876       DCHECK_EQ(1, node->InputCount());
877       Node* input = node->InputAt(0);
878       if (HasReplacementLow(input)) {
879         input = GetReplacementLow(input);
880       }
881       // Sign extend low node to Int32
882       input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
883 
884       // We use SAR to preserve the sign in the high word.
885       ReplaceNode(
886           node, input,
887           graph()->NewNode(machine()->Word32Sar(), input,
888                            graph()->NewNode(common()->Int32Constant(31))));
889       node->NullAllInputs();
890       break;
891     }
892     case IrOpcode::kWord64AtomicLoad: {
893       DCHECK_EQ(4, node->InputCount());
894       MachineType type = AtomicOpType(node->op());
895       DefaultLowering(node, true);
896       if (type == MachineType::Uint64()) {
897         NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
898         ReplaceNodeWithProjections(node);
899       } else {
900         NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
901         ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
902       }
903       break;
904     }
905     case IrOpcode::kWord64AtomicStore: {
906       DCHECK_EQ(5, node->InputCount());
907       MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
908       if (rep == MachineRepresentation::kWord64) {
909         LowerMemoryBaseAndIndex(node);
910         Node* value = node->InputAt(2);
911         node->ReplaceInput(2, GetReplacementLow(value));
912         node->InsertInput(zone(), 3, GetReplacementHigh(value));
913         NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
914       } else {
915         DefaultLowering(node, true);
916         NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
917       }
918       break;
919     }
920 #define ATOMIC_CASE(name)                                                   \
921   case IrOpcode::kWord64Atomic##name: {                                     \
922     MachineType type = AtomicOpType(node->op());                            \
923     if (type == MachineType::Uint64()) {                                    \
924       LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name());    \
925     } else {                                                                \
926       LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
927     }                                                                       \
928     break;                                                                  \
929   }
930       ATOMIC_CASE(Add)
931       ATOMIC_CASE(Sub)
932       ATOMIC_CASE(And)
933       ATOMIC_CASE(Or)
934       ATOMIC_CASE(Xor)
935       ATOMIC_CASE(Exchange)
936 #undef ATOMIC_CASE
937     case IrOpcode::kWord64AtomicCompareExchange: {
938       MachineType type = AtomicOpType(node->op());
939       if (type == MachineType::Uint64()) {
940         LowerMemoryBaseAndIndex(node);
941         Node* old_value = node->InputAt(2);
942         Node* new_value = node->InputAt(3);
943         node->ReplaceInput(2, GetReplacementLow(old_value));
944         node->ReplaceInput(3, GetReplacementHigh(old_value));
945         node->InsertInput(zone(), 4, GetReplacementLow(new_value));
946         node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
947         NodeProperties::ChangeOp(node,
948                                  machine()->Word32AtomicPairCompareExchange());
949         ReplaceNodeWithProjections(node);
950       } else {
951         DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
952                type == MachineType::Uint8());
953         DefaultLowering(node, true);
954         NodeProperties::ChangeOp(node,
955                                  machine()->Word32AtomicCompareExchange(type));
956         ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
957       }
958       break;
959     }
960     case IrOpcode::kI64x2Splat: {
961       DCHECK_EQ(1, node->InputCount());
962       Node* input = node->InputAt(0);
963       node->ReplaceInput(0, GetReplacementLow(input));
964       node->AppendInput(zone(), GetReplacementHigh(input));
965       NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
966       break;
967     }
968     case IrOpcode::kI64x2ExtractLane: {
969       DCHECK_EQ(1, node->InputCount());
970       Node* input = node->InputAt(0);
971       int32_t lane = OpParameter<int32_t>(node->op());
972       ReplaceNode(
973           node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
974           graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
975       break;
976     }
977     case IrOpcode::kI64x2ReplaceLane: {
978       DCHECK_EQ(2, node->InputCount());
979       int32_t lane = OpParameter<int32_t>(node->op());
980       Node* input = node->InputAt(1);
981       node->ReplaceInput(1, GetReplacementLow(input));
982       node->AppendInput(zone(), GetReplacementHigh(input));
983       NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
984       break;
985     }
986 
987     default: { DefaultLowering(node); }
988   }
989 }  // NOLINT(readability/fn_size)
990 
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)991 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
992                                     const Operator* low_word_op) {
993   DCHECK_EQ(2, node->InputCount());
994   Node* left = node->InputAt(0);
995   Node* right = node->InputAt(1);
996   Node* replacement = graph()->NewNode(
997       machine()->Word32Or(),
998       graph()->NewNode(high_word_op, GetReplacementHigh(left),
999                        GetReplacementHigh(right)),
1000       graph()->NewNode(
1001           machine()->Word32And(),
1002           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
1003                            GetReplacementHigh(right)),
1004           graph()->NewNode(low_word_op, GetReplacementLow(left),
1005                            GetReplacementLow(right))));
1006 
1007   ReplaceNode(node, replacement, nullptr);
1008 }
1009 
DefaultLowering(Node * node,bool low_word_only)1010 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
1011   bool something_changed = false;
1012   for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1013     Node* input = node->InputAt(i);
1014     if (HasReplacementLow(input)) {
1015       something_changed = true;
1016       node->ReplaceInput(i, GetReplacementLow(input));
1017     }
1018     if (!low_word_only && HasReplacementHigh(input)) {
1019       something_changed = true;
1020       node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
1021     }
1022   }
1023   return something_changed;
1024 }
1025 
LowerCallDescriptor(const CallDescriptor * call_descriptor)1026 const CallDescriptor* Int64Lowering::LowerCallDescriptor(
1027     const CallDescriptor* call_descriptor) {
1028   if (special_case_) {
1029     auto replacement = special_case_->replacements.find(call_descriptor);
1030     if (replacement != special_case_->replacements.end()) {
1031       return replacement->second;
1032     }
1033   }
1034   return GetI32WasmCallDescriptor(zone(), call_descriptor);
1035 }
1036 
ReplaceNode(Node * old,Node * new_low,Node * new_high)1037 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
1038   // if new_low == nullptr, then also new_high == nullptr.
1039   DCHECK(new_low != nullptr || new_high == nullptr);
1040   replacements_[old->id()].low = new_low;
1041   replacements_[old->id()].high = new_high;
1042 }
1043 
HasReplacementLow(Node * node)1044 bool Int64Lowering::HasReplacementLow(Node* node) {
1045   return replacements_[node->id()].low != nullptr;
1046 }
1047 
GetReplacementLow(Node * node)1048 Node* Int64Lowering::GetReplacementLow(Node* node) {
1049   Node* result = replacements_[node->id()].low;
1050   DCHECK(result);
1051   return result;
1052 }
1053 
HasReplacementHigh(Node * node)1054 bool Int64Lowering::HasReplacementHigh(Node* node) {
1055   return replacements_[node->id()].high != nullptr;
1056 }
1057 
GetReplacementHigh(Node * node)1058 Node* Int64Lowering::GetReplacementHigh(Node* node) {
1059   Node* result = replacements_[node->id()].high;
1060   DCHECK(result);
1061   return result;
1062 }
1063 
PreparePhiReplacement(Node * phi)1064 void Int64Lowering::PreparePhiReplacement(Node* phi) {
1065   MachineRepresentation rep = PhiRepresentationOf(phi->op());
1066   if (rep == MachineRepresentation::kWord64) {
1067     // We have to create the replacements for a phi node before we actually
1068     // lower the phi to break potential cycles in the graph. The replacements of
1069     // input nodes do not exist yet, so we use a placeholder node to pass the
1070     // graph verifier.
1071     int value_count = phi->op()->ValueInputCount();
1072     Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1073     Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1074     for (int i = 0; i < value_count; i++) {
1075       inputs_low[i] = placeholder_;
1076       inputs_high[i] = placeholder_;
1077     }
1078     inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1079     inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1080     ReplaceNode(phi,
1081                 graph()->NewNode(
1082                     common()->Phi(MachineRepresentation::kWord32, value_count),
1083                     value_count + 1, inputs_low, false),
1084                 graph()->NewNode(
1085                     common()->Phi(MachineRepresentation::kWord32, value_count),
1086                     value_count + 1, inputs_high, false));
1087   }
1088 }
1089 
ReplaceNodeWithProjections(Node * node)1090 void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1091   DCHECK(node != nullptr);
1092   Node* low_node =
1093       graph()->NewNode(common()->Projection(0), node, graph()->start());
1094   Node* high_node =
1095       graph()->NewNode(common()->Projection(1), node, graph()->start());
1096   ReplaceNode(node, low_node, high_node);
1097 }
1098 
LowerMemoryBaseAndIndex(Node * node)1099 void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
1100   DCHECK(node != nullptr);
1101   // Low word only replacements for memory operands for 32-bit address space.
1102   Node* base = node->InputAt(0);
1103   Node* index = node->InputAt(1);
1104   if (HasReplacementLow(base)) {
1105     node->ReplaceInput(0, GetReplacementLow(base));
1106   }
1107   if (HasReplacementLow(index)) {
1108     node->ReplaceInput(1, GetReplacementLow(index));
1109   }
1110 }
1111 
1112 }  // namespace compiler
1113 }  // namespace internal
1114 }  // namespace v8
1115