1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/int64-lowering.h"
6 
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19 
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23 
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature,std::unique_ptr<Int64LoweringSpecialCase> special_case)24 Int64Lowering::Int64Lowering(
25     Graph* graph, MachineOperatorBuilder* machine,
26     CommonOperatorBuilder* common, Zone* zone,
27     Signature<MachineRepresentation>* signature,
28     std::unique_ptr<Int64LoweringSpecialCase> special_case)
29     : zone_(zone),
30       graph_(graph),
31       machine_(machine),
32       common_(common),
33       state_(graph, 3),
34       stack_(zone),
35       replacements_(nullptr),
36       signature_(signature),
37       placeholder_(
38           graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
39       special_case_(std::move(special_case)) {
40   DCHECK_NOT_NULL(graph);
41   DCHECK_NOT_NULL(graph->end());
42   replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
43   memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
44 }
45 
LowerGraph()46 void Int64Lowering::LowerGraph() {
47   if (!machine()->Is32()) {
48     return;
49   }
50   stack_.push_back({graph()->end(), 0});
51   state_.Set(graph()->end(), State::kOnStack);
52 
53   while (!stack_.empty()) {
54     NodeState& top = stack_.back();
55     if (top.input_index == top.node->InputCount()) {
56       // All inputs of top have already been lowered, now lower top.
57       stack_.pop_back();
58       state_.Set(top.node, State::kVisited);
59       LowerNode(top.node);
60     } else {
61       // Push the next input onto the stack.
62       Node* input = top.node->InputAt(top.input_index++);
63       if (state_.Get(input) == State::kUnvisited) {
64         if (input->opcode() == IrOpcode::kPhi) {
65           // To break cycles with phi nodes we push phis on a separate stack so
66           // that they are processed after all other nodes.
67           PreparePhiReplacement(input);
68           stack_.push_front({input, 0});
69         } else if (input->opcode() == IrOpcode::kEffectPhi ||
70                    input->opcode() == IrOpcode::kLoop) {
71           stack_.push_front({input, 0});
72         } else {
73           stack_.push_back({input, 0});
74         }
75         state_.Set(input, State::kOnStack);
76       }
77     }
78   }
79 }
80 
81 namespace {
82 
GetReturnIndexAfterLowering(const CallDescriptor * call_descriptor,int old_index)83 int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
84                                 int old_index) {
85   int result = old_index;
86   for (int i = 0; i < old_index; i++) {
87     if (call_descriptor->GetReturnType(i).representation() ==
88         MachineRepresentation::kWord64) {
89       result++;
90     }
91   }
92   return result;
93 }
94 
GetReturnCountAfterLowering(const CallDescriptor * call_descriptor)95 int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
96   return GetReturnIndexAfterLowering(
97       call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
98 }
99 
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)100 int GetParameterIndexAfterLowering(
101     Signature<MachineRepresentation>* signature, int old_index) {
102   int result = old_index;
103   // Be robust towards special indexes (>= param count).
104   int max_to_check =
105       std::min(old_index, static_cast<int>(signature->parameter_count()));
106   for (int i = 0; i < max_to_check; i++) {
107     if (signature->GetParam(i) == MachineRepresentation::kWord64) {
108       result++;
109     }
110   }
111   return result;
112 }
113 
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)114 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
115   int result = static_cast<int>(signature->return_count());
116   for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
117     if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
118       result++;
119     }
120   }
121   return result;
122 }
123 
124 }  // namespace
125 
LowerWord64AtomicBinop(Node * node,const Operator * op)126 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
127   DCHECK_EQ(5, node->InputCount());
128   LowerMemoryBaseAndIndex(node);
129   Node* value = node->InputAt(2);
130   node->ReplaceInput(2, GetReplacementLow(value));
131   node->InsertInput(zone(), 3, GetReplacementHigh(value));
132   NodeProperties::ChangeOp(node, op);
133   ReplaceNodeWithProjections(node);
134 }
135 
LowerWord64AtomicNarrowOp(Node * node,const Operator * op)136 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
137   DefaultLowering(node, true);
138   NodeProperties::ChangeOp(node, op);
139   ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
140 }
141 
142 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)143 int Int64Lowering::GetParameterCountAfterLowering(
144     Signature<MachineRepresentation>* signature) {
145   // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
146   // after lowering.
147   return GetParameterIndexAfterLowering(
148       signature, static_cast<int>(signature->parameter_count()));
149 }
150 
GetIndexNodes(Node * index,Node ** index_low,Node ** index_high)151 void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
152                                   Node** index_high) {
153 #if defined(V8_TARGET_LITTLE_ENDIAN)
154   *index_low = index;
155   *index_high = graph()->NewNode(machine()->Int32Add(), index,
156                                  graph()->NewNode(common()->Int32Constant(4)));
157 #elif defined(V8_TARGET_BIG_ENDIAN)
158   *index_low = graph()->NewNode(machine()->Int32Add(), index,
159                                 graph()->NewNode(common()->Int32Constant(4)));
160   *index_high = index;
161 #endif
162 }
163 
LowerNode(Node * node)164 void Int64Lowering::LowerNode(Node* node) {
165   switch (node->opcode()) {
166     case IrOpcode::kInt64Constant: {
167       int64_t value = OpParameter<int64_t>(node->op());
168       Node* low_node = graph()->NewNode(
169           common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
170       Node* high_node = graph()->NewNode(
171           common()->Int32Constant(static_cast<int32_t>(value >> 32)));
172       ReplaceNode(node, low_node, high_node);
173       break;
174     }
175     case IrOpcode::kLoad:
176     case IrOpcode::kUnalignedLoad: {
177       MachineRepresentation rep;
178       if (node->opcode() == IrOpcode::kLoad) {
179         rep = LoadRepresentationOf(node->op()).representation();
180       } else {
181         DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
182         rep = LoadRepresentationOf(node->op()).representation();
183       }
184 
185       if (rep == MachineRepresentation::kWord64) {
186         LowerMemoryBaseAndIndex(node);
187         Node* base = node->InputAt(0);
188         Node* index = node->InputAt(1);
189         Node* index_low;
190         Node* index_high;
191         GetIndexNodes(index, &index_low, &index_high);
192         const Operator* load_op;
193 
194         if (node->opcode() == IrOpcode::kLoad) {
195           load_op = machine()->Load(MachineType::Int32());
196         } else {
197           DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
198           load_op = machine()->UnalignedLoad(MachineType::Int32());
199         }
200 
201         Node* high_node;
202         if (node->InputCount() > 2) {
203           Node* effect_high = node->InputAt(2);
204           Node* control_high = node->InputAt(3);
205           high_node = graph()->NewNode(load_op, base, index_high, effect_high,
206                                        control_high);
207           // change the effect change from old_node --> old_effect to
208           // old_node --> high_node --> old_effect.
209           node->ReplaceInput(2, high_node);
210         } else {
211           high_node = graph()->NewNode(load_op, base, index_high);
212         }
213         node->ReplaceInput(1, index_low);
214         NodeProperties::ChangeOp(node, load_op);
215         ReplaceNode(node, node, high_node);
216       } else {
217         DefaultLowering(node);
218       }
219       break;
220     }
221     case IrOpcode::kStore:
222     case IrOpcode::kUnalignedStore: {
223       MachineRepresentation rep;
224       if (node->opcode() == IrOpcode::kStore) {
225         rep = StoreRepresentationOf(node->op()).representation();
226       } else {
227         DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
228         rep = UnalignedStoreRepresentationOf(node->op());
229       }
230 
231       if (rep == MachineRepresentation::kWord64) {
232         // We change the original store node to store the low word, and create
233         // a new store node to store the high word. The effect and control edges
234         // are copied from the original store to the new store node, the effect
235         // edge of the original store is redirected to the new store.
236         LowerMemoryBaseAndIndex(node);
237         Node* base = node->InputAt(0);
238         Node* index = node->InputAt(1);
239         Node* index_low;
240         Node* index_high;
241         GetIndexNodes(index, &index_low, &index_high);
242         Node* value = node->InputAt(2);
243         DCHECK(HasReplacementLow(value));
244         DCHECK(HasReplacementHigh(value));
245 
246         const Operator* store_op;
247         if (node->opcode() == IrOpcode::kStore) {
248           WriteBarrierKind write_barrier_kind =
249               StoreRepresentationOf(node->op()).write_barrier_kind();
250           store_op = machine()->Store(StoreRepresentation(
251               MachineRepresentation::kWord32, write_barrier_kind));
252         } else {
253           DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
254           store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
255         }
256 
257         Node* high_node;
258         if (node->InputCount() > 3) {
259           Node* effect_high = node->InputAt(3);
260           Node* control_high = node->InputAt(4);
261           high_node = graph()->NewNode(store_op, base, index_high,
262                                        GetReplacementHigh(value), effect_high,
263                                        control_high);
264           node->ReplaceInput(3, high_node);
265 
266         } else {
267           high_node = graph()->NewNode(store_op, base, index_high,
268                                        GetReplacementHigh(value));
269         }
270 
271         node->ReplaceInput(1, index_low);
272         node->ReplaceInput(2, GetReplacementLow(value));
273         NodeProperties::ChangeOp(node, store_op);
274         ReplaceNode(node, node, high_node);
275       } else {
276         DefaultLowering(node, true);
277       }
278       break;
279     }
280     case IrOpcode::kStart: {
281       int parameter_count = GetParameterCountAfterLowering(signature());
282       // Only exchange the node if the parameter count actually changed.
283       if (parameter_count != static_cast<int>(signature()->parameter_count())) {
284         int delta =
285             parameter_count - static_cast<int>(signature()->parameter_count());
286         int new_output_count = node->op()->ValueOutputCount() + delta;
287         NodeProperties::ChangeOp(node, common()->Start(new_output_count));
288       }
289       break;
290     }
291     case IrOpcode::kParameter: {
292       DCHECK_EQ(1, node->InputCount());
293       int param_count = static_cast<int>(signature()->parameter_count());
294       // Only exchange the node if the parameter count actually changed. We do
295       // not even have to do the default lowering because the the start node,
296       // the only input of a parameter node, only changes if the parameter count
297       // changes.
298       if (GetParameterCountAfterLowering(signature()) != param_count) {
299         int old_index = ParameterIndexOf(node->op());
300         // Adjust old_index to be compliant with the signature.
301         --old_index;
302         int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303         // Adjust new_index to consider the instance parameter.
304         ++new_index;
305         NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306 
307         if (old_index < 0 || old_index >= param_count) {
308           // Special parameters (JS closure/context) don't have kWord64
309           // representation anyway.
310           break;
311         }
312 
313         if (signature()->GetParam(old_index) ==
314             MachineRepresentation::kWord64) {
315           Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
316                                              graph()->start());
317           ReplaceNode(node, node, high_node);
318         }
319       }
320       break;
321     }
322     case IrOpcode::kReturn: {
323       int input_count = node->InputCount();
324       DefaultLowering(node);
325       if (input_count != node->InputCount()) {
326         int new_return_count = GetReturnCountAfterLowering(signature());
327         if (static_cast<int>(signature()->return_count()) != new_return_count) {
328           NodeProperties::ChangeOp(node, common()->Return(new_return_count));
329         }
330       }
331       break;
332     }
333     case IrOpcode::kTailCall: {
334       auto call_descriptor =
335           const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
336       bool returns_require_lowering =
337           GetReturnCountAfterLowering(call_descriptor) !=
338           static_cast<int>(call_descriptor->ReturnCount());
339       if (DefaultLowering(node) || returns_require_lowering) {
340         // Tail calls do not have return values, so adjusting the call
341         // descriptor is enough.
342         NodeProperties::ChangeOp(
343             node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
344       }
345       break;
346     }
347     case IrOpcode::kCall: {
348       auto call_descriptor = CallDescriptorOf(node->op());
349 
350       bool returns_require_lowering =
351           GetReturnCountAfterLowering(call_descriptor) !=
352           static_cast<int>(call_descriptor->ReturnCount());
353       if (DefaultLowering(node) || returns_require_lowering) {
354         // We have to adjust the call descriptor.
355         NodeProperties::ChangeOp(
356             node, common()->Call(LowerCallDescriptor(call_descriptor)));
357       }
358       if (returns_require_lowering) {
359         size_t return_arity = call_descriptor->ReturnCount();
360         if (return_arity == 1) {
361           // We access the additional return values through projections.
362           ReplaceNodeWithProjections(node);
363         } else {
364           ZoneVector<Node*> projections(return_arity, zone());
365           NodeProperties::CollectValueProjections(node, projections.data(),
366                                                   return_arity);
367           for (size_t old_index = 0, new_index = 0; old_index < return_arity;
368                ++old_index, ++new_index) {
369             Node* use_node = projections[old_index];
370             DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
371             DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
372                                                   static_cast<int>(old_index)),
373                       static_cast<int>(new_index));
374             if (new_index != old_index) {
375               NodeProperties::ChangeOp(
376                   use_node, common()->Projection(new_index));
377             }
378             if (call_descriptor->GetReturnType(old_index).representation() ==
379                 MachineRepresentation::kWord64) {
380               Node* high_node = graph()->NewNode(
381                   common()->Projection(new_index + 1), node,
382                   graph()->start());
383               ReplaceNode(use_node, use_node, high_node);
384               ++new_index;
385             }
386           }
387         }
388       }
389       break;
390     }
391     case IrOpcode::kWord64And: {
392       DCHECK_EQ(2, node->InputCount());
393       Node* left = node->InputAt(0);
394       Node* right = node->InputAt(1);
395 
396       Node* low_node =
397           graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
398                            GetReplacementLow(right));
399       Node* high_node =
400           graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
401                            GetReplacementHigh(right));
402       ReplaceNode(node, low_node, high_node);
403       break;
404     }
405     case IrOpcode::kTruncateInt64ToInt32: {
406       DCHECK_EQ(1, node->InputCount());
407       Node* input = node->InputAt(0);
408       ReplaceNode(node, GetReplacementLow(input), nullptr);
409       node->NullAllInputs();
410       break;
411     }
412     case IrOpcode::kInt64Add: {
413       DCHECK_EQ(2, node->InputCount());
414 
415       Node* right = node->InputAt(1);
416       node->ReplaceInput(1, GetReplacementLow(right));
417       node->AppendInput(zone(), GetReplacementHigh(right));
418 
419       Node* left = node->InputAt(0);
420       node->ReplaceInput(0, GetReplacementLow(left));
421       node->InsertInput(zone(), 1, GetReplacementHigh(left));
422 
423       NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
424       // We access the additional return values through projections.
425       ReplaceNodeWithProjections(node);
426       break;
427     }
428     case IrOpcode::kInt64Sub: {
429       DCHECK_EQ(2, node->InputCount());
430 
431       Node* right = node->InputAt(1);
432       node->ReplaceInput(1, GetReplacementLow(right));
433       node->AppendInput(zone(), GetReplacementHigh(right));
434 
435       Node* left = node->InputAt(0);
436       node->ReplaceInput(0, GetReplacementLow(left));
437       node->InsertInput(zone(), 1, GetReplacementHigh(left));
438 
439       NodeProperties::ChangeOp(node, machine()->Int32PairSub());
440       // We access the additional return values through projections.
441       ReplaceNodeWithProjections(node);
442       break;
443     }
444     case IrOpcode::kInt64Mul: {
445       DCHECK_EQ(2, node->InputCount());
446 
447       Node* right = node->InputAt(1);
448       node->ReplaceInput(1, GetReplacementLow(right));
449       node->AppendInput(zone(), GetReplacementHigh(right));
450 
451       Node* left = node->InputAt(0);
452       node->ReplaceInput(0, GetReplacementLow(left));
453       node->InsertInput(zone(), 1, GetReplacementHigh(left));
454 
455       NodeProperties::ChangeOp(node, machine()->Int32PairMul());
456       // We access the additional return values through projections.
457       ReplaceNodeWithProjections(node);
458       break;
459     }
460     case IrOpcode::kWord64Or: {
461       DCHECK_EQ(2, node->InputCount());
462       Node* left = node->InputAt(0);
463       Node* right = node->InputAt(1);
464 
465       Node* low_node =
466           graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
467                            GetReplacementLow(right));
468       Node* high_node =
469           graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
470                            GetReplacementHigh(right));
471       ReplaceNode(node, low_node, high_node);
472       break;
473     }
474     case IrOpcode::kWord64Xor: {
475       DCHECK_EQ(2, node->InputCount());
476       Node* left = node->InputAt(0);
477       Node* right = node->InputAt(1);
478 
479       Node* low_node =
480           graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
481                            GetReplacementLow(right));
482       Node* high_node =
483           graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
484                            GetReplacementHigh(right));
485       ReplaceNode(node, low_node, high_node);
486       break;
487     }
488     case IrOpcode::kWord64Shl: {
489       // TODO(turbofan): if the shift count >= 32, then we can set the low word
490       // of the output to 0 and just calculate the high word.
491       DCHECK_EQ(2, node->InputCount());
492       Node* shift = node->InputAt(1);
493       if (HasReplacementLow(shift)) {
494         // We do not have to care about the high word replacement, because
495         // the shift can only be between 0 and 63 anyways.
496         node->ReplaceInput(1, GetReplacementLow(shift));
497       }
498 
499       Node* value = node->InputAt(0);
500       node->ReplaceInput(0, GetReplacementLow(value));
501       node->InsertInput(zone(), 1, GetReplacementHigh(value));
502 
503       NodeProperties::ChangeOp(node, machine()->Word32PairShl());
504       // We access the additional return values through projections.
505       ReplaceNodeWithProjections(node);
506       break;
507     }
508     case IrOpcode::kWord64Shr: {
509       // TODO(turbofan): if the shift count >= 32, then we can set the low word
510       // of the output to 0 and just calculate the high word.
511       DCHECK_EQ(2, node->InputCount());
512       Node* shift = node->InputAt(1);
513       if (HasReplacementLow(shift)) {
514         // We do not have to care about the high word replacement, because
515         // the shift can only be between 0 and 63 anyways.
516         node->ReplaceInput(1, GetReplacementLow(shift));
517       }
518 
519       Node* value = node->InputAt(0);
520       node->ReplaceInput(0, GetReplacementLow(value));
521       node->InsertInput(zone(), 1, GetReplacementHigh(value));
522 
523       NodeProperties::ChangeOp(node, machine()->Word32PairShr());
524       // We access the additional return values through projections.
525       ReplaceNodeWithProjections(node);
526       break;
527     }
528     case IrOpcode::kWord64Sar: {
529       // TODO(turbofan): if the shift count >= 32, then we can set the low word
530       // of the output to 0 and just calculate the high word.
531       DCHECK_EQ(2, node->InputCount());
532       Node* shift = node->InputAt(1);
533       if (HasReplacementLow(shift)) {
534         // We do not have to care about the high word replacement, because
535         // the shift can only be between 0 and 63 anyways.
536         node->ReplaceInput(1, GetReplacementLow(shift));
537       }
538 
539       Node* value = node->InputAt(0);
540       node->ReplaceInput(0, GetReplacementLow(value));
541       node->InsertInput(zone(), 1, GetReplacementHigh(value));
542 
543       NodeProperties::ChangeOp(node, machine()->Word32PairSar());
544       // We access the additional return values through projections.
545       ReplaceNodeWithProjections(node);
546       break;
547     }
548     case IrOpcode::kWord64Equal: {
549       DCHECK_EQ(2, node->InputCount());
550       Node* left = node->InputAt(0);
551       Node* right = node->InputAt(1);
552 
553       // TODO(wasm): Use explicit comparisons and && here?
554       Node* replacement = graph()->NewNode(
555           machine()->Word32Equal(),
556           graph()->NewNode(
557               machine()->Word32Or(),
558               graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
559                                GetReplacementLow(right)),
560               graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
561                                GetReplacementHigh(right))),
562           graph()->NewNode(common()->Int32Constant(0)));
563 
564       ReplaceNode(node, replacement, nullptr);
565       break;
566     }
567     case IrOpcode::kInt64LessThan: {
568       LowerComparison(node, machine()->Int32LessThan(),
569                       machine()->Uint32LessThan());
570       break;
571     }
572     case IrOpcode::kInt64LessThanOrEqual: {
573       LowerComparison(node, machine()->Int32LessThan(),
574                       machine()->Uint32LessThanOrEqual());
575       break;
576     }
577     case IrOpcode::kUint64LessThan: {
578       LowerComparison(node, machine()->Uint32LessThan(),
579                       machine()->Uint32LessThan());
580       break;
581     }
582     case IrOpcode::kUint64LessThanOrEqual: {
583       LowerComparison(node, machine()->Uint32LessThan(),
584                       machine()->Uint32LessThanOrEqual());
585       break;
586     }
587     case IrOpcode::kSignExtendWord32ToInt64:
588     case IrOpcode::kChangeInt32ToInt64: {
589       DCHECK_EQ(1, node->InputCount());
590       Node* input = node->InputAt(0);
591       if (HasReplacementLow(input)) {
592         input = GetReplacementLow(input);
593       }
594       // We use SAR to preserve the sign in the high word.
595       ReplaceNode(
596           node, input,
597           graph()->NewNode(machine()->Word32Sar(), input,
598                            graph()->NewNode(common()->Int32Constant(31))));
599       node->NullAllInputs();
600       break;
601     }
602     case IrOpcode::kChangeUint32ToUint64: {
603       DCHECK_EQ(1, node->InputCount());
604       Node* input = node->InputAt(0);
605       if (HasReplacementLow(input)) {
606         input = GetReplacementLow(input);
607       }
608       ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
609       node->NullAllInputs();
610       break;
611     }
612     case IrOpcode::kBitcastInt64ToFloat64: {
613       DCHECK_EQ(1, node->InputCount());
614       Node* input = node->InputAt(0);
615       Node* stack_slot = graph()->NewNode(
616           machine()->StackSlot(MachineRepresentation::kWord64));
617 
618       Node* store_high_word = graph()->NewNode(
619           machine()->Store(
620               StoreRepresentation(MachineRepresentation::kWord32,
621                                   WriteBarrierKind::kNoWriteBarrier)),
622           stack_slot,
623           graph()->NewNode(
624               common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
625           GetReplacementHigh(input), graph()->start(), graph()->start());
626 
627       Node* store_low_word = graph()->NewNode(
628           machine()->Store(
629               StoreRepresentation(MachineRepresentation::kWord32,
630                                   WriteBarrierKind::kNoWriteBarrier)),
631           stack_slot,
632           graph()->NewNode(
633               common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
634           GetReplacementLow(input), store_high_word, graph()->start());
635 
636       Node* load =
637           graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
638                            graph()->NewNode(common()->Int32Constant(0)),
639                            store_low_word, graph()->start());
640 
641       ReplaceNode(node, load, nullptr);
642       break;
643     }
644     case IrOpcode::kBitcastFloat64ToInt64: {
645       DCHECK_EQ(1, node->InputCount());
646       Node* input = node->InputAt(0);
647       if (HasReplacementLow(input)) {
648         input = GetReplacementLow(input);
649       }
650       Node* stack_slot = graph()->NewNode(
651           machine()->StackSlot(MachineRepresentation::kWord64));
652       Node* store = graph()->NewNode(
653           machine()->Store(
654               StoreRepresentation(MachineRepresentation::kFloat64,
655                                   WriteBarrierKind::kNoWriteBarrier)),
656           stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
657           graph()->start(), graph()->start());
658 
659       Node* high_node = graph()->NewNode(
660           machine()->Load(MachineType::Int32()), stack_slot,
661           graph()->NewNode(
662               common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
663           store, graph()->start());
664 
665       Node* low_node = graph()->NewNode(
666           machine()->Load(MachineType::Int32()), stack_slot,
667           graph()->NewNode(
668               common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
669           store, graph()->start());
670       ReplaceNode(node, low_node, high_node);
671       break;
672     }
673     case IrOpcode::kWord64Rol:
674       DCHECK(machine()->Word32Rol().IsSupported());
675       V8_FALLTHROUGH;
676     case IrOpcode::kWord64Ror: {
677       DCHECK_EQ(2, node->InputCount());
678       Node* input = node->InputAt(0);
679       Node* shift = HasReplacementLow(node->InputAt(1))
680                         ? GetReplacementLow(node->InputAt(1))
681                         : node->InputAt(1);
682       Int32Matcher m(shift);
683       if (m.HasResolvedValue()) {
684         // Precondition: 0 <= shift < 64.
685         int32_t shift_value = m.ResolvedValue() & 0x3F;
686         if (shift_value == 0) {
687           ReplaceNode(node, GetReplacementLow(input),
688                       GetReplacementHigh(input));
689         } else if (shift_value == 32) {
690           ReplaceNode(node, GetReplacementHigh(input),
691                       GetReplacementLow(input));
692         } else {
693           Node* low_input;
694           Node* high_input;
695           if (shift_value < 32) {
696             low_input = GetReplacementLow(input);
697             high_input = GetReplacementHigh(input);
698           } else {
699             low_input = GetReplacementHigh(input);
700             high_input = GetReplacementLow(input);
701           }
702           int32_t masked_shift_value = shift_value & 0x1F;
703           Node* masked_shift =
704               graph()->NewNode(common()->Int32Constant(masked_shift_value));
705           Node* inv_shift = graph()->NewNode(
706               common()->Int32Constant(32 - masked_shift_value));
707 
708           auto* op1 = machine()->Word32Shr();
709           auto* op2 = machine()->Word32Shl();
710           bool is_ror = node->opcode() == IrOpcode::kWord64Ror;
711           if (!is_ror) std::swap(op1, op2);
712 
713           Node* low_node =
714               graph()->NewNode(machine()->Word32Or(),
715                                graph()->NewNode(op1, low_input, masked_shift),
716                                graph()->NewNode(op2, high_input, inv_shift));
717           Node* high_node =
718               graph()->NewNode(machine()->Word32Or(),
719                                graph()->NewNode(op1, high_input, masked_shift),
720                                graph()->NewNode(op2, low_input, inv_shift));
721           ReplaceNode(node, low_node, high_node);
722         }
723       } else {
724         Node* safe_shift = shift;
725         if (!machine()->Word32ShiftIsSafe()) {
726           safe_shift =
727               graph()->NewNode(machine()->Word32And(), shift,
728                                graph()->NewNode(common()->Int32Constant(0x1F)));
729         }
730 
731         bool is_ror = node->opcode() == IrOpcode::kWord64Ror;
732         Node* inv_mask =
733             is_ror ? graph()->NewNode(
734                          machine()->Word32Xor(),
735                          graph()->NewNode(
736                              machine()->Word32Shr(),
737                              graph()->NewNode(common()->Int32Constant(-1)),
738                              safe_shift),
739                          graph()->NewNode(common()->Int32Constant(-1)))
740                    : graph()->NewNode(
741                          machine()->Word32Shl(),
742                          graph()->NewNode(common()->Int32Constant(-1)),
743                          safe_shift);
744 
745         Node* bit_mask =
746             graph()->NewNode(machine()->Word32Xor(), inv_mask,
747                              graph()->NewNode(common()->Int32Constant(-1)));
748 
749         // We have to mask the shift value for this comparison. If
750         // !machine()->Word32ShiftIsSafe() then the masking should already be
751         // part of the graph.
752         Node* masked_shift6 = shift;
753         if (machine()->Word32ShiftIsSafe()) {
754           masked_shift6 =
755               graph()->NewNode(machine()->Word32And(), shift,
756                                graph()->NewNode(common()->Int32Constant(0x3F)));
757         }
758 
759         Diamond lt32(
760             graph(), common(),
761             graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
762                              graph()->NewNode(common()->Int32Constant(32))));
763 
764         // The low word and the high word can be swapped either at the input or
765         // at the output. We swap the inputs so that shift does not have to be
766         // kept for so long in a register.
767         Node* input_low =
768             lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
769                      GetReplacementHigh(input));
770         Node* input_high =
771             lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
772                      GetReplacementLow(input));
773 
774         const Operator* oper =
775             is_ror ? machine()->Word32Ror() : machine()->Word32Rol().op();
776 
777         Node* rotate_low = graph()->NewNode(oper, input_low, safe_shift);
778         Node* rotate_high = graph()->NewNode(oper, input_high, safe_shift);
779 
780         auto* mask1 = bit_mask;
781         auto* mask2 = inv_mask;
782         if (!is_ror) std::swap(mask1, mask2);
783 
784         Node* low_node = graph()->NewNode(
785             machine()->Word32Or(),
786             graph()->NewNode(machine()->Word32And(), rotate_low, mask1),
787             graph()->NewNode(machine()->Word32And(), rotate_high, mask2));
788         Node* high_node = graph()->NewNode(
789             machine()->Word32Or(),
790             graph()->NewNode(machine()->Word32And(), rotate_high, mask1),
791             graph()->NewNode(machine()->Word32And(), rotate_low, mask2));
792         ReplaceNode(node, low_node, high_node);
793       }
794       break;
795     }
796     case IrOpcode::kWord64Clz: {
797       DCHECK_EQ(1, node->InputCount());
798       Node* input = node->InputAt(0);
799       Diamond d(
800           graph(), common(),
801           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
802                            graph()->NewNode(common()->Int32Constant(0))));
803 
804       Node* low_node = d.Phi(
805           MachineRepresentation::kWord32,
806           graph()->NewNode(machine()->Int32Add(),
807                            graph()->NewNode(machine()->Word32Clz(),
808                                             GetReplacementLow(input)),
809                            graph()->NewNode(common()->Int32Constant(32))),
810           graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
811       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
812       break;
813     }
814     case IrOpcode::kWord64Ctz: {
815       DCHECK_EQ(1, node->InputCount());
816       DCHECK(machine()->Word32Ctz().IsSupported());
817       Node* input = node->InputAt(0);
818       Diamond d(
819           graph(), common(),
820           graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
821                            graph()->NewNode(common()->Int32Constant(0))));
822       Node* low_node =
823           d.Phi(MachineRepresentation::kWord32,
824                 graph()->NewNode(machine()->Int32Add(),
825                                  graph()->NewNode(machine()->Word32Ctz().op(),
826                                                   GetReplacementHigh(input)),
827                                  graph()->NewNode(common()->Int32Constant(32))),
828                 graph()->NewNode(machine()->Word32Ctz().op(),
829                                  GetReplacementLow(input)));
830       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
831       break;
832     }
833     case IrOpcode::kWord64Popcnt: {
834       DCHECK_EQ(1, node->InputCount());
835       Node* input = node->InputAt(0);
836       // We assume that a Word64Popcnt node only has been created if
837       // Word32Popcnt is actually supported.
838       DCHECK(machine()->Word32Popcnt().IsSupported());
839       ReplaceNode(node, graph()->NewNode(
840                             machine()->Int32Add(),
841                             graph()->NewNode(machine()->Word32Popcnt().op(),
842                                              GetReplacementLow(input)),
843                             graph()->NewNode(machine()->Word32Popcnt().op(),
844                                              GetReplacementHigh(input))),
845                   graph()->NewNode(common()->Int32Constant(0)));
846       break;
847     }
848     case IrOpcode::kPhi: {
849       MachineRepresentation rep = PhiRepresentationOf(node->op());
850       if (rep == MachineRepresentation::kWord64) {
851         // The replacement nodes have already been created, we only have to
852         // replace placeholder nodes.
853         Node* low_node = GetReplacementLow(node);
854         Node* high_node = GetReplacementHigh(node);
855         for (int i = 0; i < node->op()->ValueInputCount(); i++) {
856           low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
857           high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
858         }
859       } else {
860         DefaultLowering(node);
861       }
862       break;
863     }
864     case IrOpcode::kWord64ReverseBytes: {
865       Node* input = node->InputAt(0);
866       ReplaceNode(node,
867                   graph()->NewNode(machine()->Word32ReverseBytes(),
868                                    GetReplacementHigh(input)),
869                   graph()->NewNode(machine()->Word32ReverseBytes(),
870                                    GetReplacementLow(input)));
871       break;
872     }
873     case IrOpcode::kSignExtendWord8ToInt64: {
874       DCHECK_EQ(1, node->InputCount());
875       Node* input = node->InputAt(0);
876       if (HasReplacementLow(input)) {
877         input = GetReplacementLow(input);
878       }
879       // Sign extend low node to Int32
880       input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
881 
882       // We use SAR to preserve the sign in the high word.
883       ReplaceNode(
884           node, input,
885           graph()->NewNode(machine()->Word32Sar(), input,
886                            graph()->NewNode(common()->Int32Constant(31))));
887       node->NullAllInputs();
888       break;
889     }
890     case IrOpcode::kSignExtendWord16ToInt64: {
891       DCHECK_EQ(1, node->InputCount());
892       Node* input = node->InputAt(0);
893       if (HasReplacementLow(input)) {
894         input = GetReplacementLow(input);
895       }
896       // Sign extend low node to Int32
897       input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
898 
899       // We use SAR to preserve the sign in the high word.
900       ReplaceNode(
901           node, input,
902           graph()->NewNode(machine()->Word32Sar(), input,
903                            graph()->NewNode(common()->Int32Constant(31))));
904       node->NullAllInputs();
905       break;
906     }
907     case IrOpcode::kWord64AtomicLoad: {
908       DCHECK_EQ(4, node->InputCount());
909       MachineType type = AtomicOpType(node->op());
910       DefaultLowering(node, true);
911       if (type == MachineType::Uint64()) {
912         NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
913         ReplaceNodeWithProjections(node);
914       } else {
915         NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
916         ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
917       }
918       break;
919     }
920     case IrOpcode::kWord64AtomicStore: {
921       DCHECK_EQ(5, node->InputCount());
922       MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
923       if (rep == MachineRepresentation::kWord64) {
924         LowerMemoryBaseAndIndex(node);
925         Node* value = node->InputAt(2);
926         node->ReplaceInput(2, GetReplacementLow(value));
927         node->InsertInput(zone(), 3, GetReplacementHigh(value));
928         NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
929       } else {
930         DefaultLowering(node, true);
931         NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
932       }
933       break;
934     }
935 #define ATOMIC_CASE(name)                                                   \
936   case IrOpcode::kWord64Atomic##name: {                                     \
937     MachineType type = AtomicOpType(node->op());                            \
938     if (type == MachineType::Uint64()) {                                    \
939       LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name());    \
940     } else {                                                                \
941       LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
942     }                                                                       \
943     break;                                                                  \
944   }
945       ATOMIC_CASE(Add)
946       ATOMIC_CASE(Sub)
947       ATOMIC_CASE(And)
948       ATOMIC_CASE(Or)
949       ATOMIC_CASE(Xor)
950       ATOMIC_CASE(Exchange)
951 #undef ATOMIC_CASE
952     case IrOpcode::kWord64AtomicCompareExchange: {
953       MachineType type = AtomicOpType(node->op());
954       if (type == MachineType::Uint64()) {
955         LowerMemoryBaseAndIndex(node);
956         Node* old_value = node->InputAt(2);
957         Node* new_value = node->InputAt(3);
958         node->ReplaceInput(2, GetReplacementLow(old_value));
959         node->ReplaceInput(3, GetReplacementHigh(old_value));
960         node->InsertInput(zone(), 4, GetReplacementLow(new_value));
961         node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
962         NodeProperties::ChangeOp(node,
963                                  machine()->Word32AtomicPairCompareExchange());
964         ReplaceNodeWithProjections(node);
965       } else {
966         DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
967                type == MachineType::Uint8());
968         DefaultLowering(node, true);
969         NodeProperties::ChangeOp(node,
970                                  machine()->Word32AtomicCompareExchange(type));
971         ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
972       }
973       break;
974     }
975     case IrOpcode::kI64x2Splat: {
976       DCHECK_EQ(1, node->InputCount());
977       Node* input = node->InputAt(0);
978       node->ReplaceInput(0, GetReplacementLow(input));
979       node->AppendInput(zone(), GetReplacementHigh(input));
980       NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
981       break;
982     }
983     case IrOpcode::kI64x2ExtractLane: {
984       DCHECK_EQ(1, node->InputCount());
985       Node* input = node->InputAt(0);
986       int32_t lane = OpParameter<int32_t>(node->op());
987       ReplaceNode(
988           node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
989           graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
990       break;
991     }
992     case IrOpcode::kI64x2ReplaceLane: {
993       DCHECK_EQ(2, node->InputCount());
994       int32_t lane = OpParameter<int32_t>(node->op());
995       Node* input = node->InputAt(1);
996       node->ReplaceInput(1, GetReplacementLow(input));
997       node->AppendInput(zone(), GetReplacementHigh(input));
998       NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
999       break;
1000     }
1001 
1002     default: { DefaultLowering(node); }
1003   }
1004 }  // NOLINT(readability/fn_size)
1005 
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)1006 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
1007                                     const Operator* low_word_op) {
1008   DCHECK_EQ(2, node->InputCount());
1009   Node* left = node->InputAt(0);
1010   Node* right = node->InputAt(1);
1011   Node* replacement = graph()->NewNode(
1012       machine()->Word32Or(),
1013       graph()->NewNode(high_word_op, GetReplacementHigh(left),
1014                        GetReplacementHigh(right)),
1015       graph()->NewNode(
1016           machine()->Word32And(),
1017           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
1018                            GetReplacementHigh(right)),
1019           graph()->NewNode(low_word_op, GetReplacementLow(left),
1020                            GetReplacementLow(right))));
1021 
1022   ReplaceNode(node, replacement, nullptr);
1023 }
1024 
DefaultLowering(Node * node,bool low_word_only)1025 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
1026   bool something_changed = false;
1027   for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1028     Node* input = node->InputAt(i);
1029     if (HasReplacementLow(input)) {
1030       something_changed = true;
1031       node->ReplaceInput(i, GetReplacementLow(input));
1032     }
1033     if (!low_word_only && HasReplacementHigh(input)) {
1034       something_changed = true;
1035       node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
1036     }
1037   }
1038   return something_changed;
1039 }
1040 
LowerCallDescriptor(const CallDescriptor * call_descriptor)1041 const CallDescriptor* Int64Lowering::LowerCallDescriptor(
1042     const CallDescriptor* call_descriptor) {
1043   if (special_case_) {
1044     auto replacement = special_case_->replacements.find(call_descriptor);
1045     if (replacement != special_case_->replacements.end()) {
1046       return replacement->second;
1047     }
1048   }
1049   return GetI32WasmCallDescriptor(zone(), call_descriptor);
1050 }
1051 
ReplaceNode(Node * old,Node * new_low,Node * new_high)1052 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
1053   // if new_low == nullptr, then also new_high == nullptr.
1054   DCHECK(new_low != nullptr || new_high == nullptr);
1055   replacements_[old->id()].low = new_low;
1056   replacements_[old->id()].high = new_high;
1057 }
1058 
HasReplacementLow(Node * node)1059 bool Int64Lowering::HasReplacementLow(Node* node) {
1060   return replacements_[node->id()].low != nullptr;
1061 }
1062 
GetReplacementLow(Node * node)1063 Node* Int64Lowering::GetReplacementLow(Node* node) {
1064   Node* result = replacements_[node->id()].low;
1065   DCHECK(result);
1066   return result;
1067 }
1068 
HasReplacementHigh(Node * node)1069 bool Int64Lowering::HasReplacementHigh(Node* node) {
1070   return replacements_[node->id()].high != nullptr;
1071 }
1072 
GetReplacementHigh(Node * node)1073 Node* Int64Lowering::GetReplacementHigh(Node* node) {
1074   Node* result = replacements_[node->id()].high;
1075   DCHECK(result);
1076   return result;
1077 }
1078 
PreparePhiReplacement(Node * phi)1079 void Int64Lowering::PreparePhiReplacement(Node* phi) {
1080   MachineRepresentation rep = PhiRepresentationOf(phi->op());
1081   if (rep == MachineRepresentation::kWord64) {
1082     // We have to create the replacements for a phi node before we actually
1083     // lower the phi to break potential cycles in the graph. The replacements of
1084     // input nodes do not exist yet, so we use a placeholder node to pass the
1085     // graph verifier.
1086     int value_count = phi->op()->ValueInputCount();
1087     Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1088     Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1089     for (int i = 0; i < value_count; i++) {
1090       inputs_low[i] = placeholder_;
1091       inputs_high[i] = placeholder_;
1092     }
1093     inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1094     inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1095     ReplaceNode(phi,
1096                 graph()->NewNode(
1097                     common()->Phi(MachineRepresentation::kWord32, value_count),
1098                     value_count + 1, inputs_low, false),
1099                 graph()->NewNode(
1100                     common()->Phi(MachineRepresentation::kWord32, value_count),
1101                     value_count + 1, inputs_high, false));
1102   }
1103 }
1104 
ReplaceNodeWithProjections(Node * node)1105 void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1106   DCHECK(node != nullptr);
1107   Node* low_node =
1108       graph()->NewNode(common()->Projection(0), node, graph()->start());
1109   Node* high_node =
1110       graph()->NewNode(common()->Projection(1), node, graph()->start());
1111   ReplaceNode(node, low_node, high_node);
1112 }
1113 
LowerMemoryBaseAndIndex(Node * node)1114 void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
1115   DCHECK(node != nullptr);
1116   // Low word only replacements for memory operands for 32-bit address space.
1117   Node* base = node->InputAt(0);
1118   Node* index = node->InputAt(1);
1119   if (HasReplacementLow(base)) {
1120     node->ReplaceInput(0, GetReplacementLow(base));
1121   }
1122   if (HasReplacementLow(index)) {
1123     node->ReplaceInput(1, GetReplacementLow(index));
1124   }
1125 }
1126 
1127 }  // namespace compiler
1128 }  // namespace internal
1129 }  // namespace v8
1130