1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/int64-lowering.h"
6 
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19 
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23 
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature)24 Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
25                              CommonOperatorBuilder* common, Zone* zone,
26                              Signature<MachineRepresentation>* signature)
27     : zone_(zone),
28       graph_(graph),
29       machine_(machine),
30       common_(common),
31       state_(graph, 3),
32       stack_(zone),
33       replacements_(nullptr),
34       signature_(signature),
35       placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
36                                   graph->start())) {
37   DCHECK_NOT_NULL(graph);
38   DCHECK_NOT_NULL(graph->end());
39   replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
40   memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
41 }
42 
LowerGraph()43 void Int64Lowering::LowerGraph() {
44   if (!machine()->Is32()) {
45     return;
46   }
47   stack_.push_back({graph()->end(), 0});
48   state_.Set(graph()->end(), State::kOnStack);
49 
50   while (!stack_.empty()) {
51     NodeState& top = stack_.back();
52     if (top.input_index == top.node->InputCount()) {
53       // All inputs of top have already been lowered, now lower top.
54       stack_.pop_back();
55       state_.Set(top.node, State::kVisited);
56       LowerNode(top.node);
57     } else {
58       // Push the next input onto the stack.
59       Node* input = top.node->InputAt(top.input_index++);
60       if (state_.Get(input) == State::kUnvisited) {
61         if (input->opcode() == IrOpcode::kPhi) {
62           // To break cycles with phi nodes we push phis on a separate stack so
63           // that they are processed after all other nodes.
64           PreparePhiReplacement(input);
65           stack_.push_front({input, 0});
66         } else if (input->opcode() == IrOpcode::kEffectPhi ||
67                    input->opcode() == IrOpcode::kLoop) {
68           stack_.push_front({input, 0});
69         } else {
70           stack_.push_back({input, 0});
71         }
72         state_.Set(input, State::kOnStack);
73       }
74     }
75   }
76 }
77 
78 namespace {
79 
GetReturnIndexAfterLowering(CallDescriptor * call_descriptor,int old_index)80 int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor,
81                                 int old_index) {
82   int result = old_index;
83   for (int i = 0; i < old_index; i++) {
84     if (call_descriptor->GetReturnType(i).representation() ==
85         MachineRepresentation::kWord64) {
86       result++;
87     }
88   }
89   return result;
90 }
91 
GetReturnCountAfterLowering(CallDescriptor * call_descriptor)92 int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) {
93   return GetReturnIndexAfterLowering(
94       call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
95 }
96 
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)97 int GetParameterIndexAfterLowering(
98     Signature<MachineRepresentation>* signature, int old_index) {
99   int result = old_index;
100   for (int i = 0; i < old_index; i++) {
101     if (signature->GetParam(i) == MachineRepresentation::kWord64) {
102       result++;
103     }
104   }
105   return result;
106 }
107 
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)108 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
109   int result = static_cast<int>(signature->return_count());
110   for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
111     if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
112       result++;
113     }
114   }
115   return result;
116 }
117 
118 }  // namespace
119 
120 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)121 int Int64Lowering::GetParameterCountAfterLowering(
122     Signature<MachineRepresentation>* signature) {
123   // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
124   // after lowering.
125   return GetParameterIndexAfterLowering(
126       signature, static_cast<int>(signature->parameter_count()));
127 }
128 
GetIndexNodes(Node * index,Node * & index_low,Node * & index_high)129 void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
130                                   Node*& index_high) {
131   if (HasReplacementLow(index)) {
132     index = GetReplacementLow(index);
133   }
134 #if defined(V8_TARGET_LITTLE_ENDIAN)
135   index_low = index;
136   index_high = graph()->NewNode(machine()->Int32Add(), index,
137                                 graph()->NewNode(common()->Int32Constant(4)));
138 #elif defined(V8_TARGET_BIG_ENDIAN)
139   index_low = graph()->NewNode(machine()->Int32Add(), index,
140                                graph()->NewNode(common()->Int32Constant(4)));
141   index_high = index;
142 #endif
143 }
144 
LowerNode(Node * node)145 void Int64Lowering::LowerNode(Node* node) {
146   switch (node->opcode()) {
147     case IrOpcode::kInt64Constant: {
148       int64_t value = OpParameter<int64_t>(node->op());
149       Node* low_node = graph()->NewNode(
150           common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
151       Node* high_node = graph()->NewNode(
152           common()->Int32Constant(static_cast<int32_t>(value >> 32)));
153       ReplaceNode(node, low_node, high_node);
154       break;
155     }
156     case IrOpcode::kLoad:
157     case IrOpcode::kUnalignedLoad: {
158       MachineRepresentation rep;
159       if (node->opcode() == IrOpcode::kLoad) {
160         rep = LoadRepresentationOf(node->op()).representation();
161       } else {
162         DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
163         rep = LoadRepresentationOf(node->op()).representation();
164       }
165 
166       if (rep == MachineRepresentation::kWord64) {
167         Node* base = node->InputAt(0);
168         Node* index = node->InputAt(1);
169         Node* index_low;
170         Node* index_high;
171         GetIndexNodes(index, index_low, index_high);
172         const Operator* load_op;
173 
174         if (node->opcode() == IrOpcode::kLoad) {
175           load_op = machine()->Load(MachineType::Int32());
176         } else {
177           DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
178           load_op = machine()->UnalignedLoad(MachineType::Int32());
179         }
180 
181         Node* high_node;
182         if (node->InputCount() > 2) {
183           Node* effect_high = node->InputAt(2);
184           Node* control_high = node->InputAt(3);
185           high_node = graph()->NewNode(load_op, base, index_high, effect_high,
186                                        control_high);
187           // change the effect change from old_node --> old_effect to
188           // old_node --> high_node --> old_effect.
189           node->ReplaceInput(2, high_node);
190         } else {
191           high_node = graph()->NewNode(load_op, base, index_high);
192         }
193         node->ReplaceInput(1, index_low);
194         NodeProperties::ChangeOp(node, load_op);
195         ReplaceNode(node, node, high_node);
196       } else {
197         DefaultLowering(node);
198       }
199       break;
200     }
201     case IrOpcode::kStore:
202     case IrOpcode::kUnalignedStore: {
203       MachineRepresentation rep;
204       if (node->opcode() == IrOpcode::kStore) {
205         rep = StoreRepresentationOf(node->op()).representation();
206       } else {
207         DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
208         rep = UnalignedStoreRepresentationOf(node->op());
209       }
210 
211       if (rep == MachineRepresentation::kWord64) {
212         // We change the original store node to store the low word, and create
213         // a new store node to store the high word. The effect and control edges
214         // are copied from the original store to the new store node, the effect
215         // edge of the original store is redirected to the new store.
216         Node* base = node->InputAt(0);
217         Node* index = node->InputAt(1);
218         Node* index_low;
219         Node* index_high;
220         GetIndexNodes(index, index_low, index_high);
221         Node* value = node->InputAt(2);
222         DCHECK(HasReplacementLow(value));
223         DCHECK(HasReplacementHigh(value));
224 
225         const Operator* store_op;
226         if (node->opcode() == IrOpcode::kStore) {
227           WriteBarrierKind write_barrier_kind =
228               StoreRepresentationOf(node->op()).write_barrier_kind();
229           store_op = machine()->Store(StoreRepresentation(
230               MachineRepresentation::kWord32, write_barrier_kind));
231         } else {
232           DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
233           store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
234         }
235 
236         Node* high_node;
237         if (node->InputCount() > 3) {
238           Node* effect_high = node->InputAt(3);
239           Node* control_high = node->InputAt(4);
240           high_node = graph()->NewNode(store_op, base, index_high,
241                                        GetReplacementHigh(value), effect_high,
242                                        control_high);
243           node->ReplaceInput(3, high_node);
244 
245         } else {
246           high_node = graph()->NewNode(store_op, base, index_high,
247                                        GetReplacementHigh(value));
248         }
249 
250         node->ReplaceInput(1, index_low);
251         node->ReplaceInput(2, GetReplacementLow(value));
252         NodeProperties::ChangeOp(node, store_op);
253         ReplaceNode(node, node, high_node);
254       } else {
255         DefaultLowering(node, true);
256       }
257       break;
258     }
259     case IrOpcode::kStart: {
260       int parameter_count = GetParameterCountAfterLowering(signature());
261       // Only exchange the node if the parameter count actually changed.
262       if (parameter_count != static_cast<int>(signature()->parameter_count())) {
263         int delta =
264             parameter_count - static_cast<int>(signature()->parameter_count());
265         int new_output_count = node->op()->ValueOutputCount() + delta;
266         NodeProperties::ChangeOp(node, common()->Start(new_output_count));
267       }
268       break;
269     }
270     case IrOpcode::kParameter: {
271       DCHECK_EQ(1, node->InputCount());
272       // Only exchange the node if the parameter count actually changed. We do
273       // not even have to do the default lowering because the the start node,
274       // the only input of a parameter node, only changes if the parameter count
275       // changes.
276       if (GetParameterCountAfterLowering(signature()) !=
277           static_cast<int>(signature()->parameter_count())) {
278         int old_index = ParameterIndexOf(node->op());
279         // TODO(wasm): Make this part not wasm specific.
280         // Prevent special lowering of the instance parameter.
281         if (old_index == wasm::kWasmInstanceParameterIndex) {
282           DefaultLowering(node);
283           break;
284         }
285         // Adjust old_index to be compliant with the signature.
286         --old_index;
287         int new_index = GetParameterIndexAfterLowering(signature(), old_index);
288         // Adjust new_index to consider the instance parameter.
289         ++new_index;
290         NodeProperties::ChangeOp(node, common()->Parameter(new_index));
291 
292         if (signature()->GetParam(old_index) ==
293             MachineRepresentation::kWord64) {
294           Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
295                                              graph()->start());
296           ReplaceNode(node, node, high_node);
297         }
298       }
299       break;
300     }
301     case IrOpcode::kReturn: {
302       int input_count = node->InputCount();
303       DefaultLowering(node);
304       if (input_count != node->InputCount()) {
305         int new_return_count = GetReturnCountAfterLowering(signature());
306         if (static_cast<int>(signature()->return_count()) != new_return_count) {
307           NodeProperties::ChangeOp(node, common()->Return(new_return_count));
308         }
309       }
310       break;
311     }
312     case IrOpcode::kTailCall: {
313       auto call_descriptor =
314           const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
315       bool returns_require_lowering =
316           GetReturnCountAfterLowering(call_descriptor) !=
317           static_cast<int>(call_descriptor->ReturnCount());
318       if (DefaultLowering(node) || returns_require_lowering) {
319         // Tail calls do not have return values, so adjusting the call
320         // descriptor is enough.
321         auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor);
322         NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
323       }
324       break;
325     }
326     case IrOpcode::kCall: {
327       auto call_descriptor =
328           const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
329       bool returns_require_lowering =
330           GetReturnCountAfterLowering(call_descriptor) !=
331           static_cast<int>(call_descriptor->ReturnCount());
332       if (DefaultLowering(node) || returns_require_lowering) {
333         // We have to adjust the call descriptor.
334         NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor(
335                                            zone(), call_descriptor)));
336       }
337       if (returns_require_lowering) {
338         size_t return_arity = call_descriptor->ReturnCount();
339         if (return_arity == 1) {
340           // We access the additional return values through projections.
341           Node* low_node =
342               graph()->NewNode(common()->Projection(0), node, graph()->start());
343           Node* high_node =
344               graph()->NewNode(common()->Projection(1), node, graph()->start());
345           ReplaceNode(node, low_node, high_node);
346         } else {
347           ZoneVector<Node*> projections(return_arity, zone());
348           NodeProperties::CollectValueProjections(node, projections.data(),
349                                                   return_arity);
350           for (size_t old_index = 0, new_index = 0; old_index < return_arity;
351                ++old_index, ++new_index) {
352             Node* use_node = projections[old_index];
353             DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
354             DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
355                                                   static_cast<int>(old_index)),
356                       static_cast<int>(new_index));
357             if (new_index != old_index) {
358               NodeProperties::ChangeOp(
359                   use_node, common()->Projection(new_index));
360             }
361             if (call_descriptor->GetReturnType(old_index).representation() ==
362                 MachineRepresentation::kWord64) {
363               Node* high_node = graph()->NewNode(
364                   common()->Projection(new_index + 1), node,
365                   graph()->start());
366               ReplaceNode(use_node, use_node, high_node);
367               ++new_index;
368             }
369           }
370         }
371       }
372       break;
373     }
374     case IrOpcode::kWord64And: {
375       DCHECK_EQ(2, node->InputCount());
376       Node* left = node->InputAt(0);
377       Node* right = node->InputAt(1);
378 
379       Node* low_node =
380           graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
381                            GetReplacementLow(right));
382       Node* high_node =
383           graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
384                            GetReplacementHigh(right));
385       ReplaceNode(node, low_node, high_node);
386       break;
387     }
388     case IrOpcode::kTruncateInt64ToInt32: {
389       DCHECK_EQ(1, node->InputCount());
390       Node* input = node->InputAt(0);
391       ReplaceNode(node, GetReplacementLow(input), nullptr);
392       node->NullAllInputs();
393       break;
394     }
395     case IrOpcode::kInt64Add: {
396       DCHECK_EQ(2, node->InputCount());
397 
398       Node* right = node->InputAt(1);
399       node->ReplaceInput(1, GetReplacementLow(right));
400       node->AppendInput(zone(), GetReplacementHigh(right));
401 
402       Node* left = node->InputAt(0);
403       node->ReplaceInput(0, GetReplacementLow(left));
404       node->InsertInput(zone(), 1, GetReplacementHigh(left));
405 
406       NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
407       // We access the additional return values through projections.
408       Node* low_node =
409           graph()->NewNode(common()->Projection(0), node, graph()->start());
410       Node* high_node =
411           graph()->NewNode(common()->Projection(1), node, graph()->start());
412       ReplaceNode(node, low_node, high_node);
413       break;
414     }
415     case IrOpcode::kInt64Sub: {
416       DCHECK_EQ(2, node->InputCount());
417 
418       Node* right = node->InputAt(1);
419       node->ReplaceInput(1, GetReplacementLow(right));
420       node->AppendInput(zone(), GetReplacementHigh(right));
421 
422       Node* left = node->InputAt(0);
423       node->ReplaceInput(0, GetReplacementLow(left));
424       node->InsertInput(zone(), 1, GetReplacementHigh(left));
425 
426       NodeProperties::ChangeOp(node, machine()->Int32PairSub());
427       // We access the additional return values through projections.
428       Node* low_node =
429           graph()->NewNode(common()->Projection(0), node, graph()->start());
430       Node* high_node =
431           graph()->NewNode(common()->Projection(1), node, graph()->start());
432       ReplaceNode(node, low_node, high_node);
433       break;
434     }
435     case IrOpcode::kInt64Mul: {
436       DCHECK_EQ(2, node->InputCount());
437 
438       Node* right = node->InputAt(1);
439       node->ReplaceInput(1, GetReplacementLow(right));
440       node->AppendInput(zone(), GetReplacementHigh(right));
441 
442       Node* left = node->InputAt(0);
443       node->ReplaceInput(0, GetReplacementLow(left));
444       node->InsertInput(zone(), 1, GetReplacementHigh(left));
445 
446       NodeProperties::ChangeOp(node, machine()->Int32PairMul());
447       // We access the additional return values through projections.
448       Node* low_node =
449           graph()->NewNode(common()->Projection(0), node, graph()->start());
450       Node* high_node =
451           graph()->NewNode(common()->Projection(1), node, graph()->start());
452       ReplaceNode(node, low_node, high_node);
453       break;
454     }
455     case IrOpcode::kWord64Or: {
456       DCHECK_EQ(2, node->InputCount());
457       Node* left = node->InputAt(0);
458       Node* right = node->InputAt(1);
459 
460       Node* low_node =
461           graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
462                            GetReplacementLow(right));
463       Node* high_node =
464           graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
465                            GetReplacementHigh(right));
466       ReplaceNode(node, low_node, high_node);
467       break;
468     }
469     case IrOpcode::kWord64Xor: {
470       DCHECK_EQ(2, node->InputCount());
471       Node* left = node->InputAt(0);
472       Node* right = node->InputAt(1);
473 
474       Node* low_node =
475           graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
476                            GetReplacementLow(right));
477       Node* high_node =
478           graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
479                            GetReplacementHigh(right));
480       ReplaceNode(node, low_node, high_node);
481       break;
482     }
483     case IrOpcode::kWord64Shl: {
484       // TODO(turbofan): if the shift count >= 32, then we can set the low word
485       // of the output to 0 and just calculate the high word.
486       DCHECK_EQ(2, node->InputCount());
487       Node* shift = node->InputAt(1);
488       if (HasReplacementLow(shift)) {
489         // We do not have to care about the high word replacement, because
490         // the shift can only be between 0 and 63 anyways.
491         node->ReplaceInput(1, GetReplacementLow(shift));
492       }
493 
494       Node* value = node->InputAt(0);
495       node->ReplaceInput(0, GetReplacementLow(value));
496       node->InsertInput(zone(), 1, GetReplacementHigh(value));
497 
498       NodeProperties::ChangeOp(node, machine()->Word32PairShl());
499       // We access the additional return values through projections.
500       Node* low_node =
501           graph()->NewNode(common()->Projection(0), node, graph()->start());
502       Node* high_node =
503           graph()->NewNode(common()->Projection(1), node, graph()->start());
504       ReplaceNode(node, low_node, high_node);
505       break;
506     }
507     case IrOpcode::kWord64Shr: {
508       // TODO(turbofan): if the shift count >= 32, then we can set the low word
509       // of the output to 0 and just calculate the high word.
510       DCHECK_EQ(2, node->InputCount());
511       Node* shift = node->InputAt(1);
512       if (HasReplacementLow(shift)) {
513         // We do not have to care about the high word replacement, because
514         // the shift can only be between 0 and 63 anyways.
515         node->ReplaceInput(1, GetReplacementLow(shift));
516       }
517 
518       Node* value = node->InputAt(0);
519       node->ReplaceInput(0, GetReplacementLow(value));
520       node->InsertInput(zone(), 1, GetReplacementHigh(value));
521 
522       NodeProperties::ChangeOp(node, machine()->Word32PairShr());
523       // We access the additional return values through projections.
524       Node* low_node =
525           graph()->NewNode(common()->Projection(0), node, graph()->start());
526       Node* high_node =
527           graph()->NewNode(common()->Projection(1), node, graph()->start());
528       ReplaceNode(node, low_node, high_node);
529       break;
530     }
531     case IrOpcode::kWord64Sar: {
532       // TODO(turbofan): if the shift count >= 32, then we can set the low word
533       // of the output to 0 and just calculate the high word.
534       DCHECK_EQ(2, node->InputCount());
535       Node* shift = node->InputAt(1);
536       if (HasReplacementLow(shift)) {
537         // We do not have to care about the high word replacement, because
538         // the shift can only be between 0 and 63 anyways.
539         node->ReplaceInput(1, GetReplacementLow(shift));
540       }
541 
542       Node* value = node->InputAt(0);
543       node->ReplaceInput(0, GetReplacementLow(value));
544       node->InsertInput(zone(), 1, GetReplacementHigh(value));
545 
546       NodeProperties::ChangeOp(node, machine()->Word32PairSar());
547       // We access the additional return values through projections.
548       Node* low_node =
549           graph()->NewNode(common()->Projection(0), node, graph()->start());
550       Node* high_node =
551           graph()->NewNode(common()->Projection(1), node, graph()->start());
552       ReplaceNode(node, low_node, high_node);
553       break;
554     }
555     case IrOpcode::kWord64Equal: {
556       DCHECK_EQ(2, node->InputCount());
557       Node* left = node->InputAt(0);
558       Node* right = node->InputAt(1);
559 
560       // TODO(wasm): Use explicit comparisons and && here?
561       Node* replacement = graph()->NewNode(
562           machine()->Word32Equal(),
563           graph()->NewNode(
564               machine()->Word32Or(),
565               graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
566                                GetReplacementLow(right)),
567               graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
568                                GetReplacementHigh(right))),
569           graph()->NewNode(common()->Int32Constant(0)));
570 
571       ReplaceNode(node, replacement, nullptr);
572       break;
573     }
574     case IrOpcode::kInt64LessThan: {
575       LowerComparison(node, machine()->Int32LessThan(),
576                       machine()->Uint32LessThan());
577       break;
578     }
579     case IrOpcode::kInt64LessThanOrEqual: {
580       LowerComparison(node, machine()->Int32LessThan(),
581                       machine()->Uint32LessThanOrEqual());
582       break;
583     }
584     case IrOpcode::kUint64LessThan: {
585       LowerComparison(node, machine()->Uint32LessThan(),
586                       machine()->Uint32LessThan());
587       break;
588     }
589     case IrOpcode::kUint64LessThanOrEqual: {
590       LowerComparison(node, machine()->Uint32LessThan(),
591                       machine()->Uint32LessThanOrEqual());
592       break;
593     }
594     case IrOpcode::kSignExtendWord32ToInt64:
595     case IrOpcode::kChangeInt32ToInt64: {
596       DCHECK_EQ(1, node->InputCount());
597       Node* input = node->InputAt(0);
598       if (HasReplacementLow(input)) {
599         input = GetReplacementLow(input);
600       }
601       // We use SAR to preserve the sign in the high word.
602       ReplaceNode(
603           node, input,
604           graph()->NewNode(machine()->Word32Sar(), input,
605                            graph()->NewNode(common()->Int32Constant(31))));
606       node->NullAllInputs();
607       break;
608     }
609     case IrOpcode::kChangeUint32ToUint64: {
610       DCHECK_EQ(1, node->InputCount());
611       Node* input = node->InputAt(0);
612       if (HasReplacementLow(input)) {
613         input = GetReplacementLow(input);
614       }
615       ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
616       node->NullAllInputs();
617       break;
618     }
619     case IrOpcode::kBitcastInt64ToFloat64: {
620       DCHECK_EQ(1, node->InputCount());
621       Node* input = node->InputAt(0);
622       Node* stack_slot = graph()->NewNode(
623           machine()->StackSlot(MachineRepresentation::kWord64));
624 
625       Node* store_high_word = graph()->NewNode(
626           machine()->Store(
627               StoreRepresentation(MachineRepresentation::kWord32,
628                                   WriteBarrierKind::kNoWriteBarrier)),
629           stack_slot,
630           graph()->NewNode(
631               common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
632           GetReplacementHigh(input), graph()->start(), graph()->start());
633 
634       Node* store_low_word = graph()->NewNode(
635           machine()->Store(
636               StoreRepresentation(MachineRepresentation::kWord32,
637                                   WriteBarrierKind::kNoWriteBarrier)),
638           stack_slot,
639           graph()->NewNode(
640               common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
641           GetReplacementLow(input), store_high_word, graph()->start());
642 
643       Node* load =
644           graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
645                            graph()->NewNode(common()->Int32Constant(0)),
646                            store_low_word, graph()->start());
647 
648       ReplaceNode(node, load, nullptr);
649       break;
650     }
651     case IrOpcode::kBitcastFloat64ToInt64: {
652       DCHECK_EQ(1, node->InputCount());
653       Node* input = node->InputAt(0);
654       if (HasReplacementLow(input)) {
655         input = GetReplacementLow(input);
656       }
657       Node* stack_slot = graph()->NewNode(
658           machine()->StackSlot(MachineRepresentation::kWord64));
659       Node* store = graph()->NewNode(
660           machine()->Store(
661               StoreRepresentation(MachineRepresentation::kFloat64,
662                                   WriteBarrierKind::kNoWriteBarrier)),
663           stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
664           graph()->start(), graph()->start());
665 
666       Node* high_node = graph()->NewNode(
667           machine()->Load(MachineType::Int32()), stack_slot,
668           graph()->NewNode(
669               common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
670           store, graph()->start());
671 
672       Node* low_node = graph()->NewNode(
673           machine()->Load(MachineType::Int32()), stack_slot,
674           graph()->NewNode(
675               common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
676           store, graph()->start());
677       ReplaceNode(node, low_node, high_node);
678       break;
679     }
680     case IrOpcode::kWord64Ror: {
681       DCHECK_EQ(2, node->InputCount());
682       Node* input = node->InputAt(0);
683       Node* shift = HasReplacementLow(node->InputAt(1))
684                         ? GetReplacementLow(node->InputAt(1))
685                         : node->InputAt(1);
686       Int32Matcher m(shift);
687       if (m.HasValue()) {
688         // Precondition: 0 <= shift < 64.
689         int32_t shift_value = m.Value() & 0x3F;
690         if (shift_value == 0) {
691           ReplaceNode(node, GetReplacementLow(input),
692                       GetReplacementHigh(input));
693         } else if (shift_value == 32) {
694           ReplaceNode(node, GetReplacementHigh(input),
695                       GetReplacementLow(input));
696         } else {
697           Node* low_input;
698           Node* high_input;
699           if (shift_value < 32) {
700             low_input = GetReplacementLow(input);
701             high_input = GetReplacementHigh(input);
702           } else {
703             low_input = GetReplacementHigh(input);
704             high_input = GetReplacementLow(input);
705           }
706           int32_t masked_shift_value = shift_value & 0x1F;
707           Node* masked_shift =
708               graph()->NewNode(common()->Int32Constant(masked_shift_value));
709           Node* inv_shift = graph()->NewNode(
710               common()->Int32Constant(32 - masked_shift_value));
711 
712           Node* low_node = graph()->NewNode(
713               machine()->Word32Or(),
714               graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
715               graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
716           Node* high_node = graph()->NewNode(
717               machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
718                                                       high_input, masked_shift),
719               graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
720           ReplaceNode(node, low_node, high_node);
721         }
722       } else {
723         Node* safe_shift = shift;
724         if (!machine()->Word32ShiftIsSafe()) {
725           safe_shift =
726               graph()->NewNode(machine()->Word32And(), shift,
727                                graph()->NewNode(common()->Int32Constant(0x1F)));
728         }
729 
730         // By creating this bit-mask with SAR and SHL we do not have to deal
731         // with shift == 0 as a special case.
732         Node* inv_mask = graph()->NewNode(
733             machine()->Word32Shl(),
734             graph()->NewNode(machine()->Word32Sar(),
735                              graph()->NewNode(common()->Int32Constant(
736                                  std::numeric_limits<int32_t>::min())),
737                              safe_shift),
738             graph()->NewNode(common()->Int32Constant(1)));
739 
740         Node* bit_mask =
741             graph()->NewNode(machine()->Word32Xor(), inv_mask,
742                              graph()->NewNode(common()->Int32Constant(-1)));
743 
744         // We have to mask the shift value for this comparison. If
745         // !machine()->Word32ShiftIsSafe() then the masking should already be
746         // part of the graph.
747         Node* masked_shift6 = shift;
748         if (machine()->Word32ShiftIsSafe()) {
749           masked_shift6 =
750               graph()->NewNode(machine()->Word32And(), shift,
751                                graph()->NewNode(common()->Int32Constant(0x3F)));
752         }
753 
754         Diamond lt32(
755             graph(), common(),
756             graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
757                              graph()->NewNode(common()->Int32Constant(32))));
758 
759         // The low word and the high word can be swapped either at the input or
760         // at the output. We swap the inputs so that shift does not have to be
761         // kept for so long in a register.
762         Node* input_low =
763             lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
764                      GetReplacementHigh(input));
765         Node* input_high =
766             lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
767                      GetReplacementLow(input));
768 
769         Node* rotate_low =
770             graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
771         Node* rotate_high =
772             graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
773 
774         Node* low_node = graph()->NewNode(
775             machine()->Word32Or(),
776             graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
777             graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
778 
779         Node* high_node = graph()->NewNode(
780             machine()->Word32Or(),
781             graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
782             graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
783 
784         ReplaceNode(node, low_node, high_node);
785       }
786       break;
787     }
788     case IrOpcode::kWord64Clz: {
789       DCHECK_EQ(1, node->InputCount());
790       Node* input = node->InputAt(0);
791       Diamond d(
792           graph(), common(),
793           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
794                            graph()->NewNode(common()->Int32Constant(0))));
795 
796       Node* low_node = d.Phi(
797           MachineRepresentation::kWord32,
798           graph()->NewNode(machine()->Int32Add(),
799                            graph()->NewNode(machine()->Word32Clz(),
800                                             GetReplacementLow(input)),
801                            graph()->NewNode(common()->Int32Constant(32))),
802           graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
803       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
804       break;
805     }
806     case IrOpcode::kWord64Ctz: {
807       DCHECK_EQ(1, node->InputCount());
808       DCHECK(machine()->Word32Ctz().IsSupported());
809       Node* input = node->InputAt(0);
810       Diamond d(
811           graph(), common(),
812           graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
813                            graph()->NewNode(common()->Int32Constant(0))));
814       Node* low_node =
815           d.Phi(MachineRepresentation::kWord32,
816                 graph()->NewNode(machine()->Int32Add(),
817                                  graph()->NewNode(machine()->Word32Ctz().op(),
818                                                   GetReplacementHigh(input)),
819                                  graph()->NewNode(common()->Int32Constant(32))),
820                 graph()->NewNode(machine()->Word32Ctz().op(),
821                                  GetReplacementLow(input)));
822       ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
823       break;
824     }
825     case IrOpcode::kWord64Popcnt: {
826       DCHECK_EQ(1, node->InputCount());
827       Node* input = node->InputAt(0);
828       // We assume that a Word64Popcnt node only has been created if
829       // Word32Popcnt is actually supported.
830       DCHECK(machine()->Word32Popcnt().IsSupported());
831       ReplaceNode(node, graph()->NewNode(
832                             machine()->Int32Add(),
833                             graph()->NewNode(machine()->Word32Popcnt().op(),
834                                              GetReplacementLow(input)),
835                             graph()->NewNode(machine()->Word32Popcnt().op(),
836                                              GetReplacementHigh(input))),
837                   graph()->NewNode(common()->Int32Constant(0)));
838       break;
839     }
840     case IrOpcode::kPhi: {
841       MachineRepresentation rep = PhiRepresentationOf(node->op());
842       if (rep == MachineRepresentation::kWord64) {
843         // The replacement nodes have already been created, we only have to
844         // replace placeholder nodes.
845         Node* low_node = GetReplacementLow(node);
846         Node* high_node = GetReplacementHigh(node);
847         for (int i = 0; i < node->op()->ValueInputCount(); i++) {
848           low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
849           high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
850         }
851       } else {
852         DefaultLowering(node);
853       }
854       break;
855     }
856     case IrOpcode::kWord64ReverseBytes: {
857       Node* input = node->InputAt(0);
858       ReplaceNode(node, graph()->NewNode(machine()->Word32ReverseBytes().op(),
859                                          GetReplacementHigh(input)),
860                   graph()->NewNode(machine()->Word32ReverseBytes().op(),
861                                    GetReplacementLow(input)));
862       break;
863     }
864     case IrOpcode::kSignExtendWord8ToInt64: {
865       DCHECK_EQ(1, node->InputCount());
866       Node* input = node->InputAt(0);
867       if (HasReplacementLow(input)) {
868         input = GetReplacementLow(input);
869       }
870       // Sign extend low node to Int32
871       input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
872 
873       // We use SAR to preserve the sign in the high word.
874       ReplaceNode(
875           node, input,
876           graph()->NewNode(machine()->Word32Sar(), input,
877                            graph()->NewNode(common()->Int32Constant(31))));
878       node->NullAllInputs();
879       break;
880     }
881     case IrOpcode::kSignExtendWord16ToInt64: {
882       DCHECK_EQ(1, node->InputCount());
883       Node* input = node->InputAt(0);
884       if (HasReplacementLow(input)) {
885         input = GetReplacementLow(input);
886       }
887       // Sign extend low node to Int32
888       input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
889 
890       // We use SAR to preserve the sign in the high word.
891       ReplaceNode(
892           node, input,
893           graph()->NewNode(machine()->Word32Sar(), input,
894                            graph()->NewNode(common()->Int32Constant(31))));
895       node->NullAllInputs();
896       break;
897     }
898 
899     default: { DefaultLowering(node); }
900   }
901 }  // NOLINT(readability/fn_size)
902 
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)903 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
904                                     const Operator* low_word_op) {
905   DCHECK_EQ(2, node->InputCount());
906   Node* left = node->InputAt(0);
907   Node* right = node->InputAt(1);
908   Node* replacement = graph()->NewNode(
909       machine()->Word32Or(),
910       graph()->NewNode(high_word_op, GetReplacementHigh(left),
911                        GetReplacementHigh(right)),
912       graph()->NewNode(
913           machine()->Word32And(),
914           graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
915                            GetReplacementHigh(right)),
916           graph()->NewNode(low_word_op, GetReplacementLow(left),
917                            GetReplacementLow(right))));
918 
919   ReplaceNode(node, replacement, nullptr);
920 }
921 
DefaultLowering(Node * node,bool low_word_only)922 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
923   bool something_changed = false;
924   for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
925     Node* input = node->InputAt(i);
926     if (HasReplacementLow(input)) {
927       something_changed = true;
928       node->ReplaceInput(i, GetReplacementLow(input));
929     }
930     if (!low_word_only && HasReplacementHigh(input)) {
931       something_changed = true;
932       node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
933     }
934   }
935   return something_changed;
936 }
937 
ReplaceNode(Node * old,Node * new_low,Node * new_high)938 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
939   // if new_low == nullptr, then also new_high == nullptr.
940   DCHECK(new_low != nullptr || new_high == nullptr);
941   replacements_[old->id()].low = new_low;
942   replacements_[old->id()].high = new_high;
943 }
944 
HasReplacementLow(Node * node)945 bool Int64Lowering::HasReplacementLow(Node* node) {
946   return replacements_[node->id()].low != nullptr;
947 }
948 
GetReplacementLow(Node * node)949 Node* Int64Lowering::GetReplacementLow(Node* node) {
950   Node* result = replacements_[node->id()].low;
951   DCHECK(result);
952   return result;
953 }
954 
HasReplacementHigh(Node * node)955 bool Int64Lowering::HasReplacementHigh(Node* node) {
956   return replacements_[node->id()].high != nullptr;
957 }
958 
GetReplacementHigh(Node * node)959 Node* Int64Lowering::GetReplacementHigh(Node* node) {
960   Node* result = replacements_[node->id()].high;
961   DCHECK(result);
962   return result;
963 }
964 
PreparePhiReplacement(Node * phi)965 void Int64Lowering::PreparePhiReplacement(Node* phi) {
966   MachineRepresentation rep = PhiRepresentationOf(phi->op());
967   if (rep == MachineRepresentation::kWord64) {
968     // We have to create the replacements for a phi node before we actually
969     // lower the phi to break potential cycles in the graph. The replacements of
970     // input nodes do not exist yet, so we use a placeholder node to pass the
971     // graph verifier.
972     int value_count = phi->op()->ValueInputCount();
973     Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
974     Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
975     for (int i = 0; i < value_count; i++) {
976       inputs_low[i] = placeholder_;
977       inputs_high[i] = placeholder_;
978     }
979     inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
980     inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
981     ReplaceNode(phi,
982                 graph()->NewNode(
983                     common()->Phi(MachineRepresentation::kWord32, value_count),
984                     value_count + 1, inputs_low, false),
985                 graph()->NewNode(
986                     common()->Phi(MachineRepresentation::kWord32, value_count),
987                     value_count + 1, inputs_high, false));
988   }
989 }
990 }  // namespace compiler
991 }  // namespace internal
992 }  // namespace v8
993