1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/int64-lowering.h"
6
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature,std::unique_ptr<Int64LoweringSpecialCase> special_case)24 Int64Lowering::Int64Lowering(
25 Graph* graph, MachineOperatorBuilder* machine,
26 CommonOperatorBuilder* common, Zone* zone,
27 Signature<MachineRepresentation>* signature,
28 std::unique_ptr<Int64LoweringSpecialCase> special_case)
29 : zone_(zone),
30 graph_(graph),
31 machine_(machine),
32 common_(common),
33 state_(graph, 3),
34 stack_(zone),
35 replacements_(nullptr),
36 signature_(signature),
37 placeholder_(
38 graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
39 special_case_(std::move(special_case)) {
40 DCHECK_NOT_NULL(graph);
41 DCHECK_NOT_NULL(graph->end());
42 replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
43 memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
44 }
45
LowerGraph()46 void Int64Lowering::LowerGraph() {
47 if (!machine()->Is32()) {
48 return;
49 }
50 stack_.push_back({graph()->end(), 0});
51 state_.Set(graph()->end(), State::kOnStack);
52
53 while (!stack_.empty()) {
54 NodeState& top = stack_.back();
55 if (top.input_index == top.node->InputCount()) {
56 // All inputs of top have already been lowered, now lower top.
57 stack_.pop_back();
58 state_.Set(top.node, State::kVisited);
59 LowerNode(top.node);
60 } else {
61 // Push the next input onto the stack.
62 Node* input = top.node->InputAt(top.input_index++);
63 if (state_.Get(input) == State::kUnvisited) {
64 if (input->opcode() == IrOpcode::kPhi) {
65 // To break cycles with phi nodes we push phis on a separate stack so
66 // that they are processed after all other nodes.
67 PreparePhiReplacement(input);
68 stack_.push_front({input, 0});
69 } else if (input->opcode() == IrOpcode::kEffectPhi ||
70 input->opcode() == IrOpcode::kLoop) {
71 stack_.push_front({input, 0});
72 } else {
73 stack_.push_back({input, 0});
74 }
75 state_.Set(input, State::kOnStack);
76 }
77 }
78 }
79 }
80
81 namespace {
82
GetReturnIndexAfterLowering(const CallDescriptor * call_descriptor,int old_index)83 int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
84 int old_index) {
85 int result = old_index;
86 for (int i = 0; i < old_index; i++) {
87 if (call_descriptor->GetReturnType(i).representation() ==
88 MachineRepresentation::kWord64) {
89 result++;
90 }
91 }
92 return result;
93 }
94
GetReturnCountAfterLowering(const CallDescriptor * call_descriptor)95 int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
96 return GetReturnIndexAfterLowering(
97 call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
98 }
99
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)100 int GetParameterIndexAfterLowering(
101 Signature<MachineRepresentation>* signature, int old_index) {
102 int result = old_index;
103 // Be robust towards special indexes (>= param count).
104 int max_to_check =
105 std::min(old_index, static_cast<int>(signature->parameter_count()));
106 for (int i = 0; i < max_to_check; i++) {
107 if (signature->GetParam(i) == MachineRepresentation::kWord64) {
108 result++;
109 }
110 }
111 return result;
112 }
113
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)114 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
115 int result = static_cast<int>(signature->return_count());
116 for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
117 if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
118 result++;
119 }
120 }
121 return result;
122 }
123
124 } // namespace
125
LowerWord64AtomicBinop(Node * node,const Operator * op)126 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
127 DCHECK_EQ(5, node->InputCount());
128 LowerMemoryBaseAndIndex(node);
129 Node* value = node->InputAt(2);
130 node->ReplaceInput(2, GetReplacementLow(value));
131 node->InsertInput(zone(), 3, GetReplacementHigh(value));
132 NodeProperties::ChangeOp(node, op);
133 ReplaceNodeWithProjections(node);
134 }
135
LowerWord64AtomicNarrowOp(Node * node,const Operator * op)136 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
137 DefaultLowering(node, true);
138 NodeProperties::ChangeOp(node, op);
139 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
140 }
141
142 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)143 int Int64Lowering::GetParameterCountAfterLowering(
144 Signature<MachineRepresentation>* signature) {
145 // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
146 // after lowering.
147 return GetParameterIndexAfterLowering(
148 signature, static_cast<int>(signature->parameter_count()));
149 }
150
GetIndexNodes(Node * index,Node ** index_low,Node ** index_high)151 void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
152 Node** index_high) {
153 #if defined(V8_TARGET_LITTLE_ENDIAN)
154 *index_low = index;
155 *index_high = graph()->NewNode(machine()->Int32Add(), index,
156 graph()->NewNode(common()->Int32Constant(4)));
157 #elif defined(V8_TARGET_BIG_ENDIAN)
158 *index_low = graph()->NewNode(machine()->Int32Add(), index,
159 graph()->NewNode(common()->Int32Constant(4)));
160 *index_high = index;
161 #endif
162 }
163
LowerNode(Node * node)164 void Int64Lowering::LowerNode(Node* node) {
165 switch (node->opcode()) {
166 case IrOpcode::kInt64Constant: {
167 int64_t value = OpParameter<int64_t>(node->op());
168 Node* low_node = graph()->NewNode(
169 common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
170 Node* high_node = graph()->NewNode(
171 common()->Int32Constant(static_cast<int32_t>(value >> 32)));
172 ReplaceNode(node, low_node, high_node);
173 break;
174 }
175 case IrOpcode::kLoad:
176 case IrOpcode::kUnalignedLoad: {
177 MachineRepresentation rep;
178 if (node->opcode() == IrOpcode::kLoad) {
179 rep = LoadRepresentationOf(node->op()).representation();
180 } else {
181 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
182 rep = LoadRepresentationOf(node->op()).representation();
183 }
184
185 if (rep == MachineRepresentation::kWord64) {
186 LowerMemoryBaseAndIndex(node);
187 Node* base = node->InputAt(0);
188 Node* index = node->InputAt(1);
189 Node* index_low;
190 Node* index_high;
191 GetIndexNodes(index, &index_low, &index_high);
192 const Operator* load_op;
193
194 if (node->opcode() == IrOpcode::kLoad) {
195 load_op = machine()->Load(MachineType::Int32());
196 } else {
197 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
198 load_op = machine()->UnalignedLoad(MachineType::Int32());
199 }
200
201 Node* high_node;
202 if (node->InputCount() > 2) {
203 Node* effect_high = node->InputAt(2);
204 Node* control_high = node->InputAt(3);
205 high_node = graph()->NewNode(load_op, base, index_high, effect_high,
206 control_high);
207 // change the effect change from old_node --> old_effect to
208 // old_node --> high_node --> old_effect.
209 node->ReplaceInput(2, high_node);
210 } else {
211 high_node = graph()->NewNode(load_op, base, index_high);
212 }
213 node->ReplaceInput(1, index_low);
214 NodeProperties::ChangeOp(node, load_op);
215 ReplaceNode(node, node, high_node);
216 } else {
217 DefaultLowering(node);
218 }
219 break;
220 }
221 case IrOpcode::kStore:
222 case IrOpcode::kUnalignedStore: {
223 MachineRepresentation rep;
224 if (node->opcode() == IrOpcode::kStore) {
225 rep = StoreRepresentationOf(node->op()).representation();
226 } else {
227 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
228 rep = UnalignedStoreRepresentationOf(node->op());
229 }
230
231 if (rep == MachineRepresentation::kWord64) {
232 // We change the original store node to store the low word, and create
233 // a new store node to store the high word. The effect and control edges
234 // are copied from the original store to the new store node, the effect
235 // edge of the original store is redirected to the new store.
236 LowerMemoryBaseAndIndex(node);
237 Node* base = node->InputAt(0);
238 Node* index = node->InputAt(1);
239 Node* index_low;
240 Node* index_high;
241 GetIndexNodes(index, &index_low, &index_high);
242 Node* value = node->InputAt(2);
243 DCHECK(HasReplacementLow(value));
244 DCHECK(HasReplacementHigh(value));
245
246 const Operator* store_op;
247 if (node->opcode() == IrOpcode::kStore) {
248 WriteBarrierKind write_barrier_kind =
249 StoreRepresentationOf(node->op()).write_barrier_kind();
250 store_op = machine()->Store(StoreRepresentation(
251 MachineRepresentation::kWord32, write_barrier_kind));
252 } else {
253 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
254 store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
255 }
256
257 Node* high_node;
258 if (node->InputCount() > 3) {
259 Node* effect_high = node->InputAt(3);
260 Node* control_high = node->InputAt(4);
261 high_node = graph()->NewNode(store_op, base, index_high,
262 GetReplacementHigh(value), effect_high,
263 control_high);
264 node->ReplaceInput(3, high_node);
265
266 } else {
267 high_node = graph()->NewNode(store_op, base, index_high,
268 GetReplacementHigh(value));
269 }
270
271 node->ReplaceInput(1, index_low);
272 node->ReplaceInput(2, GetReplacementLow(value));
273 NodeProperties::ChangeOp(node, store_op);
274 ReplaceNode(node, node, high_node);
275 } else {
276 DefaultLowering(node, true);
277 }
278 break;
279 }
280 case IrOpcode::kStart: {
281 int parameter_count = GetParameterCountAfterLowering(signature());
282 // Only exchange the node if the parameter count actually changed.
283 if (parameter_count != static_cast<int>(signature()->parameter_count())) {
284 int delta =
285 parameter_count - static_cast<int>(signature()->parameter_count());
286 int new_output_count = node->op()->ValueOutputCount() + delta;
287 NodeProperties::ChangeOp(node, common()->Start(new_output_count));
288 }
289 break;
290 }
291 case IrOpcode::kParameter: {
292 DCHECK_EQ(1, node->InputCount());
293 int param_count = static_cast<int>(signature()->parameter_count());
294 // Only exchange the node if the parameter count actually changed. We do
295 // not even have to do the default lowering because the the start node,
296 // the only input of a parameter node, only changes if the parameter count
297 // changes.
298 if (GetParameterCountAfterLowering(signature()) != param_count) {
299 int old_index = ParameterIndexOf(node->op());
300 // Adjust old_index to be compliant with the signature.
301 --old_index;
302 int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303 // Adjust new_index to consider the instance parameter.
304 ++new_index;
305 NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306
307 if (old_index < 0 || old_index >= param_count) {
308 // Special parameters (JS closure/context) don't have kWord64
309 // representation anyway.
310 break;
311 }
312
313 if (signature()->GetParam(old_index) ==
314 MachineRepresentation::kWord64) {
315 Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
316 graph()->start());
317 ReplaceNode(node, node, high_node);
318 }
319 }
320 break;
321 }
322 case IrOpcode::kReturn: {
323 int input_count = node->InputCount();
324 DefaultLowering(node);
325 if (input_count != node->InputCount()) {
326 int new_return_count = GetReturnCountAfterLowering(signature());
327 if (static_cast<int>(signature()->return_count()) != new_return_count) {
328 NodeProperties::ChangeOp(node, common()->Return(new_return_count));
329 }
330 }
331 break;
332 }
333 case IrOpcode::kTailCall: {
334 auto call_descriptor =
335 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
336 bool returns_require_lowering =
337 GetReturnCountAfterLowering(call_descriptor) !=
338 static_cast<int>(call_descriptor->ReturnCount());
339 if (DefaultLowering(node) || returns_require_lowering) {
340 // Tail calls do not have return values, so adjusting the call
341 // descriptor is enough.
342 NodeProperties::ChangeOp(
343 node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
344 }
345 break;
346 }
347 case IrOpcode::kCall: {
348 auto call_descriptor = CallDescriptorOf(node->op());
349
350 bool returns_require_lowering =
351 GetReturnCountAfterLowering(call_descriptor) !=
352 static_cast<int>(call_descriptor->ReturnCount());
353 if (DefaultLowering(node) || returns_require_lowering) {
354 // We have to adjust the call descriptor.
355 NodeProperties::ChangeOp(
356 node, common()->Call(LowerCallDescriptor(call_descriptor)));
357 }
358 if (returns_require_lowering) {
359 size_t return_arity = call_descriptor->ReturnCount();
360 if (return_arity == 1) {
361 // We access the additional return values through projections.
362 ReplaceNodeWithProjections(node);
363 } else {
364 ZoneVector<Node*> projections(return_arity, zone());
365 NodeProperties::CollectValueProjections(node, projections.data(),
366 return_arity);
367 for (size_t old_index = 0, new_index = 0; old_index < return_arity;
368 ++old_index, ++new_index) {
369 Node* use_node = projections[old_index];
370 DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
371 DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
372 static_cast<int>(old_index)),
373 static_cast<int>(new_index));
374 if (new_index != old_index) {
375 NodeProperties::ChangeOp(
376 use_node, common()->Projection(new_index));
377 }
378 if (call_descriptor->GetReturnType(old_index).representation() ==
379 MachineRepresentation::kWord64) {
380 Node* high_node = graph()->NewNode(
381 common()->Projection(new_index + 1), node,
382 graph()->start());
383 ReplaceNode(use_node, use_node, high_node);
384 ++new_index;
385 }
386 }
387 }
388 }
389 break;
390 }
391 case IrOpcode::kWord64And: {
392 DCHECK_EQ(2, node->InputCount());
393 Node* left = node->InputAt(0);
394 Node* right = node->InputAt(1);
395
396 Node* low_node =
397 graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
398 GetReplacementLow(right));
399 Node* high_node =
400 graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
401 GetReplacementHigh(right));
402 ReplaceNode(node, low_node, high_node);
403 break;
404 }
405 case IrOpcode::kTruncateInt64ToInt32: {
406 DCHECK_EQ(1, node->InputCount());
407 Node* input = node->InputAt(0);
408 ReplaceNode(node, GetReplacementLow(input), nullptr);
409 node->NullAllInputs();
410 break;
411 }
412 case IrOpcode::kInt64Add: {
413 DCHECK_EQ(2, node->InputCount());
414
415 Node* right = node->InputAt(1);
416 node->ReplaceInput(1, GetReplacementLow(right));
417 node->AppendInput(zone(), GetReplacementHigh(right));
418
419 Node* left = node->InputAt(0);
420 node->ReplaceInput(0, GetReplacementLow(left));
421 node->InsertInput(zone(), 1, GetReplacementHigh(left));
422
423 NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
424 // We access the additional return values through projections.
425 ReplaceNodeWithProjections(node);
426 break;
427 }
428 case IrOpcode::kInt64Sub: {
429 DCHECK_EQ(2, node->InputCount());
430
431 Node* right = node->InputAt(1);
432 node->ReplaceInput(1, GetReplacementLow(right));
433 node->AppendInput(zone(), GetReplacementHigh(right));
434
435 Node* left = node->InputAt(0);
436 node->ReplaceInput(0, GetReplacementLow(left));
437 node->InsertInput(zone(), 1, GetReplacementHigh(left));
438
439 NodeProperties::ChangeOp(node, machine()->Int32PairSub());
440 // We access the additional return values through projections.
441 ReplaceNodeWithProjections(node);
442 break;
443 }
444 case IrOpcode::kInt64Mul: {
445 DCHECK_EQ(2, node->InputCount());
446
447 Node* right = node->InputAt(1);
448 node->ReplaceInput(1, GetReplacementLow(right));
449 node->AppendInput(zone(), GetReplacementHigh(right));
450
451 Node* left = node->InputAt(0);
452 node->ReplaceInput(0, GetReplacementLow(left));
453 node->InsertInput(zone(), 1, GetReplacementHigh(left));
454
455 NodeProperties::ChangeOp(node, machine()->Int32PairMul());
456 // We access the additional return values through projections.
457 ReplaceNodeWithProjections(node);
458 break;
459 }
460 case IrOpcode::kWord64Or: {
461 DCHECK_EQ(2, node->InputCount());
462 Node* left = node->InputAt(0);
463 Node* right = node->InputAt(1);
464
465 Node* low_node =
466 graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
467 GetReplacementLow(right));
468 Node* high_node =
469 graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
470 GetReplacementHigh(right));
471 ReplaceNode(node, low_node, high_node);
472 break;
473 }
474 case IrOpcode::kWord64Xor: {
475 DCHECK_EQ(2, node->InputCount());
476 Node* left = node->InputAt(0);
477 Node* right = node->InputAt(1);
478
479 Node* low_node =
480 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
481 GetReplacementLow(right));
482 Node* high_node =
483 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
484 GetReplacementHigh(right));
485 ReplaceNode(node, low_node, high_node);
486 break;
487 }
488 case IrOpcode::kWord64Shl: {
489 // TODO(turbofan): if the shift count >= 32, then we can set the low word
490 // of the output to 0 and just calculate the high word.
491 DCHECK_EQ(2, node->InputCount());
492 Node* shift = node->InputAt(1);
493 if (HasReplacementLow(shift)) {
494 // We do not have to care about the high word replacement, because
495 // the shift can only be between 0 and 63 anyways.
496 node->ReplaceInput(1, GetReplacementLow(shift));
497 }
498
499 Node* value = node->InputAt(0);
500 node->ReplaceInput(0, GetReplacementLow(value));
501 node->InsertInput(zone(), 1, GetReplacementHigh(value));
502
503 NodeProperties::ChangeOp(node, machine()->Word32PairShl());
504 // We access the additional return values through projections.
505 ReplaceNodeWithProjections(node);
506 break;
507 }
508 case IrOpcode::kWord64Shr: {
509 // TODO(turbofan): if the shift count >= 32, then we can set the low word
510 // of the output to 0 and just calculate the high word.
511 DCHECK_EQ(2, node->InputCount());
512 Node* shift = node->InputAt(1);
513 if (HasReplacementLow(shift)) {
514 // We do not have to care about the high word replacement, because
515 // the shift can only be between 0 and 63 anyways.
516 node->ReplaceInput(1, GetReplacementLow(shift));
517 }
518
519 Node* value = node->InputAt(0);
520 node->ReplaceInput(0, GetReplacementLow(value));
521 node->InsertInput(zone(), 1, GetReplacementHigh(value));
522
523 NodeProperties::ChangeOp(node, machine()->Word32PairShr());
524 // We access the additional return values through projections.
525 ReplaceNodeWithProjections(node);
526 break;
527 }
528 case IrOpcode::kWord64Sar: {
529 // TODO(turbofan): if the shift count >= 32, then we can set the low word
530 // of the output to 0 and just calculate the high word.
531 DCHECK_EQ(2, node->InputCount());
532 Node* shift = node->InputAt(1);
533 if (HasReplacementLow(shift)) {
534 // We do not have to care about the high word replacement, because
535 // the shift can only be between 0 and 63 anyways.
536 node->ReplaceInput(1, GetReplacementLow(shift));
537 }
538
539 Node* value = node->InputAt(0);
540 node->ReplaceInput(0, GetReplacementLow(value));
541 node->InsertInput(zone(), 1, GetReplacementHigh(value));
542
543 NodeProperties::ChangeOp(node, machine()->Word32PairSar());
544 // We access the additional return values through projections.
545 ReplaceNodeWithProjections(node);
546 break;
547 }
548 case IrOpcode::kWord64Equal: {
549 DCHECK_EQ(2, node->InputCount());
550 Node* left = node->InputAt(0);
551 Node* right = node->InputAt(1);
552
553 // TODO(wasm): Use explicit comparisons and && here?
554 Node* replacement = graph()->NewNode(
555 machine()->Word32Equal(),
556 graph()->NewNode(
557 machine()->Word32Or(),
558 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
559 GetReplacementLow(right)),
560 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
561 GetReplacementHigh(right))),
562 graph()->NewNode(common()->Int32Constant(0)));
563
564 ReplaceNode(node, replacement, nullptr);
565 break;
566 }
567 case IrOpcode::kInt64LessThan: {
568 LowerComparison(node, machine()->Int32LessThan(),
569 machine()->Uint32LessThan());
570 break;
571 }
572 case IrOpcode::kInt64LessThanOrEqual: {
573 LowerComparison(node, machine()->Int32LessThan(),
574 machine()->Uint32LessThanOrEqual());
575 break;
576 }
577 case IrOpcode::kUint64LessThan: {
578 LowerComparison(node, machine()->Uint32LessThan(),
579 machine()->Uint32LessThan());
580 break;
581 }
582 case IrOpcode::kUint64LessThanOrEqual: {
583 LowerComparison(node, machine()->Uint32LessThan(),
584 machine()->Uint32LessThanOrEqual());
585 break;
586 }
587 case IrOpcode::kSignExtendWord32ToInt64:
588 case IrOpcode::kChangeInt32ToInt64: {
589 DCHECK_EQ(1, node->InputCount());
590 Node* input = node->InputAt(0);
591 if (HasReplacementLow(input)) {
592 input = GetReplacementLow(input);
593 }
594 // We use SAR to preserve the sign in the high word.
595 ReplaceNode(
596 node, input,
597 graph()->NewNode(machine()->Word32Sar(), input,
598 graph()->NewNode(common()->Int32Constant(31))));
599 node->NullAllInputs();
600 break;
601 }
602 case IrOpcode::kChangeUint32ToUint64: {
603 DCHECK_EQ(1, node->InputCount());
604 Node* input = node->InputAt(0);
605 if (HasReplacementLow(input)) {
606 input = GetReplacementLow(input);
607 }
608 ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
609 node->NullAllInputs();
610 break;
611 }
612 case IrOpcode::kBitcastInt64ToFloat64: {
613 DCHECK_EQ(1, node->InputCount());
614 Node* input = node->InputAt(0);
615 Node* stack_slot = graph()->NewNode(
616 machine()->StackSlot(MachineRepresentation::kWord64));
617
618 Node* store_high_word = graph()->NewNode(
619 machine()->Store(
620 StoreRepresentation(MachineRepresentation::kWord32,
621 WriteBarrierKind::kNoWriteBarrier)),
622 stack_slot,
623 graph()->NewNode(
624 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
625 GetReplacementHigh(input), graph()->start(), graph()->start());
626
627 Node* store_low_word = graph()->NewNode(
628 machine()->Store(
629 StoreRepresentation(MachineRepresentation::kWord32,
630 WriteBarrierKind::kNoWriteBarrier)),
631 stack_slot,
632 graph()->NewNode(
633 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
634 GetReplacementLow(input), store_high_word, graph()->start());
635
636 Node* load =
637 graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
638 graph()->NewNode(common()->Int32Constant(0)),
639 store_low_word, graph()->start());
640
641 ReplaceNode(node, load, nullptr);
642 break;
643 }
644 case IrOpcode::kBitcastFloat64ToInt64: {
645 DCHECK_EQ(1, node->InputCount());
646 Node* input = node->InputAt(0);
647 if (HasReplacementLow(input)) {
648 input = GetReplacementLow(input);
649 }
650 Node* stack_slot = graph()->NewNode(
651 machine()->StackSlot(MachineRepresentation::kWord64));
652 Node* store = graph()->NewNode(
653 machine()->Store(
654 StoreRepresentation(MachineRepresentation::kFloat64,
655 WriteBarrierKind::kNoWriteBarrier)),
656 stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
657 graph()->start(), graph()->start());
658
659 Node* high_node = graph()->NewNode(
660 machine()->Load(MachineType::Int32()), stack_slot,
661 graph()->NewNode(
662 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
663 store, graph()->start());
664
665 Node* low_node = graph()->NewNode(
666 machine()->Load(MachineType::Int32()), stack_slot,
667 graph()->NewNode(
668 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
669 store, graph()->start());
670 ReplaceNode(node, low_node, high_node);
671 break;
672 }
673 case IrOpcode::kWord64Rol:
674 DCHECK(machine()->Word32Rol().IsSupported());
675 V8_FALLTHROUGH;
676 case IrOpcode::kWord64Ror: {
677 DCHECK_EQ(2, node->InputCount());
678 Node* input = node->InputAt(0);
679 Node* shift = HasReplacementLow(node->InputAt(1))
680 ? GetReplacementLow(node->InputAt(1))
681 : node->InputAt(1);
682 Int32Matcher m(shift);
683 if (m.HasResolvedValue()) {
684 // Precondition: 0 <= shift < 64.
685 int32_t shift_value = m.ResolvedValue() & 0x3F;
686 if (shift_value == 0) {
687 ReplaceNode(node, GetReplacementLow(input),
688 GetReplacementHigh(input));
689 } else if (shift_value == 32) {
690 ReplaceNode(node, GetReplacementHigh(input),
691 GetReplacementLow(input));
692 } else {
693 Node* low_input;
694 Node* high_input;
695 if (shift_value < 32) {
696 low_input = GetReplacementLow(input);
697 high_input = GetReplacementHigh(input);
698 } else {
699 low_input = GetReplacementHigh(input);
700 high_input = GetReplacementLow(input);
701 }
702 int32_t masked_shift_value = shift_value & 0x1F;
703 Node* masked_shift =
704 graph()->NewNode(common()->Int32Constant(masked_shift_value));
705 Node* inv_shift = graph()->NewNode(
706 common()->Int32Constant(32 - masked_shift_value));
707
708 auto* op1 = machine()->Word32Shr();
709 auto* op2 = machine()->Word32Shl();
710 bool is_ror = node->opcode() == IrOpcode::kWord64Ror;
711 if (!is_ror) std::swap(op1, op2);
712
713 Node* low_node =
714 graph()->NewNode(machine()->Word32Or(),
715 graph()->NewNode(op1, low_input, masked_shift),
716 graph()->NewNode(op2, high_input, inv_shift));
717 Node* high_node =
718 graph()->NewNode(machine()->Word32Or(),
719 graph()->NewNode(op1, high_input, masked_shift),
720 graph()->NewNode(op2, low_input, inv_shift));
721 ReplaceNode(node, low_node, high_node);
722 }
723 } else {
724 Node* safe_shift = shift;
725 if (!machine()->Word32ShiftIsSafe()) {
726 safe_shift =
727 graph()->NewNode(machine()->Word32And(), shift,
728 graph()->NewNode(common()->Int32Constant(0x1F)));
729 }
730
731 bool is_ror = node->opcode() == IrOpcode::kWord64Ror;
732 Node* inv_mask =
733 is_ror ? graph()->NewNode(
734 machine()->Word32Xor(),
735 graph()->NewNode(
736 machine()->Word32Shr(),
737 graph()->NewNode(common()->Int32Constant(-1)),
738 safe_shift),
739 graph()->NewNode(common()->Int32Constant(-1)))
740 : graph()->NewNode(
741 machine()->Word32Shl(),
742 graph()->NewNode(common()->Int32Constant(-1)),
743 safe_shift);
744
745 Node* bit_mask =
746 graph()->NewNode(machine()->Word32Xor(), inv_mask,
747 graph()->NewNode(common()->Int32Constant(-1)));
748
749 // We have to mask the shift value for this comparison. If
750 // !machine()->Word32ShiftIsSafe() then the masking should already be
751 // part of the graph.
752 Node* masked_shift6 = shift;
753 if (machine()->Word32ShiftIsSafe()) {
754 masked_shift6 =
755 graph()->NewNode(machine()->Word32And(), shift,
756 graph()->NewNode(common()->Int32Constant(0x3F)));
757 }
758
759 Diamond lt32(
760 graph(), common(),
761 graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
762 graph()->NewNode(common()->Int32Constant(32))));
763
764 // The low word and the high word can be swapped either at the input or
765 // at the output. We swap the inputs so that shift does not have to be
766 // kept for so long in a register.
767 Node* input_low =
768 lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
769 GetReplacementHigh(input));
770 Node* input_high =
771 lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
772 GetReplacementLow(input));
773
774 const Operator* oper =
775 is_ror ? machine()->Word32Ror() : machine()->Word32Rol().op();
776
777 Node* rotate_low = graph()->NewNode(oper, input_low, safe_shift);
778 Node* rotate_high = graph()->NewNode(oper, input_high, safe_shift);
779
780 auto* mask1 = bit_mask;
781 auto* mask2 = inv_mask;
782 if (!is_ror) std::swap(mask1, mask2);
783
784 Node* low_node = graph()->NewNode(
785 machine()->Word32Or(),
786 graph()->NewNode(machine()->Word32And(), rotate_low, mask1),
787 graph()->NewNode(machine()->Word32And(), rotate_high, mask2));
788 Node* high_node = graph()->NewNode(
789 machine()->Word32Or(),
790 graph()->NewNode(machine()->Word32And(), rotate_high, mask1),
791 graph()->NewNode(machine()->Word32And(), rotate_low, mask2));
792 ReplaceNode(node, low_node, high_node);
793 }
794 break;
795 }
796 case IrOpcode::kWord64Clz: {
797 DCHECK_EQ(1, node->InputCount());
798 Node* input = node->InputAt(0);
799 Diamond d(
800 graph(), common(),
801 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
802 graph()->NewNode(common()->Int32Constant(0))));
803
804 Node* low_node = d.Phi(
805 MachineRepresentation::kWord32,
806 graph()->NewNode(machine()->Int32Add(),
807 graph()->NewNode(machine()->Word32Clz(),
808 GetReplacementLow(input)),
809 graph()->NewNode(common()->Int32Constant(32))),
810 graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
811 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
812 break;
813 }
814 case IrOpcode::kWord64Ctz: {
815 DCHECK_EQ(1, node->InputCount());
816 DCHECK(machine()->Word32Ctz().IsSupported());
817 Node* input = node->InputAt(0);
818 Diamond d(
819 graph(), common(),
820 graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
821 graph()->NewNode(common()->Int32Constant(0))));
822 Node* low_node =
823 d.Phi(MachineRepresentation::kWord32,
824 graph()->NewNode(machine()->Int32Add(),
825 graph()->NewNode(machine()->Word32Ctz().op(),
826 GetReplacementHigh(input)),
827 graph()->NewNode(common()->Int32Constant(32))),
828 graph()->NewNode(machine()->Word32Ctz().op(),
829 GetReplacementLow(input)));
830 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
831 break;
832 }
833 case IrOpcode::kWord64Popcnt: {
834 DCHECK_EQ(1, node->InputCount());
835 Node* input = node->InputAt(0);
836 // We assume that a Word64Popcnt node only has been created if
837 // Word32Popcnt is actually supported.
838 DCHECK(machine()->Word32Popcnt().IsSupported());
839 ReplaceNode(node, graph()->NewNode(
840 machine()->Int32Add(),
841 graph()->NewNode(machine()->Word32Popcnt().op(),
842 GetReplacementLow(input)),
843 graph()->NewNode(machine()->Word32Popcnt().op(),
844 GetReplacementHigh(input))),
845 graph()->NewNode(common()->Int32Constant(0)));
846 break;
847 }
848 case IrOpcode::kPhi: {
849 MachineRepresentation rep = PhiRepresentationOf(node->op());
850 if (rep == MachineRepresentation::kWord64) {
851 // The replacement nodes have already been created, we only have to
852 // replace placeholder nodes.
853 Node* low_node = GetReplacementLow(node);
854 Node* high_node = GetReplacementHigh(node);
855 for (int i = 0; i < node->op()->ValueInputCount(); i++) {
856 low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
857 high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
858 }
859 } else {
860 DefaultLowering(node);
861 }
862 break;
863 }
864 case IrOpcode::kWord64ReverseBytes: {
865 Node* input = node->InputAt(0);
866 ReplaceNode(node,
867 graph()->NewNode(machine()->Word32ReverseBytes(),
868 GetReplacementHigh(input)),
869 graph()->NewNode(machine()->Word32ReverseBytes(),
870 GetReplacementLow(input)));
871 break;
872 }
873 case IrOpcode::kSignExtendWord8ToInt64: {
874 DCHECK_EQ(1, node->InputCount());
875 Node* input = node->InputAt(0);
876 if (HasReplacementLow(input)) {
877 input = GetReplacementLow(input);
878 }
879 // Sign extend low node to Int32
880 input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
881
882 // We use SAR to preserve the sign in the high word.
883 ReplaceNode(
884 node, input,
885 graph()->NewNode(machine()->Word32Sar(), input,
886 graph()->NewNode(common()->Int32Constant(31))));
887 node->NullAllInputs();
888 break;
889 }
890 case IrOpcode::kSignExtendWord16ToInt64: {
891 DCHECK_EQ(1, node->InputCount());
892 Node* input = node->InputAt(0);
893 if (HasReplacementLow(input)) {
894 input = GetReplacementLow(input);
895 }
896 // Sign extend low node to Int32
897 input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
898
899 // We use SAR to preserve the sign in the high word.
900 ReplaceNode(
901 node, input,
902 graph()->NewNode(machine()->Word32Sar(), input,
903 graph()->NewNode(common()->Int32Constant(31))));
904 node->NullAllInputs();
905 break;
906 }
907 case IrOpcode::kWord64AtomicLoad: {
908 DCHECK_EQ(4, node->InputCount());
909 MachineType type = AtomicOpType(node->op());
910 DefaultLowering(node, true);
911 if (type == MachineType::Uint64()) {
912 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
913 ReplaceNodeWithProjections(node);
914 } else {
915 NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
916 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
917 }
918 break;
919 }
920 case IrOpcode::kWord64AtomicStore: {
921 DCHECK_EQ(5, node->InputCount());
922 MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
923 if (rep == MachineRepresentation::kWord64) {
924 LowerMemoryBaseAndIndex(node);
925 Node* value = node->InputAt(2);
926 node->ReplaceInput(2, GetReplacementLow(value));
927 node->InsertInput(zone(), 3, GetReplacementHigh(value));
928 NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
929 } else {
930 DefaultLowering(node, true);
931 NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
932 }
933 break;
934 }
935 #define ATOMIC_CASE(name) \
936 case IrOpcode::kWord64Atomic##name: { \
937 MachineType type = AtomicOpType(node->op()); \
938 if (type == MachineType::Uint64()) { \
939 LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \
940 } else { \
941 LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
942 } \
943 break; \
944 }
945 ATOMIC_CASE(Add)
946 ATOMIC_CASE(Sub)
947 ATOMIC_CASE(And)
948 ATOMIC_CASE(Or)
949 ATOMIC_CASE(Xor)
950 ATOMIC_CASE(Exchange)
951 #undef ATOMIC_CASE
952 case IrOpcode::kWord64AtomicCompareExchange: {
953 MachineType type = AtomicOpType(node->op());
954 if (type == MachineType::Uint64()) {
955 LowerMemoryBaseAndIndex(node);
956 Node* old_value = node->InputAt(2);
957 Node* new_value = node->InputAt(3);
958 node->ReplaceInput(2, GetReplacementLow(old_value));
959 node->ReplaceInput(3, GetReplacementHigh(old_value));
960 node->InsertInput(zone(), 4, GetReplacementLow(new_value));
961 node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
962 NodeProperties::ChangeOp(node,
963 machine()->Word32AtomicPairCompareExchange());
964 ReplaceNodeWithProjections(node);
965 } else {
966 DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
967 type == MachineType::Uint8());
968 DefaultLowering(node, true);
969 NodeProperties::ChangeOp(node,
970 machine()->Word32AtomicCompareExchange(type));
971 ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
972 }
973 break;
974 }
975 case IrOpcode::kI64x2Splat: {
976 DCHECK_EQ(1, node->InputCount());
977 Node* input = node->InputAt(0);
978 node->ReplaceInput(0, GetReplacementLow(input));
979 node->AppendInput(zone(), GetReplacementHigh(input));
980 NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
981 break;
982 }
983 case IrOpcode::kI64x2ExtractLane: {
984 DCHECK_EQ(1, node->InputCount());
985 Node* input = node->InputAt(0);
986 int32_t lane = OpParameter<int32_t>(node->op());
987 ReplaceNode(
988 node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
989 graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
990 break;
991 }
992 case IrOpcode::kI64x2ReplaceLane: {
993 DCHECK_EQ(2, node->InputCount());
994 int32_t lane = OpParameter<int32_t>(node->op());
995 Node* input = node->InputAt(1);
996 node->ReplaceInput(1, GetReplacementLow(input));
997 node->AppendInput(zone(), GetReplacementHigh(input));
998 NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
999 break;
1000 }
1001
1002 default: { DefaultLowering(node); }
1003 }
1004 } // NOLINT(readability/fn_size)
1005
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)1006 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
1007 const Operator* low_word_op) {
1008 DCHECK_EQ(2, node->InputCount());
1009 Node* left = node->InputAt(0);
1010 Node* right = node->InputAt(1);
1011 Node* replacement = graph()->NewNode(
1012 machine()->Word32Or(),
1013 graph()->NewNode(high_word_op, GetReplacementHigh(left),
1014 GetReplacementHigh(right)),
1015 graph()->NewNode(
1016 machine()->Word32And(),
1017 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
1018 GetReplacementHigh(right)),
1019 graph()->NewNode(low_word_op, GetReplacementLow(left),
1020 GetReplacementLow(right))));
1021
1022 ReplaceNode(node, replacement, nullptr);
1023 }
1024
DefaultLowering(Node * node,bool low_word_only)1025 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
1026 bool something_changed = false;
1027 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1028 Node* input = node->InputAt(i);
1029 if (HasReplacementLow(input)) {
1030 something_changed = true;
1031 node->ReplaceInput(i, GetReplacementLow(input));
1032 }
1033 if (!low_word_only && HasReplacementHigh(input)) {
1034 something_changed = true;
1035 node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
1036 }
1037 }
1038 return something_changed;
1039 }
1040
LowerCallDescriptor(const CallDescriptor * call_descriptor)1041 const CallDescriptor* Int64Lowering::LowerCallDescriptor(
1042 const CallDescriptor* call_descriptor) {
1043 if (special_case_) {
1044 auto replacement = special_case_->replacements.find(call_descriptor);
1045 if (replacement != special_case_->replacements.end()) {
1046 return replacement->second;
1047 }
1048 }
1049 return GetI32WasmCallDescriptor(zone(), call_descriptor);
1050 }
1051
ReplaceNode(Node * old,Node * new_low,Node * new_high)1052 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
1053 // if new_low == nullptr, then also new_high == nullptr.
1054 DCHECK(new_low != nullptr || new_high == nullptr);
1055 replacements_[old->id()].low = new_low;
1056 replacements_[old->id()].high = new_high;
1057 }
1058
HasReplacementLow(Node * node)1059 bool Int64Lowering::HasReplacementLow(Node* node) {
1060 return replacements_[node->id()].low != nullptr;
1061 }
1062
GetReplacementLow(Node * node)1063 Node* Int64Lowering::GetReplacementLow(Node* node) {
1064 Node* result = replacements_[node->id()].low;
1065 DCHECK(result);
1066 return result;
1067 }
1068
HasReplacementHigh(Node * node)1069 bool Int64Lowering::HasReplacementHigh(Node* node) {
1070 return replacements_[node->id()].high != nullptr;
1071 }
1072
GetReplacementHigh(Node * node)1073 Node* Int64Lowering::GetReplacementHigh(Node* node) {
1074 Node* result = replacements_[node->id()].high;
1075 DCHECK(result);
1076 return result;
1077 }
1078
PreparePhiReplacement(Node * phi)1079 void Int64Lowering::PreparePhiReplacement(Node* phi) {
1080 MachineRepresentation rep = PhiRepresentationOf(phi->op());
1081 if (rep == MachineRepresentation::kWord64) {
1082 // We have to create the replacements for a phi node before we actually
1083 // lower the phi to break potential cycles in the graph. The replacements of
1084 // input nodes do not exist yet, so we use a placeholder node to pass the
1085 // graph verifier.
1086 int value_count = phi->op()->ValueInputCount();
1087 Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1088 Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1089 for (int i = 0; i < value_count; i++) {
1090 inputs_low[i] = placeholder_;
1091 inputs_high[i] = placeholder_;
1092 }
1093 inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1094 inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1095 ReplaceNode(phi,
1096 graph()->NewNode(
1097 common()->Phi(MachineRepresentation::kWord32, value_count),
1098 value_count + 1, inputs_low, false),
1099 graph()->NewNode(
1100 common()->Phi(MachineRepresentation::kWord32, value_count),
1101 value_count + 1, inputs_high, false));
1102 }
1103 }
1104
ReplaceNodeWithProjections(Node * node)1105 void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1106 DCHECK(node != nullptr);
1107 Node* low_node =
1108 graph()->NewNode(common()->Projection(0), node, graph()->start());
1109 Node* high_node =
1110 graph()->NewNode(common()->Projection(1), node, graph()->start());
1111 ReplaceNode(node, low_node, high_node);
1112 }
1113
LowerMemoryBaseAndIndex(Node * node)1114 void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
1115 DCHECK(node != nullptr);
1116 // Low word only replacements for memory operands for 32-bit address space.
1117 Node* base = node->InputAt(0);
1118 Node* index = node->InputAt(1);
1119 if (HasReplacementLow(base)) {
1120 node->ReplaceInput(0, GetReplacementLow(base));
1121 }
1122 if (HasReplacementLow(index)) {
1123 node->ReplaceInput(1, GetReplacementLow(index));
1124 }
1125 }
1126
1127 } // namespace compiler
1128 } // namespace internal
1129 } // namespace v8
1130