1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/int64-lowering.h"
6
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23
Int64Lowering(Graph * graph,MachineOperatorBuilder * machine,CommonOperatorBuilder * common,Zone * zone,Signature<MachineRepresentation> * signature)24 Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
25 CommonOperatorBuilder* common, Zone* zone,
26 Signature<MachineRepresentation>* signature)
27 : zone_(zone),
28 graph_(graph),
29 machine_(machine),
30 common_(common),
31 state_(graph, 3),
32 stack_(zone),
33 replacements_(nullptr),
34 signature_(signature),
35 placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
36 graph->start())) {
37 DCHECK_NOT_NULL(graph);
38 DCHECK_NOT_NULL(graph->end());
39 replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
40 memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
41 }
42
LowerGraph()43 void Int64Lowering::LowerGraph() {
44 if (!machine()->Is32()) {
45 return;
46 }
47 stack_.push_back({graph()->end(), 0});
48 state_.Set(graph()->end(), State::kOnStack);
49
50 while (!stack_.empty()) {
51 NodeState& top = stack_.back();
52 if (top.input_index == top.node->InputCount()) {
53 // All inputs of top have already been lowered, now lower top.
54 stack_.pop_back();
55 state_.Set(top.node, State::kVisited);
56 LowerNode(top.node);
57 } else {
58 // Push the next input onto the stack.
59 Node* input = top.node->InputAt(top.input_index++);
60 if (state_.Get(input) == State::kUnvisited) {
61 if (input->opcode() == IrOpcode::kPhi) {
62 // To break cycles with phi nodes we push phis on a separate stack so
63 // that they are processed after all other nodes.
64 PreparePhiReplacement(input);
65 stack_.push_front({input, 0});
66 } else if (input->opcode() == IrOpcode::kEffectPhi ||
67 input->opcode() == IrOpcode::kLoop) {
68 stack_.push_front({input, 0});
69 } else {
70 stack_.push_back({input, 0});
71 }
72 state_.Set(input, State::kOnStack);
73 }
74 }
75 }
76 }
77
78 namespace {
79
GetReturnIndexAfterLowering(CallDescriptor * call_descriptor,int old_index)80 int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor,
81 int old_index) {
82 int result = old_index;
83 for (int i = 0; i < old_index; i++) {
84 if (call_descriptor->GetReturnType(i).representation() ==
85 MachineRepresentation::kWord64) {
86 result++;
87 }
88 }
89 return result;
90 }
91
GetReturnCountAfterLowering(CallDescriptor * call_descriptor)92 int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) {
93 return GetReturnIndexAfterLowering(
94 call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
95 }
96
GetParameterIndexAfterLowering(Signature<MachineRepresentation> * signature,int old_index)97 int GetParameterIndexAfterLowering(
98 Signature<MachineRepresentation>* signature, int old_index) {
99 int result = old_index;
100 for (int i = 0; i < old_index; i++) {
101 if (signature->GetParam(i) == MachineRepresentation::kWord64) {
102 result++;
103 }
104 }
105 return result;
106 }
107
GetReturnCountAfterLowering(Signature<MachineRepresentation> * signature)108 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
109 int result = static_cast<int>(signature->return_count());
110 for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
111 if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
112 result++;
113 }
114 }
115 return result;
116 }
117
118 } // namespace
119
120 // static
GetParameterCountAfterLowering(Signature<MachineRepresentation> * signature)121 int Int64Lowering::GetParameterCountAfterLowering(
122 Signature<MachineRepresentation>* signature) {
123 // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
124 // after lowering.
125 return GetParameterIndexAfterLowering(
126 signature, static_cast<int>(signature->parameter_count()));
127 }
128
GetIndexNodes(Node * index,Node * & index_low,Node * & index_high)129 void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
130 Node*& index_high) {
131 if (HasReplacementLow(index)) {
132 index = GetReplacementLow(index);
133 }
134 #if defined(V8_TARGET_LITTLE_ENDIAN)
135 index_low = index;
136 index_high = graph()->NewNode(machine()->Int32Add(), index,
137 graph()->NewNode(common()->Int32Constant(4)));
138 #elif defined(V8_TARGET_BIG_ENDIAN)
139 index_low = graph()->NewNode(machine()->Int32Add(), index,
140 graph()->NewNode(common()->Int32Constant(4)));
141 index_high = index;
142 #endif
143 }
144
LowerNode(Node * node)145 void Int64Lowering::LowerNode(Node* node) {
146 switch (node->opcode()) {
147 case IrOpcode::kInt64Constant: {
148 int64_t value = OpParameter<int64_t>(node->op());
149 Node* low_node = graph()->NewNode(
150 common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
151 Node* high_node = graph()->NewNode(
152 common()->Int32Constant(static_cast<int32_t>(value >> 32)));
153 ReplaceNode(node, low_node, high_node);
154 break;
155 }
156 case IrOpcode::kLoad:
157 case IrOpcode::kUnalignedLoad: {
158 MachineRepresentation rep;
159 if (node->opcode() == IrOpcode::kLoad) {
160 rep = LoadRepresentationOf(node->op()).representation();
161 } else {
162 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
163 rep = LoadRepresentationOf(node->op()).representation();
164 }
165
166 if (rep == MachineRepresentation::kWord64) {
167 Node* base = node->InputAt(0);
168 Node* index = node->InputAt(1);
169 Node* index_low;
170 Node* index_high;
171 GetIndexNodes(index, index_low, index_high);
172 const Operator* load_op;
173
174 if (node->opcode() == IrOpcode::kLoad) {
175 load_op = machine()->Load(MachineType::Int32());
176 } else {
177 DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
178 load_op = machine()->UnalignedLoad(MachineType::Int32());
179 }
180
181 Node* high_node;
182 if (node->InputCount() > 2) {
183 Node* effect_high = node->InputAt(2);
184 Node* control_high = node->InputAt(3);
185 high_node = graph()->NewNode(load_op, base, index_high, effect_high,
186 control_high);
187 // change the effect change from old_node --> old_effect to
188 // old_node --> high_node --> old_effect.
189 node->ReplaceInput(2, high_node);
190 } else {
191 high_node = graph()->NewNode(load_op, base, index_high);
192 }
193 node->ReplaceInput(1, index_low);
194 NodeProperties::ChangeOp(node, load_op);
195 ReplaceNode(node, node, high_node);
196 } else {
197 DefaultLowering(node);
198 }
199 break;
200 }
201 case IrOpcode::kStore:
202 case IrOpcode::kUnalignedStore: {
203 MachineRepresentation rep;
204 if (node->opcode() == IrOpcode::kStore) {
205 rep = StoreRepresentationOf(node->op()).representation();
206 } else {
207 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
208 rep = UnalignedStoreRepresentationOf(node->op());
209 }
210
211 if (rep == MachineRepresentation::kWord64) {
212 // We change the original store node to store the low word, and create
213 // a new store node to store the high word. The effect and control edges
214 // are copied from the original store to the new store node, the effect
215 // edge of the original store is redirected to the new store.
216 Node* base = node->InputAt(0);
217 Node* index = node->InputAt(1);
218 Node* index_low;
219 Node* index_high;
220 GetIndexNodes(index, index_low, index_high);
221 Node* value = node->InputAt(2);
222 DCHECK(HasReplacementLow(value));
223 DCHECK(HasReplacementHigh(value));
224
225 const Operator* store_op;
226 if (node->opcode() == IrOpcode::kStore) {
227 WriteBarrierKind write_barrier_kind =
228 StoreRepresentationOf(node->op()).write_barrier_kind();
229 store_op = machine()->Store(StoreRepresentation(
230 MachineRepresentation::kWord32, write_barrier_kind));
231 } else {
232 DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
233 store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
234 }
235
236 Node* high_node;
237 if (node->InputCount() > 3) {
238 Node* effect_high = node->InputAt(3);
239 Node* control_high = node->InputAt(4);
240 high_node = graph()->NewNode(store_op, base, index_high,
241 GetReplacementHigh(value), effect_high,
242 control_high);
243 node->ReplaceInput(3, high_node);
244
245 } else {
246 high_node = graph()->NewNode(store_op, base, index_high,
247 GetReplacementHigh(value));
248 }
249
250 node->ReplaceInput(1, index_low);
251 node->ReplaceInput(2, GetReplacementLow(value));
252 NodeProperties::ChangeOp(node, store_op);
253 ReplaceNode(node, node, high_node);
254 } else {
255 DefaultLowering(node, true);
256 }
257 break;
258 }
259 case IrOpcode::kStart: {
260 int parameter_count = GetParameterCountAfterLowering(signature());
261 // Only exchange the node if the parameter count actually changed.
262 if (parameter_count != static_cast<int>(signature()->parameter_count())) {
263 int delta =
264 parameter_count - static_cast<int>(signature()->parameter_count());
265 int new_output_count = node->op()->ValueOutputCount() + delta;
266 NodeProperties::ChangeOp(node, common()->Start(new_output_count));
267 }
268 break;
269 }
270 case IrOpcode::kParameter: {
271 DCHECK_EQ(1, node->InputCount());
272 // Only exchange the node if the parameter count actually changed. We do
273 // not even have to do the default lowering because the the start node,
274 // the only input of a parameter node, only changes if the parameter count
275 // changes.
276 if (GetParameterCountAfterLowering(signature()) !=
277 static_cast<int>(signature()->parameter_count())) {
278 int old_index = ParameterIndexOf(node->op());
279 // TODO(wasm): Make this part not wasm specific.
280 // Prevent special lowering of the instance parameter.
281 if (old_index == wasm::kWasmInstanceParameterIndex) {
282 DefaultLowering(node);
283 break;
284 }
285 // Adjust old_index to be compliant with the signature.
286 --old_index;
287 int new_index = GetParameterIndexAfterLowering(signature(), old_index);
288 // Adjust new_index to consider the instance parameter.
289 ++new_index;
290 NodeProperties::ChangeOp(node, common()->Parameter(new_index));
291
292 if (signature()->GetParam(old_index) ==
293 MachineRepresentation::kWord64) {
294 Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
295 graph()->start());
296 ReplaceNode(node, node, high_node);
297 }
298 }
299 break;
300 }
301 case IrOpcode::kReturn: {
302 int input_count = node->InputCount();
303 DefaultLowering(node);
304 if (input_count != node->InputCount()) {
305 int new_return_count = GetReturnCountAfterLowering(signature());
306 if (static_cast<int>(signature()->return_count()) != new_return_count) {
307 NodeProperties::ChangeOp(node, common()->Return(new_return_count));
308 }
309 }
310 break;
311 }
312 case IrOpcode::kTailCall: {
313 auto call_descriptor =
314 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
315 bool returns_require_lowering =
316 GetReturnCountAfterLowering(call_descriptor) !=
317 static_cast<int>(call_descriptor->ReturnCount());
318 if (DefaultLowering(node) || returns_require_lowering) {
319 // Tail calls do not have return values, so adjusting the call
320 // descriptor is enough.
321 auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor);
322 NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
323 }
324 break;
325 }
326 case IrOpcode::kCall: {
327 auto call_descriptor =
328 const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
329 bool returns_require_lowering =
330 GetReturnCountAfterLowering(call_descriptor) !=
331 static_cast<int>(call_descriptor->ReturnCount());
332 if (DefaultLowering(node) || returns_require_lowering) {
333 // We have to adjust the call descriptor.
334 NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor(
335 zone(), call_descriptor)));
336 }
337 if (returns_require_lowering) {
338 size_t return_arity = call_descriptor->ReturnCount();
339 if (return_arity == 1) {
340 // We access the additional return values through projections.
341 Node* low_node =
342 graph()->NewNode(common()->Projection(0), node, graph()->start());
343 Node* high_node =
344 graph()->NewNode(common()->Projection(1), node, graph()->start());
345 ReplaceNode(node, low_node, high_node);
346 } else {
347 ZoneVector<Node*> projections(return_arity, zone());
348 NodeProperties::CollectValueProjections(node, projections.data(),
349 return_arity);
350 for (size_t old_index = 0, new_index = 0; old_index < return_arity;
351 ++old_index, ++new_index) {
352 Node* use_node = projections[old_index];
353 DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
354 DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
355 static_cast<int>(old_index)),
356 static_cast<int>(new_index));
357 if (new_index != old_index) {
358 NodeProperties::ChangeOp(
359 use_node, common()->Projection(new_index));
360 }
361 if (call_descriptor->GetReturnType(old_index).representation() ==
362 MachineRepresentation::kWord64) {
363 Node* high_node = graph()->NewNode(
364 common()->Projection(new_index + 1), node,
365 graph()->start());
366 ReplaceNode(use_node, use_node, high_node);
367 ++new_index;
368 }
369 }
370 }
371 }
372 break;
373 }
374 case IrOpcode::kWord64And: {
375 DCHECK_EQ(2, node->InputCount());
376 Node* left = node->InputAt(0);
377 Node* right = node->InputAt(1);
378
379 Node* low_node =
380 graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
381 GetReplacementLow(right));
382 Node* high_node =
383 graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
384 GetReplacementHigh(right));
385 ReplaceNode(node, low_node, high_node);
386 break;
387 }
388 case IrOpcode::kTruncateInt64ToInt32: {
389 DCHECK_EQ(1, node->InputCount());
390 Node* input = node->InputAt(0);
391 ReplaceNode(node, GetReplacementLow(input), nullptr);
392 node->NullAllInputs();
393 break;
394 }
395 case IrOpcode::kInt64Add: {
396 DCHECK_EQ(2, node->InputCount());
397
398 Node* right = node->InputAt(1);
399 node->ReplaceInput(1, GetReplacementLow(right));
400 node->AppendInput(zone(), GetReplacementHigh(right));
401
402 Node* left = node->InputAt(0);
403 node->ReplaceInput(0, GetReplacementLow(left));
404 node->InsertInput(zone(), 1, GetReplacementHigh(left));
405
406 NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
407 // We access the additional return values through projections.
408 Node* low_node =
409 graph()->NewNode(common()->Projection(0), node, graph()->start());
410 Node* high_node =
411 graph()->NewNode(common()->Projection(1), node, graph()->start());
412 ReplaceNode(node, low_node, high_node);
413 break;
414 }
415 case IrOpcode::kInt64Sub: {
416 DCHECK_EQ(2, node->InputCount());
417
418 Node* right = node->InputAt(1);
419 node->ReplaceInput(1, GetReplacementLow(right));
420 node->AppendInput(zone(), GetReplacementHigh(right));
421
422 Node* left = node->InputAt(0);
423 node->ReplaceInput(0, GetReplacementLow(left));
424 node->InsertInput(zone(), 1, GetReplacementHigh(left));
425
426 NodeProperties::ChangeOp(node, machine()->Int32PairSub());
427 // We access the additional return values through projections.
428 Node* low_node =
429 graph()->NewNode(common()->Projection(0), node, graph()->start());
430 Node* high_node =
431 graph()->NewNode(common()->Projection(1), node, graph()->start());
432 ReplaceNode(node, low_node, high_node);
433 break;
434 }
435 case IrOpcode::kInt64Mul: {
436 DCHECK_EQ(2, node->InputCount());
437
438 Node* right = node->InputAt(1);
439 node->ReplaceInput(1, GetReplacementLow(right));
440 node->AppendInput(zone(), GetReplacementHigh(right));
441
442 Node* left = node->InputAt(0);
443 node->ReplaceInput(0, GetReplacementLow(left));
444 node->InsertInput(zone(), 1, GetReplacementHigh(left));
445
446 NodeProperties::ChangeOp(node, machine()->Int32PairMul());
447 // We access the additional return values through projections.
448 Node* low_node =
449 graph()->NewNode(common()->Projection(0), node, graph()->start());
450 Node* high_node =
451 graph()->NewNode(common()->Projection(1), node, graph()->start());
452 ReplaceNode(node, low_node, high_node);
453 break;
454 }
455 case IrOpcode::kWord64Or: {
456 DCHECK_EQ(2, node->InputCount());
457 Node* left = node->InputAt(0);
458 Node* right = node->InputAt(1);
459
460 Node* low_node =
461 graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
462 GetReplacementLow(right));
463 Node* high_node =
464 graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
465 GetReplacementHigh(right));
466 ReplaceNode(node, low_node, high_node);
467 break;
468 }
469 case IrOpcode::kWord64Xor: {
470 DCHECK_EQ(2, node->InputCount());
471 Node* left = node->InputAt(0);
472 Node* right = node->InputAt(1);
473
474 Node* low_node =
475 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
476 GetReplacementLow(right));
477 Node* high_node =
478 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
479 GetReplacementHigh(right));
480 ReplaceNode(node, low_node, high_node);
481 break;
482 }
483 case IrOpcode::kWord64Shl: {
484 // TODO(turbofan): if the shift count >= 32, then we can set the low word
485 // of the output to 0 and just calculate the high word.
486 DCHECK_EQ(2, node->InputCount());
487 Node* shift = node->InputAt(1);
488 if (HasReplacementLow(shift)) {
489 // We do not have to care about the high word replacement, because
490 // the shift can only be between 0 and 63 anyways.
491 node->ReplaceInput(1, GetReplacementLow(shift));
492 }
493
494 Node* value = node->InputAt(0);
495 node->ReplaceInput(0, GetReplacementLow(value));
496 node->InsertInput(zone(), 1, GetReplacementHigh(value));
497
498 NodeProperties::ChangeOp(node, machine()->Word32PairShl());
499 // We access the additional return values through projections.
500 Node* low_node =
501 graph()->NewNode(common()->Projection(0), node, graph()->start());
502 Node* high_node =
503 graph()->NewNode(common()->Projection(1), node, graph()->start());
504 ReplaceNode(node, low_node, high_node);
505 break;
506 }
507 case IrOpcode::kWord64Shr: {
508 // TODO(turbofan): if the shift count >= 32, then we can set the low word
509 // of the output to 0 and just calculate the high word.
510 DCHECK_EQ(2, node->InputCount());
511 Node* shift = node->InputAt(1);
512 if (HasReplacementLow(shift)) {
513 // We do not have to care about the high word replacement, because
514 // the shift can only be between 0 and 63 anyways.
515 node->ReplaceInput(1, GetReplacementLow(shift));
516 }
517
518 Node* value = node->InputAt(0);
519 node->ReplaceInput(0, GetReplacementLow(value));
520 node->InsertInput(zone(), 1, GetReplacementHigh(value));
521
522 NodeProperties::ChangeOp(node, machine()->Word32PairShr());
523 // We access the additional return values through projections.
524 Node* low_node =
525 graph()->NewNode(common()->Projection(0), node, graph()->start());
526 Node* high_node =
527 graph()->NewNode(common()->Projection(1), node, graph()->start());
528 ReplaceNode(node, low_node, high_node);
529 break;
530 }
531 case IrOpcode::kWord64Sar: {
532 // TODO(turbofan): if the shift count >= 32, then we can set the low word
533 // of the output to 0 and just calculate the high word.
534 DCHECK_EQ(2, node->InputCount());
535 Node* shift = node->InputAt(1);
536 if (HasReplacementLow(shift)) {
537 // We do not have to care about the high word replacement, because
538 // the shift can only be between 0 and 63 anyways.
539 node->ReplaceInput(1, GetReplacementLow(shift));
540 }
541
542 Node* value = node->InputAt(0);
543 node->ReplaceInput(0, GetReplacementLow(value));
544 node->InsertInput(zone(), 1, GetReplacementHigh(value));
545
546 NodeProperties::ChangeOp(node, machine()->Word32PairSar());
547 // We access the additional return values through projections.
548 Node* low_node =
549 graph()->NewNode(common()->Projection(0), node, graph()->start());
550 Node* high_node =
551 graph()->NewNode(common()->Projection(1), node, graph()->start());
552 ReplaceNode(node, low_node, high_node);
553 break;
554 }
555 case IrOpcode::kWord64Equal: {
556 DCHECK_EQ(2, node->InputCount());
557 Node* left = node->InputAt(0);
558 Node* right = node->InputAt(1);
559
560 // TODO(wasm): Use explicit comparisons and && here?
561 Node* replacement = graph()->NewNode(
562 machine()->Word32Equal(),
563 graph()->NewNode(
564 machine()->Word32Or(),
565 graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
566 GetReplacementLow(right)),
567 graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
568 GetReplacementHigh(right))),
569 graph()->NewNode(common()->Int32Constant(0)));
570
571 ReplaceNode(node, replacement, nullptr);
572 break;
573 }
574 case IrOpcode::kInt64LessThan: {
575 LowerComparison(node, machine()->Int32LessThan(),
576 machine()->Uint32LessThan());
577 break;
578 }
579 case IrOpcode::kInt64LessThanOrEqual: {
580 LowerComparison(node, machine()->Int32LessThan(),
581 machine()->Uint32LessThanOrEqual());
582 break;
583 }
584 case IrOpcode::kUint64LessThan: {
585 LowerComparison(node, machine()->Uint32LessThan(),
586 machine()->Uint32LessThan());
587 break;
588 }
589 case IrOpcode::kUint64LessThanOrEqual: {
590 LowerComparison(node, machine()->Uint32LessThan(),
591 machine()->Uint32LessThanOrEqual());
592 break;
593 }
594 case IrOpcode::kSignExtendWord32ToInt64:
595 case IrOpcode::kChangeInt32ToInt64: {
596 DCHECK_EQ(1, node->InputCount());
597 Node* input = node->InputAt(0);
598 if (HasReplacementLow(input)) {
599 input = GetReplacementLow(input);
600 }
601 // We use SAR to preserve the sign in the high word.
602 ReplaceNode(
603 node, input,
604 graph()->NewNode(machine()->Word32Sar(), input,
605 graph()->NewNode(common()->Int32Constant(31))));
606 node->NullAllInputs();
607 break;
608 }
609 case IrOpcode::kChangeUint32ToUint64: {
610 DCHECK_EQ(1, node->InputCount());
611 Node* input = node->InputAt(0);
612 if (HasReplacementLow(input)) {
613 input = GetReplacementLow(input);
614 }
615 ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
616 node->NullAllInputs();
617 break;
618 }
619 case IrOpcode::kBitcastInt64ToFloat64: {
620 DCHECK_EQ(1, node->InputCount());
621 Node* input = node->InputAt(0);
622 Node* stack_slot = graph()->NewNode(
623 machine()->StackSlot(MachineRepresentation::kWord64));
624
625 Node* store_high_word = graph()->NewNode(
626 machine()->Store(
627 StoreRepresentation(MachineRepresentation::kWord32,
628 WriteBarrierKind::kNoWriteBarrier)),
629 stack_slot,
630 graph()->NewNode(
631 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
632 GetReplacementHigh(input), graph()->start(), graph()->start());
633
634 Node* store_low_word = graph()->NewNode(
635 machine()->Store(
636 StoreRepresentation(MachineRepresentation::kWord32,
637 WriteBarrierKind::kNoWriteBarrier)),
638 stack_slot,
639 graph()->NewNode(
640 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
641 GetReplacementLow(input), store_high_word, graph()->start());
642
643 Node* load =
644 graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
645 graph()->NewNode(common()->Int32Constant(0)),
646 store_low_word, graph()->start());
647
648 ReplaceNode(node, load, nullptr);
649 break;
650 }
651 case IrOpcode::kBitcastFloat64ToInt64: {
652 DCHECK_EQ(1, node->InputCount());
653 Node* input = node->InputAt(0);
654 if (HasReplacementLow(input)) {
655 input = GetReplacementLow(input);
656 }
657 Node* stack_slot = graph()->NewNode(
658 machine()->StackSlot(MachineRepresentation::kWord64));
659 Node* store = graph()->NewNode(
660 machine()->Store(
661 StoreRepresentation(MachineRepresentation::kFloat64,
662 WriteBarrierKind::kNoWriteBarrier)),
663 stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
664 graph()->start(), graph()->start());
665
666 Node* high_node = graph()->NewNode(
667 machine()->Load(MachineType::Int32()), stack_slot,
668 graph()->NewNode(
669 common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
670 store, graph()->start());
671
672 Node* low_node = graph()->NewNode(
673 machine()->Load(MachineType::Int32()), stack_slot,
674 graph()->NewNode(
675 common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
676 store, graph()->start());
677 ReplaceNode(node, low_node, high_node);
678 break;
679 }
680 case IrOpcode::kWord64Ror: {
681 DCHECK_EQ(2, node->InputCount());
682 Node* input = node->InputAt(0);
683 Node* shift = HasReplacementLow(node->InputAt(1))
684 ? GetReplacementLow(node->InputAt(1))
685 : node->InputAt(1);
686 Int32Matcher m(shift);
687 if (m.HasValue()) {
688 // Precondition: 0 <= shift < 64.
689 int32_t shift_value = m.Value() & 0x3F;
690 if (shift_value == 0) {
691 ReplaceNode(node, GetReplacementLow(input),
692 GetReplacementHigh(input));
693 } else if (shift_value == 32) {
694 ReplaceNode(node, GetReplacementHigh(input),
695 GetReplacementLow(input));
696 } else {
697 Node* low_input;
698 Node* high_input;
699 if (shift_value < 32) {
700 low_input = GetReplacementLow(input);
701 high_input = GetReplacementHigh(input);
702 } else {
703 low_input = GetReplacementHigh(input);
704 high_input = GetReplacementLow(input);
705 }
706 int32_t masked_shift_value = shift_value & 0x1F;
707 Node* masked_shift =
708 graph()->NewNode(common()->Int32Constant(masked_shift_value));
709 Node* inv_shift = graph()->NewNode(
710 common()->Int32Constant(32 - masked_shift_value));
711
712 Node* low_node = graph()->NewNode(
713 machine()->Word32Or(),
714 graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
715 graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
716 Node* high_node = graph()->NewNode(
717 machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
718 high_input, masked_shift),
719 graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
720 ReplaceNode(node, low_node, high_node);
721 }
722 } else {
723 Node* safe_shift = shift;
724 if (!machine()->Word32ShiftIsSafe()) {
725 safe_shift =
726 graph()->NewNode(machine()->Word32And(), shift,
727 graph()->NewNode(common()->Int32Constant(0x1F)));
728 }
729
730 // By creating this bit-mask with SAR and SHL we do not have to deal
731 // with shift == 0 as a special case.
732 Node* inv_mask = graph()->NewNode(
733 machine()->Word32Shl(),
734 graph()->NewNode(machine()->Word32Sar(),
735 graph()->NewNode(common()->Int32Constant(
736 std::numeric_limits<int32_t>::min())),
737 safe_shift),
738 graph()->NewNode(common()->Int32Constant(1)));
739
740 Node* bit_mask =
741 graph()->NewNode(machine()->Word32Xor(), inv_mask,
742 graph()->NewNode(common()->Int32Constant(-1)));
743
744 // We have to mask the shift value for this comparison. If
745 // !machine()->Word32ShiftIsSafe() then the masking should already be
746 // part of the graph.
747 Node* masked_shift6 = shift;
748 if (machine()->Word32ShiftIsSafe()) {
749 masked_shift6 =
750 graph()->NewNode(machine()->Word32And(), shift,
751 graph()->NewNode(common()->Int32Constant(0x3F)));
752 }
753
754 Diamond lt32(
755 graph(), common(),
756 graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
757 graph()->NewNode(common()->Int32Constant(32))));
758
759 // The low word and the high word can be swapped either at the input or
760 // at the output. We swap the inputs so that shift does not have to be
761 // kept for so long in a register.
762 Node* input_low =
763 lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
764 GetReplacementHigh(input));
765 Node* input_high =
766 lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
767 GetReplacementLow(input));
768
769 Node* rotate_low =
770 graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
771 Node* rotate_high =
772 graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
773
774 Node* low_node = graph()->NewNode(
775 machine()->Word32Or(),
776 graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
777 graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
778
779 Node* high_node = graph()->NewNode(
780 machine()->Word32Or(),
781 graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
782 graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
783
784 ReplaceNode(node, low_node, high_node);
785 }
786 break;
787 }
788 case IrOpcode::kWord64Clz: {
789 DCHECK_EQ(1, node->InputCount());
790 Node* input = node->InputAt(0);
791 Diamond d(
792 graph(), common(),
793 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
794 graph()->NewNode(common()->Int32Constant(0))));
795
796 Node* low_node = d.Phi(
797 MachineRepresentation::kWord32,
798 graph()->NewNode(machine()->Int32Add(),
799 graph()->NewNode(machine()->Word32Clz(),
800 GetReplacementLow(input)),
801 graph()->NewNode(common()->Int32Constant(32))),
802 graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
803 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
804 break;
805 }
806 case IrOpcode::kWord64Ctz: {
807 DCHECK_EQ(1, node->InputCount());
808 DCHECK(machine()->Word32Ctz().IsSupported());
809 Node* input = node->InputAt(0);
810 Diamond d(
811 graph(), common(),
812 graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
813 graph()->NewNode(common()->Int32Constant(0))));
814 Node* low_node =
815 d.Phi(MachineRepresentation::kWord32,
816 graph()->NewNode(machine()->Int32Add(),
817 graph()->NewNode(machine()->Word32Ctz().op(),
818 GetReplacementHigh(input)),
819 graph()->NewNode(common()->Int32Constant(32))),
820 graph()->NewNode(machine()->Word32Ctz().op(),
821 GetReplacementLow(input)));
822 ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
823 break;
824 }
825 case IrOpcode::kWord64Popcnt: {
826 DCHECK_EQ(1, node->InputCount());
827 Node* input = node->InputAt(0);
828 // We assume that a Word64Popcnt node only has been created if
829 // Word32Popcnt is actually supported.
830 DCHECK(machine()->Word32Popcnt().IsSupported());
831 ReplaceNode(node, graph()->NewNode(
832 machine()->Int32Add(),
833 graph()->NewNode(machine()->Word32Popcnt().op(),
834 GetReplacementLow(input)),
835 graph()->NewNode(machine()->Word32Popcnt().op(),
836 GetReplacementHigh(input))),
837 graph()->NewNode(common()->Int32Constant(0)));
838 break;
839 }
840 case IrOpcode::kPhi: {
841 MachineRepresentation rep = PhiRepresentationOf(node->op());
842 if (rep == MachineRepresentation::kWord64) {
843 // The replacement nodes have already been created, we only have to
844 // replace placeholder nodes.
845 Node* low_node = GetReplacementLow(node);
846 Node* high_node = GetReplacementHigh(node);
847 for (int i = 0; i < node->op()->ValueInputCount(); i++) {
848 low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
849 high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
850 }
851 } else {
852 DefaultLowering(node);
853 }
854 break;
855 }
856 case IrOpcode::kWord64ReverseBytes: {
857 Node* input = node->InputAt(0);
858 ReplaceNode(node, graph()->NewNode(machine()->Word32ReverseBytes().op(),
859 GetReplacementHigh(input)),
860 graph()->NewNode(machine()->Word32ReverseBytes().op(),
861 GetReplacementLow(input)));
862 break;
863 }
864 case IrOpcode::kSignExtendWord8ToInt64: {
865 DCHECK_EQ(1, node->InputCount());
866 Node* input = node->InputAt(0);
867 if (HasReplacementLow(input)) {
868 input = GetReplacementLow(input);
869 }
870 // Sign extend low node to Int32
871 input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
872
873 // We use SAR to preserve the sign in the high word.
874 ReplaceNode(
875 node, input,
876 graph()->NewNode(machine()->Word32Sar(), input,
877 graph()->NewNode(common()->Int32Constant(31))));
878 node->NullAllInputs();
879 break;
880 }
881 case IrOpcode::kSignExtendWord16ToInt64: {
882 DCHECK_EQ(1, node->InputCount());
883 Node* input = node->InputAt(0);
884 if (HasReplacementLow(input)) {
885 input = GetReplacementLow(input);
886 }
887 // Sign extend low node to Int32
888 input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
889
890 // We use SAR to preserve the sign in the high word.
891 ReplaceNode(
892 node, input,
893 graph()->NewNode(machine()->Word32Sar(), input,
894 graph()->NewNode(common()->Int32Constant(31))));
895 node->NullAllInputs();
896 break;
897 }
898
899 default: { DefaultLowering(node); }
900 }
901 } // NOLINT(readability/fn_size)
902
LowerComparison(Node * node,const Operator * high_word_op,const Operator * low_word_op)903 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
904 const Operator* low_word_op) {
905 DCHECK_EQ(2, node->InputCount());
906 Node* left = node->InputAt(0);
907 Node* right = node->InputAt(1);
908 Node* replacement = graph()->NewNode(
909 machine()->Word32Or(),
910 graph()->NewNode(high_word_op, GetReplacementHigh(left),
911 GetReplacementHigh(right)),
912 graph()->NewNode(
913 machine()->Word32And(),
914 graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
915 GetReplacementHigh(right)),
916 graph()->NewNode(low_word_op, GetReplacementLow(left),
917 GetReplacementLow(right))));
918
919 ReplaceNode(node, replacement, nullptr);
920 }
921
DefaultLowering(Node * node,bool low_word_only)922 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
923 bool something_changed = false;
924 for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
925 Node* input = node->InputAt(i);
926 if (HasReplacementLow(input)) {
927 something_changed = true;
928 node->ReplaceInput(i, GetReplacementLow(input));
929 }
930 if (!low_word_only && HasReplacementHigh(input)) {
931 something_changed = true;
932 node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
933 }
934 }
935 return something_changed;
936 }
937
ReplaceNode(Node * old,Node * new_low,Node * new_high)938 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
939 // if new_low == nullptr, then also new_high == nullptr.
940 DCHECK(new_low != nullptr || new_high == nullptr);
941 replacements_[old->id()].low = new_low;
942 replacements_[old->id()].high = new_high;
943 }
944
HasReplacementLow(Node * node)945 bool Int64Lowering::HasReplacementLow(Node* node) {
946 return replacements_[node->id()].low != nullptr;
947 }
948
GetReplacementLow(Node * node)949 Node* Int64Lowering::GetReplacementLow(Node* node) {
950 Node* result = replacements_[node->id()].low;
951 DCHECK(result);
952 return result;
953 }
954
HasReplacementHigh(Node * node)955 bool Int64Lowering::HasReplacementHigh(Node* node) {
956 return replacements_[node->id()].high != nullptr;
957 }
958
GetReplacementHigh(Node * node)959 Node* Int64Lowering::GetReplacementHigh(Node* node) {
960 Node* result = replacements_[node->id()].high;
961 DCHECK(result);
962 return result;
963 }
964
PreparePhiReplacement(Node * phi)965 void Int64Lowering::PreparePhiReplacement(Node* phi) {
966 MachineRepresentation rep = PhiRepresentationOf(phi->op());
967 if (rep == MachineRepresentation::kWord64) {
968 // We have to create the replacements for a phi node before we actually
969 // lower the phi to break potential cycles in the graph. The replacements of
970 // input nodes do not exist yet, so we use a placeholder node to pass the
971 // graph verifier.
972 int value_count = phi->op()->ValueInputCount();
973 Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
974 Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
975 for (int i = 0; i < value_count; i++) {
976 inputs_low[i] = placeholder_;
977 inputs_high[i] = placeholder_;
978 }
979 inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
980 inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
981 ReplaceNode(phi,
982 graph()->NewNode(
983 common()->Phi(MachineRepresentation::kWord32, value_count),
984 value_count + 1, inputs_low, false),
985 graph()->NewNode(
986 common()->Phi(MachineRepresentation::kWord32, value_count),
987 value_count + 1, inputs_high, false));
988 }
989 }
990 } // namespace compiler
991 } // namespace internal
992 } // namespace v8
993