1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/wasm-compiler.h"
6
7 #include <memory>
8
9 #include "src/assembler-inl.h"
10 #include "src/base/optional.h"
11 #include "src/base/platform/elapsed-timer.h"
12 #include "src/base/platform/platform.h"
13 #include "src/base/v8-fallthrough.h"
14 #include "src/builtins/builtins.h"
15 #include "src/code-factory.h"
16 #include "src/compiler.h"
17 #include "src/compiler/access-builder.h"
18 #include "src/compiler/code-generator.h"
19 #include "src/compiler/common-operator.h"
20 #include "src/compiler/compiler-source-position-table.h"
21 #include "src/compiler/diamond.h"
22 #include "src/compiler/graph-visualizer.h"
23 #include "src/compiler/graph.h"
24 #include "src/compiler/instruction-selector.h"
25 #include "src/compiler/int64-lowering.h"
26 #include "src/compiler/js-graph.h"
27 #include "src/compiler/js-operator.h"
28 #include "src/compiler/linkage.h"
29 #include "src/compiler/machine-operator.h"
30 #include "src/compiler/node-matchers.h"
31 #include "src/compiler/node-origin-table.h"
32 #include "src/compiler/pipeline.h"
33 #include "src/compiler/simd-scalar-lowering.h"
34 #include "src/compiler/zone-stats.h"
35 #include "src/heap/factory.h"
36 #include "src/isolate-inl.h"
37 #include "src/log-inl.h"
38 #include "src/trap-handler/trap-handler.h"
39 #include "src/wasm/function-body-decoder.h"
40 #include "src/wasm/function-compiler.h"
41 #include "src/wasm/memory-tracing.h"
42 #include "src/wasm/wasm-code-manager.h"
43 #include "src/wasm/wasm-limits.h"
44 #include "src/wasm/wasm-linkage.h"
45 #include "src/wasm/wasm-module.h"
46 #include "src/wasm/wasm-objects-inl.h"
47 #include "src/wasm/wasm-opcodes.h"
48 #include "src/wasm/wasm-text.h"
49
50 namespace v8 {
51 namespace internal {
52 namespace compiler {
53
54 // TODO(titzer): pull WASM_64 up to a common header.
55 #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
56 #define WASM_64 1
57 #else
58 #define WASM_64 0
59 #endif
60
61 #define FATAL_UNSUPPORTED_OPCODE(opcode) \
62 FATAL("Unsupported opcode 0x%x:%s", (opcode), \
63 wasm::WasmOpcodes::OpcodeName(opcode));
64
65 #define WASM_INSTANCE_OBJECT_OFFSET(name) \
66 (WasmInstanceObject::k##name##Offset - kHeapObjectTag)
67
68 #define LOAD_INSTANCE_FIELD(name, type) \
69 graph()->NewNode( \
70 mcgraph()->machine()->Load(type), instance_node_.get(), \
71 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(name)), *effect_, \
72 *control_)
73
74 #define LOAD_FIXED_ARRAY_SLOT(array_node, index) \
75 graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()), \
76 array_node, \
77 mcgraph()->Int32Constant(FixedArrayOffsetMinusTag(index)), \
78 *effect_, *control_);
79
FixedArrayOffsetMinusTag(uint32_t index)80 int FixedArrayOffsetMinusTag(uint32_t index) {
81 auto access = AccessBuilder::ForFixedArraySlot(index);
82 return access.offset - access.tag();
83 }
84
85 namespace {
86
87 constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2;
88
MergeControlToEnd(MachineGraph * mcgraph,Node * node)89 void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
90 Graph* g = mcgraph->graph();
91 if (g->end()) {
92 NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
93 } else {
94 g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
95 }
96 }
97
ContainsSimd(wasm::FunctionSig * sig)98 bool ContainsSimd(wasm::FunctionSig* sig) {
99 for (auto type : sig->all()) {
100 if (type == wasm::kWasmS128) return true;
101 }
102 return false;
103 }
104
ContainsInt64(wasm::FunctionSig * sig)105 bool ContainsInt64(wasm::FunctionSig* sig) {
106 for (auto type : sig->all()) {
107 if (type == wasm::kWasmI64) return true;
108 }
109 return false;
110 }
111 } // namespace
112
WasmGraphBuilder(Isolate * isolate,wasm::ModuleEnv * env,Zone * zone,MachineGraph * mcgraph,Handle<Code> centry_stub,Handle<Oddball> anyref_null,wasm::FunctionSig * sig,compiler::SourcePositionTable * source_position_table)113 WasmGraphBuilder::WasmGraphBuilder(
114 Isolate* isolate, wasm::ModuleEnv* env, Zone* zone, MachineGraph* mcgraph,
115 Handle<Code> centry_stub, Handle<Oddball> anyref_null,
116 wasm::FunctionSig* sig,
117 compiler::SourcePositionTable* source_position_table)
118 : isolate_(isolate),
119 zone_(zone),
120 mcgraph_(mcgraph),
121 env_(env),
122 centry_stub_(centry_stub),
123 anyref_null_(anyref_null),
124 cur_buffer_(def_buffer_),
125 cur_bufsize_(kDefaultBufferSize),
126 has_simd_(ContainsSimd(sig)),
127 untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
128 sig_(sig),
129 source_position_table_(source_position_table) {
130 DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
131 DCHECK_NOT_NULL(mcgraph_);
132 }
133
Error()134 Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
135
Start(unsigned params)136 Node* WasmGraphBuilder::Start(unsigned params) {
137 Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
138 graph()->SetStart(start);
139 return start;
140 }
141
Param(unsigned index)142 Node* WasmGraphBuilder::Param(unsigned index) {
143 return graph()->NewNode(mcgraph()->common()->Parameter(index),
144 graph()->start());
145 }
146
Loop(Node * entry)147 Node* WasmGraphBuilder::Loop(Node* entry) {
148 return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
149 }
150
Terminate(Node * effect,Node * control)151 Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) {
152 Node* terminate =
153 graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
154 MergeControlToEnd(mcgraph(), terminate);
155 return terminate;
156 }
157
IsPhiWithMerge(Node * phi,Node * merge)158 bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
159 return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
160 NodeProperties::GetControlInput(phi) == merge;
161 }
162
ThrowsException(Node * node,Node ** if_success,Node ** if_exception)163 bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
164 Node** if_exception) {
165 if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
166 return false;
167 }
168
169 *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
170 *if_exception =
171 graph()->NewNode(mcgraph()->common()->IfException(), node, node);
172
173 return true;
174 }
175
AppendToMerge(Node * merge,Node * from)176 void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
177 DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
178 merge->AppendInput(mcgraph()->zone(), from);
179 int new_size = merge->InputCount();
180 NodeProperties::ChangeOp(
181 merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
182 }
183
AppendToPhi(Node * phi,Node * from)184 void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
185 DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
186 int new_size = phi->InputCount();
187 phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
188 NodeProperties::ChangeOp(
189 phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
190 }
191
Merge(unsigned count,Node ** controls)192 Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
193 return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
194 }
195
Phi(wasm::ValueType type,unsigned count,Node ** vals,Node * control)196 Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
197 Node* control) {
198 DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
199 Node** buf = Realloc(vals, count, count + 1);
200 buf[count] = control;
201 return graph()->NewNode(
202 mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
203 count),
204 count + 1, buf);
205 }
206
EffectPhi(unsigned count,Node ** effects,Node * control)207 Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
208 Node* control) {
209 DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
210 Node** buf = Realloc(effects, count, count + 1);
211 buf[count] = control;
212 return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
213 buf);
214 }
215
RefNull()216 Node* WasmGraphBuilder::RefNull() {
217 if (!anyref_null_node_.is_set()) {
218 anyref_null_node_.set(
219 graph()->NewNode(mcgraph()->common()->HeapConstant(anyref_null_)));
220 }
221 return anyref_null_node_.get();
222 }
223
CEntryStub()224 Node* WasmGraphBuilder::CEntryStub() {
225 if (!centry_stub_node_.is_set()) {
226 centry_stub_node_.set(
227 graph()->NewNode(mcgraph()->common()->HeapConstant(centry_stub_)));
228 }
229 return centry_stub_node_.get();
230 }
231
NoContextConstant()232 Node* WasmGraphBuilder::NoContextConstant() {
233 // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
234 return mcgraph()->IntPtrConstant(0);
235 }
236
Uint32Constant(uint32_t value)237 Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
238 return mcgraph()->Uint32Constant(value);
239 }
240
Int32Constant(int32_t value)241 Node* WasmGraphBuilder::Int32Constant(int32_t value) {
242 return mcgraph()->Int32Constant(value);
243 }
244
Int64Constant(int64_t value)245 Node* WasmGraphBuilder::Int64Constant(int64_t value) {
246 return mcgraph()->Int64Constant(value);
247 }
248
IntPtrConstant(intptr_t value)249 Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
250 return mcgraph()->IntPtrConstant(value);
251 }
252
StackCheck(wasm::WasmCodePosition position,Node ** effect,Node ** control)253 void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
254 Node** effect, Node** control) {
255 DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks.
256 if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
257 return;
258 }
259 if (effect == nullptr) effect = effect_;
260 if (control == nullptr) control = control_;
261
262 Node* limit =
263 graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
264 mcgraph()->ExternalConstant(
265 ExternalReference::address_of_stack_limit(isolate_)),
266 mcgraph()->IntPtrConstant(0), *effect, *control);
267 *effect = limit;
268 Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
269
270 Node* check =
271 graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
272
273 Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
274 stack_check.Chain(*control);
275
276 if (stack_check_call_operator_ == nullptr) {
277 // Build and cache the stack check call operator and the constant
278 // representing the stackcheck code.
279 Handle<Code> code = BUILTIN_CODE(isolate_, WasmStackGuard);
280 CallInterfaceDescriptor idesc = WasmRuntimeCallDescriptor(isolate_);
281 auto call_descriptor = Linkage::GetStubCallDescriptor(
282 isolate_, mcgraph()->zone(), idesc, 0, CallDescriptor::kNoFlags,
283 Operator::kNoProperties, MachineType::AnyTagged(), 1,
284 Linkage::kNoContext);
285 stack_check_builtin_code_node_.set(
286 graph()->NewNode(mcgraph()->common()->HeapConstant(code)));
287 stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
288 }
289
290 Node* call = graph()->NewNode(stack_check_call_operator_,
291 stack_check_builtin_code_node_.get(), *effect,
292 stack_check.if_false);
293
294 SetSourcePosition(call, position);
295
296 Node* ephi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), *effect,
297 call, stack_check.merge);
298
299 *control = stack_check.merge;
300 *effect = ephi;
301 }
302
PatchInStackCheckIfNeeded()303 void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
304 if (!needs_stack_check_) return;
305
306 Node* start = graph()->start();
307 // Place a stack check which uses a dummy node as control and effect.
308 Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
309 Node* control = dummy;
310 Node* effect = dummy;
311 // The function-prologue stack check is associated with position 0, which
312 // is never a position of any instruction in the function.
313 StackCheck(0, &effect, &control);
314
315 // In testing, no steck checks were emitted. Nothing to rewire then.
316 if (effect == dummy) return;
317
318 // Now patch all control uses of {start} to use {control} and all effect uses
319 // to use {effect} instead. Then rewire the dummy node to use start instead.
320 NodeProperties::ReplaceUses(start, start, effect, control);
321 NodeProperties::ReplaceUses(dummy, nullptr, start, start);
322 }
323
Binop(wasm::WasmOpcode opcode,Node * left,Node * right,wasm::WasmCodePosition position)324 Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
325 wasm::WasmCodePosition position) {
326 const Operator* op;
327 MachineOperatorBuilder* m = mcgraph()->machine();
328 switch (opcode) {
329 case wasm::kExprI32Add:
330 op = m->Int32Add();
331 break;
332 case wasm::kExprI32Sub:
333 op = m->Int32Sub();
334 break;
335 case wasm::kExprI32Mul:
336 op = m->Int32Mul();
337 break;
338 case wasm::kExprI32DivS:
339 return BuildI32DivS(left, right, position);
340 case wasm::kExprI32DivU:
341 return BuildI32DivU(left, right, position);
342 case wasm::kExprI32RemS:
343 return BuildI32RemS(left, right, position);
344 case wasm::kExprI32RemU:
345 return BuildI32RemU(left, right, position);
346 case wasm::kExprI32And:
347 op = m->Word32And();
348 break;
349 case wasm::kExprI32Ior:
350 op = m->Word32Or();
351 break;
352 case wasm::kExprI32Xor:
353 op = m->Word32Xor();
354 break;
355 case wasm::kExprI32Shl:
356 op = m->Word32Shl();
357 right = MaskShiftCount32(right);
358 break;
359 case wasm::kExprI32ShrU:
360 op = m->Word32Shr();
361 right = MaskShiftCount32(right);
362 break;
363 case wasm::kExprI32ShrS:
364 op = m->Word32Sar();
365 right = MaskShiftCount32(right);
366 break;
367 case wasm::kExprI32Ror:
368 op = m->Word32Ror();
369 right = MaskShiftCount32(right);
370 break;
371 case wasm::kExprI32Rol:
372 right = MaskShiftCount32(right);
373 return BuildI32Rol(left, right);
374 case wasm::kExprI32Eq:
375 op = m->Word32Equal();
376 break;
377 case wasm::kExprI32Ne:
378 return Invert(Binop(wasm::kExprI32Eq, left, right));
379 case wasm::kExprI32LtS:
380 op = m->Int32LessThan();
381 break;
382 case wasm::kExprI32LeS:
383 op = m->Int32LessThanOrEqual();
384 break;
385 case wasm::kExprI32LtU:
386 op = m->Uint32LessThan();
387 break;
388 case wasm::kExprI32LeU:
389 op = m->Uint32LessThanOrEqual();
390 break;
391 case wasm::kExprI32GtS:
392 op = m->Int32LessThan();
393 std::swap(left, right);
394 break;
395 case wasm::kExprI32GeS:
396 op = m->Int32LessThanOrEqual();
397 std::swap(left, right);
398 break;
399 case wasm::kExprI32GtU:
400 op = m->Uint32LessThan();
401 std::swap(left, right);
402 break;
403 case wasm::kExprI32GeU:
404 op = m->Uint32LessThanOrEqual();
405 std::swap(left, right);
406 break;
407 case wasm::kExprI64And:
408 op = m->Word64And();
409 break;
410 case wasm::kExprI64Add:
411 op = m->Int64Add();
412 break;
413 case wasm::kExprI64Sub:
414 op = m->Int64Sub();
415 break;
416 case wasm::kExprI64Mul:
417 op = m->Int64Mul();
418 break;
419 case wasm::kExprI64DivS:
420 return BuildI64DivS(left, right, position);
421 case wasm::kExprI64DivU:
422 return BuildI64DivU(left, right, position);
423 case wasm::kExprI64RemS:
424 return BuildI64RemS(left, right, position);
425 case wasm::kExprI64RemU:
426 return BuildI64RemU(left, right, position);
427 case wasm::kExprI64Ior:
428 op = m->Word64Or();
429 break;
430 case wasm::kExprI64Xor:
431 op = m->Word64Xor();
432 break;
433 case wasm::kExprI64Shl:
434 op = m->Word64Shl();
435 right = MaskShiftCount64(right);
436 break;
437 case wasm::kExprI64ShrU:
438 op = m->Word64Shr();
439 right = MaskShiftCount64(right);
440 break;
441 case wasm::kExprI64ShrS:
442 op = m->Word64Sar();
443 right = MaskShiftCount64(right);
444 break;
445 case wasm::kExprI64Eq:
446 op = m->Word64Equal();
447 break;
448 case wasm::kExprI64Ne:
449 return Invert(Binop(wasm::kExprI64Eq, left, right));
450 case wasm::kExprI64LtS:
451 op = m->Int64LessThan();
452 break;
453 case wasm::kExprI64LeS:
454 op = m->Int64LessThanOrEqual();
455 break;
456 case wasm::kExprI64LtU:
457 op = m->Uint64LessThan();
458 break;
459 case wasm::kExprI64LeU:
460 op = m->Uint64LessThanOrEqual();
461 break;
462 case wasm::kExprI64GtS:
463 op = m->Int64LessThan();
464 std::swap(left, right);
465 break;
466 case wasm::kExprI64GeS:
467 op = m->Int64LessThanOrEqual();
468 std::swap(left, right);
469 break;
470 case wasm::kExprI64GtU:
471 op = m->Uint64LessThan();
472 std::swap(left, right);
473 break;
474 case wasm::kExprI64GeU:
475 op = m->Uint64LessThanOrEqual();
476 std::swap(left, right);
477 break;
478 case wasm::kExprI64Ror:
479 op = m->Word64Ror();
480 right = MaskShiftCount64(right);
481 break;
482 case wasm::kExprI64Rol:
483 return BuildI64Rol(left, right);
484 case wasm::kExprF32CopySign:
485 return BuildF32CopySign(left, right);
486 case wasm::kExprF64CopySign:
487 return BuildF64CopySign(left, right);
488 case wasm::kExprF32Add:
489 op = m->Float32Add();
490 break;
491 case wasm::kExprF32Sub:
492 op = m->Float32Sub();
493 break;
494 case wasm::kExprF32Mul:
495 op = m->Float32Mul();
496 break;
497 case wasm::kExprF32Div:
498 op = m->Float32Div();
499 break;
500 case wasm::kExprF32Eq:
501 op = m->Float32Equal();
502 break;
503 case wasm::kExprF32Ne:
504 return Invert(Binop(wasm::kExprF32Eq, left, right));
505 case wasm::kExprF32Lt:
506 op = m->Float32LessThan();
507 break;
508 case wasm::kExprF32Ge:
509 op = m->Float32LessThanOrEqual();
510 std::swap(left, right);
511 break;
512 case wasm::kExprF32Gt:
513 op = m->Float32LessThan();
514 std::swap(left, right);
515 break;
516 case wasm::kExprF32Le:
517 op = m->Float32LessThanOrEqual();
518 break;
519 case wasm::kExprF64Add:
520 op = m->Float64Add();
521 break;
522 case wasm::kExprF64Sub:
523 op = m->Float64Sub();
524 break;
525 case wasm::kExprF64Mul:
526 op = m->Float64Mul();
527 break;
528 case wasm::kExprF64Div:
529 op = m->Float64Div();
530 break;
531 case wasm::kExprF64Eq:
532 op = m->Float64Equal();
533 break;
534 case wasm::kExprF64Ne:
535 return Invert(Binop(wasm::kExprF64Eq, left, right));
536 case wasm::kExprF64Lt:
537 op = m->Float64LessThan();
538 break;
539 case wasm::kExprF64Le:
540 op = m->Float64LessThanOrEqual();
541 break;
542 case wasm::kExprF64Gt:
543 op = m->Float64LessThan();
544 std::swap(left, right);
545 break;
546 case wasm::kExprF64Ge:
547 op = m->Float64LessThanOrEqual();
548 std::swap(left, right);
549 break;
550 case wasm::kExprF32Min:
551 op = m->Float32Min();
552 break;
553 case wasm::kExprF64Min:
554 op = m->Float64Min();
555 break;
556 case wasm::kExprF32Max:
557 op = m->Float32Max();
558 break;
559 case wasm::kExprF64Max:
560 op = m->Float64Max();
561 break;
562 case wasm::kExprF64Pow:
563 return BuildF64Pow(left, right);
564 case wasm::kExprF64Atan2:
565 op = m->Float64Atan2();
566 break;
567 case wasm::kExprF64Mod:
568 return BuildF64Mod(left, right);
569 case wasm::kExprI32AsmjsDivS:
570 return BuildI32AsmjsDivS(left, right);
571 case wasm::kExprI32AsmjsDivU:
572 return BuildI32AsmjsDivU(left, right);
573 case wasm::kExprI32AsmjsRemS:
574 return BuildI32AsmjsRemS(left, right);
575 case wasm::kExprI32AsmjsRemU:
576 return BuildI32AsmjsRemU(left, right);
577 case wasm::kExprI32AsmjsStoreMem8:
578 return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
579 case wasm::kExprI32AsmjsStoreMem16:
580 return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
581 case wasm::kExprI32AsmjsStoreMem:
582 return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
583 case wasm::kExprF32AsmjsStoreMem:
584 return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
585 case wasm::kExprF64AsmjsStoreMem:
586 return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
587 default:
588 FATAL_UNSUPPORTED_OPCODE(opcode);
589 }
590 return graph()->NewNode(op, left, right);
591 }
592
Unop(wasm::WasmOpcode opcode,Node * input,wasm::WasmCodePosition position)593 Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
594 wasm::WasmCodePosition position) {
595 const Operator* op;
596 MachineOperatorBuilder* m = mcgraph()->machine();
597 switch (opcode) {
598 case wasm::kExprI32Eqz:
599 op = m->Word32Equal();
600 return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
601 case wasm::kExprF32Abs:
602 op = m->Float32Abs();
603 break;
604 case wasm::kExprF32Neg: {
605 op = m->Float32Neg();
606 break;
607 }
608 case wasm::kExprF32Sqrt:
609 op = m->Float32Sqrt();
610 break;
611 case wasm::kExprF64Abs:
612 op = m->Float64Abs();
613 break;
614 case wasm::kExprF64Neg: {
615 op = m->Float64Neg();
616 break;
617 }
618 case wasm::kExprF64Sqrt:
619 op = m->Float64Sqrt();
620 break;
621 case wasm::kExprI32SConvertF32:
622 case wasm::kExprI32UConvertF32:
623 case wasm::kExprI32SConvertF64:
624 case wasm::kExprI32UConvertF64:
625 case wasm::kExprI32SConvertSatF64:
626 case wasm::kExprI32UConvertSatF64:
627 case wasm::kExprI32SConvertSatF32:
628 case wasm::kExprI32UConvertSatF32:
629 return BuildIntConvertFloat(input, position, opcode);
630 case wasm::kExprI32AsmjsSConvertF64:
631 return BuildI32AsmjsSConvertF64(input);
632 case wasm::kExprI32AsmjsUConvertF64:
633 return BuildI32AsmjsUConvertF64(input);
634 case wasm::kExprF32ConvertF64:
635 op = m->TruncateFloat64ToFloat32();
636 break;
637 case wasm::kExprF64SConvertI32:
638 op = m->ChangeInt32ToFloat64();
639 break;
640 case wasm::kExprF64UConvertI32:
641 op = m->ChangeUint32ToFloat64();
642 break;
643 case wasm::kExprF32SConvertI32:
644 op = m->RoundInt32ToFloat32();
645 break;
646 case wasm::kExprF32UConvertI32:
647 op = m->RoundUint32ToFloat32();
648 break;
649 case wasm::kExprI32AsmjsSConvertF32:
650 return BuildI32AsmjsSConvertF32(input);
651 case wasm::kExprI32AsmjsUConvertF32:
652 return BuildI32AsmjsUConvertF32(input);
653 case wasm::kExprF64ConvertF32:
654 op = m->ChangeFloat32ToFloat64();
655 break;
656 case wasm::kExprF32ReinterpretI32:
657 op = m->BitcastInt32ToFloat32();
658 break;
659 case wasm::kExprI32ReinterpretF32:
660 op = m->BitcastFloat32ToInt32();
661 break;
662 case wasm::kExprI32Clz:
663 op = m->Word32Clz();
664 break;
665 case wasm::kExprI32Ctz: {
666 if (m->Word32Ctz().IsSupported()) {
667 op = m->Word32Ctz().op();
668 break;
669 } else if (m->Word32ReverseBits().IsSupported()) {
670 Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
671 Node* result = graph()->NewNode(m->Word32Clz(), reversed);
672 return result;
673 } else {
674 return BuildI32Ctz(input);
675 }
676 }
677 case wasm::kExprI32Popcnt: {
678 if (m->Word32Popcnt().IsSupported()) {
679 op = m->Word32Popcnt().op();
680 break;
681 } else {
682 return BuildI32Popcnt(input);
683 }
684 }
685 case wasm::kExprF32Floor: {
686 if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
687 op = m->Float32RoundDown().op();
688 break;
689 }
690 case wasm::kExprF32Ceil: {
691 if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
692 op = m->Float32RoundUp().op();
693 break;
694 }
695 case wasm::kExprF32Trunc: {
696 if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
697 op = m->Float32RoundTruncate().op();
698 break;
699 }
700 case wasm::kExprF32NearestInt: {
701 if (!m->Float32RoundTiesEven().IsSupported())
702 return BuildF32NearestInt(input);
703 op = m->Float32RoundTiesEven().op();
704 break;
705 }
706 case wasm::kExprF64Floor: {
707 if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
708 op = m->Float64RoundDown().op();
709 break;
710 }
711 case wasm::kExprF64Ceil: {
712 if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
713 op = m->Float64RoundUp().op();
714 break;
715 }
716 case wasm::kExprF64Trunc: {
717 if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
718 op = m->Float64RoundTruncate().op();
719 break;
720 }
721 case wasm::kExprF64NearestInt: {
722 if (!m->Float64RoundTiesEven().IsSupported())
723 return BuildF64NearestInt(input);
724 op = m->Float64RoundTiesEven().op();
725 break;
726 }
727 case wasm::kExprF64Acos: {
728 return BuildF64Acos(input);
729 }
730 case wasm::kExprF64Asin: {
731 return BuildF64Asin(input);
732 }
733 case wasm::kExprF64Atan:
734 op = m->Float64Atan();
735 break;
736 case wasm::kExprF64Cos: {
737 op = m->Float64Cos();
738 break;
739 }
740 case wasm::kExprF64Sin: {
741 op = m->Float64Sin();
742 break;
743 }
744 case wasm::kExprF64Tan: {
745 op = m->Float64Tan();
746 break;
747 }
748 case wasm::kExprF64Exp: {
749 op = m->Float64Exp();
750 break;
751 }
752 case wasm::kExprF64Log:
753 op = m->Float64Log();
754 break;
755 case wasm::kExprI32ConvertI64:
756 op = m->TruncateInt64ToInt32();
757 break;
758 case wasm::kExprI64SConvertI32:
759 op = m->ChangeInt32ToInt64();
760 break;
761 case wasm::kExprI64UConvertI32:
762 op = m->ChangeUint32ToUint64();
763 break;
764 case wasm::kExprF64ReinterpretI64:
765 op = m->BitcastInt64ToFloat64();
766 break;
767 case wasm::kExprI64ReinterpretF64:
768 op = m->BitcastFloat64ToInt64();
769 break;
770 case wasm::kExprI64Clz:
771 op = m->Word64Clz();
772 break;
773 case wasm::kExprI64Ctz: {
774 OptionalOperator ctz64 = m->Word64Ctz();
775 if (ctz64.IsSupported()) {
776 op = ctz64.op();
777 break;
778 } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
779 op = ctz64.placeholder();
780 break;
781 } else if (m->Word64ReverseBits().IsSupported()) {
782 Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
783 Node* result = graph()->NewNode(m->Word64Clz(), reversed);
784 return result;
785 } else {
786 return BuildI64Ctz(input);
787 }
788 }
789 case wasm::kExprI64Popcnt: {
790 OptionalOperator popcnt64 = m->Word64Popcnt();
791 if (popcnt64.IsSupported()) {
792 op = popcnt64.op();
793 } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
794 op = popcnt64.placeholder();
795 } else {
796 return BuildI64Popcnt(input);
797 }
798 break;
799 }
800 case wasm::kExprI64Eqz:
801 op = m->Word64Equal();
802 return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
803 case wasm::kExprF32SConvertI64:
804 if (m->Is32()) {
805 return BuildF32SConvertI64(input);
806 }
807 op = m->RoundInt64ToFloat32();
808 break;
809 case wasm::kExprF32UConvertI64:
810 if (m->Is32()) {
811 return BuildF32UConvertI64(input);
812 }
813 op = m->RoundUint64ToFloat32();
814 break;
815 case wasm::kExprF64SConvertI64:
816 if (m->Is32()) {
817 return BuildF64SConvertI64(input);
818 }
819 op = m->RoundInt64ToFloat64();
820 break;
821 case wasm::kExprF64UConvertI64:
822 if (m->Is32()) {
823 return BuildF64UConvertI64(input);
824 }
825 op = m->RoundUint64ToFloat64();
826 break;
827 case wasm::kExprI32SExtendI8:
828 op = m->SignExtendWord8ToInt32();
829 break;
830 case wasm::kExprI32SExtendI16:
831 op = m->SignExtendWord16ToInt32();
832 break;
833 case wasm::kExprI64SExtendI8:
834 op = m->SignExtendWord8ToInt64();
835 break;
836 case wasm::kExprI64SExtendI16:
837 op = m->SignExtendWord16ToInt64();
838 break;
839 case wasm::kExprI64SExtendI32:
840 op = m->SignExtendWord32ToInt64();
841 break;
842 case wasm::kExprI64SConvertF32:
843 case wasm::kExprI64UConvertF32:
844 case wasm::kExprI64SConvertF64:
845 case wasm::kExprI64UConvertF64:
846 case wasm::kExprI64SConvertSatF32:
847 case wasm::kExprI64UConvertSatF32:
848 case wasm::kExprI64SConvertSatF64:
849 case wasm::kExprI64UConvertSatF64:
850 return mcgraph()->machine()->Is32()
851 ? BuildCcallConvertFloat(input, position, opcode)
852 : BuildIntConvertFloat(input, position, opcode);
853 case wasm::kExprRefIsNull:
854 return graph()->NewNode(m->WordEqual(), input, RefNull());
855 case wasm::kExprI32AsmjsLoadMem8S:
856 return BuildAsmjsLoadMem(MachineType::Int8(), input);
857 case wasm::kExprI32AsmjsLoadMem8U:
858 return BuildAsmjsLoadMem(MachineType::Uint8(), input);
859 case wasm::kExprI32AsmjsLoadMem16S:
860 return BuildAsmjsLoadMem(MachineType::Int16(), input);
861 case wasm::kExprI32AsmjsLoadMem16U:
862 return BuildAsmjsLoadMem(MachineType::Uint16(), input);
863 case wasm::kExprI32AsmjsLoadMem:
864 return BuildAsmjsLoadMem(MachineType::Int32(), input);
865 case wasm::kExprF32AsmjsLoadMem:
866 return BuildAsmjsLoadMem(MachineType::Float32(), input);
867 case wasm::kExprF64AsmjsLoadMem:
868 return BuildAsmjsLoadMem(MachineType::Float64(), input);
869 default:
870 FATAL_UNSUPPORTED_OPCODE(opcode);
871 }
872 return graph()->NewNode(op, input);
873 }
874
Float32Constant(float value)875 Node* WasmGraphBuilder::Float32Constant(float value) {
876 return mcgraph()->Float32Constant(value);
877 }
878
Float64Constant(double value)879 Node* WasmGraphBuilder::Float64Constant(double value) {
880 return mcgraph()->Float64Constant(value);
881 }
882
883 namespace {
Branch(MachineGraph * mcgraph,Node * cond,Node ** true_node,Node ** false_node,Node * control,BranchHint hint)884 Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
885 Node** false_node, Node* control, BranchHint hint) {
886 DCHECK_NOT_NULL(cond);
887 DCHECK_NOT_NULL(control);
888 Node* branch =
889 mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
890 *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
891 *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
892 return branch;
893 }
894 } // namespace
895
BranchNoHint(Node * cond,Node ** true_node,Node ** false_node)896 Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
897 Node** false_node) {
898 return Branch(mcgraph(), cond, true_node, false_node, *control_,
899 BranchHint::kNone);
900 }
901
BranchExpectTrue(Node * cond,Node ** true_node,Node ** false_node)902 Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
903 Node** false_node) {
904 return Branch(mcgraph(), cond, true_node, false_node, *control_,
905 BranchHint::kTrue);
906 }
907
BranchExpectFalse(Node * cond,Node ** true_node,Node ** false_node)908 Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
909 Node** false_node) {
910 return Branch(mcgraph(), cond, true_node, false_node, *control_,
911 BranchHint::kFalse);
912 }
913
GetBuiltinIdForTrap(wasm::TrapReason reason)914 Builtins::Name WasmGraphBuilder::GetBuiltinIdForTrap(wasm::TrapReason reason) {
915 // TODO(wasm): "!env_" should not happen when compiling an actual wasm
916 // function.
917 if (!env_ || !env_->runtime_exception_support) {
918 // We use Builtins::builtin_count as a marker to tell the code generator
919 // to generate a call to a testing c-function instead of a runtime
920 // function. This code should only be called from a cctest.
921 return Builtins::builtin_count;
922 }
923
924 switch (reason) {
925 #define TRAPREASON_TO_MESSAGE(name) \
926 case wasm::k##name: \
927 return Builtins::kThrowWasm##name;
928 FOREACH_WASM_TRAPREASON(TRAPREASON_TO_MESSAGE)
929 #undef TRAPREASON_TO_MESSAGE
930 default:
931 UNREACHABLE();
932 }
933 }
934
TrapIfTrue(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)935 Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
936 wasm::WasmCodePosition position) {
937 Builtins::Name trap_id = GetBuiltinIdForTrap(reason);
938 Node* node = graph()->NewNode(mcgraph()->common()->TrapIf(trap_id), cond,
939 Effect(), Control());
940 *control_ = node;
941 SetSourcePosition(node, position);
942 return node;
943 }
944
TrapIfFalse(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)945 Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
946 wasm::WasmCodePosition position) {
947 Builtins::Name trap_id = GetBuiltinIdForTrap(reason);
948
949 Node* node = graph()->NewNode(mcgraph()->common()->TrapUnless(trap_id), cond,
950 Effect(), Control());
951 *control_ = node;
952 SetSourcePosition(node, position);
953 return node;
954 }
955
956 // Add a check that traps if {node} is equal to {val}.
TrapIfEq32(wasm::TrapReason reason,Node * node,int32_t val,wasm::WasmCodePosition position)957 Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
958 int32_t val,
959 wasm::WasmCodePosition position) {
960 Int32Matcher m(node);
961 if (m.HasValue() && !m.Is(val)) return graph()->start();
962 if (val == 0) {
963 return TrapIfFalse(reason, node, position);
964 } else {
965 return TrapIfTrue(reason,
966 graph()->NewNode(mcgraph()->machine()->Word32Equal(),
967 node, mcgraph()->Int32Constant(val)),
968 position);
969 }
970 }
971
972 // Add a check that traps if {node} is zero.
ZeroCheck32(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)973 Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
974 wasm::WasmCodePosition position) {
975 return TrapIfEq32(reason, node, 0, position);
976 }
977
978 // Add a check that traps if {node} is equal to {val}.
TrapIfEq64(wasm::TrapReason reason,Node * node,int64_t val,wasm::WasmCodePosition position)979 Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
980 int64_t val,
981 wasm::WasmCodePosition position) {
982 Int64Matcher m(node);
983 if (m.HasValue() && !m.Is(val)) return graph()->start();
984 return TrapIfTrue(reason,
985 graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
986 mcgraph()->Int64Constant(val)),
987 position);
988 }
989
990 // Add a check that traps if {node} is zero.
ZeroCheck64(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)991 Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
992 wasm::WasmCodePosition position) {
993 return TrapIfEq64(reason, node, 0, position);
994 }
995
Switch(unsigned count,Node * key)996 Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
997 return graph()->NewNode(mcgraph()->common()->Switch(count), key, *control_);
998 }
999
IfValue(int32_t value,Node * sw)1000 Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1001 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1002 return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1003 }
1004
IfDefault(Node * sw)1005 Node* WasmGraphBuilder::IfDefault(Node* sw) {
1006 DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1007 return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1008 }
1009
Return(unsigned count,Node ** vals)1010 Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1011 DCHECK_NOT_NULL(*control_);
1012 DCHECK_NOT_NULL(*effect_);
1013
1014 static const int kStackAllocatedNodeBufferSize = 8;
1015 Node* stack_buffer[kStackAllocatedNodeBufferSize];
1016 std::vector<Node*> heap_buffer;
1017
1018 Node** buf = stack_buffer;
1019 if (count + 3 > kStackAllocatedNodeBufferSize) {
1020 heap_buffer.resize(count + 3);
1021 buf = heap_buffer.data();
1022 }
1023
1024 buf[0] = mcgraph()->Int32Constant(0);
1025 memcpy(buf + 1, vals, sizeof(void*) * count);
1026 buf[count + 1] = *effect_;
1027 buf[count + 2] = *control_;
1028 Node* ret =
1029 graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1030
1031 MergeControlToEnd(mcgraph(), ret);
1032 return ret;
1033 }
1034
ReturnVoid()1035 Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1036
Unreachable(wasm::WasmCodePosition position)1037 Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1038 TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1039 ReturnVoid();
1040 return nullptr;
1041 }
1042
MaskShiftCount32(Node * node)1043 Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1044 static const int32_t kMask32 = 0x1F;
1045 if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1046 // Shifts by constants are so common we pattern-match them here.
1047 Int32Matcher match(node);
1048 if (match.HasValue()) {
1049 int32_t masked = (match.Value() & kMask32);
1050 if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1051 } else {
1052 node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1053 mcgraph()->Int32Constant(kMask32));
1054 }
1055 }
1056 return node;
1057 }
1058
MaskShiftCount64(Node * node)1059 Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1060 static const int64_t kMask64 = 0x3F;
1061 if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1062 // Shifts by constants are so common we pattern-match them here.
1063 Int64Matcher match(node);
1064 if (match.HasValue()) {
1065 int64_t masked = (match.Value() & kMask64);
1066 if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1067 } else {
1068 node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1069 mcgraph()->Int64Constant(kMask64));
1070 }
1071 }
1072 return node;
1073 }
1074
ReverseBytesSupported(MachineOperatorBuilder * m,size_t size_in_bytes)1075 static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1076 size_t size_in_bytes) {
1077 switch (size_in_bytes) {
1078 case 4:
1079 case 16:
1080 return m->Word32ReverseBytes().IsSupported();
1081 case 8:
1082 return m->Word64ReverseBytes().IsSupported();
1083 default:
1084 break;
1085 }
1086 return false;
1087 }
1088
BuildChangeEndiannessStore(Node * node,MachineRepresentation mem_rep,wasm::ValueType wasmtype)1089 Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1090 Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1091 Node* result;
1092 Node* value = node;
1093 MachineOperatorBuilder* m = mcgraph()->machine();
1094 int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1095 int valueSizeInBits = 8 * valueSizeInBytes;
1096 bool isFloat = false;
1097
1098 switch (wasmtype) {
1099 case wasm::kWasmF64:
1100 value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1101 isFloat = true;
1102 V8_FALLTHROUGH;
1103 case wasm::kWasmI64:
1104 result = mcgraph()->Int64Constant(0);
1105 break;
1106 case wasm::kWasmF32:
1107 value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1108 isFloat = true;
1109 V8_FALLTHROUGH;
1110 case wasm::kWasmI32:
1111 result = mcgraph()->Int32Constant(0);
1112 break;
1113 case wasm::kWasmS128:
1114 DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1115 break;
1116 default:
1117 UNREACHABLE();
1118 break;
1119 }
1120
1121 if (mem_rep == MachineRepresentation::kWord8) {
1122 // No need to change endianness for byte size, return original node
1123 return node;
1124 }
1125 if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1126 // In case we store lower part of WasmI64 expression, we can truncate
1127 // upper 32bits
1128 value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1129 valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1130 valueSizeInBits = 8 * valueSizeInBytes;
1131 if (mem_rep == MachineRepresentation::kWord16) {
1132 value =
1133 graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1134 }
1135 } else if (wasmtype == wasm::kWasmI32 &&
1136 mem_rep == MachineRepresentation::kWord16) {
1137 value =
1138 graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1139 }
1140
1141 int i;
1142 uint32_t shiftCount;
1143
1144 if (ReverseBytesSupported(m, valueSizeInBytes)) {
1145 switch (valueSizeInBytes) {
1146 case 4:
1147 result = graph()->NewNode(m->Word32ReverseBytes().op(), value);
1148 break;
1149 case 8:
1150 result = graph()->NewNode(m->Word64ReverseBytes().op(), value);
1151 break;
1152 case 16: {
1153 Node* byte_reversed_lanes[4];
1154 for (int lane = 0; lane < 4; lane++) {
1155 byte_reversed_lanes[lane] = graph()->NewNode(
1156 m->Word32ReverseBytes().op(),
1157 graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1158 value));
1159 }
1160
1161 // This is making a copy of the value.
1162 result =
1163 graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1164
1165 for (int lane = 0; lane < 4; lane++) {
1166 result =
1167 graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1168 result, byte_reversed_lanes[lane]);
1169 }
1170
1171 break;
1172 }
1173 default:
1174 UNREACHABLE();
1175 break;
1176 }
1177 } else {
1178 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1179 i += 8, shiftCount -= 16) {
1180 Node* shiftLower;
1181 Node* shiftHigher;
1182 Node* lowerByte;
1183 Node* higherByte;
1184
1185 DCHECK_LT(0, shiftCount);
1186 DCHECK_EQ(0, (shiftCount + 8) % 16);
1187
1188 if (valueSizeInBits > 32) {
1189 shiftLower = graph()->NewNode(m->Word64Shl(), value,
1190 mcgraph()->Int64Constant(shiftCount));
1191 shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1192 mcgraph()->Int64Constant(shiftCount));
1193 lowerByte = graph()->NewNode(
1194 m->Word64And(), shiftLower,
1195 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1196 << (valueSizeInBits - 8 - i)));
1197 higherByte = graph()->NewNode(
1198 m->Word64And(), shiftHigher,
1199 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1200 result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1201 result = graph()->NewNode(m->Word64Or(), result, higherByte);
1202 } else {
1203 shiftLower = graph()->NewNode(m->Word32Shl(), value,
1204 mcgraph()->Int32Constant(shiftCount));
1205 shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1206 mcgraph()->Int32Constant(shiftCount));
1207 lowerByte = graph()->NewNode(
1208 m->Word32And(), shiftLower,
1209 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1210 << (valueSizeInBits - 8 - i)));
1211 higherByte = graph()->NewNode(
1212 m->Word32And(), shiftHigher,
1213 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1214 result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1215 result = graph()->NewNode(m->Word32Or(), result, higherByte);
1216 }
1217 }
1218 }
1219
1220 if (isFloat) {
1221 switch (wasmtype) {
1222 case wasm::kWasmF64:
1223 result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1224 break;
1225 case wasm::kWasmF32:
1226 result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1227 break;
1228 default:
1229 UNREACHABLE();
1230 break;
1231 }
1232 }
1233
1234 return result;
1235 }
1236
BuildChangeEndiannessLoad(Node * node,MachineType memtype,wasm::ValueType wasmtype)1237 Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1238 MachineType memtype,
1239 wasm::ValueType wasmtype) {
1240 Node* result;
1241 Node* value = node;
1242 MachineOperatorBuilder* m = mcgraph()->machine();
1243 int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1244 int valueSizeInBits = 8 * valueSizeInBytes;
1245 bool isFloat = false;
1246
1247 switch (memtype.representation()) {
1248 case MachineRepresentation::kFloat64:
1249 value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1250 isFloat = true;
1251 V8_FALLTHROUGH;
1252 case MachineRepresentation::kWord64:
1253 result = mcgraph()->Int64Constant(0);
1254 break;
1255 case MachineRepresentation::kFloat32:
1256 value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1257 isFloat = true;
1258 V8_FALLTHROUGH;
1259 case MachineRepresentation::kWord32:
1260 case MachineRepresentation::kWord16:
1261 result = mcgraph()->Int32Constant(0);
1262 break;
1263 case MachineRepresentation::kWord8:
1264 // No need to change endianness for byte size, return original node
1265 return node;
1266 break;
1267 case MachineRepresentation::kSimd128:
1268 DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1269 break;
1270 default:
1271 UNREACHABLE();
1272 break;
1273 }
1274
1275 int i;
1276 uint32_t shiftCount;
1277
1278 if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1279 switch (valueSizeInBytes) {
1280 case 2:
1281 result =
1282 graph()->NewNode(m->Word32ReverseBytes().op(),
1283 graph()->NewNode(m->Word32Shl(), value,
1284 mcgraph()->Int32Constant(16)));
1285 break;
1286 case 4:
1287 result = graph()->NewNode(m->Word32ReverseBytes().op(), value);
1288 break;
1289 case 8:
1290 result = graph()->NewNode(m->Word64ReverseBytes().op(), value);
1291 break;
1292 case 16: {
1293 Node* byte_reversed_lanes[4];
1294 for (int lane = 0; lane < 4; lane++) {
1295 byte_reversed_lanes[lane] = graph()->NewNode(
1296 m->Word32ReverseBytes().op(),
1297 graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1298 value));
1299 }
1300
1301 // This is making a copy of the value.
1302 result =
1303 graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1304
1305 for (int lane = 0; lane < 4; lane++) {
1306 result =
1307 graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1308 result, byte_reversed_lanes[lane]);
1309 }
1310
1311 break;
1312 }
1313 default:
1314 UNREACHABLE();
1315 }
1316 } else {
1317 for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1318 i += 8, shiftCount -= 16) {
1319 Node* shiftLower;
1320 Node* shiftHigher;
1321 Node* lowerByte;
1322 Node* higherByte;
1323
1324 DCHECK_LT(0, shiftCount);
1325 DCHECK_EQ(0, (shiftCount + 8) % 16);
1326
1327 if (valueSizeInBits > 32) {
1328 shiftLower = graph()->NewNode(m->Word64Shl(), value,
1329 mcgraph()->Int64Constant(shiftCount));
1330 shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1331 mcgraph()->Int64Constant(shiftCount));
1332 lowerByte = graph()->NewNode(
1333 m->Word64And(), shiftLower,
1334 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1335 << (valueSizeInBits - 8 - i)));
1336 higherByte = graph()->NewNode(
1337 m->Word64And(), shiftHigher,
1338 mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1339 result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1340 result = graph()->NewNode(m->Word64Or(), result, higherByte);
1341 } else {
1342 shiftLower = graph()->NewNode(m->Word32Shl(), value,
1343 mcgraph()->Int32Constant(shiftCount));
1344 shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1345 mcgraph()->Int32Constant(shiftCount));
1346 lowerByte = graph()->NewNode(
1347 m->Word32And(), shiftLower,
1348 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1349 << (valueSizeInBits - 8 - i)));
1350 higherByte = graph()->NewNode(
1351 m->Word32And(), shiftHigher,
1352 mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1353 result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1354 result = graph()->NewNode(m->Word32Or(), result, higherByte);
1355 }
1356 }
1357 }
1358
1359 if (isFloat) {
1360 switch (memtype.representation()) {
1361 case MachineRepresentation::kFloat64:
1362 result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1363 break;
1364 case MachineRepresentation::kFloat32:
1365 result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1366 break;
1367 default:
1368 UNREACHABLE();
1369 break;
1370 }
1371 }
1372
1373 // We need to sign extend the value
1374 if (memtype.IsSigned()) {
1375 DCHECK(!isFloat);
1376 if (valueSizeInBits < 32) {
1377 Node* shiftBitCount;
1378 // Perform sign extension using following trick
1379 // result = (x << machine_width - type_width) >> (machine_width -
1380 // type_width)
1381 if (wasmtype == wasm::kWasmI64) {
1382 shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1383 result = graph()->NewNode(
1384 m->Word64Sar(),
1385 graph()->NewNode(m->Word64Shl(),
1386 graph()->NewNode(m->ChangeInt32ToInt64(), result),
1387 shiftBitCount),
1388 shiftBitCount);
1389 } else if (wasmtype == wasm::kWasmI32) {
1390 shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1391 result = graph()->NewNode(
1392 m->Word32Sar(),
1393 graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1394 shiftBitCount);
1395 }
1396 }
1397 }
1398
1399 return result;
1400 }
1401
BuildF32CopySign(Node * left,Node * right)1402 Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1403 Node* result = Unop(
1404 wasm::kExprF32ReinterpretI32,
1405 Binop(wasm::kExprI32Ior,
1406 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1407 mcgraph()->Int32Constant(0x7FFFFFFF)),
1408 Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1409 mcgraph()->Int32Constant(0x80000000))));
1410
1411 return result;
1412 }
1413
BuildF64CopySign(Node * left,Node * right)1414 Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1415 #if WASM_64
1416 Node* result = Unop(
1417 wasm::kExprF64ReinterpretI64,
1418 Binop(wasm::kExprI64Ior,
1419 Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1420 mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1421 Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1422 mcgraph()->Int64Constant(0x8000000000000000))));
1423
1424 return result;
1425 #else
1426 MachineOperatorBuilder* m = mcgraph()->machine();
1427
1428 Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1429 Node* high_word_right =
1430 graph()->NewNode(m->Float64ExtractHighWord32(), right);
1431
1432 Node* new_high_word = Binop(wasm::kExprI32Ior,
1433 Binop(wasm::kExprI32And, high_word_left,
1434 mcgraph()->Int32Constant(0x7FFFFFFF)),
1435 Binop(wasm::kExprI32And, high_word_right,
1436 mcgraph()->Int32Constant(0x80000000)));
1437
1438 return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1439 #endif
1440 }
1441
1442 namespace {
1443
IntConvertType(wasm::WasmOpcode opcode)1444 MachineType IntConvertType(wasm::WasmOpcode opcode) {
1445 switch (opcode) {
1446 case wasm::kExprI32SConvertF32:
1447 case wasm::kExprI32SConvertF64:
1448 case wasm::kExprI32SConvertSatF32:
1449 case wasm::kExprI32SConvertSatF64:
1450 return MachineType::Int32();
1451 case wasm::kExprI32UConvertF32:
1452 case wasm::kExprI32UConvertF64:
1453 case wasm::kExprI32UConvertSatF32:
1454 case wasm::kExprI32UConvertSatF64:
1455 return MachineType::Uint32();
1456 case wasm::kExprI64SConvertF32:
1457 case wasm::kExprI64SConvertF64:
1458 case wasm::kExprI64SConvertSatF32:
1459 case wasm::kExprI64SConvertSatF64:
1460 return MachineType::Int64();
1461 case wasm::kExprI64UConvertF32:
1462 case wasm::kExprI64UConvertF64:
1463 case wasm::kExprI64UConvertSatF32:
1464 case wasm::kExprI64UConvertSatF64:
1465 return MachineType::Uint64();
1466 default:
1467 UNREACHABLE();
1468 }
1469 }
1470
FloatConvertType(wasm::WasmOpcode opcode)1471 MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1472 switch (opcode) {
1473 case wasm::kExprI32SConvertF32:
1474 case wasm::kExprI32UConvertF32:
1475 case wasm::kExprI32SConvertSatF32:
1476 case wasm::kExprI64SConvertF32:
1477 case wasm::kExprI64UConvertF32:
1478 case wasm::kExprI32UConvertSatF32:
1479 case wasm::kExprI64SConvertSatF32:
1480 case wasm::kExprI64UConvertSatF32:
1481 return MachineType::Float32();
1482 case wasm::kExprI32SConvertF64:
1483 case wasm::kExprI32UConvertF64:
1484 case wasm::kExprI64SConvertF64:
1485 case wasm::kExprI64UConvertF64:
1486 case wasm::kExprI32SConvertSatF64:
1487 case wasm::kExprI32UConvertSatF64:
1488 case wasm::kExprI64SConvertSatF64:
1489 case wasm::kExprI64UConvertSatF64:
1490 return MachineType::Float64();
1491 default:
1492 UNREACHABLE();
1493 }
1494 }
1495
ConvertOp(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1496 const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1497 switch (opcode) {
1498 case wasm::kExprI32SConvertF32:
1499 case wasm::kExprI32SConvertSatF32:
1500 return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1501 case wasm::kExprI32UConvertF32:
1502 case wasm::kExprI32UConvertSatF32:
1503 return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1504 case wasm::kExprI32SConvertF64:
1505 case wasm::kExprI32SConvertSatF64:
1506 return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1507 case wasm::kExprI32UConvertF64:
1508 case wasm::kExprI32UConvertSatF64:
1509 return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1510 case wasm::kExprI64SConvertF32:
1511 case wasm::kExprI64SConvertSatF32:
1512 return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1513 case wasm::kExprI64UConvertF32:
1514 case wasm::kExprI64UConvertSatF32:
1515 return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1516 case wasm::kExprI64SConvertF64:
1517 case wasm::kExprI64SConvertSatF64:
1518 return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1519 case wasm::kExprI64UConvertF64:
1520 case wasm::kExprI64UConvertSatF64:
1521 return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1522 default:
1523 UNREACHABLE();
1524 }
1525 }
1526
ConvertBackOp(wasm::WasmOpcode opcode)1527 wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1528 switch (opcode) {
1529 case wasm::kExprI32SConvertF32:
1530 case wasm::kExprI32SConvertSatF32:
1531 return wasm::kExprF32SConvertI32;
1532 case wasm::kExprI32UConvertF32:
1533 case wasm::kExprI32UConvertSatF32:
1534 return wasm::kExprF32UConvertI32;
1535 case wasm::kExprI32SConvertF64:
1536 case wasm::kExprI32SConvertSatF64:
1537 return wasm::kExprF64SConvertI32;
1538 case wasm::kExprI32UConvertF64:
1539 case wasm::kExprI32UConvertSatF64:
1540 return wasm::kExprF64UConvertI32;
1541 default:
1542 UNREACHABLE();
1543 }
1544 }
1545
IsTrappingConvertOp(wasm::WasmOpcode opcode)1546 bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1547 switch (opcode) {
1548 case wasm::kExprI32SConvertF32:
1549 case wasm::kExprI32UConvertF32:
1550 case wasm::kExprI32SConvertF64:
1551 case wasm::kExprI32UConvertF64:
1552 case wasm::kExprI64SConvertF32:
1553 case wasm::kExprI64UConvertF32:
1554 case wasm::kExprI64SConvertF64:
1555 case wasm::kExprI64UConvertF64:
1556 return true;
1557 case wasm::kExprI32SConvertSatF64:
1558 case wasm::kExprI32UConvertSatF64:
1559 case wasm::kExprI32SConvertSatF32:
1560 case wasm::kExprI32UConvertSatF32:
1561 case wasm::kExprI64SConvertSatF32:
1562 case wasm::kExprI64UConvertSatF32:
1563 case wasm::kExprI64SConvertSatF64:
1564 case wasm::kExprI64UConvertSatF64:
1565 return false;
1566 default:
1567 UNREACHABLE();
1568 }
1569 }
1570
Zero(WasmGraphBuilder * builder,const MachineType & ty)1571 Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1572 switch (ty.representation()) {
1573 case MachineRepresentation::kWord32:
1574 return builder->Int32Constant(0);
1575 case MachineRepresentation::kWord64:
1576 return builder->Int64Constant(0);
1577 case MachineRepresentation::kFloat32:
1578 return builder->Float32Constant(0.0);
1579 case MachineRepresentation::kFloat64:
1580 return builder->Float64Constant(0.0);
1581 default:
1582 UNREACHABLE();
1583 }
1584 }
1585
Min(WasmGraphBuilder * builder,const MachineType & ty)1586 Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1587 switch (ty.semantic()) {
1588 case MachineSemantic::kInt32:
1589 return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1590 case MachineSemantic::kUint32:
1591 return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1592 case MachineSemantic::kInt64:
1593 return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1594 case MachineSemantic::kUint64:
1595 return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1596 default:
1597 UNREACHABLE();
1598 }
1599 }
1600
Max(WasmGraphBuilder * builder,const MachineType & ty)1601 Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1602 switch (ty.semantic()) {
1603 case MachineSemantic::kInt32:
1604 return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1605 case MachineSemantic::kUint32:
1606 return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1607 case MachineSemantic::kInt64:
1608 return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1609 case MachineSemantic::kUint64:
1610 return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1611 default:
1612 UNREACHABLE();
1613 }
1614 }
1615
TruncOp(const MachineType & ty)1616 wasm::WasmOpcode TruncOp(const MachineType& ty) {
1617 switch (ty.representation()) {
1618 case MachineRepresentation::kFloat32:
1619 return wasm::kExprF32Trunc;
1620 case MachineRepresentation::kFloat64:
1621 return wasm::kExprF64Trunc;
1622 default:
1623 UNREACHABLE();
1624 }
1625 }
1626
NeOp(const MachineType & ty)1627 wasm::WasmOpcode NeOp(const MachineType& ty) {
1628 switch (ty.representation()) {
1629 case MachineRepresentation::kFloat32:
1630 return wasm::kExprF32Ne;
1631 case MachineRepresentation::kFloat64:
1632 return wasm::kExprF64Ne;
1633 default:
1634 UNREACHABLE();
1635 }
1636 }
1637
LtOp(const MachineType & ty)1638 wasm::WasmOpcode LtOp(const MachineType& ty) {
1639 switch (ty.representation()) {
1640 case MachineRepresentation::kFloat32:
1641 return wasm::kExprF32Lt;
1642 case MachineRepresentation::kFloat64:
1643 return wasm::kExprF64Lt;
1644 default:
1645 UNREACHABLE();
1646 }
1647 }
1648
ConvertTrapTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1649 Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1650 const MachineType& int_ty, const MachineType& float_ty,
1651 Node* trunc, Node* converted_value) {
1652 if (int_ty.representation() == MachineRepresentation::kWord32) {
1653 Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1654 return builder->Binop(NeOp(float_ty), trunc, check);
1655 }
1656 return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1657 trunc, builder->graph()->start());
1658 }
1659
ConvertSaturateTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1660 Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1661 const MachineType& int_ty,
1662 const MachineType& float_ty, Node* trunc,
1663 Node* converted_value) {
1664 Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1665 converted_value);
1666 if (int_ty.representation() == MachineRepresentation::kWord64) {
1667 test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1668 }
1669 return test;
1670 }
1671
1672 } // namespace
1673
BuildIntConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1674 Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1675 wasm::WasmCodePosition position,
1676 wasm::WasmOpcode opcode) {
1677 const MachineType int_ty = IntConvertType(opcode);
1678 const MachineType float_ty = FloatConvertType(opcode);
1679 const Operator* conv_op = ConvertOp(this, opcode);
1680 Node* trunc = nullptr;
1681 Node* converted_value = nullptr;
1682 const bool is_int32 =
1683 int_ty.representation() == MachineRepresentation::kWord32;
1684 if (is_int32) {
1685 trunc = Unop(TruncOp(float_ty), input);
1686 converted_value = graph()->NewNode(conv_op, trunc);
1687 } else {
1688 trunc = graph()->NewNode(conv_op, input);
1689 converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1690 trunc, graph()->start());
1691 }
1692 if (IsTrappingConvertOp(opcode)) {
1693 Node* test =
1694 ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1695 if (is_int32) {
1696 TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1697 } else {
1698 ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1699 }
1700 return converted_value;
1701 }
1702 Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1703 converted_value);
1704 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1705 tl_d.Chain(Control());
1706 Node* nan_test = Binop(NeOp(float_ty), input, input);
1707 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1708 nan_d.Nest(tl_d, true);
1709 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1710 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1711 sat_d.Nest(nan_d, false);
1712 Node* sat_val =
1713 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1714 Node* nan_val =
1715 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1716 return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1717 }
1718
BuildI32AsmjsSConvertF32(Node * input)1719 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1720 MachineOperatorBuilder* m = mcgraph()->machine();
1721 // asm.js must use the wacky JS semantics.
1722 input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1723 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1724 }
1725
BuildI32AsmjsSConvertF64(Node * input)1726 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1727 MachineOperatorBuilder* m = mcgraph()->machine();
1728 // asm.js must use the wacky JS semantics.
1729 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1730 }
1731
BuildI32AsmjsUConvertF32(Node * input)1732 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1733 MachineOperatorBuilder* m = mcgraph()->machine();
1734 // asm.js must use the wacky JS semantics.
1735 input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1736 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1737 }
1738
BuildI32AsmjsUConvertF64(Node * input)1739 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1740 MachineOperatorBuilder* m = mcgraph()->machine();
1741 // asm.js must use the wacky JS semantics.
1742 return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1743 }
1744
BuildBitCountingCall(Node * input,ExternalReference ref,MachineRepresentation input_type)1745 Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1746 MachineRepresentation input_type) {
1747 Node* stack_slot_param =
1748 graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1749
1750 const Operator* store_op = mcgraph()->machine()->Store(
1751 StoreRepresentation(input_type, kNoWriteBarrier));
1752 *effect_ =
1753 graph()->NewNode(store_op, stack_slot_param, mcgraph()->Int32Constant(0),
1754 input, *effect_, *control_);
1755
1756 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1757 MachineSignature sig(1, 1, sig_types);
1758
1759 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1760
1761 return BuildCCall(&sig, function, stack_slot_param);
1762 }
1763
BuildI32Ctz(Node * input)1764 Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1765 return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1766 MachineRepresentation::kWord32);
1767 }
1768
BuildI64Ctz(Node * input)1769 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1770 return Unop(wasm::kExprI64UConvertI32,
1771 BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1772 MachineRepresentation::kWord64));
1773 }
1774
BuildI32Popcnt(Node * input)1775 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1776 return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1777 MachineRepresentation::kWord32);
1778 }
1779
BuildI64Popcnt(Node * input)1780 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1781 return Unop(
1782 wasm::kExprI64UConvertI32,
1783 BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1784 MachineRepresentation::kWord64));
1785 }
1786
BuildF32Trunc(Node * input)1787 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1788 MachineType type = MachineType::Float32();
1789 ExternalReference ref = ExternalReference::wasm_f32_trunc();
1790
1791 return BuildCFuncInstruction(ref, type, input);
1792 }
1793
BuildF32Floor(Node * input)1794 Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1795 MachineType type = MachineType::Float32();
1796 ExternalReference ref = ExternalReference::wasm_f32_floor();
1797 return BuildCFuncInstruction(ref, type, input);
1798 }
1799
BuildF32Ceil(Node * input)1800 Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1801 MachineType type = MachineType::Float32();
1802 ExternalReference ref = ExternalReference::wasm_f32_ceil();
1803 return BuildCFuncInstruction(ref, type, input);
1804 }
1805
BuildF32NearestInt(Node * input)1806 Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1807 MachineType type = MachineType::Float32();
1808 ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1809 return BuildCFuncInstruction(ref, type, input);
1810 }
1811
BuildF64Trunc(Node * input)1812 Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1813 MachineType type = MachineType::Float64();
1814 ExternalReference ref = ExternalReference::wasm_f64_trunc();
1815 return BuildCFuncInstruction(ref, type, input);
1816 }
1817
BuildF64Floor(Node * input)1818 Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1819 MachineType type = MachineType::Float64();
1820 ExternalReference ref = ExternalReference::wasm_f64_floor();
1821 return BuildCFuncInstruction(ref, type, input);
1822 }
1823
BuildF64Ceil(Node * input)1824 Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1825 MachineType type = MachineType::Float64();
1826 ExternalReference ref = ExternalReference::wasm_f64_ceil();
1827 return BuildCFuncInstruction(ref, type, input);
1828 }
1829
BuildF64NearestInt(Node * input)1830 Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1831 MachineType type = MachineType::Float64();
1832 ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1833 return BuildCFuncInstruction(ref, type, input);
1834 }
1835
BuildF64Acos(Node * input)1836 Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1837 MachineType type = MachineType::Float64();
1838 ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1839 return BuildCFuncInstruction(ref, type, input);
1840 }
1841
BuildF64Asin(Node * input)1842 Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1843 MachineType type = MachineType::Float64();
1844 ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1845 return BuildCFuncInstruction(ref, type, input);
1846 }
1847
BuildF64Pow(Node * left,Node * right)1848 Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1849 MachineType type = MachineType::Float64();
1850 ExternalReference ref = ExternalReference::wasm_float64_pow();
1851 return BuildCFuncInstruction(ref, type, left, right);
1852 }
1853
BuildF64Mod(Node * left,Node * right)1854 Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1855 MachineType type = MachineType::Float64();
1856 ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1857 return BuildCFuncInstruction(ref, type, left, right);
1858 }
1859
BuildCFuncInstruction(ExternalReference ref,MachineType type,Node * input0,Node * input1)1860 Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1861 MachineType type, Node* input0,
1862 Node* input1) {
1863 // We do truncation by calling a C function which calculates the result.
1864 // The input is passed to the C function as a byte buffer holding the two
1865 // input doubles. We reserve this byte buffer as a stack slot, store the
1866 // parameters in this buffer slots, pass a pointer to the buffer to the C
1867 // function, and after calling the C function we collect the return value from
1868 // the buffer.
1869
1870 const int type_size = ElementSizeInBytes(type.representation());
1871 const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1872 Node* stack_slot =
1873 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1874
1875 const Operator* store_op = mcgraph()->machine()->Store(
1876 StoreRepresentation(type.representation(), kNoWriteBarrier));
1877 *effect_ = graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1878 input0, *effect_, *control_);
1879
1880 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1881
1882 if (input1 != nullptr) {
1883 *effect_ = graph()->NewNode(store_op, stack_slot,
1884 mcgraph()->Int32Constant(type_size), input1,
1885 *effect_, *control_);
1886 }
1887
1888 MachineType sig_types[] = {MachineType::Pointer()};
1889 MachineSignature sig(0, 1, sig_types);
1890 BuildCCall(&sig, function, stack_slot);
1891
1892 const Operator* load_op = mcgraph()->machine()->Load(type);
1893 Node* load = graph()->NewNode(
1894 load_op, stack_slot, mcgraph()->Int32Constant(0), *effect_, *control_);
1895 *effect_ = load;
1896 return load;
1897 }
1898
BuildF32SConvertI64(Node * input)1899 Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1900 // TODO(titzer/bradnelson): Check handlng of asm.js case.
1901 return BuildIntToFloatConversionInstruction(
1902 input, ExternalReference::wasm_int64_to_float32(),
1903 MachineRepresentation::kWord64, MachineType::Float32());
1904 }
BuildF32UConvertI64(Node * input)1905 Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1906 // TODO(titzer/bradnelson): Check handlng of asm.js case.
1907 return BuildIntToFloatConversionInstruction(
1908 input, ExternalReference::wasm_uint64_to_float32(),
1909 MachineRepresentation::kWord64, MachineType::Float32());
1910 }
BuildF64SConvertI64(Node * input)1911 Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1912 return BuildIntToFloatConversionInstruction(
1913 input, ExternalReference::wasm_int64_to_float64(),
1914 MachineRepresentation::kWord64, MachineType::Float64());
1915 }
BuildF64UConvertI64(Node * input)1916 Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1917 return BuildIntToFloatConversionInstruction(
1918 input, ExternalReference::wasm_uint64_to_float64(),
1919 MachineRepresentation::kWord64, MachineType::Float64());
1920 }
1921
BuildIntToFloatConversionInstruction(Node * input,ExternalReference ref,MachineRepresentation parameter_representation,const MachineType result_type)1922 Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1923 Node* input, ExternalReference ref,
1924 MachineRepresentation parameter_representation,
1925 const MachineType result_type) {
1926 int stack_slot_size =
1927 std::max(ElementSizeInBytes(parameter_representation),
1928 ElementSizeInBytes(result_type.representation()));
1929 Node* stack_slot =
1930 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1931 const Operator* store_op = mcgraph()->machine()->Store(
1932 StoreRepresentation(parameter_representation, kNoWriteBarrier));
1933 *effect_ = graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1934 input, *effect_, *control_);
1935 MachineType sig_types[] = {MachineType::Pointer()};
1936 MachineSignature sig(0, 1, sig_types);
1937 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1938 BuildCCall(&sig, function, stack_slot);
1939 const Operator* load_op = mcgraph()->machine()->Load(result_type);
1940 Node* load = graph()->NewNode(
1941 load_op, stack_slot, mcgraph()->Int32Constant(0), *effect_, *control_);
1942 *effect_ = load;
1943 return load;
1944 }
1945
1946 namespace {
1947
convert_ccall_ref(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1948 ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1949 wasm::WasmOpcode opcode) {
1950 switch (opcode) {
1951 case wasm::kExprI64SConvertF32:
1952 case wasm::kExprI64SConvertSatF32:
1953 return ExternalReference::wasm_float32_to_int64();
1954 case wasm::kExprI64UConvertF32:
1955 case wasm::kExprI64UConvertSatF32:
1956 return ExternalReference::wasm_float32_to_uint64();
1957 case wasm::kExprI64SConvertF64:
1958 case wasm::kExprI64SConvertSatF64:
1959 return ExternalReference::wasm_float64_to_int64();
1960 case wasm::kExprI64UConvertF64:
1961 case wasm::kExprI64UConvertSatF64:
1962 return ExternalReference::wasm_float64_to_uint64();
1963 default:
1964 UNREACHABLE();
1965 }
1966 }
1967
1968 } // namespace
1969
BuildCcallConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1970 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
1971 wasm::WasmCodePosition position,
1972 wasm::WasmOpcode opcode) {
1973 const MachineType int_ty = IntConvertType(opcode);
1974 const MachineType float_ty = FloatConvertType(opcode);
1975 ExternalReference call_ref = convert_ccall_ref(this, opcode);
1976 int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
1977 ElementSizeInBytes(float_ty.representation()));
1978 Node* stack_slot =
1979 graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1980 const Operator* store_op = mcgraph()->machine()->Store(
1981 StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
1982 *effect_ = graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
1983 *effect_, *control_);
1984 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1985 MachineSignature sig(1, 1, sig_types);
1986 Node* function =
1987 graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
1988 Node* overflow = BuildCCall(&sig, function, stack_slot);
1989 if (IsTrappingConvertOp(opcode)) {
1990 ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
1991 const Operator* load_op = mcgraph()->machine()->Load(int_ty);
1992 Node* load = graph()->NewNode(load_op, stack_slot, Int32Constant(0),
1993 *effect_, *control_);
1994 *effect_ = load;
1995 return load;
1996 }
1997 Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
1998 Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1999 tl_d.Chain(Control());
2000 Node* nan_test = Binop(NeOp(float_ty), input, input);
2001 Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2002 nan_d.Nest(tl_d, true);
2003 Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2004 Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2005 sat_d.Nest(nan_d, false);
2006 Node* sat_val =
2007 sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2008 const Operator* load_op = mcgraph()->machine()->Load(int_ty);
2009 Node* load = graph()->NewNode(load_op, stack_slot, Int32Constant(0), *effect_,
2010 *control_);
2011 Node* nan_val =
2012 nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2013 return tl_d.Phi(int_ty.representation(), nan_val, load);
2014 }
2015
GrowMemory(Node * input)2016 Node* WasmGraphBuilder::GrowMemory(Node* input) {
2017 SetNeedsStackCheck();
2018 Diamond check_input_range(
2019 graph(), mcgraph()->common(),
2020 graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(), input,
2021 mcgraph()->Uint32Constant(FLAG_wasm_max_mem_pages)),
2022 BranchHint::kTrue);
2023
2024 check_input_range.Chain(*control_);
2025
2026 Node* parameters[] = {BuildChangeUint31ToSmi(input)};
2027 Node* old_effect = *effect_;
2028 *control_ = check_input_range.if_true;
2029 Node* call = BuildCallToRuntime(Runtime::kWasmGrowMemory, parameters,
2030 arraysize(parameters));
2031
2032 Node* result = BuildChangeSmiToInt32(call);
2033
2034 result = check_input_range.Phi(MachineRepresentation::kWord32, result,
2035 mcgraph()->Int32Constant(-1));
2036 *effect_ = graph()->NewNode(mcgraph()->common()->EffectPhi(2), *effect_,
2037 old_effect, check_input_range.merge);
2038 *control_ = check_input_range.merge;
2039 return result;
2040 }
2041
GetExceptionEncodedSize(const wasm::WasmException * exception) const2042 uint32_t WasmGraphBuilder::GetExceptionEncodedSize(
2043 const wasm::WasmException* exception) const {
2044 const wasm::WasmExceptionSig* sig = exception->sig;
2045 uint32_t encoded_size = 0;
2046 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2047 size_t byte_size = static_cast<size_t>(
2048 wasm::ValueTypes::ElementSizeInBytes(sig->GetParam(i)));
2049 DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0);
2050 DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement);
2051 encoded_size += byte_size / kBytesPerExceptionValuesArrayElement;
2052 }
2053 return encoded_size;
2054 }
2055
Throw(uint32_t tag,const wasm::WasmException * exception,const Vector<Node * > values)2056 Node* WasmGraphBuilder::Throw(uint32_t tag,
2057 const wasm::WasmException* exception,
2058 const Vector<Node*> values) {
2059 SetNeedsStackCheck();
2060 uint32_t encoded_size = GetExceptionEncodedSize(exception);
2061 Node* create_parameters[] = {
2062 BuildChangeUint31ToSmi(ConvertExceptionTagToRuntimeId(tag)),
2063 BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2064 BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2065 arraysize(create_parameters));
2066 uint32_t index = 0;
2067 const wasm::WasmExceptionSig* sig = exception->sig;
2068 MachineOperatorBuilder* m = mcgraph()->machine();
2069 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2070 Node* value = values[i];
2071 switch (sig->GetParam(i)) {
2072 case wasm::kWasmF32:
2073 value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2074 V8_FALLTHROUGH;
2075 case wasm::kWasmI32:
2076 BuildEncodeException32BitValue(&index, value);
2077 break;
2078 case wasm::kWasmF64:
2079 value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2080 V8_FALLTHROUGH;
2081 case wasm::kWasmI64: {
2082 Node* upper32 = graph()->NewNode(
2083 m->TruncateInt64ToInt32(),
2084 Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2085 BuildEncodeException32BitValue(&index, upper32);
2086 Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2087 BuildEncodeException32BitValue(&index, lower32);
2088 break;
2089 }
2090 default:
2091 UNREACHABLE();
2092 }
2093 }
2094 DCHECK_EQ(encoded_size, index);
2095 return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2096 }
2097
BuildEncodeException32BitValue(uint32_t * index,Node * value)2098 void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index,
2099 Node* value) {
2100 MachineOperatorBuilder* machine = mcgraph()->machine();
2101 Node* upper_parameters[] = {
2102 BuildChangeUint31ToSmi(Int32Constant(*index)),
2103 BuildChangeUint31ToSmi(
2104 graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))),
2105 };
2106 BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters,
2107 arraysize(upper_parameters));
2108 ++(*index);
2109 Node* lower_parameters[] = {
2110 BuildChangeUint31ToSmi(Int32Constant(*index)),
2111 BuildChangeUint31ToSmi(graph()->NewNode(machine->Word32And(), value,
2112 Int32Constant(0xFFFFu))),
2113 };
2114 BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters,
2115 arraysize(lower_parameters));
2116 ++(*index);
2117 }
2118
BuildDecodeException32BitValue(Node * const * values,uint32_t * index)2119 Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values,
2120 uint32_t* index) {
2121 MachineOperatorBuilder* machine = mcgraph()->machine();
2122 Node* upper = BuildChangeSmiToInt32(values[*index]);
2123 (*index)++;
2124 upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2125 Node* lower = BuildChangeSmiToInt32(values[*index]);
2126 (*index)++;
2127 Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2128 return value;
2129 }
2130
Rethrow()2131 Node* WasmGraphBuilder::Rethrow() {
2132 SetNeedsStackCheck();
2133 Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2134 return result;
2135 }
2136
ConvertExceptionTagToRuntimeId(uint32_t tag)2137 Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) {
2138 // TODO(kschimpf): Handle exceptions from different modules, when they are
2139 // linked at runtime.
2140 return Uint32Constant(tag);
2141 }
2142
GetExceptionRuntimeId()2143 Node* WasmGraphBuilder::GetExceptionRuntimeId() {
2144 SetNeedsStackCheck();
2145 return BuildChangeSmiToInt32(
2146 BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0));
2147 }
2148
GetExceptionValues(const wasm::WasmException * except_decl)2149 Node** WasmGraphBuilder::GetExceptionValues(
2150 const wasm::WasmException* except_decl) {
2151 // TODO(kschimpf): We need to move this code to the function-body-decoder.cc
2152 // in order to build landing-pad (exception) edges in case the runtime
2153 // call causes an exception.
2154
2155 // Start by getting the encoded values from the exception.
2156 uint32_t encoded_size = GetExceptionEncodedSize(except_decl);
2157 Node** values = Buffer(encoded_size);
2158 for (uint32_t i = 0; i < encoded_size; ++i) {
2159 Node* parameters[] = {BuildChangeUint31ToSmi(Uint32Constant(i))};
2160 values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement,
2161 parameters, arraysize(parameters));
2162 }
2163
2164 // Now convert the leading entries to the corresponding parameter values.
2165 uint32_t index = 0;
2166 const wasm::WasmExceptionSig* sig = except_decl->sig;
2167 for (size_t i = 0; i < sig->parameter_count(); ++i) {
2168 Node* value = BuildDecodeException32BitValue(values, &index);
2169 switch (wasm::ValueType type = sig->GetParam(i)) {
2170 case wasm::kWasmF32: {
2171 value = Unop(wasm::kExprF32ReinterpretI32, value);
2172 break;
2173 }
2174 case wasm::kWasmI32:
2175 break;
2176 case wasm::kWasmF64:
2177 case wasm::kWasmI64: {
2178 Node* upper =
2179 Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value),
2180 Int64Constant(32));
2181 Node* lower = Unop(wasm::kExprI64UConvertI32,
2182 BuildDecodeException32BitValue(values, &index));
2183 value = Binop(wasm::kExprI64Ior, upper, lower);
2184 if (type == wasm::kWasmF64) {
2185 value = Unop(wasm::kExprF64ReinterpretI64, value);
2186 }
2187 break;
2188 }
2189 default:
2190 UNREACHABLE();
2191 }
2192 values[i] = value;
2193 }
2194 DCHECK_EQ(index, encoded_size);
2195 return values;
2196 }
2197
BuildI32DivS(Node * left,Node * right,wasm::WasmCodePosition position)2198 Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2199 wasm::WasmCodePosition position) {
2200 MachineOperatorBuilder* m = mcgraph()->machine();
2201 ZeroCheck32(wasm::kTrapDivByZero, right, position);
2202 Node* before = *control_;
2203 Node* denom_is_m1;
2204 Node* denom_is_not_m1;
2205 BranchExpectFalse(
2206 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2207 &denom_is_m1, &denom_is_not_m1);
2208 *control_ = denom_is_m1;
2209 TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2210 if (*control_ != denom_is_m1) {
2211 *control_ = graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2212 *control_);
2213 } else {
2214 *control_ = before;
2215 }
2216 return graph()->NewNode(m->Int32Div(), left, right, *control_);
2217 }
2218
BuildI32RemS(Node * left,Node * right,wasm::WasmCodePosition position)2219 Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2220 wasm::WasmCodePosition position) {
2221 MachineOperatorBuilder* m = mcgraph()->machine();
2222
2223 ZeroCheck32(wasm::kTrapRemByZero, right, position);
2224
2225 Diamond d(
2226 graph(), mcgraph()->common(),
2227 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2228 BranchHint::kFalse);
2229 d.Chain(*control_);
2230
2231 return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2232 graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2233 }
2234
BuildI32DivU(Node * left,Node * right,wasm::WasmCodePosition position)2235 Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2236 wasm::WasmCodePosition position) {
2237 MachineOperatorBuilder* m = mcgraph()->machine();
2238 return graph()->NewNode(m->Uint32Div(), left, right,
2239 ZeroCheck32(wasm::kTrapDivByZero, right, position));
2240 }
2241
BuildI32RemU(Node * left,Node * right,wasm::WasmCodePosition position)2242 Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2243 wasm::WasmCodePosition position) {
2244 MachineOperatorBuilder* m = mcgraph()->machine();
2245 return graph()->NewNode(m->Uint32Mod(), left, right,
2246 ZeroCheck32(wasm::kTrapRemByZero, right, position));
2247 }
2248
BuildI32AsmjsDivS(Node * left,Node * right)2249 Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2250 MachineOperatorBuilder* m = mcgraph()->machine();
2251
2252 Int32Matcher mr(right);
2253 if (mr.HasValue()) {
2254 if (mr.Value() == 0) {
2255 return mcgraph()->Int32Constant(0);
2256 } else if (mr.Value() == -1) {
2257 // The result is the negation of the left input.
2258 return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2259 }
2260 return graph()->NewNode(m->Int32Div(), left, right, *control_);
2261 }
2262
2263 // asm.js semantics return 0 on divide or mod by zero.
2264 if (m->Int32DivIsSafe()) {
2265 // The hardware instruction does the right thing (e.g. arm).
2266 return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2267 }
2268
2269 // Check denominator for zero.
2270 Diamond z(
2271 graph(), mcgraph()->common(),
2272 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2273 BranchHint::kFalse);
2274
2275 // Check numerator for -1. (avoid minint / -1 case).
2276 Diamond n(
2277 graph(), mcgraph()->common(),
2278 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2279 BranchHint::kFalse);
2280
2281 Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2282 Node* neg =
2283 graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2284
2285 return n.Phi(
2286 MachineRepresentation::kWord32, neg,
2287 z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2288 }
2289
BuildI32AsmjsRemS(Node * left,Node * right)2290 Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2291 CommonOperatorBuilder* c = mcgraph()->common();
2292 MachineOperatorBuilder* m = mcgraph()->machine();
2293 Node* const zero = mcgraph()->Int32Constant(0);
2294
2295 Int32Matcher mr(right);
2296 if (mr.HasValue()) {
2297 if (mr.Value() == 0 || mr.Value() == -1) {
2298 return zero;
2299 }
2300 return graph()->NewNode(m->Int32Mod(), left, right, *control_);
2301 }
2302
2303 // General case for signed integer modulus, with optimization for (unknown)
2304 // power of 2 right hand side.
2305 //
2306 // if 0 < right then
2307 // msk = right - 1
2308 // if right & msk != 0 then
2309 // left % right
2310 // else
2311 // if left < 0 then
2312 // -(-left & msk)
2313 // else
2314 // left & msk
2315 // else
2316 // if right < -1 then
2317 // left % right
2318 // else
2319 // zero
2320 //
2321 // Note: We do not use the Diamond helper class here, because it really hurts
2322 // readability with nested diamonds.
2323 Node* const minus_one = mcgraph()->Int32Constant(-1);
2324
2325 const Operator* const merge_op = c->Merge(2);
2326 const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2327
2328 Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2329 Node* branch0 =
2330 graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2331
2332 Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2333 Node* true0;
2334 {
2335 Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2336
2337 Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2338 Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2339
2340 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2341 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2342
2343 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2344 Node* false1;
2345 {
2346 Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2347 Node* branch2 =
2348 graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2349
2350 Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2351 Node* true2 = graph()->NewNode(
2352 m->Int32Sub(), zero,
2353 graph()->NewNode(m->Word32And(),
2354 graph()->NewNode(m->Int32Sub(), zero, left), msk));
2355
2356 Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2357 Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2358
2359 if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2360 false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2361 }
2362
2363 if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2364 true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2365 }
2366
2367 Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2368 Node* false0;
2369 {
2370 Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2371 Node* branch1 =
2372 graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2373
2374 Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2375 Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2376
2377 Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2378 Node* false1 = zero;
2379
2380 if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2381 false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2382 }
2383
2384 Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2385 return graph()->NewNode(phi_op, true0, false0, merge0);
2386 }
2387
BuildI32AsmjsDivU(Node * left,Node * right)2388 Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2389 MachineOperatorBuilder* m = mcgraph()->machine();
2390 // asm.js semantics return 0 on divide or mod by zero.
2391 if (m->Uint32DivIsSafe()) {
2392 // The hardware instruction does the right thing (e.g. arm).
2393 return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2394 }
2395
2396 // Explicit check for x % 0.
2397 Diamond z(
2398 graph(), mcgraph()->common(),
2399 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2400 BranchHint::kFalse);
2401
2402 return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2403 graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2404 z.if_false));
2405 }
2406
BuildI32AsmjsRemU(Node * left,Node * right)2407 Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2408 MachineOperatorBuilder* m = mcgraph()->machine();
2409 // asm.js semantics return 0 on divide or mod by zero.
2410 // Explicit check for x % 0.
2411 Diamond z(
2412 graph(), mcgraph()->common(),
2413 graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2414 BranchHint::kFalse);
2415
2416 Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2417 z.if_false);
2418 return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2419 rem);
2420 }
2421
BuildI64DivS(Node * left,Node * right,wasm::WasmCodePosition position)2422 Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2423 wasm::WasmCodePosition position) {
2424 if (mcgraph()->machine()->Is32()) {
2425 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2426 MachineType::Int64(), wasm::kTrapDivByZero, position);
2427 }
2428 ZeroCheck64(wasm::kTrapDivByZero, right, position);
2429 Node* before = *control_;
2430 Node* denom_is_m1;
2431 Node* denom_is_not_m1;
2432 BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2433 mcgraph()->Int64Constant(-1)),
2434 &denom_is_m1, &denom_is_not_m1);
2435 *control_ = denom_is_m1;
2436 TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2437 std::numeric_limits<int64_t>::min(), position);
2438 if (*control_ != denom_is_m1) {
2439 *control_ = graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2440 *control_);
2441 } else {
2442 *control_ = before;
2443 }
2444 return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2445 *control_);
2446 }
2447
BuildI64RemS(Node * left,Node * right,wasm::WasmCodePosition position)2448 Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2449 wasm::WasmCodePosition position) {
2450 if (mcgraph()->machine()->Is32()) {
2451 return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2452 MachineType::Int64(), wasm::kTrapRemByZero, position);
2453 }
2454 ZeroCheck64(wasm::kTrapRemByZero, right, position);
2455 Diamond d(mcgraph()->graph(), mcgraph()->common(),
2456 graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2457 mcgraph()->Int64Constant(-1)));
2458
2459 d.Chain(*control_);
2460
2461 Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2462 d.if_false);
2463
2464 return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2465 rem);
2466 }
2467
BuildI64DivU(Node * left,Node * right,wasm::WasmCodePosition position)2468 Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2469 wasm::WasmCodePosition position) {
2470 if (mcgraph()->machine()->Is32()) {
2471 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2472 MachineType::Int64(), wasm::kTrapDivByZero, position);
2473 }
2474 return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2475 ZeroCheck64(wasm::kTrapDivByZero, right, position));
2476 }
BuildI64RemU(Node * left,Node * right,wasm::WasmCodePosition position)2477 Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2478 wasm::WasmCodePosition position) {
2479 if (mcgraph()->machine()->Is32()) {
2480 return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2481 MachineType::Int64(), wasm::kTrapRemByZero, position);
2482 }
2483 return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2484 ZeroCheck64(wasm::kTrapRemByZero, right, position));
2485 }
2486
BuildDiv64Call(Node * left,Node * right,ExternalReference ref,MachineType result_type,wasm::TrapReason trap_zero,wasm::WasmCodePosition position)2487 Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2488 ExternalReference ref,
2489 MachineType result_type,
2490 wasm::TrapReason trap_zero,
2491 wasm::WasmCodePosition position) {
2492 Node* stack_slot =
2493 graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2494
2495 const Operator* store_op = mcgraph()->machine()->Store(
2496 StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2497 *effect_ = graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2498 left, *effect_, *control_);
2499 *effect_ = graph()->NewNode(store_op, stack_slot,
2500 mcgraph()->Int32Constant(sizeof(double)), right,
2501 *effect_, *control_);
2502
2503 MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2504 MachineSignature sig(1, 1, sig_types);
2505
2506 Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2507 Node* call = BuildCCall(&sig, function, stack_slot);
2508
2509 ZeroCheck32(trap_zero, call, position);
2510 TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2511 const Operator* load_op = mcgraph()->machine()->Load(result_type);
2512 Node* load = graph()->NewNode(
2513 load_op, stack_slot, mcgraph()->Int32Constant(0), *effect_, *control_);
2514 *effect_ = load;
2515 return load;
2516 }
2517
2518 template <typename... Args>
BuildCCall(MachineSignature * sig,Node * function,Args...args)2519 Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2520 Args... args) {
2521 DCHECK_LE(sig->return_count(), 1);
2522 DCHECK_EQ(sizeof...(args), sig->parameter_count());
2523 Node* const call_args[] = {function, args..., *effect_, *control_};
2524
2525 auto call_descriptor =
2526 Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2527
2528 const Operator* op = mcgraph()->common()->Call(call_descriptor);
2529 Node* call = graph()->NewNode(op, arraysize(call_args), call_args);
2530 *effect_ = call;
2531 return call;
2532 }
2533
BuildWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,Node * instance_node,UseRetpoline use_retpoline)2534 Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2535 Node*** rets,
2536 wasm::WasmCodePosition position,
2537 Node* instance_node,
2538 UseRetpoline use_retpoline) {
2539 if (instance_node == nullptr) {
2540 DCHECK_NOT_NULL(instance_node_);
2541 instance_node = instance_node_.get();
2542 }
2543 SetNeedsStackCheck();
2544 const size_t params = sig->parameter_count();
2545 const size_t extra = 3; // instance_node, effect, and control.
2546 const size_t count = 1 + params + extra;
2547
2548 // Reallocate the buffer to make space for extra inputs.
2549 args = Realloc(args, 1 + params, count);
2550
2551 // Make room for the instance_node parameter at index 1, just after code.
2552 memmove(&args[2], &args[1], params * sizeof(Node*));
2553 args[1] = instance_node;
2554
2555 // Add effect and control inputs.
2556 args[params + 2] = *effect_;
2557 args[params + 3] = *control_;
2558
2559 auto call_descriptor =
2560 GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2561 const Operator* op = mcgraph()->common()->Call(call_descriptor);
2562 Node* call = graph()->NewNode(op, static_cast<int>(count), args);
2563 DCHECK(position == wasm::kNoCodePosition || position > 0);
2564 if (position > 0) SetSourcePosition(call, position);
2565
2566 *effect_ = call;
2567 size_t ret_count = sig->return_count();
2568 if (ret_count == 0) return call; // No return value.
2569
2570 *rets = Buffer(ret_count);
2571 if (ret_count == 1) {
2572 // Only a single return value.
2573 (*rets)[0] = call;
2574 } else {
2575 // Create projections for all return values.
2576 for (size_t i = 0; i < ret_count; i++) {
2577 (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2578 graph()->start());
2579 }
2580 }
2581 return call;
2582 }
2583
BuildImportWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,int func_index)2584 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args,
2585 Node*** rets,
2586 wasm::WasmCodePosition position,
2587 int func_index) {
2588 // Load the instance from the imported_instances array at a known offset.
2589 Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances,
2590 MachineType::TaggedPointer());
2591 Node* instance_node = LOAD_FIXED_ARRAY_SLOT(imported_instances, func_index);
2592
2593 // Load the target from the imported_targets array at a known offset.
2594 Node* imported_targets =
2595 LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2596 Node* target_node = graph()->NewNode(
2597 mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2598 mcgraph()->Int32Constant(func_index * sizeof(Address)),
2599 mcgraph()->graph()->start(), mcgraph()->graph()->start());
2600 args[0] = target_node;
2601 return BuildWasmCall(sig, args, rets, position, instance_node,
2602 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2603 }
2604
CallDirect(uint32_t index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2605 Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2606 wasm::WasmCodePosition position) {
2607 DCHECK_NULL(args[0]);
2608 wasm::FunctionSig* sig = env_->module->functions[index].sig;
2609
2610 if (env_ && index < env_->module->num_imported_functions) {
2611 // Call to an imported function.
2612 return BuildImportWasmCall(sig, args, rets, position, index);
2613 }
2614
2615 // A direct call to a wasm function defined in this module.
2616 // Just encode the function index. This will be patched at instantiation.
2617 Address code = static_cast<Address>(index);
2618 args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2619
2620 return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2621 }
2622
CallIndirect(uint32_t sig_index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2623 Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2624 Node*** rets,
2625 wasm::WasmCodePosition position) {
2626 DCHECK_NOT_NULL(args[0]);
2627 DCHECK_NOT_NULL(env_);
2628
2629 // Assume only one table for now.
2630 wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2631
2632 Node* ift_size =
2633 LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2634
2635 MachineOperatorBuilder* machine = mcgraph()->machine();
2636 Node* key = args[0];
2637
2638 // Bounds check against the table size.
2639 Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2640 TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2641
2642 // Mask the key to prevent SSCA.
2643 if (untrusted_code_mitigations_) {
2644 // mask = ((key - size) & ~key) >> 31
2645 Node* neg_key =
2646 graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2647 Node* masked_diff = graph()->NewNode(
2648 machine->Word32And(),
2649 graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2650 Node* mask =
2651 graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2652 key = graph()->NewNode(machine->Word32And(), key, mask);
2653 }
2654
2655 // Load signature from the table and check.
2656 Node* ift_sig_ids =
2657 LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2658
2659 int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2660 Node* scaled_key =
2661 graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2));
2662 const Operator* add = nullptr;
2663 if (machine->Is64()) {
2664 scaled_key = graph()->NewNode(machine->ChangeUint32ToUint64(), scaled_key);
2665 add = machine->Int64Add();
2666 } else {
2667 add = machine->Int32Add();
2668 }
2669
2670 Node* loaded_sig =
2671 graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
2672 scaled_key, *effect_, *control_);
2673 Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2674 Int32Constant(expected_sig_id));
2675
2676 TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2677
2678 Node* ift_targets =
2679 LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2680 Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableInstances,
2681 MachineType::TaggedPointer());
2682
2683 scaled_key = graph()->NewNode(machine->Word32Shl(), key,
2684 Int32Constant(kPointerSizeLog2));
2685
2686 Node* target = graph()->NewNode(machine->Load(MachineType::Pointer()),
2687 ift_targets, scaled_key, *effect_, *control_);
2688
2689 auto access = AccessBuilder::ForFixedArrayElement();
2690 Node* target_instance = graph()->NewNode(
2691 machine->Load(MachineType::TaggedPointer()),
2692 graph()->NewNode(add, ift_instances, scaled_key),
2693 Int32Constant(access.header_size - access.tag()), *effect_, *control_);
2694
2695 args[0] = target;
2696
2697 return BuildWasmCall(sig, args, rets, position, target_instance,
2698 untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2699 }
2700
BuildI32Rol(Node * left,Node * right)2701 Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2702 // Implement Rol by Ror since TurboFan does not have Rol opcode.
2703 // TODO(weiliang): support Word32Rol opcode in TurboFan.
2704 Int32Matcher m(right);
2705 if (m.HasValue()) {
2706 return Binop(wasm::kExprI32Ror, left,
2707 mcgraph()->Int32Constant(32 - m.Value()));
2708 } else {
2709 return Binop(wasm::kExprI32Ror, left,
2710 Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2711 }
2712 }
2713
BuildI64Rol(Node * left,Node * right)2714 Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2715 // Implement Rol by Ror since TurboFan does not have Rol opcode.
2716 // TODO(weiliang): support Word64Rol opcode in TurboFan.
2717 Int64Matcher m(right);
2718 if (m.HasValue()) {
2719 return Binop(wasm::kExprI64Ror, left,
2720 mcgraph()->Int64Constant(64 - m.Value()));
2721 } else {
2722 return Binop(wasm::kExprI64Ror, left,
2723 Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2724 }
2725 }
2726
Invert(Node * node)2727 Node* WasmGraphBuilder::Invert(Node* node) {
2728 return Unop(wasm::kExprI32Eqz, node);
2729 }
2730
CanCover(Node * value,IrOpcode::Value opcode)2731 bool CanCover(Node* value, IrOpcode::Value opcode) {
2732 if (value->opcode() != opcode) return false;
2733 bool first = true;
2734 for (Edge const edge : value->use_edges()) {
2735 if (NodeProperties::IsControlEdge(edge)) continue;
2736 if (NodeProperties::IsEffectEdge(edge)) continue;
2737 DCHECK(NodeProperties::IsValueEdge(edge));
2738 if (!first) return false;
2739 first = false;
2740 }
2741 return true;
2742 }
2743
BuildChangeInt32ToSmi(Node * value)2744 Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2745 if (mcgraph()->machine()->Is64()) {
2746 value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2747 }
2748 return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2749 BuildSmiShiftBitsConstant());
2750 }
2751
BuildChangeUint31ToSmi(Node * value)2752 Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2753 return graph()->NewNode(mcgraph()->machine()->WordShl(),
2754 Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2755 }
2756
BuildSmiShiftBitsConstant()2757 Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2758 return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2759 }
2760
BuildChangeSmiToInt32(Node * value)2761 Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2762 value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2763 BuildSmiShiftBitsConstant());
2764 if (mcgraph()->machine()->Is64()) {
2765 value =
2766 graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2767 }
2768 return value;
2769 }
2770
InitInstanceCache(WasmInstanceCacheNodes * instance_cache)2771 void WasmGraphBuilder::InitInstanceCache(
2772 WasmInstanceCacheNodes* instance_cache) {
2773 DCHECK_NOT_NULL(instance_node_);
2774 DCHECK_NOT_NULL(*control_);
2775 DCHECK_NOT_NULL(*effect_);
2776
2777 // Load the memory start.
2778 Node* mem_start = graph()->NewNode(
2779 mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2780 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2781 *effect_, *control_);
2782 *effect_ = mem_start;
2783 instance_cache->mem_start = mem_start;
2784
2785 // Load the memory size.
2786 Node* mem_size = graph()->NewNode(
2787 mcgraph()->machine()->Load(MachineType::Uint32()), instance_node_.get(),
2788 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2789 *effect_, *control_);
2790 *effect_ = mem_size;
2791 instance_cache->mem_size = mem_size;
2792
2793 if (untrusted_code_mitigations_) {
2794 // Load the memory mask.
2795 Node* mem_mask = graph()->NewNode(
2796 mcgraph()->machine()->Load(MachineType::Uint32()), instance_node_.get(),
2797 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryMask)),
2798 *effect_, *control_);
2799 *effect_ = mem_mask;
2800 instance_cache->mem_mask = mem_mask;
2801 } else {
2802 // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2803 instance_cache->mem_mask = nullptr;
2804 }
2805 }
2806
PrepareInstanceCacheForLoop(WasmInstanceCacheNodes * instance_cache,Node * control)2807 void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2808 WasmInstanceCacheNodes* instance_cache, Node* control) {
2809 #define INTRODUCE_PHI(field, rep) \
2810 instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2811 instance_cache->field, control);
2812
2813 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2814 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2815 if (untrusted_code_mitigations_) {
2816 INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2817 }
2818
2819 #undef INTRODUCE_PHI
2820 }
2821
NewInstanceCacheMerge(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2822 void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2823 WasmInstanceCacheNodes* from,
2824 Node* merge) {
2825 #define INTRODUCE_PHI(field, rep) \
2826 if (to->field != from->field) { \
2827 Node* vals[] = {to->field, from->field, merge}; \
2828 to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2829 }
2830
2831 INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2832 INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2833 if (untrusted_code_mitigations_) {
2834 INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2835 }
2836
2837 #undef INTRODUCE_PHI
2838 }
2839
MergeInstanceCacheInto(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2840 void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2841 WasmInstanceCacheNodes* from,
2842 Node* merge) {
2843 to->mem_size = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge,
2844 to->mem_size, from->mem_size);
2845 to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2846 merge, to->mem_start, from->mem_start);
2847 if (untrusted_code_mitigations_) {
2848 to->mem_mask = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge,
2849 to->mem_mask, from->mem_mask);
2850 }
2851 }
2852
CreateOrMergeIntoPhi(MachineRepresentation rep,Node * merge,Node * tnode,Node * fnode)2853 Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
2854 Node* merge, Node* tnode,
2855 Node* fnode) {
2856 if (IsPhiWithMerge(tnode, merge)) {
2857 AppendToPhi(tnode, fnode);
2858 } else if (tnode != fnode) {
2859 uint32_t count = merge->InputCount();
2860 // + 1 for the merge node.
2861 Node** vals = Buffer(count + 1);
2862 for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
2863 vals[count - 1] = fnode;
2864 vals[count] = merge;
2865 return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
2866 vals);
2867 }
2868 return tnode;
2869 }
2870
CreateOrMergeIntoEffectPhi(Node * merge,Node * tnode,Node * fnode)2871 Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
2872 Node* fnode) {
2873 if (IsPhiWithMerge(tnode, merge)) {
2874 AppendToPhi(tnode, fnode);
2875 } else if (tnode != fnode) {
2876 uint32_t count = merge->InputCount();
2877 Node** effects = Buffer(count);
2878 for (uint32_t j = 0; j < count - 1; j++) {
2879 effects[j] = tnode;
2880 }
2881 effects[count - 1] = fnode;
2882 tnode = EffectPhi(count, effects, merge);
2883 }
2884 return tnode;
2885 }
2886
GetGlobalBaseAndOffset(MachineType mem_type,const wasm::WasmGlobal & global,Node ** base_node,Node ** offset_node)2887 void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
2888 const wasm::WasmGlobal& global,
2889 Node** base_node,
2890 Node** offset_node) {
2891 DCHECK_NOT_NULL(instance_node_);
2892 if (global.mutability && global.imported) {
2893 DCHECK(FLAG_experimental_wasm_mut_global);
2894 if (imported_mutable_globals_ == nullptr) {
2895 // Load imported_mutable_globals_ from the instance object at runtime.
2896 imported_mutable_globals_ = graph()->NewNode(
2897 mcgraph()->machine()->Load(MachineType::UintPtr()),
2898 instance_node_.get(),
2899 mcgraph()->Int32Constant(
2900 WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
2901 graph()->start(), graph()->start());
2902 }
2903 *base_node = graph()->NewNode(
2904 mcgraph()->machine()->Load(MachineType::UintPtr()),
2905 imported_mutable_globals_.get(),
2906 mcgraph()->Int32Constant(global.index * sizeof(Address)), *effect_,
2907 *control_);
2908 *offset_node = mcgraph()->Int32Constant(0);
2909 *effect_ = *base_node;
2910 } else {
2911 if (globals_start_ == nullptr) {
2912 // Load globals_start from the instance object at runtime.
2913 // TODO(wasm): we currently generate only one load of the {globals_start}
2914 // start per graph, which means it can be placed anywhere by the
2915 // scheduler. This is legal because the globals_start should never change.
2916 // However, in some cases (e.g. if the instance object is already in a
2917 // register), it is slightly more efficient to reload this value from the
2918 // instance object. Since this depends on register allocation, it is not
2919 // possible to express in the graph, and would essentially constitute a
2920 // "mem2reg" optimization in TurboFan.
2921 globals_start_ = graph()->NewNode(
2922 mcgraph()->machine()->Load(MachineType::UintPtr()),
2923 instance_node_.get(),
2924 mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
2925 graph()->start(), graph()->start());
2926 }
2927 *base_node = globals_start_.get();
2928 *offset_node = mcgraph()->Int32Constant(global.offset);
2929
2930 if (mem_type == MachineType::Simd128() && global.offset != 0) {
2931 // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
2932 *base_node =
2933 graph()->NewNode(kPointerSize == 4 ? mcgraph()->machine()->Int32Add()
2934 : mcgraph()->machine()->Int64Add(),
2935 *base_node, *offset_node);
2936 *offset_node = mcgraph()->Int32Constant(0);
2937 }
2938 }
2939 }
2940
MemBuffer(uint32_t offset)2941 Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
2942 DCHECK_NOT_NULL(instance_cache_);
2943 Node* mem_start = instance_cache_->mem_start;
2944 DCHECK_NOT_NULL(mem_start);
2945 if (offset == 0) return mem_start;
2946 return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
2947 mcgraph()->IntPtrConstant(offset));
2948 }
2949
CurrentMemoryPages()2950 Node* WasmGraphBuilder::CurrentMemoryPages() {
2951 // CurrentMemoryPages can not be called from asm.js.
2952 DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin());
2953 DCHECK_NOT_NULL(instance_cache_);
2954 Node* mem_size = instance_cache_->mem_size;
2955 DCHECK_NOT_NULL(mem_size);
2956 return graph()->NewNode(
2957 mcgraph()->machine()->Word32Shr(), mem_size,
2958 mcgraph()->Int32Constant(WhichPowerOf2(wasm::kWasmPageSize)));
2959 }
2960
2961 // Only call this function for code which is not reused across instantiations,
2962 // as we do not patch the embedded js_context.
BuildCallToRuntimeWithContext(Runtime::FunctionId f,Node * js_context,Node ** parameters,int parameter_count)2963 Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f,
2964 Node* js_context,
2965 Node** parameters,
2966 int parameter_count) {
2967 const Runtime::Function* fun = Runtime::FunctionForId(f);
2968 auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
2969 mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
2970 CallDescriptor::kNoFlags);
2971 // CEntryStubConstant nodes have to be created and cached in the main
2972 // thread. At the moment this is only done for CEntryStubConstant(1).
2973 DCHECK_EQ(1, fun->result_size);
2974 // At the moment we only allow 4 parameters. If more parameters are needed,
2975 // increase this constant accordingly.
2976 static const int kMaxParams = 4;
2977 DCHECK_GE(kMaxParams, parameter_count);
2978 Node* inputs[kMaxParams + 6];
2979 int count = 0;
2980 inputs[count++] = CEntryStub();
2981 for (int i = 0; i < parameter_count; i++) {
2982 inputs[count++] = parameters[i];
2983 }
2984 inputs[count++] =
2985 mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
2986 inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity
2987 inputs[count++] = js_context; // js_context
2988 inputs[count++] = *effect_;
2989 inputs[count++] = *control_;
2990
2991 Node* node = mcgraph()->graph()->NewNode(
2992 mcgraph()->common()->Call(call_descriptor), count, inputs);
2993 *effect_ = node;
2994
2995 return node;
2996 }
2997
BuildCallToRuntime(Runtime::FunctionId f,Node ** parameters,int parameter_count)2998 Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
2999 Node** parameters,
3000 int parameter_count) {
3001 return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3002 parameter_count);
3003 }
3004
GetGlobal(uint32_t index)3005 Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3006 MachineType mem_type =
3007 wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3008 Node* base = nullptr;
3009 Node* offset = nullptr;
3010 GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3011 &offset);
3012 Node* node = graph()->NewNode(mcgraph()->machine()->Load(mem_type), base,
3013 offset, *effect_, *control_);
3014 *effect_ = node;
3015 return node;
3016 }
3017
SetGlobal(uint32_t index,Node * val)3018 Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3019 MachineType mem_type =
3020 wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3021 Node* base = nullptr;
3022 Node* offset = nullptr;
3023 GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3024 &offset);
3025 const Operator* op = mcgraph()->machine()->Store(
3026 StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3027 Node* node = graph()->NewNode(op, base, offset, val, *effect_, *control_);
3028 *effect_ = node;
3029 return node;
3030 }
3031
BoundsCheckMem(uint8_t access_size,Node * index,uint32_t offset,wasm::WasmCodePosition position,EnforceBoundsCheck enforce_check)3032 Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3033 uint32_t offset,
3034 wasm::WasmCodePosition position,
3035 EnforceBoundsCheck enforce_check) {
3036 if (FLAG_wasm_no_bounds_checks) return Uint32ToUintptr(index);
3037 DCHECK_NOT_NULL(instance_cache_);
3038 Node* mem_size = instance_cache_->mem_size;
3039 DCHECK_NOT_NULL(mem_size);
3040
3041 auto m = mcgraph()->machine();
3042 if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3043 // Simply zero out the 32-bits on 64-bit targets and let the trap handler
3044 // do its job.
3045 return Uint32ToUintptr(index);
3046 }
3047
3048 uint32_t min_size = env_->module->initial_pages * wasm::kWasmPageSize;
3049 uint32_t max_size =
3050 (env_->module->has_maximum_pages ? env_->module->maximum_pages
3051 : wasm::kV8MaxWasmMemoryPages) *
3052 wasm::kWasmPageSize;
3053
3054 if (access_size > max_size || offset > max_size - access_size) {
3055 // The access will be out of bounds, even for the largest memory.
3056 TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3057 return mcgraph()->IntPtrConstant(0);
3058 }
3059 DCHECK_LE(1, access_size);
3060 // This computation cannot overflow, since
3061 // {offset <= max_size - access_size <= kMaxUint32 - access_size}.
3062 // It also cannot underflow, since {access_size >= 1}.
3063 uint32_t end_offset = offset + access_size - 1;
3064 Node* end_offset_node = Int32Constant(end_offset);
3065
3066 // The accessed memory is [index + offset, index + end_offset].
3067 // Check that the last read byte (at {index + end_offset}) is in bounds.
3068 // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3069 // compute {effective_size} as {mem_size - end_offset)}.
3070 // {effective_size} is >= 1 if condition 1) holds.
3071 // 2) Check that {index + end_offset < mem_size} by
3072 // - computing {effective_size} as {mem_size - end_offset} and
3073 // - checking that {index < effective_size}.
3074
3075 if (end_offset >= min_size) {
3076 // The end offset is larger than the smallest memory.
3077 // Dynamically check the end offset against the actual memory size, which
3078 // is not known at compile time.
3079 Node* cond = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
3080 end_offset_node, mem_size);
3081 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3082 } else {
3083 // The end offset is within the bounds of the smallest memory, so only
3084 // one check is required. Check to see if the index is also a constant.
3085 Uint32Matcher match(index);
3086 if (match.HasValue()) {
3087 uint32_t index_val = match.Value();
3088 if (index_val < min_size - end_offset) {
3089 // The input index is a constant and everything is statically within
3090 // bounds of the smallest possible memory.
3091 return Uint32ToUintptr(index);
3092 }
3093 }
3094 }
3095
3096 // This produces a positive number, since {end_offset < min_size <= mem_size}.
3097 Node* effective_size = graph()->NewNode(mcgraph()->machine()->Int32Sub(),
3098 mem_size, end_offset_node);
3099
3100 // Introduce the actual bounds check.
3101 Node* cond = graph()->NewNode(m->Uint32LessThan(), index, effective_size);
3102 TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3103
3104 if (untrusted_code_mitigations_) {
3105 // In the fallthrough case, condition the index with the memory mask.
3106 Node* mem_mask = instance_cache_->mem_mask;
3107 DCHECK_NOT_NULL(mem_mask);
3108 index = graph()->NewNode(m->Word32And(), index, mem_mask);
3109 }
3110 return Uint32ToUintptr(index);
3111 }
3112
GetSafeLoadOperator(int offset,wasm::ValueType type)3113 const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3114 wasm::ValueType type) {
3115 int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3116 MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3117 if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3118 wasm::ValueTypes::MachineRepresentationFor(type))) {
3119 return mcgraph()->machine()->Load(mach_type);
3120 }
3121 return mcgraph()->machine()->UnalignedLoad(mach_type);
3122 }
3123
GetSafeStoreOperator(int offset,wasm::ValueType type)3124 const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3125 wasm::ValueType type) {
3126 int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3127 MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3128 if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3129 StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3130 return mcgraph()->machine()->Store(store_rep);
3131 }
3132 UnalignedStoreRepresentation store_rep(rep);
3133 return mcgraph()->machine()->UnalignedStore(store_rep);
3134 }
3135
TraceMemoryOperation(bool is_store,MachineRepresentation rep,Node * index,uint32_t offset,wasm::WasmCodePosition position)3136 Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3137 MachineRepresentation rep,
3138 Node* index, uint32_t offset,
3139 wasm::WasmCodePosition position) {
3140 int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3141 Node* info = graph()->NewNode(
3142 mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3143
3144 Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3145 Int32Constant(offset), index);
3146 auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3147 *effect_ = graph()->NewNode(
3148 mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3149 info, mcgraph()->Int32Constant(offset), data, *effect_, *control_);
3150 };
3151 // Store address, is_store, and mem_rep.
3152 store(offsetof(wasm::MemoryTracingInfo, address),
3153 MachineRepresentation::kWord32, address);
3154 store(offsetof(wasm::MemoryTracingInfo, is_store),
3155 MachineRepresentation::kWord8,
3156 mcgraph()->Int32Constant(is_store ? 1 : 0));
3157 store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3158 MachineRepresentation::kWord8,
3159 mcgraph()->Int32Constant(static_cast<int>(rep)));
3160
3161 Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3162 SetSourcePosition(call, position);
3163 return call;
3164 }
3165
LoadMem(wasm::ValueType type,MachineType memtype,Node * index,uint32_t offset,uint32_t alignment,wasm::WasmCodePosition position)3166 Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3167 Node* index, uint32_t offset,
3168 uint32_t alignment,
3169 wasm::WasmCodePosition position) {
3170 Node* load;
3171
3172 // Wasm semantics throw on OOB. Introduce explicit bounds check and
3173 // conditioning when not using the trap handler.
3174 index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3175 position, kCanOmitBoundsCheck);
3176
3177 if (memtype.representation() == MachineRepresentation::kWord8 ||
3178 mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3179 if (use_trap_handler()) {
3180 load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3181 MemBuffer(offset), index, *effect_, *control_);
3182 SetSourcePosition(load, position);
3183 } else {
3184 load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3185 MemBuffer(offset), index, *effect_, *control_);
3186 }
3187 } else {
3188 // TODO(eholk): Support unaligned loads with trap handlers.
3189 DCHECK(!use_trap_handler());
3190 load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3191 MemBuffer(offset), index, *effect_, *control_);
3192 }
3193
3194 *effect_ = load;
3195
3196 #if defined(V8_TARGET_BIG_ENDIAN)
3197 load = BuildChangeEndiannessLoad(load, memtype, type);
3198 #endif
3199
3200 if (type == wasm::kWasmI64 &&
3201 ElementSizeInBytes(memtype.representation()) < 8) {
3202 // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3203 if (memtype.IsSigned()) {
3204 // sign extend
3205 load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3206 } else {
3207 // zero extend
3208 load =
3209 graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3210 }
3211 }
3212
3213 if (FLAG_wasm_trace_memory) {
3214 TraceMemoryOperation(false, memtype.representation(), index, offset,
3215 position);
3216 }
3217
3218 return load;
3219 }
3220
StoreMem(MachineRepresentation mem_rep,Node * index,uint32_t offset,uint32_t alignment,Node * val,wasm::WasmCodePosition position,wasm::ValueType type)3221 Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3222 uint32_t offset, uint32_t alignment, Node* val,
3223 wasm::WasmCodePosition position,
3224 wasm::ValueType type) {
3225 Node* store;
3226
3227 index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3228 position, kCanOmitBoundsCheck);
3229
3230 #if defined(V8_TARGET_BIG_ENDIAN)
3231 val = BuildChangeEndiannessStore(val, mem_rep, type);
3232 #endif
3233
3234 if (mem_rep == MachineRepresentation::kWord8 ||
3235 mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3236 if (use_trap_handler()) {
3237 store =
3238 graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3239 MemBuffer(offset), index, val, *effect_, *control_);
3240 SetSourcePosition(store, position);
3241 } else {
3242 StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3243 store =
3244 graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3245 index, val, *effect_, *control_);
3246 }
3247 } else {
3248 // TODO(eholk): Support unaligned stores with trap handlers.
3249 DCHECK(!use_trap_handler());
3250 UnalignedStoreRepresentation rep(mem_rep);
3251 store =
3252 graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3253 MemBuffer(offset), index, val, *effect_, *control_);
3254 }
3255
3256 *effect_ = store;
3257
3258 if (FLAG_wasm_trace_memory) {
3259 TraceMemoryOperation(true, mem_rep, index, offset, position);
3260 }
3261
3262 return store;
3263 }
3264
3265 namespace {
GetAsmJsOOBValue(MachineRepresentation rep,MachineGraph * mcgraph)3266 Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3267 switch (rep) {
3268 case MachineRepresentation::kWord8:
3269 case MachineRepresentation::kWord16:
3270 case MachineRepresentation::kWord32:
3271 return mcgraph->Int32Constant(0);
3272 case MachineRepresentation::kWord64:
3273 return mcgraph->Int64Constant(0);
3274 case MachineRepresentation::kFloat32:
3275 return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3276 case MachineRepresentation::kFloat64:
3277 return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3278 default:
3279 UNREACHABLE();
3280 }
3281 }
3282 } // namespace
3283
BuildAsmjsLoadMem(MachineType type,Node * index)3284 Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3285 DCHECK_NOT_NULL(instance_cache_);
3286 Node* mem_start = instance_cache_->mem_start;
3287 Node* mem_size = instance_cache_->mem_size;
3288 DCHECK_NOT_NULL(mem_start);
3289 DCHECK_NOT_NULL(mem_size);
3290
3291 // Asm.js semantics are defined along the lines of typed arrays, hence OOB
3292 // reads return {undefined} coerced to the result type (0 for integers, NaN
3293 // for float and double).
3294 // Note that we check against the memory size ignoring the size of the
3295 // stored value, which is conservative if misaligned. Technically, asm.js
3296 // should never have misaligned accesses.
3297 Diamond bounds_check(
3298 graph(), mcgraph()->common(),
3299 graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3300 BranchHint::kTrue);
3301 bounds_check.Chain(*control_);
3302
3303 if (untrusted_code_mitigations_) {
3304 // Condition the index with the memory mask.
3305 Node* mem_mask = instance_cache_->mem_mask;
3306 DCHECK_NOT_NULL(mem_mask);
3307 index =
3308 graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3309 }
3310
3311 index = Uint32ToUintptr(index);
3312 Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3313 index, *effect_, bounds_check.if_true);
3314 Node* value_phi =
3315 bounds_check.Phi(type.representation(), load,
3316 GetAsmJsOOBValue(type.representation(), mcgraph()));
3317 Node* effect_phi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), load,
3318 *effect_, bounds_check.merge);
3319 *effect_ = effect_phi;
3320 *control_ = bounds_check.merge;
3321 return value_phi;
3322 }
3323
Uint32ToUintptr(Node * node)3324 Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3325 if (mcgraph()->machine()->Is32()) return node;
3326 return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3327 }
3328
BuildAsmjsStoreMem(MachineType type,Node * index,Node * val)3329 Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3330 Node* val) {
3331 DCHECK_NOT_NULL(instance_cache_);
3332 Node* mem_start = instance_cache_->mem_start;
3333 Node* mem_size = instance_cache_->mem_size;
3334 DCHECK_NOT_NULL(mem_start);
3335 DCHECK_NOT_NULL(mem_size);
3336
3337 // Asm.js semantics are to ignore OOB writes.
3338 // Note that we check against the memory size ignoring the size of the
3339 // stored value, which is conservative if misaligned. Technically, asm.js
3340 // should never have misaligned accesses.
3341 Diamond bounds_check(
3342 graph(), mcgraph()->common(),
3343 graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3344 BranchHint::kTrue);
3345 bounds_check.Chain(*control_);
3346
3347 if (untrusted_code_mitigations_) {
3348 // Condition the index with the memory mask.
3349 Node* mem_mask = instance_cache_->mem_mask;
3350 DCHECK_NOT_NULL(mem_mask);
3351 index =
3352 graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3353 }
3354
3355 index = Uint32ToUintptr(index);
3356 const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3357 type.representation(), WriteBarrierKind::kNoWriteBarrier));
3358 Node* store = graph()->NewNode(store_op, mem_start, index, val, *effect_,
3359 bounds_check.if_true);
3360 Node* effect_phi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), store,
3361 *effect_, bounds_check.merge);
3362 *effect_ = effect_phi;
3363 *control_ = bounds_check.merge;
3364 return val;
3365 }
3366
PrintDebugName(Node * node)3367 void WasmGraphBuilder::PrintDebugName(Node* node) {
3368 PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3369 }
3370
graph()3371 Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3372
3373 namespace {
CreateMachineSignature(Zone * zone,wasm::FunctionSig * sig)3374 Signature<MachineRepresentation>* CreateMachineSignature(
3375 Zone* zone, wasm::FunctionSig* sig) {
3376 Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3377 sig->parameter_count());
3378 for (auto ret : sig->returns()) {
3379 builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3380 }
3381
3382 for (auto param : sig->parameters()) {
3383 builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3384 }
3385 return builder.Build();
3386 }
3387 } // namespace
3388
LowerInt64()3389 void WasmGraphBuilder::LowerInt64() {
3390 if (mcgraph()->machine()->Is64()) return;
3391 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3392 mcgraph()->zone(),
3393 CreateMachineSignature(mcgraph()->zone(), sig_));
3394 r.LowerGraph();
3395 }
3396
SimdScalarLoweringForTesting()3397 void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3398 SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3399 .LowerGraph();
3400 }
3401
SetSourcePosition(Node * node,wasm::WasmCodePosition position)3402 void WasmGraphBuilder::SetSourcePosition(Node* node,
3403 wasm::WasmCodePosition position) {
3404 DCHECK_NE(position, wasm::kNoCodePosition);
3405 if (source_position_table_)
3406 source_position_table_->SetSourcePosition(node, SourcePosition(position));
3407 }
3408
S128Zero()3409 Node* WasmGraphBuilder::S128Zero() {
3410 has_simd_ = true;
3411 return graph()->NewNode(mcgraph()->machine()->S128Zero());
3412 }
3413
SimdOp(wasm::WasmOpcode opcode,Node * const * inputs)3414 Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3415 has_simd_ = true;
3416 switch (opcode) {
3417 case wasm::kExprF32x4Splat:
3418 return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3419 case wasm::kExprF32x4SConvertI32x4:
3420 return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3421 inputs[0]);
3422 case wasm::kExprF32x4UConvertI32x4:
3423 return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3424 inputs[0]);
3425 case wasm::kExprF32x4Abs:
3426 return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3427 case wasm::kExprF32x4Neg:
3428 return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3429 case wasm::kExprF32x4RecipApprox:
3430 return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3431 inputs[0]);
3432 case wasm::kExprF32x4RecipSqrtApprox:
3433 return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3434 inputs[0]);
3435 case wasm::kExprF32x4Add:
3436 return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3437 inputs[1]);
3438 case wasm::kExprF32x4AddHoriz:
3439 return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3440 inputs[1]);
3441 case wasm::kExprF32x4Sub:
3442 return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3443 inputs[1]);
3444 case wasm::kExprF32x4Mul:
3445 return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3446 inputs[1]);
3447 case wasm::kExprF32x4Min:
3448 return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3449 inputs[1]);
3450 case wasm::kExprF32x4Max:
3451 return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3452 inputs[1]);
3453 case wasm::kExprF32x4Eq:
3454 return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3455 inputs[1]);
3456 case wasm::kExprF32x4Ne:
3457 return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3458 inputs[1]);
3459 case wasm::kExprF32x4Lt:
3460 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3461 inputs[1]);
3462 case wasm::kExprF32x4Le:
3463 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3464 inputs[1]);
3465 case wasm::kExprF32x4Gt:
3466 return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3467 inputs[0]);
3468 case wasm::kExprF32x4Ge:
3469 return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3470 inputs[0]);
3471 case wasm::kExprI32x4Splat:
3472 return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3473 case wasm::kExprI32x4SConvertF32x4:
3474 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3475 inputs[0]);
3476 case wasm::kExprI32x4UConvertF32x4:
3477 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3478 inputs[0]);
3479 case wasm::kExprI32x4SConvertI16x8Low:
3480 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3481 inputs[0]);
3482 case wasm::kExprI32x4SConvertI16x8High:
3483 return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3484 inputs[0]);
3485 case wasm::kExprI32x4Neg:
3486 return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3487 case wasm::kExprI32x4Add:
3488 return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3489 inputs[1]);
3490 case wasm::kExprI32x4AddHoriz:
3491 return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3492 inputs[1]);
3493 case wasm::kExprI32x4Sub:
3494 return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3495 inputs[1]);
3496 case wasm::kExprI32x4Mul:
3497 return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3498 inputs[1]);
3499 case wasm::kExprI32x4MinS:
3500 return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3501 inputs[1]);
3502 case wasm::kExprI32x4MaxS:
3503 return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3504 inputs[1]);
3505 case wasm::kExprI32x4Eq:
3506 return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3507 inputs[1]);
3508 case wasm::kExprI32x4Ne:
3509 return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3510 inputs[1]);
3511 case wasm::kExprI32x4LtS:
3512 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3513 inputs[0]);
3514 case wasm::kExprI32x4LeS:
3515 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3516 inputs[0]);
3517 case wasm::kExprI32x4GtS:
3518 return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3519 inputs[1]);
3520 case wasm::kExprI32x4GeS:
3521 return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3522 inputs[1]);
3523 case wasm::kExprI32x4UConvertI16x8Low:
3524 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3525 inputs[0]);
3526 case wasm::kExprI32x4UConvertI16x8High:
3527 return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3528 inputs[0]);
3529 case wasm::kExprI32x4MinU:
3530 return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3531 inputs[1]);
3532 case wasm::kExprI32x4MaxU:
3533 return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3534 inputs[1]);
3535 case wasm::kExprI32x4LtU:
3536 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3537 inputs[0]);
3538 case wasm::kExprI32x4LeU:
3539 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3540 inputs[0]);
3541 case wasm::kExprI32x4GtU:
3542 return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3543 inputs[1]);
3544 case wasm::kExprI32x4GeU:
3545 return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3546 inputs[1]);
3547 case wasm::kExprI16x8Splat:
3548 return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3549 case wasm::kExprI16x8SConvertI8x16Low:
3550 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3551 inputs[0]);
3552 case wasm::kExprI16x8SConvertI8x16High:
3553 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3554 inputs[0]);
3555 case wasm::kExprI16x8Neg:
3556 return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3557 case wasm::kExprI16x8SConvertI32x4:
3558 return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3559 inputs[0], inputs[1]);
3560 case wasm::kExprI16x8Add:
3561 return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3562 inputs[1]);
3563 case wasm::kExprI16x8AddSaturateS:
3564 return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3565 inputs[0], inputs[1]);
3566 case wasm::kExprI16x8AddHoriz:
3567 return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3568 inputs[1]);
3569 case wasm::kExprI16x8Sub:
3570 return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3571 inputs[1]);
3572 case wasm::kExprI16x8SubSaturateS:
3573 return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3574 inputs[0], inputs[1]);
3575 case wasm::kExprI16x8Mul:
3576 return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3577 inputs[1]);
3578 case wasm::kExprI16x8MinS:
3579 return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3580 inputs[1]);
3581 case wasm::kExprI16x8MaxS:
3582 return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3583 inputs[1]);
3584 case wasm::kExprI16x8Eq:
3585 return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3586 inputs[1]);
3587 case wasm::kExprI16x8Ne:
3588 return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3589 inputs[1]);
3590 case wasm::kExprI16x8LtS:
3591 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3592 inputs[0]);
3593 case wasm::kExprI16x8LeS:
3594 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3595 inputs[0]);
3596 case wasm::kExprI16x8GtS:
3597 return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3598 inputs[1]);
3599 case wasm::kExprI16x8GeS:
3600 return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3601 inputs[1]);
3602 case wasm::kExprI16x8UConvertI8x16Low:
3603 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3604 inputs[0]);
3605 case wasm::kExprI16x8UConvertI8x16High:
3606 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3607 inputs[0]);
3608 case wasm::kExprI16x8UConvertI32x4:
3609 return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3610 inputs[0], inputs[1]);
3611 case wasm::kExprI16x8AddSaturateU:
3612 return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3613 inputs[0], inputs[1]);
3614 case wasm::kExprI16x8SubSaturateU:
3615 return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3616 inputs[0], inputs[1]);
3617 case wasm::kExprI16x8MinU:
3618 return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3619 inputs[1]);
3620 case wasm::kExprI16x8MaxU:
3621 return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3622 inputs[1]);
3623 case wasm::kExprI16x8LtU:
3624 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3625 inputs[0]);
3626 case wasm::kExprI16x8LeU:
3627 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3628 inputs[0]);
3629 case wasm::kExprI16x8GtU:
3630 return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3631 inputs[1]);
3632 case wasm::kExprI16x8GeU:
3633 return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3634 inputs[1]);
3635 case wasm::kExprI8x16Splat:
3636 return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3637 case wasm::kExprI8x16Neg:
3638 return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3639 case wasm::kExprI8x16SConvertI16x8:
3640 return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3641 inputs[0], inputs[1]);
3642 case wasm::kExprI8x16Add:
3643 return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3644 inputs[1]);
3645 case wasm::kExprI8x16AddSaturateS:
3646 return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3647 inputs[0], inputs[1]);
3648 case wasm::kExprI8x16Sub:
3649 return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3650 inputs[1]);
3651 case wasm::kExprI8x16SubSaturateS:
3652 return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3653 inputs[0], inputs[1]);
3654 case wasm::kExprI8x16Mul:
3655 return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3656 inputs[1]);
3657 case wasm::kExprI8x16MinS:
3658 return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3659 inputs[1]);
3660 case wasm::kExprI8x16MaxS:
3661 return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3662 inputs[1]);
3663 case wasm::kExprI8x16Eq:
3664 return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3665 inputs[1]);
3666 case wasm::kExprI8x16Ne:
3667 return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3668 inputs[1]);
3669 case wasm::kExprI8x16LtS:
3670 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3671 inputs[0]);
3672 case wasm::kExprI8x16LeS:
3673 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3674 inputs[0]);
3675 case wasm::kExprI8x16GtS:
3676 return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3677 inputs[1]);
3678 case wasm::kExprI8x16GeS:
3679 return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3680 inputs[1]);
3681 case wasm::kExprI8x16UConvertI16x8:
3682 return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3683 inputs[0], inputs[1]);
3684 case wasm::kExprI8x16AddSaturateU:
3685 return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3686 inputs[0], inputs[1]);
3687 case wasm::kExprI8x16SubSaturateU:
3688 return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3689 inputs[0], inputs[1]);
3690 case wasm::kExprI8x16MinU:
3691 return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3692 inputs[1]);
3693 case wasm::kExprI8x16MaxU:
3694 return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
3695 inputs[1]);
3696 case wasm::kExprI8x16LtU:
3697 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
3698 inputs[0]);
3699 case wasm::kExprI8x16LeU:
3700 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
3701 inputs[0]);
3702 case wasm::kExprI8x16GtU:
3703 return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
3704 inputs[1]);
3705 case wasm::kExprI8x16GeU:
3706 return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
3707 inputs[1]);
3708 case wasm::kExprS128And:
3709 return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
3710 inputs[1]);
3711 case wasm::kExprS128Or:
3712 return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
3713 inputs[1]);
3714 case wasm::kExprS128Xor:
3715 return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
3716 inputs[1]);
3717 case wasm::kExprS128Not:
3718 return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
3719 case wasm::kExprS128Select:
3720 return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[0],
3721 inputs[1], inputs[2]);
3722 case wasm::kExprS1x4AnyTrue:
3723 return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
3724 case wasm::kExprS1x4AllTrue:
3725 return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
3726 case wasm::kExprS1x8AnyTrue:
3727 return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
3728 case wasm::kExprS1x8AllTrue:
3729 return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
3730 case wasm::kExprS1x16AnyTrue:
3731 return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
3732 case wasm::kExprS1x16AllTrue:
3733 return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
3734 default:
3735 FATAL_UNSUPPORTED_OPCODE(opcode);
3736 }
3737 }
3738
SimdLaneOp(wasm::WasmOpcode opcode,uint8_t lane,Node * const * inputs)3739 Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
3740 Node* const* inputs) {
3741 has_simd_ = true;
3742 switch (opcode) {
3743 case wasm::kExprF32x4ExtractLane:
3744 return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
3745 inputs[0]);
3746 case wasm::kExprF32x4ReplaceLane:
3747 return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
3748 inputs[0], inputs[1]);
3749 case wasm::kExprI32x4ExtractLane:
3750 return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
3751 inputs[0]);
3752 case wasm::kExprI32x4ReplaceLane:
3753 return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
3754 inputs[0], inputs[1]);
3755 case wasm::kExprI16x8ExtractLane:
3756 return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
3757 inputs[0]);
3758 case wasm::kExprI16x8ReplaceLane:
3759 return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
3760 inputs[0], inputs[1]);
3761 case wasm::kExprI8x16ExtractLane:
3762 return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
3763 inputs[0]);
3764 case wasm::kExprI8x16ReplaceLane:
3765 return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
3766 inputs[0], inputs[1]);
3767 default:
3768 FATAL_UNSUPPORTED_OPCODE(opcode);
3769 }
3770 }
3771
SimdShiftOp(wasm::WasmOpcode opcode,uint8_t shift,Node * const * inputs)3772 Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
3773 Node* const* inputs) {
3774 has_simd_ = true;
3775 switch (opcode) {
3776 case wasm::kExprI32x4Shl:
3777 return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
3778 case wasm::kExprI32x4ShrS:
3779 return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
3780 inputs[0]);
3781 case wasm::kExprI32x4ShrU:
3782 return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
3783 inputs[0]);
3784 case wasm::kExprI16x8Shl:
3785 return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
3786 case wasm::kExprI16x8ShrS:
3787 return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
3788 inputs[0]);
3789 case wasm::kExprI16x8ShrU:
3790 return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
3791 inputs[0]);
3792 case wasm::kExprI8x16Shl:
3793 return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
3794 case wasm::kExprI8x16ShrS:
3795 return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
3796 inputs[0]);
3797 case wasm::kExprI8x16ShrU:
3798 return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
3799 inputs[0]);
3800 default:
3801 FATAL_UNSUPPORTED_OPCODE(opcode);
3802 }
3803 }
3804
Simd8x16ShuffleOp(const uint8_t shuffle[16],Node * const * inputs)3805 Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
3806 Node* const* inputs) {
3807 has_simd_ = true;
3808 return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
3809 inputs[0], inputs[1]);
3810 }
3811
3812 #define ATOMIC_BINOP_LIST(V) \
3813 V(I32AtomicAdd, Add, Uint32, Word32) \
3814 V(I64AtomicAdd, Add, Uint64, Word64) \
3815 V(I32AtomicAdd8U, Add, Uint8, Word32) \
3816 V(I32AtomicAdd16U, Add, Uint16, Word32) \
3817 V(I64AtomicAdd8U, Add, Uint8, Word64) \
3818 V(I64AtomicAdd16U, Add, Uint16, Word64) \
3819 V(I64AtomicAdd32U, Add, Uint32, Word64) \
3820 V(I32AtomicSub, Sub, Uint32, Word32) \
3821 V(I64AtomicSub, Sub, Uint64, Word64) \
3822 V(I32AtomicSub8U, Sub, Uint8, Word32) \
3823 V(I32AtomicSub16U, Sub, Uint16, Word32) \
3824 V(I64AtomicSub8U, Sub, Uint8, Word64) \
3825 V(I64AtomicSub16U, Sub, Uint16, Word64) \
3826 V(I64AtomicSub32U, Sub, Uint32, Word64) \
3827 V(I32AtomicAnd, And, Uint32, Word32) \
3828 V(I64AtomicAnd, And, Uint64, Word64) \
3829 V(I32AtomicAnd8U, And, Uint8, Word32) \
3830 V(I64AtomicAnd16U, And, Uint16, Word64) \
3831 V(I32AtomicAnd16U, And, Uint16, Word32) \
3832 V(I64AtomicAnd8U, And, Uint8, Word64) \
3833 V(I64AtomicAnd32U, And, Uint32, Word64) \
3834 V(I32AtomicOr, Or, Uint32, Word32) \
3835 V(I64AtomicOr, Or, Uint64, Word64) \
3836 V(I32AtomicOr8U, Or, Uint8, Word32) \
3837 V(I32AtomicOr16U, Or, Uint16, Word32) \
3838 V(I64AtomicOr8U, Or, Uint8, Word64) \
3839 V(I64AtomicOr16U, Or, Uint16, Word64) \
3840 V(I64AtomicOr32U, Or, Uint32, Word64) \
3841 V(I32AtomicXor, Xor, Uint32, Word32) \
3842 V(I64AtomicXor, Xor, Uint64, Word64) \
3843 V(I32AtomicXor8U, Xor, Uint8, Word32) \
3844 V(I32AtomicXor16U, Xor, Uint16, Word32) \
3845 V(I64AtomicXor8U, Xor, Uint8, Word64) \
3846 V(I64AtomicXor16U, Xor, Uint16, Word64) \
3847 V(I64AtomicXor32U, Xor, Uint32, Word64) \
3848 V(I32AtomicExchange, Exchange, Uint32, Word32) \
3849 V(I64AtomicExchange, Exchange, Uint64, Word64) \
3850 V(I32AtomicExchange8U, Exchange, Uint8, Word32) \
3851 V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
3852 V(I64AtomicExchange8U, Exchange, Uint8, Word64) \
3853 V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
3854 V(I64AtomicExchange32U, Exchange, Uint32, Word64)
3855
3856 #define ATOMIC_CMP_EXCHG_LIST(V) \
3857 V(I32AtomicCompareExchange, Uint32, Word32) \
3858 V(I64AtomicCompareExchange, Uint64, Word64) \
3859 V(I32AtomicCompareExchange8U, Uint8, Word32) \
3860 V(I32AtomicCompareExchange16U, Uint16, Word32) \
3861 V(I64AtomicCompareExchange8U, Uint8, Word64) \
3862 V(I64AtomicCompareExchange16U, Uint16, Word64) \
3863 V(I64AtomicCompareExchange32U, Uint32, Word64)
3864
3865 #define ATOMIC_LOAD_LIST(V) \
3866 V(I32AtomicLoad, Uint32, Word32) \
3867 V(I64AtomicLoad, Uint64, Word64) \
3868 V(I32AtomicLoad8U, Uint8, Word32) \
3869 V(I32AtomicLoad16U, Uint16, Word32) \
3870 V(I64AtomicLoad8U, Uint8, Word64) \
3871 V(I64AtomicLoad16U, Uint16, Word64) \
3872 V(I64AtomicLoad32U, Uint32, Word64)
3873
3874 #define ATOMIC_STORE_LIST(V) \
3875 V(I32AtomicStore, Uint32, kWord32, Word32) \
3876 V(I64AtomicStore, Uint64, kWord64, Word64) \
3877 V(I32AtomicStore8U, Uint8, kWord8, Word32) \
3878 V(I32AtomicStore16U, Uint16, kWord16, Word32) \
3879 V(I64AtomicStore8U, Uint8, kWord8, Word64) \
3880 V(I64AtomicStore16U, Uint16, kWord16, Word64) \
3881 V(I64AtomicStore32U, Uint32, kWord32, Word64)
3882
AtomicOp(wasm::WasmOpcode opcode,Node * const * inputs,uint32_t alignment,uint32_t offset,wasm::WasmCodePosition position)3883 Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
3884 uint32_t alignment, uint32_t offset,
3885 wasm::WasmCodePosition position) {
3886 // TODO(gdeepti): Add alignment validation, traps on misalignment
3887 Node* node;
3888 switch (opcode) {
3889 #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \
3890 case wasm::kExpr##Name: { \
3891 Node* index = \
3892 BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()), \
3893 inputs[0], offset, position, kNeedsBoundsCheck); \
3894 node = graph()->NewNode( \
3895 mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
3896 MemBuffer(offset), index, inputs[1], *effect_, *control_); \
3897 break; \
3898 }
3899 ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
3900 #undef BUILD_ATOMIC_BINOP
3901
3902 #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \
3903 case wasm::kExpr##Name: { \
3904 Node* index = \
3905 BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()), \
3906 inputs[0], offset, position, kNeedsBoundsCheck); \
3907 node = graph()->NewNode( \
3908 mcgraph()->machine()->Prefix##AtomicCompareExchange( \
3909 MachineType::Type()), \
3910 MemBuffer(offset), index, inputs[1], inputs[2], *effect_, *control_); \
3911 break; \
3912 }
3913 ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
3914 #undef BUILD_ATOMIC_CMP_EXCHG
3915
3916 #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \
3917 case wasm::kExpr##Name: { \
3918 Node* index = \
3919 BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()), \
3920 inputs[0], offset, position, kNeedsBoundsCheck); \
3921 node = graph()->NewNode( \
3922 mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \
3923 MemBuffer(offset), index, *effect_, *control_); \
3924 break; \
3925 }
3926 ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
3927 #undef BUILD_ATOMIC_LOAD_OP
3928
3929 #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \
3930 case wasm::kExpr##Name: { \
3931 Node* index = \
3932 BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()), \
3933 inputs[0], offset, position, kNeedsBoundsCheck); \
3934 node = graph()->NewNode( \
3935 mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
3936 MemBuffer(offset), index, inputs[1], *effect_, *control_); \
3937 break; \
3938 }
3939 ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
3940 #undef BUILD_ATOMIC_STORE_OP
3941 default:
3942 FATAL_UNSUPPORTED_OPCODE(opcode);
3943 }
3944 *effect_ = node;
3945 return node;
3946 }
3947
3948 #undef ATOMIC_BINOP_LIST
3949 #undef ATOMIC_CMP_EXCHG_LIST
3950 #undef ATOMIC_LOAD_LIST
3951 #undef ATOMIC_STORE_LIST
3952
3953 namespace {
must_record_function_compilation(Isolate * isolate)3954 bool must_record_function_compilation(Isolate* isolate) {
3955 return isolate->logger()->is_listening_to_code_events() ||
3956 isolate->is_profiling();
3957 }
3958
3959 PRINTF_FORMAT(4, 5)
RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,Isolate * isolate,Handle<Code> code,const char * format,...)3960 void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
3961 Isolate* isolate, Handle<Code> code,
3962 const char* format, ...) {
3963 DCHECK(must_record_function_compilation(isolate));
3964
3965 ScopedVector<char> buffer(128);
3966 va_list arguments;
3967 va_start(arguments, format);
3968 int len = VSNPrintF(buffer, format, arguments);
3969 CHECK_LT(0, len);
3970 va_end(arguments);
3971 Handle<String> name_str =
3972 isolate->factory()->NewStringFromAsciiChecked(buffer.start());
3973 PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
3974 }
3975
3976 class WasmWrapperGraphBuilder : public WasmGraphBuilder {
3977 public:
WasmWrapperGraphBuilder(Zone * zone,wasm::ModuleEnv * env,JSGraph * jsgraph,wasm::FunctionSig * sig,compiler::SourcePositionTable * spt)3978 WasmWrapperGraphBuilder(Zone* zone, wasm::ModuleEnv* env, JSGraph* jsgraph,
3979 wasm::FunctionSig* sig,
3980 compiler::SourcePositionTable* spt)
3981 : WasmGraphBuilder(jsgraph->isolate(), env, zone, jsgraph,
3982 CodeFactory::CEntry(jsgraph->isolate()),
3983 jsgraph->isolate()->factory()->null_value(), sig, spt),
3984 jsgraph_(jsgraph) {}
3985
BuildAllocateHeapNumberWithValue(Node * value,Node * control)3986 Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
3987 MachineOperatorBuilder* machine = mcgraph()->machine();
3988 CommonOperatorBuilder* common = mcgraph()->common();
3989 // The AllocateHeapNumber builtin does not use the js_context, so we can
3990 // safely pass in Smi zero here.
3991 Callable callable =
3992 Builtins::CallableFor(isolate_, Builtins::kAllocateHeapNumber);
3993 Node* target = jsgraph()->HeapConstant(callable.code());
3994 Node* js_context = jsgraph()->NoContextConstant();
3995 Node* begin_region = graph()->NewNode(
3996 common->BeginRegion(RegionObservability::kNotObservable), *effect_);
3997 if (!allocate_heap_number_operator_.is_set()) {
3998 auto call_descriptor = Linkage::GetStubCallDescriptor(
3999 isolate_, mcgraph()->zone(), callable.descriptor(), 0,
4000 CallDescriptor::kNoFlags, Operator::kNoThrow);
4001 allocate_heap_number_operator_.set(common->Call(call_descriptor));
4002 }
4003 Node* heap_number =
4004 graph()->NewNode(allocate_heap_number_operator_.get(), target,
4005 js_context, begin_region, control);
4006 Node* store =
4007 graph()->NewNode(machine->Store(StoreRepresentation(
4008 MachineRepresentation::kFloat64, kNoWriteBarrier)),
4009 heap_number, BuildHeapNumberValueIndexConstant(),
4010 value, heap_number, control);
4011 Node* finish_region =
4012 graph()->NewNode(common->FinishRegion(), heap_number, store);
4013 *effect_ = finish_region;
4014 return finish_region;
4015 }
4016
BuildChangeSmiToFloat64(Node * value)4017 Node* BuildChangeSmiToFloat64(Node* value) {
4018 return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4019 BuildChangeSmiToInt32(value));
4020 }
4021
BuildTestNotSmi(Node * value)4022 Node* BuildTestNotSmi(Node* value) {
4023 STATIC_ASSERT(kSmiTag == 0);
4024 STATIC_ASSERT(kSmiTagMask == 1);
4025 return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4026 mcgraph()->IntPtrConstant(kSmiTagMask));
4027 }
4028
BuildLoadHeapNumberValue(Node * value,Node * control)4029 Node* BuildLoadHeapNumberValue(Node* value, Node* control) {
4030 return graph()->NewNode(mcgraph()->machine()->Load(MachineType::Float64()),
4031 value, BuildHeapNumberValueIndexConstant(),
4032 graph()->start(), control);
4033 }
4034
BuildHeapNumberValueIndexConstant()4035 Node* BuildHeapNumberValueIndexConstant() {
4036 return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4037 }
4038
BuildChangeInt32ToTagged(Node * value)4039 Node* BuildChangeInt32ToTagged(Node* value) {
4040 MachineOperatorBuilder* machine = mcgraph()->machine();
4041 CommonOperatorBuilder* common = mcgraph()->common();
4042
4043 if (machine->Is64()) {
4044 return BuildChangeInt32ToSmi(value);
4045 }
4046
4047 Node* effect = *effect_;
4048 Node* control = *control_;
4049 Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4050 graph()->start());
4051
4052 Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4053 Node* branch =
4054 graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4055
4056 Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4057 Node* vtrue = BuildAllocateHeapNumberWithValue(
4058 graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4059 Node* etrue = *effect_;
4060
4061 Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4062 Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4063
4064 Node* merge = graph()->NewNode(common->Merge(2), if_true, if_false);
4065 Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4066 vtrue, vfalse, merge);
4067 *effect_ = graph()->NewNode(common->EffectPhi(2), etrue, effect, merge);
4068 *control_ = merge;
4069 return phi;
4070 }
4071
BuildChangeFloat64ToTagged(Node * value)4072 Node* BuildChangeFloat64ToTagged(Node* value) {
4073 MachineOperatorBuilder* machine = mcgraph()->machine();
4074 CommonOperatorBuilder* common = mcgraph()->common();
4075
4076 Node* effect = *effect_;
4077 Node* control = *control_;
4078 Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4079 Node* check_same = graph()->NewNode(
4080 machine->Float64Equal(), value,
4081 graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4082 Node* branch_same = graph()->NewNode(common->Branch(), check_same, control);
4083
4084 Node* if_smi = graph()->NewNode(common->IfTrue(), branch_same);
4085 Node* vsmi;
4086 Node* if_box = graph()->NewNode(common->IfFalse(), branch_same);
4087 Node* vbox;
4088
4089 // We only need to check for -0 if the {value} can potentially contain -0.
4090 Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4091 mcgraph()->Int32Constant(0));
4092 Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4093 check_zero, if_smi);
4094
4095 Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4096 Node* if_notzero = graph()->NewNode(common->IfFalse(), branch_zero);
4097
4098 // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4099 Node* check_negative = graph()->NewNode(
4100 machine->Int32LessThan(),
4101 graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4102 mcgraph()->Int32Constant(0));
4103 Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4104 check_negative, if_zero);
4105
4106 Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4107 Node* if_notnegative = graph()->NewNode(common->IfFalse(), branch_negative);
4108
4109 // We need to create a box for negative 0.
4110 if_smi = graph()->NewNode(common->Merge(2), if_notzero, if_notnegative);
4111 if_box = graph()->NewNode(common->Merge(2), if_box, if_negative);
4112
4113 // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4114 // 32-bit machines we need to deal with potential overflow and fallback to
4115 // boxing.
4116 if (machine->Is64()) {
4117 vsmi = BuildChangeInt32ToSmi(value32);
4118 } else {
4119 Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4120 value32, if_smi);
4121
4122 Node* check_ovf =
4123 graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4124 Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4125 check_ovf, if_smi);
4126
4127 Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4128 if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4129
4130 if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4131 vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4132 }
4133
4134 // Allocate the box for the {value}.
4135 vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4136 Node* ebox = *effect_;
4137
4138 Node* merge = graph()->NewNode(common->Merge(2), if_smi, if_box);
4139 value = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4140 vsmi, vbox, merge);
4141 *effect_ = graph()->NewNode(common->EffectPhi(2), effect, ebox, merge);
4142 *control_ = merge;
4143 return value;
4144 }
4145
AddArgumentNodes(Node ** args,int pos,int param_count,wasm::FunctionSig * sig)4146 int AddArgumentNodes(Node** args, int pos, int param_count,
4147 wasm::FunctionSig* sig) {
4148 // Convert wasm numbers to JS values.
4149 for (int i = 0; i < param_count; ++i) {
4150 Node* param =
4151 Param(i + 1); // Start from index 1 to drop the instance_node.
4152 args[pos++] = ToJS(param, sig->GetParam(i));
4153 }
4154 return pos;
4155 }
4156
BuildJavaScriptToNumber(Node * node,Node * js_context)4157 Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4158 Callable callable = Builtins::CallableFor(isolate_, Builtins::kToNumber);
4159 auto call_descriptor = Linkage::GetStubCallDescriptor(
4160 isolate_, mcgraph()->zone(), callable.descriptor(), 0,
4161 CallDescriptor::kNoFlags, Operator::kNoProperties);
4162 Node* stub_code = jsgraph()->HeapConstant(callable.code());
4163
4164 Node* result =
4165 graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4166 node, js_context, *effect_, *control_);
4167
4168 SetSourcePosition(result, 1);
4169
4170 *effect_ = result;
4171
4172 return result;
4173 }
4174
BuildChangeTaggedToFloat64(Node * value)4175 Node* BuildChangeTaggedToFloat64(Node* value) {
4176 MachineOperatorBuilder* machine = mcgraph()->machine();
4177 CommonOperatorBuilder* common = mcgraph()->common();
4178
4179 Node* check = BuildTestNotSmi(value);
4180 Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), check,
4181 graph()->start());
4182
4183 Node* if_not_smi = graph()->NewNode(common->IfTrue(), branch);
4184
4185 Node* vnot_smi;
4186 Node* check_undefined = graph()->NewNode(machine->WordEqual(), value,
4187 jsgraph()->UndefinedConstant());
4188 Node* branch_undefined = graph()->NewNode(
4189 common->Branch(BranchHint::kFalse), check_undefined, if_not_smi);
4190
4191 Node* if_undefined = graph()->NewNode(common->IfTrue(), branch_undefined);
4192 Node* vundefined =
4193 mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4194
4195 Node* if_not_undefined =
4196 graph()->NewNode(common->IfFalse(), branch_undefined);
4197 Node* vheap_number = BuildLoadHeapNumberValue(value, if_not_undefined);
4198
4199 if_not_smi =
4200 graph()->NewNode(common->Merge(2), if_undefined, if_not_undefined);
4201 vnot_smi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2),
4202 vundefined, vheap_number, if_not_smi);
4203
4204 Node* if_smi = graph()->NewNode(common->IfFalse(), branch);
4205 Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4206
4207 Node* merge = graph()->NewNode(common->Merge(2), if_not_smi, if_smi);
4208 Node* phi =
4209 graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2),
4210 vnot_smi, vfrom_smi, merge);
4211
4212 return phi;
4213 }
4214
ToJS(Node * node,wasm::ValueType type)4215 Node* ToJS(Node* node, wasm::ValueType type) {
4216 switch (type) {
4217 case wasm::kWasmI32:
4218 return BuildChangeInt32ToTagged(node);
4219 case wasm::kWasmS128:
4220 case wasm::kWasmI64:
4221 UNREACHABLE();
4222 case wasm::kWasmF32:
4223 node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4224 node);
4225 return BuildChangeFloat64ToTagged(node);
4226 case wasm::kWasmF64:
4227 return BuildChangeFloat64ToTagged(node);
4228 case wasm::kWasmAnyRef:
4229 return node;
4230 case wasm::kWasmStmt:
4231 return jsgraph()->UndefinedConstant();
4232 default:
4233 UNREACHABLE();
4234 }
4235 }
4236
FromJS(Node * node,Node * js_context,wasm::ValueType type)4237 Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4238 DCHECK_NE(wasm::kWasmStmt, type);
4239
4240 // The parameter is of type AnyRef, we take it as is.
4241 if (type == wasm::kWasmAnyRef) {
4242 return node;
4243 }
4244
4245 // Do a JavaScript ToNumber.
4246 Node* num = BuildJavaScriptToNumber(node, js_context);
4247
4248 // Change representation.
4249 SimplifiedOperatorBuilder simplified(mcgraph()->zone());
4250 num = BuildChangeTaggedToFloat64(num);
4251
4252 switch (type) {
4253 case wasm::kWasmI32: {
4254 num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4255 num);
4256 break;
4257 }
4258 case wasm::kWasmS128:
4259 case wasm::kWasmI64:
4260 UNREACHABLE();
4261 case wasm::kWasmF32:
4262 num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4263 num);
4264 break;
4265 case wasm::kWasmF64:
4266 break;
4267 default:
4268 UNREACHABLE();
4269 }
4270 return num;
4271 }
4272
BuildModifyThreadInWasmFlag(bool new_value)4273 Node* BuildModifyThreadInWasmFlag(bool new_value) {
4274 // TODO(eholk): generate code to modify the thread-local storage directly,
4275 // rather than calling the runtime.
4276 if (!use_trap_handler()) {
4277 return *control_;
4278 }
4279
4280 // Using two functions instead of taking the new value as a parameter saves
4281 // one instruction on each call to set up the parameter.
4282 ExternalReference ref =
4283 new_value ? ExternalReference::wasm_set_thread_in_wasm_flag()
4284 : ExternalReference::wasm_clear_thread_in_wasm_flag();
4285 MachineSignature sig(0, 0, nullptr);
4286 return BuildCCall(
4287 &sig, graph()->NewNode(mcgraph()->common()->ExternalConstant(ref)));
4288 }
4289
BuildLoadInstanceFromExportedFunction(Node * closure)4290 Node* BuildLoadInstanceFromExportedFunction(Node* closure) {
4291 Node* shared = *effect_ = graph()->NewNode(
4292 jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4293 jsgraph()->Int32Constant(JSFunction::kSharedFunctionInfoOffset -
4294 kHeapObjectTag),
4295 *effect_, *control_);
4296 Node* function_data = *effect_ = graph()->NewNode(
4297 jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4298 jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4299 kHeapObjectTag),
4300 *effect_, *control_);
4301 Node* instance = *effect_ = graph()->NewNode(
4302 jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4303 jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4304 kHeapObjectTag),
4305 *effect_, *control_);
4306 return instance;
4307 }
4308
BuildJSToWasmWrapper(uint32_t wasm_func_index,Address call_target)4309 void BuildJSToWasmWrapper(uint32_t wasm_func_index, Address call_target) {
4310 const int wasm_count = static_cast<int>(sig_->parameter_count());
4311
4312 // Build the start and the JS parameter nodes.
4313 Node* start = Start(wasm_count + 5);
4314 *control_ = start;
4315 *effect_ = start;
4316
4317 // Create the js_closure and js_context parameters.
4318 Node* js_closure =
4319 graph()->NewNode(jsgraph()->common()->Parameter(
4320 Linkage::kJSCallClosureParamIndex, "%closure"),
4321 graph()->start());
4322 Node* js_context = graph()->NewNode(
4323 mcgraph()->common()->Parameter(
4324 Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
4325 graph()->start());
4326
4327 // Create the instance_node node to pass as parameter. It is loaded from the
4328 // an actual reference to an instance or a placeholder reference,
4329 // called {WasmExportedFunction} via the {WasmExportedFunctionData}
4330 // structure. since JSToWasm wrappers can be compiled at module compile time
4331 // and patched at instance build time.
4332 DCHECK_NULL(instance_node_);
4333 instance_node_ = BuildLoadInstanceFromExportedFunction(js_closure);
4334
4335 if (!wasm::IsJSCompatibleSignature(sig_)) {
4336 // Throw a TypeError. Use the js_context of the calling javascript
4337 // function (passed as a parameter), such that the generated code is
4338 // js_context independent.
4339 BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4340 nullptr, 0);
4341 Return(jsgraph()->UndefinedConstant());
4342 return;
4343 }
4344
4345 const int args_count = wasm_count + 1; // +1 for wasm_code.
4346 Node** args = Buffer(args_count);
4347 Node** rets;
4348
4349 // Convert JS parameters to wasm numbers.
4350 for (int i = 0; i < wasm_count; ++i) {
4351 Node* param = Param(i + 1);
4352 Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
4353 args[i + 1] = wasm_param;
4354 }
4355
4356 // Set the ThreadInWasm flag before we do the actual call.
4357 BuildModifyThreadInWasmFlag(true);
4358
4359 if (env_ && wasm_func_index < env_->module->num_imported_functions) {
4360 // Call to an imported function.
4361 DCHECK_EQ(kNullAddress, call_target);
4362 BuildImportWasmCall(sig_, args, &rets, wasm::kNoCodePosition,
4363 wasm_func_index);
4364 } else {
4365 // Call to a wasm function defined in this module.
4366 DCHECK_NE(kNullAddress, call_target);
4367 args[0] = mcgraph()->RelocatableIntPtrConstant(
4368 call_target, RelocInfo::JS_TO_WASM_CALL);
4369
4370 BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
4371 kNoRetpoline);
4372 }
4373
4374 // Clear the ThreadInWasm flag.
4375 BuildModifyThreadInWasmFlag(false);
4376
4377 Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
4378 : ToJS(rets[0], sig_->GetReturn());
4379 Return(jsval);
4380 }
4381
BuildWasmToJSWrapper(Handle<JSReceiver> target,int index)4382 bool BuildWasmToJSWrapper(Handle<JSReceiver> target, int index) {
4383 DCHECK(target->IsCallable());
4384
4385 int wasm_count = static_cast<int>(sig_->parameter_count());
4386
4387 // Build the start and the parameter nodes.
4388 CallDescriptor* call_descriptor;
4389 Node* start = Start(wasm_count + 3);
4390 *effect_ = start;
4391 *control_ = start;
4392
4393 instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4394 Node* callables_node = LOAD_INSTANCE_FIELD(ImportedFunctionCallables,
4395 MachineType::TaggedPointer());
4396 Node* callable_node = LOAD_FIXED_ARRAY_SLOT(callables_node, index);
4397 Node* undefined_node = jsgraph()->UndefinedConstant();
4398 Node* native_context =
4399 LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
4400
4401 if (!wasm::IsJSCompatibleSignature(sig_)) {
4402 // Throw a TypeError.
4403 BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
4404 native_context, nullptr, 0);
4405 // We don't need to return a value here, as the runtime call will not
4406 // return anyway (the c entry stub will trigger stack unwinding).
4407 ReturnVoid();
4408 return false;
4409 }
4410
4411 Node** args = Buffer(wasm_count + 9);
4412 Node* call = nullptr;
4413
4414 BuildModifyThreadInWasmFlag(false);
4415
4416 if (target->IsJSFunction()) {
4417 Handle<JSFunction> function = Handle<JSFunction>::cast(target);
4418 FieldAccess field_access = AccessBuilder::ForJSFunctionContext();
4419 Node* function_context = graph()->NewNode(
4420 mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4421 callable_node,
4422 mcgraph()->Int32Constant(field_access.offset - field_access.tag()),
4423 *effect_, *control_);
4424
4425 if (!IsClassConstructor(function->shared()->kind())) {
4426 if (function->shared()->internal_formal_parameter_count() ==
4427 wasm_count) {
4428 int pos = 0;
4429 args[pos++] = callable_node; // target callable.
4430 // Receiver.
4431 if (is_sloppy(function->shared()->language_mode()) &&
4432 !function->shared()->native()) {
4433 Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4434 native_context, Context::GLOBAL_PROXY_INDEX);
4435 args[pos++] = global_proxy;
4436 } else {
4437 args[pos++] = undefined_node;
4438 }
4439
4440 call_descriptor = Linkage::GetJSCallDescriptor(
4441 graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
4442
4443 // Convert wasm numbers to JS values.
4444 pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4445
4446 args[pos++] = undefined_node; // new target
4447 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4448 args[pos++] = function_context;
4449 args[pos++] = *effect_;
4450 args[pos++] = *control_;
4451
4452 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4453 pos, args);
4454 } else if (function->shared()->internal_formal_parameter_count() >= 0) {
4455 Callable callable = CodeFactory::ArgumentAdaptor(isolate_);
4456 int pos = 0;
4457 args[pos++] = jsgraph()->HeapConstant(callable.code());
4458 args[pos++] = callable_node; // target callable
4459 args[pos++] = undefined_node; // new target
4460 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4461 args[pos++] = mcgraph()->Int32Constant(
4462 function->shared()->internal_formal_parameter_count());
4463 // Receiver.
4464 if (is_sloppy(function->shared()->language_mode()) &&
4465 !function->shared()->native()) {
4466 Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4467 native_context, Context::GLOBAL_PROXY_INDEX);
4468 args[pos++] = global_proxy;
4469 } else {
4470 args[pos++] = undefined_node;
4471 }
4472
4473 // Convert wasm numbers to JS values.
4474 pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4475 args[pos++] = function_context;
4476 args[pos++] = *effect_;
4477 args[pos++] = *control_;
4478 call = graph()->NewNode(
4479 mcgraph()->common()->Call(Linkage::GetStubCallDescriptor(
4480 isolate_, mcgraph()->zone(), callable.descriptor(),
4481 1 + wasm_count, CallDescriptor::kNoFlags)),
4482 pos, args);
4483 }
4484 }
4485 }
4486
4487 // We cannot call the target directly, we have to use the Call builtin.
4488 if (!call) {
4489 int pos = 0;
4490 // We cannot call the target directly, we have to use the Call builtin.
4491 Callable callable = CodeFactory::Call(isolate_);
4492 args[pos++] = jsgraph()->HeapConstant(callable.code());
4493 args[pos++] = callable_node;
4494 args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4495 args[pos++] = undefined_node; // receiver
4496
4497 call_descriptor = Linkage::GetStubCallDescriptor(
4498 isolate_, graph()->zone(), callable.descriptor(), wasm_count + 1,
4499 CallDescriptor::kNoFlags);
4500
4501 // Convert wasm numbers to JS values.
4502 pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4503
4504 // The native_context is sufficient here, because all kind of callables
4505 // which depend on the context provide their own context. The context here
4506 // is only needed if the target is a constructor to throw a TypeError, if
4507 // the target is a native function, or if the target is a callable
4508 // JSObject, which can only be constructed by the runtime.
4509 args[pos++] = native_context;
4510 args[pos++] = *effect_;
4511 args[pos++] = *control_;
4512
4513 call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4514 args);
4515 }
4516
4517 *effect_ = call;
4518 SetSourcePosition(call, 0);
4519
4520 // Convert the return value back.
4521 Node* val = sig_->return_count() == 0
4522 ? mcgraph()->Int32Constant(0)
4523 : FromJS(call, native_context, sig_->GetReturn());
4524
4525 BuildModifyThreadInWasmFlag(true);
4526
4527 Return(val);
4528 return true;
4529 }
4530
BuildWasmInterpreterEntry(uint32_t func_index)4531 void BuildWasmInterpreterEntry(uint32_t func_index) {
4532 int param_count = static_cast<int>(sig_->parameter_count());
4533
4534 // Build the start and the parameter nodes.
4535 Node* start = Start(param_count + 3);
4536 *effect_ = start;
4537 *control_ = start;
4538
4539 // Compute size for the argument buffer.
4540 int args_size_bytes = 0;
4541 for (wasm::ValueType type : sig_->parameters()) {
4542 args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
4543 }
4544
4545 // The return value is also passed via this buffer:
4546 DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
4547 // TODO(wasm): Handle multi-value returns.
4548 DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
4549 int return_size_bytes =
4550 sig_->return_count() == 0
4551 ? 0
4552 : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
4553
4554 // Get a stack slot for the arguments.
4555 Node* arg_buffer =
4556 args_size_bytes == 0 && return_size_bytes == 0
4557 ? mcgraph()->IntPtrConstant(0)
4558 : graph()->NewNode(mcgraph()->machine()->StackSlot(
4559 std::max(args_size_bytes, return_size_bytes), 8));
4560
4561 // Now store all our arguments to the buffer.
4562 int offset = 0;
4563
4564 for (int i = 0; i < param_count; ++i) {
4565 wasm::ValueType type = sig_->GetParam(i);
4566 // Start from the parameter with index 1 to drop the instance_node.
4567 *effect_ = graph()->NewNode(GetSafeStoreOperator(offset, type),
4568 arg_buffer, Int32Constant(offset),
4569 Param(i + 1), *effect_, *control_);
4570 offset += wasm::ValueTypes::ElementSizeInBytes(type);
4571 }
4572 DCHECK_EQ(args_size_bytes, offset);
4573
4574 // We are passing the raw arg_buffer here. To the GC and other parts, it
4575 // looks like a Smi (lowest bit not set). In the runtime function however,
4576 // don't call Smi::value on it, but just cast it to a byte pointer.
4577 Node* parameters[] = {
4578 jsgraph()->SmiConstant(func_index), arg_buffer,
4579 };
4580 BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
4581 arraysize(parameters));
4582
4583 // Read back the return value.
4584 if (sig_->return_count() == 0) {
4585 Return(Int32Constant(0));
4586 } else {
4587 // TODO(wasm): Implement multi-return.
4588 DCHECK_EQ(1, sig_->return_count());
4589 MachineType load_rep =
4590 wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
4591 Node* val =
4592 graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
4593 Int32Constant(0), *effect_, *control_);
4594 Return(val);
4595 }
4596
4597 if (ContainsInt64(sig_)) LowerInt64();
4598 }
4599
BuildCWasmEntry()4600 void BuildCWasmEntry() {
4601 // Build the start and the JS parameter nodes.
4602 Node* start = Start(CWasmEntryParameters::kNumParameters + 5);
4603 *control_ = start;
4604 *effect_ = start;
4605
4606 // Create parameter nodes (offset by 1 for the receiver parameter).
4607 Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1);
4608 MachineOperatorBuilder* machine = mcgraph()->machine();
4609 Node* code_obj = graph()->NewNode(
4610 machine->Load(MachineType::Pointer()), foreign_code_obj,
4611 Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag),
4612 *effect_, *control_);
4613 Node* instance_node = Param(CWasmEntryParameters::kWasmInstance + 1);
4614 Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
4615
4616 int wasm_arg_count = static_cast<int>(sig_->parameter_count());
4617 int arg_count = wasm_arg_count + 4; // code, instance_node, control, effect
4618 Node** args = Buffer(arg_count);
4619
4620 int pos = 0;
4621 args[pos++] = code_obj;
4622 args[pos++] = instance_node;
4623
4624 int offset = 0;
4625 for (wasm::ValueType type : sig_->parameters()) {
4626 Node* arg_load =
4627 graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
4628 Int32Constant(offset), *effect_, *control_);
4629 *effect_ = arg_load;
4630 args[pos++] = arg_load;
4631 offset += wasm::ValueTypes::ElementSizeInBytes(type);
4632 }
4633
4634 args[pos++] = *effect_;
4635 args[pos++] = *control_;
4636 DCHECK_EQ(arg_count, pos);
4637
4638 // Call the wasm code.
4639 auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
4640
4641 Node* call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4642 arg_count, args);
4643 *effect_ = call;
4644
4645 // Store the return value.
4646 DCHECK_GE(1, sig_->return_count());
4647 if (sig_->return_count() == 1) {
4648 StoreRepresentation store_rep(
4649 wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
4650 kNoWriteBarrier);
4651 Node* store =
4652 graph()->NewNode(mcgraph()->machine()->Store(store_rep), arg_buffer,
4653 Int32Constant(0), call, *effect_, *control_);
4654 *effect_ = store;
4655 }
4656 Return(jsgraph()->SmiConstant(0));
4657
4658 if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
4659 MachineRepresentation sig_reps[] = {
4660 MachineRepresentation::kWord32, // return value
4661 MachineRepresentation::kTagged, // receiver
4662 MachineRepresentation::kTagged, // arg0 (code)
4663 MachineRepresentation::kTagged // arg1 (buffer)
4664 };
4665 Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
4666 Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
4667 mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
4668 r.LowerGraph();
4669 }
4670 }
4671
jsgraph()4672 JSGraph* jsgraph() { return jsgraph_; }
4673
4674 private:
4675 JSGraph* jsgraph_;
4676 SetOncePointer<const Operator> allocate_heap_number_operator_;
4677 };
4678 } // namespace
4679
CompileJSToWasmWrapper(Isolate * isolate,wasm::WasmModule * module,Address call_target,uint32_t index,wasm::UseTrapHandler use_trap_handler)4680 Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module,
4681 Address call_target, uint32_t index,
4682 wasm::UseTrapHandler use_trap_handler) {
4683 const wasm::WasmFunction* func = &module->functions[index];
4684
4685 //----------------------------------------------------------------------------
4686 // Create the Graph
4687 //----------------------------------------------------------------------------
4688 Zone zone(isolate->allocator(), ZONE_NAME);
4689 Graph graph(&zone);
4690 CommonOperatorBuilder common(&zone);
4691 MachineOperatorBuilder machine(
4692 &zone, MachineType::PointerRepresentation(),
4693 InstructionSelector::SupportedMachineOperatorFlags(),
4694 InstructionSelector::AlignmentRequirements());
4695 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4696
4697 Node* control = nullptr;
4698 Node* effect = nullptr;
4699
4700 wasm::ModuleEnv env(module, use_trap_handler, wasm::kRuntimeExceptionSupport);
4701 WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, func->sig, nullptr);
4702 builder.set_control_ptr(&control);
4703 builder.set_effect_ptr(&effect);
4704 builder.BuildJSToWasmWrapper(index, call_target);
4705
4706 //----------------------------------------------------------------------------
4707 // Run the compilation pipeline.
4708 //----------------------------------------------------------------------------
4709 #ifdef DEBUG
4710 EmbeddedVector<char, 32> func_name;
4711 static unsigned id = 0;
4712 func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++));
4713 #else
4714 Vector<const char> func_name = CStrVector("js-to-wasm");
4715 #endif
4716
4717 OptimizedCompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION);
4718
4719 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4720 OFStream os(stdout);
4721 os << "-- Graph after change lowering -- " << std::endl;
4722 os << AsRPO(graph);
4723 }
4724
4725 // Schedule and compile to machine code.
4726 int params =
4727 static_cast<int>(module->functions[index].sig->parameter_count());
4728 CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
4729 &zone, false, params + 1, CallDescriptor::kNoFlags);
4730
4731 Handle<Code> code =
4732 Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph);
4733 #ifdef ENABLE_DISASSEMBLER
4734 if (FLAG_print_opt_code && !code.is_null()) {
4735 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4736 OFStream os(tracing_scope.file());
4737 code->Disassemble(func_name.start(), os);
4738 }
4739 #endif
4740
4741 if (must_record_function_compilation(isolate)) {
4742 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4743 "%.*s", func_name.length(), func_name.start());
4744 }
4745
4746 return code;
4747 }
4748
4749 namespace {
4750
ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper)4751 void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) {
4752 #ifdef DEBUG
4753 // We expect the only embedded objects to be those originating from
4754 // a snapshot, which are immovable.
4755 DisallowHeapAllocation no_gc;
4756 if (wrapper.is_null()) return;
4757 static constexpr int kAllGCRefs = (1 << (RelocInfo::LAST_GCED_ENUM + 1)) - 1;
4758 for (RelocIterator it(*wrapper, kAllGCRefs); !it.done(); it.next()) {
4759 RelocInfo::Mode mode = it.rinfo()->rmode();
4760 Object* target = nullptr;
4761 switch (mode) {
4762 case RelocInfo::CODE_TARGET:
4763 // this would be either one of the stubs or builtins, because
4764 // we didn't link yet.
4765 target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
4766 break;
4767 case RelocInfo::EMBEDDED_OBJECT:
4768 target = it.rinfo()->target_object();
4769 break;
4770 default:
4771 UNREACHABLE();
4772 }
4773 DCHECK_NOT_NULL(target);
4774 DCHECK(target->IsSmi() || Heap::IsImmovable(HeapObject::cast(target)));
4775 }
4776 #endif
4777 }
4778
4779 } // namespace
4780
CompileWasmToJSWrapper(Isolate * isolate,Handle<JSReceiver> target,wasm::FunctionSig * sig,uint32_t index,wasm::ModuleOrigin origin,wasm::UseTrapHandler use_trap_handler)4781 Handle<Code> CompileWasmToJSWrapper(Isolate* isolate, Handle<JSReceiver> target,
4782 wasm::FunctionSig* sig, uint32_t index,
4783 wasm::ModuleOrigin origin,
4784 wasm::UseTrapHandler use_trap_handler) {
4785 //----------------------------------------------------------------------------
4786 // Create the Graph
4787 //----------------------------------------------------------------------------
4788 Zone zone(isolate->allocator(), ZONE_NAME);
4789 Graph graph(&zone);
4790 CommonOperatorBuilder common(&zone);
4791 MachineOperatorBuilder machine(
4792 &zone, MachineType::PointerRepresentation(),
4793 InstructionSelector::SupportedMachineOperatorFlags(),
4794 InstructionSelector::AlignmentRequirements());
4795 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4796
4797 Node* control = nullptr;
4798 Node* effect = nullptr;
4799
4800 SourcePositionTable* source_position_table =
4801 origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph)
4802 : nullptr;
4803
4804 wasm::ModuleEnv env(nullptr, use_trap_handler,
4805 wasm::kRuntimeExceptionSupport);
4806
4807 WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig,
4808 source_position_table);
4809 builder.set_control_ptr(&control);
4810 builder.set_effect_ptr(&effect);
4811 builder.BuildWasmToJSWrapper(target, index);
4812
4813 #ifdef DEBUG
4814 EmbeddedVector<char, 32> func_name;
4815 static unsigned id = 0;
4816 func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++));
4817 #else
4818 Vector<const char> func_name = CStrVector("wasm-to-js");
4819 #endif
4820
4821 OptimizedCompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION);
4822
4823 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4824 OFStream os(stdout);
4825 os << "-- Graph after change lowering -- " << std::endl;
4826 os << AsRPO(graph);
4827 }
4828
4829 // Schedule and compile to machine code.
4830 CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4831 if (machine.Is32()) {
4832 incoming = GetI32WasmCallDescriptor(&zone, incoming);
4833 }
4834
4835 Handle<Code> code = Pipeline::GenerateCodeForTesting(
4836 &info, isolate, incoming, &graph, nullptr, source_position_table);
4837 ValidateImportWrapperReferencesImmovables(code);
4838
4839 #ifdef ENABLE_DISASSEMBLER
4840 if (FLAG_print_opt_code && !code.is_null()) {
4841 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4842 OFStream os(tracing_scope.file());
4843 code->Disassemble(func_name.start(), os);
4844 }
4845 #endif
4846
4847 if (must_record_function_compilation(isolate)) {
4848 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4849 "%.*s", func_name.length(), func_name.start());
4850 }
4851
4852 return code;
4853 }
4854
CompileWasmInterpreterEntry(Isolate * isolate,uint32_t func_index,wasm::FunctionSig * sig)4855 Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index,
4856 wasm::FunctionSig* sig) {
4857 //----------------------------------------------------------------------------
4858 // Create the Graph
4859 //----------------------------------------------------------------------------
4860 Zone zone(isolate->allocator(), ZONE_NAME);
4861 Graph graph(&zone);
4862 CommonOperatorBuilder common(&zone);
4863 MachineOperatorBuilder machine(
4864 &zone, MachineType::PointerRepresentation(),
4865 InstructionSelector::SupportedMachineOperatorFlags(),
4866 InstructionSelector::AlignmentRequirements());
4867 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4868
4869 Node* control = nullptr;
4870 Node* effect = nullptr;
4871
4872 WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr);
4873 builder.set_control_ptr(&control);
4874 builder.set_effect_ptr(&effect);
4875 builder.BuildWasmInterpreterEntry(func_index);
4876
4877 Handle<Code> code = Handle<Code>::null();
4878 {
4879 // Schedule and compile to machine code.
4880 CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4881 if (machine.Is32()) {
4882 incoming = GetI32WasmCallDescriptor(&zone, incoming);
4883 }
4884 #ifdef DEBUG
4885 EmbeddedVector<char, 32> func_name;
4886 func_name.Truncate(
4887 SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
4888 #else
4889 Vector<const char> func_name = CStrVector("wasm-interpreter-entry");
4890 #endif
4891
4892 OptimizedCompilationInfo info(func_name, &zone,
4893 Code::WASM_INTERPRETER_ENTRY);
4894
4895 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4896 OFStream os(stdout);
4897 os << "-- Wasm interpreter entry graph -- " << std::endl;
4898 os << AsRPO(graph);
4899 }
4900
4901 code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph,
4902 nullptr);
4903 #ifdef ENABLE_DISASSEMBLER
4904 if (FLAG_print_opt_code && !code.is_null()) {
4905 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4906 OFStream os(tracing_scope.file());
4907 code->Disassemble(func_name.start(), os);
4908 }
4909 #endif
4910
4911 if (must_record_function_compilation(isolate)) {
4912 RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4913 "%.*s", func_name.length(), func_name.start());
4914 }
4915 }
4916
4917 return code;
4918 }
4919
CompileCWasmEntry(Isolate * isolate,wasm::FunctionSig * sig)4920 Handle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
4921 Zone zone(isolate->allocator(), ZONE_NAME);
4922 Graph graph(&zone);
4923 CommonOperatorBuilder common(&zone);
4924 MachineOperatorBuilder machine(
4925 &zone, MachineType::PointerRepresentation(),
4926 InstructionSelector::SupportedMachineOperatorFlags(),
4927 InstructionSelector::AlignmentRequirements());
4928 JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4929
4930 Node* control = nullptr;
4931 Node* effect = nullptr;
4932
4933 WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr);
4934 builder.set_control_ptr(&control);
4935 builder.set_effect_ptr(&effect);
4936 builder.BuildCWasmEntry();
4937
4938 // Schedule and compile to machine code.
4939 CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
4940 &zone, false, CWasmEntryParameters::kNumParameters + 1,
4941 CallDescriptor::kNoFlags);
4942
4943 // Build a name in the form "c-wasm-entry:<params>:<returns>".
4944 static constexpr size_t kMaxNameLen = 128;
4945 char debug_name[kMaxNameLen] = "c-wasm-entry:";
4946 size_t name_len = strlen(debug_name);
4947 auto append_name_char = [&](char c) {
4948 if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c;
4949 };
4950 for (wasm::ValueType t : sig->parameters()) {
4951 append_name_char(wasm::ValueTypes::ShortNameOf(t));
4952 }
4953 append_name_char(':');
4954 for (wasm::ValueType t : sig->returns()) {
4955 append_name_char(wasm::ValueTypes::ShortNameOf(t));
4956 }
4957 debug_name[name_len] = '\0';
4958 Vector<const char> debug_name_vec(debug_name, name_len);
4959
4960 OptimizedCompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY);
4961
4962 if (info.trace_turbo_graph_enabled()) { // Simple textual RPO.
4963 OFStream os(stdout);
4964 os << "-- C Wasm entry graph -- " << std::endl;
4965 os << AsRPO(graph);
4966 }
4967
4968 Handle<Code> code =
4969 Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph);
4970 #ifdef ENABLE_DISASSEMBLER
4971 if (FLAG_print_opt_code && !code.is_null()) {
4972 CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4973 OFStream os(tracing_scope.file());
4974 code->Disassemble(debug_name, os);
4975 }
4976 #endif
4977
4978 return code;
4979 }
4980
WasmCompilationData(wasm::RuntimeExceptionSupport runtime_exception_support)4981 WasmCompilationData::WasmCompilationData(
4982 wasm::RuntimeExceptionSupport runtime_exception_support)
4983 : protected_instructions_(
4984 new std::vector<trap_handler::ProtectedInstructionData>()),
4985 runtime_exception_support_(runtime_exception_support) {}
4986
AddProtectedInstruction(uint32_t instr_offset,uint32_t landing_offset)4987 void WasmCompilationData::AddProtectedInstruction(uint32_t instr_offset,
4988 uint32_t landing_offset) {
4989 protected_instructions_->emplace_back(
4990 trap_handler::ProtectedInstructionData{instr_offset, landing_offset});
4991 }
4992
TurbofanWasmCompilationUnit(wasm::WasmCompilationUnit * wasm_unit)4993 TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
4994 wasm::WasmCompilationUnit* wasm_unit)
4995 : wasm_unit_(wasm_unit),
4996 wasm_compilation_data_(wasm_unit->env_->runtime_exception_support) {}
4997
4998 // Clears unique_ptrs, but (part of) the type is forward declared in the header.
4999 TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5000
BuildGraphForWasmFunction(double * decode_ms)5001 SourcePositionTable* TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5002 double* decode_ms) {
5003 base::ElapsedTimer decode_timer;
5004 if (FLAG_trace_wasm_decode_time) {
5005 decode_timer.Start();
5006 }
5007
5008 // Create a TF graph during decoding.
5009 SourcePositionTable* source_position_table =
5010 new (mcgraph_->zone()) SourcePositionTable(mcgraph_->graph());
5011 // We get the handle for {null_value()} directly from the isolate although we
5012 // are on a background task because the handle is stored in the isolate
5013 // anyways, and it is immortal and immovable.
5014 WasmGraphBuilder builder(wasm_unit_->isolate_, wasm_unit_->env_,
5015 mcgraph_->zone(), mcgraph_, wasm_unit_->centry_stub_,
5016 wasm_unit_->isolate_->factory()->null_value(),
5017 wasm_unit_->func_body_.sig, source_position_table);
5018 graph_construction_result_ = wasm::BuildTFGraph(
5019 wasm_unit_->isolate_->allocator(), &builder, wasm_unit_->func_body_);
5020 if (graph_construction_result_.failed()) {
5021 if (FLAG_trace_wasm_compiler) {
5022 OFStream os(stdout);
5023 os << "Compilation failed: " << graph_construction_result_.error_msg()
5024 << std::endl;
5025 }
5026 return nullptr;
5027 }
5028
5029 builder.LowerInt64();
5030
5031 if (builder.has_simd() &&
5032 (!CpuFeatures::SupportsWasmSimd128() || wasm_unit_->lower_simd_)) {
5033 SimdScalarLowering(
5034 mcgraph_,
5035 CreateMachineSignature(mcgraph_->zone(), wasm_unit_->func_body_.sig))
5036 .LowerGraph();
5037 }
5038
5039 if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5040 wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5041 PrintRawWasmCode(wasm_unit_->isolate_->allocator(), wasm_unit_->func_body_,
5042 wasm_unit_->env_->module, wasm::kPrintLocals);
5043 }
5044 if (FLAG_trace_wasm_decode_time) {
5045 *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5046 }
5047 return source_position_table;
5048 }
5049
5050 namespace {
GetDebugName(Zone * zone,wasm::WasmName name,int index)5051 Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) {
5052 if (!name.is_empty()) {
5053 return name;
5054 }
5055 #ifdef DEBUG
5056 constexpr int kBufferLength = 15;
5057
5058 EmbeddedVector<char, kBufferLength> name_vector;
5059 int name_len = SNPrintF(name_vector, "wasm#%d", index);
5060 DCHECK(name_len > 0 && name_len < name_vector.length());
5061
5062 char* index_name = zone->NewArray<char>(name_len);
5063 memcpy(index_name, name_vector.start(), name_len);
5064 return Vector<const char>(index_name, name_len);
5065 #else
5066 return {};
5067 #endif
5068 }
5069
5070 } // namespace
5071
ExecuteCompilation()5072 void TurbofanWasmCompilationUnit::ExecuteCompilation() {
5073 double decode_ms = 0;
5074 size_t node_count = 0;
5075
5076 // Scope for the {graph_zone}.
5077 {
5078 Zone graph_zone(wasm_unit_->isolate_->allocator(), ZONE_NAME);
5079 mcgraph_ = new (&graph_zone)
5080 MachineGraph(new (&graph_zone) Graph(&graph_zone),
5081 new (&graph_zone) CommonOperatorBuilder(&graph_zone),
5082 new (&graph_zone) MachineOperatorBuilder(
5083 &graph_zone, MachineType::PointerRepresentation(),
5084 InstructionSelector::SupportedMachineOperatorFlags(),
5085 InstructionSelector::AlignmentRequirements()));
5086 SourcePositionTable* source_positions =
5087 BuildGraphForWasmFunction(&decode_ms);
5088
5089 if (graph_construction_result_.failed()) {
5090 ok_ = false;
5091 return;
5092 }
5093
5094 base::ElapsedTimer pipeline_timer;
5095 if (FLAG_trace_wasm_decode_time) {
5096 node_count = mcgraph_->graph()->NodeCount();
5097 pipeline_timer.Start();
5098 }
5099
5100 compilation_zone_.reset(
5101 new Zone(wasm_unit_->isolate_->allocator(), ZONE_NAME));
5102
5103 // Run the compiler pipeline to generate machine code.
5104 auto call_descriptor = GetWasmCallDescriptor(compilation_zone_.get(),
5105 wasm_unit_->func_body_.sig);
5106 if (mcgraph_->machine()->Is32()) {
5107 call_descriptor =
5108 GetI32WasmCallDescriptor(compilation_zone_.get(), call_descriptor);
5109 }
5110 info_.reset(new OptimizedCompilationInfo(
5111 GetDebugName(compilation_zone_.get(), wasm_unit_->func_name_,
5112 wasm_unit_->func_index_),
5113 compilation_zone_.get(), Code::WASM_FUNCTION));
5114
5115 NodeOriginTable* node_origins = info_->trace_turbo_graph_enabled()
5116 ? new (&graph_zone)
5117 NodeOriginTable(mcgraph_->graph())
5118 : nullptr;
5119
5120 job_.reset(Pipeline::NewWasmCompilationJob(
5121 info_.get(), wasm_unit_->isolate_, mcgraph_, call_descriptor,
5122 source_positions, node_origins, &wasm_compilation_data_,
5123 wasm_unit_->env_->module->origin()));
5124 ok_ = job_->ExecuteJob() == CompilationJob::SUCCEEDED;
5125 // TODO(bradnelson): Improve histogram handling of size_t.
5126 wasm_unit_->counters_->wasm_compile_function_peak_memory_bytes()->AddSample(
5127 static_cast<int>(mcgraph_->graph()->zone()->allocation_size()));
5128
5129 if (FLAG_trace_wasm_decode_time) {
5130 double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
5131 PrintF(
5132 "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
5133 "%0.3f ms pipeline\n",
5134 static_cast<unsigned>(wasm_unit_->func_body_.end -
5135 wasm_unit_->func_body_.start),
5136 decode_ms, node_count, pipeline_ms);
5137 }
5138 // The graph zone is about to get out of scope. Avoid invalid references.
5139 mcgraph_ = nullptr;
5140 }
5141
5142 // Record the memory cost this unit places on the system until
5143 // it is finalized.
5144 wasm_unit_->memory_cost_ = job_->AllocatedMemory();
5145 }
5146
FinishCompilation(wasm::ErrorThrower * thrower)5147 wasm::WasmCode* TurbofanWasmCompilationUnit::FinishCompilation(
5148 wasm::ErrorThrower* thrower) {
5149 if (!ok_) {
5150 if (graph_construction_result_.failed()) {
5151 // Add the function as another context for the exception.
5152 EmbeddedVector<char, 128> message;
5153 if (wasm_unit_->func_name_.start() == nullptr) {
5154 SNPrintF(message, "Compiling wasm function #%d failed",
5155 wasm_unit_->func_index_);
5156 } else {
5157 wasm::TruncatedUserString<> trunc_name(wasm_unit_->func_name_);
5158 SNPrintF(message, "Compiling wasm function #%d:%.*s failed",
5159 wasm_unit_->func_index_, trunc_name.length(),
5160 trunc_name.start());
5161 }
5162 thrower->CompileFailed(message.start(), graph_construction_result_);
5163 }
5164
5165 return nullptr;
5166 }
5167 base::ElapsedTimer codegen_timer;
5168 if (FLAG_trace_wasm_decode_time) {
5169 codegen_timer.Start();
5170 }
5171
5172 if (job_->FinalizeJob(wasm_unit_->isolate_) != CompilationJob::SUCCEEDED) {
5173 return nullptr;
5174 }
5175
5176 // TODO(mtrofin): when we crystalize a design in lieu of WasmCodeDesc, that
5177 // works for both wasm and non-wasm, we can simplify AddCode to just take
5178 // that as a parameter.
5179 const CodeDesc& desc = job_->compilation_info()->wasm_code_desc()->code_desc;
5180 wasm::WasmCode* code = wasm_unit_->native_module_->AddCode(
5181 desc, job_->compilation_info()->wasm_code_desc()->frame_slot_count,
5182 wasm_unit_->func_index_,
5183 job_->compilation_info()->wasm_code_desc()->safepoint_table_offset,
5184 job_->compilation_info()->wasm_code_desc()->handler_table_offset,
5185 wasm_compilation_data_.ReleaseProtectedInstructions(),
5186 job_->compilation_info()->wasm_code_desc()->source_positions_table,
5187 wasm::WasmCode::kTurbofan);
5188 if (!code) return code;
5189 if (FLAG_trace_wasm_decode_time) {
5190 double codegen_ms = codegen_timer.Elapsed().InMillisecondsF();
5191 PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n",
5192 static_cast<unsigned>(wasm_unit_->func_body_.end -
5193 wasm_unit_->func_body_.start),
5194 codegen_ms);
5195 }
5196
5197 return code;
5198 }
5199
5200 namespace {
5201 // Helper for allocating either an GP or FP reg, or the next stack slot.
5202 class LinkageLocationAllocator {
5203 public:
5204 template <size_t kNumGpRegs, size_t kNumFpRegs>
LinkageLocationAllocator(const Register (& gp)[kNumGpRegs],const DoubleRegister (& fp)[kNumFpRegs])5205 constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
5206 const DoubleRegister (&fp)[kNumFpRegs])
5207 : allocator_(wasm::LinkageAllocator(gp, fp)) {}
5208
Next(MachineRepresentation type)5209 LinkageLocation Next(MachineRepresentation type) {
5210 MachineType mach_type = MachineType::TypeForRepresentation(type);
5211 if (type == MachineRepresentation::kFloat32 ||
5212 type == MachineRepresentation::kFloat64) {
5213 if (allocator_.has_more_fp_regs()) {
5214 DoubleRegister reg = allocator_.NextFpReg();
5215 #if V8_TARGET_ARCH_ARM
5216 // Allocate floats using a double register, but modify the code to
5217 // reflect how ARM FP registers alias.
5218 // TODO(bbudge) Modify wasm linkage to allow use of all float regs.
5219 if (type == MachineRepresentation::kFloat32) {
5220 int float_reg_code = reg.code() * 2;
5221 DCHECK_GT(RegisterConfiguration::kMaxFPRegisters, float_reg_code);
5222 return LinkageLocation::ForRegister(
5223 DoubleRegister::from_code(float_reg_code).code(), mach_type);
5224 }
5225 #endif
5226 return LinkageLocation::ForRegister(reg.code(), mach_type);
5227 }
5228 } else if (allocator_.has_more_gp_regs()) {
5229 return LinkageLocation::ForRegister(allocator_.NextGpReg().code(),
5230 mach_type);
5231 }
5232 // Cannot use register; use stack slot.
5233 int index = -1 - allocator_.NextStackSlot(type);
5234 return LinkageLocation::ForCallerFrameSlot(index, mach_type);
5235 }
5236
SetStackOffset(int offset)5237 void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
NumStackSlots() const5238 int NumStackSlots() const { return allocator_.NumStackSlots(); }
5239
5240 private:
5241 wasm::LinkageAllocator allocator_;
5242 };
5243 } // namespace
5244
5245 // General code uses the above configuration data.
GetWasmCallDescriptor(Zone * zone,wasm::FunctionSig * fsig,WasmGraphBuilder::UseRetpoline use_retpoline)5246 CallDescriptor* GetWasmCallDescriptor(
5247 Zone* zone, wasm::FunctionSig* fsig,
5248 WasmGraphBuilder::UseRetpoline use_retpoline) {
5249 // The '+ 1' here is to accomodate the instance object as first parameter.
5250 LocationSignature::Builder locations(zone, fsig->return_count(),
5251 fsig->parameter_count() + 1);
5252
5253 // Add register and/or stack parameter(s).
5254 LinkageLocationAllocator params(wasm::kGpParamRegisters,
5255 wasm::kFpParamRegisters);
5256
5257 // The instance object.
5258 locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
5259
5260 const int parameter_count = static_cast<int>(fsig->parameter_count());
5261 for (int i = 0; i < parameter_count; i++) {
5262 MachineRepresentation param =
5263 wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
5264 auto l = params.Next(param);
5265 locations.AddParam(l);
5266 }
5267
5268 // Add return location(s).
5269 LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5270 wasm::kFpReturnRegisters);
5271 rets.SetStackOffset(params.NumStackSlots());
5272
5273 const int return_count = static_cast<int>(locations.return_count_);
5274 for (int i = 0; i < return_count; i++) {
5275 MachineRepresentation ret =
5276 wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
5277 auto l = rets.Next(ret);
5278 locations.AddReturn(l);
5279 }
5280
5281 const RegList kCalleeSaveRegisters = 0;
5282 const RegList kCalleeSaveFPRegisters = 0;
5283
5284 // The target for wasm calls is always a code object.
5285 MachineType target_type = MachineType::Pointer();
5286 LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
5287
5288 CallDescriptor::Kind kind = CallDescriptor::kCallWasmFunction;
5289
5290 CallDescriptor::Flags flags =
5291 use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
5292 return new (zone) CallDescriptor( // --
5293 kind, // kind
5294 target_type, // target MachineType
5295 target_loc, // target location
5296 locations.Build(), // location_sig
5297 params.NumStackSlots(), // stack_parameter_count
5298 compiler::Operator::kNoProperties, // properties
5299 kCalleeSaveRegisters, // callee-saved registers
5300 kCalleeSaveFPRegisters, // callee-saved fp regs
5301 flags, // flags
5302 "wasm-call", // debug name
5303 0, // allocatable registers
5304 rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
5305 }
5306
5307 namespace {
ReplaceTypeInCallDescriptorWith(Zone * zone,CallDescriptor * call_descriptor,size_t num_replacements,MachineType input_type,MachineRepresentation output_type)5308 CallDescriptor* ReplaceTypeInCallDescriptorWith(
5309 Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
5310 MachineType input_type, MachineRepresentation output_type) {
5311 size_t parameter_count = call_descriptor->ParameterCount();
5312 size_t return_count = call_descriptor->ReturnCount();
5313 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5314 if (call_descriptor->GetParameterType(i) == input_type) {
5315 parameter_count += num_replacements - 1;
5316 }
5317 }
5318 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5319 if (call_descriptor->GetReturnType(i) == input_type) {
5320 return_count += num_replacements - 1;
5321 }
5322 }
5323 if (parameter_count == call_descriptor->ParameterCount() &&
5324 return_count == call_descriptor->ReturnCount()) {
5325 return call_descriptor;
5326 }
5327
5328 LocationSignature::Builder locations(zone, return_count, parameter_count);
5329
5330 LinkageLocationAllocator params(wasm::kGpParamRegisters,
5331 wasm::kFpParamRegisters);
5332 for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5333 if (call_descriptor->GetParameterType(i) == input_type) {
5334 for (size_t j = 0; j < num_replacements; j++) {
5335 locations.AddParam(params.Next(output_type));
5336 }
5337 } else {
5338 locations.AddParam(
5339 params.Next(call_descriptor->GetParameterType(i).representation()));
5340 }
5341 }
5342
5343 LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5344 wasm::kFpReturnRegisters);
5345 rets.SetStackOffset(params.NumStackSlots());
5346 for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5347 if (call_descriptor->GetReturnType(i) == input_type) {
5348 for (size_t j = 0; j < num_replacements; j++) {
5349 locations.AddReturn(rets.Next(output_type));
5350 }
5351 } else {
5352 locations.AddReturn(
5353 rets.Next(call_descriptor->GetReturnType(i).representation()));
5354 }
5355 }
5356
5357 return new (zone) CallDescriptor( // --
5358 call_descriptor->kind(), // kind
5359 call_descriptor->GetInputType(0), // target MachineType
5360 call_descriptor->GetInputLocation(0), // target location
5361 locations.Build(), // location_sig
5362 params.NumStackSlots(), // stack_parameter_count
5363 call_descriptor->properties(), // properties
5364 call_descriptor->CalleeSavedRegisters(), // callee-saved registers
5365 call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
5366 call_descriptor->flags(), // flags
5367 call_descriptor->debug_name(), // debug name
5368 call_descriptor->AllocatableRegisters(), // allocatable registers
5369 rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
5370 }
5371 } // namespace
5372
GetI32WasmCallDescriptor(Zone * zone,CallDescriptor * call_descriptor)5373 CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
5374 CallDescriptor* call_descriptor) {
5375 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
5376 MachineType::Int64(),
5377 MachineRepresentation::kWord32);
5378 }
5379
GetI32WasmCallDescriptorForSimd(Zone * zone,CallDescriptor * call_descriptor)5380 CallDescriptor* GetI32WasmCallDescriptorForSimd(
5381 Zone* zone, CallDescriptor* call_descriptor) {
5382 return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
5383 MachineType::Simd128(),
5384 MachineRepresentation::kWord32);
5385 }
5386
5387 #undef WASM_64
5388 #undef FATAL_UNSUPPORTED_OPCODE
5389 #undef WASM_INSTANCE_OBJECT_OFFSET
5390 #undef LOAD_INSTANCE_FIELD
5391 #undef LOAD_FIXED_ARRAY_SLOT
5392
5393 } // namespace compiler
5394 } // namespace internal
5395 } // namespace v8
5396