1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/wasm-compiler.h"
6 
7 #include <memory>
8 
9 #include "src/assembler-inl.h"
10 #include "src/base/optional.h"
11 #include "src/base/platform/elapsed-timer.h"
12 #include "src/base/platform/platform.h"
13 #include "src/base/v8-fallthrough.h"
14 #include "src/builtins/builtins.h"
15 #include "src/code-factory.h"
16 #include "src/compiler.h"
17 #include "src/compiler/access-builder.h"
18 #include "src/compiler/code-generator.h"
19 #include "src/compiler/common-operator.h"
20 #include "src/compiler/compiler-source-position-table.h"
21 #include "src/compiler/diamond.h"
22 #include "src/compiler/graph-visualizer.h"
23 #include "src/compiler/graph.h"
24 #include "src/compiler/instruction-selector.h"
25 #include "src/compiler/int64-lowering.h"
26 #include "src/compiler/js-graph.h"
27 #include "src/compiler/js-operator.h"
28 #include "src/compiler/linkage.h"
29 #include "src/compiler/machine-operator.h"
30 #include "src/compiler/node-matchers.h"
31 #include "src/compiler/node-origin-table.h"
32 #include "src/compiler/pipeline.h"
33 #include "src/compiler/simd-scalar-lowering.h"
34 #include "src/compiler/zone-stats.h"
35 #include "src/heap/factory.h"
36 #include "src/isolate-inl.h"
37 #include "src/log-inl.h"
38 #include "src/trap-handler/trap-handler.h"
39 #include "src/wasm/function-body-decoder.h"
40 #include "src/wasm/function-compiler.h"
41 #include "src/wasm/memory-tracing.h"
42 #include "src/wasm/wasm-code-manager.h"
43 #include "src/wasm/wasm-limits.h"
44 #include "src/wasm/wasm-linkage.h"
45 #include "src/wasm/wasm-module.h"
46 #include "src/wasm/wasm-objects-inl.h"
47 #include "src/wasm/wasm-opcodes.h"
48 #include "src/wasm/wasm-text.h"
49 
50 namespace v8 {
51 namespace internal {
52 namespace compiler {
53 
54 // TODO(titzer): pull WASM_64 up to a common header.
55 #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
56 #define WASM_64 1
57 #else
58 #define WASM_64 0
59 #endif
60 
61 #define FATAL_UNSUPPORTED_OPCODE(opcode)        \
62   FATAL("Unsupported opcode 0x%x:%s", (opcode), \
63         wasm::WasmOpcodes::OpcodeName(opcode));
64 
65 #define WASM_INSTANCE_OBJECT_OFFSET(name) \
66   (WasmInstanceObject::k##name##Offset - kHeapObjectTag)
67 
68 #define LOAD_INSTANCE_FIELD(name, type)                                      \
69   graph()->NewNode(                                                          \
70       mcgraph()->machine()->Load(type), instance_node_.get(),                \
71       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(name)), *effect_, \
72       *control_)
73 
74 #define LOAD_FIXED_ARRAY_SLOT(array_node, index)                              \
75   graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),  \
76                    array_node,                                                \
77                    mcgraph()->Int32Constant(FixedArrayOffsetMinusTag(index)), \
78                    *effect_, *control_);
79 
FixedArrayOffsetMinusTag(uint32_t index)80 int FixedArrayOffsetMinusTag(uint32_t index) {
81   auto access = AccessBuilder::ForFixedArraySlot(index);
82   return access.offset - access.tag();
83 }
84 
85 namespace {
86 
87 constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2;
88 
MergeControlToEnd(MachineGraph * mcgraph,Node * node)89 void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
90   Graph* g = mcgraph->graph();
91   if (g->end()) {
92     NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
93   } else {
94     g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
95   }
96 }
97 
ContainsSimd(wasm::FunctionSig * sig)98 bool ContainsSimd(wasm::FunctionSig* sig) {
99   for (auto type : sig->all()) {
100     if (type == wasm::kWasmS128) return true;
101   }
102   return false;
103 }
104 
ContainsInt64(wasm::FunctionSig * sig)105 bool ContainsInt64(wasm::FunctionSig* sig) {
106   for (auto type : sig->all()) {
107     if (type == wasm::kWasmI64) return true;
108   }
109   return false;
110 }
111 }  // namespace
112 
WasmGraphBuilder(Isolate * isolate,wasm::ModuleEnv * env,Zone * zone,MachineGraph * mcgraph,Handle<Code> centry_stub,Handle<Oddball> anyref_null,wasm::FunctionSig * sig,compiler::SourcePositionTable * source_position_table)113 WasmGraphBuilder::WasmGraphBuilder(
114     Isolate* isolate, wasm::ModuleEnv* env, Zone* zone, MachineGraph* mcgraph,
115     Handle<Code> centry_stub, Handle<Oddball> anyref_null,
116     wasm::FunctionSig* sig,
117     compiler::SourcePositionTable* source_position_table)
118     : isolate_(isolate),
119       zone_(zone),
120       mcgraph_(mcgraph),
121       env_(env),
122       centry_stub_(centry_stub),
123       anyref_null_(anyref_null),
124       cur_buffer_(def_buffer_),
125       cur_bufsize_(kDefaultBufferSize),
126       has_simd_(ContainsSimd(sig)),
127       untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
128       sig_(sig),
129       source_position_table_(source_position_table) {
130   DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
131   DCHECK_NOT_NULL(mcgraph_);
132 }
133 
Error()134 Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
135 
Start(unsigned params)136 Node* WasmGraphBuilder::Start(unsigned params) {
137   Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
138   graph()->SetStart(start);
139   return start;
140 }
141 
Param(unsigned index)142 Node* WasmGraphBuilder::Param(unsigned index) {
143   return graph()->NewNode(mcgraph()->common()->Parameter(index),
144                           graph()->start());
145 }
146 
Loop(Node * entry)147 Node* WasmGraphBuilder::Loop(Node* entry) {
148   return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
149 }
150 
Terminate(Node * effect,Node * control)151 Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) {
152   Node* terminate =
153       graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
154   MergeControlToEnd(mcgraph(), terminate);
155   return terminate;
156 }
157 
IsPhiWithMerge(Node * phi,Node * merge)158 bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
159   return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
160          NodeProperties::GetControlInput(phi) == merge;
161 }
162 
ThrowsException(Node * node,Node ** if_success,Node ** if_exception)163 bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
164                                        Node** if_exception) {
165   if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
166     return false;
167   }
168 
169   *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
170   *if_exception =
171       graph()->NewNode(mcgraph()->common()->IfException(), node, node);
172 
173   return true;
174 }
175 
AppendToMerge(Node * merge,Node * from)176 void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
177   DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
178   merge->AppendInput(mcgraph()->zone(), from);
179   int new_size = merge->InputCount();
180   NodeProperties::ChangeOp(
181       merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
182 }
183 
AppendToPhi(Node * phi,Node * from)184 void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
185   DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
186   int new_size = phi->InputCount();
187   phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
188   NodeProperties::ChangeOp(
189       phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
190 }
191 
Merge(unsigned count,Node ** controls)192 Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
193   return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
194 }
195 
Phi(wasm::ValueType type,unsigned count,Node ** vals,Node * control)196 Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
197                             Node* control) {
198   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
199   Node** buf = Realloc(vals, count, count + 1);
200   buf[count] = control;
201   return graph()->NewNode(
202       mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
203                                count),
204       count + 1, buf);
205 }
206 
EffectPhi(unsigned count,Node ** effects,Node * control)207 Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
208                                   Node* control) {
209   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
210   Node** buf = Realloc(effects, count, count + 1);
211   buf[count] = control;
212   return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
213                           buf);
214 }
215 
RefNull()216 Node* WasmGraphBuilder::RefNull() {
217   if (!anyref_null_node_.is_set()) {
218     anyref_null_node_.set(
219         graph()->NewNode(mcgraph()->common()->HeapConstant(anyref_null_)));
220   }
221   return anyref_null_node_.get();
222 }
223 
CEntryStub()224 Node* WasmGraphBuilder::CEntryStub() {
225   if (!centry_stub_node_.is_set()) {
226     centry_stub_node_.set(
227         graph()->NewNode(mcgraph()->common()->HeapConstant(centry_stub_)));
228   }
229   return centry_stub_node_.get();
230 }
231 
NoContextConstant()232 Node* WasmGraphBuilder::NoContextConstant() {
233   // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
234   return mcgraph()->IntPtrConstant(0);
235 }
236 
Uint32Constant(uint32_t value)237 Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
238   return mcgraph()->Uint32Constant(value);
239 }
240 
Int32Constant(int32_t value)241 Node* WasmGraphBuilder::Int32Constant(int32_t value) {
242   return mcgraph()->Int32Constant(value);
243 }
244 
Int64Constant(int64_t value)245 Node* WasmGraphBuilder::Int64Constant(int64_t value) {
246   return mcgraph()->Int64Constant(value);
247 }
248 
IntPtrConstant(intptr_t value)249 Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
250   return mcgraph()->IntPtrConstant(value);
251 }
252 
StackCheck(wasm::WasmCodePosition position,Node ** effect,Node ** control)253 void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
254                                   Node** effect, Node** control) {
255   DCHECK_NOT_NULL(env_);  // Wrappers don't get stack checks.
256   if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
257     return;
258   }
259   if (effect == nullptr) effect = effect_;
260   if (control == nullptr) control = control_;
261 
262   Node* limit =
263       graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
264                        mcgraph()->ExternalConstant(
265                            ExternalReference::address_of_stack_limit(isolate_)),
266                        mcgraph()->IntPtrConstant(0), *effect, *control);
267   *effect = limit;
268   Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
269 
270   Node* check =
271       graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
272 
273   Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
274   stack_check.Chain(*control);
275 
276   if (stack_check_call_operator_ == nullptr) {
277     // Build and cache the stack check call operator and the constant
278     // representing the stackcheck code.
279     Handle<Code> code = BUILTIN_CODE(isolate_, WasmStackGuard);
280     CallInterfaceDescriptor idesc = WasmRuntimeCallDescriptor(isolate_);
281     auto call_descriptor = Linkage::GetStubCallDescriptor(
282         isolate_, mcgraph()->zone(), idesc, 0, CallDescriptor::kNoFlags,
283         Operator::kNoProperties, MachineType::AnyTagged(), 1,
284         Linkage::kNoContext);
285     stack_check_builtin_code_node_.set(
286         graph()->NewNode(mcgraph()->common()->HeapConstant(code)));
287     stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
288   }
289 
290   Node* call = graph()->NewNode(stack_check_call_operator_,
291                                 stack_check_builtin_code_node_.get(), *effect,
292                                 stack_check.if_false);
293 
294   SetSourcePosition(call, position);
295 
296   Node* ephi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), *effect,
297                                 call, stack_check.merge);
298 
299   *control = stack_check.merge;
300   *effect = ephi;
301 }
302 
PatchInStackCheckIfNeeded()303 void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
304   if (!needs_stack_check_) return;
305 
306   Node* start = graph()->start();
307   // Place a stack check which uses a dummy node as control and effect.
308   Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
309   Node* control = dummy;
310   Node* effect = dummy;
311   // The function-prologue stack check is associated with position 0, which
312   // is never a position of any instruction in the function.
313   StackCheck(0, &effect, &control);
314 
315   // In testing, no steck checks were emitted. Nothing to rewire then.
316   if (effect == dummy) return;
317 
318   // Now patch all control uses of {start} to use {control} and all effect uses
319   // to use {effect} instead. Then rewire the dummy node to use start instead.
320   NodeProperties::ReplaceUses(start, start, effect, control);
321   NodeProperties::ReplaceUses(dummy, nullptr, start, start);
322 }
323 
Binop(wasm::WasmOpcode opcode,Node * left,Node * right,wasm::WasmCodePosition position)324 Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
325                               wasm::WasmCodePosition position) {
326   const Operator* op;
327   MachineOperatorBuilder* m = mcgraph()->machine();
328   switch (opcode) {
329     case wasm::kExprI32Add:
330       op = m->Int32Add();
331       break;
332     case wasm::kExprI32Sub:
333       op = m->Int32Sub();
334       break;
335     case wasm::kExprI32Mul:
336       op = m->Int32Mul();
337       break;
338     case wasm::kExprI32DivS:
339       return BuildI32DivS(left, right, position);
340     case wasm::kExprI32DivU:
341       return BuildI32DivU(left, right, position);
342     case wasm::kExprI32RemS:
343       return BuildI32RemS(left, right, position);
344     case wasm::kExprI32RemU:
345       return BuildI32RemU(left, right, position);
346     case wasm::kExprI32And:
347       op = m->Word32And();
348       break;
349     case wasm::kExprI32Ior:
350       op = m->Word32Or();
351       break;
352     case wasm::kExprI32Xor:
353       op = m->Word32Xor();
354       break;
355     case wasm::kExprI32Shl:
356       op = m->Word32Shl();
357       right = MaskShiftCount32(right);
358       break;
359     case wasm::kExprI32ShrU:
360       op = m->Word32Shr();
361       right = MaskShiftCount32(right);
362       break;
363     case wasm::kExprI32ShrS:
364       op = m->Word32Sar();
365       right = MaskShiftCount32(right);
366       break;
367     case wasm::kExprI32Ror:
368       op = m->Word32Ror();
369       right = MaskShiftCount32(right);
370       break;
371     case wasm::kExprI32Rol:
372       right = MaskShiftCount32(right);
373       return BuildI32Rol(left, right);
374     case wasm::kExprI32Eq:
375       op = m->Word32Equal();
376       break;
377     case wasm::kExprI32Ne:
378       return Invert(Binop(wasm::kExprI32Eq, left, right));
379     case wasm::kExprI32LtS:
380       op = m->Int32LessThan();
381       break;
382     case wasm::kExprI32LeS:
383       op = m->Int32LessThanOrEqual();
384       break;
385     case wasm::kExprI32LtU:
386       op = m->Uint32LessThan();
387       break;
388     case wasm::kExprI32LeU:
389       op = m->Uint32LessThanOrEqual();
390       break;
391     case wasm::kExprI32GtS:
392       op = m->Int32LessThan();
393       std::swap(left, right);
394       break;
395     case wasm::kExprI32GeS:
396       op = m->Int32LessThanOrEqual();
397       std::swap(left, right);
398       break;
399     case wasm::kExprI32GtU:
400       op = m->Uint32LessThan();
401       std::swap(left, right);
402       break;
403     case wasm::kExprI32GeU:
404       op = m->Uint32LessThanOrEqual();
405       std::swap(left, right);
406       break;
407     case wasm::kExprI64And:
408       op = m->Word64And();
409       break;
410     case wasm::kExprI64Add:
411       op = m->Int64Add();
412       break;
413     case wasm::kExprI64Sub:
414       op = m->Int64Sub();
415       break;
416     case wasm::kExprI64Mul:
417       op = m->Int64Mul();
418       break;
419     case wasm::kExprI64DivS:
420       return BuildI64DivS(left, right, position);
421     case wasm::kExprI64DivU:
422       return BuildI64DivU(left, right, position);
423     case wasm::kExprI64RemS:
424       return BuildI64RemS(left, right, position);
425     case wasm::kExprI64RemU:
426       return BuildI64RemU(left, right, position);
427     case wasm::kExprI64Ior:
428       op = m->Word64Or();
429       break;
430     case wasm::kExprI64Xor:
431       op = m->Word64Xor();
432       break;
433     case wasm::kExprI64Shl:
434       op = m->Word64Shl();
435       right = MaskShiftCount64(right);
436       break;
437     case wasm::kExprI64ShrU:
438       op = m->Word64Shr();
439       right = MaskShiftCount64(right);
440       break;
441     case wasm::kExprI64ShrS:
442       op = m->Word64Sar();
443       right = MaskShiftCount64(right);
444       break;
445     case wasm::kExprI64Eq:
446       op = m->Word64Equal();
447       break;
448     case wasm::kExprI64Ne:
449       return Invert(Binop(wasm::kExprI64Eq, left, right));
450     case wasm::kExprI64LtS:
451       op = m->Int64LessThan();
452       break;
453     case wasm::kExprI64LeS:
454       op = m->Int64LessThanOrEqual();
455       break;
456     case wasm::kExprI64LtU:
457       op = m->Uint64LessThan();
458       break;
459     case wasm::kExprI64LeU:
460       op = m->Uint64LessThanOrEqual();
461       break;
462     case wasm::kExprI64GtS:
463       op = m->Int64LessThan();
464       std::swap(left, right);
465       break;
466     case wasm::kExprI64GeS:
467       op = m->Int64LessThanOrEqual();
468       std::swap(left, right);
469       break;
470     case wasm::kExprI64GtU:
471       op = m->Uint64LessThan();
472       std::swap(left, right);
473       break;
474     case wasm::kExprI64GeU:
475       op = m->Uint64LessThanOrEqual();
476       std::swap(left, right);
477       break;
478     case wasm::kExprI64Ror:
479       op = m->Word64Ror();
480       right = MaskShiftCount64(right);
481       break;
482     case wasm::kExprI64Rol:
483       return BuildI64Rol(left, right);
484     case wasm::kExprF32CopySign:
485       return BuildF32CopySign(left, right);
486     case wasm::kExprF64CopySign:
487       return BuildF64CopySign(left, right);
488     case wasm::kExprF32Add:
489       op = m->Float32Add();
490       break;
491     case wasm::kExprF32Sub:
492       op = m->Float32Sub();
493       break;
494     case wasm::kExprF32Mul:
495       op = m->Float32Mul();
496       break;
497     case wasm::kExprF32Div:
498       op = m->Float32Div();
499       break;
500     case wasm::kExprF32Eq:
501       op = m->Float32Equal();
502       break;
503     case wasm::kExprF32Ne:
504       return Invert(Binop(wasm::kExprF32Eq, left, right));
505     case wasm::kExprF32Lt:
506       op = m->Float32LessThan();
507       break;
508     case wasm::kExprF32Ge:
509       op = m->Float32LessThanOrEqual();
510       std::swap(left, right);
511       break;
512     case wasm::kExprF32Gt:
513       op = m->Float32LessThan();
514       std::swap(left, right);
515       break;
516     case wasm::kExprF32Le:
517       op = m->Float32LessThanOrEqual();
518       break;
519     case wasm::kExprF64Add:
520       op = m->Float64Add();
521       break;
522     case wasm::kExprF64Sub:
523       op = m->Float64Sub();
524       break;
525     case wasm::kExprF64Mul:
526       op = m->Float64Mul();
527       break;
528     case wasm::kExprF64Div:
529       op = m->Float64Div();
530       break;
531     case wasm::kExprF64Eq:
532       op = m->Float64Equal();
533       break;
534     case wasm::kExprF64Ne:
535       return Invert(Binop(wasm::kExprF64Eq, left, right));
536     case wasm::kExprF64Lt:
537       op = m->Float64LessThan();
538       break;
539     case wasm::kExprF64Le:
540       op = m->Float64LessThanOrEqual();
541       break;
542     case wasm::kExprF64Gt:
543       op = m->Float64LessThan();
544       std::swap(left, right);
545       break;
546     case wasm::kExprF64Ge:
547       op = m->Float64LessThanOrEqual();
548       std::swap(left, right);
549       break;
550     case wasm::kExprF32Min:
551       op = m->Float32Min();
552       break;
553     case wasm::kExprF64Min:
554       op = m->Float64Min();
555       break;
556     case wasm::kExprF32Max:
557       op = m->Float32Max();
558       break;
559     case wasm::kExprF64Max:
560       op = m->Float64Max();
561       break;
562     case wasm::kExprF64Pow:
563       return BuildF64Pow(left, right);
564     case wasm::kExprF64Atan2:
565       op = m->Float64Atan2();
566       break;
567     case wasm::kExprF64Mod:
568       return BuildF64Mod(left, right);
569     case wasm::kExprI32AsmjsDivS:
570       return BuildI32AsmjsDivS(left, right);
571     case wasm::kExprI32AsmjsDivU:
572       return BuildI32AsmjsDivU(left, right);
573     case wasm::kExprI32AsmjsRemS:
574       return BuildI32AsmjsRemS(left, right);
575     case wasm::kExprI32AsmjsRemU:
576       return BuildI32AsmjsRemU(left, right);
577     case wasm::kExprI32AsmjsStoreMem8:
578       return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
579     case wasm::kExprI32AsmjsStoreMem16:
580       return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
581     case wasm::kExprI32AsmjsStoreMem:
582       return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
583     case wasm::kExprF32AsmjsStoreMem:
584       return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
585     case wasm::kExprF64AsmjsStoreMem:
586       return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
587     default:
588       FATAL_UNSUPPORTED_OPCODE(opcode);
589   }
590   return graph()->NewNode(op, left, right);
591 }
592 
Unop(wasm::WasmOpcode opcode,Node * input,wasm::WasmCodePosition position)593 Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
594                              wasm::WasmCodePosition position) {
595   const Operator* op;
596   MachineOperatorBuilder* m = mcgraph()->machine();
597   switch (opcode) {
598     case wasm::kExprI32Eqz:
599       op = m->Word32Equal();
600       return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
601     case wasm::kExprF32Abs:
602       op = m->Float32Abs();
603       break;
604     case wasm::kExprF32Neg: {
605       op = m->Float32Neg();
606       break;
607     }
608     case wasm::kExprF32Sqrt:
609       op = m->Float32Sqrt();
610       break;
611     case wasm::kExprF64Abs:
612       op = m->Float64Abs();
613       break;
614     case wasm::kExprF64Neg: {
615       op = m->Float64Neg();
616       break;
617     }
618     case wasm::kExprF64Sqrt:
619       op = m->Float64Sqrt();
620       break;
621     case wasm::kExprI32SConvertF32:
622     case wasm::kExprI32UConvertF32:
623     case wasm::kExprI32SConvertF64:
624     case wasm::kExprI32UConvertF64:
625     case wasm::kExprI32SConvertSatF64:
626     case wasm::kExprI32UConvertSatF64:
627     case wasm::kExprI32SConvertSatF32:
628     case wasm::kExprI32UConvertSatF32:
629       return BuildIntConvertFloat(input, position, opcode);
630     case wasm::kExprI32AsmjsSConvertF64:
631       return BuildI32AsmjsSConvertF64(input);
632     case wasm::kExprI32AsmjsUConvertF64:
633       return BuildI32AsmjsUConvertF64(input);
634     case wasm::kExprF32ConvertF64:
635       op = m->TruncateFloat64ToFloat32();
636       break;
637     case wasm::kExprF64SConvertI32:
638       op = m->ChangeInt32ToFloat64();
639       break;
640     case wasm::kExprF64UConvertI32:
641       op = m->ChangeUint32ToFloat64();
642       break;
643     case wasm::kExprF32SConvertI32:
644       op = m->RoundInt32ToFloat32();
645       break;
646     case wasm::kExprF32UConvertI32:
647       op = m->RoundUint32ToFloat32();
648       break;
649     case wasm::kExprI32AsmjsSConvertF32:
650       return BuildI32AsmjsSConvertF32(input);
651     case wasm::kExprI32AsmjsUConvertF32:
652       return BuildI32AsmjsUConvertF32(input);
653     case wasm::kExprF64ConvertF32:
654       op = m->ChangeFloat32ToFloat64();
655       break;
656     case wasm::kExprF32ReinterpretI32:
657       op = m->BitcastInt32ToFloat32();
658       break;
659     case wasm::kExprI32ReinterpretF32:
660       op = m->BitcastFloat32ToInt32();
661       break;
662     case wasm::kExprI32Clz:
663       op = m->Word32Clz();
664       break;
665     case wasm::kExprI32Ctz: {
666       if (m->Word32Ctz().IsSupported()) {
667         op = m->Word32Ctz().op();
668         break;
669       } else if (m->Word32ReverseBits().IsSupported()) {
670         Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
671         Node* result = graph()->NewNode(m->Word32Clz(), reversed);
672         return result;
673       } else {
674         return BuildI32Ctz(input);
675       }
676     }
677     case wasm::kExprI32Popcnt: {
678       if (m->Word32Popcnt().IsSupported()) {
679         op = m->Word32Popcnt().op();
680         break;
681       } else {
682         return BuildI32Popcnt(input);
683       }
684     }
685     case wasm::kExprF32Floor: {
686       if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
687       op = m->Float32RoundDown().op();
688       break;
689     }
690     case wasm::kExprF32Ceil: {
691       if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
692       op = m->Float32RoundUp().op();
693       break;
694     }
695     case wasm::kExprF32Trunc: {
696       if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
697       op = m->Float32RoundTruncate().op();
698       break;
699     }
700     case wasm::kExprF32NearestInt: {
701       if (!m->Float32RoundTiesEven().IsSupported())
702         return BuildF32NearestInt(input);
703       op = m->Float32RoundTiesEven().op();
704       break;
705     }
706     case wasm::kExprF64Floor: {
707       if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
708       op = m->Float64RoundDown().op();
709       break;
710     }
711     case wasm::kExprF64Ceil: {
712       if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
713       op = m->Float64RoundUp().op();
714       break;
715     }
716     case wasm::kExprF64Trunc: {
717       if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
718       op = m->Float64RoundTruncate().op();
719       break;
720     }
721     case wasm::kExprF64NearestInt: {
722       if (!m->Float64RoundTiesEven().IsSupported())
723         return BuildF64NearestInt(input);
724       op = m->Float64RoundTiesEven().op();
725       break;
726     }
727     case wasm::kExprF64Acos: {
728       return BuildF64Acos(input);
729     }
730     case wasm::kExprF64Asin: {
731       return BuildF64Asin(input);
732     }
733     case wasm::kExprF64Atan:
734       op = m->Float64Atan();
735       break;
736     case wasm::kExprF64Cos: {
737       op = m->Float64Cos();
738       break;
739     }
740     case wasm::kExprF64Sin: {
741       op = m->Float64Sin();
742       break;
743     }
744     case wasm::kExprF64Tan: {
745       op = m->Float64Tan();
746       break;
747     }
748     case wasm::kExprF64Exp: {
749       op = m->Float64Exp();
750       break;
751     }
752     case wasm::kExprF64Log:
753       op = m->Float64Log();
754       break;
755     case wasm::kExprI32ConvertI64:
756       op = m->TruncateInt64ToInt32();
757       break;
758     case wasm::kExprI64SConvertI32:
759       op = m->ChangeInt32ToInt64();
760       break;
761     case wasm::kExprI64UConvertI32:
762       op = m->ChangeUint32ToUint64();
763       break;
764     case wasm::kExprF64ReinterpretI64:
765       op = m->BitcastInt64ToFloat64();
766       break;
767     case wasm::kExprI64ReinterpretF64:
768       op = m->BitcastFloat64ToInt64();
769       break;
770     case wasm::kExprI64Clz:
771       op = m->Word64Clz();
772       break;
773     case wasm::kExprI64Ctz: {
774       OptionalOperator ctz64 = m->Word64Ctz();
775       if (ctz64.IsSupported()) {
776         op = ctz64.op();
777         break;
778       } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
779         op = ctz64.placeholder();
780         break;
781       } else if (m->Word64ReverseBits().IsSupported()) {
782         Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
783         Node* result = graph()->NewNode(m->Word64Clz(), reversed);
784         return result;
785       } else {
786         return BuildI64Ctz(input);
787       }
788     }
789     case wasm::kExprI64Popcnt: {
790       OptionalOperator popcnt64 = m->Word64Popcnt();
791       if (popcnt64.IsSupported()) {
792         op = popcnt64.op();
793       } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
794         op = popcnt64.placeholder();
795       } else {
796         return BuildI64Popcnt(input);
797       }
798       break;
799     }
800     case wasm::kExprI64Eqz:
801       op = m->Word64Equal();
802       return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
803     case wasm::kExprF32SConvertI64:
804       if (m->Is32()) {
805         return BuildF32SConvertI64(input);
806       }
807       op = m->RoundInt64ToFloat32();
808       break;
809     case wasm::kExprF32UConvertI64:
810       if (m->Is32()) {
811         return BuildF32UConvertI64(input);
812       }
813       op = m->RoundUint64ToFloat32();
814       break;
815     case wasm::kExprF64SConvertI64:
816       if (m->Is32()) {
817         return BuildF64SConvertI64(input);
818       }
819       op = m->RoundInt64ToFloat64();
820       break;
821     case wasm::kExprF64UConvertI64:
822       if (m->Is32()) {
823         return BuildF64UConvertI64(input);
824       }
825       op = m->RoundUint64ToFloat64();
826       break;
827     case wasm::kExprI32SExtendI8:
828       op = m->SignExtendWord8ToInt32();
829       break;
830     case wasm::kExprI32SExtendI16:
831       op = m->SignExtendWord16ToInt32();
832       break;
833     case wasm::kExprI64SExtendI8:
834       op = m->SignExtendWord8ToInt64();
835       break;
836     case wasm::kExprI64SExtendI16:
837       op = m->SignExtendWord16ToInt64();
838       break;
839     case wasm::kExprI64SExtendI32:
840       op = m->SignExtendWord32ToInt64();
841       break;
842     case wasm::kExprI64SConvertF32:
843     case wasm::kExprI64UConvertF32:
844     case wasm::kExprI64SConvertF64:
845     case wasm::kExprI64UConvertF64:
846     case wasm::kExprI64SConvertSatF32:
847     case wasm::kExprI64UConvertSatF32:
848     case wasm::kExprI64SConvertSatF64:
849     case wasm::kExprI64UConvertSatF64:
850       return mcgraph()->machine()->Is32()
851                  ? BuildCcallConvertFloat(input, position, opcode)
852                  : BuildIntConvertFloat(input, position, opcode);
853     case wasm::kExprRefIsNull:
854       return graph()->NewNode(m->WordEqual(), input, RefNull());
855     case wasm::kExprI32AsmjsLoadMem8S:
856       return BuildAsmjsLoadMem(MachineType::Int8(), input);
857     case wasm::kExprI32AsmjsLoadMem8U:
858       return BuildAsmjsLoadMem(MachineType::Uint8(), input);
859     case wasm::kExprI32AsmjsLoadMem16S:
860       return BuildAsmjsLoadMem(MachineType::Int16(), input);
861     case wasm::kExprI32AsmjsLoadMem16U:
862       return BuildAsmjsLoadMem(MachineType::Uint16(), input);
863     case wasm::kExprI32AsmjsLoadMem:
864       return BuildAsmjsLoadMem(MachineType::Int32(), input);
865     case wasm::kExprF32AsmjsLoadMem:
866       return BuildAsmjsLoadMem(MachineType::Float32(), input);
867     case wasm::kExprF64AsmjsLoadMem:
868       return BuildAsmjsLoadMem(MachineType::Float64(), input);
869     default:
870       FATAL_UNSUPPORTED_OPCODE(opcode);
871   }
872   return graph()->NewNode(op, input);
873 }
874 
Float32Constant(float value)875 Node* WasmGraphBuilder::Float32Constant(float value) {
876   return mcgraph()->Float32Constant(value);
877 }
878 
Float64Constant(double value)879 Node* WasmGraphBuilder::Float64Constant(double value) {
880   return mcgraph()->Float64Constant(value);
881 }
882 
883 namespace {
Branch(MachineGraph * mcgraph,Node * cond,Node ** true_node,Node ** false_node,Node * control,BranchHint hint)884 Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
885              Node** false_node, Node* control, BranchHint hint) {
886   DCHECK_NOT_NULL(cond);
887   DCHECK_NOT_NULL(control);
888   Node* branch =
889       mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
890   *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
891   *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
892   return branch;
893 }
894 }  // namespace
895 
BranchNoHint(Node * cond,Node ** true_node,Node ** false_node)896 Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
897                                      Node** false_node) {
898   return Branch(mcgraph(), cond, true_node, false_node, *control_,
899                 BranchHint::kNone);
900 }
901 
BranchExpectTrue(Node * cond,Node ** true_node,Node ** false_node)902 Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
903                                          Node** false_node) {
904   return Branch(mcgraph(), cond, true_node, false_node, *control_,
905                 BranchHint::kTrue);
906 }
907 
BranchExpectFalse(Node * cond,Node ** true_node,Node ** false_node)908 Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
909                                           Node** false_node) {
910   return Branch(mcgraph(), cond, true_node, false_node, *control_,
911                 BranchHint::kFalse);
912 }
913 
GetBuiltinIdForTrap(wasm::TrapReason reason)914 Builtins::Name WasmGraphBuilder::GetBuiltinIdForTrap(wasm::TrapReason reason) {
915   // TODO(wasm): "!env_" should not happen when compiling an actual wasm
916   // function.
917   if (!env_ || !env_->runtime_exception_support) {
918     // We use Builtins::builtin_count as a marker to tell the code generator
919     // to generate a call to a testing c-function instead of a runtime
920     // function. This code should only be called from a cctest.
921     return Builtins::builtin_count;
922   }
923 
924   switch (reason) {
925 #define TRAPREASON_TO_MESSAGE(name) \
926   case wasm::k##name:               \
927     return Builtins::kThrowWasm##name;
928     FOREACH_WASM_TRAPREASON(TRAPREASON_TO_MESSAGE)
929 #undef TRAPREASON_TO_MESSAGE
930     default:
931       UNREACHABLE();
932   }
933 }
934 
TrapIfTrue(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)935 Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
936                                    wasm::WasmCodePosition position) {
937   Builtins::Name trap_id = GetBuiltinIdForTrap(reason);
938   Node* node = graph()->NewNode(mcgraph()->common()->TrapIf(trap_id), cond,
939                                 Effect(), Control());
940   *control_ = node;
941   SetSourcePosition(node, position);
942   return node;
943 }
944 
TrapIfFalse(wasm::TrapReason reason,Node * cond,wasm::WasmCodePosition position)945 Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
946                                     wasm::WasmCodePosition position) {
947   Builtins::Name trap_id = GetBuiltinIdForTrap(reason);
948 
949   Node* node = graph()->NewNode(mcgraph()->common()->TrapUnless(trap_id), cond,
950                                 Effect(), Control());
951   *control_ = node;
952   SetSourcePosition(node, position);
953   return node;
954 }
955 
956 // Add a check that traps if {node} is equal to {val}.
TrapIfEq32(wasm::TrapReason reason,Node * node,int32_t val,wasm::WasmCodePosition position)957 Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
958                                    int32_t val,
959                                    wasm::WasmCodePosition position) {
960   Int32Matcher m(node);
961   if (m.HasValue() && !m.Is(val)) return graph()->start();
962   if (val == 0) {
963     return TrapIfFalse(reason, node, position);
964   } else {
965     return TrapIfTrue(reason,
966                       graph()->NewNode(mcgraph()->machine()->Word32Equal(),
967                                        node, mcgraph()->Int32Constant(val)),
968                       position);
969   }
970 }
971 
972 // Add a check that traps if {node} is zero.
ZeroCheck32(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)973 Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
974                                     wasm::WasmCodePosition position) {
975   return TrapIfEq32(reason, node, 0, position);
976 }
977 
978 // Add a check that traps if {node} is equal to {val}.
TrapIfEq64(wasm::TrapReason reason,Node * node,int64_t val,wasm::WasmCodePosition position)979 Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
980                                    int64_t val,
981                                    wasm::WasmCodePosition position) {
982   Int64Matcher m(node);
983   if (m.HasValue() && !m.Is(val)) return graph()->start();
984   return TrapIfTrue(reason,
985                     graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
986                                      mcgraph()->Int64Constant(val)),
987                     position);
988 }
989 
990 // Add a check that traps if {node} is zero.
ZeroCheck64(wasm::TrapReason reason,Node * node,wasm::WasmCodePosition position)991 Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
992                                     wasm::WasmCodePosition position) {
993   return TrapIfEq64(reason, node, 0, position);
994 }
995 
Switch(unsigned count,Node * key)996 Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
997   return graph()->NewNode(mcgraph()->common()->Switch(count), key, *control_);
998 }
999 
IfValue(int32_t value,Node * sw)1000 Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1001   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1002   return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1003 }
1004 
IfDefault(Node * sw)1005 Node* WasmGraphBuilder::IfDefault(Node* sw) {
1006   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1007   return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1008 }
1009 
Return(unsigned count,Node ** vals)1010 Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1011   DCHECK_NOT_NULL(*control_);
1012   DCHECK_NOT_NULL(*effect_);
1013 
1014   static const int kStackAllocatedNodeBufferSize = 8;
1015   Node* stack_buffer[kStackAllocatedNodeBufferSize];
1016   std::vector<Node*> heap_buffer;
1017 
1018   Node** buf = stack_buffer;
1019   if (count + 3 > kStackAllocatedNodeBufferSize) {
1020     heap_buffer.resize(count + 3);
1021     buf = heap_buffer.data();
1022   }
1023 
1024   buf[0] = mcgraph()->Int32Constant(0);
1025   memcpy(buf + 1, vals, sizeof(void*) * count);
1026   buf[count + 1] = *effect_;
1027   buf[count + 2] = *control_;
1028   Node* ret =
1029       graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1030 
1031   MergeControlToEnd(mcgraph(), ret);
1032   return ret;
1033 }
1034 
ReturnVoid()1035 Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1036 
Unreachable(wasm::WasmCodePosition position)1037 Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1038   TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1039   ReturnVoid();
1040   return nullptr;
1041 }
1042 
MaskShiftCount32(Node * node)1043 Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1044   static const int32_t kMask32 = 0x1F;
1045   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1046     // Shifts by constants are so common we pattern-match them here.
1047     Int32Matcher match(node);
1048     if (match.HasValue()) {
1049       int32_t masked = (match.Value() & kMask32);
1050       if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1051     } else {
1052       node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1053                               mcgraph()->Int32Constant(kMask32));
1054     }
1055   }
1056   return node;
1057 }
1058 
MaskShiftCount64(Node * node)1059 Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1060   static const int64_t kMask64 = 0x3F;
1061   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1062     // Shifts by constants are so common we pattern-match them here.
1063     Int64Matcher match(node);
1064     if (match.HasValue()) {
1065       int64_t masked = (match.Value() & kMask64);
1066       if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1067     } else {
1068       node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1069                               mcgraph()->Int64Constant(kMask64));
1070     }
1071   }
1072   return node;
1073 }
1074 
ReverseBytesSupported(MachineOperatorBuilder * m,size_t size_in_bytes)1075 static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1076                                   size_t size_in_bytes) {
1077   switch (size_in_bytes) {
1078     case 4:
1079     case 16:
1080       return m->Word32ReverseBytes().IsSupported();
1081     case 8:
1082       return m->Word64ReverseBytes().IsSupported();
1083     default:
1084       break;
1085   }
1086   return false;
1087 }
1088 
BuildChangeEndiannessStore(Node * node,MachineRepresentation mem_rep,wasm::ValueType wasmtype)1089 Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1090     Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1091   Node* result;
1092   Node* value = node;
1093   MachineOperatorBuilder* m = mcgraph()->machine();
1094   int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1095   int valueSizeInBits = 8 * valueSizeInBytes;
1096   bool isFloat = false;
1097 
1098   switch (wasmtype) {
1099     case wasm::kWasmF64:
1100       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1101       isFloat = true;
1102       V8_FALLTHROUGH;
1103     case wasm::kWasmI64:
1104       result = mcgraph()->Int64Constant(0);
1105       break;
1106     case wasm::kWasmF32:
1107       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1108       isFloat = true;
1109       V8_FALLTHROUGH;
1110     case wasm::kWasmI32:
1111       result = mcgraph()->Int32Constant(0);
1112       break;
1113     case wasm::kWasmS128:
1114       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1115       break;
1116     default:
1117       UNREACHABLE();
1118       break;
1119   }
1120 
1121   if (mem_rep == MachineRepresentation::kWord8) {
1122     // No need to change endianness for byte size, return original node
1123     return node;
1124   }
1125   if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1126     // In case we store lower part of WasmI64 expression, we can truncate
1127     // upper 32bits
1128     value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1129     valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1130     valueSizeInBits = 8 * valueSizeInBytes;
1131     if (mem_rep == MachineRepresentation::kWord16) {
1132       value =
1133           graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1134     }
1135   } else if (wasmtype == wasm::kWasmI32 &&
1136              mem_rep == MachineRepresentation::kWord16) {
1137     value =
1138         graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1139   }
1140 
1141   int i;
1142   uint32_t shiftCount;
1143 
1144   if (ReverseBytesSupported(m, valueSizeInBytes)) {
1145     switch (valueSizeInBytes) {
1146       case 4:
1147         result = graph()->NewNode(m->Word32ReverseBytes().op(), value);
1148         break;
1149       case 8:
1150         result = graph()->NewNode(m->Word64ReverseBytes().op(), value);
1151         break;
1152       case 16: {
1153         Node* byte_reversed_lanes[4];
1154         for (int lane = 0; lane < 4; lane++) {
1155           byte_reversed_lanes[lane] = graph()->NewNode(
1156               m->Word32ReverseBytes().op(),
1157               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1158                                value));
1159         }
1160 
1161         // This is making a copy of the value.
1162         result =
1163             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1164 
1165         for (int lane = 0; lane < 4; lane++) {
1166           result =
1167               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1168                                result, byte_reversed_lanes[lane]);
1169         }
1170 
1171         break;
1172       }
1173       default:
1174         UNREACHABLE();
1175         break;
1176     }
1177   } else {
1178     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1179          i += 8, shiftCount -= 16) {
1180       Node* shiftLower;
1181       Node* shiftHigher;
1182       Node* lowerByte;
1183       Node* higherByte;
1184 
1185       DCHECK_LT(0, shiftCount);
1186       DCHECK_EQ(0, (shiftCount + 8) % 16);
1187 
1188       if (valueSizeInBits > 32) {
1189         shiftLower = graph()->NewNode(m->Word64Shl(), value,
1190                                       mcgraph()->Int64Constant(shiftCount));
1191         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1192                                        mcgraph()->Int64Constant(shiftCount));
1193         lowerByte = graph()->NewNode(
1194             m->Word64And(), shiftLower,
1195             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1196                                      << (valueSizeInBits - 8 - i)));
1197         higherByte = graph()->NewNode(
1198             m->Word64And(), shiftHigher,
1199             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1200         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1201         result = graph()->NewNode(m->Word64Or(), result, higherByte);
1202       } else {
1203         shiftLower = graph()->NewNode(m->Word32Shl(), value,
1204                                       mcgraph()->Int32Constant(shiftCount));
1205         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1206                                        mcgraph()->Int32Constant(shiftCount));
1207         lowerByte = graph()->NewNode(
1208             m->Word32And(), shiftLower,
1209             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1210                                      << (valueSizeInBits - 8 - i)));
1211         higherByte = graph()->NewNode(
1212             m->Word32And(), shiftHigher,
1213             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1214         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1215         result = graph()->NewNode(m->Word32Or(), result, higherByte);
1216       }
1217     }
1218   }
1219 
1220   if (isFloat) {
1221     switch (wasmtype) {
1222       case wasm::kWasmF64:
1223         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1224         break;
1225       case wasm::kWasmF32:
1226         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1227         break;
1228       default:
1229         UNREACHABLE();
1230         break;
1231     }
1232   }
1233 
1234   return result;
1235 }
1236 
BuildChangeEndiannessLoad(Node * node,MachineType memtype,wasm::ValueType wasmtype)1237 Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1238                                                   MachineType memtype,
1239                                                   wasm::ValueType wasmtype) {
1240   Node* result;
1241   Node* value = node;
1242   MachineOperatorBuilder* m = mcgraph()->machine();
1243   int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1244   int valueSizeInBits = 8 * valueSizeInBytes;
1245   bool isFloat = false;
1246 
1247   switch (memtype.representation()) {
1248     case MachineRepresentation::kFloat64:
1249       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1250       isFloat = true;
1251       V8_FALLTHROUGH;
1252     case MachineRepresentation::kWord64:
1253       result = mcgraph()->Int64Constant(0);
1254       break;
1255     case MachineRepresentation::kFloat32:
1256       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1257       isFloat = true;
1258       V8_FALLTHROUGH;
1259     case MachineRepresentation::kWord32:
1260     case MachineRepresentation::kWord16:
1261       result = mcgraph()->Int32Constant(0);
1262       break;
1263     case MachineRepresentation::kWord8:
1264       // No need to change endianness for byte size, return original node
1265       return node;
1266       break;
1267     case MachineRepresentation::kSimd128:
1268       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1269       break;
1270     default:
1271       UNREACHABLE();
1272       break;
1273   }
1274 
1275   int i;
1276   uint32_t shiftCount;
1277 
1278   if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1279     switch (valueSizeInBytes) {
1280       case 2:
1281         result =
1282             graph()->NewNode(m->Word32ReverseBytes().op(),
1283                              graph()->NewNode(m->Word32Shl(), value,
1284                                               mcgraph()->Int32Constant(16)));
1285         break;
1286       case 4:
1287         result = graph()->NewNode(m->Word32ReverseBytes().op(), value);
1288         break;
1289       case 8:
1290         result = graph()->NewNode(m->Word64ReverseBytes().op(), value);
1291         break;
1292       case 16: {
1293         Node* byte_reversed_lanes[4];
1294         for (int lane = 0; lane < 4; lane++) {
1295           byte_reversed_lanes[lane] = graph()->NewNode(
1296               m->Word32ReverseBytes().op(),
1297               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1298                                value));
1299         }
1300 
1301         // This is making a copy of the value.
1302         result =
1303             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1304 
1305         for (int lane = 0; lane < 4; lane++) {
1306           result =
1307               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1308                                result, byte_reversed_lanes[lane]);
1309         }
1310 
1311         break;
1312       }
1313       default:
1314         UNREACHABLE();
1315     }
1316   } else {
1317     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1318          i += 8, shiftCount -= 16) {
1319       Node* shiftLower;
1320       Node* shiftHigher;
1321       Node* lowerByte;
1322       Node* higherByte;
1323 
1324       DCHECK_LT(0, shiftCount);
1325       DCHECK_EQ(0, (shiftCount + 8) % 16);
1326 
1327       if (valueSizeInBits > 32) {
1328         shiftLower = graph()->NewNode(m->Word64Shl(), value,
1329                                       mcgraph()->Int64Constant(shiftCount));
1330         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1331                                        mcgraph()->Int64Constant(shiftCount));
1332         lowerByte = graph()->NewNode(
1333             m->Word64And(), shiftLower,
1334             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1335                                      << (valueSizeInBits - 8 - i)));
1336         higherByte = graph()->NewNode(
1337             m->Word64And(), shiftHigher,
1338             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1339         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1340         result = graph()->NewNode(m->Word64Or(), result, higherByte);
1341       } else {
1342         shiftLower = graph()->NewNode(m->Word32Shl(), value,
1343                                       mcgraph()->Int32Constant(shiftCount));
1344         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1345                                        mcgraph()->Int32Constant(shiftCount));
1346         lowerByte = graph()->NewNode(
1347             m->Word32And(), shiftLower,
1348             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1349                                      << (valueSizeInBits - 8 - i)));
1350         higherByte = graph()->NewNode(
1351             m->Word32And(), shiftHigher,
1352             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1353         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1354         result = graph()->NewNode(m->Word32Or(), result, higherByte);
1355       }
1356     }
1357   }
1358 
1359   if (isFloat) {
1360     switch (memtype.representation()) {
1361       case MachineRepresentation::kFloat64:
1362         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1363         break;
1364       case MachineRepresentation::kFloat32:
1365         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1366         break;
1367       default:
1368         UNREACHABLE();
1369         break;
1370     }
1371   }
1372 
1373   // We need to sign extend the value
1374   if (memtype.IsSigned()) {
1375     DCHECK(!isFloat);
1376     if (valueSizeInBits < 32) {
1377       Node* shiftBitCount;
1378       // Perform sign extension using following trick
1379       // result = (x << machine_width - type_width) >> (machine_width -
1380       // type_width)
1381       if (wasmtype == wasm::kWasmI64) {
1382         shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1383         result = graph()->NewNode(
1384             m->Word64Sar(),
1385             graph()->NewNode(m->Word64Shl(),
1386                              graph()->NewNode(m->ChangeInt32ToInt64(), result),
1387                              shiftBitCount),
1388             shiftBitCount);
1389       } else if (wasmtype == wasm::kWasmI32) {
1390         shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1391         result = graph()->NewNode(
1392             m->Word32Sar(),
1393             graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1394             shiftBitCount);
1395       }
1396     }
1397   }
1398 
1399   return result;
1400 }
1401 
BuildF32CopySign(Node * left,Node * right)1402 Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1403   Node* result = Unop(
1404       wasm::kExprF32ReinterpretI32,
1405       Binop(wasm::kExprI32Ior,
1406             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1407                   mcgraph()->Int32Constant(0x7FFFFFFF)),
1408             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1409                   mcgraph()->Int32Constant(0x80000000))));
1410 
1411   return result;
1412 }
1413 
BuildF64CopySign(Node * left,Node * right)1414 Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1415 #if WASM_64
1416   Node* result = Unop(
1417       wasm::kExprF64ReinterpretI64,
1418       Binop(wasm::kExprI64Ior,
1419             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1420                   mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1421             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1422                   mcgraph()->Int64Constant(0x8000000000000000))));
1423 
1424   return result;
1425 #else
1426   MachineOperatorBuilder* m = mcgraph()->machine();
1427 
1428   Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1429   Node* high_word_right =
1430       graph()->NewNode(m->Float64ExtractHighWord32(), right);
1431 
1432   Node* new_high_word = Binop(wasm::kExprI32Ior,
1433                               Binop(wasm::kExprI32And, high_word_left,
1434                                     mcgraph()->Int32Constant(0x7FFFFFFF)),
1435                               Binop(wasm::kExprI32And, high_word_right,
1436                                     mcgraph()->Int32Constant(0x80000000)));
1437 
1438   return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1439 #endif
1440 }
1441 
1442 namespace {
1443 
IntConvertType(wasm::WasmOpcode opcode)1444 MachineType IntConvertType(wasm::WasmOpcode opcode) {
1445   switch (opcode) {
1446     case wasm::kExprI32SConvertF32:
1447     case wasm::kExprI32SConvertF64:
1448     case wasm::kExprI32SConvertSatF32:
1449     case wasm::kExprI32SConvertSatF64:
1450       return MachineType::Int32();
1451     case wasm::kExprI32UConvertF32:
1452     case wasm::kExprI32UConvertF64:
1453     case wasm::kExprI32UConvertSatF32:
1454     case wasm::kExprI32UConvertSatF64:
1455       return MachineType::Uint32();
1456     case wasm::kExprI64SConvertF32:
1457     case wasm::kExprI64SConvertF64:
1458     case wasm::kExprI64SConvertSatF32:
1459     case wasm::kExprI64SConvertSatF64:
1460       return MachineType::Int64();
1461     case wasm::kExprI64UConvertF32:
1462     case wasm::kExprI64UConvertF64:
1463     case wasm::kExprI64UConvertSatF32:
1464     case wasm::kExprI64UConvertSatF64:
1465       return MachineType::Uint64();
1466     default:
1467       UNREACHABLE();
1468   }
1469 }
1470 
FloatConvertType(wasm::WasmOpcode opcode)1471 MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1472   switch (opcode) {
1473     case wasm::kExprI32SConvertF32:
1474     case wasm::kExprI32UConvertF32:
1475     case wasm::kExprI32SConvertSatF32:
1476     case wasm::kExprI64SConvertF32:
1477     case wasm::kExprI64UConvertF32:
1478     case wasm::kExprI32UConvertSatF32:
1479     case wasm::kExprI64SConvertSatF32:
1480     case wasm::kExprI64UConvertSatF32:
1481       return MachineType::Float32();
1482     case wasm::kExprI32SConvertF64:
1483     case wasm::kExprI32UConvertF64:
1484     case wasm::kExprI64SConvertF64:
1485     case wasm::kExprI64UConvertF64:
1486     case wasm::kExprI32SConvertSatF64:
1487     case wasm::kExprI32UConvertSatF64:
1488     case wasm::kExprI64SConvertSatF64:
1489     case wasm::kExprI64UConvertSatF64:
1490       return MachineType::Float64();
1491     default:
1492       UNREACHABLE();
1493   }
1494 }
1495 
ConvertOp(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1496 const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1497   switch (opcode) {
1498     case wasm::kExprI32SConvertF32:
1499     case wasm::kExprI32SConvertSatF32:
1500       return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1501     case wasm::kExprI32UConvertF32:
1502     case wasm::kExprI32UConvertSatF32:
1503       return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1504     case wasm::kExprI32SConvertF64:
1505     case wasm::kExprI32SConvertSatF64:
1506       return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1507     case wasm::kExprI32UConvertF64:
1508     case wasm::kExprI32UConvertSatF64:
1509       return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1510     case wasm::kExprI64SConvertF32:
1511     case wasm::kExprI64SConvertSatF32:
1512       return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1513     case wasm::kExprI64UConvertF32:
1514     case wasm::kExprI64UConvertSatF32:
1515       return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1516     case wasm::kExprI64SConvertF64:
1517     case wasm::kExprI64SConvertSatF64:
1518       return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1519     case wasm::kExprI64UConvertF64:
1520     case wasm::kExprI64UConvertSatF64:
1521       return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1522     default:
1523       UNREACHABLE();
1524   }
1525 }
1526 
ConvertBackOp(wasm::WasmOpcode opcode)1527 wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1528   switch (opcode) {
1529     case wasm::kExprI32SConvertF32:
1530     case wasm::kExprI32SConvertSatF32:
1531       return wasm::kExprF32SConvertI32;
1532     case wasm::kExprI32UConvertF32:
1533     case wasm::kExprI32UConvertSatF32:
1534       return wasm::kExprF32UConvertI32;
1535     case wasm::kExprI32SConvertF64:
1536     case wasm::kExprI32SConvertSatF64:
1537       return wasm::kExprF64SConvertI32;
1538     case wasm::kExprI32UConvertF64:
1539     case wasm::kExprI32UConvertSatF64:
1540       return wasm::kExprF64UConvertI32;
1541     default:
1542       UNREACHABLE();
1543   }
1544 }
1545 
IsTrappingConvertOp(wasm::WasmOpcode opcode)1546 bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1547   switch (opcode) {
1548     case wasm::kExprI32SConvertF32:
1549     case wasm::kExprI32UConvertF32:
1550     case wasm::kExprI32SConvertF64:
1551     case wasm::kExprI32UConvertF64:
1552     case wasm::kExprI64SConvertF32:
1553     case wasm::kExprI64UConvertF32:
1554     case wasm::kExprI64SConvertF64:
1555     case wasm::kExprI64UConvertF64:
1556       return true;
1557     case wasm::kExprI32SConvertSatF64:
1558     case wasm::kExprI32UConvertSatF64:
1559     case wasm::kExprI32SConvertSatF32:
1560     case wasm::kExprI32UConvertSatF32:
1561     case wasm::kExprI64SConvertSatF32:
1562     case wasm::kExprI64UConvertSatF32:
1563     case wasm::kExprI64SConvertSatF64:
1564     case wasm::kExprI64UConvertSatF64:
1565       return false;
1566     default:
1567       UNREACHABLE();
1568   }
1569 }
1570 
Zero(WasmGraphBuilder * builder,const MachineType & ty)1571 Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1572   switch (ty.representation()) {
1573     case MachineRepresentation::kWord32:
1574       return builder->Int32Constant(0);
1575     case MachineRepresentation::kWord64:
1576       return builder->Int64Constant(0);
1577     case MachineRepresentation::kFloat32:
1578       return builder->Float32Constant(0.0);
1579     case MachineRepresentation::kFloat64:
1580       return builder->Float64Constant(0.0);
1581     default:
1582       UNREACHABLE();
1583   }
1584 }
1585 
Min(WasmGraphBuilder * builder,const MachineType & ty)1586 Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1587   switch (ty.semantic()) {
1588     case MachineSemantic::kInt32:
1589       return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1590     case MachineSemantic::kUint32:
1591       return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1592     case MachineSemantic::kInt64:
1593       return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1594     case MachineSemantic::kUint64:
1595       return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1596     default:
1597       UNREACHABLE();
1598   }
1599 }
1600 
Max(WasmGraphBuilder * builder,const MachineType & ty)1601 Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1602   switch (ty.semantic()) {
1603     case MachineSemantic::kInt32:
1604       return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1605     case MachineSemantic::kUint32:
1606       return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1607     case MachineSemantic::kInt64:
1608       return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1609     case MachineSemantic::kUint64:
1610       return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1611     default:
1612       UNREACHABLE();
1613   }
1614 }
1615 
TruncOp(const MachineType & ty)1616 wasm::WasmOpcode TruncOp(const MachineType& ty) {
1617   switch (ty.representation()) {
1618     case MachineRepresentation::kFloat32:
1619       return wasm::kExprF32Trunc;
1620     case MachineRepresentation::kFloat64:
1621       return wasm::kExprF64Trunc;
1622     default:
1623       UNREACHABLE();
1624   }
1625 }
1626 
NeOp(const MachineType & ty)1627 wasm::WasmOpcode NeOp(const MachineType& ty) {
1628   switch (ty.representation()) {
1629     case MachineRepresentation::kFloat32:
1630       return wasm::kExprF32Ne;
1631     case MachineRepresentation::kFloat64:
1632       return wasm::kExprF64Ne;
1633     default:
1634       UNREACHABLE();
1635   }
1636 }
1637 
LtOp(const MachineType & ty)1638 wasm::WasmOpcode LtOp(const MachineType& ty) {
1639   switch (ty.representation()) {
1640     case MachineRepresentation::kFloat32:
1641       return wasm::kExprF32Lt;
1642     case MachineRepresentation::kFloat64:
1643       return wasm::kExprF64Lt;
1644     default:
1645       UNREACHABLE();
1646   }
1647 }
1648 
ConvertTrapTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1649 Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1650                       const MachineType& int_ty, const MachineType& float_ty,
1651                       Node* trunc, Node* converted_value) {
1652   if (int_ty.representation() == MachineRepresentation::kWord32) {
1653     Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1654     return builder->Binop(NeOp(float_ty), trunc, check);
1655   }
1656   return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1657                                    trunc, builder->graph()->start());
1658 }
1659 
ConvertSaturateTest(WasmGraphBuilder * builder,wasm::WasmOpcode opcode,const MachineType & int_ty,const MachineType & float_ty,Node * trunc,Node * converted_value)1660 Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1661                           const MachineType& int_ty,
1662                           const MachineType& float_ty, Node* trunc,
1663                           Node* converted_value) {
1664   Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1665                                converted_value);
1666   if (int_ty.representation() == MachineRepresentation::kWord64) {
1667     test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1668   }
1669   return test;
1670 }
1671 
1672 }  // namespace
1673 
BuildIntConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1674 Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1675                                              wasm::WasmCodePosition position,
1676                                              wasm::WasmOpcode opcode) {
1677   const MachineType int_ty = IntConvertType(opcode);
1678   const MachineType float_ty = FloatConvertType(opcode);
1679   const Operator* conv_op = ConvertOp(this, opcode);
1680   Node* trunc = nullptr;
1681   Node* converted_value = nullptr;
1682   const bool is_int32 =
1683       int_ty.representation() == MachineRepresentation::kWord32;
1684   if (is_int32) {
1685     trunc = Unop(TruncOp(float_ty), input);
1686     converted_value = graph()->NewNode(conv_op, trunc);
1687   } else {
1688     trunc = graph()->NewNode(conv_op, input);
1689     converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1690                                        trunc, graph()->start());
1691   }
1692   if (IsTrappingConvertOp(opcode)) {
1693     Node* test =
1694         ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1695     if (is_int32) {
1696       TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1697     } else {
1698       ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1699     }
1700     return converted_value;
1701   }
1702   Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1703                                    converted_value);
1704   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1705   tl_d.Chain(Control());
1706   Node* nan_test = Binop(NeOp(float_ty), input, input);
1707   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1708   nan_d.Nest(tl_d, true);
1709   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1710   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1711   sat_d.Nest(nan_d, false);
1712   Node* sat_val =
1713       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1714   Node* nan_val =
1715       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1716   return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1717 }
1718 
BuildI32AsmjsSConvertF32(Node * input)1719 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1720   MachineOperatorBuilder* m = mcgraph()->machine();
1721   // asm.js must use the wacky JS semantics.
1722   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1723   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1724 }
1725 
BuildI32AsmjsSConvertF64(Node * input)1726 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1727   MachineOperatorBuilder* m = mcgraph()->machine();
1728   // asm.js must use the wacky JS semantics.
1729   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1730 }
1731 
BuildI32AsmjsUConvertF32(Node * input)1732 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1733   MachineOperatorBuilder* m = mcgraph()->machine();
1734   // asm.js must use the wacky JS semantics.
1735   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1736   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1737 }
1738 
BuildI32AsmjsUConvertF64(Node * input)1739 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1740   MachineOperatorBuilder* m = mcgraph()->machine();
1741   // asm.js must use the wacky JS semantics.
1742   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1743 }
1744 
BuildBitCountingCall(Node * input,ExternalReference ref,MachineRepresentation input_type)1745 Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1746                                              MachineRepresentation input_type) {
1747   Node* stack_slot_param =
1748       graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1749 
1750   const Operator* store_op = mcgraph()->machine()->Store(
1751       StoreRepresentation(input_type, kNoWriteBarrier));
1752   *effect_ =
1753       graph()->NewNode(store_op, stack_slot_param, mcgraph()->Int32Constant(0),
1754                        input, *effect_, *control_);
1755 
1756   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1757   MachineSignature sig(1, 1, sig_types);
1758 
1759   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1760 
1761   return BuildCCall(&sig, function, stack_slot_param);
1762 }
1763 
BuildI32Ctz(Node * input)1764 Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1765   return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1766                               MachineRepresentation::kWord32);
1767 }
1768 
BuildI64Ctz(Node * input)1769 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1770   return Unop(wasm::kExprI64UConvertI32,
1771               BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1772                                    MachineRepresentation::kWord64));
1773 }
1774 
BuildI32Popcnt(Node * input)1775 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1776   return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1777                               MachineRepresentation::kWord32);
1778 }
1779 
BuildI64Popcnt(Node * input)1780 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1781   return Unop(
1782       wasm::kExprI64UConvertI32,
1783       BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1784                            MachineRepresentation::kWord64));
1785 }
1786 
BuildF32Trunc(Node * input)1787 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1788   MachineType type = MachineType::Float32();
1789   ExternalReference ref = ExternalReference::wasm_f32_trunc();
1790 
1791   return BuildCFuncInstruction(ref, type, input);
1792 }
1793 
BuildF32Floor(Node * input)1794 Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1795   MachineType type = MachineType::Float32();
1796   ExternalReference ref = ExternalReference::wasm_f32_floor();
1797   return BuildCFuncInstruction(ref, type, input);
1798 }
1799 
BuildF32Ceil(Node * input)1800 Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1801   MachineType type = MachineType::Float32();
1802   ExternalReference ref = ExternalReference::wasm_f32_ceil();
1803   return BuildCFuncInstruction(ref, type, input);
1804 }
1805 
BuildF32NearestInt(Node * input)1806 Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1807   MachineType type = MachineType::Float32();
1808   ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1809   return BuildCFuncInstruction(ref, type, input);
1810 }
1811 
BuildF64Trunc(Node * input)1812 Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1813   MachineType type = MachineType::Float64();
1814   ExternalReference ref = ExternalReference::wasm_f64_trunc();
1815   return BuildCFuncInstruction(ref, type, input);
1816 }
1817 
BuildF64Floor(Node * input)1818 Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1819   MachineType type = MachineType::Float64();
1820   ExternalReference ref = ExternalReference::wasm_f64_floor();
1821   return BuildCFuncInstruction(ref, type, input);
1822 }
1823 
BuildF64Ceil(Node * input)1824 Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1825   MachineType type = MachineType::Float64();
1826   ExternalReference ref = ExternalReference::wasm_f64_ceil();
1827   return BuildCFuncInstruction(ref, type, input);
1828 }
1829 
BuildF64NearestInt(Node * input)1830 Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1831   MachineType type = MachineType::Float64();
1832   ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1833   return BuildCFuncInstruction(ref, type, input);
1834 }
1835 
BuildF64Acos(Node * input)1836 Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1837   MachineType type = MachineType::Float64();
1838   ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1839   return BuildCFuncInstruction(ref, type, input);
1840 }
1841 
BuildF64Asin(Node * input)1842 Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1843   MachineType type = MachineType::Float64();
1844   ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1845   return BuildCFuncInstruction(ref, type, input);
1846 }
1847 
BuildF64Pow(Node * left,Node * right)1848 Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1849   MachineType type = MachineType::Float64();
1850   ExternalReference ref = ExternalReference::wasm_float64_pow();
1851   return BuildCFuncInstruction(ref, type, left, right);
1852 }
1853 
BuildF64Mod(Node * left,Node * right)1854 Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1855   MachineType type = MachineType::Float64();
1856   ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1857   return BuildCFuncInstruction(ref, type, left, right);
1858 }
1859 
BuildCFuncInstruction(ExternalReference ref,MachineType type,Node * input0,Node * input1)1860 Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1861                                               MachineType type, Node* input0,
1862                                               Node* input1) {
1863   // We do truncation by calling a C function which calculates the result.
1864   // The input is passed to the C function as a byte buffer holding the two
1865   // input doubles. We reserve this byte buffer as a stack slot, store the
1866   // parameters in this buffer slots, pass a pointer to the buffer to the C
1867   // function, and after calling the C function we collect the return value from
1868   // the buffer.
1869 
1870   const int type_size = ElementSizeInBytes(type.representation());
1871   const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1872   Node* stack_slot =
1873       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1874 
1875   const Operator* store_op = mcgraph()->machine()->Store(
1876       StoreRepresentation(type.representation(), kNoWriteBarrier));
1877   *effect_ = graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1878                               input0, *effect_, *control_);
1879 
1880   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1881 
1882   if (input1 != nullptr) {
1883     *effect_ = graph()->NewNode(store_op, stack_slot,
1884                                 mcgraph()->Int32Constant(type_size), input1,
1885                                 *effect_, *control_);
1886   }
1887 
1888   MachineType sig_types[] = {MachineType::Pointer()};
1889   MachineSignature sig(0, 1, sig_types);
1890   BuildCCall(&sig, function, stack_slot);
1891 
1892   const Operator* load_op = mcgraph()->machine()->Load(type);
1893   Node* load = graph()->NewNode(
1894       load_op, stack_slot, mcgraph()->Int32Constant(0), *effect_, *control_);
1895   *effect_ = load;
1896   return load;
1897 }
1898 
BuildF32SConvertI64(Node * input)1899 Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1900   // TODO(titzer/bradnelson): Check handlng of asm.js case.
1901   return BuildIntToFloatConversionInstruction(
1902       input, ExternalReference::wasm_int64_to_float32(),
1903       MachineRepresentation::kWord64, MachineType::Float32());
1904 }
BuildF32UConvertI64(Node * input)1905 Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1906   // TODO(titzer/bradnelson): Check handlng of asm.js case.
1907   return BuildIntToFloatConversionInstruction(
1908       input, ExternalReference::wasm_uint64_to_float32(),
1909       MachineRepresentation::kWord64, MachineType::Float32());
1910 }
BuildF64SConvertI64(Node * input)1911 Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1912   return BuildIntToFloatConversionInstruction(
1913       input, ExternalReference::wasm_int64_to_float64(),
1914       MachineRepresentation::kWord64, MachineType::Float64());
1915 }
BuildF64UConvertI64(Node * input)1916 Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1917   return BuildIntToFloatConversionInstruction(
1918       input, ExternalReference::wasm_uint64_to_float64(),
1919       MachineRepresentation::kWord64, MachineType::Float64());
1920 }
1921 
BuildIntToFloatConversionInstruction(Node * input,ExternalReference ref,MachineRepresentation parameter_representation,const MachineType result_type)1922 Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1923     Node* input, ExternalReference ref,
1924     MachineRepresentation parameter_representation,
1925     const MachineType result_type) {
1926   int stack_slot_size =
1927       std::max(ElementSizeInBytes(parameter_representation),
1928                ElementSizeInBytes(result_type.representation()));
1929   Node* stack_slot =
1930       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1931   const Operator* store_op = mcgraph()->machine()->Store(
1932       StoreRepresentation(parameter_representation, kNoWriteBarrier));
1933   *effect_ = graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1934                               input, *effect_, *control_);
1935   MachineType sig_types[] = {MachineType::Pointer()};
1936   MachineSignature sig(0, 1, sig_types);
1937   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1938   BuildCCall(&sig, function, stack_slot);
1939   const Operator* load_op = mcgraph()->machine()->Load(result_type);
1940   Node* load = graph()->NewNode(
1941       load_op, stack_slot, mcgraph()->Int32Constant(0), *effect_, *control_);
1942   *effect_ = load;
1943   return load;
1944 }
1945 
1946 namespace {
1947 
convert_ccall_ref(WasmGraphBuilder * builder,wasm::WasmOpcode opcode)1948 ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1949                                     wasm::WasmOpcode opcode) {
1950   switch (opcode) {
1951     case wasm::kExprI64SConvertF32:
1952     case wasm::kExprI64SConvertSatF32:
1953       return ExternalReference::wasm_float32_to_int64();
1954     case wasm::kExprI64UConvertF32:
1955     case wasm::kExprI64UConvertSatF32:
1956       return ExternalReference::wasm_float32_to_uint64();
1957     case wasm::kExprI64SConvertF64:
1958     case wasm::kExprI64SConvertSatF64:
1959       return ExternalReference::wasm_float64_to_int64();
1960     case wasm::kExprI64UConvertF64:
1961     case wasm::kExprI64UConvertSatF64:
1962       return ExternalReference::wasm_float64_to_uint64();
1963     default:
1964       UNREACHABLE();
1965   }
1966 }
1967 
1968 }  // namespace
1969 
BuildCcallConvertFloat(Node * input,wasm::WasmCodePosition position,wasm::WasmOpcode opcode)1970 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
1971                                                wasm::WasmCodePosition position,
1972                                                wasm::WasmOpcode opcode) {
1973   const MachineType int_ty = IntConvertType(opcode);
1974   const MachineType float_ty = FloatConvertType(opcode);
1975   ExternalReference call_ref = convert_ccall_ref(this, opcode);
1976   int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
1977                                  ElementSizeInBytes(float_ty.representation()));
1978   Node* stack_slot =
1979       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1980   const Operator* store_op = mcgraph()->machine()->Store(
1981       StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
1982   *effect_ = graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
1983                               *effect_, *control_);
1984   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1985   MachineSignature sig(1, 1, sig_types);
1986   Node* function =
1987       graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
1988   Node* overflow = BuildCCall(&sig, function, stack_slot);
1989   if (IsTrappingConvertOp(opcode)) {
1990     ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
1991     const Operator* load_op = mcgraph()->machine()->Load(int_ty);
1992     Node* load = graph()->NewNode(load_op, stack_slot, Int32Constant(0),
1993                                   *effect_, *control_);
1994     *effect_ = load;
1995     return load;
1996   }
1997   Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
1998   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1999   tl_d.Chain(Control());
2000   Node* nan_test = Binop(NeOp(float_ty), input, input);
2001   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2002   nan_d.Nest(tl_d, true);
2003   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2004   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2005   sat_d.Nest(nan_d, false);
2006   Node* sat_val =
2007       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2008   const Operator* load_op = mcgraph()->machine()->Load(int_ty);
2009   Node* load = graph()->NewNode(load_op, stack_slot, Int32Constant(0), *effect_,
2010                                 *control_);
2011   Node* nan_val =
2012       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2013   return tl_d.Phi(int_ty.representation(), nan_val, load);
2014 }
2015 
GrowMemory(Node * input)2016 Node* WasmGraphBuilder::GrowMemory(Node* input) {
2017   SetNeedsStackCheck();
2018   Diamond check_input_range(
2019       graph(), mcgraph()->common(),
2020       graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(), input,
2021                        mcgraph()->Uint32Constant(FLAG_wasm_max_mem_pages)),
2022       BranchHint::kTrue);
2023 
2024   check_input_range.Chain(*control_);
2025 
2026   Node* parameters[] = {BuildChangeUint31ToSmi(input)};
2027   Node* old_effect = *effect_;
2028   *control_ = check_input_range.if_true;
2029   Node* call = BuildCallToRuntime(Runtime::kWasmGrowMemory, parameters,
2030                                   arraysize(parameters));
2031 
2032   Node* result = BuildChangeSmiToInt32(call);
2033 
2034   result = check_input_range.Phi(MachineRepresentation::kWord32, result,
2035                                  mcgraph()->Int32Constant(-1));
2036   *effect_ = graph()->NewNode(mcgraph()->common()->EffectPhi(2), *effect_,
2037                               old_effect, check_input_range.merge);
2038   *control_ = check_input_range.merge;
2039   return result;
2040 }
2041 
GetExceptionEncodedSize(const wasm::WasmException * exception) const2042 uint32_t WasmGraphBuilder::GetExceptionEncodedSize(
2043     const wasm::WasmException* exception) const {
2044   const wasm::WasmExceptionSig* sig = exception->sig;
2045   uint32_t encoded_size = 0;
2046   for (size_t i = 0; i < sig->parameter_count(); ++i) {
2047     size_t byte_size = static_cast<size_t>(
2048         wasm::ValueTypes::ElementSizeInBytes(sig->GetParam(i)));
2049     DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0);
2050     DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement);
2051     encoded_size += byte_size / kBytesPerExceptionValuesArrayElement;
2052   }
2053   return encoded_size;
2054 }
2055 
Throw(uint32_t tag,const wasm::WasmException * exception,const Vector<Node * > values)2056 Node* WasmGraphBuilder::Throw(uint32_t tag,
2057                               const wasm::WasmException* exception,
2058                               const Vector<Node*> values) {
2059   SetNeedsStackCheck();
2060   uint32_t encoded_size = GetExceptionEncodedSize(exception);
2061   Node* create_parameters[] = {
2062       BuildChangeUint31ToSmi(ConvertExceptionTagToRuntimeId(tag)),
2063       BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2064   BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2065                      arraysize(create_parameters));
2066   uint32_t index = 0;
2067   const wasm::WasmExceptionSig* sig = exception->sig;
2068   MachineOperatorBuilder* m = mcgraph()->machine();
2069   for (size_t i = 0; i < sig->parameter_count(); ++i) {
2070     Node* value = values[i];
2071     switch (sig->GetParam(i)) {
2072       case wasm::kWasmF32:
2073         value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2074         V8_FALLTHROUGH;
2075       case wasm::kWasmI32:
2076         BuildEncodeException32BitValue(&index, value);
2077         break;
2078       case wasm::kWasmF64:
2079         value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2080         V8_FALLTHROUGH;
2081       case wasm::kWasmI64: {
2082         Node* upper32 = graph()->NewNode(
2083             m->TruncateInt64ToInt32(),
2084             Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2085         BuildEncodeException32BitValue(&index, upper32);
2086         Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2087         BuildEncodeException32BitValue(&index, lower32);
2088         break;
2089       }
2090       default:
2091         UNREACHABLE();
2092     }
2093   }
2094   DCHECK_EQ(encoded_size, index);
2095   return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2096 }
2097 
BuildEncodeException32BitValue(uint32_t * index,Node * value)2098 void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index,
2099                                                       Node* value) {
2100   MachineOperatorBuilder* machine = mcgraph()->machine();
2101   Node* upper_parameters[] = {
2102       BuildChangeUint31ToSmi(Int32Constant(*index)),
2103       BuildChangeUint31ToSmi(
2104           graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))),
2105   };
2106   BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters,
2107                      arraysize(upper_parameters));
2108   ++(*index);
2109   Node* lower_parameters[] = {
2110       BuildChangeUint31ToSmi(Int32Constant(*index)),
2111       BuildChangeUint31ToSmi(graph()->NewNode(machine->Word32And(), value,
2112                                               Int32Constant(0xFFFFu))),
2113   };
2114   BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters,
2115                      arraysize(lower_parameters));
2116   ++(*index);
2117 }
2118 
BuildDecodeException32BitValue(Node * const * values,uint32_t * index)2119 Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values,
2120                                                        uint32_t* index) {
2121   MachineOperatorBuilder* machine = mcgraph()->machine();
2122   Node* upper = BuildChangeSmiToInt32(values[*index]);
2123   (*index)++;
2124   upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2125   Node* lower = BuildChangeSmiToInt32(values[*index]);
2126   (*index)++;
2127   Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2128   return value;
2129 }
2130 
Rethrow()2131 Node* WasmGraphBuilder::Rethrow() {
2132   SetNeedsStackCheck();
2133   Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0);
2134   return result;
2135 }
2136 
ConvertExceptionTagToRuntimeId(uint32_t tag)2137 Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) {
2138   // TODO(kschimpf): Handle exceptions from different modules, when they are
2139   // linked at runtime.
2140   return Uint32Constant(tag);
2141 }
2142 
GetExceptionRuntimeId()2143 Node* WasmGraphBuilder::GetExceptionRuntimeId() {
2144   SetNeedsStackCheck();
2145   return BuildChangeSmiToInt32(
2146       BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0));
2147 }
2148 
GetExceptionValues(const wasm::WasmException * except_decl)2149 Node** WasmGraphBuilder::GetExceptionValues(
2150     const wasm::WasmException* except_decl) {
2151   // TODO(kschimpf): We need to move this code to the function-body-decoder.cc
2152   // in order to build landing-pad (exception) edges in case the runtime
2153   // call causes an exception.
2154 
2155   // Start by getting the encoded values from the exception.
2156   uint32_t encoded_size = GetExceptionEncodedSize(except_decl);
2157   Node** values = Buffer(encoded_size);
2158   for (uint32_t i = 0; i < encoded_size; ++i) {
2159     Node* parameters[] = {BuildChangeUint31ToSmi(Uint32Constant(i))};
2160     values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement,
2161                                    parameters, arraysize(parameters));
2162   }
2163 
2164   // Now convert the leading entries to the corresponding parameter values.
2165   uint32_t index = 0;
2166   const wasm::WasmExceptionSig* sig = except_decl->sig;
2167   for (size_t i = 0; i < sig->parameter_count(); ++i) {
2168     Node* value = BuildDecodeException32BitValue(values, &index);
2169     switch (wasm::ValueType type = sig->GetParam(i)) {
2170       case wasm::kWasmF32: {
2171         value = Unop(wasm::kExprF32ReinterpretI32, value);
2172         break;
2173       }
2174       case wasm::kWasmI32:
2175         break;
2176       case wasm::kWasmF64:
2177       case wasm::kWasmI64: {
2178         Node* upper =
2179             Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value),
2180                   Int64Constant(32));
2181         Node* lower = Unop(wasm::kExprI64UConvertI32,
2182                            BuildDecodeException32BitValue(values, &index));
2183         value = Binop(wasm::kExprI64Ior, upper, lower);
2184         if (type == wasm::kWasmF64) {
2185           value = Unop(wasm::kExprF64ReinterpretI64, value);
2186         }
2187         break;
2188       }
2189       default:
2190         UNREACHABLE();
2191     }
2192     values[i] = value;
2193   }
2194   DCHECK_EQ(index, encoded_size);
2195   return values;
2196 }
2197 
BuildI32DivS(Node * left,Node * right,wasm::WasmCodePosition position)2198 Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2199                                      wasm::WasmCodePosition position) {
2200   MachineOperatorBuilder* m = mcgraph()->machine();
2201   ZeroCheck32(wasm::kTrapDivByZero, right, position);
2202   Node* before = *control_;
2203   Node* denom_is_m1;
2204   Node* denom_is_not_m1;
2205   BranchExpectFalse(
2206       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2207       &denom_is_m1, &denom_is_not_m1);
2208   *control_ = denom_is_m1;
2209   TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2210   if (*control_ != denom_is_m1) {
2211     *control_ = graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2212                                  *control_);
2213   } else {
2214     *control_ = before;
2215   }
2216   return graph()->NewNode(m->Int32Div(), left, right, *control_);
2217 }
2218 
BuildI32RemS(Node * left,Node * right,wasm::WasmCodePosition position)2219 Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2220                                      wasm::WasmCodePosition position) {
2221   MachineOperatorBuilder* m = mcgraph()->machine();
2222 
2223   ZeroCheck32(wasm::kTrapRemByZero, right, position);
2224 
2225   Diamond d(
2226       graph(), mcgraph()->common(),
2227       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2228       BranchHint::kFalse);
2229   d.Chain(*control_);
2230 
2231   return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2232                graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2233 }
2234 
BuildI32DivU(Node * left,Node * right,wasm::WasmCodePosition position)2235 Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2236                                      wasm::WasmCodePosition position) {
2237   MachineOperatorBuilder* m = mcgraph()->machine();
2238   return graph()->NewNode(m->Uint32Div(), left, right,
2239                           ZeroCheck32(wasm::kTrapDivByZero, right, position));
2240 }
2241 
BuildI32RemU(Node * left,Node * right,wasm::WasmCodePosition position)2242 Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2243                                      wasm::WasmCodePosition position) {
2244   MachineOperatorBuilder* m = mcgraph()->machine();
2245   return graph()->NewNode(m->Uint32Mod(), left, right,
2246                           ZeroCheck32(wasm::kTrapRemByZero, right, position));
2247 }
2248 
BuildI32AsmjsDivS(Node * left,Node * right)2249 Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2250   MachineOperatorBuilder* m = mcgraph()->machine();
2251 
2252   Int32Matcher mr(right);
2253   if (mr.HasValue()) {
2254     if (mr.Value() == 0) {
2255       return mcgraph()->Int32Constant(0);
2256     } else if (mr.Value() == -1) {
2257       // The result is the negation of the left input.
2258       return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2259     }
2260     return graph()->NewNode(m->Int32Div(), left, right, *control_);
2261   }
2262 
2263   // asm.js semantics return 0 on divide or mod by zero.
2264   if (m->Int32DivIsSafe()) {
2265     // The hardware instruction does the right thing (e.g. arm).
2266     return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2267   }
2268 
2269   // Check denominator for zero.
2270   Diamond z(
2271       graph(), mcgraph()->common(),
2272       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2273       BranchHint::kFalse);
2274 
2275   // Check numerator for -1. (avoid minint / -1 case).
2276   Diamond n(
2277       graph(), mcgraph()->common(),
2278       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2279       BranchHint::kFalse);
2280 
2281   Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2282   Node* neg =
2283       graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2284 
2285   return n.Phi(
2286       MachineRepresentation::kWord32, neg,
2287       z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2288 }
2289 
BuildI32AsmjsRemS(Node * left,Node * right)2290 Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2291   CommonOperatorBuilder* c = mcgraph()->common();
2292   MachineOperatorBuilder* m = mcgraph()->machine();
2293   Node* const zero = mcgraph()->Int32Constant(0);
2294 
2295   Int32Matcher mr(right);
2296   if (mr.HasValue()) {
2297     if (mr.Value() == 0 || mr.Value() == -1) {
2298       return zero;
2299     }
2300     return graph()->NewNode(m->Int32Mod(), left, right, *control_);
2301   }
2302 
2303   // General case for signed integer modulus, with optimization for (unknown)
2304   // power of 2 right hand side.
2305   //
2306   //   if 0 < right then
2307   //     msk = right - 1
2308   //     if right & msk != 0 then
2309   //       left % right
2310   //     else
2311   //       if left < 0 then
2312   //         -(-left & msk)
2313   //       else
2314   //         left & msk
2315   //   else
2316   //     if right < -1 then
2317   //       left % right
2318   //     else
2319   //       zero
2320   //
2321   // Note: We do not use the Diamond helper class here, because it really hurts
2322   // readability with nested diamonds.
2323   Node* const minus_one = mcgraph()->Int32Constant(-1);
2324 
2325   const Operator* const merge_op = c->Merge(2);
2326   const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2327 
2328   Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2329   Node* branch0 =
2330       graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2331 
2332   Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2333   Node* true0;
2334   {
2335     Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2336 
2337     Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2338     Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2339 
2340     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2341     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2342 
2343     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2344     Node* false1;
2345     {
2346       Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2347       Node* branch2 =
2348           graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2349 
2350       Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2351       Node* true2 = graph()->NewNode(
2352           m->Int32Sub(), zero,
2353           graph()->NewNode(m->Word32And(),
2354                            graph()->NewNode(m->Int32Sub(), zero, left), msk));
2355 
2356       Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2357       Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2358 
2359       if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2360       false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2361     }
2362 
2363     if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2364     true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2365   }
2366 
2367   Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2368   Node* false0;
2369   {
2370     Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2371     Node* branch1 =
2372         graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2373 
2374     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2375     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2376 
2377     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2378     Node* false1 = zero;
2379 
2380     if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2381     false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2382   }
2383 
2384   Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2385   return graph()->NewNode(phi_op, true0, false0, merge0);
2386 }
2387 
BuildI32AsmjsDivU(Node * left,Node * right)2388 Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2389   MachineOperatorBuilder* m = mcgraph()->machine();
2390   // asm.js semantics return 0 on divide or mod by zero.
2391   if (m->Uint32DivIsSafe()) {
2392     // The hardware instruction does the right thing (e.g. arm).
2393     return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2394   }
2395 
2396   // Explicit check for x % 0.
2397   Diamond z(
2398       graph(), mcgraph()->common(),
2399       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2400       BranchHint::kFalse);
2401 
2402   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2403                graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2404                                 z.if_false));
2405 }
2406 
BuildI32AsmjsRemU(Node * left,Node * right)2407 Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2408   MachineOperatorBuilder* m = mcgraph()->machine();
2409   // asm.js semantics return 0 on divide or mod by zero.
2410   // Explicit check for x % 0.
2411   Diamond z(
2412       graph(), mcgraph()->common(),
2413       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2414       BranchHint::kFalse);
2415 
2416   Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2417                                z.if_false);
2418   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2419                rem);
2420 }
2421 
BuildI64DivS(Node * left,Node * right,wasm::WasmCodePosition position)2422 Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2423                                      wasm::WasmCodePosition position) {
2424   if (mcgraph()->machine()->Is32()) {
2425     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2426                           MachineType::Int64(), wasm::kTrapDivByZero, position);
2427   }
2428   ZeroCheck64(wasm::kTrapDivByZero, right, position);
2429   Node* before = *control_;
2430   Node* denom_is_m1;
2431   Node* denom_is_not_m1;
2432   BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2433                                      mcgraph()->Int64Constant(-1)),
2434                     &denom_is_m1, &denom_is_not_m1);
2435   *control_ = denom_is_m1;
2436   TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2437              std::numeric_limits<int64_t>::min(), position);
2438   if (*control_ != denom_is_m1) {
2439     *control_ = graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2440                                  *control_);
2441   } else {
2442     *control_ = before;
2443   }
2444   return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2445                           *control_);
2446 }
2447 
BuildI64RemS(Node * left,Node * right,wasm::WasmCodePosition position)2448 Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2449                                      wasm::WasmCodePosition position) {
2450   if (mcgraph()->machine()->Is32()) {
2451     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2452                           MachineType::Int64(), wasm::kTrapRemByZero, position);
2453   }
2454   ZeroCheck64(wasm::kTrapRemByZero, right, position);
2455   Diamond d(mcgraph()->graph(), mcgraph()->common(),
2456             graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2457                              mcgraph()->Int64Constant(-1)));
2458 
2459   d.Chain(*control_);
2460 
2461   Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2462                                d.if_false);
2463 
2464   return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2465                rem);
2466 }
2467 
BuildI64DivU(Node * left,Node * right,wasm::WasmCodePosition position)2468 Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2469                                      wasm::WasmCodePosition position) {
2470   if (mcgraph()->machine()->Is32()) {
2471     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2472                           MachineType::Int64(), wasm::kTrapDivByZero, position);
2473   }
2474   return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2475                           ZeroCheck64(wasm::kTrapDivByZero, right, position));
2476 }
BuildI64RemU(Node * left,Node * right,wasm::WasmCodePosition position)2477 Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2478                                      wasm::WasmCodePosition position) {
2479   if (mcgraph()->machine()->Is32()) {
2480     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2481                           MachineType::Int64(), wasm::kTrapRemByZero, position);
2482   }
2483   return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2484                           ZeroCheck64(wasm::kTrapRemByZero, right, position));
2485 }
2486 
BuildDiv64Call(Node * left,Node * right,ExternalReference ref,MachineType result_type,wasm::TrapReason trap_zero,wasm::WasmCodePosition position)2487 Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2488                                        ExternalReference ref,
2489                                        MachineType result_type,
2490                                        wasm::TrapReason trap_zero,
2491                                        wasm::WasmCodePosition position) {
2492   Node* stack_slot =
2493       graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2494 
2495   const Operator* store_op = mcgraph()->machine()->Store(
2496       StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2497   *effect_ = graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2498                               left, *effect_, *control_);
2499   *effect_ = graph()->NewNode(store_op, stack_slot,
2500                               mcgraph()->Int32Constant(sizeof(double)), right,
2501                               *effect_, *control_);
2502 
2503   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2504   MachineSignature sig(1, 1, sig_types);
2505 
2506   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2507   Node* call = BuildCCall(&sig, function, stack_slot);
2508 
2509   ZeroCheck32(trap_zero, call, position);
2510   TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2511   const Operator* load_op = mcgraph()->machine()->Load(result_type);
2512   Node* load = graph()->NewNode(
2513       load_op, stack_slot, mcgraph()->Int32Constant(0), *effect_, *control_);
2514   *effect_ = load;
2515   return load;
2516 }
2517 
2518 template <typename... Args>
BuildCCall(MachineSignature * sig,Node * function,Args...args)2519 Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2520                                    Args... args) {
2521   DCHECK_LE(sig->return_count(), 1);
2522   DCHECK_EQ(sizeof...(args), sig->parameter_count());
2523   Node* const call_args[] = {function, args..., *effect_, *control_};
2524 
2525   auto call_descriptor =
2526       Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2527 
2528   const Operator* op = mcgraph()->common()->Call(call_descriptor);
2529   Node* call = graph()->NewNode(op, arraysize(call_args), call_args);
2530   *effect_ = call;
2531   return call;
2532 }
2533 
BuildWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,Node * instance_node,UseRetpoline use_retpoline)2534 Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2535                                       Node*** rets,
2536                                       wasm::WasmCodePosition position,
2537                                       Node* instance_node,
2538                                       UseRetpoline use_retpoline) {
2539   if (instance_node == nullptr) {
2540     DCHECK_NOT_NULL(instance_node_);
2541     instance_node = instance_node_.get();
2542   }
2543   SetNeedsStackCheck();
2544   const size_t params = sig->parameter_count();
2545   const size_t extra = 3;  // instance_node, effect, and control.
2546   const size_t count = 1 + params + extra;
2547 
2548   // Reallocate the buffer to make space for extra inputs.
2549   args = Realloc(args, 1 + params, count);
2550 
2551   // Make room for the instance_node parameter at index 1, just after code.
2552   memmove(&args[2], &args[1], params * sizeof(Node*));
2553   args[1] = instance_node;
2554 
2555   // Add effect and control inputs.
2556   args[params + 2] = *effect_;
2557   args[params + 3] = *control_;
2558 
2559   auto call_descriptor =
2560       GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2561   const Operator* op = mcgraph()->common()->Call(call_descriptor);
2562   Node* call = graph()->NewNode(op, static_cast<int>(count), args);
2563   DCHECK(position == wasm::kNoCodePosition || position > 0);
2564   if (position > 0) SetSourcePosition(call, position);
2565 
2566   *effect_ = call;
2567   size_t ret_count = sig->return_count();
2568   if (ret_count == 0) return call;  // No return value.
2569 
2570   *rets = Buffer(ret_count);
2571   if (ret_count == 1) {
2572     // Only a single return value.
2573     (*rets)[0] = call;
2574   } else {
2575     // Create projections for all return values.
2576     for (size_t i = 0; i < ret_count; i++) {
2577       (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2578                                     graph()->start());
2579     }
2580   }
2581   return call;
2582 }
2583 
BuildImportWasmCall(wasm::FunctionSig * sig,Node ** args,Node *** rets,wasm::WasmCodePosition position,int func_index)2584 Node* WasmGraphBuilder::BuildImportWasmCall(wasm::FunctionSig* sig, Node** args,
2585                                             Node*** rets,
2586                                             wasm::WasmCodePosition position,
2587                                             int func_index) {
2588   // Load the instance from the imported_instances array at a known offset.
2589   Node* imported_instances = LOAD_INSTANCE_FIELD(ImportedFunctionInstances,
2590                                                  MachineType::TaggedPointer());
2591   Node* instance_node = LOAD_FIXED_ARRAY_SLOT(imported_instances, func_index);
2592 
2593   // Load the target from the imported_targets array at a known offset.
2594   Node* imported_targets =
2595       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2596   Node* target_node = graph()->NewNode(
2597       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2598       mcgraph()->Int32Constant(func_index * sizeof(Address)),
2599       mcgraph()->graph()->start(), mcgraph()->graph()->start());
2600   args[0] = target_node;
2601   return BuildWasmCall(sig, args, rets, position, instance_node,
2602                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2603 }
2604 
CallDirect(uint32_t index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2605 Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2606                                    wasm::WasmCodePosition position) {
2607   DCHECK_NULL(args[0]);
2608   wasm::FunctionSig* sig = env_->module->functions[index].sig;
2609 
2610   if (env_ && index < env_->module->num_imported_functions) {
2611     // Call to an imported function.
2612     return BuildImportWasmCall(sig, args, rets, position, index);
2613   }
2614 
2615   // A direct call to a wasm function defined in this module.
2616   // Just encode the function index. This will be patched at instantiation.
2617   Address code = static_cast<Address>(index);
2618   args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2619 
2620   return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2621 }
2622 
CallIndirect(uint32_t sig_index,Node ** args,Node *** rets,wasm::WasmCodePosition position)2623 Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2624                                      Node*** rets,
2625                                      wasm::WasmCodePosition position) {
2626   DCHECK_NOT_NULL(args[0]);
2627   DCHECK_NOT_NULL(env_);
2628 
2629   // Assume only one table for now.
2630   wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2631 
2632   Node* ift_size =
2633       LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2634 
2635   MachineOperatorBuilder* machine = mcgraph()->machine();
2636   Node* key = args[0];
2637 
2638   // Bounds check against the table size.
2639   Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2640   TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2641 
2642   // Mask the key to prevent SSCA.
2643   if (untrusted_code_mitigations_) {
2644     // mask = ((key - size) & ~key) >> 31
2645     Node* neg_key =
2646         graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2647     Node* masked_diff = graph()->NewNode(
2648         machine->Word32And(),
2649         graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2650     Node* mask =
2651         graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2652     key = graph()->NewNode(machine->Word32And(), key, mask);
2653   }
2654 
2655   // Load signature from the table and check.
2656   Node* ift_sig_ids =
2657       LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2658 
2659   int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2660   Node* scaled_key =
2661       graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2));
2662   const Operator* add = nullptr;
2663   if (machine->Is64()) {
2664     scaled_key = graph()->NewNode(machine->ChangeUint32ToUint64(), scaled_key);
2665     add = machine->Int64Add();
2666   } else {
2667     add = machine->Int32Add();
2668   }
2669 
2670   Node* loaded_sig =
2671       graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
2672                        scaled_key, *effect_, *control_);
2673   Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2674                                      Int32Constant(expected_sig_id));
2675 
2676   TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2677 
2678   Node* ift_targets =
2679       LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2680   Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableInstances,
2681                                             MachineType::TaggedPointer());
2682 
2683   scaled_key = graph()->NewNode(machine->Word32Shl(), key,
2684                                 Int32Constant(kPointerSizeLog2));
2685 
2686   Node* target = graph()->NewNode(machine->Load(MachineType::Pointer()),
2687                                   ift_targets, scaled_key, *effect_, *control_);
2688 
2689   auto access = AccessBuilder::ForFixedArrayElement();
2690   Node* target_instance = graph()->NewNode(
2691       machine->Load(MachineType::TaggedPointer()),
2692       graph()->NewNode(add, ift_instances, scaled_key),
2693       Int32Constant(access.header_size - access.tag()), *effect_, *control_);
2694 
2695   args[0] = target;
2696 
2697   return BuildWasmCall(sig, args, rets, position, target_instance,
2698                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2699 }
2700 
BuildI32Rol(Node * left,Node * right)2701 Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2702   // Implement Rol by Ror since TurboFan does not have Rol opcode.
2703   // TODO(weiliang): support Word32Rol opcode in TurboFan.
2704   Int32Matcher m(right);
2705   if (m.HasValue()) {
2706     return Binop(wasm::kExprI32Ror, left,
2707                  mcgraph()->Int32Constant(32 - m.Value()));
2708   } else {
2709     return Binop(wasm::kExprI32Ror, left,
2710                  Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2711   }
2712 }
2713 
BuildI64Rol(Node * left,Node * right)2714 Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2715   // Implement Rol by Ror since TurboFan does not have Rol opcode.
2716   // TODO(weiliang): support Word64Rol opcode in TurboFan.
2717   Int64Matcher m(right);
2718   if (m.HasValue()) {
2719     return Binop(wasm::kExprI64Ror, left,
2720                  mcgraph()->Int64Constant(64 - m.Value()));
2721   } else {
2722     return Binop(wasm::kExprI64Ror, left,
2723                  Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2724   }
2725 }
2726 
Invert(Node * node)2727 Node* WasmGraphBuilder::Invert(Node* node) {
2728   return Unop(wasm::kExprI32Eqz, node);
2729 }
2730 
CanCover(Node * value,IrOpcode::Value opcode)2731 bool CanCover(Node* value, IrOpcode::Value opcode) {
2732   if (value->opcode() != opcode) return false;
2733   bool first = true;
2734   for (Edge const edge : value->use_edges()) {
2735     if (NodeProperties::IsControlEdge(edge)) continue;
2736     if (NodeProperties::IsEffectEdge(edge)) continue;
2737     DCHECK(NodeProperties::IsValueEdge(edge));
2738     if (!first) return false;
2739     first = false;
2740   }
2741   return true;
2742 }
2743 
BuildChangeInt32ToSmi(Node * value)2744 Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2745   if (mcgraph()->machine()->Is64()) {
2746     value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2747   }
2748   return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2749                           BuildSmiShiftBitsConstant());
2750 }
2751 
BuildChangeUint31ToSmi(Node * value)2752 Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2753   return graph()->NewNode(mcgraph()->machine()->WordShl(),
2754                           Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2755 }
2756 
BuildSmiShiftBitsConstant()2757 Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2758   return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2759 }
2760 
BuildChangeSmiToInt32(Node * value)2761 Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2762   value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2763                            BuildSmiShiftBitsConstant());
2764   if (mcgraph()->machine()->Is64()) {
2765     value =
2766         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2767   }
2768   return value;
2769 }
2770 
InitInstanceCache(WasmInstanceCacheNodes * instance_cache)2771 void WasmGraphBuilder::InitInstanceCache(
2772     WasmInstanceCacheNodes* instance_cache) {
2773   DCHECK_NOT_NULL(instance_node_);
2774   DCHECK_NOT_NULL(*control_);
2775   DCHECK_NOT_NULL(*effect_);
2776 
2777   // Load the memory start.
2778   Node* mem_start = graph()->NewNode(
2779       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2780       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2781       *effect_, *control_);
2782   *effect_ = mem_start;
2783   instance_cache->mem_start = mem_start;
2784 
2785   // Load the memory size.
2786   Node* mem_size = graph()->NewNode(
2787       mcgraph()->machine()->Load(MachineType::Uint32()), instance_node_.get(),
2788       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2789       *effect_, *control_);
2790   *effect_ = mem_size;
2791   instance_cache->mem_size = mem_size;
2792 
2793   if (untrusted_code_mitigations_) {
2794     // Load the memory mask.
2795     Node* mem_mask = graph()->NewNode(
2796         mcgraph()->machine()->Load(MachineType::Uint32()), instance_node_.get(),
2797         mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryMask)),
2798         *effect_, *control_);
2799     *effect_ = mem_mask;
2800     instance_cache->mem_mask = mem_mask;
2801   } else {
2802     // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2803     instance_cache->mem_mask = nullptr;
2804   }
2805 }
2806 
PrepareInstanceCacheForLoop(WasmInstanceCacheNodes * instance_cache,Node * control)2807 void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2808     WasmInstanceCacheNodes* instance_cache, Node* control) {
2809 #define INTRODUCE_PHI(field, rep)                                            \
2810   instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2811                                            instance_cache->field, control);
2812 
2813   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2814   INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2815   if (untrusted_code_mitigations_) {
2816     INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2817   }
2818 
2819 #undef INTRODUCE_PHI
2820 }
2821 
NewInstanceCacheMerge(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2822 void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2823                                              WasmInstanceCacheNodes* from,
2824                                              Node* merge) {
2825 #define INTRODUCE_PHI(field, rep)                                            \
2826   if (to->field != from->field) {                                            \
2827     Node* vals[] = {to->field, from->field, merge};                          \
2828     to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2829   }
2830 
2831   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2832   INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2833   if (untrusted_code_mitigations_) {
2834     INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2835   }
2836 
2837 #undef INTRODUCE_PHI
2838 }
2839 
MergeInstanceCacheInto(WasmInstanceCacheNodes * to,WasmInstanceCacheNodes * from,Node * merge)2840 void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2841                                               WasmInstanceCacheNodes* from,
2842                                               Node* merge) {
2843   to->mem_size = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge,
2844                                       to->mem_size, from->mem_size);
2845   to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2846                                        merge, to->mem_start, from->mem_start);
2847   if (untrusted_code_mitigations_) {
2848     to->mem_mask = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge,
2849                                         to->mem_mask, from->mem_mask);
2850   }
2851 }
2852 
CreateOrMergeIntoPhi(MachineRepresentation rep,Node * merge,Node * tnode,Node * fnode)2853 Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
2854                                              Node* merge, Node* tnode,
2855                                              Node* fnode) {
2856   if (IsPhiWithMerge(tnode, merge)) {
2857     AppendToPhi(tnode, fnode);
2858   } else if (tnode != fnode) {
2859     uint32_t count = merge->InputCount();
2860     // + 1 for the merge node.
2861     Node** vals = Buffer(count + 1);
2862     for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
2863     vals[count - 1] = fnode;
2864     vals[count] = merge;
2865     return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
2866                             vals);
2867   }
2868   return tnode;
2869 }
2870 
CreateOrMergeIntoEffectPhi(Node * merge,Node * tnode,Node * fnode)2871 Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
2872                                                    Node* fnode) {
2873   if (IsPhiWithMerge(tnode, merge)) {
2874     AppendToPhi(tnode, fnode);
2875   } else if (tnode != fnode) {
2876     uint32_t count = merge->InputCount();
2877     Node** effects = Buffer(count);
2878     for (uint32_t j = 0; j < count - 1; j++) {
2879       effects[j] = tnode;
2880     }
2881     effects[count - 1] = fnode;
2882     tnode = EffectPhi(count, effects, merge);
2883   }
2884   return tnode;
2885 }
2886 
GetGlobalBaseAndOffset(MachineType mem_type,const wasm::WasmGlobal & global,Node ** base_node,Node ** offset_node)2887 void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
2888                                               const wasm::WasmGlobal& global,
2889                                               Node** base_node,
2890                                               Node** offset_node) {
2891   DCHECK_NOT_NULL(instance_node_);
2892   if (global.mutability && global.imported) {
2893     DCHECK(FLAG_experimental_wasm_mut_global);
2894     if (imported_mutable_globals_ == nullptr) {
2895       // Load imported_mutable_globals_ from the instance object at runtime.
2896       imported_mutable_globals_ = graph()->NewNode(
2897           mcgraph()->machine()->Load(MachineType::UintPtr()),
2898           instance_node_.get(),
2899           mcgraph()->Int32Constant(
2900               WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
2901           graph()->start(), graph()->start());
2902     }
2903     *base_node = graph()->NewNode(
2904         mcgraph()->machine()->Load(MachineType::UintPtr()),
2905         imported_mutable_globals_.get(),
2906         mcgraph()->Int32Constant(global.index * sizeof(Address)), *effect_,
2907         *control_);
2908     *offset_node = mcgraph()->Int32Constant(0);
2909     *effect_ = *base_node;
2910   } else {
2911     if (globals_start_ == nullptr) {
2912       // Load globals_start from the instance object at runtime.
2913       // TODO(wasm): we currently generate only one load of the {globals_start}
2914       // start per graph, which means it can be placed anywhere by the
2915       // scheduler. This is legal because the globals_start should never change.
2916       // However, in some cases (e.g. if the instance object is already in a
2917       // register), it is slightly more efficient to reload this value from the
2918       // instance object. Since this depends on register allocation, it is not
2919       // possible to express in the graph, and would essentially constitute a
2920       // "mem2reg" optimization in TurboFan.
2921       globals_start_ = graph()->NewNode(
2922           mcgraph()->machine()->Load(MachineType::UintPtr()),
2923           instance_node_.get(),
2924           mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
2925           graph()->start(), graph()->start());
2926     }
2927     *base_node = globals_start_.get();
2928     *offset_node = mcgraph()->Int32Constant(global.offset);
2929 
2930     if (mem_type == MachineType::Simd128() && global.offset != 0) {
2931       // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
2932       *base_node =
2933           graph()->NewNode(kPointerSize == 4 ? mcgraph()->machine()->Int32Add()
2934                                              : mcgraph()->machine()->Int64Add(),
2935                            *base_node, *offset_node);
2936       *offset_node = mcgraph()->Int32Constant(0);
2937     }
2938   }
2939 }
2940 
MemBuffer(uint32_t offset)2941 Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
2942   DCHECK_NOT_NULL(instance_cache_);
2943   Node* mem_start = instance_cache_->mem_start;
2944   DCHECK_NOT_NULL(mem_start);
2945   if (offset == 0) return mem_start;
2946   return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
2947                           mcgraph()->IntPtrConstant(offset));
2948 }
2949 
CurrentMemoryPages()2950 Node* WasmGraphBuilder::CurrentMemoryPages() {
2951   // CurrentMemoryPages can not be called from asm.js.
2952   DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin());
2953   DCHECK_NOT_NULL(instance_cache_);
2954   Node* mem_size = instance_cache_->mem_size;
2955   DCHECK_NOT_NULL(mem_size);
2956   return graph()->NewNode(
2957       mcgraph()->machine()->Word32Shr(), mem_size,
2958       mcgraph()->Int32Constant(WhichPowerOf2(wasm::kWasmPageSize)));
2959 }
2960 
2961 // Only call this function for code which is not reused across instantiations,
2962 // as we do not patch the embedded js_context.
BuildCallToRuntimeWithContext(Runtime::FunctionId f,Node * js_context,Node ** parameters,int parameter_count)2963 Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f,
2964                                                       Node* js_context,
2965                                                       Node** parameters,
2966                                                       int parameter_count) {
2967   const Runtime::Function* fun = Runtime::FunctionForId(f);
2968   auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
2969       mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
2970       CallDescriptor::kNoFlags);
2971   // CEntryStubConstant nodes have to be created and cached in the main
2972   // thread. At the moment this is only done for CEntryStubConstant(1).
2973   DCHECK_EQ(1, fun->result_size);
2974   // At the moment we only allow 4 parameters. If more parameters are needed,
2975   // increase this constant accordingly.
2976   static const int kMaxParams = 4;
2977   DCHECK_GE(kMaxParams, parameter_count);
2978   Node* inputs[kMaxParams + 6];
2979   int count = 0;
2980   inputs[count++] = CEntryStub();
2981   for (int i = 0; i < parameter_count; i++) {
2982     inputs[count++] = parameters[i];
2983   }
2984   inputs[count++] =
2985       mcgraph()->ExternalConstant(ExternalReference::Create(f));  // ref
2986   inputs[count++] = mcgraph()->Int32Constant(fun->nargs);         // arity
2987   inputs[count++] = js_context;                                   // js_context
2988   inputs[count++] = *effect_;
2989   inputs[count++] = *control_;
2990 
2991   Node* node = mcgraph()->graph()->NewNode(
2992       mcgraph()->common()->Call(call_descriptor), count, inputs);
2993   *effect_ = node;
2994 
2995   return node;
2996 }
2997 
BuildCallToRuntime(Runtime::FunctionId f,Node ** parameters,int parameter_count)2998 Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
2999                                            Node** parameters,
3000                                            int parameter_count) {
3001   return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3002                                        parameter_count);
3003 }
3004 
GetGlobal(uint32_t index)3005 Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3006   MachineType mem_type =
3007       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3008   Node* base = nullptr;
3009   Node* offset = nullptr;
3010   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3011                          &offset);
3012   Node* node = graph()->NewNode(mcgraph()->machine()->Load(mem_type), base,
3013                                 offset, *effect_, *control_);
3014   *effect_ = node;
3015   return node;
3016 }
3017 
SetGlobal(uint32_t index,Node * val)3018 Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3019   MachineType mem_type =
3020       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3021   Node* base = nullptr;
3022   Node* offset = nullptr;
3023   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3024                          &offset);
3025   const Operator* op = mcgraph()->machine()->Store(
3026       StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3027   Node* node = graph()->NewNode(op, base, offset, val, *effect_, *control_);
3028   *effect_ = node;
3029   return node;
3030 }
3031 
BoundsCheckMem(uint8_t access_size,Node * index,uint32_t offset,wasm::WasmCodePosition position,EnforceBoundsCheck enforce_check)3032 Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3033                                        uint32_t offset,
3034                                        wasm::WasmCodePosition position,
3035                                        EnforceBoundsCheck enforce_check) {
3036   if (FLAG_wasm_no_bounds_checks) return Uint32ToUintptr(index);
3037   DCHECK_NOT_NULL(instance_cache_);
3038   Node* mem_size = instance_cache_->mem_size;
3039   DCHECK_NOT_NULL(mem_size);
3040 
3041   auto m = mcgraph()->machine();
3042   if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3043     // Simply zero out the 32-bits on 64-bit targets and let the trap handler
3044     // do its job.
3045     return Uint32ToUintptr(index);
3046   }
3047 
3048   uint32_t min_size = env_->module->initial_pages * wasm::kWasmPageSize;
3049   uint32_t max_size =
3050       (env_->module->has_maximum_pages ? env_->module->maximum_pages
3051                                        : wasm::kV8MaxWasmMemoryPages) *
3052       wasm::kWasmPageSize;
3053 
3054   if (access_size > max_size || offset > max_size - access_size) {
3055     // The access will be out of bounds, even for the largest memory.
3056     TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3057     return mcgraph()->IntPtrConstant(0);
3058   }
3059   DCHECK_LE(1, access_size);
3060   // This computation cannot overflow, since
3061   //   {offset <= max_size - access_size <= kMaxUint32 - access_size}.
3062   // It also cannot underflow, since {access_size >= 1}.
3063   uint32_t end_offset = offset + access_size - 1;
3064   Node* end_offset_node = Int32Constant(end_offset);
3065 
3066   // The accessed memory is [index + offset, index + end_offset].
3067   // Check that the last read byte (at {index + end_offset}) is in bounds.
3068   // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3069   //    compute {effective_size} as {mem_size - end_offset)}.
3070   //    {effective_size} is >= 1 if condition 1) holds.
3071   // 2) Check that {index + end_offset < mem_size} by
3072   //    - computing {effective_size} as {mem_size - end_offset} and
3073   //    - checking that {index < effective_size}.
3074 
3075   if (end_offset >= min_size) {
3076     // The end offset is larger than the smallest memory.
3077     // Dynamically check the end offset against the actual memory size, which
3078     // is not known at compile time.
3079     Node* cond = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
3080                                   end_offset_node, mem_size);
3081     TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3082   } else {
3083     // The end offset is within the bounds of the smallest memory, so only
3084     // one check is required. Check to see if the index is also a constant.
3085     Uint32Matcher match(index);
3086     if (match.HasValue()) {
3087       uint32_t index_val = match.Value();
3088       if (index_val < min_size - end_offset) {
3089         // The input index is a constant and everything is statically within
3090         // bounds of the smallest possible memory.
3091         return Uint32ToUintptr(index);
3092       }
3093     }
3094   }
3095 
3096   // This produces a positive number, since {end_offset < min_size <= mem_size}.
3097   Node* effective_size = graph()->NewNode(mcgraph()->machine()->Int32Sub(),
3098                                           mem_size, end_offset_node);
3099 
3100   // Introduce the actual bounds check.
3101   Node* cond = graph()->NewNode(m->Uint32LessThan(), index, effective_size);
3102   TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3103 
3104   if (untrusted_code_mitigations_) {
3105     // In the fallthrough case, condition the index with the memory mask.
3106     Node* mem_mask = instance_cache_->mem_mask;
3107     DCHECK_NOT_NULL(mem_mask);
3108     index = graph()->NewNode(m->Word32And(), index, mem_mask);
3109   }
3110   return Uint32ToUintptr(index);
3111 }
3112 
GetSafeLoadOperator(int offset,wasm::ValueType type)3113 const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3114                                                       wasm::ValueType type) {
3115   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3116   MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3117   if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3118                             wasm::ValueTypes::MachineRepresentationFor(type))) {
3119     return mcgraph()->machine()->Load(mach_type);
3120   }
3121   return mcgraph()->machine()->UnalignedLoad(mach_type);
3122 }
3123 
GetSafeStoreOperator(int offset,wasm::ValueType type)3124 const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3125                                                        wasm::ValueType type) {
3126   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3127   MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3128   if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3129     StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3130     return mcgraph()->machine()->Store(store_rep);
3131   }
3132   UnalignedStoreRepresentation store_rep(rep);
3133   return mcgraph()->machine()->UnalignedStore(store_rep);
3134 }
3135 
TraceMemoryOperation(bool is_store,MachineRepresentation rep,Node * index,uint32_t offset,wasm::WasmCodePosition position)3136 Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3137                                              MachineRepresentation rep,
3138                                              Node* index, uint32_t offset,
3139                                              wasm::WasmCodePosition position) {
3140   int kAlign = 4;  // Ensure that the LSB is 0, such that this looks like a Smi.
3141   Node* info = graph()->NewNode(
3142       mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3143 
3144   Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3145                                    Int32Constant(offset), index);
3146   auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3147     *effect_ = graph()->NewNode(
3148         mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3149         info, mcgraph()->Int32Constant(offset), data, *effect_, *control_);
3150   };
3151   // Store address, is_store, and mem_rep.
3152   store(offsetof(wasm::MemoryTracingInfo, address),
3153         MachineRepresentation::kWord32, address);
3154   store(offsetof(wasm::MemoryTracingInfo, is_store),
3155         MachineRepresentation::kWord8,
3156         mcgraph()->Int32Constant(is_store ? 1 : 0));
3157   store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3158         MachineRepresentation::kWord8,
3159         mcgraph()->Int32Constant(static_cast<int>(rep)));
3160 
3161   Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3162   SetSourcePosition(call, position);
3163   return call;
3164 }
3165 
LoadMem(wasm::ValueType type,MachineType memtype,Node * index,uint32_t offset,uint32_t alignment,wasm::WasmCodePosition position)3166 Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3167                                 Node* index, uint32_t offset,
3168                                 uint32_t alignment,
3169                                 wasm::WasmCodePosition position) {
3170   Node* load;
3171 
3172   // Wasm semantics throw on OOB. Introduce explicit bounds check and
3173   // conditioning when not using the trap handler.
3174   index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3175                          position, kCanOmitBoundsCheck);
3176 
3177   if (memtype.representation() == MachineRepresentation::kWord8 ||
3178       mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3179     if (use_trap_handler()) {
3180       load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3181                               MemBuffer(offset), index, *effect_, *control_);
3182       SetSourcePosition(load, position);
3183     } else {
3184       load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3185                               MemBuffer(offset), index, *effect_, *control_);
3186     }
3187   } else {
3188     // TODO(eholk): Support unaligned loads with trap handlers.
3189     DCHECK(!use_trap_handler());
3190     load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3191                             MemBuffer(offset), index, *effect_, *control_);
3192   }
3193 
3194   *effect_ = load;
3195 
3196 #if defined(V8_TARGET_BIG_ENDIAN)
3197   load = BuildChangeEndiannessLoad(load, memtype, type);
3198 #endif
3199 
3200   if (type == wasm::kWasmI64 &&
3201       ElementSizeInBytes(memtype.representation()) < 8) {
3202     // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3203     if (memtype.IsSigned()) {
3204       // sign extend
3205       load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3206     } else {
3207       // zero extend
3208       load =
3209           graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3210     }
3211   }
3212 
3213   if (FLAG_wasm_trace_memory) {
3214     TraceMemoryOperation(false, memtype.representation(), index, offset,
3215                          position);
3216   }
3217 
3218   return load;
3219 }
3220 
StoreMem(MachineRepresentation mem_rep,Node * index,uint32_t offset,uint32_t alignment,Node * val,wasm::WasmCodePosition position,wasm::ValueType type)3221 Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3222                                  uint32_t offset, uint32_t alignment, Node* val,
3223                                  wasm::WasmCodePosition position,
3224                                  wasm::ValueType type) {
3225   Node* store;
3226 
3227   index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3228                          position, kCanOmitBoundsCheck);
3229 
3230 #if defined(V8_TARGET_BIG_ENDIAN)
3231   val = BuildChangeEndiannessStore(val, mem_rep, type);
3232 #endif
3233 
3234   if (mem_rep == MachineRepresentation::kWord8 ||
3235       mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3236     if (use_trap_handler()) {
3237       store =
3238           graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3239                            MemBuffer(offset), index, val, *effect_, *control_);
3240       SetSourcePosition(store, position);
3241     } else {
3242       StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3243       store =
3244           graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3245                            index, val, *effect_, *control_);
3246     }
3247   } else {
3248     // TODO(eholk): Support unaligned stores with trap handlers.
3249     DCHECK(!use_trap_handler());
3250     UnalignedStoreRepresentation rep(mem_rep);
3251     store =
3252         graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3253                          MemBuffer(offset), index, val, *effect_, *control_);
3254   }
3255 
3256   *effect_ = store;
3257 
3258   if (FLAG_wasm_trace_memory) {
3259     TraceMemoryOperation(true, mem_rep, index, offset, position);
3260   }
3261 
3262   return store;
3263 }
3264 
3265 namespace {
GetAsmJsOOBValue(MachineRepresentation rep,MachineGraph * mcgraph)3266 Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3267   switch (rep) {
3268     case MachineRepresentation::kWord8:
3269     case MachineRepresentation::kWord16:
3270     case MachineRepresentation::kWord32:
3271       return mcgraph->Int32Constant(0);
3272     case MachineRepresentation::kWord64:
3273       return mcgraph->Int64Constant(0);
3274     case MachineRepresentation::kFloat32:
3275       return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3276     case MachineRepresentation::kFloat64:
3277       return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3278     default:
3279       UNREACHABLE();
3280   }
3281 }
3282 }  // namespace
3283 
BuildAsmjsLoadMem(MachineType type,Node * index)3284 Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3285   DCHECK_NOT_NULL(instance_cache_);
3286   Node* mem_start = instance_cache_->mem_start;
3287   Node* mem_size = instance_cache_->mem_size;
3288   DCHECK_NOT_NULL(mem_start);
3289   DCHECK_NOT_NULL(mem_size);
3290 
3291   // Asm.js semantics are defined along the lines of typed arrays, hence OOB
3292   // reads return {undefined} coerced to the result type (0 for integers, NaN
3293   // for float and double).
3294   // Note that we check against the memory size ignoring the size of the
3295   // stored value, which is conservative if misaligned. Technically, asm.js
3296   // should never have misaligned accesses.
3297   Diamond bounds_check(
3298       graph(), mcgraph()->common(),
3299       graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3300       BranchHint::kTrue);
3301   bounds_check.Chain(*control_);
3302 
3303   if (untrusted_code_mitigations_) {
3304     // Condition the index with the memory mask.
3305     Node* mem_mask = instance_cache_->mem_mask;
3306     DCHECK_NOT_NULL(mem_mask);
3307     index =
3308         graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3309   }
3310 
3311   index = Uint32ToUintptr(index);
3312   Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3313                                 index, *effect_, bounds_check.if_true);
3314   Node* value_phi =
3315       bounds_check.Phi(type.representation(), load,
3316                        GetAsmJsOOBValue(type.representation(), mcgraph()));
3317   Node* effect_phi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), load,
3318                                       *effect_, bounds_check.merge);
3319   *effect_ = effect_phi;
3320   *control_ = bounds_check.merge;
3321   return value_phi;
3322 }
3323 
Uint32ToUintptr(Node * node)3324 Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3325   if (mcgraph()->machine()->Is32()) return node;
3326   return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3327 }
3328 
BuildAsmjsStoreMem(MachineType type,Node * index,Node * val)3329 Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3330                                            Node* val) {
3331   DCHECK_NOT_NULL(instance_cache_);
3332   Node* mem_start = instance_cache_->mem_start;
3333   Node* mem_size = instance_cache_->mem_size;
3334   DCHECK_NOT_NULL(mem_start);
3335   DCHECK_NOT_NULL(mem_size);
3336 
3337   // Asm.js semantics are to ignore OOB writes.
3338   // Note that we check against the memory size ignoring the size of the
3339   // stored value, which is conservative if misaligned. Technically, asm.js
3340   // should never have misaligned accesses.
3341   Diamond bounds_check(
3342       graph(), mcgraph()->common(),
3343       graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3344       BranchHint::kTrue);
3345   bounds_check.Chain(*control_);
3346 
3347   if (untrusted_code_mitigations_) {
3348     // Condition the index with the memory mask.
3349     Node* mem_mask = instance_cache_->mem_mask;
3350     DCHECK_NOT_NULL(mem_mask);
3351     index =
3352         graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3353   }
3354 
3355   index = Uint32ToUintptr(index);
3356   const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3357       type.representation(), WriteBarrierKind::kNoWriteBarrier));
3358   Node* store = graph()->NewNode(store_op, mem_start, index, val, *effect_,
3359                                  bounds_check.if_true);
3360   Node* effect_phi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), store,
3361                                       *effect_, bounds_check.merge);
3362   *effect_ = effect_phi;
3363   *control_ = bounds_check.merge;
3364   return val;
3365 }
3366 
PrintDebugName(Node * node)3367 void WasmGraphBuilder::PrintDebugName(Node* node) {
3368   PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3369 }
3370 
graph()3371 Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3372 
3373 namespace {
CreateMachineSignature(Zone * zone,wasm::FunctionSig * sig)3374 Signature<MachineRepresentation>* CreateMachineSignature(
3375     Zone* zone, wasm::FunctionSig* sig) {
3376   Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3377                                                     sig->parameter_count());
3378   for (auto ret : sig->returns()) {
3379     builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3380   }
3381 
3382   for (auto param : sig->parameters()) {
3383     builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3384   }
3385   return builder.Build();
3386 }
3387 }  // namespace
3388 
LowerInt64()3389 void WasmGraphBuilder::LowerInt64() {
3390   if (mcgraph()->machine()->Is64()) return;
3391   Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3392                   mcgraph()->zone(),
3393                   CreateMachineSignature(mcgraph()->zone(), sig_));
3394   r.LowerGraph();
3395 }
3396 
SimdScalarLoweringForTesting()3397 void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3398   SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3399       .LowerGraph();
3400 }
3401 
SetSourcePosition(Node * node,wasm::WasmCodePosition position)3402 void WasmGraphBuilder::SetSourcePosition(Node* node,
3403                                          wasm::WasmCodePosition position) {
3404   DCHECK_NE(position, wasm::kNoCodePosition);
3405   if (source_position_table_)
3406     source_position_table_->SetSourcePosition(node, SourcePosition(position));
3407 }
3408 
S128Zero()3409 Node* WasmGraphBuilder::S128Zero() {
3410   has_simd_ = true;
3411   return graph()->NewNode(mcgraph()->machine()->S128Zero());
3412 }
3413 
SimdOp(wasm::WasmOpcode opcode,Node * const * inputs)3414 Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3415   has_simd_ = true;
3416   switch (opcode) {
3417     case wasm::kExprF32x4Splat:
3418       return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3419     case wasm::kExprF32x4SConvertI32x4:
3420       return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3421                               inputs[0]);
3422     case wasm::kExprF32x4UConvertI32x4:
3423       return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3424                               inputs[0]);
3425     case wasm::kExprF32x4Abs:
3426       return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3427     case wasm::kExprF32x4Neg:
3428       return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3429     case wasm::kExprF32x4RecipApprox:
3430       return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3431                               inputs[0]);
3432     case wasm::kExprF32x4RecipSqrtApprox:
3433       return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3434                               inputs[0]);
3435     case wasm::kExprF32x4Add:
3436       return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3437                               inputs[1]);
3438     case wasm::kExprF32x4AddHoriz:
3439       return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3440                               inputs[1]);
3441     case wasm::kExprF32x4Sub:
3442       return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3443                               inputs[1]);
3444     case wasm::kExprF32x4Mul:
3445       return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3446                               inputs[1]);
3447     case wasm::kExprF32x4Min:
3448       return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3449                               inputs[1]);
3450     case wasm::kExprF32x4Max:
3451       return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3452                               inputs[1]);
3453     case wasm::kExprF32x4Eq:
3454       return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3455                               inputs[1]);
3456     case wasm::kExprF32x4Ne:
3457       return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3458                               inputs[1]);
3459     case wasm::kExprF32x4Lt:
3460       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3461                               inputs[1]);
3462     case wasm::kExprF32x4Le:
3463       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3464                               inputs[1]);
3465     case wasm::kExprF32x4Gt:
3466       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3467                               inputs[0]);
3468     case wasm::kExprF32x4Ge:
3469       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3470                               inputs[0]);
3471     case wasm::kExprI32x4Splat:
3472       return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3473     case wasm::kExprI32x4SConvertF32x4:
3474       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3475                               inputs[0]);
3476     case wasm::kExprI32x4UConvertF32x4:
3477       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3478                               inputs[0]);
3479     case wasm::kExprI32x4SConvertI16x8Low:
3480       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3481                               inputs[0]);
3482     case wasm::kExprI32x4SConvertI16x8High:
3483       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3484                               inputs[0]);
3485     case wasm::kExprI32x4Neg:
3486       return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3487     case wasm::kExprI32x4Add:
3488       return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3489                               inputs[1]);
3490     case wasm::kExprI32x4AddHoriz:
3491       return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3492                               inputs[1]);
3493     case wasm::kExprI32x4Sub:
3494       return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3495                               inputs[1]);
3496     case wasm::kExprI32x4Mul:
3497       return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3498                               inputs[1]);
3499     case wasm::kExprI32x4MinS:
3500       return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3501                               inputs[1]);
3502     case wasm::kExprI32x4MaxS:
3503       return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3504                               inputs[1]);
3505     case wasm::kExprI32x4Eq:
3506       return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3507                               inputs[1]);
3508     case wasm::kExprI32x4Ne:
3509       return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3510                               inputs[1]);
3511     case wasm::kExprI32x4LtS:
3512       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3513                               inputs[0]);
3514     case wasm::kExprI32x4LeS:
3515       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3516                               inputs[0]);
3517     case wasm::kExprI32x4GtS:
3518       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3519                               inputs[1]);
3520     case wasm::kExprI32x4GeS:
3521       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3522                               inputs[1]);
3523     case wasm::kExprI32x4UConvertI16x8Low:
3524       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3525                               inputs[0]);
3526     case wasm::kExprI32x4UConvertI16x8High:
3527       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3528                               inputs[0]);
3529     case wasm::kExprI32x4MinU:
3530       return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3531                               inputs[1]);
3532     case wasm::kExprI32x4MaxU:
3533       return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3534                               inputs[1]);
3535     case wasm::kExprI32x4LtU:
3536       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3537                               inputs[0]);
3538     case wasm::kExprI32x4LeU:
3539       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3540                               inputs[0]);
3541     case wasm::kExprI32x4GtU:
3542       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3543                               inputs[1]);
3544     case wasm::kExprI32x4GeU:
3545       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3546                               inputs[1]);
3547     case wasm::kExprI16x8Splat:
3548       return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3549     case wasm::kExprI16x8SConvertI8x16Low:
3550       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3551                               inputs[0]);
3552     case wasm::kExprI16x8SConvertI8x16High:
3553       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3554                               inputs[0]);
3555     case wasm::kExprI16x8Neg:
3556       return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3557     case wasm::kExprI16x8SConvertI32x4:
3558       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3559                               inputs[0], inputs[1]);
3560     case wasm::kExprI16x8Add:
3561       return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3562                               inputs[1]);
3563     case wasm::kExprI16x8AddSaturateS:
3564       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3565                               inputs[0], inputs[1]);
3566     case wasm::kExprI16x8AddHoriz:
3567       return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3568                               inputs[1]);
3569     case wasm::kExprI16x8Sub:
3570       return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3571                               inputs[1]);
3572     case wasm::kExprI16x8SubSaturateS:
3573       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3574                               inputs[0], inputs[1]);
3575     case wasm::kExprI16x8Mul:
3576       return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3577                               inputs[1]);
3578     case wasm::kExprI16x8MinS:
3579       return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3580                               inputs[1]);
3581     case wasm::kExprI16x8MaxS:
3582       return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3583                               inputs[1]);
3584     case wasm::kExprI16x8Eq:
3585       return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3586                               inputs[1]);
3587     case wasm::kExprI16x8Ne:
3588       return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3589                               inputs[1]);
3590     case wasm::kExprI16x8LtS:
3591       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3592                               inputs[0]);
3593     case wasm::kExprI16x8LeS:
3594       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3595                               inputs[0]);
3596     case wasm::kExprI16x8GtS:
3597       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3598                               inputs[1]);
3599     case wasm::kExprI16x8GeS:
3600       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3601                               inputs[1]);
3602     case wasm::kExprI16x8UConvertI8x16Low:
3603       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3604                               inputs[0]);
3605     case wasm::kExprI16x8UConvertI8x16High:
3606       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3607                               inputs[0]);
3608     case wasm::kExprI16x8UConvertI32x4:
3609       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3610                               inputs[0], inputs[1]);
3611     case wasm::kExprI16x8AddSaturateU:
3612       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3613                               inputs[0], inputs[1]);
3614     case wasm::kExprI16x8SubSaturateU:
3615       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3616                               inputs[0], inputs[1]);
3617     case wasm::kExprI16x8MinU:
3618       return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3619                               inputs[1]);
3620     case wasm::kExprI16x8MaxU:
3621       return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3622                               inputs[1]);
3623     case wasm::kExprI16x8LtU:
3624       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3625                               inputs[0]);
3626     case wasm::kExprI16x8LeU:
3627       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3628                               inputs[0]);
3629     case wasm::kExprI16x8GtU:
3630       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3631                               inputs[1]);
3632     case wasm::kExprI16x8GeU:
3633       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3634                               inputs[1]);
3635     case wasm::kExprI8x16Splat:
3636       return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3637     case wasm::kExprI8x16Neg:
3638       return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3639     case wasm::kExprI8x16SConvertI16x8:
3640       return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3641                               inputs[0], inputs[1]);
3642     case wasm::kExprI8x16Add:
3643       return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3644                               inputs[1]);
3645     case wasm::kExprI8x16AddSaturateS:
3646       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3647                               inputs[0], inputs[1]);
3648     case wasm::kExprI8x16Sub:
3649       return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3650                               inputs[1]);
3651     case wasm::kExprI8x16SubSaturateS:
3652       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3653                               inputs[0], inputs[1]);
3654     case wasm::kExprI8x16Mul:
3655       return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3656                               inputs[1]);
3657     case wasm::kExprI8x16MinS:
3658       return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3659                               inputs[1]);
3660     case wasm::kExprI8x16MaxS:
3661       return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3662                               inputs[1]);
3663     case wasm::kExprI8x16Eq:
3664       return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3665                               inputs[1]);
3666     case wasm::kExprI8x16Ne:
3667       return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3668                               inputs[1]);
3669     case wasm::kExprI8x16LtS:
3670       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3671                               inputs[0]);
3672     case wasm::kExprI8x16LeS:
3673       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3674                               inputs[0]);
3675     case wasm::kExprI8x16GtS:
3676       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3677                               inputs[1]);
3678     case wasm::kExprI8x16GeS:
3679       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3680                               inputs[1]);
3681     case wasm::kExprI8x16UConvertI16x8:
3682       return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3683                               inputs[0], inputs[1]);
3684     case wasm::kExprI8x16AddSaturateU:
3685       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3686                               inputs[0], inputs[1]);
3687     case wasm::kExprI8x16SubSaturateU:
3688       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3689                               inputs[0], inputs[1]);
3690     case wasm::kExprI8x16MinU:
3691       return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3692                               inputs[1]);
3693     case wasm::kExprI8x16MaxU:
3694       return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
3695                               inputs[1]);
3696     case wasm::kExprI8x16LtU:
3697       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
3698                               inputs[0]);
3699     case wasm::kExprI8x16LeU:
3700       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
3701                               inputs[0]);
3702     case wasm::kExprI8x16GtU:
3703       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
3704                               inputs[1]);
3705     case wasm::kExprI8x16GeU:
3706       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
3707                               inputs[1]);
3708     case wasm::kExprS128And:
3709       return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
3710                               inputs[1]);
3711     case wasm::kExprS128Or:
3712       return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
3713                               inputs[1]);
3714     case wasm::kExprS128Xor:
3715       return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
3716                               inputs[1]);
3717     case wasm::kExprS128Not:
3718       return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
3719     case wasm::kExprS128Select:
3720       return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[0],
3721                               inputs[1], inputs[2]);
3722     case wasm::kExprS1x4AnyTrue:
3723       return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
3724     case wasm::kExprS1x4AllTrue:
3725       return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
3726     case wasm::kExprS1x8AnyTrue:
3727       return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
3728     case wasm::kExprS1x8AllTrue:
3729       return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
3730     case wasm::kExprS1x16AnyTrue:
3731       return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
3732     case wasm::kExprS1x16AllTrue:
3733       return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
3734     default:
3735       FATAL_UNSUPPORTED_OPCODE(opcode);
3736   }
3737 }
3738 
SimdLaneOp(wasm::WasmOpcode opcode,uint8_t lane,Node * const * inputs)3739 Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
3740                                    Node* const* inputs) {
3741   has_simd_ = true;
3742   switch (opcode) {
3743     case wasm::kExprF32x4ExtractLane:
3744       return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
3745                               inputs[0]);
3746     case wasm::kExprF32x4ReplaceLane:
3747       return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
3748                               inputs[0], inputs[1]);
3749     case wasm::kExprI32x4ExtractLane:
3750       return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
3751                               inputs[0]);
3752     case wasm::kExprI32x4ReplaceLane:
3753       return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
3754                               inputs[0], inputs[1]);
3755     case wasm::kExprI16x8ExtractLane:
3756       return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
3757                               inputs[0]);
3758     case wasm::kExprI16x8ReplaceLane:
3759       return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
3760                               inputs[0], inputs[1]);
3761     case wasm::kExprI8x16ExtractLane:
3762       return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
3763                               inputs[0]);
3764     case wasm::kExprI8x16ReplaceLane:
3765       return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
3766                               inputs[0], inputs[1]);
3767     default:
3768       FATAL_UNSUPPORTED_OPCODE(opcode);
3769   }
3770 }
3771 
SimdShiftOp(wasm::WasmOpcode opcode,uint8_t shift,Node * const * inputs)3772 Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
3773                                     Node* const* inputs) {
3774   has_simd_ = true;
3775   switch (opcode) {
3776     case wasm::kExprI32x4Shl:
3777       return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
3778     case wasm::kExprI32x4ShrS:
3779       return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
3780                               inputs[0]);
3781     case wasm::kExprI32x4ShrU:
3782       return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
3783                               inputs[0]);
3784     case wasm::kExprI16x8Shl:
3785       return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
3786     case wasm::kExprI16x8ShrS:
3787       return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
3788                               inputs[0]);
3789     case wasm::kExprI16x8ShrU:
3790       return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
3791                               inputs[0]);
3792     case wasm::kExprI8x16Shl:
3793       return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
3794     case wasm::kExprI8x16ShrS:
3795       return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
3796                               inputs[0]);
3797     case wasm::kExprI8x16ShrU:
3798       return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
3799                               inputs[0]);
3800     default:
3801       FATAL_UNSUPPORTED_OPCODE(opcode);
3802   }
3803 }
3804 
Simd8x16ShuffleOp(const uint8_t shuffle[16],Node * const * inputs)3805 Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
3806                                           Node* const* inputs) {
3807   has_simd_ = true;
3808   return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
3809                           inputs[0], inputs[1]);
3810 }
3811 
3812 #define ATOMIC_BINOP_LIST(V)                        \
3813   V(I32AtomicAdd, Add, Uint32, Word32)              \
3814   V(I64AtomicAdd, Add, Uint64, Word64)              \
3815   V(I32AtomicAdd8U, Add, Uint8, Word32)             \
3816   V(I32AtomicAdd16U, Add, Uint16, Word32)           \
3817   V(I64AtomicAdd8U, Add, Uint8, Word64)             \
3818   V(I64AtomicAdd16U, Add, Uint16, Word64)           \
3819   V(I64AtomicAdd32U, Add, Uint32, Word64)           \
3820   V(I32AtomicSub, Sub, Uint32, Word32)              \
3821   V(I64AtomicSub, Sub, Uint64, Word64)              \
3822   V(I32AtomicSub8U, Sub, Uint8, Word32)             \
3823   V(I32AtomicSub16U, Sub, Uint16, Word32)           \
3824   V(I64AtomicSub8U, Sub, Uint8, Word64)             \
3825   V(I64AtomicSub16U, Sub, Uint16, Word64)           \
3826   V(I64AtomicSub32U, Sub, Uint32, Word64)           \
3827   V(I32AtomicAnd, And, Uint32, Word32)              \
3828   V(I64AtomicAnd, And, Uint64, Word64)              \
3829   V(I32AtomicAnd8U, And, Uint8, Word32)             \
3830   V(I64AtomicAnd16U, And, Uint16, Word64)           \
3831   V(I32AtomicAnd16U, And, Uint16, Word32)           \
3832   V(I64AtomicAnd8U, And, Uint8, Word64)             \
3833   V(I64AtomicAnd32U, And, Uint32, Word64)           \
3834   V(I32AtomicOr, Or, Uint32, Word32)                \
3835   V(I64AtomicOr, Or, Uint64, Word64)                \
3836   V(I32AtomicOr8U, Or, Uint8, Word32)               \
3837   V(I32AtomicOr16U, Or, Uint16, Word32)             \
3838   V(I64AtomicOr8U, Or, Uint8, Word64)               \
3839   V(I64AtomicOr16U, Or, Uint16, Word64)             \
3840   V(I64AtomicOr32U, Or, Uint32, Word64)             \
3841   V(I32AtomicXor, Xor, Uint32, Word32)              \
3842   V(I64AtomicXor, Xor, Uint64, Word64)              \
3843   V(I32AtomicXor8U, Xor, Uint8, Word32)             \
3844   V(I32AtomicXor16U, Xor, Uint16, Word32)           \
3845   V(I64AtomicXor8U, Xor, Uint8, Word64)             \
3846   V(I64AtomicXor16U, Xor, Uint16, Word64)           \
3847   V(I64AtomicXor32U, Xor, Uint32, Word64)           \
3848   V(I32AtomicExchange, Exchange, Uint32, Word32)    \
3849   V(I64AtomicExchange, Exchange, Uint64, Word64)    \
3850   V(I32AtomicExchange8U, Exchange, Uint8, Word32)   \
3851   V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
3852   V(I64AtomicExchange8U, Exchange, Uint8, Word64)   \
3853   V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
3854   V(I64AtomicExchange32U, Exchange, Uint32, Word64)
3855 
3856 #define ATOMIC_CMP_EXCHG_LIST(V)                 \
3857   V(I32AtomicCompareExchange, Uint32, Word32)    \
3858   V(I64AtomicCompareExchange, Uint64, Word64)    \
3859   V(I32AtomicCompareExchange8U, Uint8, Word32)   \
3860   V(I32AtomicCompareExchange16U, Uint16, Word32) \
3861   V(I64AtomicCompareExchange8U, Uint8, Word64)   \
3862   V(I64AtomicCompareExchange16U, Uint16, Word64) \
3863   V(I64AtomicCompareExchange32U, Uint32, Word64)
3864 
3865 #define ATOMIC_LOAD_LIST(V)           \
3866   V(I32AtomicLoad, Uint32, Word32)    \
3867   V(I64AtomicLoad, Uint64, Word64)    \
3868   V(I32AtomicLoad8U, Uint8, Word32)   \
3869   V(I32AtomicLoad16U, Uint16, Word32) \
3870   V(I64AtomicLoad8U, Uint8, Word64)   \
3871   V(I64AtomicLoad16U, Uint16, Word64) \
3872   V(I64AtomicLoad32U, Uint32, Word64)
3873 
3874 #define ATOMIC_STORE_LIST(V)                    \
3875   V(I32AtomicStore, Uint32, kWord32, Word32)    \
3876   V(I64AtomicStore, Uint64, kWord64, Word64)    \
3877   V(I32AtomicStore8U, Uint8, kWord8, Word32)    \
3878   V(I32AtomicStore16U, Uint16, kWord16, Word32) \
3879   V(I64AtomicStore8U, Uint8, kWord8, Word64)    \
3880   V(I64AtomicStore16U, Uint16, kWord16, Word64) \
3881   V(I64AtomicStore32U, Uint32, kWord32, Word64)
3882 
AtomicOp(wasm::WasmOpcode opcode,Node * const * inputs,uint32_t alignment,uint32_t offset,wasm::WasmCodePosition position)3883 Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
3884                                  uint32_t alignment, uint32_t offset,
3885                                  wasm::WasmCodePosition position) {
3886   // TODO(gdeepti): Add alignment validation, traps on misalignment
3887   Node* node;
3888   switch (opcode) {
3889 #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix)                     \
3890   case wasm::kExpr##Name: {                                                   \
3891     Node* index =                                                             \
3892         BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()),        \
3893                        inputs[0], offset, position, kNeedsBoundsCheck);       \
3894     node = graph()->NewNode(                                                  \
3895         mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
3896         MemBuffer(offset), index, inputs[1], *effect_, *control_);            \
3897     break;                                                                    \
3898   }
3899     ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
3900 #undef BUILD_ATOMIC_BINOP
3901 
3902 #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix)                            \
3903   case wasm::kExpr##Name: {                                                   \
3904     Node* index =                                                             \
3905         BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()),        \
3906                        inputs[0], offset, position, kNeedsBoundsCheck);       \
3907     node = graph()->NewNode(                                                  \
3908         mcgraph()->machine()->Prefix##AtomicCompareExchange(                  \
3909             MachineType::Type()),                                             \
3910         MemBuffer(offset), index, inputs[1], inputs[2], *effect_, *control_); \
3911     break;                                                                    \
3912   }
3913     ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
3914 #undef BUILD_ATOMIC_CMP_EXCHG
3915 
3916 #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix)                        \
3917   case wasm::kExpr##Name: {                                             \
3918     Node* index =                                                       \
3919         BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()),  \
3920                        inputs[0], offset, position, kNeedsBoundsCheck); \
3921     node = graph()->NewNode(                                            \
3922         mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()),  \
3923         MemBuffer(offset), index, *effect_, *control_);                 \
3924     break;                                                              \
3925   }
3926     ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
3927 #undef BUILD_ATOMIC_LOAD_OP
3928 
3929 #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix)                         \
3930   case wasm::kExpr##Name: {                                                    \
3931     Node* index =                                                              \
3932         BoundsCheckMem(wasm::ValueTypes::MemSize(MachineType::Type()),         \
3933                        inputs[0], offset, position, kNeedsBoundsCheck);        \
3934     node = graph()->NewNode(                                                   \
3935         mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
3936         MemBuffer(offset), index, inputs[1], *effect_, *control_);             \
3937     break;                                                                     \
3938   }
3939     ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
3940 #undef BUILD_ATOMIC_STORE_OP
3941     default:
3942       FATAL_UNSUPPORTED_OPCODE(opcode);
3943   }
3944   *effect_ = node;
3945   return node;
3946 }
3947 
3948 #undef ATOMIC_BINOP_LIST
3949 #undef ATOMIC_CMP_EXCHG_LIST
3950 #undef ATOMIC_LOAD_LIST
3951 #undef ATOMIC_STORE_LIST
3952 
3953 namespace {
must_record_function_compilation(Isolate * isolate)3954 bool must_record_function_compilation(Isolate* isolate) {
3955   return isolate->logger()->is_listening_to_code_events() ||
3956          isolate->is_profiling();
3957 }
3958 
3959 PRINTF_FORMAT(4, 5)
RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,Isolate * isolate,Handle<Code> code,const char * format,...)3960 void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
3961                                Isolate* isolate, Handle<Code> code,
3962                                const char* format, ...) {
3963   DCHECK(must_record_function_compilation(isolate));
3964 
3965   ScopedVector<char> buffer(128);
3966   va_list arguments;
3967   va_start(arguments, format);
3968   int len = VSNPrintF(buffer, format, arguments);
3969   CHECK_LT(0, len);
3970   va_end(arguments);
3971   Handle<String> name_str =
3972       isolate->factory()->NewStringFromAsciiChecked(buffer.start());
3973   PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
3974 }
3975 
3976 class WasmWrapperGraphBuilder : public WasmGraphBuilder {
3977  public:
WasmWrapperGraphBuilder(Zone * zone,wasm::ModuleEnv * env,JSGraph * jsgraph,wasm::FunctionSig * sig,compiler::SourcePositionTable * spt)3978   WasmWrapperGraphBuilder(Zone* zone, wasm::ModuleEnv* env, JSGraph* jsgraph,
3979                           wasm::FunctionSig* sig,
3980                           compiler::SourcePositionTable* spt)
3981       : WasmGraphBuilder(jsgraph->isolate(), env, zone, jsgraph,
3982                          CodeFactory::CEntry(jsgraph->isolate()),
3983                          jsgraph->isolate()->factory()->null_value(), sig, spt),
3984         jsgraph_(jsgraph) {}
3985 
BuildAllocateHeapNumberWithValue(Node * value,Node * control)3986   Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
3987     MachineOperatorBuilder* machine = mcgraph()->machine();
3988     CommonOperatorBuilder* common = mcgraph()->common();
3989     // The AllocateHeapNumber builtin does not use the js_context, so we can
3990     // safely pass in Smi zero here.
3991     Callable callable =
3992         Builtins::CallableFor(isolate_, Builtins::kAllocateHeapNumber);
3993     Node* target = jsgraph()->HeapConstant(callable.code());
3994     Node* js_context = jsgraph()->NoContextConstant();
3995     Node* begin_region = graph()->NewNode(
3996         common->BeginRegion(RegionObservability::kNotObservable), *effect_);
3997     if (!allocate_heap_number_operator_.is_set()) {
3998       auto call_descriptor = Linkage::GetStubCallDescriptor(
3999           isolate_, mcgraph()->zone(), callable.descriptor(), 0,
4000           CallDescriptor::kNoFlags, Operator::kNoThrow);
4001       allocate_heap_number_operator_.set(common->Call(call_descriptor));
4002     }
4003     Node* heap_number =
4004         graph()->NewNode(allocate_heap_number_operator_.get(), target,
4005                          js_context, begin_region, control);
4006     Node* store =
4007         graph()->NewNode(machine->Store(StoreRepresentation(
4008                              MachineRepresentation::kFloat64, kNoWriteBarrier)),
4009                          heap_number, BuildHeapNumberValueIndexConstant(),
4010                          value, heap_number, control);
4011     Node* finish_region =
4012         graph()->NewNode(common->FinishRegion(), heap_number, store);
4013     *effect_ = finish_region;
4014     return finish_region;
4015   }
4016 
BuildChangeSmiToFloat64(Node * value)4017   Node* BuildChangeSmiToFloat64(Node* value) {
4018     return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4019                             BuildChangeSmiToInt32(value));
4020   }
4021 
BuildTestNotSmi(Node * value)4022   Node* BuildTestNotSmi(Node* value) {
4023     STATIC_ASSERT(kSmiTag == 0);
4024     STATIC_ASSERT(kSmiTagMask == 1);
4025     return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4026                             mcgraph()->IntPtrConstant(kSmiTagMask));
4027   }
4028 
BuildLoadHeapNumberValue(Node * value,Node * control)4029   Node* BuildLoadHeapNumberValue(Node* value, Node* control) {
4030     return graph()->NewNode(mcgraph()->machine()->Load(MachineType::Float64()),
4031                             value, BuildHeapNumberValueIndexConstant(),
4032                             graph()->start(), control);
4033   }
4034 
BuildHeapNumberValueIndexConstant()4035   Node* BuildHeapNumberValueIndexConstant() {
4036     return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4037   }
4038 
BuildChangeInt32ToTagged(Node * value)4039   Node* BuildChangeInt32ToTagged(Node* value) {
4040     MachineOperatorBuilder* machine = mcgraph()->machine();
4041     CommonOperatorBuilder* common = mcgraph()->common();
4042 
4043     if (machine->Is64()) {
4044       return BuildChangeInt32ToSmi(value);
4045     }
4046 
4047     Node* effect = *effect_;
4048     Node* control = *control_;
4049     Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4050                                  graph()->start());
4051 
4052     Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4053     Node* branch =
4054         graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4055 
4056     Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4057     Node* vtrue = BuildAllocateHeapNumberWithValue(
4058         graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4059     Node* etrue = *effect_;
4060 
4061     Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4062     Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4063 
4064     Node* merge = graph()->NewNode(common->Merge(2), if_true, if_false);
4065     Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4066                                  vtrue, vfalse, merge);
4067     *effect_ = graph()->NewNode(common->EffectPhi(2), etrue, effect, merge);
4068     *control_ = merge;
4069     return phi;
4070   }
4071 
BuildChangeFloat64ToTagged(Node * value)4072   Node* BuildChangeFloat64ToTagged(Node* value) {
4073     MachineOperatorBuilder* machine = mcgraph()->machine();
4074     CommonOperatorBuilder* common = mcgraph()->common();
4075 
4076     Node* effect = *effect_;
4077     Node* control = *control_;
4078     Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4079     Node* check_same = graph()->NewNode(
4080         machine->Float64Equal(), value,
4081         graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4082     Node* branch_same = graph()->NewNode(common->Branch(), check_same, control);
4083 
4084     Node* if_smi = graph()->NewNode(common->IfTrue(), branch_same);
4085     Node* vsmi;
4086     Node* if_box = graph()->NewNode(common->IfFalse(), branch_same);
4087     Node* vbox;
4088 
4089     // We only need to check for -0 if the {value} can potentially contain -0.
4090     Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4091                                         mcgraph()->Int32Constant(0));
4092     Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4093                                          check_zero, if_smi);
4094 
4095     Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4096     Node* if_notzero = graph()->NewNode(common->IfFalse(), branch_zero);
4097 
4098     // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4099     Node* check_negative = graph()->NewNode(
4100         machine->Int32LessThan(),
4101         graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4102         mcgraph()->Int32Constant(0));
4103     Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4104                                              check_negative, if_zero);
4105 
4106     Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4107     Node* if_notnegative = graph()->NewNode(common->IfFalse(), branch_negative);
4108 
4109     // We need to create a box for negative 0.
4110     if_smi = graph()->NewNode(common->Merge(2), if_notzero, if_notnegative);
4111     if_box = graph()->NewNode(common->Merge(2), if_box, if_negative);
4112 
4113     // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4114     // 32-bit machines we need to deal with potential overflow and fallback to
4115     // boxing.
4116     if (machine->Is64()) {
4117       vsmi = BuildChangeInt32ToSmi(value32);
4118     } else {
4119       Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4120                                        value32, if_smi);
4121 
4122       Node* check_ovf =
4123           graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4124       Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4125                                           check_ovf, if_smi);
4126 
4127       Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4128       if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4129 
4130       if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4131       vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4132     }
4133 
4134     // Allocate the box for the {value}.
4135     vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4136     Node* ebox = *effect_;
4137 
4138     Node* merge = graph()->NewNode(common->Merge(2), if_smi, if_box);
4139     value = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4140                              vsmi, vbox, merge);
4141     *effect_ = graph()->NewNode(common->EffectPhi(2), effect, ebox, merge);
4142     *control_ = merge;
4143     return value;
4144   }
4145 
AddArgumentNodes(Node ** args,int pos,int param_count,wasm::FunctionSig * sig)4146   int AddArgumentNodes(Node** args, int pos, int param_count,
4147                        wasm::FunctionSig* sig) {
4148     // Convert wasm numbers to JS values.
4149     for (int i = 0; i < param_count; ++i) {
4150       Node* param =
4151           Param(i + 1);  // Start from index 1 to drop the instance_node.
4152       args[pos++] = ToJS(param, sig->GetParam(i));
4153     }
4154     return pos;
4155   }
4156 
BuildJavaScriptToNumber(Node * node,Node * js_context)4157   Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4158     Callable callable = Builtins::CallableFor(isolate_, Builtins::kToNumber);
4159     auto call_descriptor = Linkage::GetStubCallDescriptor(
4160         isolate_, mcgraph()->zone(), callable.descriptor(), 0,
4161         CallDescriptor::kNoFlags, Operator::kNoProperties);
4162     Node* stub_code = jsgraph()->HeapConstant(callable.code());
4163 
4164     Node* result =
4165         graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4166                          node, js_context, *effect_, *control_);
4167 
4168     SetSourcePosition(result, 1);
4169 
4170     *effect_ = result;
4171 
4172     return result;
4173   }
4174 
BuildChangeTaggedToFloat64(Node * value)4175   Node* BuildChangeTaggedToFloat64(Node* value) {
4176     MachineOperatorBuilder* machine = mcgraph()->machine();
4177     CommonOperatorBuilder* common = mcgraph()->common();
4178 
4179     Node* check = BuildTestNotSmi(value);
4180     Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), check,
4181                                     graph()->start());
4182 
4183     Node* if_not_smi = graph()->NewNode(common->IfTrue(), branch);
4184 
4185     Node* vnot_smi;
4186     Node* check_undefined = graph()->NewNode(machine->WordEqual(), value,
4187                                              jsgraph()->UndefinedConstant());
4188     Node* branch_undefined = graph()->NewNode(
4189         common->Branch(BranchHint::kFalse), check_undefined, if_not_smi);
4190 
4191     Node* if_undefined = graph()->NewNode(common->IfTrue(), branch_undefined);
4192     Node* vundefined =
4193         mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4194 
4195     Node* if_not_undefined =
4196         graph()->NewNode(common->IfFalse(), branch_undefined);
4197     Node* vheap_number = BuildLoadHeapNumberValue(value, if_not_undefined);
4198 
4199     if_not_smi =
4200         graph()->NewNode(common->Merge(2), if_undefined, if_not_undefined);
4201     vnot_smi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2),
4202                                 vundefined, vheap_number, if_not_smi);
4203 
4204     Node* if_smi = graph()->NewNode(common->IfFalse(), branch);
4205     Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4206 
4207     Node* merge = graph()->NewNode(common->Merge(2), if_not_smi, if_smi);
4208     Node* phi =
4209         graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2),
4210                          vnot_smi, vfrom_smi, merge);
4211 
4212     return phi;
4213   }
4214 
ToJS(Node * node,wasm::ValueType type)4215   Node* ToJS(Node* node, wasm::ValueType type) {
4216     switch (type) {
4217       case wasm::kWasmI32:
4218         return BuildChangeInt32ToTagged(node);
4219       case wasm::kWasmS128:
4220       case wasm::kWasmI64:
4221         UNREACHABLE();
4222       case wasm::kWasmF32:
4223         node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4224                                 node);
4225         return BuildChangeFloat64ToTagged(node);
4226       case wasm::kWasmF64:
4227         return BuildChangeFloat64ToTagged(node);
4228       case wasm::kWasmAnyRef:
4229         return node;
4230       case wasm::kWasmStmt:
4231         return jsgraph()->UndefinedConstant();
4232       default:
4233         UNREACHABLE();
4234     }
4235   }
4236 
FromJS(Node * node,Node * js_context,wasm::ValueType type)4237   Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4238     DCHECK_NE(wasm::kWasmStmt, type);
4239 
4240     // The parameter is of type AnyRef, we take it as is.
4241     if (type == wasm::kWasmAnyRef) {
4242       return node;
4243     }
4244 
4245     // Do a JavaScript ToNumber.
4246     Node* num = BuildJavaScriptToNumber(node, js_context);
4247 
4248     // Change representation.
4249     SimplifiedOperatorBuilder simplified(mcgraph()->zone());
4250     num = BuildChangeTaggedToFloat64(num);
4251 
4252     switch (type) {
4253       case wasm::kWasmI32: {
4254         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4255                                num);
4256         break;
4257       }
4258       case wasm::kWasmS128:
4259       case wasm::kWasmI64:
4260         UNREACHABLE();
4261       case wasm::kWasmF32:
4262         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4263                                num);
4264         break;
4265       case wasm::kWasmF64:
4266         break;
4267       default:
4268         UNREACHABLE();
4269     }
4270     return num;
4271   }
4272 
BuildModifyThreadInWasmFlag(bool new_value)4273   Node* BuildModifyThreadInWasmFlag(bool new_value) {
4274     // TODO(eholk): generate code to modify the thread-local storage directly,
4275     // rather than calling the runtime.
4276     if (!use_trap_handler()) {
4277       return *control_;
4278     }
4279 
4280     // Using two functions instead of taking the new value as a parameter saves
4281     // one instruction on each call to set up the parameter.
4282     ExternalReference ref =
4283         new_value ? ExternalReference::wasm_set_thread_in_wasm_flag()
4284                   : ExternalReference::wasm_clear_thread_in_wasm_flag();
4285     MachineSignature sig(0, 0, nullptr);
4286     return BuildCCall(
4287         &sig, graph()->NewNode(mcgraph()->common()->ExternalConstant(ref)));
4288   }
4289 
BuildLoadInstanceFromExportedFunction(Node * closure)4290   Node* BuildLoadInstanceFromExportedFunction(Node* closure) {
4291     Node* shared = *effect_ = graph()->NewNode(
4292         jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4293         jsgraph()->Int32Constant(JSFunction::kSharedFunctionInfoOffset -
4294                                  kHeapObjectTag),
4295         *effect_, *control_);
4296     Node* function_data = *effect_ = graph()->NewNode(
4297         jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4298         jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4299                                  kHeapObjectTag),
4300         *effect_, *control_);
4301     Node* instance = *effect_ = graph()->NewNode(
4302         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4303         jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4304                                  kHeapObjectTag),
4305         *effect_, *control_);
4306     return instance;
4307   }
4308 
BuildJSToWasmWrapper(uint32_t wasm_func_index,Address call_target)4309   void BuildJSToWasmWrapper(uint32_t wasm_func_index, Address call_target) {
4310     const int wasm_count = static_cast<int>(sig_->parameter_count());
4311 
4312     // Build the start and the JS parameter nodes.
4313     Node* start = Start(wasm_count + 5);
4314     *control_ = start;
4315     *effect_ = start;
4316 
4317     // Create the js_closure and js_context parameters.
4318     Node* js_closure =
4319         graph()->NewNode(jsgraph()->common()->Parameter(
4320                              Linkage::kJSCallClosureParamIndex, "%closure"),
4321                          graph()->start());
4322     Node* js_context = graph()->NewNode(
4323         mcgraph()->common()->Parameter(
4324             Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
4325         graph()->start());
4326 
4327     // Create the instance_node node to pass as parameter. It is loaded from the
4328     // an actual reference to an instance or a placeholder reference,
4329     // called {WasmExportedFunction} via the {WasmExportedFunctionData}
4330     // structure. since JSToWasm wrappers can be compiled at module compile time
4331     // and patched at instance build time.
4332     DCHECK_NULL(instance_node_);
4333     instance_node_ = BuildLoadInstanceFromExportedFunction(js_closure);
4334 
4335     if (!wasm::IsJSCompatibleSignature(sig_)) {
4336       // Throw a TypeError. Use the js_context of the calling javascript
4337       // function (passed as a parameter), such that the generated code is
4338       // js_context independent.
4339       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4340                                     nullptr, 0);
4341       Return(jsgraph()->UndefinedConstant());
4342       return;
4343     }
4344 
4345     const int args_count = wasm_count + 1;  // +1 for wasm_code.
4346     Node** args = Buffer(args_count);
4347     Node** rets;
4348 
4349     // Convert JS parameters to wasm numbers.
4350     for (int i = 0; i < wasm_count; ++i) {
4351       Node* param = Param(i + 1);
4352       Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
4353       args[i + 1] = wasm_param;
4354     }
4355 
4356     // Set the ThreadInWasm flag before we do the actual call.
4357     BuildModifyThreadInWasmFlag(true);
4358 
4359     if (env_ && wasm_func_index < env_->module->num_imported_functions) {
4360       // Call to an imported function.
4361       DCHECK_EQ(kNullAddress, call_target);
4362       BuildImportWasmCall(sig_, args, &rets, wasm::kNoCodePosition,
4363                           wasm_func_index);
4364     } else {
4365       // Call to a wasm function defined in this module.
4366       DCHECK_NE(kNullAddress, call_target);
4367       args[0] = mcgraph()->RelocatableIntPtrConstant(
4368           call_target, RelocInfo::JS_TO_WASM_CALL);
4369 
4370       BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
4371                     kNoRetpoline);
4372     }
4373 
4374     // Clear the ThreadInWasm flag.
4375     BuildModifyThreadInWasmFlag(false);
4376 
4377     Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
4378                                             : ToJS(rets[0], sig_->GetReturn());
4379     Return(jsval);
4380   }
4381 
BuildWasmToJSWrapper(Handle<JSReceiver> target,int index)4382   bool BuildWasmToJSWrapper(Handle<JSReceiver> target, int index) {
4383     DCHECK(target->IsCallable());
4384 
4385     int wasm_count = static_cast<int>(sig_->parameter_count());
4386 
4387     // Build the start and the parameter nodes.
4388     CallDescriptor* call_descriptor;
4389     Node* start = Start(wasm_count + 3);
4390     *effect_ = start;
4391     *control_ = start;
4392 
4393     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4394     Node* callables_node = LOAD_INSTANCE_FIELD(ImportedFunctionCallables,
4395                                                MachineType::TaggedPointer());
4396     Node* callable_node = LOAD_FIXED_ARRAY_SLOT(callables_node, index);
4397     Node* undefined_node = jsgraph()->UndefinedConstant();
4398     Node* native_context =
4399         LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
4400 
4401     if (!wasm::IsJSCompatibleSignature(sig_)) {
4402       // Throw a TypeError.
4403       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
4404                                     native_context, nullptr, 0);
4405       // We don't need to return a value here, as the runtime call will not
4406       // return anyway (the c entry stub will trigger stack unwinding).
4407       ReturnVoid();
4408       return false;
4409     }
4410 
4411     Node** args = Buffer(wasm_count + 9);
4412     Node* call = nullptr;
4413 
4414     BuildModifyThreadInWasmFlag(false);
4415 
4416     if (target->IsJSFunction()) {
4417       Handle<JSFunction> function = Handle<JSFunction>::cast(target);
4418       FieldAccess field_access = AccessBuilder::ForJSFunctionContext();
4419       Node* function_context = graph()->NewNode(
4420           mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4421           callable_node,
4422           mcgraph()->Int32Constant(field_access.offset - field_access.tag()),
4423           *effect_, *control_);
4424 
4425       if (!IsClassConstructor(function->shared()->kind())) {
4426         if (function->shared()->internal_formal_parameter_count() ==
4427             wasm_count) {
4428           int pos = 0;
4429           args[pos++] = callable_node;  // target callable.
4430           // Receiver.
4431           if (is_sloppy(function->shared()->language_mode()) &&
4432               !function->shared()->native()) {
4433             Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4434                 native_context, Context::GLOBAL_PROXY_INDEX);
4435             args[pos++] = global_proxy;
4436           } else {
4437             args[pos++] = undefined_node;
4438           }
4439 
4440           call_descriptor = Linkage::GetJSCallDescriptor(
4441               graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
4442 
4443           // Convert wasm numbers to JS values.
4444           pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4445 
4446           args[pos++] = undefined_node;                        // new target
4447           args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
4448           args[pos++] = function_context;
4449           args[pos++] = *effect_;
4450           args[pos++] = *control_;
4451 
4452           call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4453                                   pos, args);
4454         } else if (function->shared()->internal_formal_parameter_count() >= 0) {
4455           Callable callable = CodeFactory::ArgumentAdaptor(isolate_);
4456           int pos = 0;
4457           args[pos++] = jsgraph()->HeapConstant(callable.code());
4458           args[pos++] = callable_node;   // target callable
4459           args[pos++] = undefined_node;  // new target
4460           args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
4461           args[pos++] = mcgraph()->Int32Constant(
4462               function->shared()->internal_formal_parameter_count());
4463           // Receiver.
4464           if (is_sloppy(function->shared()->language_mode()) &&
4465               !function->shared()->native()) {
4466             Node* global_proxy = LOAD_FIXED_ARRAY_SLOT(
4467                 native_context, Context::GLOBAL_PROXY_INDEX);
4468             args[pos++] = global_proxy;
4469           } else {
4470             args[pos++] = undefined_node;
4471           }
4472 
4473           // Convert wasm numbers to JS values.
4474           pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4475           args[pos++] = function_context;
4476           args[pos++] = *effect_;
4477           args[pos++] = *control_;
4478           call = graph()->NewNode(
4479               mcgraph()->common()->Call(Linkage::GetStubCallDescriptor(
4480                   isolate_, mcgraph()->zone(), callable.descriptor(),
4481                   1 + wasm_count, CallDescriptor::kNoFlags)),
4482               pos, args);
4483         }
4484       }
4485     }
4486 
4487     // We cannot call the target directly, we have to use the Call builtin.
4488     if (!call) {
4489       int pos = 0;
4490       // We cannot call the target directly, we have to use the Call builtin.
4491       Callable callable = CodeFactory::Call(isolate_);
4492       args[pos++] = jsgraph()->HeapConstant(callable.code());
4493       args[pos++] = callable_node;
4494       args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
4495       args[pos++] = undefined_node;                        // receiver
4496 
4497       call_descriptor = Linkage::GetStubCallDescriptor(
4498           isolate_, graph()->zone(), callable.descriptor(), wasm_count + 1,
4499           CallDescriptor::kNoFlags);
4500 
4501       // Convert wasm numbers to JS values.
4502       pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4503 
4504       // The native_context is sufficient here, because all kind of callables
4505       // which depend on the context provide their own context. The context here
4506       // is only needed if the target is a constructor to throw a TypeError, if
4507       // the target is a native function, or if the target is a callable
4508       // JSObject, which can only be constructed by the runtime.
4509       args[pos++] = native_context;
4510       args[pos++] = *effect_;
4511       args[pos++] = *control_;
4512 
4513       call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4514                               args);
4515     }
4516 
4517     *effect_ = call;
4518     SetSourcePosition(call, 0);
4519 
4520     // Convert the return value back.
4521     Node* val = sig_->return_count() == 0
4522                     ? mcgraph()->Int32Constant(0)
4523                     : FromJS(call, native_context, sig_->GetReturn());
4524 
4525     BuildModifyThreadInWasmFlag(true);
4526 
4527     Return(val);
4528     return true;
4529   }
4530 
BuildWasmInterpreterEntry(uint32_t func_index)4531   void BuildWasmInterpreterEntry(uint32_t func_index) {
4532     int param_count = static_cast<int>(sig_->parameter_count());
4533 
4534     // Build the start and the parameter nodes.
4535     Node* start = Start(param_count + 3);
4536     *effect_ = start;
4537     *control_ = start;
4538 
4539     // Compute size for the argument buffer.
4540     int args_size_bytes = 0;
4541     for (wasm::ValueType type : sig_->parameters()) {
4542       args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
4543     }
4544 
4545     // The return value is also passed via this buffer:
4546     DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
4547     // TODO(wasm): Handle multi-value returns.
4548     DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
4549     int return_size_bytes =
4550         sig_->return_count() == 0
4551             ? 0
4552             : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
4553 
4554     // Get a stack slot for the arguments.
4555     Node* arg_buffer =
4556         args_size_bytes == 0 && return_size_bytes == 0
4557             ? mcgraph()->IntPtrConstant(0)
4558             : graph()->NewNode(mcgraph()->machine()->StackSlot(
4559                   std::max(args_size_bytes, return_size_bytes), 8));
4560 
4561     // Now store all our arguments to the buffer.
4562     int offset = 0;
4563 
4564     for (int i = 0; i < param_count; ++i) {
4565       wasm::ValueType type = sig_->GetParam(i);
4566       // Start from the parameter with index 1 to drop the instance_node.
4567       *effect_ = graph()->NewNode(GetSafeStoreOperator(offset, type),
4568                                   arg_buffer, Int32Constant(offset),
4569                                   Param(i + 1), *effect_, *control_);
4570       offset += wasm::ValueTypes::ElementSizeInBytes(type);
4571     }
4572     DCHECK_EQ(args_size_bytes, offset);
4573 
4574     // We are passing the raw arg_buffer here. To the GC and other parts, it
4575     // looks like a Smi (lowest bit not set). In the runtime function however,
4576     // don't call Smi::value on it, but just cast it to a byte pointer.
4577     Node* parameters[] = {
4578         jsgraph()->SmiConstant(func_index), arg_buffer,
4579     };
4580     BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
4581                        arraysize(parameters));
4582 
4583     // Read back the return value.
4584     if (sig_->return_count() == 0) {
4585       Return(Int32Constant(0));
4586     } else {
4587       // TODO(wasm): Implement multi-return.
4588       DCHECK_EQ(1, sig_->return_count());
4589       MachineType load_rep =
4590           wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
4591       Node* val =
4592           graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
4593                            Int32Constant(0), *effect_, *control_);
4594       Return(val);
4595     }
4596 
4597     if (ContainsInt64(sig_)) LowerInt64();
4598   }
4599 
BuildCWasmEntry()4600   void BuildCWasmEntry() {
4601     // Build the start and the JS parameter nodes.
4602     Node* start = Start(CWasmEntryParameters::kNumParameters + 5);
4603     *control_ = start;
4604     *effect_ = start;
4605 
4606     // Create parameter nodes (offset by 1 for the receiver parameter).
4607     Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1);
4608     MachineOperatorBuilder* machine = mcgraph()->machine();
4609     Node* code_obj = graph()->NewNode(
4610         machine->Load(MachineType::Pointer()), foreign_code_obj,
4611         Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag),
4612         *effect_, *control_);
4613     Node* instance_node = Param(CWasmEntryParameters::kWasmInstance + 1);
4614     Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
4615 
4616     int wasm_arg_count = static_cast<int>(sig_->parameter_count());
4617     int arg_count = wasm_arg_count + 4;  // code, instance_node, control, effect
4618     Node** args = Buffer(arg_count);
4619 
4620     int pos = 0;
4621     args[pos++] = code_obj;
4622     args[pos++] = instance_node;
4623 
4624     int offset = 0;
4625     for (wasm::ValueType type : sig_->parameters()) {
4626       Node* arg_load =
4627           graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
4628                            Int32Constant(offset), *effect_, *control_);
4629       *effect_ = arg_load;
4630       args[pos++] = arg_load;
4631       offset += wasm::ValueTypes::ElementSizeInBytes(type);
4632     }
4633 
4634     args[pos++] = *effect_;
4635     args[pos++] = *control_;
4636     DCHECK_EQ(arg_count, pos);
4637 
4638     // Call the wasm code.
4639     auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
4640 
4641     Node* call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4642                                   arg_count, args);
4643     *effect_ = call;
4644 
4645     // Store the return value.
4646     DCHECK_GE(1, sig_->return_count());
4647     if (sig_->return_count() == 1) {
4648       StoreRepresentation store_rep(
4649           wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
4650           kNoWriteBarrier);
4651       Node* store =
4652           graph()->NewNode(mcgraph()->machine()->Store(store_rep), arg_buffer,
4653                            Int32Constant(0), call, *effect_, *control_);
4654       *effect_ = store;
4655     }
4656     Return(jsgraph()->SmiConstant(0));
4657 
4658     if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
4659       MachineRepresentation sig_reps[] = {
4660           MachineRepresentation::kWord32,  // return value
4661           MachineRepresentation::kTagged,  // receiver
4662           MachineRepresentation::kTagged,  // arg0 (code)
4663           MachineRepresentation::kTagged   // arg1 (buffer)
4664       };
4665       Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
4666       Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
4667                       mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
4668       r.LowerGraph();
4669     }
4670   }
4671 
jsgraph()4672   JSGraph* jsgraph() { return jsgraph_; }
4673 
4674  private:
4675   JSGraph* jsgraph_;
4676   SetOncePointer<const Operator> allocate_heap_number_operator_;
4677 };
4678 }  // namespace
4679 
CompileJSToWasmWrapper(Isolate * isolate,wasm::WasmModule * module,Address call_target,uint32_t index,wasm::UseTrapHandler use_trap_handler)4680 Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module,
4681                                     Address call_target, uint32_t index,
4682                                     wasm::UseTrapHandler use_trap_handler) {
4683   const wasm::WasmFunction* func = &module->functions[index];
4684 
4685   //----------------------------------------------------------------------------
4686   // Create the Graph
4687   //----------------------------------------------------------------------------
4688   Zone zone(isolate->allocator(), ZONE_NAME);
4689   Graph graph(&zone);
4690   CommonOperatorBuilder common(&zone);
4691   MachineOperatorBuilder machine(
4692       &zone, MachineType::PointerRepresentation(),
4693       InstructionSelector::SupportedMachineOperatorFlags(),
4694       InstructionSelector::AlignmentRequirements());
4695   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4696 
4697   Node* control = nullptr;
4698   Node* effect = nullptr;
4699 
4700   wasm::ModuleEnv env(module, use_trap_handler, wasm::kRuntimeExceptionSupport);
4701   WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, func->sig, nullptr);
4702   builder.set_control_ptr(&control);
4703   builder.set_effect_ptr(&effect);
4704   builder.BuildJSToWasmWrapper(index, call_target);
4705 
4706   //----------------------------------------------------------------------------
4707   // Run the compilation pipeline.
4708   //----------------------------------------------------------------------------
4709 #ifdef DEBUG
4710   EmbeddedVector<char, 32> func_name;
4711   static unsigned id = 0;
4712   func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++));
4713 #else
4714   Vector<const char> func_name = CStrVector("js-to-wasm");
4715 #endif
4716 
4717   OptimizedCompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION);
4718 
4719   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4720     OFStream os(stdout);
4721     os << "-- Graph after change lowering -- " << std::endl;
4722     os << AsRPO(graph);
4723   }
4724 
4725   // Schedule and compile to machine code.
4726   int params =
4727       static_cast<int>(module->functions[index].sig->parameter_count());
4728   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
4729       &zone, false, params + 1, CallDescriptor::kNoFlags);
4730 
4731   Handle<Code> code =
4732       Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph);
4733 #ifdef ENABLE_DISASSEMBLER
4734   if (FLAG_print_opt_code && !code.is_null()) {
4735     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4736     OFStream os(tracing_scope.file());
4737     code->Disassemble(func_name.start(), os);
4738   }
4739 #endif
4740 
4741   if (must_record_function_compilation(isolate)) {
4742     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4743                               "%.*s", func_name.length(), func_name.start());
4744   }
4745 
4746   return code;
4747 }
4748 
4749 namespace {
4750 
ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper)4751 void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) {
4752 #ifdef DEBUG
4753   // We expect the only embedded objects to be those originating from
4754   // a snapshot, which are immovable.
4755   DisallowHeapAllocation no_gc;
4756   if (wrapper.is_null()) return;
4757   static constexpr int kAllGCRefs = (1 << (RelocInfo::LAST_GCED_ENUM + 1)) - 1;
4758   for (RelocIterator it(*wrapper, kAllGCRefs); !it.done(); it.next()) {
4759     RelocInfo::Mode mode = it.rinfo()->rmode();
4760     Object* target = nullptr;
4761     switch (mode) {
4762       case RelocInfo::CODE_TARGET:
4763         // this would be either one of the stubs or builtins, because
4764         // we didn't link yet.
4765         target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
4766         break;
4767       case RelocInfo::EMBEDDED_OBJECT:
4768         target = it.rinfo()->target_object();
4769         break;
4770       default:
4771         UNREACHABLE();
4772     }
4773     DCHECK_NOT_NULL(target);
4774     DCHECK(target->IsSmi() || Heap::IsImmovable(HeapObject::cast(target)));
4775   }
4776 #endif
4777 }
4778 
4779 }  // namespace
4780 
CompileWasmToJSWrapper(Isolate * isolate,Handle<JSReceiver> target,wasm::FunctionSig * sig,uint32_t index,wasm::ModuleOrigin origin,wasm::UseTrapHandler use_trap_handler)4781 Handle<Code> CompileWasmToJSWrapper(Isolate* isolate, Handle<JSReceiver> target,
4782                                     wasm::FunctionSig* sig, uint32_t index,
4783                                     wasm::ModuleOrigin origin,
4784                                     wasm::UseTrapHandler use_trap_handler) {
4785   //----------------------------------------------------------------------------
4786   // Create the Graph
4787   //----------------------------------------------------------------------------
4788   Zone zone(isolate->allocator(), ZONE_NAME);
4789   Graph graph(&zone);
4790   CommonOperatorBuilder common(&zone);
4791   MachineOperatorBuilder machine(
4792       &zone, MachineType::PointerRepresentation(),
4793       InstructionSelector::SupportedMachineOperatorFlags(),
4794       InstructionSelector::AlignmentRequirements());
4795   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4796 
4797   Node* control = nullptr;
4798   Node* effect = nullptr;
4799 
4800   SourcePositionTable* source_position_table =
4801       origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph)
4802                                    : nullptr;
4803 
4804   wasm::ModuleEnv env(nullptr, use_trap_handler,
4805                       wasm::kRuntimeExceptionSupport);
4806 
4807   WasmWrapperGraphBuilder builder(&zone, &env, &jsgraph, sig,
4808                                   source_position_table);
4809   builder.set_control_ptr(&control);
4810   builder.set_effect_ptr(&effect);
4811   builder.BuildWasmToJSWrapper(target, index);
4812 
4813 #ifdef DEBUG
4814   EmbeddedVector<char, 32> func_name;
4815   static unsigned id = 0;
4816   func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++));
4817 #else
4818   Vector<const char> func_name = CStrVector("wasm-to-js");
4819 #endif
4820 
4821   OptimizedCompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION);
4822 
4823   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4824     OFStream os(stdout);
4825     os << "-- Graph after change lowering -- " << std::endl;
4826     os << AsRPO(graph);
4827   }
4828 
4829   // Schedule and compile to machine code.
4830   CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4831   if (machine.Is32()) {
4832     incoming = GetI32WasmCallDescriptor(&zone, incoming);
4833   }
4834 
4835   Handle<Code> code = Pipeline::GenerateCodeForTesting(
4836       &info, isolate, incoming, &graph, nullptr, source_position_table);
4837   ValidateImportWrapperReferencesImmovables(code);
4838 
4839 #ifdef ENABLE_DISASSEMBLER
4840   if (FLAG_print_opt_code && !code.is_null()) {
4841     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4842     OFStream os(tracing_scope.file());
4843     code->Disassemble(func_name.start(), os);
4844   }
4845 #endif
4846 
4847   if (must_record_function_compilation(isolate)) {
4848     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4849                               "%.*s", func_name.length(), func_name.start());
4850   }
4851 
4852   return code;
4853 }
4854 
CompileWasmInterpreterEntry(Isolate * isolate,uint32_t func_index,wasm::FunctionSig * sig)4855 Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index,
4856                                          wasm::FunctionSig* sig) {
4857   //----------------------------------------------------------------------------
4858   // Create the Graph
4859   //----------------------------------------------------------------------------
4860   Zone zone(isolate->allocator(), ZONE_NAME);
4861   Graph graph(&zone);
4862   CommonOperatorBuilder common(&zone);
4863   MachineOperatorBuilder machine(
4864       &zone, MachineType::PointerRepresentation(),
4865       InstructionSelector::SupportedMachineOperatorFlags(),
4866       InstructionSelector::AlignmentRequirements());
4867   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4868 
4869   Node* control = nullptr;
4870   Node* effect = nullptr;
4871 
4872   WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr);
4873   builder.set_control_ptr(&control);
4874   builder.set_effect_ptr(&effect);
4875   builder.BuildWasmInterpreterEntry(func_index);
4876 
4877   Handle<Code> code = Handle<Code>::null();
4878   {
4879     // Schedule and compile to machine code.
4880     CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
4881     if (machine.Is32()) {
4882       incoming = GetI32WasmCallDescriptor(&zone, incoming);
4883     }
4884 #ifdef DEBUG
4885     EmbeddedVector<char, 32> func_name;
4886     func_name.Truncate(
4887         SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
4888 #else
4889     Vector<const char> func_name = CStrVector("wasm-interpreter-entry");
4890 #endif
4891 
4892     OptimizedCompilationInfo info(func_name, &zone,
4893                                   Code::WASM_INTERPRETER_ENTRY);
4894 
4895     if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4896       OFStream os(stdout);
4897       os << "-- Wasm interpreter entry graph -- " << std::endl;
4898       os << AsRPO(graph);
4899     }
4900 
4901     code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph,
4902                                             nullptr);
4903 #ifdef ENABLE_DISASSEMBLER
4904     if (FLAG_print_opt_code && !code.is_null()) {
4905       CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4906       OFStream os(tracing_scope.file());
4907       code->Disassemble(func_name.start(), os);
4908     }
4909 #endif
4910 
4911     if (must_record_function_compilation(isolate)) {
4912       RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code,
4913                                 "%.*s", func_name.length(), func_name.start());
4914     }
4915   }
4916 
4917   return code;
4918 }
4919 
CompileCWasmEntry(Isolate * isolate,wasm::FunctionSig * sig)4920 Handle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
4921   Zone zone(isolate->allocator(), ZONE_NAME);
4922   Graph graph(&zone);
4923   CommonOperatorBuilder common(&zone);
4924   MachineOperatorBuilder machine(
4925       &zone, MachineType::PointerRepresentation(),
4926       InstructionSelector::SupportedMachineOperatorFlags(),
4927       InstructionSelector::AlignmentRequirements());
4928   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4929 
4930   Node* control = nullptr;
4931   Node* effect = nullptr;
4932 
4933   WasmWrapperGraphBuilder builder(&zone, nullptr, &jsgraph, sig, nullptr);
4934   builder.set_control_ptr(&control);
4935   builder.set_effect_ptr(&effect);
4936   builder.BuildCWasmEntry();
4937 
4938   // Schedule and compile to machine code.
4939   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
4940       &zone, false, CWasmEntryParameters::kNumParameters + 1,
4941       CallDescriptor::kNoFlags);
4942 
4943   // Build a name in the form "c-wasm-entry:<params>:<returns>".
4944   static constexpr size_t kMaxNameLen = 128;
4945   char debug_name[kMaxNameLen] = "c-wasm-entry:";
4946   size_t name_len = strlen(debug_name);
4947   auto append_name_char = [&](char c) {
4948     if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c;
4949   };
4950   for (wasm::ValueType t : sig->parameters()) {
4951     append_name_char(wasm::ValueTypes::ShortNameOf(t));
4952   }
4953   append_name_char(':');
4954   for (wasm::ValueType t : sig->returns()) {
4955     append_name_char(wasm::ValueTypes::ShortNameOf(t));
4956   }
4957   debug_name[name_len] = '\0';
4958   Vector<const char> debug_name_vec(debug_name, name_len);
4959 
4960   OptimizedCompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY);
4961 
4962   if (info.trace_turbo_graph_enabled()) {  // Simple textual RPO.
4963     OFStream os(stdout);
4964     os << "-- C Wasm entry graph -- " << std::endl;
4965     os << AsRPO(graph);
4966   }
4967 
4968   Handle<Code> code =
4969       Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph);
4970 #ifdef ENABLE_DISASSEMBLER
4971   if (FLAG_print_opt_code && !code.is_null()) {
4972     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
4973     OFStream os(tracing_scope.file());
4974     code->Disassemble(debug_name, os);
4975   }
4976 #endif
4977 
4978   return code;
4979 }
4980 
WasmCompilationData(wasm::RuntimeExceptionSupport runtime_exception_support)4981 WasmCompilationData::WasmCompilationData(
4982     wasm::RuntimeExceptionSupport runtime_exception_support)
4983     : protected_instructions_(
4984           new std::vector<trap_handler::ProtectedInstructionData>()),
4985       runtime_exception_support_(runtime_exception_support) {}
4986 
AddProtectedInstruction(uint32_t instr_offset,uint32_t landing_offset)4987 void WasmCompilationData::AddProtectedInstruction(uint32_t instr_offset,
4988                                                   uint32_t landing_offset) {
4989   protected_instructions_->emplace_back(
4990       trap_handler::ProtectedInstructionData{instr_offset, landing_offset});
4991 }
4992 
TurbofanWasmCompilationUnit(wasm::WasmCompilationUnit * wasm_unit)4993 TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
4994     wasm::WasmCompilationUnit* wasm_unit)
4995     : wasm_unit_(wasm_unit),
4996       wasm_compilation_data_(wasm_unit->env_->runtime_exception_support) {}
4997 
4998 // Clears unique_ptrs, but (part of) the type is forward declared in the header.
4999 TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5000 
BuildGraphForWasmFunction(double * decode_ms)5001 SourcePositionTable* TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5002     double* decode_ms) {
5003   base::ElapsedTimer decode_timer;
5004   if (FLAG_trace_wasm_decode_time) {
5005     decode_timer.Start();
5006   }
5007 
5008   // Create a TF graph during decoding.
5009   SourcePositionTable* source_position_table =
5010       new (mcgraph_->zone()) SourcePositionTable(mcgraph_->graph());
5011   // We get the handle for {null_value()} directly from the isolate although we
5012   // are on a background task because the handle is stored in the isolate
5013   // anyways, and it is immortal and immovable.
5014   WasmGraphBuilder builder(wasm_unit_->isolate_, wasm_unit_->env_,
5015                            mcgraph_->zone(), mcgraph_, wasm_unit_->centry_stub_,
5016                            wasm_unit_->isolate_->factory()->null_value(),
5017                            wasm_unit_->func_body_.sig, source_position_table);
5018   graph_construction_result_ = wasm::BuildTFGraph(
5019       wasm_unit_->isolate_->allocator(), &builder, wasm_unit_->func_body_);
5020   if (graph_construction_result_.failed()) {
5021     if (FLAG_trace_wasm_compiler) {
5022       OFStream os(stdout);
5023       os << "Compilation failed: " << graph_construction_result_.error_msg()
5024          << std::endl;
5025     }
5026     return nullptr;
5027   }
5028 
5029   builder.LowerInt64();
5030 
5031   if (builder.has_simd() &&
5032       (!CpuFeatures::SupportsWasmSimd128() || wasm_unit_->lower_simd_)) {
5033     SimdScalarLowering(
5034         mcgraph_,
5035         CreateMachineSignature(mcgraph_->zone(), wasm_unit_->func_body_.sig))
5036         .LowerGraph();
5037   }
5038 
5039   if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5040       wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5041     PrintRawWasmCode(wasm_unit_->isolate_->allocator(), wasm_unit_->func_body_,
5042                      wasm_unit_->env_->module, wasm::kPrintLocals);
5043   }
5044   if (FLAG_trace_wasm_decode_time) {
5045     *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5046   }
5047   return source_position_table;
5048 }
5049 
5050 namespace {
GetDebugName(Zone * zone,wasm::WasmName name,int index)5051 Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) {
5052   if (!name.is_empty()) {
5053     return name;
5054   }
5055 #ifdef DEBUG
5056   constexpr int kBufferLength = 15;
5057 
5058   EmbeddedVector<char, kBufferLength> name_vector;
5059   int name_len = SNPrintF(name_vector, "wasm#%d", index);
5060   DCHECK(name_len > 0 && name_len < name_vector.length());
5061 
5062   char* index_name = zone->NewArray<char>(name_len);
5063   memcpy(index_name, name_vector.start(), name_len);
5064   return Vector<const char>(index_name, name_len);
5065 #else
5066   return {};
5067 #endif
5068 }
5069 
5070 }  // namespace
5071 
ExecuteCompilation()5072 void TurbofanWasmCompilationUnit::ExecuteCompilation() {
5073   double decode_ms = 0;
5074   size_t node_count = 0;
5075 
5076   // Scope for the {graph_zone}.
5077   {
5078     Zone graph_zone(wasm_unit_->isolate_->allocator(), ZONE_NAME);
5079     mcgraph_ = new (&graph_zone)
5080         MachineGraph(new (&graph_zone) Graph(&graph_zone),
5081                      new (&graph_zone) CommonOperatorBuilder(&graph_zone),
5082                      new (&graph_zone) MachineOperatorBuilder(
5083                          &graph_zone, MachineType::PointerRepresentation(),
5084                          InstructionSelector::SupportedMachineOperatorFlags(),
5085                          InstructionSelector::AlignmentRequirements()));
5086     SourcePositionTable* source_positions =
5087         BuildGraphForWasmFunction(&decode_ms);
5088 
5089     if (graph_construction_result_.failed()) {
5090       ok_ = false;
5091       return;
5092     }
5093 
5094     base::ElapsedTimer pipeline_timer;
5095     if (FLAG_trace_wasm_decode_time) {
5096       node_count = mcgraph_->graph()->NodeCount();
5097       pipeline_timer.Start();
5098     }
5099 
5100     compilation_zone_.reset(
5101         new Zone(wasm_unit_->isolate_->allocator(), ZONE_NAME));
5102 
5103     // Run the compiler pipeline to generate machine code.
5104     auto call_descriptor = GetWasmCallDescriptor(compilation_zone_.get(),
5105                                                  wasm_unit_->func_body_.sig);
5106     if (mcgraph_->machine()->Is32()) {
5107       call_descriptor =
5108           GetI32WasmCallDescriptor(compilation_zone_.get(), call_descriptor);
5109     }
5110     info_.reset(new OptimizedCompilationInfo(
5111         GetDebugName(compilation_zone_.get(), wasm_unit_->func_name_,
5112                      wasm_unit_->func_index_),
5113         compilation_zone_.get(), Code::WASM_FUNCTION));
5114 
5115     NodeOriginTable* node_origins = info_->trace_turbo_graph_enabled()
5116                                         ? new (&graph_zone)
5117                                               NodeOriginTable(mcgraph_->graph())
5118                                         : nullptr;
5119 
5120     job_.reset(Pipeline::NewWasmCompilationJob(
5121         info_.get(), wasm_unit_->isolate_, mcgraph_, call_descriptor,
5122         source_positions, node_origins, &wasm_compilation_data_,
5123         wasm_unit_->env_->module->origin()));
5124     ok_ = job_->ExecuteJob() == CompilationJob::SUCCEEDED;
5125     // TODO(bradnelson): Improve histogram handling of size_t.
5126     wasm_unit_->counters_->wasm_compile_function_peak_memory_bytes()->AddSample(
5127         static_cast<int>(mcgraph_->graph()->zone()->allocation_size()));
5128 
5129     if (FLAG_trace_wasm_decode_time) {
5130       double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
5131       PrintF(
5132           "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
5133           "%0.3f ms pipeline\n",
5134           static_cast<unsigned>(wasm_unit_->func_body_.end -
5135                                 wasm_unit_->func_body_.start),
5136           decode_ms, node_count, pipeline_ms);
5137     }
5138     // The graph zone is about to get out of scope. Avoid invalid references.
5139     mcgraph_ = nullptr;
5140   }
5141 
5142   // Record the memory cost this unit places on the system until
5143   // it is finalized.
5144   wasm_unit_->memory_cost_ = job_->AllocatedMemory();
5145 }
5146 
FinishCompilation(wasm::ErrorThrower * thrower)5147 wasm::WasmCode* TurbofanWasmCompilationUnit::FinishCompilation(
5148     wasm::ErrorThrower* thrower) {
5149   if (!ok_) {
5150     if (graph_construction_result_.failed()) {
5151       // Add the function as another context for the exception.
5152       EmbeddedVector<char, 128> message;
5153       if (wasm_unit_->func_name_.start() == nullptr) {
5154         SNPrintF(message, "Compiling wasm function #%d failed",
5155                  wasm_unit_->func_index_);
5156       } else {
5157         wasm::TruncatedUserString<> trunc_name(wasm_unit_->func_name_);
5158         SNPrintF(message, "Compiling wasm function #%d:%.*s failed",
5159                  wasm_unit_->func_index_, trunc_name.length(),
5160                  trunc_name.start());
5161       }
5162       thrower->CompileFailed(message.start(), graph_construction_result_);
5163     }
5164 
5165     return nullptr;
5166   }
5167   base::ElapsedTimer codegen_timer;
5168   if (FLAG_trace_wasm_decode_time) {
5169     codegen_timer.Start();
5170   }
5171 
5172   if (job_->FinalizeJob(wasm_unit_->isolate_) != CompilationJob::SUCCEEDED) {
5173     return nullptr;
5174   }
5175 
5176   // TODO(mtrofin): when we crystalize a design in lieu of WasmCodeDesc, that
5177   // works for both wasm and non-wasm, we can simplify AddCode to just take
5178   // that as a parameter.
5179   const CodeDesc& desc = job_->compilation_info()->wasm_code_desc()->code_desc;
5180   wasm::WasmCode* code = wasm_unit_->native_module_->AddCode(
5181       desc, job_->compilation_info()->wasm_code_desc()->frame_slot_count,
5182       wasm_unit_->func_index_,
5183       job_->compilation_info()->wasm_code_desc()->safepoint_table_offset,
5184       job_->compilation_info()->wasm_code_desc()->handler_table_offset,
5185       wasm_compilation_data_.ReleaseProtectedInstructions(),
5186       job_->compilation_info()->wasm_code_desc()->source_positions_table,
5187       wasm::WasmCode::kTurbofan);
5188   if (!code) return code;
5189   if (FLAG_trace_wasm_decode_time) {
5190     double codegen_ms = codegen_timer.Elapsed().InMillisecondsF();
5191     PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n",
5192            static_cast<unsigned>(wasm_unit_->func_body_.end -
5193                                  wasm_unit_->func_body_.start),
5194            codegen_ms);
5195   }
5196 
5197   return code;
5198 }
5199 
5200 namespace {
5201 // Helper for allocating either an GP or FP reg, or the next stack slot.
5202 class LinkageLocationAllocator {
5203  public:
5204   template <size_t kNumGpRegs, size_t kNumFpRegs>
LinkageLocationAllocator(const Register (& gp)[kNumGpRegs],const DoubleRegister (& fp)[kNumFpRegs])5205   constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
5206                                      const DoubleRegister (&fp)[kNumFpRegs])
5207       : allocator_(wasm::LinkageAllocator(gp, fp)) {}
5208 
Next(MachineRepresentation type)5209   LinkageLocation Next(MachineRepresentation type) {
5210     MachineType mach_type = MachineType::TypeForRepresentation(type);
5211     if (type == MachineRepresentation::kFloat32 ||
5212         type == MachineRepresentation::kFloat64) {
5213       if (allocator_.has_more_fp_regs()) {
5214         DoubleRegister reg = allocator_.NextFpReg();
5215 #if V8_TARGET_ARCH_ARM
5216         // Allocate floats using a double register, but modify the code to
5217         // reflect how ARM FP registers alias.
5218         // TODO(bbudge) Modify wasm linkage to allow use of all float regs.
5219         if (type == MachineRepresentation::kFloat32) {
5220           int float_reg_code = reg.code() * 2;
5221           DCHECK_GT(RegisterConfiguration::kMaxFPRegisters, float_reg_code);
5222           return LinkageLocation::ForRegister(
5223               DoubleRegister::from_code(float_reg_code).code(), mach_type);
5224         }
5225 #endif
5226         return LinkageLocation::ForRegister(reg.code(), mach_type);
5227       }
5228     } else if (allocator_.has_more_gp_regs()) {
5229       return LinkageLocation::ForRegister(allocator_.NextGpReg().code(),
5230                                           mach_type);
5231     }
5232     // Cannot use register; use stack slot.
5233     int index = -1 - allocator_.NextStackSlot(type);
5234     return LinkageLocation::ForCallerFrameSlot(index, mach_type);
5235   }
5236 
SetStackOffset(int offset)5237   void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
NumStackSlots() const5238   int NumStackSlots() const { return allocator_.NumStackSlots(); }
5239 
5240  private:
5241   wasm::LinkageAllocator allocator_;
5242 };
5243 }  // namespace
5244 
5245 // General code uses the above configuration data.
GetWasmCallDescriptor(Zone * zone,wasm::FunctionSig * fsig,WasmGraphBuilder::UseRetpoline use_retpoline)5246 CallDescriptor* GetWasmCallDescriptor(
5247     Zone* zone, wasm::FunctionSig* fsig,
5248     WasmGraphBuilder::UseRetpoline use_retpoline) {
5249   // The '+ 1' here is to accomodate the instance object as first parameter.
5250   LocationSignature::Builder locations(zone, fsig->return_count(),
5251                                        fsig->parameter_count() + 1);
5252 
5253   // Add register and/or stack parameter(s).
5254   LinkageLocationAllocator params(wasm::kGpParamRegisters,
5255                                   wasm::kFpParamRegisters);
5256 
5257   // The instance object.
5258   locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
5259 
5260   const int parameter_count = static_cast<int>(fsig->parameter_count());
5261   for (int i = 0; i < parameter_count; i++) {
5262     MachineRepresentation param =
5263         wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
5264     auto l = params.Next(param);
5265     locations.AddParam(l);
5266   }
5267 
5268   // Add return location(s).
5269   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5270                                 wasm::kFpReturnRegisters);
5271   rets.SetStackOffset(params.NumStackSlots());
5272 
5273   const int return_count = static_cast<int>(locations.return_count_);
5274   for (int i = 0; i < return_count; i++) {
5275     MachineRepresentation ret =
5276         wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
5277     auto l = rets.Next(ret);
5278     locations.AddReturn(l);
5279   }
5280 
5281   const RegList kCalleeSaveRegisters = 0;
5282   const RegList kCalleeSaveFPRegisters = 0;
5283 
5284   // The target for wasm calls is always a code object.
5285   MachineType target_type = MachineType::Pointer();
5286   LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
5287 
5288   CallDescriptor::Kind kind = CallDescriptor::kCallWasmFunction;
5289 
5290   CallDescriptor::Flags flags =
5291       use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
5292   return new (zone) CallDescriptor(                    // --
5293       kind,                                            // kind
5294       target_type,                                     // target MachineType
5295       target_loc,                                      // target location
5296       locations.Build(),                               // location_sig
5297       params.NumStackSlots(),                          // stack_parameter_count
5298       compiler::Operator::kNoProperties,               // properties
5299       kCalleeSaveRegisters,                            // callee-saved registers
5300       kCalleeSaveFPRegisters,                          // callee-saved fp regs
5301       flags,                                           // flags
5302       "wasm-call",                                     // debug name
5303       0,                                               // allocatable registers
5304       rets.NumStackSlots() - params.NumStackSlots());  // stack_return_count
5305 }
5306 
5307 namespace {
ReplaceTypeInCallDescriptorWith(Zone * zone,CallDescriptor * call_descriptor,size_t num_replacements,MachineType input_type,MachineRepresentation output_type)5308 CallDescriptor* ReplaceTypeInCallDescriptorWith(
5309     Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
5310     MachineType input_type, MachineRepresentation output_type) {
5311   size_t parameter_count = call_descriptor->ParameterCount();
5312   size_t return_count = call_descriptor->ReturnCount();
5313   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5314     if (call_descriptor->GetParameterType(i) == input_type) {
5315       parameter_count += num_replacements - 1;
5316     }
5317   }
5318   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5319     if (call_descriptor->GetReturnType(i) == input_type) {
5320       return_count += num_replacements - 1;
5321     }
5322   }
5323   if (parameter_count == call_descriptor->ParameterCount() &&
5324       return_count == call_descriptor->ReturnCount()) {
5325     return call_descriptor;
5326   }
5327 
5328   LocationSignature::Builder locations(zone, return_count, parameter_count);
5329 
5330   LinkageLocationAllocator params(wasm::kGpParamRegisters,
5331                                   wasm::kFpParamRegisters);
5332   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5333     if (call_descriptor->GetParameterType(i) == input_type) {
5334       for (size_t j = 0; j < num_replacements; j++) {
5335         locations.AddParam(params.Next(output_type));
5336       }
5337     } else {
5338       locations.AddParam(
5339           params.Next(call_descriptor->GetParameterType(i).representation()));
5340     }
5341   }
5342 
5343   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5344                                 wasm::kFpReturnRegisters);
5345   rets.SetStackOffset(params.NumStackSlots());
5346   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5347     if (call_descriptor->GetReturnType(i) == input_type) {
5348       for (size_t j = 0; j < num_replacements; j++) {
5349         locations.AddReturn(rets.Next(output_type));
5350       }
5351     } else {
5352       locations.AddReturn(
5353           rets.Next(call_descriptor->GetReturnType(i).representation()));
5354     }
5355   }
5356 
5357   return new (zone) CallDescriptor(                    // --
5358       call_descriptor->kind(),                         // kind
5359       call_descriptor->GetInputType(0),                // target MachineType
5360       call_descriptor->GetInputLocation(0),            // target location
5361       locations.Build(),                               // location_sig
5362       params.NumStackSlots(),                          // stack_parameter_count
5363       call_descriptor->properties(),                   // properties
5364       call_descriptor->CalleeSavedRegisters(),         // callee-saved registers
5365       call_descriptor->CalleeSavedFPRegisters(),       // callee-saved fp regs
5366       call_descriptor->flags(),                        // flags
5367       call_descriptor->debug_name(),                   // debug name
5368       call_descriptor->AllocatableRegisters(),         // allocatable registers
5369       rets.NumStackSlots() - params.NumStackSlots());  // stack_return_count
5370 }
5371 }  // namespace
5372 
GetI32WasmCallDescriptor(Zone * zone,CallDescriptor * call_descriptor)5373 CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
5374                                          CallDescriptor* call_descriptor) {
5375   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
5376                                          MachineType::Int64(),
5377                                          MachineRepresentation::kWord32);
5378 }
5379 
GetI32WasmCallDescriptorForSimd(Zone * zone,CallDescriptor * call_descriptor)5380 CallDescriptor* GetI32WasmCallDescriptorForSimd(
5381     Zone* zone, CallDescriptor* call_descriptor) {
5382   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
5383                                          MachineType::Simd128(),
5384                                          MachineRepresentation::kWord32);
5385 }
5386 
5387 #undef WASM_64
5388 #undef FATAL_UNSUPPORTED_OPCODE
5389 #undef WASM_INSTANCE_OBJECT_OFFSET
5390 #undef LOAD_INSTANCE_FIELD
5391 #undef LOAD_FIXED_ARRAY_SLOT
5392 
5393 }  // namespace compiler
5394 }  // namespace internal
5395 }  // namespace v8
5396