1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/wasm/module-compiler.h"
6
7 #include "src/api.h"
8 #include "src/asmjs/asm-js.h"
9 #include "src/assembler-inl.h"
10 #include "src/base/optional.h"
11 #include "src/base/template-utils.h"
12 #include "src/base/utils/random-number-generator.h"
13 #include "src/code-factory.h"
14 #include "src/code-stubs.h"
15 #include "src/compiler/wasm-compiler.h"
16 #include "src/counters.h"
17 #include "src/identity-map.h"
18 #include "src/property-descriptor.h"
19 #include "src/trap-handler/trap-handler.h"
20 #include "src/wasm/module-decoder.h"
21 #include "src/wasm/streaming-decoder.h"
22 #include "src/wasm/wasm-code-manager.h"
23 #include "src/wasm/wasm-code-specialization.h"
24 #include "src/wasm/wasm-engine.h"
25 #include "src/wasm/wasm-js.h"
26 #include "src/wasm/wasm-memory.h"
27 #include "src/wasm/wasm-objects-inl.h"
28 #include "src/wasm/wasm-result.h"
29
30 #define TRACE(...) \
31 do { \
32 if (FLAG_trace_wasm_instances) PrintF(__VA_ARGS__); \
33 } while (false)
34
35 #define TRACE_CHAIN(instance) \
36 do { \
37 instance->PrintInstancesChain(); \
38 } while (false)
39
40 #define TRACE_COMPILE(...) \
41 do { \
42 if (FLAG_trace_wasm_compiler) PrintF(__VA_ARGS__); \
43 } while (false)
44
45 #define TRACE_STREAMING(...) \
46 do { \
47 if (FLAG_trace_wasm_streaming) PrintF(__VA_ARGS__); \
48 } while (false)
49
50 #define TRACE_LAZY(...) \
51 do { \
52 if (FLAG_trace_wasm_lazy_compilation) PrintF(__VA_ARGS__); \
53 } while (false)
54
55 namespace v8 {
56 namespace internal {
57 namespace wasm {
58
59 enum class CompilationEvent : uint8_t {
60 kFinishedBaselineCompilation,
61 kFinishedTopTierCompilation,
62 kFailedCompilation,
63 kDestroyed
64 };
65
66 enum class CompileMode : uint8_t { kRegular, kTiering };
67
68 // The CompilationState keeps track of the compilation state of the
69 // owning NativeModule, i.e. which functions are left to be compiled.
70 // It contains a task manager to allow parallel and asynchronous background
71 // compilation of functions.
72 class CompilationState {
73 public:
74 CompilationState(internal::Isolate* isolate, ModuleEnv& env);
75 ~CompilationState();
76
77 // Needs to be set before {AddCompilationUnits} is run, which triggers
78 // background compilation.
79 void SetNumberOfFunctionsToCompile(size_t num_functions);
80 void AddCallback(
81 std::function<void(CompilationEvent, ErrorThrower*)> callback);
82
83 // Inserts new functions to compile and kicks off compilation.
84 void AddCompilationUnits(
85 std::vector<std::unique_ptr<WasmCompilationUnit>>& baseline_units,
86 std::vector<std::unique_ptr<WasmCompilationUnit>>& tiering_units);
87 std::unique_ptr<WasmCompilationUnit> GetNextCompilationUnit();
88 std::unique_ptr<WasmCompilationUnit> GetNextExecutedUnit();
89
90 bool HasCompilationUnitToFinish();
91
92 void OnError(ErrorThrower* thrower);
93 void OnFinishedUnit();
94 void ScheduleUnitForFinishing(std::unique_ptr<WasmCompilationUnit> unit,
95 WasmCompilationUnit::CompilationMode mode);
96
97 void CancelAndWait();
98 void OnBackgroundTaskStopped();
99 void RestartBackgroundTasks(size_t max = std::numeric_limits<size_t>::max());
100 // Only one foreground thread (finisher) is allowed to run at a time.
101 // {SetFinisherIsRunning} returns whether the flag changed its state.
102 bool SetFinisherIsRunning(bool value);
103 void ScheduleFinisherTask();
104
105 bool StopBackgroundCompilationTaskForThrottling();
106
107 void Abort();
108
isolate() const109 Isolate* isolate() const { return isolate_; }
110
failed() const111 bool failed() const {
112 base::LockGuard<base::Mutex> guard(&mutex_);
113 return failed_;
114 }
115
baseline_compilation_finished() const116 bool baseline_compilation_finished() const {
117 return baseline_compilation_finished_;
118 }
119
compile_mode() const120 CompileMode compile_mode() const { return compile_mode_; }
121
module_env()122 ModuleEnv* module_env() { return &module_env_; }
123
wire_bytes() const124 const ModuleWireBytes& wire_bytes() const { return wire_bytes_; }
125
SetWireBytes(const ModuleWireBytes & wire_bytes)126 void SetWireBytes(const ModuleWireBytes& wire_bytes) {
127 DCHECK_NULL(bytes_copy_);
128 DCHECK_EQ(0, wire_bytes_.length());
129 bytes_copy_ = std::unique_ptr<byte[]>(new byte[wire_bytes.length()]);
130 memcpy(bytes_copy_.get(), wire_bytes.start(), wire_bytes.length());
131 wire_bytes_ = ModuleWireBytes(bytes_copy_.get(),
132 bytes_copy_.get() + wire_bytes.length());
133 }
134
135 private:
136 void NotifyOnEvent(CompilationEvent event, ErrorThrower* thrower);
137
finish_units()138 std::vector<std::unique_ptr<WasmCompilationUnit>>& finish_units() {
139 return baseline_compilation_finished_ ? tiering_finish_units_
140 : baseline_finish_units_;
141 }
142
143 Isolate* const isolate_;
144 ModuleEnv module_env_;
145 const size_t max_memory_;
146 const CompileMode compile_mode_;
147 bool baseline_compilation_finished_ = false;
148
149 // TODO(wasm): eventually we want to get rid of this
150 // additional copy (see AsyncCompileJob).
151 std::unique_ptr<byte[]> bytes_copy_;
152 ModuleWireBytes wire_bytes_;
153
154 // This mutex protects all information of this CompilationState which is being
155 // accessed concurrently.
156 mutable base::Mutex mutex_;
157
158 //////////////////////////////////////////////////////////////////////////////
159 // Protected by {mutex_}:
160
161 std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_compilation_units_;
162 std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_compilation_units_;
163
164 bool finisher_is_running_ = false;
165 bool failed_ = false;
166 size_t num_background_tasks_ = 0;
167
168 std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_finish_units_;
169 std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_finish_units_;
170
171 size_t allocated_memory_ = 0;
172
173 // End of fields protected by {mutex_}.
174 //////////////////////////////////////////////////////////////////////////////
175
176 // TODO(mstarzinger): We should make sure this allows at most one callback
177 // to exist for each {CompilationState} because reifying the error object on
178 // the given {ErrorThrower} can be done at most once.
179 std::vector<std::function<void(CompilationEvent, ErrorThrower*)>> callbacks_;
180
181 // When canceling the background_task_manager_, use {CancelAndWait} on
182 // the CompilationState in order to cleanly clean up.
183 CancelableTaskManager background_task_manager_;
184 CancelableTaskManager foreground_task_manager_;
185 std::shared_ptr<v8::TaskRunner> background_task_runner_;
186 std::shared_ptr<v8::TaskRunner> foreground_task_runner_;
187
188 const size_t max_background_tasks_ = 0;
189
190 size_t outstanding_units_ = 0;
191 size_t num_tiering_units_ = 0;
192 };
193
194 namespace {
195
196 class JSToWasmWrapperCache {
197 public:
CloneOrCompileJSToWasmWrapper(Isolate * isolate,wasm::WasmModule * module,Address call_target,uint32_t index,wasm::UseTrapHandler use_trap_handler)198 Handle<Code> CloneOrCompileJSToWasmWrapper(
199 Isolate* isolate, wasm::WasmModule* module, Address call_target,
200 uint32_t index, wasm::UseTrapHandler use_trap_handler) {
201 const bool is_import = index < module->num_imported_functions;
202 DCHECK_EQ(is_import, call_target == kNullAddress);
203 const wasm::WasmFunction* func = &module->functions[index];
204 // We cannot cache js-to-wasm wrappers for imports, as they hard-code the
205 // function index.
206 if (!is_import) {
207 int cached_idx = sig_map_.Find(func->sig);
208 if (cached_idx >= 0) {
209 Handle<Code> code =
210 isolate->factory()->CopyCode(code_cache_[cached_idx]);
211 // Now patch the call to wasm code.
212 RelocIterator it(*code,
213 RelocInfo::ModeMask(RelocInfo::JS_TO_WASM_CALL));
214 // If there is no reloc info, then it's an incompatible signature or
215 // calls an import.
216 if (!it.done()) it.rinfo()->set_js_to_wasm_address(call_target);
217 return code;
218 }
219 }
220
221 Handle<Code> code = compiler::CompileJSToWasmWrapper(
222 isolate, module, call_target, index, use_trap_handler);
223 if (!is_import) {
224 uint32_t new_cache_idx = sig_map_.FindOrInsert(func->sig);
225 DCHECK_EQ(code_cache_.size(), new_cache_idx);
226 USE(new_cache_idx);
227 code_cache_.push_back(code);
228 }
229 return code;
230 }
231
232 private:
233 // sig_map_ maps signatures to an index in code_cache_.
234 wasm::SignatureMap sig_map_;
235 std::vector<Handle<Code>> code_cache_;
236 };
237
238 // A helper class to simplify instantiating a module from a compiled module.
239 // It closes over the {Isolate}, the {ErrorThrower}, the {WasmCompiledModule},
240 // etc.
241 class InstanceBuilder {
242 public:
243 InstanceBuilder(Isolate* isolate, ErrorThrower* thrower,
244 Handle<WasmModuleObject> module_object,
245 MaybeHandle<JSReceiver> ffi,
246 MaybeHandle<JSArrayBuffer> memory);
247
248 // Build an instance, in all of its glory.
249 MaybeHandle<WasmInstanceObject> Build();
250 // Run the start function, if any.
251 bool ExecuteStartFunction();
252
253 private:
254 // Represents the initialized state of a table.
255 struct TableInstance {
256 Handle<WasmTableObject> table_object; // WebAssembly.Table instance
257 Handle<FixedArray> js_wrappers; // JSFunctions exported
258 size_t table_size;
259 };
260
261 // A pre-evaluated value to use in import binding.
262 struct SanitizedImport {
263 Handle<String> module_name;
264 Handle<String> import_name;
265 Handle<Object> value;
266 };
267
268 Isolate* isolate_;
269 WasmModule* const module_;
270 const std::shared_ptr<Counters> async_counters_;
271 ErrorThrower* thrower_;
272 Handle<WasmModuleObject> module_object_;
273 MaybeHandle<JSReceiver> ffi_;
274 MaybeHandle<JSArrayBuffer> memory_;
275 Handle<JSArrayBuffer> globals_;
276 Handle<WasmCompiledModule> compiled_module_;
277 std::vector<TableInstance> table_instances_;
278 std::vector<Handle<JSFunction>> js_wrappers_;
279 Handle<WasmExportedFunction> start_function_;
280 JSToWasmWrapperCache js_to_wasm_cache_;
281 std::vector<SanitizedImport> sanitized_imports_;
282
async_counters() const283 const std::shared_ptr<Counters>& async_counters() const {
284 return async_counters_;
285 }
286
counters() const287 Counters* counters() const { return async_counters().get(); }
288
use_trap_handler() const289 wasm::UseTrapHandler use_trap_handler() const {
290 return compiled_module_->GetNativeModule()->use_trap_handler()
291 ? kUseTrapHandler
292 : kNoTrapHandler;
293 }
294
295 // Helper routines to print out errors with imports.
296 #define ERROR_THROWER_WITH_MESSAGE(TYPE) \
297 void Report##TYPE(const char* error, uint32_t index, \
298 Handle<String> module_name, Handle<String> import_name) { \
299 thrower_->TYPE("Import #%d module=\"%s\" function=\"%s\" error: %s", \
300 index, module_name->ToCString().get(), \
301 import_name->ToCString().get(), error); \
302 } \
303 \
304 MaybeHandle<Object> Report##TYPE(const char* error, uint32_t index, \
305 Handle<String> module_name) { \
306 thrower_->TYPE("Import #%d module=\"%s\" error: %s", index, \
307 module_name->ToCString().get(), error); \
308 return MaybeHandle<Object>(); \
309 }
310
311 ERROR_THROWER_WITH_MESSAGE(LinkError)
312 ERROR_THROWER_WITH_MESSAGE(TypeError)
313
314 #undef ERROR_THROWER_WITH_MESSAGE
315
316 // Look up an import value in the {ffi_} object.
317 MaybeHandle<Object> LookupImport(uint32_t index, Handle<String> module_name,
318 Handle<String> import_name);
319
320 // Look up an import value in the {ffi_} object specifically for linking an
321 // asm.js module. This only performs non-observable lookups, which allows
322 // falling back to JavaScript proper (and hence re-executing all lookups) if
323 // module instantiation fails.
324 MaybeHandle<Object> LookupImportAsm(uint32_t index,
325 Handle<String> import_name);
326
327 uint32_t EvalUint32InitExpr(const WasmInitExpr& expr);
328
329 // Load data segments into the memory.
330 void LoadDataSegments(Handle<WasmInstanceObject> instance);
331
332 void WriteGlobalValue(WasmGlobal& global, double value);
333 void WriteGlobalValue(WasmGlobal& global, Handle<WasmGlobalObject> value);
334
335 void SanitizeImports();
336
337 // Process the imports, including functions, tables, globals, and memory, in
338 // order, loading them from the {ffi_} object. Returns the number of imported
339 // functions.
340 int ProcessImports(Handle<WasmInstanceObject> instance);
341
342 template <typename T>
343 T* GetRawGlobalPtr(WasmGlobal& global);
344
345 // Process initialization of globals.
346 void InitGlobals();
347
348 // Allocate memory for a module instance as a new JSArrayBuffer.
349 Handle<JSArrayBuffer> AllocateMemory(uint32_t num_pages);
350
351 bool NeedsWrappers() const;
352
353 // Process the exports, creating wrappers for functions, tables, memories,
354 // and globals.
355 void ProcessExports(Handle<WasmInstanceObject> instance);
356
357 void InitializeTables(Handle<WasmInstanceObject> instance);
358
359 void LoadTableSegments(Handle<WasmInstanceObject> instance);
360 };
361
362 } // namespace
363
InstantiateToInstanceObject(Isolate * isolate,ErrorThrower * thrower,Handle<WasmModuleObject> module_object,MaybeHandle<JSReceiver> imports,MaybeHandle<JSArrayBuffer> memory)364 MaybeHandle<WasmInstanceObject> InstantiateToInstanceObject(
365 Isolate* isolate, ErrorThrower* thrower,
366 Handle<WasmModuleObject> module_object, MaybeHandle<JSReceiver> imports,
367 MaybeHandle<JSArrayBuffer> memory) {
368 InstanceBuilder builder(isolate, thrower, module_object, imports, memory);
369 auto instance = builder.Build();
370 if (!instance.is_null() && builder.ExecuteStartFunction()) {
371 return instance;
372 }
373 return {};
374 }
375
376 // A helper class to prevent pathological patching behavior for indirect
377 // references to code which must be updated after lazy compiles.
378 // Utilizes a reverse mapping to prevent O(n^2) behavior.
379 class IndirectPatcher {
380 public:
Patch(Handle<WasmInstanceObject> caller_instance,Handle<WasmInstanceObject> target_instance,int func_index,Address old_target,Address new_target)381 void Patch(Handle<WasmInstanceObject> caller_instance,
382 Handle<WasmInstanceObject> target_instance, int func_index,
383 Address old_target, Address new_target) {
384 TRACE_LAZY(
385 "IndirectPatcher::Patch(caller=%p, target=%p, func_index=%i, "
386 "old_target=%" PRIuPTR ", new_target=%" PRIuPTR ")\n",
387 *caller_instance, *target_instance, func_index, old_target, new_target);
388 if (mapping_.size() == 0 || misses_ >= kMaxMisses) {
389 BuildMapping(caller_instance);
390 }
391 // Patch entries for the given function index.
392 WasmCodeManager* code_manager =
393 caller_instance->GetIsolate()->wasm_engine()->code_manager();
394 USE(code_manager);
395 auto& entries = mapping_[func_index];
396 int patched = 0;
397 for (auto index : entries) {
398 if (index < 0) {
399 // Imported function entry.
400 int i = -1 - index;
401 ImportedFunctionEntry entry(caller_instance, i);
402 if (entry.target() == old_target) {
403 DCHECK_EQ(
404 func_index,
405 code_manager->GetCodeFromStartAddress(entry.target())->index());
406 entry.set_wasm_to_wasm(*target_instance, new_target);
407 patched++;
408 }
409 } else {
410 // Indirect function table entry.
411 int i = index;
412 IndirectFunctionTableEntry entry(caller_instance, i);
413 if (entry.target() == old_target) {
414 DCHECK_EQ(
415 func_index,
416 code_manager->GetCodeFromStartAddress(entry.target())->index());
417 entry.set(entry.sig_id(), *target_instance, new_target);
418 patched++;
419 }
420 }
421 }
422 if (patched == 0) misses_++;
423 }
424
425 private:
BuildMapping(Handle<WasmInstanceObject> caller_instance)426 void BuildMapping(Handle<WasmInstanceObject> caller_instance) {
427 mapping_.clear();
428 misses_ = 0;
429 TRACE_LAZY("BuildMapping for (caller=%p)...\n", *caller_instance);
430 Isolate* isolate = caller_instance->GetIsolate();
431 WasmCodeManager* code_manager = isolate->wasm_engine()->code_manager();
432 uint32_t num_imported_functions =
433 caller_instance->module()->num_imported_functions;
434 // Process the imported function entries.
435 for (unsigned i = 0; i < num_imported_functions; i++) {
436 ImportedFunctionEntry entry(caller_instance, i);
437 WasmCode* code = code_manager->GetCodeFromStartAddress(entry.target());
438 if (code->kind() != WasmCode::kLazyStub) continue;
439 TRACE_LAZY(" +import[%u] -> #%d (%p)\n", i, code->index(),
440 code->instructions().start());
441 DCHECK(!entry.is_js_receiver_entry());
442 WasmInstanceObject* target_instance = entry.instance();
443 WasmCode* new_code =
444 target_instance->compiled_module()->GetNativeModule()->code(
445 code->index());
446 if (new_code->kind() != WasmCode::kLazyStub) {
447 // Patch an imported function entry which is already compiled.
448 entry.set_wasm_to_wasm(target_instance, new_code->instruction_start());
449 } else {
450 int key = code->index();
451 int index = -1 - i;
452 mapping_[key].push_back(index);
453 }
454 }
455 // Process the indirect function table entries.
456 size_t ift_size = caller_instance->indirect_function_table_size();
457 for (unsigned i = 0; i < ift_size; i++) {
458 IndirectFunctionTableEntry entry(caller_instance, i);
459 if (entry.target() == kNullAddress) continue; // null IFT entry
460 WasmCode* code = code_manager->GetCodeFromStartAddress(entry.target());
461 if (code->kind() != WasmCode::kLazyStub) continue;
462 TRACE_LAZY(" +indirect[%u] -> #%d (lazy:%p)\n", i, code->index(),
463 code->instructions().start());
464 WasmInstanceObject* target_instance = entry.instance();
465 WasmCode* new_code =
466 target_instance->compiled_module()->GetNativeModule()->code(
467 code->index());
468 if (new_code->kind() != WasmCode::kLazyStub) {
469 // Patch an indirect function table entry which is already compiled.
470 entry.set(entry.sig_id(), target_instance,
471 new_code->instruction_start());
472 } else {
473 int key = code->index();
474 int index = i;
475 mapping_[key].push_back(index);
476 }
477 }
478 }
479
480 static constexpr int kMaxMisses = 5; // maximum misses before rebuilding
481 std::unordered_map<int, std::vector<int>> mapping_;
482 int misses_ = 0;
483 };
484
CreateModuleEnvFromModuleObject(Isolate * isolate,Handle<WasmModuleObject> module_object)485 ModuleEnv CreateModuleEnvFromModuleObject(
486 Isolate* isolate, Handle<WasmModuleObject> module_object) {
487 WasmModule* module = module_object->shared()->module();
488 wasm::UseTrapHandler use_trap_handler =
489 module_object->compiled_module()->GetNativeModule()->use_trap_handler()
490 ? kUseTrapHandler
491 : kNoTrapHandler;
492 return ModuleEnv(module, use_trap_handler, wasm::kRuntimeExceptionSupport);
493 }
494
LazyCompileFunction(Isolate * isolate,Handle<WasmModuleObject> module_object,int func_index)495 const wasm::WasmCode* LazyCompileFunction(
496 Isolate* isolate, Handle<WasmModuleObject> module_object, int func_index) {
497 base::ElapsedTimer compilation_timer;
498 NativeModule* native_module =
499 module_object->compiled_module()->GetNativeModule();
500 wasm::WasmCode* existing_code =
501 native_module->code(static_cast<uint32_t>(func_index));
502 if (existing_code != nullptr &&
503 existing_code->kind() == wasm::WasmCode::kFunction) {
504 TRACE_LAZY("Function %d already compiled.\n", func_index);
505 return existing_code;
506 }
507
508 compilation_timer.Start();
509 // TODO(wasm): Refactor this to only get the name if it is really needed for
510 // tracing / debugging.
511 std::string func_name;
512 {
513 WasmName name = Vector<const char>::cast(
514 module_object->shared()->GetRawFunctionName(func_index));
515 // Copy to std::string, because the underlying string object might move on
516 // the heap.
517 func_name.assign(name.start(), static_cast<size_t>(name.length()));
518 }
519
520 TRACE_LAZY("Compiling function %s, %d.\n", func_name.c_str(), func_index);
521
522 ModuleEnv module_env =
523 CreateModuleEnvFromModuleObject(isolate, module_object);
524
525 const uint8_t* module_start =
526 module_object->shared()->module_bytes()->GetChars();
527
528 const WasmFunction* func = &module_env.module->functions[func_index];
529 FunctionBody body{func->sig, func->code.offset(),
530 module_start + func->code.offset(),
531 module_start + func->code.end_offset()};
532
533 ErrorThrower thrower(isolate, "WasmLazyCompile");
534 WasmCompilationUnit unit(isolate, &module_env, native_module, body,
535 CStrVector(func_name.c_str()), func_index,
536 CodeFactory::CEntry(isolate));
537 unit.ExecuteCompilation();
538 wasm::WasmCode* wasm_code = unit.FinishCompilation(&thrower);
539
540 if (wasm::WasmCode::ShouldBeLogged(isolate)) wasm_code->LogCode(isolate);
541
542 // If there is a pending error, something really went wrong. The module was
543 // verified before starting execution with lazy compilation.
544 // This might be OOM, but then we cannot continue execution anyway.
545 // TODO(clemensh): According to the spec, we can actually skip validation at
546 // module creation time, and return a function that always traps here.
547 CHECK(!thrower.error());
548
549 // Now specialize the generated code for this instance.
550 CodeSpecialization code_specialization;
551 code_specialization.RelocateDirectCalls(native_module);
552 code_specialization.ApplyToWasmCode(wasm_code, SKIP_ICACHE_FLUSH);
553 int64_t func_size =
554 static_cast<int64_t>(func->code.end_offset() - func->code.offset());
555 int64_t compilation_time = compilation_timer.Elapsed().InMicroseconds();
556
557 auto counters = isolate->counters();
558 counters->wasm_lazily_compiled_functions()->Increment();
559
560 Assembler::FlushICache(wasm_code->instructions().start(),
561 wasm_code->instructions().size());
562 counters->wasm_generated_code_size()->Increment(
563 static_cast<int>(wasm_code->instructions().size()));
564 counters->wasm_reloc_size()->Increment(
565 static_cast<int>(wasm_code->reloc_info().size()));
566
567 counters->wasm_lazy_compilation_throughput()->AddSample(
568 compilation_time != 0 ? static_cast<int>(func_size / compilation_time)
569 : 0);
570
571 if (trap_handler::IsTrapHandlerEnabled()) {
572 wasm_code->RegisterTrapHandlerData();
573 }
574 return wasm_code;
575 }
576
577 namespace {
578
AdvanceSourcePositionTableIterator(SourcePositionTableIterator & iterator,int offset)579 int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
580 int offset) {
581 DCHECK(!iterator.done());
582 int byte_pos;
583 do {
584 byte_pos = iterator.source_position().ScriptOffset();
585 iterator.Advance();
586 } while (!iterator.done() && iterator.code_offset() <= offset);
587 return byte_pos;
588 }
589
LazyCompileFromJsToWasm(Isolate * isolate,Handle<WasmInstanceObject> instance,Handle<Code> js_to_wasm_caller,uint32_t callee_func_index)590 const wasm::WasmCode* LazyCompileFromJsToWasm(
591 Isolate* isolate, Handle<WasmInstanceObject> instance,
592 Handle<Code> js_to_wasm_caller, uint32_t callee_func_index) {
593 Decoder decoder(nullptr, nullptr);
594 Handle<WasmModuleObject> module_object(instance->module_object());
595 NativeModule* native_module = instance->compiled_module()->GetNativeModule();
596
597 TRACE_LAZY(
598 "Starting lazy compilation (func %u, js_to_wasm: true, patch caller: "
599 "true). \n",
600 callee_func_index);
601 LazyCompileFunction(isolate, module_object, callee_func_index);
602 {
603 DisallowHeapAllocation no_gc;
604 CodeSpaceMemoryModificationScope modification_scope(isolate->heap());
605 RelocIterator it(*js_to_wasm_caller,
606 RelocInfo::ModeMask(RelocInfo::JS_TO_WASM_CALL));
607 DCHECK(!it.done());
608 const wasm::WasmCode* callee_compiled =
609 native_module->code(callee_func_index);
610 DCHECK_NOT_NULL(callee_compiled);
611 DCHECK_EQ(WasmCode::kLazyStub,
612 isolate->wasm_engine()
613 ->code_manager()
614 ->GetCodeFromStartAddress(it.rinfo()->js_to_wasm_address())
615 ->kind());
616 it.rinfo()->set_js_to_wasm_address(callee_compiled->instruction_start());
617 TRACE_LAZY("Patched 1 location in js-to-wasm %p.\n", *js_to_wasm_caller);
618
619 #ifdef DEBUG
620 it.next();
621 DCHECK(it.done());
622 #endif
623 }
624
625 wasm::WasmCode* ret = native_module->code(callee_func_index);
626 DCHECK_NOT_NULL(ret);
627 DCHECK_EQ(wasm::WasmCode::kFunction, ret->kind());
628 return ret;
629 }
630
LazyCompileIndirectCall(Isolate * isolate,Handle<WasmInstanceObject> instance,uint32_t func_index)631 const wasm::WasmCode* LazyCompileIndirectCall(
632 Isolate* isolate, Handle<WasmInstanceObject> instance,
633 uint32_t func_index) {
634 TRACE_LAZY(
635 "Starting lazy compilation (func %u, js_to_wasm: false, patch caller: "
636 "false). \n",
637 func_index);
638 Handle<WasmModuleObject> module_object(instance->module_object());
639 return LazyCompileFunction(isolate, module_object, func_index);
640 }
641
LazyCompileDirectCall(Isolate * isolate,Handle<WasmInstanceObject> instance,const wasm::WasmCode * wasm_caller,int32_t caller_ret_offset)642 const wasm::WasmCode* LazyCompileDirectCall(Isolate* isolate,
643 Handle<WasmInstanceObject> instance,
644 const wasm::WasmCode* wasm_caller,
645 int32_t caller_ret_offset) {
646 DCHECK_LE(0, caller_ret_offset);
647
648 Decoder decoder(nullptr, nullptr);
649
650 // Gather all the targets of direct calls inside the code of {wasm_caller}
651 // and place their function indexes in {direct_callees}.
652 std::vector<int32_t> direct_callees;
653 // The last one before {caller_ret_offset} must be the call that triggered
654 // this lazy compilation.
655 int callee_pos = -1;
656 uint32_t num_non_compiled_callees = 0; // For stats.
657 {
658 DisallowHeapAllocation no_gc;
659 Handle<WasmSharedModuleData> shared(
660 wasm_caller->native_module()->shared_module_data(), isolate);
661 SeqOneByteString* module_bytes = shared->module_bytes();
662 uint32_t caller_func_index = wasm_caller->index();
663 SourcePositionTableIterator source_pos_iterator(
664 wasm_caller->source_positions());
665
666 const byte* func_bytes =
667 module_bytes->GetChars() +
668 shared->module()->functions[caller_func_index].code.offset();
669 for (RelocIterator it(wasm_caller->instructions(),
670 wasm_caller->reloc_info(),
671 wasm_caller->constant_pool(),
672 RelocInfo::ModeMask(RelocInfo::WASM_CALL));
673 !it.done(); it.next()) {
674 // TODO(clemensh): Introduce safe_cast<T, bool> which (D)CHECKS
675 // (depending on the bool) against limits of T and then static_casts.
676 size_t offset_l = it.rinfo()->pc() - wasm_caller->instruction_start();
677 DCHECK_GE(kMaxInt, offset_l);
678 int offset = static_cast<int>(offset_l);
679 int byte_pos =
680 AdvanceSourcePositionTableIterator(source_pos_iterator, offset);
681
682 WasmCode* callee = isolate->wasm_engine()->code_manager()->LookupCode(
683 it.rinfo()->target_address());
684 if (callee->kind() == WasmCode::kLazyStub) {
685 // The callee has not been compiled.
686 ++num_non_compiled_callees;
687 int32_t callee_func_index =
688 ExtractDirectCallIndex(decoder, func_bytes + byte_pos);
689 DCHECK_LT(callee_func_index,
690 wasm_caller->native_module()->function_count());
691 // {caller_ret_offset} points to one instruction after the call.
692 // Remember the last called function before that offset.
693 if (offset < caller_ret_offset) {
694 callee_pos = static_cast<int>(direct_callees.size());
695 }
696 direct_callees.push_back(callee_func_index);
697 } else {
698 // If the callee is not the lazy compile stub, assume this callee
699 // has already been compiled.
700 direct_callees.push_back(-1);
701 continue;
702 }
703 }
704
705 TRACE_LAZY("Found %d non-compiled callees in function=%p.\n",
706 num_non_compiled_callees, wasm_caller);
707 USE(num_non_compiled_callees);
708 }
709 CHECK_LE(0, callee_pos);
710
711 // TODO(wasm): compile all functions in non_compiled_callees in
712 // background, wait for direct_callees[callee_pos].
713 auto callee_func_index = direct_callees[callee_pos];
714 TRACE_LAZY(
715 "Starting lazy compilation (function=%p retaddr=+%d direct_callees[%d] "
716 "-> %d).\n",
717 wasm_caller, caller_ret_offset, callee_pos, callee_func_index);
718
719 Handle<WasmModuleObject> module_object(instance->module_object());
720 NativeModule* native_module = instance->compiled_module()->GetNativeModule();
721 const WasmCode* ret =
722 LazyCompileFunction(isolate, module_object, callee_func_index);
723 DCHECK_NOT_NULL(ret);
724
725 int patched = 0;
726 {
727 // Now patch the code in {wasm_caller} with all functions which are now
728 // compiled. This will pick up any other compiled functions, not only {ret}.
729 size_t pos = 0;
730 for (RelocIterator
731 it(wasm_caller->instructions(), wasm_caller->reloc_info(),
732 wasm_caller->constant_pool(),
733 RelocInfo::ModeMask(RelocInfo::WASM_CALL));
734 !it.done(); it.next(), ++pos) {
735 auto callee_index = direct_callees[pos];
736 if (callee_index < 0) continue; // callee already compiled.
737 const WasmCode* callee_compiled = native_module->code(callee_index);
738 if (callee_compiled->kind() != WasmCode::kFunction) continue;
739 DCHECK_EQ(WasmCode::kLazyStub,
740 isolate->wasm_engine()
741 ->code_manager()
742 ->GetCodeFromStartAddress(it.rinfo()->wasm_call_address())
743 ->kind());
744 it.rinfo()->set_wasm_call_address(callee_compiled->instruction_start());
745 ++patched;
746 }
747 DCHECK_EQ(direct_callees.size(), pos);
748 }
749
750 DCHECK_LT(0, patched);
751 TRACE_LAZY("Patched %d calls(s) in %p.\n", patched, wasm_caller);
752 USE(patched);
753
754 return ret;
755 }
756
757 } // namespace
758
CompileLazy(Isolate * isolate,Handle<WasmInstanceObject> target_instance)759 Address CompileLazy(Isolate* isolate,
760 Handle<WasmInstanceObject> target_instance) {
761 HistogramTimerScope lazy_time_scope(
762 isolate->counters()->wasm_lazy_compilation_time());
763
764 //==========================================================================
765 // Begin stack walk.
766 //==========================================================================
767 StackFrameIterator it(isolate);
768
769 //==========================================================================
770 // First frame: C entry stub.
771 //==========================================================================
772 DCHECK(!it.done());
773 DCHECK_EQ(StackFrame::EXIT, it.frame()->type());
774 it.Advance();
775
776 //==========================================================================
777 // Second frame: WasmCompileLazy builtin.
778 //==========================================================================
779 DCHECK(!it.done());
780 int target_func_index = -1;
781 bool indirectly_called = false;
782 const wasm::WasmCode* lazy_stub =
783 isolate->wasm_engine()->code_manager()->LookupCode(it.frame()->pc());
784 CHECK_EQ(wasm::WasmCode::kLazyStub, lazy_stub->kind());
785 if (!lazy_stub->IsAnonymous()) {
786 // If the lazy stub is not "anonymous", then its copy encodes the target
787 // function index. Used for import and indirect calls.
788 target_func_index = lazy_stub->index();
789 indirectly_called = true;
790 }
791 it.Advance();
792
793 //==========================================================================
794 // Third frame: The calling wasm code (direct or indirect), or js-to-wasm
795 // wrapper.
796 //==========================================================================
797 DCHECK(!it.done());
798 DCHECK(it.frame()->is_js_to_wasm() || it.frame()->is_wasm_compiled());
799 Handle<Code> js_to_wasm_caller_code;
800 Handle<WasmInstanceObject> caller_instance;
801 const WasmCode* wasm_caller_code = nullptr;
802 int32_t caller_ret_offset = -1;
803 if (it.frame()->is_js_to_wasm()) {
804 js_to_wasm_caller_code = handle(it.frame()->LookupCode(), isolate);
805 // This wasn't actually an indirect call, but a JS->wasm call.
806 indirectly_called = false;
807 } else {
808 caller_instance =
809 handle(WasmCompiledFrame::cast(it.frame())->wasm_instance(), isolate);
810 wasm_caller_code =
811 isolate->wasm_engine()->code_manager()->LookupCode(it.frame()->pc());
812 auto offset = it.frame()->pc() - wasm_caller_code->instruction_start();
813 caller_ret_offset = static_cast<int32_t>(offset);
814 DCHECK_EQ(offset, caller_ret_offset);
815 }
816
817 //==========================================================================
818 // Begin compilation.
819 //==========================================================================
820 Handle<WasmCompiledModule> compiled_module(
821 target_instance->compiled_module());
822
823 NativeModule* native_module = compiled_module->GetNativeModule();
824 DCHECK(!native_module->lazy_compile_frozen());
825
826 NativeModuleModificationScope native_module_modification_scope(native_module);
827
828 const wasm::WasmCode* result = nullptr;
829
830 if (!js_to_wasm_caller_code.is_null()) {
831 result = LazyCompileFromJsToWasm(isolate, target_instance,
832 js_to_wasm_caller_code, target_func_index);
833 DCHECK_NOT_NULL(result);
834 DCHECK_EQ(target_func_index, result->index());
835 } else {
836 DCHECK_NOT_NULL(wasm_caller_code);
837 if (target_func_index < 0) {
838 result = LazyCompileDirectCall(isolate, target_instance, wasm_caller_code,
839 caller_ret_offset);
840 DCHECK_NOT_NULL(result);
841 } else {
842 result =
843 LazyCompileIndirectCall(isolate, target_instance, target_func_index);
844 DCHECK_NOT_NULL(result);
845 }
846 }
847
848 //==========================================================================
849 // Update import and indirect function tables in the caller.
850 //==========================================================================
851 if (indirectly_called) {
852 DCHECK(!caller_instance.is_null());
853 if (!caller_instance->has_managed_indirect_patcher()) {
854 auto patcher = Managed<IndirectPatcher>::Allocate(isolate);
855 caller_instance->set_managed_indirect_patcher(*patcher);
856 }
857 IndirectPatcher* patcher = Managed<IndirectPatcher>::cast(
858 caller_instance->managed_indirect_patcher())
859 ->raw();
860 Address old_target = lazy_stub->instruction_start();
861 patcher->Patch(caller_instance, target_instance, target_func_index,
862 old_target, result->instruction_start());
863 }
864
865 return result->instruction_start();
866 }
867
868 namespace {
compile_lazy(const WasmModule * module)869 bool compile_lazy(const WasmModule* module) {
870 return FLAG_wasm_lazy_compilation ||
871 (FLAG_asm_wasm_lazy_compilation && module->is_asm_js());
872 }
873
FlushICache(const wasm::NativeModule * native_module)874 void FlushICache(const wasm::NativeModule* native_module) {
875 for (uint32_t i = native_module->num_imported_functions(),
876 e = native_module->function_count();
877 i < e; ++i) {
878 const wasm::WasmCode* code = native_module->code(i);
879 if (code == nullptr) continue;
880 Assembler::FlushICache(code->instructions().start(),
881 code->instructions().size());
882 }
883 }
884
FlushICache(Handle<FixedArray> functions)885 void FlushICache(Handle<FixedArray> functions) {
886 for (int i = 0, e = functions->length(); i < e; ++i) {
887 if (!functions->get(i)->IsCode()) continue;
888 Code* code = Code::cast(functions->get(i));
889 Assembler::FlushICache(code->raw_instruction_start(),
890 code->raw_instruction_size());
891 }
892 }
893
raw_buffer_ptr(MaybeHandle<JSArrayBuffer> buffer,int offset)894 byte* raw_buffer_ptr(MaybeHandle<JSArrayBuffer> buffer, int offset) {
895 return static_cast<byte*>(buffer.ToHandleChecked()->backing_store()) + offset;
896 }
897
RecordStats(const Code * code,Counters * counters)898 void RecordStats(const Code* code, Counters* counters) {
899 counters->wasm_generated_code_size()->Increment(code->body_size());
900 counters->wasm_reloc_size()->Increment(code->relocation_info()->length());
901 }
902
RecordStats(const wasm::WasmCode * code,Counters * counters)903 void RecordStats(const wasm::WasmCode* code, Counters* counters) {
904 counters->wasm_generated_code_size()->Increment(
905 static_cast<int>(code->instructions().size()));
906 counters->wasm_reloc_size()->Increment(
907 static_cast<int>(code->reloc_info().size()));
908 }
909
RecordStats(const wasm::NativeModule * native_module,Counters * counters)910 void RecordStats(const wasm::NativeModule* native_module, Counters* counters) {
911 for (uint32_t i = native_module->num_imported_functions(),
912 e = native_module->function_count();
913 i < e; ++i) {
914 const wasm::WasmCode* code = native_module->code(i);
915 if (code != nullptr) RecordStats(code, counters);
916 }
917 }
918
in_bounds(uint32_t offset,size_t size,size_t upper)919 bool in_bounds(uint32_t offset, size_t size, size_t upper) {
920 return offset + size <= upper && offset + size >= offset;
921 }
922
923 using WasmInstanceMap =
924 IdentityMap<Handle<WasmInstanceObject>, FreeStoreAllocationPolicy>;
925
MonotonicallyIncreasingTimeInMs()926 double MonotonicallyIncreasingTimeInMs() {
927 return V8::GetCurrentPlatform()->MonotonicallyIncreasingTime() *
928 base::Time::kMillisecondsPerSecond;
929 }
930
CreateDefaultModuleEnv(WasmModule * module)931 ModuleEnv CreateDefaultModuleEnv(WasmModule* module) {
932 // TODO(kschimpf): Add module-specific policy handling here (see v8:7143)?
933 UseTrapHandler use_trap_handler =
934 trap_handler::IsTrapHandlerEnabled() ? kUseTrapHandler : kNoTrapHandler;
935 return ModuleEnv(module, use_trap_handler, kRuntimeExceptionSupport);
936 }
937
NewCompiledModule(Isolate * isolate,WasmModule * module,ModuleEnv & env)938 Handle<WasmCompiledModule> NewCompiledModule(Isolate* isolate,
939 WasmModule* module,
940 ModuleEnv& env) {
941 Handle<WasmCompiledModule> compiled_module =
942 WasmCompiledModule::New(isolate, module, env);
943 return compiled_module;
944 }
945
GetMaxUsableMemorySize(Isolate * isolate)946 size_t GetMaxUsableMemorySize(Isolate* isolate) {
947 return isolate->heap()->memory_allocator()->code_range()->valid()
948 ? isolate->heap()->memory_allocator()->code_range()->size()
949 : isolate->heap()->code_space()->Capacity();
950 }
951
952 // The CompilationUnitBuilder builds compilation units and stores them in an
953 // internal buffer. The buffer is moved into the working queue of the
954 // CompilationState when {Commit} is called.
955 class CompilationUnitBuilder {
956 public:
CompilationUnitBuilder(NativeModule * native_module,Handle<Code> centry_stub)957 explicit CompilationUnitBuilder(NativeModule* native_module,
958 Handle<Code> centry_stub)
959 : native_module_(native_module),
960 compilation_state_(native_module->compilation_state()),
961 centry_stub_(centry_stub) {}
962
AddUnit(const WasmFunction * function,uint32_t buffer_offset,Vector<const uint8_t> bytes,WasmName name)963 void AddUnit(const WasmFunction* function, uint32_t buffer_offset,
964 Vector<const uint8_t> bytes, WasmName name) {
965 switch (compilation_state_->compile_mode()) {
966 case CompileMode::kTiering:
967 tiering_units_.emplace_back(
968 CreateUnit(function, buffer_offset, bytes, name,
969 WasmCompilationUnit::CompilationMode::kTurbofan));
970 baseline_units_.emplace_back(
971 CreateUnit(function, buffer_offset, bytes, name,
972 WasmCompilationUnit::CompilationMode::kLiftoff));
973 return;
974 case CompileMode::kRegular:
975 baseline_units_.emplace_back(
976 CreateUnit(function, buffer_offset, bytes, name,
977 WasmCompilationUnit::GetDefaultCompilationMode()));
978 return;
979 }
980 UNREACHABLE();
981 }
982
Commit()983 bool Commit() {
984 if (baseline_units_.empty() && tiering_units_.empty()) return false;
985 compilation_state_->AddCompilationUnits(baseline_units_, tiering_units_);
986 Clear();
987 return true;
988 }
989
Clear()990 void Clear() {
991 baseline_units_.clear();
992 tiering_units_.clear();
993 }
994
995 private:
CreateUnit(const WasmFunction * function,uint32_t buffer_offset,Vector<const uint8_t> bytes,WasmName name,WasmCompilationUnit::CompilationMode mode)996 std::unique_ptr<WasmCompilationUnit> CreateUnit(
997 const WasmFunction* function, uint32_t buffer_offset,
998 Vector<const uint8_t> bytes, WasmName name,
999 WasmCompilationUnit::CompilationMode mode) {
1000 return base::make_unique<WasmCompilationUnit>(
1001 compilation_state_->isolate(), compilation_state_->module_env(),
1002 native_module_,
1003 wasm::FunctionBody{function->sig, buffer_offset, bytes.begin(),
1004 bytes.end()},
1005 name, function->func_index, centry_stub_, mode,
1006 compilation_state_->isolate()->async_counters().get());
1007 }
1008
1009 NativeModule* native_module_;
1010 CompilationState* compilation_state_;
1011 Handle<Code> centry_stub_;
1012 std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_units_;
1013 std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_units_;
1014 };
1015
1016 // Run by each compilation task and by the main thread (i.e. in both
1017 // foreground and background threads). The no_finisher_callback is called
1018 // within the result_mutex_ lock when no finishing task is running, i.e. when
1019 // the finisher_is_running_ flag is not set.
FetchAndExecuteCompilationUnit(CompilationState * compilation_state)1020 bool FetchAndExecuteCompilationUnit(CompilationState* compilation_state) {
1021 DisallowHeapAllocation no_allocation;
1022 DisallowHandleAllocation no_handles;
1023 DisallowHandleDereference no_deref;
1024 DisallowCodeDependencyChange no_dependency_change;
1025
1026 std::unique_ptr<WasmCompilationUnit> unit =
1027 compilation_state->GetNextCompilationUnit();
1028 if (unit == nullptr) return false;
1029
1030 // TODO(kimanh): We need to find out in which mode the unit
1031 // should be compiled in before compiling it, as it might fallback
1032 // to Turbofan if it cannot be compiled using Liftoff. This can be removed
1033 // later as soon as Liftoff can compile any function. Then, we can directly
1034 // access {unit->mode()} within {ScheduleUnitForFinishing()}.
1035 WasmCompilationUnit::CompilationMode mode = unit->mode();
1036 unit->ExecuteCompilation();
1037 compilation_state->ScheduleUnitForFinishing(std::move(unit), mode);
1038
1039 return true;
1040 }
1041
GetNumFunctionsToCompile(const WasmModule * wasm_module)1042 size_t GetNumFunctionsToCompile(const WasmModule* wasm_module) {
1043 // TODO(kimanh): Remove, FLAG_skip_compiling_wasm_funcs: previously used for
1044 // debugging, and now not necessarily working anymore.
1045 uint32_t start =
1046 wasm_module->num_imported_functions + FLAG_skip_compiling_wasm_funcs;
1047 uint32_t num_funcs = static_cast<uint32_t>(wasm_module->functions.size());
1048 uint32_t funcs_to_compile = start > num_funcs ? 0 : num_funcs - start;
1049 return funcs_to_compile;
1050 }
1051
InitializeCompilationUnits(const std::vector<WasmFunction> & functions,const ModuleWireBytes & wire_bytes,const WasmModule * wasm_module,Handle<Code> centry_stub,NativeModule * native_module)1052 void InitializeCompilationUnits(const std::vector<WasmFunction>& functions,
1053 const ModuleWireBytes& wire_bytes,
1054 const WasmModule* wasm_module,
1055 Handle<Code> centry_stub,
1056 NativeModule* native_module) {
1057 uint32_t start =
1058 wasm_module->num_imported_functions + FLAG_skip_compiling_wasm_funcs;
1059 uint32_t num_funcs = static_cast<uint32_t>(functions.size());
1060
1061 CompilationUnitBuilder builder(native_module, centry_stub);
1062 for (uint32_t i = start; i < num_funcs; ++i) {
1063 const WasmFunction* func = &functions[i];
1064 uint32_t buffer_offset = func->code.offset();
1065 Vector<const uint8_t> bytes(wire_bytes.start() + func->code.offset(),
1066 func->code.end_offset() - func->code.offset());
1067
1068 WasmName name = wire_bytes.GetName(func, wasm_module);
1069 DCHECK_NOT_NULL(native_module);
1070 builder.AddUnit(func, buffer_offset, bytes, name);
1071 }
1072 builder.Commit();
1073 }
1074
FinishCompilationUnits(CompilationState * compilation_state,ErrorThrower * thrower)1075 void FinishCompilationUnits(CompilationState* compilation_state,
1076 ErrorThrower* thrower) {
1077 while (true) {
1078 if (compilation_state->failed()) break;
1079 std::unique_ptr<WasmCompilationUnit> unit =
1080 compilation_state->GetNextExecutedUnit();
1081 if (unit == nullptr) break;
1082 wasm::WasmCode* result = unit->FinishCompilation(thrower);
1083
1084 if (thrower->error()) {
1085 compilation_state->Abort();
1086 break;
1087 }
1088
1089 // Update the compilation state.
1090 compilation_state->OnFinishedUnit();
1091 DCHECK_IMPLIES(result == nullptr, thrower->error());
1092 if (result == nullptr) break;
1093 }
1094 if (!compilation_state->failed()) {
1095 compilation_state->RestartBackgroundTasks();
1096 }
1097 }
1098
UpdateAllCompiledModulesWithTopTierCode(Handle<WasmModuleObject> module_object)1099 void UpdateAllCompiledModulesWithTopTierCode(
1100 Handle<WasmModuleObject> module_object) {
1101 WasmModule* module = module_object->shared()->module();
1102 DCHECK_GT(module->functions.size() - module->num_imported_functions, 0);
1103 USE(module);
1104
1105 CodeSpaceMemoryModificationScope modification_scope(
1106 module_object->GetIsolate()->heap());
1107
1108 NativeModule* native_module =
1109 module_object->compiled_module()->GetNativeModule();
1110
1111 // Link.
1112 CodeSpecialization code_specialization;
1113 code_specialization.RelocateDirectCalls(native_module);
1114 code_specialization.ApplyToWholeModule(native_module, module_object);
1115 }
1116
CompileInParallel(Isolate * isolate,NativeModule * native_module,const ModuleWireBytes & wire_bytes,ModuleEnv * module_env,Handle<WasmModuleObject> module_object,Handle<Code> centry_stub,ErrorThrower * thrower)1117 void CompileInParallel(Isolate* isolate, NativeModule* native_module,
1118 const ModuleWireBytes& wire_bytes, ModuleEnv* module_env,
1119 Handle<WasmModuleObject> module_object,
1120 Handle<Code> centry_stub, ErrorThrower* thrower) {
1121 const WasmModule* module = module_env->module;
1122 // Data structures for the parallel compilation.
1123
1124 //-----------------------------------------------------------------------
1125 // For parallel compilation:
1126 // 1) The main thread allocates a compilation unit for each wasm function
1127 // and stores them in the vector {compilation_units} within the
1128 // {compilation_state}. By adding units to the {compilation_state}, new
1129 // {BackgroundCompileTasks} instances are spawned which run on
1130 // the background threads.
1131 // 2.a) The background threads and the main thread pick one compilation
1132 // unit at a time and execute the parallel phase of the compilation
1133 // unit. After finishing the execution of the parallel phase, the
1134 // result is enqueued in {baseline_finish_units_}.
1135 // 2.b) If {baseline_finish_units_} contains a compilation unit, the main
1136 // thread dequeues it and finishes the compilation.
1137 // 3) After the parallel phase of all compilation units has started, the
1138 // main thread continues to finish all compilation units as long as
1139 // baseline-compilation units are left to be processed.
1140 // 4) If tier-up is enabled, the main thread restarts background tasks
1141 // that take care of compiling and finishing the top-tier compilation
1142 // units.
1143
1144 // Turn on the {CanonicalHandleScope} so that the background threads can
1145 // use the node cache.
1146 CanonicalHandleScope canonical(isolate);
1147
1148 CompilationState* compilation_state = native_module->compilation_state();
1149 // Make sure that no foreground task is spawned for finishing
1150 // the compilation units. This foreground thread will be
1151 // responsible for finishing compilation.
1152 compilation_state->SetFinisherIsRunning(true);
1153 size_t functions_count = GetNumFunctionsToCompile(module);
1154 compilation_state->SetNumberOfFunctionsToCompile(functions_count);
1155 compilation_state->SetWireBytes(wire_bytes);
1156
1157 DeferredHandles* deferred_handles = nullptr;
1158 Handle<Code> centry_deferred = centry_stub;
1159 Handle<WasmModuleObject> module_object_deferred;
1160 if (compilation_state->compile_mode() == CompileMode::kTiering) {
1161 // Open a deferred handle scope for the centry_stub, in order to allow
1162 // for background tiering compilation.
1163 DeferredHandleScope deferred(isolate);
1164 centry_deferred = Handle<Code>(*centry_stub, isolate);
1165 module_object_deferred = handle(*module_object, isolate);
1166 deferred_handles = deferred.Detach();
1167 }
1168 compilation_state->AddCallback(
1169 [module_object_deferred, deferred_handles](
1170 // Callback is called from a foreground thread.
1171 CompilationEvent event, ErrorThrower* thrower) mutable {
1172 switch (event) {
1173 case CompilationEvent::kFinishedBaselineCompilation:
1174 // Nothing to do, since we are finishing baseline compilation
1175 // in this foreground thread.
1176 return;
1177 case CompilationEvent::kFinishedTopTierCompilation:
1178 UpdateAllCompiledModulesWithTopTierCode(module_object_deferred);
1179 // TODO(wasm): Currently compilation has to finish before the
1180 // {deferred_handles} can be removed. We need to make sure that
1181 // we can clean it up at a time when the native module
1182 // should die (but currently cannot, since it's kept alive
1183 // through the {deferred_handles} themselves).
1184 delete deferred_handles;
1185 deferred_handles = nullptr;
1186 return;
1187 case CompilationEvent::kFailedCompilation:
1188 // If baseline compilation failed, we will reflect this without
1189 // a callback, in this thread through {thrower}.
1190 // Tier-up compilation should not fail if baseline compilation
1191 // did not fail.
1192 DCHECK(!module_object_deferred->compiled_module()
1193 ->GetNativeModule()
1194 ->compilation_state()
1195 ->baseline_compilation_finished());
1196 delete deferred_handles;
1197 deferred_handles = nullptr;
1198 return;
1199 case CompilationEvent::kDestroyed:
1200 if (deferred_handles) delete deferred_handles;
1201 return;
1202 }
1203 UNREACHABLE();
1204 });
1205
1206 // 1) The main thread allocates a compilation unit for each wasm function
1207 // and stores them in the vector {compilation_units} within the
1208 // {compilation_state}. By adding units to the {compilation_state}, new
1209 // {BackgroundCompileTask} instances are spawned which run on
1210 // background threads.
1211 InitializeCompilationUnits(module->functions, compilation_state->wire_bytes(),
1212 module, centry_deferred, native_module);
1213
1214 // 2.a) The background threads and the main thread pick one compilation
1215 // unit at a time and execute the parallel phase of the compilation
1216 // unit. After finishing the execution of the parallel phase, the
1217 // result is enqueued in {baseline_finish_units_}.
1218 // The foreground task bypasses waiting on memory threshold, because
1219 // its results will immediately be converted to code (below).
1220 while (FetchAndExecuteCompilationUnit(compilation_state) &&
1221 !compilation_state->baseline_compilation_finished()) {
1222 // 2.b) If {baseline_finish_units_} contains a compilation unit, the main
1223 // thread dequeues it and finishes the compilation unit. Compilation
1224 // units are finished concurrently to the background threads to save
1225 // memory.
1226 FinishCompilationUnits(compilation_state, thrower);
1227
1228 if (compilation_state->failed()) break;
1229 }
1230
1231 while (!compilation_state->failed()) {
1232 // 3) After the parallel phase of all compilation units has started, the
1233 // main thread continues to finish compilation units as long as
1234 // baseline compilation units are left to be processed. If compilation
1235 // already failed, all background tasks have already been canceled
1236 // in {FinishCompilationUnits}, and there are no units to finish.
1237 FinishCompilationUnits(compilation_state, thrower);
1238
1239 if (compilation_state->baseline_compilation_finished()) break;
1240 }
1241
1242 // 4) If tiering-compilation is enabled, we need to set the finisher
1243 // to false, such that the background threads will spawn a foreground
1244 // thread to finish the top-tier compilation units.
1245 if (!compilation_state->failed() &&
1246 compilation_state->compile_mode() == CompileMode::kTiering) {
1247 compilation_state->SetFinisherIsRunning(false);
1248 compilation_state->RestartBackgroundTasks();
1249 }
1250 }
1251
CompileSequentially(Isolate * isolate,NativeModule * native_module,const ModuleWireBytes & wire_bytes,ModuleEnv * module_env,ErrorThrower * thrower)1252 void CompileSequentially(Isolate* isolate, NativeModule* native_module,
1253 const ModuleWireBytes& wire_bytes,
1254 ModuleEnv* module_env, ErrorThrower* thrower) {
1255 DCHECK(!thrower->error());
1256
1257 const WasmModule* module = module_env->module;
1258 for (uint32_t i = FLAG_skip_compiling_wasm_funcs;
1259 i < module->functions.size(); ++i) {
1260 const WasmFunction& func = module->functions[i];
1261 if (func.imported) continue; // Imports are compiled at instantiation time.
1262
1263 // Compile the function.
1264 wasm::WasmCode* code = WasmCompilationUnit::CompileWasmFunction(
1265 native_module, thrower, isolate, wire_bytes, module_env, &func);
1266 if (code == nullptr) {
1267 TruncatedUserString<> name(wire_bytes.GetName(&func, module));
1268 thrower->CompileError("Compilation of #%d:%.*s failed.", i, name.length(),
1269 name.start());
1270 break;
1271 }
1272 }
1273 }
1274
ValidateSequentially(Isolate * isolate,const ModuleWireBytes & wire_bytes,ModuleEnv * module_env,ErrorThrower * thrower)1275 void ValidateSequentially(Isolate* isolate, const ModuleWireBytes& wire_bytes,
1276 ModuleEnv* module_env, ErrorThrower* thrower) {
1277 DCHECK(!thrower->error());
1278
1279 const WasmModule* module = module_env->module;
1280 for (uint32_t i = 0; i < module->functions.size(); ++i) {
1281 const WasmFunction& func = module->functions[i];
1282 if (func.imported) continue;
1283
1284 const byte* base = wire_bytes.start();
1285 FunctionBody body{func.sig, func.code.offset(), base + func.code.offset(),
1286 base + func.code.end_offset()};
1287 DecodeResult result = VerifyWasmCodeWithStats(
1288 isolate->allocator(), module, body, module->is_wasm(),
1289 isolate->async_counters().get());
1290 if (result.failed()) {
1291 TruncatedUserString<> name(wire_bytes.GetName(&func, module));
1292 thrower->CompileError("Compiling function #%d:%.*s failed: %s @+%u", i,
1293 name.length(), name.start(),
1294 result.error_msg().c_str(), result.error_offset());
1295 break;
1296 }
1297 }
1298 }
1299
CompileToModuleObjectInternal(Isolate * isolate,ErrorThrower * thrower,std::unique_ptr<WasmModule> module,const ModuleWireBytes & wire_bytes,Handle<Script> asm_js_script,Vector<const byte> asm_js_offset_table_bytes)1300 MaybeHandle<WasmModuleObject> CompileToModuleObjectInternal(
1301 Isolate* isolate, ErrorThrower* thrower, std::unique_ptr<WasmModule> module,
1302 const ModuleWireBytes& wire_bytes, Handle<Script> asm_js_script,
1303 Vector<const byte> asm_js_offset_table_bytes) {
1304 WasmModule* wasm_module = module.get();
1305 Handle<Code> centry_stub = CodeFactory::CEntry(isolate);
1306 TimedHistogramScope wasm_compile_module_time_scope(
1307 wasm_module->is_wasm()
1308 ? isolate->async_counters()->wasm_compile_wasm_module_time()
1309 : isolate->async_counters()->wasm_compile_asm_module_time());
1310 // TODO(6792): No longer needed once WebAssembly code is off heap. Use
1311 // base::Optional to be able to close the scope before notifying the debugger.
1312 base::Optional<CodeSpaceMemoryModificationScope> modification_scope(
1313 base::in_place_t(), isolate->heap());
1314
1315 // Check whether lazy compilation is enabled for this module.
1316 bool lazy_compile = compile_lazy(wasm_module);
1317
1318 Factory* factory = isolate->factory();
1319 // Create heap objects for script, module bytes and asm.js offset table to
1320 // be stored in the shared module data.
1321 Handle<Script> script;
1322 Handle<ByteArray> asm_js_offset_table;
1323 if (asm_js_script.is_null()) {
1324 script = CreateWasmScript(isolate, wire_bytes);
1325 } else {
1326 script = asm_js_script;
1327 asm_js_offset_table =
1328 isolate->factory()->NewByteArray(asm_js_offset_table_bytes.length());
1329 asm_js_offset_table->copy_in(0, asm_js_offset_table_bytes.start(),
1330 asm_js_offset_table_bytes.length());
1331 }
1332 // TODO(wasm): only save the sections necessary to deserialize a
1333 // {WasmModule}. E.g. function bodies could be omitted.
1334 Handle<String> module_bytes =
1335 factory
1336 ->NewStringFromOneByte({wire_bytes.start(), wire_bytes.length()},
1337 TENURED)
1338 .ToHandleChecked();
1339 DCHECK(module_bytes->IsSeqOneByteString());
1340
1341 // The {managed_module} will take ownership of the {WasmModule} object,
1342 // and it will be destroyed when the GC reclaims the wrapper object.
1343 Handle<Managed<WasmModule>> managed_module =
1344 Managed<WasmModule>::FromUniquePtr(isolate, std::move(module));
1345
1346 // Create the shared module data.
1347 // TODO(clemensh): For the same module (same bytes / same hash), we should
1348 // only have one WasmSharedModuleData. Otherwise, we might only set
1349 // breakpoints on a (potentially empty) subset of the instances.
1350
1351 Handle<WasmSharedModuleData> shared = WasmSharedModuleData::New(
1352 isolate, managed_module, Handle<SeqOneByteString>::cast(module_bytes),
1353 script, asm_js_offset_table);
1354
1355 int export_wrappers_size =
1356 static_cast<int>(wasm_module->num_exported_functions);
1357 Handle<FixedArray> export_wrappers =
1358 factory->NewFixedArray(static_cast<int>(export_wrappers_size), TENURED);
1359 Handle<Code> init_builtin = BUILTIN_CODE(isolate, Illegal);
1360 for (int i = 0, e = export_wrappers->length(); i < e; ++i) {
1361 export_wrappers->set(i, *init_builtin);
1362 }
1363 ModuleEnv env = CreateDefaultModuleEnv(wasm_module);
1364
1365 // Create the compiled module object and populate with compiled functions
1366 // and information needed at instantiation time. This object needs to be
1367 // serializable. Instantiation may occur off a deserialized version of this
1368 // object.
1369 Handle<WasmCompiledModule> compiled_module =
1370 NewCompiledModule(isolate, shared->module(), env);
1371 NativeModule* native_module = compiled_module->GetNativeModule();
1372 compiled_module->GetNativeModule()->SetSharedModuleData(shared);
1373 Handle<WasmModuleObject> module_object =
1374 WasmModuleObject::New(isolate, compiled_module, export_wrappers, shared);
1375 if (lazy_compile) {
1376 if (wasm_module->is_wasm()) {
1377 // Validate wasm modules for lazy compilation. Don't validate asm.js
1378 // modules, they are valid by construction (otherwise a CHECK will fail
1379 // during lazy compilation).
1380 // TODO(clemensh): According to the spec, we can actually skip validation
1381 // at module creation time, and return a function that always traps at
1382 // (lazy) compilation time.
1383 ValidateSequentially(isolate, wire_bytes, &env, thrower);
1384 if (thrower->error()) return {};
1385 }
1386
1387 native_module->SetLazyBuiltin(BUILTIN_CODE(isolate, WasmCompileLazy));
1388 } else {
1389 size_t funcs_to_compile =
1390 wasm_module->functions.size() - wasm_module->num_imported_functions;
1391 bool compile_parallel =
1392 !FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks > 0 &&
1393 funcs_to_compile > 1 &&
1394 V8::GetCurrentPlatform()->NumberOfWorkerThreads() > 0;
1395
1396 if (compile_parallel) {
1397 CompileInParallel(isolate, native_module, wire_bytes, &env, module_object,
1398 centry_stub, thrower);
1399 } else {
1400 CompileSequentially(isolate, native_module, wire_bytes, &env, thrower);
1401 }
1402 if (thrower->error()) return {};
1403
1404 RecordStats(native_module, isolate->async_counters().get());
1405 }
1406
1407 // Compile JS->wasm wrappers for exported functions.
1408 CompileJsToWasmWrappers(isolate, module_object,
1409 isolate->async_counters().get());
1410
1411 // If we created a wasm script, finish it now and make it public to the
1412 // debugger.
1413 if (asm_js_script.is_null()) {
1414 // Close the CodeSpaceMemoryModificationScope before calling into the
1415 // debugger.
1416 modification_scope.reset();
1417 isolate->debug()->OnAfterCompile(script);
1418 }
1419
1420 return module_object;
1421 }
1422
1423 // The runnable task that finishes compilation in foreground (e.g. updating
1424 // the NativeModule, the code table, etc.).
1425 class FinishCompileTask : public CancelableTask {
1426 public:
FinishCompileTask(CompilationState * compilation_state,CancelableTaskManager * task_manager)1427 explicit FinishCompileTask(CompilationState* compilation_state,
1428 CancelableTaskManager* task_manager)
1429 : CancelableTask(task_manager), compilation_state_(compilation_state) {}
1430
RunInternal()1431 void RunInternal() override {
1432 Isolate* isolate = compilation_state_->isolate();
1433 HandleScope scope(isolate);
1434 SaveContext saved_context(isolate);
1435 isolate->set_context(nullptr);
1436
1437 TRACE_COMPILE("(4a) Finishing compilation units...\n");
1438 if (compilation_state_->failed()) {
1439 compilation_state_->SetFinisherIsRunning(false);
1440 return;
1441 }
1442
1443 // We execute for 1 ms and then reschedule the task, same as the GC.
1444 double deadline = MonotonicallyIncreasingTimeInMs() + 1.0;
1445 while (true) {
1446 compilation_state_->RestartBackgroundTasks();
1447
1448 std::unique_ptr<WasmCompilationUnit> unit =
1449 compilation_state_->GetNextExecutedUnit();
1450
1451 if (unit == nullptr) {
1452 // It might happen that a background task just scheduled a unit to be
1453 // finished, but did not start a finisher task since the flag was still
1454 // set. Check for this case, and continue if there is more work.
1455 compilation_state_->SetFinisherIsRunning(false);
1456 if (compilation_state_->HasCompilationUnitToFinish() &&
1457 compilation_state_->SetFinisherIsRunning(true)) {
1458 continue;
1459 }
1460 break;
1461 }
1462
1463 ErrorThrower thrower(compilation_state_->isolate(), "AsyncCompile");
1464 wasm::WasmCode* result = unit->FinishCompilation(&thrower);
1465
1466 NativeModule* native_module = unit->native_module();
1467 if (thrower.error()) {
1468 DCHECK_NULL(result);
1469 compilation_state_->OnError(&thrower);
1470 compilation_state_->SetFinisherIsRunning(false);
1471 thrower.Reset();
1472 break;
1473 }
1474
1475 if (compilation_state_->baseline_compilation_finished()) {
1476 // If Liftoff compilation finishes it will directly start executing.
1477 // As soon as we have Turbofan-compiled code available, it will
1478 // directly be used by Liftoff-compiled code. Therefore we need
1479 // to patch the compiled Turbofan function directly after finishing it.
1480 DCHECK_EQ(CompileMode::kTiering, compilation_state_->compile_mode());
1481 DCHECK(!result->is_liftoff());
1482 CodeSpecialization code_specialization;
1483 code_specialization.RelocateDirectCalls(native_module);
1484 code_specialization.ApplyToWasmCode(result);
1485
1486 if (wasm::WasmCode::ShouldBeLogged(isolate)) result->LogCode(isolate);
1487
1488 // Update the counters to include the top-tier code.
1489 RecordStats(result,
1490 compilation_state_->isolate()->async_counters().get());
1491 }
1492
1493 // Update the compilation state, and possibly notify
1494 // threads waiting for events.
1495 compilation_state_->OnFinishedUnit();
1496
1497 if (deadline < MonotonicallyIncreasingTimeInMs()) {
1498 // We reached the deadline. We reschedule this task and return
1499 // immediately. Since we rescheduled this task already, we do not set
1500 // the FinisherIsRunning flag to false.
1501 compilation_state_->ScheduleFinisherTask();
1502 return;
1503 }
1504 }
1505 }
1506
1507 private:
1508 CompilationState* compilation_state_;
1509 };
1510
1511 // The runnable task that performs compilations in the background.
1512 class BackgroundCompileTask : public CancelableTask {
1513 public:
BackgroundCompileTask(CompilationState * compilation_state,CancelableTaskManager * task_manager)1514 explicit BackgroundCompileTask(CompilationState* compilation_state,
1515 CancelableTaskManager* task_manager)
1516 : CancelableTask(task_manager), compilation_state_(compilation_state) {}
1517
RunInternal()1518 void RunInternal() override {
1519 TRACE_COMPILE("(3b) Compiling...\n");
1520 // The number of currently running background tasks is reduced either in
1521 // {StopBackgroundCompilationTaskForThrottling} or in
1522 // {OnBackgroundTaskStopped}.
1523 while (!compilation_state_->StopBackgroundCompilationTaskForThrottling()) {
1524 if (compilation_state_->failed() ||
1525 !FetchAndExecuteCompilationUnit(compilation_state_)) {
1526 compilation_state_->OnBackgroundTaskStopped();
1527 break;
1528 }
1529 }
1530 }
1531
1532 private:
1533 CompilationState* compilation_state_;
1534 };
1535 } // namespace
1536
CompileToModuleObject(Isolate * isolate,ErrorThrower * thrower,std::unique_ptr<WasmModule> module,const ModuleWireBytes & wire_bytes,Handle<Script> asm_js_script,Vector<const byte> asm_js_offset_table_bytes)1537 MaybeHandle<WasmModuleObject> CompileToModuleObject(
1538 Isolate* isolate, ErrorThrower* thrower, std::unique_ptr<WasmModule> module,
1539 const ModuleWireBytes& wire_bytes, Handle<Script> asm_js_script,
1540 Vector<const byte> asm_js_offset_table_bytes) {
1541 return CompileToModuleObjectInternal(isolate, thrower, std::move(module),
1542 wire_bytes, asm_js_script,
1543 asm_js_offset_table_bytes);
1544 }
1545
InstanceBuilder(Isolate * isolate,ErrorThrower * thrower,Handle<WasmModuleObject> module_object,MaybeHandle<JSReceiver> ffi,MaybeHandle<JSArrayBuffer> memory)1546 InstanceBuilder::InstanceBuilder(Isolate* isolate, ErrorThrower* thrower,
1547 Handle<WasmModuleObject> module_object,
1548 MaybeHandle<JSReceiver> ffi,
1549 MaybeHandle<JSArrayBuffer> memory)
1550 : isolate_(isolate),
1551 module_(module_object->shared()->module()),
1552 async_counters_(isolate->async_counters()),
1553 thrower_(thrower),
1554 module_object_(module_object),
1555 ffi_(ffi),
1556 memory_(memory) {
1557 sanitized_imports_.reserve(module_->import_table.size());
1558 }
1559
1560 // Build an instance, in all of its glory.
Build()1561 MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() {
1562 // Check that an imports argument was provided, if the module requires it.
1563 // No point in continuing otherwise.
1564 if (!module_->import_table.empty() && ffi_.is_null()) {
1565 thrower_->TypeError(
1566 "Imports argument must be present and must be an object");
1567 return {};
1568 }
1569
1570 SanitizeImports();
1571 if (thrower_->error()) return {};
1572
1573 // TODO(6792): No longer needed once WebAssembly code is off heap.
1574 CodeSpaceMemoryModificationScope modification_scope(isolate_->heap());
1575 // From here on, we expect the build pipeline to run without exiting to JS.
1576 DisallowJavascriptExecution no_js(isolate_);
1577 // Record build time into correct bucket, then build instance.
1578 TimedHistogramScope wasm_instantiate_module_time_scope(
1579 module_->is_wasm() ? counters()->wasm_instantiate_wasm_module_time()
1580 : counters()->wasm_instantiate_asm_module_time());
1581 Factory* factory = isolate_->factory();
1582
1583 //--------------------------------------------------------------------------
1584 // Reuse the compiled module (if no owner), otherwise clone.
1585 //--------------------------------------------------------------------------
1586 wasm::NativeModule* native_module = nullptr;
1587 // Root the old instance, if any, in case later allocation causes GC,
1588 // to prevent the finalizer running for the old instance.
1589 MaybeHandle<WasmInstanceObject> old_instance;
1590
1591 TRACE("Starting new module instantiation\n");
1592 {
1593 Handle<WasmCompiledModule> original =
1594 handle(module_object_->compiled_module());
1595 if (original->has_instance()) {
1596 old_instance = handle(original->owning_instance());
1597 // Clone, but don't insert yet the clone in the instances chain.
1598 // We do that last. Since we are holding on to the old instance,
1599 // the owner + original state used for cloning and patching
1600 // won't be mutated by possible finalizer runs.
1601 TRACE("Cloning from %zu\n", original->GetNativeModule()->instance_id);
1602 compiled_module_ = WasmCompiledModule::Clone(isolate_, original);
1603 native_module = compiled_module_->GetNativeModule();
1604 RecordStats(native_module, counters());
1605 } else {
1606 // No instance owned the original compiled module.
1607 compiled_module_ = original;
1608 native_module = compiled_module_->GetNativeModule();
1609 TRACE("Reusing existing instance %zu\n",
1610 compiled_module_->GetNativeModule()->instance_id);
1611 }
1612 }
1613 DCHECK_NOT_NULL(native_module);
1614 wasm::NativeModuleModificationScope native_modification_scope(native_module);
1615
1616 //--------------------------------------------------------------------------
1617 // Create the WebAssembly.Instance object.
1618 //--------------------------------------------------------------------------
1619 Handle<WasmInstanceObject> instance =
1620 WasmInstanceObject::New(isolate_, module_object_, compiled_module_);
1621 Handle<WeakCell> weak_instance = factory->NewWeakCell(instance);
1622
1623 //--------------------------------------------------------------------------
1624 // Set up the globals for the new instance.
1625 //--------------------------------------------------------------------------
1626 MaybeHandle<JSArrayBuffer> old_globals;
1627 uint32_t globals_size = module_->globals_size;
1628 if (globals_size > 0) {
1629 void* backing_store =
1630 isolate_->array_buffer_allocator()->Allocate(globals_size);
1631 if (backing_store == nullptr) {
1632 thrower_->RangeError("Out of memory: wasm globals");
1633 return {};
1634 }
1635 globals_ =
1636 isolate_->factory()->NewJSArrayBuffer(SharedFlag::kNotShared, TENURED);
1637 constexpr bool is_external = false;
1638 constexpr bool is_wasm_memory = false;
1639 JSArrayBuffer::Setup(globals_, isolate_, is_external, backing_store,
1640 globals_size, SharedFlag::kNotShared, is_wasm_memory);
1641 if (globals_.is_null()) {
1642 thrower_->RangeError("Out of memory: wasm globals");
1643 return {};
1644 }
1645 instance->set_globals_start(
1646 reinterpret_cast<byte*>(globals_->backing_store()));
1647 instance->set_globals_buffer(*globals_);
1648 }
1649
1650 //--------------------------------------------------------------------------
1651 // Set up the array of references to imported globals' array buffers.
1652 //--------------------------------------------------------------------------
1653 if (module_->num_imported_mutable_globals > 0) {
1654 DCHECK(FLAG_experimental_wasm_mut_global);
1655 // TODO(binji): This allocates one slot for each mutable global, which is
1656 // more than required if multiple globals are imported from the same
1657 // module.
1658 Handle<FixedArray> buffers_array = isolate_->factory()->NewFixedArray(
1659 module_->num_imported_mutable_globals, TENURED);
1660 instance->set_imported_mutable_globals_buffers(*buffers_array);
1661 }
1662
1663 //--------------------------------------------------------------------------
1664 // Reserve the metadata for indirect function tables.
1665 //--------------------------------------------------------------------------
1666 int function_table_count = static_cast<int>(module_->function_tables.size());
1667 table_instances_.reserve(module_->function_tables.size());
1668 for (int index = 0; index < function_table_count; ++index) {
1669 table_instances_.emplace_back();
1670 }
1671
1672 //--------------------------------------------------------------------------
1673 // Process the imports for the module.
1674 //--------------------------------------------------------------------------
1675 int num_imported_functions = ProcessImports(instance);
1676 if (num_imported_functions < 0) return {};
1677
1678 //--------------------------------------------------------------------------
1679 // Process the initialization for the module's globals.
1680 //--------------------------------------------------------------------------
1681 InitGlobals();
1682
1683 //--------------------------------------------------------------------------
1684 // Initialize the indirect tables.
1685 //--------------------------------------------------------------------------
1686 if (function_table_count > 0) {
1687 InitializeTables(instance);
1688 }
1689
1690 //--------------------------------------------------------------------------
1691 // Allocate the memory array buffer.
1692 //--------------------------------------------------------------------------
1693 uint32_t initial_pages = module_->initial_pages;
1694 (module_->is_wasm() ? counters()->wasm_wasm_min_mem_pages_count()
1695 : counters()->wasm_asm_min_mem_pages_count())
1696 ->AddSample(initial_pages);
1697
1698 if (!memory_.is_null()) {
1699 // Set externally passed ArrayBuffer non neuterable.
1700 Handle<JSArrayBuffer> memory = memory_.ToHandleChecked();
1701 memory->set_is_neuterable(false);
1702
1703 DCHECK_IMPLIES(use_trap_handler(),
1704 module_->is_asm_js() || memory->is_wasm_memory() ||
1705 memory->backing_store() == nullptr ||
1706 // TODO(836800) Remove once is_wasm_memory transfers over
1707 // post-message.
1708 (FLAG_experimental_wasm_threads && memory->is_shared()));
1709 } else if (initial_pages > 0 || use_trap_handler()) {
1710 // We need to unconditionally create a guard region if using trap handlers,
1711 // even when the size is zero to prevent null-dereference issues
1712 // (e.g. https://crbug.com/769637).
1713 // Allocate memory if the initial size is more than 0 pages.
1714 memory_ = AllocateMemory(initial_pages);
1715 if (memory_.is_null()) return {}; // failed to allocate memory
1716 }
1717
1718 //--------------------------------------------------------------------------
1719 // Create the WebAssembly.Memory object.
1720 //--------------------------------------------------------------------------
1721 if (module_->has_memory) {
1722 if (!instance->has_memory_object()) {
1723 // No memory object exists. Create one.
1724 Handle<WasmMemoryObject> memory_object = WasmMemoryObject::New(
1725 isolate_, memory_,
1726 module_->maximum_pages != 0 ? module_->maximum_pages : -1);
1727 instance->set_memory_object(*memory_object);
1728 }
1729
1730 // Add the instance object to the list of instances for this memory.
1731 Handle<WasmMemoryObject> memory_object(instance->memory_object(), isolate_);
1732 WasmMemoryObject::AddInstance(isolate_, memory_object, instance);
1733
1734 if (!memory_.is_null()) {
1735 // Double-check the {memory} array buffer matches the instance.
1736 Handle<JSArrayBuffer> memory = memory_.ToHandleChecked();
1737 uint32_t mem_size = 0;
1738 CHECK(memory->byte_length()->ToUint32(&mem_size));
1739 CHECK_EQ(instance->memory_size(), mem_size);
1740 CHECK_EQ(instance->memory_start(), memory->backing_store());
1741 }
1742 }
1743
1744 //--------------------------------------------------------------------------
1745 // Check that indirect function table segments are within bounds.
1746 //--------------------------------------------------------------------------
1747 for (WasmTableInit& table_init : module_->table_inits) {
1748 DCHECK(table_init.table_index < table_instances_.size());
1749 uint32_t base = EvalUint32InitExpr(table_init.offset);
1750 size_t table_size = table_instances_[table_init.table_index].table_size;
1751 if (!in_bounds(base, table_init.entries.size(), table_size)) {
1752 thrower_->LinkError("table initializer is out of bounds");
1753 return {};
1754 }
1755 }
1756
1757 //--------------------------------------------------------------------------
1758 // Check that memory segments are within bounds.
1759 //--------------------------------------------------------------------------
1760 for (WasmDataSegment& seg : module_->data_segments) {
1761 uint32_t base = EvalUint32InitExpr(seg.dest_addr);
1762 if (!in_bounds(base, seg.source.length(), instance->memory_size())) {
1763 thrower_->LinkError("data segment is out of bounds");
1764 return {};
1765 }
1766 }
1767
1768 //--------------------------------------------------------------------------
1769 // Set up the exports object for the new instance.
1770 //--------------------------------------------------------------------------
1771 ProcessExports(instance);
1772 if (thrower_->error()) return {};
1773
1774 //--------------------------------------------------------------------------
1775 // Initialize the indirect function tables.
1776 //--------------------------------------------------------------------------
1777 if (function_table_count > 0) {
1778 LoadTableSegments(instance);
1779 }
1780
1781 //--------------------------------------------------------------------------
1782 // Initialize the memory by loading data segments.
1783 //--------------------------------------------------------------------------
1784 if (module_->data_segments.size() > 0) {
1785 LoadDataSegments(instance);
1786 }
1787
1788 //--------------------------------------------------------------------------
1789 // Patch all code with the relocations registered in code_specialization.
1790 //--------------------------------------------------------------------------
1791 CodeSpecialization code_specialization;
1792 code_specialization.RelocateDirectCalls(native_module);
1793 code_specialization.ApplyToWholeModule(native_module, module_object_,
1794 SKIP_ICACHE_FLUSH);
1795 FlushICache(native_module);
1796 FlushICache(handle(module_object_->export_wrappers()));
1797
1798 //--------------------------------------------------------------------------
1799 // Unpack and notify signal handler of protected instructions.
1800 //--------------------------------------------------------------------------
1801 if (use_trap_handler()) {
1802 native_module->UnpackAndRegisterProtectedInstructions();
1803 }
1804
1805 //--------------------------------------------------------------------------
1806 // Insert the compiled module into the weak list of compiled modules.
1807 //--------------------------------------------------------------------------
1808 {
1809 if (!old_instance.is_null()) {
1810 // Publish the new instance to the instances chain.
1811 DisallowHeapAllocation no_gc;
1812 compiled_module_->InsertInChain(*module_object_);
1813 }
1814 module_object_->set_compiled_module(*compiled_module_);
1815 compiled_module_->set_weak_owning_instance(*weak_instance);
1816 WasmInstanceObject::InstallFinalizer(isolate_, instance);
1817 }
1818
1819 //--------------------------------------------------------------------------
1820 // Debugging support.
1821 //--------------------------------------------------------------------------
1822 // Set all breakpoints that were set on the shared module.
1823 WasmSharedModuleData::SetBreakpointsOnNewInstance(
1824 handle(module_object_->shared(), isolate_), instance);
1825
1826 if (FLAG_wasm_interpret_all && module_->is_wasm()) {
1827 Handle<WasmDebugInfo> debug_info =
1828 WasmInstanceObject::GetOrCreateDebugInfo(instance);
1829 std::vector<int> func_indexes;
1830 for (int func_index = num_imported_functions,
1831 num_wasm_functions = static_cast<int>(module_->functions.size());
1832 func_index < num_wasm_functions; ++func_index) {
1833 func_indexes.push_back(func_index);
1834 }
1835 WasmDebugInfo::RedirectToInterpreter(
1836 debug_info, Vector<int>(func_indexes.data(),
1837 static_cast<int>(func_indexes.size())));
1838 }
1839
1840 //--------------------------------------------------------------------------
1841 // Create a wrapper for the start function.
1842 //--------------------------------------------------------------------------
1843 if (module_->start_function_index >= 0) {
1844 int start_index = module_->start_function_index;
1845 Handle<WasmInstanceObject> start_function_instance = instance;
1846 Address start_call_address =
1847 static_cast<uint32_t>(start_index) < module_->num_imported_functions
1848 ? kNullAddress
1849 : native_module->GetCallTargetForFunction(start_index);
1850 FunctionSig* sig = module_->functions[start_index].sig;
1851 Handle<Code> wrapper_code = js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper(
1852 isolate_, module_, start_call_address, start_index, use_trap_handler());
1853 // TODO(clemensh): Don't generate an exported function for the start
1854 // function. Use CWasmEntry instead.
1855 start_function_ = WasmExportedFunction::New(
1856 isolate_, start_function_instance, MaybeHandle<String>(), start_index,
1857 static_cast<int>(sig->parameter_count()), wrapper_code);
1858 }
1859
1860 DCHECK(!isolate_->has_pending_exception());
1861 TRACE("Successfully built instance %zu\n",
1862 compiled_module_->GetNativeModule()->instance_id);
1863 TRACE_CHAIN(module_object_->compiled_module());
1864 return instance;
1865 }
1866
ExecuteStartFunction()1867 bool InstanceBuilder::ExecuteStartFunction() {
1868 if (start_function_.is_null()) return true; // No start function.
1869
1870 HandleScope scope(isolate_);
1871 // Call the JS function.
1872 Handle<Object> undefined = isolate_->factory()->undefined_value();
1873 MaybeHandle<Object> retval =
1874 Execution::Call(isolate_, start_function_, undefined, 0, nullptr);
1875
1876 if (retval.is_null()) {
1877 DCHECK(isolate_->has_pending_exception());
1878 return false;
1879 }
1880 return true;
1881 }
1882
1883 // Look up an import value in the {ffi_} object.
LookupImport(uint32_t index,Handle<String> module_name,Handle<String> import_name)1884 MaybeHandle<Object> InstanceBuilder::LookupImport(uint32_t index,
1885 Handle<String> module_name,
1886
1887 Handle<String> import_name) {
1888 // We pre-validated in the js-api layer that the ffi object is present, and
1889 // a JSObject, if the module has imports.
1890 DCHECK(!ffi_.is_null());
1891
1892 // Look up the module first.
1893 MaybeHandle<Object> result =
1894 Object::GetPropertyOrElement(ffi_.ToHandleChecked(), module_name);
1895 if (result.is_null()) {
1896 return ReportTypeError("module not found", index, module_name);
1897 }
1898
1899 Handle<Object> module = result.ToHandleChecked();
1900
1901 // Look up the value in the module.
1902 if (!module->IsJSReceiver()) {
1903 return ReportTypeError("module is not an object or function", index,
1904 module_name);
1905 }
1906
1907 result = Object::GetPropertyOrElement(module, import_name);
1908 if (result.is_null()) {
1909 ReportLinkError("import not found", index, module_name, import_name);
1910 return MaybeHandle<JSFunction>();
1911 }
1912
1913 return result;
1914 }
1915
1916 // Look up an import value in the {ffi_} object specifically for linking an
1917 // asm.js module. This only performs non-observable lookups, which allows
1918 // falling back to JavaScript proper (and hence re-executing all lookups) if
1919 // module instantiation fails.
LookupImportAsm(uint32_t index,Handle<String> import_name)1920 MaybeHandle<Object> InstanceBuilder::LookupImportAsm(
1921 uint32_t index, Handle<String> import_name) {
1922 // Check that a foreign function interface object was provided.
1923 if (ffi_.is_null()) {
1924 return ReportLinkError("missing imports object", index, import_name);
1925 }
1926
1927 // Perform lookup of the given {import_name} without causing any observable
1928 // side-effect. We only accept accesses that resolve to data properties,
1929 // which is indicated by the asm.js spec in section 7 ("Linking") as well.
1930 Handle<Object> result;
1931 LookupIterator it = LookupIterator::PropertyOrElement(
1932 isolate_, ffi_.ToHandleChecked(), import_name);
1933 switch (it.state()) {
1934 case LookupIterator::ACCESS_CHECK:
1935 case LookupIterator::INTEGER_INDEXED_EXOTIC:
1936 case LookupIterator::INTERCEPTOR:
1937 case LookupIterator::JSPROXY:
1938 case LookupIterator::ACCESSOR:
1939 case LookupIterator::TRANSITION:
1940 return ReportLinkError("not a data property", index, import_name);
1941 case LookupIterator::NOT_FOUND:
1942 // Accepting missing properties as undefined does not cause any
1943 // observable difference from JavaScript semantics, we are lenient.
1944 result = isolate_->factory()->undefined_value();
1945 break;
1946 case LookupIterator::DATA:
1947 result = it.GetDataValue();
1948 break;
1949 }
1950
1951 return result;
1952 }
1953
EvalUint32InitExpr(const WasmInitExpr & expr)1954 uint32_t InstanceBuilder::EvalUint32InitExpr(const WasmInitExpr& expr) {
1955 switch (expr.kind) {
1956 case WasmInitExpr::kI32Const:
1957 return expr.val.i32_const;
1958 case WasmInitExpr::kGlobalIndex: {
1959 uint32_t offset = module_->globals[expr.val.global_index].offset;
1960 return *reinterpret_cast<uint32_t*>(raw_buffer_ptr(globals_, offset));
1961 }
1962 default:
1963 UNREACHABLE();
1964 }
1965 }
1966
1967 // Load data segments into the memory.
LoadDataSegments(Handle<WasmInstanceObject> instance)1968 void InstanceBuilder::LoadDataSegments(Handle<WasmInstanceObject> instance) {
1969 Handle<SeqOneByteString> module_bytes(
1970 module_object_->shared()->module_bytes(), isolate_);
1971 for (const WasmDataSegment& segment : module_->data_segments) {
1972 uint32_t source_size = segment.source.length();
1973 // Segments of size == 0 are just nops.
1974 if (source_size == 0) continue;
1975 uint32_t dest_offset = EvalUint32InitExpr(segment.dest_addr);
1976 DCHECK(in_bounds(dest_offset, source_size, instance->memory_size()));
1977 byte* dest = instance->memory_start() + dest_offset;
1978 const byte* src = reinterpret_cast<const byte*>(
1979 module_bytes->GetCharsAddress() + segment.source.offset());
1980 memcpy(dest, src, source_size);
1981 }
1982 }
1983
WriteGlobalValue(WasmGlobal & global,double num)1984 void InstanceBuilder::WriteGlobalValue(WasmGlobal& global, double num) {
1985 TRACE("init [globals_start=%p + %u] = %lf, type = %s\n",
1986 reinterpret_cast<void*>(raw_buffer_ptr(globals_, 0)), global.offset,
1987 num, ValueTypes::TypeName(global.type));
1988 switch (global.type) {
1989 case kWasmI32:
1990 *GetRawGlobalPtr<int32_t>(global) = static_cast<int32_t>(num);
1991 break;
1992 case kWasmI64:
1993 // TODO(titzer): initialization of imported i64 globals.
1994 UNREACHABLE();
1995 break;
1996 case kWasmF32:
1997 *GetRawGlobalPtr<float>(global) = static_cast<float>(num);
1998 break;
1999 case kWasmF64:
2000 *GetRawGlobalPtr<double>(global) = static_cast<double>(num);
2001 break;
2002 default:
2003 UNREACHABLE();
2004 }
2005 }
2006
WriteGlobalValue(WasmGlobal & global,Handle<WasmGlobalObject> value)2007 void InstanceBuilder::WriteGlobalValue(WasmGlobal& global,
2008 Handle<WasmGlobalObject> value) {
2009 TRACE("init [globals_start=%p + %u] = ",
2010 reinterpret_cast<void*>(raw_buffer_ptr(globals_, 0)), global.offset);
2011 switch (global.type) {
2012 case kWasmI32: {
2013 int32_t num = value->GetI32();
2014 *GetRawGlobalPtr<int32_t>(global) = num;
2015 TRACE("%d", num);
2016 break;
2017 }
2018 case kWasmI64: {
2019 int64_t num = value->GetI64();
2020 *GetRawGlobalPtr<int64_t>(global) = num;
2021 TRACE("%" PRId64, num);
2022 break;
2023 }
2024 case kWasmF32: {
2025 float num = value->GetF32();
2026 *GetRawGlobalPtr<float>(global) = num;
2027 TRACE("%f", num);
2028 break;
2029 }
2030 case kWasmF64: {
2031 double num = value->GetF64();
2032 *GetRawGlobalPtr<double>(global) = num;
2033 TRACE("%lf", num);
2034 break;
2035 }
2036 default:
2037 UNREACHABLE();
2038 }
2039 TRACE(", type = %s (from WebAssembly.Global)\n",
2040 ValueTypes::TypeName(global.type));
2041 }
2042
SanitizeImports()2043 void InstanceBuilder::SanitizeImports() {
2044 Handle<SeqOneByteString> module_bytes(
2045 module_object_->shared()->module_bytes());
2046 for (size_t index = 0; index < module_->import_table.size(); ++index) {
2047 WasmImport& import = module_->import_table[index];
2048
2049 Handle<String> module_name;
2050 MaybeHandle<String> maybe_module_name =
2051 WasmSharedModuleData::ExtractUtf8StringFromModuleBytes(
2052 isolate_, module_bytes, import.module_name);
2053 if (!maybe_module_name.ToHandle(&module_name)) {
2054 thrower_->LinkError("Could not resolve module name for import %zu",
2055 index);
2056 return;
2057 }
2058
2059 Handle<String> import_name;
2060 MaybeHandle<String> maybe_import_name =
2061 WasmSharedModuleData::ExtractUtf8StringFromModuleBytes(
2062 isolate_, module_bytes, import.field_name);
2063 if (!maybe_import_name.ToHandle(&import_name)) {
2064 thrower_->LinkError("Could not resolve import name for import %zu",
2065 index);
2066 return;
2067 }
2068
2069 int int_index = static_cast<int>(index);
2070 MaybeHandle<Object> result =
2071 module_->is_asm_js()
2072 ? LookupImportAsm(int_index, import_name)
2073 : LookupImport(int_index, module_name, import_name);
2074 if (thrower_->error()) {
2075 thrower_->LinkError("Could not find value for import %zu", index);
2076 return;
2077 }
2078 Handle<Object> value = result.ToHandleChecked();
2079 sanitized_imports_.push_back({module_name, import_name, value});
2080 }
2081 }
2082
2083 // Process the imports, including functions, tables, globals, and memory, in
2084 // order, loading them from the {ffi_} object. Returns the number of imported
2085 // functions.
ProcessImports(Handle<WasmInstanceObject> instance)2086 int InstanceBuilder::ProcessImports(Handle<WasmInstanceObject> instance) {
2087 int num_imported_functions = 0;
2088 int num_imported_tables = 0;
2089 int num_imported_mutable_globals = 0;
2090
2091 DCHECK_EQ(module_->import_table.size(), sanitized_imports_.size());
2092 for (int index = 0; index < static_cast<int>(module_->import_table.size());
2093 ++index) {
2094 WasmImport& import = module_->import_table[index];
2095
2096 Handle<String> module_name = sanitized_imports_[index].module_name;
2097 Handle<String> import_name = sanitized_imports_[index].import_name;
2098 Handle<Object> value = sanitized_imports_[index].value;
2099 NativeModule* native_module =
2100 instance->compiled_module()->GetNativeModule();
2101
2102 switch (import.kind) {
2103 case kExternalFunction: {
2104 // Function imports must be callable.
2105 if (!value->IsCallable()) {
2106 ReportLinkError("function import requires a callable", index,
2107 module_name, import_name);
2108 return -1;
2109 }
2110 uint32_t func_index = import.index;
2111 DCHECK_EQ(num_imported_functions, func_index);
2112 FunctionSig* expected_sig = module_->functions[func_index].sig;
2113 if (WasmExportedFunction::IsWasmExportedFunction(*value)) {
2114 // The imported function is a WASM function from another instance.
2115 Handle<WasmExportedFunction> imported_function(
2116 WasmExportedFunction::cast(*value), isolate_);
2117 Handle<WasmInstanceObject> imported_instance(
2118 imported_function->instance(), isolate_);
2119 FunctionSig* imported_sig =
2120 imported_instance->module()
2121 ->functions[imported_function->function_index()]
2122 .sig;
2123 if (!imported_sig->Equals(expected_sig)) {
2124 ReportLinkError(
2125 "imported function does not match the expected type", index,
2126 module_name, import_name);
2127 return -1;
2128 }
2129 // The import reference is the instance object itself.
2130 ImportedFunctionEntry entry(instance, func_index);
2131 Address imported_target = imported_function->GetWasmCallTarget();
2132 entry.set_wasm_to_wasm(*imported_instance, imported_target);
2133 } else {
2134 // The imported function is a callable.
2135 Handle<JSReceiver> js_receiver(JSReceiver::cast(*value), isolate_);
2136 Handle<Code> wrapper_code = compiler::CompileWasmToJSWrapper(
2137 isolate_, js_receiver, expected_sig, func_index,
2138 module_->origin(), use_trap_handler());
2139 RecordStats(*wrapper_code, counters());
2140
2141 WasmCode* wasm_code = native_module->AddCodeCopy(
2142 wrapper_code, wasm::WasmCode::kWasmToJsWrapper, func_index);
2143 ImportedFunctionEntry entry(instance, func_index);
2144 entry.set_wasm_to_js(*js_receiver, wasm_code);
2145 }
2146 num_imported_functions++;
2147 break;
2148 }
2149 case kExternalTable: {
2150 if (!value->IsWasmTableObject()) {
2151 ReportLinkError("table import requires a WebAssembly.Table", index,
2152 module_name, import_name);
2153 return -1;
2154 }
2155 uint32_t table_num = import.index;
2156 DCHECK_EQ(table_num, num_imported_tables);
2157 WasmIndirectFunctionTable& table = module_->function_tables[table_num];
2158 TableInstance& table_instance = table_instances_[table_num];
2159 table_instance.table_object = Handle<WasmTableObject>::cast(value);
2160 instance->set_table_object(*table_instance.table_object);
2161 table_instance.js_wrappers = Handle<FixedArray>(
2162 table_instance.table_object->functions(), isolate_);
2163
2164 int imported_table_size = table_instance.js_wrappers->length();
2165 if (imported_table_size < static_cast<int>(table.initial_size)) {
2166 thrower_->LinkError(
2167 "table import %d is smaller than initial %d, got %u", index,
2168 table.initial_size, imported_table_size);
2169 return -1;
2170 }
2171
2172 if (table.has_maximum_size) {
2173 int64_t imported_maximum_size =
2174 table_instance.table_object->maximum_length()->Number();
2175 if (imported_maximum_size < 0) {
2176 thrower_->LinkError(
2177 "table import %d has no maximum length, expected %d", index,
2178 table.maximum_size);
2179 return -1;
2180 }
2181 if (imported_maximum_size > table.maximum_size) {
2182 thrower_->LinkError(
2183 " table import %d has a larger maximum size %" PRIx64
2184 " than the module's declared maximum %u",
2185 index, imported_maximum_size, table.maximum_size);
2186 return -1;
2187 }
2188 }
2189
2190 // Allocate a new dispatch table.
2191 if (!instance->has_indirect_function_table()) {
2192 WasmInstanceObject::EnsureIndirectFunctionTableWithMinimumSize(
2193 instance, imported_table_size);
2194 table_instances_[table_num].table_size = imported_table_size;
2195 }
2196 // Initialize the dispatch table with the (foreign) JS functions
2197 // that are already in the table.
2198 for (int i = 0; i < imported_table_size; ++i) {
2199 Handle<Object> val(table_instance.js_wrappers->get(i), isolate_);
2200 // TODO(mtrofin): this is the same logic as WasmTableObject::Set:
2201 // insert in the local table a wrapper from the other module, and add
2202 // a reference to the owning instance of the other module.
2203 if (!val->IsJSFunction()) continue;
2204 if (!WasmExportedFunction::IsWasmExportedFunction(*val)) {
2205 thrower_->LinkError("table import %d[%d] is not a wasm function",
2206 index, i);
2207 return -1;
2208 }
2209 // Look up the signature's canonical id. If there is no canonical
2210 // id, then the signature does not appear at all in this module,
2211 // so putting {-1} in the table will cause checks to always fail.
2212 auto target = Handle<WasmExportedFunction>::cast(val);
2213 Handle<WasmInstanceObject> imported_instance =
2214 handle(target->instance());
2215 Address exported_call_target = target->GetWasmCallTarget();
2216 FunctionSig* sig = imported_instance->module()
2217 ->functions[target->function_index()]
2218 .sig;
2219 IndirectFunctionTableEntry(instance, i)
2220 .set(module_->signature_map.Find(sig), *imported_instance,
2221 exported_call_target);
2222 }
2223 num_imported_tables++;
2224 break;
2225 }
2226 case kExternalMemory: {
2227 // Validation should have failed if more than one memory object was
2228 // provided.
2229 DCHECK(!instance->has_memory_object());
2230 if (!value->IsWasmMemoryObject()) {
2231 ReportLinkError("memory import must be a WebAssembly.Memory object",
2232 index, module_name, import_name);
2233 return -1;
2234 }
2235 auto memory = Handle<WasmMemoryObject>::cast(value);
2236 instance->set_memory_object(*memory);
2237 Handle<JSArrayBuffer> buffer(memory->array_buffer(), isolate_);
2238 memory_ = buffer;
2239 uint32_t imported_cur_pages = static_cast<uint32_t>(
2240 buffer->byte_length()->Number() / kWasmPageSize);
2241 if (imported_cur_pages < module_->initial_pages) {
2242 thrower_->LinkError(
2243 "memory import %d is smaller than initial %u, got %u", index,
2244 module_->initial_pages, imported_cur_pages);
2245 }
2246 int32_t imported_maximum_pages = memory->maximum_pages();
2247 if (module_->has_maximum_pages) {
2248 if (imported_maximum_pages < 0) {
2249 thrower_->LinkError(
2250 "memory import %d has no maximum limit, expected at most %u",
2251 index, imported_maximum_pages);
2252 return -1;
2253 }
2254 if (static_cast<uint32_t>(imported_maximum_pages) >
2255 module_->maximum_pages) {
2256 thrower_->LinkError(
2257 "memory import %d has a larger maximum size %u than the "
2258 "module's declared maximum %u",
2259 index, imported_maximum_pages, module_->maximum_pages);
2260 return -1;
2261 }
2262 }
2263 if (module_->has_shared_memory != buffer->is_shared()) {
2264 thrower_->LinkError(
2265 "mismatch in shared state of memory, declared = %d, imported = "
2266 "%d",
2267 module_->has_shared_memory, buffer->is_shared());
2268 return -1;
2269 }
2270
2271 break;
2272 }
2273 case kExternalGlobal: {
2274 // Immutable global imports are converted to numbers and written into
2275 // the {globals_} array buffer.
2276 //
2277 // Mutable global imports instead have their backing array buffers
2278 // referenced by this instance, and store the address of the imported
2279 // global in the {imported_mutable_globals_} array.
2280 WasmGlobal& global = module_->globals[import.index];
2281
2282 // The mutable-global proposal allows importing i64 values, but only if
2283 // they are passed as a WebAssembly.Global object.
2284 if (global.type == kWasmI64 && !(FLAG_experimental_wasm_mut_global &&
2285 value->IsWasmGlobalObject())) {
2286 ReportLinkError("global import cannot have type i64", index,
2287 module_name, import_name);
2288 return -1;
2289 }
2290 if (module_->is_asm_js()) {
2291 // Accepting {JSFunction} on top of just primitive values here is a
2292 // workaround to support legacy asm.js code with broken binding. Note
2293 // that using {NaN} (or Smi::kZero) here is what using the observable
2294 // conversion via {ToPrimitive} would produce as well.
2295 // TODO(mstarzinger): Still observable if Function.prototype.valueOf
2296 // or friends are patched, we might need to check for that as well.
2297 if (value->IsJSFunction()) value = isolate_->factory()->nan_value();
2298 if (value->IsPrimitive() && !value->IsSymbol()) {
2299 if (global.type == kWasmI32) {
2300 value = Object::ToInt32(isolate_, value).ToHandleChecked();
2301 } else {
2302 value = Object::ToNumber(value).ToHandleChecked();
2303 }
2304 }
2305 }
2306 if (FLAG_experimental_wasm_mut_global) {
2307 if (value->IsWasmGlobalObject()) {
2308 auto global_object = Handle<WasmGlobalObject>::cast(value);
2309 if (global_object->type() != global.type) {
2310 ReportLinkError(
2311 "imported global does not match the expected type", index,
2312 module_name, import_name);
2313 return -1;
2314 }
2315 if (global_object->is_mutable() != global.mutability) {
2316 ReportLinkError(
2317 "imported global does not match the expected mutability",
2318 index, module_name, import_name);
2319 return -1;
2320 }
2321 if (global.mutability) {
2322 Handle<JSArrayBuffer> buffer(global_object->array_buffer(),
2323 isolate_);
2324 int index = num_imported_mutable_globals++;
2325 instance->imported_mutable_globals_buffers()->set(index, *buffer);
2326 // It is safe in this case to store the raw pointer to the buffer
2327 // since the backing store of the JSArrayBuffer will not be
2328 // relocated.
2329 instance->imported_mutable_globals()[index] =
2330 reinterpret_cast<Address>(
2331 raw_buffer_ptr(buffer, global_object->offset()));
2332 } else {
2333 WriteGlobalValue(global, global_object);
2334 }
2335 } else if (value->IsNumber()) {
2336 if (global.mutability) {
2337 ReportLinkError(
2338 "imported mutable global must be a WebAssembly.Global object",
2339 index, module_name, import_name);
2340 return -1;
2341 }
2342 WriteGlobalValue(global, value->Number());
2343 } else {
2344 ReportLinkError(
2345 "global import must be a number or WebAssembly.Global object",
2346 index, module_name, import_name);
2347 return -1;
2348 }
2349 } else {
2350 if (value->IsNumber()) {
2351 WriteGlobalValue(global, value->Number());
2352 } else {
2353 ReportLinkError("global import must be a number", index,
2354 module_name, import_name);
2355 return -1;
2356 }
2357 }
2358 break;
2359 }
2360 default:
2361 UNREACHABLE();
2362 break;
2363 }
2364 }
2365
2366 DCHECK_EQ(module_->num_imported_mutable_globals,
2367 num_imported_mutable_globals);
2368
2369 return num_imported_functions;
2370 }
2371
2372 template <typename T>
GetRawGlobalPtr(WasmGlobal & global)2373 T* InstanceBuilder::GetRawGlobalPtr(WasmGlobal& global) {
2374 return reinterpret_cast<T*>(raw_buffer_ptr(globals_, global.offset));
2375 }
2376
2377 // Process initialization of globals.
InitGlobals()2378 void InstanceBuilder::InitGlobals() {
2379 for (auto global : module_->globals) {
2380 if (global.mutability && global.imported) {
2381 DCHECK(FLAG_experimental_wasm_mut_global);
2382 continue;
2383 }
2384
2385 switch (global.init.kind) {
2386 case WasmInitExpr::kI32Const:
2387 *GetRawGlobalPtr<int32_t>(global) = global.init.val.i32_const;
2388 break;
2389 case WasmInitExpr::kI64Const:
2390 *GetRawGlobalPtr<int64_t>(global) = global.init.val.i64_const;
2391 break;
2392 case WasmInitExpr::kF32Const:
2393 *GetRawGlobalPtr<float>(global) = global.init.val.f32_const;
2394 break;
2395 case WasmInitExpr::kF64Const:
2396 *GetRawGlobalPtr<double>(global) = global.init.val.f64_const;
2397 break;
2398 case WasmInitExpr::kGlobalIndex: {
2399 // Initialize with another global.
2400 uint32_t new_offset = global.offset;
2401 uint32_t old_offset =
2402 module_->globals[global.init.val.global_index].offset;
2403 TRACE("init [globals+%u] = [globals+%d]\n", global.offset, old_offset);
2404 size_t size = (global.type == kWasmI64 || global.type == kWasmF64)
2405 ? sizeof(double)
2406 : sizeof(int32_t);
2407 memcpy(raw_buffer_ptr(globals_, new_offset),
2408 raw_buffer_ptr(globals_, old_offset), size);
2409 break;
2410 }
2411 case WasmInitExpr::kNone:
2412 // Happens with imported globals.
2413 break;
2414 default:
2415 UNREACHABLE();
2416 break;
2417 }
2418 }
2419 }
2420
2421 // Allocate memory for a module instance as a new JSArrayBuffer.
AllocateMemory(uint32_t num_pages)2422 Handle<JSArrayBuffer> InstanceBuilder::AllocateMemory(uint32_t num_pages) {
2423 if (num_pages > FLAG_wasm_max_mem_pages) {
2424 thrower_->RangeError("Out of memory: wasm memory too large");
2425 return Handle<JSArrayBuffer>::null();
2426 }
2427 const bool is_shared_memory =
2428 module_->has_shared_memory && i::FLAG_experimental_wasm_threads;
2429 i::SharedFlag shared_flag =
2430 is_shared_memory ? i::SharedFlag::kShared : i::SharedFlag::kNotShared;
2431 Handle<JSArrayBuffer> mem_buffer;
2432 if (!NewArrayBuffer(isolate_, num_pages * kWasmPageSize, shared_flag)
2433 .ToHandle(&mem_buffer)) {
2434 thrower_->RangeError("Out of memory: wasm memory");
2435 }
2436 return mem_buffer;
2437 }
2438
NeedsWrappers() const2439 bool InstanceBuilder::NeedsWrappers() const {
2440 if (module_->num_exported_functions > 0) return true;
2441 for (auto& table_instance : table_instances_) {
2442 if (!table_instance.js_wrappers.is_null()) return true;
2443 }
2444 for (auto& table : module_->function_tables) {
2445 if (table.exported) return true;
2446 }
2447 return false;
2448 }
2449
2450 // Process the exports, creating wrappers for functions, tables, memories,
2451 // and globals.
ProcessExports(Handle<WasmInstanceObject> instance)2452 void InstanceBuilder::ProcessExports(Handle<WasmInstanceObject> instance) {
2453 Handle<FixedArray> export_wrappers(module_object_->export_wrappers(),
2454 isolate_);
2455 if (NeedsWrappers()) {
2456 // Fill the table to cache the exported JSFunction wrappers.
2457 js_wrappers_.insert(js_wrappers_.begin(), module_->functions.size(),
2458 Handle<JSFunction>::null());
2459
2460 // If an imported WebAssembly function gets exported, the exported function
2461 // has to be identical to to imported function. Therefore we put all
2462 // imported WebAssembly functions into the js_wrappers_ list.
2463 for (int index = 0, end = static_cast<int>(module_->import_table.size());
2464 index < end; ++index) {
2465 WasmImport& import = module_->import_table[index];
2466 if (import.kind == kExternalFunction) {
2467 Handle<Object> value = sanitized_imports_[index].value;
2468 if (WasmExportedFunction::IsWasmExportedFunction(*value)) {
2469 js_wrappers_[import.index] = Handle<JSFunction>::cast(value);
2470 }
2471 }
2472 }
2473 }
2474
2475 Handle<JSObject> exports_object;
2476 if (module_->is_wasm()) {
2477 // Create the "exports" object.
2478 exports_object = isolate_->factory()->NewJSObjectWithNullProto();
2479 } else if (module_->is_asm_js()) {
2480 Handle<JSFunction> object_function = Handle<JSFunction>(
2481 isolate_->native_context()->object_function(), isolate_);
2482 exports_object = isolate_->factory()->NewJSObject(object_function);
2483 } else {
2484 UNREACHABLE();
2485 }
2486 instance->set_exports_object(*exports_object);
2487
2488 Handle<String> single_function_name =
2489 isolate_->factory()->InternalizeUtf8String(AsmJs::kSingleFunctionName);
2490
2491 PropertyDescriptor desc;
2492 desc.set_writable(module_->is_asm_js());
2493 desc.set_enumerable(true);
2494 desc.set_configurable(module_->is_asm_js());
2495
2496 // Process each export in the export table.
2497 int export_index = 0; // Index into {export_wrappers}.
2498 for (WasmExport& exp : module_->export_table) {
2499 Handle<String> name =
2500 WasmSharedModuleData::ExtractUtf8StringFromModuleBytes(
2501 isolate_, handle(module_object_->shared(), isolate_), exp.name)
2502 .ToHandleChecked();
2503 Handle<JSObject> export_to;
2504 if (module_->is_asm_js() && exp.kind == kExternalFunction &&
2505 String::Equals(name, single_function_name)) {
2506 export_to = instance;
2507 } else {
2508 export_to = exports_object;
2509 }
2510
2511 switch (exp.kind) {
2512 case kExternalFunction: {
2513 // Wrap and export the code as a JSFunction.
2514 WasmFunction& function = module_->functions[exp.index];
2515 Handle<JSFunction> js_function = js_wrappers_[exp.index];
2516 if (js_function.is_null()) {
2517 // Wrap the exported code as a JSFunction.
2518 Handle<Code> export_code =
2519 export_wrappers->GetValueChecked<Code>(isolate_, export_index);
2520 MaybeHandle<String> func_name;
2521 if (module_->is_asm_js()) {
2522 // For modules arising from asm.js, honor the names section.
2523 WireBytesRef func_name_ref = module_->LookupName(
2524 module_object_->shared()->module_bytes(), function.func_index);
2525 func_name =
2526 WasmSharedModuleData::ExtractUtf8StringFromModuleBytes(
2527 isolate_, handle(module_object_->shared(), isolate_),
2528 func_name_ref)
2529 .ToHandleChecked();
2530 }
2531 js_function = WasmExportedFunction::New(
2532 isolate_, instance, func_name, function.func_index,
2533 static_cast<int>(function.sig->parameter_count()), export_code);
2534 js_wrappers_[exp.index] = js_function;
2535 }
2536 desc.set_value(js_function);
2537 export_index++;
2538 break;
2539 }
2540 case kExternalTable: {
2541 // Export a table as a WebAssembly.Table object.
2542 TableInstance& table_instance = table_instances_[exp.index];
2543 WasmIndirectFunctionTable& table = module_->function_tables[exp.index];
2544 if (table_instance.table_object.is_null()) {
2545 uint32_t maximum = table.has_maximum_size ? table.maximum_size
2546 : FLAG_wasm_max_table_size;
2547 table_instance.table_object =
2548 WasmTableObject::New(isolate_, table.initial_size, maximum,
2549 &table_instance.js_wrappers);
2550 }
2551 desc.set_value(table_instance.table_object);
2552 break;
2553 }
2554 case kExternalMemory: {
2555 // Export the memory as a WebAssembly.Memory object. A WasmMemoryObject
2556 // should already be available if the module has memory, since we always
2557 // create or import it when building an WasmInstanceObject.
2558 DCHECK(instance->has_memory_object());
2559 desc.set_value(
2560 Handle<WasmMemoryObject>(instance->memory_object(), isolate_));
2561 break;
2562 }
2563 case kExternalGlobal: {
2564 WasmGlobal& global = module_->globals[exp.index];
2565 if (FLAG_experimental_wasm_mut_global) {
2566 Handle<JSArrayBuffer> globals_buffer(instance->globals_buffer(),
2567 isolate_);
2568 // Since the global's array buffer is always provided, allocation
2569 // should never fail.
2570 Handle<WasmGlobalObject> global_obj =
2571 WasmGlobalObject::New(isolate_, globals_buffer, global.type,
2572 global.offset, global.mutability)
2573 .ToHandleChecked();
2574 desc.set_value(global_obj);
2575 } else {
2576 // Export the value of the global variable as a number.
2577 double num = 0;
2578 switch (global.type) {
2579 case kWasmI32:
2580 num = *GetRawGlobalPtr<int32_t>(global);
2581 break;
2582 case kWasmF32:
2583 num = *GetRawGlobalPtr<float>(global);
2584 break;
2585 case kWasmF64:
2586 num = *GetRawGlobalPtr<double>(global);
2587 break;
2588 case kWasmI64:
2589 thrower_->LinkError(
2590 "export of globals of type I64 is not allowed.");
2591 return;
2592 default:
2593 UNREACHABLE();
2594 }
2595 desc.set_value(isolate_->factory()->NewNumber(num));
2596 }
2597 break;
2598 }
2599 default:
2600 UNREACHABLE();
2601 break;
2602 }
2603
2604 v8::Maybe<bool> status = JSReceiver::DefineOwnProperty(
2605 isolate_, export_to, name, &desc, kThrowOnError);
2606 if (!status.IsJust()) {
2607 TruncatedUserString<> trunc_name(name->GetCharVector<uint8_t>());
2608 thrower_->LinkError("export of %.*s failed.", trunc_name.length(),
2609 trunc_name.start());
2610 return;
2611 }
2612 }
2613 DCHECK_EQ(export_index, export_wrappers->length());
2614
2615 if (module_->is_wasm()) {
2616 v8::Maybe<bool> success =
2617 JSReceiver::SetIntegrityLevel(exports_object, FROZEN, kDontThrow);
2618 DCHECK(success.FromMaybe(false));
2619 USE(success);
2620 }
2621 }
2622
InitializeTables(Handle<WasmInstanceObject> instance)2623 void InstanceBuilder::InitializeTables(Handle<WasmInstanceObject> instance) {
2624 size_t table_count = module_->function_tables.size();
2625 for (size_t index = 0; index < table_count; ++index) {
2626 WasmIndirectFunctionTable& table = module_->function_tables[index];
2627 TableInstance& table_instance = table_instances_[index];
2628
2629 if (!instance->has_indirect_function_table()) {
2630 WasmInstanceObject::EnsureIndirectFunctionTableWithMinimumSize(
2631 instance, table.initial_size);
2632 table_instance.table_size = table.initial_size;
2633 }
2634 }
2635 }
2636
LoadTableSegments(Handle<WasmInstanceObject> instance)2637 void InstanceBuilder::LoadTableSegments(Handle<WasmInstanceObject> instance) {
2638 NativeModule* native_module = compiled_module_->GetNativeModule();
2639 int function_table_count = static_cast<int>(module_->function_tables.size());
2640 for (int index = 0; index < function_table_count; ++index) {
2641 TableInstance& table_instance = table_instances_[index];
2642
2643 // TODO(titzer): this does redundant work if there are multiple tables,
2644 // since initializations are not sorted by table index.
2645 for (auto& table_init : module_->table_inits) {
2646 uint32_t base = EvalUint32InitExpr(table_init.offset);
2647 uint32_t num_entries = static_cast<uint32_t>(table_init.entries.size());
2648 DCHECK(in_bounds(base, num_entries, table_instance.table_size));
2649 for (uint32_t i = 0; i < num_entries; ++i) {
2650 uint32_t func_index = table_init.entries[i];
2651 WasmFunction* function = &module_->functions[func_index];
2652 int table_index = static_cast<int>(i + base);
2653
2654 // Update the local dispatch table first.
2655 uint32_t sig_id = module_->signature_ids[function->sig_index];
2656 Handle<WasmInstanceObject> target_instance = instance;
2657 Address call_target;
2658 const bool is_import = func_index < module_->num_imported_functions;
2659 if (is_import) {
2660 // For imported calls, take target instance and address from the
2661 // import table.
2662 ImportedFunctionEntry entry(instance, func_index);
2663 target_instance = handle(entry.instance(), isolate_);
2664 call_target = entry.target();
2665 } else {
2666 call_target = native_module->GetCallTargetForFunction(func_index);
2667 }
2668 IndirectFunctionTableEntry(instance, table_index)
2669 .set(sig_id, *target_instance, call_target);
2670
2671 if (!table_instance.table_object.is_null()) {
2672 // Update the table object's other dispatch tables.
2673 if (js_wrappers_[func_index].is_null()) {
2674 // No JSFunction entry yet exists for this function. Create one.
2675 // TODO(titzer): We compile JS->wasm wrappers for functions are
2676 // not exported but are in an exported table. This should be done
2677 // at module compile time and cached instead.
2678
2679 Handle<Code> wrapper_code =
2680 js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper(
2681 isolate_, module_, is_import ? kNullAddress : call_target,
2682 func_index, use_trap_handler());
2683 MaybeHandle<String> func_name;
2684 if (module_->is_asm_js()) {
2685 // For modules arising from asm.js, honor the names section.
2686 WireBytesRef func_name_ref = module_->LookupName(
2687 module_object_->shared()->module_bytes(), func_index);
2688 func_name =
2689 WasmSharedModuleData::ExtractUtf8StringFromModuleBytes(
2690 isolate_, handle(module_object_->shared(), isolate_),
2691 func_name_ref)
2692 .ToHandleChecked();
2693 }
2694 Handle<WasmExportedFunction> js_function =
2695 WasmExportedFunction::New(
2696 isolate_, instance, func_name, func_index,
2697 static_cast<int>(function->sig->parameter_count()),
2698 wrapper_code);
2699 js_wrappers_[func_index] = js_function;
2700 }
2701 table_instance.js_wrappers->set(table_index,
2702 *js_wrappers_[func_index]);
2703 // UpdateDispatchTables() updates all other dispatch tables, since
2704 // we have not yet added the dispatch table we are currently building.
2705 WasmTableObject::UpdateDispatchTables(
2706 isolate_, table_instance.table_object, table_index, function->sig,
2707 target_instance, call_target);
2708 }
2709 }
2710 }
2711
2712 // Add the new dispatch table at the end to avoid redundant lookups.
2713 if (!table_instance.table_object.is_null()) {
2714 // Add the new dispatch table to the WebAssembly.Table object.
2715 WasmTableObject::AddDispatchTable(isolate_, table_instance.table_object,
2716 instance, index);
2717 }
2718 }
2719 }
2720
AsyncCompileJob(Isolate * isolate,std::unique_ptr<byte[]> bytes_copy,size_t length,Handle<Context> context,Handle<JSPromise> promise)2721 AsyncCompileJob::AsyncCompileJob(Isolate* isolate,
2722 std::unique_ptr<byte[]> bytes_copy,
2723 size_t length, Handle<Context> context,
2724 Handle<JSPromise> promise)
2725 : isolate_(isolate),
2726 async_counters_(isolate->async_counters()),
2727 bytes_copy_(std::move(bytes_copy)),
2728 wire_bytes_(bytes_copy_.get(), bytes_copy_.get() + length) {
2729 v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate);
2730 v8::Platform* platform = V8::GetCurrentPlatform();
2731 foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate);
2732 background_task_runner_ = platform->GetWorkerThreadsTaskRunner(v8_isolate);
2733 // The handles for the context and promise must be deferred.
2734 DeferredHandleScope deferred(isolate);
2735 context_ = Handle<Context>(*context);
2736 module_promise_ = Handle<JSPromise>(*promise);
2737 deferred_handles_.push_back(deferred.Detach());
2738 }
2739
Start()2740 void AsyncCompileJob::Start() {
2741 DoAsync<DecodeModule>(); // --
2742 }
2743
Abort()2744 void AsyncCompileJob::Abort() {
2745 background_task_manager_.CancelAndWait();
2746 if (!compiled_module_.is_null()) {
2747 compiled_module_->GetNativeModule()->compilation_state()->Abort();
2748 }
2749 if (num_pending_foreground_tasks_ == 0) {
2750 // No task is pending, we can just remove the AsyncCompileJob.
2751 isolate_->wasm_engine()->RemoveCompileJob(this);
2752 } else {
2753 // There is still a compilation task in the task queue. We enter the
2754 // AbortCompilation state and wait for this compilation task to abort the
2755 // AsyncCompileJob.
2756 NextStep<AbortCompilation>();
2757 }
2758 }
2759
2760 class AsyncStreamingProcessor final : public StreamingProcessor {
2761 public:
2762 explicit AsyncStreamingProcessor(AsyncCompileJob* job);
2763
2764 bool ProcessModuleHeader(Vector<const uint8_t> bytes,
2765 uint32_t offset) override;
2766
2767 bool ProcessSection(SectionCode section_code, Vector<const uint8_t> bytes,
2768 uint32_t offset) override;
2769
2770 bool ProcessCodeSectionHeader(size_t functions_count,
2771 uint32_t offset) override;
2772
2773 bool ProcessFunctionBody(Vector<const uint8_t> bytes,
2774 uint32_t offset) override;
2775
2776 void OnFinishedChunk() override;
2777
2778 void OnFinishedStream(std::unique_ptr<uint8_t[]> bytes,
2779 size_t length) override;
2780
2781 void OnError(DecodeResult result) override;
2782
2783 void OnAbort() override;
2784
2785 private:
2786 // Finishes the AsyncCompileJob with an error.
2787 void FinishAsyncCompileJobWithError(ResultBase result);
2788
2789 void CommitCompilationUnits();
2790
2791 ModuleDecoder decoder_;
2792 AsyncCompileJob* job_;
2793 std::unique_ptr<CompilationUnitBuilder> compilation_unit_builder_;
2794 uint32_t next_function_ = 0;
2795 };
2796
CreateStreamingDecoder()2797 std::shared_ptr<StreamingDecoder> AsyncCompileJob::CreateStreamingDecoder() {
2798 DCHECK_NULL(stream_);
2799 stream_.reset(
2800 new StreamingDecoder(base::make_unique<AsyncStreamingProcessor>(this)));
2801 return stream_;
2802 }
2803
~AsyncCompileJob()2804 AsyncCompileJob::~AsyncCompileJob() {
2805 background_task_manager_.CancelAndWait();
2806 for (auto d : deferred_handles_) delete d;
2807 }
2808
FinishCompile()2809 void AsyncCompileJob::FinishCompile() {
2810 RecordStats(compiled_module_->GetNativeModule(), counters());
2811
2812 // Create heap objects for script and module bytes to be stored in the
2813 // shared module data. Asm.js is not compiled asynchronously.
2814 Handle<Script> script = CreateWasmScript(isolate_, wire_bytes_);
2815 Handle<ByteArray> asm_js_offset_table;
2816 // TODO(wasm): Improve efficiency of storing module wire bytes.
2817 // 1. Only store relevant sections, not function bodies
2818 // 2. Don't make a second copy of the bytes here; reuse the copy made
2819 // for asynchronous compilation and store it as an external one
2820 // byte string for serialization/deserialization.
2821 Handle<String> module_bytes =
2822 isolate_->factory()
2823 ->NewStringFromOneByte({wire_bytes_.start(), wire_bytes_.length()},
2824 TENURED)
2825 .ToHandleChecked();
2826 DCHECK(module_bytes->IsSeqOneByteString());
2827 int export_wrapper_size = static_cast<int>(module_->num_exported_functions);
2828 Handle<FixedArray> export_wrappers =
2829 isolate_->factory()->NewFixedArray(export_wrapper_size, TENURED);
2830
2831 // The {managed_module} will take ownership of the {WasmModule} object,
2832 // and it will be destroyed when the GC reclaims the wrapper object.
2833 Handle<Managed<WasmModule>> managed_module =
2834 Managed<WasmModule>::FromUniquePtr(isolate_, std::move(module_));
2835
2836 // Create the shared module data.
2837 // TODO(clemensh): For the same module (same bytes / same hash), we should
2838 // only have one WasmSharedModuleData. Otherwise, we might only set
2839 // breakpoints on a (potentially empty) subset of the instances.
2840 Handle<WasmSharedModuleData> shared = WasmSharedModuleData::New(
2841 isolate_, managed_module, Handle<SeqOneByteString>::cast(module_bytes),
2842 script, asm_js_offset_table);
2843 compiled_module_->GetNativeModule()->SetSharedModuleData(shared);
2844
2845 // Create the module object.
2846 module_object_ = WasmModuleObject::New(isolate_, compiled_module_,
2847 export_wrappers, shared);
2848 {
2849 DeferredHandleScope deferred(isolate_);
2850 module_object_ = handle(*module_object_, isolate_);
2851 deferred_handles_.push_back(deferred.Detach());
2852 }
2853
2854 // Finish the wasm script now and make it public to the debugger.
2855 isolate_->debug()->OnAfterCompile(script);
2856
2857 // TODO(wasm): compiling wrappers should be made async as well.
2858 DoSync<CompileWrappers>();
2859 }
2860
AsyncCompileFailed(Handle<Object> error_reason)2861 void AsyncCompileJob::AsyncCompileFailed(Handle<Object> error_reason) {
2862 if (stream_) stream_->NotifyError();
2863 // {job} keeps the {this} pointer alive.
2864 std::shared_ptr<AsyncCompileJob> job =
2865 isolate_->wasm_engine()->RemoveCompileJob(this);
2866 MaybeHandle<Object> promise_result =
2867 JSPromise::Reject(module_promise_, error_reason);
2868 CHECK_EQ(promise_result.is_null(), isolate_->has_pending_exception());
2869 }
2870
AsyncCompileSucceeded(Handle<Object> result)2871 void AsyncCompileJob::AsyncCompileSucceeded(Handle<Object> result) {
2872 MaybeHandle<Object> promise_result =
2873 JSPromise::Resolve(module_promise_, result);
2874 CHECK_EQ(promise_result.is_null(), isolate_->has_pending_exception());
2875 }
2876
2877 // A closure to run a compilation step (either as foreground or background
2878 // task) and schedule the next step(s), if any.
2879 class AsyncCompileJob::CompileStep {
2880 public:
CompileStep(int num_background_tasks=0)2881 explicit CompileStep(int num_background_tasks = 0)
2882 : num_background_tasks_(num_background_tasks) {}
2883
~CompileStep()2884 virtual ~CompileStep() {}
2885
Run(bool on_foreground)2886 void Run(bool on_foreground) {
2887 if (on_foreground) {
2888 HandleScope scope(job_->isolate_);
2889 --job_->num_pending_foreground_tasks_;
2890 DCHECK_EQ(0, job_->num_pending_foreground_tasks_);
2891 SaveContext saved_context(job_->isolate_);
2892 job_->isolate_->set_context(*job_->context_);
2893 RunInForeground();
2894 } else {
2895 RunInBackground();
2896 }
2897 }
2898
RunInForeground()2899 virtual void RunInForeground() { UNREACHABLE(); }
RunInBackground()2900 virtual void RunInBackground() { UNREACHABLE(); }
2901
NumberOfBackgroundTasks()2902 int NumberOfBackgroundTasks() { return num_background_tasks_; }
2903
2904 AsyncCompileJob* job_ = nullptr;
2905 const int num_background_tasks_;
2906 };
2907
2908 class AsyncCompileJob::CompileTask : public CancelableTask {
2909 public:
CompileTask(AsyncCompileJob * job,bool on_foreground)2910 CompileTask(AsyncCompileJob* job, bool on_foreground)
2911 // We only manage the background tasks with the {CancelableTaskManager} of
2912 // the {AsyncCompileJob}. Foreground tasks are managed by the system's
2913 // {CancelableTaskManager}. Background tasks cannot spawn tasks managed by
2914 // their own task manager.
2915 : CancelableTask(on_foreground ? job->isolate_->cancelable_task_manager()
2916 : &job->background_task_manager_),
2917 job_(job),
2918 on_foreground_(on_foreground) {}
2919
RunInternal()2920 void RunInternal() override { job_->step_->Run(on_foreground_); }
2921
2922 private:
2923 AsyncCompileJob* job_;
2924 bool on_foreground_;
2925 };
2926
StartForegroundTask()2927 void AsyncCompileJob::StartForegroundTask() {
2928 ++num_pending_foreground_tasks_;
2929 DCHECK_EQ(1, num_pending_foreground_tasks_);
2930
2931 foreground_task_runner_->PostTask(base::make_unique<CompileTask>(this, true));
2932 }
2933
2934 template <typename Step, typename... Args>
DoSync(Args &&...args)2935 void AsyncCompileJob::DoSync(Args&&... args) {
2936 NextStep<Step>(std::forward<Args>(args)...);
2937 StartForegroundTask();
2938 }
2939
StartBackgroundTask()2940 void AsyncCompileJob::StartBackgroundTask() {
2941 // If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
2942 // tasks. This is used to make timing deterministic.
2943 v8::TaskRunner* task_runner = FLAG_wasm_num_compilation_tasks > 0
2944 ? background_task_runner_.get()
2945 : foreground_task_runner_.get();
2946 task_runner->PostTask(base::make_unique<CompileTask>(this, false));
2947 }
2948
2949 template <typename Step, typename... Args>
DoAsync(Args &&...args)2950 void AsyncCompileJob::DoAsync(Args&&... args) {
2951 NextStep<Step>(std::forward<Args>(args)...);
2952 int end = step_->NumberOfBackgroundTasks();
2953 for (int i = 0; i < end; ++i) {
2954 StartBackgroundTask();
2955 }
2956 }
2957
2958 template <typename Step, typename... Args>
NextStep(Args &&...args)2959 void AsyncCompileJob::NextStep(Args&&... args) {
2960 step_.reset(new Step(std::forward<Args>(args)...));
2961 step_->job_ = this;
2962 }
2963
2964 //==========================================================================
2965 // Step 1: (async) Decode the module.
2966 //==========================================================================
2967 class AsyncCompileJob::DecodeModule : public AsyncCompileJob::CompileStep {
2968 public:
DecodeModule()2969 DecodeModule() : CompileStep(1) {}
2970
RunInBackground()2971 void RunInBackground() override {
2972 ModuleResult result;
2973 {
2974 DisallowHandleAllocation no_handle;
2975 DisallowHeapAllocation no_allocation;
2976 // Decode the module bytes.
2977 TRACE_COMPILE("(1) Decoding module...\n");
2978 result = AsyncDecodeWasmModule(job_->isolate_, job_->wire_bytes_.start(),
2979 job_->wire_bytes_.end(), false,
2980 kWasmOrigin, job_->async_counters());
2981 }
2982 if (result.failed()) {
2983 // Decoding failure; reject the promise and clean up.
2984 job_->DoSync<DecodeFail>(std::move(result));
2985 } else {
2986 // Decode passed.
2987 job_->module_ = std::move(result.val);
2988 job_->DoSync<PrepareAndStartCompile>(job_->module_.get(), true);
2989 }
2990 }
2991 };
2992
2993 //==========================================================================
2994 // Step 1b: (sync) Fail decoding the module.
2995 //==========================================================================
2996 class AsyncCompileJob::DecodeFail : public CompileStep {
2997 public:
DecodeFail(ModuleResult result)2998 explicit DecodeFail(ModuleResult result) : result_(std::move(result)) {}
2999
3000 private:
3001 ModuleResult result_;
RunInForeground()3002 void RunInForeground() override {
3003 TRACE_COMPILE("(1b) Decoding failed.\n");
3004 ErrorThrower thrower(job_->isolate_, "AsyncCompile");
3005 thrower.CompileFailed("Wasm decoding failed", result_);
3006 // {job_} is deleted in AsyncCompileFailed, therefore the {return}.
3007 return job_->AsyncCompileFailed(thrower.Reify());
3008 }
3009 };
3010
3011 //==========================================================================
3012 // Step 2 (sync): Create heap-allocated data and start compile.
3013 //==========================================================================
3014 class AsyncCompileJob::PrepareAndStartCompile : public CompileStep {
3015 public:
PrepareAndStartCompile(WasmModule * module,bool start_compilation)3016 explicit PrepareAndStartCompile(WasmModule* module, bool start_compilation)
3017 : module_(module), start_compilation_(start_compilation) {}
3018
3019 private:
3020 WasmModule* module_;
3021 bool start_compilation_;
3022
RunInForeground()3023 void RunInForeground() override {
3024 TRACE_COMPILE("(2) Prepare and start compile...\n");
3025
3026 // Make sure all compilation tasks stopped running. Decoding (async step)
3027 // is done.
3028 job_->background_task_manager_.CancelAndWait();
3029
3030 Isolate* isolate = job_->isolate_;
3031
3032 job_->centry_stub_ = CodeFactory::CEntry(isolate);
3033
3034 DCHECK_LE(module_->num_imported_functions, module_->functions.size());
3035 // Create the compiled module object and populate with compiled functions
3036 // and information needed at instantiation time. This object needs to be
3037 // serializable. Instantiation may occur off a deserialized version of
3038 // this object.
3039 ModuleEnv env = CreateDefaultModuleEnv(module_);
3040 job_->compiled_module_ = NewCompiledModule(job_->isolate_, module_, env);
3041
3042 {
3043 DeferredHandleScope deferred(job_->isolate_);
3044 job_->compiled_module_ = handle(*job_->compiled_module_, job_->isolate_);
3045 job_->deferred_handles_.push_back(deferred.Detach());
3046 }
3047 size_t num_functions =
3048 module_->functions.size() - module_->num_imported_functions;
3049
3050 if (num_functions == 0) {
3051 // Tiering has nothing to do if module is empty.
3052 job_->tiering_completed_ = true;
3053
3054 // Degenerate case of an empty module.
3055 job_->FinishCompile();
3056 return;
3057 }
3058
3059 CompilationState* compilation_state =
3060 job_->compiled_module_->GetNativeModule()->compilation_state();
3061 {
3062 // Instance field {job_} cannot be captured by copy, therefore
3063 // we need to add a local helper variable {job}. We want to
3064 // capture the {job} pointer by copy, as it otherwise is dependent
3065 // on the current step we are in.
3066 AsyncCompileJob* job = job_;
3067 compilation_state->AddCallback(
3068 [job](CompilationEvent event, ErrorThrower* thrower) {
3069 // Callback is called from a foreground thread.
3070 switch (event) {
3071 case CompilationEvent::kFinishedBaselineCompilation:
3072 if (job->DecrementAndCheckFinisherCount()) {
3073 SaveContext saved_context(job->isolate());
3074 // TODO(mstarzinger): Make {AsyncCompileJob::context} point
3075 // to the native context and also rename to {native_context}.
3076 job->isolate()->set_context(job->context_->native_context());
3077 job->FinishCompile();
3078 }
3079 return;
3080 case CompilationEvent::kFinishedTopTierCompilation:
3081 // It is only safe to schedule the UpdateToTopTierCompiledCode
3082 // step if no foreground task is currently pending, and no
3083 // finisher is outstanding (streaming compilation).
3084 if (job->num_pending_foreground_tasks_ == 0 &&
3085 job->outstanding_finishers_.Value() == 0) {
3086 job->DoSync<UpdateToTopTierCompiledCode>();
3087 }
3088 // If a foreground task was pending or a finsher was pending,
3089 // we will rely on FinishModule to switch the step to
3090 // UpdateToTopTierCompiledCode.
3091 job->tiering_completed_ = true;
3092 return;
3093 case CompilationEvent::kFailedCompilation: {
3094 // Tier-up compilation should not fail if baseline compilation
3095 // did not fail.
3096 DCHECK(!job->compiled_module_->GetNativeModule()
3097 ->compilation_state()
3098 ->baseline_compilation_finished());
3099
3100 SaveContext saved_context(job->isolate());
3101 job->isolate()->set_context(job->context_->native_context());
3102 Handle<Object> error = thrower->Reify();
3103
3104 DeferredHandleScope deferred(job->isolate());
3105 error = handle(*error, job->isolate());
3106 job->deferred_handles_.push_back(deferred.Detach());
3107 job->DoSync<CompileFailed>(error);
3108 return;
3109 }
3110 case CompilationEvent::kDestroyed:
3111 // Nothing to do.
3112 return;
3113 }
3114 UNREACHABLE();
3115 });
3116 }
3117 if (start_compilation_) {
3118 // TODO(ahaas): Try to remove the {start_compilation_} check when
3119 // streaming decoding is done in the background. If
3120 // InitializeCompilationUnits always returns 0 for streaming compilation,
3121 // then DoAsync would do the same as NextStep already.
3122
3123 size_t functions_count = GetNumFunctionsToCompile(env.module);
3124 compilation_state->SetNumberOfFunctionsToCompile(functions_count);
3125 // Add compilation units and kick off compilation.
3126 InitializeCompilationUnits(module_->functions, job_->wire_bytes_,
3127 env.module, job_->centry_stub_,
3128 job_->compiled_module_->GetNativeModule());
3129 }
3130 }
3131 };
3132
3133 //==========================================================================
3134 // Step 4b (sync): Compilation failed. Reject Promise.
3135 //==========================================================================
3136 class AsyncCompileJob::CompileFailed : public CompileStep {
3137 public:
CompileFailed(Handle<Object> error_reason)3138 explicit CompileFailed(Handle<Object> error_reason)
3139 : error_reason_(error_reason) {}
3140
RunInForeground()3141 void RunInForeground() override {
3142 TRACE_COMPILE("(4b) Compilation Failed...\n");
3143 return job_->AsyncCompileFailed(error_reason_);
3144 }
3145
3146 private:
3147 Handle<Object> error_reason_;
3148 };
3149
3150 //==========================================================================
3151 // Step 5 (sync): Compile JS->wasm wrappers.
3152 //==========================================================================
3153 class AsyncCompileJob::CompileWrappers : public CompileStep {
3154 // TODO(wasm): Compile all wrappers here, including the start function wrapper
3155 // and the wrappers for the function table elements.
RunInForeground()3156 void RunInForeground() override {
3157 TRACE_COMPILE("(5) Compile wrappers...\n");
3158 // TODO(6792): No longer needed once WebAssembly code is off heap.
3159 CodeSpaceMemoryModificationScope modification_scope(job_->isolate_->heap());
3160 // Compile JS->wasm wrappers for exported functions.
3161 CompileJsToWasmWrappers(job_->isolate_, job_->module_object_,
3162 job_->counters());
3163 job_->DoSync<FinishModule>();
3164 }
3165 };
3166
3167 //==========================================================================
3168 // Step 6 (sync): Finish the module and resolve the promise.
3169 //==========================================================================
3170 class AsyncCompileJob::FinishModule : public CompileStep {
RunInForeground()3171 void RunInForeground() override {
3172 TRACE_COMPILE("(6) Finish module...\n");
3173 job_->AsyncCompileSucceeded(job_->module_object_);
3174
3175 WasmModule* module = job_->module_object_->shared()->module();
3176 size_t num_functions =
3177 module->functions.size() - module->num_imported_functions;
3178 if (job_->compiled_module_->GetNativeModule()
3179 ->compilation_state()
3180 ->compile_mode() == CompileMode::kRegular ||
3181 num_functions == 0) {
3182 // If we do not tier up, the async compile job is done here and
3183 // can be deleted.
3184 job_->isolate_->wasm_engine()->RemoveCompileJob(job_);
3185 return;
3186 }
3187 // If background tiering compilation finished before we resolved the
3188 // promise, switch to patching now. Otherwise, patching will be scheduled
3189 // by a callback.
3190 DCHECK_EQ(CompileMode::kTiering, job_->compiled_module_->GetNativeModule()
3191 ->compilation_state()
3192 ->compile_mode());
3193 if (job_->tiering_completed_) {
3194 job_->DoSync<UpdateToTopTierCompiledCode>();
3195 }
3196 }
3197 };
3198
3199 //==========================================================================
3200 // Step 7 (sync): Update with top tier code.
3201 //==========================================================================
3202 class AsyncCompileJob::UpdateToTopTierCompiledCode : public CompileStep {
RunInForeground()3203 void RunInForeground() override {
3204 TRACE_COMPILE("(7) Update native module to use optimized code...\n");
3205
3206 UpdateAllCompiledModulesWithTopTierCode(job_->module_object_);
3207 job_->isolate_->wasm_engine()->RemoveCompileJob(job_);
3208 }
3209 };
3210
3211 class AsyncCompileJob::AbortCompilation : public CompileStep {
RunInForeground()3212 void RunInForeground() override {
3213 TRACE_COMPILE("Abort asynchronous compilation ...\n");
3214 job_->isolate_->wasm_engine()->RemoveCompileJob(job_);
3215 }
3216 };
3217
AsyncStreamingProcessor(AsyncCompileJob * job)3218 AsyncStreamingProcessor::AsyncStreamingProcessor(AsyncCompileJob* job)
3219 : job_(job), compilation_unit_builder_(nullptr) {}
3220
FinishAsyncCompileJobWithError(ResultBase error)3221 void AsyncStreamingProcessor::FinishAsyncCompileJobWithError(ResultBase error) {
3222 // Make sure all background tasks stopped executing before we change the state
3223 // of the AsyncCompileJob to DecodeFail.
3224 job_->background_task_manager_.CancelAndWait();
3225
3226 // Create a ModuleResult from the result we got as parameter. Since there was
3227 // no error, we don't have to provide a real wasm module to the ModuleResult.
3228 ModuleResult result(nullptr);
3229 result.MoveErrorFrom(error);
3230
3231 // Check if there is already a CompiledModule, in which case we have to clean
3232 // up the CompilationState as well.
3233 if (!job_->compiled_module_.is_null()) {
3234 job_->compiled_module_->GetNativeModule()->compilation_state()->Abort();
3235
3236 if (job_->num_pending_foreground_tasks_ == 0) {
3237 job_->DoSync<AsyncCompileJob::DecodeFail>(std::move(result));
3238 } else {
3239 job_->NextStep<AsyncCompileJob::DecodeFail>(std::move(result));
3240 }
3241
3242 // Clear the {compilation_unit_builder_} if it exists. This is needed
3243 // because there is a check in the destructor of the
3244 // {CompilationUnitBuilder} that it is empty.
3245 if (compilation_unit_builder_) compilation_unit_builder_->Clear();
3246 } else {
3247 job_->DoSync<AsyncCompileJob::DecodeFail>(std::move(result));
3248 }
3249 }
3250
3251 // Process the module header.
ProcessModuleHeader(Vector<const uint8_t> bytes,uint32_t offset)3252 bool AsyncStreamingProcessor::ProcessModuleHeader(Vector<const uint8_t> bytes,
3253 uint32_t offset) {
3254 TRACE_STREAMING("Process module header...\n");
3255 decoder_.StartDecoding(job_->isolate());
3256 decoder_.DecodeModuleHeader(bytes, offset);
3257 if (!decoder_.ok()) {
3258 FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false));
3259 return false;
3260 }
3261 return true;
3262 }
3263
3264 // Process all sections except for the code section.
ProcessSection(SectionCode section_code,Vector<const uint8_t> bytes,uint32_t offset)3265 bool AsyncStreamingProcessor::ProcessSection(SectionCode section_code,
3266 Vector<const uint8_t> bytes,
3267 uint32_t offset) {
3268 TRACE_STREAMING("Process section %d ...\n", section_code);
3269 if (compilation_unit_builder_) {
3270 // We reached a section after the code section, we do not need the
3271 // compilation_unit_builder_ anymore.
3272 CommitCompilationUnits();
3273 compilation_unit_builder_.reset();
3274 }
3275 if (section_code == SectionCode::kUnknownSectionCode) {
3276 Decoder decoder(bytes, offset);
3277 section_code = ModuleDecoder::IdentifyUnknownSection(
3278 decoder, bytes.start() + bytes.length());
3279 if (section_code == SectionCode::kUnknownSectionCode) {
3280 // Skip unknown sections that we do not know how to handle.
3281 return true;
3282 }
3283 // Remove the unknown section tag from the payload bytes.
3284 offset += decoder.position();
3285 bytes = bytes.SubVector(decoder.position(), bytes.size());
3286 }
3287 constexpr bool verify_functions = false;
3288 decoder_.DecodeSection(section_code, bytes, offset, verify_functions);
3289 if (!decoder_.ok()) {
3290 FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false));
3291 return false;
3292 }
3293 return true;
3294 }
3295
3296 // Start the code section.
ProcessCodeSectionHeader(size_t functions_count,uint32_t offset)3297 bool AsyncStreamingProcessor::ProcessCodeSectionHeader(size_t functions_count,
3298 uint32_t offset) {
3299 TRACE_STREAMING("Start the code section with %zu functions...\n",
3300 functions_count);
3301 if (!decoder_.CheckFunctionsCount(static_cast<uint32_t>(functions_count),
3302 offset)) {
3303 FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false));
3304 return false;
3305 }
3306 job_->NextStep<AsyncCompileJob::PrepareAndStartCompile>(decoder_.module(),
3307 false);
3308 // Execute the PrepareAndStartCompile step immediately and not in a separate
3309 // task. The step expects to be run on a separate foreground thread though, so
3310 // we to increment {num_pending_foreground_tasks_} to look like one.
3311 ++job_->num_pending_foreground_tasks_;
3312 DCHECK_EQ(1, job_->num_pending_foreground_tasks_);
3313 constexpr bool on_foreground = true;
3314 job_->step_->Run(on_foreground);
3315
3316 NativeModule* native_module = job_->compiled_module_->GetNativeModule();
3317 native_module->compilation_state()->SetNumberOfFunctionsToCompile(
3318 functions_count);
3319
3320 // Set outstanding_finishers_ to 2, because both the AsyncCompileJob and the
3321 // AsyncStreamingProcessor have to finish.
3322 job_->outstanding_finishers_.SetValue(2);
3323 compilation_unit_builder_.reset(
3324 new CompilationUnitBuilder(native_module, job_->centry_stub_));
3325 return true;
3326 }
3327
3328 // Process a function body.
ProcessFunctionBody(Vector<const uint8_t> bytes,uint32_t offset)3329 bool AsyncStreamingProcessor::ProcessFunctionBody(Vector<const uint8_t> bytes,
3330 uint32_t offset) {
3331 TRACE_STREAMING("Process function body %d ...\n", next_function_);
3332
3333 if (next_function_ >= FLAG_skip_compiling_wasm_funcs) {
3334 decoder_.DecodeFunctionBody(
3335 next_function_, static_cast<uint32_t>(bytes.length()), offset, false);
3336
3337 uint32_t index = next_function_ + decoder_.module()->num_imported_functions;
3338 const WasmFunction* func = &decoder_.module()->functions[index];
3339 WasmName name = {nullptr, 0};
3340 compilation_unit_builder_->AddUnit(func, offset, bytes, name);
3341 }
3342 ++next_function_;
3343 // This method always succeeds. The return value is necessary to comply with
3344 // the StreamingProcessor interface.
3345 return true;
3346 }
3347
CommitCompilationUnits()3348 void AsyncStreamingProcessor::CommitCompilationUnits() {
3349 DCHECK(compilation_unit_builder_);
3350 compilation_unit_builder_->Commit();
3351 }
3352
OnFinishedChunk()3353 void AsyncStreamingProcessor::OnFinishedChunk() {
3354 TRACE_STREAMING("FinishChunk...\n");
3355 if (compilation_unit_builder_) CommitCompilationUnits();
3356 }
3357
3358 // Finish the processing of the stream.
OnFinishedStream(std::unique_ptr<uint8_t[]> bytes,size_t length)3359 void AsyncStreamingProcessor::OnFinishedStream(std::unique_ptr<uint8_t[]> bytes,
3360 size_t length) {
3361 TRACE_STREAMING("Finish stream...\n");
3362 job_->bytes_copy_ = std::move(bytes);
3363 job_->wire_bytes_ = ModuleWireBytes(job_->bytes_copy_.get(),
3364 job_->bytes_copy_.get() + length);
3365 ModuleResult result = decoder_.FinishDecoding(false);
3366 DCHECK(result.ok());
3367 job_->module_ = std::move(result.val);
3368 if (job_->DecrementAndCheckFinisherCount()) {
3369 if (job_->compiled_module_.is_null()) {
3370 // We are processing a WebAssembly module without code section. We need to
3371 // prepare compilation first before we can finish it.
3372 // {PrepareAndStartCompile} will call {FinishCompile} by itself if there
3373 // is no code section.
3374 job_->DoSync<AsyncCompileJob::PrepareAndStartCompile>(job_->module_.get(),
3375 true);
3376 } else {
3377 job_->FinishCompile();
3378 }
3379 }
3380 }
3381
3382 // Report an error detected in the StreamingDecoder.
OnError(DecodeResult result)3383 void AsyncStreamingProcessor::OnError(DecodeResult result) {
3384 TRACE_STREAMING("Stream error...\n");
3385 FinishAsyncCompileJobWithError(std::move(result));
3386 }
3387
OnAbort()3388 void AsyncStreamingProcessor::OnAbort() {
3389 TRACE_STREAMING("Abort stream...\n");
3390 job_->Abort();
3391 }
3392
operator ()(CompilationState * compilation_state) const3393 void CompilationStateDeleter::operator()(
3394 CompilationState* compilation_state) const {
3395 delete compilation_state;
3396 }
3397
NewCompilationState(Isolate * isolate,ModuleEnv & env)3398 std::unique_ptr<CompilationState, CompilationStateDeleter> NewCompilationState(
3399 Isolate* isolate, ModuleEnv& env) {
3400 return std::unique_ptr<CompilationState, CompilationStateDeleter>(
3401 new CompilationState(isolate, env));
3402 }
3403
GetModuleEnv(CompilationState * compilation_state)3404 ModuleEnv* GetModuleEnv(CompilationState* compilation_state) {
3405 return compilation_state->module_env();
3406 }
3407
CompilationState(internal::Isolate * isolate,ModuleEnv & env)3408 CompilationState::CompilationState(internal::Isolate* isolate, ModuleEnv& env)
3409 : isolate_(isolate),
3410 module_env_(env),
3411 max_memory_(GetMaxUsableMemorySize(isolate) / 2),
3412 // TODO(clemensh): Fix fuzzers such that {env.module} is always non-null.
3413 compile_mode_(FLAG_wasm_tier_up && (!env.module || env.module->is_wasm())
3414 ? CompileMode::kTiering
3415 : CompileMode::kRegular),
3416 wire_bytes_(ModuleWireBytes(nullptr, nullptr)),
3417 max_background_tasks_(std::max(
3418 1, std::min(FLAG_wasm_num_compilation_tasks,
3419 V8::GetCurrentPlatform()->NumberOfWorkerThreads()))) {
3420 DCHECK_LT(0, max_memory_);
3421 v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate_);
3422 v8::Platform* platform = V8::GetCurrentPlatform();
3423 foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate);
3424 background_task_runner_ = platform->GetWorkerThreadsTaskRunner(v8_isolate);
3425
3426 // Register task manager for clean shutdown in case of an isolate shutdown.
3427 isolate_->wasm_engine()->Register(&background_task_manager_);
3428 isolate_->wasm_engine()->Register(&foreground_task_manager_);
3429 }
3430
~CompilationState()3431 CompilationState::~CompilationState() {
3432 CancelAndWait();
3433 foreground_task_manager_.CancelAndWait();
3434 isolate_->wasm_engine()->Unregister(&foreground_task_manager_);
3435 NotifyOnEvent(CompilationEvent::kDestroyed, nullptr);
3436 }
3437
SetNumberOfFunctionsToCompile(size_t num_functions)3438 void CompilationState::SetNumberOfFunctionsToCompile(size_t num_functions) {
3439 DCHECK(!failed());
3440 outstanding_units_ = num_functions;
3441
3442 if (compile_mode_ == CompileMode::kTiering) {
3443 outstanding_units_ += num_functions;
3444 num_tiering_units_ = num_functions;
3445 }
3446 }
3447
AddCallback(std::function<void (CompilationEvent,ErrorThrower *)> callback)3448 void CompilationState::AddCallback(
3449 std::function<void(CompilationEvent, ErrorThrower*)> callback) {
3450 callbacks_.push_back(callback);
3451 }
3452
AddCompilationUnits(std::vector<std::unique_ptr<WasmCompilationUnit>> & baseline_units,std::vector<std::unique_ptr<WasmCompilationUnit>> & tiering_units)3453 void CompilationState::AddCompilationUnits(
3454 std::vector<std::unique_ptr<WasmCompilationUnit>>& baseline_units,
3455 std::vector<std::unique_ptr<WasmCompilationUnit>>& tiering_units) {
3456 {
3457 base::LockGuard<base::Mutex> guard(&mutex_);
3458
3459 if (compile_mode_ == CompileMode::kTiering) {
3460 DCHECK_EQ(baseline_units.size(), tiering_units.size());
3461 DCHECK_EQ(tiering_units.back()->mode(),
3462 WasmCompilationUnit::CompilationMode::kTurbofan);
3463 tiering_compilation_units_.insert(
3464 tiering_compilation_units_.end(),
3465 std::make_move_iterator(tiering_units.begin()),
3466 std::make_move_iterator(tiering_units.end()));
3467 } else {
3468 DCHECK(tiering_compilation_units_.empty());
3469 }
3470
3471 baseline_compilation_units_.insert(
3472 baseline_compilation_units_.end(),
3473 std::make_move_iterator(baseline_units.begin()),
3474 std::make_move_iterator(baseline_units.end()));
3475 }
3476
3477 RestartBackgroundTasks();
3478 }
3479
3480 std::unique_ptr<WasmCompilationUnit>
GetNextCompilationUnit()3481 CompilationState::GetNextCompilationUnit() {
3482 base::LockGuard<base::Mutex> guard(&mutex_);
3483
3484 std::vector<std::unique_ptr<WasmCompilationUnit>>& units =
3485 baseline_compilation_units_.empty() ? tiering_compilation_units_
3486 : baseline_compilation_units_;
3487
3488 if (!units.empty()) {
3489 std::unique_ptr<WasmCompilationUnit> unit = std::move(units.back());
3490 units.pop_back();
3491 return unit;
3492 }
3493
3494 return std::unique_ptr<WasmCompilationUnit>();
3495 }
3496
GetNextExecutedUnit()3497 std::unique_ptr<WasmCompilationUnit> CompilationState::GetNextExecutedUnit() {
3498 base::LockGuard<base::Mutex> guard(&mutex_);
3499 std::vector<std::unique_ptr<WasmCompilationUnit>>& units = finish_units();
3500 if (units.empty()) return {};
3501 std::unique_ptr<WasmCompilationUnit> ret = std::move(units.back());
3502 units.pop_back();
3503 allocated_memory_ -= ret->memory_cost();
3504 return ret;
3505 }
3506
HasCompilationUnitToFinish()3507 bool CompilationState::HasCompilationUnitToFinish() {
3508 base::LockGuard<base::Mutex> guard(&mutex_);
3509 return !finish_units().empty();
3510 }
3511
OnError(ErrorThrower * thrower)3512 void CompilationState::OnError(ErrorThrower* thrower) {
3513 Abort();
3514 DCHECK(thrower->error());
3515 NotifyOnEvent(CompilationEvent::kFailedCompilation, thrower);
3516 }
3517
OnFinishedUnit()3518 void CompilationState::OnFinishedUnit() {
3519 DCHECK_GT(outstanding_units_, 0);
3520 --outstanding_units_;
3521
3522 if (outstanding_units_ == 0) {
3523 CancelAndWait();
3524 baseline_compilation_finished_ = true;
3525
3526 DCHECK(compile_mode_ == CompileMode::kRegular ||
3527 compile_mode_ == CompileMode::kTiering);
3528 NotifyOnEvent(compile_mode_ == CompileMode::kRegular
3529 ? CompilationEvent::kFinishedBaselineCompilation
3530 : CompilationEvent::kFinishedTopTierCompilation,
3531 nullptr);
3532
3533 } else if (outstanding_units_ == num_tiering_units_) {
3534 DCHECK_EQ(compile_mode_, CompileMode::kTiering);
3535 baseline_compilation_finished_ = true;
3536
3537 // TODO(wasm): For streaming compilation, we want to start top tier
3538 // compilation before all functions have been compiled with Liftoff, e.g.
3539 // in the case when all received functions have been compiled with Liftoff
3540 // and we are waiting for new functions to compile.
3541
3542 // If we are in {kRegular} mode, {num_tiering_units_} is 0, therefore
3543 // this case is already caught by the previous check.
3544 NotifyOnEvent(CompilationEvent::kFinishedBaselineCompilation, nullptr);
3545 RestartBackgroundTasks();
3546 }
3547 }
3548
ScheduleUnitForFinishing(std::unique_ptr<WasmCompilationUnit> unit,WasmCompilationUnit::CompilationMode mode)3549 void CompilationState::ScheduleUnitForFinishing(
3550 std::unique_ptr<WasmCompilationUnit> unit,
3551 WasmCompilationUnit::CompilationMode mode) {
3552 size_t cost = unit->memory_cost();
3553 base::LockGuard<base::Mutex> guard(&mutex_);
3554 if (compile_mode_ == CompileMode::kTiering &&
3555 mode == WasmCompilationUnit::CompilationMode::kTurbofan) {
3556 tiering_finish_units_.push_back(std::move(unit));
3557 } else {
3558 baseline_finish_units_.push_back(std::move(unit));
3559 }
3560 allocated_memory_ += cost;
3561
3562 if (!finisher_is_running_ && !failed_) {
3563 ScheduleFinisherTask();
3564 // We set the flag here so that not more than one finisher is started.
3565 finisher_is_running_ = true;
3566 }
3567 }
3568
CancelAndWait()3569 void CompilationState::CancelAndWait() {
3570 background_task_manager_.CancelAndWait();
3571 isolate_->wasm_engine()->Unregister(&background_task_manager_);
3572 }
3573
OnBackgroundTaskStopped()3574 void CompilationState::OnBackgroundTaskStopped() {
3575 base::LockGuard<base::Mutex> guard(&mutex_);
3576 DCHECK_LE(1, num_background_tasks_);
3577 --num_background_tasks_;
3578 }
3579
RestartBackgroundTasks(size_t max)3580 void CompilationState::RestartBackgroundTasks(size_t max) {
3581 size_t num_restart;
3582 {
3583 base::LockGuard<base::Mutex> guard(&mutex_);
3584 // No need to restart tasks if compilation already failed.
3585 if (failed_) return;
3586
3587 bool should_increase_workload = allocated_memory_ <= max_memory_ / 2;
3588 if (!should_increase_workload) return;
3589 DCHECK_LE(num_background_tasks_, max_background_tasks_);
3590 if (num_background_tasks_ == max_background_tasks_) return;
3591 size_t num_compilation_units =
3592 baseline_compilation_units_.size() + tiering_compilation_units_.size();
3593 size_t stopped_tasks = max_background_tasks_ - num_background_tasks_;
3594 num_restart = std::min(max, std::min(num_compilation_units, stopped_tasks));
3595 num_background_tasks_ += num_restart;
3596 }
3597
3598 // If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
3599 // tasks. This is used to make timing deterministic.
3600 v8::TaskRunner* task_runner = FLAG_wasm_num_compilation_tasks > 0
3601 ? background_task_runner_.get()
3602 : foreground_task_runner_.get();
3603 for (; num_restart > 0; --num_restart) {
3604 task_runner->PostTask(base::make_unique<BackgroundCompileTask>(
3605 this, &background_task_manager_));
3606 }
3607 }
3608
SetFinisherIsRunning(bool value)3609 bool CompilationState::SetFinisherIsRunning(bool value) {
3610 base::LockGuard<base::Mutex> guard(&mutex_);
3611 if (finisher_is_running_ == value) return false;
3612 finisher_is_running_ = value;
3613 return true;
3614 }
3615
ScheduleFinisherTask()3616 void CompilationState::ScheduleFinisherTask() {
3617 foreground_task_runner_->PostTask(
3618 base::make_unique<FinishCompileTask>(this, &foreground_task_manager_));
3619 }
3620
StopBackgroundCompilationTaskForThrottling()3621 bool CompilationState::StopBackgroundCompilationTaskForThrottling() {
3622 base::LockGuard<base::Mutex> guard(&mutex_);
3623 DCHECK_LE(1, num_background_tasks_);
3624 bool can_accept_work = allocated_memory_ < max_memory_;
3625 if (can_accept_work) return false;
3626 --num_background_tasks_;
3627 return true;
3628 }
3629
Abort()3630 void CompilationState::Abort() {
3631 {
3632 base::LockGuard<base::Mutex> guard(&mutex_);
3633 failed_ = true;
3634 }
3635 CancelAndWait();
3636 }
3637
NotifyOnEvent(CompilationEvent event,ErrorThrower * thrower)3638 void CompilationState::NotifyOnEvent(CompilationEvent event,
3639 ErrorThrower* thrower) {
3640 for (auto& callback_function : callbacks_) {
3641 callback_function(event, thrower);
3642 }
3643 }
3644
CompileJsToWasmWrappers(Isolate * isolate,Handle<WasmModuleObject> module_object,Counters * counters)3645 void CompileJsToWasmWrappers(Isolate* isolate,
3646 Handle<WasmModuleObject> module_object,
3647 Counters* counters) {
3648 JSToWasmWrapperCache js_to_wasm_cache;
3649 int wrapper_index = 0;
3650 Handle<FixedArray> export_wrappers(module_object->export_wrappers(), isolate);
3651 NativeModule* native_module =
3652 module_object->compiled_module()->GetNativeModule();
3653 wasm::UseTrapHandler use_trap_handler =
3654 native_module->use_trap_handler() ? kUseTrapHandler : kNoTrapHandler;
3655 WasmModule* module = native_module->shared_module_data()->module();
3656 for (auto exp : module->export_table) {
3657 if (exp.kind != kExternalFunction) continue;
3658 Address call_target =
3659 exp.index < module->num_imported_functions
3660 ? kNullAddress
3661 : native_module->GetCallTargetForFunction(exp.index);
3662 Handle<Code> wrapper_code = js_to_wasm_cache.CloneOrCompileJSToWasmWrapper(
3663 isolate, module, call_target, exp.index, use_trap_handler);
3664 export_wrappers->set(wrapper_index, *wrapper_code);
3665 RecordStats(*wrapper_code, counters);
3666 ++wrapper_index;
3667 }
3668 }
3669
CreateWasmScript(Isolate * isolate,const ModuleWireBytes & wire_bytes)3670 Handle<Script> CreateWasmScript(Isolate* isolate,
3671 const ModuleWireBytes& wire_bytes) {
3672 Handle<Script> script =
3673 isolate->factory()->NewScript(isolate->factory()->empty_string());
3674 script->set_context_data(isolate->native_context()->debug_context_id());
3675 script->set_type(Script::TYPE_WASM);
3676
3677 int hash = StringHasher::HashSequentialString(
3678 reinterpret_cast<const char*>(wire_bytes.start()),
3679 static_cast<int>(wire_bytes.length()), kZeroHashSeed);
3680
3681 const int kBufferSize = 32;
3682 char buffer[kBufferSize];
3683 int url_chars = SNPrintF(ArrayVector(buffer), "wasm://wasm/%08x", hash);
3684 DCHECK(url_chars >= 0 && url_chars < kBufferSize);
3685 MaybeHandle<String> url_str = isolate->factory()->NewStringFromOneByte(
3686 Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), url_chars),
3687 TENURED);
3688 script->set_source_url(*url_str.ToHandleChecked());
3689
3690 int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash);
3691 DCHECK(name_chars >= 0 && name_chars < kBufferSize);
3692 MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte(
3693 Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), name_chars),
3694 TENURED);
3695 script->set_name(*name_str.ToHandleChecked());
3696
3697 return script;
3698 }
3699
3700 } // namespace wasm
3701 } // namespace internal
3702 } // namespace v8
3703
3704 #undef TRACE
3705 #undef TRACE_CHAIN
3706 #undef TRACE_COMPILE
3707 #undef TRACE_STREAMING
3708 #undef TRACE_LAZY
3709