Line data Source code
1 : // Copyright 2017 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/wasm/module-compiler.h"
6 :
7 : #include "src/api.h"
8 : #include "src/asmjs/asm-js.h"
9 : #include "src/base/enum-set.h"
10 : #include "src/base/optional.h"
11 : #include "src/base/template-utils.h"
12 : #include "src/base/utils/random-number-generator.h"
13 : #include "src/compiler/wasm-compiler.h"
14 : #include "src/counters.h"
15 : #include "src/heap/heap-inl.h" // For CodeSpaceMemoryModificationScope.
16 : #include "src/identity-map.h"
17 : #include "src/property-descriptor.h"
18 : #include "src/task-utils.h"
19 : #include "src/tracing/trace-event.h"
20 : #include "src/trap-handler/trap-handler.h"
21 : #include "src/wasm/js-to-wasm-wrapper-cache-inl.h"
22 : #include "src/wasm/module-decoder.h"
23 : #include "src/wasm/streaming-decoder.h"
24 : #include "src/wasm/wasm-code-manager.h"
25 : #include "src/wasm/wasm-engine.h"
26 : #include "src/wasm/wasm-import-wrapper-cache-inl.h"
27 : #include "src/wasm/wasm-js.h"
28 : #include "src/wasm/wasm-limits.h"
29 : #include "src/wasm/wasm-memory.h"
30 : #include "src/wasm/wasm-objects-inl.h"
31 : #include "src/wasm/wasm-result.h"
32 : #include "src/wasm/wasm-serialization.h"
33 :
34 : #define TRACE_COMPILE(...) \
35 : do { \
36 : if (FLAG_trace_wasm_compiler) PrintF(__VA_ARGS__); \
37 : } while (false)
38 :
39 : #define TRACE_STREAMING(...) \
40 : do { \
41 : if (FLAG_trace_wasm_streaming) PrintF(__VA_ARGS__); \
42 : } while (false)
43 :
44 : #define TRACE_LAZY(...) \
45 : do { \
46 : if (FLAG_trace_wasm_lazy_compilation) PrintF(__VA_ARGS__); \
47 : } while (false)
48 :
49 : namespace v8 {
50 : namespace internal {
51 : namespace wasm {
52 :
53 : namespace {
54 :
55 : enum class CompileMode : uint8_t { kRegular, kTiering };
56 :
57 : // Background compile jobs hold a shared pointer to this token. The token is
58 : // used to notify them that they should stop. As soon as they see this (after
59 : // finishing their current compilation unit), they will stop.
60 : // This allows to already remove the NativeModule without having to synchronize
61 : // on background compile jobs.
62 2483804 : class BackgroundCompileToken {
63 : public:
64 : explicit BackgroundCompileToken(
65 : const std::shared_ptr<NativeModule>& native_module)
66 1241902 : : native_module_(native_module) {}
67 :
68 1242555 : void Cancel() {
69 1242555 : base::SharedMutexGuard<base::kExclusive> mutex_guard(&mutex_);
70 : native_module_.reset();
71 1242555 : }
72 :
73 : private:
74 : friend class BackgroundCompileScope;
75 : base::SharedMutex mutex_;
76 : std::weak_ptr<NativeModule> native_module_;
77 :
78 : std::shared_ptr<NativeModule> StartScope() {
79 226576 : mutex_.LockShared();
80 : return native_module_.lock();
81 : }
82 :
83 230797 : void ExitScope() { mutex_.UnlockShared(); }
84 : };
85 :
86 : class CompilationStateImpl;
87 :
88 : // Keep these scopes short, as they hold the mutex of the token, which
89 : // sequentializes all these scopes. The mutex is also acquired from foreground
90 : // tasks, which should not be blocked for a long time.
91 : class BackgroundCompileScope {
92 : public:
93 226576 : explicit BackgroundCompileScope(
94 : const std::shared_ptr<BackgroundCompileToken>& token)
95 457436 : : token_(token.get()), native_module_(token->StartScope()) {}
96 :
97 461551 : ~BackgroundCompileScope() { token_->ExitScope(); }
98 :
99 : bool cancelled() const { return native_module_ == nullptr; }
100 :
101 : NativeModule* native_module() {
102 : DCHECK(!cancelled());
103 : return native_module_.get();
104 : }
105 :
106 : inline CompilationStateImpl* compilation_state();
107 :
108 : private:
109 : BackgroundCompileToken* const token_;
110 : // Keep the native module alive while in this scope.
111 : std::shared_ptr<NativeModule> const native_module_;
112 : };
113 :
114 : // The {CompilationStateImpl} keeps track of the compilation state of the
115 : // owning NativeModule, i.e. which functions are left to be compiled.
116 : // It contains a task manager to allow parallel and asynchronous background
117 : // compilation of functions.
118 : // Its public interface {CompilationState} lives in compilation-environment.h.
119 3725706 : class CompilationStateImpl {
120 : public:
121 : CompilationStateImpl(const std::shared_ptr<NativeModule>& native_module,
122 : std::shared_ptr<Counters> async_counters);
123 :
124 : // Cancel all background compilation and wait for all tasks to finish. Call
125 : // this before destructing this object.
126 : void AbortCompilation();
127 :
128 : // Set the number of compilations unit expected to be executed. Needs to be
129 : // set before {AddCompilationUnits} is run, which triggers background
130 : // compilation.
131 : void SetNumberOfFunctionsToCompile(int num_functions);
132 :
133 : // Add the callback function to be called on compilation events. Needs to be
134 : // set before {AddCompilationUnits} is run to ensure that it receives all
135 : // events. The callback object must support being deleted from any thread.
136 : void AddCallback(CompilationState::callback_t);
137 :
138 : // Inserts new functions to compile and kicks off compilation.
139 : void AddCompilationUnits(
140 : std::vector<std::unique_ptr<WasmCompilationUnit>>& baseline_units,
141 : std::vector<std::unique_ptr<WasmCompilationUnit>>& tiering_units);
142 : std::unique_ptr<WasmCompilationUnit> GetNextCompilationUnit();
143 :
144 : void OnFinishedUnit(ExecutionTier, WasmCode*);
145 :
146 : void ReportDetectedFeatures(const WasmFeatures& detected);
147 : void OnBackgroundTaskStopped(const WasmFeatures& detected);
148 : void PublishDetectedFeatures(Isolate* isolate, const WasmFeatures& detected);
149 : void RestartBackgroundCompileTask();
150 : void RestartBackgroundTasks();
151 :
152 : void SetError();
153 :
154 : bool failed() const {
155 : return compile_failed_.load(std::memory_order_relaxed);
156 : }
157 :
158 5397613 : bool baseline_compilation_finished() const {
159 5397613 : base::MutexGuard guard(&callbacks_mutex_);
160 10780306 : return outstanding_baseline_units_ == 0 ||
161 10687399 : (compile_mode_ == CompileMode::kTiering &&
162 10702319 : outstanding_tiering_units_ == 0);
163 : }
164 :
165 : CompileMode compile_mode() const { return compile_mode_; }
166 10272 : WasmFeatures* detected_features() { return &detected_features_; }
167 :
168 2339109 : void SetWireBytesStorage(
169 : std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
170 2339109 : base::MutexGuard guard(&mutex_);
171 : wire_bytes_storage_ = wire_bytes_storage;
172 2339109 : }
173 :
174 : std::shared_ptr<WireBytesStorage> GetWireBytesStorage() const {
175 909599 : base::MutexGuard guard(&mutex_);
176 : DCHECK_NOT_NULL(wire_bytes_storage_);
177 : return wire_bytes_storage_;
178 : }
179 :
180 : private:
181 : NativeModule* const native_module_;
182 : const std::shared_ptr<BackgroundCompileToken> background_compile_token_;
183 : const CompileMode compile_mode_;
184 : const std::shared_ptr<Counters> async_counters_;
185 :
186 : // Compilation error, atomically updated. This flag can be updated and read
187 : // using relaxed semantics.
188 : std::atomic<bool> compile_failed_{false};
189 :
190 : // This mutex protects all information of this {CompilationStateImpl} which is
191 : // being accessed concurrently.
192 : mutable base::Mutex mutex_;
193 :
194 : //////////////////////////////////////////////////////////////////////////////
195 : // Protected by {mutex_}:
196 :
197 : std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_compilation_units_;
198 : std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_compilation_units_;
199 :
200 : int num_background_tasks_ = 0;
201 :
202 : // Features detected to be used in this module. Features can be detected
203 : // as a module is being compiled.
204 : WasmFeatures detected_features_ = kNoWasmFeatures;
205 :
206 : // Abstraction over the storage of the wire bytes. Held in a shared_ptr so
207 : // that background compilation jobs can keep the storage alive while
208 : // compiling.
209 : std::shared_ptr<WireBytesStorage> wire_bytes_storage_;
210 :
211 : // End of fields protected by {mutex_}.
212 : //////////////////////////////////////////////////////////////////////////////
213 :
214 : // This mutex protects the callbacks vector, and the counters used to
215 : // determine which callbacks to call. The counters plus the callbacks
216 : // themselves need to be synchronized to ensure correct order of events.
217 : mutable base::Mutex callbacks_mutex_;
218 :
219 : //////////////////////////////////////////////////////////////////////////////
220 : // Protected by {callbacks_mutex_}:
221 :
222 : // Callback functions to be called on compilation events.
223 : std::vector<CompilationState::callback_t> callbacks_;
224 :
225 : int outstanding_baseline_units_ = 0;
226 : int outstanding_tiering_units_ = 0;
227 :
228 : // End of fields protected by {callbacks_mutex_}.
229 : //////////////////////////////////////////////////////////////////////////////
230 :
231 : const int max_background_tasks_ = 0;
232 : };
233 :
234 : CompilationStateImpl* Impl(CompilationState* compilation_state) {
235 : return reinterpret_cast<CompilationStateImpl*>(compilation_state);
236 : }
237 : const CompilationStateImpl* Impl(const CompilationState* compilation_state) {
238 : return reinterpret_cast<const CompilationStateImpl*>(compilation_state);
239 : }
240 :
241 : CompilationStateImpl* BackgroundCompileScope::compilation_state() {
242 : return Impl(native_module()->compilation_state());
243 : }
244 :
245 : void UpdateFeatureUseCounts(Isolate* isolate, const WasmFeatures& detected) {
246 144223 : if (detected.threads) {
247 1444 : isolate->CountUsage(v8::Isolate::UseCounterFeature::kWasmThreadOpcodes);
248 : }
249 : }
250 :
251 : } // namespace
252 :
253 : //////////////////////////////////////////////////////
254 : // PIMPL implementation of {CompilationState}.
255 :
256 1241902 : CompilationState::~CompilationState() { Impl(this)->~CompilationStateImpl(); }
257 :
258 1241901 : void CompilationState::AbortCompilation() { Impl(this)->AbortCompilation(); }
259 :
260 7129 : void CompilationState::SetError() { Impl(this)->SetError(); }
261 :
262 2339109 : void CompilationState::SetWireBytesStorage(
263 : std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
264 4678218 : Impl(this)->SetWireBytesStorage(std::move(wire_bytes_storage));
265 2339109 : }
266 :
267 863017 : std::shared_ptr<WireBytesStorage> CompilationState::GetWireBytesStorage()
268 : const {
269 863017 : return Impl(this)->GetWireBytesStorage();
270 : }
271 :
272 1263 : void CompilationState::AddCallback(CompilationState::callback_t callback) {
273 2526 : return Impl(this)->AddCallback(std::move(callback));
274 : }
275 :
276 32 : bool CompilationState::failed() const { return Impl(this)->failed(); }
277 :
278 0 : void CompilationState::OnFinishedUnit(ExecutionTier tier, WasmCode* code) {
279 0 : Impl(this)->OnFinishedUnit(tier, code);
280 0 : }
281 :
282 : // static
283 1241902 : std::unique_ptr<CompilationState> CompilationState::New(
284 : const std::shared_ptr<NativeModule>& native_module,
285 : std::shared_ptr<Counters> async_counters) {
286 : return std::unique_ptr<CompilationState>(reinterpret_cast<CompilationState*>(
287 2483804 : new CompilationStateImpl(native_module, std::move(async_counters))));
288 : }
289 :
290 : // End of PIMPL implementation of {CompilationState}.
291 : //////////////////////////////////////////////////////
292 :
293 8062 : void CompileLazy(Isolate* isolate, NativeModule* native_module,
294 : uint32_t func_index) {
295 : Counters* counters = isolate->counters();
296 : HistogramTimerScope lazy_time_scope(counters->wasm_lazy_compilation_time());
297 :
298 : DCHECK(!native_module->lazy_compile_frozen());
299 :
300 : base::ElapsedTimer compilation_timer;
301 :
302 16124 : NativeModuleModificationScope native_module_modification_scope(native_module);
303 :
304 : DCHECK(!native_module->has_code(static_cast<uint32_t>(func_index)));
305 :
306 : compilation_timer.Start();
307 :
308 : TRACE_LAZY("Compiling wasm-function#%d.\n", func_index);
309 :
310 : const uint8_t* module_start = native_module->wire_bytes().start();
311 :
312 8062 : const WasmFunction* func = &native_module->module()->functions[func_index];
313 : FunctionBody func_body{func->sig, func->code.offset(),
314 : module_start + func->code.offset(),
315 : module_start + func->code.end_offset()};
316 :
317 : ExecutionTier tier =
318 8062 : WasmCompilationUnit::GetDefaultExecutionTier(native_module->module());
319 16124 : WasmCompilationUnit unit(isolate->wasm_engine(), func_index, tier);
320 8062 : CompilationEnv env = native_module->CreateCompilationEnv();
321 : WasmCompilationResult result = unit.ExecuteCompilation(
322 8062 : &env, native_module->compilation_state()->GetWireBytesStorage(),
323 : isolate->counters(),
324 16124 : Impl(native_module->compilation_state())->detected_features());
325 8062 : WasmCode* code = native_module->AddCompiledCode(std::move(result));
326 :
327 : // During lazy compilation, we should never get compilation errors. The module
328 : // was verified before starting execution with lazy compilation.
329 : // This might be OOM, but then we cannot continue execution anyway.
330 : // TODO(clemensh): According to the spec, we can actually skip validation at
331 : // module creation time, and return a function that always traps here.
332 8062 : CHECK(!native_module->compilation_state()->failed());
333 :
334 : // The code we just produced should be the one that was requested.
335 : DCHECK_EQ(func_index, code->index());
336 :
337 8062 : if (WasmCode::ShouldBeLogged(isolate)) code->LogCode(isolate);
338 :
339 : int64_t func_size =
340 8062 : static_cast<int64_t>(func->code.end_offset() - func->code.offset());
341 8062 : int64_t compilation_time = compilation_timer.Elapsed().InMicroseconds();
342 :
343 8062 : counters->wasm_lazily_compiled_functions()->Increment();
344 :
345 16124 : counters->wasm_lazy_compilation_throughput()->AddSample(
346 8062 : compilation_time != 0 ? static_cast<int>(func_size / compilation_time)
347 8062 : : 0);
348 8062 : }
349 :
350 : namespace {
351 :
352 96 : ExecutionTier apply_hint_to_execution_tier(WasmCompilationHintTier hint,
353 : ExecutionTier default_tier) {
354 96 : switch (hint) {
355 : case WasmCompilationHintTier::kDefault:
356 64 : return default_tier;
357 : case WasmCompilationHintTier::kInterpreter:
358 : return ExecutionTier::kInterpreter;
359 : case WasmCompilationHintTier::kBaseline:
360 8 : return ExecutionTier::kBaseline;
361 : case WasmCompilationHintTier::kOptimized:
362 8 : return ExecutionTier::kOptimized;
363 : }
364 0 : UNREACHABLE();
365 : }
366 :
367 : // The {CompilationUnitBuilder} builds compilation units and stores them in an
368 : // internal buffer. The buffer is moved into the working queue of the
369 : // {CompilationStateImpl} when {Commit} is called.
370 6617 : class CompilationUnitBuilder {
371 : public:
372 : explicit CompilationUnitBuilder(NativeModule* native_module,
373 : WasmEngine* wasm_engine)
374 : : native_module_(native_module),
375 : wasm_engine_(wasm_engine),
376 6617 : default_tier_(WasmCompilationUnit::GetDefaultExecutionTier(
377 13234 : native_module->module())) {}
378 :
379 103513 : void AddUnit(uint32_t func_index) {
380 103513 : switch (compilation_state()->compile_mode()) {
381 : case CompileMode::kRegular:
382 744 : baseline_units_.emplace_back(CreateUnit(func_index, default_tier_));
383 372 : return;
384 : case CompileMode::kTiering:
385 :
386 : // Default tiering behaviour.
387 : ExecutionTier first_tier = ExecutionTier::kBaseline;
388 : ExecutionTier second_tier = ExecutionTier::kOptimized;
389 :
390 : // Check if compilation hints override default tiering behaviour.
391 103141 : if (native_module_->enabled_features().compilation_hints) {
392 : // Find compilation hint.
393 252 : CHECK_LE(native_module_->num_imported_functions(), func_index);
394 : uint32_t hint_index =
395 252 : func_index - native_module_->num_imported_functions();
396 : const std::vector<WasmCompilationHint>& compilation_hints =
397 : native_module_->module()->compilation_hints;
398 504 : if (hint_index < compilation_hints.size()) {
399 48 : WasmCompilationHint hint = compilation_hints[hint_index];
400 :
401 : // Apply compilation hint.
402 : first_tier =
403 48 : apply_hint_to_execution_tier(hint.first_tier, first_tier);
404 : second_tier =
405 48 : apply_hint_to_execution_tier(hint.second_tier, second_tier);
406 : }
407 : }
408 :
409 : // Create compilation units and suppress duplicate compilation.
410 103141 : baseline_units_.emplace_back(
411 103141 : CreateUnit(func_index, std::move(first_tier)));
412 : static_assert(ExecutionTier::kInterpreter < ExecutionTier::kBaseline &&
413 : ExecutionTier::kBaseline < ExecutionTier::kOptimized,
414 : "Assume an order on execution tiers");
415 103141 : if (first_tier < second_tier) {
416 103133 : tiering_units_.emplace_back(
417 103133 : CreateUnit(func_index, std::move(second_tier)));
418 : }
419 : return;
420 : }
421 0 : UNREACHABLE();
422 : }
423 :
424 6693 : bool Commit() {
425 6693 : if (baseline_units_.empty() && tiering_units_.empty()) return false;
426 13258 : compilation_state()->AddCompilationUnits(baseline_units_, tiering_units_);
427 6629 : Clear();
428 6629 : return true;
429 : }
430 :
431 6681 : void Clear() {
432 : baseline_units_.clear();
433 : tiering_units_.clear();
434 6681 : }
435 :
436 : private:
437 : std::unique_ptr<WasmCompilationUnit> CreateUnit(uint32_t func_index,
438 : ExecutionTier tier) {
439 : return base::make_unique<WasmCompilationUnit>(wasm_engine_, func_index,
440 206646 : tier);
441 : }
442 :
443 : CompilationStateImpl* compilation_state() const {
444 : return Impl(native_module_->compilation_state());
445 : }
446 :
447 : NativeModule* const native_module_;
448 : WasmEngine* const wasm_engine_;
449 : const ExecutionTier default_tier_;
450 : std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_units_;
451 : std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_units_;
452 : };
453 :
454 : bool compile_lazy(const WasmModule* module) {
455 144648 : return FLAG_wasm_lazy_compilation ||
456 144160 : (FLAG_asm_wasm_lazy_compilation && module->origin == kAsmJsOrigin);
457 : }
458 :
459 220385 : void RecordStats(const Code code, Counters* counters) {
460 440770 : counters->wasm_generated_code_size()->Increment(code->body_size());
461 220385 : counters->wasm_reloc_size()->Increment(code->relocation_info()->length());
462 220386 : }
463 :
464 229988 : double MonotonicallyIncreasingTimeInMs() {
465 229988 : return V8::GetCurrentPlatform()->MonotonicallyIncreasingTime() *
466 229920 : base::Time::kMillisecondsPerSecond;
467 : }
468 :
469 : // Run by each compilation task and by the main thread (i.e. in both
470 : // foreground and background threads).
471 5359929 : bool FetchAndExecuteCompilationUnit(CompilationEnv* env,
472 : NativeModule* native_module,
473 : CompilationStateImpl* compilation_state,
474 : WasmFeatures* detected,
475 : Counters* counters) {
476 : DisallowHeapAccess no_heap_access;
477 :
478 : std::unique_ptr<WasmCompilationUnit> unit =
479 10719858 : compilation_state->GetNextCompilationUnit();
480 5359929 : if (unit == nullptr) return false;
481 :
482 : WasmCompilationResult result = unit->ExecuteCompilation(
483 20718 : env, compilation_state->GetWireBytesStorage(), counters, detected);
484 :
485 6906 : if (result.succeeded()) {
486 6858 : WasmCode* code = native_module->AddCompiledCode(std::move(result));
487 6858 : compilation_state->OnFinishedUnit(result.requested_tier, code);
488 : } else {
489 48 : compilation_state->SetError();
490 : }
491 :
492 : return true;
493 : }
494 :
495 6401 : void InitializeCompilationUnits(NativeModule* native_module,
496 : WasmEngine* wasm_engine) {
497 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
498 : const WasmModule* module = native_module->module();
499 : CompilationUnitBuilder builder(native_module, wasm_engine);
500 6401 : uint32_t start = module->num_imported_functions;
501 6401 : uint32_t end = start + module->num_declared_functions;
502 212523 : for (uint32_t i = start; i < end; ++i) {
503 103061 : builder.AddUnit(i);
504 : }
505 6401 : builder.Commit();
506 6401 : }
507 :
508 5162 : void CompileInParallel(Isolate* isolate, NativeModule* native_module) {
509 : // Data structures for the parallel compilation.
510 :
511 : //-----------------------------------------------------------------------
512 : // For parallel compilation:
513 : // 1) The main thread allocates a compilation unit for each wasm function
514 : // and stores them in the vector {compilation_units} within the
515 : // {compilation_state}. By adding units to the {compilation_state}, new
516 : // {BackgroundCompileTasks} instances are spawned which run on
517 : // the background threads.
518 : // 2) The background threads and the main thread pick one compilation unit at
519 : // a time and execute the parallel phase of the compilation unit.
520 :
521 : // Turn on the {CanonicalHandleScope} so that the background threads can
522 : // use the node cache.
523 10324 : CanonicalHandleScope canonical(isolate);
524 :
525 : CompilationStateImpl* compilation_state =
526 : Impl(native_module->compilation_state());
527 : DCHECK_GE(kMaxInt, native_module->module()->num_declared_functions);
528 : int num_wasm_functions =
529 5162 : static_cast<int>(native_module->module()->num_declared_functions);
530 5162 : compilation_state->SetNumberOfFunctionsToCompile(num_wasm_functions);
531 :
532 : // 1) The main thread allocates a compilation unit for each wasm function
533 : // and stores them in the vector {compilation_units} within the
534 : // {compilation_state}. By adding units to the {compilation_state}, new
535 : // {BackgroundCompileTask} instances are spawned which run on
536 : // background threads.
537 5162 : InitializeCompilationUnits(native_module, isolate->wasm_engine());
538 :
539 : // 2) The background threads and the main thread pick one compilation unit at
540 : // a time and execute the parallel phase of the compilation unit.
541 5162 : WasmFeatures detected_features;
542 5162 : CompilationEnv env = native_module->CreateCompilationEnv();
543 : // TODO(wasm): This might already execute TurboFan units on the main thread,
544 : // while waiting for baseline compilation to finish. This can introduce
545 : // additional delay.
546 : // TODO(wasm): This is a busy-wait loop once all units have started executing
547 : // in background threads. Replace by a semaphore / barrier.
548 16089735 : while (!compilation_state->failed() &&
549 5364715 : !compilation_state->baseline_compilation_finished()) {
550 : FetchAndExecuteCompilationUnit(&env, native_module, compilation_state,
551 5359929 : &detected_features, isolate->counters());
552 : }
553 :
554 : // Publish features from the foreground and background tasks.
555 5162 : compilation_state->PublishDetectedFeatures(isolate, detected_features);
556 5162 : }
557 :
558 136857 : void CompileSequentially(Isolate* isolate, NativeModule* native_module) {
559 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
560 : const WasmModule* module = native_module->module();
561 136857 : WasmFeatures detected = kNoWasmFeatures;
562 : auto* comp_state = Impl(native_module->compilation_state());
563 : ExecutionTier tier =
564 136857 : WasmCompilationUnit::GetDefaultExecutionTier(native_module->module());
565 372554 : for (const WasmFunction& func : module->functions) {
566 242829 : if (func.imported) continue; // Imports are compiled at instantiation time.
567 :
568 : // Compile the function.
569 : WasmCompilationUnit::CompileWasmFunction(isolate, native_module, &detected,
570 131469 : &func, tier);
571 131463 : if (comp_state->failed()) break;
572 : }
573 : UpdateFeatureUseCounts(isolate, detected);
574 136854 : }
575 :
576 7757 : void ValidateSequentially(Isolate* isolate, NativeModule* native_module,
577 : ErrorThrower* thrower) {
578 : DCHECK(!thrower->error());
579 :
580 : ModuleWireBytes wire_bytes{native_module->wire_bytes()};
581 : const WasmModule* module = native_module->module();
582 7757 : uint32_t start = module->num_imported_functions;
583 7757 : uint32_t end = start + module->num_declared_functions;
584 9253 : for (uint32_t i = start; i < end; ++i) {
585 8337 : const WasmFunction* func = &module->functions[i];
586 :
587 : Vector<const uint8_t> code = wire_bytes.GetFunctionBytes(func);
588 8337 : FunctionBody body{func->sig, func->code.offset(), code.start(), code.end()};
589 : DecodeResult result;
590 : {
591 8337 : auto time_counter = SELECT_WASM_COUNTER(
592 : isolate->counters(), module->origin, wasm_decode, function_time);
593 :
594 : TimedHistogramScope wasm_decode_function_time_scope(time_counter);
595 8337 : WasmFeatures detected;
596 16674 : result = VerifyWasmCode(isolate->allocator(),
597 : native_module->enabled_features(), module,
598 : &detected, body);
599 : }
600 8337 : if (result.failed()) {
601 7589 : WasmName name = wire_bytes.GetNameOrNull(func, module);
602 7589 : if (name.start() == nullptr) {
603 : thrower->CompileError("Compiling function #%d failed: %s @+%u", i,
604 : result.error().message().c_str(),
605 7397 : result.error().offset());
606 : } else {
607 192 : TruncatedUserString<> name(wire_bytes.GetNameOrNull(func, module));
608 : thrower->CompileError("Compiling function #%d:\"%.*s\" failed: %s @+%u",
609 : i, name.length(), name.start(),
610 : result.error().message().c_str(),
611 192 : result.error().offset());
612 : }
613 : break;
614 : }
615 : }
616 7757 : }
617 :
618 144648 : void CompileNativeModule(Isolate* isolate, ErrorThrower* thrower,
619 : const WasmModule* wasm_module,
620 : NativeModule* native_module) {
621 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
622 :
623 144648 : if (compile_lazy(wasm_module)) {
624 2629 : if (wasm_module->origin == kWasmOrigin) {
625 : // Validate wasm modules for lazy compilation. Don't validate asm.js
626 : // modules, they are valid by construction (otherwise a CHECK will fail
627 : // during lazy compilation).
628 : // TODO(clemensh): According to the spec, we can actually skip validation
629 : // at module creation time, and return a function that always traps at
630 : // (lazy) compilation time.
631 176 : ValidateSequentially(isolate, native_module, thrower);
632 176 : if (thrower->error()) return;
633 : }
634 :
635 2621 : native_module->SetLazyBuiltin();
636 : } else {
637 : size_t funcs_to_compile =
638 142019 : wasm_module->functions.size() - wasm_module->num_imported_functions;
639 : bool compile_parallel =
640 284030 : !FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks > 0 &&
641 147181 : funcs_to_compile > 1 &&
642 5162 : V8::GetCurrentPlatform()->NumberOfWorkerThreads() > 0;
643 :
644 142019 : if (compile_parallel) {
645 5162 : CompileInParallel(isolate, native_module);
646 : } else {
647 136857 : CompileSequentially(isolate, native_module);
648 : }
649 : auto* compilation_state = Impl(native_module->compilation_state());
650 142019 : if (compilation_state->failed()) {
651 7505 : ValidateSequentially(isolate, native_module, thrower);
652 7505 : CHECK(thrower->error());
653 : }
654 : }
655 : }
656 :
657 : // The runnable task that performs compilations in the background.
658 98617 : class BackgroundCompileTask : public CancelableTask {
659 : public:
660 : explicit BackgroundCompileTask(CancelableTaskManager* manager,
661 : std::shared_ptr<BackgroundCompileToken> token,
662 : std::shared_ptr<Counters> async_counters)
663 : : CancelableTask(manager),
664 : token_(std::move(token)),
665 32898 : async_counters_(std::move(async_counters)) {}
666 :
667 32819 : void RunInternal() override {
668 : TRACE_COMPILE("(3b) Compiling...\n");
669 66075 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
670 : "BackgroundCompileTask::RunInternal");
671 :
672 32823 : double deadline = MonotonicallyIncreasingTimeInMs() + 50.0;
673 :
674 : // These fields are initialized in a {BackgroundCompileScope} before
675 : // starting compilation.
676 32808 : base::Optional<CompilationEnv> env;
677 32808 : std::shared_ptr<WireBytesStorage> wire_bytes;
678 : std::shared_ptr<const WasmModule> module;
679 33240 : std::unique_ptr<WasmCompilationUnit> unit;
680 32808 : WasmFeatures detected_features = kNoWasmFeatures;
681 :
682 : // Preparation (synchronized): Initialize the fields above and get the first
683 : // compilation unit.
684 : {
685 47093 : BackgroundCompileScope compile_scope(token_);
686 51464 : if (compile_scope.cancelled()) return;
687 63289 : env.emplace(compile_scope.native_module()->CreateCompilationEnv());
688 31680 : wire_bytes = compile_scope.compilation_state()->GetWireBytesStorage();
689 31679 : module = compile_scope.native_module()->shared_module();
690 63351 : unit = compile_scope.compilation_state()->GetNextCompilationUnit();
691 31672 : if (unit == nullptr) {
692 : compile_scope.compilation_state()->OnBackgroundTaskStopped(
693 17387 : detected_features);
694 17391 : return;
695 : }
696 : }
697 :
698 433 : std::vector<WasmCompilationResult> results_to_publish;
699 :
700 : auto publish_results =
701 315375 : [&results_to_publish](BackgroundCompileScope* compile_scope) {
702 124767 : if (results_to_publish.empty()) return;
703 : // TODO(clemensh): Refactor {OnFinishedUnit} and remove this.
704 : std::vector<ExecutionTier> requested_tiers;
705 101045 : requested_tiers.reserve(results_to_publish.size());
706 398288 : for (auto& result : results_to_publish) {
707 195780 : requested_tiers.push_back(result.requested_tier);
708 : }
709 :
710 : std::vector<WasmCode*> generated_code =
711 : compile_scope->native_module()->AddCompiledCode(
712 101233 : VectorOf(results_to_publish));
713 : results_to_publish.clear();
714 :
715 : // Account for the finished compilation units.
716 : // TODO(clemensh): This takes a lock on each invokation. Only do this
717 : // once and pass accumulated counts.
718 : DCHECK_EQ(generated_code.size(), requested_tiers.size());
719 494933 : for (size_t i = 0; i < generated_code.size(); ++i) {
720 393650 : compile_scope->compilation_state()->OnFinishedUnit(
721 196825 : requested_tiers[i], generated_code[i]);
722 : }
723 14191 : };
724 :
725 : bool compilation_failed = false;
726 : while (true) {
727 : // (asynchronous): Execute the compilation.
728 :
729 : WasmCompilationResult result = unit->ExecuteCompilation(
730 381382 : &env.value(), wire_bytes, async_counters_.get(), &detected_features);
731 193855 : results_to_publish.emplace_back(std::move(result));
732 :
733 : // (synchronized): Publish the compilation result and get the next unit.
734 : {
735 377637 : BackgroundCompileScope compile_scope(token_);
736 211834 : if (compile_scope.cancelled()) return;
737 197692 : if (!results_to_publish.back().succeeded()) {
738 : // Compile error.
739 433 : compile_scope.compilation_state()->SetError();
740 : compile_scope.compilation_state()->OnBackgroundTaskStopped(
741 433 : detected_features);
742 : compilation_failed = true;
743 433 : break;
744 : }
745 : // Publish TurboFan units immediately to reduce peak memory consumption.
746 197259 : if (result.requested_tier == ExecutionTier::kOptimized) {
747 99487 : publish_results(&compile_scope);
748 : }
749 :
750 197330 : if (deadline < MonotonicallyIncreasingTimeInMs()) {
751 65 : publish_results(&compile_scope);
752 : compile_scope.compilation_state()->ReportDetectedFeatures(
753 65 : detected_features);
754 65 : compile_scope.compilation_state()->RestartBackgroundCompileTask();
755 65 : return;
756 : }
757 :
758 : // Get next unit.
759 394059 : unit = compile_scope.compilation_state()->GetNextCompilationUnit();
760 197156 : if (unit == nullptr) {
761 13496 : publish_results(&compile_scope);
762 : compile_scope.compilation_state()->OnBackgroundTaskStopped(
763 13497 : detected_features);
764 13501 : return;
765 : }
766 : }
767 : }
768 : // We only get here if compilation failed. Other exits return directly.
769 : DCHECK(compilation_failed);
770 : USE(compilation_failed);
771 433 : token_->Cancel();
772 : }
773 :
774 : private:
775 : std::shared_ptr<BackgroundCompileToken> token_;
776 : std::shared_ptr<Counters> async_counters_;
777 : };
778 :
779 : } // namespace
780 :
781 144596 : std::shared_ptr<NativeModule> CompileToNativeModule(
782 : Isolate* isolate, const WasmFeatures& enabled, ErrorThrower* thrower,
783 : std::shared_ptr<const WasmModule> module, const ModuleWireBytes& wire_bytes,
784 : Handle<FixedArray>* export_wrappers_out) {
785 : const WasmModule* wasm_module = module.get();
786 144596 : TimedHistogramScope wasm_compile_module_time_scope(SELECT_WASM_COUNTER(
787 144596 : isolate->counters(), wasm_module->origin, wasm_compile, module_time));
788 :
789 : // Embedder usage count for declared shared memories.
790 144605 : if (wasm_module->has_shared_memory) {
791 1484 : isolate->CountUsage(v8::Isolate::UseCounterFeature::kWasmSharedMemory);
792 : }
793 144607 : int export_wrapper_size = static_cast<int>(module->num_exported_functions);
794 :
795 : // TODO(wasm): only save the sections necessary to deserialize a
796 : // {WasmModule}. E.g. function bodies could be omitted.
797 : OwnedVector<uint8_t> wire_bytes_copy =
798 144607 : OwnedVector<uint8_t>::Of(wire_bytes.module_bytes());
799 :
800 : // Create and compile the native module.
801 : size_t code_size_estimate =
802 144610 : wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
803 :
804 : // Create a new {NativeModule} first.
805 : auto native_module = isolate->wasm_engine()->NewNativeModule(
806 : isolate, enabled, code_size_estimate,
807 289215 : wasm::NativeModule::kCanAllocateMoreMemory, std::move(module));
808 289220 : native_module->SetWireBytes(std::move(wire_bytes_copy));
809 144610 : native_module->SetRuntimeStubs(isolate);
810 :
811 144610 : CompileNativeModule(isolate, thrower, wasm_module, native_module.get());
812 144608 : if (thrower->error()) return {};
813 :
814 : // Compile JS->wasm wrappers for exported functions.
815 : *export_wrappers_out = isolate->factory()->NewFixedArray(
816 137095 : export_wrapper_size, AllocationType::kOld);
817 : CompileJsToWasmWrappers(isolate, native_module->module(),
818 137097 : *export_wrappers_out);
819 :
820 : // Log the code within the generated module for profiling.
821 137097 : native_module->LogWasmCodes(isolate);
822 :
823 : return native_module;
824 : }
825 :
826 38 : void CompileNativeModuleWithExplicitBoundsChecks(Isolate* isolate,
827 : ErrorThrower* thrower,
828 : const WasmModule* wasm_module,
829 : NativeModule* native_module) {
830 38 : native_module->DisableTrapHandler();
831 38 : CompileNativeModule(isolate, thrower, wasm_module, native_module);
832 38 : }
833 :
834 2543 : AsyncCompileJob::AsyncCompileJob(
835 : Isolate* isolate, const WasmFeatures& enabled,
836 : std::unique_ptr<byte[]> bytes_copy, size_t length, Handle<Context> context,
837 : std::shared_ptr<CompilationResultResolver> resolver)
838 : : isolate_(isolate),
839 : enabled_features_(enabled),
840 : bytes_copy_(std::move(bytes_copy)),
841 : wire_bytes_(bytes_copy_.get(), bytes_copy_.get() + length),
842 10172 : resolver_(std::move(resolver)) {
843 : v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate);
844 2543 : v8::Platform* platform = V8::GetCurrentPlatform();
845 5086 : foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate);
846 : native_context_ =
847 5086 : isolate->global_handles()->Create(context->native_context());
848 : DCHECK(native_context_->IsNativeContext());
849 2543 : }
850 :
851 2163 : void AsyncCompileJob::Start() {
852 4326 : DoAsync<DecodeModule>(isolate_->counters()); // --
853 2163 : }
854 :
855 56 : void AsyncCompileJob::Abort() {
856 : // Removing this job will trigger the destructor, which will cancel all
857 : // compilation.
858 56 : isolate_->wasm_engine()->RemoveCompileJob(this);
859 56 : }
860 :
861 1520 : class AsyncStreamingProcessor final : public StreamingProcessor {
862 : public:
863 : explicit AsyncStreamingProcessor(AsyncCompileJob* job);
864 :
865 : bool ProcessModuleHeader(Vector<const uint8_t> bytes,
866 : uint32_t offset) override;
867 :
868 : bool ProcessSection(SectionCode section_code, Vector<const uint8_t> bytes,
869 : uint32_t offset) override;
870 :
871 : bool ProcessCodeSectionHeader(int functions_count, uint32_t offset,
872 : std::shared_ptr<WireBytesStorage>) override;
873 :
874 : bool ProcessFunctionBody(Vector<const uint8_t> bytes,
875 : uint32_t offset) override;
876 :
877 : void OnFinishedChunk() override;
878 :
879 : void OnFinishedStream(OwnedVector<uint8_t> bytes) override;
880 :
881 : void OnError(const WasmError&) override;
882 :
883 : void OnAbort() override;
884 :
885 : bool Deserialize(Vector<const uint8_t> wire_bytes,
886 : Vector<const uint8_t> module_bytes) override;
887 :
888 : private:
889 : // Finishes the AsyncCompileJob with an error.
890 : void FinishAsyncCompileJobWithError(const WasmError&);
891 :
892 : void CommitCompilationUnits();
893 :
894 : ModuleDecoder decoder_;
895 : AsyncCompileJob* job_;
896 : std::unique_ptr<CompilationUnitBuilder> compilation_unit_builder_;
897 : int num_functions_ = 0;
898 : };
899 :
900 380 : std::shared_ptr<StreamingDecoder> AsyncCompileJob::CreateStreamingDecoder() {
901 : DCHECK_NULL(stream_);
902 1140 : stream_.reset(
903 1520 : new StreamingDecoder(base::make_unique<AsyncStreamingProcessor>(this)));
904 380 : return stream_;
905 : }
906 :
907 7629 : AsyncCompileJob::~AsyncCompileJob() {
908 : // Note: This destructor always runs on the foreground thread of the isolate.
909 2543 : background_task_manager_.CancelAndWait();
910 : // If the runtime objects were not created yet, then initial compilation did
911 : // not finish yet. In this case we can abort compilation.
912 4918 : if (native_module_ && module_object_.is_null()) {
913 165 : Impl(native_module_->compilation_state())->AbortCompilation();
914 : }
915 : // Tell the streaming decoder that the AsyncCompileJob is not available
916 : // anymore.
917 : // TODO(ahaas): Is this notification really necessary? Check
918 : // https://crbug.com/888170.
919 2543 : if (stream_) stream_->NotifyCompilationEnded();
920 : CancelPendingForegroundTask();
921 2543 : isolate_->global_handles()->Destroy(native_context_.location());
922 2543 : if (!module_object_.is_null()) {
923 2210 : isolate_->global_handles()->Destroy(module_object_.location());
924 : }
925 2543 : }
926 :
927 2371 : void AsyncCompileJob::CreateNativeModule(
928 : std::shared_ptr<const WasmModule> module) {
929 : // Embedder usage count for declared shared memories.
930 2371 : if (module->has_shared_memory) {
931 0 : isolate_->CountUsage(v8::Isolate::UseCounterFeature::kWasmSharedMemory);
932 : }
933 :
934 : // TODO(wasm): Improve efficiency of storing module wire bytes. Only store
935 : // relevant sections, not function bodies
936 :
937 : // Create the module object and populate with compiled functions and
938 : // information needed at instantiation time.
939 : // TODO(clemensh): For the same module (same bytes / same hash), we should
940 : // only have one {WasmModuleObject}. Otherwise, we might only set
941 : // breakpoints on a (potentially empty) subset of the instances.
942 : // Create the module object.
943 :
944 : size_t code_size_estimate =
945 2371 : wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
946 7113 : native_module_ = isolate_->wasm_engine()->NewNativeModule(
947 : isolate_, enabled_features_, code_size_estimate,
948 : wasm::NativeModule::kCanAllocateMoreMemory, std::move(module));
949 4742 : native_module_->SetWireBytes({std::move(bytes_copy_), wire_bytes_.length()});
950 2371 : native_module_->SetRuntimeStubs(isolate_);
951 :
952 2371 : if (stream_) stream_->NotifyNativeModuleCreated(native_module_);
953 2371 : }
954 :
955 2206 : void AsyncCompileJob::PrepareRuntimeObjects() {
956 : // Create heap objects for script and module bytes to be stored in the
957 : // module object. Asm.js is not compiled asynchronously.
958 : const WasmModule* module = native_module_->module();
959 : Handle<Script> script =
960 2206 : CreateWasmScript(isolate_, wire_bytes_, module->source_map_url);
961 :
962 : size_t code_size_estimate =
963 2206 : wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module);
964 : Handle<WasmModuleObject> module_object = WasmModuleObject::New(
965 4412 : isolate_, native_module_, script, code_size_estimate);
966 :
967 4412 : module_object_ = isolate_->global_handles()->Create(*module_object);
968 2206 : }
969 :
970 : // This function assumes that it is executed in a HandleScope, and that a
971 : // context is set on the isolate.
972 2210 : void AsyncCompileJob::FinishCompile() {
973 : bool is_after_deserialization = !module_object_.is_null();
974 2210 : if (!is_after_deserialization) {
975 2206 : PrepareRuntimeObjects();
976 : }
977 : DCHECK(!isolate_->context().is_null());
978 : // Finish the wasm script now and make it public to the debugger.
979 2210 : Handle<Script> script(module_object_->script(), isolate_);
980 6630 : if (script->type() == Script::TYPE_WASM &&
981 4420 : module_object_->module()->source_map_url.size() != 0) {
982 0 : MaybeHandle<String> src_map_str = isolate_->factory()->NewStringFromUtf8(
983 0 : CStrVector(module_object_->module()->source_map_url.c_str()),
984 0 : AllocationType::kOld);
985 0 : script->set_source_mapping_url(*src_map_str.ToHandleChecked());
986 : }
987 2210 : isolate_->debug()->OnAfterCompile(script);
988 :
989 : // We can only update the feature counts once the entire compile is done.
990 : auto compilation_state =
991 4420 : Impl(module_object_->native_module()->compilation_state());
992 : compilation_state->PublishDetectedFeatures(
993 2210 : isolate_, *compilation_state->detected_features());
994 :
995 : // TODO(bbudge) Allow deserialization without wrapper compilation, so we can
996 : // just compile wrappers here.
997 2210 : if (!is_after_deserialization) {
998 : // TODO(wasm): compiling wrappers should be made async.
999 2206 : CompileWrappers();
1000 : }
1001 2210 : FinishModule();
1002 2210 : }
1003 :
1004 189 : void AsyncCompileJob::DecodeFailed(const WasmError& error) {
1005 378 : ErrorThrower thrower(isolate_, "WebAssembly.compile()");
1006 : thrower.CompileFailed(error);
1007 : // {job} keeps the {this} pointer alive.
1008 : std::shared_ptr<AsyncCompileJob> job =
1009 378 : isolate_->wasm_engine()->RemoveCompileJob(this);
1010 189 : resolver_->OnCompilationFailed(thrower.Reify());
1011 189 : }
1012 :
1013 76 : void AsyncCompileJob::AsyncCompileFailed() {
1014 152 : ErrorThrower thrower(isolate_, "WebAssembly.compile()");
1015 76 : ValidateSequentially(isolate_, native_module_.get(), &thrower);
1016 : DCHECK(thrower.error());
1017 : // {job} keeps the {this} pointer alive.
1018 : std::shared_ptr<AsyncCompileJob> job =
1019 152 : isolate_->wasm_engine()->RemoveCompileJob(this);
1020 76 : resolver_->OnCompilationFailed(thrower.Reify());
1021 76 : }
1022 :
1023 0 : void AsyncCompileJob::AsyncCompileSucceeded(Handle<WasmModuleObject> result) {
1024 2210 : resolver_->OnCompilationSucceeded(result);
1025 0 : }
1026 :
1027 : class AsyncCompileJob::CompilationStateCallback {
1028 : public:
1029 : explicit CompilationStateCallback(AsyncCompileJob* job) : job_(job) {}
1030 :
1031 2664 : void operator()(CompilationEvent event) {
1032 : // This callback is only being called from a foreground task.
1033 2664 : switch (event) {
1034 : case CompilationEvent::kFinishedBaselineCompilation:
1035 : DCHECK(!last_event_.has_value());
1036 2588 : if (job_->DecrementAndCheckFinisherCount()) {
1037 1259 : job_->DoSync<CompileFinished>();
1038 : }
1039 : break;
1040 : case CompilationEvent::kFinishedTopTierCompilation:
1041 : DCHECK_EQ(CompilationEvent::kFinishedBaselineCompilation, last_event_);
1042 : // At this point, the job will already be gone, thus do not access it
1043 : // here.
1044 : break;
1045 : case CompilationEvent::kFailedCompilation: {
1046 : DCHECK(!last_event_.has_value());
1047 160 : if (job_->DecrementAndCheckFinisherCount()) {
1048 59 : job_->DoSync<CompileFailed>();
1049 : }
1050 : break;
1051 : }
1052 : default:
1053 0 : UNREACHABLE();
1054 : }
1055 : #ifdef DEBUG
1056 : last_event_ = event;
1057 : #endif
1058 2664 : }
1059 :
1060 : private:
1061 : AsyncCompileJob* job_;
1062 : #ifdef DEBUG
1063 : // This will be modified by different threads, but they externally
1064 : // synchronize, so no explicit synchronization (currently) needed here.
1065 : base::Optional<CompilationEvent> last_event_;
1066 : #endif
1067 : };
1068 :
1069 : // A closure to run a compilation step (either as foreground or background
1070 : // task) and schedule the next step(s), if any.
1071 6024 : class AsyncCompileJob::CompileStep {
1072 : public:
1073 6024 : virtual ~CompileStep() = default;
1074 :
1075 6021 : void Run(AsyncCompileJob* job, bool on_foreground) {
1076 6021 : if (on_foreground) {
1077 3858 : HandleScope scope(job->isolate_);
1078 3858 : SaveAndSwitchContext saved_context(job->isolate_, *job->native_context_);
1079 3858 : RunInForeground(job);
1080 : } else {
1081 2163 : RunInBackground(job);
1082 : }
1083 6021 : }
1084 :
1085 0 : virtual void RunInForeground(AsyncCompileJob*) { UNREACHABLE(); }
1086 0 : virtual void RunInBackground(AsyncCompileJob*) { UNREACHABLE(); }
1087 : };
1088 :
1089 : class AsyncCompileJob::CompileTask : public CancelableTask {
1090 : public:
1091 : CompileTask(AsyncCompileJob* job, bool on_foreground)
1092 : // We only manage the background tasks with the {CancelableTaskManager} of
1093 : // the {AsyncCompileJob}. Foreground tasks are managed by the system's
1094 : // {CancelableTaskManager}. Background tasks cannot spawn tasks managed by
1095 : // their own task manager.
1096 3861 : : CancelableTask(on_foreground ? job->isolate_->cancelable_task_manager()
1097 : : &job->background_task_manager_),
1098 : job_(job),
1099 8187 : on_foreground_(on_foreground) {}
1100 :
1101 18072 : ~CompileTask() override {
1102 6024 : if (job_ != nullptr && on_foreground_) ResetPendingForegroundTask();
1103 12048 : }
1104 :
1105 6020 : void RunInternal() final {
1106 6020 : if (!job_) return;
1107 6021 : if (on_foreground_) ResetPendingForegroundTask();
1108 12042 : job_->step_->Run(job_, on_foreground_);
1109 : // After execution, reset {job_} such that we don't try to reset the pending
1110 : // foreground task when the task is deleted.
1111 6021 : job_ = nullptr;
1112 : }
1113 :
1114 : void Cancel() {
1115 : DCHECK_NOT_NULL(job_);
1116 3 : job_ = nullptr;
1117 : }
1118 :
1119 : private:
1120 : // {job_} will be cleared to cancel a pending task.
1121 : AsyncCompileJob* job_;
1122 : bool on_foreground_;
1123 :
1124 : void ResetPendingForegroundTask() const {
1125 : DCHECK_EQ(this, job_->pending_foreground_task_);
1126 3858 : job_->pending_foreground_task_ = nullptr;
1127 : }
1128 : };
1129 :
1130 3645 : void AsyncCompileJob::StartForegroundTask() {
1131 : DCHECK_NULL(pending_foreground_task_);
1132 :
1133 3645 : auto new_task = base::make_unique<CompileTask>(this, true);
1134 3645 : pending_foreground_task_ = new_task.get();
1135 10935 : foreground_task_runner_->PostTask(std::move(new_task));
1136 3645 : }
1137 :
1138 216 : void AsyncCompileJob::ExecuteForegroundTaskImmediately() {
1139 : DCHECK_NULL(pending_foreground_task_);
1140 :
1141 216 : auto new_task = base::make_unique<CompileTask>(this, true);
1142 216 : pending_foreground_task_ = new_task.get();
1143 216 : new_task->Run();
1144 216 : }
1145 :
1146 0 : void AsyncCompileJob::CancelPendingForegroundTask() {
1147 2543 : if (!pending_foreground_task_) return;
1148 : pending_foreground_task_->Cancel();
1149 3 : pending_foreground_task_ = nullptr;
1150 : }
1151 :
1152 2163 : void AsyncCompileJob::StartBackgroundTask() {
1153 2163 : auto task = base::make_unique<CompileTask>(this, false);
1154 :
1155 : // If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
1156 : // tasks. This is used to make timing deterministic.
1157 2163 : if (FLAG_wasm_num_compilation_tasks > 0) {
1158 6489 : V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
1159 : } else {
1160 0 : foreground_task_runner_->PostTask(std::move(task));
1161 : }
1162 2163 : }
1163 :
1164 : template <typename Step,
1165 : AsyncCompileJob::UseExistingForegroundTask use_existing_fg_task,
1166 : typename... Args>
1167 56 : void AsyncCompileJob::DoSync(Args&&... args) {
1168 3645 : NextStep<Step>(std::forward<Args>(args)...);
1169 56 : if (use_existing_fg_task && pending_foreground_task_ != nullptr) return;
1170 3645 : StartForegroundTask();
1171 : }
1172 :
1173 : template <typename Step, typename... Args>
1174 : void AsyncCompileJob::DoImmediately(Args&&... args) {
1175 216 : NextStep<Step>(std::forward<Args>(args)...);
1176 216 : ExecuteForegroundTaskImmediately();
1177 : }
1178 :
1179 : template <typename Step, typename... Args>
1180 : void AsyncCompileJob::DoAsync(Args&&... args) {
1181 2163 : NextStep<Step>(std::forward<Args>(args)...);
1182 2163 : StartBackgroundTask();
1183 : }
1184 :
1185 : template <typename Step, typename... Args>
1186 6024 : void AsyncCompileJob::NextStep(Args&&... args) {
1187 6429 : step_.reset(new Step(std::forward<Args>(args)...));
1188 6024 : }
1189 :
1190 : //==========================================================================
1191 : // Step 1: (async) Decode the module.
1192 : //==========================================================================
1193 4326 : class AsyncCompileJob::DecodeModule : public AsyncCompileJob::CompileStep {
1194 : public:
1195 2163 : explicit DecodeModule(Counters* counters) : counters_(counters) {}
1196 :
1197 2163 : void RunInBackground(AsyncCompileJob* job) override {
1198 2163 : ModuleResult result;
1199 : {
1200 : DisallowHandleAllocation no_handle;
1201 : DisallowHeapAllocation no_allocation;
1202 : // Decode the module bytes.
1203 : TRACE_COMPILE("(1) Decoding module...\n");
1204 6489 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
1205 : "AsyncCompileJob::DecodeModule");
1206 4326 : result = DecodeWasmModule(
1207 : job->enabled_features_, job->wire_bytes_.start(),
1208 2163 : job->wire_bytes_.end(), false, kWasmOrigin, counters_,
1209 2163 : job->isolate()->wasm_engine()->allocator());
1210 : }
1211 2163 : if (result.failed()) {
1212 : // Decoding failure; reject the promise and clean up.
1213 : job->DoSync<DecodeFail>(std::move(result).error());
1214 : } else {
1215 : // Decode passed.
1216 4276 : job->DoSync<PrepareAndStartCompile>(std::move(result).value(), true);
1217 : }
1218 2163 : }
1219 :
1220 : private:
1221 : Counters* const counters_;
1222 : };
1223 :
1224 : //==========================================================================
1225 : // Step 1b: (sync) Fail decoding the module.
1226 : //==========================================================================
1227 378 : class AsyncCompileJob::DecodeFail : public CompileStep {
1228 : public:
1229 189 : explicit DecodeFail(WasmError error) : error_(std::move(error)) {}
1230 :
1231 : private:
1232 : WasmError error_;
1233 :
1234 189 : void RunInForeground(AsyncCompileJob* job) override {
1235 : TRACE_COMPILE("(1b) Decoding failed.\n");
1236 : // {job_} is deleted in DecodeFailed, therefore the {return}.
1237 189 : return job->DecodeFailed(error_);
1238 : }
1239 : };
1240 :
1241 : //==========================================================================
1242 : // Step 2 (sync): Create heap-allocated data and start compile.
1243 : //==========================================================================
1244 4708 : class AsyncCompileJob::PrepareAndStartCompile : public CompileStep {
1245 : public:
1246 : PrepareAndStartCompile(std::shared_ptr<const WasmModule> module,
1247 : bool start_compilation)
1248 4708 : : module_(std::move(module)), start_compilation_(start_compilation) {}
1249 :
1250 : private:
1251 : std::shared_ptr<const WasmModule> module_;
1252 : bool start_compilation_;
1253 :
1254 2351 : void RunInForeground(AsyncCompileJob* job) override {
1255 : TRACE_COMPILE("(2) Prepare and start compile...\n");
1256 :
1257 : // Make sure all compilation tasks stopped running. Decoding (async step)
1258 : // is done.
1259 2351 : job->background_task_manager_.CancelAndWait();
1260 :
1261 4702 : job->CreateNativeModule(module_);
1262 :
1263 : size_t num_functions =
1264 2351 : module_->functions.size() - module_->num_imported_functions;
1265 :
1266 2351 : if (num_functions == 0) {
1267 : // Degenerate case of an empty module.
1268 896 : job->FinishCompile();
1269 896 : return;
1270 : }
1271 :
1272 : CompilationStateImpl* compilation_state =
1273 : Impl(job->native_module_->compilation_state());
1274 2910 : compilation_state->AddCallback(CompilationStateCallback{job});
1275 1455 : if (start_compilation_) {
1276 : // TODO(ahaas): Try to remove the {start_compilation_} check when
1277 : // streaming decoding is done in the background. If
1278 : // InitializeCompilationUnits always returns 0 for streaming compilation,
1279 : // then DoAsync would do the same as NextStep already.
1280 :
1281 1239 : compilation_state->SetNumberOfFunctionsToCompile(
1282 2478 : module_->num_declared_functions);
1283 : // Add compilation units and kick off compilation.
1284 : InitializeCompilationUnits(job->native_module_.get(),
1285 1239 : job->isolate()->wasm_engine());
1286 : }
1287 : }
1288 : };
1289 :
1290 : //==========================================================================
1291 : // Step 3a (sync): Compilation failed.
1292 : //==========================================================================
1293 177 : class AsyncCompileJob::CompileFailed : public CompileStep {
1294 : private:
1295 59 : void RunInForeground(AsyncCompileJob* job) override {
1296 : TRACE_COMPILE("(3a) Compilation failed\n");
1297 : DCHECK(job->native_module_->compilation_state()->failed());
1298 :
1299 : // {job_} is deleted in AsyncCompileFailed, therefore the {return}.
1300 59 : return job->AsyncCompileFailed();
1301 : }
1302 : };
1303 :
1304 : namespace {
1305 1259 : class SampleTopTierCodeSizeCallback {
1306 : public:
1307 : explicit SampleTopTierCodeSizeCallback(
1308 : std::weak_ptr<NativeModule> native_module)
1309 : : native_module_(std::move(native_module)) {}
1310 :
1311 351 : void operator()(CompilationEvent event) {
1312 : // This callback is registered after baseline compilation finished, so the
1313 : // only possible event to follow is {kFinishedTopTierCompilation}.
1314 : DCHECK_EQ(CompilationEvent::kFinishedTopTierCompilation, event);
1315 351 : if (std::shared_ptr<NativeModule> native_module = native_module_.lock()) {
1316 : native_module->engine()->SampleTopTierCodeSizeInAllIsolates(
1317 351 : native_module);
1318 : }
1319 351 : }
1320 :
1321 : private:
1322 : std::weak_ptr<NativeModule> native_module_;
1323 : };
1324 : } // namespace
1325 :
1326 : //==========================================================================
1327 : // Step 3b (sync): Compilation finished.
1328 : //==========================================================================
1329 3777 : class AsyncCompileJob::CompileFinished : public CompileStep {
1330 : private:
1331 1259 : void RunInForeground(AsyncCompileJob* job) override {
1332 : TRACE_COMPILE("(3b) Compilation finished\n");
1333 : DCHECK(!job->native_module_->compilation_state()->failed());
1334 : // Sample the generated code size when baseline compilation finished.
1335 1259 : job->native_module_->SampleCodeSize(job->isolate_->counters(),
1336 1259 : NativeModule::kAfterBaseline);
1337 : // Also, set a callback to sample the code size after top-tier compilation
1338 : // finished. This callback will *not* keep the NativeModule alive.
1339 2518 : job->native_module_->compilation_state()->AddCallback(
1340 1259 : SampleTopTierCodeSizeCallback{job->native_module_});
1341 : // Then finalize and publish the generated module.
1342 1259 : job->FinishCompile();
1343 1259 : }
1344 : };
1345 :
1346 2206 : void AsyncCompileJob::CompileWrappers() {
1347 : // TODO(wasm): Compile all wrappers here, including the start function wrapper
1348 : // and the wrappers for the function table elements.
1349 : TRACE_COMPILE("(5) Compile wrappers...\n");
1350 : // Compile JS->wasm wrappers for exported functions.
1351 4412 : CompileJsToWasmWrappers(isolate_, module_object_->native_module()->module(),
1352 4412 : handle(module_object_->export_wrappers(), isolate_));
1353 2206 : }
1354 :
1355 2210 : void AsyncCompileJob::FinishModule() {
1356 : TRACE_COMPILE("(6) Finish module...\n");
1357 : AsyncCompileSucceeded(module_object_);
1358 2210 : isolate_->wasm_engine()->RemoveCompileJob(this);
1359 2210 : }
1360 :
1361 0 : AsyncStreamingProcessor::AsyncStreamingProcessor(AsyncCompileJob* job)
1362 : : decoder_(job->enabled_features_),
1363 : job_(job),
1364 760 : compilation_unit_builder_(nullptr) {}
1365 :
1366 164 : void AsyncStreamingProcessor::FinishAsyncCompileJobWithError(
1367 : const WasmError& error) {
1368 : DCHECK(error.has_error());
1369 : // Make sure all background tasks stopped executing before we change the state
1370 : // of the AsyncCompileJob to DecodeFail.
1371 164 : job_->background_task_manager_.CancelAndWait();
1372 :
1373 : // Check if there is already a CompiledModule, in which case we have to clean
1374 : // up the CompilationStateImpl as well.
1375 164 : if (job_->native_module_) {
1376 56 : Impl(job_->native_module_->compilation_state())->AbortCompilation();
1377 :
1378 56 : job_->DoSync<AsyncCompileJob::DecodeFail,
1379 56 : AsyncCompileJob::kUseExistingForegroundTask>(error);
1380 :
1381 : // Clear the {compilation_unit_builder_} if it exists. This is needed
1382 : // because there is a check in the destructor of the
1383 : // {CompilationUnitBuilder} that it is empty.
1384 56 : if (compilation_unit_builder_) compilation_unit_builder_->Clear();
1385 : } else {
1386 : job_->DoSync<AsyncCompileJob::DecodeFail>(error);
1387 : }
1388 164 : }
1389 :
1390 : // Process the module header.
1391 352 : bool AsyncStreamingProcessor::ProcessModuleHeader(Vector<const uint8_t> bytes,
1392 : uint32_t offset) {
1393 : TRACE_STREAMING("Process module header...\n");
1394 352 : decoder_.StartDecoding(job_->isolate()->counters(),
1395 704 : job_->isolate()->wasm_engine()->allocator());
1396 352 : decoder_.DecodeModuleHeader(bytes, offset);
1397 352 : if (!decoder_.ok()) {
1398 16 : FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false).error());
1399 16 : return false;
1400 : }
1401 : return true;
1402 : }
1403 :
1404 : // Process all sections except for the code section.
1405 804 : bool AsyncStreamingProcessor::ProcessSection(SectionCode section_code,
1406 : Vector<const uint8_t> bytes,
1407 : uint32_t offset) {
1408 : TRACE_STREAMING("Process section %d ...\n", section_code);
1409 804 : if (compilation_unit_builder_) {
1410 : // We reached a section after the code section, we do not need the
1411 : // compilation_unit_builder_ anymore.
1412 : CommitCompilationUnits();
1413 : compilation_unit_builder_.reset();
1414 : }
1415 804 : if (section_code == SectionCode::kUnknownSectionCode) {
1416 : Decoder decoder(bytes, offset);
1417 : section_code = ModuleDecoder::IdentifyUnknownSection(
1418 72 : decoder, bytes.start() + bytes.length());
1419 72 : if (section_code == SectionCode::kUnknownSectionCode) {
1420 : // Skip unknown sections that we do not know how to handle.
1421 : return true;
1422 : }
1423 : // Remove the unknown section tag from the payload bytes.
1424 72 : offset += decoder.position();
1425 72 : bytes = bytes.SubVector(decoder.position(), bytes.size());
1426 : }
1427 : constexpr bool verify_functions = false;
1428 804 : decoder_.DecodeSection(section_code, bytes, offset, verify_functions);
1429 804 : if (!decoder_.ok()) {
1430 20 : FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false).error());
1431 20 : return false;
1432 : }
1433 : return true;
1434 : }
1435 :
1436 : // Start the code section.
1437 228 : bool AsyncStreamingProcessor::ProcessCodeSectionHeader(
1438 : int functions_count, uint32_t offset,
1439 : std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
1440 : TRACE_STREAMING("Start the code section with %d functions...\n",
1441 : functions_count);
1442 228 : if (!decoder_.CheckFunctionsCount(static_cast<uint32_t>(functions_count),
1443 : offset)) {
1444 12 : FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false).error());
1445 12 : return false;
1446 : }
1447 : // Execute the PrepareAndStartCompile step immediately and not in a separate
1448 : // task.
1449 648 : job_->DoImmediately<AsyncCompileJob::PrepareAndStartCompile>(
1450 216 : decoder_.shared_module(), false);
1451 432 : job_->native_module_->compilation_state()->SetWireBytesStorage(
1452 216 : std::move(wire_bytes_storage));
1453 :
1454 216 : auto* compilation_state = Impl(job_->native_module_->compilation_state());
1455 216 : compilation_state->SetNumberOfFunctionsToCompile(functions_count);
1456 :
1457 : // Set outstanding_finishers_ to 2, because both the AsyncCompileJob and the
1458 : // AsyncStreamingProcessor have to finish.
1459 216 : job_->outstanding_finishers_.store(2);
1460 216 : compilation_unit_builder_.reset(new CompilationUnitBuilder(
1461 216 : job_->native_module_.get(), job_->isolate()->wasm_engine()));
1462 : return true;
1463 : }
1464 :
1465 : // Process a function body.
1466 452 : bool AsyncStreamingProcessor::ProcessFunctionBody(Vector<const uint8_t> bytes,
1467 : uint32_t offset) {
1468 : TRACE_STREAMING("Process function body %d ...\n", num_functions_);
1469 :
1470 904 : decoder_.DecodeFunctionBody(
1471 904 : num_functions_, static_cast<uint32_t>(bytes.length()), offset, false);
1472 :
1473 904 : int index = num_functions_ + decoder_.module()->num_imported_functions;
1474 452 : compilation_unit_builder_->AddUnit(index);
1475 452 : ++num_functions_;
1476 : // This method always succeeds. The return value is necessary to comply with
1477 : // the StreamingProcessor interface.
1478 452 : return true;
1479 : }
1480 :
1481 0 : void AsyncStreamingProcessor::CommitCompilationUnits() {
1482 : DCHECK(compilation_unit_builder_);
1483 292 : compilation_unit_builder_->Commit();
1484 0 : }
1485 :
1486 356 : void AsyncStreamingProcessor::OnFinishedChunk() {
1487 : TRACE_STREAMING("FinishChunk...\n");
1488 356 : if (compilation_unit_builder_) CommitCompilationUnits();
1489 356 : }
1490 :
1491 : // Finish the processing of the stream.
1492 152 : void AsyncStreamingProcessor::OnFinishedStream(OwnedVector<uint8_t> bytes) {
1493 : TRACE_STREAMING("Finish stream...\n");
1494 452 : ModuleResult result = decoder_.FinishDecoding(false);
1495 152 : if (result.failed()) {
1496 4 : FinishAsyncCompileJobWithError(result.error());
1497 4 : return;
1498 : }
1499 : // We have to open a HandleScope and prepare the Context for
1500 : // CreateNativeModule, PrepareRuntimeObjects and FinishCompile as this is a
1501 : // callback from the embedder.
1502 148 : HandleScope scope(job_->isolate_);
1503 296 : SaveAndSwitchContext saved_context(job_->isolate_, *job_->native_context_);
1504 :
1505 148 : bool needs_finish = job_->DecrementAndCheckFinisherCount();
1506 148 : if (job_->native_module_ == nullptr) {
1507 : // We are processing a WebAssembly module without code section. Create the
1508 : // runtime objects now (would otherwise happen in {PrepareAndStartCompile}).
1509 40 : job_->CreateNativeModule(std::move(result).value());
1510 : DCHECK(needs_finish);
1511 : }
1512 296 : job_->wire_bytes_ = ModuleWireBytes(bytes.as_vector());
1513 296 : job_->native_module_->SetWireBytes(std::move(bytes));
1514 148 : if (needs_finish) {
1515 136 : if (job_->native_module_->compilation_state()->failed()) {
1516 17 : job_->AsyncCompileFailed();
1517 : } else {
1518 51 : job_->FinishCompile();
1519 : }
1520 : }
1521 : }
1522 :
1523 : // Report an error detected in the StreamingDecoder.
1524 112 : void AsyncStreamingProcessor::OnError(const WasmError& error) {
1525 : TRACE_STREAMING("Stream error...\n");
1526 112 : FinishAsyncCompileJobWithError(error);
1527 112 : }
1528 :
1529 56 : void AsyncStreamingProcessor::OnAbort() {
1530 : TRACE_STREAMING("Abort stream...\n");
1531 56 : job_->Abort();
1532 56 : }
1533 :
1534 8 : bool AsyncStreamingProcessor::Deserialize(Vector<const uint8_t> module_bytes,
1535 : Vector<const uint8_t> wire_bytes) {
1536 : // DeserializeNativeModule and FinishCompile assume that they are executed in
1537 : // a HandleScope, and that a context is set on the isolate.
1538 8 : HandleScope scope(job_->isolate_);
1539 16 : SaveAndSwitchContext saved_context(job_->isolate_, *job_->native_context_);
1540 :
1541 : MaybeHandle<WasmModuleObject> result =
1542 8 : DeserializeNativeModule(job_->isolate_, module_bytes, wire_bytes);
1543 8 : if (result.is_null()) return false;
1544 :
1545 : job_->module_object_ =
1546 8 : job_->isolate_->global_handles()->Create(*result.ToHandleChecked());
1547 12 : job_->native_module_ = job_->module_object_->shared_native_module();
1548 4 : auto owned_wire_bytes = OwnedVector<uint8_t>::Of(wire_bytes);
1549 8 : job_->wire_bytes_ = ModuleWireBytes(owned_wire_bytes.as_vector());
1550 8 : job_->native_module_->SetWireBytes(std::move(owned_wire_bytes));
1551 4 : job_->FinishCompile();
1552 : return true;
1553 : }
1554 :
1555 1241902 : CompilationStateImpl::CompilationStateImpl(
1556 : const std::shared_ptr<NativeModule>& native_module,
1557 : std::shared_ptr<Counters> async_counters)
1558 : : native_module_(native_module.get()),
1559 : background_compile_token_(
1560 : std::make_shared<BackgroundCompileToken>(native_module)),
1561 1241492 : compile_mode_(FLAG_wasm_tier_up &&
1562 1241492 : native_module->module()->origin == kWasmOrigin
1563 : ? CompileMode::kTiering
1564 : : CompileMode::kRegular),
1565 : async_counters_(std::move(async_counters)),
1566 : max_background_tasks_(std::max(
1567 : 1, std::min(FLAG_wasm_num_compilation_tasks,
1568 11177118 : V8::GetCurrentPlatform()->NumberOfWorkerThreads()))) {}
1569 :
1570 1242122 : void CompilationStateImpl::AbortCompilation() {
1571 1242122 : background_compile_token_->Cancel();
1572 : // No more callbacks after abort.
1573 1242122 : base::MutexGuard callbacks_guard(&callbacks_mutex_);
1574 : callbacks_.clear();
1575 1242122 : }
1576 :
1577 6617 : void CompilationStateImpl::SetNumberOfFunctionsToCompile(int num_functions) {
1578 : DCHECK(!failed());
1579 6617 : base::MutexGuard guard(&callbacks_mutex_);
1580 6617 : outstanding_baseline_units_ = num_functions;
1581 :
1582 6617 : if (compile_mode_ == CompileMode::kTiering) {
1583 6505 : outstanding_tiering_units_ = num_functions;
1584 : }
1585 6617 : }
1586 :
1587 2718 : void CompilationStateImpl::AddCallback(CompilationState::callback_t callback) {
1588 2718 : base::MutexGuard callbacks_guard(&callbacks_mutex_);
1589 2718 : callbacks_.emplace_back(std::move(callback));
1590 2718 : }
1591 :
1592 6629 : void CompilationStateImpl::AddCompilationUnits(
1593 : std::vector<std::unique_ptr<WasmCompilationUnit>>& baseline_units,
1594 : std::vector<std::unique_ptr<WasmCompilationUnit>>& tiering_units) {
1595 : {
1596 6629 : base::MutexGuard guard(&mutex_);
1597 :
1598 : DCHECK_LE(tiering_units.size(), baseline_units.size());
1599 : DCHECK_IMPLIES(compile_mode_ == CompileMode::kTiering &&
1600 : !native_module_->enabled_features().compilation_hints,
1601 : tiering_units.size() == baseline_units.size());
1602 : DCHECK_IMPLIES(compile_mode_ == CompileMode::kTiering &&
1603 : !native_module_->enabled_features().compilation_hints,
1604 : tiering_units.back()->tier() == ExecutionTier::kOptimized);
1605 : DCHECK_IMPLIES(compile_mode_ == CompileMode::kRegular,
1606 : tiering_compilation_units_.empty());
1607 :
1608 : baseline_compilation_units_.insert(
1609 : baseline_compilation_units_.end(),
1610 : std::make_move_iterator(baseline_units.begin()),
1611 6629 : std::make_move_iterator(baseline_units.end()));
1612 6629 : if (!tiering_units.empty()) {
1613 : tiering_compilation_units_.insert(
1614 : tiering_compilation_units_.end(),
1615 : std::make_move_iterator(tiering_units.begin()),
1616 6517 : std::make_move_iterator(tiering_units.end()));
1617 : }
1618 : }
1619 :
1620 6629 : RestartBackgroundTasks();
1621 6629 : }
1622 :
1623 : std::unique_ptr<WasmCompilationUnit>
1624 5587964 : CompilationStateImpl::GetNextCompilationUnit() {
1625 5587964 : base::MutexGuard guard(&mutex_);
1626 :
1627 : std::vector<std::unique_ptr<WasmCompilationUnit>>& units =
1628 : baseline_compilation_units_.empty() ? tiering_compilation_units_
1629 5588880 : : baseline_compilation_units_;
1630 :
1631 5588880 : if (!units.empty()) {
1632 204965 : std::unique_ptr<WasmCompilationUnit> unit = std::move(units.back());
1633 : units.pop_back();
1634 : return unit;
1635 : }
1636 :
1637 : return std::unique_ptr<WasmCompilationUnit>();
1638 : }
1639 :
1640 203560 : void CompilationStateImpl::OnFinishedUnit(ExecutionTier tier, WasmCode* code) {
1641 : // This mutex guarantees that events happen in the right order.
1642 203560 : base::MutexGuard guard(&callbacks_mutex_);
1643 :
1644 : // If we are *not* compiling in tiering mode, then all units are counted as
1645 : // baseline units.
1646 203708 : bool is_tiering_mode = compile_mode_ == CompileMode::kTiering;
1647 203708 : bool is_tiering_unit = is_tiering_mode && tier == ExecutionTier::kOptimized;
1648 :
1649 : // Sanity check: If we are not in tiering mode, there cannot be outstanding
1650 : // tiering units.
1651 : DCHECK_IMPLIES(!is_tiering_mode, outstanding_tiering_units_ == 0);
1652 :
1653 : bool baseline_finished = false;
1654 : bool tiering_finished = false;
1655 203708 : if (is_tiering_unit) {
1656 : DCHECK_LT(0, outstanding_tiering_units_);
1657 101613 : --outstanding_tiering_units_;
1658 101613 : tiering_finished = outstanding_tiering_units_ == 0;
1659 : // If baseline compilation has not finished yet, then also trigger
1660 : // {kFinishedBaselineCompilation}.
1661 101613 : baseline_finished = tiering_finished && outstanding_baseline_units_ > 0;
1662 : } else {
1663 : DCHECK_LT(0, outstanding_baseline_units_);
1664 102095 : --outstanding_baseline_units_;
1665 : // If we are in tiering mode and tiering finished before, then do not
1666 : // trigger baseline finished.
1667 102095 : baseline_finished = outstanding_baseline_units_ == 0 &&
1668 5968 : (!is_tiering_mode || outstanding_tiering_units_ > 0);
1669 : // If we are not tiering, then we also trigger the "top tier finished"
1670 : // event when baseline compilation is finished.
1671 102095 : tiering_finished = baseline_finished && !is_tiering_mode;
1672 : }
1673 :
1674 203708 : if (baseline_finished) {
1675 7378 : for (auto& callback : callbacks_)
1676 : callback(CompilationEvent::kFinishedBaselineCompilation);
1677 : }
1678 203708 : if (tiering_finished) {
1679 7698 : for (auto& callback : callbacks_)
1680 : callback(CompilationEvent::kFinishedTopTierCompilation);
1681 : // Clear the callbacks because no more events will be delivered.
1682 : callbacks_.clear();
1683 : }
1684 :
1685 203708 : if (code != nullptr) native_module_->engine()->LogCode(code);
1686 203705 : }
1687 :
1688 32898 : void CompilationStateImpl::RestartBackgroundCompileTask() {
1689 : auto task =
1690 32898 : native_module_->engine()->NewBackgroundCompileTask<BackgroundCompileTask>(
1691 65796 : background_compile_token_, async_counters_);
1692 :
1693 32898 : if (baseline_compilation_finished()) {
1694 20392 : V8::GetCurrentPlatform()->CallLowPriorityTaskOnWorkerThread(
1695 20392 : std::move(task));
1696 : } else {
1697 68106 : V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
1698 : }
1699 32898 : }
1700 :
1701 65 : void CompilationStateImpl::ReportDetectedFeatures(
1702 : const WasmFeatures& detected) {
1703 65 : base::MutexGuard guard(&mutex_);
1704 65 : UnionFeaturesInto(&detected_features_, detected);
1705 65 : }
1706 :
1707 31305 : void CompilationStateImpl::OnBackgroundTaskStopped(
1708 : const WasmFeatures& detected) {
1709 31305 : base::MutexGuard guard(&mutex_);
1710 : DCHECK_LE(1, num_background_tasks_);
1711 31325 : --num_background_tasks_;
1712 31325 : UnionFeaturesInto(&detected_features_, detected);
1713 31323 : }
1714 :
1715 7372 : void CompilationStateImpl::PublishDetectedFeatures(
1716 : Isolate* isolate, const WasmFeatures& detected) {
1717 : // Notifying the isolate of the feature counts must take place under
1718 : // the mutex, because even if we have finished baseline compilation,
1719 : // tiering compilations may still occur in the background.
1720 7372 : base::MutexGuard guard(&mutex_);
1721 7372 : UnionFeaturesInto(&detected_features_, detected);
1722 : UpdateFeatureUseCounts(isolate, detected_features_);
1723 7372 : }
1724 :
1725 6629 : void CompilationStateImpl::RestartBackgroundTasks() {
1726 : int num_restart;
1727 : {
1728 6629 : base::MutexGuard guard(&mutex_);
1729 : // No need to restart tasks if compilation already failed.
1730 6629 : if (failed()) return;
1731 :
1732 : DCHECK_LE(num_background_tasks_, max_background_tasks_);
1733 6625 : if (num_background_tasks_ == max_background_tasks_) return;
1734 : size_t num_compilation_units =
1735 6625 : baseline_compilation_units_.size() + tiering_compilation_units_.size();
1736 6625 : num_restart = max_background_tasks_ - num_background_tasks_;
1737 : DCHECK_LE(0, num_restart);
1738 6625 : if (num_compilation_units < static_cast<size_t>(num_restart)) {
1739 4414 : num_restart = static_cast<int>(num_compilation_units);
1740 : }
1741 6625 : num_background_tasks_ += num_restart;
1742 : }
1743 :
1744 72291 : for (; num_restart > 0; --num_restart) {
1745 32833 : RestartBackgroundCompileTask();
1746 : }
1747 : }
1748 :
1749 7610 : void CompilationStateImpl::SetError() {
1750 7610 : bool expected = false;
1751 7610 : if (!compile_failed_.compare_exchange_strong(expected, true,
1752 : std::memory_order_relaxed)) {
1753 25 : return; // Already failed before.
1754 : }
1755 :
1756 7585 : base::MutexGuard callbacks_guard(&callbacks_mutex_);
1757 7665 : for (auto& callback : callbacks_) {
1758 : callback(CompilationEvent::kFailedCompilation);
1759 : }
1760 : // No more callbacks after an error.
1761 : callbacks_.clear();
1762 : }
1763 :
1764 139645 : void CompileJsToWasmWrappers(Isolate* isolate, const WasmModule* module,
1765 : Handle<FixedArray> export_wrappers) {
1766 : JSToWasmWrapperCache js_to_wasm_cache;
1767 : int wrapper_index = 0;
1768 :
1769 : // TODO(6792): Wrappers below are allocated with {Factory::NewCode}. As an
1770 : // optimization we keep the code space unlocked to avoid repeated unlocking
1771 : // because many such wrapper are allocated in sequence below.
1772 279292 : CodeSpaceMemoryModificationScope modification_scope(isolate->heap());
1773 362785 : for (auto exp : module->export_table) {
1774 223138 : if (exp.kind != kExternalFunction) continue;
1775 220386 : auto& function = module->functions[exp.index];
1776 : Handle<Code> wrapper_code = js_to_wasm_cache.GetOrCompileJSToWasmWrapper(
1777 220386 : isolate, function.sig, function.imported);
1778 440768 : export_wrappers->set(wrapper_index, *wrapper_code);
1779 220385 : RecordStats(*wrapper_code, isolate->counters());
1780 220386 : ++wrapper_index;
1781 : }
1782 139647 : }
1783 :
1784 137189 : Handle<Script> CreateWasmScript(Isolate* isolate,
1785 : const ModuleWireBytes& wire_bytes,
1786 : const std::string& source_map_url) {
1787 : Handle<Script> script =
1788 137189 : isolate->factory()->NewScript(isolate->factory()->empty_string());
1789 411569 : script->set_context_data(isolate->native_context()->debug_context_id());
1790 : script->set_type(Script::TYPE_WASM);
1791 :
1792 137189 : int hash = StringHasher::HashSequentialString(
1793 : reinterpret_cast<const char*>(wire_bytes.start()),
1794 137189 : static_cast<int>(wire_bytes.length()), kZeroHashSeed);
1795 :
1796 : const int kBufferSize = 32;
1797 : char buffer[kBufferSize];
1798 :
1799 137189 : int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash);
1800 : DCHECK(name_chars >= 0 && name_chars < kBufferSize);
1801 : MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte(
1802 137189 : VectorOf(reinterpret_cast<uint8_t*>(buffer), name_chars),
1803 137189 : AllocationType::kOld);
1804 274378 : script->set_name(*name_str.ToHandleChecked());
1805 :
1806 137189 : if (source_map_url.size() != 0) {
1807 : MaybeHandle<String> src_map_str = isolate->factory()->NewStringFromUtf8(
1808 4 : CStrVector(source_map_url.c_str()), AllocationType::kOld);
1809 8 : script->set_source_mapping_url(*src_map_str.ToHandleChecked());
1810 : }
1811 137189 : return script;
1812 : }
1813 :
1814 : } // namespace wasm
1815 : } // namespace internal
1816 120216 : } // namespace v8
1817 :
1818 : #undef TRACE_COMPILE
1819 : #undef TRACE_STREAMING
1820 : #undef TRACE_LAZY
|
