Line data Source code
1 : // Copyright 2017 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/wasm/module-compiler.h"
6 :
7 : #include "src/api.h"
8 : #include "src/asmjs/asm-js.h"
9 : #include "src/base/enum-set.h"
10 : #include "src/base/template-utils.h"
11 : #include "src/base/utils/random-number-generator.h"
12 : #include "src/compiler/wasm-compiler.h"
13 : #include "src/counters.h"
14 : #include "src/identity-map.h"
15 : #include "src/property-descriptor.h"
16 : #include "src/task-utils.h"
17 : #include "src/tracing/trace-event.h"
18 : #include "src/trap-handler/trap-handler.h"
19 : #include "src/wasm/js-to-wasm-wrapper-cache-inl.h"
20 : #include "src/wasm/module-decoder.h"
21 : #include "src/wasm/streaming-decoder.h"
22 : #include "src/wasm/wasm-code-manager.h"
23 : #include "src/wasm/wasm-engine.h"
24 : #include "src/wasm/wasm-import-wrapper-cache-inl.h"
25 : #include "src/wasm/wasm-js.h"
26 : #include "src/wasm/wasm-limits.h"
27 : #include "src/wasm/wasm-memory.h"
28 : #include "src/wasm/wasm-objects-inl.h"
29 : #include "src/wasm/wasm-result.h"
30 : #include "src/wasm/wasm-serialization.h"
31 :
32 : #define TRACE_COMPILE(...) \
33 : do { \
34 : if (FLAG_trace_wasm_compiler) PrintF(__VA_ARGS__); \
35 : } while (false)
36 :
37 : #define TRACE_STREAMING(...) \
38 : do { \
39 : if (FLAG_trace_wasm_streaming) PrintF(__VA_ARGS__); \
40 : } while (false)
41 :
42 : #define TRACE_LAZY(...) \
43 : do { \
44 : if (FLAG_trace_wasm_lazy_compilation) PrintF(__VA_ARGS__); \
45 : } while (false)
46 :
47 : namespace v8 {
48 : namespace internal {
49 : namespace wasm {
50 :
51 : namespace {
52 :
53 : enum class CompileMode : uint8_t { kRegular, kTiering };
54 :
55 : // The {CompilationStateImpl} keeps track of the compilation state of the
56 : // owning NativeModule, i.e. which functions are left to be compiled.
57 : // It contains a task manager to allow parallel and asynchronous background
58 : // compilation of functions.
59 : // It's public interface {CompilationState} lives in compilation-environment.h.
60 : class CompilationStateImpl {
61 : public:
62 : CompilationStateImpl(internal::Isolate*, NativeModule*);
63 : ~CompilationStateImpl();
64 :
65 : // Cancel all background compilation and wait for all tasks to finish. Call
66 : // this before destructing this object.
67 : void CancelAndWait();
68 :
69 : // Set the number of compilations unit expected to be executed. Needs to be
70 : // set before {AddCompilationUnits} is run, which triggers background
71 : // compilation.
72 : void SetNumberOfFunctionsToCompile(size_t num_functions);
73 :
74 : // Add the callback function to be called on compilation events. Needs to be
75 : // set before {AddCompilationUnits} is run.
76 : void AddCallback(CompilationState::callback_t);
77 :
78 : // Inserts new functions to compile and kicks off compilation.
79 : void AddCompilationUnits(
80 : std::vector<std::unique_ptr<WasmCompilationUnit>>& baseline_units,
81 : std::vector<std::unique_ptr<WasmCompilationUnit>>& tiering_units);
82 : std::unique_ptr<WasmCompilationUnit> GetNextCompilationUnit();
83 : std::unique_ptr<WasmCompilationUnit> GetNextExecutedUnit();
84 :
85 : bool HasCompilationUnitToFinish();
86 :
87 : void OnFinishedUnit(ExecutionTier, WasmCode*);
88 :
89 : void ReportDetectedFeatures(const WasmFeatures& detected);
90 : void OnBackgroundTaskStopped(const WasmFeatures& detected);
91 : void PublishDetectedFeatures(Isolate* isolate, const WasmFeatures& detected);
92 : void RestartBackgroundCompileTask();
93 : void RestartBackgroundTasks(size_t max = std::numeric_limits<size_t>::max());
94 : // Only one foreground thread (finisher) is allowed to run at a time.
95 : // {SetFinisherIsRunning} returns whether the flag changed its state.
96 : bool SetFinisherIsRunning(bool value);
97 : void ScheduleFinisherTask();
98 :
99 : void Abort();
100 :
101 : void SetError(uint32_t func_index, const WasmError& error);
102 :
103 : Isolate* isolate() const { return isolate_; }
104 :
105 : bool failed() const {
106 9679 : return compile_error_.load(std::memory_order_relaxed) != nullptr;
107 : }
108 :
109 686702 : bool baseline_compilation_finished() const {
110 686702 : base::MutexGuard guard(&mutex_);
111 1342886 : return outstanding_baseline_units_ == 0 ||
112 1281896 : (compile_mode_ == CompileMode::kTiering &&
113 1312414 : outstanding_tiering_units_ == 0);
114 : }
115 :
116 : CompileMode compile_mode() const { return compile_mode_; }
117 : WasmFeatures* detected_features() { return &detected_features_; }
118 :
119 : // Call {GetCompileError} from foreground threads only, since we access
120 : // NativeModule::wire_bytes, which is set from the foreground thread once the
121 : // stream has finished.
122 13342 : WasmError GetCompileError() {
123 : CompilationError* error = compile_error_.load(std::memory_order_acquire);
124 : DCHECK_NOT_NULL(error);
125 13342 : std::ostringstream error_msg;
126 13342 : error_msg << "Compiling wasm function \"";
127 13342 : wasm::ModuleWireBytes wire_bytes(native_module_->wire_bytes());
128 : wasm::WireBytesRef name_ref = native_module_->module()->LookupFunctionName(
129 26684 : wire_bytes, error->func_index);
130 13342 : if (name_ref.is_set()) {
131 279 : wasm::WasmName name = wire_bytes.GetNameOrNull(name_ref);
132 558 : error_msg.write(name.start(), name.length());
133 : } else {
134 26126 : error_msg << "wasm-function[" << error->func_index << "]";
135 : }
136 26684 : error_msg << "\" failed: " << error->error.message();
137 13342 : return WasmError{error->error.offset(), error_msg.str()};
138 : }
139 :
140 2900482 : void SetWireBytesStorage(
141 : std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
142 2900482 : base::MutexGuard guard(&mutex_);
143 : wire_bytes_storage_ = wire_bytes_storage;
144 2900482 : }
145 :
146 1754498 : std::shared_ptr<WireBytesStorage> GetWireBytesStorage() const {
147 1754498 : base::MutexGuard guard(&mutex_);
148 : DCHECK_NOT_NULL(wire_bytes_storage_);
149 1754499 : return wire_bytes_storage_;
150 : }
151 :
152 : private:
153 : struct CompilationError {
154 : uint32_t const func_index;
155 : WasmError const error;
156 : CompilationError(uint32_t func_index, WasmError error)
157 6930 : : func_index(func_index), error(std::move(error)) {}
158 : };
159 :
160 0 : class LogCodesTask : public CancelableTask {
161 : public:
162 : LogCodesTask(CancelableTaskManager* manager,
163 : CompilationStateImpl* compilation_state, Isolate* isolate)
164 : : CancelableTask(manager),
165 : compilation_state_(compilation_state),
166 0 : isolate_(isolate) {
167 : // This task should only be created if we should actually log code.
168 : DCHECK(WasmCode::ShouldBeLogged(isolate));
169 : }
170 :
171 : // Hold the compilation state {mutex_} when calling this method.
172 0 : void AddCode(WasmCode* code) { code_to_log_.push_back(code); }
173 :
174 0 : void RunInternal() override {
175 : // Remove this task from the {CompilationStateImpl}. The next compilation
176 : // that finishes will allocate and schedule a new task.
177 : {
178 0 : base::MutexGuard guard(&compilation_state_->mutex_);
179 : DCHECK_EQ(this, compilation_state_->log_codes_task_);
180 0 : compilation_state_->log_codes_task_ = nullptr;
181 : }
182 : // If by now we shouldn't log code any more, don't log it.
183 0 : if (!WasmCode::ShouldBeLogged(isolate_)) return;
184 0 : for (WasmCode* code : code_to_log_) {
185 0 : code->LogCode(isolate_);
186 : }
187 : }
188 :
189 : private:
190 : CompilationStateImpl* const compilation_state_;
191 : Isolate* const isolate_;
192 : std::vector<WasmCode*> code_to_log_;
193 : };
194 :
195 170 : class FreeCallbacksTask : public CancelableTask {
196 : public:
197 : explicit FreeCallbacksTask(CompilationStateImpl* comp_state)
198 : : CancelableTask(&comp_state->foreground_task_manager_),
199 85 : compilation_state_(comp_state) {}
200 :
201 55 : void RunInternal() override { compilation_state_->callbacks_.clear(); }
202 :
203 : private:
204 : CompilationStateImpl* const compilation_state_;
205 : };
206 :
207 : void NotifyOnEvent(CompilationEvent event, const WasmError* error);
208 :
209 : std::vector<std::unique_ptr<WasmCompilationUnit>>& finish_units() {
210 324591 : return baseline_compilation_finished() ? tiering_finish_units_
211 324591 : : baseline_finish_units_;
212 : }
213 :
214 : // TODO(mstarzinger): Get rid of the Isolate field to make sure the
215 : // {CompilationStateImpl} can be shared across multiple Isolates.
216 : Isolate* const isolate_;
217 : NativeModule* const native_module_;
218 : const CompileMode compile_mode_;
219 : // Store the value of {WasmCode::ShouldBeLogged()} at creation time of the
220 : // compilation state.
221 : // TODO(wasm): We might lose log events if logging is enabled while
222 : // compilation is running.
223 : bool const should_log_code_;
224 :
225 : // Compilation error, atomically updated, but at most once (nullptr -> error).
226 : // Uses acquire-release semantics (acquire on load, release on update).
227 : // For checking whether an error is set, relaxed semantics can be used.
228 : std::atomic<CompilationError*> compile_error_{nullptr};
229 :
230 : // This mutex protects all information of this {CompilationStateImpl} which is
231 : // being accessed concurrently.
232 : mutable base::Mutex mutex_;
233 :
234 : //////////////////////////////////////////////////////////////////////////////
235 : // Protected by {mutex_}:
236 :
237 : std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_compilation_units_;
238 : std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_compilation_units_;
239 :
240 : bool finisher_is_running_ = false;
241 : size_t num_background_tasks_ = 0;
242 :
243 : std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_finish_units_;
244 : std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_finish_units_;
245 :
246 : // Features detected to be used in this module. Features can be detected
247 : // as a module is being compiled.
248 : WasmFeatures detected_features_ = kNoWasmFeatures;
249 :
250 : // The foreground task to log finished wasm code. Is {nullptr} if no such task
251 : // is currently scheduled.
252 : LogCodesTask* log_codes_task_ = nullptr;
253 :
254 : // Abstraction over the storage of the wire bytes. Held in a shared_ptr so
255 : // that background compilation jobs can keep the storage alive while
256 : // compiling.
257 : std::shared_ptr<WireBytesStorage> wire_bytes_storage_;
258 :
259 : size_t outstanding_baseline_units_ = 0;
260 : size_t outstanding_tiering_units_ = 0;
261 :
262 : // End of fields protected by {mutex_}.
263 : //////////////////////////////////////////////////////////////////////////////
264 :
265 : // Callback functions to be called on compilation events. Only accessible from
266 : // the foreground thread.
267 : std::vector<CompilationState::callback_t> callbacks_;
268 :
269 : // Remember whether {Abort()} was called. When set from the foreground this
270 : // ensures no more callbacks will be called afterwards. No guarantees when set
271 : // from the background. Only needs to be atomic so that it can be set from
272 : // foreground and background.
273 : std::atomic<bool> aborted_{false};
274 :
275 : CancelableTaskManager background_task_manager_;
276 : CancelableTaskManager foreground_task_manager_;
277 : std::shared_ptr<v8::TaskRunner> foreground_task_runner_;
278 :
279 : const size_t max_background_tasks_ = 0;
280 : };
281 :
282 : void UpdateFeatureUseCounts(Isolate* isolate, const WasmFeatures& detected) {
283 159412 : if (detected.threads) {
284 1367 : isolate->CountUsage(v8::Isolate::UseCounterFeature::kWasmThreadOpcodes);
285 : }
286 : }
287 :
288 : CompilationStateImpl* Impl(CompilationState* compilation_state) {
289 : return reinterpret_cast<CompilationStateImpl*>(compilation_state);
290 : }
291 : const CompilationStateImpl* Impl(const CompilationState* compilation_state) {
292 : return reinterpret_cast<const CompilationStateImpl*>(compilation_state);
293 : }
294 :
295 : } // namespace
296 :
297 : //////////////////////////////////////////////////////
298 : // PIMPL implementation of {CompilationState}.
299 :
300 1530590 : CompilationState::~CompilationState() { Impl(this)->~CompilationStateImpl(); }
301 :
302 3061180 : void CompilationState::CancelAndWait() { Impl(this)->CancelAndWait(); }
303 :
304 6739 : void CompilationState::SetError(uint32_t func_index, const WasmError& error) {
305 6739 : Impl(this)->SetError(func_index, error);
306 6739 : }
307 :
308 2900482 : void CompilationState::SetWireBytesStorage(
309 : std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
310 5800964 : Impl(this)->SetWireBytesStorage(std::move(wire_bytes_storage));
311 2900482 : }
312 :
313 1516483 : std::shared_ptr<WireBytesStorage> CompilationState::GetWireBytesStorage()
314 : const {
315 1526144 : return Impl(this)->GetWireBytesStorage();
316 : }
317 :
318 5 : void CompilationState::AddCallback(CompilationState::callback_t callback) {
319 10 : return Impl(this)->AddCallback(std::move(callback));
320 : }
321 :
322 36 : bool CompilationState::failed() const { return Impl(this)->failed(); }
323 :
324 : // static
325 1530590 : std::unique_ptr<CompilationState> CompilationState::New(
326 : Isolate* isolate, NativeModule* native_module) {
327 : return std::unique_ptr<CompilationState>(reinterpret_cast<CompilationState*>(
328 3061180 : new CompilationStateImpl(isolate, native_module)));
329 : }
330 :
331 : // End of PIMPL implementation of {CompilationState}.
332 : //////////////////////////////////////////////////////
333 :
334 9661 : WasmCode* LazyCompileFunction(Isolate* isolate, NativeModule* native_module,
335 : int func_index) {
336 : base::ElapsedTimer compilation_timer;
337 : DCHECK(!native_module->has_code(static_cast<uint32_t>(func_index)));
338 :
339 : compilation_timer.Start();
340 :
341 : TRACE_LAZY("Compiling wasm-function#%d.\n", func_index);
342 :
343 : const uint8_t* module_start = native_module->wire_bytes().start();
344 :
345 19322 : const WasmFunction* func = &native_module->module()->functions[func_index];
346 : FunctionBody func_body{func->sig, func->code.offset(),
347 : module_start + func->code.offset(),
348 : module_start + func->code.end_offset()};
349 :
350 : WasmCompilationUnit unit(
351 : isolate->wasm_engine(), func_index,
352 19322 : WasmCompilationUnit::GetDefaultExecutionTier(native_module->module()));
353 9661 : CompilationEnv env = native_module->CreateCompilationEnv();
354 : unit.ExecuteCompilation(
355 : &env, native_module,
356 : native_module->compilation_state()->GetWireBytesStorage(),
357 : isolate->counters(),
358 28983 : Impl(native_module->compilation_state())->detected_features());
359 :
360 : // During lazy compilation, we should never get compilation errors. The module
361 : // was verified before starting execution with lazy compilation.
362 : // This might be OOM, but then we cannot continue execution anyway.
363 : // TODO(clemensh): According to the spec, we can actually skip validation at
364 : // module creation time, and return a function that always traps here.
365 9661 : CHECK(!native_module->compilation_state()->failed());
366 :
367 9661 : WasmCode* code = unit.result();
368 :
369 9661 : if (WasmCode::ShouldBeLogged(isolate)) code->LogCode(isolate);
370 :
371 : int64_t func_size =
372 9661 : static_cast<int64_t>(func->code.end_offset() - func->code.offset());
373 9661 : int64_t compilation_time = compilation_timer.Elapsed().InMicroseconds();
374 :
375 : auto counters = isolate->counters();
376 9661 : counters->wasm_lazily_compiled_functions()->Increment();
377 :
378 : counters->wasm_lazy_compilation_throughput()->AddSample(
379 9661 : compilation_time != 0 ? static_cast<int>(func_size / compilation_time)
380 19322 : : 0);
381 :
382 9661 : return code;
383 : }
384 :
385 9661 : Address CompileLazy(Isolate* isolate, NativeModule* native_module,
386 : uint32_t func_index) {
387 : HistogramTimerScope lazy_time_scope(
388 9661 : isolate->counters()->wasm_lazy_compilation_time());
389 :
390 : DCHECK(!native_module->lazy_compile_frozen());
391 :
392 19322 : NativeModuleModificationScope native_module_modification_scope(native_module);
393 :
394 9661 : WasmCode* result = LazyCompileFunction(isolate, native_module, func_index);
395 : DCHECK_NOT_NULL(result);
396 : DCHECK_EQ(func_index, result->index());
397 :
398 9661 : return result->instruction_start();
399 : }
400 :
401 : namespace {
402 :
403 : // The {CompilationUnitBuilder} builds compilation units and stores them in an
404 : // internal buffer. The buffer is moved into the working queue of the
405 : // {CompilationStateImpl} when {Commit} is called.
406 6852 : class CompilationUnitBuilder {
407 : public:
408 : explicit CompilationUnitBuilder(NativeModule* native_module,
409 : WasmEngine* wasm_engine)
410 : : native_module_(native_module),
411 : wasm_engine_(wasm_engine),
412 : default_tier_(WasmCompilationUnit::GetDefaultExecutionTier(
413 7081 : native_module->module())) {}
414 :
415 114696 : void AddUnit(uint32_t func_index) {
416 114696 : switch (compilation_state()->compile_mode()) {
417 : case CompileMode::kTiering:
418 : tiering_units_.emplace_back(
419 229020 : CreateUnit(func_index, ExecutionTier::kOptimized));
420 : baseline_units_.emplace_back(
421 229020 : CreateUnit(func_index, ExecutionTier::kBaseline));
422 114510 : return;
423 : case CompileMode::kRegular:
424 558 : baseline_units_.emplace_back(CreateUnit(func_index, default_tier_));
425 186 : return;
426 : }
427 0 : UNREACHABLE();
428 : }
429 :
430 13824 : bool Commit() {
431 7017 : if (baseline_units_.empty() && tiering_units_.empty()) return false;
432 13754 : compilation_state()->AddCompilationUnits(baseline_units_, tiering_units_);
433 6877 : Clear();
434 6877 : return true;
435 : }
436 :
437 6927 : void Clear() {
438 : baseline_units_.clear();
439 : tiering_units_.clear();
440 6927 : }
441 :
442 : private:
443 : std::unique_ptr<WasmCompilationUnit> CreateUnit(uint32_t func_index,
444 : ExecutionTier tier) {
445 : return base::make_unique<WasmCompilationUnit>(wasm_engine_, func_index,
446 229206 : tier);
447 : }
448 :
449 : CompilationStateImpl* compilation_state() const {
450 : return Impl(native_module_->compilation_state());
451 : }
452 :
453 : NativeModule* const native_module_;
454 : WasmEngine* const wasm_engine_;
455 : const ExecutionTier default_tier_;
456 : std::vector<std::unique_ptr<WasmCompilationUnit>> baseline_units_;
457 : std::vector<std::unique_ptr<WasmCompilationUnit>> tiering_units_;
458 : };
459 :
460 : bool compile_lazy(const WasmModule* module) {
461 160209 : return FLAG_wasm_lazy_compilation ||
462 159660 : (FLAG_asm_wasm_lazy_compilation && module->origin == kAsmJsOrigin);
463 : }
464 :
465 247021 : void RecordStats(const Code code, Counters* counters) {
466 247021 : counters->wasm_generated_code_size()->Increment(code->body_size());
467 494039 : counters->wasm_reloc_size()->Increment(code->relocation_info()->length());
468 247022 : }
469 :
470 257612 : double MonotonicallyIncreasingTimeInMs() {
471 257612 : return V8::GetCurrentPlatform()->MonotonicallyIncreasingTime() *
472 257611 : base::Time::kMillisecondsPerSecond;
473 : }
474 :
475 : // Run by each compilation task and by the main thread (i.e. in both
476 : // foreground and background threads). The no_finisher_callback is called
477 : // within the result_mutex_ lock when no finishing task is running, i.e. when
478 : // the finisher_is_running_ flag is not set.
479 265807 : bool FetchAndExecuteCompilationUnit(CompilationEnv* env,
480 : NativeModule* native_module,
481 : CompilationStateImpl* compilation_state,
482 : WasmFeatures* detected,
483 : Counters* counters) {
484 : DisallowHeapAccess no_heap_access;
485 :
486 : std::unique_ptr<WasmCompilationUnit> unit =
487 265807 : compilation_state->GetNextCompilationUnit();
488 265918 : if (unit == nullptr) return false;
489 :
490 : // Get the tier before starting compilation, as compilation can switch tiers
491 : // if baseline bails out.
492 228354 : ExecutionTier tier = unit->tier();
493 : unit->ExecuteCompilation(env, native_module,
494 : compilation_state->GetWireBytesStorage(), counters,
495 685066 : detected);
496 228356 : compilation_state->OnFinishedUnit(tier, unit->result());
497 :
498 228360 : return true;
499 : }
500 :
501 6623 : void InitializeCompilationUnits(NativeModule* native_module,
502 : WasmEngine* wasm_engine) {
503 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
504 : const WasmModule* module = native_module->module();
505 : CompilationUnitBuilder builder(native_module, wasm_engine);
506 6623 : uint32_t start = module->num_imported_functions;
507 6623 : uint32_t end = start + module->num_declared_functions;
508 120816 : for (uint32_t i = start; i < end; ++i) {
509 114193 : builder.AddUnit(i);
510 : }
511 6623 : builder.Commit();
512 6623 : }
513 :
514 324707 : void FinishCompilationUnits(CompilationStateImpl* compilation_state) {
515 649414 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"), "FinishCompilationUnits");
516 324707 : while (!compilation_state->failed()) {
517 : std::unique_ptr<WasmCompilationUnit> unit =
518 324591 : compilation_state->GetNextExecutedUnit();
519 324591 : if (unit == nullptr) break;
520 324707 : }
521 324707 : }
522 :
523 5273 : void CompileInParallel(Isolate* isolate, NativeModule* native_module) {
524 : // Data structures for the parallel compilation.
525 :
526 : //-----------------------------------------------------------------------
527 : // For parallel compilation:
528 : // 1) The main thread allocates a compilation unit for each wasm function
529 : // and stores them in the vector {compilation_units} within the
530 : // {compilation_state}. By adding units to the {compilation_state}, new
531 : // {BackgroundCompileTasks} instances are spawned which run on
532 : // the background threads.
533 : // 2) The background threads and the main thread pick one compilation unit at
534 : // a time and execute the parallel phase of the compilation unit.
535 : // 3) After the parallel phase of all compilation units has started, the
536 : // main thread continues to finish all compilation units as long as
537 : // baseline-compilation units are left to be processed.
538 : // 4) If tier-up is enabled, the main thread restarts background tasks
539 : // that take care of compiling and finishing the top-tier compilation
540 : // units.
541 :
542 : // Turn on the {CanonicalHandleScope} so that the background threads can
543 : // use the node cache.
544 5273 : CanonicalHandleScope canonical(isolate);
545 :
546 5156 : CompilationStateImpl* compilation_state =
547 : Impl(native_module->compilation_state());
548 : // Make sure that no foreground task is spawned for finishing
549 : // the compilation units. This foreground thread will be
550 : // responsible for finishing compilation.
551 5273 : compilation_state->SetFinisherIsRunning(true);
552 : uint32_t num_wasm_functions =
553 5273 : native_module->num_functions() - native_module->num_imported_functions();
554 5273 : compilation_state->SetNumberOfFunctionsToCompile(num_wasm_functions);
555 :
556 : // 1) The main thread allocates a compilation unit for each wasm function
557 : // and stores them in the vector {compilation_units} within the
558 : // {compilation_state}. By adding units to the {compilation_state}, new
559 : // {BackgroundCompileTask} instances are spawned which run on
560 : // background threads.
561 5273 : InitializeCompilationUnits(native_module, isolate->wasm_engine());
562 :
563 : // 2) The background threads and the main thread pick one compilation unit at
564 : // a time and execute the parallel phase of the compilation unit.
565 5273 : WasmFeatures detected_features;
566 5273 : CompilationEnv env = native_module->CreateCompilationEnv();
567 14240 : while (FetchAndExecuteCompilationUnit(&env, native_module, compilation_state,
568 : &detected_features,
569 14116 : isolate->counters()) &&
570 5149 : !compilation_state->baseline_compilation_finished()) {
571 : // TODO(clemensh): Refactor ownership of the AsyncCompileJob and remove
572 : // this.
573 3810 : FinishCompilationUnits(compilation_state);
574 :
575 3810 : if (compilation_state->failed()) break;
576 : }
577 :
578 321014 : while (!compilation_state->failed()) {
579 : // 3) After the parallel phase of all compilation units has started, the
580 : // main thread continues to finish compilation units as long as
581 : // baseline compilation units are left to be processed. If compilation
582 : // already failed, all background tasks have already been canceled
583 : // in {FinishCompilationUnits}, and there are no units to finish.
584 320897 : FinishCompilationUnits(compilation_state);
585 :
586 320897 : if (compilation_state->baseline_compilation_finished()) break;
587 : }
588 :
589 : // Publish features from the foreground and background tasks.
590 5273 : compilation_state->PublishDetectedFeatures(isolate, detected_features);
591 :
592 : // 4) If tiering-compilation is enabled, we need to set the finisher
593 : // to false, such that the background threads will spawn a foreground
594 : // thread to finish the top-tier compilation units.
595 10429 : if (!compilation_state->failed() &&
596 : compilation_state->compile_mode() == CompileMode::kTiering) {
597 5123 : compilation_state->SetFinisherIsRunning(false);
598 5273 : }
599 5273 : }
600 :
601 151758 : void CompileSequentially(Isolate* isolate, NativeModule* native_module,
602 : ErrorThrower* thrower) {
603 : DCHECK(!thrower->error());
604 :
605 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
606 : const WasmModule* module = native_module->module();
607 151758 : WasmFeatures detected = kNoWasmFeatures;
608 : auto* comp_state = Impl(native_module->compilation_state());
609 : ExecutionTier tier =
610 151758 : WasmCompilationUnit::GetDefaultExecutionTier(native_module->module());
611 567657 : for (const WasmFunction& func : module->functions) {
612 270702 : if (func.imported) continue; // Imports are compiled at instantiation time.
613 :
614 : // Compile the function.
615 : WasmCompilationUnit::CompileWasmFunction(isolate, native_module, &detected,
616 145808 : &func, tier);
617 145807 : if (comp_state->failed()) {
618 13124 : thrower->CompileFailed(comp_state->GetCompileError());
619 6562 : break;
620 : }
621 : }
622 : UpdateFeatureUseCounts(isolate, detected);
623 151760 : }
624 :
625 702 : void ValidateSequentially(Isolate* isolate, NativeModule* native_module,
626 : ErrorThrower* thrower) {
627 : DCHECK(!thrower->error());
628 :
629 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
630 : const WasmModule* module = native_module->module();
631 198 : uint32_t start = module->num_imported_functions;
632 198 : uint32_t end = start + module->num_declared_functions;
633 693 : for (uint32_t i = start; i < end; ++i) {
634 504 : const WasmFunction& func = module->functions[i];
635 :
636 : const byte* base = wire_bytes.start();
637 : FunctionBody body{func.sig, func.code.offset(), base + func.code.offset(),
638 504 : base + func.code.end_offset()};
639 : DecodeResult result;
640 : {
641 1008 : auto time_counter = SELECT_WASM_COUNTER(
642 : isolate->counters(), module->origin, wasm_decode, function_time);
643 :
644 : TimedHistogramScope wasm_decode_function_time_scope(time_counter);
645 504 : WasmFeatures detected;
646 1008 : result = VerifyWasmCode(isolate->allocator(),
647 504 : native_module->enabled_features(), module,
648 : &detected, body);
649 : }
650 504 : if (result.failed()) {
651 9 : TruncatedUserString<> name(wire_bytes.GetNameOrNull(&func, module));
652 : thrower->CompileError("Compiling function #%d:%.*s failed: %s @+%u", i,
653 : name.length(), name.start(),
654 : result.error().message().c_str(),
655 18 : result.error().offset());
656 : break;
657 : }
658 : }
659 198 : }
660 :
661 160407 : void CompileNativeModule(Isolate* isolate, ErrorThrower* thrower,
662 : const WasmModule* wasm_module,
663 : NativeModule* native_module) {
664 : ModuleWireBytes wire_bytes(native_module->wire_bytes());
665 :
666 160209 : if (compile_lazy(wasm_module)) {
667 3178 : if (wasm_module->origin == kWasmOrigin) {
668 : // Validate wasm modules for lazy compilation. Don't validate asm.js
669 : // modules, they are valid by construction (otherwise a CHECK will fail
670 : // during lazy compilation).
671 : // TODO(clemensh): According to the spec, we can actually skip validation
672 : // at module creation time, and return a function that always traps at
673 : // (lazy) compilation time.
674 198 : ValidateSequentially(isolate, native_module, thrower);
675 160407 : if (thrower->error()) return;
676 : }
677 :
678 3169 : native_module->SetLazyBuiltin(BUILTIN_CODE(isolate, WasmCompileLazy));
679 : } else {
680 : size_t funcs_to_compile =
681 314062 : wasm_module->functions.size() - wasm_module->num_imported_functions;
682 : bool compile_parallel =
683 314053 : !FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks > 0 &&
684 162304 : funcs_to_compile > 1 &&
685 5273 : V8::GetCurrentPlatform()->NumberOfWorkerThreads() > 0;
686 :
687 157031 : if (compile_parallel) {
688 5273 : CompileInParallel(isolate, native_module);
689 : } else {
690 151758 : CompileSequentially(isolate, native_module, thrower);
691 : }
692 : auto* compilation_state = Impl(native_module->compilation_state());
693 157032 : if (compilation_state->failed()) {
694 13358 : thrower->CompileFailed(compilation_state->GetCompileError());
695 : }
696 : }
697 : }
698 :
699 : // The runnable task that finishes compilation in foreground (e.g. updating
700 : // the NativeModule, the code table, etc.).
701 : class FinishCompileTask : public CancelableTask {
702 : public:
703 : explicit FinishCompileTask(CompilationStateImpl* compilation_state,
704 : CancelableTaskManager* task_manager)
705 : : CancelableTask(task_manager), compilation_state_(compilation_state) {}
706 :
707 : void RunInternal() override {
708 : Isolate* isolate = compilation_state_->isolate();
709 : HandleScope scope(isolate);
710 : SaveContext saved_context(isolate);
711 : isolate->set_context(Context());
712 :
713 : TRACE_COMPILE("(4a) Finishing compilation units...\n");
714 : if (compilation_state_->failed()) {
715 : compilation_state_->SetFinisherIsRunning(false);
716 : return;
717 : }
718 :
719 : // We execute for 1 ms and then reschedule the task, same as the GC.
720 : double deadline = MonotonicallyIncreasingTimeInMs() + 1.0;
721 : while (true) {
722 : compilation_state_->RestartBackgroundTasks();
723 :
724 : std::unique_ptr<WasmCompilationUnit> unit =
725 : compilation_state_->GetNextExecutedUnit();
726 :
727 : if (unit == nullptr) {
728 : // It might happen that a background task just scheduled a unit to be
729 : // finished, but did not start a finisher task since the flag was still
730 : // set. Check for this case, and continue if there is more work.
731 : compilation_state_->SetFinisherIsRunning(false);
732 : if (compilation_state_->HasCompilationUnitToFinish() &&
733 : compilation_state_->SetFinisherIsRunning(true)) {
734 : continue;
735 : }
736 : break;
737 : }
738 :
739 : if (compilation_state_->failed()) break;
740 :
741 : if (deadline < MonotonicallyIncreasingTimeInMs()) {
742 : // We reached the deadline. We reschedule this task and return
743 : // immediately. Since we rescheduled this task already, we do not set
744 : // the FinisherIsRunning flag to false.
745 : compilation_state_->ScheduleFinisherTask();
746 : return;
747 : }
748 : }
749 : }
750 :
751 : private:
752 : CompilationStateImpl* compilation_state_;
753 : };
754 :
755 : // The runnable task that performs compilations in the background.
756 69258 : class BackgroundCompileTask : public CancelableTask {
757 : public:
758 : explicit BackgroundCompileTask(CancelableTaskManager* task_manager,
759 : NativeModule* native_module,
760 : Counters* counters)
761 : : CancelableTask(task_manager),
762 : native_module_(native_module),
763 34635 : counters_(counters) {}
764 :
765 34416 : void RunInternal() override {
766 : TRACE_COMPILE("(3b) Compiling...\n");
767 68835 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
768 : "BackgroundCompileTask::RunInternal");
769 : // The number of currently running background tasks is reduced in
770 : // {OnBackgroundTaskStopped}.
771 34419 : CompilationEnv env = native_module_->CreateCompilationEnv();
772 34410 : auto* compilation_state = Impl(native_module_->compilation_state());
773 34410 : WasmFeatures detected_features = kNoWasmFeatures;
774 34410 : double deadline = MonotonicallyIncreasingTimeInMs() + 50.0;
775 291979 : while (!compilation_state->failed()) {
776 256954 : if (!FetchAndExecuteCompilationUnit(&env, native_module_,
777 : compilation_state, &detected_features,
778 256901 : counters_)) {
779 : break;
780 : }
781 223208 : if (deadline < MonotonicallyIncreasingTimeInMs()) {
782 59 : compilation_state->ReportDetectedFeatures(detected_features);
783 59 : compilation_state->RestartBackgroundCompileTask();
784 34499 : return;
785 : }
786 : }
787 34407 : compilation_state->OnBackgroundTaskStopped(detected_features);
788 : }
789 :
790 : private:
791 : NativeModule* const native_module_;
792 : Counters* const counters_;
793 : };
794 :
795 : } // namespace
796 :
797 160153 : std::unique_ptr<NativeModule> CompileToNativeModule(
798 160165 : Isolate* isolate, const WasmFeatures& enabled, ErrorThrower* thrower,
799 : std::shared_ptr<const WasmModule> module, const ModuleWireBytes& wire_bytes,
800 : Handle<FixedArray>* export_wrappers_out) {
801 480473 : const WasmModule* wasm_module = module.get();
802 160153 : TimedHistogramScope wasm_compile_module_time_scope(SELECT_WASM_COUNTER(
803 320306 : isolate->counters(), wasm_module->origin, wasm_compile, module_time));
804 :
805 : // Embedder usage count for declared shared memories.
806 160163 : if (wasm_module->has_shared_memory) {
807 1386 : isolate->CountUsage(v8::Isolate::UseCounterFeature::kWasmSharedMemory);
808 : }
809 160159 : int export_wrapper_size = static_cast<int>(module->num_exported_functions);
810 :
811 : // TODO(wasm): only save the sections necessary to deserialize a
812 : // {WasmModule}. E.g. function bodies could be omitted.
813 : OwnedVector<uint8_t> wire_bytes_copy =
814 160159 : OwnedVector<uint8_t>::Of(wire_bytes.module_bytes());
815 :
816 : // Create and compile the native module.
817 : size_t code_size_estimate =
818 160161 : wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
819 :
820 : // Create a new {NativeModule} first.
821 : auto native_module = isolate->wasm_engine()->code_manager()->NewNativeModule(
822 : isolate, enabled, code_size_estimate,
823 320322 : wasm::NativeModule::kCanAllocateMoreMemory, std::move(module));
824 320334 : native_module->SetWireBytes(std::move(wire_bytes_copy));
825 160167 : native_module->SetRuntimeStubs(isolate);
826 :
827 160167 : CompileNativeModule(isolate, thrower, wasm_module, native_module.get());
828 160165 : if (thrower->error()) return {};
829 :
830 : // Compile JS->wasm wrappers for exported functions.
831 : *export_wrappers_out =
832 153477 : isolate->factory()->NewFixedArray(export_wrapper_size, TENURED);
833 : CompileJsToWasmWrappers(isolate, native_module->module(),
834 153478 : *export_wrappers_out);
835 :
836 : // Log the code within the generated module for profiling.
837 153479 : native_module->LogWasmCodes(isolate);
838 :
839 : return native_module;
840 : }
841 :
842 43 : void CompileNativeModuleWithExplicitBoundsChecks(Isolate* isolate,
843 : ErrorThrower* thrower,
844 : const WasmModule* wasm_module,
845 : NativeModule* native_module) {
846 43 : native_module->DisableTrapHandler();
847 43 : CompileNativeModule(isolate, thrower, wasm_module, native_module);
848 43 : }
849 :
850 2685 : AsyncCompileJob::AsyncCompileJob(
851 : Isolate* isolate, const WasmFeatures& enabled,
852 : std::unique_ptr<byte[]> bytes_copy, size_t length, Handle<Context> context,
853 : std::shared_ptr<CompilationResultResolver> resolver)
854 : : isolate_(isolate),
855 : enabled_features_(enabled),
856 : bytes_copy_(std::move(bytes_copy)),
857 : wire_bytes_(bytes_copy_.get(), bytes_copy_.get() + length),
858 10740 : resolver_(std::move(resolver)) {
859 : v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate);
860 2685 : v8::Platform* platform = V8::GetCurrentPlatform();
861 5370 : foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate);
862 : // The handle for the context must be deferred.
863 2685 : DeferredHandleScope deferred(isolate);
864 5370 : native_context_ = Handle<Context>(context->native_context(), isolate);
865 : DCHECK(native_context_->IsNativeContext());
866 5370 : deferred_handles_.push_back(deferred.Detach());
867 2685 : }
868 :
869 2261 : void AsyncCompileJob::Start() {
870 6783 : DoAsync<DecodeModule>(isolate_->counters()); // --
871 2261 : }
872 :
873 65 : void AsyncCompileJob::Abort() {
874 : // Removing this job will trigger the destructor, which will cancel all
875 : // compilation.
876 195 : isolate_->wasm_engine()->RemoveCompileJob(this);
877 65 : }
878 :
879 1696 : class AsyncStreamingProcessor final : public StreamingProcessor {
880 : public:
881 : explicit AsyncStreamingProcessor(AsyncCompileJob* job);
882 :
883 : bool ProcessModuleHeader(Vector<const uint8_t> bytes,
884 : uint32_t offset) override;
885 :
886 : bool ProcessSection(SectionCode section_code, Vector<const uint8_t> bytes,
887 : uint32_t offset) override;
888 :
889 : bool ProcessCodeSectionHeader(size_t functions_count, uint32_t offset,
890 : std::shared_ptr<WireBytesStorage>) override;
891 :
892 : bool ProcessFunctionBody(Vector<const uint8_t> bytes,
893 : uint32_t offset) override;
894 :
895 : void OnFinishedChunk() override;
896 :
897 : void OnFinishedStream(OwnedVector<uint8_t> bytes) override;
898 :
899 : void OnError(const WasmError&) override;
900 :
901 : void OnAbort() override;
902 :
903 : bool Deserialize(Vector<const uint8_t> wire_bytes,
904 : Vector<const uint8_t> module_bytes) override;
905 :
906 : private:
907 : // Finishes the AsyncCompileJob with an error.
908 : void FinishAsyncCompileJobWithError(const WasmError&);
909 :
910 : void CommitCompilationUnits();
911 :
912 : ModuleDecoder decoder_;
913 : AsyncCompileJob* job_;
914 : std::unique_ptr<CompilationUnitBuilder> compilation_unit_builder_;
915 : uint32_t next_function_ = 0;
916 : };
917 :
918 424 : std::shared_ptr<StreamingDecoder> AsyncCompileJob::CreateStreamingDecoder() {
919 : DCHECK_NULL(stream_);
920 : stream_.reset(
921 1696 : new StreamingDecoder(base::make_unique<AsyncStreamingProcessor>(this)));
922 424 : return stream_;
923 : }
924 :
925 5370 : AsyncCompileJob::~AsyncCompileJob() {
926 2685 : background_task_manager_.CancelAndWait();
927 : // If the runtime objects were not created yet, then initial compilation did
928 : // not finish yet. In this case we can abort compilation.
929 5202 : if (native_module_ && module_object_.is_null()) {
930 136 : Impl(native_module_->compilation_state())->Abort();
931 : }
932 : // Tell the streaming decoder that the AsyncCompileJob is not available
933 : // anymore.
934 : // TODO(ahaas): Is this notification really necessary? Check
935 : // https://crbug.com/888170.
936 2685 : if (stream_) stream_->NotifyCompilationEnded();
937 : CancelPendingForegroundTask();
938 10482 : for (auto d : deferred_handles_) delete d;
939 2685 : }
940 :
941 2512 : void AsyncCompileJob::CreateNativeModule(
942 : std::shared_ptr<const WasmModule> module) {
943 : // Embedder usage count for declared shared memories.
944 5024 : if (module->has_shared_memory) {
945 0 : isolate_->CountUsage(v8::Isolate::UseCounterFeature::kWasmSharedMemory);
946 : }
947 :
948 : // TODO(wasm): Improve efficiency of storing module wire bytes. Only store
949 : // relevant sections, not function bodies
950 :
951 : // Create the module object and populate with compiled functions and
952 : // information needed at instantiation time.
953 : // TODO(clemensh): For the same module (same bytes / same hash), we should
954 : // only have one {WasmModuleObject}. Otherwise, we might only set
955 : // breakpoints on a (potentially empty) subset of the instances.
956 : // Create the module object.
957 :
958 : size_t code_size_estimate =
959 2512 : wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get());
960 10048 : native_module_ = isolate_->wasm_engine()->code_manager()->NewNativeModule(
961 : isolate_, enabled_features_, code_size_estimate,
962 : wasm::NativeModule::kCanAllocateMoreMemory, std::move(module));
963 7536 : native_module_->SetWireBytes({std::move(bytes_copy_), wire_bytes_.length()});
964 5024 : native_module_->SetRuntimeStubs(isolate_);
965 :
966 2512 : if (stream_) stream_->NotifyNativeModuleCreated(native_module_);
967 2512 : }
968 :
969 2376 : void AsyncCompileJob::PrepareRuntimeObjects() {
970 : // Create heap objects for script and module bytes to be stored in the
971 : // module object. Asm.js is not compiled asynchronously.
972 2376 : const WasmModule* module = native_module_->module();
973 : Handle<Script> script =
974 2376 : CreateWasmScript(isolate_, wire_bytes_, module->source_map_url);
975 :
976 : size_t code_size_estimate =
977 2376 : wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module);
978 : module_object_ = WasmModuleObject::New(isolate_, native_module_, script,
979 4752 : code_size_estimate);
980 :
981 : {
982 2376 : DeferredHandleScope deferred(isolate_);
983 4752 : module_object_ = handle(*module_object_, isolate_);
984 4752 : deferred_handles_.push_back(deferred.Detach());
985 : }
986 2376 : }
987 :
988 : // This function assumes that it is executed in a HandleScope, and that a
989 : // context is set on the isolate.
990 2381 : void AsyncCompileJob::FinishCompile() {
991 2381 : bool is_after_deserialization = !module_object_.is_null();
992 2381 : if (!is_after_deserialization) {
993 2376 : PrepareRuntimeObjects();
994 : }
995 : DCHECK(!isolate_->context().is_null());
996 : // Finish the wasm script now and make it public to the debugger.
997 9524 : Handle<Script> script(module_object_->script(), isolate_);
998 7143 : if (script->type() == Script::TYPE_WASM &&
999 4762 : module_object_->module()->source_map_url.size() != 0) {
1000 : MaybeHandle<String> src_map_str = isolate_->factory()->NewStringFromUtf8(
1001 0 : CStrVector(module_object_->module()->source_map_url.c_str()), TENURED);
1002 0 : script->set_source_mapping_url(*src_map_str.ToHandleChecked());
1003 : }
1004 4762 : isolate_->debug()->OnAfterCompile(script);
1005 :
1006 : // We can only update the feature counts once the entire compile is done.
1007 : auto compilation_state =
1008 4762 : Impl(module_object_->native_module()->compilation_state());
1009 : compilation_state->PublishDetectedFeatures(
1010 2381 : isolate_, *compilation_state->detected_features());
1011 :
1012 : // TODO(bbudge) Allow deserialization without wrapper compilation, so we can
1013 : // just compile wrappers here.
1014 2381 : if (!is_after_deserialization) {
1015 : // TODO(wasm): compiling wrappers should be made async.
1016 2376 : CompileWrappers();
1017 : }
1018 2381 : FinishModule();
1019 2381 : }
1020 :
1021 228 : void AsyncCompileJob::AsyncCompileFailed(Handle<Object> error_reason) {
1022 : // {job} keeps the {this} pointer alive.
1023 : std::shared_ptr<AsyncCompileJob> job =
1024 684 : isolate_->wasm_engine()->RemoveCompileJob(this);
1025 228 : resolver_->OnCompilationFailed(error_reason);
1026 228 : }
1027 :
1028 0 : void AsyncCompileJob::AsyncCompileSucceeded(Handle<WasmModuleObject> result) {
1029 2381 : resolver_->OnCompilationSucceeded(result);
1030 0 : }
1031 :
1032 : class AsyncCompileJob::CompilationStateCallback {
1033 : public:
1034 : explicit CompilationStateCallback(AsyncCompileJob* job) : job_(job) {}
1035 :
1036 2921 : void operator()(CompilationEvent event, const WasmError* error) {
1037 : // This callback is only being called from a foreground task.
1038 2921 : switch (event) {
1039 : case CompilationEvent::kFinishedBaselineCompilation:
1040 : DCHECK(!last_event_.has_value());
1041 5982 : if (job_->DecrementAndCheckFinisherCount()) {
1042 2856 : SaveContext saved_context(job_->isolate());
1043 2856 : job_->isolate()->set_context(*job_->native_context_);
1044 1428 : job_->FinishCompile();
1045 : }
1046 : break;
1047 : case CompilationEvent::kFinishedTopTierCompilation:
1048 : DCHECK_EQ(CompilationEvent::kFinishedBaselineCompilation, last_event_);
1049 : // This callback should not react to top tier finished callbacks, since
1050 : // the job might already be gone then.
1051 : break;
1052 : case CompilationEvent::kFailedCompilation:
1053 : DCHECK(!last_event_.has_value());
1054 : DCHECK_NOT_NULL(error);
1055 : // Tier-up compilation should not fail if baseline compilation
1056 : // did not fail.
1057 : DCHECK(!Impl(job_->native_module_->compilation_state())
1058 : ->baseline_compilation_finished());
1059 :
1060 : {
1061 92 : SaveContext saved_context(job_->isolate());
1062 92 : job_->isolate()->set_context(*job_->native_context_);
1063 92 : ErrorThrower thrower(job_->isolate(), "AsyncCompilation");
1064 : thrower.CompileFailed(nullptr, *error);
1065 46 : Handle<Object> error = thrower.Reify();
1066 :
1067 138 : DeferredHandleScope deferred(job_->isolate());
1068 92 : error = handle(*error, job_->isolate());
1069 92 : job_->deferred_handles_.push_back(deferred.Detach());
1070 :
1071 92 : job_->DoSync<CompileFailed, kUseExistingForegroundTask>(error);
1072 : }
1073 :
1074 46 : break;
1075 : default:
1076 0 : UNREACHABLE();
1077 : }
1078 : #ifdef DEBUG
1079 : last_event_ = event;
1080 : #endif
1081 2921 : }
1082 :
1083 : private:
1084 : AsyncCompileJob* job_;
1085 : #ifdef DEBUG
1086 : base::Optional<CompilationEvent> last_event_;
1087 : #endif
1088 : };
1089 :
1090 : // A closure to run a compilation step (either as foreground or background
1091 : // task) and schedule the next step(s), if any.
1092 4978 : class AsyncCompileJob::CompileStep {
1093 : public:
1094 4977 : virtual ~CompileStep() = default;
1095 :
1096 4977 : void Run(AsyncCompileJob* job, bool on_foreground) {
1097 4977 : if (on_foreground) {
1098 2716 : HandleScope scope(job->isolate_);
1099 5432 : SaveContext saved_context(job->isolate_);
1100 2716 : job->isolate_->set_context(*job->native_context_);
1101 2716 : RunInForeground(job);
1102 : } else {
1103 2261 : RunInBackground(job);
1104 : }
1105 4977 : }
1106 :
1107 0 : virtual void RunInForeground(AsyncCompileJob*) { UNREACHABLE(); }
1108 0 : virtual void RunInBackground(AsyncCompileJob*) { UNREACHABLE(); }
1109 : };
1110 :
1111 : class AsyncCompileJob::CompileTask : public CancelableTask {
1112 : public:
1113 : CompileTask(AsyncCompileJob* job, bool on_foreground)
1114 : // We only manage the background tasks with the {CancelableTaskManager} of
1115 : // the {AsyncCompileJob}. Foreground tasks are managed by the system's
1116 : // {CancelableTaskManager}. Background tasks cannot spawn tasks managed by
1117 : // their own task manager.
1118 2717 : : CancelableTask(on_foreground ? job->isolate_->cancelable_task_manager()
1119 : : &job->background_task_manager_),
1120 : job_(job),
1121 7239 : on_foreground_(on_foreground) {}
1122 :
1123 9956 : ~CompileTask() override {
1124 4978 : if (job_ != nullptr && on_foreground_) ResetPendingForegroundTask();
1125 9956 : }
1126 :
1127 4977 : void RunInternal() final {
1128 9954 : if (!job_) return;
1129 4977 : if (on_foreground_) ResetPendingForegroundTask();
1130 9954 : job_->step_->Run(job_, on_foreground_);
1131 : // After execution, reset {job_} such that we don't try to reset the pending
1132 : // foreground task when the task is deleted.
1133 4977 : job_ = nullptr;
1134 : }
1135 :
1136 : void Cancel() {
1137 : DCHECK_NOT_NULL(job_);
1138 1 : job_ = nullptr;
1139 : }
1140 :
1141 : private:
1142 : // {job_} will be cleared to cancel a pending task.
1143 : AsyncCompileJob* job_;
1144 : bool on_foreground_;
1145 :
1146 : void ResetPendingForegroundTask() const {
1147 : DCHECK_EQ(this, job_->pending_foreground_task_);
1148 2716 : job_->pending_foreground_task_ = nullptr;
1149 : }
1150 : };
1151 :
1152 2488 : void AsyncCompileJob::StartForegroundTask() {
1153 : DCHECK_NULL(pending_foreground_task_);
1154 :
1155 2488 : auto new_task = base::make_unique<CompileTask>(this, true);
1156 2488 : pending_foreground_task_ = new_task.get();
1157 7464 : foreground_task_runner_->PostTask(std::move(new_task));
1158 2488 : }
1159 :
1160 229 : void AsyncCompileJob::ExecuteForegroundTaskImmediately() {
1161 : DCHECK_NULL(pending_foreground_task_);
1162 :
1163 229 : auto new_task = base::make_unique<CompileTask>(this, true);
1164 229 : pending_foreground_task_ = new_task.get();
1165 229 : new_task->Run();
1166 229 : }
1167 :
1168 0 : void AsyncCompileJob::CancelPendingForegroundTask() {
1169 2685 : if (!pending_foreground_task_) return;
1170 : pending_foreground_task_->Cancel();
1171 1 : pending_foreground_task_ = nullptr;
1172 : }
1173 :
1174 2261 : void AsyncCompileJob::StartBackgroundTask() {
1175 2261 : auto task = base::make_unique<CompileTask>(this, false);
1176 :
1177 : // If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
1178 : // tasks. This is used to make timing deterministic.
1179 2261 : if (FLAG_wasm_num_compilation_tasks > 0) {
1180 6783 : V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
1181 : } else {
1182 0 : foreground_task_runner_->PostTask(std::move(task));
1183 : }
1184 2261 : }
1185 :
1186 : template <typename Step,
1187 : AsyncCompileJob::UseExistingForegroundTask use_existing_fg_task,
1188 : typename... Args>
1189 101 : void AsyncCompileJob::DoSync(Args&&... args) {
1190 2486 : NextStep<Step>(std::forward<Args>(args)...);
1191 202 : if (use_existing_fg_task && pending_foreground_task_ != nullptr) return;
1192 2488 : StartForegroundTask();
1193 : }
1194 :
1195 : template <typename Step, typename... Args>
1196 : void AsyncCompileJob::DoImmediately(Args&&... args) {
1197 229 : NextStep<Step>(std::forward<Args>(args)...);
1198 229 : ExecuteForegroundTaskImmediately();
1199 : }
1200 :
1201 : template <typename Step, typename... Args>
1202 : void AsyncCompileJob::DoAsync(Args&&... args) {
1203 2261 : NextStep<Step>(std::forward<Args>(args)...);
1204 2261 : StartBackgroundTask();
1205 : }
1206 :
1207 : template <typename Step, typename... Args>
1208 4976 : void AsyncCompileJob::NextStep(Args&&... args) {
1209 5157 : step_.reset(new Step(std::forward<Args>(args)...));
1210 4979 : }
1211 :
1212 : //==========================================================================
1213 : // Step 1: (async) Decode the module.
1214 : //==========================================================================
1215 4520 : class AsyncCompileJob::DecodeModule : public AsyncCompileJob::CompileStep {
1216 : public:
1217 2261 : explicit DecodeModule(Counters* counters) : counters_(counters) {}
1218 :
1219 4522 : void RunInBackground(AsyncCompileJob* job) override {
1220 : ModuleResult result;
1221 : {
1222 : DisallowHandleAllocation no_handle;
1223 : DisallowHeapAllocation no_allocation;
1224 : // Decode the module bytes.
1225 : TRACE_COMPILE("(1) Decoding module...\n");
1226 4522 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
1227 : "AsyncCompileJob::DecodeModule");
1228 6781 : result = DecodeWasmModule(
1229 : job->enabled_features_, job->wire_bytes_.start(),
1230 : job->wire_bytes_.end(), false, kWasmOrigin, counters_,
1231 4518 : job->isolate()->wasm_engine()->allocator());
1232 : }
1233 2259 : if (result.failed()) {
1234 : // Decoding failure; reject the promise and clean up.
1235 : job->DoSync<DecodeFail>(std::move(result).error());
1236 : } else {
1237 : // Decode passed.
1238 4518 : job->DoSync<PrepareAndStartCompile>(std::move(result).value(), true);
1239 2261 : }
1240 2261 : }
1241 :
1242 : private:
1243 : Counters* const counters_;
1244 : };
1245 :
1246 : //==========================================================================
1247 : // Step 1b: (sync) Fail decoding the module.
1248 : //==========================================================================
1249 364 : class AsyncCompileJob::DecodeFail : public CompileStep {
1250 : public:
1251 182 : explicit DecodeFail(WasmError error) : error_(std::move(error)) {}
1252 :
1253 : private:
1254 : WasmError error_;
1255 :
1256 182 : void RunInForeground(AsyncCompileJob* job) override {
1257 : TRACE_COMPILE("(1b) Decoding failed.\n");
1258 182 : ErrorThrower thrower(job->isolate_, "AsyncCompile");
1259 : thrower.CompileFailed("Wasm decoding failed", error_);
1260 : // {job_} is deleted in AsyncCompileFailed, therefore the {return}.
1261 182 : return job->AsyncCompileFailed(thrower.Reify());
1262 : }
1263 : };
1264 :
1265 : //==========================================================================
1266 : // Step 2 (sync): Create heap-allocated data and start compile.
1267 : //==========================================================================
1268 4978 : class AsyncCompileJob::PrepareAndStartCompile : public CompileStep {
1269 : public:
1270 : PrepareAndStartCompile(std::shared_ptr<const WasmModule> module,
1271 : bool start_compilation)
1272 4978 : : module_(std::move(module)), start_compilation_(start_compilation) {}
1273 :
1274 : private:
1275 : std::shared_ptr<const WasmModule> module_;
1276 : bool start_compilation_;
1277 :
1278 3838 : void RunInForeground(AsyncCompileJob* job) override {
1279 : TRACE_COMPILE("(2) Prepare and start compile...\n");
1280 :
1281 : // Make sure all compilation tasks stopped running. Decoding (async step)
1282 : // is done.
1283 2488 : job->background_task_manager_.CancelAndWait();
1284 :
1285 4976 : job->CreateNativeModule(module_);
1286 :
1287 : size_t num_functions =
1288 6326 : module_->functions.size() - module_->num_imported_functions;
1289 :
1290 2488 : if (num_functions == 0) {
1291 : // Degenerate case of an empty module.
1292 909 : job->FinishCompile();
1293 3397 : return;
1294 : }
1295 :
1296 : CompilationStateImpl* compilation_state =
1297 2929 : Impl(job->native_module_->compilation_state());
1298 1579 : compilation_state->AddCallback(CompilationStateCallback{job});
1299 1579 : if (start_compilation_) {
1300 : // TODO(ahaas): Try to remove the {start_compilation_} check when
1301 : // streaming decoding is done in the background. If
1302 : // InitializeCompilationUnits always returns 0 for streaming compilation,
1303 : // then DoAsync would do the same as NextStep already.
1304 :
1305 : compilation_state->SetNumberOfFunctionsToCompile(
1306 1350 : module_->num_declared_functions);
1307 : // Add compilation units and kick off compilation.
1308 : InitializeCompilationUnits(job->native_module_.get(),
1309 1350 : job->isolate()->wasm_engine());
1310 : }
1311 : }
1312 : };
1313 :
1314 : //==========================================================================
1315 : // Step 4b (sync): Compilation failed. Reject Promise.
1316 : //==========================================================================
1317 92 : class AsyncCompileJob::CompileFailed : public CompileStep {
1318 : public:
1319 : explicit CompileFailed(Handle<Object> error_reason)
1320 46 : : error_reason_(error_reason) {}
1321 :
1322 46 : void RunInForeground(AsyncCompileJob* job) override {
1323 : TRACE_COMPILE("(4b) Compilation Failed...\n");
1324 46 : return job->AsyncCompileFailed(error_reason_);
1325 : }
1326 :
1327 : private:
1328 : Handle<Object> error_reason_;
1329 : };
1330 :
1331 2376 : void AsyncCompileJob::CompileWrappers() {
1332 : // TODO(wasm): Compile all wrappers here, including the start function wrapper
1333 : // and the wrappers for the function table elements.
1334 : TRACE_COMPILE("(5) Compile wrappers...\n");
1335 : // Compile JS->wasm wrappers for exported functions.
1336 : CompileJsToWasmWrappers(isolate_, module_object_->native_module()->module(),
1337 9504 : handle(module_object_->export_wrappers(), isolate_));
1338 2376 : }
1339 :
1340 2381 : void AsyncCompileJob::FinishModule() {
1341 : TRACE_COMPILE("(6) Finish module...\n");
1342 : AsyncCompileSucceeded(module_object_);
1343 :
1344 2381 : size_t num_functions = native_module_->num_functions() -
1345 : native_module_->num_imported_functions();
1346 2381 : auto* compilation_state = Impl(native_module_->compilation_state());
1347 2381 : if (compilation_state->compile_mode() == CompileMode::kRegular ||
1348 : num_functions == 0) {
1349 : // If we do not tier up, the async compile job is done here and
1350 : // can be deleted.
1351 2853 : isolate_->wasm_engine()->RemoveCompileJob(this);
1352 3332 : return;
1353 : }
1354 : DCHECK_EQ(CompileMode::kTiering, compilation_state->compile_mode());
1355 1430 : if (compilation_state->baseline_compilation_finished()) {
1356 4290 : isolate_->wasm_engine()->RemoveCompileJob(this);
1357 : }
1358 : }
1359 :
1360 0 : AsyncStreamingProcessor::AsyncStreamingProcessor(AsyncCompileJob* job)
1361 : : decoder_(job->enabled_features_),
1362 : job_(job),
1363 424 : compilation_unit_builder_(nullptr) {}
1364 :
1365 181 : void AsyncStreamingProcessor::FinishAsyncCompileJobWithError(
1366 : const WasmError& error) {
1367 : DCHECK(error.has_error());
1368 : // Make sure all background tasks stopped executing before we change the state
1369 : // of the AsyncCompileJob to DecodeFail.
1370 181 : job_->background_task_manager_.CancelAndWait();
1371 :
1372 : // Check if there is already a CompiledModule, in which case we have to clean
1373 : // up the CompilationStateImpl as well.
1374 181 : if (job_->native_module_) {
1375 55 : Impl(job_->native_module_->compilation_state())->Abort();
1376 :
1377 : job_->DoSync<AsyncCompileJob::DecodeFail,
1378 55 : AsyncCompileJob::kUseExistingForegroundTask>(error);
1379 :
1380 : // Clear the {compilation_unit_builder_} if it exists. This is needed
1381 : // because there is a check in the destructor of the
1382 : // {CompilationUnitBuilder} that it is empty.
1383 55 : if (compilation_unit_builder_) compilation_unit_builder_->Clear();
1384 : } else {
1385 : job_->DoSync<AsyncCompileJob::DecodeFail>(error);
1386 : }
1387 181 : }
1388 :
1389 : // Process the module header.
1390 389 : bool AsyncStreamingProcessor::ProcessModuleHeader(Vector<const uint8_t> bytes,
1391 : uint32_t offset) {
1392 : TRACE_STREAMING("Process module header...\n");
1393 : decoder_.StartDecoding(job_->isolate()->counters(),
1394 1167 : job_->isolate()->wasm_engine()->allocator());
1395 389 : decoder_.DecodeModuleHeader(bytes, offset);
1396 389 : if (!decoder_.ok()) {
1397 18 : FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false).error());
1398 18 : return false;
1399 : }
1400 : return true;
1401 : }
1402 :
1403 : // Process all sections except for the code section.
1404 860 : bool AsyncStreamingProcessor::ProcessSection(SectionCode section_code,
1405 : Vector<const uint8_t> bytes,
1406 : uint32_t offset) {
1407 : TRACE_STREAMING("Process section %d ...\n", section_code);
1408 860 : if (compilation_unit_builder_) {
1409 : // We reached a section after the code section, we do not need the
1410 : // compilation_unit_builder_ anymore.
1411 : CommitCompilationUnits();
1412 : compilation_unit_builder_.reset();
1413 : }
1414 860 : if (section_code == SectionCode::kUnknownSectionCode) {
1415 : Decoder decoder(bytes, offset);
1416 : section_code = ModuleDecoder::IdentifyUnknownSection(
1417 65 : decoder, bytes.start() + bytes.length());
1418 65 : if (section_code == SectionCode::kUnknownSectionCode) {
1419 : // Skip unknown sections that we do not know how to handle.
1420 : return true;
1421 : }
1422 : // Remove the unknown section tag from the payload bytes.
1423 130 : offset += decoder.position();
1424 65 : bytes = bytes.SubVector(decoder.position(), bytes.size());
1425 : }
1426 : constexpr bool verify_functions = false;
1427 860 : decoder_.DecodeSection(section_code, bytes, offset, verify_functions);
1428 860 : if (!decoder_.ok()) {
1429 24 : FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false).error());
1430 24 : return false;
1431 : }
1432 : return true;
1433 : }
1434 :
1435 : // Start the code section.
1436 243 : bool AsyncStreamingProcessor::ProcessCodeSectionHeader(
1437 : size_t functions_count, uint32_t offset,
1438 : std::shared_ptr<WireBytesStorage> wire_bytes_storage) {
1439 : TRACE_STREAMING("Start the code section with %zu functions...\n",
1440 : functions_count);
1441 243 : if (!decoder_.CheckFunctionsCount(static_cast<uint32_t>(functions_count),
1442 243 : offset)) {
1443 14 : FinishAsyncCompileJobWithError(decoder_.FinishDecoding(false).error());
1444 14 : return false;
1445 : }
1446 : // Execute the PrepareAndStartCompile step immediately and not in a separate
1447 : // task.
1448 : job_->DoImmediately<AsyncCompileJob::PrepareAndStartCompile>(
1449 687 : decoder_.shared_module(), false);
1450 : job_->native_module_->compilation_state()->SetWireBytesStorage(
1451 1145 : std::move(wire_bytes_storage));
1452 :
1453 229 : auto* compilation_state = Impl(job_->native_module_->compilation_state());
1454 229 : compilation_state->SetNumberOfFunctionsToCompile(functions_count);
1455 :
1456 : // Set outstanding_finishers_ to 2, because both the AsyncCompileJob and the
1457 : // AsyncStreamingProcessor have to finish.
1458 229 : job_->outstanding_finishers_.store(2);
1459 : compilation_unit_builder_.reset(new CompilationUnitBuilder(
1460 458 : job_->native_module_.get(), job_->isolate()->wasm_engine()));
1461 : return true;
1462 : }
1463 :
1464 : // Process a function body.
1465 503 : bool AsyncStreamingProcessor::ProcessFunctionBody(Vector<const uint8_t> bytes,
1466 : uint32_t offset) {
1467 : TRACE_STREAMING("Process function body %d ...\n", next_function_);
1468 :
1469 : decoder_.DecodeFunctionBody(
1470 1006 : next_function_, static_cast<uint32_t>(bytes.length()), offset, false);
1471 :
1472 1006 : uint32_t index = next_function_ + decoder_.module()->num_imported_functions;
1473 503 : compilation_unit_builder_->AddUnit(index);
1474 503 : ++next_function_;
1475 : // This method always succeeds. The return value is necessary to comply with
1476 : // the StreamingProcessor interface.
1477 503 : return true;
1478 : }
1479 :
1480 0 : void AsyncStreamingProcessor::CommitCompilationUnits() {
1481 : DCHECK(compilation_unit_builder_);
1482 324 : compilation_unit_builder_->Commit();
1483 0 : }
1484 :
1485 403 : void AsyncStreamingProcessor::OnFinishedChunk() {
1486 : TRACE_STREAMING("FinishChunk...\n");
1487 403 : if (compilation_unit_builder_) CommitCompilationUnits();
1488 403 : }
1489 :
1490 : // Finish the processing of the stream.
1491 158 : void AsyncStreamingProcessor::OnFinishedStream(OwnedVector<uint8_t> bytes) {
1492 : TRACE_STREAMING("Finish stream...\n");
1493 316 : ModuleResult result = decoder_.FinishDecoding(false);
1494 158 : if (result.failed()) {
1495 5 : FinishAsyncCompileJobWithError(result.error());
1496 163 : return;
1497 : }
1498 : // We have to open a HandleScope and prepare the Context for
1499 : // CreateNativeModule, PrepareRuntimeObjects and FinishCompile as this is a
1500 : // callback from the embedder.
1501 153 : HandleScope scope(job_->isolate_);
1502 306 : SaveContext saved_context(job_->isolate_);
1503 306 : job_->isolate_->set_context(*job_->native_context_);
1504 :
1505 153 : bool needs_finish = job_->DecrementAndCheckFinisherCount();
1506 306 : if (job_->native_module_ == nullptr) {
1507 : // We are processing a WebAssembly module without code section. Create the
1508 : // runtime objects now (would otherwise happen in {PrepareAndStartCompile}).
1509 48 : job_->CreateNativeModule(std::move(result).value());
1510 : DCHECK(needs_finish);
1511 : }
1512 306 : job_->wire_bytes_ = ModuleWireBytes(bytes.as_vector());
1513 306 : job_->native_module_->SetWireBytes(std::move(bytes));
1514 153 : if (needs_finish) {
1515 39 : job_->FinishCompile();
1516 153 : }
1517 : }
1518 :
1519 : // Report an error detected in the StreamingDecoder.
1520 120 : void AsyncStreamingProcessor::OnError(const WasmError& error) {
1521 : TRACE_STREAMING("Stream error...\n");
1522 120 : FinishAsyncCompileJobWithError(error);
1523 120 : }
1524 :
1525 65 : void AsyncStreamingProcessor::OnAbort() {
1526 : TRACE_STREAMING("Abort stream...\n");
1527 65 : job_->Abort();
1528 65 : }
1529 :
1530 10 : bool AsyncStreamingProcessor::Deserialize(Vector<const uint8_t> module_bytes,
1531 : Vector<const uint8_t> wire_bytes) {
1532 : // DeserializeNativeModule and FinishCompile assume that they are executed in
1533 : // a HandleScope, and that a context is set on the isolate.
1534 10 : HandleScope scope(job_->isolate_);
1535 20 : SaveContext saved_context(job_->isolate_);
1536 20 : job_->isolate_->set_context(*job_->native_context_);
1537 :
1538 : MaybeHandle<WasmModuleObject> result =
1539 10 : DeserializeNativeModule(job_->isolate_, module_bytes, wire_bytes);
1540 10 : if (result.is_null()) return false;
1541 :
1542 10 : job_->module_object_ = result.ToHandleChecked();
1543 : {
1544 5 : DeferredHandleScope deferred(job_->isolate_);
1545 10 : job_->module_object_ = handle(*job_->module_object_, job_->isolate_);
1546 10 : job_->deferred_handles_.push_back(deferred.Detach());
1547 : }
1548 15 : job_->native_module_ = job_->module_object_->shared_native_module();
1549 5 : auto owned_wire_bytes = OwnedVector<uint8_t>::Of(wire_bytes);
1550 10 : job_->wire_bytes_ = ModuleWireBytes(owned_wire_bytes.as_vector());
1551 10 : job_->native_module_->SetWireBytes(std::move(owned_wire_bytes));
1552 5 : job_->FinishCompile();
1553 : return true;
1554 : }
1555 :
1556 1530590 : CompilationStateImpl::CompilationStateImpl(internal::Isolate* isolate,
1557 : NativeModule* native_module)
1558 : : isolate_(isolate),
1559 : native_module_(native_module),
1560 1530488 : compile_mode_(FLAG_wasm_tier_up &&
1561 1530488 : native_module->module()->origin == kWasmOrigin
1562 : ? CompileMode::kTiering
1563 : : CompileMode::kRegular),
1564 1530590 : should_log_code_(WasmCode::ShouldBeLogged(isolate)),
1565 : max_background_tasks_(std::max(
1566 : 1, std::min(FLAG_wasm_num_compilation_tasks,
1567 13775306 : V8::GetCurrentPlatform()->NumberOfWorkerThreads()))) {
1568 1530588 : v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate_);
1569 1530588 : v8::Platform* platform = V8::GetCurrentPlatform();
1570 3061179 : foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate);
1571 1530590 : }
1572 :
1573 4591770 : CompilationStateImpl::~CompilationStateImpl() {
1574 : DCHECK(background_task_manager_.canceled());
1575 : DCHECK(foreground_task_manager_.canceled());
1576 : CompilationError* error = compile_error_.load(std::memory_order_acquire);
1577 1537405 : if (error != nullptr) delete error;
1578 1530590 : }
1579 :
1580 : void CompilationStateImpl::CancelAndWait() {
1581 1530590 : background_task_manager_.CancelAndWait();
1582 1530590 : foreground_task_manager_.CancelAndWait();
1583 : }
1584 :
1585 6852 : void CompilationStateImpl::SetNumberOfFunctionsToCompile(size_t num_functions) {
1586 : DCHECK(!failed());
1587 6852 : base::MutexGuard guard(&mutex_);
1588 6852 : outstanding_baseline_units_ = num_functions;
1589 :
1590 6852 : if (compile_mode_ == CompileMode::kTiering) {
1591 6801 : outstanding_tiering_units_ = num_functions;
1592 : }
1593 6852 : }
1594 :
1595 : void CompilationStateImpl::AddCallback(CompilationState::callback_t callback) {
1596 1584 : callbacks_.emplace_back(std::move(callback));
1597 : }
1598 :
1599 6877 : void CompilationStateImpl::AddCompilationUnits(
1600 : std::vector<std::unique_ptr<WasmCompilationUnit>>& baseline_units,
1601 : std::vector<std::unique_ptr<WasmCompilationUnit>>& tiering_units) {
1602 : {
1603 6877 : base::MutexGuard guard(&mutex_);
1604 :
1605 6877 : if (compile_mode_ == CompileMode::kTiering) {
1606 : DCHECK_EQ(baseline_units.size(), tiering_units.size());
1607 : DCHECK_EQ(tiering_units.back()->tier(), ExecutionTier::kOptimized);
1608 : tiering_compilation_units_.insert(
1609 : tiering_compilation_units_.end(),
1610 : std::make_move_iterator(tiering_units.begin()),
1611 6826 : std::make_move_iterator(tiering_units.end()));
1612 : } else {
1613 : DCHECK(tiering_compilation_units_.empty());
1614 : }
1615 :
1616 : baseline_compilation_units_.insert(
1617 : baseline_compilation_units_.end(),
1618 : std::make_move_iterator(baseline_units.begin()),
1619 6877 : std::make_move_iterator(baseline_units.end()));
1620 : }
1621 :
1622 6877 : RestartBackgroundTasks();
1623 6877 : }
1624 :
1625 : std::unique_ptr<WasmCompilationUnit>
1626 265798 : CompilationStateImpl::GetNextCompilationUnit() {
1627 265798 : base::MutexGuard guard(&mutex_);
1628 :
1629 : std::vector<std::unique_ptr<WasmCompilationUnit>>& units =
1630 : baseline_compilation_units_.empty() ? tiering_compilation_units_
1631 265928 : : baseline_compilation_units_;
1632 :
1633 265928 : if (!units.empty()) {
1634 : std::unique_ptr<WasmCompilationUnit> unit = std::move(units.back());
1635 228363 : units.pop_back();
1636 : return unit;
1637 : }
1638 :
1639 : return std::unique_ptr<WasmCompilationUnit>();
1640 : }
1641 :
1642 : std::unique_ptr<WasmCompilationUnit>
1643 324591 : CompilationStateImpl::GetNextExecutedUnit() {
1644 : std::vector<std::unique_ptr<WasmCompilationUnit>>& units = finish_units();
1645 324591 : base::MutexGuard guard(&mutex_);
1646 324591 : if (units.empty()) return {};
1647 : std::unique_ptr<WasmCompilationUnit> ret = std::move(units.back());
1648 0 : units.pop_back();
1649 : return ret;
1650 : }
1651 :
1652 : bool CompilationStateImpl::HasCompilationUnitToFinish() {
1653 : return !finish_units().empty();
1654 : }
1655 :
1656 228357 : void CompilationStateImpl::OnFinishedUnit(ExecutionTier tier, WasmCode* code) {
1657 : // This mutex guarantees that events happen in the right order.
1658 228357 : base::MutexGuard guard(&mutex_);
1659 :
1660 456724 : if (failed()) return;
1661 :
1662 : // If we are *not* compiling in tiering mode, then all units are counted as
1663 : // baseline units.
1664 228063 : bool is_tiering_mode = compile_mode_ == CompileMode::kTiering;
1665 228063 : bool is_tiering_unit = is_tiering_mode && tier == ExecutionTier::kOptimized;
1666 :
1667 : // Sanity check: If we are not in tiering mode, there cannot be outstanding
1668 : // tiering units.
1669 : DCHECK_IMPLIES(!is_tiering_mode, outstanding_tiering_units_ == 0);
1670 :
1671 : // Bitset of events to deliver.
1672 : base::EnumSet<CompilationEvent> events;
1673 :
1674 228063 : if (is_tiering_unit) {
1675 : DCHECK_LT(0, outstanding_tiering_units_);
1676 113799 : --outstanding_tiering_units_;
1677 113799 : if (outstanding_tiering_units_ == 0) {
1678 : // If baseline compilation has not finished yet, then also trigger
1679 : // {kFinishedBaselineCompilation}.
1680 6547 : if (outstanding_baseline_units_ > 0) {
1681 : events.Add(CompilationEvent::kFinishedBaselineCompilation);
1682 : }
1683 : events.Add(CompilationEvent::kFinishedTopTierCompilation);
1684 : }
1685 : } else {
1686 : DCHECK_LT(0, outstanding_baseline_units_);
1687 114264 : --outstanding_baseline_units_;
1688 114264 : if (outstanding_baseline_units_ == 0) {
1689 : events.Add(CompilationEvent::kFinishedBaselineCompilation);
1690 : // If we are not tiering, then we also trigger the "top tier finished"
1691 : // event when baseline compilation is finished.
1692 6604 : if (!is_tiering_mode) {
1693 : events.Add(CompilationEvent::kFinishedTopTierCompilation);
1694 : }
1695 : }
1696 : }
1697 :
1698 228063 : if (!events.empty()) {
1699 11398 : auto notify_events = [this, events] {
1700 34194 : for (auto event : {CompilationEvent::kFinishedBaselineCompilation,
1701 22796 : CompilationEvent::kFinishedTopTierCompilation}) {
1702 22796 : if (!events.contains(event)) continue;
1703 11727 : NotifyOnEvent(event, nullptr);
1704 : }
1705 11398 : };
1706 13151 : foreground_task_runner_->PostTask(
1707 78906 : MakeCancelableTask(&foreground_task_manager_, notify_events));
1708 : }
1709 :
1710 228063 : if (should_log_code_ && code != nullptr) {
1711 0 : if (log_codes_task_ == nullptr) {
1712 : auto new_task = base::make_unique<LogCodesTask>(&foreground_task_manager_,
1713 0 : this, isolate_);
1714 0 : log_codes_task_ = new_task.get();
1715 0 : foreground_task_runner_->PostTask(std::move(new_task));
1716 : }
1717 0 : log_codes_task_->AddCode(code);
1718 : }
1719 : }
1720 :
1721 34635 : void CompilationStateImpl::RestartBackgroundCompileTask() {
1722 : auto task = base::make_unique<BackgroundCompileTask>(
1723 69270 : &background_task_manager_, native_module_, isolate_->counters());
1724 :
1725 : // If --wasm-num-compilation-tasks=0 is passed, do only spawn foreground
1726 : // tasks. This is used to make timing deterministic.
1727 34635 : if (FLAG_wasm_num_compilation_tasks == 0) {
1728 0 : foreground_task_runner_->PostTask(std::move(task));
1729 34635 : return;
1730 : }
1731 :
1732 34635 : if (baseline_compilation_finished()) {
1733 17844 : V8::GetCurrentPlatform()->CallLowPriorityTaskOnWorkerThread(
1734 53532 : std::move(task));
1735 : } else {
1736 50373 : V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
1737 : }
1738 : }
1739 :
1740 59 : void CompilationStateImpl::ReportDetectedFeatures(
1741 : const WasmFeatures& detected) {
1742 59 : base::MutexGuard guard(&mutex_);
1743 59 : UnionFeaturesInto(&detected_features_, detected);
1744 59 : }
1745 :
1746 34383 : void CompilationStateImpl::OnBackgroundTaskStopped(
1747 : const WasmFeatures& detected) {
1748 34383 : base::MutexGuard guard(&mutex_);
1749 : DCHECK_LE(1, num_background_tasks_);
1750 34384 : --num_background_tasks_;
1751 34384 : UnionFeaturesInto(&detected_features_, detected);
1752 34383 : }
1753 :
1754 7654 : void CompilationStateImpl::PublishDetectedFeatures(
1755 : Isolate* isolate, const WasmFeatures& detected) {
1756 : // Notifying the isolate of the feature counts must take place under
1757 : // the mutex, because even if we have finished baseline compilation,
1758 : // tiering compilations may still occur in the background.
1759 7654 : base::MutexGuard guard(&mutex_);
1760 7654 : UnionFeaturesInto(&detected_features_, detected);
1761 : UpdateFeatureUseCounts(isolate, detected_features_);
1762 7654 : }
1763 :
1764 6877 : void CompilationStateImpl::RestartBackgroundTasks(size_t max) {
1765 : size_t num_restart;
1766 : {
1767 6877 : base::MutexGuard guard(&mutex_);
1768 : // No need to restart tasks if compilation already failed.
1769 6877 : if (failed()) return;
1770 :
1771 : DCHECK_LE(num_background_tasks_, max_background_tasks_);
1772 6877 : if (num_background_tasks_ == max_background_tasks_) return;
1773 : size_t num_compilation_units =
1774 20631 : baseline_compilation_units_.size() + tiering_compilation_units_.size();
1775 6877 : size_t stopped_tasks = max_background_tasks_ - num_background_tasks_;
1776 6877 : num_restart = std::min(max, std::min(num_compilation_units, stopped_tasks));
1777 6877 : num_background_tasks_ += num_restart;
1778 : }
1779 :
1780 34576 : for (; num_restart > 0; --num_restart) {
1781 34576 : RestartBackgroundCompileTask();
1782 : }
1783 : }
1784 :
1785 10396 : bool CompilationStateImpl::SetFinisherIsRunning(bool value) {
1786 10396 : base::MutexGuard guard(&mutex_);
1787 10396 : if (finisher_is_running_ == value) return false;
1788 10396 : finisher_is_running_ = value;
1789 10396 : return true;
1790 : }
1791 :
1792 : void CompilationStateImpl::ScheduleFinisherTask() {
1793 : foreground_task_runner_->PostTask(
1794 : base::make_unique<FinishCompileTask>(this, &foreground_task_manager_));
1795 : }
1796 :
1797 191 : void CompilationStateImpl::Abort() {
1798 382 : SetError(0, WasmError{0, "Compilation aborted"});
1799 191 : background_task_manager_.CancelAndWait();
1800 : // No more callbacks after abort. Don't free the std::function objects here,
1801 : // since this might clear references in the embedder, which is only allowed on
1802 : // the main thread.
1803 : aborted_.store(true);
1804 191 : if (!callbacks_.empty()) {
1805 85 : foreground_task_runner_->PostTask(
1806 340 : base::make_unique<FreeCallbacksTask>(this));
1807 : }
1808 191 : }
1809 :
1810 6930 : void CompilationStateImpl::SetError(uint32_t func_index,
1811 : const WasmError& error) {
1812 : DCHECK(error.has_error());
1813 : std::unique_ptr<CompilationError> compile_error =
1814 6930 : base::make_unique<CompilationError>(func_index, error);
1815 6930 : CompilationError* expected = nullptr;
1816 : bool set = compile_error_.compare_exchange_strong(
1817 6930 : expected, compile_error.get(), std::memory_order_acq_rel);
1818 : // Ignore all but the first error. If the previous value is not nullptr, just
1819 : // return (and free the allocated error).
1820 13860 : if (!set) return;
1821 : // If set successfully, give up ownership.
1822 : compile_error.release();
1823 : // Schedule a foreground task to call the callback and notify users about the
1824 : // compile error.
1825 6815 : foreground_task_runner_->PostTask(
1826 101 : MakeCancelableTask(&foreground_task_manager_, [this] {
1827 101 : WasmError error = GetCompileError();
1828 101 : NotifyOnEvent(CompilationEvent::kFailedCompilation, &error);
1829 40991 : }));
1830 : }
1831 :
1832 11828 : void CompilationStateImpl::NotifyOnEvent(CompilationEvent event,
1833 : const WasmError* error) {
1834 23656 : if (aborted_.load()) return;
1835 11773 : HandleScope scope(isolate_);
1836 26477 : for (auto& callback : callbacks_) callback(event, error);
1837 : // If no more events are expected after this one, clear the callbacks to free
1838 : // memory. We can safely do this here, as this method is only called from
1839 : // foreground tasks.
1840 11773 : if (event >= CompilationEvent::kFirstFinalEvent) callbacks_.clear();
1841 : }
1842 :
1843 156206 : void CompileJsToWasmWrappers(Isolate* isolate, const WasmModule* module,
1844 : Handle<FixedArray> export_wrappers) {
1845 : JSToWasmWrapperCache js_to_wasm_cache;
1846 : int wrapper_index = 0;
1847 :
1848 : // TODO(6792): Wrappers below are allocated with {Factory::NewCode}. As an
1849 : // optimization we keep the code space unlocked to avoid repeated unlocking
1850 : // because many such wrapper are allocated in sequence below.
1851 312414 : CodeSpaceMemoryModificationScope modification_scope(isolate->heap());
1852 562246 : for (auto exp : module->export_table) {
1853 249832 : if (exp.kind != kExternalFunction) continue;
1854 247022 : auto& function = module->functions[exp.index];
1855 : Handle<Code> wrapper_code = js_to_wasm_cache.GetOrCompileJSToWasmWrapper(
1856 247022 : isolate, function.sig, function.imported);
1857 494038 : export_wrappers->set(wrapper_index, *wrapper_code);
1858 247022 : RecordStats(*wrapper_code, isolate->counters());
1859 247022 : ++wrapper_index;
1860 : }
1861 156207 : }
1862 :
1863 153227 : Handle<Script> CreateWasmScript(Isolate* isolate,
1864 : const ModuleWireBytes& wire_bytes,
1865 : const std::string& source_map_url) {
1866 : Handle<Script> script =
1867 153227 : isolate->factory()->NewScript(isolate->factory()->empty_string());
1868 459681 : script->set_context_data(isolate->native_context()->debug_context_id());
1869 : script->set_type(Script::TYPE_WASM);
1870 :
1871 : int hash = StringHasher::HashSequentialString(
1872 : reinterpret_cast<const char*>(wire_bytes.start()),
1873 153227 : static_cast<int>(wire_bytes.length()), kZeroHashSeed);
1874 :
1875 : const int kBufferSize = 32;
1876 : char buffer[kBufferSize];
1877 :
1878 153227 : int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash);
1879 : DCHECK(name_chars >= 0 && name_chars < kBufferSize);
1880 : MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte(
1881 153224 : VectorOf(reinterpret_cast<uint8_t*>(buffer), name_chars), TENURED);
1882 306452 : script->set_name(*name_str.ToHandleChecked());
1883 :
1884 153227 : if (source_map_url.size() != 0) {
1885 : MaybeHandle<String> src_map_str = isolate->factory()->NewStringFromUtf8(
1886 5 : CStrVector(source_map_url.c_str()), TENURED);
1887 10 : script->set_source_mapping_url(*src_map_str.ToHandleChecked());
1888 : }
1889 153227 : return script;
1890 : }
1891 :
1892 : } // namespace wasm
1893 : } // namespace internal
1894 183867 : } // namespace v8
1895 :
1896 : #undef TRACE_COMPILE
1897 : #undef TRACE_STREAMING
1898 : #undef TRACE_LAZY
|
