Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include <memory>
6 :
7 : #include "src/assembler-inl.h"
8 : #include "src/base/atomic-utils.h"
9 : #include "src/code-stubs.h"
10 : #include "src/compiler/wasm-compiler.h"
11 : #include "src/debug/interface-types.h"
12 : #include "src/frames-inl.h"
13 : #include "src/objects.h"
14 : #include "src/property-descriptor.h"
15 : #include "src/simulator.h"
16 : #include "src/snapshot/snapshot.h"
17 : #include "src/trap-handler/trap-handler.h"
18 : #include "src/v8.h"
19 :
20 : #include "src/asmjs/asm-wasm-builder.h"
21 : #include "src/wasm/function-body-decoder.h"
22 : #include "src/wasm/module-decoder.h"
23 : #include "src/wasm/wasm-code-specialization.h"
24 : #include "src/wasm/wasm-js.h"
25 : #include "src/wasm/wasm-limits.h"
26 : #include "src/wasm/wasm-module.h"
27 : #include "src/wasm/wasm-objects.h"
28 : #include "src/wasm/wasm-result.h"
29 :
30 : using namespace v8::internal;
31 : using namespace v8::internal::wasm;
32 : namespace base = v8::base;
33 :
34 : #define TRACE(...) \
35 : do { \
36 : if (FLAG_trace_wasm_instances) PrintF(__VA_ARGS__); \
37 : } while (false)
38 :
39 : #define TRACE_CHAIN(instance) \
40 : do { \
41 : instance->PrintInstancesChain(); \
42 : } while (false)
43 :
44 : #define TRACE_COMPILE(...) \
45 : do { \
46 : if (FLAG_trace_wasm_compiler) PrintF(__VA_ARGS__); \
47 : } while (false)
48 :
49 : namespace {
50 :
51 : static const int kInvalidSigIndex = -1;
52 :
53 12359 : byte* raw_buffer_ptr(MaybeHandle<JSArrayBuffer> buffer, int offset) {
54 12359 : return static_cast<byte*>(buffer.ToHandleChecked()->backing_store()) + offset;
55 : }
56 :
57 1241 : static void MemoryFinalizer(const v8::WeakCallbackInfo<void>& data) {
58 : DisallowHeapAllocation no_gc;
59 : JSArrayBuffer** p = reinterpret_cast<JSArrayBuffer**>(data.GetParameter());
60 634 : JSArrayBuffer* buffer = *p;
61 :
62 634 : if (!buffer->was_neutered()) {
63 : void* memory = buffer->backing_store();
64 : DCHECK(memory != nullptr);
65 : base::OS::Free(memory,
66 1214 : RoundUp(kWasmMaxHeapOffset, base::OS::CommitPageSize()));
67 :
68 : data.GetIsolate()->AdjustAmountOfExternalAllocatedMemory(
69 607 : -buffer->byte_length()->Number());
70 : }
71 :
72 634 : GlobalHandles::Destroy(reinterpret_cast<Object**>(p));
73 634 : }
74 :
75 784935 : static void RecordStats(Isolate* isolate, Code* code) {
76 523290 : isolate->counters()->wasm_generated_code_size()->Increment(code->body_size());
77 : isolate->counters()->wasm_reloc_size()->Increment(
78 261645 : code->relocation_info()->length());
79 261645 : }
80 :
81 32147 : static void RecordStats(Isolate* isolate, Handle<FixedArray> functions) {
82 : DisallowHeapAllocation no_gc;
83 295930 : for (int i = 0; i < functions->length(); ++i) {
84 115818 : RecordStats(isolate, Code::cast(functions->get(i)));
85 : }
86 32147 : }
87 :
88 22772 : void* TryAllocateBackingStore(Isolate* isolate, size_t size,
89 : bool enable_guard_regions, bool& is_external) {
90 11702 : is_external = false;
91 : // TODO(eholk): Right now enable_guard_regions has no effect on 32-bit
92 : // systems. It may be safer to fail instead, given that other code might do
93 : // things that would be unsafe if they expected guard pages where there
94 : // weren't any.
95 11702 : if (enable_guard_regions && kGuardRegionsSupported) {
96 : // TODO(eholk): On Windows we want to make sure we don't commit the guard
97 : // pages yet.
98 :
99 : // We always allocate the largest possible offset into the heap, so the
100 : // addressable memory after the guard page can be made inaccessible.
101 : const size_t alloc_size =
102 632 : RoundUp(kWasmMaxHeapOffset, base::OS::CommitPageSize());
103 : DCHECK_EQ(0, size % base::OS::CommitPageSize());
104 :
105 : // AllocateGuarded makes the whole region inaccessible by default.
106 632 : void* memory = base::OS::AllocateGuarded(alloc_size);
107 632 : if (memory == nullptr) {
108 : return nullptr;
109 : }
110 :
111 : // Make the part we care about accessible.
112 632 : base::OS::Unprotect(memory, size);
113 :
114 : reinterpret_cast<v8::Isolate*>(isolate)
115 632 : ->AdjustAmountOfExternalAllocatedMemory(size);
116 :
117 632 : is_external = true;
118 632 : return memory;
119 : } else {
120 11070 : void* memory = isolate->array_buffer_allocator()->Allocate(size);
121 11070 : return memory;
122 : }
123 : }
124 :
125 47286 : void FlushICache(Isolate* isolate, Handle<FixedArray> code_table) {
126 536458 : for (int i = 0; i < code_table->length(); ++i) {
127 220943 : Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i);
128 220943 : Assembler::FlushICache(isolate, code->instruction_start(),
129 441886 : code->instruction_size());
130 : }
131 47286 : }
132 :
133 12185 : Handle<Script> CreateWasmScript(Isolate* isolate,
134 : const ModuleWireBytes& wire_bytes) {
135 : Handle<Script> script =
136 12185 : isolate->factory()->NewScript(isolate->factory()->empty_string());
137 24370 : script->set_context_data(isolate->native_context()->debug_context_id());
138 : script->set_type(Script::TYPE_WASM);
139 :
140 : int hash = StringHasher::HashSequentialString(
141 : reinterpret_cast<const char*>(wire_bytes.start()), wire_bytes.length(),
142 12185 : kZeroHashSeed);
143 :
144 : const int kBufferSize = 32;
145 : char buffer[kBufferSize];
146 12185 : int url_chars = SNPrintF(ArrayVector(buffer), "wasm://wasm/%08x", hash);
147 : DCHECK(url_chars >= 0 && url_chars < kBufferSize);
148 : MaybeHandle<String> url_str = isolate->factory()->NewStringFromOneByte(
149 : Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), url_chars),
150 12185 : TENURED);
151 12185 : script->set_source_url(*url_str.ToHandleChecked());
152 :
153 12185 : int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash);
154 : DCHECK(name_chars >= 0 && name_chars < kBufferSize);
155 : MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte(
156 : Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), name_chars),
157 12185 : TENURED);
158 12185 : script->set_name(*name_str.ToHandleChecked());
159 :
160 12185 : return script;
161 : }
162 :
163 127800 : class JSToWasmWrapperCache {
164 : public:
165 49301 : Handle<Code> CloneOrCompileJSToWasmWrapper(Isolate* isolate,
166 : const wasm::WasmModule* module,
167 : Handle<Code> wasm_code,
168 : uint32_t index) {
169 49301 : const wasm::WasmFunction* func = &module->functions[index];
170 49301 : int cached_idx = sig_map_.Find(func->sig);
171 49301 : if (cached_idx >= 0) {
172 46668 : Handle<Code> code = isolate->factory()->CopyCode(code_cache_[cached_idx]);
173 : // Now patch the call to wasm code.
174 52894 : for (RelocIterator it(*code, RelocInfo::kCodeTargetMask);; it.next()) {
175 : DCHECK(!it.done());
176 : Code* target =
177 52894 : Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
178 84349 : if (target->kind() == Code::WASM_FUNCTION ||
179 31455 : target->kind() == Code::WASM_TO_JS_FUNCTION ||
180 84274 : target->builtin_index() == Builtins::kIllegal ||
181 : target->builtin_index() == Builtins::kWasmCompileLazy) {
182 : it.rinfo()->set_target_address(isolate,
183 23334 : wasm_code->instruction_start());
184 : break;
185 : }
186 29560 : }
187 23334 : return code;
188 : }
189 :
190 : Handle<Code> code =
191 25967 : compiler::CompileJSToWasmWrapper(isolate, module, wasm_code, index);
192 25967 : uint32_t new_cache_idx = sig_map_.FindOrInsert(func->sig);
193 : DCHECK_EQ(code_cache_.size(), new_cache_idx);
194 : USE(new_cache_idx);
195 25967 : code_cache_.push_back(code);
196 25967 : return code;
197 : }
198 :
199 : private:
200 : // sig_map_ maps signatures to an index in code_cache_.
201 : wasm::SignatureMap sig_map_;
202 : std::vector<Handle<Code>> code_cache_;
203 : };
204 :
205 : // Ensure that the code object in <code_table> at offset <func_index> has
206 : // deoptimization data attached. This is needed for lazy compile stubs which are
207 : // called from JS_TO_WASM functions or via exported function tables. The deopt
208 : // data is used to determine which function this lazy compile stub belongs to.
209 56559 : Handle<Code> EnsureExportedLazyDeoptData(Isolate* isolate,
210 : Handle<WasmInstanceObject> instance,
211 : Handle<FixedArray> code_table,
212 : int func_index) {
213 : Handle<Code> code(Code::cast(code_table->get(func_index)), isolate);
214 56559 : if (code->builtin_index() != Builtins::kWasmCompileLazy) {
215 : // No special deopt data needed for compiled functions, and imported
216 : // functions, which map to Illegal at this point (they get compiled at
217 : // instantiation time).
218 : DCHECK(code->kind() == Code::WASM_FUNCTION ||
219 : code->kind() == Code::WASM_TO_JS_FUNCTION ||
220 : code->builtin_index() == Builtins::kIllegal);
221 39052 : return code;
222 : }
223 : // deopt_data:
224 : // #0: weak instance
225 : // #1: func_index
226 : // might be extended later for table exports (see
227 : // EnsureTableExportLazyDeoptData).
228 : Handle<FixedArray> deopt_data(code->deoptimization_data());
229 : DCHECK_EQ(0, deopt_data->length() % 2);
230 17507 : if (deopt_data->length() == 0) {
231 13651 : code = isolate->factory()->CopyCode(code);
232 13651 : code_table->set(func_index, *code);
233 13651 : deopt_data = isolate->factory()->NewFixedArray(2, TENURED);
234 13651 : code->set_deoptimization_data(*deopt_data);
235 13651 : if (!instance.is_null()) {
236 : Handle<WeakCell> weak_instance =
237 5224 : isolate->factory()->NewWeakCell(instance);
238 5224 : deopt_data->set(0, *weak_instance);
239 : }
240 : deopt_data->set(1, Smi::FromInt(func_index));
241 : }
242 : DCHECK_IMPLIES(!instance.is_null(),
243 : WeakCell::cast(code->deoptimization_data()->get(0))->value() ==
244 : *instance);
245 : DCHECK_EQ(func_index,
246 : Smi::cast(code->deoptimization_data()->get(1))->value());
247 17507 : return code;
248 : }
249 :
250 : // Ensure that the code object in <code_table> at offset <func_index> has
251 : // deoptimization data attached. This is needed for lazy compile stubs which are
252 : // called from JS_TO_WASM functions or via exported function tables. The deopt
253 : // data is used to determine which function this lazy compile stub belongs to.
254 8610 : Handle<Code> EnsureTableExportLazyDeoptData(
255 : Isolate* isolate, Handle<WasmInstanceObject> instance,
256 : Handle<FixedArray> code_table, int func_index,
257 : Handle<FixedArray> export_table, int export_index,
258 : std::unordered_map<uint32_t, uint32_t>& table_export_count) {
259 : Handle<Code> code =
260 8610 : EnsureExportedLazyDeoptData(isolate, instance, code_table, func_index);
261 8610 : if (code->builtin_index() != Builtins::kWasmCompileLazy) return code;
262 :
263 : // deopt_data:
264 : // #0: weak instance
265 : // #1: func_index
266 : // [#2: export table
267 : // #3: export table index]
268 : // [#4: export table
269 : // #5: export table index]
270 : // ...
271 : // table_export_count counts down and determines the index for the new export
272 : // table entry.
273 11884 : auto table_export_entry = table_export_count.find(func_index);
274 : DCHECK(table_export_entry != table_export_count.end());
275 : DCHECK_LT(0, table_export_entry->second);
276 5942 : uint32_t this_idx = 2 * table_export_entry->second;
277 5942 : --table_export_entry->second;
278 : Handle<FixedArray> deopt_data(code->deoptimization_data());
279 : DCHECK_EQ(0, deopt_data->length() % 2);
280 5942 : if (deopt_data->length() == 2) {
281 : // Then only the "header" (#0 and #1) exists. Extend for the export table
282 : // entries (make space for this_idx + 2 elements).
283 : deopt_data = isolate->factory()->CopyFixedArrayAndGrow(deopt_data, this_idx,
284 4128 : TENURED);
285 4128 : code->set_deoptimization_data(*deopt_data);
286 : }
287 : DCHECK_LE(this_idx + 2, deopt_data->length());
288 : DCHECK(deopt_data->get(this_idx)->IsUndefined(isolate));
289 : DCHECK(deopt_data->get(this_idx + 1)->IsUndefined(isolate));
290 11884 : deopt_data->set(this_idx, *export_table);
291 5942 : deopt_data->set(this_idx + 1, Smi::FromInt(export_index));
292 5942 : return code;
293 : }
294 :
295 4614 : bool compile_lazy(const WasmModule* module) {
296 16819 : return FLAG_wasm_lazy_compilation ||
297 4614 : (FLAG_asm_wasm_lazy_compilation && module->is_asm_js());
298 : }
299 :
300 : // A helper for compiling an entire module.
301 55672 : class CompilationHelper {
302 : public:
303 27836 : CompilationHelper(Isolate* isolate, WasmModule* module)
304 83508 : : isolate_(isolate), module_(module) {}
305 :
306 : // The actual runnable task that performs compilations in the background.
307 28624 : class CompilationTask : public CancelableTask {
308 : public:
309 : CompilationHelper* helper_;
310 : explicit CompilationTask(CompilationHelper* helper)
311 14322 : : CancelableTask(helper->isolate_), helper_(helper) {}
312 :
313 12394 : void RunInternal() override {
314 12394 : while (helper_->FetchAndExecuteCompilationUnit()) {
315 : }
316 24802 : helper_->module_->pending_tasks.get()->Signal();
317 12397 : }
318 : };
319 :
320 : Isolate* isolate_;
321 : WasmModule* module_;
322 : std::vector<compiler::WasmCompilationUnit*> compilation_units_;
323 : std::queue<compiler::WasmCompilationUnit*> executed_units_;
324 : base::Mutex result_mutex_;
325 : base::AtomicNumber<size_t> next_unit_;
326 : size_t num_background_tasks_ = 0;
327 :
328 : // Run by each compilation task and by the main thread.
329 41115 : bool FetchAndExecuteCompilationUnit() {
330 : DisallowHeapAllocation no_allocation;
331 : DisallowHandleAllocation no_handles;
332 : DisallowHandleDereference no_deref;
333 : DisallowCodeDependencyChange no_dependency_change;
334 :
335 : // - 1 because AtomicIncrement returns the value after the atomic increment.
336 41118 : size_t index = next_unit_.Increment(1) - 1;
337 82236 : if (index >= compilation_units_.size()) {
338 : return false;
339 : }
340 :
341 26466 : compiler::WasmCompilationUnit* unit = compilation_units_.at(index);
342 26466 : unit->ExecuteCompilation();
343 26470 : base::LockGuard<base::Mutex> guard(&result_mutex_);
344 : executed_units_.push(unit);
345 : return true;
346 : }
347 :
348 2256 : size_t InitializeParallelCompilation(
349 28728 : const std::vector<WasmFunction>& functions, ModuleBytesEnv& module_env) {
350 2256 : uint32_t start = module_env.module_env.module->num_imported_functions +
351 2256 : FLAG_skip_compiling_wasm_funcs;
352 2256 : uint32_t num_funcs = static_cast<uint32_t>(functions.size());
353 2256 : uint32_t funcs_to_compile = start > num_funcs ? 0 : num_funcs - start;
354 2256 : compilation_units_.reserve(funcs_to_compile);
355 28728 : for (uint32_t i = start; i < num_funcs; ++i) {
356 26472 : const WasmFunction* func = &functions[i];
357 : compilation_units_.push_back(
358 52944 : new compiler::WasmCompilationUnit(isolate_, &module_env, func));
359 : }
360 2256 : return funcs_to_compile;
361 : }
362 :
363 : void InitializeHandles() {
364 55740 : for (auto unit : compilation_units_) {
365 26472 : unit->InitializeHandles();
366 : }
367 : }
368 :
369 2046 : uint32_t* StartCompilationTasks() {
370 : num_background_tasks_ =
371 : Min(static_cast<size_t>(FLAG_wasm_num_compilation_tasks),
372 4092 : V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads());
373 2046 : uint32_t* task_ids = new uint32_t[num_background_tasks_];
374 16368 : for (size_t i = 0; i < num_background_tasks_; ++i) {
375 14322 : CompilationTask* task = new CompilationTask(this);
376 14322 : task_ids[i] = task->id();
377 14322 : V8::GetCurrentPlatform()->CallOnBackgroundThread(
378 14322 : task, v8::Platform::kShortRunningTask);
379 : }
380 2046 : return task_ids;
381 : }
382 :
383 2046 : void WaitForCompilationTasks(uint32_t* task_ids) {
384 16368 : for (size_t i = 0; i < num_background_tasks_; ++i) {
385 : // If the task has not started yet, then we abort it. Otherwise we wait
386 : // for it to finish.
387 14322 : if (isolate_->cancelable_task_manager()->TryAbort(task_ids[i]) !=
388 : CancelableTaskManager::kTaskAborted) {
389 24812 : module_->pending_tasks.get()->Wait();
390 : }
391 : }
392 2046 : }
393 :
394 29387 : void FinishCompilationUnits(std::vector<Handle<Code>>& results,
395 : ErrorThrower* thrower) {
396 : while (true) {
397 29387 : int func_index = -1;
398 29387 : Handle<Code> result = FinishCompilationUnit(thrower, &func_index);
399 29387 : if (func_index < 0) break;
400 52524 : results[func_index] = result;
401 26262 : }
402 3125 : }
403 :
404 29597 : Handle<Code> FinishCompilationUnit(ErrorThrower* thrower, int* func_index) {
405 26472 : compiler::WasmCompilationUnit* unit = nullptr;
406 : {
407 29597 : base::LockGuard<base::Mutex> guard(&result_mutex_);
408 29597 : if (executed_units_.empty()) return Handle<Code>::null();
409 26472 : unit = executed_units_.front();
410 : executed_units_.pop();
411 : }
412 26472 : *func_index = unit->func_index();
413 26472 : Handle<Code> result = unit->FinishCompilation(thrower);
414 26472 : delete unit;
415 26472 : return result;
416 : }
417 :
418 2046 : void CompileInParallel(ModuleBytesEnv* module_env,
419 : std::vector<Handle<Code>>& results,
420 : ErrorThrower* thrower) {
421 2046 : const WasmModule* module = module_env->module_env.module;
422 : // Data structures for the parallel compilation.
423 :
424 : //-----------------------------------------------------------------------
425 : // For parallel compilation:
426 : // 1) The main thread allocates a compilation unit for each wasm function
427 : // and stores them in the vector {compilation_units}.
428 : // 2) The main thread spawns {CompilationTask} instances which run on
429 : // the background threads.
430 : // 3.a) The background threads and the main thread pick one compilation
431 : // unit at a time and execute the parallel phase of the compilation
432 : // unit. After finishing the execution of the parallel phase, the
433 : // result is enqueued in {executed_units}.
434 : // 3.b) If {executed_units} contains a compilation unit, the main thread
435 : // dequeues it and finishes the compilation.
436 : // 4) After the parallel phase of all compilation units has started, the
437 : // main thread waits for all {CompilationTask} instances to finish.
438 : // 5) The main thread finishes the compilation.
439 :
440 : // Turn on the {CanonicalHandleScope} so that the background threads can
441 : // use the node cache.
442 2046 : CanonicalHandleScope canonical(isolate_);
443 :
444 : // 1) The main thread allocates a compilation unit for each wasm function
445 : // and stores them in the vector {compilation_units}.
446 2046 : InitializeParallelCompilation(module->functions, *module_env);
447 : InitializeHandles();
448 :
449 : // Objects for the synchronization with the background threads.
450 : base::AtomicNumber<size_t> next_unit(
451 : static_cast<size_t>(FLAG_skip_compiling_wasm_funcs));
452 :
453 : // 2) The main thread spawns {CompilationTask} instances which run on
454 : // the background threads.
455 2046 : std::unique_ptr<uint32_t[]> task_ids(StartCompilationTasks());
456 :
457 : // 3.a) The background threads and the main thread pick one compilation
458 : // unit at a time and execute the parallel phase of the compilation
459 : // unit. After finishing the execution of the parallel phase, the
460 : // result is enqueued in {executed_units}.
461 3125 : while (FetchAndExecuteCompilationUnit()) {
462 : // 3.b) If {executed_units} contains a compilation unit, the main thread
463 : // dequeues it and finishes the compilation unit. Compilation units
464 : // are finished concurrently to the background threads to save
465 : // memory.
466 1079 : FinishCompilationUnits(results, thrower);
467 : }
468 : // 4) After the parallel phase of all compilation units has started, the
469 : // main thread waits for all {CompilationTask} instances to finish.
470 2046 : WaitForCompilationTasks(task_ids.get());
471 : // Finish the compilation of the remaining compilation units.
472 4092 : FinishCompilationUnits(results, thrower);
473 2046 : }
474 :
475 9689 : void CompileSequentially(ModuleBytesEnv* module_env,
476 8371 : std::vector<Handle<Code>>& results,
477 : ErrorThrower* thrower) {
478 : DCHECK(!thrower->error());
479 :
480 9689 : const WasmModule* module = module_env->module_env.module;
481 46782 : for (uint32_t i = FLAG_skip_compiling_wasm_funcs;
482 23391 : i < module->functions.size(); ++i) {
483 23554 : const WasmFunction& func = module->functions[i];
484 13865 : if (func.imported)
485 5331 : continue; // Imports are compiled at instantiation time.
486 :
487 : // Compile the function.
488 : Handle<Code> code = compiler::WasmCompilationUnit::CompileWasmFunction(
489 8534 : thrower, isolate_, module_env, &func);
490 8534 : if (code.is_null()) {
491 163 : WasmName str = module_env->wire_bytes.GetName(&func);
492 : thrower->CompileError("Compilation of #%d:%.*s failed.", i,
493 163 : str.length(), str.start());
494 : break;
495 : }
496 8371 : results[i] = code;
497 : }
498 9689 : }
499 :
500 15351 : MaybeHandle<WasmModuleObject> CompileToModuleObject(
501 : ErrorThrower* thrower, const ModuleWireBytes& wire_bytes,
502 : Handle<Script> asm_js_script,
503 : Vector<const byte> asm_js_offset_table_bytes) {
504 57488 : Factory* factory = isolate_->factory();
505 : // The {module_wrapper} will take ownership of the {WasmModule} object,
506 : // and it will be destroyed when the GC reclaims the wrapper object.
507 : Handle<WasmModuleWrapper> module_wrapper =
508 46053 : WasmModuleWrapper::New(isolate_, module_);
509 15351 : WasmInstance temp_instance(module_);
510 15351 : temp_instance.context = isolate_->native_context();
511 15351 : temp_instance.mem_size = WasmModule::kPageSize * module_->min_mem_pages;
512 15351 : temp_instance.mem_start = nullptr;
513 15351 : temp_instance.globals_start = nullptr;
514 :
515 : // Initialize the indirect tables with placeholders.
516 : int function_table_count =
517 30702 : static_cast<int>(module_->function_tables.size());
518 : Handle<FixedArray> function_tables =
519 15351 : factory->NewFixedArray(function_table_count, TENURED);
520 : Handle<FixedArray> signature_tables =
521 15351 : factory->NewFixedArray(function_table_count, TENURED);
522 949 : for (int i = 0; i < function_table_count; ++i) {
523 1898 : temp_instance.function_tables[i] = factory->NewFixedArray(1, TENURED);
524 1898 : temp_instance.signature_tables[i] = factory->NewFixedArray(1, TENURED);
525 1898 : function_tables->set(i, *temp_instance.function_tables[i]);
526 1898 : signature_tables->set(i, *temp_instance.signature_tables[i]);
527 : }
528 :
529 : HistogramTimerScope wasm_compile_module_time_scope(
530 15351 : module_->is_wasm()
531 11735 : ? isolate_->counters()->wasm_compile_wasm_module_time()
532 30702 : : isolate_->counters()->wasm_compile_asm_module_time());
533 :
534 15351 : ModuleBytesEnv module_env(module_, &temp_instance, wire_bytes);
535 :
536 : // The {code_table} array contains import wrappers and functions (which
537 : // are both included in {functions.size()}, and export wrappers.
538 98741 : int code_table_size = static_cast<int>(module_->functions.size() +
539 15351 : module_->num_exported_functions);
540 : Handle<FixedArray> code_table =
541 15351 : factory->NewFixedArray(static_cast<int>(code_table_size), TENURED);
542 :
543 : // Check whether lazy compilation is enabled for this module.
544 15351 : bool lazy_compile = compile_lazy(module_);
545 :
546 : // If lazy compile: Initialize the code table with the lazy compile builtin.
547 : // Otherwise: Initialize with the illegal builtin. All call sites will be
548 : // patched at instantiation.
549 : Handle<Code> init_builtin = lazy_compile
550 3616 : ? isolate_->builtins()->WasmCompileLazy()
551 18967 : : isolate_->builtins()->Illegal();
552 91792 : for (int i = 0, e = static_cast<int>(module_->functions.size()); i < e;
553 : ++i) {
554 61090 : code_table->set(i, *init_builtin);
555 122180 : temp_instance.function_code[i] = init_builtin;
556 : }
557 :
558 11735 : (module_->is_wasm() ? isolate_->counters()->wasm_functions_per_wasm_module()
559 3616 : : isolate_->counters()->wasm_functions_per_asm_module())
560 61404 : ->AddSample(static_cast<int>(module_->functions.size()));
561 :
562 15351 : if (!lazy_compile) {
563 11735 : CompilationHelper helper(isolate_, module_);
564 : size_t funcs_to_compile =
565 23470 : module_->functions.size() - module_->num_imported_functions;
566 11735 : if (!FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks != 0 &&
567 : funcs_to_compile > 1) {
568 : // Avoid a race condition by collecting results into a second vector.
569 2046 : std::vector<Handle<Code>> results(temp_instance.function_code);
570 2046 : helper.CompileInParallel(&module_env, results, thrower);
571 : temp_instance.function_code.swap(results);
572 : } else {
573 : helper.CompileSequentially(&module_env, temp_instance.function_code,
574 9689 : thrower);
575 : }
576 12035 : if (thrower->error()) return {};
577 : }
578 :
579 : // At this point, compilation has completed. Update the code table.
580 150786 : for (size_t i = FLAG_skip_compiling_wasm_funcs;
581 75393 : i < temp_instance.function_code.size(); ++i) {
582 : Code* code = *temp_instance.function_code[i];
583 120684 : code_table->set(static_cast<int>(i), code);
584 60342 : RecordStats(isolate_, code);
585 : }
586 :
587 : // Create heap objects for script, module bytes and asm.js offset table to
588 : // be
589 : // stored in the shared module data.
590 : Handle<Script> script;
591 : Handle<ByteArray> asm_js_offset_table;
592 15051 : if (asm_js_script.is_null()) {
593 11435 : script = CreateWasmScript(isolate_, wire_bytes);
594 : } else {
595 : script = asm_js_script;
596 : asm_js_offset_table =
597 3616 : isolate_->factory()->NewByteArray(asm_js_offset_table_bytes.length());
598 : asm_js_offset_table->copy_in(0, asm_js_offset_table_bytes.start(),
599 3616 : asm_js_offset_table_bytes.length());
600 : }
601 : // TODO(wasm): only save the sections necessary to deserialize a
602 : // {WasmModule}. E.g. function bodies could be omitted.
603 : Handle<String> module_bytes =
604 : factory
605 : ->NewStringFromOneByte({wire_bytes.start(), wire_bytes.length()},
606 : TENURED)
607 30102 : .ToHandleChecked();
608 : DCHECK(module_bytes->IsSeqOneByteString());
609 :
610 : // Create the shared module data.
611 : // TODO(clemensh): For the same module (same bytes / same hash), we should
612 : // only have one WasmSharedModuleData. Otherwise, we might only set
613 : // breakpoints on a (potentially empty) subset of the instances.
614 :
615 : Handle<WasmSharedModuleData> shared = WasmSharedModuleData::New(
616 : isolate_, module_wrapper, Handle<SeqOneByteString>::cast(module_bytes),
617 15051 : script, asm_js_offset_table);
618 15051 : if (lazy_compile) WasmSharedModuleData::PrepareForLazyCompilation(shared);
619 :
620 : // Create the compiled module object, and populate with compiled functions
621 : // and information needed at instantiation time. This object needs to be
622 : // serializable. Instantiation may occur off a deserialized version of this
623 : // object.
624 : Handle<WasmCompiledModule> compiled_module = WasmCompiledModule::New(
625 15051 : isolate_, shared, code_table, function_tables, signature_tables);
626 :
627 : // If we created a wasm script, finish it now and make it public to the
628 : // debugger.
629 15051 : if (asm_js_script.is_null()) {
630 11435 : script->set_wasm_compiled_module(*compiled_module);
631 22870 : isolate_->debug()->OnAfterCompile(script);
632 : }
633 :
634 : // Compile JS->WASM wrappers for exported functions.
635 15051 : JSToWasmWrapperCache js_to_wasm_cache;
636 : int func_index = 0;
637 114026 : for (auto exp : module_->export_table) {
638 42971 : if (exp.kind != kExternalFunction) continue;
639 : Handle<Code> wasm_code = EnsureExportedLazyDeoptData(
640 40953 : isolate_, Handle<WasmInstanceObject>::null(), code_table, exp.index);
641 : Handle<Code> wrapper_code =
642 : js_to_wasm_cache.CloneOrCompileJSToWasmWrapper(isolate_, module_,
643 40953 : wasm_code, exp.index);
644 : int export_index =
645 81906 : static_cast<int>(module_->functions.size() + func_index);
646 40953 : code_table->set(export_index, *wrapper_code);
647 40953 : RecordStats(isolate_, *wrapper_code);
648 40953 : func_index++;
649 : }
650 :
651 30402 : return WasmModuleObject::New(isolate_, compiled_module);
652 : }
653 : };
654 :
655 15898 : static void MemoryInstanceFinalizer(Isolate* isolate,
656 : WasmInstanceObject* instance) {
657 : DisallowHeapAllocation no_gc;
658 : // If the memory object is destroyed, nothing needs to be done here.
659 15898 : if (!instance->has_memory_object()) return;
660 : Handle<WasmInstanceWrapper> instance_wrapper =
661 1141 : handle(instance->instance_wrapper());
662 : DCHECK(WasmInstanceWrapper::IsWasmInstanceWrapper(*instance_wrapper));
663 : DCHECK(instance_wrapper->has_instance());
664 1141 : bool has_prev = instance_wrapper->has_previous();
665 1141 : bool has_next = instance_wrapper->has_next();
666 1141 : Handle<WasmMemoryObject> memory_object(instance->memory_object());
667 :
668 1141 : if (!has_prev && !has_next) {
669 811 : memory_object->ResetInstancesLink(isolate);
670 811 : return;
671 : } else {
672 : Handle<WasmInstanceWrapper> next_wrapper, prev_wrapper;
673 330 : if (!has_prev) {
674 : Handle<WasmInstanceWrapper> next_wrapper =
675 61 : instance_wrapper->next_wrapper();
676 : next_wrapper->reset_previous_wrapper();
677 : // As this is the first link in the memory object, destroying
678 : // without updating memory object would corrupt the instance chain in
679 : // the memory object.
680 61 : memory_object->set_instances_link(*next_wrapper);
681 269 : } else if (!has_next) {
682 414 : instance_wrapper->previous_wrapper()->reset_next_wrapper();
683 : } else {
684 : DCHECK(has_next && has_prev);
685 : Handle<WasmInstanceWrapper> prev_wrapper =
686 62 : instance_wrapper->previous_wrapper();
687 : Handle<WasmInstanceWrapper> next_wrapper =
688 62 : instance_wrapper->next_wrapper();
689 : prev_wrapper->set_next_wrapper(*next_wrapper);
690 : next_wrapper->set_previous_wrapper(*prev_wrapper);
691 : }
692 : // Reset to avoid dangling pointers
693 : instance_wrapper->reset();
694 : }
695 : }
696 :
697 31796 : static void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) {
698 : DisallowHeapAllocation no_gc;
699 : JSObject** p = reinterpret_cast<JSObject**>(data.GetParameter());
700 15898 : WasmInstanceObject* owner = reinterpret_cast<WasmInstanceObject*>(*p);
701 : Isolate* isolate = reinterpret_cast<Isolate*>(data.GetIsolate());
702 : // If a link to shared memory instances exists, update the list of memory
703 : // instances before the instance is destroyed.
704 15898 : if (owner->has_instance_wrapper()) MemoryInstanceFinalizer(isolate, owner);
705 15898 : WasmCompiledModule* compiled_module = owner->compiled_module();
706 : TRACE("Finalizing %d {\n", compiled_module->instance_id());
707 : DCHECK(compiled_module->has_weak_wasm_module());
708 : WeakCell* weak_wasm_module = compiled_module->ptr_to_weak_wasm_module();
709 :
710 15898 : if (trap_handler::UseTrapHandler()) {
711 3097 : Handle<FixedArray> code_table = compiled_module->code_table();
712 35312 : for (int i = 0; i < code_table->length(); ++i) {
713 14559 : Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i);
714 : int index = code->trap_handler_index()->value();
715 14559 : if (index >= 0) {
716 188 : trap_handler::ReleaseHandlerData(index);
717 188 : code->set_trap_handler_index(Smi::FromInt(-1));
718 : }
719 : }
720 : }
721 :
722 : // weak_wasm_module may have been cleared, meaning the module object
723 : // was GC-ed. In that case, there won't be any new instances created,
724 : // and we don't need to maintain the links between instances.
725 15898 : if (!weak_wasm_module->cleared()) {
726 : JSObject* wasm_module = JSObject::cast(weak_wasm_module->value());
727 : WasmCompiledModule* current_template =
728 : WasmCompiledModule::cast(wasm_module->GetEmbedderField(0));
729 :
730 : TRACE("chain before {\n");
731 2744 : TRACE_CHAIN(current_template);
732 : TRACE("}\n");
733 :
734 : DCHECK(!current_template->has_weak_prev_instance());
735 2744 : WeakCell* next = compiled_module->maybe_ptr_to_weak_next_instance();
736 2744 : WeakCell* prev = compiled_module->maybe_ptr_to_weak_prev_instance();
737 :
738 2744 : if (current_template == compiled_module) {
739 1969 : if (next == nullptr) {
740 1643 : WasmCompiledModule::Reset(isolate, compiled_module);
741 : } else {
742 : DCHECK(next->value()->IsFixedArray());
743 326 : wasm_module->SetEmbedderField(0, next->value());
744 : DCHECK_NULL(prev);
745 : WasmCompiledModule::cast(next->value())->reset_weak_prev_instance();
746 : }
747 : } else {
748 : DCHECK(!(prev == nullptr && next == nullptr));
749 : // the only reason prev or next would be cleared is if the
750 : // respective objects got collected, but if that happened,
751 : // we would have relinked the list.
752 775 : if (prev != nullptr) {
753 : DCHECK(!prev->cleared());
754 775 : if (next == nullptr) {
755 : WasmCompiledModule::cast(prev->value())->reset_weak_next_instance();
756 : } else {
757 : WasmCompiledModule::cast(prev->value())
758 : ->set_ptr_to_weak_next_instance(next);
759 : }
760 : }
761 775 : if (next != nullptr) {
762 : DCHECK(!next->cleared());
763 176 : if (prev == nullptr) {
764 : WasmCompiledModule::cast(next->value())->reset_weak_prev_instance();
765 : } else {
766 : WasmCompiledModule::cast(next->value())
767 : ->set_ptr_to_weak_prev_instance(prev);
768 : }
769 : }
770 : }
771 : TRACE("chain after {\n");
772 2744 : TRACE_CHAIN(WasmCompiledModule::cast(wasm_module->GetEmbedderField(0)));
773 : TRACE("}\n");
774 : }
775 : compiled_module->reset_weak_owning_instance();
776 15898 : GlobalHandles::Destroy(reinterpret_cast<Object**>(p));
777 : TRACE("}\n");
778 15898 : }
779 :
780 136345 : int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
781 : int offset) {
782 : DCHECK(!iterator.done());
783 : int byte_pos;
784 55302 : do {
785 : byte_pos = iterator.source_position().ScriptOffset();
786 55302 : iterator.Advance();
787 110604 : } while (!iterator.done() && iterator.code_offset() <= offset);
788 25741 : return byte_pos;
789 : }
790 :
791 25741 : int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) {
792 : DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc));
793 : // Read the leb128 encoded u32 value (up to 5 bytes starting at pc + 1).
794 25741 : decoder.Reset(pc + 1, pc + 6);
795 : uint32_t call_idx = decoder.consume_u32v("call index");
796 : DCHECK(decoder.ok());
797 : DCHECK_GE(kMaxInt, call_idx);
798 25741 : return static_cast<int>(call_idx);
799 : }
800 :
801 37488 : void RecordLazyCodeStats(Isolate* isolate, Code* code) {
802 12496 : isolate->counters()->wasm_lazily_compiled_functions()->Increment();
803 24992 : isolate->counters()->wasm_generated_code_size()->Increment(code->body_size());
804 : isolate->counters()->wasm_reloc_size()->Increment(
805 12496 : code->relocation_info()->length());
806 12496 : }
807 :
808 : } // namespace
809 :
810 12337 : Handle<JSArrayBuffer> wasm::SetupArrayBuffer(Isolate* isolate,
811 : void* backing_store, size_t size,
812 : bool is_external,
813 : bool enable_guard_regions) {
814 11702 : Handle<JSArrayBuffer> buffer = isolate->factory()->NewJSArrayBuffer();
815 : JSArrayBuffer::Setup(buffer, isolate, is_external, backing_store,
816 11702 : static_cast<int>(size));
817 : buffer->set_is_neuterable(false);
818 : buffer->set_has_guard_region(enable_guard_regions);
819 :
820 11702 : if (is_external) {
821 : // We mark the buffer as external if we allocated it here with guard
822 : // pages. That means we need to arrange for it to be freed.
823 :
824 : // TODO(eholk): Finalizers may not run when the main thread is shutting
825 : // down, which means we may leak memory here.
826 : Handle<Object> global_handle = isolate->global_handles()->Create(*buffer);
827 : GlobalHandles::MakeWeak(global_handle.location(), global_handle.location(),
828 635 : &MemoryFinalizer, v8::WeakCallbackType::kFinalizer);
829 : }
830 11702 : return buffer;
831 : }
832 :
833 11657 : Handle<JSArrayBuffer> wasm::NewArrayBuffer(Isolate* isolate, size_t size,
834 : bool enable_guard_regions) {
835 11657 : if (size > (FLAG_wasm_max_mem_pages * WasmModule::kPageSize)) {
836 : // TODO(titzer): lift restriction on maximum memory allocated here.
837 : return Handle<JSArrayBuffer>::null();
838 : }
839 :
840 : enable_guard_regions = enable_guard_regions && kGuardRegionsSupported;
841 :
842 : bool is_external; // Set by TryAllocateBackingStore
843 : void* memory =
844 11657 : TryAllocateBackingStore(isolate, size, enable_guard_regions, is_external);
845 :
846 11657 : if (memory == nullptr) {
847 : return Handle<JSArrayBuffer>::null();
848 : }
849 :
850 : #if DEBUG
851 : // Double check the API allocator actually zero-initialized the memory.
852 : const byte* bytes = reinterpret_cast<const byte*>(memory);
853 : for (size_t i = 0; i < size; ++i) {
854 : DCHECK_EQ(0, bytes[i]);
855 : }
856 : #endif
857 :
858 : return SetupArrayBuffer(isolate, memory, size, is_external,
859 11657 : enable_guard_regions);
860 : }
861 :
862 6187 : void wasm::UnpackAndRegisterProtectedInstructions(
863 7741 : Isolate* isolate, Handle<FixedArray> code_table) {
864 50366 : for (int i = 0; i < code_table->length(); ++i) {
865 : Handle<Code> code;
866 : // This is sometimes undefined when we're called from cctests.
867 37992 : if (!code_table->GetValue<Code>(isolate, i).ToHandle(&code)) {
868 11255 : continue;
869 : }
870 :
871 18087 : if (code->kind() != Code::WASM_FUNCTION) {
872 : continue;
873 : }
874 :
875 7741 : const intptr_t base = reinterpret_cast<intptr_t>(code->entry());
876 :
877 7741 : Zone zone(isolate->allocator(), "Wasm Module");
878 : ZoneVector<trap_handler::ProtectedInstructionData> unpacked(&zone);
879 : const int mode_mask =
880 : RelocInfo::ModeMask(RelocInfo::WASM_PROTECTED_INSTRUCTION_LANDING);
881 9428 : for (RelocIterator it(*code, mode_mask); !it.done(); it.next()) {
882 : trap_handler::ProtectedInstructionData data;
883 1687 : data.instr_offset = it.rinfo()->data();
884 1687 : data.landing_offset = reinterpret_cast<intptr_t>(it.rinfo()->pc()) - base;
885 1687 : unpacked.emplace_back(data);
886 : }
887 15482 : if (unpacked.size() > 0) {
888 : int size = code->CodeSize();
889 : const int index = RegisterHandlerData(reinterpret_cast<void*>(base), size,
890 336 : unpacked.size(), &unpacked[0]);
891 : // TODO(eholk): if index is negative, fail.
892 : DCHECK(index >= 0);
893 336 : code->set_trap_handler_index(Smi::FromInt(index));
894 : }
895 7741 : }
896 6187 : }
897 :
898 466 : std::ostream& wasm::operator<<(std::ostream& os, const WasmFunctionName& name) {
899 466 : os << "#" << name.function_->func_index;
900 466 : if (name.function_->name_offset > 0) {
901 0 : if (name.name_.start()) {
902 0 : os << ":";
903 0 : os.write(name.name_.start(), name.name_.length());
904 : }
905 : } else {
906 466 : os << "?";
907 : }
908 466 : return os;
909 : }
910 :
911 162254 : WasmInstanceObject* wasm::GetOwningWasmInstance(Code* code) {
912 : DisallowHeapAllocation no_gc;
913 : DCHECK(code->kind() == Code::WASM_FUNCTION ||
914 : code->kind() == Code::WASM_INTERPRETER_ENTRY);
915 : FixedArray* deopt_data = code->deoptimization_data();
916 : DCHECK_EQ(code->kind() == Code::WASM_INTERPRETER_ENTRY ? 1 : 2,
917 : deopt_data->length());
918 : Object* weak_link = deopt_data->get(0);
919 : DCHECK(weak_link->IsWeakCell());
920 : WeakCell* cell = WeakCell::cast(weak_link);
921 162254 : if (cell->cleared()) return nullptr;
922 162254 : return WasmInstanceObject::cast(cell->value());
923 : }
924 :
925 59658 : WasmModule::WasmModule(Zone* owned)
926 238632 : : owned_zone(owned), pending_tasks(new base::Semaphore(0)) {}
927 :
928 : namespace {
929 :
930 39906 : WasmFunction* GetWasmFunctionForImportWrapper(Isolate* isolate,
931 : Handle<Object> target) {
932 39906 : if (target->IsJSFunction()) {
933 : Handle<JSFunction> func = Handle<JSFunction>::cast(target);
934 39861 : if (func->code()->kind() == Code::JS_TO_WASM_FUNCTION) {
935 : auto exported = Handle<WasmExportedFunction>::cast(func);
936 3171 : Handle<WasmInstanceObject> other_instance(exported->instance(), isolate);
937 3171 : int func_index = exported->function_index();
938 6342 : return &other_instance->module()->functions[func_index];
939 : }
940 : }
941 : return nullptr;
942 : }
943 :
944 3126 : static Handle<Code> UnwrapImportWrapper(Handle<Object> import_wrapper) {
945 : Handle<JSFunction> func = Handle<JSFunction>::cast(import_wrapper);
946 3126 : Handle<Code> export_wrapper_code = handle(func->code());
947 : int mask = RelocInfo::ModeMask(RelocInfo::CODE_TARGET);
948 3981 : for (RelocIterator it(*export_wrapper_code, mask);; it.next()) {
949 : DCHECK(!it.done());
950 7962 : Code* target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
951 5997 : if (target->kind() != Code::WASM_FUNCTION &&
952 4842 : target->kind() != Code::WASM_TO_JS_FUNCTION &&
953 : target->kind() != Code::WASM_INTERPRETER_ENTRY)
954 : continue;
955 : // There should only be this one call to wasm code.
956 : #ifdef DEBUG
957 : for (it.next(); !it.done(); it.next()) {
958 : Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
959 : DCHECK(code->kind() != Code::WASM_FUNCTION &&
960 : code->kind() != Code::WASM_TO_JS_FUNCTION &&
961 : code->kind() != Code::WASM_INTERPRETER_ENTRY);
962 : }
963 : #endif
964 : return handle(target);
965 855 : }
966 : UNREACHABLE();
967 : return Handle<Code>::null();
968 : }
969 :
970 36831 : Handle<Code> CompileImportWrapper(Isolate* isolate, int index, FunctionSig* sig,
971 : Handle<JSReceiver> target,
972 : Handle<String> module_name,
973 : MaybeHandle<String> import_name,
974 : ModuleOrigin origin) {
975 36831 : WasmFunction* other_func = GetWasmFunctionForImportWrapper(isolate, target);
976 36831 : if (other_func) {
977 96 : if (!sig->Equals(other_func->sig)) return Handle<Code>::null();
978 : // Signature matched. Unwrap the JS->WASM wrapper and return the raw
979 : // WASM function code.
980 51 : return UnwrapImportWrapper(target);
981 : }
982 : // No wasm function or being debugged. Compile a new wrapper for the new
983 : // signature.
984 : return compiler::CompileWasmToJSWrapper(isolate, target, sig, index,
985 36735 : module_name, import_name, origin);
986 : }
987 :
988 4772 : void UpdateDispatchTablesInternal(Isolate* isolate,
989 : Handle<FixedArray> dispatch_tables, int index,
990 : WasmFunction* function, Handle<Code> code) {
991 : DCHECK_EQ(0, dispatch_tables->length() % 4);
992 18934 : for (int i = 0; i < dispatch_tables->length(); i += 4) {
993 4695 : int table_index = Smi::cast(dispatch_tables->get(i + 1))->value();
994 : Handle<FixedArray> function_table(
995 4695 : FixedArray::cast(dispatch_tables->get(i + 2)), isolate);
996 : Handle<FixedArray> signature_table(
997 4695 : FixedArray::cast(dispatch_tables->get(i + 3)), isolate);
998 4695 : if (function) {
999 : // TODO(titzer): the signature might need to be copied to avoid
1000 : // a dangling pointer in the signature map.
1001 : Handle<WasmInstanceObject> instance(
1002 3045 : WasmInstanceObject::cast(dispatch_tables->get(i)), isolate);
1003 9135 : auto func_table = instance->module()->function_tables[table_index];
1004 3045 : uint32_t sig_index = func_table.map.FindOrInsert(function->sig);
1005 3045 : signature_table->set(index, Smi::FromInt(static_cast<int>(sig_index)));
1006 3045 : function_table->set(index, *code);
1007 : } else {
1008 : signature_table->set(index, Smi::FromInt(-1));
1009 : function_table->set(index, Smi::kZero);
1010 : }
1011 : }
1012 4772 : }
1013 :
1014 : } // namespace
1015 :
1016 3480 : void wasm::UpdateDispatchTables(Isolate* isolate,
1017 : Handle<FixedArray> dispatch_tables, int index,
1018 : Handle<JSFunction> function) {
1019 3480 : if (function.is_null()) {
1020 : UpdateDispatchTablesInternal(isolate, dispatch_tables, index, nullptr,
1021 2250 : Handle<Code>::null());
1022 : } else {
1023 : UpdateDispatchTablesInternal(
1024 : isolate, dispatch_tables, index,
1025 : GetWasmFunctionForImportWrapper(isolate, function),
1026 2460 : UnwrapImportWrapper(function));
1027 : }
1028 3480 : }
1029 :
1030 : // A helper class to simplify instantiating a module from a compiled module.
1031 : // It closes over the {Isolate}, the {ErrorThrower}, the {WasmCompiledModule},
1032 : // etc.
1033 96198 : class InstantiationHelper {
1034 : public:
1035 48099 : InstantiationHelper(Isolate* isolate, ErrorThrower* thrower,
1036 : Handle<WasmModuleObject> module_object,
1037 : MaybeHandle<JSReceiver> ffi,
1038 : MaybeHandle<JSArrayBuffer> memory)
1039 : : isolate_(isolate),
1040 48099 : module_(module_object->compiled_module()->module()),
1041 : thrower_(thrower),
1042 : module_object_(module_object),
1043 : ffi_(ffi.is_null() ? Handle<JSReceiver>::null()
1044 : : ffi.ToHandleChecked()),
1045 : memory_(memory.is_null() ? Handle<JSArrayBuffer>::null()
1046 282110 : : memory.ToHandleChecked()) {}
1047 :
1048 : // Build an instance, in all of its glory.
1049 54606 : MaybeHandle<WasmInstanceObject> Build() {
1050 : // Check that an imports argument was provided, if the module requires it.
1051 : // No point in continuing otherwise.
1052 228842 : if (!module_->import_table.empty() && ffi_.is_null()) {
1053 : thrower_->TypeError(
1054 122 : "Imports argument must be present and must be an object");
1055 : return {};
1056 : }
1057 :
1058 : // Record build time into correct bucket, then build instance.
1059 : HistogramTimerScope wasm_instantiate_module_time_scope(
1060 : module_->is_wasm()
1061 190542 : ? isolate_->counters()->wasm_instantiate_wasm_module_time()
1062 95954 : : isolate_->counters()->wasm_instantiate_asm_module_time());
1063 47977 : Factory* factory = isolate_->factory();
1064 :
1065 : //--------------------------------------------------------------------------
1066 : // Reuse the compiled module (if no owner), otherwise clone.
1067 : //--------------------------------------------------------------------------
1068 : Handle<FixedArray> code_table;
1069 : // We keep around a copy of the old code table, because we'll be replacing
1070 : // imports for the new instance, and then we need the old imports to be
1071 : // able to relocate.
1072 : Handle<FixedArray> old_code_table;
1073 : MaybeHandle<WasmInstanceObject> owner;
1074 :
1075 : TRACE("Starting new module instantiation\n");
1076 : {
1077 : // Root the owner, if any, before doing any allocations, which
1078 : // may trigger GC.
1079 : // Both owner and original template need to be in sync. Even
1080 : // after we lose the original template handle, the code
1081 : // objects we copied from it have data relative to the
1082 : // instance - such as globals addresses.
1083 : Handle<WasmCompiledModule> original;
1084 : {
1085 : DisallowHeapAllocation no_gc;
1086 47977 : original = handle(module_object_->compiled_module());
1087 47977 : if (original->has_weak_owning_instance()) {
1088 : owner = handle(WasmInstanceObject::cast(
1089 96441 : original->weak_owning_instance()->value()));
1090 : }
1091 : }
1092 : DCHECK(!original.is_null());
1093 47977 : if (original->has_weak_owning_instance()) {
1094 : // Clone, but don't insert yet the clone in the instances chain.
1095 : // We do that last. Since we are holding on to the owner instance,
1096 : // the owner + original state used for cloning and patching
1097 : // won't be mutated by possible finalizer runs.
1098 : DCHECK(!owner.is_null());
1099 : TRACE("Cloning from %d\n", original->instance_id());
1100 32147 : old_code_table = original->code_table();
1101 32147 : compiled_module_ = WasmCompiledModule::Clone(isolate_, original);
1102 32147 : code_table = compiled_module_->code_table();
1103 : // Avoid creating too many handles in the outer scope.
1104 32147 : HandleScope scope(isolate_);
1105 :
1106 : // Clone the code for WASM functions and exports.
1107 295930 : for (int i = 0; i < code_table->length(); ++i) {
1108 115818 : Handle<Code> orig_code(Code::cast(code_table->get(i)), isolate_);
1109 115818 : switch (orig_code->kind()) {
1110 : case Code::WASM_TO_JS_FUNCTION:
1111 : // Imports will be overwritten with newly compiled wrappers.
1112 : break;
1113 : case Code::BUILTIN:
1114 : DCHECK_EQ(Builtins::kWasmCompileLazy, orig_code->builtin_index());
1115 : // If this code object has deoptimization data, then we need a
1116 : // unique copy to attach updated deoptimization data.
1117 6811 : if (orig_code->deoptimization_data()->length() > 0) {
1118 3894 : Handle<Code> code = factory->CopyCode(orig_code);
1119 : Handle<FixedArray> deopt_data =
1120 3894 : factory->NewFixedArray(2, TENURED);
1121 : deopt_data->set(1, Smi::FromInt(i));
1122 3894 : code->set_deoptimization_data(*deopt_data);
1123 3894 : code_table->set(i, *code);
1124 : }
1125 : break;
1126 : case Code::JS_TO_WASM_FUNCTION:
1127 : case Code::WASM_FUNCTION: {
1128 78932 : Handle<Code> code = factory->CopyCode(orig_code);
1129 78932 : code_table->set(i, *code);
1130 : break;
1131 : }
1132 : default:
1133 0 : UNREACHABLE();
1134 : }
1135 : }
1136 32147 : RecordStats(isolate_, code_table);
1137 : } else {
1138 : // There was no owner, so we can reuse the original.
1139 15830 : compiled_module_ = original;
1140 : old_code_table =
1141 15830 : factory->CopyFixedArray(compiled_module_->code_table());
1142 15830 : code_table = compiled_module_->code_table();
1143 : TRACE("Reusing existing instance %d\n",
1144 : compiled_module_->instance_id());
1145 : }
1146 47977 : compiled_module_->set_native_context(isolate_->native_context());
1147 : }
1148 :
1149 : //--------------------------------------------------------------------------
1150 : // Allocate the instance object.
1151 : //--------------------------------------------------------------------------
1152 95954 : Zone instantiation_zone(isolate_->allocator(), ZONE_NAME);
1153 95954 : CodeSpecialization code_specialization(isolate_, &instantiation_zone);
1154 : Handle<WasmInstanceObject> instance =
1155 47977 : WasmInstanceObject::New(isolate_, compiled_module_);
1156 :
1157 : //--------------------------------------------------------------------------
1158 : // Set up the globals for the new instance.
1159 : //--------------------------------------------------------------------------
1160 : MaybeHandle<JSArrayBuffer> old_globals;
1161 47977 : uint32_t globals_size = module_->globals_size;
1162 47977 : if (globals_size > 0) {
1163 : const bool enable_guard_regions = false;
1164 : Handle<JSArrayBuffer> global_buffer =
1165 2860 : NewArrayBuffer(isolate_, globals_size, enable_guard_regions);
1166 2860 : globals_ = global_buffer;
1167 2860 : if (globals_.is_null()) {
1168 0 : thrower_->RangeError("Out of memory: wasm globals");
1169 : return {};
1170 : }
1171 2860 : Address old_globals_start = compiled_module_->GetGlobalsStartOrNull();
1172 : Address new_globals_start =
1173 : static_cast<Address>(global_buffer->backing_store());
1174 2860 : code_specialization.RelocateGlobals(old_globals_start, new_globals_start);
1175 : // The address of the backing buffer for the golbals is in native memory
1176 : // and, thus, not moving. We need it saved for
1177 : // serialization/deserialization purposes - so that the other end
1178 : // understands how to relocate the references. We still need to save the
1179 : // JSArrayBuffer on the instance, to keep it all alive.
1180 : WasmCompiledModule::SetGlobalsStartAddressFrom(factory, compiled_module_,
1181 2860 : global_buffer);
1182 2860 : instance->set_globals_buffer(*global_buffer);
1183 : }
1184 :
1185 : //--------------------------------------------------------------------------
1186 : // Prepare for initialization of function tables.
1187 : //--------------------------------------------------------------------------
1188 : int function_table_count =
1189 95954 : static_cast<int>(module_->function_tables.size());
1190 49095 : table_instances_.reserve(module_->function_tables.size());
1191 49610 : for (int index = 0; index < function_table_count; ++index) {
1192 : table_instances_.push_back(
1193 : {Handle<WasmTableObject>::null(), Handle<FixedArray>::null(),
1194 3266 : Handle<FixedArray>::null(), Handle<FixedArray>::null()});
1195 : }
1196 :
1197 : //--------------------------------------------------------------------------
1198 : // Process the imports for the module.
1199 : //--------------------------------------------------------------------------
1200 47977 : int num_imported_functions = ProcessImports(code_table, instance);
1201 47977 : if (num_imported_functions < 0) return {};
1202 :
1203 : //--------------------------------------------------------------------------
1204 : // Process the initialization for the module's globals.
1205 : //--------------------------------------------------------------------------
1206 47302 : InitGlobals();
1207 :
1208 : //--------------------------------------------------------------------------
1209 : // Set up the indirect function tables for the new instance.
1210 : //--------------------------------------------------------------------------
1211 47302 : if (function_table_count > 0)
1212 1468 : InitializeTables(instance, &code_specialization);
1213 :
1214 : //--------------------------------------------------------------------------
1215 : // Set up the memory for the new instance.
1216 : //--------------------------------------------------------------------------
1217 47302 : uint32_t min_mem_pages = module_->min_mem_pages;
1218 40815 : (module_->is_wasm() ? isolate_->counters()->wasm_wasm_min_mem_pages_count()
1219 6487 : : isolate_->counters()->wasm_asm_min_mem_pages_count())
1220 141906 : ->AddSample(min_mem_pages);
1221 :
1222 47302 : if (!memory_.is_null()) {
1223 : // Set externally passed ArrayBuffer non neuterable.
1224 : memory_->set_is_neuterable(false);
1225 :
1226 : DCHECK_IMPLIES(EnableGuardRegions(),
1227 : module_->is_asm_js() || memory_->has_guard_region());
1228 44107 : } else if (min_mem_pages > 0) {
1229 6507 : memory_ = AllocateMemory(min_mem_pages);
1230 6507 : if (memory_.is_null()) return {}; // failed to allocate memory
1231 : }
1232 :
1233 : //--------------------------------------------------------------------------
1234 : // Check that indirect function table segments are within bounds.
1235 : //--------------------------------------------------------------------------
1236 143024 : for (WasmTableInit& table_init : module_->table_inits) {
1237 : DCHECK(table_init.table_index < table_instances_.size());
1238 1118 : uint32_t base = EvalUint32InitExpr(table_init.offset);
1239 : uint32_t table_size =
1240 2236 : table_instances_[table_init.table_index].function_table->length();
1241 2236 : if (!in_bounds(base, static_cast<uint32_t>(table_init.entries.size()),
1242 1118 : table_size)) {
1243 0 : thrower_->LinkError("table initializer is out of bounds");
1244 : return {};
1245 : }
1246 : }
1247 :
1248 : //--------------------------------------------------------------------------
1249 : // Check that memory segments are within bounds.
1250 : //--------------------------------------------------------------------------
1251 94890 : for (WasmDataSegment& seg : module_->data_segments) {
1252 302 : uint32_t base = EvalUint32InitExpr(seg.dest_addr);
1253 : uint32_t mem_size = memory_.is_null()
1254 588 : ? 0 : static_cast<uint32_t>(memory_->byte_length()->Number());
1255 604 : if (!in_bounds(base, seg.source_size, mem_size)) {
1256 16 : thrower_->LinkError("data segment is out of bounds");
1257 : return {};
1258 : }
1259 : }
1260 :
1261 : //--------------------------------------------------------------------------
1262 : // Initialize memory.
1263 : //--------------------------------------------------------------------------
1264 47286 : if (!memory_.is_null()) {
1265 : Address mem_start = static_cast<Address>(memory_->backing_store());
1266 : uint32_t mem_size =
1267 9694 : static_cast<uint32_t>(memory_->byte_length()->Number());
1268 9694 : LoadDataSegments(mem_start, mem_size);
1269 :
1270 9694 : uint32_t old_mem_size = compiled_module_->mem_size();
1271 9694 : Address old_mem_start = compiled_module_->GetEmbeddedMemStartOrNull();
1272 : // We might get instantiated again with the same memory. No patching
1273 : // needed in this case.
1274 9694 : if (old_mem_start != mem_start || old_mem_size != mem_size) {
1275 : code_specialization.RelocateMemoryReferences(
1276 9436 : old_mem_start, old_mem_size, mem_start, mem_size);
1277 : }
1278 : // Just like with globals, we need to keep both the JSArrayBuffer
1279 : // and save the start pointer.
1280 9694 : instance->set_memory_buffer(*memory_);
1281 : WasmCompiledModule::SetSpecializationMemInfoFrom(
1282 9694 : factory, compiled_module_, memory_);
1283 : }
1284 :
1285 : //--------------------------------------------------------------------------
1286 : // Set up the runtime support for the new instance.
1287 : //--------------------------------------------------------------------------
1288 47286 : Handle<WeakCell> weak_link = factory->NewWeakCell(instance);
1289 :
1290 198898 : for (int i = num_imported_functions + FLAG_skip_compiling_wasm_funcs,
1291 102618 : num_functions = static_cast<int>(module_->functions.size());
1292 : i < num_functions; ++i) {
1293 104326 : Handle<Code> code = handle(Code::cast(code_table->get(i)), isolate_);
1294 104326 : if (code->kind() == Code::WASM_FUNCTION) {
1295 78947 : Handle<FixedArray> deopt_data = factory->NewFixedArray(2, TENURED);
1296 78947 : deopt_data->set(0, *weak_link);
1297 : deopt_data->set(1, Smi::FromInt(i));
1298 78947 : code->set_deoptimization_data(*deopt_data);
1299 : continue;
1300 : }
1301 : DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index());
1302 25379 : if (code->deoptimization_data()->length() == 0) continue;
1303 : DCHECK_LE(2, code->deoptimization_data()->length());
1304 : DCHECK_EQ(i, Smi::cast(code->deoptimization_data()->get(1))->value());
1305 11871 : code->deoptimization_data()->set(0, *weak_link);
1306 : }
1307 :
1308 : //--------------------------------------------------------------------------
1309 : // Set up the exports object for the new instance.
1310 : //--------------------------------------------------------------------------
1311 47286 : ProcessExports(code_table, instance, compiled_module_);
1312 :
1313 : //--------------------------------------------------------------------------
1314 : // Add instance to Memory object
1315 : //--------------------------------------------------------------------------
1316 : DCHECK(wasm::IsWasmInstance(*instance));
1317 47286 : if (instance->has_memory_object()) {
1318 3484 : instance->memory_object()->AddInstance(isolate_, instance);
1319 : }
1320 :
1321 : //--------------------------------------------------------------------------
1322 : // Initialize the indirect function tables.
1323 : //--------------------------------------------------------------------------
1324 47286 : if (function_table_count > 0) LoadTableSegments(code_table, instance);
1325 :
1326 : // Patch all code with the relocations registered in code_specialization.
1327 47286 : code_specialization.RelocateDirectCalls(instance);
1328 47286 : code_specialization.ApplyToWholeInstance(*instance, SKIP_ICACHE_FLUSH);
1329 :
1330 47286 : FlushICache(isolate_, code_table);
1331 :
1332 : //--------------------------------------------------------------------------
1333 : // Unpack and notify signal handler of protected instructions.
1334 : //--------------------------------------------------------------------------
1335 47286 : if (trap_handler::UseTrapHandler()) {
1336 3168 : UnpackAndRegisterProtectedInstructions(isolate_, code_table);
1337 : }
1338 :
1339 : //--------------------------------------------------------------------------
1340 : // Set up and link the new instance.
1341 : //--------------------------------------------------------------------------
1342 : {
1343 : Handle<Object> global_handle =
1344 94572 : isolate_->global_handles()->Create(*instance);
1345 47286 : Handle<WeakCell> link_to_clone = factory->NewWeakCell(compiled_module_);
1346 47286 : Handle<WeakCell> link_to_owning_instance = factory->NewWeakCell(instance);
1347 : MaybeHandle<WeakCell> link_to_original;
1348 : MaybeHandle<WasmCompiledModule> original;
1349 47286 : if (!owner.is_null()) {
1350 : // prepare the data needed for publishing in a chain, but don't link
1351 : // just yet, because
1352 : // we want all the publishing to happen free from GC interruptions, and
1353 : // so we do it in
1354 : // one GC-free scope afterwards.
1355 32147 : original = handle(owner.ToHandleChecked()->compiled_module());
1356 32147 : link_to_original = factory->NewWeakCell(original.ToHandleChecked());
1357 : }
1358 : // Publish the new instance to the instances chain.
1359 : {
1360 : DisallowHeapAllocation no_gc;
1361 47286 : if (!link_to_original.is_null()) {
1362 : compiled_module_->set_weak_next_instance(
1363 : link_to_original.ToHandleChecked());
1364 : original.ToHandleChecked()->set_weak_prev_instance(link_to_clone);
1365 : compiled_module_->set_weak_wasm_module(
1366 32147 : original.ToHandleChecked()->weak_wasm_module());
1367 : }
1368 47286 : module_object_->SetEmbedderField(0, *compiled_module_);
1369 : compiled_module_->set_weak_owning_instance(link_to_owning_instance);
1370 : GlobalHandles::MakeWeak(global_handle.location(),
1371 : global_handle.location(), &InstanceFinalizer,
1372 47286 : v8::WeakCallbackType::kFinalizer);
1373 : }
1374 : }
1375 :
1376 : //--------------------------------------------------------------------------
1377 : // Debugging support.
1378 : //--------------------------------------------------------------------------
1379 : // Set all breakpoints that were set on the shared module.
1380 : WasmSharedModuleData::SetBreakpointsOnNewInstance(
1381 47286 : compiled_module_->shared(), instance);
1382 :
1383 47286 : if (FLAG_wasm_interpret_all) {
1384 : Handle<WasmDebugInfo> debug_info =
1385 1050 : WasmInstanceObject::GetOrCreateDebugInfo(instance);
1386 : std::vector<int> func_indexes;
1387 5640 : for (int func_index = num_imported_functions,
1388 2100 : num_wasm_functions = static_cast<int>(module_->functions.size());
1389 2295 : func_index < num_wasm_functions; ++func_index) {
1390 1245 : func_indexes.push_back(func_index);
1391 : }
1392 : WasmDebugInfo::RedirectToInterpreter(
1393 : debug_info, Vector<int>(func_indexes.data(),
1394 3150 : static_cast<int>(func_indexes.size())));
1395 : }
1396 :
1397 : //--------------------------------------------------------------------------
1398 : // Run the start function if one was specified.
1399 : //--------------------------------------------------------------------------
1400 47286 : if (module_->start_function_index >= 0) {
1401 6996 : HandleScope scope(isolate_);
1402 : int start_index = module_->start_function_index;
1403 : Handle<Code> startup_code = EnsureExportedLazyDeoptData(
1404 6996 : isolate_, instance, code_table, start_index);
1405 20988 : FunctionSig* sig = module_->functions[start_index].sig;
1406 : Handle<Code> wrapper_code =
1407 : js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper(
1408 6996 : isolate_, module_, startup_code, start_index);
1409 : Handle<WasmExportedFunction> startup_fct = WasmExportedFunction::New(
1410 : isolate_, instance, MaybeHandle<String>(), start_index,
1411 13992 : static_cast<int>(sig->parameter_count()), wrapper_code);
1412 6996 : RecordStats(isolate_, *startup_code);
1413 : // Call the JS function.
1414 : Handle<Object> undefined = factory->undefined_value();
1415 : MaybeHandle<Object> retval =
1416 6996 : Execution::Call(isolate_, startup_fct, undefined, 0, nullptr);
1417 :
1418 6996 : if (retval.is_null()) {
1419 : DCHECK(isolate_->has_pending_exception());
1420 32 : isolate_->OptionalRescheduleException(false);
1421 : // It's unfortunate that the new instance is already linked in the
1422 : // chain. However, we need to set up everything before executing the
1423 : // start function, such that stack trace information can be generated
1424 : // correctly already in the start function.
1425 : return {};
1426 : }
1427 : }
1428 :
1429 : DCHECK(!isolate_->has_pending_exception());
1430 : TRACE("Finishing instance %d\n", compiled_module_->instance_id());
1431 47254 : TRACE_CHAIN(module_object_->compiled_module());
1432 47977 : return instance;
1433 : }
1434 :
1435 : private:
1436 : // Represents the initialized state of a table.
1437 : struct TableInstance {
1438 : Handle<WasmTableObject> table_object; // WebAssembly.Table instance
1439 : Handle<FixedArray> js_wrappers; // JSFunctions exported
1440 : Handle<FixedArray> function_table; // internal code array
1441 : Handle<FixedArray> signature_table; // internal sig array
1442 : };
1443 :
1444 : Isolate* isolate_;
1445 : WasmModule* const module_;
1446 : ErrorThrower* thrower_;
1447 : Handle<WasmModuleObject> module_object_;
1448 : Handle<JSReceiver> ffi_; // TODO(titzer): Use MaybeHandle
1449 : Handle<JSArrayBuffer> memory_; // TODO(titzer): Use MaybeHandle
1450 : Handle<JSArrayBuffer> globals_;
1451 : Handle<WasmCompiledModule> compiled_module_;
1452 : std::vector<TableInstance> table_instances_;
1453 : std::vector<Handle<JSFunction>> js_wrappers_;
1454 : JSToWasmWrapperCache js_to_wasm_cache_;
1455 :
1456 : // Helper routines to print out errors with imports.
1457 : #define ERROR_THROWER_WITH_MESSAGE(TYPE) \
1458 : void Report##TYPE(const char* error, uint32_t index, \
1459 : Handle<String> module_name, Handle<String> import_name) { \
1460 : thrower_->TYPE("Import #%d module=\"%.*s\" function=\"%.*s\" error: %s", \
1461 : index, module_name->length(), \
1462 : module_name->ToCString().get(), import_name->length(), \
1463 : import_name->ToCString().get(), error); \
1464 : } \
1465 : \
1466 : MaybeHandle<Object> Report##TYPE(const char* error, uint32_t index, \
1467 : Handle<String> module_name) { \
1468 : thrower_->TYPE("Import #%d module=\"%.*s\" error: %s", index, \
1469 : module_name->length(), module_name->ToCString().get(), \
1470 : error); \
1471 : return MaybeHandle<Object>(); \
1472 : }
1473 :
1474 1440 : ERROR_THROWER_WITH_MESSAGE(LinkError)
1475 1275 : ERROR_THROWER_WITH_MESSAGE(TypeError)
1476 :
1477 : // Look up an import value in the {ffi_} object.
1478 40207 : MaybeHandle<Object> LookupImport(uint32_t index, Handle<String> module_name,
1479 : Handle<String> import_name) {
1480 : // We pre-validated in the js-api layer that the ffi object is present, and
1481 : // a JSObject, if the module has imports.
1482 : DCHECK(!ffi_.is_null());
1483 :
1484 : // Look up the module first.
1485 : MaybeHandle<Object> result =
1486 40207 : Object::GetPropertyOrElement(ffi_, module_name);
1487 40207 : if (result.is_null()) {
1488 0 : return ReportTypeError("module not found", index, module_name);
1489 : }
1490 :
1491 : Handle<Object> module = result.ToHandleChecked();
1492 :
1493 : // Look up the value in the module.
1494 40207 : if (!module->IsJSReceiver()) {
1495 : return ReportTypeError("module is not an object or function", index,
1496 255 : module_name);
1497 : }
1498 :
1499 39952 : result = Object::GetPropertyOrElement(module, import_name);
1500 39952 : if (result.is_null()) {
1501 0 : ReportLinkError("import not found", index, module_name, import_name);
1502 : return MaybeHandle<JSFunction>();
1503 : }
1504 :
1505 39952 : return result;
1506 : }
1507 :
1508 2816 : uint32_t EvalUint32InitExpr(const WasmInitExpr& expr) {
1509 2816 : switch (expr.kind) {
1510 : case WasmInitExpr::kI32Const:
1511 1256 : return expr.val.i32_const;
1512 : case WasmInitExpr::kGlobalIndex: {
1513 3120 : uint32_t offset = module_->globals[expr.val.global_index].offset;
1514 3120 : return *reinterpret_cast<uint32_t*>(raw_buffer_ptr(globals_, offset));
1515 : }
1516 : default:
1517 0 : UNREACHABLE();
1518 : return 0;
1519 : }
1520 : }
1521 :
1522 : bool in_bounds(uint32_t offset, uint32_t size, uint32_t upper) {
1523 1420 : return offset + size <= upper && offset + size >= offset;
1524 : }
1525 :
1526 : // Load data segments into the memory.
1527 9694 : void LoadDataSegments(Address mem_addr, size_t mem_size) {
1528 : Handle<SeqOneByteString> module_bytes(compiled_module_->module_bytes(),
1529 19388 : isolate_);
1530 29360 : for (const WasmDataSegment& segment : module_->data_segments) {
1531 278 : uint32_t source_size = segment.source_size;
1532 : // Segments of size == 0 are just nops.
1533 278 : if (source_size == 0) continue;
1534 278 : uint32_t dest_offset = EvalUint32InitExpr(segment.dest_addr);
1535 : DCHECK(in_bounds(dest_offset, source_size,
1536 : static_cast<uint32_t>(mem_size)));
1537 278 : byte* dest = mem_addr + dest_offset;
1538 : const byte* src = reinterpret_cast<const byte*>(
1539 278 : module_bytes->GetCharsAddress() + segment.source_offset);
1540 278 : memcpy(dest, src, source_size);
1541 : }
1542 9694 : }
1543 :
1544 2372 : void WriteGlobalValue(WasmGlobal& global, Handle<Object> value) {
1545 : double num = value->Number();
1546 : TRACE("init [globals+%u] = %lf, type = %s\n", global.offset, num,
1547 : WasmOpcodes::TypeName(global.type));
1548 1186 : switch (global.type) {
1549 : case kWasmI32:
1550 1003 : *GetRawGlobalPtr<int32_t>(global) = static_cast<int32_t>(num);
1551 1003 : break;
1552 : case kWasmI64:
1553 : // TODO(titzer): initialization of imported i64 globals.
1554 0 : UNREACHABLE();
1555 : break;
1556 : case kWasmF32:
1557 45 : *GetRawGlobalPtr<float>(global) = static_cast<float>(num);
1558 45 : break;
1559 : case kWasmF64:
1560 138 : *GetRawGlobalPtr<double>(global) = static_cast<double>(num);
1561 138 : break;
1562 : default:
1563 0 : UNREACHABLE();
1564 : }
1565 1186 : }
1566 :
1567 : // Process the imports, including functions, tables, globals, and memory, in
1568 : // order, loading them from the {ffi_} object. Returns the number of imported
1569 : // functions.
1570 47977 : int ProcessImports(Handle<FixedArray> code_table,
1571 : Handle<WasmInstanceObject> instance) {
1572 : int num_imported_functions = 0;
1573 : int num_imported_tables = 0;
1574 262527 : for (int index = 0; index < static_cast<int>(module_->import_table.size());
1575 : ++index) {
1576 165778 : WasmImport& import = module_->import_table[index];
1577 :
1578 : Handle<String> module_name;
1579 : MaybeHandle<String> maybe_module_name =
1580 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1581 : isolate_, compiled_module_, import.module_name_offset,
1582 40207 : import.module_name_length);
1583 40207 : if (!maybe_module_name.ToHandle(&module_name)) return -1;
1584 :
1585 : Handle<String> import_name;
1586 : MaybeHandle<String> maybe_import_name =
1587 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1588 : isolate_, compiled_module_, import.field_name_offset,
1589 40207 : import.field_name_length);
1590 40207 : if (!maybe_import_name.ToHandle(&import_name)) return -1;
1591 :
1592 : MaybeHandle<Object> result =
1593 40207 : LookupImport(index, module_name, import_name);
1594 80414 : if (thrower_->error()) return -1;
1595 : Handle<Object> value = result.ToHandleChecked();
1596 :
1597 39952 : switch (import.kind) {
1598 : case kExternalFunction: {
1599 : // Function imports must be callable.
1600 36921 : if (!value->IsCallable()) {
1601 : ReportLinkError("function import requires a callable", index,
1602 90 : module_name, import_name);
1603 90 : return -1;
1604 : }
1605 :
1606 : Handle<Code> import_wrapper = CompileImportWrapper(
1607 36831 : isolate_, index, module_->functions[import.index].sig,
1608 : Handle<JSReceiver>::cast(value), module_name, import_name,
1609 110493 : module_->get_origin());
1610 36831 : if (import_wrapper.is_null()) {
1611 : ReportLinkError(
1612 : "imported function does not match the expected type", index,
1613 45 : module_name, import_name);
1614 45 : return -1;
1615 : }
1616 36786 : code_table->set(num_imported_functions, *import_wrapper);
1617 36786 : RecordStats(isolate_, *import_wrapper);
1618 36786 : num_imported_functions++;
1619 : break;
1620 : }
1621 : case kExternalTable: {
1622 765 : if (!WasmJs::IsWasmTableObject(isolate_, value)) {
1623 : ReportLinkError("table import requires a WebAssembly.Table", index,
1624 0 : module_name, import_name);
1625 0 : return -1;
1626 : }
1627 : WasmIndirectFunctionTable& table =
1628 765 : module_->function_tables[num_imported_tables];
1629 765 : TableInstance& table_instance = table_instances_[num_imported_tables];
1630 765 : table_instance.table_object = Handle<WasmTableObject>::cast(value);
1631 : table_instance.js_wrappers = Handle<FixedArray>(
1632 2295 : table_instance.table_object->functions(), isolate_);
1633 :
1634 : int imported_cur_size = table_instance.js_wrappers->length();
1635 765 : if (imported_cur_size < static_cast<int>(table.min_size)) {
1636 : thrower_->LinkError(
1637 : "table import %d is smaller than minimum %d, got %u", index,
1638 45 : table.min_size, imported_cur_size);
1639 45 : return -1;
1640 : }
1641 :
1642 720 : if (table.has_max) {
1643 : int64_t imported_max_size =
1644 705 : table_instance.table_object->maximum_length();
1645 705 : if (imported_max_size < 0) {
1646 : thrower_->LinkError(
1647 : "table import %d has no maximum length, expected %d", index,
1648 15 : table.max_size);
1649 15 : return -1;
1650 : }
1651 690 : if (imported_max_size > table.max_size) {
1652 : thrower_->LinkError(
1653 : "table import %d has maximum larger than maximum %d, "
1654 : "got %" PRIx64,
1655 75 : index, table.max_size, imported_max_size);
1656 75 : return -1;
1657 : }
1658 : }
1659 :
1660 : // Allocate a new dispatch table and signature table.
1661 : int table_size = imported_cur_size;
1662 : table_instance.function_table =
1663 630 : isolate_->factory()->NewFixedArray(table_size);
1664 : table_instance.signature_table =
1665 630 : isolate_->factory()->NewFixedArray(table_size);
1666 5970 : for (int i = 0; i < table_size; ++i) {
1667 : table_instance.signature_table->set(i,
1668 : Smi::FromInt(kInvalidSigIndex));
1669 : }
1670 : // Initialize the dispatch table with the (foreign) JS functions
1671 : // that are already in the table.
1672 5340 : for (int i = 0; i < table_size; ++i) {
1673 5340 : Handle<Object> val(table_instance.js_wrappers->get(i), isolate_);
1674 5340 : if (!val->IsJSFunction()) continue;
1675 : WasmFunction* function =
1676 1845 : GetWasmFunctionForImportWrapper(isolate_, val);
1677 1845 : if (function == nullptr) {
1678 : thrower_->LinkError("table import %d[%d] is not a WASM function",
1679 0 : index, i);
1680 : return -1;
1681 : }
1682 1845 : int sig_index = table.map.FindOrInsert(function->sig);
1683 : table_instance.signature_table->set(i, Smi::FromInt(sig_index));
1684 3690 : table_instance.function_table->set(i, *UnwrapImportWrapper(val));
1685 : }
1686 :
1687 630 : num_imported_tables++;
1688 630 : break;
1689 : }
1690 : case kExternalMemory: {
1691 : // Validation should have failed if more than one memory object was
1692 : // provided.
1693 : DCHECK(!instance->has_memory_object());
1694 1035 : if (!WasmJs::IsWasmMemoryObject(isolate_, value)) {
1695 : ReportLinkError("memory import must be a WebAssembly.Memory object",
1696 60 : index, module_name, import_name);
1697 60 : return -1;
1698 : }
1699 : auto memory = Handle<WasmMemoryObject>::cast(value);
1700 : DCHECK(WasmJs::IsWasmMemoryObject(isolate_, memory));
1701 975 : instance->set_memory_object(*memory);
1702 2925 : memory_ = Handle<JSArrayBuffer>(memory->buffer(), isolate_);
1703 : uint32_t imported_cur_pages = static_cast<uint32_t>(
1704 975 : memory_->byte_length()->Number() / WasmModule::kPageSize);
1705 975 : if (imported_cur_pages < module_->min_mem_pages) {
1706 : thrower_->LinkError(
1707 : "memory import %d is smaller than maximum %u, got %u", index,
1708 0 : module_->min_mem_pages, imported_cur_pages);
1709 : }
1710 975 : int32_t imported_max_pages = memory->maximum_pages();
1711 975 : if (module_->has_max_mem) {
1712 60 : if (imported_max_pages < 0) {
1713 : thrower_->LinkError(
1714 : "memory import %d has no maximum limit, expected at most %u",
1715 15 : index, imported_max_pages);
1716 15 : return -1;
1717 : }
1718 45 : if (static_cast<uint32_t>(imported_max_pages) >
1719 : module_->max_mem_pages) {
1720 : thrower_->LinkError(
1721 : "memory import %d has larger maximum than maximum %u, got %d",
1722 30 : index, module_->max_mem_pages, imported_max_pages);
1723 30 : return -1;
1724 : }
1725 : }
1726 : break;
1727 : }
1728 : case kExternalGlobal: {
1729 : // Global imports are converted to numbers and written into the
1730 : // {globals_} array buffer.
1731 3648 : if (module_->globals[import.index].type == kWasmI64) {
1732 : ReportLinkError("global import cannot have type i64", index,
1733 0 : module_name, import_name);
1734 0 : return -1;
1735 : }
1736 1231 : if (module_->is_asm_js() && FLAG_fast_validate_asm) {
1737 256 : if (module_->globals[import.index].type == kWasmI32) {
1738 326 : value = Object::ToInt32(isolate_, value).ToHandleChecked();
1739 : } else {
1740 186 : value = Object::ToNumber(value).ToHandleChecked();
1741 : }
1742 : }
1743 1231 : if (!value->IsNumber()) {
1744 : ReportLinkError("global import must be a number", index,
1745 45 : module_name, import_name);
1746 45 : return -1;
1747 : }
1748 2372 : WriteGlobalValue(module_->globals[import.index], value);
1749 1186 : break;
1750 : }
1751 : default:
1752 0 : UNREACHABLE();
1753 : break;
1754 : }
1755 : }
1756 : return num_imported_functions;
1757 : }
1758 :
1759 : template <typename T>
1760 : T* GetRawGlobalPtr(WasmGlobal& global) {
1761 21238 : return reinterpret_cast<T*>(raw_buffer_ptr(globals_, global.offset));
1762 : }
1763 :
1764 : // Process initialization of globals.
1765 47302 : void InitGlobals() {
1766 152519 : for (auto global : module_->globals) {
1767 10613 : switch (global.init.kind) {
1768 : case WasmInitExpr::kI32Const:
1769 5560 : *GetRawGlobalPtr<int32_t>(global) = global.init.val.i32_const;
1770 5560 : break;
1771 : case WasmInitExpr::kI64Const:
1772 32 : *GetRawGlobalPtr<int64_t>(global) = global.init.val.i64_const;
1773 32 : break;
1774 : case WasmInitExpr::kF32Const:
1775 2748 : *GetRawGlobalPtr<float>(global) = global.init.val.f32_const;
1776 2748 : break;
1777 : case WasmInitExpr::kF64Const:
1778 997 : *GetRawGlobalPtr<double>(global) = global.init.val.f64_const;
1779 997 : break;
1780 : case WasmInitExpr::kGlobalIndex: {
1781 : // Initialize with another global.
1782 : uint32_t new_offset = global.offset;
1783 : uint32_t old_offset =
1784 180 : module_->globals[global.init.val.global_index].offset;
1785 : TRACE("init [globals+%u] = [globals+%d]\n", global.offset,
1786 : old_offset);
1787 90 : size_t size = (global.type == kWasmI64 || global.type == kWasmF64)
1788 : ? sizeof(double)
1789 90 : : sizeof(int32_t);
1790 270 : memcpy(raw_buffer_ptr(globals_, new_offset),
1791 270 : raw_buffer_ptr(globals_, old_offset), size);
1792 90 : break;
1793 : }
1794 : case WasmInitExpr::kNone:
1795 : // Happens with imported globals.
1796 : break;
1797 : default:
1798 0 : UNREACHABLE();
1799 : break;
1800 : }
1801 : }
1802 47302 : }
1803 :
1804 : // Allocate memory for a module instance as a new JSArrayBuffer.
1805 6507 : Handle<JSArrayBuffer> AllocateMemory(uint32_t min_mem_pages) {
1806 6507 : if (min_mem_pages > FLAG_wasm_max_mem_pages) {
1807 0 : thrower_->RangeError("Out of memory: wasm memory too large");
1808 : return Handle<JSArrayBuffer>::null();
1809 : }
1810 : const bool enable_guard_regions = EnableGuardRegions();
1811 : Handle<JSArrayBuffer> mem_buffer = NewArrayBuffer(
1812 6507 : isolate_, min_mem_pages * WasmModule::kPageSize, enable_guard_regions);
1813 :
1814 6507 : if (mem_buffer.is_null()) {
1815 0 : thrower_->RangeError("Out of memory: wasm memory");
1816 : }
1817 6507 : return mem_buffer;
1818 : }
1819 :
1820 47286 : bool NeedsWrappers() {
1821 47286 : if (module_->num_exported_functions > 0) return true;
1822 2107 : for (auto table_instance : table_instances_) {
1823 333 : if (!table_instance.js_wrappers.is_null()) return true;
1824 : }
1825 1654 : for (auto table : module_->function_tables) {
1826 213 : if (table.exported) return true;
1827 0 : }
1828 614 : return false;
1829 : }
1830 :
1831 : // Process the exports, creating wrappers for functions, tables, memories,
1832 : // and globals.
1833 47286 : void ProcessExports(Handle<FixedArray> code_table,
1834 : Handle<WasmInstanceObject> instance,
1835 : Handle<WasmCompiledModule> compiled_module) {
1836 47286 : if (NeedsWrappers()) {
1837 : // Fill the table to cache the exported JSFunction wrappers.
1838 : js_wrappers_.insert(js_wrappers_.begin(), module_->functions.size(),
1839 761900 : Handle<JSFunction>::null());
1840 : }
1841 :
1842 : Handle<JSObject> exports_object;
1843 94572 : if (module_->is_wasm()) {
1844 : // Create the "exports" object.
1845 40799 : exports_object = isolate_->factory()->NewJSObjectWithNullProto();
1846 6487 : } else if (module_->is_asm_js()) {
1847 : Handle<JSFunction> object_function = Handle<JSFunction>(
1848 12974 : isolate_->native_context()->object_function(), isolate_);
1849 6487 : exports_object = isolate_->factory()->NewJSObject(object_function);
1850 : } else {
1851 0 : UNREACHABLE();
1852 : }
1853 : Handle<String> exports_name =
1854 47286 : isolate_->factory()->InternalizeUtf8String("exports");
1855 47286 : JSObject::AddProperty(instance, exports_name, exports_object, NONE);
1856 :
1857 : Handle<String> foreign_init_name =
1858 : isolate_->factory()->InternalizeUtf8String(
1859 47286 : wasm::AsmWasmBuilder::foreign_init_name);
1860 : Handle<String> single_function_name =
1861 : isolate_->factory()->InternalizeUtf8String(
1862 47286 : wasm::AsmWasmBuilder::single_function_name);
1863 :
1864 : PropertyDescriptor desc;
1865 47286 : desc.set_writable(module_->is_asm_js());
1866 : desc.set_enumerable(true);
1867 : desc.set_configurable(module_->is_asm_js());
1868 :
1869 : // Store weak references to all exported functions.
1870 : Handle<FixedArray> weak_exported_functions;
1871 47286 : if (compiled_module->has_weak_exported_functions()) {
1872 1222 : weak_exported_functions = compiled_module->weak_exported_functions();
1873 : } else {
1874 : int export_count = 0;
1875 216983 : for (WasmExport& exp : module_->export_table) {
1876 78791 : if (exp.kind == kExternalFunction) ++export_count;
1877 : }
1878 : weak_exported_functions =
1879 46064 : isolate_->factory()->NewFixedArray(export_count);
1880 : compiled_module->set_weak_exported_functions(weak_exported_functions);
1881 : }
1882 :
1883 : // Process each export in the export table.
1884 : int export_index = 0; // Index into {weak_exported_functions}.
1885 223163 : for (WasmExport& exp : module_->export_table) {
1886 : Handle<String> name =
1887 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1888 : isolate_, compiled_module_, exp.name_offset, exp.name_length)
1889 162610 : .ToHandleChecked();
1890 : Handle<JSObject> export_to;
1891 261440 : if (module_->is_asm_js() && exp.kind == kExternalFunction &&
1892 29254 : (String::Equals(name, foreign_init_name) ||
1893 11729 : String::Equals(name, single_function_name))) {
1894 : export_to = instance;
1895 : } else {
1896 : export_to = exports_object;
1897 : }
1898 :
1899 81305 : switch (exp.kind) {
1900 : case kExternalFunction: {
1901 : // Wrap and export the code as a JSFunction.
1902 79831 : WasmFunction& function = module_->functions[exp.index];
1903 : int func_index =
1904 79831 : static_cast<int>(module_->functions.size() + export_index);
1905 79831 : Handle<JSFunction> js_function = js_wrappers_[exp.index];
1906 79831 : if (js_function.is_null()) {
1907 : // Wrap the exported code as a JSFunction.
1908 : Handle<Code> export_code =
1909 159228 : code_table->GetValueChecked<Code>(isolate_, func_index);
1910 : MaybeHandle<String> func_name;
1911 159228 : if (module_->is_asm_js()) {
1912 : // For modules arising from asm.js, honor the names section.
1913 : func_name = WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1914 : isolate_, compiled_module_, function.name_offset,
1915 : function.name_length)
1916 34964 : .ToHandleChecked();
1917 : }
1918 : js_function = WasmExportedFunction::New(
1919 : isolate_, instance, func_name, function.func_index,
1920 79614 : static_cast<int>(function.sig->parameter_count()), export_code);
1921 159228 : js_wrappers_[exp.index] = js_function;
1922 : }
1923 : desc.set_value(js_function);
1924 : Handle<WeakCell> weak_export =
1925 79831 : isolate_->factory()->NewWeakCell(js_function);
1926 : DCHECK_GT(weak_exported_functions->length(), export_index);
1927 79831 : weak_exported_functions->set(export_index, *weak_export);
1928 79831 : export_index++;
1929 : break;
1930 : }
1931 : case kExternalTable: {
1932 : // Export a table as a WebAssembly.Table object.
1933 337 : TableInstance& table_instance = table_instances_[exp.index];
1934 : WasmIndirectFunctionTable& table =
1935 337 : module_->function_tables[exp.index];
1936 337 : if (table_instance.table_object.is_null()) {
1937 : uint32_t maximum =
1938 337 : table.has_max ? table.max_size : FLAG_wasm_max_table_size;
1939 : table_instance.table_object = WasmTableObject::New(
1940 337 : isolate_, table.min_size, maximum, &table_instance.js_wrappers);
1941 : }
1942 : desc.set_value(table_instance.table_object);
1943 337 : break;
1944 : }
1945 : case kExternalMemory: {
1946 : // Export the memory as a WebAssembly.Memory object.
1947 : Handle<WasmMemoryObject> memory_object;
1948 1025 : if (!instance->has_memory_object()) {
1949 : // If there was no imported WebAssembly.Memory object, create one.
1950 : memory_object = WasmMemoryObject::New(
1951 : isolate_,
1952 812 : (instance->has_memory_buffer())
1953 797 : ? handle(instance->memory_buffer())
1954 : : Handle<JSArrayBuffer>::null(),
1955 2436 : (module_->max_mem_pages != 0) ? module_->max_mem_pages : -1);
1956 812 : instance->set_memory_object(*memory_object);
1957 : } else {
1958 : memory_object =
1959 426 : Handle<WasmMemoryObject>(instance->memory_object(), isolate_);
1960 : DCHECK(WasmJs::IsWasmMemoryObject(isolate_, memory_object));
1961 426 : memory_object->ResetInstancesLink(isolate_);
1962 : }
1963 :
1964 : desc.set_value(memory_object);
1965 : break;
1966 : }
1967 : case kExternalGlobal: {
1968 : // Export the value of the global variable as a number.
1969 208 : WasmGlobal& global = module_->globals[exp.index];
1970 : double num = 0;
1971 112 : switch (global.type) {
1972 : case kWasmI32:
1973 34 : num = *GetRawGlobalPtr<int32_t>(global);
1974 34 : break;
1975 : case kWasmF32:
1976 31 : num = *GetRawGlobalPtr<float>(global);
1977 31 : break;
1978 : case kWasmF64:
1979 31 : num = *GetRawGlobalPtr<double>(global);
1980 31 : break;
1981 : case kWasmI64:
1982 : thrower_->LinkError(
1983 16 : "export of globals of type I64 is not allowed.");
1984 16 : break;
1985 : default:
1986 0 : UNREACHABLE();
1987 : }
1988 112 : desc.set_value(isolate_->factory()->NewNumber(num));
1989 : break;
1990 : }
1991 : default:
1992 0 : UNREACHABLE();
1993 : break;
1994 : }
1995 :
1996 : v8::Maybe<bool> status = JSReceiver::DefineOwnProperty(
1997 81305 : isolate_, export_to, name, &desc, Object::THROW_ON_ERROR);
1998 81305 : if (!status.IsJust()) {
1999 : thrower_->LinkError("export of %.*s failed.", name->length(),
2000 0 : name->ToCString().get());
2001 47286 : return;
2002 : }
2003 : }
2004 : DCHECK_EQ(export_index, weak_exported_functions->length());
2005 :
2006 94572 : if (module_->is_wasm()) {
2007 : v8::Maybe<bool> success = JSReceiver::SetIntegrityLevel(
2008 40799 : exports_object, FROZEN, Object::DONT_THROW);
2009 : DCHECK(success.FromMaybe(false));
2010 : USE(success);
2011 : }
2012 : }
2013 :
2014 1468 : void InitializeTables(Handle<WasmInstanceObject> instance,
2015 : CodeSpecialization* code_specialization) {
2016 : int function_table_count =
2017 4404 : static_cast<int>(module_->function_tables.size());
2018 : Handle<FixedArray> new_function_tables =
2019 1468 : isolate_->factory()->NewFixedArray(function_table_count);
2020 : Handle<FixedArray> new_signature_tables =
2021 1468 : isolate_->factory()->NewFixedArray(function_table_count);
2022 2936 : for (int index = 0; index < function_table_count; ++index) {
2023 1468 : WasmIndirectFunctionTable& table = module_->function_tables[index];
2024 1468 : TableInstance& table_instance = table_instances_[index];
2025 1468 : int table_size = static_cast<int>(table.min_size);
2026 :
2027 1468 : if (table_instance.function_table.is_null()) {
2028 : // Create a new dispatch table if necessary.
2029 : table_instance.function_table =
2030 838 : isolate_->factory()->NewFixedArray(table_size);
2031 : table_instance.signature_table =
2032 838 : isolate_->factory()->NewFixedArray(table_size);
2033 11160 : for (int i = 0; i < table_size; ++i) {
2034 : // Fill the table with invalid signature indexes so that
2035 : // uninitialized entries will always fail the signature check.
2036 : table_instance.signature_table->set(i,
2037 : Smi::FromInt(kInvalidSigIndex));
2038 : }
2039 : } else {
2040 : // Table is imported, patch table bounds check
2041 : DCHECK(table_size <= table_instance.function_table->length());
2042 630 : if (table_size < table_instance.function_table->length()) {
2043 : code_specialization->PatchTableSize(
2044 45 : table_size, table_instance.function_table->length());
2045 : }
2046 : }
2047 :
2048 : new_function_tables->set(static_cast<int>(index),
2049 1468 : *table_instance.function_table);
2050 : new_signature_tables->set(static_cast<int>(index),
2051 1468 : *table_instance.signature_table);
2052 : }
2053 :
2054 : FixedArray* old_function_tables =
2055 : compiled_module_->ptr_to_function_tables();
2056 : DCHECK_EQ(old_function_tables->length(), new_function_tables->length());
2057 2936 : for (int i = 0, e = new_function_tables->length(); i < e; ++i) {
2058 : code_specialization->RelocateObject(
2059 : handle(old_function_tables->get(i), isolate_),
2060 4404 : handle(new_function_tables->get(i), isolate_));
2061 : }
2062 : FixedArray* old_signature_tables =
2063 : compiled_module_->ptr_to_signature_tables();
2064 : DCHECK_EQ(old_signature_tables->length(), new_signature_tables->length());
2065 2936 : for (int i = 0, e = new_signature_tables->length(); i < e; ++i) {
2066 : code_specialization->RelocateObject(
2067 : handle(old_signature_tables->get(i), isolate_),
2068 4404 : handle(new_signature_tables->get(i), isolate_));
2069 : }
2070 :
2071 : compiled_module_->set_function_tables(new_function_tables);
2072 : compiled_module_->set_signature_tables(new_signature_tables);
2073 1468 : }
2074 :
2075 1468 : void LoadTableSegments(Handle<FixedArray> code_table,
2076 : Handle<WasmInstanceObject> instance) {
2077 : int function_table_count =
2078 5621 : static_cast<int>(module_->function_tables.size());
2079 2936 : for (int index = 0; index < function_table_count; ++index) {
2080 1468 : WasmIndirectFunctionTable& table = module_->function_tables[index];
2081 1468 : TableInstance& table_instance = table_instances_[index];
2082 :
2083 : Handle<FixedArray> all_dispatch_tables;
2084 1468 : if (!table_instance.table_object.is_null()) {
2085 : // Get the existing dispatch table(s) with the WebAssembly.Table object.
2086 : all_dispatch_tables = WasmTableObject::AddDispatchTable(
2087 : isolate_, table_instance.table_object,
2088 : Handle<WasmInstanceObject>::null(), index,
2089 967 : Handle<FixedArray>::null(), Handle<FixedArray>::null());
2090 : }
2091 :
2092 : // Count the number of table exports for each function (needed for lazy
2093 : // compilation).
2094 1468 : std::unordered_map<uint32_t, uint32_t> num_table_exports;
2095 2936 : if (compile_lazy(module_)) {
2096 450 : for (auto table_init : module_->table_inits) {
2097 6705 : for (uint32_t func_index : table_init.entries) {
2098 : Code* code =
2099 6405 : Code::cast(code_table->get(static_cast<int>(func_index)));
2100 : // Only increase the counter for lazy compile builtins (it's not
2101 : // needed otherwise).
2102 6405 : if (code->is_wasm_code()) continue;
2103 : DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index());
2104 5942 : ++num_table_exports[func_index];
2105 : }
2106 : }
2107 : }
2108 :
2109 : // TODO(titzer): this does redundant work if there are multiple tables,
2110 : // since initializations are not sorted by table index.
2111 5522 : for (auto table_init : module_->table_inits) {
2112 1118 : uint32_t base = EvalUint32InitExpr(table_init.offset);
2113 : DCHECK(in_bounds(base, static_cast<uint32_t>(table_init.entries.size()),
2114 : table_instance.function_table->length()));
2115 10846 : for (int i = 0, e = static_cast<int>(table_init.entries.size()); i < e;
2116 : ++i) {
2117 17220 : uint32_t func_index = table_init.entries[i];
2118 8610 : WasmFunction* function = &module_->functions[func_index];
2119 8610 : int table_index = static_cast<int>(i + base);
2120 9827 : int32_t sig_index = table.map.Find(function->sig);
2121 : DCHECK_GE(sig_index, 0);
2122 : table_instance.signature_table->set(table_index,
2123 : Smi::FromInt(sig_index));
2124 : Handle<Code> wasm_code = EnsureTableExportLazyDeoptData(
2125 : isolate_, instance, code_table, func_index,
2126 8610 : table_instance.function_table, table_index, num_table_exports);
2127 8610 : table_instance.function_table->set(table_index, *wasm_code);
2128 :
2129 8610 : if (!all_dispatch_tables.is_null()) {
2130 5093 : if (js_wrappers_[func_index].is_null()) {
2131 : // No JSFunction entry yet exists for this function. Create one.
2132 : // TODO(titzer): We compile JS->WASM wrappers for functions are
2133 : // not exported but are in an exported table. This should be done
2134 : // at module compile time and cached instead.
2135 :
2136 : Handle<Code> wrapper_code =
2137 : js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper(
2138 1217 : isolate_, module_, wasm_code, func_index);
2139 : MaybeHandle<String> func_name;
2140 2434 : if (module_->is_asm_js()) {
2141 : // For modules arising from asm.js, honor the names section.
2142 : func_name =
2143 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2144 : isolate_, compiled_module_, function->name_offset,
2145 : function->name_length)
2146 0 : .ToHandleChecked();
2147 : }
2148 : Handle<WasmExportedFunction> js_function =
2149 : WasmExportedFunction::New(
2150 : isolate_, instance, func_name, func_index,
2151 1217 : static_cast<int>(function->sig->parameter_count()),
2152 1217 : wrapper_code);
2153 1217 : js_wrappers_[func_index] = js_function;
2154 : }
2155 : table_instance.js_wrappers->set(table_index,
2156 1292 : *js_wrappers_[func_index]);
2157 :
2158 : UpdateDispatchTablesInternal(isolate_, all_dispatch_tables,
2159 1292 : table_index, function, wasm_code);
2160 : }
2161 : }
2162 : }
2163 :
2164 : #ifdef DEBUG
2165 : // Check that the count of table exports was accurate. The entries are
2166 : // decremented on each export, so all should be zero now.
2167 : for (auto e : num_table_exports) {
2168 : DCHECK_EQ(0, e.second);
2169 : }
2170 : #endif
2171 :
2172 : // TODO(titzer): we add the new dispatch table at the end to avoid
2173 : // redundant work and also because the new instance is not yet fully
2174 : // initialized.
2175 1468 : if (!table_instance.table_object.is_null()) {
2176 : // Add the new dispatch table to the WebAssembly.Table object.
2177 : all_dispatch_tables = WasmTableObject::AddDispatchTable(
2178 : isolate_, table_instance.table_object, instance, index,
2179 967 : table_instance.function_table, table_instance.signature_table);
2180 : }
2181 : }
2182 1468 : }
2183 : };
2184 :
2185 0 : bool wasm::IsWasmInstance(Object* object) {
2186 60 : return WasmInstanceObject::IsWasmInstanceObject(object);
2187 : }
2188 :
2189 1806 : Handle<Script> wasm::GetScript(Handle<JSObject> instance) {
2190 : WasmCompiledModule* compiled_module =
2191 1806 : WasmInstanceObject::cast(*instance)->compiled_module();
2192 3612 : return handle(compiled_module->script());
2193 : }
2194 :
2195 12373 : bool wasm::IsWasmCodegenAllowed(Isolate* isolate, Handle<Context> context) {
2196 12394 : return isolate->allow_code_gen_callback() == nullptr ||
2197 195 : isolate->allow_code_gen_callback()(v8::Utils::ToLocal(context));
2198 : }
2199 :
2200 756 : void wasm::DetachWebAssemblyMemoryBuffer(Isolate* isolate,
2201 : Handle<JSArrayBuffer> buffer) {
2202 : int64_t byte_length =
2203 : buffer->byte_length()->IsNumber()
2204 : ? static_cast<uint32_t>(buffer->byte_length()->Number())
2205 840 : : 0;
2206 840 : if (buffer.is_null() || byte_length == 0) return;
2207 : const bool has_guard_regions = buffer->has_guard_region();
2208 : const bool is_external = buffer->is_external();
2209 : void* backing_store = buffer->backing_store();
2210 : DCHECK(!buffer->is_neuterable());
2211 360 : if (!has_guard_regions && !is_external) {
2212 : buffer->set_is_external(true);
2213 336 : isolate->heap()->UnregisterArrayBuffer(*buffer);
2214 : }
2215 : buffer->set_is_neuterable(true);
2216 360 : buffer->Neuter();
2217 360 : if (has_guard_regions) {
2218 : base::OS::Free(backing_store, RoundUp(i::wasm::kWasmMaxHeapOffset,
2219 48 : base::OS::CommitPageSize()));
2220 : reinterpret_cast<v8::Isolate*>(isolate)
2221 24 : ->AdjustAmountOfExternalAllocatedMemory(-byte_length);
2222 336 : } else if (!has_guard_regions && !is_external) {
2223 336 : isolate->array_buffer_allocator()->Free(backing_store, byte_length);
2224 : }
2225 : }
2226 :
2227 255 : void wasm::GrowDispatchTables(Isolate* isolate,
2228 : Handle<FixedArray> dispatch_tables,
2229 : uint32_t old_size, uint32_t count) {
2230 : DCHECK_EQ(0, dispatch_tables->length() % 4);
2231 :
2232 255 : Zone specialization_zone(isolate->allocator(), ZONE_NAME);
2233 1020 : for (int i = 0; i < dispatch_tables->length(); i += 4) {
2234 : Handle<FixedArray> old_function_table(
2235 255 : FixedArray::cast(dispatch_tables->get(i + 2)));
2236 : Handle<FixedArray> old_signature_table(
2237 255 : FixedArray::cast(dispatch_tables->get(i + 3)));
2238 : Handle<FixedArray> new_function_table =
2239 255 : isolate->factory()->CopyFixedArrayAndGrow(old_function_table, count);
2240 : Handle<FixedArray> new_signature_table =
2241 255 : isolate->factory()->CopyFixedArrayAndGrow(old_signature_table, count);
2242 :
2243 : // Update dispatch tables with new function/signature tables
2244 255 : dispatch_tables->set(i + 2, *new_function_table);
2245 255 : dispatch_tables->set(i + 3, *new_signature_table);
2246 :
2247 : // Patch the code of the respective instance.
2248 255 : CodeSpecialization code_specialization(isolate, &specialization_zone);
2249 255 : code_specialization.PatchTableSize(old_size, old_size + count);
2250 255 : code_specialization.RelocateObject(old_function_table, new_function_table);
2251 : code_specialization.RelocateObject(old_signature_table,
2252 255 : new_signature_table);
2253 : code_specialization.ApplyToWholeInstance(
2254 255 : WasmInstanceObject::cast(dispatch_tables->get(i)));
2255 510 : }
2256 255 : }
2257 :
2258 90 : void testing::ValidateInstancesChain(Isolate* isolate,
2259 : Handle<WasmModuleObject> module_obj,
2260 : int instance_count) {
2261 90 : CHECK_GE(instance_count, 0);
2262 : DisallowHeapAllocation no_gc;
2263 90 : WasmCompiledModule* compiled_module = module_obj->compiled_module();
2264 90 : CHECK_EQ(JSObject::cast(compiled_module->ptr_to_weak_wasm_module()->value()),
2265 : *module_obj);
2266 : Object* prev = nullptr;
2267 90 : int found_instances = compiled_module->has_weak_owning_instance() ? 1 : 0;
2268 : WasmCompiledModule* current_instance = compiled_module;
2269 240 : while (current_instance->has_weak_next_instance()) {
2270 75 : CHECK((prev == nullptr && !current_instance->has_weak_prev_instance()) ||
2271 : current_instance->ptr_to_weak_prev_instance()->value() == prev);
2272 60 : CHECK_EQ(current_instance->ptr_to_weak_wasm_module()->value(), *module_obj);
2273 60 : CHECK(IsWasmInstance(
2274 : current_instance->ptr_to_weak_owning_instance()->value()));
2275 : prev = current_instance;
2276 : current_instance = WasmCompiledModule::cast(
2277 : current_instance->ptr_to_weak_next_instance()->value());
2278 60 : ++found_instances;
2279 60 : CHECK_LE(found_instances, instance_count);
2280 : }
2281 90 : CHECK_EQ(found_instances, instance_count);
2282 90 : }
2283 :
2284 30 : void testing::ValidateModuleState(Isolate* isolate,
2285 : Handle<WasmModuleObject> module_obj) {
2286 : DisallowHeapAllocation no_gc;
2287 30 : WasmCompiledModule* compiled_module = module_obj->compiled_module();
2288 30 : CHECK(compiled_module->has_weak_wasm_module());
2289 30 : CHECK_EQ(compiled_module->ptr_to_weak_wasm_module()->value(), *module_obj);
2290 30 : CHECK(!compiled_module->has_weak_prev_instance());
2291 30 : CHECK(!compiled_module->has_weak_next_instance());
2292 30 : CHECK(!compiled_module->has_weak_owning_instance());
2293 30 : }
2294 :
2295 15 : void testing::ValidateOrphanedInstance(Isolate* isolate,
2296 : Handle<WasmInstanceObject> instance) {
2297 : DisallowHeapAllocation no_gc;
2298 15 : WasmCompiledModule* compiled_module = instance->compiled_module();
2299 15 : CHECK(compiled_module->has_weak_wasm_module());
2300 15 : CHECK(compiled_module->ptr_to_weak_wasm_module()->cleared());
2301 15 : }
2302 :
2303 60 : Handle<JSArray> wasm::GetImports(Isolate* isolate,
2304 : Handle<WasmModuleObject> module_object) {
2305 : Handle<WasmCompiledModule> compiled_module(module_object->compiled_module(),
2306 60 : isolate);
2307 : Factory* factory = isolate->factory();
2308 :
2309 60 : Handle<String> module_string = factory->InternalizeUtf8String("module");
2310 60 : Handle<String> name_string = factory->InternalizeUtf8String("name");
2311 60 : Handle<String> kind_string = factory->InternalizeUtf8String("kind");
2312 :
2313 60 : Handle<String> function_string = factory->InternalizeUtf8String("function");
2314 60 : Handle<String> table_string = factory->InternalizeUtf8String("table");
2315 60 : Handle<String> memory_string = factory->InternalizeUtf8String("memory");
2316 60 : Handle<String> global_string = factory->InternalizeUtf8String("global");
2317 :
2318 : // Create the result array.
2319 60 : WasmModule* module = compiled_module->module();
2320 240 : int num_imports = static_cast<int>(module->import_table.size());
2321 60 : Handle<JSArray> array_object = factory->NewJSArray(FAST_ELEMENTS, 0, 0);
2322 60 : Handle<FixedArray> storage = factory->NewFixedArray(num_imports);
2323 60 : JSArray::SetContent(array_object, storage);
2324 : array_object->set_length(Smi::FromInt(num_imports));
2325 :
2326 : Handle<JSFunction> object_function =
2327 120 : Handle<JSFunction>(isolate->native_context()->object_function(), isolate);
2328 :
2329 : // Populate the result array.
2330 180 : for (int index = 0; index < num_imports; ++index) {
2331 120 : WasmImport& import = module->import_table[index];
2332 :
2333 120 : Handle<JSObject> entry = factory->NewJSObject(object_function);
2334 :
2335 : Handle<String> import_kind;
2336 120 : switch (import.kind) {
2337 : case kExternalFunction:
2338 30 : import_kind = function_string;
2339 30 : break;
2340 : case kExternalTable:
2341 30 : import_kind = table_string;
2342 30 : break;
2343 : case kExternalMemory:
2344 30 : import_kind = memory_string;
2345 30 : break;
2346 : case kExternalGlobal:
2347 30 : import_kind = global_string;
2348 30 : break;
2349 : default:
2350 0 : UNREACHABLE();
2351 : }
2352 :
2353 : MaybeHandle<String> import_module =
2354 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2355 : isolate, compiled_module, import.module_name_offset,
2356 120 : import.module_name_length);
2357 :
2358 : MaybeHandle<String> import_name =
2359 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2360 : isolate, compiled_module, import.field_name_offset,
2361 120 : import.field_name_length);
2362 :
2363 : JSObject::AddProperty(entry, module_string, import_module.ToHandleChecked(),
2364 120 : NONE);
2365 : JSObject::AddProperty(entry, name_string, import_name.ToHandleChecked(),
2366 120 : NONE);
2367 120 : JSObject::AddProperty(entry, kind_string, import_kind, NONE);
2368 :
2369 120 : storage->set(index, *entry);
2370 : }
2371 :
2372 60 : return array_object;
2373 : }
2374 :
2375 60 : Handle<JSArray> wasm::GetExports(Isolate* isolate,
2376 : Handle<WasmModuleObject> module_object) {
2377 : Handle<WasmCompiledModule> compiled_module(module_object->compiled_module(),
2378 60 : isolate);
2379 : Factory* factory = isolate->factory();
2380 :
2381 60 : Handle<String> name_string = factory->InternalizeUtf8String("name");
2382 60 : Handle<String> kind_string = factory->InternalizeUtf8String("kind");
2383 :
2384 60 : Handle<String> function_string = factory->InternalizeUtf8String("function");
2385 60 : Handle<String> table_string = factory->InternalizeUtf8String("table");
2386 60 : Handle<String> memory_string = factory->InternalizeUtf8String("memory");
2387 60 : Handle<String> global_string = factory->InternalizeUtf8String("global");
2388 :
2389 : // Create the result array.
2390 60 : WasmModule* module = compiled_module->module();
2391 240 : int num_exports = static_cast<int>(module->export_table.size());
2392 60 : Handle<JSArray> array_object = factory->NewJSArray(FAST_ELEMENTS, 0, 0);
2393 60 : Handle<FixedArray> storage = factory->NewFixedArray(num_exports);
2394 60 : JSArray::SetContent(array_object, storage);
2395 : array_object->set_length(Smi::FromInt(num_exports));
2396 :
2397 : Handle<JSFunction> object_function =
2398 120 : Handle<JSFunction>(isolate->native_context()->object_function(), isolate);
2399 :
2400 : // Populate the result array.
2401 180 : for (int index = 0; index < num_exports; ++index) {
2402 120 : WasmExport& exp = module->export_table[index];
2403 :
2404 : Handle<String> export_kind;
2405 120 : switch (exp.kind) {
2406 : case kExternalFunction:
2407 30 : export_kind = function_string;
2408 30 : break;
2409 : case kExternalTable:
2410 30 : export_kind = table_string;
2411 30 : break;
2412 : case kExternalMemory:
2413 30 : export_kind = memory_string;
2414 30 : break;
2415 : case kExternalGlobal:
2416 30 : export_kind = global_string;
2417 30 : break;
2418 : default:
2419 0 : UNREACHABLE();
2420 : }
2421 :
2422 120 : Handle<JSObject> entry = factory->NewJSObject(object_function);
2423 :
2424 : MaybeHandle<String> export_name =
2425 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2426 120 : isolate, compiled_module, exp.name_offset, exp.name_length);
2427 :
2428 : JSObject::AddProperty(entry, name_string, export_name.ToHandleChecked(),
2429 120 : NONE);
2430 120 : JSObject::AddProperty(entry, kind_string, export_kind, NONE);
2431 :
2432 120 : storage->set(index, *entry);
2433 : }
2434 :
2435 60 : return array_object;
2436 : }
2437 :
2438 60 : Handle<JSArray> wasm::GetCustomSections(Isolate* isolate,
2439 : Handle<WasmModuleObject> module_object,
2440 : Handle<String> name,
2441 : ErrorThrower* thrower) {
2442 : Handle<WasmCompiledModule> compiled_module(module_object->compiled_module(),
2443 60 : isolate);
2444 : Factory* factory = isolate->factory();
2445 :
2446 : std::vector<CustomSectionOffset> custom_sections;
2447 : {
2448 : DisallowHeapAllocation no_gc; // for raw access to string bytes.
2449 : Handle<SeqOneByteString> module_bytes(compiled_module->module_bytes(),
2450 60 : isolate);
2451 : const byte* start =
2452 60 : reinterpret_cast<const byte*>(module_bytes->GetCharsAddress());
2453 60 : const byte* end = start + module_bytes->length();
2454 120 : custom_sections = DecodeCustomSections(start, end);
2455 : }
2456 :
2457 : std::vector<Handle<Object>> matching_sections;
2458 :
2459 : // Gather matching sections.
2460 300 : for (auto section : custom_sections) {
2461 : MaybeHandle<String> section_name =
2462 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2463 180 : isolate, compiled_module, section.name_offset, section.name_length);
2464 :
2465 180 : if (!name->Equals(*section_name.ToHandleChecked())) continue;
2466 :
2467 : // Make a copy of the payload data in the section.
2468 : bool is_external; // Set by TryAllocateBackingStore
2469 : void* memory = TryAllocateBackingStore(isolate, section.payload_length,
2470 45 : false, is_external);
2471 :
2472 : Handle<Object> section_data = factory->undefined_value();
2473 45 : if (memory) {
2474 45 : Handle<JSArrayBuffer> buffer = isolate->factory()->NewJSArrayBuffer();
2475 : JSArrayBuffer::Setup(buffer, isolate, is_external, memory,
2476 45 : static_cast<int>(section.payload_length));
2477 : DisallowHeapAllocation no_gc; // for raw access to string bytes.
2478 : Handle<SeqOneByteString> module_bytes(compiled_module->module_bytes(),
2479 45 : isolate);
2480 : const byte* start =
2481 45 : reinterpret_cast<const byte*>(module_bytes->GetCharsAddress());
2482 45 : memcpy(memory, start + section.payload_offset, section.payload_length);
2483 45 : section_data = buffer;
2484 : } else {
2485 0 : thrower->RangeError("out of memory allocating custom section data");
2486 0 : return Handle<JSArray>();
2487 : }
2488 :
2489 45 : matching_sections.push_back(section_data);
2490 : }
2491 :
2492 120 : int num_custom_sections = static_cast<int>(matching_sections.size());
2493 60 : Handle<JSArray> array_object = factory->NewJSArray(FAST_ELEMENTS, 0, 0);
2494 60 : Handle<FixedArray> storage = factory->NewFixedArray(num_custom_sections);
2495 60 : JSArray::SetContent(array_object, storage);
2496 : array_object->set_length(Smi::FromInt(num_custom_sections));
2497 :
2498 105 : for (int i = 0; i < num_custom_sections; i++) {
2499 90 : storage->set(i, *matching_sections[i]);
2500 : }
2501 :
2502 60 : return array_object;
2503 : }
2504 :
2505 1110 : bool wasm::SyncValidate(Isolate* isolate, ErrorThrower* thrower,
2506 : const ModuleWireBytes& bytes) {
2507 2220 : if (bytes.start() == nullptr || bytes.length() == 0) return false;
2508 : ModuleResult result =
2509 2220 : DecodeWasmModule(isolate, bytes.start(), bytes.end(), true, kWasmOrigin);
2510 1110 : if (result.val) delete result.val;
2511 : return result.ok();
2512 : }
2513 :
2514 3616 : MaybeHandle<WasmModuleObject> wasm::SyncCompileTranslatedAsmJs(
2515 : Isolate* isolate, ErrorThrower* thrower, const ModuleWireBytes& bytes,
2516 : Handle<Script> asm_js_script,
2517 3616 : Vector<const byte> asm_js_offset_table_bytes) {
2518 :
2519 : ModuleResult result = DecodeWasmModule(isolate, bytes.start(), bytes.end(),
2520 7232 : false, kAsmJsOrigin);
2521 3616 : if (result.failed()) {
2522 : // TODO(titzer): use Result<std::unique_ptr<const WasmModule*>>?
2523 0 : if (result.val) delete result.val;
2524 : thrower->CompileFailed("Wasm decoding failed", result);
2525 : return {};
2526 : }
2527 :
2528 7232 : CompilationHelper helper(isolate, const_cast<WasmModule*>(result.val));
2529 : return helper.CompileToModuleObject(thrower, bytes, asm_js_script,
2530 3616 : asm_js_offset_table_bytes);
2531 : }
2532 :
2533 12199 : MaybeHandle<WasmModuleObject> wasm::SyncCompile(Isolate* isolate,
2534 : ErrorThrower* thrower,
2535 : const ModuleWireBytes& bytes) {
2536 24398 : if (!IsWasmCodegenAllowed(isolate, isolate->native_context())) {
2537 14 : thrower->CompileError("Wasm code generation disallowed in this context");
2538 : return {};
2539 : }
2540 :
2541 : ModuleResult result =
2542 24370 : DecodeWasmModule(isolate, bytes.start(), bytes.end(), false, kWasmOrigin);
2543 12185 : if (result.failed()) {
2544 450 : if (result.val) delete result.val;
2545 : thrower->CompileFailed("Wasm decoding failed", result);
2546 : return {};
2547 : }
2548 :
2549 23470 : CompilationHelper helper(isolate, const_cast<WasmModule*>(result.val));
2550 : return helper.CompileToModuleObject(thrower, bytes, Handle<Script>(),
2551 11735 : Vector<const byte>());
2552 : }
2553 :
2554 48099 : MaybeHandle<WasmInstanceObject> wasm::SyncInstantiate(
2555 : Isolate* isolate, ErrorThrower* thrower,
2556 : Handle<WasmModuleObject> module_object, MaybeHandle<JSReceiver> imports,
2557 : MaybeHandle<JSArrayBuffer> memory) {
2558 48099 : InstantiationHelper helper(isolate, thrower, module_object, imports, memory);
2559 48099 : return helper.Build();
2560 : }
2561 :
2562 : namespace {
2563 :
2564 15 : void RejectPromise(Isolate* isolate, Handle<Context> context,
2565 : ErrorThrower* thrower, Handle<JSPromise> promise) {
2566 : v8::Local<v8::Promise::Resolver> resolver =
2567 : v8::Utils::PromiseToLocal(promise).As<v8::Promise::Resolver>();
2568 : auto maybe = resolver->Reject(v8::Utils::ToLocal(context),
2569 15 : v8::Utils::ToLocal(thrower->Reify()));
2570 15 : CHECK_IMPLIES(!maybe.FromMaybe(false), isolate->has_scheduled_exception());
2571 15 : }
2572 :
2573 750 : void ResolvePromise(Isolate* isolate, Handle<Context> context,
2574 : Handle<JSPromise> promise, Handle<Object> result) {
2575 : v8::Local<v8::Promise::Resolver> resolver =
2576 : v8::Utils::PromiseToLocal(promise).As<v8::Promise::Resolver>();
2577 : auto maybe = resolver->Resolve(v8::Utils::ToLocal(context),
2578 750 : v8::Utils::ToLocal(result));
2579 750 : CHECK_IMPLIES(!maybe.FromMaybe(false), isolate->has_scheduled_exception());
2580 750 : }
2581 :
2582 : } // namespace
2583 :
2584 0 : void wasm::AsyncInstantiate(Isolate* isolate, Handle<JSPromise> promise,
2585 : Handle<WasmModuleObject> module_object,
2586 : MaybeHandle<JSReceiver> imports) {
2587 : ErrorThrower thrower(isolate, nullptr);
2588 : MaybeHandle<WasmInstanceObject> instance_object = SyncInstantiate(
2589 0 : isolate, &thrower, module_object, imports, Handle<JSArrayBuffer>::null());
2590 0 : if (thrower.error()) {
2591 0 : RejectPromise(isolate, handle(isolate->context()), &thrower, promise);
2592 0 : return;
2593 : }
2594 : ResolvePromise(isolate, handle(isolate->context()), promise,
2595 0 : instance_object.ToHandleChecked());
2596 : }
2597 :
2598 : // Encapsulates all the state and steps of an asynchronous compilation.
2599 : // An asynchronous compile job consists of a number of tasks that are executed
2600 : // as foreground and background tasks. Any phase that touches the V8 heap or
2601 : // allocates on the V8 heap (e.g. creating the module object) must be a
2602 : // foreground task. All other tasks (e.g. decoding and validating, the majority
2603 : // of the work of compilation) can be background tasks.
2604 : // TODO(wasm): factor out common parts of this with the synchronous pipeline.
2605 : //
2606 : // Note: In predictable mode, DoSync and DoAsync execute the referenced function
2607 : // immediately before returning. Thus we handle the predictable mode specially,
2608 : // e.g. when we synchronizing tasks or when we delete the AyncCompileJob.
2609 : class AsyncCompileJob {
2610 : // TODO(ahaas): Fix https://bugs.chromium.org/p/v8/issues/detail?id=6263 to
2611 : // make sure that d8 does not shut down before the AsyncCompileJob is
2612 : // finished.
2613 : public:
2614 765 : explicit AsyncCompileJob(Isolate* isolate, std::unique_ptr<byte[]> bytes_copy,
2615 : int length, Handle<Context> context,
2616 : Handle<JSPromise> promise)
2617 : : isolate_(isolate),
2618 : thrower_(isolate, "AsyncCompile"),
2619 : bytes_copy_(std::move(bytes_copy)),
2620 3825 : wire_bytes_(bytes_copy_.get(), bytes_copy_.get() + length) {
2621 : // The handles for the context and promise must be deferred.
2622 765 : DeferredHandleScope deferred(isolate);
2623 765 : context_ = Handle<Context>(*context);
2624 765 : module_promise_ = Handle<JSPromise>(*promise);
2625 1530 : deferred_handles_.push_back(deferred.Detach());
2626 765 : }
2627 :
2628 : void Start() {
2629 765 : DoAsync(&AsyncCompileJob::DecodeModule); // --
2630 : }
2631 :
2632 1530 : ~AsyncCompileJob() {
2633 3795 : for (auto d : deferred_handles_) delete d;
2634 765 : }
2635 :
2636 : private:
2637 : Isolate* isolate_;
2638 : ErrorThrower thrower_;
2639 : std::unique_ptr<byte[]> bytes_copy_;
2640 : ModuleWireBytes wire_bytes_;
2641 : Handle<Context> context_;
2642 : Handle<JSPromise> module_promise_;
2643 : WasmModule* module_ = nullptr;
2644 : ModuleResult result_;
2645 : std::unique_ptr<CompilationHelper> helper_ = nullptr;
2646 : std::unique_ptr<ModuleBytesEnv> module_bytes_env_ = nullptr;
2647 :
2648 : volatile bool failed_ = false;
2649 : std::vector<DeferredHandles*> deferred_handles_;
2650 : Handle<WasmModuleWrapper> module_wrapper_;
2651 : Handle<WasmModuleObject> module_object_;
2652 : Handle<FixedArray> function_tables_;
2653 : Handle<FixedArray> signature_tables_;
2654 : Handle<WasmCompiledModule> compiled_module_;
2655 : Handle<FixedArray> code_table_;
2656 : std::unique_ptr<WasmInstance> temp_instance_ = nullptr;
2657 : size_t outstanding_units_ = 0;
2658 : size_t num_background_tasks_ = 0;
2659 :
2660 750 : void ReopenHandlesInDeferredScope() {
2661 750 : DeferredHandleScope deferred(isolate_);
2662 1500 : module_wrapper_ = handle(*module_wrapper_, isolate_);
2663 1500 : function_tables_ = handle(*function_tables_, isolate_);
2664 1500 : signature_tables_ = handle(*signature_tables_, isolate_);
2665 1500 : code_table_ = handle(*code_table_, isolate_);
2666 1500 : temp_instance_->ReopenHandles(isolate_);
2667 : helper_->InitializeHandles();
2668 1500 : deferred_handles_.push_back(deferred.Detach());
2669 750 : }
2670 :
2671 15 : void AsyncCompileFailed() {
2672 15 : RejectPromise(isolate_, context_, &thrower_, module_promise_);
2673 : // The AsyncCompileJob is finished, we resolved the promise, we do not need
2674 : // the data anymore. We can delete the AsyncCompileJob object.
2675 15 : if (!FLAG_verify_predictable) delete this;
2676 15 : }
2677 :
2678 750 : void AsyncCompileSucceeded(Handle<Object> result) {
2679 750 : ResolvePromise(isolate_, context_, module_promise_, result);
2680 : // The AsyncCompileJob is finished, we resolved the promise, we do not need
2681 : // the data anymore. We can delete the AsyncCompileJob object.
2682 750 : if (!FLAG_verify_predictable) delete this;
2683 750 : }
2684 :
2685 : //==========================================================================
2686 : // Step 1: (async) Decode the module.
2687 : //==========================================================================
2688 765 : void DecodeModule() {
2689 : {
2690 : DisallowHandleAllocation no_handle;
2691 : DisallowHeapAllocation no_allocation;
2692 : // Decode the module bytes.
2693 765 : TRACE_COMPILE("(1) Decoding module...\n");
2694 1530 : result_ = DecodeWasmModule(isolate_, wire_bytes_.start(),
2695 : wire_bytes_.end(), true, kWasmOrigin);
2696 : }
2697 765 : if (result_.failed()) {
2698 : // Decoding failure; reject the promise and clean up.
2699 15 : if (result_.val) delete result_.val;
2700 15 : DoSync(&AsyncCompileJob::DecodeFail);
2701 : } else {
2702 : // Decode passed.
2703 750 : module_ = const_cast<WasmModule*>(result_.val);
2704 750 : DoSync(&AsyncCompileJob::PrepareAndStartCompile);
2705 : }
2706 765 : }
2707 :
2708 : //==========================================================================
2709 : // Step 1b: (sync) Fail decoding the module.
2710 : //==========================================================================
2711 15 : void DecodeFail() {
2712 15 : HandleScope scope(isolate_);
2713 15 : thrower_.CompileFailed("Wasm decoding failed", result_);
2714 : // {this} is deleted in AsyncCompileFailed, therefore the {return}.
2715 30 : return AsyncCompileFailed();
2716 : }
2717 :
2718 : //==========================================================================
2719 : // Step 2 (sync): Create heap-allocated data and start compile.
2720 : //==========================================================================
2721 750 : void PrepareAndStartCompile() {
2722 750 : TRACE_COMPILE("(2) Prepare and start compile...\n");
2723 1500 : HandleScope scope(isolate_);
2724 :
2725 : Factory* factory = isolate_->factory();
2726 : // The {module_wrapper} will take ownership of the {WasmModule} object,
2727 : // and it will be destroyed when the GC reclaims the wrapper object.
2728 750 : module_wrapper_ = WasmModuleWrapper::New(isolate_, module_);
2729 750 : temp_instance_ = std::unique_ptr<WasmInstance>(new WasmInstance(module_));
2730 750 : temp_instance_->context = context_;
2731 750 : temp_instance_->mem_size = WasmModule::kPageSize * module_->min_mem_pages;
2732 750 : temp_instance_->mem_start = nullptr;
2733 750 : temp_instance_->globals_start = nullptr;
2734 :
2735 : // Initialize the indirect tables with placeholders.
2736 : int function_table_count =
2737 1500 : static_cast<int>(module_->function_tables.size());
2738 750 : function_tables_ = factory->NewFixedArray(function_table_count, TENURED);
2739 750 : signature_tables_ = factory->NewFixedArray(function_table_count, TENURED);
2740 855 : for (int i = 0; i < function_table_count; ++i) {
2741 315 : temp_instance_->function_tables[i] = factory->NewFixedArray(1, TENURED);
2742 210 : temp_instance_->signature_tables[i] = factory->NewFixedArray(1, TENURED);
2743 210 : function_tables_->set(i, *temp_instance_->function_tables[i]);
2744 210 : signature_tables_->set(i, *temp_instance_->signature_tables[i]);
2745 : }
2746 :
2747 : // The {code_table} array contains import wrappers and functions (which
2748 : // are both included in {functions.size()}, and export wrappers.
2749 : // The results of compilation will be written into it.
2750 2865 : int code_table_size = static_cast<int>(module_->functions.size() +
2751 750 : module_->num_exported_functions);
2752 750 : code_table_ = factory->NewFixedArray(code_table_size, TENURED);
2753 :
2754 : // Initialize {code_table_} with the illegal builtin. All call sites
2755 : // will be patched at instantiation.
2756 750 : Handle<Code> illegal_builtin = isolate_->builtins()->Illegal();
2757 : // TODO(wasm): Fix this for lazy compilation.
2758 2730 : for (uint32_t i = 0; i < module_->functions.size(); ++i) {
2759 1230 : code_table_->set(static_cast<int>(i), *illegal_builtin);
2760 1230 : temp_instance_->function_code[i] = illegal_builtin;
2761 : }
2762 :
2763 : isolate_->counters()->wasm_functions_per_wasm_module()->AddSample(
2764 1500 : static_cast<int>(module_->functions.size()));
2765 :
2766 750 : helper_.reset(new CompilationHelper(isolate_, module_));
2767 :
2768 : DCHECK_LE(module_->num_imported_functions, module_->functions.size());
2769 : size_t num_functions =
2770 1500 : module_->functions.size() - module_->num_imported_functions;
2771 750 : if (num_functions == 0) {
2772 540 : ReopenHandlesInDeferredScope();
2773 : // Degenerate case of an empty module.
2774 1290 : return DoSync(&AsyncCompileJob::FinishCompile);
2775 : }
2776 :
2777 : // Start asynchronous compilation tasks.
2778 : num_background_tasks_ =
2779 : Max(static_cast<size_t>(1),
2780 : Min(num_functions,
2781 : Min(static_cast<size_t>(FLAG_wasm_num_compilation_tasks),
2782 210 : V8::GetCurrentPlatform()
2783 420 : ->NumberOfAvailableBackgroundThreads())));
2784 210 : module_bytes_env_ = std::unique_ptr<ModuleBytesEnv>(
2785 210 : new ModuleBytesEnv(module_, temp_instance_.get(), wire_bytes_));
2786 : outstanding_units_ = helper_->InitializeParallelCompilation(
2787 420 : module_->functions, *module_bytes_env_);
2788 :
2789 : // Reopen all handles which should survive in the DeferredHandleScope.
2790 210 : ReopenHandlesInDeferredScope();
2791 420 : for (size_t i = 0; i < num_background_tasks_; ++i) {
2792 210 : DoAsync(&AsyncCompileJob::ExecuteCompilationUnits);
2793 : }
2794 : }
2795 :
2796 : //==========================================================================
2797 : // Step 3 (async x K tasks): Execute compilation units.
2798 : //==========================================================================
2799 210 : void ExecuteCompilationUnits() {
2800 210 : TRACE_COMPILE("(3) Compiling...\n");
2801 420 : while (!failed_) {
2802 : {
2803 : DisallowHandleAllocation no_handle;
2804 : DisallowHeapAllocation no_allocation;
2805 420 : if (!helper_->FetchAndExecuteCompilationUnit()) break;
2806 : }
2807 : // TODO(ahaas): Create one FinishCompilationUnit job for all compilation
2808 : // units.
2809 210 : DoSync(&AsyncCompileJob::FinishCompilationUnit);
2810 : // TODO(ahaas): Limit the number of outstanding compilation units to be
2811 : // finished to reduce memory overhead.
2812 : }
2813 : // Special handling for predictable mode, see above.
2814 : if (!FLAG_verify_predictable)
2815 420 : helper_->module_->pending_tasks.get()->Signal();
2816 210 : }
2817 :
2818 : //==========================================================================
2819 : // Step 4 (sync x each function): Finish a single compilation unit.
2820 : //==========================================================================
2821 210 : void FinishCompilationUnit() {
2822 210 : TRACE_COMPILE("(4a) Finishing compilation unit...\n");
2823 210 : HandleScope scope(isolate_);
2824 420 : if (failed_) return; // already failed
2825 :
2826 210 : int func_index = -1;
2827 : Handle<Code> result =
2828 420 : helper_->FinishCompilationUnit(&thrower_, &func_index);
2829 210 : if (thrower_.error()) {
2830 0 : failed_ = true;
2831 : } else {
2832 : DCHECK(func_index >= 0);
2833 420 : code_table_->set(func_index, *(result));
2834 : }
2835 210 : if (failed_ || --outstanding_units_ == 0) {
2836 : // All compilation units are done. We still need to wait for the
2837 : // background tasks to shut down and only then is it safe to finish the
2838 : // compile and delete this job. We can wait for that to happen also
2839 : // in a background task.
2840 210 : DoAsync(&AsyncCompileJob::WaitForBackgroundTasks);
2841 : }
2842 : }
2843 :
2844 : //==========================================================================
2845 : // Step 4b (async): Wait for all background tasks to finish.
2846 : //==========================================================================
2847 210 : void WaitForBackgroundTasks() {
2848 210 : TRACE_COMPILE("(4b) Waiting for background tasks...\n");
2849 : // Special handling for predictable mode, see above.
2850 : if (!FLAG_verify_predictable) {
2851 210 : for (size_t i = 0; i < num_background_tasks_; ++i) {
2852 : // We wait for it to finish.
2853 420 : module_->pending_tasks.get()->Wait();
2854 : }
2855 : }
2856 210 : if (!failed_) {
2857 210 : DoSync(&AsyncCompileJob::FinishCompile);
2858 : } else {
2859 : // Call AsyncCompileFailed with DoSync because it has to happen on the
2860 : // main thread.
2861 0 : DoSync(&AsyncCompileJob::AsyncCompileFailed);
2862 : }
2863 210 : }
2864 :
2865 : //==========================================================================
2866 : // Step 5 (sync): Finish heap-allocated data structures.
2867 : //==========================================================================
2868 750 : void FinishCompile() {
2869 750 : TRACE_COMPILE("(5) Finish compile...\n");
2870 1500 : HandleScope scope(isolate_);
2871 1500 : SaveContext saved_context(isolate_);
2872 750 : isolate_->set_context(*context_);
2873 : // At this point, compilation has completed. Update the code table.
2874 2730 : for (size_t i = FLAG_skip_compiling_wasm_funcs;
2875 1365 : i < temp_instance_->function_code.size(); ++i) {
2876 615 : Code* code = Code::cast(code_table_->get(static_cast<int>(i)));
2877 615 : RecordStats(isolate_, code);
2878 : }
2879 :
2880 : // Create heap objects for script and module bytes to be stored in the
2881 : // shared module data. Asm.js is not compiled asynchronously.
2882 750 : Handle<Script> script = CreateWasmScript(isolate_, wire_bytes_);
2883 : Handle<ByteArray> asm_js_offset_table;
2884 : // TODO(wasm): Improve efficiency of storing module wire bytes.
2885 : // 1. Only store relevant sections, not function bodies
2886 : // 2. Don't make a second copy of the bytes here; reuse the copy made
2887 : // for asynchronous compilation and store it as an external one
2888 : // byte string for serialization/deserialization.
2889 : Handle<String> module_bytes =
2890 : isolate_->factory()
2891 : ->NewStringFromOneByte({wire_bytes_.start(), wire_bytes_.length()},
2892 : TENURED)
2893 1500 : .ToHandleChecked();
2894 : DCHECK(module_bytes->IsSeqOneByteString());
2895 :
2896 : // Create the shared module data.
2897 : // TODO(clemensh): For the same module (same bytes / same hash), we should
2898 : // only have one WasmSharedModuleData. Otherwise, we might only set
2899 : // breakpoints on a (potentially empty) subset of the instances.
2900 :
2901 : Handle<WasmSharedModuleData> shared = WasmSharedModuleData::New(
2902 : isolate_, module_wrapper_, Handle<SeqOneByteString>::cast(module_bytes),
2903 750 : script, asm_js_offset_table);
2904 :
2905 : // Create the compiled module object and populate with compiled functions
2906 : // and information needed at instantiation time. This object needs to be
2907 : // serializable. Instantiation may occur off a deserialized version of this
2908 : // object.
2909 : compiled_module_ = WasmCompiledModule::New(
2910 750 : isolate_, shared, code_table_, function_tables_, signature_tables_);
2911 :
2912 : // Finish the WASM script now and make it public to the debugger.
2913 750 : script->set_wasm_compiled_module(*compiled_module_);
2914 1500 : isolate_->debug()->OnAfterCompile(script);
2915 :
2916 1500 : DeferredHandleScope deferred(isolate_);
2917 1500 : compiled_module_ = handle(*compiled_module_, isolate_);
2918 1500 : deferred_handles_.push_back(deferred.Detach());
2919 : // TODO(wasm): compiling wrappers should be made async as well.
2920 750 : DoSync(&AsyncCompileJob::CompileWrappers);
2921 750 : }
2922 :
2923 : //==========================================================================
2924 : // Step 6 (sync): Compile JS->WASM wrappers.
2925 : //==========================================================================
2926 750 : void CompileWrappers() {
2927 750 : TRACE_COMPILE("(6) Compile wrappers...\n");
2928 : // Compile JS->WASM wrappers for exported functions.
2929 750 : HandleScope scope(isolate_);
2930 750 : JSToWasmWrapperCache js_to_wasm_cache;
2931 : int func_index = 0;
2932 2445 : for (auto exp : module_->export_table) {
2933 255 : if (exp.kind != kExternalFunction) continue;
2934 135 : Handle<Code> wasm_code(Code::cast(code_table_->get(exp.index)), isolate_);
2935 : Handle<Code> wrapper_code =
2936 : js_to_wasm_cache.CloneOrCompileJSToWasmWrapper(isolate_, module_,
2937 135 : wasm_code, exp.index);
2938 : int export_index =
2939 270 : static_cast<int>(module_->functions.size() + func_index);
2940 135 : code_table_->set(export_index, *wrapper_code);
2941 135 : RecordStats(isolate_, *wrapper_code);
2942 135 : func_index++;
2943 : }
2944 :
2945 750 : DoSync(&AsyncCompileJob::FinishModule);
2946 750 : }
2947 :
2948 : //==========================================================================
2949 : // Step 7 (sync): Finish the module and resolve the promise.
2950 : //==========================================================================
2951 750 : void FinishModule() {
2952 750 : TRACE_COMPILE("(7) Finish module...\n");
2953 750 : HandleScope scope(isolate_);
2954 1500 : SaveContext saved_context(isolate_);
2955 750 : isolate_->set_context(*context_);
2956 : Handle<WasmModuleObject> result =
2957 750 : WasmModuleObject::New(isolate_, compiled_module_);
2958 : // {this} is deleted in AsyncCompileSucceeded, therefore the {return}.
2959 1500 : return AsyncCompileSucceeded(result);
2960 : }
2961 :
2962 : // Run the given member method as an asynchronous task.
2963 1185 : void DoAsync(void (AsyncCompileJob::*func)()) {
2964 1185 : auto task = new AsyncCompileTask(this, func);
2965 1185 : V8::GetCurrentPlatform()->CallOnBackgroundThread(
2966 1185 : task, v8::Platform::kShortRunningTask);
2967 1185 : }
2968 :
2969 : // Run the given member method as a synchronous task.
2970 3225 : void DoSync(void (AsyncCompileJob::*func)()) {
2971 3225 : V8::GetCurrentPlatform()->CallOnForegroundThread(
2972 : reinterpret_cast<v8::Isolate*>(isolate_),
2973 6450 : new AsyncCompileTask(this, func));
2974 3225 : }
2975 :
2976 : // A helper closure to run a particular member method as a task.
2977 8818 : class AsyncCompileTask : NON_EXPORTED_BASE(public v8::Task) {
2978 : public:
2979 : AsyncCompileJob* job_;
2980 : void (AsyncCompileJob::*func_)();
2981 : AsyncCompileTask(AsyncCompileJob* job, void (AsyncCompileJob::*func)())
2982 4410 : : v8::Task(), job_(job), func_(func) {}
2983 :
2984 4410 : void Run() override {
2985 4410 : (job_->*func_)(); // run the task.
2986 4410 : }
2987 : };
2988 : };
2989 :
2990 1530 : void wasm::AsyncCompile(Isolate* isolate, Handle<JSPromise> promise,
2991 : const ModuleWireBytes& bytes) {
2992 765 : if (!FLAG_wasm_async_compilation) {
2993 : ErrorThrower thrower(isolate, "WasmCompile");
2994 : // Compile the module.
2995 : MaybeHandle<WasmModuleObject> module_object =
2996 0 : SyncCompile(isolate, &thrower, bytes);
2997 0 : if (thrower.error()) {
2998 0 : RejectPromise(isolate, handle(isolate->context()), &thrower, promise);
2999 0 : return;
3000 : }
3001 : Handle<WasmModuleObject> module = module_object.ToHandleChecked();
3002 0 : ResolvePromise(isolate, handle(isolate->context()), promise, module);
3003 0 : return;
3004 : }
3005 :
3006 : // Make a copy of the wire bytes in case the user program changes them
3007 : // during asynchronous compilation.
3008 765 : std::unique_ptr<byte[]> copy(new byte[bytes.length()]);
3009 765 : memcpy(copy.get(), bytes.start(), bytes.length());
3010 : auto job = new AsyncCompileJob(isolate, std::move(copy), bytes.length(),
3011 1530 : handle(isolate->context()), promise);
3012 : job->Start();
3013 : // Special handling for predictable mode, see above.
3014 : if (FLAG_verify_predictable) delete job;
3015 : }
3016 :
3017 12699 : Handle<Code> wasm::CompileLazy(Isolate* isolate) {
3018 : HistogramTimerScope lazy_time_scope(
3019 12699 : isolate->counters()->wasm_lazy_compilation_time());
3020 :
3021 : // Find the wasm frame which triggered the lazy compile, to get the wasm
3022 : // instance.
3023 12699 : StackFrameIterator it(isolate);
3024 : // First frame: C entry stub.
3025 : DCHECK(!it.done());
3026 : DCHECK_EQ(StackFrame::EXIT, it.frame()->type());
3027 12699 : it.Advance();
3028 : // Second frame: WasmCompileLazy builtin.
3029 : DCHECK(!it.done());
3030 12699 : Handle<Code> lazy_compile_code(it.frame()->LookupCode(), isolate);
3031 : DCHECK_EQ(Builtins::kWasmCompileLazy, lazy_compile_code->builtin_index());
3032 : Handle<WasmInstanceObject> instance;
3033 : Handle<FixedArray> exp_deopt_data;
3034 : int func_index = -1;
3035 12699 : if (lazy_compile_code->deoptimization_data()->length() > 0) {
3036 : // Then it's an indirect call or via JS->WASM wrapper.
3037 : DCHECK_LE(2, lazy_compile_code->deoptimization_data()->length());
3038 : exp_deopt_data = handle(lazy_compile_code->deoptimization_data(), isolate);
3039 : auto* weak_cell = WeakCell::cast(exp_deopt_data->get(0));
3040 10107 : instance = handle(WasmInstanceObject::cast(weak_cell->value()), isolate);
3041 : func_index = Smi::cast(exp_deopt_data->get(1))->value();
3042 : }
3043 12699 : it.Advance();
3044 : // Third frame: The calling wasm code or js-to-wasm wrapper.
3045 : DCHECK(!it.done());
3046 : DCHECK(it.frame()->is_js_to_wasm() || it.frame()->is_wasm_compiled());
3047 12699 : Handle<Code> caller_code = handle(it.frame()->LookupCode(), isolate);
3048 25398 : if (it.frame()->is_js_to_wasm()) {
3049 : DCHECK(!instance.is_null());
3050 3124 : } else if (instance.is_null()) {
3051 2592 : instance = handle(wasm::GetOwningWasmInstance(*caller_code), isolate);
3052 : } else {
3053 : DCHECK(*instance == wasm::GetOwningWasmInstance(*caller_code));
3054 : }
3055 : int offset =
3056 38097 : static_cast<int>(it.frame()->pc() - caller_code->instruction_start());
3057 : // Only patch the caller code if this is *no* indirect call.
3058 : // exp_deopt_data will be null if the called function is not exported at all,
3059 : // and its length will be <= 2 if all entries in tables were already patched.
3060 : // Note that this check is conservative: If the first call to an exported
3061 : // function is direct, we will just patch the export tables, and only on the
3062 : // second call we will patch the caller.
3063 3124 : bool patch_caller = caller_code->kind() == Code::JS_TO_WASM_FUNCTION ||
3064 13231 : exp_deopt_data.is_null() || exp_deopt_data->length() <= 2;
3065 :
3066 : MaybeHandle<Code> maybe_compiled_code = WasmCompiledModule::CompileLazy(
3067 12699 : isolate, instance, caller_code, offset, func_index, patch_caller);
3068 12699 : if (maybe_compiled_code.is_null()) {
3069 : DCHECK(isolate->has_pending_exception());
3070 0 : return isolate->builtins()->Illegal();
3071 : }
3072 : Handle<Code> compiled_code = maybe_compiled_code.ToHandleChecked();
3073 22806 : if (!exp_deopt_data.is_null() && exp_deopt_data->length() > 2) {
3074 : // See EnsureExportedLazyDeoptData: exp_deopt_data[2...(len-1)] are pairs of
3075 : // <export_table, index> followed by undefined values.
3076 : // Use this information here to patch all export tables.
3077 : DCHECK_EQ(0, exp_deopt_data->length() % 2);
3078 770 : for (int idx = 2, end = exp_deopt_data->length(); idx < end; idx += 2) {
3079 392 : if (exp_deopt_data->get(idx)->IsUndefined(isolate)) break;
3080 : FixedArray* exp_table = FixedArray::cast(exp_deopt_data->get(idx));
3081 392 : int exp_index = Smi::cast(exp_deopt_data->get(idx + 1))->value();
3082 : DCHECK(exp_table->get(exp_index) == *lazy_compile_code);
3083 392 : exp_table->set(exp_index, *compiled_code);
3084 : }
3085 : // After processing, remove the list of exported entries, such that we don't
3086 : // do the patching redundantly.
3087 : Handle<FixedArray> new_deopt_data =
3088 378 : isolate->factory()->CopyFixedArrayUpTo(exp_deopt_data, 2, TENURED);
3089 378 : lazy_compile_code->set_deoptimization_data(*new_deopt_data);
3090 : }
3091 :
3092 12699 : return compiled_code;
3093 : }
3094 :
3095 12699 : bool LazyCompilationOrchestrator::CompileFunction(
3096 12496 : Isolate* isolate, Handle<WasmInstanceObject> instance, int func_index) {
3097 : Handle<WasmCompiledModule> compiled_module(instance->compiled_module(),
3098 12699 : isolate);
3099 25398 : if (Code::cast(compiled_module->code_table()->get(func_index))->kind() ==
3100 : Code::WASM_FUNCTION) {
3101 : return true;
3102 : }
3103 : size_t num_function_tables =
3104 12496 : compiled_module->module()->function_tables.size();
3105 : // Store a vector of handles to be embedded in the generated code.
3106 : // TODO(clemensh): For concurrent compilation, these will have to live in a
3107 : // DeferredHandleScope.
3108 12496 : std::vector<Handle<FixedArray>> fun_tables(num_function_tables);
3109 12496 : std::vector<Handle<FixedArray>> sig_tables(num_function_tables);
3110 15553 : for (size_t i = 0; i < num_function_tables; ++i) {
3111 : Object* fun_table =
3112 9171 : compiled_module->function_tables()->get(static_cast<int>(i));
3113 6114 : fun_tables[i] = handle(FixedArray::cast(fun_table), isolate);
3114 : Object* sig_table =
3115 6114 : compiled_module->signature_tables()->get(static_cast<int>(i));
3116 6114 : sig_tables[i] = handle(FixedArray::cast(sig_table), isolate);
3117 : }
3118 12496 : wasm::ModuleEnv module_env(compiled_module->module(), &fun_tables,
3119 : &sig_tables);
3120 12496 : uint8_t* module_start = compiled_module->module_bytes()->GetChars();
3121 12496 : const WasmFunction* func = &module_env.module->functions[func_index];
3122 : wasm::FunctionBody body{func->sig, module_start,
3123 : module_start + func->code_start_offset,
3124 12496 : module_start + func->code_end_offset};
3125 : // TODO(wasm): Refactor this to only get the name if it is really needed for
3126 : // tracing / debugging.
3127 : std::string func_name;
3128 : {
3129 : wasm::WasmName name = Vector<const char>::cast(
3130 24992 : compiled_module->GetRawFunctionName(func_index));
3131 : // Copy to std::string, because the underlying string object might move on
3132 : // the heap.
3133 12496 : func_name.assign(name.start(), static_cast<size_t>(name.length()));
3134 : }
3135 12496 : ErrorThrower thrower(isolate, "WasmLazyCompile");
3136 : compiler::WasmCompilationUnit unit(isolate, &module_env, body,
3137 24992 : CStrVector(func_name.c_str()), func_index);
3138 12496 : unit.InitializeHandles();
3139 12496 : unit.ExecuteCompilation();
3140 12496 : Handle<Code> code = unit.FinishCompilation(&thrower);
3141 :
3142 12496 : Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(2, TENURED);
3143 12496 : Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance);
3144 : // TODO(wasm): Introduce constants for the indexes in wasm deopt data.
3145 12496 : deopt_data->set(0, *weak_instance);
3146 : deopt_data->set(1, Smi::FromInt(func_index));
3147 12496 : code->set_deoptimization_data(*deopt_data);
3148 :
3149 12496 : if (thrower.error()) {
3150 0 : if (!isolate->has_pending_exception()) isolate->Throw(*thrower.Reify());
3151 : return false;
3152 : }
3153 :
3154 : DCHECK_EQ(Builtins::kWasmCompileLazy,
3155 : Code::cast(compiled_module->code_table()->get(func_index))
3156 : ->builtin_index());
3157 24992 : compiled_module->code_table()->set(func_index, *code);
3158 :
3159 : // Now specialize the generated code for this instance.
3160 24992 : Zone specialization_zone(isolate->allocator(), ZONE_NAME);
3161 24992 : CodeSpecialization code_specialization(isolate, &specialization_zone);
3162 12496 : if (module_env.module->globals_size) {
3163 : Address globals_start =
3164 4679 : reinterpret_cast<Address>(compiled_module->globals_start());
3165 4679 : code_specialization.RelocateGlobals(nullptr, globals_start);
3166 : }
3167 12496 : if (instance->has_memory_buffer()) {
3168 : Address mem_start =
3169 12496 : reinterpret_cast<Address>(instance->memory_buffer()->backing_store());
3170 24992 : int mem_size = instance->memory_buffer()->byte_length()->Number();
3171 : DCHECK_IMPLIES(mem_size == 0, mem_start == nullptr);
3172 12496 : if (mem_size > 0) {
3173 : code_specialization.RelocateMemoryReferences(nullptr, 0, mem_start,
3174 12483 : mem_size);
3175 : }
3176 : }
3177 12496 : code_specialization.RelocateDirectCalls(instance);
3178 12496 : code_specialization.ApplyToWasmCode(*code, SKIP_ICACHE_FLUSH);
3179 12496 : Assembler::FlushICache(isolate, code->instruction_start(),
3180 24992 : code->instruction_size());
3181 12496 : RecordLazyCodeStats(isolate, *code);
3182 : return true;
3183 : }
3184 :
3185 12699 : MaybeHandle<Code> LazyCompilationOrchestrator::CompileLazy(
3186 : Isolate* isolate, Handle<WasmInstanceObject> instance, Handle<Code> caller,
3187 : int call_offset, int exported_func_index, bool patch_caller) {
3188 : struct NonCompiledFunction {
3189 : int offset;
3190 : int func_index;
3191 : };
3192 : std::vector<NonCompiledFunction> non_compiled_functions;
3193 : int func_to_return_idx = exported_func_index;
3194 : wasm::Decoder decoder(nullptr, nullptr);
3195 12699 : bool is_js_to_wasm = caller->kind() == Code::JS_TO_WASM_FUNCTION;
3196 : Handle<WasmCompiledModule> compiled_module(instance->compiled_module(),
3197 12699 : isolate);
3198 :
3199 12699 : if (is_js_to_wasm) {
3200 19150 : non_compiled_functions.push_back({0, exported_func_index});
3201 3124 : } else if (patch_caller) {
3202 : DisallowHeapAllocation no_gc;
3203 2753 : SeqOneByteString* module_bytes = compiled_module->module_bytes();
3204 : SourcePositionTableIterator source_pos_iterator(
3205 2753 : caller->SourcePositionTable());
3206 : DCHECK_EQ(2, caller->deoptimization_data()->length());
3207 : int caller_func_index =
3208 : Smi::cast(caller->deoptimization_data()->get(1))->value();
3209 : const byte* func_bytes =
3210 5506 : module_bytes->GetChars() + compiled_module->module()
3211 5506 : ->functions[caller_func_index]
3212 2753 : .code_start_offset;
3213 81263 : for (RelocIterator it(*caller, RelocInfo::kCodeTargetMask); !it.done();
3214 75757 : it.next()) {
3215 : Code* callee =
3216 75757 : Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
3217 75757 : if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue;
3218 : // TODO(clemensh): Introduce safe_cast<T, bool> which (D)CHECKS
3219 : // (depending on the bool) against limits of T and then static_casts.
3220 51482 : size_t offset_l = it.rinfo()->pc() - caller->instruction_start();
3221 : DCHECK_GE(kMaxInt, offset_l);
3222 25741 : int offset = static_cast<int>(offset_l);
3223 : int byte_pos =
3224 25741 : AdvanceSourcePositionTableIterator(source_pos_iterator, offset);
3225 : int called_func_index =
3226 25741 : ExtractDirectCallIndex(decoder, func_bytes + byte_pos);
3227 51482 : non_compiled_functions.push_back({offset, called_func_index});
3228 : // Call offset one instruction after the call. Remember the last called
3229 : // function before that offset.
3230 25741 : if (offset < call_offset) func_to_return_idx = called_func_index;
3231 : }
3232 : }
3233 :
3234 : // TODO(clemensh): compile all functions in non_compiled_functions in
3235 : // background, wait for func_to_return_idx.
3236 12699 : if (!CompileFunction(isolate, instance, func_to_return_idx)) {
3237 : return {};
3238 : }
3239 :
3240 12699 : if (is_js_to_wasm || patch_caller) {
3241 : DisallowHeapAllocation no_gc;
3242 : // Now patch the code object with all functions which are now compiled.
3243 : int idx = 0;
3244 121506 : for (RelocIterator it(*caller, RelocInfo::kCodeTargetMask); !it.done();
3245 96850 : it.next()) {
3246 : Code* callee =
3247 96850 : Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
3248 96850 : if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue;
3249 : DCHECK_GT(non_compiled_functions.size(), idx);
3250 70632 : int called_func_index = non_compiled_functions[idx].func_index;
3251 : // Check that the callee agrees with our assumed called_func_index.
3252 : DCHECK_IMPLIES(
3253 : callee->deoptimization_data()->length() > 0,
3254 : Smi::cast(callee->deoptimization_data()->get(1))->value() ==
3255 : called_func_index);
3256 : if (is_js_to_wasm) {
3257 : DCHECK_EQ(func_to_return_idx, called_func_index);
3258 : } else {
3259 : DCHECK_EQ(non_compiled_functions[idx].offset,
3260 : it.rinfo()->pc() - caller->instruction_start());
3261 : }
3262 35316 : ++idx;
3263 : Handle<Code> callee_compiled(
3264 70632 : Code::cast(compiled_module->code_table()->get(called_func_index)));
3265 35316 : if (callee_compiled->builtin_index() == Builtins::kWasmCompileLazy) {
3266 : DCHECK_NE(func_to_return_idx, called_func_index);
3267 : continue;
3268 : }
3269 : DCHECK_EQ(Code::WASM_FUNCTION, callee_compiled->kind());
3270 : it.rinfo()->set_target_address(isolate,
3271 14785 : callee_compiled->instruction_start());
3272 : }
3273 : DCHECK_EQ(non_compiled_functions.size(), idx);
3274 : }
3275 :
3276 : Code* ret =
3277 25398 : Code::cast(compiled_module->code_table()->get(func_to_return_idx));
3278 : DCHECK_EQ(Code::WASM_FUNCTION, ret->kind());
3279 : return handle(ret, isolate);
3280 : }
|
