Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include <memory>
6 :
7 : #include "src/assembler-inl.h"
8 : #include "src/base/adapters.h"
9 : #include "src/base/atomic-utils.h"
10 : #include "src/code-stubs.h"
11 : #include "src/compiler/wasm-compiler.h"
12 : #include "src/debug/interface-types.h"
13 : #include "src/frames-inl.h"
14 : #include "src/objects.h"
15 : #include "src/property-descriptor.h"
16 : #include "src/simulator.h"
17 : #include "src/snapshot/snapshot.h"
18 : #include "src/trap-handler/trap-handler.h"
19 : #include "src/v8.h"
20 :
21 : #include "src/asmjs/asm-wasm-builder.h"
22 : #include "src/wasm/function-body-decoder.h"
23 : #include "src/wasm/module-decoder.h"
24 : #include "src/wasm/wasm-code-specialization.h"
25 : #include "src/wasm/wasm-js.h"
26 : #include "src/wasm/wasm-limits.h"
27 : #include "src/wasm/wasm-module.h"
28 : #include "src/wasm/wasm-objects.h"
29 : #include "src/wasm/wasm-result.h"
30 :
31 : using namespace v8::internal;
32 : using namespace v8::internal::wasm;
33 : namespace base = v8::base;
34 :
35 : #define TRACE(...) \
36 : do { \
37 : if (FLAG_trace_wasm_instances) PrintF(__VA_ARGS__); \
38 : } while (false)
39 :
40 : #define TRACE_CHAIN(instance) \
41 : do { \
42 : instance->PrintInstancesChain(); \
43 : } while (false)
44 :
45 : #define TRACE_COMPILE(...) \
46 : do { \
47 : if (FLAG_trace_wasm_compiler) PrintF(__VA_ARGS__); \
48 : } while (false)
49 :
50 : namespace {
51 :
52 : static const int kInvalidSigIndex = -1;
53 :
54 12359 : byte* raw_buffer_ptr(MaybeHandle<JSArrayBuffer> buffer, int offset) {
55 12359 : return static_cast<byte*>(buffer.ToHandleChecked()->backing_store()) + offset;
56 : }
57 :
58 1239 : static void MemoryFinalizer(const v8::WeakCallbackInfo<void>& data) {
59 : DisallowHeapAllocation no_gc;
60 : JSArrayBuffer** p = reinterpret_cast<JSArrayBuffer**>(data.GetParameter());
61 633 : JSArrayBuffer* buffer = *p;
62 :
63 633 : if (!buffer->was_neutered()) {
64 : void* memory = buffer->backing_store();
65 : DCHECK(memory != nullptr);
66 : base::OS::Free(memory,
67 1212 : RoundUp(kWasmMaxHeapOffset, base::OS::CommitPageSize()));
68 :
69 : data.GetIsolate()->AdjustAmountOfExternalAllocatedMemory(
70 606 : -buffer->byte_length()->Number());
71 : }
72 :
73 633 : GlobalHandles::Destroy(reinterpret_cast<Object**>(p));
74 633 : }
75 :
76 784371 : static void RecordStats(Isolate* isolate, Code* code) {
77 522914 : isolate->counters()->wasm_generated_code_size()->Increment(code->body_size());
78 : isolate->counters()->wasm_reloc_size()->Increment(
79 261457 : code->relocation_info()->length());
80 261457 : }
81 :
82 32123 : static void RecordStats(Isolate* isolate, Handle<FixedArray> functions) {
83 : DisallowHeapAllocation no_gc;
84 295642 : for (int i = 0; i < functions->length(); ++i) {
85 115698 : RecordStats(isolate, Code::cast(functions->get(i)));
86 : }
87 32123 : }
88 :
89 22759 : void* TryAllocateBackingStore(Isolate* isolate, size_t size,
90 : bool enable_guard_regions, bool& is_external) {
91 11695 : is_external = false;
92 : // TODO(eholk): Right now enable_guard_regions has no effect on 32-bit
93 : // systems. It may be safer to fail instead, given that other code might do
94 : // things that would be unsafe if they expected guard pages where there
95 : // weren't any.
96 11695 : if (enable_guard_regions && kGuardRegionsSupported) {
97 : // TODO(eholk): On Windows we want to make sure we don't commit the guard
98 : // pages yet.
99 :
100 : // We always allocate the largest possible offset into the heap, so the
101 : // addressable memory after the guard page can be made inaccessible.
102 : const size_t alloc_size =
103 631 : RoundUp(kWasmMaxHeapOffset, base::OS::CommitPageSize());
104 : DCHECK_EQ(0, size % base::OS::CommitPageSize());
105 :
106 : // AllocateGuarded makes the whole region inaccessible by default.
107 631 : void* memory = base::OS::AllocateGuarded(alloc_size);
108 631 : if (memory == nullptr) {
109 : return nullptr;
110 : }
111 :
112 : // Make the part we care about accessible.
113 631 : base::OS::Unprotect(memory, size);
114 :
115 : reinterpret_cast<v8::Isolate*>(isolate)
116 631 : ->AdjustAmountOfExternalAllocatedMemory(size);
117 :
118 631 : is_external = true;
119 631 : return memory;
120 : } else {
121 11064 : void* memory = isolate->array_buffer_allocator()->Allocate(size);
122 11064 : return memory;
123 : }
124 : }
125 :
126 47279 : void FlushICache(Isolate* isolate, Handle<FixedArray> code_table) {
127 536322 : for (int i = 0; i < code_table->length(); ++i) {
128 220882 : Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i);
129 220882 : Assembler::FlushICache(isolate, code->instruction_start(),
130 441764 : code->instruction_size());
131 : }
132 47279 : }
133 :
134 12185 : Handle<Script> CreateWasmScript(Isolate* isolate,
135 : const ModuleWireBytes& wire_bytes) {
136 : Handle<Script> script =
137 12185 : isolate->factory()->NewScript(isolate->factory()->empty_string());
138 24370 : script->set_context_data(isolate->native_context()->debug_context_id());
139 : script->set_type(Script::TYPE_WASM);
140 :
141 : int hash = StringHasher::HashSequentialString(
142 : reinterpret_cast<const char*>(wire_bytes.start()), wire_bytes.length(),
143 12185 : kZeroHashSeed);
144 :
145 : const int kBufferSize = 32;
146 : char buffer[kBufferSize];
147 12185 : int url_chars = SNPrintF(ArrayVector(buffer), "wasm://wasm/%08x", hash);
148 : DCHECK(url_chars >= 0 && url_chars < kBufferSize);
149 : MaybeHandle<String> url_str = isolate->factory()->NewStringFromOneByte(
150 : Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), url_chars),
151 12185 : TENURED);
152 12185 : script->set_source_url(*url_str.ToHandleChecked());
153 :
154 12185 : int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash);
155 : DCHECK(name_chars >= 0 && name_chars < kBufferSize);
156 : MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte(
157 : Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), name_chars),
158 12185 : TENURED);
159 12185 : script->set_name(*name_str.ToHandleChecked());
160 :
161 12185 : return script;
162 : }
163 :
164 127772 : class JSToWasmWrapperCache {
165 : public:
166 49253 : Handle<Code> CloneOrCompileJSToWasmWrapper(Isolate* isolate,
167 : const wasm::WasmModule* module,
168 : Handle<Code> wasm_code,
169 : uint32_t index) {
170 49253 : const wasm::WasmFunction* func = &module->functions[index];
171 49253 : int cached_idx = sig_map_.Find(func->sig);
172 49253 : if (cached_idx >= 0) {
173 46600 : Handle<Code> code = isolate->factory()->CopyCode(code_cache_[cached_idx]);
174 : // Now patch the call to wasm code.
175 52855 : for (RelocIterator it(*code, RelocInfo::kCodeTargetMask);; it.next()) {
176 : DCHECK(!it.done());
177 : Code* target =
178 52855 : Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
179 84271 : if (target->kind() == Code::WASM_FUNCTION ||
180 31416 : target->kind() == Code::WASM_TO_JS_FUNCTION ||
181 84196 : target->builtin_index() == Builtins::kIllegal ||
182 : target->builtin_index() == Builtins::kWasmCompileLazy) {
183 : it.rinfo()->set_target_address(isolate,
184 23300 : wasm_code->instruction_start());
185 : break;
186 : }
187 29555 : }
188 23300 : return code;
189 : }
190 :
191 : Handle<Code> code =
192 25953 : compiler::CompileJSToWasmWrapper(isolate, module, wasm_code, index);
193 25953 : uint32_t new_cache_idx = sig_map_.FindOrInsert(func->sig);
194 : DCHECK_EQ(code_cache_.size(), new_cache_idx);
195 : USE(new_cache_idx);
196 25953 : code_cache_.push_back(code);
197 25953 : return code;
198 : }
199 :
200 : private:
201 : // sig_map_ maps signatures to an index in code_cache_.
202 : wasm::SignatureMap sig_map_;
203 : std::vector<Handle<Code>> code_cache_;
204 : };
205 :
206 : // Ensure that the code object in <code_table> at offset <func_index> has
207 : // deoptimization data attached. This is needed for lazy compile stubs which are
208 : // called from JS_TO_WASM functions or via exported function tables. The deopt
209 : // data is used to determine which function this lazy compile stub belongs to.
210 56511 : Handle<Code> EnsureExportedLazyDeoptData(Isolate* isolate,
211 : Handle<WasmInstanceObject> instance,
212 : Handle<FixedArray> code_table,
213 : int func_index) {
214 : Handle<Code> code(Code::cast(code_table->get(func_index)), isolate);
215 56511 : if (code->builtin_index() != Builtins::kWasmCompileLazy) {
216 : // No special deopt data needed for compiled functions, and imported
217 : // functions, which map to Illegal at this point (they get compiled at
218 : // instantiation time).
219 : DCHECK(code->kind() == Code::WASM_FUNCTION ||
220 : code->kind() == Code::WASM_TO_JS_FUNCTION ||
221 : code->builtin_index() == Builtins::kIllegal);
222 39052 : return code;
223 : }
224 : // deopt_data:
225 : // #0: weak instance
226 : // #1: func_index
227 : // might be extended later for table exports (see
228 : // EnsureTableExportLazyDeoptData).
229 : Handle<FixedArray> deopt_data(code->deoptimization_data());
230 : DCHECK_EQ(0, deopt_data->length() % 2);
231 17459 : if (deopt_data->length() == 0) {
232 13631 : code = isolate->factory()->CopyCode(code);
233 13631 : code_table->set(func_index, *code);
234 13631 : deopt_data = isolate->factory()->NewFixedArray(2, TENURED);
235 13631 : code->set_deoptimization_data(*deopt_data);
236 13631 : if (!instance.is_null()) {
237 : Handle<WeakCell> weak_instance =
238 5217 : isolate->factory()->NewWeakCell(instance);
239 5217 : deopt_data->set(0, *weak_instance);
240 : }
241 : deopt_data->set(1, Smi::FromInt(func_index));
242 : }
243 : DCHECK_IMPLIES(!instance.is_null(),
244 : WeakCell::cast(code->deoptimization_data()->get(0))->value() ==
245 : *instance);
246 : DCHECK_EQ(func_index,
247 : Smi::cast(code->deoptimization_data()->get(1))->value());
248 17459 : return code;
249 : }
250 :
251 : // Ensure that the code object in <code_table> at offset <func_index> has
252 : // deoptimization data attached. This is needed for lazy compile stubs which are
253 : // called from JS_TO_WASM functions or via exported function tables. The deopt
254 : // data is used to determine which function this lazy compile stub belongs to.
255 8610 : Handle<Code> EnsureTableExportLazyDeoptData(
256 : Isolate* isolate, Handle<WasmInstanceObject> instance,
257 : Handle<FixedArray> code_table, int func_index,
258 : Handle<FixedArray> export_table, int export_index,
259 : std::unordered_map<uint32_t, uint32_t>& table_export_count) {
260 : Handle<Code> code =
261 8610 : EnsureExportedLazyDeoptData(isolate, instance, code_table, func_index);
262 8610 : if (code->builtin_index() != Builtins::kWasmCompileLazy) return code;
263 :
264 : // deopt_data:
265 : // #0: weak instance
266 : // #1: func_index
267 : // [#2: export table
268 : // #3: export table index]
269 : // [#4: export table
270 : // #5: export table index]
271 : // ...
272 : // table_export_count counts down and determines the index for the new export
273 : // table entry.
274 11884 : auto table_export_entry = table_export_count.find(func_index);
275 : DCHECK(table_export_entry != table_export_count.end());
276 : DCHECK_LT(0, table_export_entry->second);
277 5942 : uint32_t this_idx = 2 * table_export_entry->second;
278 5942 : --table_export_entry->second;
279 : Handle<FixedArray> deopt_data(code->deoptimization_data());
280 : DCHECK_EQ(0, deopt_data->length() % 2);
281 5942 : if (deopt_data->length() == 2) {
282 : // Then only the "header" (#0 and #1) exists. Extend for the export table
283 : // entries (make space for this_idx + 2 elements).
284 : deopt_data = isolate->factory()->CopyFixedArrayAndGrow(deopt_data, this_idx,
285 4128 : TENURED);
286 4128 : code->set_deoptimization_data(*deopt_data);
287 : }
288 : DCHECK_LE(this_idx + 2, deopt_data->length());
289 : DCHECK(deopt_data->get(this_idx)->IsUndefined(isolate));
290 : DCHECK(deopt_data->get(this_idx + 1)->IsUndefined(isolate));
291 11884 : deopt_data->set(this_idx, *export_table);
292 5942 : deopt_data->set(this_idx + 1, Smi::FromInt(export_index));
293 5942 : return code;
294 : }
295 :
296 4607 : bool compile_lazy(const WasmModule* module) {
297 16812 : return FLAG_wasm_lazy_compilation ||
298 4607 : (FLAG_asm_wasm_lazy_compilation && module->is_asm_js());
299 : }
300 :
301 : // A helper for compiling an entire module.
302 55658 : class CompilationHelper {
303 : public:
304 27829 : CompilationHelper(Isolate* isolate, WasmModule* module)
305 83487 : : isolate_(isolate), module_(module) {}
306 :
307 : // The actual runnable task that performs compilations in the background.
308 28619 : class CompilationTask : public CancelableTask {
309 : public:
310 : CompilationHelper* helper_;
311 : explicit CompilationTask(CompilationHelper* helper)
312 14322 : : CancelableTask(helper->isolate_), helper_(helper) {}
313 :
314 12375 : void RunInternal() override {
315 12375 : while (helper_->FetchAndExecuteCompilationUnit()) {
316 : }
317 24768 : helper_->module_->pending_tasks.get()->Signal();
318 12372 : }
319 : };
320 :
321 : Isolate* isolate_;
322 : WasmModule* module_;
323 : std::vector<compiler::WasmCompilationUnit*> compilation_units_;
324 : std::queue<compiler::WasmCompilationUnit*> executed_units_;
325 : base::Mutex result_mutex_;
326 : base::AtomicNumber<size_t> next_unit_;
327 : size_t num_background_tasks_ = 0;
328 :
329 : // Run by each compilation task and by the main thread.
330 41103 : bool FetchAndExecuteCompilationUnit() {
331 : DisallowHeapAllocation no_allocation;
332 : DisallowHandleAllocation no_handles;
333 : DisallowHandleDereference no_deref;
334 : DisallowCodeDependencyChange no_dependency_change;
335 :
336 : // - 1 because AtomicIncrement returns the value after the atomic increment.
337 41107 : size_t index = next_unit_.Increment(1) - 1;
338 82214 : if (index >= compilation_units_.size()) {
339 : return false;
340 : }
341 :
342 26468 : compiler::WasmCompilationUnit* unit = compilation_units_.at(index);
343 26468 : unit->ExecuteCompilation();
344 26468 : base::LockGuard<base::Mutex> guard(&result_mutex_);
345 : executed_units_.push(unit);
346 : return true;
347 : }
348 :
349 2256 : size_t InitializeParallelCompilation(
350 28728 : const std::vector<WasmFunction>& functions, ModuleBytesEnv& module_env) {
351 2256 : uint32_t start = module_env.module_env.module->num_imported_functions +
352 2256 : FLAG_skip_compiling_wasm_funcs;
353 2256 : uint32_t num_funcs = static_cast<uint32_t>(functions.size());
354 2256 : uint32_t funcs_to_compile = start > num_funcs ? 0 : num_funcs - start;
355 2256 : compilation_units_.reserve(funcs_to_compile);
356 28728 : for (uint32_t i = start; i < num_funcs; ++i) {
357 26472 : const WasmFunction* func = &functions[i];
358 : compilation_units_.push_back(
359 52944 : new compiler::WasmCompilationUnit(isolate_, &module_env, func));
360 : }
361 2256 : return funcs_to_compile;
362 : }
363 :
364 : void InitializeHandles() {
365 55740 : for (auto unit : compilation_units_) {
366 26472 : unit->InitializeHandles();
367 : }
368 : }
369 :
370 2046 : uint32_t* StartCompilationTasks() {
371 : num_background_tasks_ =
372 : Min(static_cast<size_t>(FLAG_wasm_num_compilation_tasks),
373 4092 : V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads());
374 2046 : uint32_t* task_ids = new uint32_t[num_background_tasks_];
375 16368 : for (size_t i = 0; i < num_background_tasks_; ++i) {
376 14322 : CompilationTask* task = new CompilationTask(this);
377 14322 : task_ids[i] = task->id();
378 14322 : V8::GetCurrentPlatform()->CallOnBackgroundThread(
379 14322 : task, v8::Platform::kShortRunningTask);
380 : }
381 2046 : return task_ids;
382 : }
383 :
384 2046 : void WaitForCompilationTasks(uint32_t* task_ids) {
385 16368 : for (size_t i = 0; i < num_background_tasks_; ++i) {
386 : // If the task has not started yet, then we abort it. Otherwise we wait
387 : // for it to finish.
388 14322 : if (isolate_->cancelable_task_manager()->TryAbort(task_ids[i]) !=
389 : CancelableTaskManager::kTaskAborted) {
390 24772 : module_->pending_tasks.get()->Wait();
391 : }
392 : }
393 2046 : }
394 :
395 29502 : void FinishCompilationUnits(std::vector<Handle<Code>>& results,
396 : ErrorThrower* thrower) {
397 : while (true) {
398 29502 : int func_index = -1;
399 29502 : Handle<Code> result = FinishCompilationUnit(thrower, &func_index);
400 29502 : if (func_index < 0) break;
401 52524 : results[func_index] = result;
402 26262 : }
403 3240 : }
404 :
405 29712 : Handle<Code> FinishCompilationUnit(ErrorThrower* thrower, int* func_index) {
406 26472 : compiler::WasmCompilationUnit* unit = nullptr;
407 : {
408 29712 : base::LockGuard<base::Mutex> guard(&result_mutex_);
409 29712 : if (executed_units_.empty()) return Handle<Code>::null();
410 26472 : unit = executed_units_.front();
411 : executed_units_.pop();
412 : }
413 26472 : *func_index = unit->func_index();
414 26472 : Handle<Code> result = unit->FinishCompilation(thrower);
415 26472 : delete unit;
416 26472 : return result;
417 : }
418 :
419 2046 : void CompileInParallel(ModuleBytesEnv* module_env,
420 : std::vector<Handle<Code>>& results,
421 : ErrorThrower* thrower) {
422 2046 : const WasmModule* module = module_env->module_env.module;
423 : // Data structures for the parallel compilation.
424 :
425 : //-----------------------------------------------------------------------
426 : // For parallel compilation:
427 : // 1) The main thread allocates a compilation unit for each wasm function
428 : // and stores them in the vector {compilation_units}.
429 : // 2) The main thread spawns {CompilationTask} instances which run on
430 : // the background threads.
431 : // 3.a) The background threads and the main thread pick one compilation
432 : // unit at a time and execute the parallel phase of the compilation
433 : // unit. After finishing the execution of the parallel phase, the
434 : // result is enqueued in {executed_units}.
435 : // 3.b) If {executed_units} contains a compilation unit, the main thread
436 : // dequeues it and finishes the compilation.
437 : // 4) After the parallel phase of all compilation units has started, the
438 : // main thread waits for all {CompilationTask} instances to finish.
439 : // 5) The main thread finishes the compilation.
440 :
441 : // Turn on the {CanonicalHandleScope} so that the background threads can
442 : // use the node cache.
443 2046 : CanonicalHandleScope canonical(isolate_);
444 :
445 : // 1) The main thread allocates a compilation unit for each wasm function
446 : // and stores them in the vector {compilation_units}.
447 2046 : InitializeParallelCompilation(module->functions, *module_env);
448 : InitializeHandles();
449 :
450 : // Objects for the synchronization with the background threads.
451 : base::AtomicNumber<size_t> next_unit(
452 : static_cast<size_t>(FLAG_skip_compiling_wasm_funcs));
453 :
454 : // 2) The main thread spawns {CompilationTask} instances which run on
455 : // the background threads.
456 2046 : std::unique_ptr<uint32_t[]> task_ids(StartCompilationTasks());
457 :
458 : // 3.a) The background threads and the main thread pick one compilation
459 : // unit at a time and execute the parallel phase of the compilation
460 : // unit. After finishing the execution of the parallel phase, the
461 : // result is enqueued in {executed_units}.
462 3240 : while (FetchAndExecuteCompilationUnit()) {
463 : // 3.b) If {executed_units} contains a compilation unit, the main thread
464 : // dequeues it and finishes the compilation unit. Compilation units
465 : // are finished concurrently to the background threads to save
466 : // memory.
467 1194 : FinishCompilationUnits(results, thrower);
468 : }
469 : // 4) After the parallel phase of all compilation units has started, the
470 : // main thread waits for all {CompilationTask} instances to finish.
471 2046 : WaitForCompilationTasks(task_ids.get());
472 : // Finish the compilation of the remaining compilation units.
473 4092 : FinishCompilationUnits(results, thrower);
474 2046 : }
475 :
476 9689 : void CompileSequentially(ModuleBytesEnv* module_env,
477 8371 : std::vector<Handle<Code>>& results,
478 : ErrorThrower* thrower) {
479 : DCHECK(!thrower->error());
480 :
481 9689 : const WasmModule* module = module_env->module_env.module;
482 46782 : for (uint32_t i = FLAG_skip_compiling_wasm_funcs;
483 23391 : i < module->functions.size(); ++i) {
484 23554 : const WasmFunction& func = module->functions[i];
485 13865 : if (func.imported)
486 5331 : continue; // Imports are compiled at instantiation time.
487 :
488 : // Compile the function.
489 : Handle<Code> code = compiler::WasmCompilationUnit::CompileWasmFunction(
490 8534 : thrower, isolate_, module_env, &func);
491 8534 : if (code.is_null()) {
492 163 : WasmName str = module_env->wire_bytes.GetName(&func);
493 : thrower->CompileError("Compilation of #%d:%.*s failed.", i,
494 163 : str.length(), str.start());
495 : break;
496 : }
497 8371 : results[i] = code;
498 : }
499 9689 : }
500 :
501 15344 : MaybeHandle<WasmModuleObject> CompileToModuleObject(
502 : ErrorThrower* thrower, const ModuleWireBytes& wire_bytes,
503 : Handle<Script> asm_js_script,
504 : Vector<const byte> asm_js_offset_table_bytes) {
505 57467 : Factory* factory = isolate_->factory();
506 : // The {module_wrapper} will take ownership of the {WasmModule} object,
507 : // and it will be destroyed when the GC reclaims the wrapper object.
508 : Handle<WasmModuleWrapper> module_wrapper =
509 46032 : WasmModuleWrapper::New(isolate_, module_);
510 15344 : WasmInstance temp_instance(module_);
511 15344 : temp_instance.context = isolate_->native_context();
512 15344 : temp_instance.mem_size = WasmModule::kPageSize * module_->min_mem_pages;
513 15344 : temp_instance.mem_start = nullptr;
514 15344 : temp_instance.globals_start = nullptr;
515 :
516 : // Initialize the indirect tables with placeholders.
517 : int function_table_count =
518 30688 : static_cast<int>(module_->function_tables.size());
519 : Handle<FixedArray> function_tables =
520 15344 : factory->NewFixedArray(function_table_count, TENURED);
521 : Handle<FixedArray> signature_tables =
522 15344 : factory->NewFixedArray(function_table_count, TENURED);
523 949 : for (int i = 0; i < function_table_count; ++i) {
524 1898 : temp_instance.function_tables[i] = factory->NewFixedArray(1, TENURED);
525 1898 : temp_instance.signature_tables[i] = factory->NewFixedArray(1, TENURED);
526 1898 : function_tables->set(i, *temp_instance.function_tables[i]);
527 1898 : signature_tables->set(i, *temp_instance.signature_tables[i]);
528 : }
529 :
530 : HistogramTimerScope wasm_compile_module_time_scope(
531 15344 : module_->is_wasm()
532 11735 : ? isolate_->counters()->wasm_compile_wasm_module_time()
533 30688 : : isolate_->counters()->wasm_compile_asm_module_time());
534 :
535 15344 : ModuleBytesEnv module_env(module_, &temp_instance, wire_bytes);
536 :
537 : // The {code_table} array contains import wrappers and functions (which
538 : // are both included in {functions.size()}, and export wrappers.
539 98679 : int code_table_size = static_cast<int>(module_->functions.size() +
540 15344 : module_->num_exported_functions);
541 : Handle<FixedArray> code_table =
542 15344 : factory->NewFixedArray(static_cast<int>(code_table_size), TENURED);
543 :
544 : // Check whether lazy compilation is enabled for this module.
545 15344 : bool lazy_compile = compile_lazy(module_);
546 :
547 : // If lazy compile: Initialize the code table with the lazy compile builtin.
548 : // Otherwise: Initialize with the illegal builtin. All call sites will be
549 : // patched at instantiation.
550 : Handle<Code> init_builtin = lazy_compile
551 3609 : ? isolate_->builtins()->WasmCompileLazy()
552 18953 : : isolate_->builtins()->Illegal();
553 91758 : for (int i = 0, e = static_cast<int>(module_->functions.size()); i < e;
554 : ++i) {
555 61070 : code_table->set(i, *init_builtin);
556 122140 : temp_instance.function_code[i] = init_builtin;
557 : }
558 :
559 11735 : (module_->is_wasm() ? isolate_->counters()->wasm_functions_per_wasm_module()
560 3609 : : isolate_->counters()->wasm_functions_per_asm_module())
561 61376 : ->AddSample(static_cast<int>(module_->functions.size()));
562 :
563 15344 : if (!lazy_compile) {
564 11735 : CompilationHelper helper(isolate_, module_);
565 : size_t funcs_to_compile =
566 23470 : module_->functions.size() - module_->num_imported_functions;
567 11735 : if (!FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks != 0 &&
568 : funcs_to_compile > 1) {
569 : // Avoid a race condition by collecting results into a second vector.
570 2046 : std::vector<Handle<Code>> results(temp_instance.function_code);
571 2046 : helper.CompileInParallel(&module_env, results, thrower);
572 : temp_instance.function_code.swap(results);
573 : } else {
574 : helper.CompileSequentially(&module_env, temp_instance.function_code,
575 9689 : thrower);
576 : }
577 12035 : if (thrower->error()) return {};
578 : }
579 :
580 : // At this point, compilation has completed. Update the code table.
581 150732 : for (size_t i = FLAG_skip_compiling_wasm_funcs;
582 75366 : i < temp_instance.function_code.size(); ++i) {
583 : Code* code = *temp_instance.function_code[i];
584 120644 : code_table->set(static_cast<int>(i), code);
585 60322 : RecordStats(isolate_, code);
586 : }
587 :
588 : // Create heap objects for script, module bytes and asm.js offset table to
589 : // be
590 : // stored in the shared module data.
591 : Handle<Script> script;
592 : Handle<ByteArray> asm_js_offset_table;
593 15044 : if (asm_js_script.is_null()) {
594 11435 : script = CreateWasmScript(isolate_, wire_bytes);
595 : } else {
596 : script = asm_js_script;
597 : asm_js_offset_table =
598 3609 : isolate_->factory()->NewByteArray(asm_js_offset_table_bytes.length());
599 : asm_js_offset_table->copy_in(0, asm_js_offset_table_bytes.start(),
600 3609 : asm_js_offset_table_bytes.length());
601 : }
602 : // TODO(wasm): only save the sections necessary to deserialize a
603 : // {WasmModule}. E.g. function bodies could be omitted.
604 : Handle<String> module_bytes =
605 : factory
606 : ->NewStringFromOneByte({wire_bytes.start(), wire_bytes.length()},
607 : TENURED)
608 30088 : .ToHandleChecked();
609 : DCHECK(module_bytes->IsSeqOneByteString());
610 :
611 : // Create the shared module data.
612 : // TODO(clemensh): For the same module (same bytes / same hash), we should
613 : // only have one WasmSharedModuleData. Otherwise, we might only set
614 : // breakpoints on a (potentially empty) subset of the instances.
615 :
616 : Handle<WasmSharedModuleData> shared = WasmSharedModuleData::New(
617 : isolate_, module_wrapper, Handle<SeqOneByteString>::cast(module_bytes),
618 15044 : script, asm_js_offset_table);
619 15044 : if (lazy_compile) WasmSharedModuleData::PrepareForLazyCompilation(shared);
620 :
621 : // Create the compiled module object, and populate with compiled functions
622 : // and information needed at instantiation time. This object needs to be
623 : // serializable. Instantiation may occur off a deserialized version of this
624 : // object.
625 : Handle<WasmCompiledModule> compiled_module = WasmCompiledModule::New(
626 15044 : isolate_, shared, code_table, function_tables, signature_tables);
627 :
628 : // If we created a wasm script, finish it now and make it public to the
629 : // debugger.
630 15044 : if (asm_js_script.is_null()) {
631 11435 : script->set_wasm_compiled_module(*compiled_module);
632 22870 : isolate_->debug()->OnAfterCompile(script);
633 : }
634 :
635 : // Compile JS->WASM wrappers for exported functions.
636 15044 : JSToWasmWrapperCache js_to_wasm_cache;
637 : int func_index = 0;
638 113930 : for (auto exp : module_->export_table) {
639 42930 : if (exp.kind != kExternalFunction) continue;
640 : Handle<Code> wasm_code = EnsureExportedLazyDeoptData(
641 40912 : isolate_, Handle<WasmInstanceObject>::null(), code_table, exp.index);
642 : Handle<Code> wrapper_code =
643 : js_to_wasm_cache.CloneOrCompileJSToWasmWrapper(isolate_, module_,
644 40912 : wasm_code, exp.index);
645 : int export_index =
646 40912 : static_cast<int>(module_->functions.size() + func_index);
647 40912 : code_table->set(export_index, *wrapper_code);
648 40912 : RecordStats(isolate_, *wrapper_code);
649 40912 : func_index++;
650 : }
651 :
652 30388 : return WasmModuleObject::New(isolate_, compiled_module);
653 : }
654 : };
655 :
656 15949 : static void MemoryInstanceFinalizer(Isolate* isolate,
657 : WasmInstanceObject* instance) {
658 : DisallowHeapAllocation no_gc;
659 : // If the memory object is destroyed, nothing needs to be done here.
660 15949 : if (!instance->has_memory_object()) return;
661 : Handle<WasmInstanceWrapper> instance_wrapper =
662 1137 : handle(instance->instance_wrapper());
663 : DCHECK(WasmInstanceWrapper::IsWasmInstanceWrapper(*instance_wrapper));
664 : DCHECK(instance_wrapper->has_instance());
665 1137 : bool has_prev = instance_wrapper->has_previous();
666 1137 : bool has_next = instance_wrapper->has_next();
667 1137 : Handle<WasmMemoryObject> memory_object(instance->memory_object());
668 :
669 1137 : if (!has_prev && !has_next) {
670 807 : memory_object->ResetInstancesLink(isolate);
671 807 : return;
672 : } else {
673 : Handle<WasmInstanceWrapper> next_wrapper, prev_wrapper;
674 330 : if (!has_prev) {
675 : Handle<WasmInstanceWrapper> next_wrapper =
676 61 : instance_wrapper->next_wrapper();
677 : next_wrapper->reset_previous_wrapper();
678 : // As this is the first link in the memory object, destroying
679 : // without updating memory object would corrupt the instance chain in
680 : // the memory object.
681 61 : memory_object->set_instances_link(*next_wrapper);
682 269 : } else if (!has_next) {
683 414 : instance_wrapper->previous_wrapper()->reset_next_wrapper();
684 : } else {
685 : DCHECK(has_next && has_prev);
686 : Handle<WasmInstanceWrapper> prev_wrapper =
687 62 : instance_wrapper->previous_wrapper();
688 : Handle<WasmInstanceWrapper> next_wrapper =
689 62 : instance_wrapper->next_wrapper();
690 : prev_wrapper->set_next_wrapper(*next_wrapper);
691 : next_wrapper->set_previous_wrapper(*prev_wrapper);
692 : }
693 : // Reset to avoid dangling pointers
694 : instance_wrapper->reset();
695 : }
696 : }
697 :
698 31898 : static void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) {
699 : DisallowHeapAllocation no_gc;
700 : JSObject** p = reinterpret_cast<JSObject**>(data.GetParameter());
701 15949 : WasmInstanceObject* owner = reinterpret_cast<WasmInstanceObject*>(*p);
702 : Isolate* isolate = reinterpret_cast<Isolate*>(data.GetIsolate());
703 : // If a link to shared memory instances exists, update the list of memory
704 : // instances before the instance is destroyed.
705 15949 : if (owner->has_instance_wrapper()) MemoryInstanceFinalizer(isolate, owner);
706 15949 : WasmCompiledModule* compiled_module = owner->compiled_module();
707 : TRACE("Finalizing %d {\n", compiled_module->instance_id());
708 : DCHECK(compiled_module->has_weak_wasm_module());
709 : WeakCell* weak_wasm_module = compiled_module->ptr_to_weak_wasm_module();
710 :
711 15949 : if (trap_handler::UseTrapHandler()) {
712 3096 : Handle<FixedArray> code_table = compiled_module->code_table();
713 35292 : for (int i = 0; i < code_table->length(); ++i) {
714 14550 : Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i);
715 : int index = code->trap_handler_index()->value();
716 14550 : if (index >= 0) {
717 188 : trap_handler::ReleaseHandlerData(index);
718 188 : code->set_trap_handler_index(Smi::FromInt(-1));
719 : }
720 : }
721 : }
722 :
723 : // weak_wasm_module may have been cleared, meaning the module object
724 : // was GC-ed. In that case, there won't be any new instances created,
725 : // and we don't need to maintain the links between instances.
726 15949 : if (!weak_wasm_module->cleared()) {
727 : JSObject* wasm_module = JSObject::cast(weak_wasm_module->value());
728 : WasmCompiledModule* current_template =
729 : WasmCompiledModule::cast(wasm_module->GetEmbedderField(0));
730 :
731 : TRACE("chain before {\n");
732 2786 : TRACE_CHAIN(current_template);
733 : TRACE("}\n");
734 :
735 : DCHECK(!current_template->has_weak_prev_instance());
736 2786 : WeakCell* next = compiled_module->maybe_ptr_to_weak_next_instance();
737 2786 : WeakCell* prev = compiled_module->maybe_ptr_to_weak_prev_instance();
738 :
739 2786 : if (current_template == compiled_module) {
740 2014 : if (next == nullptr) {
741 1665 : WasmCompiledModule::Reset(isolate, compiled_module);
742 : } else {
743 : DCHECK(next->value()->IsFixedArray());
744 349 : wasm_module->SetEmbedderField(0, next->value());
745 : DCHECK_NULL(prev);
746 : WasmCompiledModule::cast(next->value())->reset_weak_prev_instance();
747 : }
748 : } else {
749 : DCHECK(!(prev == nullptr && next == nullptr));
750 : // the only reason prev or next would be cleared is if the
751 : // respective objects got collected, but if that happened,
752 : // we would have relinked the list.
753 772 : if (prev != nullptr) {
754 : DCHECK(!prev->cleared());
755 772 : if (next == nullptr) {
756 : WasmCompiledModule::cast(prev->value())->reset_weak_next_instance();
757 : } else {
758 : WasmCompiledModule::cast(prev->value())
759 : ->set_ptr_to_weak_next_instance(next);
760 : }
761 : }
762 772 : if (next != nullptr) {
763 : DCHECK(!next->cleared());
764 180 : if (prev == nullptr) {
765 : WasmCompiledModule::cast(next->value())->reset_weak_prev_instance();
766 : } else {
767 : WasmCompiledModule::cast(next->value())
768 : ->set_ptr_to_weak_prev_instance(prev);
769 : }
770 : }
771 : }
772 : TRACE("chain after {\n");
773 2786 : TRACE_CHAIN(WasmCompiledModule::cast(wasm_module->GetEmbedderField(0)));
774 : TRACE("}\n");
775 : }
776 : compiled_module->reset_weak_owning_instance();
777 15949 : GlobalHandles::Destroy(reinterpret_cast<Object**>(p));
778 : TRACE("}\n");
779 15949 : }
780 :
781 136345 : int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
782 : int offset) {
783 : DCHECK(!iterator.done());
784 : int byte_pos;
785 55302 : do {
786 : byte_pos = iterator.source_position().ScriptOffset();
787 55302 : iterator.Advance();
788 110604 : } while (!iterator.done() && iterator.code_offset() <= offset);
789 25741 : return byte_pos;
790 : }
791 :
792 25741 : int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) {
793 : DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc));
794 : // Read the leb128 encoded u32 value (up to 5 bytes starting at pc + 1).
795 25741 : decoder.Reset(pc + 1, pc + 6);
796 : uint32_t call_idx = decoder.consume_u32v("call index");
797 : DCHECK(decoder.ok());
798 : DCHECK_GE(kMaxInt, call_idx);
799 25741 : return static_cast<int>(call_idx);
800 : }
801 :
802 37449 : void RecordLazyCodeStats(Isolate* isolate, Code* code) {
803 12483 : isolate->counters()->wasm_lazily_compiled_functions()->Increment();
804 24966 : isolate->counters()->wasm_generated_code_size()->Increment(code->body_size());
805 : isolate->counters()->wasm_reloc_size()->Increment(
806 12483 : code->relocation_info()->length());
807 12483 : }
808 :
809 : } // namespace
810 :
811 12329 : Handle<JSArrayBuffer> wasm::SetupArrayBuffer(Isolate* isolate,
812 : void* backing_store, size_t size,
813 : bool is_external,
814 : bool enable_guard_regions) {
815 11695 : Handle<JSArrayBuffer> buffer = isolate->factory()->NewJSArrayBuffer();
816 : JSArrayBuffer::Setup(buffer, isolate, is_external, backing_store,
817 11695 : static_cast<int>(size));
818 : buffer->set_is_neuterable(false);
819 : buffer->set_has_guard_region(enable_guard_regions);
820 :
821 11695 : if (is_external) {
822 : // We mark the buffer as external if we allocated it here with guard
823 : // pages. That means we need to arrange for it to be freed.
824 :
825 : // TODO(eholk): Finalizers may not run when the main thread is shutting
826 : // down, which means we may leak memory here.
827 : Handle<Object> global_handle = isolate->global_handles()->Create(*buffer);
828 : GlobalHandles::MakeWeak(global_handle.location(), global_handle.location(),
829 634 : &MemoryFinalizer, v8::WeakCallbackType::kFinalizer);
830 : }
831 11695 : return buffer;
832 : }
833 :
834 11650 : Handle<JSArrayBuffer> wasm::NewArrayBuffer(Isolate* isolate, size_t size,
835 : bool enable_guard_regions) {
836 11650 : if (size > (FLAG_wasm_max_mem_pages * WasmModule::kPageSize)) {
837 : // TODO(titzer): lift restriction on maximum memory allocated here.
838 : return Handle<JSArrayBuffer>::null();
839 : }
840 :
841 : enable_guard_regions = enable_guard_regions && kGuardRegionsSupported;
842 :
843 : bool is_external; // Set by TryAllocateBackingStore
844 : void* memory =
845 11650 : TryAllocateBackingStore(isolate, size, enable_guard_regions, is_external);
846 :
847 11650 : if (memory == nullptr) {
848 : return Handle<JSArrayBuffer>::null();
849 : }
850 :
851 : #if DEBUG
852 : // Double check the API allocator actually zero-initialized the memory.
853 : const byte* bytes = reinterpret_cast<const byte*>(memory);
854 : for (size_t i = 0; i < size; ++i) {
855 : DCHECK_EQ(0, bytes[i]);
856 : }
857 : #endif
858 :
859 : return SetupArrayBuffer(isolate, memory, size, is_external,
860 11650 : enable_guard_regions);
861 : }
862 :
863 6186 : void wasm::UnpackAndRegisterProtectedInstructions(
864 7741 : Isolate* isolate, Handle<FixedArray> code_table) {
865 50346 : for (int i = 0; i < code_table->length(); ++i) {
866 : Handle<Code> code;
867 : // This is sometimes undefined when we're called from cctests.
868 37974 : if (!code_table->GetValue<Code>(isolate, i).ToHandle(&code)) {
869 11246 : continue;
870 : }
871 :
872 18078 : if (code->kind() != Code::WASM_FUNCTION) {
873 : continue;
874 : }
875 :
876 7741 : const intptr_t base = reinterpret_cast<intptr_t>(code->entry());
877 :
878 7741 : Zone zone(isolate->allocator(), "Wasm Module");
879 : ZoneVector<trap_handler::ProtectedInstructionData> unpacked(&zone);
880 : const int mode_mask =
881 : RelocInfo::ModeMask(RelocInfo::WASM_PROTECTED_INSTRUCTION_LANDING);
882 9428 : for (RelocIterator it(*code, mode_mask); !it.done(); it.next()) {
883 : trap_handler::ProtectedInstructionData data;
884 1687 : data.instr_offset = it.rinfo()->data();
885 1687 : data.landing_offset = reinterpret_cast<intptr_t>(it.rinfo()->pc()) - base;
886 1687 : unpacked.emplace_back(data);
887 : }
888 15482 : if (unpacked.size() > 0) {
889 : int size = code->CodeSize();
890 : const int index = RegisterHandlerData(reinterpret_cast<void*>(base), size,
891 336 : unpacked.size(), &unpacked[0]);
892 : // TODO(eholk): if index is negative, fail.
893 : DCHECK(index >= 0);
894 336 : code->set_trap_handler_index(Smi::FromInt(index));
895 : }
896 7741 : }
897 6186 : }
898 :
899 466 : std::ostream& wasm::operator<<(std::ostream& os, const WasmFunctionName& name) {
900 466 : os << "#" << name.function_->func_index;
901 466 : if (name.function_->name_offset > 0) {
902 0 : if (name.name_.start()) {
903 0 : os << ":";
904 0 : os.write(name.name_.start(), name.name_.length());
905 : }
906 : } else {
907 466 : os << "?";
908 : }
909 466 : return os;
910 : }
911 :
912 162287 : WasmInstanceObject* wasm::GetOwningWasmInstance(Code* code) {
913 : DisallowHeapAllocation no_gc;
914 : DCHECK(code->kind() == Code::WASM_FUNCTION ||
915 : code->kind() == Code::WASM_INTERPRETER_ENTRY);
916 : FixedArray* deopt_data = code->deoptimization_data();
917 : DCHECK_EQ(code->kind() == Code::WASM_INTERPRETER_ENTRY ? 1 : 2,
918 : deopt_data->length());
919 : Object* weak_link = deopt_data->get(0);
920 : DCHECK(weak_link->IsWeakCell());
921 : WeakCell* cell = WeakCell::cast(weak_link);
922 162287 : if (cell->cleared()) return nullptr;
923 162287 : return WasmInstanceObject::cast(cell->value());
924 : }
925 :
926 59651 : WasmModule::WasmModule(Zone* owned)
927 238604 : : owned_zone(owned), pending_tasks(new base::Semaphore(0)) {}
928 :
929 : namespace {
930 :
931 39906 : WasmFunction* GetWasmFunctionForImportWrapper(Isolate* isolate,
932 : Handle<Object> target) {
933 39906 : if (target->IsJSFunction()) {
934 : Handle<JSFunction> func = Handle<JSFunction>::cast(target);
935 39861 : if (func->code()->kind() == Code::JS_TO_WASM_FUNCTION) {
936 : auto exported = Handle<WasmExportedFunction>::cast(func);
937 3171 : Handle<WasmInstanceObject> other_instance(exported->instance(), isolate);
938 3171 : int func_index = exported->function_index();
939 6342 : return &other_instance->module()->functions[func_index];
940 : }
941 : }
942 : return nullptr;
943 : }
944 :
945 3126 : static Handle<Code> UnwrapImportWrapper(Handle<Object> import_wrapper) {
946 : Handle<JSFunction> func = Handle<JSFunction>::cast(import_wrapper);
947 3126 : Handle<Code> export_wrapper_code = handle(func->code());
948 : int mask = RelocInfo::ModeMask(RelocInfo::CODE_TARGET);
949 3981 : for (RelocIterator it(*export_wrapper_code, mask);; it.next()) {
950 : DCHECK(!it.done());
951 7962 : Code* target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
952 5997 : if (target->kind() != Code::WASM_FUNCTION &&
953 4842 : target->kind() != Code::WASM_TO_JS_FUNCTION &&
954 : target->kind() != Code::WASM_INTERPRETER_ENTRY)
955 : continue;
956 : // There should only be this one call to wasm code.
957 : #ifdef DEBUG
958 : for (it.next(); !it.done(); it.next()) {
959 : Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
960 : DCHECK(code->kind() != Code::WASM_FUNCTION &&
961 : code->kind() != Code::WASM_TO_JS_FUNCTION &&
962 : code->kind() != Code::WASM_INTERPRETER_ENTRY);
963 : }
964 : #endif
965 : return handle(target);
966 855 : }
967 : UNREACHABLE();
968 : return Handle<Code>::null();
969 : }
970 :
971 36831 : Handle<Code> CompileImportWrapper(Isolate* isolate, int index, FunctionSig* sig,
972 : Handle<JSReceiver> target,
973 : Handle<String> module_name,
974 : MaybeHandle<String> import_name,
975 : ModuleOrigin origin) {
976 36831 : WasmFunction* other_func = GetWasmFunctionForImportWrapper(isolate, target);
977 36831 : if (other_func) {
978 96 : if (!sig->Equals(other_func->sig)) return Handle<Code>::null();
979 : // Signature matched. Unwrap the JS->WASM wrapper and return the raw
980 : // WASM function code.
981 51 : return UnwrapImportWrapper(target);
982 : }
983 : // No wasm function or being debugged. Compile a new wrapper for the new
984 : // signature.
985 : return compiler::CompileWasmToJSWrapper(isolate, target, sig, index,
986 36735 : module_name, import_name, origin);
987 : }
988 :
989 4772 : void UpdateDispatchTablesInternal(Isolate* isolate,
990 : Handle<FixedArray> dispatch_tables, int index,
991 : WasmFunction* function, Handle<Code> code) {
992 : DCHECK_EQ(0, dispatch_tables->length() % 4);
993 18934 : for (int i = 0; i < dispatch_tables->length(); i += 4) {
994 4695 : int table_index = Smi::cast(dispatch_tables->get(i + 1))->value();
995 : Handle<FixedArray> function_table(
996 4695 : FixedArray::cast(dispatch_tables->get(i + 2)), isolate);
997 : Handle<FixedArray> signature_table(
998 4695 : FixedArray::cast(dispatch_tables->get(i + 3)), isolate);
999 4695 : if (function) {
1000 : // TODO(titzer): the signature might need to be copied to avoid
1001 : // a dangling pointer in the signature map.
1002 : Handle<WasmInstanceObject> instance(
1003 3045 : WasmInstanceObject::cast(dispatch_tables->get(i)), isolate);
1004 9135 : auto func_table = instance->module()->function_tables[table_index];
1005 3045 : uint32_t sig_index = func_table.map.FindOrInsert(function->sig);
1006 3045 : signature_table->set(index, Smi::FromInt(static_cast<int>(sig_index)));
1007 3045 : function_table->set(index, *code);
1008 : } else {
1009 : signature_table->set(index, Smi::FromInt(-1));
1010 : function_table->set(index, Smi::kZero);
1011 : }
1012 : }
1013 4772 : }
1014 :
1015 : } // namespace
1016 :
1017 3480 : void wasm::UpdateDispatchTables(Isolate* isolate,
1018 : Handle<FixedArray> dispatch_tables, int index,
1019 : Handle<JSFunction> function) {
1020 3480 : if (function.is_null()) {
1021 : UpdateDispatchTablesInternal(isolate, dispatch_tables, index, nullptr,
1022 2250 : Handle<Code>::null());
1023 : } else {
1024 : UpdateDispatchTablesInternal(
1025 : isolate, dispatch_tables, index,
1026 : GetWasmFunctionForImportWrapper(isolate, function),
1027 2460 : UnwrapImportWrapper(function));
1028 : }
1029 3480 : }
1030 :
1031 : // A helper class to simplify instantiating a module from a compiled module.
1032 : // It closes over the {Isolate}, the {ErrorThrower}, the {WasmCompiledModule},
1033 : // etc.
1034 96184 : class InstantiationHelper {
1035 : public:
1036 48092 : InstantiationHelper(Isolate* isolate, ErrorThrower* thrower,
1037 : Handle<WasmModuleObject> module_object,
1038 : MaybeHandle<JSReceiver> ffi,
1039 : MaybeHandle<JSArrayBuffer> memory)
1040 : : isolate_(isolate),
1041 48092 : module_(module_object->compiled_module()->module()),
1042 : thrower_(thrower),
1043 : module_object_(module_object),
1044 : ffi_(ffi.is_null() ? Handle<JSReceiver>::null()
1045 : : ffi.ToHandleChecked()),
1046 : memory_(memory.is_null() ? Handle<JSArrayBuffer>::null()
1047 282075 : : memory.ToHandleChecked()) {}
1048 :
1049 : // Build an instance, in all of its glory.
1050 54592 : MaybeHandle<WasmInstanceObject> Build() {
1051 : // Check that an imports argument was provided, if the module requires it.
1052 : // No point in continuing otherwise.
1053 228814 : if (!module_->import_table.empty() && ffi_.is_null()) {
1054 : thrower_->TypeError(
1055 122 : "Imports argument must be present and must be an object");
1056 : return {};
1057 : }
1058 :
1059 : // Record build time into correct bucket, then build instance.
1060 : HistogramTimerScope wasm_instantiate_module_time_scope(
1061 : module_->is_wasm()
1062 190514 : ? isolate_->counters()->wasm_instantiate_wasm_module_time()
1063 95940 : : isolate_->counters()->wasm_instantiate_asm_module_time());
1064 47970 : Factory* factory = isolate_->factory();
1065 :
1066 : //--------------------------------------------------------------------------
1067 : // Reuse the compiled module (if no owner), otherwise clone.
1068 : //--------------------------------------------------------------------------
1069 : Handle<FixedArray> code_table;
1070 : // We keep around a copy of the old code table, because we'll be replacing
1071 : // imports for the new instance, and then we need the old imports to be
1072 : // able to relocate.
1073 : Handle<FixedArray> old_code_table;
1074 : MaybeHandle<WasmInstanceObject> owner;
1075 :
1076 : TRACE("Starting new module instantiation\n");
1077 : {
1078 : // Root the owner, if any, before doing any allocations, which
1079 : // may trigger GC.
1080 : // Both owner and original template need to be in sync. Even
1081 : // after we lose the original template handle, the code
1082 : // objects we copied from it have data relative to the
1083 : // instance - such as globals addresses.
1084 : Handle<WasmCompiledModule> original;
1085 : {
1086 : DisallowHeapAllocation no_gc;
1087 47970 : original = handle(module_object_->compiled_module());
1088 47970 : if (original->has_weak_owning_instance()) {
1089 : owner = handle(WasmInstanceObject::cast(
1090 96369 : original->weak_owning_instance()->value()));
1091 : }
1092 : }
1093 : DCHECK(!original.is_null());
1094 47970 : if (original->has_weak_owning_instance()) {
1095 : // Clone, but don't insert yet the clone in the instances chain.
1096 : // We do that last. Since we are holding on to the owner instance,
1097 : // the owner + original state used for cloning and patching
1098 : // won't be mutated by possible finalizer runs.
1099 : DCHECK(!owner.is_null());
1100 : TRACE("Cloning from %d\n", original->instance_id());
1101 32123 : old_code_table = original->code_table();
1102 32123 : compiled_module_ = WasmCompiledModule::Clone(isolate_, original);
1103 32123 : code_table = compiled_module_->code_table();
1104 : // Avoid creating too many handles in the outer scope.
1105 32123 : HandleScope scope(isolate_);
1106 :
1107 : // Clone the code for WASM functions and exports.
1108 295642 : for (int i = 0; i < code_table->length(); ++i) {
1109 115698 : Handle<Code> orig_code(Code::cast(code_table->get(i)), isolate_);
1110 115698 : switch (orig_code->kind()) {
1111 : case Code::WASM_TO_JS_FUNCTION:
1112 : // Imports will be overwritten with newly compiled wrappers.
1113 : break;
1114 : case Code::BUILTIN:
1115 : DCHECK_EQ(Builtins::kWasmCompileLazy, orig_code->builtin_index());
1116 : // If this code object has deoptimization data, then we need a
1117 : // unique copy to attach updated deoptimization data.
1118 6811 : if (orig_code->deoptimization_data()->length() > 0) {
1119 3894 : Handle<Code> code = factory->CopyCode(orig_code);
1120 : Handle<FixedArray> deopt_data =
1121 3894 : factory->NewFixedArray(2, TENURED);
1122 : deopt_data->set(1, Smi::FromInt(i));
1123 3894 : code->set_deoptimization_data(*deopt_data);
1124 3894 : code_table->set(i, *code);
1125 : }
1126 : break;
1127 : case Code::JS_TO_WASM_FUNCTION:
1128 : case Code::WASM_FUNCTION: {
1129 78812 : Handle<Code> code = factory->CopyCode(orig_code);
1130 78812 : code_table->set(i, *code);
1131 : break;
1132 : }
1133 : default:
1134 0 : UNREACHABLE();
1135 : }
1136 : }
1137 32123 : RecordStats(isolate_, code_table);
1138 : } else {
1139 : // There was no owner, so we can reuse the original.
1140 15847 : compiled_module_ = original;
1141 : old_code_table =
1142 15847 : factory->CopyFixedArray(compiled_module_->code_table());
1143 15847 : code_table = compiled_module_->code_table();
1144 : TRACE("Reusing existing instance %d\n",
1145 : compiled_module_->instance_id());
1146 : }
1147 47970 : compiled_module_->set_native_context(isolate_->native_context());
1148 : }
1149 :
1150 : //--------------------------------------------------------------------------
1151 : // Allocate the instance object.
1152 : //--------------------------------------------------------------------------
1153 95940 : Zone instantiation_zone(isolate_->allocator(), ZONE_NAME);
1154 95940 : CodeSpecialization code_specialization(isolate_, &instantiation_zone);
1155 : Handle<WasmInstanceObject> instance =
1156 47970 : WasmInstanceObject::New(isolate_, compiled_module_);
1157 :
1158 : //--------------------------------------------------------------------------
1159 : // Set up the globals for the new instance.
1160 : //--------------------------------------------------------------------------
1161 : MaybeHandle<JSArrayBuffer> old_globals;
1162 47970 : uint32_t globals_size = module_->globals_size;
1163 47970 : if (globals_size > 0) {
1164 : const bool enable_guard_regions = false;
1165 : Handle<JSArrayBuffer> global_buffer =
1166 2860 : NewArrayBuffer(isolate_, globals_size, enable_guard_regions);
1167 2860 : globals_ = global_buffer;
1168 2860 : if (globals_.is_null()) {
1169 0 : thrower_->RangeError("Out of memory: wasm globals");
1170 : return {};
1171 : }
1172 2860 : Address old_globals_start = compiled_module_->GetGlobalsStartOrNull();
1173 : Address new_globals_start =
1174 : static_cast<Address>(global_buffer->backing_store());
1175 2860 : code_specialization.RelocateGlobals(old_globals_start, new_globals_start);
1176 : // The address of the backing buffer for the golbals is in native memory
1177 : // and, thus, not moving. We need it saved for
1178 : // serialization/deserialization purposes - so that the other end
1179 : // understands how to relocate the references. We still need to save the
1180 : // JSArrayBuffer on the instance, to keep it all alive.
1181 : WasmCompiledModule::SetGlobalsStartAddressFrom(factory, compiled_module_,
1182 2860 : global_buffer);
1183 2860 : instance->set_globals_buffer(*global_buffer);
1184 : }
1185 :
1186 : //--------------------------------------------------------------------------
1187 : // Prepare for initialization of function tables.
1188 : //--------------------------------------------------------------------------
1189 : int function_table_count =
1190 95940 : static_cast<int>(module_->function_tables.size());
1191 49088 : table_instances_.reserve(module_->function_tables.size());
1192 49603 : for (int index = 0; index < function_table_count; ++index) {
1193 : table_instances_.push_back(
1194 : {Handle<WasmTableObject>::null(), Handle<FixedArray>::null(),
1195 3266 : Handle<FixedArray>::null(), Handle<FixedArray>::null()});
1196 : }
1197 :
1198 : //--------------------------------------------------------------------------
1199 : // Process the imports for the module.
1200 : //--------------------------------------------------------------------------
1201 47970 : int num_imported_functions = ProcessImports(code_table, instance);
1202 47970 : if (num_imported_functions < 0) return {};
1203 :
1204 : //--------------------------------------------------------------------------
1205 : // Process the initialization for the module's globals.
1206 : //--------------------------------------------------------------------------
1207 47295 : InitGlobals();
1208 :
1209 : //--------------------------------------------------------------------------
1210 : // Set up the indirect function tables for the new instance.
1211 : //--------------------------------------------------------------------------
1212 47295 : if (function_table_count > 0)
1213 1468 : InitializeTables(instance, &code_specialization);
1214 :
1215 : //--------------------------------------------------------------------------
1216 : // Set up the memory for the new instance.
1217 : //--------------------------------------------------------------------------
1218 47295 : uint32_t min_mem_pages = module_->min_mem_pages;
1219 40815 : (module_->is_wasm() ? isolate_->counters()->wasm_wasm_min_mem_pages_count()
1220 6480 : : isolate_->counters()->wasm_asm_min_mem_pages_count())
1221 141885 : ->AddSample(min_mem_pages);
1222 :
1223 47295 : if (!memory_.is_null()) {
1224 : // Set externally passed ArrayBuffer non neuterable.
1225 : memory_->set_is_neuterable(false);
1226 :
1227 : DCHECK_IMPLIES(EnableGuardRegions(),
1228 : module_->is_asm_js() || memory_->has_guard_region());
1229 44100 : } else if (min_mem_pages > 0) {
1230 6500 : memory_ = AllocateMemory(min_mem_pages);
1231 6500 : if (memory_.is_null()) return {}; // failed to allocate memory
1232 : }
1233 :
1234 : //--------------------------------------------------------------------------
1235 : // Check that indirect function table segments are within bounds.
1236 : //--------------------------------------------------------------------------
1237 143003 : for (WasmTableInit& table_init : module_->table_inits) {
1238 : DCHECK(table_init.table_index < table_instances_.size());
1239 1118 : uint32_t base = EvalUint32InitExpr(table_init.offset);
1240 : uint32_t table_size =
1241 2236 : table_instances_[table_init.table_index].function_table->length();
1242 2236 : if (!in_bounds(base, static_cast<uint32_t>(table_init.entries.size()),
1243 1118 : table_size)) {
1244 0 : thrower_->LinkError("table initializer is out of bounds");
1245 : return {};
1246 : }
1247 : }
1248 :
1249 : //--------------------------------------------------------------------------
1250 : // Check that memory segments are within bounds.
1251 : //--------------------------------------------------------------------------
1252 94876 : for (WasmDataSegment& seg : module_->data_segments) {
1253 302 : uint32_t base = EvalUint32InitExpr(seg.dest_addr);
1254 : uint32_t mem_size = memory_.is_null()
1255 588 : ? 0 : static_cast<uint32_t>(memory_->byte_length()->Number());
1256 604 : if (!in_bounds(base, seg.source_size, mem_size)) {
1257 16 : thrower_->LinkError("data segment is out of bounds");
1258 : return {};
1259 : }
1260 : }
1261 :
1262 : //--------------------------------------------------------------------------
1263 : // Initialize memory.
1264 : //--------------------------------------------------------------------------
1265 47279 : if (!memory_.is_null()) {
1266 : Address mem_start = static_cast<Address>(memory_->backing_store());
1267 : uint32_t mem_size =
1268 9687 : static_cast<uint32_t>(memory_->byte_length()->Number());
1269 9687 : LoadDataSegments(mem_start, mem_size);
1270 :
1271 9687 : uint32_t old_mem_size = compiled_module_->mem_size();
1272 9687 : Address old_mem_start = compiled_module_->GetEmbeddedMemStartOrNull();
1273 : // We might get instantiated again with the same memory. No patching
1274 : // needed in this case.
1275 9687 : if (old_mem_start != mem_start || old_mem_size != mem_size) {
1276 : code_specialization.RelocateMemoryReferences(
1277 9429 : old_mem_start, old_mem_size, mem_start, mem_size);
1278 : }
1279 : // Just like with globals, we need to keep both the JSArrayBuffer
1280 : // and save the start pointer.
1281 9687 : instance->set_memory_buffer(*memory_);
1282 : WasmCompiledModule::SetSpecializationMemInfoFrom(
1283 9687 : factory, compiled_module_, memory_);
1284 : }
1285 :
1286 : //--------------------------------------------------------------------------
1287 : // Set up the runtime support for the new instance.
1288 : //--------------------------------------------------------------------------
1289 47279 : Handle<WeakCell> weak_link = factory->NewWeakCell(instance);
1290 :
1291 198864 : for (int i = num_imported_functions + FLAG_skip_compiling_wasm_funcs,
1292 102597 : num_functions = static_cast<int>(module_->functions.size());
1293 : i < num_functions; ++i) {
1294 104306 : Handle<Code> code = handle(Code::cast(code_table->get(i)), isolate_);
1295 104306 : if (code->kind() == Code::WASM_FUNCTION) {
1296 78947 : Handle<FixedArray> deopt_data = factory->NewFixedArray(2, TENURED);
1297 78947 : deopt_data->set(0, *weak_link);
1298 : deopt_data->set(1, Smi::FromInt(i));
1299 78947 : code->set_deoptimization_data(*deopt_data);
1300 : continue;
1301 : }
1302 : DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index());
1303 25359 : if (code->deoptimization_data()->length() == 0) continue;
1304 : DCHECK_LE(2, code->deoptimization_data()->length());
1305 : DCHECK_EQ(i, Smi::cast(code->deoptimization_data()->get(1))->value());
1306 11858 : code->deoptimization_data()->set(0, *weak_link);
1307 : }
1308 :
1309 : //--------------------------------------------------------------------------
1310 : // Set up the exports object for the new instance.
1311 : //--------------------------------------------------------------------------
1312 47279 : ProcessExports(code_table, instance, compiled_module_);
1313 :
1314 : //--------------------------------------------------------------------------
1315 : // Add instance to Memory object
1316 : //--------------------------------------------------------------------------
1317 : DCHECK(wasm::IsWasmInstance(*instance));
1318 47279 : if (instance->has_memory_object()) {
1319 3484 : instance->memory_object()->AddInstance(isolate_, instance);
1320 : }
1321 :
1322 : //--------------------------------------------------------------------------
1323 : // Initialize the indirect function tables.
1324 : //--------------------------------------------------------------------------
1325 47279 : if (function_table_count > 0) LoadTableSegments(code_table, instance);
1326 :
1327 : // Patch all code with the relocations registered in code_specialization.
1328 47279 : code_specialization.RelocateDirectCalls(instance);
1329 47279 : code_specialization.ApplyToWholeInstance(*instance, SKIP_ICACHE_FLUSH);
1330 :
1331 47279 : FlushICache(isolate_, code_table);
1332 :
1333 : //--------------------------------------------------------------------------
1334 : // Unpack and notify signal handler of protected instructions.
1335 : //--------------------------------------------------------------------------
1336 47279 : if (trap_handler::UseTrapHandler()) {
1337 3167 : UnpackAndRegisterProtectedInstructions(isolate_, code_table);
1338 : }
1339 :
1340 : //--------------------------------------------------------------------------
1341 : // Set up and link the new instance.
1342 : //--------------------------------------------------------------------------
1343 : {
1344 : Handle<Object> global_handle =
1345 94558 : isolate_->global_handles()->Create(*instance);
1346 47279 : Handle<WeakCell> link_to_clone = factory->NewWeakCell(compiled_module_);
1347 47279 : Handle<WeakCell> link_to_owning_instance = factory->NewWeakCell(instance);
1348 : MaybeHandle<WeakCell> link_to_original;
1349 : MaybeHandle<WasmCompiledModule> original;
1350 47279 : if (!owner.is_null()) {
1351 : // prepare the data needed for publishing in a chain, but don't link
1352 : // just yet, because
1353 : // we want all the publishing to happen free from GC interruptions, and
1354 : // so we do it in
1355 : // one GC-free scope afterwards.
1356 32123 : original = handle(owner.ToHandleChecked()->compiled_module());
1357 32123 : link_to_original = factory->NewWeakCell(original.ToHandleChecked());
1358 : }
1359 : // Publish the new instance to the instances chain.
1360 : {
1361 : DisallowHeapAllocation no_gc;
1362 47279 : if (!link_to_original.is_null()) {
1363 : compiled_module_->set_weak_next_instance(
1364 : link_to_original.ToHandleChecked());
1365 : original.ToHandleChecked()->set_weak_prev_instance(link_to_clone);
1366 : compiled_module_->set_weak_wasm_module(
1367 32123 : original.ToHandleChecked()->weak_wasm_module());
1368 : }
1369 47279 : module_object_->SetEmbedderField(0, *compiled_module_);
1370 : compiled_module_->set_weak_owning_instance(link_to_owning_instance);
1371 : GlobalHandles::MakeWeak(global_handle.location(),
1372 : global_handle.location(), &InstanceFinalizer,
1373 47279 : v8::WeakCallbackType::kFinalizer);
1374 : }
1375 : }
1376 :
1377 : //--------------------------------------------------------------------------
1378 : // Debugging support.
1379 : //--------------------------------------------------------------------------
1380 : // Set all breakpoints that were set on the shared module.
1381 : WasmSharedModuleData::SetBreakpointsOnNewInstance(
1382 47279 : compiled_module_->shared(), instance);
1383 :
1384 47279 : if (FLAG_wasm_interpret_all) {
1385 : Handle<WasmDebugInfo> debug_info =
1386 1050 : WasmInstanceObject::GetOrCreateDebugInfo(instance);
1387 : std::vector<int> func_indexes;
1388 5640 : for (int func_index = num_imported_functions,
1389 2100 : num_wasm_functions = static_cast<int>(module_->functions.size());
1390 2295 : func_index < num_wasm_functions; ++func_index) {
1391 1245 : func_indexes.push_back(func_index);
1392 : }
1393 : WasmDebugInfo::RedirectToInterpreter(
1394 : debug_info, Vector<int>(func_indexes.data(),
1395 3150 : static_cast<int>(func_indexes.size())));
1396 : }
1397 :
1398 : //--------------------------------------------------------------------------
1399 : // Run the start function if one was specified.
1400 : //--------------------------------------------------------------------------
1401 47279 : if (module_->start_function_index >= 0) {
1402 6989 : HandleScope scope(isolate_);
1403 : int start_index = module_->start_function_index;
1404 : Handle<Code> startup_code = EnsureExportedLazyDeoptData(
1405 6989 : isolate_, instance, code_table, start_index);
1406 20967 : FunctionSig* sig = module_->functions[start_index].sig;
1407 : Handle<Code> wrapper_code =
1408 : js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper(
1409 6989 : isolate_, module_, startup_code, start_index);
1410 : Handle<WasmExportedFunction> startup_fct = WasmExportedFunction::New(
1411 : isolate_, instance, MaybeHandle<String>(), start_index,
1412 13978 : static_cast<int>(sig->parameter_count()), wrapper_code);
1413 6989 : RecordStats(isolate_, *startup_code);
1414 : // Call the JS function.
1415 : Handle<Object> undefined = factory->undefined_value();
1416 : MaybeHandle<Object> retval =
1417 6989 : Execution::Call(isolate_, startup_fct, undefined, 0, nullptr);
1418 :
1419 6989 : if (retval.is_null()) {
1420 : DCHECK(isolate_->has_pending_exception());
1421 32 : isolate_->OptionalRescheduleException(false);
1422 : // It's unfortunate that the new instance is already linked in the
1423 : // chain. However, we need to set up everything before executing the
1424 : // start function, such that stack trace information can be generated
1425 : // correctly already in the start function.
1426 : return {};
1427 : }
1428 : }
1429 :
1430 : DCHECK(!isolate_->has_pending_exception());
1431 : TRACE("Finishing instance %d\n", compiled_module_->instance_id());
1432 47247 : TRACE_CHAIN(module_object_->compiled_module());
1433 47970 : return instance;
1434 : }
1435 :
1436 : private:
1437 : // Represents the initialized state of a table.
1438 : struct TableInstance {
1439 : Handle<WasmTableObject> table_object; // WebAssembly.Table instance
1440 : Handle<FixedArray> js_wrappers; // JSFunctions exported
1441 : Handle<FixedArray> function_table; // internal code array
1442 : Handle<FixedArray> signature_table; // internal sig array
1443 : };
1444 :
1445 : Isolate* isolate_;
1446 : WasmModule* const module_;
1447 : ErrorThrower* thrower_;
1448 : Handle<WasmModuleObject> module_object_;
1449 : Handle<JSReceiver> ffi_; // TODO(titzer): Use MaybeHandle
1450 : Handle<JSArrayBuffer> memory_; // TODO(titzer): Use MaybeHandle
1451 : Handle<JSArrayBuffer> globals_;
1452 : Handle<WasmCompiledModule> compiled_module_;
1453 : std::vector<TableInstance> table_instances_;
1454 : std::vector<Handle<JSFunction>> js_wrappers_;
1455 : JSToWasmWrapperCache js_to_wasm_cache_;
1456 :
1457 : // Helper routines to print out errors with imports.
1458 : #define ERROR_THROWER_WITH_MESSAGE(TYPE) \
1459 : void Report##TYPE(const char* error, uint32_t index, \
1460 : Handle<String> module_name, Handle<String> import_name) { \
1461 : thrower_->TYPE("Import #%d module=\"%.*s\" function=\"%.*s\" error: %s", \
1462 : index, module_name->length(), \
1463 : module_name->ToCString().get(), import_name->length(), \
1464 : import_name->ToCString().get(), error); \
1465 : } \
1466 : \
1467 : MaybeHandle<Object> Report##TYPE(const char* error, uint32_t index, \
1468 : Handle<String> module_name) { \
1469 : thrower_->TYPE("Import #%d module=\"%.*s\" error: %s", index, \
1470 : module_name->length(), module_name->ToCString().get(), \
1471 : error); \
1472 : return MaybeHandle<Object>(); \
1473 : }
1474 :
1475 1440 : ERROR_THROWER_WITH_MESSAGE(LinkError)
1476 1275 : ERROR_THROWER_WITH_MESSAGE(TypeError)
1477 :
1478 : // Look up an import value in the {ffi_} object.
1479 40207 : MaybeHandle<Object> LookupImport(uint32_t index, Handle<String> module_name,
1480 : Handle<String> import_name) {
1481 : // We pre-validated in the js-api layer that the ffi object is present, and
1482 : // a JSObject, if the module has imports.
1483 : DCHECK(!ffi_.is_null());
1484 :
1485 : // Look up the module first.
1486 : MaybeHandle<Object> result =
1487 40207 : Object::GetPropertyOrElement(ffi_, module_name);
1488 40207 : if (result.is_null()) {
1489 0 : return ReportTypeError("module not found", index, module_name);
1490 : }
1491 :
1492 : Handle<Object> module = result.ToHandleChecked();
1493 :
1494 : // Look up the value in the module.
1495 40207 : if (!module->IsJSReceiver()) {
1496 : return ReportTypeError("module is not an object or function", index,
1497 255 : module_name);
1498 : }
1499 :
1500 39952 : result = Object::GetPropertyOrElement(module, import_name);
1501 39952 : if (result.is_null()) {
1502 0 : ReportLinkError("import not found", index, module_name, import_name);
1503 : return MaybeHandle<JSFunction>();
1504 : }
1505 :
1506 39952 : return result;
1507 : }
1508 :
1509 2816 : uint32_t EvalUint32InitExpr(const WasmInitExpr& expr) {
1510 2816 : switch (expr.kind) {
1511 : case WasmInitExpr::kI32Const:
1512 1256 : return expr.val.i32_const;
1513 : case WasmInitExpr::kGlobalIndex: {
1514 3120 : uint32_t offset = module_->globals[expr.val.global_index].offset;
1515 3120 : return *reinterpret_cast<uint32_t*>(raw_buffer_ptr(globals_, offset));
1516 : }
1517 : default:
1518 0 : UNREACHABLE();
1519 : return 0;
1520 : }
1521 : }
1522 :
1523 : bool in_bounds(uint32_t offset, uint32_t size, uint32_t upper) {
1524 1420 : return offset + size <= upper && offset + size >= offset;
1525 : }
1526 :
1527 : // Load data segments into the memory.
1528 9687 : void LoadDataSegments(Address mem_addr, size_t mem_size) {
1529 : Handle<SeqOneByteString> module_bytes(compiled_module_->module_bytes(),
1530 19374 : isolate_);
1531 29339 : for (const WasmDataSegment& segment : module_->data_segments) {
1532 278 : uint32_t source_size = segment.source_size;
1533 : // Segments of size == 0 are just nops.
1534 278 : if (source_size == 0) continue;
1535 278 : uint32_t dest_offset = EvalUint32InitExpr(segment.dest_addr);
1536 : DCHECK(in_bounds(dest_offset, source_size,
1537 : static_cast<uint32_t>(mem_size)));
1538 278 : byte* dest = mem_addr + dest_offset;
1539 : const byte* src = reinterpret_cast<const byte*>(
1540 278 : module_bytes->GetCharsAddress() + segment.source_offset);
1541 278 : memcpy(dest, src, source_size);
1542 : }
1543 9687 : }
1544 :
1545 2372 : void WriteGlobalValue(WasmGlobal& global, Handle<Object> value) {
1546 : double num = value->Number();
1547 : TRACE("init [globals+%u] = %lf, type = %s\n", global.offset, num,
1548 : WasmOpcodes::TypeName(global.type));
1549 1186 : switch (global.type) {
1550 : case kWasmI32:
1551 1003 : *GetRawGlobalPtr<int32_t>(global) = static_cast<int32_t>(num);
1552 1003 : break;
1553 : case kWasmI64:
1554 : // TODO(titzer): initialization of imported i64 globals.
1555 0 : UNREACHABLE();
1556 : break;
1557 : case kWasmF32:
1558 45 : *GetRawGlobalPtr<float>(global) = static_cast<float>(num);
1559 45 : break;
1560 : case kWasmF64:
1561 138 : *GetRawGlobalPtr<double>(global) = static_cast<double>(num);
1562 138 : break;
1563 : default:
1564 0 : UNREACHABLE();
1565 : }
1566 1186 : }
1567 :
1568 : // Process the imports, including functions, tables, globals, and memory, in
1569 : // order, loading them from the {ffi_} object. Returns the number of imported
1570 : // functions.
1571 47970 : int ProcessImports(Handle<FixedArray> code_table,
1572 : Handle<WasmInstanceObject> instance) {
1573 : int num_imported_functions = 0;
1574 : int num_imported_tables = 0;
1575 262506 : for (int index = 0; index < static_cast<int>(module_->import_table.size());
1576 : ++index) {
1577 165771 : WasmImport& import = module_->import_table[index];
1578 :
1579 : Handle<String> module_name;
1580 : MaybeHandle<String> maybe_module_name =
1581 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1582 : isolate_, compiled_module_, import.module_name_offset,
1583 40207 : import.module_name_length);
1584 40207 : if (!maybe_module_name.ToHandle(&module_name)) return -1;
1585 :
1586 : Handle<String> import_name;
1587 : MaybeHandle<String> maybe_import_name =
1588 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1589 : isolate_, compiled_module_, import.field_name_offset,
1590 40207 : import.field_name_length);
1591 40207 : if (!maybe_import_name.ToHandle(&import_name)) return -1;
1592 :
1593 : MaybeHandle<Object> result =
1594 40207 : LookupImport(index, module_name, import_name);
1595 80414 : if (thrower_->error()) return -1;
1596 : Handle<Object> value = result.ToHandleChecked();
1597 :
1598 39952 : switch (import.kind) {
1599 : case kExternalFunction: {
1600 : // Function imports must be callable.
1601 36921 : if (!value->IsCallable()) {
1602 : ReportLinkError("function import requires a callable", index,
1603 90 : module_name, import_name);
1604 90 : return -1;
1605 : }
1606 :
1607 : Handle<Code> import_wrapper = CompileImportWrapper(
1608 36831 : isolate_, index, module_->functions[import.index].sig,
1609 : Handle<JSReceiver>::cast(value), module_name, import_name,
1610 110493 : module_->get_origin());
1611 36831 : if (import_wrapper.is_null()) {
1612 : ReportLinkError(
1613 : "imported function does not match the expected type", index,
1614 45 : module_name, import_name);
1615 45 : return -1;
1616 : }
1617 36786 : code_table->set(num_imported_functions, *import_wrapper);
1618 36786 : RecordStats(isolate_, *import_wrapper);
1619 36786 : num_imported_functions++;
1620 : break;
1621 : }
1622 : case kExternalTable: {
1623 765 : if (!WasmJs::IsWasmTableObject(isolate_, value)) {
1624 : ReportLinkError("table import requires a WebAssembly.Table", index,
1625 0 : module_name, import_name);
1626 0 : return -1;
1627 : }
1628 : WasmIndirectFunctionTable& table =
1629 765 : module_->function_tables[num_imported_tables];
1630 765 : TableInstance& table_instance = table_instances_[num_imported_tables];
1631 765 : table_instance.table_object = Handle<WasmTableObject>::cast(value);
1632 : table_instance.js_wrappers = Handle<FixedArray>(
1633 2295 : table_instance.table_object->functions(), isolate_);
1634 :
1635 : int imported_cur_size = table_instance.js_wrappers->length();
1636 765 : if (imported_cur_size < static_cast<int>(table.min_size)) {
1637 : thrower_->LinkError(
1638 : "table import %d is smaller than minimum %d, got %u", index,
1639 45 : table.min_size, imported_cur_size);
1640 45 : return -1;
1641 : }
1642 :
1643 720 : if (table.has_max) {
1644 : int64_t imported_max_size =
1645 705 : table_instance.table_object->maximum_length();
1646 705 : if (imported_max_size < 0) {
1647 : thrower_->LinkError(
1648 : "table import %d has no maximum length, expected %d", index,
1649 15 : table.max_size);
1650 15 : return -1;
1651 : }
1652 690 : if (imported_max_size > table.max_size) {
1653 : thrower_->LinkError(
1654 : "table import %d has maximum larger than maximum %d, "
1655 : "got %" PRIx64,
1656 75 : index, table.max_size, imported_max_size);
1657 75 : return -1;
1658 : }
1659 : }
1660 :
1661 : // Allocate a new dispatch table and signature table.
1662 : int table_size = imported_cur_size;
1663 : table_instance.function_table =
1664 630 : isolate_->factory()->NewFixedArray(table_size);
1665 : table_instance.signature_table =
1666 630 : isolate_->factory()->NewFixedArray(table_size);
1667 5970 : for (int i = 0; i < table_size; ++i) {
1668 : table_instance.signature_table->set(i,
1669 : Smi::FromInt(kInvalidSigIndex));
1670 : }
1671 : // Initialize the dispatch table with the (foreign) JS functions
1672 : // that are already in the table.
1673 5340 : for (int i = 0; i < table_size; ++i) {
1674 5340 : Handle<Object> val(table_instance.js_wrappers->get(i), isolate_);
1675 5340 : if (!val->IsJSFunction()) continue;
1676 : WasmFunction* function =
1677 1845 : GetWasmFunctionForImportWrapper(isolate_, val);
1678 1845 : if (function == nullptr) {
1679 : thrower_->LinkError("table import %d[%d] is not a WASM function",
1680 0 : index, i);
1681 : return -1;
1682 : }
1683 1845 : int sig_index = table.map.FindOrInsert(function->sig);
1684 : table_instance.signature_table->set(i, Smi::FromInt(sig_index));
1685 3690 : table_instance.function_table->set(i, *UnwrapImportWrapper(val));
1686 : }
1687 :
1688 630 : num_imported_tables++;
1689 630 : break;
1690 : }
1691 : case kExternalMemory: {
1692 : // Validation should have failed if more than one memory object was
1693 : // provided.
1694 : DCHECK(!instance->has_memory_object());
1695 1035 : if (!WasmJs::IsWasmMemoryObject(isolate_, value)) {
1696 : ReportLinkError("memory import must be a WebAssembly.Memory object",
1697 60 : index, module_name, import_name);
1698 60 : return -1;
1699 : }
1700 : auto memory = Handle<WasmMemoryObject>::cast(value);
1701 : DCHECK(WasmJs::IsWasmMemoryObject(isolate_, memory));
1702 975 : instance->set_memory_object(*memory);
1703 2925 : memory_ = Handle<JSArrayBuffer>(memory->buffer(), isolate_);
1704 : uint32_t imported_cur_pages = static_cast<uint32_t>(
1705 975 : memory_->byte_length()->Number() / WasmModule::kPageSize);
1706 975 : if (imported_cur_pages < module_->min_mem_pages) {
1707 : thrower_->LinkError(
1708 : "memory import %d is smaller than maximum %u, got %u", index,
1709 0 : module_->min_mem_pages, imported_cur_pages);
1710 : }
1711 975 : int32_t imported_max_pages = memory->maximum_pages();
1712 975 : if (module_->has_max_mem) {
1713 60 : if (imported_max_pages < 0) {
1714 : thrower_->LinkError(
1715 : "memory import %d has no maximum limit, expected at most %u",
1716 15 : index, imported_max_pages);
1717 15 : return -1;
1718 : }
1719 45 : if (static_cast<uint32_t>(imported_max_pages) >
1720 : module_->max_mem_pages) {
1721 : thrower_->LinkError(
1722 : "memory import %d has larger maximum than maximum %u, got %d",
1723 30 : index, module_->max_mem_pages, imported_max_pages);
1724 30 : return -1;
1725 : }
1726 : }
1727 : break;
1728 : }
1729 : case kExternalGlobal: {
1730 : // Global imports are converted to numbers and written into the
1731 : // {globals_} array buffer.
1732 3648 : if (module_->globals[import.index].type == kWasmI64) {
1733 : ReportLinkError("global import cannot have type i64", index,
1734 0 : module_name, import_name);
1735 0 : return -1;
1736 : }
1737 1231 : if (module_->is_asm_js() && FLAG_fast_validate_asm) {
1738 256 : if (module_->globals[import.index].type == kWasmI32) {
1739 326 : value = Object::ToInt32(isolate_, value).ToHandleChecked();
1740 : } else {
1741 186 : value = Object::ToNumber(value).ToHandleChecked();
1742 : }
1743 : }
1744 1231 : if (!value->IsNumber()) {
1745 : ReportLinkError("global import must be a number", index,
1746 45 : module_name, import_name);
1747 45 : return -1;
1748 : }
1749 2372 : WriteGlobalValue(module_->globals[import.index], value);
1750 1186 : break;
1751 : }
1752 : default:
1753 0 : UNREACHABLE();
1754 : break;
1755 : }
1756 : }
1757 : return num_imported_functions;
1758 : }
1759 :
1760 : template <typename T>
1761 : T* GetRawGlobalPtr(WasmGlobal& global) {
1762 21238 : return reinterpret_cast<T*>(raw_buffer_ptr(globals_, global.offset));
1763 : }
1764 :
1765 : // Process initialization of globals.
1766 47295 : void InitGlobals() {
1767 152498 : for (auto global : module_->globals) {
1768 10613 : switch (global.init.kind) {
1769 : case WasmInitExpr::kI32Const:
1770 5560 : *GetRawGlobalPtr<int32_t>(global) = global.init.val.i32_const;
1771 5560 : break;
1772 : case WasmInitExpr::kI64Const:
1773 32 : *GetRawGlobalPtr<int64_t>(global) = global.init.val.i64_const;
1774 32 : break;
1775 : case WasmInitExpr::kF32Const:
1776 2748 : *GetRawGlobalPtr<float>(global) = global.init.val.f32_const;
1777 2748 : break;
1778 : case WasmInitExpr::kF64Const:
1779 997 : *GetRawGlobalPtr<double>(global) = global.init.val.f64_const;
1780 997 : break;
1781 : case WasmInitExpr::kGlobalIndex: {
1782 : // Initialize with another global.
1783 : uint32_t new_offset = global.offset;
1784 : uint32_t old_offset =
1785 180 : module_->globals[global.init.val.global_index].offset;
1786 : TRACE("init [globals+%u] = [globals+%d]\n", global.offset,
1787 : old_offset);
1788 90 : size_t size = (global.type == kWasmI64 || global.type == kWasmF64)
1789 : ? sizeof(double)
1790 90 : : sizeof(int32_t);
1791 270 : memcpy(raw_buffer_ptr(globals_, new_offset),
1792 270 : raw_buffer_ptr(globals_, old_offset), size);
1793 90 : break;
1794 : }
1795 : case WasmInitExpr::kNone:
1796 : // Happens with imported globals.
1797 : break;
1798 : default:
1799 0 : UNREACHABLE();
1800 : break;
1801 : }
1802 : }
1803 47295 : }
1804 :
1805 : // Allocate memory for a module instance as a new JSArrayBuffer.
1806 6500 : Handle<JSArrayBuffer> AllocateMemory(uint32_t min_mem_pages) {
1807 6500 : if (min_mem_pages > FLAG_wasm_max_mem_pages) {
1808 0 : thrower_->RangeError("Out of memory: wasm memory too large");
1809 : return Handle<JSArrayBuffer>::null();
1810 : }
1811 : const bool enable_guard_regions = EnableGuardRegions();
1812 : Handle<JSArrayBuffer> mem_buffer = NewArrayBuffer(
1813 6500 : isolate_, min_mem_pages * WasmModule::kPageSize, enable_guard_regions);
1814 :
1815 6500 : if (mem_buffer.is_null()) {
1816 0 : thrower_->RangeError("Out of memory: wasm memory");
1817 : }
1818 6500 : return mem_buffer;
1819 : }
1820 :
1821 47279 : bool NeedsWrappers() {
1822 47279 : if (module_->num_exported_functions > 0) return true;
1823 2107 : for (auto table_instance : table_instances_) {
1824 333 : if (!table_instance.js_wrappers.is_null()) return true;
1825 : }
1826 1654 : for (auto table : module_->function_tables) {
1827 213 : if (table.exported) return true;
1828 0 : }
1829 614 : return false;
1830 : }
1831 :
1832 : // Process the exports, creating wrappers for functions, tables, memories,
1833 : // and globals.
1834 47279 : void ProcessExports(Handle<FixedArray> code_table,
1835 : Handle<WasmInstanceObject> instance,
1836 : Handle<WasmCompiledModule> compiled_module) {
1837 47279 : if (NeedsWrappers()) {
1838 : // Fill the table to cache the exported JSFunction wrappers.
1839 : js_wrappers_.insert(js_wrappers_.begin(), module_->functions.size(),
1840 842932 : Handle<JSFunction>::null());
1841 : }
1842 :
1843 : Handle<JSObject> exports_object;
1844 94558 : if (module_->is_wasm()) {
1845 : // Create the "exports" object.
1846 40799 : exports_object = isolate_->factory()->NewJSObjectWithNullProto();
1847 6480 : } else if (module_->is_asm_js()) {
1848 : Handle<JSFunction> object_function = Handle<JSFunction>(
1849 12960 : isolate_->native_context()->object_function(), isolate_);
1850 6480 : exports_object = isolate_->factory()->NewJSObject(object_function);
1851 : } else {
1852 0 : UNREACHABLE();
1853 : }
1854 : Handle<String> exports_name =
1855 47279 : isolate_->factory()->InternalizeUtf8String("exports");
1856 47279 : JSObject::AddProperty(instance, exports_name, exports_object, NONE);
1857 :
1858 : Handle<String> foreign_init_name =
1859 : isolate_->factory()->InternalizeUtf8String(
1860 47279 : wasm::AsmWasmBuilder::foreign_init_name);
1861 : Handle<String> single_function_name =
1862 : isolate_->factory()->InternalizeUtf8String(
1863 47279 : wasm::AsmWasmBuilder::single_function_name);
1864 :
1865 : PropertyDescriptor desc;
1866 47279 : desc.set_writable(module_->is_asm_js());
1867 : desc.set_enumerable(true);
1868 : desc.set_configurable(module_->is_asm_js());
1869 :
1870 : // Count up export indexes.
1871 : int export_index = 0;
1872 175822 : for (auto exp : module_->export_table) {
1873 81264 : if (exp.kind == kExternalFunction) {
1874 79790 : ++export_index;
1875 : }
1876 : }
1877 :
1878 : // Store weak references to all exported functions.
1879 : Handle<FixedArray> weak_exported_functions;
1880 47279 : if (compiled_module->has_weak_exported_functions()) {
1881 1246 : weak_exported_functions = compiled_module->weak_exported_functions();
1882 : } else {
1883 : weak_exported_functions =
1884 46033 : isolate_->factory()->NewFixedArray(export_index);
1885 : compiled_module->set_weak_exported_functions(weak_exported_functions);
1886 : }
1887 : DCHECK_EQ(export_index, weak_exported_functions->length());
1888 :
1889 : // Process each export in the export table (go in reverse so asm.js
1890 : // can skip duplicates).
1891 257086 : for (auto exp : base::Reversed(module_->export_table)) {
1892 : Handle<String> name =
1893 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1894 : isolate_, compiled_module_, exp.name_offset, exp.name_length)
1895 162528 : .ToHandleChecked();
1896 : Handle<JSObject> export_to;
1897 261276 : if (module_->is_asm_js() && exp.kind == kExternalFunction &&
1898 29178 : (String::Equals(name, foreign_init_name) ||
1899 11694 : String::Equals(name, single_function_name))) {
1900 : export_to = instance;
1901 : } else {
1902 : export_to = exports_object;
1903 : }
1904 :
1905 81264 : switch (exp.kind) {
1906 : case kExternalFunction: {
1907 : // Wrap and export the code as a JSFunction.
1908 79790 : WasmFunction& function = module_->functions[exp.index];
1909 : int func_index =
1910 79790 : static_cast<int>(module_->functions.size() + --export_index);
1911 79790 : Handle<JSFunction> js_function = js_wrappers_[exp.index];
1912 79790 : if (js_function.is_null()) {
1913 : // Wrap the exported code as a JSFunction.
1914 : Handle<Code> export_code =
1915 159202 : code_table->GetValueChecked<Code>(isolate_, func_index);
1916 : MaybeHandle<String> func_name;
1917 159202 : if (module_->is_asm_js()) {
1918 : // For modules arising from asm.js, honor the names section.
1919 : func_name = WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
1920 : isolate_, compiled_module_, function.name_offset,
1921 : function.name_length)
1922 34938 : .ToHandleChecked();
1923 : }
1924 : js_function = WasmExportedFunction::New(
1925 : isolate_, instance, func_name, function.func_index,
1926 79601 : static_cast<int>(function.sig->parameter_count()), export_code);
1927 79601 : js_wrappers_[exp.index] = js_function;
1928 : }
1929 : desc.set_value(js_function);
1930 : Handle<WeakCell> weak_export =
1931 79790 : isolate_->factory()->NewWeakCell(js_function);
1932 : DCHECK_GT(weak_exported_functions->length(), export_index);
1933 79790 : weak_exported_functions->set(export_index, *weak_export);
1934 : break;
1935 : }
1936 : case kExternalTable: {
1937 : // Export a table as a WebAssembly.Table object.
1938 337 : TableInstance& table_instance = table_instances_[exp.index];
1939 : WasmIndirectFunctionTable& table =
1940 337 : module_->function_tables[exp.index];
1941 337 : if (table_instance.table_object.is_null()) {
1942 : uint32_t maximum =
1943 337 : table.has_max ? table.max_size : FLAG_wasm_max_table_size;
1944 : table_instance.table_object = WasmTableObject::New(
1945 337 : isolate_, table.min_size, maximum, &table_instance.js_wrappers);
1946 : }
1947 : desc.set_value(table_instance.table_object);
1948 337 : break;
1949 : }
1950 : case kExternalMemory: {
1951 : // Export the memory as a WebAssembly.Memory object.
1952 : Handle<WasmMemoryObject> memory_object;
1953 1025 : if (!instance->has_memory_object()) {
1954 : // If there was no imported WebAssembly.Memory object, create one.
1955 : memory_object = WasmMemoryObject::New(
1956 : isolate_,
1957 812 : (instance->has_memory_buffer())
1958 797 : ? handle(instance->memory_buffer())
1959 : : Handle<JSArrayBuffer>::null(),
1960 2436 : (module_->max_mem_pages != 0) ? module_->max_mem_pages : -1);
1961 812 : instance->set_memory_object(*memory_object);
1962 : } else {
1963 : memory_object =
1964 426 : Handle<WasmMemoryObject>(instance->memory_object(), isolate_);
1965 : DCHECK(WasmJs::IsWasmMemoryObject(isolate_, memory_object));
1966 426 : memory_object->ResetInstancesLink(isolate_);
1967 : }
1968 :
1969 : desc.set_value(memory_object);
1970 : break;
1971 : }
1972 : case kExternalGlobal: {
1973 : // Export the value of the global variable as a number.
1974 208 : WasmGlobal& global = module_->globals[exp.index];
1975 : double num = 0;
1976 112 : switch (global.type) {
1977 : case kWasmI32:
1978 34 : num = *GetRawGlobalPtr<int32_t>(global);
1979 34 : break;
1980 : case kWasmF32:
1981 31 : num = *GetRawGlobalPtr<float>(global);
1982 31 : break;
1983 : case kWasmF64:
1984 31 : num = *GetRawGlobalPtr<double>(global);
1985 31 : break;
1986 : case kWasmI64:
1987 : thrower_->LinkError(
1988 16 : "export of globals of type I64 is not allowed.");
1989 16 : break;
1990 : default:
1991 0 : UNREACHABLE();
1992 : }
1993 112 : desc.set_value(isolate_->factory()->NewNumber(num));
1994 : break;
1995 : }
1996 : default:
1997 0 : UNREACHABLE();
1998 : break;
1999 : }
2000 :
2001 : // Skip duplicates for asm.js.
2002 162528 : if (module_->is_asm_js()) {
2003 17484 : v8::Maybe<bool> status = JSReceiver::HasOwnProperty(export_to, name);
2004 17484 : if (status.FromMaybe(false)) {
2005 7 : continue;
2006 : }
2007 : }
2008 : v8::Maybe<bool> status = JSReceiver::DefineOwnProperty(
2009 81257 : isolate_, export_to, name, &desc, Object::THROW_ON_ERROR);
2010 81257 : if (!status.IsJust()) {
2011 : thrower_->LinkError("export of %.*s failed.", name->length(),
2012 0 : name->ToCString().get());
2013 47279 : return;
2014 : }
2015 : }
2016 :
2017 94558 : if (module_->is_wasm()) {
2018 : v8::Maybe<bool> success = JSReceiver::SetIntegrityLevel(
2019 40799 : exports_object, FROZEN, Object::DONT_THROW);
2020 : DCHECK(success.FromMaybe(false));
2021 : USE(success);
2022 : }
2023 : }
2024 :
2025 1468 : void InitializeTables(Handle<WasmInstanceObject> instance,
2026 : CodeSpecialization* code_specialization) {
2027 : int function_table_count =
2028 4404 : static_cast<int>(module_->function_tables.size());
2029 : Handle<FixedArray> new_function_tables =
2030 1468 : isolate_->factory()->NewFixedArray(function_table_count);
2031 : Handle<FixedArray> new_signature_tables =
2032 1468 : isolate_->factory()->NewFixedArray(function_table_count);
2033 2936 : for (int index = 0; index < function_table_count; ++index) {
2034 1468 : WasmIndirectFunctionTable& table = module_->function_tables[index];
2035 1468 : TableInstance& table_instance = table_instances_[index];
2036 1468 : int table_size = static_cast<int>(table.min_size);
2037 :
2038 1468 : if (table_instance.function_table.is_null()) {
2039 : // Create a new dispatch table if necessary.
2040 : table_instance.function_table =
2041 838 : isolate_->factory()->NewFixedArray(table_size);
2042 : table_instance.signature_table =
2043 838 : isolate_->factory()->NewFixedArray(table_size);
2044 11160 : for (int i = 0; i < table_size; ++i) {
2045 : // Fill the table with invalid signature indexes so that
2046 : // uninitialized entries will always fail the signature check.
2047 : table_instance.signature_table->set(i,
2048 : Smi::FromInt(kInvalidSigIndex));
2049 : }
2050 : } else {
2051 : // Table is imported, patch table bounds check
2052 : DCHECK(table_size <= table_instance.function_table->length());
2053 630 : if (table_size < table_instance.function_table->length()) {
2054 : code_specialization->PatchTableSize(
2055 45 : table_size, table_instance.function_table->length());
2056 : }
2057 : }
2058 :
2059 : new_function_tables->set(static_cast<int>(index),
2060 1468 : *table_instance.function_table);
2061 : new_signature_tables->set(static_cast<int>(index),
2062 1468 : *table_instance.signature_table);
2063 : }
2064 :
2065 : FixedArray* old_function_tables =
2066 : compiled_module_->ptr_to_function_tables();
2067 : DCHECK_EQ(old_function_tables->length(), new_function_tables->length());
2068 2936 : for (int i = 0, e = new_function_tables->length(); i < e; ++i) {
2069 : code_specialization->RelocateObject(
2070 : handle(old_function_tables->get(i), isolate_),
2071 4404 : handle(new_function_tables->get(i), isolate_));
2072 : }
2073 : FixedArray* old_signature_tables =
2074 : compiled_module_->ptr_to_signature_tables();
2075 : DCHECK_EQ(old_signature_tables->length(), new_signature_tables->length());
2076 2936 : for (int i = 0, e = new_signature_tables->length(); i < e; ++i) {
2077 : code_specialization->RelocateObject(
2078 : handle(old_signature_tables->get(i), isolate_),
2079 4404 : handle(new_signature_tables->get(i), isolate_));
2080 : }
2081 :
2082 : compiled_module_->set_function_tables(new_function_tables);
2083 : compiled_module_->set_signature_tables(new_signature_tables);
2084 1468 : }
2085 :
2086 1468 : void LoadTableSegments(Handle<FixedArray> code_table,
2087 : Handle<WasmInstanceObject> instance) {
2088 : int function_table_count =
2089 5621 : static_cast<int>(module_->function_tables.size());
2090 2936 : for (int index = 0; index < function_table_count; ++index) {
2091 1468 : WasmIndirectFunctionTable& table = module_->function_tables[index];
2092 1468 : TableInstance& table_instance = table_instances_[index];
2093 :
2094 : Handle<FixedArray> all_dispatch_tables;
2095 1468 : if (!table_instance.table_object.is_null()) {
2096 : // Get the existing dispatch table(s) with the WebAssembly.Table object.
2097 : all_dispatch_tables = WasmTableObject::AddDispatchTable(
2098 : isolate_, table_instance.table_object,
2099 : Handle<WasmInstanceObject>::null(), index,
2100 967 : Handle<FixedArray>::null(), Handle<FixedArray>::null());
2101 : }
2102 :
2103 : // Count the number of table exports for each function (needed for lazy
2104 : // compilation).
2105 1468 : std::unordered_map<uint32_t, uint32_t> num_table_exports;
2106 2936 : if (compile_lazy(module_)) {
2107 450 : for (auto table_init : module_->table_inits) {
2108 6705 : for (uint32_t func_index : table_init.entries) {
2109 : Code* code =
2110 6405 : Code::cast(code_table->get(static_cast<int>(func_index)));
2111 : // Only increase the counter for lazy compile builtins (it's not
2112 : // needed otherwise).
2113 6405 : if (code->is_wasm_code()) continue;
2114 : DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index());
2115 5942 : ++num_table_exports[func_index];
2116 : }
2117 : }
2118 : }
2119 :
2120 : // TODO(titzer): this does redundant work if there are multiple tables,
2121 : // since initializations are not sorted by table index.
2122 5522 : for (auto table_init : module_->table_inits) {
2123 1118 : uint32_t base = EvalUint32InitExpr(table_init.offset);
2124 : DCHECK(in_bounds(base, static_cast<uint32_t>(table_init.entries.size()),
2125 : table_instance.function_table->length()));
2126 10846 : for (int i = 0, e = static_cast<int>(table_init.entries.size()); i < e;
2127 : ++i) {
2128 17220 : uint32_t func_index = table_init.entries[i];
2129 8610 : WasmFunction* function = &module_->functions[func_index];
2130 8610 : int table_index = static_cast<int>(i + base);
2131 9827 : int32_t sig_index = table.map.Find(function->sig);
2132 : DCHECK_GE(sig_index, 0);
2133 : table_instance.signature_table->set(table_index,
2134 : Smi::FromInt(sig_index));
2135 : Handle<Code> wasm_code = EnsureTableExportLazyDeoptData(
2136 : isolate_, instance, code_table, func_index,
2137 8610 : table_instance.function_table, table_index, num_table_exports);
2138 8610 : table_instance.function_table->set(table_index, *wasm_code);
2139 :
2140 8610 : if (!all_dispatch_tables.is_null()) {
2141 5093 : if (js_wrappers_[func_index].is_null()) {
2142 : // No JSFunction entry yet exists for this function. Create one.
2143 : // TODO(titzer): We compile JS->WASM wrappers for functions are
2144 : // not exported but are in an exported table. This should be done
2145 : // at module compile time and cached instead.
2146 :
2147 : Handle<Code> wrapper_code =
2148 : js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper(
2149 1217 : isolate_, module_, wasm_code, func_index);
2150 : MaybeHandle<String> func_name;
2151 2434 : if (module_->is_asm_js()) {
2152 : // For modules arising from asm.js, honor the names section.
2153 : func_name =
2154 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2155 : isolate_, compiled_module_, function->name_offset,
2156 : function->name_length)
2157 0 : .ToHandleChecked();
2158 : }
2159 : Handle<WasmExportedFunction> js_function =
2160 : WasmExportedFunction::New(
2161 : isolate_, instance, func_name, func_index,
2162 1217 : static_cast<int>(function->sig->parameter_count()),
2163 1217 : wrapper_code);
2164 1217 : js_wrappers_[func_index] = js_function;
2165 : }
2166 : table_instance.js_wrappers->set(table_index,
2167 1292 : *js_wrappers_[func_index]);
2168 :
2169 : UpdateDispatchTablesInternal(isolate_, all_dispatch_tables,
2170 1292 : table_index, function, wasm_code);
2171 : }
2172 : }
2173 : }
2174 :
2175 : #ifdef DEBUG
2176 : // Check that the count of table exports was accurate. The entries are
2177 : // decremented on each export, so all should be zero now.
2178 : for (auto e : num_table_exports) {
2179 : DCHECK_EQ(0, e.second);
2180 : }
2181 : #endif
2182 :
2183 : // TODO(titzer): we add the new dispatch table at the end to avoid
2184 : // redundant work and also because the new instance is not yet fully
2185 : // initialized.
2186 1468 : if (!table_instance.table_object.is_null()) {
2187 : // Add the new dispatch table to the WebAssembly.Table object.
2188 : all_dispatch_tables = WasmTableObject::AddDispatchTable(
2189 : isolate_, table_instance.table_object, instance, index,
2190 967 : table_instance.function_table, table_instance.signature_table);
2191 : }
2192 : }
2193 1468 : }
2194 : };
2195 :
2196 0 : bool wasm::IsWasmInstance(Object* object) {
2197 60 : return WasmInstanceObject::IsWasmInstanceObject(object);
2198 : }
2199 :
2200 1806 : Handle<Script> wasm::GetScript(Handle<JSObject> instance) {
2201 : WasmCompiledModule* compiled_module =
2202 1806 : WasmInstanceObject::cast(*instance)->compiled_module();
2203 3612 : return handle(compiled_module->script());
2204 : }
2205 :
2206 12373 : bool wasm::IsWasmCodegenAllowed(Isolate* isolate, Handle<Context> context) {
2207 12394 : return isolate->allow_code_gen_callback() == nullptr ||
2208 195 : isolate->allow_code_gen_callback()(v8::Utils::ToLocal(context));
2209 : }
2210 :
2211 756 : void wasm::DetachWebAssemblyMemoryBuffer(Isolate* isolate,
2212 : Handle<JSArrayBuffer> buffer) {
2213 : int64_t byte_length =
2214 : buffer->byte_length()->IsNumber()
2215 : ? static_cast<uint32_t>(buffer->byte_length()->Number())
2216 840 : : 0;
2217 840 : if (buffer.is_null() || byte_length == 0) return;
2218 : const bool has_guard_regions = buffer->has_guard_region();
2219 : const bool is_external = buffer->is_external();
2220 : void* backing_store = buffer->backing_store();
2221 : DCHECK(!buffer->is_neuterable());
2222 360 : if (!has_guard_regions && !is_external) {
2223 : buffer->set_is_external(true);
2224 336 : isolate->heap()->UnregisterArrayBuffer(*buffer);
2225 : }
2226 : buffer->set_is_neuterable(true);
2227 360 : buffer->Neuter();
2228 360 : if (has_guard_regions) {
2229 : base::OS::Free(backing_store, RoundUp(i::wasm::kWasmMaxHeapOffset,
2230 48 : base::OS::CommitPageSize()));
2231 : reinterpret_cast<v8::Isolate*>(isolate)
2232 24 : ->AdjustAmountOfExternalAllocatedMemory(-byte_length);
2233 336 : } else if (!has_guard_regions && !is_external) {
2234 336 : isolate->array_buffer_allocator()->Free(backing_store, byte_length);
2235 : }
2236 : }
2237 :
2238 255 : void wasm::GrowDispatchTables(Isolate* isolate,
2239 : Handle<FixedArray> dispatch_tables,
2240 : uint32_t old_size, uint32_t count) {
2241 : DCHECK_EQ(0, dispatch_tables->length() % 4);
2242 :
2243 255 : Zone specialization_zone(isolate->allocator(), ZONE_NAME);
2244 1020 : for (int i = 0; i < dispatch_tables->length(); i += 4) {
2245 : Handle<FixedArray> old_function_table(
2246 255 : FixedArray::cast(dispatch_tables->get(i + 2)));
2247 : Handle<FixedArray> old_signature_table(
2248 255 : FixedArray::cast(dispatch_tables->get(i + 3)));
2249 : Handle<FixedArray> new_function_table =
2250 255 : isolate->factory()->CopyFixedArrayAndGrow(old_function_table, count);
2251 : Handle<FixedArray> new_signature_table =
2252 255 : isolate->factory()->CopyFixedArrayAndGrow(old_signature_table, count);
2253 :
2254 : // Update dispatch tables with new function/signature tables
2255 255 : dispatch_tables->set(i + 2, *new_function_table);
2256 255 : dispatch_tables->set(i + 3, *new_signature_table);
2257 :
2258 : // Patch the code of the respective instance.
2259 255 : CodeSpecialization code_specialization(isolate, &specialization_zone);
2260 255 : code_specialization.PatchTableSize(old_size, old_size + count);
2261 255 : code_specialization.RelocateObject(old_function_table, new_function_table);
2262 : code_specialization.RelocateObject(old_signature_table,
2263 255 : new_signature_table);
2264 : code_specialization.ApplyToWholeInstance(
2265 255 : WasmInstanceObject::cast(dispatch_tables->get(i)));
2266 510 : }
2267 255 : }
2268 :
2269 90 : void testing::ValidateInstancesChain(Isolate* isolate,
2270 : Handle<WasmModuleObject> module_obj,
2271 : int instance_count) {
2272 90 : CHECK_GE(instance_count, 0);
2273 : DisallowHeapAllocation no_gc;
2274 90 : WasmCompiledModule* compiled_module = module_obj->compiled_module();
2275 90 : CHECK_EQ(JSObject::cast(compiled_module->ptr_to_weak_wasm_module()->value()),
2276 : *module_obj);
2277 : Object* prev = nullptr;
2278 90 : int found_instances = compiled_module->has_weak_owning_instance() ? 1 : 0;
2279 : WasmCompiledModule* current_instance = compiled_module;
2280 240 : while (current_instance->has_weak_next_instance()) {
2281 75 : CHECK((prev == nullptr && !current_instance->has_weak_prev_instance()) ||
2282 : current_instance->ptr_to_weak_prev_instance()->value() == prev);
2283 60 : CHECK_EQ(current_instance->ptr_to_weak_wasm_module()->value(), *module_obj);
2284 60 : CHECK(IsWasmInstance(
2285 : current_instance->ptr_to_weak_owning_instance()->value()));
2286 : prev = current_instance;
2287 : current_instance = WasmCompiledModule::cast(
2288 : current_instance->ptr_to_weak_next_instance()->value());
2289 60 : ++found_instances;
2290 60 : CHECK_LE(found_instances, instance_count);
2291 : }
2292 90 : CHECK_EQ(found_instances, instance_count);
2293 90 : }
2294 :
2295 30 : void testing::ValidateModuleState(Isolate* isolate,
2296 : Handle<WasmModuleObject> module_obj) {
2297 : DisallowHeapAllocation no_gc;
2298 30 : WasmCompiledModule* compiled_module = module_obj->compiled_module();
2299 30 : CHECK(compiled_module->has_weak_wasm_module());
2300 30 : CHECK_EQ(compiled_module->ptr_to_weak_wasm_module()->value(), *module_obj);
2301 30 : CHECK(!compiled_module->has_weak_prev_instance());
2302 30 : CHECK(!compiled_module->has_weak_next_instance());
2303 30 : CHECK(!compiled_module->has_weak_owning_instance());
2304 30 : }
2305 :
2306 15 : void testing::ValidateOrphanedInstance(Isolate* isolate,
2307 : Handle<WasmInstanceObject> instance) {
2308 : DisallowHeapAllocation no_gc;
2309 15 : WasmCompiledModule* compiled_module = instance->compiled_module();
2310 15 : CHECK(compiled_module->has_weak_wasm_module());
2311 15 : CHECK(compiled_module->ptr_to_weak_wasm_module()->cleared());
2312 15 : }
2313 :
2314 60 : Handle<JSArray> wasm::GetImports(Isolate* isolate,
2315 : Handle<WasmModuleObject> module_object) {
2316 : Handle<WasmCompiledModule> compiled_module(module_object->compiled_module(),
2317 60 : isolate);
2318 : Factory* factory = isolate->factory();
2319 :
2320 60 : Handle<String> module_string = factory->InternalizeUtf8String("module");
2321 60 : Handle<String> name_string = factory->InternalizeUtf8String("name");
2322 60 : Handle<String> kind_string = factory->InternalizeUtf8String("kind");
2323 :
2324 60 : Handle<String> function_string = factory->InternalizeUtf8String("function");
2325 60 : Handle<String> table_string = factory->InternalizeUtf8String("table");
2326 60 : Handle<String> memory_string = factory->InternalizeUtf8String("memory");
2327 60 : Handle<String> global_string = factory->InternalizeUtf8String("global");
2328 :
2329 : // Create the result array.
2330 60 : WasmModule* module = compiled_module->module();
2331 240 : int num_imports = static_cast<int>(module->import_table.size());
2332 60 : Handle<JSArray> array_object = factory->NewJSArray(FAST_ELEMENTS, 0, 0);
2333 60 : Handle<FixedArray> storage = factory->NewFixedArray(num_imports);
2334 60 : JSArray::SetContent(array_object, storage);
2335 : array_object->set_length(Smi::FromInt(num_imports));
2336 :
2337 : Handle<JSFunction> object_function =
2338 120 : Handle<JSFunction>(isolate->native_context()->object_function(), isolate);
2339 :
2340 : // Populate the result array.
2341 180 : for (int index = 0; index < num_imports; ++index) {
2342 120 : WasmImport& import = module->import_table[index];
2343 :
2344 120 : Handle<JSObject> entry = factory->NewJSObject(object_function);
2345 :
2346 : Handle<String> import_kind;
2347 120 : switch (import.kind) {
2348 : case kExternalFunction:
2349 30 : import_kind = function_string;
2350 30 : break;
2351 : case kExternalTable:
2352 30 : import_kind = table_string;
2353 30 : break;
2354 : case kExternalMemory:
2355 30 : import_kind = memory_string;
2356 30 : break;
2357 : case kExternalGlobal:
2358 30 : import_kind = global_string;
2359 30 : break;
2360 : default:
2361 0 : UNREACHABLE();
2362 : }
2363 :
2364 : MaybeHandle<String> import_module =
2365 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2366 : isolate, compiled_module, import.module_name_offset,
2367 120 : import.module_name_length);
2368 :
2369 : MaybeHandle<String> import_name =
2370 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2371 : isolate, compiled_module, import.field_name_offset,
2372 120 : import.field_name_length);
2373 :
2374 : JSObject::AddProperty(entry, module_string, import_module.ToHandleChecked(),
2375 120 : NONE);
2376 : JSObject::AddProperty(entry, name_string, import_name.ToHandleChecked(),
2377 120 : NONE);
2378 120 : JSObject::AddProperty(entry, kind_string, import_kind, NONE);
2379 :
2380 120 : storage->set(index, *entry);
2381 : }
2382 :
2383 60 : return array_object;
2384 : }
2385 :
2386 60 : Handle<JSArray> wasm::GetExports(Isolate* isolate,
2387 : Handle<WasmModuleObject> module_object) {
2388 : Handle<WasmCompiledModule> compiled_module(module_object->compiled_module(),
2389 60 : isolate);
2390 : Factory* factory = isolate->factory();
2391 :
2392 60 : Handle<String> name_string = factory->InternalizeUtf8String("name");
2393 60 : Handle<String> kind_string = factory->InternalizeUtf8String("kind");
2394 :
2395 60 : Handle<String> function_string = factory->InternalizeUtf8String("function");
2396 60 : Handle<String> table_string = factory->InternalizeUtf8String("table");
2397 60 : Handle<String> memory_string = factory->InternalizeUtf8String("memory");
2398 60 : Handle<String> global_string = factory->InternalizeUtf8String("global");
2399 :
2400 : // Create the result array.
2401 60 : WasmModule* module = compiled_module->module();
2402 240 : int num_exports = static_cast<int>(module->export_table.size());
2403 60 : Handle<JSArray> array_object = factory->NewJSArray(FAST_ELEMENTS, 0, 0);
2404 60 : Handle<FixedArray> storage = factory->NewFixedArray(num_exports);
2405 60 : JSArray::SetContent(array_object, storage);
2406 : array_object->set_length(Smi::FromInt(num_exports));
2407 :
2408 : Handle<JSFunction> object_function =
2409 120 : Handle<JSFunction>(isolate->native_context()->object_function(), isolate);
2410 :
2411 : // Populate the result array.
2412 180 : for (int index = 0; index < num_exports; ++index) {
2413 120 : WasmExport& exp = module->export_table[index];
2414 :
2415 : Handle<String> export_kind;
2416 120 : switch (exp.kind) {
2417 : case kExternalFunction:
2418 30 : export_kind = function_string;
2419 30 : break;
2420 : case kExternalTable:
2421 30 : export_kind = table_string;
2422 30 : break;
2423 : case kExternalMemory:
2424 30 : export_kind = memory_string;
2425 30 : break;
2426 : case kExternalGlobal:
2427 30 : export_kind = global_string;
2428 30 : break;
2429 : default:
2430 0 : UNREACHABLE();
2431 : }
2432 :
2433 120 : Handle<JSObject> entry = factory->NewJSObject(object_function);
2434 :
2435 : MaybeHandle<String> export_name =
2436 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2437 120 : isolate, compiled_module, exp.name_offset, exp.name_length);
2438 :
2439 : JSObject::AddProperty(entry, name_string, export_name.ToHandleChecked(),
2440 120 : NONE);
2441 120 : JSObject::AddProperty(entry, kind_string, export_kind, NONE);
2442 :
2443 120 : storage->set(index, *entry);
2444 : }
2445 :
2446 60 : return array_object;
2447 : }
2448 :
2449 60 : Handle<JSArray> wasm::GetCustomSections(Isolate* isolate,
2450 : Handle<WasmModuleObject> module_object,
2451 : Handle<String> name,
2452 : ErrorThrower* thrower) {
2453 : Handle<WasmCompiledModule> compiled_module(module_object->compiled_module(),
2454 60 : isolate);
2455 : Factory* factory = isolate->factory();
2456 :
2457 : std::vector<CustomSectionOffset> custom_sections;
2458 : {
2459 : DisallowHeapAllocation no_gc; // for raw access to string bytes.
2460 : Handle<SeqOneByteString> module_bytes(compiled_module->module_bytes(),
2461 60 : isolate);
2462 : const byte* start =
2463 60 : reinterpret_cast<const byte*>(module_bytes->GetCharsAddress());
2464 60 : const byte* end = start + module_bytes->length();
2465 120 : custom_sections = DecodeCustomSections(start, end);
2466 : }
2467 :
2468 : std::vector<Handle<Object>> matching_sections;
2469 :
2470 : // Gather matching sections.
2471 300 : for (auto section : custom_sections) {
2472 : MaybeHandle<String> section_name =
2473 : WasmCompiledModule::ExtractUtf8StringFromModuleBytes(
2474 180 : isolate, compiled_module, section.name_offset, section.name_length);
2475 :
2476 180 : if (!name->Equals(*section_name.ToHandleChecked())) continue;
2477 :
2478 : // Make a copy of the payload data in the section.
2479 : bool is_external; // Set by TryAllocateBackingStore
2480 : void* memory = TryAllocateBackingStore(isolate, section.payload_length,
2481 45 : false, is_external);
2482 :
2483 : Handle<Object> section_data = factory->undefined_value();
2484 45 : if (memory) {
2485 45 : Handle<JSArrayBuffer> buffer = isolate->factory()->NewJSArrayBuffer();
2486 : JSArrayBuffer::Setup(buffer, isolate, is_external, memory,
2487 45 : static_cast<int>(section.payload_length));
2488 : DisallowHeapAllocation no_gc; // for raw access to string bytes.
2489 : Handle<SeqOneByteString> module_bytes(compiled_module->module_bytes(),
2490 45 : isolate);
2491 : const byte* start =
2492 45 : reinterpret_cast<const byte*>(module_bytes->GetCharsAddress());
2493 45 : memcpy(memory, start + section.payload_offset, section.payload_length);
2494 45 : section_data = buffer;
2495 : } else {
2496 0 : thrower->RangeError("out of memory allocating custom section data");
2497 0 : return Handle<JSArray>();
2498 : }
2499 :
2500 45 : matching_sections.push_back(section_data);
2501 : }
2502 :
2503 120 : int num_custom_sections = static_cast<int>(matching_sections.size());
2504 60 : Handle<JSArray> array_object = factory->NewJSArray(FAST_ELEMENTS, 0, 0);
2505 60 : Handle<FixedArray> storage = factory->NewFixedArray(num_custom_sections);
2506 60 : JSArray::SetContent(array_object, storage);
2507 : array_object->set_length(Smi::FromInt(num_custom_sections));
2508 :
2509 105 : for (int i = 0; i < num_custom_sections; i++) {
2510 90 : storage->set(i, *matching_sections[i]);
2511 : }
2512 :
2513 60 : return array_object;
2514 : }
2515 :
2516 1110 : bool wasm::SyncValidate(Isolate* isolate, ErrorThrower* thrower,
2517 : const ModuleWireBytes& bytes) {
2518 2220 : if (bytes.start() == nullptr || bytes.length() == 0) return false;
2519 : ModuleResult result =
2520 2220 : DecodeWasmModule(isolate, bytes.start(), bytes.end(), true, kWasmOrigin);
2521 1110 : if (result.val) delete result.val;
2522 : return result.ok();
2523 : }
2524 :
2525 3609 : MaybeHandle<WasmModuleObject> wasm::SyncCompileTranslatedAsmJs(
2526 : Isolate* isolate, ErrorThrower* thrower, const ModuleWireBytes& bytes,
2527 : Handle<Script> asm_js_script,
2528 3609 : Vector<const byte> asm_js_offset_table_bytes) {
2529 :
2530 : ModuleResult result = DecodeWasmModule(isolate, bytes.start(), bytes.end(),
2531 7218 : false, kAsmJsOrigin);
2532 3609 : if (result.failed()) {
2533 : // TODO(titzer): use Result<std::unique_ptr<const WasmModule*>>?
2534 0 : if (result.val) delete result.val;
2535 : thrower->CompileFailed("Wasm decoding failed", result);
2536 : return {};
2537 : }
2538 :
2539 7218 : CompilationHelper helper(isolate, const_cast<WasmModule*>(result.val));
2540 : return helper.CompileToModuleObject(thrower, bytes, asm_js_script,
2541 3609 : asm_js_offset_table_bytes);
2542 : }
2543 :
2544 12199 : MaybeHandle<WasmModuleObject> wasm::SyncCompile(Isolate* isolate,
2545 : ErrorThrower* thrower,
2546 : const ModuleWireBytes& bytes) {
2547 24398 : if (!IsWasmCodegenAllowed(isolate, isolate->native_context())) {
2548 14 : thrower->CompileError("Wasm code generation disallowed in this context");
2549 : return {};
2550 : }
2551 :
2552 : ModuleResult result =
2553 24370 : DecodeWasmModule(isolate, bytes.start(), bytes.end(), false, kWasmOrigin);
2554 12185 : if (result.failed()) {
2555 450 : if (result.val) delete result.val;
2556 : thrower->CompileFailed("Wasm decoding failed", result);
2557 : return {};
2558 : }
2559 :
2560 23470 : CompilationHelper helper(isolate, const_cast<WasmModule*>(result.val));
2561 : return helper.CompileToModuleObject(thrower, bytes, Handle<Script>(),
2562 11735 : Vector<const byte>());
2563 : }
2564 :
2565 48092 : MaybeHandle<WasmInstanceObject> wasm::SyncInstantiate(
2566 : Isolate* isolate, ErrorThrower* thrower,
2567 : Handle<WasmModuleObject> module_object, MaybeHandle<JSReceiver> imports,
2568 : MaybeHandle<JSArrayBuffer> memory) {
2569 48092 : InstantiationHelper helper(isolate, thrower, module_object, imports, memory);
2570 48092 : return helper.Build();
2571 : }
2572 :
2573 : namespace {
2574 :
2575 15 : void RejectPromise(Isolate* isolate, Handle<Context> context,
2576 : ErrorThrower* thrower, Handle<JSPromise> promise) {
2577 : v8::Local<v8::Promise::Resolver> resolver =
2578 : v8::Utils::PromiseToLocal(promise).As<v8::Promise::Resolver>();
2579 : auto maybe = resolver->Reject(v8::Utils::ToLocal(context),
2580 15 : v8::Utils::ToLocal(thrower->Reify()));
2581 15 : CHECK_IMPLIES(!maybe.FromMaybe(false), isolate->has_scheduled_exception());
2582 15 : }
2583 :
2584 750 : void ResolvePromise(Isolate* isolate, Handle<Context> context,
2585 : Handle<JSPromise> promise, Handle<Object> result) {
2586 : v8::Local<v8::Promise::Resolver> resolver =
2587 : v8::Utils::PromiseToLocal(promise).As<v8::Promise::Resolver>();
2588 : auto maybe = resolver->Resolve(v8::Utils::ToLocal(context),
2589 750 : v8::Utils::ToLocal(result));
2590 750 : CHECK_IMPLIES(!maybe.FromMaybe(false), isolate->has_scheduled_exception());
2591 750 : }
2592 :
2593 : } // namespace
2594 :
2595 0 : void wasm::AsyncInstantiate(Isolate* isolate, Handle<JSPromise> promise,
2596 : Handle<WasmModuleObject> module_object,
2597 : MaybeHandle<JSReceiver> imports) {
2598 : ErrorThrower thrower(isolate, nullptr);
2599 : MaybeHandle<WasmInstanceObject> instance_object = SyncInstantiate(
2600 0 : isolate, &thrower, module_object, imports, Handle<JSArrayBuffer>::null());
2601 0 : if (thrower.error()) {
2602 0 : RejectPromise(isolate, handle(isolate->context()), &thrower, promise);
2603 0 : return;
2604 : }
2605 : ResolvePromise(isolate, handle(isolate->context()), promise,
2606 0 : instance_object.ToHandleChecked());
2607 : }
2608 :
2609 : // Encapsulates all the state and steps of an asynchronous compilation.
2610 : // An asynchronous compile job consists of a number of tasks that are executed
2611 : // as foreground and background tasks. Any phase that touches the V8 heap or
2612 : // allocates on the V8 heap (e.g. creating the module object) must be a
2613 : // foreground task. All other tasks (e.g. decoding and validating, the majority
2614 : // of the work of compilation) can be background tasks.
2615 : // TODO(wasm): factor out common parts of this with the synchronous pipeline.
2616 : //
2617 : // Note: In predictable mode, DoSync and DoAsync execute the referenced function
2618 : // immediately before returning. Thus we handle the predictable mode specially,
2619 : // e.g. when we synchronizing tasks or when we delete the AyncCompileJob.
2620 : class AsyncCompileJob {
2621 : // TODO(ahaas): Fix https://bugs.chromium.org/p/v8/issues/detail?id=6263 to
2622 : // make sure that d8 does not shut down before the AsyncCompileJob is
2623 : // finished.
2624 : public:
2625 765 : explicit AsyncCompileJob(Isolate* isolate, std::unique_ptr<byte[]> bytes_copy,
2626 : int length, Handle<Context> context,
2627 : Handle<JSPromise> promise)
2628 : : isolate_(isolate),
2629 : bytes_copy_(std::move(bytes_copy)),
2630 3825 : wire_bytes_(bytes_copy_.get(), bytes_copy_.get() + length) {
2631 : // The handles for the context and promise must be deferred.
2632 765 : DeferredHandleScope deferred(isolate);
2633 765 : context_ = Handle<Context>(*context);
2634 765 : module_promise_ = Handle<JSPromise>(*promise);
2635 1530 : deferred_handles_.push_back(deferred.Detach());
2636 765 : }
2637 :
2638 : bool Start() {
2639 765 : return DoAsync(&AsyncCompileJob::DecodeModule); // --
2640 : }
2641 :
2642 765 : ~AsyncCompileJob() {
2643 3795 : for (auto d : deferred_handles_) delete d;
2644 765 : }
2645 :
2646 : private:
2647 : Isolate* isolate_;
2648 : std::unique_ptr<byte[]> bytes_copy_;
2649 : ModuleWireBytes wire_bytes_;
2650 : Handle<Context> context_;
2651 : Handle<JSPromise> module_promise_;
2652 : WasmModule* module_ = nullptr;
2653 : ModuleResult result_;
2654 : std::unique_ptr<CompilationHelper> helper_ = nullptr;
2655 : std::unique_ptr<ModuleBytesEnv> module_bytes_env_ = nullptr;
2656 :
2657 : volatile bool failed_ = false;
2658 : std::vector<DeferredHandles*> deferred_handles_;
2659 : Handle<WasmModuleWrapper> module_wrapper_;
2660 : Handle<WasmModuleObject> module_object_;
2661 : Handle<FixedArray> function_tables_;
2662 : Handle<FixedArray> signature_tables_;
2663 : Handle<WasmCompiledModule> compiled_module_;
2664 : Handle<FixedArray> code_table_;
2665 : std::unique_ptr<WasmInstance> temp_instance_ = nullptr;
2666 : size_t outstanding_units_ = 0;
2667 : size_t num_background_tasks_ = 0;
2668 :
2669 750 : void ReopenHandlesInDeferredScope() {
2670 750 : DeferredHandleScope deferred(isolate_);
2671 1500 : module_wrapper_ = handle(*module_wrapper_, isolate_);
2672 1500 : function_tables_ = handle(*function_tables_, isolate_);
2673 1500 : signature_tables_ = handle(*signature_tables_, isolate_);
2674 1500 : code_table_ = handle(*code_table_, isolate_);
2675 1500 : temp_instance_->ReopenHandles(isolate_);
2676 : helper_->InitializeHandles();
2677 1500 : deferred_handles_.push_back(deferred.Detach());
2678 750 : }
2679 :
2680 : //==========================================================================
2681 : // Step 1: (async) Decode the module.
2682 : //==========================================================================
2683 765 : bool DecodeModule() {
2684 : {
2685 : DisallowHandleAllocation no_handle;
2686 : DisallowHeapAllocation no_allocation;
2687 : // Decode the module bytes.
2688 765 : TRACE_COMPILE("(1) Decoding module...\n");
2689 1530 : result_ = DecodeWasmModule(isolate_, wire_bytes_.start(),
2690 : wire_bytes_.end(), true, kWasmOrigin);
2691 : }
2692 765 : if (result_.failed()) {
2693 : // Decoding failure; reject the promise and clean up.
2694 15 : if (result_.val) delete result_.val;
2695 15 : return DoSync(&AsyncCompileJob::DecodeFail);
2696 : } else {
2697 : // Decode passed.
2698 750 : module_ = const_cast<WasmModule*>(result_.val);
2699 750 : return DoSync(&AsyncCompileJob::PrepareAndStartCompile);
2700 : }
2701 : }
2702 :
2703 : //==========================================================================
2704 : // Step 1b: (sync) Fail decoding the module.
2705 : //==========================================================================
2706 15 : bool DecodeFail() {
2707 15 : HandleScope scope(isolate_);
2708 15 : ErrorThrower thrower(isolate_, nullptr);
2709 : thrower.CompileFailed("Wasm decoding failed", result_);
2710 15 : RejectPromise(isolate_, context_, &thrower, module_promise_);
2711 15 : return false;
2712 : }
2713 :
2714 : //==========================================================================
2715 : // Step 2 (sync): Create heap-allocated data and start compile.
2716 : //==========================================================================
2717 750 : bool PrepareAndStartCompile() {
2718 750 : TRACE_COMPILE("(2) Prepare and start compile...\n");
2719 1500 : HandleScope scope(isolate_);
2720 :
2721 : Factory* factory = isolate_->factory();
2722 : // The {module_wrapper} will take ownership of the {WasmModule} object,
2723 : // and it will be destroyed when the GC reclaims the wrapper object.
2724 750 : module_wrapper_ = WasmModuleWrapper::New(isolate_, module_);
2725 750 : temp_instance_ = std::unique_ptr<WasmInstance>(new WasmInstance(module_));
2726 750 : temp_instance_->context = context_;
2727 750 : temp_instance_->mem_size = WasmModule::kPageSize * module_->min_mem_pages;
2728 750 : temp_instance_->mem_start = nullptr;
2729 750 : temp_instance_->globals_start = nullptr;
2730 :
2731 : // Initialize the indirect tables with placeholders.
2732 : int function_table_count =
2733 1500 : static_cast<int>(module_->function_tables.size());
2734 750 : function_tables_ = factory->NewFixedArray(function_table_count, TENURED);
2735 750 : signature_tables_ = factory->NewFixedArray(function_table_count, TENURED);
2736 855 : for (int i = 0; i < function_table_count; ++i) {
2737 315 : temp_instance_->function_tables[i] = factory->NewFixedArray(1, TENURED);
2738 210 : temp_instance_->signature_tables[i] = factory->NewFixedArray(1, TENURED);
2739 210 : function_tables_->set(i, *temp_instance_->function_tables[i]);
2740 210 : signature_tables_->set(i, *temp_instance_->signature_tables[i]);
2741 : }
2742 :
2743 : // The {code_table} array contains import wrappers and functions (which
2744 : // are both included in {functions.size()}, and export wrappers.
2745 : // The results of compilation will be written into it.
2746 2865 : int code_table_size = static_cast<int>(module_->functions.size() +
2747 750 : module_->num_exported_functions);
2748 750 : code_table_ = factory->NewFixedArray(code_table_size, TENURED);
2749 :
2750 : // Initialize {code_table_} with the illegal builtin. All call sites
2751 : // will be patched at instantiation.
2752 750 : Handle<Code> illegal_builtin = isolate_->builtins()->Illegal();
2753 : // TODO(wasm): Fix this for lazy compilation.
2754 2730 : for (uint32_t i = 0; i < module_->functions.size(); ++i) {
2755 1230 : code_table_->set(static_cast<int>(i), *illegal_builtin);
2756 1230 : temp_instance_->function_code[i] = illegal_builtin;
2757 : }
2758 :
2759 : isolate_->counters()->wasm_functions_per_wasm_module()->AddSample(
2760 1500 : static_cast<int>(module_->functions.size()));
2761 :
2762 750 : helper_.reset(new CompilationHelper(isolate_, module_));
2763 :
2764 : DCHECK_LE(module_->num_imported_functions, module_->functions.size());
2765 : size_t num_functions =
2766 1500 : module_->functions.size() - module_->num_imported_functions;
2767 750 : if (num_functions == 0) {
2768 540 : ReopenHandlesInDeferredScope();
2769 : // Degenerate case of an empty module.
2770 540 : return DoSync(&AsyncCompileJob::FinishCompile);
2771 : }
2772 :
2773 : // Start asynchronous compilation tasks.
2774 : num_background_tasks_ =
2775 : Max(static_cast<size_t>(1),
2776 : Min(num_functions,
2777 : Min(static_cast<size_t>(FLAG_wasm_num_compilation_tasks),
2778 210 : V8::GetCurrentPlatform()
2779 420 : ->NumberOfAvailableBackgroundThreads())));
2780 210 : module_bytes_env_ = std::unique_ptr<ModuleBytesEnv>(
2781 210 : new ModuleBytesEnv(module_, temp_instance_.get(), wire_bytes_));
2782 : outstanding_units_ = helper_->InitializeParallelCompilation(
2783 420 : module_->functions, *module_bytes_env_);
2784 :
2785 : // Reopen all handles which should survive in the DeferredHandleScope.
2786 210 : ReopenHandlesInDeferredScope();
2787 420 : for (size_t i = 0; i < num_background_tasks_; ++i) {
2788 210 : DoAsync(&AsyncCompileJob::ExecuteCompilationUnits);
2789 : }
2790 : return true;
2791 : }
2792 :
2793 : //==========================================================================
2794 : // Step 3 (async x K tasks): Execute compilation units.
2795 : //==========================================================================
2796 210 : bool ExecuteCompilationUnits() {
2797 210 : TRACE_COMPILE("(3) Compiling...\n");
2798 420 : while (!failed_) {
2799 : {
2800 : DisallowHandleAllocation no_handle;
2801 : DisallowHeapAllocation no_allocation;
2802 420 : if (!helper_->FetchAndExecuteCompilationUnit()) break;
2803 : }
2804 : // TODO(ahaas): Create one FinishCompilationUnit job for all compilation
2805 : // units.
2806 210 : DoSync(&AsyncCompileJob::FinishCompilationUnit);
2807 : // TODO(ahaas): Limit the number of outstanding compilation units to be
2808 : // finished to reduce memory overhead.
2809 : }
2810 : // Special handling for predictable mode, see above.
2811 : if (!FLAG_verify_predictable)
2812 418 : helper_->module_->pending_tasks.get()->Signal();
2813 210 : return true;
2814 : }
2815 :
2816 : //==========================================================================
2817 : // Step 4 (sync x each function): Finish a single compilation unit.
2818 : //==========================================================================
2819 210 : bool FinishCompilationUnit() {
2820 210 : TRACE_COMPILE("(4a) Finishing compilation unit...\n");
2821 210 : HandleScope scope(isolate_);
2822 210 : if (failed_) return true; // already failed
2823 :
2824 210 : int func_index = -1;
2825 210 : ErrorThrower thrower(isolate_, nullptr);
2826 210 : Handle<Code> result = helper_->FinishCompilationUnit(&thrower, &func_index);
2827 210 : if (thrower.error()) {
2828 0 : RejectPromise(isolate_, context_, &thrower, module_promise_);
2829 0 : failed_ = true;
2830 : } else {
2831 : DCHECK(func_index >= 0);
2832 420 : code_table_->set(func_index, *(result));
2833 : }
2834 210 : if (failed_ || --outstanding_units_ == 0) {
2835 : // All compilation units are done. We still need to wait for the
2836 : // background tasks to shut down and only then is it safe to finish the
2837 : // compile and delete this job. We can wait for that to happen also
2838 : // in a background task.
2839 210 : DoAsync(&AsyncCompileJob::WaitForBackgroundTasks);
2840 : }
2841 : return true;
2842 : }
2843 :
2844 : //==========================================================================
2845 : // Step 4b (async): Wait for all background tasks to finish.
2846 : //==========================================================================
2847 210 : bool WaitForBackgroundTasks() {
2848 210 : TRACE_COMPILE("(4b) Waiting for background tasks...\n");
2849 : // Special handling for predictable mode, see above.
2850 : if (!FLAG_verify_predictable) {
2851 209 : for (size_t i = 0; i < num_background_tasks_; ++i) {
2852 : // We wait for it to finish.
2853 420 : module_->pending_tasks.get()->Wait();
2854 : }
2855 : }
2856 209 : if (failed_) {
2857 : // If {failed_}, we've already rejected the promise and there
2858 : // is nothing more to do.
2859 : return false;
2860 : } else {
2861 : // Otherwise, post a synchronous task to finish the compile.
2862 209 : DoSync(&AsyncCompileJob::FinishCompile);
2863 210 : return true;
2864 : }
2865 : }
2866 :
2867 : //==========================================================================
2868 : // Step 5 (sync): Finish heap-allocated data structures.
2869 : //==========================================================================
2870 750 : bool FinishCompile() {
2871 750 : TRACE_COMPILE("(5) Finish compile...\n");
2872 1500 : HandleScope scope(isolate_);
2873 1500 : SaveContext saved_context(isolate_);
2874 750 : isolate_->set_context(*context_);
2875 : // At this point, compilation has completed. Update the code table.
2876 2730 : for (size_t i = FLAG_skip_compiling_wasm_funcs;
2877 1365 : i < temp_instance_->function_code.size(); ++i) {
2878 615 : Code* code = Code::cast(code_table_->get(static_cast<int>(i)));
2879 615 : RecordStats(isolate_, code);
2880 : }
2881 :
2882 : // Create heap objects for script and module bytes to be stored in the
2883 : // shared module data. Asm.js is not compiled asynchronously.
2884 750 : Handle<Script> script = CreateWasmScript(isolate_, wire_bytes_);
2885 : Handle<ByteArray> asm_js_offset_table;
2886 : // TODO(wasm): Improve efficiency of storing module wire bytes.
2887 : // 1. Only store relevant sections, not function bodies
2888 : // 2. Don't make a second copy of the bytes here; reuse the copy made
2889 : // for asynchronous compilation and store it as an external one
2890 : // byte string for serialization/deserialization.
2891 : Handle<String> module_bytes =
2892 : isolate_->factory()
2893 : ->NewStringFromOneByte({wire_bytes_.start(), wire_bytes_.length()},
2894 : TENURED)
2895 1500 : .ToHandleChecked();
2896 : DCHECK(module_bytes->IsSeqOneByteString());
2897 :
2898 : // Create the shared module data.
2899 : // TODO(clemensh): For the same module (same bytes / same hash), we should
2900 : // only have one WasmSharedModuleData. Otherwise, we might only set
2901 : // breakpoints on a (potentially empty) subset of the instances.
2902 :
2903 : Handle<WasmSharedModuleData> shared = WasmSharedModuleData::New(
2904 : isolate_, module_wrapper_, Handle<SeqOneByteString>::cast(module_bytes),
2905 750 : script, asm_js_offset_table);
2906 :
2907 : // Create the compiled module object and populate with compiled functions
2908 : // and information needed at instantiation time. This object needs to be
2909 : // serializable. Instantiation may occur off a deserialized version of this
2910 : // object.
2911 : compiled_module_ = WasmCompiledModule::New(
2912 750 : isolate_, shared, code_table_, function_tables_, signature_tables_);
2913 :
2914 : // Finish the WASM script now and make it public to the debugger.
2915 750 : script->set_wasm_compiled_module(*compiled_module_);
2916 1500 : isolate_->debug()->OnAfterCompile(script);
2917 :
2918 1500 : DeferredHandleScope deferred(isolate_);
2919 1500 : compiled_module_ = handle(*compiled_module_, isolate_);
2920 1500 : deferred_handles_.push_back(deferred.Detach());
2921 : // TODO(wasm): compiling wrappers should be made async as well.
2922 1500 : return DoSync(&AsyncCompileJob::CompileWrappers);
2923 : }
2924 :
2925 : //==========================================================================
2926 : // Step 6 (sync): Compile JS->WASM wrappers.
2927 : //==========================================================================
2928 750 : bool CompileWrappers() {
2929 750 : TRACE_COMPILE("(6) Compile wrappers...\n");
2930 : // Compile JS->WASM wrappers for exported functions.
2931 750 : HandleScope scope(isolate_);
2932 750 : JSToWasmWrapperCache js_to_wasm_cache;
2933 : int func_index = 0;
2934 2445 : for (auto exp : module_->export_table) {
2935 255 : if (exp.kind != kExternalFunction) continue;
2936 135 : Handle<Code> wasm_code(Code::cast(code_table_->get(exp.index)), isolate_);
2937 : Handle<Code> wrapper_code =
2938 : js_to_wasm_cache.CloneOrCompileJSToWasmWrapper(isolate_, module_,
2939 135 : wasm_code, exp.index);
2940 : int export_index =
2941 270 : static_cast<int>(module_->functions.size() + func_index);
2942 135 : code_table_->set(export_index, *wrapper_code);
2943 135 : RecordStats(isolate_, *wrapper_code);
2944 135 : func_index++;
2945 : }
2946 :
2947 1500 : return DoSync(&AsyncCompileJob::FinishModule);
2948 : }
2949 :
2950 : //==========================================================================
2951 : // Step 7 (sync): Finish the module and resolve the promise.
2952 : //==========================================================================
2953 750 : bool FinishModule() {
2954 750 : TRACE_COMPILE("(7) Finish module...\n");
2955 750 : HandleScope scope(isolate_);
2956 1500 : SaveContext saved_context(isolate_);
2957 750 : isolate_->set_context(*context_);
2958 : Handle<WasmModuleObject> result =
2959 750 : WasmModuleObject::New(isolate_, compiled_module_);
2960 750 : ResolvePromise(isolate_, context_, module_promise_, result);
2961 750 : return false; // no more work to do.
2962 : }
2963 :
2964 : // Run the given member method as an asynchronous task.
2965 1185 : bool DoAsync(bool (AsyncCompileJob::*func)()) {
2966 1185 : auto task = new AsyncCompileTask(this, func);
2967 1185 : V8::GetCurrentPlatform()->CallOnBackgroundThread(
2968 1185 : task, v8::Platform::kShortRunningTask);
2969 1185 : return true; // more work to do.
2970 : }
2971 :
2972 : // Run the given member method as a synchronous task.
2973 3224 : bool DoSync(bool (AsyncCompileJob::*func)()) {
2974 3224 : V8::GetCurrentPlatform()->CallOnForegroundThread(
2975 : reinterpret_cast<v8::Isolate*>(isolate_),
2976 6446 : new AsyncCompileTask(this, func));
2977 3225 : return true; // more work to do.
2978 : }
2979 :
2980 : // A helper closure to run a particular member method as a task.
2981 8818 : class AsyncCompileTask : NON_EXPORTED_BASE(public v8::Task) {
2982 : public:
2983 : AsyncCompileJob* job_;
2984 : bool (AsyncCompileJob::*func_)();
2985 : AsyncCompileTask(AsyncCompileJob* job, bool (AsyncCompileJob::*func)())
2986 4410 : : v8::Task(), job_(job), func_(func) {}
2987 :
2988 4410 : void Run() {
2989 4410 : bool more = (job_->*func_)(); // run the task.
2990 4410 : if (!more) {
2991 : // If no more work, then this job is done. Predictable mode is handled
2992 : // specially though, see above.
2993 765 : if (!FLAG_verify_predictable) delete job_;
2994 : }
2995 4410 : }
2996 : };
2997 : };
2998 :
2999 1530 : void wasm::AsyncCompile(Isolate* isolate, Handle<JSPromise> promise,
3000 : const ModuleWireBytes& bytes) {
3001 765 : if (!FLAG_wasm_async_compilation) {
3002 : ErrorThrower thrower(isolate, "WasmCompile");
3003 : // Compile the module.
3004 : MaybeHandle<WasmModuleObject> module_object =
3005 0 : SyncCompile(isolate, &thrower, bytes);
3006 0 : if (thrower.error()) {
3007 0 : RejectPromise(isolate, handle(isolate->context()), &thrower, promise);
3008 0 : return;
3009 : }
3010 : Handle<WasmModuleObject> module = module_object.ToHandleChecked();
3011 0 : ResolvePromise(isolate, handle(isolate->context()), promise, module);
3012 0 : return;
3013 : }
3014 :
3015 : // Make a copy of the wire bytes in case the user program changes them
3016 : // during asynchronous compilation.
3017 765 : std::unique_ptr<byte[]> copy(new byte[bytes.length()]);
3018 765 : memcpy(copy.get(), bytes.start(), bytes.length());
3019 : auto job = new AsyncCompileJob(isolate, std::move(copy), bytes.length(),
3020 1530 : handle(isolate->context()), promise);
3021 : job->Start();
3022 : // Special handling for predictable mode, see above.
3023 : if (FLAG_verify_predictable) delete job;
3024 : }
3025 :
3026 12686 : Handle<Code> wasm::CompileLazy(Isolate* isolate) {
3027 : HistogramTimerScope lazy_time_scope(
3028 12686 : isolate->counters()->wasm_lazy_compilation_time());
3029 :
3030 : // Find the wasm frame which triggered the lazy compile, to get the wasm
3031 : // instance.
3032 12686 : StackFrameIterator it(isolate);
3033 : // First frame: C entry stub.
3034 : DCHECK(!it.done());
3035 : DCHECK_EQ(StackFrame::EXIT, it.frame()->type());
3036 12686 : it.Advance();
3037 : // Second frame: WasmCompileLazy builtin.
3038 : DCHECK(!it.done());
3039 12686 : Handle<Code> lazy_compile_code(it.frame()->LookupCode(), isolate);
3040 : DCHECK_EQ(Builtins::kWasmCompileLazy, lazy_compile_code->builtin_index());
3041 : Handle<WasmInstanceObject> instance;
3042 : Handle<FixedArray> exp_deopt_data;
3043 : int func_index = -1;
3044 12686 : if (lazy_compile_code->deoptimization_data()->length() > 0) {
3045 : // Then it's an indirect call or via JS->WASM wrapper.
3046 : DCHECK_LE(2, lazy_compile_code->deoptimization_data()->length());
3047 : exp_deopt_data = handle(lazy_compile_code->deoptimization_data(), isolate);
3048 : auto* weak_cell = WeakCell::cast(exp_deopt_data->get(0));
3049 10094 : instance = handle(WasmInstanceObject::cast(weak_cell->value()), isolate);
3050 : func_index = Smi::cast(exp_deopt_data->get(1))->value();
3051 : }
3052 12686 : it.Advance();
3053 : // Third frame: The calling wasm code or js-to-wasm wrapper.
3054 : DCHECK(!it.done());
3055 : DCHECK(it.frame()->is_js_to_wasm() || it.frame()->is_wasm_compiled());
3056 12686 : Handle<Code> caller_code = handle(it.frame()->LookupCode(), isolate);
3057 25372 : if (it.frame()->is_js_to_wasm()) {
3058 : DCHECK(!instance.is_null());
3059 3124 : } else if (instance.is_null()) {
3060 2592 : instance = handle(wasm::GetOwningWasmInstance(*caller_code), isolate);
3061 : } else {
3062 : DCHECK(*instance == wasm::GetOwningWasmInstance(*caller_code));
3063 : }
3064 : int offset =
3065 38058 : static_cast<int>(it.frame()->pc() - caller_code->instruction_start());
3066 : // Only patch the caller code if this is *no* indirect call.
3067 : // exp_deopt_data will be null if the called function is not exported at all,
3068 : // and its length will be <= 2 if all entries in tables were already patched.
3069 : // Note that this check is conservative: If the first call to an exported
3070 : // function is direct, we will just patch the export tables, and only on the
3071 : // second call we will patch the caller.
3072 3124 : bool patch_caller = caller_code->kind() == Code::JS_TO_WASM_FUNCTION ||
3073 13218 : exp_deopt_data.is_null() || exp_deopt_data->length() <= 2;
3074 :
3075 : MaybeHandle<Code> maybe_compiled_code = WasmCompiledModule::CompileLazy(
3076 12686 : isolate, instance, caller_code, offset, func_index, patch_caller);
3077 12686 : if (maybe_compiled_code.is_null()) {
3078 : DCHECK(isolate->has_pending_exception());
3079 0 : return isolate->builtins()->Illegal();
3080 : }
3081 : Handle<Code> compiled_code = maybe_compiled_code.ToHandleChecked();
3082 22780 : if (!exp_deopt_data.is_null() && exp_deopt_data->length() > 2) {
3083 : // See EnsureExportedLazyDeoptData: exp_deopt_data[2...(len-1)] are pairs of
3084 : // <export_table, index> followed by undefined values.
3085 : // Use this information here to patch all export tables.
3086 : DCHECK_EQ(0, exp_deopt_data->length() % 2);
3087 770 : for (int idx = 2, end = exp_deopt_data->length(); idx < end; idx += 2) {
3088 392 : if (exp_deopt_data->get(idx)->IsUndefined(isolate)) break;
3089 : FixedArray* exp_table = FixedArray::cast(exp_deopt_data->get(idx));
3090 392 : int exp_index = Smi::cast(exp_deopt_data->get(idx + 1))->value();
3091 : DCHECK(exp_table->get(exp_index) == *lazy_compile_code);
3092 392 : exp_table->set(exp_index, *compiled_code);
3093 : }
3094 : // After processing, remove the list of exported entries, such that we don't
3095 : // do the patching redundantly.
3096 : Handle<FixedArray> new_deopt_data =
3097 378 : isolate->factory()->CopyFixedArrayUpTo(exp_deopt_data, 2, TENURED);
3098 378 : lazy_compile_code->set_deoptimization_data(*new_deopt_data);
3099 : }
3100 :
3101 12686 : return compiled_code;
3102 : }
3103 :
3104 12686 : bool LazyCompilationOrchestrator::CompileFunction(
3105 12483 : Isolate* isolate, Handle<WasmInstanceObject> instance, int func_index) {
3106 : Handle<WasmCompiledModule> compiled_module(instance->compiled_module(),
3107 12686 : isolate);
3108 25372 : if (Code::cast(compiled_module->code_table()->get(func_index))->kind() ==
3109 : Code::WASM_FUNCTION) {
3110 : return true;
3111 : }
3112 : size_t num_function_tables =
3113 12483 : compiled_module->module()->function_tables.size();
3114 : // Store a vector of handles to be embedded in the generated code.
3115 : // TODO(clemensh): For concurrent compilation, these will have to live in a
3116 : // DeferredHandleScope.
3117 12483 : std::vector<Handle<FixedArray>> fun_tables(num_function_tables);
3118 12483 : std::vector<Handle<FixedArray>> sig_tables(num_function_tables);
3119 15540 : for (size_t i = 0; i < num_function_tables; ++i) {
3120 : Object* fun_table =
3121 9171 : compiled_module->function_tables()->get(static_cast<int>(i));
3122 6114 : fun_tables[i] = handle(FixedArray::cast(fun_table), isolate);
3123 : Object* sig_table =
3124 6114 : compiled_module->signature_tables()->get(static_cast<int>(i));
3125 6114 : sig_tables[i] = handle(FixedArray::cast(sig_table), isolate);
3126 : }
3127 12483 : wasm::ModuleEnv module_env(compiled_module->module(), &fun_tables,
3128 : &sig_tables);
3129 12483 : uint8_t* module_start = compiled_module->module_bytes()->GetChars();
3130 12483 : const WasmFunction* func = &module_env.module->functions[func_index];
3131 : wasm::FunctionBody body{func->sig, module_start,
3132 : module_start + func->code_start_offset,
3133 12483 : module_start + func->code_end_offset};
3134 : // TODO(wasm): Refactor this to only get the name if it is really needed for
3135 : // tracing / debugging.
3136 : std::string func_name;
3137 : {
3138 : wasm::WasmName name = Vector<const char>::cast(
3139 24966 : compiled_module->GetRawFunctionName(func_index));
3140 : // Copy to std::string, because the underlying string object might move on
3141 : // the heap.
3142 12483 : func_name.assign(name.start(), static_cast<size_t>(name.length()));
3143 : }
3144 12483 : ErrorThrower thrower(isolate, "WasmLazyCompile");
3145 : compiler::WasmCompilationUnit unit(isolate, &module_env, body,
3146 24966 : CStrVector(func_name.c_str()), func_index);
3147 12483 : unit.InitializeHandles();
3148 12483 : unit.ExecuteCompilation();
3149 12483 : Handle<Code> code = unit.FinishCompilation(&thrower);
3150 :
3151 12483 : Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(2, TENURED);
3152 12483 : Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance);
3153 : // TODO(wasm): Introduce constants for the indexes in wasm deopt data.
3154 12483 : deopt_data->set(0, *weak_instance);
3155 : deopt_data->set(1, Smi::FromInt(func_index));
3156 12483 : code->set_deoptimization_data(*deopt_data);
3157 :
3158 12483 : if (thrower.error()) {
3159 0 : if (!isolate->has_pending_exception()) isolate->Throw(*thrower.Reify());
3160 : return false;
3161 : }
3162 :
3163 : DCHECK_EQ(Builtins::kWasmCompileLazy,
3164 : Code::cast(compiled_module->code_table()->get(func_index))
3165 : ->builtin_index());
3166 24966 : compiled_module->code_table()->set(func_index, *code);
3167 :
3168 : // Now specialize the generated code for this instance.
3169 24966 : Zone specialization_zone(isolate->allocator(), ZONE_NAME);
3170 24966 : CodeSpecialization code_specialization(isolate, &specialization_zone);
3171 12483 : if (module_env.module->globals_size) {
3172 : Address globals_start =
3173 4679 : reinterpret_cast<Address>(compiled_module->globals_start());
3174 4679 : code_specialization.RelocateGlobals(nullptr, globals_start);
3175 : }
3176 12483 : if (instance->has_memory_buffer()) {
3177 : Address mem_start =
3178 12483 : reinterpret_cast<Address>(instance->memory_buffer()->backing_store());
3179 24966 : int mem_size = instance->memory_buffer()->byte_length()->Number();
3180 : DCHECK_IMPLIES(mem_size == 0, mem_start == nullptr);
3181 12483 : if (mem_size > 0) {
3182 : code_specialization.RelocateMemoryReferences(nullptr, 0, mem_start,
3183 12470 : mem_size);
3184 : }
3185 : }
3186 12483 : code_specialization.RelocateDirectCalls(instance);
3187 12483 : code_specialization.ApplyToWasmCode(*code, SKIP_ICACHE_FLUSH);
3188 12483 : Assembler::FlushICache(isolate, code->instruction_start(),
3189 24966 : code->instruction_size());
3190 12483 : RecordLazyCodeStats(isolate, *code);
3191 : return true;
3192 : }
3193 :
3194 12686 : MaybeHandle<Code> LazyCompilationOrchestrator::CompileLazy(
3195 : Isolate* isolate, Handle<WasmInstanceObject> instance, Handle<Code> caller,
3196 : int call_offset, int exported_func_index, bool patch_caller) {
3197 : struct NonCompiledFunction {
3198 : int offset;
3199 : int func_index;
3200 : };
3201 : std::vector<NonCompiledFunction> non_compiled_functions;
3202 : int func_to_return_idx = exported_func_index;
3203 : wasm::Decoder decoder(nullptr, nullptr);
3204 12686 : bool is_js_to_wasm = caller->kind() == Code::JS_TO_WASM_FUNCTION;
3205 : Handle<WasmCompiledModule> compiled_module(instance->compiled_module(),
3206 12686 : isolate);
3207 :
3208 12686 : if (is_js_to_wasm) {
3209 19124 : non_compiled_functions.push_back({0, exported_func_index});
3210 3124 : } else if (patch_caller) {
3211 : DisallowHeapAllocation no_gc;
3212 2753 : SeqOneByteString* module_bytes = compiled_module->module_bytes();
3213 : SourcePositionTableIterator source_pos_iterator(
3214 2753 : caller->SourcePositionTable());
3215 : DCHECK_EQ(2, caller->deoptimization_data()->length());
3216 : int caller_func_index =
3217 : Smi::cast(caller->deoptimization_data()->get(1))->value();
3218 : const byte* func_bytes =
3219 5506 : module_bytes->GetChars() + compiled_module->module()
3220 5506 : ->functions[caller_func_index]
3221 2753 : .code_start_offset;
3222 81263 : for (RelocIterator it(*caller, RelocInfo::kCodeTargetMask); !it.done();
3223 75757 : it.next()) {
3224 : Code* callee =
3225 75757 : Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
3226 75757 : if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue;
3227 : // TODO(clemensh): Introduce safe_cast<T, bool> which (D)CHECKS
3228 : // (depending on the bool) against limits of T and then static_casts.
3229 51482 : size_t offset_l = it.rinfo()->pc() - caller->instruction_start();
3230 : DCHECK_GE(kMaxInt, offset_l);
3231 25741 : int offset = static_cast<int>(offset_l);
3232 : int byte_pos =
3233 25741 : AdvanceSourcePositionTableIterator(source_pos_iterator, offset);
3234 : int called_func_index =
3235 25741 : ExtractDirectCallIndex(decoder, func_bytes + byte_pos);
3236 51482 : non_compiled_functions.push_back({offset, called_func_index});
3237 : // Call offset one instruction after the call. Remember the last called
3238 : // function before that offset.
3239 25741 : if (offset < call_offset) func_to_return_idx = called_func_index;
3240 : }
3241 : }
3242 :
3243 : // TODO(clemensh): compile all functions in non_compiled_functions in
3244 : // background, wait for func_to_return_idx.
3245 12686 : if (!CompileFunction(isolate, instance, func_to_return_idx)) {
3246 : return {};
3247 : }
3248 :
3249 12686 : if (is_js_to_wasm || patch_caller) {
3250 : DisallowHeapAllocation no_gc;
3251 : // Now patch the code object with all functions which are now compiled.
3252 : int idx = 0;
3253 121463 : for (RelocIterator it(*caller, RelocInfo::kCodeTargetMask); !it.done();
3254 96833 : it.next()) {
3255 : Code* callee =
3256 96833 : Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
3257 96833 : if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue;
3258 : DCHECK_GT(non_compiled_functions.size(), idx);
3259 70606 : int called_func_index = non_compiled_functions[idx].func_index;
3260 : // Check that the callee agrees with our assumed called_func_index.
3261 : DCHECK_IMPLIES(
3262 : callee->deoptimization_data()->length() > 0,
3263 : Smi::cast(callee->deoptimization_data()->get(1))->value() ==
3264 : called_func_index);
3265 : if (is_js_to_wasm) {
3266 : DCHECK_EQ(func_to_return_idx, called_func_index);
3267 : } else {
3268 : DCHECK_EQ(non_compiled_functions[idx].offset,
3269 : it.rinfo()->pc() - caller->instruction_start());
3270 : }
3271 35303 : ++idx;
3272 : Handle<Code> callee_compiled(
3273 70606 : Code::cast(compiled_module->code_table()->get(called_func_index)));
3274 35303 : if (callee_compiled->builtin_index() == Builtins::kWasmCompileLazy) {
3275 : DCHECK_NE(func_to_return_idx, called_func_index);
3276 : continue;
3277 : }
3278 : DCHECK_EQ(Code::WASM_FUNCTION, callee_compiled->kind());
3279 : it.rinfo()->set_target_address(isolate,
3280 14772 : callee_compiled->instruction_start());
3281 : }
3282 : DCHECK_EQ(non_compiled_functions.size(), idx);
3283 : }
3284 :
3285 : Code* ret =
3286 25372 : Code::cast(compiled_module->code_table()->get(func_to_return_idx));
3287 : DCHECK_EQ(Code::WASM_FUNCTION, ret->kind());
3288 : return handle(ret, isolate);
3289 : }
|
