Line data Source code
1 : // Copyright 2017 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/wasm/wasm-code-specialization.h"
6 :
7 : #include "src/assembler-inl.h"
8 : #include "src/objects-inl.h"
9 : #include "src/source-position-table.h"
10 : #include "src/wasm/decoder.h"
11 : #include "src/wasm/wasm-module.h"
12 : #include "src/wasm/wasm-objects-inl.h"
13 : #include "src/wasm/wasm-opcodes.h"
14 :
15 : namespace v8 {
16 : namespace internal {
17 : namespace wasm {
18 :
19 211175 : int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) {
20 : DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc));
21 211175 : decoder.Reset(pc + 1, pc + 6);
22 : uint32_t call_idx = decoder.consume_u32v("call index");
23 : DCHECK(decoder.ok());
24 : DCHECK_GE(kMaxInt, call_idx);
25 211175 : return static_cast<int>(call_idx);
26 : }
27 :
28 : namespace {
29 :
30 731277 : int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
31 : size_t offset_l) {
32 : DCHECK_GE(kMaxInt, offset_l);
33 181902 : int offset = static_cast<int>(offset_l);
34 : DCHECK(!iterator.done());
35 : int byte_pos;
36 183125 : do {
37 : byte_pos = iterator.source_position().ScriptOffset();
38 183125 : iterator.Advance();
39 366250 : } while (!iterator.done() && iterator.code_offset() <= offset);
40 181902 : return byte_pos;
41 : }
42 :
43 : class PatchDirectCallsHelper {
44 : public:
45 147155 : PatchDirectCallsHelper(WasmInstanceObject* instance, Code* code)
46 147155 : : source_pos_it(code->SourcePositionTable()), decoder(nullptr, nullptr) {
47 : FixedArray* deopt_data = code->deoptimization_data();
48 : DCHECK_EQ(2, deopt_data->length());
49 : WasmCompiledModule* comp_mod = instance->compiled_module();
50 : int func_index = Smi::ToInt(deopt_data->get(1));
51 147155 : func_bytes = comp_mod->module_bytes()->GetChars() +
52 294310 : comp_mod->module()->functions[func_index].code.offset();
53 147155 : }
54 :
55 : SourcePositionTableIterator source_pos_it;
56 : Decoder decoder;
57 : const byte* func_bytes;
58 : };
59 :
60 847192 : bool IsAtWasmDirectCallTarget(RelocIterator& it) {
61 : DCHECK(RelocInfo::IsCodeTarget(it.rinfo()->rmode()));
62 847192 : Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
63 662404 : return code->kind() == Code::WASM_FUNCTION ||
64 655591 : code->kind() == Code::WASM_TO_JS_FUNCTION ||
65 655051 : code->kind() == Code::WASM_INTERPRETER_ENTRY ||
66 1358287 : code->builtin_index() == Builtins::kIllegal ||
67 847192 : code->builtin_index() == Builtins::kWasmCompileLazy;
68 : }
69 :
70 : } // namespace
71 :
72 333564 : CodeSpecialization::CodeSpecialization(Isolate* isolate, Zone* zone) {}
73 :
74 333564 : CodeSpecialization::~CodeSpecialization() {}
75 :
76 152712 : void CodeSpecialization::RelocateWasmContextReferences(Address new_context) {
77 : DCHECK_NOT_NULL(new_context);
78 : DCHECK_NULL(new_wasm_context_address);
79 152712 : new_wasm_context_address = new_context;
80 152712 : }
81 :
82 2000 : void CodeSpecialization::PatchTableSize(uint32_t old_size, uint32_t new_size) {
83 : DCHECK(old_function_table_size == 0 && new_function_table_size == 0);
84 2000 : old_function_table_size = old_size;
85 2000 : new_function_table_size = new_size;
86 2000 : }
87 :
88 163921 : void CodeSpecialization::RelocateDirectCalls(
89 : Handle<WasmInstanceObject> instance) {
90 : DCHECK(relocate_direct_calls_instance.is_null());
91 : DCHECK(!instance.is_null());
92 163921 : relocate_direct_calls_instance = instance;
93 163921 : }
94 :
95 6128 : void CodeSpecialization::RelocatePointer(Address old_ptr, Address new_ptr) {
96 12256 : pointers_to_relocate.insert(std::make_pair(old_ptr, new_ptr));
97 6128 : }
98 :
99 153782 : bool CodeSpecialization::ApplyToWholeInstance(
100 : WasmInstanceObject* instance, ICacheFlushMode icache_flush_mode) {
101 : DisallowHeapAllocation no_gc;
102 : WasmCompiledModule* compiled_module = instance->compiled_module();
103 : FixedArray* code_table = compiled_module->ptr_to_code_table();
104 153782 : WasmModule* module = compiled_module->module();
105 153782 : std::vector<WasmFunction>* wasm_functions =
106 153782 : &compiled_module->module()->functions;
107 : DCHECK_EQ(wasm_functions->size(), code_table->length());
108 : DCHECK_EQ(compiled_module->export_wrappers()->length(),
109 : compiled_module->module()->num_exported_functions);
110 :
111 : bool changed = false;
112 153782 : int func_index = module->num_imported_functions;
113 :
114 : // Patch all wasm functions.
115 365762 : for (int num_wasm_functions = static_cast<int>(wasm_functions->size());
116 : func_index < num_wasm_functions; ++func_index) {
117 : Code* wasm_function = Code::cast(code_table->get(func_index));
118 211980 : if (wasm_function->kind() != Code::WASM_FUNCTION) continue;
119 189082 : changed |= ApplyToWasmCode(wasm_function, icache_flush_mode);
120 : }
121 :
122 : // Patch all exported functions (JS_TO_WASM_FUNCTION).
123 : int reloc_mode = 0;
124 : // We need to patch WASM_CONTEXT_REFERENCE to put the correct address.
125 153782 : if (new_wasm_context_address) {
126 : reloc_mode |= RelocInfo::ModeMask(RelocInfo::WASM_CONTEXT_REFERENCE);
127 : }
128 : // Patch CODE_TARGET if we shall relocate direct calls. If we patch direct
129 : // calls, the instance registered for that (relocate_direct_calls_instance)
130 : // should match the instance we currently patch (instance).
131 153782 : if (!relocate_direct_calls_instance.is_null()) {
132 : DCHECK_EQ(instance, *relocate_direct_calls_instance);
133 152662 : reloc_mode |= RelocInfo::ModeMask(RelocInfo::CODE_TARGET);
134 : }
135 153782 : if (!reloc_mode) return changed;
136 : int wrapper_index = 0;
137 494267 : for (auto exp : module->export_table) {
138 188943 : if (exp.kind != kExternalFunction) continue;
139 : Code* export_wrapper =
140 373706 : Code::cast(compiled_module->export_wrappers()->get(wrapper_index));
141 : DCHECK_EQ(Code::JS_TO_WASM_FUNCTION, export_wrapper->kind());
142 703644 : for (RelocIterator it(export_wrapper, reloc_mode); !it.done(); it.next()) {
143 516791 : RelocInfo::Mode mode = it.rinfo()->rmode();
144 516791 : switch (mode) {
145 : case RelocInfo::WASM_CONTEXT_REFERENCE:
146 : it.rinfo()->set_wasm_context_reference(export_wrapper->GetIsolate(),
147 : new_wasm_context_address,
148 373706 : icache_flush_mode);
149 186853 : break;
150 : case RelocInfo::CODE_TARGET: {
151 : // Ignore calls to other builtins like ToNumber.
152 329938 : if (!IsAtWasmDirectCallTarget(it)) continue;
153 186853 : Code* new_code = Code::cast(code_table->get(exp.index));
154 : it.rinfo()->set_target_address(
155 : new_code->GetIsolate(), new_code->instruction_start(),
156 373706 : UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
157 186853 : } break;
158 : default:
159 0 : UNREACHABLE();
160 : }
161 : }
162 : changed = true;
163 186853 : ++wrapper_index;
164 : }
165 : DCHECK_EQ(code_table->length(), func_index);
166 : DCHECK_EQ(compiled_module->export_wrappers()->length(), wrapper_index);
167 : return changed;
168 : }
169 :
170 201700 : bool CodeSpecialization::ApplyToWasmCode(Code* code,
171 : ICacheFlushMode icache_flush_mode) {
172 : DisallowHeapAllocation no_gc;
173 : DCHECK_EQ(Code::WASM_FUNCTION, code->kind());
174 :
175 201700 : bool patch_table_size = old_function_table_size || new_function_table_size;
176 : bool reloc_direct_calls = !relocate_direct_calls_instance.is_null();
177 : bool reloc_pointers = pointers_to_relocate.size() > 0;
178 :
179 201700 : int reloc_mode = 0;
180 : auto add_mode = [&reloc_mode](bool cond, RelocInfo::Mode mode) {
181 201700 : if (cond) reloc_mode |= RelocInfo::ModeMask(mode);
182 : };
183 : add_mode(patch_table_size, RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE);
184 : add_mode(reloc_direct_calls, RelocInfo::CODE_TARGET);
185 : add_mode(reloc_pointers, RelocInfo::WASM_GLOBAL_HANDLE);
186 :
187 : std::unique_ptr<PatchDirectCallsHelper> patch_direct_calls_helper;
188 : bool changed = false;
189 :
190 725831 : for (RelocIterator it(code, reloc_mode); !it.done(); it.next()) {
191 524131 : RelocInfo::Mode mode = it.rinfo()->rmode();
192 524131 : switch (mode) {
193 : case RelocInfo::CODE_TARGET: {
194 : DCHECK(reloc_direct_calls);
195 : // Skip everything which is not a wasm call (stack checks, traps, ...).
196 517254 : if (!IsAtWasmDirectCallTarget(it)) continue;
197 : // Iterate simultaneously over the relocation information and the source
198 : // position table. For each call in the reloc info, move the source
199 : // position iterator forward to that position to find the byte offset of
200 : // the respective call. Then extract the call index from the module wire
201 : // bytes to find the new compiled function.
202 181902 : size_t offset = it.rinfo()->pc() - code->instruction_start();
203 181902 : if (!patch_direct_calls_helper) {
204 : patch_direct_calls_helper.reset(new PatchDirectCallsHelper(
205 147155 : *relocate_direct_calls_instance, code));
206 : }
207 : int byte_pos = AdvanceSourcePositionTableIterator(
208 181902 : patch_direct_calls_helper->source_pos_it, offset);
209 : int called_func_index = ExtractDirectCallIndex(
210 : patch_direct_calls_helper->decoder,
211 181902 : patch_direct_calls_helper->func_bytes + byte_pos);
212 : FixedArray* code_table =
213 : relocate_direct_calls_instance->compiled_module()
214 : ->ptr_to_code_table();
215 : Code* new_code = Code::cast(code_table->get(called_func_index));
216 : it.rinfo()->set_target_address(new_code->GetIsolate(),
217 : new_code->instruction_start(),
218 363804 : UPDATE_WRITE_BARRIER, icache_flush_mode);
219 : changed = true;
220 181902 : } break;
221 : case RelocInfo::WASM_GLOBAL_HANDLE: {
222 : DCHECK(reloc_pointers);
223 5627 : Address old_ptr = it.rinfo()->global_handle();
224 5627 : if (pointers_to_relocate.count(old_ptr) == 1) {
225 5627 : Address new_ptr = pointers_to_relocate[old_ptr];
226 : it.rinfo()->set_global_handle(code->GetIsolate(), new_ptr,
227 5627 : icache_flush_mode);
228 : changed = true;
229 : }
230 5627 : } break;
231 : case RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE:
232 : DCHECK(patch_table_size);
233 : it.rinfo()->update_wasm_function_table_size_reference(
234 : code->GetIsolate(), old_function_table_size,
235 2500 : new_function_table_size, icache_flush_mode);
236 : changed = true;
237 1250 : break;
238 : default:
239 0 : UNREACHABLE();
240 : }
241 : }
242 :
243 201700 : return changed;
244 : }
245 :
246 : } // namespace wasm
247 : } // namespace internal
248 : } // namespace v8
|
