Line data Source code
1 : // Copyright 2017 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/wasm/wasm-code-specialization.h"
6 :
7 : #include "src/assembler-inl.h"
8 : #include "src/objects-inl.h"
9 : #include "src/source-position-table.h"
10 : #include "src/wasm/decoder.h"
11 : #include "src/wasm/wasm-module.h"
12 : #include "src/wasm/wasm-opcodes.h"
13 :
14 : using namespace v8::internal;
15 : using namespace v8::internal::wasm;
16 :
17 : namespace {
18 :
19 81819 : int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) {
20 : DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc));
21 81819 : decoder.Reset(pc + 1, pc + 6);
22 : uint32_t call_idx = decoder.consume_u32v("call index");
23 : DCHECK(decoder.ok());
24 : DCHECK_GE(kMaxInt, call_idx);
25 81819 : return static_cast<int>(call_idx);
26 : }
27 :
28 330246 : int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
29 : size_t offset_l) {
30 : DCHECK_GE(kMaxInt, offset_l);
31 81819 : int offset = static_cast<int>(offset_l);
32 : DCHECK(!iterator.done());
33 : int byte_pos;
34 82809 : do {
35 : byte_pos = iterator.source_position().ScriptOffset();
36 82809 : iterator.Advance();
37 165618 : } while (!iterator.done() && iterator.code_offset() <= offset);
38 81819 : return byte_pos;
39 : }
40 :
41 : class PatchDirectCallsHelper {
42 : public:
43 43745 : PatchDirectCallsHelper(WasmInstanceObject* instance, Code* code)
44 43745 : : source_pos_it(code->SourcePositionTable()), decoder(nullptr, nullptr) {
45 : FixedArray* deopt_data = code->deoptimization_data();
46 : DCHECK_EQ(2, deopt_data->length());
47 43745 : WasmCompiledModule* comp_mod = instance->compiled_module();
48 : int func_index = Smi::cast(deopt_data->get(1))->value();
49 43745 : func_bytes = comp_mod->module_bytes()->GetChars() +
50 87490 : comp_mod->module()->functions[func_index].code_start_offset;
51 43745 : }
52 :
53 : SourcePositionTableIterator source_pos_it;
54 : Decoder decoder;
55 : const byte* func_bytes;
56 : };
57 :
58 378258 : bool IsAtWasmDirectCallTarget(RelocIterator& it) {
59 : DCHECK(RelocInfo::IsCodeTarget(it.rinfo()->rmode()));
60 378258 : Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
61 297063 : return code->kind() == Code::WASM_FUNCTION ||
62 260665 : code->kind() == Code::WASM_TO_JS_FUNCTION ||
63 259855 : code->kind() == Code::WASM_INTERPRETER_ENTRY ||
64 607885 : code->builtin_index() == Builtins::kIllegal ||
65 378258 : code->builtin_index() == Builtins::kWasmCompileLazy;
66 : }
67 :
68 : } // namespace
69 :
70 67256 : CodeSpecialization::CodeSpecialization(Isolate* isolate, Zone* zone)
71 201768 : : objects_to_relocate(isolate->heap(), ZoneAllocationPolicy(zone)) {}
72 :
73 134512 : CodeSpecialization::~CodeSpecialization() {}
74 :
75 28355 : void CodeSpecialization::RelocateMemoryReferences(Address old_start,
76 : uint32_t old_size,
77 : Address new_start,
78 : uint32_t new_size) {
79 : DCHECK(old_mem_start == nullptr && old_mem_size == 0 &&
80 : new_mem_start == nullptr && new_mem_size == 0);
81 : DCHECK(old_start != new_start || old_size != new_size);
82 28355 : old_mem_start = old_start;
83 28355 : old_mem_size = old_size;
84 28355 : new_mem_start = new_start;
85 28355 : new_mem_size = new_size;
86 28355 : }
87 :
88 7624 : void CodeSpecialization::RelocateGlobals(Address old_start, Address new_start) {
89 : DCHECK(old_globals_start == 0 && new_globals_start == 0);
90 : DCHECK(old_start != 0 || new_start != 0);
91 7624 : old_globals_start = old_start;
92 7624 : new_globals_start = new_start;
93 7624 : }
94 :
95 300 : void CodeSpecialization::PatchTableSize(uint32_t old_size, uint32_t new_size) {
96 : DCHECK(old_function_table_size == 0 && new_function_table_size == 0);
97 : DCHECK(old_size != 0 || new_size != 0);
98 300 : old_function_table_size = old_size;
99 300 : new_function_table_size = new_size;
100 300 : }
101 :
102 59782 : void CodeSpecialization::RelocateDirectCalls(
103 : Handle<WasmInstanceObject> instance) {
104 : DCHECK(relocate_direct_calls_instance.is_null());
105 : DCHECK(!instance.is_null());
106 59782 : relocate_direct_calls_instance = instance;
107 59782 : }
108 :
109 3527 : void CodeSpecialization::RelocateObject(Handle<Object> old_obj,
110 : Handle<Object> new_obj) {
111 : DCHECK(!old_obj.is_null() && !new_obj.is_null());
112 3527 : has_objects_to_relocate = true;
113 : objects_to_relocate.Set(*old_obj, new_obj);
114 3527 : }
115 :
116 52314 : bool CodeSpecialization::ApplyToWholeInstance(
117 : WasmInstanceObject* instance, ICacheFlushMode icache_flush_mode) {
118 : DisallowHeapAllocation no_gc;
119 52314 : WasmCompiledModule* compiled_module = instance->compiled_module();
120 : FixedArray* code_table = compiled_module->ptr_to_code_table();
121 52314 : WasmModule* module = compiled_module->module();
122 52314 : std::vector<WasmFunction>* wasm_functions =
123 52314 : &compiled_module->module()->functions;
124 : DCHECK_EQ(wasm_functions->size() +
125 : compiled_module->module()->num_exported_functions,
126 : code_table->length());
127 :
128 : bool changed = false;
129 52314 : int func_index = module->num_imported_functions;
130 :
131 : // Patch all wasm functions.
132 167623 : for (int num_wasm_functions = static_cast<int>(wasm_functions->size());
133 : func_index < num_wasm_functions; ++func_index) {
134 : Code* wasm_function = Code::cast(code_table->get(func_index));
135 115309 : if (wasm_function->builtin_index() == Builtins::kWasmCompileLazy) continue;
136 89930 : changed |= ApplyToWasmCode(wasm_function, icache_flush_mode);
137 : }
138 :
139 : // Patch all exported functions (if we shall relocate direct calls).
140 52314 : if (!relocate_direct_calls_instance.is_null()) {
141 : // If we patch direct calls, the instance registered for that
142 : // (relocate_direct_calls_instance) should match the instance we currently
143 : // patch (instance).
144 : DCHECK_EQ(instance, *relocate_direct_calls_instance);
145 175877 : for (auto exp : module->export_table) {
146 81305 : if (exp.kind != kExternalFunction) continue;
147 : Code* export_wrapper = Code::cast(code_table->get(func_index));
148 : DCHECK_EQ(Code::JS_TO_WASM_FUNCTION, export_wrapper->kind());
149 : // There must be exactly one call to WASM_FUNCTION or WASM_TO_JS_FUNCTION.
150 : for (RelocIterator it(export_wrapper,
151 79831 : RelocInfo::ModeMask(RelocInfo::CODE_TARGET));
152 90958 : ; it.next()) {
153 : DCHECK(!it.done());
154 : // Ignore calls to other builtins like ToNumber.
155 170789 : if (!IsAtWasmDirectCallTarget(it)) continue;
156 79831 : Code* new_code = Code::cast(code_table->get(exp.index));
157 : it.rinfo()->set_target_address(new_code->GetIsolate(),
158 : new_code->instruction_start(),
159 159662 : UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
160 : break;
161 90958 : }
162 : changed = true;
163 79831 : func_index++;
164 : }
165 : DCHECK_EQ(code_table->length(), func_index);
166 : }
167 52314 : return changed;
168 : }
169 :
170 107813 : bool CodeSpecialization::ApplyToWasmCode(Code* code,
171 : ICacheFlushMode icache_flush_mode) {
172 : DisallowHeapAllocation no_gc;
173 : DCHECK_EQ(Code::WASM_FUNCTION, code->kind());
174 :
175 107813 : bool reloc_mem_addr = old_mem_start != new_mem_start;
176 107813 : bool reloc_mem_size = old_mem_size != new_mem_size;
177 107813 : bool reloc_globals = old_globals_start || new_globals_start;
178 107813 : bool patch_table_size = old_function_table_size || new_function_table_size;
179 : bool reloc_direct_calls = !relocate_direct_calls_instance.is_null();
180 107813 : bool reloc_objects = has_objects_to_relocate;
181 :
182 107813 : int reloc_mode = 0;
183 : auto add_mode = [&reloc_mode](bool cond, RelocInfo::Mode mode) {
184 107813 : if (cond) reloc_mode |= RelocInfo::ModeMask(mode);
185 : };
186 : add_mode(reloc_mem_addr, RelocInfo::WASM_MEMORY_REFERENCE);
187 : add_mode(reloc_mem_size, RelocInfo::WASM_MEMORY_SIZE_REFERENCE);
188 : add_mode(reloc_globals, RelocInfo::WASM_GLOBAL_REFERENCE);
189 : add_mode(patch_table_size, RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE);
190 : add_mode(reloc_direct_calls, RelocInfo::CODE_TARGET);
191 : add_mode(reloc_objects, RelocInfo::EMBEDDED_OBJECT);
192 :
193 : std::unique_ptr<PatchDirectCallsHelper> patch_direct_calls_helper;
194 : bool changed = false;
195 :
196 693875 : for (RelocIterator it(code, reloc_mode); !it.done(); it.next()) {
197 586062 : RelocInfo::Mode mode = it.rinfo()->rmode();
198 586062 : switch (mode) {
199 : case RelocInfo::WASM_MEMORY_REFERENCE:
200 : DCHECK(reloc_mem_addr);
201 : it.rinfo()->update_wasm_memory_reference(code->GetIsolate(),
202 : old_mem_start, new_mem_start,
203 128516 : icache_flush_mode);
204 : changed = true;
205 64258 : break;
206 : case RelocInfo::WASM_MEMORY_SIZE_REFERENCE:
207 : DCHECK(reloc_mem_size);
208 : it.rinfo()->update_wasm_memory_size(code->GetIsolate(), old_mem_size,
209 547472 : new_mem_size, icache_flush_mode);
210 : changed = true;
211 273736 : break;
212 : case RelocInfo::WASM_GLOBAL_REFERENCE:
213 : DCHECK(reloc_globals);
214 : it.rinfo()->update_wasm_global_reference(
215 : code->GetIsolate(), old_globals_start, new_globals_start,
216 70706 : icache_flush_mode);
217 : changed = true;
218 35353 : break;
219 : case RelocInfo::CODE_TARGET: {
220 : DCHECK(reloc_direct_calls);
221 : // Skip everything which is not a wasm call (stack checks, traps, ...).
222 207469 : if (!IsAtWasmDirectCallTarget(it)) continue;
223 : // Iterate simultaneously over the relocation information and the source
224 : // position table. For each call in the reloc info, move the source
225 : // position iterator forward to that position to find the byte offset of
226 : // the respective call. Then extract the call index from the module wire
227 : // bytes to find the new compiled function.
228 81819 : size_t offset = it.rinfo()->pc() - code->instruction_start();
229 81819 : if (!patch_direct_calls_helper) {
230 : patch_direct_calls_helper.reset(new PatchDirectCallsHelper(
231 43745 : *relocate_direct_calls_instance, code));
232 : }
233 : int byte_pos = AdvanceSourcePositionTableIterator(
234 81819 : patch_direct_calls_helper->source_pos_it, offset);
235 : int called_func_index = ExtractDirectCallIndex(
236 : patch_direct_calls_helper->decoder,
237 81819 : patch_direct_calls_helper->func_bytes + byte_pos);
238 : FixedArray* code_table =
239 : relocate_direct_calls_instance->compiled_module()
240 81819 : ->ptr_to_code_table();
241 : Code* new_code = Code::cast(code_table->get(called_func_index));
242 : it.rinfo()->set_target_address(new_code->GetIsolate(),
243 : new_code->instruction_start(),
244 163638 : UPDATE_WRITE_BARRIER, icache_flush_mode);
245 : changed = true;
246 81819 : } break;
247 : case RelocInfo::EMBEDDED_OBJECT: {
248 : DCHECK(reloc_objects);
249 5096 : Object* old = it.rinfo()->target_object();
250 : Handle<Object>* new_obj = objects_to_relocate.Find(old);
251 5096 : if (new_obj) {
252 : it.rinfo()->set_target_object(HeapObject::cast(**new_obj),
253 : UPDATE_WRITE_BARRIER,
254 : icache_flush_mode);
255 : changed = true;
256 : }
257 : } break;
258 : case RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE:
259 : DCHECK(patch_table_size);
260 : it.rinfo()->update_wasm_function_table_size_reference(
261 : code->GetIsolate(), old_function_table_size,
262 300 : new_function_table_size, icache_flush_mode);
263 : changed = true;
264 150 : break;
265 : default:
266 0 : UNREACHABLE();
267 : }
268 : }
269 :
270 107813 : return changed;
271 : }
|
