Line data Source code
1 : // Copyright 2017 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/wasm/baseline/liftoff-assembler.h"
6 :
7 : #include <sstream>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/base/optional.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/wasm-compiler.h"
13 : #include "src/macro-assembler-inl.h"
14 : #include "src/ostreams.h"
15 : #include "src/wasm/function-body-decoder-impl.h"
16 : #include "src/wasm/wasm-linkage.h"
17 : #include "src/wasm/wasm-opcodes.h"
18 :
19 : namespace v8 {
20 : namespace internal {
21 : namespace wasm {
22 :
23 : using VarState = LiftoffAssembler::VarState;
24 :
25 : namespace {
26 :
27 : #define __ asm_->
28 :
29 : #define TRACE(...) \
30 : do { \
31 : if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
32 : } while (false)
33 :
34 : class StackTransferRecipe {
35 : struct RegisterMove {
36 : LiftoffRegister src;
37 : ValueType type;
38 : constexpr RegisterMove(LiftoffRegister src, ValueType type)
39 : : src(src), type(type) {}
40 : };
41 :
42 : struct RegisterLoad {
43 : enum LoadKind : uint8_t {
44 : kConstant, // load a constant value into a register.
45 : kStack, // fill a register from a stack slot.
46 : kLowHalfStack, // fill a register from the low half of a stack slot.
47 : kHighHalfStack // fill a register from the high half of a stack slot.
48 : };
49 :
50 : LoadKind kind;
51 : ValueType type;
52 : int32_t value; // i32 constant value or stack index, depending on kind.
53 :
54 : // Named constructors.
55 : static RegisterLoad Const(WasmValue constant) {
56 53283 : if (constant.type() == kWasmI32) {
57 : return {kConstant, kWasmI32, constant.to_i32()};
58 : }
59 : DCHECK_EQ(kWasmI64, constant.type());
60 : DCHECK_EQ(constant.to_i32_unchecked(), constant.to_i64_unchecked());
61 : return {kConstant, kWasmI64, constant.to_i32_unchecked()};
62 : }
63 : static RegisterLoad Stack(int32_t stack_index, ValueType type) {
64 : return {kStack, type, stack_index};
65 : }
66 : static RegisterLoad HalfStack(int32_t stack_index, RegPairHalf half) {
67 : return {half == kLowWord ? kLowHalfStack : kHighHalfStack, kWasmI32,
68 : stack_index};
69 : }
70 :
71 : private:
72 : RegisterLoad(LoadKind kind, ValueType type, int32_t value)
73 : : kind(kind), type(type), value(value) {}
74 : };
75 :
76 : public:
77 1015272 : explicit StackTransferRecipe(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {}
78 507904 : ~StackTransferRecipe() { Execute(); }
79 :
80 : void Execute() {
81 : // First, execute register moves. Then load constants and stack values into
82 : // registers.
83 555381 : ExecuteMoves();
84 : DCHECK(move_dst_regs_.is_empty());
85 555377 : ExecuteLoads();
86 : DCHECK(load_dst_regs_.is_empty());
87 : }
88 :
89 19700 : void TransferStackSlot(const LiftoffAssembler::CacheState& dst_state,
90 : uint32_t dst_index,
91 : const LiftoffAssembler::CacheState& src_state,
92 : uint32_t src_index) {
93 19700 : const VarState& dst = dst_state.stack_state[dst_index];
94 19700 : const VarState& src = src_state.stack_state[src_index];
95 : DCHECK_EQ(dst.type(), src.type());
96 19700 : switch (dst.loc()) {
97 : case VarState::kStack:
98 1596 : switch (src.loc()) {
99 : case VarState::kStack:
100 1101 : if (src_index == dst_index) break;
101 6 : asm_->MoveStackValue(dst_index, src_index, src.type());
102 6 : break;
103 : case VarState::kRegister:
104 400 : asm_->Spill(dst_index, src.reg(), src.type());
105 400 : break;
106 : case VarState::kIntConst:
107 98 : asm_->Spill(dst_index, src.constant());
108 98 : break;
109 : }
110 : break;
111 : case VarState::kRegister:
112 17826 : LoadIntoRegister(dst.reg(), src, src_index);
113 17822 : break;
114 : case VarState::kIntConst:
115 : DCHECK_EQ(dst, src);
116 : break;
117 : }
118 19696 : }
119 :
120 515337 : void LoadIntoRegister(LiftoffRegister dst,
121 : const LiftoffAssembler::VarState& src,
122 : uint32_t src_index) {
123 515337 : switch (src.loc()) {
124 : case VarState::kStack:
125 : LoadStackSlot(dst, src_index, src.type());
126 : break;
127 : case VarState::kRegister:
128 : DCHECK_EQ(dst.reg_class(), src.reg_class());
129 455606 : if (dst != src.reg()) MoveRegister(dst, src.reg(), src.type());
130 : break;
131 : case VarState::kIntConst:
132 : LoadConstant(dst, src.constant());
133 : break;
134 : }
135 515342 : }
136 :
137 : void LoadI64HalfIntoRegister(LiftoffRegister dst,
138 : const LiftoffAssembler::VarState& src,
139 : uint32_t index, RegPairHalf half) {
140 : // Use CHECK such that the remaining code is statically dead if
141 : // {kNeedI64RegPair} is false.
142 : CHECK(kNeedI64RegPair);
143 : DCHECK_EQ(kWasmI64, src.type());
144 : switch (src.loc()) {
145 : case VarState::kStack:
146 : LoadI64HalfStackSlot(dst, index, half);
147 : break;
148 : case VarState::kRegister: {
149 : LiftoffRegister src_half =
150 : half == kLowWord ? src.reg().low() : src.reg().high();
151 : if (dst != src_half) MoveRegister(dst, src_half, kWasmI32);
152 : break;
153 : }
154 : case VarState::kIntConst:
155 : int32_t value = src.i32_const();
156 : // The high word is the sign extension of the low word.
157 : if (half == kHighWord) value = value >> 31;
158 : LoadConstant(dst, WasmValue(value));
159 : break;
160 : }
161 : }
162 :
163 409752 : void MoveRegister(LiftoffRegister dst, LiftoffRegister src, ValueType type) {
164 : DCHECK_NE(dst, src);
165 : DCHECK_EQ(dst.reg_class(), src.reg_class());
166 : DCHECK_EQ(reg_class_for(type), src.reg_class());
167 : if (src.is_pair()) {
168 : DCHECK_EQ(kWasmI64, type);
169 : if (dst.low() != src.low()) MoveRegister(dst.low(), src.low(), kWasmI32);
170 : if (dst.high() != src.high())
171 : MoveRegister(dst.high(), src.high(), kWasmI32);
172 : return;
173 : }
174 410779 : if (move_dst_regs_.has(dst)) {
175 : DCHECK_EQ(register_move(dst)->src, src);
176 : // Non-fp registers can only occur with the exact same type.
177 : DCHECK_IMPLIES(!dst.is_fp(), register_move(dst)->type == type);
178 : // It can happen that one fp register holds both the f32 zero and the f64
179 : // zero, as the initial value for local variables. Move the value as f64
180 : // in that case.
181 15 : if (type == kWasmF64) register_move(dst)->type = kWasmF64;
182 : return;
183 : }
184 : move_dst_regs_.set(dst);
185 454751 : ++*src_reg_use_count(src);
186 454751 : *register_move(dst) = {src, type};
187 : }
188 :
189 : void LoadConstant(LiftoffRegister dst, WasmValue value) {
190 : DCHECK(!load_dst_regs_.has(dst));
191 : load_dst_regs_.set(dst);
192 : if (dst.is_pair()) {
193 : DCHECK_EQ(kWasmI64, value.type());
194 : int64_t i64 = value.to_i64();
195 : *register_load(dst.low()) =
196 : RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64)));
197 : *register_load(dst.high()) =
198 : RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64 >> 32)));
199 : } else {
200 53283 : *register_load(dst) = RegisterLoad::Const(value);
201 : }
202 : }
203 :
204 : void LoadStackSlot(LiftoffRegister dst, uint32_t stack_index,
205 : ValueType type) {
206 6600 : if (load_dst_regs_.has(dst)) {
207 : // It can happen that we spilled the same register to different stack
208 : // slots, and then we reload them later into the same dst register.
209 : // In that case, it is enough to load one of the stack slots.
210 : return;
211 : }
212 : load_dst_regs_.set(dst);
213 : if (dst.is_pair()) {
214 : DCHECK_EQ(kWasmI64, type);
215 : *register_load(dst.low()) =
216 : RegisterLoad::HalfStack(stack_index, kLowWord);
217 : *register_load(dst.high()) =
218 : RegisterLoad::HalfStack(stack_index, kHighWord);
219 : } else {
220 6597 : *register_load(dst) = RegisterLoad::Stack(stack_index, type);
221 : }
222 : }
223 :
224 : void LoadI64HalfStackSlot(LiftoffRegister dst, uint32_t stack_index,
225 : RegPairHalf half) {
226 : if (load_dst_regs_.has(dst)) {
227 : // It can happen that we spilled the same register to different stack
228 : // slots, and then we reload them later into the same dst register.
229 : // In that case, it is enough to load one of the stack slots.
230 : return;
231 : }
232 : load_dst_regs_.set(dst);
233 : *register_load(dst) = RegisterLoad::HalfStack(stack_index, half);
234 : }
235 :
236 : private:
237 : using MovesStorage =
238 : std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterMove),
239 : alignof(RegisterMove)>::type;
240 : using LoadsStorage =
241 : std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterLoad),
242 : alignof(RegisterLoad)>::type;
243 :
244 : ASSERT_TRIVIALLY_COPYABLE(RegisterMove);
245 : ASSERT_TRIVIALLY_COPYABLE(RegisterLoad);
246 :
247 : MovesStorage register_moves_; // uninitialized
248 : LoadsStorage register_loads_; // uninitialized
249 : int src_reg_use_count_[kAfterMaxLiftoffRegCode] = {0};
250 : LiftoffRegList move_dst_regs_;
251 : LiftoffRegList load_dst_regs_;
252 : LiftoffAssembler* const asm_;
253 :
254 : RegisterMove* register_move(LiftoffRegister reg) {
255 1364252 : return reinterpret_cast<RegisterMove*>(®ister_moves_) +
256 1364252 : reg.liftoff_code();
257 : }
258 : RegisterLoad* register_load(LiftoffRegister reg) {
259 119913 : return reinterpret_cast<RegisterLoad*>(®ister_loads_) +
260 119913 : reg.liftoff_code();
261 : }
262 : int* src_reg_use_count(LiftoffRegister reg) {
263 1363298 : return src_reg_use_count_ + reg.liftoff_code();
264 : }
265 :
266 454585 : void ExecuteMove(LiftoffRegister dst) {
267 : RegisterMove* move = register_move(dst);
268 : DCHECK_EQ(0, *src_reg_use_count(dst));
269 454585 : asm_->Move(dst, move->src, move->type);
270 454583 : ClearExecutedMove(dst);
271 454588 : }
272 :
273 454747 : void ClearExecutedMove(LiftoffRegister dst) {
274 : DCHECK(move_dst_regs_.has(dst));
275 : move_dst_regs_.clear(dst);
276 : RegisterMove* move = register_move(dst);
277 : DCHECK_LT(0, *src_reg_use_count(move->src));
278 454747 : if (--*src_reg_use_count(move->src)) return;
279 : // src count dropped to zero. If this is a destination register, execute
280 : // that move now.
281 454683 : if (!move_dst_regs_.has(move->src)) return;
282 36792 : ExecuteMove(move->src);
283 : }
284 :
285 555318 : void ExecuteMoves() {
286 : // Execute all moves whose {dst} is not being used as src in another move.
287 : // If any src count drops to zero, also (transitively) execute the
288 : // corresponding move to that register.
289 1010070 : for (LiftoffRegister dst : move_dst_regs_) {
290 : // Check if already handled via transitivity in {ClearExecutedMove}.
291 454753 : if (!move_dst_regs_.has(dst)) continue;
292 453800 : if (*src_reg_use_count(dst)) continue;
293 417794 : ExecuteMove(dst);
294 : }
295 :
296 : // All remaining moves are parts of a cycle. Just spill the first one, then
297 : // process all remaining moves in that cycle. Repeat for all cycles.
298 555317 : uint32_t next_spill_slot = asm_->cache_state()->stack_height();
299 555647 : while (!move_dst_regs_.is_empty()) {
300 : // TODO(clemensh): Use an unused register if available.
301 : LiftoffRegister dst = move_dst_regs_.GetFirstRegSet();
302 : RegisterMove* move = register_move(dst);
303 166 : LiftoffRegister spill_reg = move->src;
304 166 : asm_->Spill(next_spill_slot, spill_reg, move->type);
305 : // Remember to reload into the destination register later.
306 166 : LoadStackSlot(dst, next_spill_slot, move->type);
307 166 : ++next_spill_slot;
308 166 : ClearExecutedMove(dst);
309 : }
310 555316 : }
311 :
312 555483 : void ExecuteLoads() {
313 615637 : for (LiftoffRegister dst : load_dst_regs_) {
314 : RegisterLoad* load = register_load(dst);
315 60033 : switch (load->kind) {
316 : case RegisterLoad::kConstant:
317 162906 : asm_->LoadConstant(dst, load->type == kWasmI64
318 2634 : ? WasmValue(int64_t{load->value})
319 106889 : : WasmValue(int32_t{load->value}));
320 : break;
321 : case RegisterLoad::kStack:
322 6596 : asm_->Fill(dst, load->value, load->type);
323 6596 : break;
324 : case RegisterLoad::kLowHalfStack:
325 : // Half of a register pair, {dst} must be a gp register.
326 : asm_->FillI64Half(dst.gp(), load->value, kLowWord);
327 : break;
328 : case RegisterLoad::kHighHalfStack:
329 : // Half of a register pair, {dst} must be a gp register.
330 : asm_->FillI64Half(dst.gp(), load->value, kHighWord);
331 : break;
332 : }
333 : }
334 555604 : load_dst_regs_ = {};
335 555604 : }
336 :
337 : DISALLOW_COPY_AND_ASSIGN(StackTransferRecipe);
338 : };
339 :
340 159660 : class RegisterReuseMap {
341 : public:
342 78 : void Add(LiftoffRegister src, LiftoffRegister dst) {
343 78 : if (auto previous = Lookup(src)) {
344 : DCHECK_EQ(previous, dst);
345 : return;
346 : }
347 72 : map_.emplace_back(src);
348 72 : map_.emplace_back(dst);
349 : }
350 :
351 : base::Optional<LiftoffRegister> Lookup(LiftoffRegister src) {
352 177 : for (auto it = map_.begin(), end = map_.end(); it != end; it += 2) {
353 45 : if (it->is_pair() == src.is_pair() && *it == src) return *(it + 1);
354 : }
355 : return {};
356 : }
357 :
358 : private:
359 : // {map_} holds pairs of <src, dst>.
360 : base::SmallVector<LiftoffRegister, 8> map_;
361 : };
362 :
363 : enum MergeKeepStackSlots : bool {
364 : kKeepStackSlots = true,
365 : kTurnStackSlotsIntoRegisters = false
366 : };
367 : enum MergeAllowConstants : bool {
368 : kConstantsAllowed = true,
369 : kConstantsNotAllowed = false
370 : };
371 : enum ReuseRegisters : bool {
372 : kReuseRegisters = true,
373 : kNoReuseRegisters = false
374 : };
375 159660 : void InitMergeRegion(LiftoffAssembler::CacheState* state,
376 : const VarState* source, VarState* target, uint32_t count,
377 : MergeKeepStackSlots keep_stack_slots,
378 : MergeAllowConstants allow_constants,
379 : ReuseRegisters reuse_registers, LiftoffRegList used_regs) {
380 : RegisterReuseMap register_reuse_map;
381 176042 : for (const VarState* source_end = source + count; source < source_end;
382 : ++source, ++target) {
383 32764 : if ((source->is_stack() && keep_stack_slots) ||
384 1659 : (source->is_const() && allow_constants)) {
385 859 : *target = *source;
386 1829 : continue;
387 : }
388 : base::Optional<LiftoffRegister> reg;
389 : // First try: Keep the same register, if it's free.
390 29570 : if (source->is_reg() && state->is_free(source->reg())) {
391 : reg = source->reg();
392 : }
393 : // Second try: Use the same register we used before (if we reuse registers).
394 15523 : if (!reg && reuse_registers) {
395 : reg = register_reuse_map.Lookup(source->reg());
396 : }
397 : // Third try: Use any free register.
398 : RegClass rc = reg_class_for(source->type());
399 17294 : if (!reg && state->has_unused_register(rc, used_regs)) {
400 : reg = state->unused_register(rc, used_regs);
401 : }
402 15523 : if (!reg) {
403 : // No free register; make this a stack slot.
404 111 : *target = VarState(source->type());
405 111 : continue;
406 : }
407 15412 : if (reuse_registers) register_reuse_map.Add(source->reg(), *reg);
408 : state->inc_used(*reg);
409 15412 : *target = VarState(source->type(), *reg);
410 : }
411 159660 : }
412 :
413 : } // namespace
414 :
415 : // TODO(clemensh): Don't copy the full parent state (this makes us N^2).
416 53222 : void LiftoffAssembler::CacheState::InitMerge(const CacheState& source,
417 : uint32_t num_locals,
418 : uint32_t arity,
419 : uint32_t stack_depth) {
420 : // |------locals------|---(in between)----|--(discarded)--|----merge----|
421 : // <-- num_locals --> <-- stack_depth -->^stack_base <-- arity -->
422 :
423 53222 : uint32_t stack_base = stack_depth + num_locals;
424 53222 : uint32_t target_height = stack_base + arity;
425 53222 : uint32_t discarded = source.stack_height() - target_height;
426 : DCHECK(stack_state.empty());
427 :
428 : DCHECK_GE(source.stack_height(), stack_base);
429 53222 : stack_state.resize_no_init(target_height);
430 :
431 : const VarState* source_begin = source.stack_state.data();
432 : VarState* target_begin = stack_state.data();
433 :
434 : // Try to keep locals and the merge region in their registers. Register used
435 : // multiple times need to be copied to another free register. Compute the list
436 : // of used registers.
437 : LiftoffRegList used_regs;
438 135640 : for (auto& src : VectorOf(source_begin, num_locals)) {
439 14600 : if (src.is_reg()) used_regs.set(src.reg());
440 : }
441 109460 : for (auto& src : VectorOf(source_begin + stack_base + discarded, arity)) {
442 1510 : if (src.is_reg()) used_regs.set(src.reg());
443 : }
444 :
445 : // Initialize the merge region. If this region moves, try to turn stack slots
446 : // into registers since we need to load the value anyways.
447 : MergeKeepStackSlots keep_merge_stack_slots =
448 53220 : discarded == 0 ? kKeepStackSlots : kTurnStackSlotsIntoRegisters;
449 53220 : InitMergeRegion(this, source_begin + stack_base + discarded,
450 : target_begin + stack_base, arity, keep_merge_stack_slots,
451 53220 : kConstantsNotAllowed, kNoReuseRegisters, used_regs);
452 :
453 : // Initialize the locals region. Here, stack slots stay stack slots (because
454 : // they do not move). Try to keep register in registers, but avoid duplicates.
455 : InitMergeRegion(this, source_begin, target_begin, num_locals, kKeepStackSlots,
456 53225 : kConstantsNotAllowed, kNoReuseRegisters, used_regs);
457 : // Sanity check: All the {used_regs} are really in use now.
458 : DCHECK_EQ(used_regs, used_registers & used_regs);
459 :
460 : // Last, initialize the section in between. Here, constants are allowed, but
461 : // registers which are already used for the merge region or locals must be
462 : // moved to other registers or spilled. If a register appears twice in the
463 : // source region, ensure to use the same register twice in the target region.
464 53223 : InitMergeRegion(this, source_begin + num_locals, target_begin + num_locals,
465 : stack_depth, kKeepStackSlots, kConstantsAllowed,
466 53223 : kReuseRegisters, used_regs);
467 53223 : }
468 :
469 53978 : void LiftoffAssembler::CacheState::Steal(const CacheState& source) {
470 : // Just use the move assignment operator.
471 53978 : *this = std::move(source);
472 53979 : }
473 :
474 2350 : void LiftoffAssembler::CacheState::Split(const CacheState& source) {
475 : // Call the private copy assignment operator.
476 2350 : *this = source;
477 2352 : }
478 :
479 : namespace {
480 :
481 : constexpr AssemblerOptions DefaultLiftoffOptions() {
482 460740 : return AssemblerOptions{};
483 : }
484 :
485 : } // namespace
486 :
487 : // TODO(clemensh): Provide a reasonably sized buffer, based on wasm function
488 : // size.
489 460740 : LiftoffAssembler::LiftoffAssembler(std::unique_ptr<AssemblerBuffer> buffer)
490 461038 : : TurboAssembler(nullptr, DefaultLiftoffOptions(), CodeObjectRequired::kNo,
491 1382816 : std::move(buffer)) {
492 : set_abort_hard(true); // Avoid calls to Abort.
493 461038 : }
494 :
495 921889 : LiftoffAssembler::~LiftoffAssembler() {
496 460733 : if (num_locals_ > kInlineLocalTypes) {
497 1746 : free(more_local_types_);
498 : }
499 461156 : }
500 :
501 1341471 : LiftoffRegister LiftoffAssembler::PopToRegister(LiftoffRegList pinned) {
502 : DCHECK(!cache_state_.stack_state.empty());
503 1341471 : VarState slot = cache_state_.stack_state.back();
504 : cache_state_.stack_state.pop_back();
505 1341471 : switch (slot.loc()) {
506 : case VarState::kStack: {
507 : LiftoffRegister reg =
508 : GetUnusedRegister(reg_class_for(slot.type()), pinned);
509 822 : Fill(reg, cache_state_.stack_height(), slot.type());
510 411 : return reg;
511 : }
512 : case VarState::kRegister:
513 : cache_state_.dec_used(slot.reg());
514 : return slot.reg();
515 : case VarState::kIntConst: {
516 : RegClass rc =
517 : kNeedI64RegPair && slot.type() == kWasmI64 ? kGpRegPair : kGpReg;
518 : LiftoffRegister reg = GetUnusedRegister(rc, pinned);
519 874278 : LoadConstant(reg, slot.constant());
520 874169 : return reg;
521 : }
522 : }
523 0 : UNREACHABLE();
524 : }
525 :
526 2036 : void LiftoffAssembler::MergeFullStackWith(const CacheState& target,
527 : const CacheState& source) {
528 : DCHECK_EQ(source.stack_height(), target.stack_height());
529 : // TODO(clemensh): Reuse the same StackTransferRecipe object to save some
530 : // allocations.
531 : StackTransferRecipe transfers(this);
532 10428 : for (uint32_t i = 0, e = source.stack_height(); i < e; ++i) {
533 4197 : transfers.TransferStackSlot(target, i, source, i);
534 : }
535 2041 : }
536 :
537 52870 : void LiftoffAssembler::MergeStackWith(const CacheState& target,
538 : uint32_t arity) {
539 : // Before: ----------------|----- (discarded) ----|--- arity ---|
540 : // ^target_stack_height ^stack_base ^stack_height
541 : // After: ----|-- arity --|
542 : // ^ ^target_stack_height
543 : // ^target_stack_base
544 : uint32_t stack_height = cache_state_.stack_height();
545 : uint32_t target_stack_height = target.stack_height();
546 : DCHECK_LE(target_stack_height, stack_height);
547 : DCHECK_LE(arity, target_stack_height);
548 52870 : uint32_t stack_base = stack_height - arity;
549 52870 : uint32_t target_stack_base = target_stack_height - arity;
550 : StackTransferRecipe transfers(this);
551 81664 : for (uint32_t i = 0; i < target_stack_base; ++i) {
552 14395 : transfers.TransferStackSlot(target, i, cache_state_, i);
553 : }
554 55112 : for (uint32_t i = 0; i < arity; ++i) {
555 1121 : transfers.TransferStackSlot(target, target_stack_base + i, cache_state_,
556 1121 : stack_base + i);
557 : }
558 52877 : }
559 :
560 930 : void LiftoffAssembler::Spill(uint32_t index) {
561 930 : auto& slot = cache_state_.stack_state[index];
562 930 : switch (slot.loc()) {
563 : case VarState::kStack:
564 : return;
565 : case VarState::kRegister:
566 615 : Spill(index, slot.reg(), slot.type());
567 : cache_state_.dec_used(slot.reg());
568 : break;
569 : case VarState::kIntConst:
570 165 : Spill(index, slot.constant());
571 166 : break;
572 : }
573 : slot.MakeStack();
574 : }
575 :
576 982 : void LiftoffAssembler::SpillLocals() {
577 2846 : for (uint32_t i = 0; i < num_locals_; ++i) {
578 930 : Spill(i);
579 : }
580 984 : }
581 :
582 27460 : void LiftoffAssembler::SpillAllRegisters() {
583 28528 : for (uint32_t i = 0, e = cache_state_.stack_height(); i < e; ++i) {
584 533 : auto& slot = cache_state_.stack_state[i];
585 533 : if (!slot.is_reg()) continue;
586 405 : Spill(i, slot.reg(), slot.type());
587 : slot.MakeStack();
588 : }
589 : cache_state_.reset_used_registers();
590 27461 : }
591 :
592 47753 : void LiftoffAssembler::PrepareCall(FunctionSig* sig,
593 : compiler::CallDescriptor* call_descriptor,
594 : Register* target,
595 : Register* target_instance) {
596 47753 : uint32_t num_params = static_cast<uint32_t>(sig->parameter_count());
597 : // Input 0 is the call target.
598 : constexpr size_t kInputShift = 1;
599 :
600 : // Spill all cache slots which are not being used as parameters.
601 : // Don't update any register use counters, they will be reset later anyway.
602 180882 : for (uint32_t idx = 0, end = cache_state_.stack_height() - num_params;
603 180882 : idx < end; ++idx) {
604 133130 : VarState& slot = cache_state_.stack_state[idx];
605 133130 : if (!slot.is_reg()) continue;
606 8611 : Spill(idx, slot.reg(), slot.type());
607 : slot.MakeStack();
608 : }
609 :
610 : LiftoffStackSlots stack_slots(this);
611 : StackTransferRecipe stack_transfers(this);
612 : LiftoffRegList param_regs;
613 :
614 : // Move the target instance (if supplied) into the correct instance register.
615 : compiler::LinkageLocation instance_loc =
616 : call_descriptor->GetInputLocation(kInputShift);
617 : DCHECK(instance_loc.IsRegister() && !instance_loc.IsAnyRegister());
618 : Register instance_reg = Register::from_code(instance_loc.AsRegister());
619 : param_regs.set(instance_reg);
620 91909 : if (target_instance && *target_instance != instance_reg) {
621 : stack_transfers.MoveRegister(LiftoffRegister(instance_reg),
622 : LiftoffRegister(*target_instance),
623 : kWasmIntPtr);
624 : }
625 :
626 : // Now move all parameter values into the right slot for the call.
627 : // Don't pop values yet, such that the stack height is still correct when
628 : // executing the {stack_transfers}.
629 : // Process parameters backwards, such that pushes of caller frame slots are
630 : // in the correct order.
631 47752 : uint32_t param_base = cache_state_.stack_height() - num_params;
632 : uint32_t call_desc_input_idx =
633 47752 : static_cast<uint32_t>(call_descriptor->InputCount());
634 159325 : for (uint32_t i = num_params; i > 0; --i) {
635 111575 : const uint32_t param = i - 1;
636 111575 : ValueType type = sig->GetParam(param);
637 : const bool is_pair = kNeedI64RegPair && type == kWasmI64;
638 : const int num_lowered_params = is_pair ? 2 : 1;
639 111575 : const uint32_t stack_idx = param_base + param;
640 111575 : const VarState& slot = cache_state_.stack_state[stack_idx];
641 : // Process both halfs of a register pair separately, because they are passed
642 : // as separate parameters. One or both of them could end up on the stack.
643 334721 : for (int lowered_idx = 0; lowered_idx < num_lowered_params; ++lowered_idx) {
644 : const RegPairHalf half =
645 : is_pair && lowered_idx == 0 ? kHighWord : kLowWord;
646 111575 : --call_desc_input_idx;
647 : compiler::LinkageLocation loc =
648 111575 : call_descriptor->GetInputLocation(call_desc_input_idx);
649 111575 : if (loc.IsRegister()) {
650 : DCHECK(!loc.IsAnyRegister());
651 : RegClass rc = is_pair ? kGpReg : reg_class_for(type);
652 : int reg_code = loc.AsRegister();
653 : #if V8_TARGET_ARCH_ARM
654 : // Liftoff assumes a one-to-one mapping between float registers and
655 : // double registers, and so does not distinguish between f32 and f64
656 : // registers. The f32 register code must therefore be halved in order to
657 : // pass the f64 code to Liftoff.
658 : DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0);
659 : LiftoffRegister reg = LiftoffRegister::from_code(
660 : rc, (type == kWasmF32) ? (reg_code / 2) : reg_code);
661 : #else
662 92536 : LiftoffRegister reg = LiftoffRegister::from_code(rc, reg_code);
663 : #endif
664 : param_regs.set(reg);
665 : if (is_pair) {
666 : stack_transfers.LoadI64HalfIntoRegister(reg, slot, stack_idx, half);
667 : } else {
668 92535 : stack_transfers.LoadIntoRegister(reg, slot, stack_idx);
669 : }
670 : } else {
671 : DCHECK(loc.IsCallerFrameSlot());
672 : stack_slots.Add(slot, stack_idx, half);
673 : }
674 : }
675 : }
676 : // {call_desc_input_idx} should point after the instance parameter now.
677 : DCHECK_EQ(call_desc_input_idx, kInputShift + 1);
678 :
679 : // If the target register overlaps with a parameter register, then move the
680 : // target to another free register, or spill to the stack.
681 91908 : if (target && param_regs.has(LiftoffRegister(*target))) {
682 : // Try to find another free register.
683 : LiftoffRegList free_regs = kGpCacheRegList.MaskOut(param_regs);
684 1027 : if (!free_regs.is_empty()) {
685 : LiftoffRegister new_target = free_regs.GetFirstRegSet();
686 : stack_transfers.MoveRegister(new_target, LiftoffRegister(*target),
687 : kWasmIntPtr);
688 1027 : *target = new_target.gp();
689 : } else {
690 0 : stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr,
691 : LiftoffRegister(*target)));
692 0 : *target = no_reg;
693 : }
694 : }
695 :
696 : // Create all the slots.
697 47750 : stack_slots.Construct();
698 : // Execute the stack transfers before filling the instance register.
699 : stack_transfers.Execute();
700 :
701 : // Pop parameters from the value stack.
702 : cache_state_.stack_state.pop_back(num_params);
703 :
704 : // Reset register use counters.
705 : cache_state_.reset_used_registers();
706 :
707 : // Reload the instance from the stack.
708 47747 : if (!target_instance) {
709 3591 : FillInstanceInto(instance_reg);
710 : }
711 47747 : }
712 :
713 47750 : void LiftoffAssembler::FinishCall(FunctionSig* sig,
714 : compiler::CallDescriptor* call_descriptor) {
715 : const size_t return_count = sig->return_count();
716 47750 : if (return_count != 0) {
717 : DCHECK_EQ(1, return_count);
718 : ValueType return_type = sig->GetReturn(0);
719 : const bool need_pair = kNeedI64RegPair && return_type == kWasmI64;
720 : DCHECK_EQ(need_pair ? 2 : 1, call_descriptor->ReturnCount());
721 : RegClass rc = need_pair ? kGpReg : reg_class_for(return_type);
722 : #if V8_TARGET_ARCH_ARM
723 : // If the return register was not d0 for f32, the code value would have to
724 : // be halved as is done for the parameter registers.
725 : DCHECK_EQ(call_descriptor->GetReturnLocation(0).AsRegister(), 0);
726 : #endif
727 : LiftoffRegister return_reg = LiftoffRegister::from_code(
728 44362 : rc, call_descriptor->GetReturnLocation(0).AsRegister());
729 : DCHECK(GetCacheRegList(rc).has(return_reg));
730 : if (need_pair) {
731 : LiftoffRegister high_reg = LiftoffRegister::from_code(
732 : rc, call_descriptor->GetReturnLocation(1).AsRegister());
733 : DCHECK(GetCacheRegList(rc).has(high_reg));
734 : return_reg = LiftoffRegister::ForPair(return_reg.gp(), high_reg.gp());
735 : }
736 : DCHECK(!cache_state_.is_used(return_reg));
737 : PushRegister(return_type, return_reg);
738 : }
739 47748 : }
740 :
741 481892 : void LiftoffAssembler::Move(LiftoffRegister dst, LiftoffRegister src,
742 : ValueType type) {
743 : DCHECK_EQ(dst.reg_class(), src.reg_class());
744 : DCHECK_NE(dst, src);
745 : if (kNeedI64RegPair && dst.is_pair()) {
746 : // Use the {StackTransferRecipe} to move pairs, as the registers in the
747 : // pairs might overlap.
748 : StackTransferRecipe(this).MoveRegister(dst, src, type);
749 481892 : } else if (dst.is_gp()) {
750 814280 : Move(dst.gp(), src.gp(), type);
751 : } else {
752 149504 : Move(dst.fp(), src.fp(), type);
753 : }
754 481899 : }
755 :
756 0 : void LiftoffAssembler::ParallelRegisterMove(
757 : Vector<ParallelRegisterMoveTuple> tuples) {
758 : StackTransferRecipe stack_transfers(this);
759 0 : for (auto tuple : tuples) {
760 0 : if (tuple.dst == tuple.src) continue;
761 0 : stack_transfers.MoveRegister(tuple.dst, tuple.src, tuple.type);
762 : }
763 0 : }
764 :
765 404978 : void LiftoffAssembler::MoveToReturnRegisters(FunctionSig* sig) {
766 : // We do not support multi-value yet.
767 : DCHECK_EQ(1, sig->return_count());
768 : ValueType return_type = sig->GetReturn(0);
769 : StackTransferRecipe stack_transfers(this);
770 : LiftoffRegister return_reg =
771 : needs_reg_pair(return_type)
772 : ? LiftoffRegister::ForPair(kGpReturnRegisters[0],
773 : kGpReturnRegisters[1])
774 : : reg_class_for(return_type) == kGpReg
775 : ? LiftoffRegister(kGpReturnRegisters[0])
776 404978 : : LiftoffRegister(kFpReturnRegisters[0]);
777 404978 : stack_transfers.LoadIntoRegister(return_reg, cache_state_.stack_state.back(),
778 404978 : cache_state_.stack_height() - 1);
779 405239 : }
780 :
781 : #ifdef ENABLE_SLOW_DCHECKS
782 : bool LiftoffAssembler::ValidateCacheState() const {
783 : uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0};
784 : LiftoffRegList used_regs;
785 : for (const VarState& var : cache_state_.stack_state) {
786 : if (!var.is_reg()) continue;
787 : LiftoffRegister reg = var.reg();
788 : if (kNeedI64RegPair && reg.is_pair()) {
789 : ++register_use_count[reg.low().liftoff_code()];
790 : ++register_use_count[reg.high().liftoff_code()];
791 : } else {
792 : ++register_use_count[reg.liftoff_code()];
793 : }
794 : used_regs.set(reg);
795 : }
796 : bool valid = memcmp(register_use_count, cache_state_.register_use_count,
797 : sizeof(register_use_count)) == 0 &&
798 : used_regs == cache_state_.used_registers;
799 : if (valid) return true;
800 : std::ostringstream os;
801 : os << "Error in LiftoffAssembler::ValidateCacheState().\n";
802 : os << "expected: used_regs " << used_regs << ", counts "
803 : << PrintCollection(register_use_count) << "\n";
804 : os << "found: used_regs " << cache_state_.used_registers << ", counts "
805 : << PrintCollection(cache_state_.register_use_count) << "\n";
806 : os << "Use --trace-wasm-decoder and --trace-liftoff to debug.";
807 : FATAL("%s", os.str().c_str());
808 : }
809 : #endif
810 :
811 59522 : LiftoffRegister LiftoffAssembler::SpillOneRegister(LiftoffRegList candidates,
812 : LiftoffRegList pinned) {
813 : // Spill one cached value to free a register.
814 : LiftoffRegister spill_reg = cache_state_.GetNextSpillReg(candidates, pinned);
815 59522 : SpillRegister(spill_reg);
816 59522 : return spill_reg;
817 : }
818 :
819 64593 : void LiftoffAssembler::SpillRegister(LiftoffRegister reg) {
820 64593 : int remaining_uses = cache_state_.get_use_count(reg);
821 : DCHECK_LT(0, remaining_uses);
822 2095069 : for (uint32_t idx = cache_state_.stack_height() - 1;; --idx) {
823 : DCHECK_GT(cache_state_.stack_height(), idx);
824 2095069 : auto* slot = &cache_state_.stack_state[idx];
825 2095069 : if (!slot->is_reg() || !slot->reg().overlaps(reg)) continue;
826 : if (slot->reg().is_pair()) {
827 : // Make sure to decrement *both* registers in a pair, because the
828 : // {clear_used} call below only clears one of them.
829 : cache_state_.dec_used(slot->reg().low());
830 : cache_state_.dec_used(slot->reg().high());
831 : }
832 65180 : Spill(idx, slot->reg(), slot->type());
833 : slot->MakeStack();
834 65180 : if (--remaining_uses == 0) break;
835 : }
836 : cache_state_.clear_used(reg);
837 64593 : }
838 :
839 460346 : void LiftoffAssembler::set_num_locals(uint32_t num_locals) {
840 : DCHECK_EQ(0, num_locals_); // only call this once.
841 460346 : num_locals_ = num_locals;
842 460346 : if (num_locals > kInlineLocalTypes) {
843 : more_local_types_ =
844 1746 : reinterpret_cast<ValueType*>(malloc(num_locals * sizeof(ValueType)));
845 : DCHECK_NOT_NULL(more_local_types_);
846 : }
847 460346 : }
848 :
849 0 : std::ostream& operator<<(std::ostream& os, VarState slot) {
850 0 : os << ValueTypes::TypeName(slot.type()) << ":";
851 0 : switch (slot.loc()) {
852 : case VarState::kStack:
853 0 : return os << "s";
854 : case VarState::kRegister:
855 0 : return os << slot.reg();
856 : case VarState::kIntConst:
857 0 : return os << "c" << slot.i32_const();
858 : }
859 0 : UNREACHABLE();
860 : }
861 :
862 : #undef __
863 : #undef TRACE
864 :
865 : } // namespace wasm
866 : } // namespace internal
867 122004 : } // namespace v8
|
