Line data Source code
1 : // Copyright 2017 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/wasm/baseline/liftoff-assembler.h"
6 :
7 : #include <sstream>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/compiler/linkage.h"
11 : #include "src/compiler/wasm-compiler.h"
12 : #include "src/macro-assembler-inl.h"
13 : #include "src/wasm/function-body-decoder-impl.h"
14 : #include "src/wasm/wasm-linkage.h"
15 : #include "src/wasm/wasm-opcodes.h"
16 :
17 : namespace v8 {
18 : namespace internal {
19 : namespace wasm {
20 :
21 : using VarState = LiftoffAssembler::VarState;
22 :
23 : namespace {
24 :
25 : #define __ asm_->
26 :
27 : #define TRACE(...) \
28 : do { \
29 : if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
30 : } while (false)
31 :
32 : class StackTransferRecipe {
33 : struct RegisterMove {
34 : LiftoffRegister src;
35 : ValueType type;
36 : constexpr RegisterMove(LiftoffRegister src, ValueType type)
37 : : src(src), type(type) {}
38 : };
39 :
40 : struct RegisterLoad {
41 : enum LoadKind : uint8_t {
42 : kConstant, // load a constant value into a register.
43 : kStack, // fill a register from a stack slot.
44 : kLowHalfStack, // fill a register from the low half of a stack slot.
45 : kHighHalfStack // fill a register from the high half of a stack slot.
46 : };
47 :
48 : LoadKind kind;
49 : ValueType type;
50 : int32_t value; // i32 constant value or stack index, depending on kind.
51 :
52 : // Named constructors.
53 : static RegisterLoad Const(WasmValue constant) {
54 111181 : if (constant.type() == kWasmI32) {
55 : return {kConstant, kWasmI32, constant.to_i32()};
56 : }
57 : DCHECK_EQ(kWasmI64, constant.type());
58 : DCHECK_EQ(constant.to_i32_unchecked(), constant.to_i64_unchecked());
59 : return {kConstant, kWasmI64, constant.to_i32_unchecked()};
60 : }
61 : static RegisterLoad Stack(int32_t stack_index, ValueType type) {
62 : return {kStack, type, stack_index};
63 : }
64 : static RegisterLoad HalfStack(int32_t stack_index, RegPairHalf half) {
65 : return {half == kLowWord ? kLowHalfStack : kHighHalfStack, kWasmI32,
66 : stack_index};
67 : }
68 :
69 : private:
70 : RegisterLoad(LoadKind kind, ValueType type, int32_t value)
71 : : kind(kind), type(type), value(value) {}
72 : };
73 :
74 : public:
75 849383 : explicit StackTransferRecipe(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {}
76 : ~StackTransferRecipe() { Execute(); }
77 :
78 : void Execute() {
79 : // First, execute register moves. Then load constants and stack values into
80 : // registers.
81 989528 : ExecuteMoves();
82 : DCHECK(move_dst_regs_.is_empty());
83 989577 : ExecuteLoads();
84 : DCHECK(load_dst_regs_.is_empty());
85 : }
86 :
87 35317 : void TransferStackSlot(const LiftoffAssembler::CacheState& dst_state,
88 : uint32_t dst_index,
89 : const LiftoffAssembler::CacheState& src_state,
90 : uint32_t src_index) {
91 70634 : const VarState& dst = dst_state.stack_state[dst_index];
92 39750 : const VarState& src = src_state.stack_state[src_index];
93 : DCHECK_EQ(dst.type(), src.type());
94 35317 : switch (dst.loc()) {
95 : case VarState::kStack:
96 3350 : switch (src.loc()) {
97 : case VarState::kStack:
98 2286 : if (src_index == dst_index) break;
99 18 : asm_->MoveStackValue(dst_index, src_index, src.type());
100 18 : break;
101 : case VarState::kRegister:
102 1019 : asm_->Spill(dst_index, src.reg(), src.type());
103 1019 : break;
104 : case VarState::KIntConst:
105 46 : asm_->Spill(dst_index, src.constant());
106 46 : break;
107 : }
108 : break;
109 : case VarState::kRegister:
110 31201 : LoadIntoRegister(dst.reg(), src, src_index);
111 31199 : break;
112 : case VarState::KIntConst:
113 : DCHECK_EQ(dst, src);
114 : break;
115 : }
116 35315 : }
117 :
118 855933 : void LoadIntoRegister(LiftoffRegister dst,
119 1648364 : const LiftoffAssembler::VarState& src,
120 : uint32_t src_index) {
121 855933 : switch (src.loc()) {
122 : case VarState::kStack:
123 : LoadStackSlot(dst, src_index, src.type());
124 : break;
125 : case VarState::kRegister:
126 : DCHECK_EQ(dst.reg_class(), src.reg_class());
127 1400356 : if (dst != src.reg()) MoveRegister(dst, src.reg(), src.type());
128 : break;
129 : case VarState::KIntConst:
130 111181 : LoadConstant(dst, src.constant());
131 : break;
132 : }
133 855938 : }
134 :
135 : void LoadI64HalfIntoRegister(LiftoffRegister dst,
136 : const LiftoffAssembler::VarState& src,
137 : uint32_t index, RegPairHalf half) {
138 : // Use CHECK such that the remaining code is statically dead if
139 : // {kNeedI64RegPair} is false.
140 : CHECK(kNeedI64RegPair);
141 : DCHECK_EQ(kWasmI64, src.type());
142 : switch (src.loc()) {
143 : case VarState::kStack:
144 : LoadI64HalfStackSlot(dst, index, half);
145 : break;
146 : case VarState::kRegister: {
147 : LiftoffRegister src_half =
148 : half == kLowWord ? src.reg().low() : src.reg().high();
149 : if (dst != src_half) MoveRegister(dst, src_half, kWasmI32);
150 : break;
151 : }
152 : case VarState::KIntConst:
153 : int32_t value = src.i32_const();
154 : // The high word is the sign extension of the low word.
155 : if (half == kHighWord) value = value >> 31;
156 : LoadConstant(dst, WasmValue(value));
157 : break;
158 : }
159 : }
160 :
161 800733 : void MoveRegister(LiftoffRegister dst, LiftoffRegister src, ValueType type) {
162 : DCHECK_NE(dst, src);
163 : DCHECK_EQ(dst.reg_class(), src.reg_class());
164 : DCHECK_EQ(reg_class_for(type), src.reg_class());
165 : if (src.is_pair()) {
166 : DCHECK_EQ(kWasmI64, type);
167 : if (dst.low() != src.low()) MoveRegister(dst.low(), src.low(), kWasmI32);
168 : if (dst.high() != src.high())
169 : MoveRegister(dst.high(), src.high(), kWasmI32);
170 : return;
171 : }
172 800733 : if (move_dst_regs_.has(dst)) {
173 : DCHECK_EQ(register_move(dst)->src, src);
174 : // Non-fp registers can only occur with the exact same type.
175 : DCHECK_IMPLIES(!dst.is_fp(), register_move(dst)->type == type);
176 : // It can happen that one fp register holds both the f32 zero and the f64
177 : // zero, as the initial value for local variables. Move the value as f64
178 : // in that case.
179 45 : if (type == kWasmF64) register_move(dst)->type = kWasmF64;
180 : return;
181 : }
182 : move_dst_regs_.set(dst);
183 800697 : ++*src_reg_use_count(src);
184 800697 : *register_move(dst) = {src, type};
185 : }
186 :
187 : void LoadConstant(LiftoffRegister dst, WasmValue value) {
188 : DCHECK(!load_dst_regs_.has(dst));
189 : load_dst_regs_.set(dst);
190 : if (dst.is_pair()) {
191 : DCHECK_EQ(kWasmI64, value.type());
192 : int64_t i64 = value.to_i64();
193 : *register_load(dst.low()) =
194 : RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64)));
195 : *register_load(dst.high()) =
196 : RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64 >> 32)));
197 : } else {
198 111181 : *register_load(dst) = RegisterLoad::Const(value);
199 : }
200 : }
201 :
202 : void LoadStackSlot(LiftoffRegister dst, uint32_t stack_index,
203 : ValueType type) {
204 13308 : if (load_dst_regs_.has(dst)) {
205 : // It can happen that we spilled the same register to different stack
206 : // slots, and then we reload them later into the same dst register.
207 : // In that case, it is enough to load one of the stack slots.
208 : return;
209 : }
210 : load_dst_regs_.set(dst);
211 : if (dst.is_pair()) {
212 : DCHECK_EQ(kWasmI64, type);
213 : *register_load(dst.low()) =
214 : RegisterLoad::HalfStack(stack_index, kLowWord);
215 : *register_load(dst.high()) =
216 : RegisterLoad::HalfStack(stack_index, kHighWord);
217 : } else {
218 13299 : *register_load(dst) = RegisterLoad::Stack(stack_index, type);
219 : }
220 : }
221 :
222 : void LoadI64HalfStackSlot(LiftoffRegister dst, uint32_t stack_index,
223 : RegPairHalf half) {
224 : if (load_dst_regs_.has(dst)) {
225 : // It can happen that we spilled the same register to different stack
226 : // slots, and then we reload them later into the same dst register.
227 : // In that case, it is enough to load one of the stack slots.
228 : return;
229 : }
230 : load_dst_regs_.set(dst);
231 : *register_load(dst) = RegisterLoad::HalfStack(stack_index, half);
232 : }
233 :
234 : private:
235 : using MovesStorage =
236 : std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterMove),
237 : alignof(RegisterMove)>::type;
238 : using LoadsStorage =
239 : std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterLoad),
240 : alignof(RegisterLoad)>::type;
241 :
242 : ASSERT_TRIVIALLY_COPYABLE(RegisterMove);
243 : ASSERT_TRIVIALLY_COPYABLE(RegisterLoad);
244 :
245 : MovesStorage register_moves_; // uninitialized
246 : LoadsStorage register_loads_; // uninitialized
247 : int src_reg_use_count_[kAfterMaxLiftoffRegCode] = {0};
248 : LiftoffRegList move_dst_regs_;
249 : LiftoffRegList load_dst_regs_;
250 : LiftoffAssembler* const asm_;
251 :
252 : RegisterMove* register_move(LiftoffRegister reg) {
253 2402144 : return reinterpret_cast<RegisterMove*>(®ister_moves_) +
254 2402144 : reg.liftoff_code();
255 : }
256 : RegisterLoad* register_load(LiftoffRegister reg) {
257 248963 : return reinterpret_cast<RegisterLoad*>(®ister_loads_) +
258 248963 : reg.liftoff_code();
259 : }
260 : int* src_reg_use_count(LiftoffRegister reg) {
261 2400270 : return src_reg_use_count_ + reg.liftoff_code();
262 : }
263 :
264 800238 : void ExecuteMove(LiftoffRegister dst) {
265 : RegisterMove* move = register_move(dst);
266 : DCHECK_EQ(0, *src_reg_use_count(dst));
267 800238 : asm_->Move(dst, move->src, move->type);
268 800229 : ClearExecutedMove(dst);
269 800252 : }
270 :
271 800714 : void ClearExecutedMove(LiftoffRegister dst) {
272 : DCHECK(move_dst_regs_.has(dst));
273 : move_dst_regs_.clear(dst);
274 : RegisterMove* move = register_move(dst);
275 : DCHECK_LT(0, *src_reg_use_count(move->src));
276 800714 : if (--*src_reg_use_count(move->src)) return;
277 : // src count dropped to zero. If this is a destination register, execute
278 : // that move now.
279 800574 : if (!move_dst_regs_.has(move->src)) return;
280 107424 : ExecuteMove(move->src);
281 : }
282 :
283 989508 : void ExecuteMoves() {
284 : // Execute all moves whose {dst} is not being used as src in another move.
285 : // If any src count drops to zero, also (transitively) execute the
286 : // corresponding move to that register.
287 2779746 : for (LiftoffRegister dst : move_dst_regs_) {
288 : // Check if already handled via transitivity in {ClearExecutedMove}.
289 800704 : if (!move_dst_regs_.has(dst)) continue;
290 798859 : if (*src_reg_use_count(dst)) continue;
291 692814 : ExecuteMove(dst);
292 : }
293 :
294 : // All remaining moves are parts of a cycle. Just spill the first one, then
295 : // process all remaining moves in that cycle. Repeat for all cycles.
296 989534 : uint32_t next_spill_slot = asm_->cache_state()->stack_height();
297 1979554 : while (!move_dst_regs_.is_empty()) {
298 : // TODO(clemensh): Use an unused register if available.
299 : LiftoffRegister dst = move_dst_regs_.GetFirstRegSet();
300 : RegisterMove* move = register_move(dst);
301 486 : LiftoffRegister spill_reg = move->src;
302 486 : asm_->Spill(next_spill_slot, spill_reg, move->type);
303 : // Remember to reload into the destination register later.
304 486 : LoadStackSlot(dst, next_spill_slot, move->type);
305 486 : ++next_spill_slot;
306 486 : ClearExecutedMove(dst);
307 : }
308 989534 : }
309 :
310 989512 : void ExecuteLoads() {
311 2103537 : for (LiftoffRegister dst : load_dst_regs_) {
312 : RegisterLoad* load = register_load(dst);
313 124483 : switch (load->kind) {
314 : case RegisterLoad::kConstant:
315 111236 : asm_->LoadConstant(dst, load->type == kWasmI64
316 : ? WasmValue(int64_t{load->value})
317 222472 : : WasmValue(int32_t{load->value}));
318 : break;
319 : case RegisterLoad::kStack:
320 13242 : asm_->Fill(dst, load->value, load->type);
321 13298 : break;
322 : case RegisterLoad::kLowHalfStack:
323 : // Half of a register pair, {dst} must be a gp register.
324 0 : asm_->FillI64Half(dst.gp(), load->value, kLowWord);
325 : break;
326 : case RegisterLoad::kHighHalfStack:
327 : // Half of a register pair, {dst} must be a gp register.
328 0 : asm_->FillI64Half(dst.gp(), load->value, kHighWord);
329 : break;
330 : }
331 : }
332 989542 : load_dst_regs_ = {};
333 989542 : }
334 :
335 : DISALLOW_COPY_AND_ASSIGN(StackTransferRecipe);
336 : };
337 :
338 : class RegisterReuseMap {
339 : public:
340 216 : void Add(LiftoffRegister src, LiftoffRegister dst) {
341 216 : if (auto previous = Lookup(src)) {
342 : DCHECK_EQ(previous, dst);
343 216 : return;
344 : }
345 198 : map_.emplace_back(src);
346 198 : map_.emplace_back(dst);
347 : }
348 :
349 : base::Optional<LiftoffRegister> Lookup(LiftoffRegister src) {
350 369 : for (auto it = map_.begin(), end = map_.end(); it != end; it += 2) {
351 90 : if (*it == src) return *(it + 1);
352 : }
353 : return {};
354 : }
355 :
356 : private:
357 : // {map_} holds pairs of <src, dst>.
358 : base::SmallVector<LiftoffRegister, 8> map_;
359 : };
360 :
361 : enum MergeKeepStackSlots : bool {
362 : kKeepStackSlots = true,
363 : kTurnStackSlotsIntoRegisters = false
364 : };
365 : enum MergeAllowConstants : bool {
366 : kConstantsAllowed = true,
367 : kConstantsNotAllowed = false
368 : };
369 : enum ReuseRegisters : bool {
370 : kReuseRegisters = true,
371 : kNoReuseRegisters = false
372 : };
373 416758 : void InitMergeRegion(LiftoffAssembler::CacheState* state,
374 75406 : const VarState* source, VarState* target, uint32_t count,
375 : MergeKeepStackSlots keep_stack_slots,
376 : MergeAllowConstants allow_constants,
377 : ReuseRegisters reuse_registers, LiftoffRegList used_regs) {
378 : RegisterReuseMap register_reuse_map;
379 443037 : for (const VarState* source_end = source + count; source < source_end;
380 : ++source, ++target) {
381 52548 : if ((source->is_stack() && keep_stack_slots) ||
382 4362 : (source->is_const() && allow_constants)) {
383 1688 : *target = *source;
384 3421 : continue;
385 : }
386 : base::Optional<LiftoffRegister> reg;
387 : // First try: Keep the same register, if it's free.
388 45297 : if (source->is_reg() && state->is_free(source->reg())) {
389 : reg = source->reg();
390 : }
391 : // Second try: Use the same register we used before (if we reuse registers).
392 24586 : if (!reg && reuse_registers) {
393 : reg = register_reuse_map.Lookup(source->reg());
394 : }
395 : // Third try: Use any free register.
396 : RegClass rc = reg_class_for(source->type());
397 29029 : if (!reg && state->has_unused_register(rc, used_regs)) {
398 : reg = state->unused_register(rc, used_regs);
399 : }
400 24586 : if (!reg) {
401 : // No free register; make this a stack slot.
402 45 : *target = VarState(source->type());
403 45 : continue;
404 : }
405 24541 : if (reuse_registers) register_reuse_map.Add(source->reg(), *reg);
406 : state->inc_used(*reg);
407 24546 : *target = VarState(source->type(), *reg);
408 : }
409 416763 : }
410 :
411 : } // namespace
412 :
413 : // TODO(clemensh): Don't copy the full parent state (this makes us N^2).
414 138914 : void LiftoffAssembler::CacheState::InitMerge(const CacheState& source,
415 : uint32_t num_locals,
416 : uint32_t arity,
417 : uint32_t stack_depth) {
418 : // |------locals------|---(in between)----|--(discarded)--|----merge----|
419 : // <-- num_locals --> <-- stack_depth -->^stack_base <-- arity -->
420 :
421 138914 : uint32_t stack_base = stack_depth + num_locals;
422 138914 : uint32_t target_height = stack_base + arity;
423 138914 : uint32_t discarded = source.stack_height() - target_height;
424 : DCHECK(stack_state.empty());
425 :
426 : DCHECK_GE(source.stack_height(), stack_base);
427 277832 : stack_state.resize_no_init(target_height);
428 :
429 138918 : const VarState* source_begin = source.stack_state.data();
430 : VarState* target_begin = stack_state.data();
431 :
432 : // Try to keep locals and the merge region in their registers. Register used
433 : // multiple times need to be copied to another free register. Compute the list
434 : // of used registers.
435 : LiftoffRegList used_regs;
436 299260 : for (auto& src : VectorOf(source_begin, num_locals)) {
437 21424 : if (src.is_reg()) used_regs.set(src.reg());
438 : }
439 281931 : for (auto& src : VectorOf(source_begin + stack_base + discarded, arity)) {
440 4095 : if (src.is_reg()) used_regs.set(src.reg());
441 : }
442 :
443 : // Initialize the merge region. If this region moves, try to turn stack slots
444 : // into registers since we need to load the value anyways.
445 : MergeKeepStackSlots keep_merge_stack_slots =
446 138918 : discarded == 0 ? kKeepStackSlots : kTurnStackSlotsIntoRegisters;
447 : InitMergeRegion(this, source_begin + stack_base + discarded,
448 : target_begin + stack_base, arity, keep_merge_stack_slots,
449 138918 : kConstantsNotAllowed, kNoReuseRegisters, used_regs);
450 :
451 : // Initialize the locals region. Here, stack slots stay stack slots (because
452 : // they do not move). Try to keep register in registers, but avoid duplicates.
453 : InitMergeRegion(this, source_begin, target_begin, num_locals, kKeepStackSlots,
454 138922 : kConstantsNotAllowed, kNoReuseRegisters, used_regs);
455 : // Sanity check: All the {used_regs} are really in use now.
456 : DCHECK_EQ(used_regs, used_registers & used_regs);
457 :
458 : // Last, initialize the section in between. Here, constants are allowed, but
459 : // registers which are already used for the merge region or locals must be
460 : // moved to other registers or spilled. If a register appears twice in the
461 : // source region, ensure to use the same register twice in the target region.
462 : InitMergeRegion(this, source_begin + num_locals, target_begin + num_locals,
463 : stack_depth, kKeepStackSlots, kConstantsAllowed,
464 138924 : kReuseRegisters, used_regs);
465 138922 : }
466 :
467 140880 : void LiftoffAssembler::CacheState::Steal(const CacheState& source) {
468 : // Just use the move assignment operator.
469 140880 : *this = std::move(source);
470 140879 : }
471 :
472 5809 : void LiftoffAssembler::CacheState::Split(const CacheState& source) {
473 : // Call the private copy assignment operator.
474 5809 : *this = source;
475 5811 : }
476 :
477 : namespace {
478 :
479 : constexpr AssemblerOptions DefaultLiftoffOptions() {
480 711839 : return AssemblerOptions{};
481 : }
482 :
483 : } // namespace
484 :
485 : // TODO(clemensh): Provide a reasonably sized buffer, based on wasm function
486 : // size.
487 711839 : LiftoffAssembler::LiftoffAssembler()
488 : : TurboAssembler(nullptr, DefaultLiftoffOptions(),
489 2135553 : CodeObjectRequired::kNo) {
490 : set_abort_hard(true); // Avoid calls to Abort.
491 711875 : }
492 :
493 711947 : LiftoffAssembler::~LiftoffAssembler() {
494 711947 : if (num_locals_ > kInlineLocalTypes) {
495 3141 : free(more_local_types_);
496 : }
497 711969 : }
498 :
499 2565413 : LiftoffRegister LiftoffAssembler::PopToRegister(LiftoffRegList pinned) {
500 : DCHECK(!cache_state_.stack_state.empty());
501 2565413 : VarState slot = cache_state_.stack_state.back();
502 : cache_state_.stack_state.pop_back();
503 2565413 : switch (slot.loc()) {
504 : case VarState::kStack: {
505 : LiftoffRegister reg =
506 : GetUnusedRegister(reg_class_for(slot.type()), pinned);
507 2440 : Fill(reg, cache_state_.stack_height(), slot.type());
508 1219 : return reg;
509 : }
510 : case VarState::kRegister:
511 : cache_state_.dec_used(slot.reg());
512 : return slot.reg();
513 : case VarState::KIntConst: {
514 : RegClass rc =
515 : kNeedI64RegPair && slot.type() == kWasmI64 ? kGpRegPair : kGpReg;
516 : LiftoffRegister reg = GetUnusedRegister(rc, pinned);
517 2927332 : LoadConstant(reg, slot.constant());
518 1463661 : return reg;
519 : }
520 : }
521 0 : UNREACHABLE();
522 : }
523 :
524 5485 : void LiftoffAssembler::MergeFullStackWith(const CacheState& target,
525 : const CacheState& source) {
526 : DCHECK_EQ(source.stack_height(), target.stack_height());
527 : // TODO(clemensh): Reuse the same StackTransferRecipe object to save some
528 : // allocations.
529 : StackTransferRecipe transfers(this);
530 16702 : for (uint32_t i = 0, e = source.stack_height(); i < e; ++i) {
531 11210 : transfers.TransferStackSlot(target, i, source, i);
532 : }
533 5494 : }
534 :
535 137982 : void LiftoffAssembler::MergeStackWith(const CacheState& target,
536 : uint32_t arity) {
537 : // Before: ----------------|----- (discarded) ----|--- arity ---|
538 : // ^target_stack_height ^stack_base ^stack_height
539 : // After: ----|-- arity --|
540 : // ^ ^target_stack_height
541 : // ^target_stack_base
542 : uint32_t stack_height = cache_state_.stack_height();
543 : uint32_t target_stack_height = target.stack_height();
544 : DCHECK_LE(target_stack_height, stack_height);
545 : DCHECK_LE(arity, target_stack_height);
546 137982 : uint32_t stack_base = stack_height - arity;
547 137982 : uint32_t target_stack_base = target_stack_height - arity;
548 : StackTransferRecipe transfers(this);
549 159007 : for (uint32_t i = 0; i < target_stack_base; ++i) {
550 21022 : transfers.TransferStackSlot(target, i, cache_state_, i);
551 : }
552 3091 : for (uint32_t i = 0; i < arity; ++i) {
553 : transfers.TransferStackSlot(target, target_stack_base + i, cache_state_,
554 3091 : stack_base + i);
555 : }
556 137987 : }
557 :
558 2107 : void LiftoffAssembler::Spill(uint32_t index) {
559 6088 : auto& slot = cache_state_.stack_state[index];
560 2107 : switch (slot.loc()) {
561 : case VarState::kStack:
562 2107 : return;
563 : case VarState::kRegister:
564 1454 : Spill(index, slot.reg(), slot.type());
565 : cache_state_.dec_used(slot.reg());
566 : break;
567 : case VarState::KIntConst:
568 420 : Spill(index, slot.constant());
569 420 : break;
570 : }
571 : slot.MakeStack();
572 : }
573 :
574 2453 : void LiftoffAssembler::SpillLocals() {
575 4560 : for (uint32_t i = 0; i < num_locals_; ++i) {
576 2107 : Spill(i);
577 : }
578 2453 : }
579 :
580 35192 : void LiftoffAssembler::SpillAllRegisters() {
581 36690 : for (uint32_t i = 0, e = cache_state_.stack_height(); i < e; ++i) {
582 4125 : auto& slot = cache_state_.stack_state[i];
583 1498 : if (!slot.is_reg()) continue;
584 1129 : Spill(i, slot.reg(), slot.type());
585 : slot.MakeStack();
586 : }
587 : cache_state_.reset_used_registers();
588 35192 : }
589 :
590 436182 : void LiftoffAssembler::PrepareCall(FunctionSig* sig,
591 140101 : compiler::CallDescriptor* call_descriptor,
592 : Register* target,
593 : Register* target_instance) {
594 140105 : uint32_t num_params = static_cast<uint32_t>(sig->parameter_count());
595 : // Input 0 is the call target.
596 : constexpr size_t kInputShift = 1;
597 :
598 : // Spill all cache slots which are not being used as parameters.
599 : // Don't update any register use counters, they will be reset later anyway.
600 955915 : for (uint32_t idx = 0, end = cache_state_.stack_height() - num_params;
601 : idx < end; ++idx) {
602 2392676 : VarState& slot = cache_state_.stack_state[idx];
603 815810 : if (!slot.is_reg()) continue;
604 464979 : Spill(idx, slot.reg(), slot.type());
605 : slot.MakeStack();
606 : }
607 :
608 : LiftoffStackSlots stack_slots(this);
609 : StackTransferRecipe stack_transfers(this);
610 : LiftoffRegList param_regs;
611 :
612 : // Move the target instance (if supplied) into the correct instance register.
613 : compiler::LinkageLocation instance_loc =
614 140105 : call_descriptor->GetInputLocation(kInputShift);
615 : DCHECK(instance_loc.IsRegister() && !instance_loc.IsAnyRegister());
616 : Register instance_reg = Register::from_code(instance_loc.AsRegister());
617 : param_regs.set(instance_reg);
618 270364 : if (target_instance && *target_instance != instance_reg) {
619 : stack_transfers.MoveRegister(LiftoffRegister(instance_reg),
620 : LiftoffRegister(*target_instance),
621 129819 : kWasmIntPtr);
622 : }
623 :
624 : // Now move all parameter values into the right slot for the call.
625 : // Don't pop values yet, such that the stack height is still correct when
626 : // executing the {stack_transfers}.
627 : // Process parameters backwards, such that pushes of caller frame slots are
628 : // in the correct order.
629 140101 : uint32_t param_base = cache_state_.stack_height() - num_params;
630 : uint32_t call_desc_input_idx =
631 140101 : static_cast<uint32_t>(call_descriptor->InputCount());
632 576283 : for (uint32_t i = num_params; i > 0; --i) {
633 296077 : const uint32_t param = i - 1;
634 296077 : ValueType type = sig->GetParam(param);
635 : const bool is_pair = kNeedI64RegPair && type == kWasmI64;
636 : const int num_lowered_params = is_pair ? 2 : 1;
637 296077 : const uint32_t stack_idx = param_base + param;
638 296077 : const VarState& slot = cache_state_.stack_state[stack_idx];
639 : // Process both halfs of a register pair separately, because they are passed
640 : // as separate parameters. One or both of them could end up on the stack.
641 592159 : for (int lowered_idx = 0; lowered_idx < num_lowered_params; ++lowered_idx) {
642 : const RegPairHalf half =
643 : is_pair && lowered_idx == 0 ? kHighWord : kLowWord;
644 296078 : --call_desc_input_idx;
645 : compiler::LinkageLocation loc =
646 296078 : call_descriptor->GetInputLocation(call_desc_input_idx);
647 296079 : if (loc.IsRegister()) {
648 : DCHECK(!loc.IsAnyRegister());
649 : RegClass rc = is_pair ? kGpReg : reg_class_for(type);
650 : int reg_code = loc.AsRegister();
651 : #if V8_TARGET_ARCH_ARM
652 : // Liftoff assumes a one-to-one mapping between float registers and
653 : // double registers, and so does not distinguish between f32 and f64
654 : // registers. The f32 register code must therefore be halved in order to
655 : // pass the f64 code to Liftoff.
656 : DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0);
657 : LiftoffRegister reg = LiftoffRegister::from_code(
658 : rc, (type == kWasmF32) ? (reg_code / 2) : reg_code);
659 : #else
660 258920 : LiftoffRegister reg = LiftoffRegister::from_code(rc, reg_code);
661 : #endif
662 : param_regs.set(reg);
663 : if (is_pair) {
664 : stack_transfers.LoadI64HalfIntoRegister(reg, slot, stack_idx, half);
665 : } else {
666 258921 : stack_transfers.LoadIntoRegister(reg, slot, stack_idx);
667 : }
668 : } else {
669 : DCHECK(loc.IsCallerFrameSlot());
670 : stack_slots.Add(slot, stack_idx, half);
671 : }
672 : }
673 : }
674 : // {call_desc_input_idx} should point after the instance parameter now.
675 : DCHECK_EQ(call_desc_input_idx, kInputShift + 1);
676 :
677 : // If the target register overlaps with a parameter register, then move the
678 : // target to another free register, or spill to the stack.
679 270372 : if (target && param_regs.has(LiftoffRegister(*target))) {
680 : // Try to find another free register.
681 : LiftoffRegList free_regs = kGpCacheRegList.MaskOut(param_regs);
682 2492 : if (!free_regs.is_empty()) {
683 : LiftoffRegister new_target = free_regs.GetFirstRegSet();
684 : stack_transfers.MoveRegister(new_target, LiftoffRegister(*target),
685 2492 : kWasmIntPtr);
686 2491 : *target = new_target.gp();
687 : } else {
688 : stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr,
689 0 : LiftoffRegister(*target)));
690 0 : *target = no_reg;
691 : }
692 : }
693 :
694 : // Create all the slots.
695 140104 : stack_slots.Construct();
696 : // Execute the stack transfers before filling the instance register.
697 : stack_transfers.Execute();
698 :
699 : // Pop parameters from the value stack.
700 140104 : cache_state_.stack_state.pop_back(num_params);
701 :
702 : // Reset register use counters.
703 : cache_state_.reset_used_registers();
704 :
705 : // Reload the instance from the stack.
706 140104 : if (!target_instance) {
707 9841 : FillInstanceInto(instance_reg);
708 : }
709 140133 : }
710 :
711 271212 : void LiftoffAssembler::FinishCall(FunctionSig* sig,
712 131094 : compiler::CallDescriptor* call_descriptor) {
713 : const size_t return_count = sig->return_count();
714 140118 : if (return_count != 0) {
715 : DCHECK_EQ(1, return_count);
716 : ValueType return_type = sig->GetReturn(0);
717 : const bool need_pair = kNeedI64RegPair && return_type == kWasmI64;
718 : DCHECK_EQ(need_pair ? 2 : 1, call_descriptor->ReturnCount());
719 : RegClass rc = need_pair ? kGpReg : reg_class_for(return_type);
720 : #if V8_TARGET_ARCH_ARM
721 : // If the return register was not d0 for f32, the code value would have to
722 : // be halved as is done for the parameter registers.
723 : DCHECK_EQ(call_descriptor->GetReturnLocation(0).AsRegister(), 0);
724 : #endif
725 : LiftoffRegister return_reg = LiftoffRegister::from_code(
726 131094 : rc, call_descriptor->GetReturnLocation(0).AsRegister());
727 : DCHECK(GetCacheRegList(rc).has(return_reg));
728 : if (need_pair) {
729 : LiftoffRegister high_reg = LiftoffRegister::from_code(
730 : rc, call_descriptor->GetReturnLocation(1).AsRegister());
731 : DCHECK(GetCacheRegList(rc).has(high_reg));
732 : return_reg = LiftoffRegister::ForPair(return_reg.gp(), high_reg.gp());
733 : }
734 : DCHECK(!cache_state_.is_used(return_reg));
735 : PushRegister(return_type, return_reg);
736 : }
737 140106 : }
738 :
739 834997 : void LiftoffAssembler::Move(LiftoffRegister dst, LiftoffRegister src,
740 : ValueType type) {
741 : DCHECK_EQ(dst.reg_class(), src.reg_class());
742 : DCHECK_NE(dst, src);
743 : if (kNeedI64RegPair && dst.is_pair()) {
744 : // Use the {StackTransferRecipe} to move pairs, as the registers in the
745 : // pairs might overlap.
746 : StackTransferRecipe(this).MoveRegister(dst, src, type);
747 834997 : } else if (dst.is_gp()) {
748 1230492 : Move(dst.gp(), src.gp(), type);
749 : } else {
750 439502 : Move(dst.fp(), src.fp(), type);
751 : }
752 835011 : }
753 :
754 0 : void LiftoffAssembler::ParallelRegisterMove(
755 : Vector<ParallelRegisterMoveTuple> tuples) {
756 : StackTransferRecipe stack_transfers(this);
757 0 : for (auto tuple : tuples) {
758 0 : if (tuple.dst == tuple.src) continue;
759 0 : stack_transfers.MoveRegister(tuple.dst, tuple.src, tuple.type);
760 : }
761 0 : }
762 :
763 565811 : void LiftoffAssembler::MoveToReturnRegisters(FunctionSig* sig) {
764 : // We do not support multi-value yet.
765 : DCHECK_EQ(1, sig->return_count());
766 : ValueType return_type = sig->GetReturn(0);
767 : StackTransferRecipe stack_transfers(this);
768 : LiftoffRegister return_reg =
769 : needs_reg_pair(return_type)
770 : ? LiftoffRegister::ForPair(kGpReturnRegisters[0],
771 : kGpReturnRegisters[1])
772 : : reg_class_for(return_type) == kGpReg
773 : ? LiftoffRegister(kGpReturnRegisters[0])
774 565811 : : LiftoffRegister(kFpReturnRegisters[0]);
775 565811 : stack_transfers.LoadIntoRegister(return_reg, cache_state_.stack_state.back(),
776 1131622 : cache_state_.stack_height() - 1);
777 565861 : }
778 :
779 : #ifdef ENABLE_SLOW_DCHECKS
780 : bool LiftoffAssembler::ValidateCacheState() const {
781 : uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0};
782 : LiftoffRegList used_regs;
783 : for (const VarState& var : cache_state_.stack_state) {
784 : if (!var.is_reg()) continue;
785 : LiftoffRegister reg = var.reg();
786 : if (kNeedI64RegPair && reg.is_pair()) {
787 : ++register_use_count[reg.low().liftoff_code()];
788 : ++register_use_count[reg.high().liftoff_code()];
789 : } else {
790 : ++register_use_count[reg.liftoff_code()];
791 : }
792 : used_regs.set(reg);
793 : }
794 : bool valid = memcmp(register_use_count, cache_state_.register_use_count,
795 : sizeof(register_use_count)) == 0 &&
796 : used_regs == cache_state_.used_registers;
797 : if (valid) return true;
798 : std::ostringstream os;
799 : os << "Error in LiftoffAssembler::ValidateCacheState().\n";
800 : os << "expected: used_regs " << used_regs << ", counts "
801 : << PrintCollection(register_use_count) << "\n";
802 : os << "found: used_regs " << cache_state_.used_registers << ", counts "
803 : << PrintCollection(cache_state_.register_use_count) << "\n";
804 : os << "Use --trace-wasm-decoder and --trace-liftoff to debug.";
805 : FATAL("%s", os.str().c_str());
806 : }
807 : #endif
808 :
809 107594 : LiftoffRegister LiftoffAssembler::SpillOneRegister(LiftoffRegList candidates,
810 : LiftoffRegList pinned) {
811 : // Spill one cached value to free a register.
812 : LiftoffRegister spill_reg = cache_state_.GetNextSpillReg(candidates, pinned);
813 107594 : SpillRegister(spill_reg);
814 107594 : return spill_reg;
815 : }
816 :
817 114438 : void LiftoffAssembler::SpillRegister(LiftoffRegister reg) {
818 114438 : int remaining_uses = cache_state_.get_use_count(reg);
819 : DCHECK_LT(0, remaining_uses);
820 5770039 : for (uint32_t idx = cache_state_.stack_height() - 1;; --idx) {
821 : DCHECK_GT(cache_state_.stack_height(), idx);
822 11655906 : auto* slot = &cache_state_.stack_state[idx];
823 5770039 : if (!slot->is_reg() || !slot->reg().overlaps(reg)) continue;
824 : if (slot->reg().is_pair()) {
825 : // Make sure to decrement *both* registers in a pair, because the
826 : // {clear_used} call below only clears one of them.
827 : cache_state_.dec_used(slot->reg().low());
828 : cache_state_.dec_used(slot->reg().high());
829 : }
830 115828 : Spill(idx, slot->reg(), slot->type());
831 : slot->MakeStack();
832 115829 : if (--remaining_uses == 0) break;
833 5655601 : }
834 : cache_state_.clear_used(reg);
835 114439 : }
836 :
837 711771 : void LiftoffAssembler::set_num_locals(uint32_t num_locals) {
838 : DCHECK_EQ(0, num_locals_); // only call this once.
839 711771 : num_locals_ = num_locals;
840 711771 : if (num_locals > kInlineLocalTypes) {
841 : more_local_types_ =
842 3141 : reinterpret_cast<ValueType*>(malloc(num_locals * sizeof(ValueType)));
843 : DCHECK_NOT_NULL(more_local_types_);
844 : }
845 711771 : }
846 :
847 0 : std::ostream& operator<<(std::ostream& os, VarState slot) {
848 0 : os << ValueTypes::TypeName(slot.type()) << ":";
849 0 : switch (slot.loc()) {
850 : case VarState::kStack:
851 0 : return os << "s";
852 : case VarState::kRegister:
853 0 : return os << slot.reg();
854 : case VarState::KIntConst:
855 0 : return os << "c" << slot.i32_const();
856 : }
857 0 : UNREACHABLE();
858 : }
859 :
860 : #undef __
861 : #undef TRACE
862 :
863 : } // namespace wasm
864 : } // namespace internal
865 183867 : } // namespace v8
|
