Line data Source code
1 : // Copyright 2013 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/backend/code-generator.h"
6 :
7 : #include "src/address-map.h"
8 : #include "src/assembler-inl.h"
9 : #include "src/base/adapters.h"
10 : #include "src/compiler/backend/code-generator-impl.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/pipeline.h"
13 : #include "src/compiler/wasm-compiler.h"
14 : #include "src/counters.h"
15 : #include "src/eh-frame.h"
16 : #include "src/frames.h"
17 : #include "src/log.h"
18 : #include "src/macro-assembler-inl.h"
19 : #include "src/objects/smi.h"
20 : #include "src/optimized-compilation-info.h"
21 : #include "src/string-constants.h"
22 :
23 : namespace v8 {
24 : namespace internal {
25 : namespace compiler {
26 :
27 : class CodeGenerator::JumpTable final : public ZoneObject {
28 : public:
29 : JumpTable(JumpTable* next, Label** targets, size_t target_count)
30 314 : : next_(next), targets_(targets), target_count_(target_count) {}
31 :
32 : Label* label() { return &label_; }
33 : JumpTable* next() const { return next_; }
34 : Label** targets() const { return targets_; }
35 : size_t target_count() const { return target_count_; }
36 :
37 : private:
38 : Label label_;
39 : JumpTable* const next_;
40 : Label** const targets_;
41 : size_t const target_count_;
42 : };
43 :
44 2141438 : CodeGenerator::CodeGenerator(
45 : Zone* codegen_zone, Frame* frame, Linkage* linkage,
46 25865183 : InstructionSequence* instructions, OptimizedCompilationInfo* info,
47 : Isolate* isolate, base::Optional<OsrHelper> osr_helper,
48 : int start_source_position, JumpOptimizationInfo* jump_opt,
49 : PoisoningMitigationLevel poisoning_level, const AssemblerOptions& options,
50 12848724 : int32_t builtin_index, std::unique_ptr<AssemblerBuffer> buffer)
51 : : zone_(codegen_zone),
52 : isolate_(isolate),
53 : frame_access_state_(nullptr),
54 : linkage_(linkage),
55 : instructions_(instructions),
56 : unwinding_info_writer_(zone()),
57 : info_(info),
58 2141224 : labels_(zone()->NewArray<Label>(instructions->InstructionBlockCount())),
59 : current_block_(RpoNumber::Invalid()),
60 : start_source_position_(start_source_position),
61 : current_source_position_(SourcePosition::Unknown()),
62 : tasm_(isolate, options, CodeObjectRequired::kNo, std::move(buffer)),
63 : resolver_(this),
64 : safepoints_(zone()),
65 : handlers_(zone()),
66 : deoptimization_exits_(zone()),
67 : deoptimization_states_(zone()),
68 : deoptimization_literals_(zone()),
69 : translations_(zone()),
70 : caller_registers_saved_(false),
71 : jump_tables_(nullptr),
72 : ools_(nullptr),
73 : osr_helper_(std::move(osr_helper)),
74 : osr_pc_offset_(-1),
75 : optimized_out_literal_id_(-1),
76 : source_position_table_builder_(
77 : SourcePositionTableBuilder::RECORD_SOURCE_POSITIONS),
78 : protected_instructions_(zone()),
79 : result_(kSuccess),
80 : poisoning_level_(poisoning_level),
81 : block_starts_(zone()),
82 17131334 : instr_starts_(zone()) {
83 43165170 : for (int i = 0; i < instructions->InstructionBlockCount(); ++i) {
84 19441091 : new (&labels_[i]) Label;
85 : }
86 2141494 : CreateFrameAccessState(frame);
87 2141374 : CHECK_EQ(info->is_osr(), osr_helper_.has_value());
88 : tasm_.set_jump_optimization_info(jump_opt);
89 : Code::Kind code_kind = info->code_kind();
90 4282748 : if (code_kind == Code::WASM_FUNCTION ||
91 2141374 : code_kind == Code::WASM_TO_JS_FUNCTION ||
92 5918029 : code_kind == Code::WASM_INTERPRETER_ENTRY ||
93 121632 : (Builtins::IsBuiltinId(builtin_index) &&
94 121632 : Builtins::IsWasmRuntimeStub(builtin_index))) {
95 : tasm_.set_abort_hard(true);
96 : }
97 : tasm_.set_builtin_index(builtin_index);
98 2141374 : }
99 :
100 271589 : bool CodeGenerator::wasm_runtime_exception_support() const {
101 : DCHECK_NOT_NULL(info_);
102 543178 : return info_->wasm_runtime_exception_support();
103 : }
104 :
105 238149 : void CodeGenerator::AddProtectedInstructionLanding(uint32_t instr_offset,
106 : uint32_t landing_offset) {
107 476357 : protected_instructions_.push_back({instr_offset, landing_offset});
108 238208 : }
109 :
110 4282903 : void CodeGenerator::CreateFrameAccessState(Frame* frame) {
111 2141467 : FinishFrame(frame);
112 2141466 : frame_access_state_ = new (zone()) FrameAccessState(frame);
113 2141466 : }
114 :
115 3306870 : CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
116 3306868 : int deoptimization_id, SourcePosition pos) {
117 3306870 : if (deoptimization_id > Deoptimizer::kMaxNumberOfEntries) {
118 : return kTooManyDeoptimizationBailouts;
119 : }
120 :
121 3306860 : DeoptimizeKind deopt_kind = GetDeoptimizationKind(deoptimization_id);
122 : DeoptimizeReason deoptimization_reason =
123 3306861 : GetDeoptimizationReason(deoptimization_id);
124 : Address deopt_entry =
125 3306869 : Deoptimizer::GetDeoptimizationEntry(tasm()->isolate(), deopt_kind);
126 3306868 : if (info()->is_source_positions_enabled()) {
127 99479 : tasm()->RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
128 : }
129 3306868 : tasm()->CallForDeoptimization(deopt_entry, deoptimization_id);
130 3306867 : return kSuccess;
131 : }
132 :
133 36 : void CodeGenerator::MaybeEmitOutOfLineConstantPool() {
134 : tasm()->MaybeEmitOutOfLineConstantPool();
135 36 : }
136 :
137 49598162 : void CodeGenerator::AssembleCode() {
138 126 : OptimizedCompilationInfo* info = this->info();
139 :
140 : // Open a frame scope to indicate that there is a frame on the stack. The
141 : // MANUAL indicates that the scope shouldn't actually generate code to set up
142 : // the frame (that is done in AssemblePrologue).
143 2141424 : FrameScope frame_scope(tasm(), StackFrame::MANUAL);
144 :
145 2141408 : if (info->is_source_positions_enabled()) {
146 11105 : AssembleSourcePosition(start_source_position());
147 : }
148 :
149 : // Check that {kJavaScriptCallCodeStartRegister} has been set correctly.
150 2141560 : if (FLAG_debug_code && (info->code_kind() == Code::OPTIMIZED_FUNCTION ||
151 : info->code_kind() == Code::BYTECODE_HANDLER)) {
152 110 : tasm()->RecordComment("-- Prologue: check code start register --");
153 110 : AssembleCodeStartRegisterCheck();
154 : }
155 :
156 : // We want to bailout only from JS functions, which are the only ones
157 : // that are optimized.
158 2141434 : if (info->IsOptimizing()) {
159 : DCHECK(linkage()->GetIncomingDescriptor()->IsJSFunctionCall());
160 456688 : tasm()->RecordComment("-- Prologue: check for deoptimization --");
161 456688 : BailoutIfDeoptimized();
162 : }
163 :
164 2141429 : InitializeSpeculationPoison();
165 :
166 : // Define deoptimization literals for all inlined functions.
167 : DCHECK_EQ(0u, deoptimization_literals_.size());
168 4348728 : for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
169 : info->inlined_functions()) {
170 66098 : if (!inlined.shared_info.equals(info->shared_info())) {
171 : int index = DefineDeoptimizationLiteral(
172 65680 : DeoptimizationLiteral(inlined.shared_info));
173 : inlined.RegisterInlinedFunctionId(index);
174 : }
175 : }
176 2141315 : inlined_function_count_ = deoptimization_literals_.size();
177 :
178 : // Define deoptimization literals for all BytecodeArrays to which we might
179 : // deopt to ensure they are strongly held by the optimized code.
180 2141315 : if (info->has_bytecode_array()) {
181 456686 : DefineDeoptimizationLiteral(DeoptimizationLiteral(info->bytecode_array()));
182 : }
183 4348712 : for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
184 : info->inlined_functions()) {
185 66098 : DefineDeoptimizationLiteral(DeoptimizationLiteral(inlined.bytecode_array));
186 : }
187 :
188 : unwinding_info_writer_.SetNumberOfInstructionBlocks(
189 2141307 : instructions()->InstructionBlockCount());
190 :
191 2141062 : if (info->trace_turbo_json_enabled()) {
192 27 : block_starts_.assign(instructions()->instruction_blocks().size(), -1);
193 6 : instr_starts_.assign(instructions()->instructions().size(), -1);
194 : }
195 :
196 : // Assemble instructions in assembly order.
197 62613043 : for (const InstructionBlock* block : instructions()->ao_blocks()) {
198 : // Align loop headers on vendor recommended boundaries.
199 19440916 : if (block->ShouldAlign() && !tasm()->jump_optimization_info()) {
200 57812 : tasm()->CodeTargetAlign();
201 : }
202 19440948 : if (info->trace_turbo_json_enabled()) {
203 57 : block_starts_[block->rpo_number().ToInt()] = tasm()->pc_offset();
204 : }
205 : // Bind a label for a block.
206 19440948 : current_block_ = block->rpo_number();
207 38881896 : unwinding_info_writer_.BeginInstructionBlock(tasm()->pc_offset(), block);
208 19440926 : if (FLAG_code_comments) {
209 1984 : std::ostringstream buffer;
210 1984 : buffer << "-- B" << block->rpo_number().ToInt() << " start";
211 1984 : if (block->IsDeferred()) buffer << " (deferred)";
212 1984 : if (!block->needs_frame()) buffer << " (no frame)";
213 1984 : if (block->must_construct_frame()) buffer << " (construct frame)";
214 1984 : if (block->must_deconstruct_frame()) buffer << " (deconstruct frame)";
215 :
216 1984 : if (block->IsLoopHeader()) {
217 18 : buffer << " (loop up to " << block->loop_end().ToInt() << ")";
218 : }
219 1984 : if (block->loop_header().IsValid()) {
220 819 : buffer << " (in loop " << block->loop_header().ToInt() << ")";
221 : }
222 1984 : buffer << " --";
223 3968 : tasm()->RecordComment(buffer.str().c_str());
224 : }
225 :
226 38881852 : frame_access_state()->MarkHasFrame(block->needs_frame());
227 :
228 19440982 : tasm()->bind(GetLabel(current_block_));
229 :
230 19441037 : TryInsertBranchPoisoning(block);
231 :
232 19440912 : if (block->must_construct_frame()) {
233 2151225 : AssembleConstructFrame();
234 : // We need to setup the root register after we assemble the prologue, to
235 : // avoid clobbering callee saved registers in case of C linkage and
236 : // using the roots.
237 : // TODO(mtrofin): investigate how we can avoid doing this repeatedly.
238 2151183 : if (linkage()->GetIncomingDescriptor()->InitializeRootRegister()) {
239 888336 : tasm()->InitializeRootRegister();
240 : }
241 : }
242 :
243 : if (FLAG_enable_embedded_constant_pool && !block->needs_frame()) {
244 : ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
245 : result_ = AssembleBlock(block);
246 : } else {
247 19440870 : result_ = AssembleBlock(block);
248 : }
249 19441192 : if (result_ != kSuccess) return;
250 19441229 : unwinding_info_writer_.EndInstructionBlock(block);
251 : }
252 :
253 : // Assemble all out-of-line code.
254 2141291 : if (ools_) {
255 140311 : tasm()->RecordComment("-- Out of line code --");
256 790674 : for (OutOfLineCode* ool = ools_; ool; ool = ool->next()) {
257 650375 : tasm()->bind(ool->entry());
258 650385 : ool->Generate();
259 650395 : if (ool->exit()->is_bound()) tasm()->jmp(ool->exit());
260 : }
261 : }
262 :
263 : // This nop operation is needed to ensure that the trampoline is not
264 : // confused with the pc of the call before deoptimization.
265 : // The test regress/regress-259 is an example of where we need it.
266 2141279 : tasm()->nop();
267 :
268 : // Assemble deoptimization exits.
269 : int last_updated = 0;
270 11924976 : for (DeoptimizationExit* exit : deoptimization_exits_) {
271 3261254 : tasm()->bind(exit->label());
272 3261258 : int trampoline_pc = tasm()->pc_offset();
273 : int deoptimization_id = exit->deoptimization_id();
274 6189283 : DeoptimizationState* ds = deoptimization_states_[deoptimization_id];
275 :
276 3261258 : if (ds->kind() == DeoptimizeKind::kLazy) {
277 : last_updated = safepoints()->UpdateDeoptimizationInfo(
278 2928025 : ds->pc_offset(), trampoline_pc, last_updated);
279 : }
280 3261262 : result_ = AssembleDeoptimizerCall(deoptimization_id, exit->pos());
281 3261265 : if (result_ != kSuccess) return;
282 : }
283 :
284 : // TODO(jgruber): Move all inlined metadata generation into a new,
285 : // architecture-independent version of FinishCode. Currently, this includes
286 : // the safepoint table, handler table, constant pool, and code comments, in
287 : // that order.
288 2141210 : FinishCode();
289 :
290 : // Emit the jump tables.
291 2141274 : if (jump_tables_) {
292 312 : tasm()->Align(kSystemPointerSize);
293 940 : for (JumpTable* table = jump_tables_; table; table = table->next()) {
294 314 : tasm()->bind(table->label());
295 314 : AssembleJumpTable(table->targets(), table->target_count());
296 : }
297 : }
298 :
299 : // The PerfJitLogger logs code up until here, excluding the safepoint
300 : // table. Resolve the unwinding info now so it is aware of the same code
301 : // size as reported by perf.
302 2141274 : unwinding_info_writer_.Finish(tasm()->pc_offset());
303 :
304 2141274 : safepoints()->Emit(tasm(), frame()->GetTotalFrameSlotCount());
305 :
306 : // Emit the exception handler table.
307 2379595 : if (!handlers_.empty()) {
308 : handler_table_offset_ = HandlerTable::EmitReturnTableStart(
309 19888 : tasm(), static_cast<int>(handlers_.size()));
310 476484 : for (size_t i = 0; i < handlers_.size(); ++i) {
311 : HandlerTable::EmitReturnEntry(tasm(), handlers_[i].pc_offset,
312 218354 : handlers_[i].handler->pos());
313 : }
314 : }
315 :
316 : tasm()->MaybeEmitOutOfLineConstantPool();
317 2141353 : tasm()->FinalizeJumpOptimizationInfo();
318 :
319 2141249 : result_ = kSuccess;
320 : }
321 :
322 32997915 : void CodeGenerator::TryInsertBranchPoisoning(const InstructionBlock* block) {
323 : // See if our predecessor was a basic block terminated by a branch_and_poison
324 : // instruction. If yes, then perform the masking based on the flags.
325 25325160 : if (block->PredecessorCount() != 1) return;
326 13556877 : RpoNumber pred_rpo = (block->predecessors())[0];
327 13556877 : const InstructionBlock* pred = instructions()->InstructionBlockAt(pred_rpo);
328 13556829 : if (pred->code_start() == pred->code_end()) return;
329 27113759 : Instruction* instr = instructions()->InstructionAt(pred->code_end() - 1);
330 13556891 : FlagsMode mode = FlagsModeField::decode(instr->opcode());
331 13556891 : switch (mode) {
332 : case kFlags_branch_and_poison: {
333 : BranchInfo branch;
334 0 : RpoNumber target = ComputeBranchInfo(&branch, instr);
335 0 : if (!target.IsValid()) {
336 : // Non-trivial branch, add the masking code.
337 0 : FlagsCondition condition = branch.condition;
338 0 : if (branch.false_label == GetLabel(block->rpo_number())) {
339 : condition = NegateFlagsCondition(condition);
340 : }
341 0 : AssembleBranchPoisoning(condition, instr);
342 : }
343 : break;
344 : }
345 : case kFlags_deoptimize_and_poison: {
346 0 : UNREACHABLE();
347 : break;
348 : }
349 : default:
350 : break;
351 : }
352 : }
353 :
354 124998 : void CodeGenerator::AssembleArchBinarySearchSwitchRange(
355 : Register input, RpoNumber def_block, std::pair<int32_t, Label*>* begin,
356 : std::pair<int32_t, Label*>* end) {
357 124998 : if (end - begin < kBinarySearchSwitchMinimalCases) {
358 282106 : while (begin != end) {
359 202246 : tasm()->JumpIfEqual(input, begin->first, begin->second);
360 202246 : ++begin;
361 : }
362 79860 : AssembleArchJump(def_block);
363 204858 : return;
364 : }
365 45138 : auto middle = begin + (end - begin) / 2;
366 45138 : Label less_label;
367 45138 : tasm()->JumpIfLessThan(input, middle->first, &less_label);
368 45138 : AssembleArchBinarySearchSwitchRange(input, def_block, middle, end);
369 45138 : tasm()->bind(&less_label);
370 45138 : AssembleArchBinarySearchSwitchRange(input, def_block, begin, middle);
371 : }
372 :
373 505465 : OwnedVector<byte> CodeGenerator::GetSourcePositionTable() {
374 505465 : return source_position_table_builder_.ToSourcePositionTableVector();
375 : }
376 :
377 : OwnedVector<trap_handler::ProtectedInstructionData>
378 505506 : CodeGenerator::GetProtectedInstructions() {
379 : return OwnedVector<trap_handler::ProtectedInstructionData>::Of(
380 505506 : protected_instructions_);
381 : }
382 :
383 12633580 : MaybeHandle<Code> CodeGenerator::FinalizeCode() {
384 1579198 : if (result_ != kSuccess) {
385 9 : tasm()->AbortedCodeGeneration();
386 9 : return MaybeHandle<Code>();
387 : }
388 :
389 : // Allocate the source position table.
390 : Handle<ByteArray> source_positions =
391 1579189 : source_position_table_builder_.ToSourcePositionTable(isolate());
392 :
393 : // Allocate deoptimization data.
394 1579192 : Handle<DeoptimizationData> deopt_data = GenerateDeoptimizationData();
395 :
396 : // Allocate and install the code.
397 1579199 : CodeDesc desc;
398 3158398 : tasm()->GetCode(isolate(), &desc, safepoints(), handler_table_offset_);
399 1579198 : if (unwinding_info_writer_.eh_frame_writer()) {
400 22 : unwinding_info_writer_.eh_frame_writer()->GetEhFrame(&desc);
401 : }
402 :
403 : MaybeHandle<Code> maybe_code = isolate()->factory()->TryNewCode(
404 : desc, info()->code_kind(), Handle<Object>(), info()->builtin_index(),
405 : source_positions, deopt_data, kMovable, true,
406 6316792 : frame()->GetTotalFrameSlotCount());
407 :
408 : Handle<Code> code;
409 1579200 : if (!maybe_code.ToHandle(&code)) {
410 0 : tasm()->AbortedCodeGeneration();
411 0 : return MaybeHandle<Code>();
412 : }
413 :
414 : isolate()->counters()->total_compiled_code_size()->Increment(
415 1579200 : code->raw_instruction_size());
416 :
417 1579272 : LOG_CODE_EVENT(isolate(),
418 : CodeLinePosInfoRecordEvent(code->raw_instruction_start(),
419 : *source_positions));
420 :
421 1579200 : return code;
422 : }
423 :
424 15725024 : bool CodeGenerator::IsNextInAssemblyOrder(RpoNumber block) const {
425 : return instructions()
426 : ->InstructionBlockAt(current_block_)
427 15725050 : ->ao_number()
428 31450075 : .IsNext(instructions()->InstructionBlockAt(block)->ao_number());
429 : }
430 :
431 5812978 : void CodeGenerator::RecordSafepoint(ReferenceMap* references,
432 : Safepoint::Kind kind,
433 5812988 : Safepoint::DeoptMode deopt_mode) {
434 5812978 : Safepoint safepoint = safepoints()->DefineSafepoint(tasm(), kind, deopt_mode);
435 : int stackSlotToSpillSlotDelta =
436 5812988 : frame()->GetTotalFrameSlotCount() - frame()->GetSpillSlotCount();
437 88947076 : for (const InstructionOperand& operand : references->reference_operands()) {
438 77320989 : if (operand.IsStackSlot()) {
439 : int index = LocationOperand::cast(operand).index();
440 : DCHECK_LE(0, index);
441 : // We might index values in the fixed part of the frame (i.e. the
442 : // closure pointer or the context pointer); these are not spill slots
443 : // and therefore don't work with the SafepointTable currently, but
444 : // we also don't need to worry about them, since the GC has special
445 : // knowledge about those fields anyway.
446 15982929 : if (index < stackSlotToSpillSlotDelta) continue;
447 15982425 : safepoint.DefinePointerSlot(index);
448 61338060 : } else if (operand.IsRegister() && (kind & Safepoint::kWithRegisters)) {
449 0 : Register reg = LocationOperand::cast(operand).GetRegister();
450 0 : safepoint.DefinePointerRegister(reg);
451 : }
452 : }
453 5813099 : }
454 :
455 8361588 : bool CodeGenerator::IsMaterializableFromRoot(Handle<HeapObject> object,
456 15880903 : RootIndex* index_return) {
457 : const CallDescriptor* incoming_descriptor =
458 : linkage()->GetIncomingDescriptor();
459 8361588 : if (incoming_descriptor->flags() & CallDescriptor::kCanUseRoots) {
460 9372187 : return isolate()->roots_table().IsRootHandle(object, index_return) &&
461 9372187 : RootsTable::IsImmortalImmovable(*index_return);
462 : }
463 : return false;
464 : }
465 :
466 19440864 : CodeGenerator::CodeGenResult CodeGenerator::AssembleBlock(
467 228399688 : const InstructionBlock* block) {
468 165227056 : for (int i = block->code_start(); i < block->code_end(); ++i) {
469 63172648 : if (info()->trace_turbo_json_enabled()) {
470 189 : instr_starts_[i] = tasm()->pc_offset();
471 : }
472 : Instruction* instr = instructions()->InstructionAt(i);
473 63172675 : CodeGenResult result = AssembleInstruction(instr, block);
474 63172516 : if (result != kSuccess) return result;
475 : }
476 : return kSuccess;
477 : }
478 :
479 36368 : bool CodeGenerator::IsValidPush(InstructionOperand source,
480 : CodeGenerator::PushTypeFlags push_type) {
481 36368 : if (source.IsImmediate() &&
482 : ((push_type & CodeGenerator::kImmediatePush) != 0)) {
483 : return true;
484 : }
485 59856 : if (source.IsRegister() &&
486 : ((push_type & CodeGenerator::kRegisterPush) != 0)) {
487 : return true;
488 : }
489 24992 : if (source.IsStackSlot() &&
490 : ((push_type & CodeGenerator::kStackSlotPush) != 0)) {
491 : return true;
492 : }
493 768 : return false;
494 : }
495 :
496 119308 : void CodeGenerator::GetPushCompatibleMoves(Instruction* instr,
497 : PushTypeFlags push_type,
498 : ZoneVector<MoveOperands*>* pushes) {
499 174212 : pushes->clear();
500 325332 : for (int i = Instruction::FIRST_GAP_POSITION;
501 : i <= Instruction::LAST_GAP_POSITION; ++i) {
502 : Instruction::GapPosition inner_pos =
503 : static_cast<Instruction::GapPosition>(i);
504 : ParallelMove* parallel_move = instr->GetParallelMove(inner_pos);
505 222320 : if (parallel_move != nullptr) {
506 682462 : for (auto move : *parallel_move) {
507 428686 : InstructionOperand source = move->source();
508 428686 : InstructionOperand destination = move->destination();
509 : int first_push_compatible_index =
510 : V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0;
511 : // If there are any moves from slots that will be overridden by pushes,
512 : // then the full gap resolver must be used since optimization with
513 : // pushes don't participate in the parallel move and might clobber
514 : // values needed for the gap resolve.
515 457618 : if (source.IsStackSlot() && LocationOperand::cast(source).index() >=
516 : first_push_compatible_index) {
517 : pushes->clear();
518 119308 : return;
519 : }
520 : // TODO(danno): Right now, only consider moves from the FIRST gap for
521 : // pushes. Theoretically, we could extract pushes for both gaps (there
522 : // are cases where this happens), but the logic for that would also have
523 : // to check to make sure that non-memory inputs to the pushes from the
524 : // LAST gap don't get clobbered in the FIRST gap.
525 412390 : if (i == Instruction::FIRST_GAP_POSITION) {
526 467394 : if (destination.IsStackSlot() &&
527 : LocationOperand::cast(destination).index() >=
528 : first_push_compatible_index) {
529 : int index = LocationOperand::cast(destination).index();
530 36368 : if (IsValidPush(source, push_type)) {
531 35600 : if (index >= static_cast<int>(pushes->size())) {
532 35600 : pushes->resize(index + 1);
533 : }
534 71200 : (*pushes)[index] = move;
535 : }
536 : }
537 : }
538 : }
539 : }
540 : }
541 :
542 : // For now, only support a set of continuous pushes at the end of the list.
543 : size_t push_count_upper_bound = pushes->size();
544 : size_t push_begin = push_count_upper_bound;
545 173096 : for (auto move : base::Reversed(*pushes)) {
546 41092 : if (move == nullptr) break;
547 28992 : push_begin--;
548 : }
549 103012 : size_t push_count = pushes->size() - push_begin;
550 : std::copy(pushes->begin() + push_begin,
551 : pushes->begin() + push_begin + push_count, pushes->begin());
552 103012 : pushes->resize(push_count);
553 : }
554 :
555 35844273 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferMove(
556 : InstructionOperand* source, InstructionOperand* destination) {
557 35844273 : if (source->IsConstant()) {
558 18559258 : if (destination->IsAnyRegister()) {
559 : return MoveType::kConstantToRegister;
560 : } else {
561 : DCHECK(destination->IsAnyStackSlot());
562 45287 : return MoveType::kConstantToStack;
563 : }
564 : }
565 : DCHECK(LocationOperand::cast(source)->IsCompatible(
566 : LocationOperand::cast(destination)));
567 17285015 : if (source->IsAnyRegister()) {
568 9053985 : if (destination->IsAnyRegister()) {
569 : return MoveType::kRegisterToRegister;
570 : } else {
571 : DCHECK(destination->IsAnyStackSlot());
572 4948630 : return MoveType::kRegisterToStack;
573 : }
574 : } else {
575 : DCHECK(source->IsAnyStackSlot());
576 8231030 : if (destination->IsAnyRegister()) {
577 : return MoveType::kStackToRegister;
578 : } else {
579 : DCHECK(destination->IsAnyStackSlot());
580 44286 : return MoveType::kStackToStack;
581 : }
582 : }
583 : }
584 :
585 75903 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferSwap(
586 : InstructionOperand* source, InstructionOperand* destination) {
587 : DCHECK(LocationOperand::cast(source)->IsCompatible(
588 : LocationOperand::cast(destination)));
589 75903 : if (source->IsAnyRegister()) {
590 72186 : if (destination->IsAnyRegister()) {
591 : return MoveType::kRegisterToRegister;
592 : } else {
593 : DCHECK(destination->IsAnyStackSlot());
594 6899 : return MoveType::kRegisterToStack;
595 : }
596 : } else {
597 : DCHECK(source->IsAnyStackSlot());
598 : DCHECK(destination->IsAnyStackSlot());
599 : return MoveType::kStackToStack;
600 : }
601 : }
602 :
603 5358488 : RpoNumber CodeGenerator::ComputeBranchInfo(BranchInfo* branch,
604 26786050 : Instruction* instr) {
605 : // Assemble a branch after this instruction.
606 : InstructionOperandConverter i(this, instr);
607 5358488 : RpoNumber true_rpo = i.InputRpo(instr->InputCount() - 2);
608 5358500 : RpoNumber false_rpo = i.InputRpo(instr->InputCount() - 1);
609 :
610 5358503 : if (true_rpo == false_rpo) {
611 2147 : return true_rpo;
612 : }
613 5356356 : FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
614 5356356 : if (IsNextInAssemblyOrder(true_rpo)) {
615 : // true block is next, can fall through if condition negated.
616 : std::swap(true_rpo, false_rpo);
617 : condition = NegateFlagsCondition(condition);
618 : }
619 5356353 : branch->condition = condition;
620 5356353 : branch->true_label = GetLabel(true_rpo);
621 5356353 : branch->false_label = GetLabel(false_rpo);
622 5356353 : branch->fallthru = IsNextInAssemblyOrder(false_rpo);
623 : return RpoNumber::Invalid();
624 : }
625 :
626 63172641 : CodeGenerator::CodeGenResult CodeGenerator::AssembleInstruction(
627 194445764 : Instruction* instr, const InstructionBlock* block) {
628 : int first_unused_stack_slot;
629 63172641 : FlagsMode mode = FlagsModeField::decode(instr->opcode());
630 63172641 : if (mode != kFlags_trap) {
631 63139378 : AssembleSourcePosition(instr);
632 : }
633 : bool adjust_stack =
634 63172802 : GetSlotAboveSPBeforeTailCall(instr, &first_unused_stack_slot);
635 63173149 : if (adjust_stack) AssembleTailCallBeforeGap(instr, first_unused_stack_slot);
636 63173149 : AssembleGaps(instr);
637 63173051 : if (adjust_stack) AssembleTailCallAfterGap(instr, first_unused_stack_slot);
638 : DCHECK_IMPLIES(
639 : block->must_deconstruct_frame(),
640 : instr != instructions()->InstructionAt(block->last_instruction_index()) ||
641 : instr->IsRet() || instr->IsJump());
642 68103231 : if (instr->IsJump() && block->must_deconstruct_frame()) {
643 48502 : AssembleDeconstructFrame();
644 : }
645 : // Assemble architecture-specific code for the instruction.
646 63173123 : CodeGenResult result = AssembleArchInstruction(instr);
647 63172419 : if (result != kSuccess) return result;
648 :
649 63172587 : FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
650 63172587 : switch (mode) {
651 : case kFlags_branch:
652 : case kFlags_branch_and_poison: {
653 : BranchInfo branch;
654 5358491 : RpoNumber target = ComputeBranchInfo(&branch, instr);
655 5358501 : if (target.IsValid()) {
656 : // redundant branch.
657 2147 : if (!IsNextInAssemblyOrder(target)) {
658 320 : AssembleArchJump(target);
659 : }
660 2147 : return kSuccess;
661 : }
662 : // Assemble architecture-specific branch.
663 5356354 : AssembleArchBranch(instr, &branch);
664 5356332 : break;
665 : }
666 : case kFlags_deoptimize:
667 : case kFlags_deoptimize_and_poison: {
668 : // Assemble a conditional eager deoptimization after this instruction.
669 : InstructionOperandConverter i(this, instr);
670 333239 : size_t frame_state_offset = MiscField::decode(instr->opcode());
671 : DeoptimizationExit* const exit =
672 333239 : AddDeoptimizationExit(instr, frame_state_offset);
673 333240 : Label continue_label;
674 : BranchInfo branch;
675 333240 : branch.condition = condition;
676 333240 : branch.true_label = exit->label();
677 333240 : branch.false_label = &continue_label;
678 333240 : branch.fallthru = true;
679 : // Assemble architecture-specific branch.
680 333240 : AssembleArchDeoptBranch(instr, &branch);
681 333238 : tasm()->bind(&continue_label);
682 333240 : if (mode == kFlags_deoptimize_and_poison) {
683 0 : AssembleBranchPoisoning(NegateFlagsCondition(branch.condition), instr);
684 : }
685 : break;
686 : }
687 : case kFlags_set: {
688 : // Assemble a boolean materialization after this instruction.
689 376860 : AssembleArchBoolean(instr, condition);
690 376859 : break;
691 : }
692 : case kFlags_trap: {
693 33448 : AssembleArchTrap(instr, condition);
694 33447 : break;
695 : }
696 : case kFlags_none: {
697 : break;
698 : }
699 : }
700 :
701 : // TODO(jarin) We should thread the flag through rather than set it.
702 63170428 : if (instr->IsCall()) {
703 : ResetSpeculationPoison();
704 : }
705 :
706 : return kSuccess;
707 : }
708 :
709 103567624 : void CodeGenerator::AssembleSourcePosition(Instruction* instr) {
710 63257831 : SourcePosition source_position = SourcePosition::Unknown();
711 121867479 : if (instr->IsNop() && instr->AreMovesRedundant()) return;
712 40309793 : if (!instructions()->GetSourcePosition(instr, &source_position)) return;
713 4648531 : AssembleSourcePosition(source_position);
714 : }
715 :
716 4660667 : void CodeGenerator::AssembleSourcePosition(SourcePosition source_position) {
717 4659630 : if (source_position == current_source_position_) return;
718 3512455 : current_source_position_ = source_position;
719 3512455 : if (!source_position.IsKnown()) return;
720 3512415 : source_position_table_builder_.AddPosition(tasm()->pc_offset(),
721 3512415 : source_position, false);
722 3512536 : if (FLAG_code_comments) {
723 : OptimizedCompilationInfo* info = this->info();
724 1037 : if (info->IsNotOptimizedFunctionOrWasmFunction()) return;
725 1037 : std::ostringstream buffer;
726 1037 : buffer << "-- ";
727 : // Turbolizer only needs the source position, as it can reconstruct
728 : // the inlining stack from other information.
729 2074 : if (info->trace_turbo_json_enabled() || !tasm()->isolate() ||
730 1037 : tasm()->isolate()->concurrent_recompilation_enabled()) {
731 153 : buffer << source_position;
732 : } else {
733 : AllowHeapAllocation allocation;
734 : AllowHandleAllocation handles;
735 : AllowHandleDereference deref;
736 1768 : buffer << source_position.InliningStack(info);
737 : }
738 1037 : buffer << " --";
739 2074 : tasm()->RecordComment(buffer.str().c_str());
740 : }
741 : }
742 :
743 63292115 : bool CodeGenerator::GetSlotAboveSPBeforeTailCall(Instruction* instr,
744 : int* slot) {
745 63172819 : if (instr->IsTailCall()) {
746 : InstructionOperandConverter g(this, instr);
747 119296 : *slot = g.InputInt32(instr->InputCount() - 1);
748 : return true;
749 : } else {
750 : return false;
751 : }
752 : }
753 :
754 458776 : StubCallMode CodeGenerator::DetermineStubCallMode() const {
755 458776 : Code::Kind code_kind = info()->code_kind();
756 458776 : return (code_kind == Code::WASM_FUNCTION ||
757 : code_kind == Code::WASM_TO_JS_FUNCTION)
758 : ? StubCallMode::kCallWasmRuntimeStub
759 458776 : : StubCallMode::kCallCodeObject;
760 : }
761 :
762 63172848 : void CodeGenerator::AssembleGaps(Instruction* instr) {
763 189518450 : for (int i = Instruction::FIRST_GAP_POSITION;
764 : i <= Instruction::LAST_GAP_POSITION; i++) {
765 : Instruction::GapPosition inner_pos =
766 : static_cast<Instruction::GapPosition>(i);
767 : ParallelMove* move = instr->GetParallelMove(inner_pos);
768 126345352 : if (move != nullptr) resolver()->Resolve(move);
769 : }
770 63173098 : }
771 :
772 : namespace {
773 :
774 456573 : Handle<PodArray<InliningPosition>> CreateInliningPositions(
775 : OptimizedCompilationInfo* info, Isolate* isolate) {
776 535815 : const OptimizedCompilationInfo::InlinedFunctionList& inlined_functions =
777 : info->inlined_functions();
778 456573 : if (inlined_functions.size() == 0) {
779 : return Handle<PodArray<InliningPosition>>::cast(
780 443310 : isolate->factory()->empty_byte_array());
781 : }
782 : Handle<PodArray<InliningPosition>> inl_positions =
783 : PodArray<InliningPosition>::New(
784 13263 : isolate, static_cast<int>(inlined_functions.size()), TENURED);
785 158484 : for (size_t i = 0; i < inlined_functions.size(); ++i) {
786 : inl_positions->set(static_cast<int>(i), inlined_functions[i].position);
787 : }
788 13263 : return inl_positions;
789 : }
790 :
791 : } // namespace
792 :
793 6307298 : Handle<DeoptimizationData> CodeGenerator::GenerateDeoptimizationData() {
794 456574 : OptimizedCompilationInfo* info = this->info();
795 1579195 : int deopt_count = static_cast<int>(deoptimization_states_.size());
796 2701818 : if (deopt_count == 0 && !info->is_osr()) {
797 1122623 : return DeoptimizationData::Empty(isolate());
798 : }
799 : Handle<DeoptimizationData> data =
800 456572 : DeoptimizationData::New(isolate(), deopt_count, TENURED);
801 :
802 : Handle<ByteArray> translation_array =
803 456571 : translations_.CreateByteArray(isolate()->factory());
804 :
805 913146 : data->SetTranslationByteArray(*translation_array);
806 : data->SetInlinedFunctionCount(
807 456574 : Smi::FromInt(static_cast<int>(inlined_function_count_)));
808 : data->SetOptimizationId(Smi::FromInt(info->optimization_id()));
809 :
810 456574 : if (info->has_shared_info()) {
811 913148 : data->SetSharedFunctionInfo(*info->shared_info());
812 : } else {
813 0 : data->SetSharedFunctionInfo(Smi::kZero);
814 : }
815 :
816 : Handle<FixedArray> literals = isolate()->factory()->NewFixedArray(
817 913148 : static_cast<int>(deoptimization_literals_.size()), TENURED);
818 4471528 : for (unsigned i = 0; i < deoptimization_literals_.size(); i++) {
819 1779190 : Handle<Object> object = deoptimization_literals_[i].Reify(isolate());
820 3558387 : literals->set(i, *object);
821 : }
822 913147 : data->SetLiteralArray(*literals);
823 :
824 : Handle<PodArray<InliningPosition>> inl_pos =
825 456573 : CreateInliningPositions(info, isolate());
826 913147 : data->SetInliningPositions(*inl_pos);
827 :
828 456574 : if (info->is_osr()) {
829 : DCHECK_LE(0, osr_pc_offset_);
830 4991 : data->SetOsrBytecodeOffset(Smi::FromInt(info_->osr_offset().ToInt()));
831 4991 : data->SetOsrPcOffset(Smi::FromInt(osr_pc_offset_));
832 : } else {
833 : BailoutId osr_offset = BailoutId::None();
834 : data->SetOsrBytecodeOffset(Smi::FromInt(osr_offset.ToInt()));
835 : data->SetOsrPcOffset(Smi::FromInt(-1));
836 : }
837 :
838 : // Populate deoptimization entries.
839 3272608 : for (int i = 0; i < deopt_count; i++) {
840 9817826 : DeoptimizationState* deoptimization_state = deoptimization_states_[i];
841 6545218 : data->SetBytecodeOffset(i, deoptimization_state->bailout_id());
842 3272609 : CHECK(deoptimization_state);
843 : data->SetTranslationIndex(
844 3272609 : i, Smi::FromInt(deoptimization_state->translation_id()));
845 3272608 : data->SetPc(i, Smi::FromInt(deoptimization_state->pc_offset()));
846 : }
847 :
848 456574 : return data;
849 : }
850 :
851 314 : Label* CodeGenerator::AddJumpTable(Label** targets, size_t target_count) {
852 628 : jump_tables_ = new (zone()) JumpTable(jump_tables_, targets, target_count);
853 314 : return jump_tables_->label();
854 : }
855 :
856 15015793 : void CodeGenerator::RecordCallPosition(Instruction* instr) {
857 5694449 : CallDescriptor::Flags flags(MiscField::decode(instr->opcode()));
858 :
859 5694449 : bool needs_frame_state = (flags & CallDescriptor::kNeedsFrameState);
860 :
861 : RecordSafepoint(
862 : instr->reference_map(), Safepoint::kSimple,
863 11388898 : needs_frame_state ? Safepoint::kLazyDeopt : Safepoint::kNoLazyDeopt);
864 :
865 5694485 : if (flags & CallDescriptor::kHasExceptionHandler) {
866 : InstructionOperandConverter i(this, instr);
867 218354 : RpoNumber handler_rpo = i.InputRpo(instr->InputCount() - 1);
868 655061 : handlers_.push_back({GetLabel(handler_rpo), tasm()->pc_offset()});
869 : }
870 :
871 5694484 : if (needs_frame_state) {
872 : MarkLazyDeoptSite();
873 : // If the frame state is present, it starts at argument 2 - after
874 : // the code address and the poison-alias index.
875 : size_t frame_state_offset = 2;
876 : FrameStateDescriptor* descriptor =
877 3190184 : GetDeoptimizationEntry(instr, frame_state_offset).descriptor();
878 3190185 : int pc_offset = tasm()->pc_offset();
879 : int deopt_state_id = BuildTranslation(instr, pc_offset, frame_state_offset,
880 3190185 : descriptor->state_combine());
881 :
882 : DeoptimizationExit* const exit = new (zone())
883 6380374 : DeoptimizationExit(deopt_state_id, current_source_position_);
884 3190187 : deoptimization_exits_.push_back(exit);
885 3190184 : safepoints()->RecordLazyDeoptimizationIndex(deopt_state_id);
886 : }
887 5694486 : }
888 :
889 8079733 : int CodeGenerator::DefineDeoptimizationLiteral(DeoptimizationLiteral literal) {
890 8079733 : int result = static_cast<int>(deoptimization_literals_.size());
891 159227334 : for (unsigned i = 0; i < deoptimization_literals_.size(); ++i) {
892 77833858 : if (deoptimization_literals_[i] == literal) return i;
893 : }
894 1779796 : deoptimization_literals_.push_back(literal);
895 1779789 : return result;
896 : }
897 :
898 6759205 : DeoptimizationEntry const& CodeGenerator::GetDeoptimizationEntry(
899 6759215 : Instruction* instr, size_t frame_state_offset) {
900 : InstructionOperandConverter i(this, instr);
901 6759205 : int const state_id = i.InputInt32(frame_state_offset);
902 6759215 : return instructions()->GetDeoptimizationEntry(state_id);
903 : }
904 :
905 3306859 : DeoptimizeKind CodeGenerator::GetDeoptimizationKind(
906 : int deoptimization_id) const {
907 : size_t const index = static_cast<size_t>(deoptimization_id);
908 : DCHECK_LT(index, deoptimization_states_.size());
909 3306861 : return deoptimization_states_[index]->kind();
910 : }
911 :
912 3306861 : DeoptimizeReason CodeGenerator::GetDeoptimizationReason(
913 : int deoptimization_id) const {
914 : size_t const index = static_cast<size_t>(deoptimization_id);
915 : DCHECK_LT(index, deoptimization_states_.size());
916 3306869 : return deoptimization_states_[index]->reason();
917 : }
918 :
919 51189143 : void CodeGenerator::TranslateStateValueDescriptor(
920 51224602 : StateValueDescriptor* desc, StateValueList* nested,
921 456492 : Translation* translation, InstructionOperandIterator* iter) {
922 : // Note:
923 : // If translation is null, we just skip the relevant instruction operands.
924 51189143 : if (desc->IsNested()) {
925 98457 : if (translation != nullptr) {
926 98457 : translation->BeginCapturedObject(static_cast<int>(nested->size()));
927 : }
928 791284 : for (auto field : *nested) {
929 : TranslateStateValueDescriptor(field.desc, field.nested, translation,
930 594370 : iter);
931 : }
932 51090686 : } else if (desc->IsArgumentsElements()) {
933 6180 : if (translation != nullptr) {
934 6180 : translation->ArgumentsElements(desc->arguments_type());
935 : }
936 51084506 : } else if (desc->IsArgumentsLength()) {
937 6500 : if (translation != nullptr) {
938 6500 : translation->ArgumentsLength(desc->arguments_type());
939 : }
940 51078006 : } else if (desc->IsDuplicate()) {
941 22779 : if (translation != nullptr) {
942 22779 : translation->DuplicateObject(static_cast<int>(desc->id()));
943 : }
944 51055227 : } else if (desc->IsPlain()) {
945 23217982 : InstructionOperand* op = iter->Advance();
946 23217982 : if (translation != nullptr) {
947 : AddTranslationForOperand(translation, iter->instruction(), op,
948 23217988 : desc->type());
949 : }
950 : } else {
951 : DCHECK(desc->IsOptimizedOut());
952 27837245 : if (translation != nullptr) {
953 27837244 : if (optimized_out_literal_id_ == -1) {
954 : optimized_out_literal_id_ = DefineDeoptimizationLiteral(
955 456492 : DeoptimizationLiteral(isolate()->factory()->optimized_out()));
956 : }
957 27837237 : translation->StoreLiteral(optimized_out_literal_id_);
958 : }
959 : }
960 51189121 : }
961 :
962 3946949 : void CodeGenerator::TranslateFrameStateDescriptorOperands(
963 : FrameStateDescriptor* desc, InstructionOperandIterator* iter,
964 : Translation* translation) {
965 : size_t index = 0;
966 : StateValueList* values = desc->GetStateValueDescriptors();
967 58488288 : for (StateValueList::iterator it = values->begin(); it != values->end();
968 : ++it, ++index) {
969 50594377 : TranslateStateValueDescriptor((*it).desc, (*it).nested, translation, iter);
970 : }
971 : DCHECK_EQ(desc->GetSize(), index);
972 3946962 : }
973 :
974 3946957 : void CodeGenerator::BuildTranslationForFrameStateDescriptor(
975 14221238 : FrameStateDescriptor* descriptor, InstructionOperandIterator* iter,
976 2100 : Translation* translation, OutputFrameStateCombine state_combine) {
977 : // Outer-most state must be added to translation first.
978 3946957 : if (descriptor->outer_state() != nullptr) {
979 : BuildTranslationForFrameStateDescriptor(descriptor->outer_state(), iter,
980 377929 : translation, state_combine);
981 : }
982 :
983 : Handle<SharedFunctionInfo> shared_info;
984 3946958 : if (!descriptor->shared_info().ToHandle(&shared_info)) {
985 2100 : if (!info()->has_shared_info()) {
986 3946962 : return; // Stub with no SharedFunctionInfo.
987 : }
988 : shared_info = info()->shared_info();
989 : }
990 : int shared_info_id =
991 3946958 : DefineDeoptimizationLiteral(DeoptimizationLiteral(shared_info));
992 :
993 3946969 : switch (descriptor->type()) {
994 : case FrameStateType::kInterpretedFunction: {
995 : int return_offset = 0;
996 : int return_count = 0;
997 3823534 : if (!state_combine.IsOutputIgnored()) {
998 2380343 : return_offset = static_cast<int>(state_combine.GetOffsetToPokeAt());
999 4760686 : return_count = static_cast<int>(iter->instruction()->OutputCount());
1000 : }
1001 : translation->BeginInterpretedFrame(
1002 : descriptor->bailout_id(), shared_info_id,
1003 : static_cast<unsigned int>(descriptor->locals_count() + 1),
1004 3823534 : return_offset, return_count);
1005 3823528 : break;
1006 : }
1007 : case FrameStateType::kArgumentsAdaptor:
1008 : translation->BeginArgumentsAdaptorFrame(
1009 : shared_info_id,
1010 83729 : static_cast<unsigned int>(descriptor->parameters_count()));
1011 83729 : break;
1012 : case FrameStateType::kConstructStub:
1013 : DCHECK(descriptor->bailout_id().IsValidForConstructStub());
1014 : translation->BeginConstructStubFrame(
1015 : descriptor->bailout_id(), shared_info_id,
1016 26155 : static_cast<unsigned int>(descriptor->parameters_count() + 1));
1017 26155 : break;
1018 : case FrameStateType::kBuiltinContinuation: {
1019 2100 : BailoutId bailout_id = descriptor->bailout_id();
1020 : int parameter_count =
1021 : static_cast<unsigned int>(descriptor->parameters_count());
1022 : translation->BeginBuiltinContinuationFrame(bailout_id, shared_info_id,
1023 2100 : parameter_count);
1024 : break;
1025 : }
1026 : case FrameStateType::kJavaScriptBuiltinContinuation: {
1027 11184 : BailoutId bailout_id = descriptor->bailout_id();
1028 : int parameter_count =
1029 : static_cast<unsigned int>(descriptor->parameters_count());
1030 : translation->BeginJavaScriptBuiltinContinuationFrame(
1031 11184 : bailout_id, shared_info_id, parameter_count);
1032 : break;
1033 : }
1034 : case FrameStateType::kJavaScriptBuiltinContinuationWithCatch: {
1035 267 : BailoutId bailout_id = descriptor->bailout_id();
1036 : int parameter_count =
1037 : static_cast<unsigned int>(descriptor->parameters_count());
1038 : translation->BeginJavaScriptBuiltinContinuationWithCatchFrame(
1039 267 : bailout_id, shared_info_id, parameter_count);
1040 : break;
1041 : }
1042 : }
1043 :
1044 3946963 : TranslateFrameStateDescriptorOperands(descriptor, iter, translation);
1045 : }
1046 :
1047 3569027 : int CodeGenerator::BuildTranslation(Instruction* instr, int pc_offset,
1048 : size_t frame_state_offset,
1049 7138072 : OutputFrameStateCombine state_combine) {
1050 7138072 : DeoptimizationEntry const& entry =
1051 3569027 : GetDeoptimizationEntry(instr, frame_state_offset);
1052 : FrameStateDescriptor* const descriptor = entry.descriptor();
1053 3569034 : frame_state_offset++;
1054 :
1055 3569034 : int update_feedback_count = entry.feedback().IsValid() ? 1 : 0;
1056 : Translation translation(&translations_,
1057 3569034 : static_cast<int>(descriptor->GetFrameCount()),
1058 3569034 : static_cast<int>(descriptor->GetJSFrameCount()),
1059 7138073 : update_feedback_count, zone());
1060 3569039 : if (entry.feedback().IsValid()) {
1061 : DeoptimizationLiteral literal =
1062 : DeoptimizationLiteral(entry.feedback().vector());
1063 38854 : int literal_id = DefineDeoptimizationLiteral(literal);
1064 38854 : translation.AddUpdateFeedback(literal_id, entry.feedback().slot().ToInt());
1065 : }
1066 : InstructionOperandIterator iter(instr, frame_state_offset);
1067 : BuildTranslationForFrameStateDescriptor(descriptor, &iter, &translation,
1068 3569039 : state_combine);
1069 :
1070 3569038 : int deoptimization_id = static_cast<int>(deoptimization_states_.size());
1071 :
1072 : deoptimization_states_.push_back(new (zone()) DeoptimizationState(
1073 3569038 : descriptor->bailout_id(), translation.index(), pc_offset, entry.kind(),
1074 7138077 : entry.reason()));
1075 :
1076 3569038 : return deoptimization_id;
1077 : }
1078 :
1079 23218128 : void CodeGenerator::AddTranslationForOperand(Translation* translation,
1080 : Instruction* instr,
1081 : InstructionOperand* op,
1082 6647561 : MachineType type) {
1083 23218128 : if (op->IsStackSlot()) {
1084 15739748 : if (type.representation() == MachineRepresentation::kBit) {
1085 21554 : translation->StoreBoolStackSlot(LocationOperand::cast(op)->index());
1086 47153179 : } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
1087 : type == MachineType::Int32()) {
1088 475418 : translation->StoreInt32StackSlot(LocationOperand::cast(op)->index());
1089 45728315 : } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
1090 : type == MachineType::Uint32()) {
1091 11119 : translation->StoreUint32StackSlot(LocationOperand::cast(op)->index());
1092 15231657 : } else if (type == MachineType::Int64()) {
1093 135 : translation->StoreInt64StackSlot(LocationOperand::cast(op)->index());
1094 : } else {
1095 15231522 : CHECK_EQ(MachineRepresentation::kTagged, type.representation());
1096 15231522 : translation->StoreStackSlot(LocationOperand::cast(op)->index());
1097 : }
1098 7478380 : } else if (op->IsFPStackSlot()) {
1099 252331 : if (type.representation() == MachineRepresentation::kFloat64) {
1100 251434 : translation->StoreDoubleStackSlot(LocationOperand::cast(op)->index());
1101 : } else {
1102 897 : CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
1103 897 : translation->StoreFloatStackSlot(LocationOperand::cast(op)->index());
1104 : }
1105 7226049 : } else if (op->IsRegister()) {
1106 : InstructionOperandConverter converter(this, instr);
1107 514983 : if (type.representation() == MachineRepresentation::kBit) {
1108 3797 : translation->StoreBoolRegister(converter.ToRegister(op));
1109 1532694 : } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
1110 : type == MachineType::Int32()) {
1111 35569 : translation->StoreInt32Register(converter.ToRegister(op));
1112 1426852 : } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
1113 : type == MachineType::Uint32()) {
1114 1603 : translation->StoreUint32Register(converter.ToRegister(op));
1115 474014 : } else if (type == MachineType::Int64()) {
1116 14 : translation->StoreInt64Register(converter.ToRegister(op));
1117 : } else {
1118 474000 : CHECK_EQ(MachineRepresentation::kTagged, type.representation());
1119 474000 : translation->StoreRegister(converter.ToRegister(op));
1120 : }
1121 6711066 : } else if (op->IsFPRegister()) {
1122 : InstructionOperandConverter converter(this, instr);
1123 64301 : if (type.representation() == MachineRepresentation::kFloat64) {
1124 64183 : translation->StoreDoubleRegister(converter.ToDoubleRegister(op));
1125 : } else {
1126 118 : CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
1127 118 : translation->StoreFloatRegister(converter.ToFloatRegister(op));
1128 : }
1129 : } else {
1130 6646765 : CHECK(op->IsImmediate());
1131 : InstructionOperandConverter converter(this, instr);
1132 6646765 : Constant constant = converter.ToConstant(op);
1133 : DeoptimizationLiteral literal;
1134 6646776 : switch (constant.type()) {
1135 : case Constant::kInt32:
1136 16534 : if (type.representation() == MachineRepresentation::kTagged) {
1137 : // When pointers are 4 bytes, we can use int32 constants to represent
1138 : // Smis.
1139 : DCHECK_EQ(4, kSystemPointerSize);
1140 10 : Smi smi(static_cast<Address>(constant.ToInt32()));
1141 : DCHECK(smi->IsSmi());
1142 5 : literal = DeoptimizationLiteral(smi->value());
1143 16529 : } else if (type.representation() == MachineRepresentation::kBit) {
1144 1566 : if (constant.ToInt32() == 0) {
1145 : literal =
1146 340 : DeoptimizationLiteral(isolate()->factory()->false_value());
1147 : } else {
1148 : DCHECK_EQ(1, constant.ToInt32());
1149 443 : literal = DeoptimizationLiteral(isolate()->factory()->true_value());
1150 : }
1151 : } else {
1152 : DCHECK(type == MachineType::Int32() ||
1153 : type == MachineType::Uint32() ||
1154 : type.representation() == MachineRepresentation::kWord32 ||
1155 : type.representation() == MachineRepresentation::kNone);
1156 : DCHECK(type.representation() != MachineRepresentation::kNone ||
1157 : constant.ToInt32() == FrameStateDescriptor::kImpossibleValue);
1158 15746 : if (type == MachineType::Uint32()) {
1159 : literal = DeoptimizationLiteral(
1160 1640 : static_cast<uint32_t>(constant.ToInt32()));
1161 : } else {
1162 29852 : literal = DeoptimizationLiteral(constant.ToInt32());
1163 : }
1164 : }
1165 : break;
1166 : case Constant::kInt64:
1167 : DCHECK_EQ(8, kSystemPointerSize);
1168 548293 : if (type.representation() == MachineRepresentation::kWord64) {
1169 : literal =
1170 556 : DeoptimizationLiteral(static_cast<double>(constant.ToInt64()));
1171 : } else {
1172 : // When pointers are 8 bytes, we can use int64 constants to represent
1173 : // Smis.
1174 : DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
1175 : Smi smi(static_cast<Address>(constant.ToInt64()));
1176 : DCHECK(smi->IsSmi());
1177 547737 : literal = DeoptimizationLiteral(smi->value());
1178 : }
1179 : break;
1180 : case Constant::kFloat32:
1181 : DCHECK(type.representation() == MachineRepresentation::kFloat32 ||
1182 : type.representation() == MachineRepresentation::kTagged);
1183 0 : literal = DeoptimizationLiteral(constant.ToFloat32());
1184 0 : break;
1185 : case Constant::kFloat64:
1186 : DCHECK(type.representation() == MachineRepresentation::kFloat64 ||
1187 : type.representation() == MachineRepresentation::kTagged);
1188 93081 : literal = DeoptimizationLiteral(constant.ToFloat64().value());
1189 93081 : break;
1190 : case Constant::kHeapObject:
1191 : DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
1192 5986378 : literal = DeoptimizationLiteral(constant.ToHeapObject());
1193 5986380 : break;
1194 : case Constant::kDelayedStringConstant:
1195 : DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
1196 2490 : literal = DeoptimizationLiteral(constant.ToDelayedStringConstant());
1197 2490 : break;
1198 : default:
1199 0 : UNREACHABLE();
1200 : }
1201 6646778 : if (literal.object().equals(info()->closure())) {
1202 3597587 : translation->StoreJSFrameFunction();
1203 : } else {
1204 3049191 : int literal_id = DefineDeoptimizationLiteral(literal);
1205 3049188 : translation->StoreLiteral(literal_id);
1206 : }
1207 : }
1208 23218111 : }
1209 :
1210 0 : void CodeGenerator::MarkLazyDeoptSite() {
1211 6380368 : last_lazy_deopt_pc_ = tasm()->pc_offset();
1212 0 : }
1213 :
1214 333237 : DeoptimizationExit* CodeGenerator::AddDeoptimizationExit(
1215 333241 : Instruction* instr, size_t frame_state_offset) {
1216 : int const deoptimization_id = BuildTranslation(
1217 333237 : instr, -1, frame_state_offset, OutputFrameStateCombine::Ignore());
1218 :
1219 : DeoptimizationExit* const exit = new (zone())
1220 666482 : DeoptimizationExit(deoptimization_id, current_source_position_);
1221 333241 : deoptimization_exits_.push_back(exit);
1222 333240 : return exit;
1223 : }
1224 :
1225 2141323 : void CodeGenerator::InitializeSpeculationPoison() {
1226 4282646 : if (poisoning_level_ == PoisoningMitigationLevel::kDontPoison) return;
1227 :
1228 : // Initialize {kSpeculationPoisonRegister} either by comparing the expected
1229 : // with the actual call target, or by unconditionally using {-1} initially.
1230 : // Masking register arguments with it only makes sense in the first case.
1231 0 : if (info()->called_with_code_start_register()) {
1232 0 : tasm()->RecordComment("-- Prologue: generate speculation poison --");
1233 0 : GenerateSpeculationPoisonFromCodeStartRegister();
1234 0 : if (info()->is_poisoning_register_arguments()) {
1235 0 : AssembleRegisterArgumentPoisoning();
1236 : }
1237 : } else {
1238 : ResetSpeculationPoison();
1239 : }
1240 : }
1241 :
1242 4991 : void CodeGenerator::ResetSpeculationPoison() {
1243 5728941 : if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison) {
1244 0 : tasm()->ResetSpeculationPoisonRegister();
1245 : }
1246 4991 : }
1247 :
1248 1300644 : OutOfLineCode::OutOfLineCode(CodeGenerator* gen)
1249 1300644 : : frame_(gen->frame()), tasm_(gen->tasm()), next_(gen->ools_) {
1250 650322 : gen->ools_ = this;
1251 650322 : }
1252 :
1253 : OutOfLineCode::~OutOfLineCode() = default;
1254 :
1255 1779192 : Handle<Object> DeoptimizationLiteral::Reify(Isolate* isolate) const {
1256 1779192 : switch (kind_) {
1257 : case DeoptimizationLiteralKind::kObject: {
1258 1701645 : return object_;
1259 : }
1260 : case DeoptimizationLiteralKind::kNumber: {
1261 76912 : return isolate->factory()->NewNumber(number_);
1262 : }
1263 : case DeoptimizationLiteralKind::kString: {
1264 635 : return string_->AllocateStringConstant(isolate);
1265 : }
1266 : }
1267 0 : UNREACHABLE();
1268 : }
1269 :
1270 : } // namespace compiler
1271 : } // namespace internal
1272 178779 : } // namespace v8
|
