Line data Source code
1 : // Copyright 2013 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/deoptimizer.h"
6 :
7 : #include <memory>
8 :
9 : #include "src/accessors.h"
10 : #include "src/assembler-inl.h"
11 : #include "src/ast/prettyprinter.h"
12 : #include "src/callable.h"
13 : #include "src/counters.h"
14 : #include "src/disasm.h"
15 : #include "src/frames-inl.h"
16 : #include "src/global-handles.h"
17 : #include "src/interpreter/interpreter.h"
18 : #include "src/macro-assembler.h"
19 : #include "src/objects/debug-objects-inl.h"
20 : #include "src/objects/heap-number-inl.h"
21 : #include "src/objects/smi.h"
22 : #include "src/register-configuration.h"
23 : #include "src/tracing/trace-event.h"
24 : #include "src/v8.h"
25 :
26 : // Has to be the last include (doesn't have include guards)
27 : #include "src/objects/object-macros.h"
28 :
29 : namespace v8 {
30 : namespace internal {
31 :
32 : // {FrameWriter} offers a stack writer abstraction for writing
33 : // FrameDescriptions. The main service the class provides is managing
34 : // {top_offset_}, i.e. the offset of the next slot to write to.
35 : class FrameWriter {
36 : public:
37 : static const int NO_INPUT_INDEX = -1;
38 146530 : FrameWriter(Deoptimizer* deoptimizer, FrameDescription* frame,
39 : CodeTracer::Scope* trace_scope)
40 : : deoptimizer_(deoptimizer),
41 : frame_(frame),
42 : trace_scope_(trace_scope),
43 293060 : top_offset_(frame->GetFrameSize()) {}
44 :
45 115812 : void PushRawValue(intptr_t value, const char* debug_hint) {
46 : PushValue(value);
47 :
48 115812 : if (trace_scope_ != nullptr) {
49 0 : DebugPrintOutputValue(value, debug_hint);
50 : }
51 115812 : }
52 :
53 2994394 : void PushRawObject(Object obj, const char* debug_hint) {
54 1497197 : intptr_t value = obj->ptr();
55 : PushValue(value);
56 1497197 : if (trace_scope_ != nullptr) {
57 3600 : DebugPrintOutputObject(obj, top_offset_, debug_hint);
58 : }
59 1497197 : }
60 :
61 146530 : void PushCallerPc(intptr_t pc) {
62 146530 : top_offset_ -= kPCOnStackSize;
63 146530 : frame_->SetCallerPc(top_offset_, pc);
64 146530 : DebugPrintOutputValue(pc, "caller's pc\n");
65 146530 : }
66 :
67 146530 : void PushCallerFp(intptr_t fp) {
68 146530 : top_offset_ -= kFPOnStackSize;
69 146530 : frame_->SetCallerFp(top_offset_, fp);
70 146530 : DebugPrintOutputValue(fp, "caller's fp\n");
71 146530 : }
72 :
73 : void PushCallerConstantPool(intptr_t cp) {
74 : top_offset_ -= kSystemPointerSize;
75 : frame_->SetCallerConstantPool(top_offset_, cp);
76 : DebugPrintOutputValue(cp, "caller's constant_pool\n");
77 : }
78 :
79 1200989 : void PushTranslatedValue(const TranslatedFrame::iterator& iterator,
80 1197989 : const char* debug_hint = "") {
81 1197989 : Object obj = iterator->GetRawValue();
82 :
83 1197989 : PushRawObject(obj, debug_hint);
84 :
85 1197989 : if (trace_scope_) {
86 3000 : PrintF(trace_scope_->file(), " (input #%d)\n", iterator.input_index());
87 : }
88 :
89 : deoptimizer_->QueueValueForMaterialization(output_address(top_offset_), obj,
90 2395978 : iterator);
91 1197989 : }
92 :
93 : unsigned top_offset() const { return top_offset_; }
94 :
95 : private:
96 : void PushValue(intptr_t value) {
97 : CHECK_GE(top_offset_, 0);
98 1613009 : top_offset_ -= kSystemPointerSize;
99 : frame_->SetFrameSlot(top_offset_, value);
100 : }
101 :
102 : Address output_address(unsigned output_offset) {
103 : Address output_address =
104 1202189 : static_cast<Address>(frame_->GetTop()) + output_offset;
105 : return output_address;
106 : }
107 :
108 293660 : void DebugPrintOutputValue(intptr_t value, const char* debug_hint = "") {
109 293060 : if (trace_scope_ != nullptr) {
110 : PrintF(trace_scope_->file(),
111 : " " V8PRIxPTR_FMT ": [top + %3d] <- " V8PRIxPTR_FMT " ; %s",
112 1200 : output_address(top_offset_), top_offset_, value, debug_hint);
113 : }
114 293060 : }
115 :
116 3600 : void DebugPrintOutputObject(Object obj, unsigned output_offset,
117 3600 : const char* debug_hint = "") {
118 3600 : if (trace_scope_ != nullptr) {
119 : PrintF(trace_scope_->file(), " " V8PRIxPTR_FMT ": [top + %3d] <- ",
120 3600 : output_address(output_offset), output_offset);
121 3600 : if (obj->IsSmi()) {
122 390 : PrintF(V8PRIxPTR_FMT " <Smi %d>", obj->ptr(), Smi::cast(obj)->value());
123 : } else {
124 6420 : obj->ShortPrint(trace_scope_->file());
125 : }
126 7200 : PrintF(trace_scope_->file(), " ; %s", debug_hint);
127 : }
128 3600 : }
129 :
130 : Deoptimizer* deoptimizer_;
131 : FrameDescription* frame_;
132 : CodeTracer::Scope* trace_scope_;
133 : unsigned top_offset_;
134 : };
135 :
136 251532 : DeoptimizerData::DeoptimizerData(Heap* heap) : heap_(heap), current_(nullptr) {
137 : Code* start = &deopt_entry_code_[0];
138 : Code* end = &deopt_entry_code_[DeoptimizerData::kLastDeoptimizeKind + 1];
139 62883 : heap_->RegisterStrongRoots(FullObjectSlot(start), FullObjectSlot(end));
140 62883 : }
141 :
142 :
143 62867 : DeoptimizerData::~DeoptimizerData() {
144 : Code* start = &deopt_entry_code_[0];
145 62867 : heap_->UnregisterStrongRoots(FullObjectSlot(start));
146 62868 : }
147 :
148 0 : Code DeoptimizerData::deopt_entry_code(DeoptimizeKind kind) {
149 4150210 : return deopt_entry_code_[static_cast<int>(kind)];
150 : }
151 :
152 0 : void DeoptimizerData::set_deopt_entry_code(DeoptimizeKind kind, Code code) {
153 45129 : deopt_entry_code_[static_cast<int>(kind)] = code;
154 0 : }
155 :
156 141231 : Code Deoptimizer::FindDeoptimizingCode(Address addr) {
157 282462 : if (function_->IsHeapObject()) {
158 : // Search all deoptimizing code in the native context of the function.
159 141231 : Isolate* isolate = isolate_;
160 141231 : Context native_context = function_->context()->native_context();
161 141231 : Object element = native_context->DeoptimizedCodeListHead();
162 8552987 : while (!element->IsUndefined(isolate)) {
163 8389329 : Code code = Code::cast(element);
164 8389329 : CHECK(code->kind() == Code::OPTIMIZED_FUNCTION);
165 8389329 : if (code->contains(addr)) return code;
166 8270525 : element = code->next_code_link();
167 : }
168 : }
169 22427 : return Code();
170 : }
171 :
172 :
173 : // We rely on this function not causing a GC. It is called from generated code
174 : // without having a real stack frame in place.
175 141231 : Deoptimizer* Deoptimizer::New(Address raw_function, DeoptimizeKind kind,
176 : unsigned bailout_id, Address from,
177 141231 : int fp_to_sp_delta, Isolate* isolate) {
178 : JSFunction function = JSFunction::cast(Object(raw_function));
179 : Deoptimizer* deoptimizer = new Deoptimizer(isolate, function, kind,
180 282462 : bailout_id, from, fp_to_sp_delta);
181 141231 : CHECK_NULL(isolate->deoptimizer_data()->current_);
182 141231 : isolate->deoptimizer_data()->current_ = deoptimizer;
183 141231 : return deoptimizer;
184 : }
185 :
186 282462 : Deoptimizer* Deoptimizer::Grab(Isolate* isolate) {
187 141231 : Deoptimizer* result = isolate->deoptimizer_data()->current_;
188 141231 : CHECK_NOT_NULL(result);
189 141231 : result->DeleteFrameDescriptions();
190 141231 : isolate->deoptimizer_data()->current_ = nullptr;
191 141231 : return result;
192 : }
193 :
194 24844 : DeoptimizedFrameInfo* Deoptimizer::DebuggerInspectableFrame(
195 : JavaScriptFrame* frame,
196 : int jsframe_index,
197 : Isolate* isolate) {
198 74532 : CHECK(frame->is_optimized());
199 :
200 24844 : TranslatedState translated_values(frame);
201 24844 : translated_values.Prepare(frame->fp());
202 :
203 : TranslatedState::iterator frame_it = translated_values.end();
204 : int counter = jsframe_index;
205 107790 : for (auto it = translated_values.begin(); it != translated_values.end();
206 : it++) {
207 194060 : if (it->kind() == TranslatedFrame::kInterpretedFunction ||
208 111086 : it->kind() == TranslatedFrame::kJavaScriptBuiltinContinuation ||
209 : it->kind() ==
210 : TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch) {
211 54806 : if (counter == 0) {
212 : frame_it = it;
213 : break;
214 : }
215 29962 : counter--;
216 : }
217 : }
218 24844 : CHECK(frame_it != translated_values.end());
219 : // We only include kJavaScriptBuiltinContinuation frames above to get the
220 : // counting right.
221 24844 : CHECK_EQ(frame_it->kind(), TranslatedFrame::kInterpretedFunction);
222 :
223 : DeoptimizedFrameInfo* info =
224 49688 : new DeoptimizedFrameInfo(&translated_values, frame_it, isolate);
225 :
226 24844 : return info;
227 : }
228 :
229 : namespace {
230 69955 : class ActivationsFinder : public ThreadVisitor {
231 : public:
232 : explicit ActivationsFinder(std::set<Code>* codes, Code topmost_optimized_code,
233 : bool safe_to_deopt_topmost_optimized_code)
234 69955 : : codes_(codes) {
235 : #ifdef DEBUG
236 : topmost_ = topmost_optimized_code;
237 : safe_to_deopt_ = safe_to_deopt_topmost_optimized_code;
238 : #endif
239 : }
240 :
241 : // Find the frames with activations of codes marked for deoptimization, search
242 : // for the trampoline to the deoptimizer call respective to each code, and use
243 : // it to replace the current pc on the stack.
244 122536 : void VisitThread(Isolate* isolate, ThreadLocalTop* top) override {
245 897806 : for (StackFrameIterator it(isolate, top); !it.done(); it.Advance()) {
246 775270 : if (it.frame()->type() == StackFrame::OPTIMIZED) {
247 309131 : Code code = it.frame()->LookupCode();
248 609692 : if (code->kind() == Code::OPTIMIZED_FUNCTION &&
249 300561 : code->marked_for_deoptimization()) {
250 263959 : codes_->erase(code);
251 : // Obtain the trampoline to the deoptimizer call.
252 527918 : SafepointEntry safepoint = code->GetSafepointEntry(it.frame()->pc());
253 263959 : int trampoline_pc = safepoint.trampoline_pc();
254 : DCHECK_IMPLIES(code == topmost_, safe_to_deopt_);
255 : // Replace the current pc on the stack with the trampoline.
256 263959 : it.frame()->set_pc(code->raw_instruction_start() + trampoline_pc);
257 : }
258 : }
259 : }
260 122536 : }
261 :
262 : private:
263 : std::set<Code>* codes_;
264 :
265 : #ifdef DEBUG
266 : Code topmost_;
267 : bool safe_to_deopt_;
268 : #endif
269 : };
270 : } // namespace
271 :
272 : // Move marked code from the optimized code list to the deoptimized code list,
273 : // and replace pc on the stack for codes marked for deoptimization.
274 69955 : void Deoptimizer::DeoptimizeMarkedCodeForContext(Context context) {
275 : DisallowHeapAllocation no_allocation;
276 :
277 69955 : Isolate* isolate = context->GetHeap()->isolate();
278 : Code topmost_optimized_code;
279 : bool safe_to_deopt_topmost_optimized_code = false;
280 : #ifdef DEBUG
281 : // Make sure all activations of optimized code can deopt at their current PC.
282 : // The topmost optimized code has special handling because it cannot be
283 : // deoptimized due to weak object dependency.
284 : for (StackFrameIterator it(isolate, isolate->thread_local_top());
285 : !it.done(); it.Advance()) {
286 : StackFrame::Type type = it.frame()->type();
287 : if (type == StackFrame::OPTIMIZED) {
288 : Code code = it.frame()->LookupCode();
289 : JSFunction function =
290 : static_cast<OptimizedFrame*>(it.frame())->function();
291 : if (FLAG_trace_deopt) {
292 : CodeTracer::Scope scope(isolate->GetCodeTracer());
293 : PrintF(scope.file(), "[deoptimizer found activation of function: ");
294 : function->PrintName(scope.file());
295 : PrintF(scope.file(), " / %" V8PRIxPTR "]\n", function.ptr());
296 : }
297 : SafepointEntry safepoint = code->GetSafepointEntry(it.frame()->pc());
298 :
299 : // Turbofan deopt is checked when we are patching addresses on stack.
300 : bool safe_if_deopt_triggered = safepoint.has_deoptimization_index();
301 : bool is_builtin_code = code->kind() == Code::BUILTIN;
302 : DCHECK(topmost_optimized_code.is_null() || safe_if_deopt_triggered ||
303 : is_builtin_code);
304 : if (topmost_optimized_code.is_null()) {
305 : topmost_optimized_code = code;
306 : safe_to_deopt_topmost_optimized_code = safe_if_deopt_triggered;
307 : }
308 : }
309 : }
310 : #endif
311 :
312 : // We will use this set to mark those Code objects that are marked for
313 : // deoptimization and have not been found in stack frames.
314 : std::set<Code> codes;
315 :
316 : // Move marked code from the optimized code list to the deoptimized code list.
317 : // Walk over all optimized code objects in this native context.
318 69955 : Code prev;
319 69955 : Object element = context->OptimizedCodeListHead();
320 1056354 : while (!element->IsUndefined(isolate)) {
321 916444 : Code code = Code::cast(element);
322 916444 : CHECK_EQ(code->kind(), Code::OPTIMIZED_FUNCTION);
323 : Object next = code->next_code_link();
324 :
325 916444 : if (code->marked_for_deoptimization()) {
326 : codes.insert(code);
327 :
328 327589 : if (!prev.is_null()) {
329 : // Skip this code in the optimized code list.
330 4152 : prev->set_next_code_link(next);
331 : } else {
332 : // There was no previous node, the next node is the new head.
333 323437 : context->SetOptimizedCodeListHead(next);
334 : }
335 :
336 : // Move the code to the _deoptimized_ code list.
337 327589 : code->set_next_code_link(context->DeoptimizedCodeListHead());
338 327589 : context->SetDeoptimizedCodeListHead(code);
339 : } else {
340 : // Not marked; preserve this element.
341 588855 : prev = code;
342 : }
343 916444 : element = next;
344 : }
345 :
346 : ActivationsFinder visitor(&codes, topmost_optimized_code,
347 : safe_to_deopt_topmost_optimized_code);
348 : // Iterate over the stack of this thread.
349 69955 : visitor.VisitThread(isolate, isolate->thread_local_top());
350 : // In addition to iterate over the stack of this thread, we also
351 : // need to consider all the other threads as they may also use
352 : // the code currently beings deoptimized.
353 69955 : isolate->thread_manager()->IterateArchivedThreads(&visitor);
354 :
355 : // If there's no activation of a code in any stack then we can remove its
356 : // deoptimization data. We do this to ensure that code objects that are
357 : // unlinked don't transitively keep objects alive unnecessarily.
358 464252 : for (Code code : codes) {
359 324342 : isolate->heap()->InvalidateCodeDeoptimizationData(code);
360 : }
361 69955 : }
362 :
363 :
364 6539 : void Deoptimizer::DeoptimizeAll(Isolate* isolate) {
365 : RuntimeCallTimerScope runtimeTimer(isolate,
366 6539 : RuntimeCallCounterId::kDeoptimizeCode);
367 : TimerEventScope<TimerEventDeoptimizeCode> timer(isolate);
368 19617 : TRACE_EVENT0("v8", "V8.DeoptimizeCode");
369 6539 : if (FLAG_trace_deopt) {
370 0 : CodeTracer::Scope scope(isolate->GetCodeTracer());
371 0 : PrintF(scope.file(), "[deoptimize all code in all contexts]\n");
372 : }
373 6539 : isolate->AbortConcurrentOptimization(BlockingBehavior::kBlock);
374 : DisallowHeapAllocation no_allocation;
375 : // For all contexts, mark all code, then deoptimize.
376 6539 : Object context = isolate->heap()->native_contexts_list();
377 38832 : while (!context->IsUndefined(isolate)) {
378 25754 : Context native_context = Context::cast(context);
379 25754 : MarkAllCodeForContext(native_context);
380 25754 : DeoptimizeMarkedCodeForContext(native_context);
381 25754 : context = native_context->next_context_link();
382 : }
383 6539 : }
384 :
385 :
386 2805 : void Deoptimizer::DeoptimizeMarkedCode(Isolate* isolate) {
387 : RuntimeCallTimerScope runtimeTimer(isolate,
388 2805 : RuntimeCallCounterId::kDeoptimizeCode);
389 : TimerEventScope<TimerEventDeoptimizeCode> timer(isolate);
390 8415 : TRACE_EVENT0("v8", "V8.DeoptimizeCode");
391 2805 : if (FLAG_trace_deopt) {
392 2 : CodeTracer::Scope scope(isolate->GetCodeTracer());
393 2 : PrintF(scope.file(), "[deoptimize marked code in all contexts]\n");
394 : }
395 : DisallowHeapAllocation no_allocation;
396 : // For all contexts, deoptimize code already marked.
397 2805 : Object context = isolate->heap()->native_contexts_list();
398 12763 : while (!context->IsUndefined(isolate)) {
399 7153 : Context native_context = Context::cast(context);
400 7153 : DeoptimizeMarkedCodeForContext(native_context);
401 7153 : context = native_context->next_context_link();
402 : }
403 2805 : }
404 :
405 25754 : void Deoptimizer::MarkAllCodeForContext(Context context) {
406 25754 : Object element = context->OptimizedCodeListHead();
407 : Isolate* isolate = context->GetIsolate();
408 337466 : while (!element->IsUndefined(isolate)) {
409 285958 : Code code = Code::cast(element);
410 285958 : CHECK_EQ(code->kind(), Code::OPTIMIZED_FUNCTION);
411 285958 : code->set_marked_for_deoptimization(true);
412 285958 : element = code->next_code_link();
413 : }
414 25754 : }
415 :
416 586113 : void Deoptimizer::DeoptimizeFunction(JSFunction function, Code code) {
417 : Isolate* isolate = function->GetIsolate();
418 : RuntimeCallTimerScope runtimeTimer(isolate,
419 586113 : RuntimeCallCounterId::kDeoptimizeCode);
420 : TimerEventScope<TimerEventDeoptimizeCode> timer(isolate);
421 1758339 : TRACE_EVENT0("v8", "V8.DeoptimizeCode");
422 586113 : function->ResetIfBytecodeFlushed();
423 586113 : if (code.is_null()) code = function->code();
424 :
425 586113 : if (code->kind() == Code::OPTIMIZED_FUNCTION) {
426 : // Mark the code for deoptimization and unlink any functions that also
427 : // refer to that code. The code cannot be shared across native contexts,
428 : // so we only need to search one.
429 37048 : code->set_marked_for_deoptimization(true);
430 : // The code in the function's optimized code feedback vector slot might
431 : // be different from the code on the function - evict it if necessary.
432 : function->feedback_vector()->EvictOptimizedCodeMarkedForDeoptimization(
433 37048 : function->shared(), "unlinking code marked for deopt");
434 37048 : if (!code->deopt_already_counted()) {
435 30182 : function->feedback_vector()->increment_deopt_count();
436 15091 : code->set_deopt_already_counted(true);
437 : }
438 37048 : DeoptimizeMarkedCodeForContext(function->context()->native_context());
439 : }
440 586113 : }
441 :
442 141231 : void Deoptimizer::ComputeOutputFrames(Deoptimizer* deoptimizer) {
443 141231 : deoptimizer->DoComputeOutputFrames();
444 141231 : }
445 :
446 606 : const char* Deoptimizer::MessageFor(DeoptimizeKind kind) {
447 606 : switch (kind) {
448 : case DeoptimizeKind::kEager:
449 : return "eager";
450 : case DeoptimizeKind::kSoft:
451 3 : return "soft";
452 : case DeoptimizeKind::kLazy:
453 0 : return "lazy";
454 : }
455 0 : FATAL("Unsupported deopt kind");
456 : return nullptr;
457 : }
458 :
459 282462 : Deoptimizer::Deoptimizer(Isolate* isolate, JSFunction function,
460 : DeoptimizeKind kind, unsigned bailout_id, Address from,
461 : int fp_to_sp_delta)
462 : : isolate_(isolate),
463 : function_(function),
464 : bailout_id_(bailout_id),
465 : deopt_kind_(kind),
466 : from_(from),
467 : fp_to_sp_delta_(fp_to_sp_delta),
468 : deoptimizing_throw_(false),
469 : catch_handler_data_(-1),
470 : catch_handler_pc_offset_(-1),
471 : input_(nullptr),
472 : output_count_(0),
473 : jsframe_count_(0),
474 : output_(nullptr),
475 : caller_frame_top_(0),
476 : caller_fp_(0),
477 : caller_pc_(0),
478 : caller_constant_pool_(0),
479 : input_frame_context_(0),
480 : stack_fp_(0),
481 282462 : trace_scope_(nullptr) {
482 141231 : if (isolate->deoptimizer_lazy_throw()) {
483 : isolate->set_deoptimizer_lazy_throw(false);
484 5431 : deoptimizing_throw_ = true;
485 : }
486 :
487 : DCHECK_NE(from, kNullAddress);
488 141231 : compiled_code_ = FindOptimizedCode();
489 : DCHECK(!compiled_code_.is_null());
490 :
491 : DCHECK(function->IsJSFunction());
492 : trace_scope_ = FLAG_trace_deopt
493 : ? new CodeTracer::Scope(isolate->GetCodeTracer())
494 282462 : : nullptr;
495 : #ifdef DEBUG
496 : DCHECK(AllowHeapAllocation::IsAllowed());
497 : disallow_heap_allocation_ = new DisallowHeapAllocation();
498 : #endif // DEBUG
499 282462 : if (compiled_code_->kind() != Code::OPTIMIZED_FUNCTION ||
500 141231 : !compiled_code_->deopt_already_counted()) {
501 : // If the function is optimized, and we haven't counted that deopt yet, then
502 : // increment the function's deopt count so that we can avoid optimising
503 : // functions that deopt too often.
504 :
505 23033 : if (deopt_kind_ == DeoptimizeKind::kSoft) {
506 : // Soft deopts shouldn't count against the overall deoptimization count
507 : // that can eventually lead to disabling optimization for a function.
508 4511 : isolate->counters()->soft_deopts_executed()->Increment();
509 18522 : } else if (!function.is_null()) {
510 37044 : function->feedback_vector()->increment_deopt_count();
511 : }
512 : }
513 141231 : if (compiled_code_->kind() == Code::OPTIMIZED_FUNCTION) {
514 141231 : compiled_code_->set_deopt_already_counted(true);
515 282462 : PROFILE(isolate_,
516 : CodeDeoptEvent(compiled_code_, kind, from_, fp_to_sp_delta_));
517 : }
518 141231 : unsigned size = ComputeInputFrameSize();
519 : int parameter_count =
520 282462 : function->shared()->internal_formal_parameter_count() + 1;
521 141231 : input_ = new (size) FrameDescription(size, parameter_count);
522 141231 : }
523 :
524 141231 : Code Deoptimizer::FindOptimizedCode() {
525 141231 : Code compiled_code = FindDeoptimizingCode(from_);
526 : return !compiled_code.is_null() ? compiled_code
527 141231 : : isolate_->FindCodeObject(from_);
528 : }
529 :
530 :
531 600 : void Deoptimizer::PrintFunctionName() {
532 1800 : if (function_->IsHeapObject() && function_->IsJSFunction()) {
533 1200 : function_->ShortPrint(trace_scope_->file());
534 : } else {
535 : PrintF(trace_scope_->file(),
536 0 : "%s", Code::Kind2String(compiled_code_->kind()));
537 : }
538 600 : }
539 :
540 282462 : Handle<JSFunction> Deoptimizer::function() const {
541 282462 : return Handle<JSFunction>(function_, isolate());
542 : }
543 0 : Handle<Code> Deoptimizer::compiled_code() const {
544 0 : return Handle<Code>(compiled_code_, isolate());
545 : }
546 :
547 141231 : Deoptimizer::~Deoptimizer() {
548 : DCHECK(input_ == nullptr && output_ == nullptr);
549 : DCHECK_NULL(disallow_heap_allocation_);
550 141531 : delete trace_scope_;
551 141231 : }
552 :
553 :
554 141231 : void Deoptimizer::DeleteFrameDescriptions() {
555 141231 : delete input_;
556 287761 : for (int i = 0; i < output_count_; ++i) {
557 146530 : if (output_[i] != input_) delete output_[i];
558 : }
559 141231 : delete[] output_;
560 141231 : input_ = nullptr;
561 141231 : output_ = nullptr;
562 : #ifdef DEBUG
563 : DCHECK(!AllowHeapAllocation::IsAllowed());
564 : DCHECK_NOT_NULL(disallow_heap_allocation_);
565 : delete disallow_heap_allocation_;
566 : disallow_heap_allocation_ = nullptr;
567 : #endif // DEBUG
568 141231 : }
569 :
570 2736623 : Address Deoptimizer::GetDeoptimizationEntry(Isolate* isolate,
571 : DeoptimizeKind kind) {
572 : DeoptimizerData* data = isolate->deoptimizer_data();
573 2736623 : CHECK_LE(kind, DeoptimizerData::kLastDeoptimizeKind);
574 2736623 : CHECK(!data->deopt_entry_code(kind).is_null());
575 2736623 : return data->deopt_entry_code(kind)->raw_instruction_start();
576 : }
577 :
578 0 : bool Deoptimizer::IsDeoptimizationEntry(Isolate* isolate, Address addr,
579 : DeoptimizeKind type) {
580 : DeoptimizerData* data = isolate->deoptimizer_data();
581 0 : CHECK_LE(type, DeoptimizerData::kLastDeoptimizeKind);
582 : Code code = data->deopt_entry_code(type);
583 0 : if (code.is_null()) return false;
584 0 : return addr == code->raw_instruction_start();
585 : }
586 :
587 0 : bool Deoptimizer::IsDeoptimizationEntry(Isolate* isolate, Address addr,
588 : DeoptimizeKind* type) {
589 0 : if (IsDeoptimizationEntry(isolate, addr, DeoptimizeKind::kEager)) {
590 0 : *type = DeoptimizeKind::kEager;
591 0 : return true;
592 : }
593 0 : if (IsDeoptimizationEntry(isolate, addr, DeoptimizeKind::kSoft)) {
594 0 : *type = DeoptimizeKind::kSoft;
595 0 : return true;
596 : }
597 0 : if (IsDeoptimizationEntry(isolate, addr, DeoptimizeKind::kLazy)) {
598 0 : *type = DeoptimizeKind::kLazy;
599 0 : return true;
600 : }
601 : return false;
602 : }
603 :
604 85 : int Deoptimizer::GetDeoptimizedCodeCount(Isolate* isolate) {
605 : int length = 0;
606 : // Count all entries in the deoptimizing code list of every context.
607 85 : Object context = isolate->heap()->native_contexts_list();
608 255 : while (!context->IsUndefined(isolate)) {
609 85 : Context native_context = Context::cast(context);
610 85 : Object element = native_context->DeoptimizedCodeListHead();
611 341 : while (!element->IsUndefined(isolate)) {
612 171 : Code code = Code::cast(element);
613 : DCHECK(code->kind() == Code::OPTIMIZED_FUNCTION);
614 171 : if (!code->marked_for_deoptimization()) {
615 0 : length++;
616 : }
617 171 : element = code->next_code_link();
618 : }
619 85 : context = Context::cast(context)->next_context_link();
620 : }
621 85 : return length;
622 : }
623 :
624 : namespace {
625 :
626 5504 : int LookupCatchHandler(TranslatedFrame* translated_frame, int* data_out) {
627 5504 : switch (translated_frame->kind()) {
628 : case TranslatedFrame::kInterpretedFunction: {
629 : int bytecode_offset = translated_frame->node_id().ToInt();
630 : HandlerTable table(
631 5458 : translated_frame->raw_shared_info()->GetBytecodeArray());
632 5458 : return table.LookupRange(bytecode_offset, data_out, nullptr);
633 : }
634 : case TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch: {
635 : return 0;
636 : }
637 : default:
638 : break;
639 : }
640 38 : return -1;
641 : }
642 :
643 : bool ShouldPadArguments(int arg_count) {
644 : return kPadArguments && (arg_count % 2 != 0);
645 : }
646 :
647 : } // namespace
648 :
649 : // We rely on this function not causing a GC. It is called from generated code
650 : // without having a real stack frame in place.
651 282462 : void Deoptimizer::DoComputeOutputFrames() {
652 : base::ElapsedTimer timer;
653 :
654 : // Determine basic deoptimization information. The optimized frame is
655 : // described by the input data.
656 : DeoptimizationData input_data =
657 282462 : DeoptimizationData::cast(compiled_code_->deoptimization_data());
658 :
659 : {
660 : // Read caller's PC, caller's FP and caller's constant pool values
661 : // from input frame. Compute caller's frame top address.
662 :
663 141231 : Register fp_reg = JavaScriptFrame::fp_register();
664 282462 : stack_fp_ = input_->GetRegister(fp_reg.code());
665 :
666 141231 : caller_frame_top_ = stack_fp_ + ComputeInputFrameAboveFpFixedSize();
667 :
668 141231 : Address fp_address = input_->GetFramePointerAddress();
669 141231 : caller_fp_ = Memory<intptr_t>(fp_address);
670 : caller_pc_ =
671 282462 : Memory<intptr_t>(fp_address + CommonFrameConstants::kCallerPCOffset);
672 : input_frame_context_ = Memory<intptr_t>(
673 282462 : fp_address + CommonFrameConstants::kContextOrFrameTypeOffset);
674 :
675 : if (FLAG_enable_embedded_constant_pool) {
676 : caller_constant_pool_ = Memory<intptr_t>(
677 : fp_address + CommonFrameConstants::kConstantPoolOffset);
678 : }
679 : }
680 :
681 141231 : if (trace_scope_ != nullptr) {
682 : timer.Start();
683 : PrintF(trace_scope_->file(), "[deoptimizing (DEOPT %s): begin ",
684 600 : MessageFor(deopt_kind_));
685 300 : PrintFunctionName();
686 : PrintF(trace_scope_->file(),
687 : " (opt #%d) @%d, FP to SP delta: %d, caller sp: " V8PRIxPTR_FMT
688 : "]\n",
689 : input_data->OptimizationId()->value(), bailout_id_, fp_to_sp_delta_,
690 900 : caller_frame_top_);
691 300 : if (deopt_kind_ == DeoptimizeKind::kEager ||
692 : deopt_kind_ == DeoptimizeKind::kSoft) {
693 : compiled_code_->PrintDeoptLocation(
694 600 : trace_scope_->file(), " ;;; deoptimize at ", from_);
695 : }
696 : }
697 :
698 141231 : BailoutId node_id = input_data->BytecodeOffset(bailout_id_);
699 141231 : ByteArray translations = input_data->TranslationByteArray();
700 : unsigned translation_index =
701 282462 : input_data->TranslationIndex(bailout_id_)->value();
702 :
703 : TranslationIterator state_iterator(translations, translation_index);
704 : translated_state_.Init(
705 : isolate_, input_->GetFramePointerAddress(), &state_iterator,
706 : input_data->LiteralArray(), input_->GetRegisterValues(),
707 141231 : trace_scope_ == nullptr ? nullptr : trace_scope_->file(),
708 141231 : function_->IsHeapObject()
709 423693 : ? function_->shared()->internal_formal_parameter_count()
710 706155 : : 0);
711 :
712 : // Do the input frame to output frame(s) translation.
713 141231 : size_t count = translated_state_.frames().size();
714 : // If we are supposed to go to the catch handler, find the catching frame
715 : // for the catch and make sure we only deoptimize upto that frame.
716 141231 : if (deoptimizing_throw_) {
717 : size_t catch_handler_frame_index = count;
718 5504 : for (size_t i = count; i-- > 0;) {
719 : catch_handler_pc_offset_ = LookupCatchHandler(
720 11008 : &(translated_state_.frames()[i]), &catch_handler_data_);
721 5504 : if (catch_handler_pc_offset_ >= 0) {
722 : catch_handler_frame_index = i;
723 : break;
724 : }
725 : }
726 5431 : CHECK_LT(catch_handler_frame_index, count);
727 5431 : count = catch_handler_frame_index + 1;
728 : }
729 :
730 : DCHECK_NULL(output_);
731 141231 : output_ = new FrameDescription*[count];
732 287761 : for (size_t i = 0; i < count; ++i) {
733 146530 : output_[i] = nullptr;
734 : }
735 141231 : output_count_ = static_cast<int>(count);
736 :
737 : // Translate each output frame.
738 : int frame_index = 0; // output_frame_index
739 287761 : for (size_t i = 0; i < count; ++i, ++frame_index) {
740 : // Read the ast node id, function, and frame height for this output frame.
741 293060 : TranslatedFrame* translated_frame = &(translated_state_.frames()[i]);
742 146530 : bool handle_exception = deoptimizing_throw_ && i == count - 1;
743 146530 : switch (translated_frame->kind()) {
744 : case TranslatedFrame::kInterpretedFunction:
745 : DoComputeInterpretedFrame(translated_frame, frame_index,
746 144429 : handle_exception);
747 144429 : jsframe_count_++;
748 144429 : break;
749 : case TranslatedFrame::kArgumentsAdaptor:
750 868 : DoComputeArgumentsAdaptorFrame(translated_frame, frame_index);
751 868 : break;
752 : case TranslatedFrame::kConstructStub:
753 540 : DoComputeConstructStubFrame(translated_frame, frame_index);
754 540 : break;
755 : case TranslatedFrame::kBuiltinContinuation:
756 : DoComputeBuiltinContinuation(translated_frame, frame_index,
757 17 : BuiltinContinuationMode::STUB);
758 17 : break;
759 : case TranslatedFrame::kJavaScriptBuiltinContinuation:
760 : DoComputeBuiltinContinuation(translated_frame, frame_index,
761 644 : BuiltinContinuationMode::JAVASCRIPT);
762 644 : break;
763 : case TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch:
764 : DoComputeBuiltinContinuation(
765 : translated_frame, frame_index,
766 : handle_exception
767 : ? BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION
768 32 : : BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH);
769 32 : break;
770 : case TranslatedFrame::kInvalid:
771 0 : FATAL("invalid frame");
772 : break;
773 : }
774 : }
775 :
776 141231 : FrameDescription* topmost = output_[count - 1];
777 : topmost->GetRegisterValues()->SetRegister(kRootRegister.code(),
778 141231 : isolate()->isolate_root());
779 :
780 : // Print some helpful diagnostic information.
781 141231 : if (trace_scope_ != nullptr) {
782 300 : double ms = timer.Elapsed().InMillisecondsF();
783 300 : int index = output_count_ - 1; // Index of the topmost frame.
784 : PrintF(trace_scope_->file(), "[deoptimizing (%s): end ",
785 600 : MessageFor(deopt_kind_));
786 300 : PrintFunctionName();
787 : PrintF(trace_scope_->file(),
788 : " @%d => node=%d, pc=" V8PRIxPTR_FMT ", caller sp=" V8PRIxPTR_FMT
789 : ", took %0.3f ms]\n",
790 300 : bailout_id_, node_id.ToInt(), output_[index]->GetPc(),
791 900 : caller_frame_top_, ms);
792 : }
793 141231 : }
794 :
795 917224 : void Deoptimizer::DoComputeInterpretedFrame(TranslatedFrame* translated_frame,
796 : int frame_index,
797 144429 : bool goto_catch_handler) {
798 144429 : SharedFunctionInfo shared = translated_frame->raw_shared_info();
799 :
800 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
801 144429 : bool is_bottommost = (0 == frame_index);
802 144429 : bool is_topmost = (output_count_ - 1 == frame_index);
803 :
804 : int bytecode_offset = translated_frame->node_id().ToInt();
805 : int height = translated_frame->height();
806 144429 : int register_count = height - 1; // Exclude accumulator.
807 : int register_stack_slot_count =
808 144429 : InterpreterFrameConstants::RegisterStackSlotCount(register_count);
809 144429 : int height_in_bytes = register_stack_slot_count * kSystemPointerSize;
810 :
811 : // The topmost frame will contain the accumulator.
812 144429 : if (is_topmost) {
813 140937 : height_in_bytes += kSystemPointerSize;
814 140937 : if (PadTopOfStackRegister()) height_in_bytes += kSystemPointerSize;
815 : }
816 :
817 : TranslatedFrame::iterator function_iterator = value_iterator++;
818 144429 : if (trace_scope_ != nullptr) {
819 300 : PrintF(trace_scope_->file(), " translating interpreted frame ");
820 300 : std::unique_ptr<char[]> name = shared->DebugName()->ToCString();
821 600 : PrintF(trace_scope_->file(), "%s", name.get());
822 : PrintF(trace_scope_->file(), " => bytecode_offset=%d, height=%d%s\n",
823 : bytecode_offset, height_in_bytes,
824 600 : goto_catch_handler ? " (throw)" : "");
825 : }
826 144429 : if (goto_catch_handler) {
827 5423 : bytecode_offset = catch_handler_pc_offset_;
828 : }
829 :
830 : // The 'fixed' part of the frame consists of the incoming parameters and
831 : // the part described by InterpreterFrameConstants. This will include
832 : // argument padding, when needed.
833 : unsigned fixed_frame_size = ComputeInterpretedFixedSize(shared);
834 144429 : unsigned output_frame_size = height_in_bytes + fixed_frame_size;
835 :
836 : // Allocate and store the output frame description.
837 : int parameter_count = shared->internal_formal_parameter_count() + 1;
838 : FrameDescription* output_frame = new (output_frame_size)
839 144429 : FrameDescription(output_frame_size, parameter_count);
840 144429 : FrameWriter frame_writer(this, output_frame, trace_scope_);
841 :
842 144429 : CHECK(frame_index >= 0 && frame_index < output_count_);
843 144429 : CHECK_NULL(output_[frame_index]);
844 144429 : output_[frame_index] = output_frame;
845 :
846 : // The top address of the frame is computed from the previous frame's top and
847 : // this frame's size.
848 : intptr_t top_address;
849 144429 : if (is_bottommost) {
850 141231 : top_address = caller_frame_top_ - output_frame_size;
851 : } else {
852 9594 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
853 : }
854 : output_frame->SetTop(top_address);
855 :
856 : // Compute the incoming parameter translation.
857 :
858 : ReadOnlyRoots roots(isolate());
859 : if (ShouldPadArguments(parameter_count)) {
860 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
861 : }
862 :
863 674394 : for (int i = 0; i < parameter_count; ++i, ++value_iterator) {
864 192768 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
865 : }
866 :
867 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
868 : frame_writer.top_offset());
869 144429 : if (trace_scope_ != nullptr) {
870 300 : PrintF(trace_scope_->file(), " -------------------------\n");
871 : }
872 :
873 : // There are no translation commands for the caller's pc and fp, the
874 : // context, the function and the bytecode offset. Synthesize
875 : // their values and set them up
876 : // explicitly.
877 : //
878 : // The caller's pc for the bottommost output frame is the same as in the
879 : // input frame. For all subsequent output frames, it can be read from the
880 : // previous one. This frame's pc can be computed from the non-optimized
881 : // function code and AST id of the bailout.
882 : const intptr_t caller_pc =
883 144429 : is_bottommost ? caller_pc_ : output_[frame_index - 1]->GetPc();
884 144429 : frame_writer.PushCallerPc(caller_pc);
885 :
886 : // The caller's frame pointer for the bottommost output frame is the same
887 : // as in the input frame. For all subsequent output frames, it can be
888 : // read from the previous one. Also compute and set this frame's frame
889 : // pointer.
890 : const intptr_t caller_fp =
891 144429 : is_bottommost ? caller_fp_ : output_[frame_index - 1]->GetFp();
892 144429 : frame_writer.PushCallerFp(caller_fp);
893 :
894 144429 : intptr_t fp_value = top_address + frame_writer.top_offset();
895 : output_frame->SetFp(fp_value);
896 144429 : if (is_topmost) {
897 140937 : Register fp_reg = InterpretedFrame::fp_register();
898 140937 : output_frame->SetRegister(fp_reg.code(), fp_value);
899 : }
900 :
901 : if (FLAG_enable_embedded_constant_pool) {
902 : // For the bottommost output frame the constant pool pointer can be gotten
903 : // from the input frame. For subsequent output frames, it can be read from
904 : // the previous frame.
905 : const intptr_t caller_cp =
906 : is_bottommost ? caller_constant_pool_
907 : : output_[frame_index - 1]->GetConstantPool();
908 : frame_writer.PushCallerConstantPool(caller_cp);
909 : }
910 :
911 : // For the bottommost output frame the context can be gotten from the input
912 : // frame. For all subsequent output frames it can be gotten from the function
913 : // so long as we don't inline functions that need local contexts.
914 :
915 : // When deoptimizing into a catch block, we need to take the context
916 : // from a register that was specified in the handler table.
917 : TranslatedFrame::iterator context_pos = value_iterator++;
918 144429 : if (goto_catch_handler) {
919 : // Skip to the translated value of the register specified
920 : // in the handler table.
921 73859 : for (int i = 0; i < catch_handler_data_ + 1; ++i) {
922 73859 : context_pos++;
923 : }
924 : }
925 : // Read the context from the translations.
926 144429 : Object context = context_pos->GetRawValue();
927 144429 : output_frame->SetContext(static_cast<intptr_t>(context->ptr()));
928 144429 : frame_writer.PushTranslatedValue(context_pos, "context");
929 :
930 : // The function was mentioned explicitly in the BEGIN_FRAME.
931 144429 : frame_writer.PushTranslatedValue(function_iterator, "function");
932 :
933 : // Set the bytecode array pointer.
934 144429 : Object bytecode_array = shared->HasBreakInfo()
935 144899 : ? shared->GetDebugInfo()->DebugBytecodeArray()
936 288858 : : shared->GetBytecodeArray();
937 144429 : frame_writer.PushRawObject(bytecode_array, "bytecode array\n");
938 :
939 : // The bytecode offset was mentioned explicitly in the BEGIN_FRAME.
940 : int raw_bytecode_offset =
941 144429 : BytecodeArray::kHeaderSize - kHeapObjectTag + bytecode_offset;
942 144429 : Smi smi_bytecode_offset = Smi::FromInt(raw_bytecode_offset);
943 144429 : frame_writer.PushRawObject(smi_bytecode_offset, "bytecode offset\n");
944 :
945 144429 : if (trace_scope_ != nullptr) {
946 300 : PrintF(trace_scope_->file(), " -------------------------\n");
947 : }
948 :
949 : // Translate the rest of the interpreter registers in the frame.
950 : // The return_value_offset is counted from the top. Here, we compute the
951 : // register index (counted from the start).
952 : int return_value_first_reg =
953 144429 : register_count - translated_frame->return_value_offset();
954 : int return_value_count = translated_frame->return_value_count();
955 1638310 : for (int i = 0; i < register_count; ++i, ++value_iterator) {
956 : // Ensure we write the return value if we have one and we are returning
957 : // normally to a lazy deopt point.
958 1205201 : if (is_topmost && !goto_catch_handler &&
959 949484 : deopt_kind_ == DeoptimizeKind::kLazy && i >= return_value_first_reg &&
960 36 : i < return_value_first_reg + return_value_count) {
961 27 : int return_index = i - return_value_first_reg;
962 27 : if (return_index == 0) {
963 : frame_writer.PushRawValue(input_->GetRegister(kReturnRegister0.code()),
964 54 : "return value 0\n");
965 : // We do not handle the situation when one return value should go into
966 : // the accumulator and another one into an ordinary register. Since
967 : // the interpreter should never create such situation, just assert
968 : // this does not happen.
969 27 : CHECK_LE(return_value_first_reg + return_value_count, register_count);
970 : } else {
971 0 : CHECK_EQ(return_index, 1);
972 : frame_writer.PushRawValue(input_->GetRegister(kReturnRegister1.code()),
973 0 : "return value 1\n");
974 : }
975 : } else {
976 : // This is not return value, just write the value from the translations.
977 674699 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
978 : }
979 : }
980 :
981 : int register_slots_written = register_count;
982 : DCHECK_LE(register_slots_written, register_stack_slot_count);
983 : // Some architectures must pad the stack frame with extra stack slots
984 : // to ensure the stack frame is aligned. Do this now.
985 144429 : while (register_slots_written < register_stack_slot_count) {
986 0 : register_slots_written++;
987 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
988 : }
989 :
990 : // Translate the accumulator register (depending on frame position).
991 144429 : if (is_topmost) {
992 140937 : if (PadTopOfStackRegister()) {
993 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
994 : }
995 : // For topmost frame, put the accumulator on the stack. The
996 : // {NotifyDeoptimized} builtin pops it off the topmost frame (possibly
997 : // after materialization).
998 140937 : if (goto_catch_handler) {
999 : // If we are lazy deopting to a catch handler, we set the accumulator to
1000 : // the exception (which lives in the result register).
1001 : intptr_t accumulator_value =
1002 5423 : input_->GetRegister(kInterpreterAccumulatorRegister.code());
1003 10846 : frame_writer.PushRawObject(Object(accumulator_value), "accumulator\n");
1004 : } else {
1005 : // If we are lazily deoptimizing make sure we store the deopt
1006 : // return value into the appropriate slot.
1007 384343 : if (deopt_kind_ == DeoptimizeKind::kLazy &&
1008 248802 : translated_frame->return_value_offset() == 0 &&
1009 : translated_frame->return_value_count() > 0) {
1010 112905 : CHECK_EQ(translated_frame->return_value_count(), 1);
1011 : frame_writer.PushRawValue(input_->GetRegister(kReturnRegister0.code()),
1012 225810 : "return value 0\n");
1013 : } else {
1014 22609 : frame_writer.PushTranslatedValue(value_iterator, "accumulator");
1015 : }
1016 : }
1017 : ++value_iterator; // Move over the accumulator.
1018 : } else {
1019 : // For non-topmost frames, skip the accumulator translation. For those
1020 : // frames, the return value from the callee will become the accumulator.
1021 : ++value_iterator;
1022 : }
1023 144429 : CHECK_EQ(translated_frame->end(), value_iterator);
1024 144429 : CHECK_EQ(0u, frame_writer.top_offset());
1025 :
1026 : // Compute this frame's PC and state. The PC will be a special builtin that
1027 : // continues the bytecode dispatch. Note that non-topmost and lazy-style
1028 : // bailout handlers also advance the bytecode offset before dispatch, hence
1029 : // simulating what normal handlers do upon completion of the operation.
1030 144429 : Builtins* builtins = isolate_->builtins();
1031 : Code dispatch_builtin =
1032 140937 : (!is_topmost || (deopt_kind_ == DeoptimizeKind::kLazy)) &&
1033 : !goto_catch_handler
1034 : ? builtins->builtin(Builtins::kInterpreterEnterBytecodeAdvance)
1035 261236 : : builtins->builtin(Builtins::kInterpreterEnterBytecodeDispatch);
1036 : output_frame->SetPc(
1037 144429 : static_cast<intptr_t>(dispatch_builtin->InstructionStart()));
1038 :
1039 : // Update constant pool.
1040 : if (FLAG_enable_embedded_constant_pool) {
1041 : intptr_t constant_pool_value =
1042 : static_cast<intptr_t>(dispatch_builtin->constant_pool());
1043 : output_frame->SetConstantPool(constant_pool_value);
1044 : if (is_topmost) {
1045 : Register constant_pool_reg =
1046 : InterpretedFrame::constant_pool_pointer_register();
1047 : output_frame->SetRegister(constant_pool_reg.code(), constant_pool_value);
1048 : }
1049 : }
1050 :
1051 : // Clear the context register. The context might be a de-materialized object
1052 : // and will be materialized by {Runtime_NotifyDeoptimized}. For additional
1053 : // safety we use Smi(0) instead of the potential {arguments_marker} here.
1054 144429 : if (is_topmost) {
1055 : intptr_t context_value = static_cast<intptr_t>(Smi::zero().ptr());
1056 140937 : Register context_reg = JavaScriptFrame::context_register();
1057 140937 : output_frame->SetRegister(context_reg.code(), context_value);
1058 : // Set the continuation for the topmost frame.
1059 140937 : Code continuation = builtins->builtin(Builtins::kNotifyDeoptimized);
1060 : output_frame->SetContinuation(
1061 140937 : static_cast<intptr_t>(continuation->InstructionStart()));
1062 : }
1063 144429 : }
1064 :
1065 868 : void Deoptimizer::DoComputeArgumentsAdaptorFrame(
1066 1736 : TranslatedFrame* translated_frame, int frame_index) {
1067 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
1068 868 : bool is_bottommost = (0 == frame_index);
1069 :
1070 868 : unsigned height = translated_frame->height();
1071 868 : unsigned height_in_bytes = height * kSystemPointerSize;
1072 : int parameter_count = height;
1073 : if (ShouldPadArguments(parameter_count))
1074 : height_in_bytes += kSystemPointerSize;
1075 :
1076 : TranslatedFrame::iterator function_iterator = value_iterator++;
1077 868 : if (trace_scope_ != nullptr) {
1078 : PrintF(trace_scope_->file(),
1079 0 : " translating arguments adaptor => height=%d\n", height_in_bytes);
1080 : }
1081 :
1082 : unsigned fixed_frame_size = ArgumentsAdaptorFrameConstants::kFixedFrameSize;
1083 868 : unsigned output_frame_size = height_in_bytes + fixed_frame_size;
1084 :
1085 : // Allocate and store the output frame description.
1086 : FrameDescription* output_frame = new (output_frame_size)
1087 868 : FrameDescription(output_frame_size, parameter_count);
1088 868 : FrameWriter frame_writer(this, output_frame, trace_scope_);
1089 :
1090 : // Arguments adaptor can not be topmost.
1091 868 : CHECK(frame_index < output_count_ - 1);
1092 868 : CHECK_NULL(output_[frame_index]);
1093 868 : output_[frame_index] = output_frame;
1094 :
1095 : // The top address of the frame is computed from the previous frame's top and
1096 : // this frame's size.
1097 : intptr_t top_address;
1098 868 : if (is_bottommost) {
1099 0 : top_address = caller_frame_top_ - output_frame_size;
1100 : } else {
1101 2604 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
1102 : }
1103 : output_frame->SetTop(top_address);
1104 :
1105 : ReadOnlyRoots roots(isolate());
1106 : if (ShouldPadArguments(parameter_count)) {
1107 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1108 : }
1109 :
1110 : // Compute the incoming parameter translation.
1111 6912 : for (int i = 0; i < parameter_count; ++i, ++value_iterator) {
1112 2588 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
1113 : }
1114 :
1115 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
1116 : frame_writer.top_offset());
1117 :
1118 : // Read caller's PC from the previous frame.
1119 : const intptr_t caller_pc =
1120 868 : is_bottommost ? caller_pc_ : output_[frame_index - 1]->GetPc();
1121 868 : frame_writer.PushCallerPc(caller_pc);
1122 :
1123 : // Read caller's FP from the previous frame, and set this frame's FP.
1124 : const intptr_t caller_fp =
1125 868 : is_bottommost ? caller_fp_ : output_[frame_index - 1]->GetFp();
1126 868 : frame_writer.PushCallerFp(caller_fp);
1127 :
1128 868 : intptr_t fp_value = top_address + frame_writer.top_offset();
1129 : output_frame->SetFp(fp_value);
1130 :
1131 : if (FLAG_enable_embedded_constant_pool) {
1132 : // Read the caller's constant pool from the previous frame.
1133 : const intptr_t caller_cp =
1134 : is_bottommost ? caller_constant_pool_
1135 : : output_[frame_index - 1]->GetConstantPool();
1136 : frame_writer.PushCallerConstantPool(caller_cp);
1137 : }
1138 :
1139 : // A marker value is used in place of the context.
1140 : intptr_t marker = StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR);
1141 868 : frame_writer.PushRawValue(marker, "context (adaptor sentinel)\n");
1142 :
1143 : // The function was mentioned explicitly in the ARGUMENTS_ADAPTOR_FRAME.
1144 868 : frame_writer.PushTranslatedValue(function_iterator, "function\n");
1145 :
1146 : // Number of incoming arguments.
1147 1736 : frame_writer.PushRawObject(Smi::FromInt(height - 1), "argc\n");
1148 :
1149 868 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1150 :
1151 868 : CHECK_EQ(translated_frame->end(), value_iterator);
1152 : DCHECK_EQ(0, frame_writer.top_offset());
1153 :
1154 868 : Builtins* builtins = isolate_->builtins();
1155 : Code adaptor_trampoline =
1156 868 : builtins->builtin(Builtins::kArgumentsAdaptorTrampoline);
1157 : intptr_t pc_value = static_cast<intptr_t>(
1158 1736 : adaptor_trampoline->InstructionStart() +
1159 1736 : isolate_->heap()->arguments_adaptor_deopt_pc_offset()->value());
1160 : output_frame->SetPc(pc_value);
1161 : if (FLAG_enable_embedded_constant_pool) {
1162 : intptr_t constant_pool_value =
1163 : static_cast<intptr_t>(adaptor_trampoline->constant_pool());
1164 : output_frame->SetConstantPool(constant_pool_value);
1165 : }
1166 868 : }
1167 :
1168 1080 : void Deoptimizer::DoComputeConstructStubFrame(TranslatedFrame* translated_frame,
1169 540 : int frame_index) {
1170 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
1171 540 : bool is_topmost = (output_count_ - 1 == frame_index);
1172 : // The construct frame could become topmost only if we inlined a constructor
1173 : // call which does a tail call (otherwise the tail callee's frame would be
1174 : // the topmost one). So it could only be the DeoptimizeKind::kLazy case.
1175 540 : CHECK(!is_topmost || deopt_kind_ == DeoptimizeKind::kLazy);
1176 :
1177 540 : Builtins* builtins = isolate_->builtins();
1178 540 : Code construct_stub = builtins->builtin(Builtins::kJSConstructStubGeneric);
1179 : BailoutId bailout_id = translated_frame->node_id();
1180 540 : unsigned height = translated_frame->height();
1181 540 : unsigned parameter_count = height - 1; // Exclude the context.
1182 540 : unsigned height_in_bytes = parameter_count * kSystemPointerSize;
1183 :
1184 : // If the construct frame appears to be topmost we should ensure that the
1185 : // value of result register is preserved during continuation execution.
1186 : // We do this here by "pushing" the result of the constructor function to the
1187 : // top of the reconstructed stack and popping it in
1188 : // {Builtins::kNotifyDeoptimized}.
1189 540 : if (is_topmost) {
1190 40 : height_in_bytes += kSystemPointerSize;
1191 40 : if (PadTopOfStackRegister()) height_in_bytes += kSystemPointerSize;
1192 : }
1193 :
1194 : if (ShouldPadArguments(parameter_count))
1195 : height_in_bytes += kSystemPointerSize;
1196 :
1197 : TranslatedFrame::iterator function_iterator = value_iterator++;
1198 540 : if (trace_scope_ != nullptr) {
1199 : PrintF(trace_scope_->file(),
1200 : " translating construct stub => bailout_id=%d (%s), height=%d\n",
1201 : bailout_id.ToInt(),
1202 : bailout_id == BailoutId::ConstructStubCreate() ? "create" : "invoke",
1203 0 : height_in_bytes);
1204 : }
1205 :
1206 : unsigned fixed_frame_size = ConstructFrameConstants::kFixedFrameSize;
1207 540 : unsigned output_frame_size = height_in_bytes + fixed_frame_size;
1208 :
1209 : // Allocate and store the output frame description.
1210 : FrameDescription* output_frame = new (output_frame_size)
1211 540 : FrameDescription(output_frame_size, parameter_count);
1212 540 : FrameWriter frame_writer(this, output_frame, trace_scope_);
1213 :
1214 : // Construct stub can not be topmost.
1215 : DCHECK(frame_index > 0 && frame_index < output_count_);
1216 : DCHECK_NULL(output_[frame_index]);
1217 540 : output_[frame_index] = output_frame;
1218 :
1219 : // The top address of the frame is computed from the previous frame's top and
1220 : // this frame's size.
1221 : intptr_t top_address;
1222 1620 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
1223 : output_frame->SetTop(top_address);
1224 :
1225 : ReadOnlyRoots roots(isolate());
1226 : if (ShouldPadArguments(parameter_count)) {
1227 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1228 : }
1229 :
1230 : // The allocated receiver of a construct stub frame is passed as the
1231 : // receiver parameter through the translation. It might be encoding
1232 : // a captured object, so we need save it for later.
1233 : TranslatedFrame::iterator receiver_iterator = value_iterator;
1234 :
1235 : // Compute the incoming parameter translation.
1236 4366 : for (unsigned i = 0; i < parameter_count; ++i, ++value_iterator) {
1237 1643 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
1238 : }
1239 :
1240 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
1241 : frame_writer.top_offset());
1242 :
1243 : // Read caller's PC from the previous frame.
1244 540 : const intptr_t caller_pc = output_[frame_index - 1]->GetPc();
1245 540 : frame_writer.PushCallerPc(caller_pc);
1246 :
1247 : // Read caller's FP from the previous frame, and set this frame's FP.
1248 540 : const intptr_t caller_fp = output_[frame_index - 1]->GetFp();
1249 540 : frame_writer.PushCallerFp(caller_fp);
1250 :
1251 540 : intptr_t fp_value = top_address + frame_writer.top_offset();
1252 : output_frame->SetFp(fp_value);
1253 540 : if (is_topmost) {
1254 40 : Register fp_reg = JavaScriptFrame::fp_register();
1255 40 : output_frame->SetRegister(fp_reg.code(), fp_value);
1256 : }
1257 :
1258 : if (FLAG_enable_embedded_constant_pool) {
1259 : // Read the caller's constant pool from the previous frame.
1260 : const intptr_t caller_cp = output_[frame_index - 1]->GetConstantPool();
1261 : frame_writer.PushCallerConstantPool(caller_cp);
1262 : }
1263 :
1264 : // A marker value is used to mark the frame.
1265 : intptr_t marker = StackFrame::TypeToMarker(StackFrame::CONSTRUCT);
1266 540 : frame_writer.PushRawValue(marker, "context (construct stub sentinel)\n");
1267 :
1268 540 : frame_writer.PushTranslatedValue(value_iterator++, "context");
1269 :
1270 : // Number of incoming arguments.
1271 1080 : frame_writer.PushRawObject(Smi::FromInt(parameter_count - 1), "argc\n");
1272 :
1273 : // The constructor function was mentioned explicitly in the
1274 : // CONSTRUCT_STUB_FRAME.
1275 540 : frame_writer.PushTranslatedValue(function_iterator, "constructor function\n");
1276 :
1277 : // The deopt info contains the implicit receiver or the new target at the
1278 : // position of the receiver. Copy it to the top of stack, with the hole value
1279 : // as padding to maintain alignment.
1280 :
1281 540 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1282 :
1283 540 : CHECK(bailout_id == BailoutId::ConstructStubCreate() ||
1284 : bailout_id == BailoutId::ConstructStubInvoke());
1285 : const char* debug_hint = bailout_id == BailoutId::ConstructStubCreate()
1286 : ? "new target\n"
1287 540 : : "allocated receiver\n";
1288 540 : frame_writer.PushTranslatedValue(receiver_iterator, debug_hint);
1289 :
1290 540 : if (is_topmost) {
1291 40 : if (PadTopOfStackRegister()) {
1292 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1293 : }
1294 : // Ensure the result is restored back when we return to the stub.
1295 : Register result_reg = kReturnRegister0;
1296 40 : intptr_t result = input_->GetRegister(result_reg.code());
1297 40 : frame_writer.PushRawValue(result, "subcall result\n");
1298 : }
1299 :
1300 540 : CHECK_EQ(translated_frame->end(), value_iterator);
1301 540 : CHECK_EQ(0u, frame_writer.top_offset());
1302 :
1303 : // Compute this frame's PC.
1304 : DCHECK(bailout_id.IsValidForConstructStub());
1305 540 : Address start = construct_stub->InstructionStart();
1306 : int pc_offset =
1307 : bailout_id == BailoutId::ConstructStubCreate()
1308 580 : ? isolate_->heap()->construct_stub_create_deopt_pc_offset()->value()
1309 1580 : : isolate_->heap()->construct_stub_invoke_deopt_pc_offset()->value();
1310 540 : intptr_t pc_value = static_cast<intptr_t>(start + pc_offset);
1311 : output_frame->SetPc(pc_value);
1312 :
1313 : // Update constant pool.
1314 : if (FLAG_enable_embedded_constant_pool) {
1315 : intptr_t constant_pool_value =
1316 : static_cast<intptr_t>(construct_stub->constant_pool());
1317 : output_frame->SetConstantPool(constant_pool_value);
1318 : if (is_topmost) {
1319 : Register constant_pool_reg =
1320 : JavaScriptFrame::constant_pool_pointer_register();
1321 : output_frame->SetRegister(constant_pool_reg.code(), constant_pool_value);
1322 : }
1323 : }
1324 :
1325 : // Clear the context register. The context might be a de-materialized object
1326 : // and will be materialized by {Runtime_NotifyDeoptimized}. For additional
1327 : // safety we use Smi(0) instead of the potential {arguments_marker} here.
1328 540 : if (is_topmost) {
1329 : intptr_t context_value = static_cast<intptr_t>(Smi::zero().ptr());
1330 40 : Register context_reg = JavaScriptFrame::context_register();
1331 40 : output_frame->SetRegister(context_reg.code(), context_value);
1332 : }
1333 :
1334 : // Set the continuation for the topmost frame.
1335 540 : if (is_topmost) {
1336 40 : Builtins* builtins = isolate_->builtins();
1337 : DCHECK_EQ(DeoptimizeKind::kLazy, deopt_kind_);
1338 40 : Code continuation = builtins->builtin(Builtins::kNotifyDeoptimized);
1339 : output_frame->SetContinuation(
1340 40 : static_cast<intptr_t>(continuation->InstructionStart()));
1341 : }
1342 540 : }
1343 :
1344 1386 : bool Deoptimizer::BuiltinContinuationModeIsJavaScript(
1345 : BuiltinContinuationMode mode) {
1346 1386 : switch (mode) {
1347 : case BuiltinContinuationMode::STUB:
1348 : return false;
1349 : case BuiltinContinuationMode::JAVASCRIPT:
1350 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1351 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1352 1352 : return true;
1353 : }
1354 0 : UNREACHABLE();
1355 : }
1356 :
1357 693 : bool Deoptimizer::BuiltinContinuationModeIsWithCatch(
1358 : BuiltinContinuationMode mode) {
1359 693 : switch (mode) {
1360 : case BuiltinContinuationMode::STUB:
1361 : case BuiltinContinuationMode::JAVASCRIPT:
1362 : return false;
1363 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1364 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1365 32 : return true;
1366 : }
1367 0 : UNREACHABLE();
1368 : }
1369 :
1370 693 : StackFrame::Type Deoptimizer::BuiltinContinuationModeToFrameType(
1371 : BuiltinContinuationMode mode) {
1372 693 : switch (mode) {
1373 : case BuiltinContinuationMode::STUB:
1374 : return StackFrame::BUILTIN_CONTINUATION;
1375 : case BuiltinContinuationMode::JAVASCRIPT:
1376 644 : return StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION;
1377 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1378 24 : return StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH;
1379 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1380 8 : return StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH;
1381 : }
1382 0 : UNREACHABLE();
1383 : }
1384 :
1385 693 : Builtins::Name Deoptimizer::TrampolineForBuiltinContinuation(
1386 : BuiltinContinuationMode mode, bool must_handle_result) {
1387 693 : switch (mode) {
1388 : case BuiltinContinuationMode::STUB:
1389 : return must_handle_result ? Builtins::kContinueToCodeStubBuiltinWithResult
1390 17 : : Builtins::kContinueToCodeStubBuiltin;
1391 : case BuiltinContinuationMode::JAVASCRIPT:
1392 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1393 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1394 : return must_handle_result
1395 : ? Builtins::kContinueToJavaScriptBuiltinWithResult
1396 676 : : Builtins::kContinueToJavaScriptBuiltin;
1397 : }
1398 0 : UNREACHABLE();
1399 : }
1400 :
1401 : // BuiltinContinuationFrames capture the machine state that is expected as input
1402 : // to a builtin, including both input register values and stack parameters. When
1403 : // the frame is reactivated (i.e. the frame below it returns), a
1404 : // ContinueToBuiltin stub restores the register state from the frame and tail
1405 : // calls to the actual target builtin, making it appear that the stub had been
1406 : // directly called by the frame above it. The input values to populate the frame
1407 : // are taken from the deopt's FrameState.
1408 : //
1409 : // Frame translation happens in two modes, EAGER and LAZY. In EAGER mode, all of
1410 : // the parameters to the Builtin are explicitly specified in the TurboFan
1411 : // FrameState node. In LAZY mode, there is always one fewer parameters specified
1412 : // in the FrameState than expected by the Builtin. In that case, construction of
1413 : // BuiltinContinuationFrame adds the final missing parameter during
1414 : // deoptimization, and that parameter is always on the stack and contains the
1415 : // value returned from the callee of the call site triggering the LAZY deopt
1416 : // (e.g. rax on x64). This requires that continuation Builtins for LAZY deopts
1417 : // must have at least one stack parameter.
1418 : //
1419 : // TO
1420 : // | .... |
1421 : // +-------------------------+
1422 : // | arg padding (arch dept) |<- at most 1*kSystemPointerSize
1423 : // +-------------------------+
1424 : // | builtin param 0 |<- FrameState input value n becomes
1425 : // +-------------------------+
1426 : // | ... |
1427 : // +-------------------------+
1428 : // | builtin param m |<- FrameState input value n+m-1, or in
1429 : // +-----needs-alignment-----+ the LAZY case, return LAZY result value
1430 : // | ContinueToBuiltin entry |
1431 : // +-------------------------+
1432 : // | | saved frame (FP) |
1433 : // | +=====needs=alignment=====+<- fpreg
1434 : // | |constant pool (if ool_cp)|
1435 : // v +-------------------------+
1436 : // |BUILTIN_CONTINUATION mark|
1437 : // +-------------------------+
1438 : // | JSFunction (or zero) |<- only if JavaScript builtin
1439 : // +-------------------------+
1440 : // | frame height above FP |
1441 : // +-------------------------+
1442 : // | context |<- this non-standard context slot contains
1443 : // +-------------------------+ the context, even for non-JS builtins.
1444 : // | builtin address |
1445 : // +-------------------------+
1446 : // | builtin input GPR reg0 |<- populated from deopt FrameState using
1447 : // +-------------------------+ the builtin's CallInterfaceDescriptor
1448 : // | ... | to map a FrameState's 0..n-1 inputs to
1449 : // +-------------------------+ the builtin's n input register params.
1450 : // | builtin input GPR regn |
1451 : // +-------------------------+
1452 : // | reg padding (arch dept) |
1453 : // +-----needs--alignment----+
1454 : // | res padding (arch dept) |<- only if {is_topmost}; result is pop'd by
1455 : // +-------------------------+<- kNotifyDeopt ASM stub and moved to acc
1456 : // | result value |<- reg, as ContinueToBuiltin stub expects.
1457 : // +-----needs-alignment-----+<- spreg
1458 : //
1459 693 : void Deoptimizer::DoComputeBuiltinContinuation(
1460 693 : TranslatedFrame* translated_frame, int frame_index,
1461 3465 : BuiltinContinuationMode mode) {
1462 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
1463 :
1464 : // The output frame must have room for all of the parameters that need to be
1465 : // passed to the builtin continuation.
1466 : const int height_in_words = translated_frame->height();
1467 :
1468 693 : BailoutId bailout_id = translated_frame->node_id();
1469 693 : Builtins::Name builtin_name = Builtins::GetBuiltinFromBailoutId(bailout_id);
1470 693 : Code builtin = isolate()->builtins()->builtin(builtin_name);
1471 : Callable continuation_callable =
1472 693 : Builtins::CallableFor(isolate(), builtin_name);
1473 : CallInterfaceDescriptor continuation_descriptor =
1474 : continuation_callable.descriptor();
1475 :
1476 693 : const bool is_bottommost = (0 == frame_index);
1477 693 : const bool is_topmost = (output_count_ - 1 == frame_index);
1478 : const bool must_handle_result =
1479 693 : !is_topmost || deopt_kind_ == DeoptimizeKind::kLazy;
1480 :
1481 9702 : const RegisterConfiguration* config(RegisterConfiguration::Default());
1482 : const int allocatable_register_count =
1483 : config->num_allocatable_general_registers();
1484 : const int padding_slot_count =
1485 : BuiltinContinuationFrameConstants::PaddingSlotCount(
1486 693 : allocatable_register_count);
1487 :
1488 : const int register_parameter_count =
1489 : continuation_descriptor.GetRegisterParameterCount();
1490 : // Make sure to account for the context by removing it from the register
1491 : // parameter count.
1492 : const int translated_stack_parameters =
1493 693 : height_in_words - register_parameter_count - 1;
1494 : const int stack_param_count =
1495 1386 : translated_stack_parameters + (must_handle_result ? 1 : 0) +
1496 1386 : (BuiltinContinuationModeIsWithCatch(mode) ? 1 : 0);
1497 : const int stack_param_pad_count =
1498 : ShouldPadArguments(stack_param_count) ? 1 : 0;
1499 :
1500 : // If the builtins frame appears to be topmost we should ensure that the
1501 : // value of result register is preserved during continuation execution.
1502 : // We do this here by "pushing" the result of callback function to the
1503 : // top of the reconstructed stack and popping it in
1504 : // {Builtins::kNotifyDeoptimized}.
1505 : const int push_result_count =
1506 693 : is_topmost ? (PadTopOfStackRegister() ? 2 : 1) : 0;
1507 :
1508 : const unsigned output_frame_size =
1509 693 : kSystemPointerSize * (stack_param_count + stack_param_pad_count +
1510 693 : allocatable_register_count + padding_slot_count +
1511 693 : push_result_count) +
1512 693 : BuiltinContinuationFrameConstants::kFixedFrameSize;
1513 :
1514 : const unsigned output_frame_size_above_fp =
1515 693 : kSystemPointerSize * (allocatable_register_count + padding_slot_count +
1516 693 : push_result_count) +
1517 : (BuiltinContinuationFrameConstants::kFixedFrameSize -
1518 : BuiltinContinuationFrameConstants::kFixedFrameSizeAboveFp);
1519 :
1520 : // Validate types of parameters. They must all be tagged except for argc for
1521 : // JS builtins.
1522 : bool has_argc = false;
1523 2721 : for (int i = 0; i < register_parameter_count; ++i) {
1524 : MachineType type = continuation_descriptor.GetParameterType(i);
1525 : int code = continuation_descriptor.GetRegisterParameter(i).code();
1526 : // Only tagged and int32 arguments are supported, and int32 only for the
1527 : // arguments count on JavaScript builtins.
1528 2028 : if (type == MachineType::Int32()) {
1529 676 : CHECK_EQ(code, kJavaScriptCallArgCountRegister.code());
1530 : has_argc = true;
1531 : } else {
1532 : // Any other argument must be a tagged value.
1533 1352 : CHECK(IsAnyTagged(type.representation()));
1534 : }
1535 : }
1536 693 : CHECK_EQ(BuiltinContinuationModeIsJavaScript(mode), has_argc);
1537 :
1538 693 : if (trace_scope_ != nullptr) {
1539 : PrintF(trace_scope_->file(),
1540 : " translating BuiltinContinuation to %s,"
1541 : " register param count %d,"
1542 : " stack param count %d\n",
1543 : Builtins::name(builtin_name), register_parameter_count,
1544 0 : stack_param_count);
1545 : }
1546 :
1547 : FrameDescription* output_frame = new (output_frame_size)
1548 693 : FrameDescription(output_frame_size, stack_param_count);
1549 693 : output_[frame_index] = output_frame;
1550 693 : FrameWriter frame_writer(this, output_frame, trace_scope_);
1551 :
1552 : // The top address of the frame is computed from the previous frame's top and
1553 : // this frame's size.
1554 : intptr_t top_address;
1555 693 : if (is_bottommost) {
1556 0 : top_address = caller_frame_top_ - output_frame_size;
1557 : } else {
1558 2079 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
1559 : }
1560 : output_frame->SetTop(top_address);
1561 :
1562 : // Get the possible JSFunction for the case that this is a
1563 : // JavaScriptBuiltinContinuationFrame, which needs the JSFunction pointer
1564 : // like a normal JavaScriptFrame.
1565 1386 : const intptr_t maybe_function = value_iterator->GetRawValue()->ptr();
1566 : ++value_iterator;
1567 :
1568 : ReadOnlyRoots roots(isolate());
1569 : if (ShouldPadArguments(stack_param_count)) {
1570 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1571 : }
1572 :
1573 8040 : for (int i = 0; i < translated_stack_parameters; ++i, ++value_iterator) {
1574 3327 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
1575 : }
1576 :
1577 693 : switch (mode) {
1578 : case BuiltinContinuationMode::STUB:
1579 : break;
1580 : case BuiltinContinuationMode::JAVASCRIPT:
1581 : break;
1582 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH: {
1583 : frame_writer.PushRawObject(roots.the_hole_value(),
1584 24 : "placeholder for exception on lazy deopt\n");
1585 24 : } break;
1586 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION: {
1587 : intptr_t accumulator_value =
1588 8 : input_->GetRegister(kInterpreterAccumulatorRegister.code());
1589 : frame_writer.PushRawObject(Object(accumulator_value),
1590 16 : "exception (from accumulator)\n");
1591 8 : } break;
1592 : }
1593 :
1594 693 : if (must_handle_result) {
1595 : frame_writer.PushRawObject(roots.the_hole_value(),
1596 485 : "placeholder for return result on lazy deopt\n");
1597 : }
1598 :
1599 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
1600 : frame_writer.top_offset());
1601 :
1602 : std::vector<TranslatedFrame::iterator> register_values;
1603 : int total_registers = config->num_general_registers();
1604 693 : register_values.resize(total_registers, {value_iterator});
1605 :
1606 5442 : for (int i = 0; i < register_parameter_count; ++i, ++value_iterator) {
1607 : int code = continuation_descriptor.GetRegisterParameter(i).code();
1608 4056 : register_values[code] = value_iterator;
1609 : }
1610 :
1611 : // The context register is always implicit in the CallInterfaceDescriptor but
1612 : // its register must be explicitly set when continuing to the builtin. Make
1613 : // sure that it's harvested from the translation and copied into the register
1614 : // set (it was automatically added at the end of the FrameState by the
1615 : // instruction selector).
1616 693 : Object context = value_iterator->GetRawValue();
1617 693 : const intptr_t value = context->ptr();
1618 : TranslatedFrame::iterator context_register_value = value_iterator++;
1619 693 : register_values[kContextRegister.code()] = context_register_value;
1620 : output_frame->SetContext(value);
1621 : output_frame->SetRegister(kContextRegister.code(), value);
1622 :
1623 : // Set caller's PC (JSFunction continuation).
1624 : const intptr_t caller_pc =
1625 693 : is_bottommost ? caller_pc_ : output_[frame_index - 1]->GetPc();
1626 693 : frame_writer.PushCallerPc(caller_pc);
1627 :
1628 : // Read caller's FP from the previous frame, and set this frame's FP.
1629 : const intptr_t caller_fp =
1630 693 : is_bottommost ? caller_fp_ : output_[frame_index - 1]->GetFp();
1631 693 : frame_writer.PushCallerFp(caller_fp);
1632 :
1633 693 : const intptr_t fp_value = top_address + frame_writer.top_offset();
1634 : output_frame->SetFp(fp_value);
1635 :
1636 : DCHECK_EQ(output_frame_size_above_fp, frame_writer.top_offset());
1637 :
1638 : if (FLAG_enable_embedded_constant_pool) {
1639 : // Read the caller's constant pool from the previous frame.
1640 : const intptr_t caller_cp =
1641 : is_bottommost ? caller_constant_pool_
1642 : : output_[frame_index - 1]->GetConstantPool();
1643 : frame_writer.PushCallerConstantPool(caller_cp);
1644 : }
1645 :
1646 : // A marker value is used in place of the context.
1647 : const intptr_t marker =
1648 1386 : StackFrame::TypeToMarker(BuiltinContinuationModeToFrameType(mode));
1649 : frame_writer.PushRawValue(marker,
1650 693 : "context (builtin continuation sentinel)\n");
1651 :
1652 693 : if (BuiltinContinuationModeIsJavaScript(mode)) {
1653 676 : frame_writer.PushRawValue(maybe_function, "JSFunction\n");
1654 : } else {
1655 17 : frame_writer.PushRawValue(0, "unused\n");
1656 : }
1657 :
1658 : // The delta from the SP to the FP; used to reconstruct SP in
1659 : // Isolate::UnwindAndFindHandler.
1660 : frame_writer.PushRawObject(Smi::FromInt(output_frame_size_above_fp),
1661 693 : "frame height at deoptimization\n");
1662 :
1663 : // The context even if this is a stub contininuation frame. We can't use the
1664 : // usual context slot, because we must store the frame marker there.
1665 : frame_writer.PushTranslatedValue(context_register_value,
1666 693 : "builtin JavaScript context\n");
1667 :
1668 : // The builtin to continue to.
1669 693 : frame_writer.PushRawObject(builtin, "builtin address\n");
1670 :
1671 9009 : for (int i = 0; i < allocatable_register_count; ++i) {
1672 : int code = config->GetAllocatableGeneralCode(i);
1673 : ScopedVector<char> str(128);
1674 8316 : if (trace_scope_ != nullptr) {
1675 0 : if (BuiltinContinuationModeIsJavaScript(mode) &&
1676 : code == kJavaScriptCallArgCountRegister.code()) {
1677 : SNPrintF(
1678 : str,
1679 : "tagged argument count %s (will be untagged by continuation)\n",
1680 0 : RegisterName(Register::from_code(code)));
1681 : } else {
1682 : SNPrintF(str, "builtin register argument %s\n",
1683 0 : RegisterName(Register::from_code(code)));
1684 : }
1685 : }
1686 : frame_writer.PushTranslatedValue(
1687 16632 : register_values[code], trace_scope_ != nullptr ? str.start() : "");
1688 : }
1689 :
1690 : // Some architectures must pad the stack frame with extra stack slots
1691 : // to ensure the stack frame is aligned.
1692 0 : for (int i = 0; i < padding_slot_count; ++i) {
1693 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1694 : }
1695 :
1696 693 : if (is_topmost) {
1697 254 : if (PadTopOfStackRegister()) {
1698 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1699 : }
1700 : // Ensure the result is restored back when we return to the stub.
1701 :
1702 254 : if (must_handle_result) {
1703 : Register result_reg = kReturnRegister0;
1704 : frame_writer.PushRawValue(input_->GetRegister(result_reg.code()),
1705 92 : "callback result\n");
1706 : } else {
1707 208 : frame_writer.PushRawObject(roots.undefined_value(), "callback result\n");
1708 : }
1709 : }
1710 :
1711 693 : CHECK_EQ(translated_frame->end(), value_iterator);
1712 693 : CHECK_EQ(0u, frame_writer.top_offset());
1713 :
1714 : // Clear the context register. The context might be a de-materialized object
1715 : // and will be materialized by {Runtime_NotifyDeoptimized}. For additional
1716 : // safety we use Smi(0) instead of the potential {arguments_marker} here.
1717 693 : if (is_topmost) {
1718 : intptr_t context_value = static_cast<intptr_t>(Smi::zero().ptr());
1719 254 : Register context_reg = JavaScriptFrame::context_register();
1720 254 : output_frame->SetRegister(context_reg.code(), context_value);
1721 : }
1722 :
1723 : // Ensure the frame pointer register points to the callee's frame. The builtin
1724 : // will build its own frame once we continue to it.
1725 693 : Register fp_reg = JavaScriptFrame::fp_register();
1726 693 : output_frame->SetRegister(fp_reg.code(), fp_value);
1727 :
1728 : Code continue_to_builtin = isolate()->builtins()->builtin(
1729 1386 : TrampolineForBuiltinContinuation(mode, must_handle_result));
1730 : output_frame->SetPc(
1731 693 : static_cast<intptr_t>(continue_to_builtin->InstructionStart()));
1732 :
1733 : Code continuation =
1734 693 : isolate()->builtins()->builtin(Builtins::kNotifyDeoptimized);
1735 : output_frame->SetContinuation(
1736 693 : static_cast<intptr_t>(continuation->InstructionStart()));
1737 693 : }
1738 :
1739 141231 : void Deoptimizer::MaterializeHeapObjects() {
1740 141231 : translated_state_.Prepare(static_cast<Address>(stack_fp_));
1741 141231 : if (FLAG_deopt_every_n_times > 0) {
1742 : // Doing a GC here will find problems with the deoptimized frames.
1743 : isolate_->heap()->CollectAllGarbage(Heap::kNoGCFlags,
1744 141254 : GarbageCollectionReason::kTesting);
1745 : }
1746 :
1747 300050 : for (auto& materialization : values_to_materialize_) {
1748 17588 : Handle<Object> value = materialization.value_->GetValue();
1749 :
1750 17588 : if (trace_scope_ != nullptr) {
1751 : PrintF("Materialization [" V8PRIxPTR_FMT "] <- " V8PRIxPTR_FMT " ; ",
1752 : static_cast<intptr_t>(materialization.output_slot_address_),
1753 0 : value->ptr());
1754 0 : value->ShortPrint(trace_scope_->file());
1755 0 : PrintF(trace_scope_->file(), "\n");
1756 : }
1757 :
1758 : *(reinterpret_cast<Address*>(materialization.output_slot_address_)) =
1759 35176 : value->ptr();
1760 : }
1761 :
1762 : translated_state_.VerifyMaterializedObjects();
1763 :
1764 141231 : bool feedback_updated = translated_state_.DoUpdateFeedback();
1765 141231 : if (trace_scope_ != nullptr && feedback_updated) {
1766 0 : PrintF(trace_scope_->file(), "Feedback updated");
1767 : compiled_code_->PrintDeoptLocation(trace_scope_->file(),
1768 0 : " from deoptimization at ", from_);
1769 : }
1770 :
1771 : isolate_->materialized_object_store()->Remove(
1772 282462 : static_cast<Address>(stack_fp_));
1773 141231 : }
1774 :
1775 1197989 : void Deoptimizer::QueueValueForMaterialization(
1776 : Address output_address, Object obj,
1777 : const TranslatedFrame::iterator& iterator) {
1778 2395978 : if (obj == ReadOnlyRoots(isolate_).arguments_marker()) {
1779 52764 : values_to_materialize_.push_back({output_address, iterator});
1780 : }
1781 1197989 : }
1782 :
1783 282462 : unsigned Deoptimizer::ComputeInputFrameAboveFpFixedSize() const {
1784 : unsigned fixed_size = CommonFrameConstants::kFixedFrameSizeAboveFp;
1785 : // TODO(jkummerow): If {function_->IsSmi()} can indeed be true, then
1786 : // {function_} should not have type {JSFunction}.
1787 564924 : if (!function_->IsSmi()) {
1788 564924 : fixed_size += ComputeIncomingArgumentSize(function_->shared());
1789 : }
1790 282462 : return fixed_size;
1791 : }
1792 :
1793 141231 : unsigned Deoptimizer::ComputeInputFrameSize() const {
1794 : // The fp-to-sp delta already takes the context, constant pool pointer and the
1795 : // function into account so we have to avoid double counting them.
1796 141231 : unsigned fixed_size_above_fp = ComputeInputFrameAboveFpFixedSize();
1797 141231 : unsigned result = fixed_size_above_fp + fp_to_sp_delta_;
1798 141231 : if (compiled_code_->kind() == Code::OPTIMIZED_FUNCTION) {
1799 : unsigned stack_slots = compiled_code_->stack_slots();
1800 : unsigned outgoing_size = 0;
1801 : // ComputeOutgoingArgumentSize(compiled_code_, bailout_id_);
1802 141231 : CHECK_EQ(fixed_size_above_fp + (stack_slots * kSystemPointerSize) -
1803 : CommonFrameConstants::kFixedFrameSizeAboveFp + outgoing_size,
1804 : result);
1805 : }
1806 141231 : return result;
1807 : }
1808 :
1809 : // static
1810 0 : unsigned Deoptimizer::ComputeInterpretedFixedSize(SharedFunctionInfo shared) {
1811 : // The fixed part of the frame consists of the return address, frame
1812 : // pointer, function, context, bytecode offset and all the incoming arguments.
1813 : return ComputeIncomingArgumentSize(shared) +
1814 144429 : InterpreterFrameConstants::kFixedFrameSize;
1815 : }
1816 :
1817 : // static
1818 0 : unsigned Deoptimizer::ComputeIncomingArgumentSize(SharedFunctionInfo shared) {
1819 426891 : int parameter_slots = shared->internal_formal_parameter_count() + 1;
1820 : if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
1821 426891 : return parameter_slots * kSystemPointerSize;
1822 : }
1823 :
1824 1368458 : void Deoptimizer::EnsureCodeForDeoptimizationEntry(Isolate* isolate,
1825 : DeoptimizeKind kind) {
1826 1368458 : CHECK(kind == DeoptimizeKind::kEager || kind == DeoptimizeKind::kSoft ||
1827 : kind == DeoptimizeKind::kLazy);
1828 1368458 : DeoptimizerData* data = isolate->deoptimizer_data();
1829 2691787 : if (!data->deopt_entry_code(kind).is_null()) return;
1830 :
1831 : MacroAssembler masm(isolate, CodeObjectRequired::kYes,
1832 135387 : NewAssemblerBuffer(16 * KB));
1833 : masm.set_emit_debug_code(false);
1834 45129 : GenerateDeoptimizationEntries(&masm, masm.isolate(), kind);
1835 45129 : CodeDesc desc;
1836 45129 : masm.GetCode(isolate, &desc);
1837 : DCHECK(!RelocInfo::RequiresRelocationAfterCodegen(desc));
1838 :
1839 : // Allocate the code as immovable since the entry addresses will be used
1840 : // directly and there is no support for relocating them.
1841 : Handle<Code> code = isolate->factory()->NewCode(
1842 : desc, Code::STUB, Handle<Object>(), Builtins::kNoBuiltinId,
1843 90252 : MaybeHandle<ByteArray>(), MaybeHandle<DeoptimizationData>(), kImmovable);
1844 45129 : CHECK(isolate->heap()->IsImmovable(*code));
1845 :
1846 45129 : CHECK(data->deopt_entry_code(kind).is_null());
1847 : data->set_deopt_entry_code(kind, *code);
1848 : }
1849 :
1850 456143 : void Deoptimizer::EnsureCodeForDeoptimizationEntries(Isolate* isolate) {
1851 456143 : EnsureCodeForDeoptimizationEntry(isolate, DeoptimizeKind::kEager);
1852 456154 : EnsureCodeForDeoptimizationEntry(isolate, DeoptimizeKind::kLazy);
1853 456154 : EnsureCodeForDeoptimizationEntry(isolate, DeoptimizeKind::kSoft);
1854 456154 : }
1855 :
1856 287761 : FrameDescription::FrameDescription(uint32_t frame_size, int parameter_count)
1857 : : frame_size_(frame_size),
1858 : parameter_count_(parameter_count),
1859 : top_(kZapUint32),
1860 : pc_(kZapUint32),
1861 : fp_(kZapUint32),
1862 : context_(kZapUint32),
1863 575522 : constant_pool_(kZapUint32) {
1864 : // Zap all the registers.
1865 4891937 : for (int r = 0; r < Register::kNumRegisters; r++) {
1866 : // TODO(jbramley): It isn't safe to use kZapUint32 here. If the register
1867 : // isn't used before the next safepoint, the GC will try to scan it as a
1868 : // tagged value. kZapUint32 looks like a valid tagged pointer, but it isn't.
1869 : #if defined(V8_OS_WIN) && defined(V8_TARGET_ARCH_ARM64)
1870 : // x18 is reserved as platform register on Windows arm64 platform
1871 : const int kPlatformRegister = 18;
1872 : if (r != kPlatformRegister) {
1873 : SetRegister(r, kZapUint32);
1874 : }
1875 : #else
1876 4604176 : SetRegister(r, kZapUint32);
1877 : #endif
1878 : }
1879 :
1880 : // Zap all the slots.
1881 3043870 : for (unsigned o = 0; o < frame_size; o += kSystemPointerSize) {
1882 : SetFrameSlot(o, kZapUint32);
1883 : }
1884 287761 : }
1885 :
1886 116670401 : void TranslationBuffer::Add(int32_t value) {
1887 : // This wouldn't handle kMinInt correctly if it ever encountered it.
1888 : DCHECK_NE(value, kMinInt);
1889 : // Encode the sign bit in the least significant bit.
1890 116670401 : bool is_negative = (value < 0);
1891 116670401 : uint32_t bits = (static_cast<uint32_t>(is_negative ? -value : value) << 1) |
1892 116670401 : static_cast<uint32_t>(is_negative);
1893 : // Encode the individual bytes using the least significant bit of
1894 : // each byte to indicate whether or not more bytes follow.
1895 120743880 : do {
1896 120741749 : uint32_t next = bits >> 7;
1897 120741749 : contents_.push_back(((bits << 1) & 0xFF) | (next != 0));
1898 : bits = next;
1899 : } while (bits != 0);
1900 116672532 : }
1901 :
1902 9444 : TranslationIterator::TranslationIterator(ByteArray buffer, int index)
1903 1479512 : : buffer_(buffer), index_(index) {
1904 : DCHECK(index >= 0 && index < buffer->length());
1905 9444 : }
1906 :
1907 60617984 : int32_t TranslationIterator::Next() {
1908 : // Run through the bytes until we reach one with a least significant
1909 : // bit of zero (marks the end).
1910 : uint32_t bits = 0;
1911 534025 : for (int i = 0; true; i += 7) {
1912 : DCHECK(HasNext());
1913 61152009 : uint8_t next = buffer_->get(index_++);
1914 61152009 : bits |= (next >> 1) << i;
1915 61152009 : if ((next & 1) == 0) break;
1916 534025 : }
1917 : // The bits encode the sign in the least significant bit.
1918 60617984 : bool is_negative = (bits & 1) == 1;
1919 60617984 : int32_t result = bits >> 1;
1920 60617984 : return is_negative ? -result : result;
1921 : }
1922 :
1923 1470068 : bool TranslationIterator::HasNext() const { return index_ < buffer_->length(); }
1924 :
1925 455969 : Handle<ByteArray> TranslationBuffer::CreateByteArray(Factory* factory) {
1926 455969 : Handle<ByteArray> result = factory->NewByteArray(CurrentIndex(), TENURED);
1927 455986 : contents_.CopyTo(result->GetDataStartAddress());
1928 455985 : return result;
1929 : }
1930 :
1931 2765 : void Translation::BeginBuiltinContinuationFrame(BailoutId bailout_id,
1932 : int literal_id,
1933 : unsigned height) {
1934 2765 : buffer_->Add(BUILTIN_CONTINUATION_FRAME);
1935 2765 : buffer_->Add(bailout_id.ToInt());
1936 2765 : buffer_->Add(literal_id);
1937 2765 : buffer_->Add(height);
1938 2765 : }
1939 :
1940 11703 : void Translation::BeginJavaScriptBuiltinContinuationFrame(BailoutId bailout_id,
1941 : int literal_id,
1942 : unsigned height) {
1943 11703 : buffer_->Add(JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME);
1944 11703 : buffer_->Add(bailout_id.ToInt());
1945 11703 : buffer_->Add(literal_id);
1946 11703 : buffer_->Add(height);
1947 11703 : }
1948 :
1949 277 : void Translation::BeginJavaScriptBuiltinContinuationWithCatchFrame(
1950 : BailoutId bailout_id, int literal_id, unsigned height) {
1951 277 : buffer_->Add(JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME);
1952 277 : buffer_->Add(bailout_id.ToInt());
1953 277 : buffer_->Add(literal_id);
1954 277 : buffer_->Add(height);
1955 277 : }
1956 :
1957 26689 : void Translation::BeginConstructStubFrame(BailoutId bailout_id, int literal_id,
1958 : unsigned height) {
1959 26689 : buffer_->Add(CONSTRUCT_STUB_FRAME);
1960 26689 : buffer_->Add(bailout_id.ToInt());
1961 26689 : buffer_->Add(literal_id);
1962 26689 : buffer_->Add(height);
1963 26689 : }
1964 :
1965 :
1966 76831 : void Translation::BeginArgumentsAdaptorFrame(int literal_id, unsigned height) {
1967 76831 : buffer_->Add(ARGUMENTS_ADAPTOR_FRAME);
1968 76831 : buffer_->Add(literal_id);
1969 76831 : buffer_->Add(height);
1970 76831 : }
1971 :
1972 3239689 : void Translation::BeginInterpretedFrame(BailoutId bytecode_offset,
1973 : int literal_id, unsigned height,
1974 : int return_value_offset,
1975 : int return_value_count) {
1976 3239689 : buffer_->Add(INTERPRETED_FRAME);
1977 3239711 : buffer_->Add(bytecode_offset.ToInt());
1978 3239722 : buffer_->Add(literal_id);
1979 3239731 : buffer_->Add(height);
1980 3239731 : buffer_->Add(return_value_offset);
1981 3239733 : buffer_->Add(return_value_count);
1982 3239731 : }
1983 :
1984 6054 : void Translation::ArgumentsElements(CreateArgumentsType type) {
1985 6054 : buffer_->Add(ARGUMENTS_ELEMENTS);
1986 6054 : buffer_->Add(static_cast<uint8_t>(type));
1987 6054 : }
1988 :
1989 6336 : void Translation::ArgumentsLength(CreateArgumentsType type) {
1990 6336 : buffer_->Add(ARGUMENTS_LENGTH);
1991 6336 : buffer_->Add(static_cast<uint8_t>(type));
1992 6336 : }
1993 :
1994 92285 : void Translation::BeginCapturedObject(int length) {
1995 92285 : buffer_->Add(CAPTURED_OBJECT);
1996 92285 : buffer_->Add(length);
1997 92285 : }
1998 :
1999 :
2000 22811 : void Translation::DuplicateObject(int object_index) {
2001 22811 : buffer_->Add(DUPLICATED_OBJECT);
2002 22811 : buffer_->Add(object_index);
2003 22811 : }
2004 :
2005 :
2006 492987 : void Translation::StoreRegister(Register reg) {
2007 492987 : buffer_->Add(REGISTER);
2008 492987 : buffer_->Add(reg.code());
2009 492991 : }
2010 :
2011 :
2012 39645 : void Translation::StoreInt32Register(Register reg) {
2013 39645 : buffer_->Add(INT32_REGISTER);
2014 39645 : buffer_->Add(reg.code());
2015 39645 : }
2016 :
2017 14 : void Translation::StoreInt64Register(Register reg) {
2018 14 : buffer_->Add(INT64_REGISTER);
2019 14 : buffer_->Add(reg.code());
2020 14 : }
2021 :
2022 1326 : void Translation::StoreUint32Register(Register reg) {
2023 1326 : buffer_->Add(UINT32_REGISTER);
2024 1326 : buffer_->Add(reg.code());
2025 1326 : }
2026 :
2027 :
2028 4676 : void Translation::StoreBoolRegister(Register reg) {
2029 4676 : buffer_->Add(BOOL_REGISTER);
2030 4676 : buffer_->Add(reg.code());
2031 4676 : }
2032 :
2033 234 : void Translation::StoreFloatRegister(FloatRegister reg) {
2034 234 : buffer_->Add(FLOAT_REGISTER);
2035 234 : buffer_->Add(reg.code());
2036 234 : }
2037 :
2038 67796 : void Translation::StoreDoubleRegister(DoubleRegister reg) {
2039 67796 : buffer_->Add(DOUBLE_REGISTER);
2040 67794 : buffer_->Add(reg.code());
2041 67792 : }
2042 :
2043 :
2044 15995755 : void Translation::StoreStackSlot(int index) {
2045 15995755 : buffer_->Add(STACK_SLOT);
2046 15995725 : buffer_->Add(index);
2047 15995720 : }
2048 :
2049 :
2050 464191 : void Translation::StoreInt32StackSlot(int index) {
2051 464191 : buffer_->Add(INT32_STACK_SLOT);
2052 464190 : buffer_->Add(index);
2053 464187 : }
2054 :
2055 265 : void Translation::StoreInt64StackSlot(int index) {
2056 265 : buffer_->Add(INT64_STACK_SLOT);
2057 265 : buffer_->Add(index);
2058 265 : }
2059 :
2060 9560 : void Translation::StoreUint32StackSlot(int index) {
2061 9560 : buffer_->Add(UINT32_STACK_SLOT);
2062 9560 : buffer_->Add(index);
2063 9560 : }
2064 :
2065 :
2066 18407 : void Translation::StoreBoolStackSlot(int index) {
2067 18407 : buffer_->Add(BOOL_STACK_SLOT);
2068 18407 : buffer_->Add(index);
2069 18407 : }
2070 :
2071 642 : void Translation::StoreFloatStackSlot(int index) {
2072 642 : buffer_->Add(FLOAT_STACK_SLOT);
2073 642 : buffer_->Add(index);
2074 642 : }
2075 :
2076 233377 : void Translation::StoreDoubleStackSlot(int index) {
2077 233377 : buffer_->Add(DOUBLE_STACK_SLOT);
2078 233378 : buffer_->Add(index);
2079 233377 : }
2080 :
2081 :
2082 24896262 : void Translation::StoreLiteral(int literal_id) {
2083 24896262 : buffer_->Add(LITERAL);
2084 24896374 : buffer_->Add(literal_id);
2085 24896374 : }
2086 :
2087 47806 : void Translation::AddUpdateFeedback(int vector_literal, int slot) {
2088 47806 : buffer_->Add(UPDATE_FEEDBACK);
2089 47806 : buffer_->Add(vector_literal);
2090 47806 : buffer_->Add(slot);
2091 47806 : }
2092 :
2093 3015906 : void Translation::StoreJSFrameFunction() {
2094 : StoreStackSlot((StandardFrameConstants::kCallerPCOffset -
2095 : StandardFrameConstants::kFunctionOffset) /
2096 3015906 : kSystemPointerSize);
2097 3015924 : }
2098 :
2099 9801 : int Translation::NumberOfOperandsFor(Opcode opcode) {
2100 9801 : switch (opcode) {
2101 : case DUPLICATED_OBJECT:
2102 : case ARGUMENTS_ELEMENTS:
2103 : case ARGUMENTS_LENGTH:
2104 : case CAPTURED_OBJECT:
2105 : case REGISTER:
2106 : case INT32_REGISTER:
2107 : case INT64_REGISTER:
2108 : case UINT32_REGISTER:
2109 : case BOOL_REGISTER:
2110 : case FLOAT_REGISTER:
2111 : case DOUBLE_REGISTER:
2112 : case STACK_SLOT:
2113 : case INT32_STACK_SLOT:
2114 : case INT64_STACK_SLOT:
2115 : case UINT32_STACK_SLOT:
2116 : case BOOL_STACK_SLOT:
2117 : case FLOAT_STACK_SLOT:
2118 : case DOUBLE_STACK_SLOT:
2119 : case LITERAL:
2120 : return 1;
2121 : case ARGUMENTS_ADAPTOR_FRAME:
2122 : case UPDATE_FEEDBACK:
2123 0 : return 2;
2124 : case BEGIN:
2125 : case CONSTRUCT_STUB_FRAME:
2126 : case BUILTIN_CONTINUATION_FRAME:
2127 : case JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME:
2128 : case JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME:
2129 0 : return 3;
2130 : case INTERPRETED_FRAME:
2131 9500 : return 5;
2132 : }
2133 0 : FATAL("Unexpected translation type");
2134 : return -1;
2135 : }
2136 :
2137 :
2138 : #if defined(OBJECT_PRINT) || defined(ENABLE_DISASSEMBLER)
2139 :
2140 : const char* Translation::StringFor(Opcode opcode) {
2141 : #define TRANSLATION_OPCODE_CASE(item) case item: return #item;
2142 : switch (opcode) {
2143 : TRANSLATION_OPCODE_LIST(TRANSLATION_OPCODE_CASE)
2144 : }
2145 : #undef TRANSLATION_OPCODE_CASE
2146 : UNREACHABLE();
2147 : }
2148 :
2149 : #endif
2150 :
2151 :
2152 1470466 : Handle<FixedArray> MaterializedObjectStore::Get(Address fp) {
2153 1470239 : int index = StackIdToIndex(fp);
2154 1470239 : if (index == -1) {
2155 : return Handle<FixedArray>::null();
2156 : }
2157 227 : Handle<FixedArray> array = GetStackEntries();
2158 227 : CHECK_GT(array->length(), index);
2159 227 : return Handle<FixedArray>::cast(Handle<Object>(array->get(index), isolate()));
2160 : }
2161 :
2162 :
2163 143 : void MaterializedObjectStore::Set(Address fp,
2164 : Handle<FixedArray> materialized_objects) {
2165 143 : int index = StackIdToIndex(fp);
2166 143 : if (index == -1) {
2167 286 : index = static_cast<int>(frame_fps_.size());
2168 143 : frame_fps_.push_back(fp);
2169 : }
2170 :
2171 143 : Handle<FixedArray> array = EnsureStackEntries(index + 1);
2172 286 : array->set(index, *materialized_objects);
2173 143 : }
2174 :
2175 :
2176 3176038 : bool MaterializedObjectStore::Remove(Address fp) {
2177 143 : auto it = std::find(frame_fps_.begin(), frame_fps_.end(), fp);
2178 3175752 : if (it == frame_fps_.end()) return false;
2179 143 : int index = static_cast<int>(std::distance(frame_fps_.begin(), it));
2180 :
2181 143 : frame_fps_.erase(it);
2182 143 : FixedArray array = isolate()->heap()->materialized_objects();
2183 :
2184 143 : CHECK_LT(index, array->length());
2185 143 : int fps_size = static_cast<int>(frame_fps_.size());
2186 286 : for (int i = index; i < fps_size; i++) {
2187 0 : array->set(i, array->get(i + 1));
2188 : }
2189 143 : array->set(fps_size, ReadOnlyRoots(isolate()).undefined_value());
2190 143 : return true;
2191 : }
2192 :
2193 :
2194 1470382 : int MaterializedObjectStore::StackIdToIndex(Address fp) {
2195 : auto it = std::find(frame_fps_.begin(), frame_fps_.end(), fp);
2196 : return it == frame_fps_.end()
2197 : ? -1
2198 1470609 : : static_cast<int>(std::distance(frame_fps_.begin(), it));
2199 : }
2200 :
2201 :
2202 370 : Handle<FixedArray> MaterializedObjectStore::GetStackEntries() {
2203 : return Handle<FixedArray>(isolate()->heap()->materialized_objects(),
2204 740 : isolate());
2205 : }
2206 :
2207 :
2208 293 : Handle<FixedArray> MaterializedObjectStore::EnsureStackEntries(int length) {
2209 143 : Handle<FixedArray> array = GetStackEntries();
2210 143 : if (array->length() >= length) {
2211 93 : return array;
2212 : }
2213 :
2214 50 : int new_length = length > 10 ? length : 10;
2215 50 : if (new_length < 2 * array->length()) {
2216 0 : new_length = 2 * array->length();
2217 : }
2218 :
2219 : Handle<FixedArray> new_array =
2220 50 : isolate()->factory()->NewFixedArray(new_length, TENURED);
2221 100 : for (int i = 0; i < array->length(); i++) {
2222 0 : new_array->set(i, array->get(i));
2223 : }
2224 50 : HeapObject undefined_value = ReadOnlyRoots(isolate()).undefined_value();
2225 100 : for (int i = array->length(); i < length; i++) {
2226 50 : new_array->set(i, undefined_value);
2227 : }
2228 50 : isolate()->heap()->SetRootMaterializedObjects(*new_array);
2229 50 : return new_array;
2230 : }
2231 :
2232 : namespace {
2233 :
2234 181062 : Handle<Object> GetValueForDebugger(TranslatedFrame::iterator it,
2235 : Isolate* isolate) {
2236 362124 : if (it->GetRawValue() == ReadOnlyRoots(isolate).arguments_marker()) {
2237 21261 : if (!it->IsMaterializableByDebugger()) {
2238 236 : return isolate->factory()->optimized_out();
2239 : }
2240 : }
2241 180826 : return it->GetValue();
2242 : }
2243 :
2244 : } // namespace
2245 :
2246 24844 : DeoptimizedFrameInfo::DeoptimizedFrameInfo(TranslatedState* state,
2247 : TranslatedState::iterator frame_it,
2248 : Isolate* isolate) {
2249 : int parameter_count =
2250 24844 : frame_it->shared_info()->internal_formal_parameter_count();
2251 : TranslatedFrame::iterator stack_it = frame_it->begin();
2252 :
2253 : // Get the function. Note that this might materialize the function.
2254 : // In case the debugger mutates this value, we should deoptimize
2255 : // the function and remember the value in the materialized value store.
2256 24844 : function_ = Handle<JSFunction>::cast(stack_it->GetValue());
2257 24844 : stack_it++; // Skip the function.
2258 24844 : stack_it++; // Skip the receiver.
2259 :
2260 : DCHECK_EQ(TranslatedFrame::kInterpretedFunction, frame_it->kind());
2261 : source_position_ = Deoptimizer::ComputeSourcePositionFromBytecodeArray(
2262 24844 : *frame_it->shared_info(), frame_it->node_id());
2263 :
2264 : DCHECK_EQ(parameter_count,
2265 : function_->shared()->internal_formal_parameter_count());
2266 :
2267 24844 : parameters_.resize(static_cast<size_t>(parameter_count));
2268 66099 : for (int i = 0; i < parameter_count; i++) {
2269 41255 : Handle<Object> parameter = GetValueForDebugger(stack_it, isolate);
2270 : SetParameter(i, parameter);
2271 41255 : stack_it++;
2272 : }
2273 :
2274 : // Get the context.
2275 24844 : context_ = GetValueForDebugger(stack_it, isolate);
2276 24844 : stack_it++;
2277 :
2278 : // Get the expression stack.
2279 24844 : int stack_height = frame_it->height();
2280 24844 : if (frame_it->kind() == TranslatedFrame::kInterpretedFunction) {
2281 : // For interpreter frames, we should not count the accumulator.
2282 : // TODO(jarin): Clean up the indexing in translated frames.
2283 24844 : stack_height--;
2284 : }
2285 24844 : expression_stack_.resize(static_cast<size_t>(stack_height));
2286 139807 : for (int i = 0; i < stack_height; i++) {
2287 114963 : Handle<Object> expression = GetValueForDebugger(stack_it, isolate);
2288 : SetExpression(i, expression);
2289 114963 : stack_it++;
2290 : }
2291 :
2292 : // For interpreter frame, skip the accumulator.
2293 24844 : if (frame_it->kind() == TranslatedFrame::kInterpretedFunction) {
2294 24844 : stack_it++;
2295 : }
2296 24844 : CHECK(stack_it == frame_it->end());
2297 24844 : }
2298 :
2299 306 : Deoptimizer::DeoptInfo Deoptimizer::GetDeoptInfo(Code code, Address pc) {
2300 306 : CHECK(code->InstructionStart() <= pc && pc <= code->InstructionEnd());
2301 : SourcePosition last_position = SourcePosition::Unknown();
2302 : DeoptimizeReason last_reason = DeoptimizeReason::kUnknown;
2303 : int last_deopt_id = kNoDeoptimizationId;
2304 : int mask = RelocInfo::ModeMask(RelocInfo::DEOPT_REASON) |
2305 : RelocInfo::ModeMask(RelocInfo::DEOPT_ID) |
2306 : RelocInfo::ModeMask(RelocInfo::DEOPT_SCRIPT_OFFSET) |
2307 : RelocInfo::ModeMask(RelocInfo::DEOPT_INLINING_ID);
2308 2574 : for (RelocIterator it(code, mask); !it.done(); it.next()) {
2309 7110 : RelocInfo* info = it.rinfo();
2310 2574 : if (info->pc() >= pc) break;
2311 2268 : if (info->rmode() == RelocInfo::DEOPT_SCRIPT_OFFSET) {
2312 756 : int script_offset = static_cast<int>(info->data());
2313 756 : it.next();
2314 : DCHECK(it.rinfo()->rmode() == RelocInfo::DEOPT_INLINING_ID);
2315 756 : int inlining_id = static_cast<int>(it.rinfo()->data());
2316 : last_position = SourcePosition(script_offset, inlining_id);
2317 1512 : } else if (info->rmode() == RelocInfo::DEOPT_ID) {
2318 756 : last_deopt_id = static_cast<int>(info->data());
2319 756 : } else if (info->rmode() == RelocInfo::DEOPT_REASON) {
2320 756 : last_reason = static_cast<DeoptimizeReason>(info->data());
2321 : }
2322 : }
2323 306 : return DeoptInfo(last_position, last_reason, last_deopt_id);
2324 : }
2325 :
2326 :
2327 : // static
2328 24844 : int Deoptimizer::ComputeSourcePositionFromBytecodeArray(
2329 : SharedFunctionInfo shared, BailoutId node_id) {
2330 : DCHECK(shared->HasBytecodeArray());
2331 49688 : return AbstractCode::cast(shared->GetBytecodeArray())
2332 49688 : ->SourcePosition(node_id.ToInt());
2333 : }
2334 :
2335 : // static
2336 0 : TranslatedValue TranslatedValue::NewDeferredObject(TranslatedState* container,
2337 : int length,
2338 : int object_index) {
2339 : TranslatedValue slot(container, kCapturedObject);
2340 151179 : slot.materialization_info_ = {object_index, length};
2341 0 : return slot;
2342 : }
2343 :
2344 :
2345 : // static
2346 0 : TranslatedValue TranslatedValue::NewDuplicateObject(TranslatedState* container,
2347 : int id) {
2348 : TranslatedValue slot(container, kDuplicatedObject);
2349 83397 : slot.materialization_info_ = {id, -1};
2350 0 : return slot;
2351 : }
2352 :
2353 :
2354 : // static
2355 0 : TranslatedValue TranslatedValue::NewFloat(TranslatedState* container,
2356 : Float32 value) {
2357 : TranslatedValue slot(container, kFloat);
2358 288 : slot.float_value_ = value;
2359 0 : return slot;
2360 : }
2361 :
2362 : // static
2363 0 : TranslatedValue TranslatedValue::NewDouble(TranslatedState* container,
2364 : Float64 value) {
2365 : TranslatedValue slot(container, kDouble);
2366 522801 : slot.double_value_ = value;
2367 0 : return slot;
2368 : }
2369 :
2370 :
2371 : // static
2372 0 : TranslatedValue TranslatedValue::NewInt32(TranslatedState* container,
2373 : int32_t value) {
2374 : TranslatedValue slot(container, kInt32);
2375 323096 : slot.int32_value_ = value;
2376 0 : return slot;
2377 : }
2378 :
2379 : // static
2380 0 : TranslatedValue TranslatedValue::NewInt64(TranslatedState* container,
2381 : int64_t value) {
2382 : TranslatedValue slot(container, kInt64);
2383 9 : slot.int64_value_ = value;
2384 0 : return slot;
2385 : }
2386 :
2387 : // static
2388 0 : TranslatedValue TranslatedValue::NewUInt32(TranslatedState* container,
2389 : uint32_t value) {
2390 : TranslatedValue slot(container, kUInt32);
2391 57177 : slot.uint32_value_ = value;
2392 0 : return slot;
2393 : }
2394 :
2395 :
2396 : // static
2397 0 : TranslatedValue TranslatedValue::NewBool(TranslatedState* container,
2398 : uint32_t value) {
2399 : TranslatedValue slot(container, kBoolBit);
2400 434 : slot.uint32_value_ = value;
2401 0 : return slot;
2402 : }
2403 :
2404 :
2405 : // static
2406 0 : TranslatedValue TranslatedValue::NewTagged(TranslatedState* container,
2407 : Object literal) {
2408 : TranslatedValue slot(container, kTagged);
2409 20319154 : slot.raw_literal_ = literal;
2410 0 : return slot;
2411 : }
2412 :
2413 : // static
2414 0 : TranslatedValue TranslatedValue::NewInvalid(TranslatedState* container) {
2415 0 : return TranslatedValue(container, kInvalid);
2416 : }
2417 :
2418 :
2419 20458527 : Isolate* TranslatedValue::isolate() const { return container_->isolate(); }
2420 :
2421 0 : Object TranslatedValue::raw_literal() const {
2422 : DCHECK_EQ(kTagged, kind());
2423 21642570 : return raw_literal_;
2424 : }
2425 :
2426 0 : int32_t TranslatedValue::int32_value() const {
2427 : DCHECK_EQ(kInt32, kind());
2428 27203 : return int32_value_;
2429 : }
2430 :
2431 0 : int64_t TranslatedValue::int64_value() const {
2432 : DCHECK_EQ(kInt64, kind());
2433 9 : return int64_value_;
2434 : }
2435 :
2436 0 : uint32_t TranslatedValue::uint32_value() const {
2437 : DCHECK(kind() == kUInt32 || kind() == kBoolBit);
2438 167 : return uint32_value_;
2439 : }
2440 :
2441 0 : Float32 TranslatedValue::float_value() const {
2442 : DCHECK_EQ(kFloat, kind());
2443 280 : return float_value_;
2444 : }
2445 :
2446 0 : Float64 TranslatedValue::double_value() const {
2447 : DCHECK_EQ(kDouble, kind());
2448 23849 : return double_value_;
2449 : }
2450 :
2451 :
2452 0 : int TranslatedValue::object_length() const {
2453 : DCHECK_EQ(kind(), kCapturedObject);
2454 1868597 : return materialization_info_.length_;
2455 : }
2456 :
2457 :
2458 0 : int TranslatedValue::object_index() const {
2459 : DCHECK(kind() == kCapturedObject || kind() == kDuplicatedObject);
2460 145634 : return materialization_info_.id_;
2461 : }
2462 :
2463 2978300 : Object TranslatedValue::GetRawValue() const {
2464 : // If we have a value, return it.
2465 1564124 : if (materialization_state() == kFinished) {
2466 : return *storage_;
2467 : }
2468 :
2469 : // Otherwise, do a best effort to get the value without allocation.
2470 1414176 : switch (kind()) {
2471 : case kTagged:
2472 : return raw_literal();
2473 :
2474 : case kInt32: {
2475 : bool is_smi = Smi::IsValid(int32_value());
2476 : if (is_smi) {
2477 27203 : return Smi::FromInt(int32_value());
2478 : }
2479 : break;
2480 : }
2481 :
2482 : case kInt64: {
2483 9 : bool is_smi = (int64_value() >= static_cast<int64_t>(Smi::kMinValue) &&
2484 : int64_value() <= static_cast<int64_t>(Smi::kMaxValue));
2485 9 : if (is_smi) {
2486 9 : return Smi::FromIntptr(static_cast<intptr_t>(int64_value()));
2487 : }
2488 : break;
2489 : }
2490 :
2491 : case kUInt32: {
2492 89 : bool is_smi = (uint32_value() <= static_cast<uintptr_t>(Smi::kMaxValue));
2493 89 : if (is_smi) {
2494 75 : return Smi::FromInt(static_cast<int32_t>(uint32_value()));
2495 : }
2496 : break;
2497 : }
2498 :
2499 : case kBoolBit: {
2500 71 : if (uint32_value() == 0) {
2501 52 : return ReadOnlyRoots(isolate()).false_value();
2502 : } else {
2503 19 : CHECK_EQ(1U, uint32_value());
2504 19 : return ReadOnlyRoots(isolate()).true_value();
2505 : }
2506 : }
2507 :
2508 : default:
2509 : break;
2510 : }
2511 :
2512 : // If we could not get the value without allocation, return the arguments
2513 : // marker.
2514 63402 : return ReadOnlyRoots(isolate()).arguments_marker();
2515 : }
2516 :
2517 0 : void TranslatedValue::set_initialized_storage(Handle<Object> storage) {
2518 : DCHECK_EQ(kUninitialized, materialization_state());
2519 20357415 : storage_ = storage;
2520 20357415 : materialization_state_ = kFinished;
2521 0 : }
2522 :
2523 3501997 : Handle<Object> TranslatedValue::GetValue() {
2524 : // If we already have a value, then get it.
2525 3420546 : if (materialization_state() == kFinished) return storage_;
2526 :
2527 : // Otherwise we have to materialize.
2528 81451 : switch (kind()) {
2529 : case TranslatedValue::kTagged:
2530 : case TranslatedValue::kInt32:
2531 : case TranslatedValue::kInt64:
2532 : case TranslatedValue::kUInt32:
2533 : case TranslatedValue::kBoolBit:
2534 : case TranslatedValue::kFloat:
2535 : case TranslatedValue::kDouble: {
2536 37374 : MaterializeSimple();
2537 37374 : return storage_;
2538 : }
2539 :
2540 : case TranslatedValue::kCapturedObject:
2541 : case TranslatedValue::kDuplicatedObject: {
2542 : // We need to materialize the object (or possibly even object graphs).
2543 : // To make the object verifier happy, we materialize in two steps.
2544 :
2545 : // 1. Allocate storage for reachable objects. This makes sure that for
2546 : // each object we have allocated space on heap. The space will be
2547 : // a byte array that will be later initialized, or a fully
2548 : // initialized object if it is safe to allocate one that will
2549 : // pass the verifier.
2550 44077 : container_->EnsureObjectAllocatedAt(this);
2551 :
2552 : // 2. Initialize the objects. If we have allocated only byte arrays
2553 : // for some objects, we now overwrite the byte arrays with the
2554 : // correct object fields. Note that this phase does not allocate
2555 : // any new objects, so it does not trigger the object verifier.
2556 44077 : return container_->InitializeObjectAt(this);
2557 : }
2558 :
2559 : case TranslatedValue::kInvalid:
2560 0 : FATAL("unexpected case");
2561 : return Handle<Object>::null();
2562 : }
2563 :
2564 0 : FATAL("internal error: value missing");
2565 : return Handle<Object>::null();
2566 : }
2567 :
2568 712959 : void TranslatedValue::MaterializeSimple() {
2569 : // If we already have materialized, return.
2570 1377646 : if (materialization_state() == kFinished) return;
2571 :
2572 37950 : Object raw_value = GetRawValue();
2573 37950 : if (raw_value != ReadOnlyRoots(isolate()).arguments_marker()) {
2574 : // We can get the value without allocation, just return it here.
2575 : set_initialized_storage(Handle<Object>(raw_value, isolate()));
2576 13814 : return;
2577 : }
2578 :
2579 24136 : switch (kind()) {
2580 : case kInt32:
2581 : set_initialized_storage(
2582 0 : Handle<Object>(isolate()->factory()->NewNumber(int32_value())));
2583 : return;
2584 :
2585 : case kInt64:
2586 : set_initialized_storage(Handle<Object>(
2587 0 : isolate()->factory()->NewNumber(static_cast<double>(int64_value()))));
2588 : return;
2589 :
2590 : case kUInt32:
2591 : set_initialized_storage(
2592 14 : Handle<Object>(isolate()->factory()->NewNumber(uint32_value())));
2593 : return;
2594 :
2595 : case kFloat: {
2596 280 : double scalar_value = float_value().get_scalar();
2597 : set_initialized_storage(
2598 280 : Handle<Object>(isolate()->factory()->NewNumber(scalar_value)));
2599 : return;
2600 : }
2601 :
2602 : case kDouble: {
2603 23849 : double scalar_value = double_value().get_scalar();
2604 : set_initialized_storage(
2605 23849 : Handle<Object>(isolate()->factory()->NewNumber(scalar_value)));
2606 : return;
2607 : }
2608 :
2609 : case kCapturedObject:
2610 : case kDuplicatedObject:
2611 : case kInvalid:
2612 : case kTagged:
2613 : case kBoolBit:
2614 0 : FATAL("internal error: unexpected materialization.");
2615 : break;
2616 : }
2617 : }
2618 :
2619 :
2620 10580 : bool TranslatedValue::IsMaterializedObject() const {
2621 10580 : switch (kind()) {
2622 : case kCapturedObject:
2623 : case kDuplicatedObject:
2624 : return true;
2625 : default:
2626 9640 : return false;
2627 : }
2628 : }
2629 :
2630 21261 : bool TranslatedValue::IsMaterializableByDebugger() const {
2631 : // At the moment, we only allow materialization of doubles.
2632 0 : return (kind() == kDouble);
2633 : }
2634 :
2635 27943469 : int TranslatedValue::GetChildrenCount() const {
2636 27988564 : if (kind() == kCapturedObject) {
2637 0 : return object_length();
2638 : } else {
2639 : return 0;
2640 : }
2641 : }
2642 :
2643 0 : uint64_t TranslatedState::GetUInt64Slot(Address fp, int slot_offset) {
2644 1044692 : return Memory<uint64_t>(fp + slot_offset);
2645 : }
2646 :
2647 0 : uint32_t TranslatedState::GetUInt32Slot(Address fp, int slot_offset) {
2648 379306 : Address address = fp + slot_offset;
2649 : #if V8_TARGET_BIG_ENDIAN && V8_HOST_ARCH_64_BIT
2650 : return Memory<uint32_t>(address + kIntSize);
2651 : #else
2652 379306 : return Memory<uint32_t>(address);
2653 : #endif
2654 : }
2655 :
2656 0 : Float32 TranslatedState::GetFloatSlot(Address fp, int slot_offset) {
2657 : #if !V8_TARGET_ARCH_S390X && !V8_TARGET_ARCH_PPC64
2658 0 : return Float32::FromBits(GetUInt32Slot(fp, slot_offset));
2659 : #else
2660 : return Float32::FromBits(Memory<uint32_t>(fp + slot_offset));
2661 : #endif
2662 : }
2663 :
2664 0 : Float64 TranslatedState::GetDoubleSlot(Address fp, int slot_offset) {
2665 0 : return Float64::FromBits(GetUInt64Slot(fp, slot_offset));
2666 : }
2667 :
2668 21457535 : void TranslatedValue::Handlify() {
2669 21457535 : if (kind() == kTagged) {
2670 : set_initialized_storage(Handle<Object>(raw_literal(), isolate()));
2671 20319154 : raw_literal_ = Object();
2672 : }
2673 21457535 : }
2674 :
2675 0 : TranslatedFrame TranslatedFrame::InterpretedFrame(
2676 : BailoutId bytecode_offset, SharedFunctionInfo shared_info, int height,
2677 : int return_value_offset, int return_value_count) {
2678 : TranslatedFrame frame(kInterpretedFunction, shared_info, height,
2679 : return_value_offset, return_value_count);
2680 1779190 : frame.node_id_ = bytecode_offset;
2681 0 : return frame;
2682 : }
2683 :
2684 0 : TranslatedFrame TranslatedFrame::ArgumentsAdaptorFrame(
2685 : SharedFunctionInfo shared_info, int height) {
2686 0 : return TranslatedFrame(kArgumentsAdaptor, shared_info, height);
2687 : }
2688 :
2689 0 : TranslatedFrame TranslatedFrame::ConstructStubFrame(
2690 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2691 : TranslatedFrame frame(kConstructStub, shared_info, height);
2692 40046 : frame.node_id_ = bailout_id;
2693 0 : return frame;
2694 : }
2695 :
2696 0 : TranslatedFrame TranslatedFrame::BuiltinContinuationFrame(
2697 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2698 : TranslatedFrame frame(kBuiltinContinuation, shared_info, height);
2699 42148 : frame.node_id_ = bailout_id;
2700 0 : return frame;
2701 : }
2702 :
2703 0 : TranslatedFrame TranslatedFrame::JavaScriptBuiltinContinuationFrame(
2704 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2705 : TranslatedFrame frame(kJavaScriptBuiltinContinuation, shared_info, height);
2706 2187 : frame.node_id_ = bailout_id;
2707 0 : return frame;
2708 : }
2709 :
2710 0 : TranslatedFrame TranslatedFrame::JavaScriptBuiltinContinuationWithCatchFrame(
2711 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2712 : TranslatedFrame frame(kJavaScriptBuiltinContinuationWithCatch, shared_info,
2713 : height);
2714 82 : frame.node_id_ = bailout_id;
2715 0 : return frame;
2716 : }
2717 :
2718 2007063 : int TranslatedFrame::GetValueCount() {
2719 2007063 : switch (kind()) {
2720 : case kInterpretedFunction: {
2721 : int parameter_count =
2722 1782190 : raw_shared_info_->internal_formal_parameter_count() + 1;
2723 : // + 2 for function and context.
2724 1782190 : return height_ + parameter_count + 2;
2725 : }
2726 :
2727 : case kArgumentsAdaptor:
2728 : case kConstructStub:
2729 : case kBuiltinContinuation:
2730 : case kJavaScriptBuiltinContinuation:
2731 : case kJavaScriptBuiltinContinuationWithCatch:
2732 224873 : return 1 + height_;
2733 :
2734 : case kInvalid:
2735 0 : UNREACHABLE();
2736 : break;
2737 : }
2738 0 : UNREACHABLE();
2739 : }
2740 :
2741 :
2742 2004063 : void TranslatedFrame::Handlify() {
2743 2004063 : if (!raw_shared_info_.is_null()) {
2744 : shared_info_ = Handle<SharedFunctionInfo>(raw_shared_info_,
2745 2004063 : raw_shared_info_->GetIsolate());
2746 2004063 : raw_shared_info_ = SharedFunctionInfo();
2747 : }
2748 23461598 : for (auto& value : values_) {
2749 21457535 : value.Handlify();
2750 : }
2751 2004063 : }
2752 :
2753 2004063 : TranslatedFrame TranslatedState::CreateNextTranslatedFrame(
2754 : TranslationIterator* iterator, FixedArray literal_array, Address fp,
2755 : FILE* trace_file) {
2756 : Translation::Opcode opcode =
2757 2004063 : static_cast<Translation::Opcode>(iterator->Next());
2758 2004063 : switch (opcode) {
2759 : case Translation::INTERPRETED_FRAME: {
2760 1779190 : BailoutId bytecode_offset = BailoutId(iterator->Next());
2761 : SharedFunctionInfo shared_info =
2762 3558380 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2763 1779190 : int height = iterator->Next();
2764 1779190 : int return_value_offset = iterator->Next();
2765 1779190 : int return_value_count = iterator->Next();
2766 1779190 : if (trace_file != nullptr) {
2767 300 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2768 300 : PrintF(trace_file, " reading input frame %s", name.get());
2769 300 : int arg_count = shared_info->internal_formal_parameter_count() + 1;
2770 : PrintF(trace_file,
2771 : " => bytecode_offset=%d, args=%d, height=%d, retval=%i(#%i); "
2772 : "inputs:\n",
2773 : bytecode_offset.ToInt(), arg_count, height, return_value_offset,
2774 300 : return_value_count);
2775 : }
2776 : return TranslatedFrame::InterpretedFrame(bytecode_offset, shared_info,
2777 : height, return_value_offset,
2778 : return_value_count);
2779 : }
2780 :
2781 : case Translation::ARGUMENTS_ADAPTOR_FRAME: {
2782 : SharedFunctionInfo shared_info =
2783 280820 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2784 140410 : int height = iterator->Next();
2785 140410 : if (trace_file != nullptr) {
2786 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2787 0 : PrintF(trace_file, " reading arguments adaptor frame %s", name.get());
2788 0 : PrintF(trace_file, " => height=%d; inputs:\n", height);
2789 : }
2790 : return TranslatedFrame::ArgumentsAdaptorFrame(shared_info, height);
2791 : }
2792 :
2793 : case Translation::CONSTRUCT_STUB_FRAME: {
2794 40046 : BailoutId bailout_id = BailoutId(iterator->Next());
2795 : SharedFunctionInfo shared_info =
2796 80092 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2797 40046 : int height = iterator->Next();
2798 40046 : if (trace_file != nullptr) {
2799 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2800 0 : PrintF(trace_file, " reading construct stub frame %s", name.get());
2801 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2802 0 : bailout_id.ToInt(), height);
2803 : }
2804 : return TranslatedFrame::ConstructStubFrame(bailout_id, shared_info,
2805 : height);
2806 : }
2807 :
2808 : case Translation::BUILTIN_CONTINUATION_FRAME: {
2809 42148 : BailoutId bailout_id = BailoutId(iterator->Next());
2810 : SharedFunctionInfo shared_info =
2811 84296 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2812 42148 : int height = iterator->Next();
2813 42148 : if (trace_file != nullptr) {
2814 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2815 : PrintF(trace_file, " reading builtin continuation frame %s",
2816 0 : name.get());
2817 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2818 0 : bailout_id.ToInt(), height);
2819 : }
2820 : // Add one to the height to account for the context which was implicitly
2821 : // added to the translation during code generation.
2822 42148 : int height_with_context = height + 1;
2823 : return TranslatedFrame::BuiltinContinuationFrame(bailout_id, shared_info,
2824 : height_with_context);
2825 : }
2826 :
2827 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME: {
2828 2187 : BailoutId bailout_id = BailoutId(iterator->Next());
2829 : SharedFunctionInfo shared_info =
2830 4374 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2831 2187 : int height = iterator->Next();
2832 2187 : if (trace_file != nullptr) {
2833 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2834 : PrintF(trace_file, " reading JavaScript builtin continuation frame %s",
2835 0 : name.get());
2836 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2837 0 : bailout_id.ToInt(), height);
2838 : }
2839 : // Add one to the height to account for the context which was implicitly
2840 : // added to the translation during code generation.
2841 2187 : int height_with_context = height + 1;
2842 : return TranslatedFrame::JavaScriptBuiltinContinuationFrame(
2843 : bailout_id, shared_info, height_with_context);
2844 : }
2845 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME: {
2846 82 : BailoutId bailout_id = BailoutId(iterator->Next());
2847 : SharedFunctionInfo shared_info =
2848 164 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2849 82 : int height = iterator->Next();
2850 82 : if (trace_file != nullptr) {
2851 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2852 : PrintF(trace_file,
2853 : " reading JavaScript builtin continuation frame with catch %s",
2854 0 : name.get());
2855 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2856 0 : bailout_id.ToInt(), height);
2857 : }
2858 : // Add one to the height to account for the context which was implicitly
2859 : // added to the translation during code generation.
2860 82 : int height_with_context = height + 1;
2861 : return TranslatedFrame::JavaScriptBuiltinContinuationWithCatchFrame(
2862 : bailout_id, shared_info, height_with_context);
2863 : }
2864 : case Translation::UPDATE_FEEDBACK:
2865 : case Translation::BEGIN:
2866 : case Translation::DUPLICATED_OBJECT:
2867 : case Translation::ARGUMENTS_ELEMENTS:
2868 : case Translation::ARGUMENTS_LENGTH:
2869 : case Translation::CAPTURED_OBJECT:
2870 : case Translation::REGISTER:
2871 : case Translation::INT32_REGISTER:
2872 : case Translation::INT64_REGISTER:
2873 : case Translation::UINT32_REGISTER:
2874 : case Translation::BOOL_REGISTER:
2875 : case Translation::FLOAT_REGISTER:
2876 : case Translation::DOUBLE_REGISTER:
2877 : case Translation::STACK_SLOT:
2878 : case Translation::INT32_STACK_SLOT:
2879 : case Translation::INT64_STACK_SLOT:
2880 : case Translation::UINT32_STACK_SLOT:
2881 : case Translation::BOOL_STACK_SLOT:
2882 : case Translation::FLOAT_STACK_SLOT:
2883 : case Translation::DOUBLE_STACK_SLOT:
2884 : case Translation::LITERAL:
2885 : break;
2886 : }
2887 0 : FATAL("We should never get here - unexpected deopt info.");
2888 : return TranslatedFrame::InvalidFrame();
2889 : }
2890 :
2891 : // static
2892 4755192 : void TranslatedFrame::AdvanceIterator(
2893 5405916 : std::deque<TranslatedValue>::iterator* iter) {
2894 : int values_to_skip = 1;
2895 14916300 : while (values_to_skip > 0) {
2896 : // Consume the current element.
2897 5405916 : values_to_skip--;
2898 : // Add all the children.
2899 5405916 : values_to_skip += (*iter)->GetChildrenCount();
2900 :
2901 5405916 : (*iter)++;
2902 : }
2903 4755192 : }
2904 :
2905 510 : Address TranslatedState::ComputeArgumentsPosition(Address input_frame_pointer,
2906 : CreateArgumentsType type,
2907 : int* length) {
2908 : Address parent_frame_pointer = *reinterpret_cast<Address*>(
2909 510 : input_frame_pointer + StandardFrameConstants::kCallerFPOffset);
2910 : intptr_t parent_frame_type = Memory<intptr_t>(
2911 1020 : parent_frame_pointer + CommonFrameConstants::kContextOrFrameTypeOffset);
2912 :
2913 : Address arguments_frame;
2914 510 : if (parent_frame_type ==
2915 : StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)) {
2916 334 : if (length)
2917 : *length = Smi::cast(*FullObjectSlot(
2918 : parent_frame_pointer +
2919 334 : ArgumentsAdaptorFrameConstants::kLengthOffset))
2920 334 : ->value();
2921 : arguments_frame = parent_frame_pointer;
2922 : } else {
2923 176 : if (length) *length = formal_parameter_count_;
2924 : arguments_frame = input_frame_pointer;
2925 : }
2926 :
2927 510 : if (type == CreateArgumentsType::kRestParameter) {
2928 : // If the actual number of arguments is less than the number of formal
2929 : // parameters, we have zero rest parameters.
2930 76 : if (length) *length = std::max(0, *length - formal_parameter_count_);
2931 : }
2932 :
2933 510 : return arguments_frame;
2934 : }
2935 :
2936 : // Creates translated values for an arguments backing store, or the backing
2937 : // store for rest parameters depending on the given {type}. The TranslatedValue
2938 : // objects for the fields are not read from the TranslationIterator, but instead
2939 : // created on-the-fly based on dynamic information in the optimized frame.
2940 251 : void TranslatedState::CreateArgumentsElementsTranslatedValues(
2941 : int frame_index, Address input_frame_pointer, CreateArgumentsType type,
2942 : FILE* trace_file) {
2943 251 : TranslatedFrame& frame = frames_[frame_index];
2944 :
2945 : int length;
2946 : Address arguments_frame =
2947 251 : ComputeArgumentsPosition(input_frame_pointer, type, &length);
2948 :
2949 251 : int object_index = static_cast<int>(object_positions_.size());
2950 251 : int value_index = static_cast<int>(frame.values_.size());
2951 251 : if (trace_file != nullptr) {
2952 : PrintF(trace_file, "arguments elements object #%d (type = %d, length = %d)",
2953 0 : object_index, static_cast<uint8_t>(type), length);
2954 : }
2955 :
2956 502 : object_positions_.push_back({frame_index, value_index});
2957 : frame.Add(TranslatedValue::NewDeferredObject(
2958 502 : this, length + FixedArray::kHeaderSize / kTaggedSize, object_index));
2959 :
2960 251 : ReadOnlyRoots roots(isolate_);
2961 251 : frame.Add(TranslatedValue::NewTagged(this, roots.fixed_array_map()));
2962 502 : frame.Add(TranslatedValue::NewInt32(this, length));
2963 :
2964 : int number_of_holes = 0;
2965 251 : if (type == CreateArgumentsType::kMappedArguments) {
2966 : // If the actual number of arguments is less than the number of formal
2967 : // parameters, we have fewer holes to fill to not overshoot the length.
2968 43 : number_of_holes = Min(formal_parameter_count_, length);
2969 : }
2970 309 : for (int i = 0; i < number_of_holes; ++i) {
2971 58 : frame.Add(TranslatedValue::NewTagged(this, roots.the_hole_value()));
2972 : }
2973 573950 : for (int i = length - number_of_holes - 1; i >= 0; --i) {
2974 : Address argument_slot = arguments_frame +
2975 573699 : CommonFrameConstants::kFixedFrameSizeAboveFp +
2976 1147398 : i * kSystemPointerSize;
2977 573699 : frame.Add(TranslatedValue::NewTagged(this, *FullObjectSlot(argument_slot)));
2978 : }
2979 251 : }
2980 :
2981 : // We can't intermix stack decoding and allocations because the deoptimization
2982 : // infrastracture is not GC safe.
2983 : // Thus we build a temporary structure in malloced space.
2984 : // The TranslatedValue objects created correspond to the static translation
2985 : // instructions from the TranslationIterator, except for
2986 : // Translation::ARGUMENTS_ELEMENTS, where the number and values of the
2987 : // FixedArray elements depend on dynamic information from the optimized frame.
2988 : // Returns the number of expected nested translations from the
2989 : // TranslationIterator.
2990 20883276 : int TranslatedState::CreateNextTranslatedValue(
2991 : int frame_index, TranslationIterator* iterator, FixedArray literal_array,
2992 : Address fp, RegisterValues* registers, FILE* trace_file) {
2993 : disasm::NameConverter converter;
2994 :
2995 20883276 : TranslatedFrame& frame = frames_[frame_index];
2996 20883276 : int value_index = static_cast<int>(frame.values_.size());
2997 :
2998 : Translation::Opcode opcode =
2999 20883276 : static_cast<Translation::Opcode>(iterator->Next());
3000 20883276 : switch (opcode) {
3001 : case Translation::BEGIN:
3002 : case Translation::INTERPRETED_FRAME:
3003 : case Translation::ARGUMENTS_ADAPTOR_FRAME:
3004 : case Translation::CONSTRUCT_STUB_FRAME:
3005 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME:
3006 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME:
3007 : case Translation::BUILTIN_CONTINUATION_FRAME:
3008 : case Translation::UPDATE_FEEDBACK:
3009 : // Peeled off before getting here.
3010 : break;
3011 :
3012 : case Translation::DUPLICATED_OBJECT: {
3013 83397 : int object_id = iterator->Next();
3014 83397 : if (trace_file != nullptr) {
3015 0 : PrintF(trace_file, "duplicated object #%d", object_id);
3016 : }
3017 83397 : object_positions_.push_back(object_positions_[object_id]);
3018 : TranslatedValue translated_value =
3019 : TranslatedValue::NewDuplicateObject(this, object_id);
3020 : frame.Add(translated_value);
3021 : return translated_value.GetChildrenCount();
3022 : }
3023 :
3024 : case Translation::ARGUMENTS_ELEMENTS: {
3025 : CreateArgumentsType arguments_type =
3026 251 : static_cast<CreateArgumentsType>(iterator->Next());
3027 : CreateArgumentsElementsTranslatedValues(frame_index, fp, arguments_type,
3028 251 : trace_file);
3029 251 : return 0;
3030 : }
3031 :
3032 : case Translation::ARGUMENTS_LENGTH: {
3033 : CreateArgumentsType arguments_type =
3034 259 : static_cast<CreateArgumentsType>(iterator->Next());
3035 : int length;
3036 259 : ComputeArgumentsPosition(fp, arguments_type, &length);
3037 259 : if (trace_file != nullptr) {
3038 : PrintF(trace_file, "arguments length field (type = %d, length = %d)",
3039 0 : static_cast<uint8_t>(arguments_type), length);
3040 : }
3041 518 : frame.Add(TranslatedValue::NewInt32(this, length));
3042 : return 0;
3043 : }
3044 :
3045 : case Translation::CAPTURED_OBJECT: {
3046 150928 : int field_count = iterator->Next();
3047 150928 : int object_index = static_cast<int>(object_positions_.size());
3048 150928 : if (trace_file != nullptr) {
3049 : PrintF(trace_file, "captured object #%d (length = %d)", object_index,
3050 0 : field_count);
3051 : }
3052 301856 : object_positions_.push_back({frame_index, value_index});
3053 : TranslatedValue translated_value =
3054 : TranslatedValue::NewDeferredObject(this, field_count, object_index);
3055 : frame.Add(translated_value);
3056 : return translated_value.GetChildrenCount();
3057 : }
3058 :
3059 : case Translation::REGISTER: {
3060 22494 : int input_reg = iterator->Next();
3061 22494 : if (registers == nullptr) {
3062 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3063 : frame.Add(translated_value);
3064 : return translated_value.GetChildrenCount();
3065 : }
3066 22494 : intptr_t value = registers->GetRegister(input_reg);
3067 22494 : if (trace_file != nullptr) {
3068 : PrintF(trace_file, V8PRIxPTR_FMT " ; %s ", value,
3069 450 : converter.NameOfCPURegister(input_reg));
3070 900 : Object(value)->ShortPrint(trace_file);
3071 : }
3072 : TranslatedValue translated_value =
3073 22494 : TranslatedValue::NewTagged(this, Object(value));
3074 : frame.Add(translated_value);
3075 22494 : return translated_value.GetChildrenCount();
3076 : }
3077 :
3078 : case Translation::INT32_REGISTER: {
3079 1010 : int input_reg = iterator->Next();
3080 1010 : if (registers == nullptr) {
3081 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3082 : frame.Add(translated_value);
3083 : return translated_value.GetChildrenCount();
3084 : }
3085 1010 : intptr_t value = registers->GetRegister(input_reg);
3086 1010 : if (trace_file != nullptr) {
3087 : PrintF(trace_file, "%" V8PRIdPTR " ; %s (int32)", value,
3088 0 : converter.NameOfCPURegister(input_reg));
3089 : }
3090 : TranslatedValue translated_value =
3091 1010 : TranslatedValue::NewInt32(this, static_cast<int32_t>(value));
3092 : frame.Add(translated_value);
3093 1010 : return translated_value.GetChildrenCount();
3094 : }
3095 :
3096 : case Translation::INT64_REGISTER: {
3097 0 : int input_reg = iterator->Next();
3098 0 : if (registers == nullptr) {
3099 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3100 : frame.Add(translated_value);
3101 : return translated_value.GetChildrenCount();
3102 : }
3103 0 : intptr_t value = registers->GetRegister(input_reg);
3104 0 : if (trace_file != nullptr) {
3105 : PrintF(trace_file, "%" V8PRIdPTR " ; %s (int64)", value,
3106 0 : converter.NameOfCPURegister(input_reg));
3107 : }
3108 : TranslatedValue translated_value =
3109 : TranslatedValue::NewInt64(this, static_cast<int64_t>(value));
3110 : frame.Add(translated_value);
3111 0 : return translated_value.GetChildrenCount();
3112 : }
3113 :
3114 : case Translation::UINT32_REGISTER: {
3115 17 : int input_reg = iterator->Next();
3116 17 : if (registers == nullptr) {
3117 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3118 : frame.Add(translated_value);
3119 : return translated_value.GetChildrenCount();
3120 : }
3121 17 : intptr_t value = registers->GetRegister(input_reg);
3122 17 : if (trace_file != nullptr) {
3123 : PrintF(trace_file, "%" V8PRIuPTR " ; %s (uint32)", value,
3124 0 : converter.NameOfCPURegister(input_reg));
3125 : }
3126 : TranslatedValue translated_value =
3127 17 : TranslatedValue::NewUInt32(this, static_cast<uint32_t>(value));
3128 : frame.Add(translated_value);
3129 17 : return translated_value.GetChildrenCount();
3130 : }
3131 :
3132 : case Translation::BOOL_REGISTER: {
3133 61 : int input_reg = iterator->Next();
3134 61 : if (registers == nullptr) {
3135 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3136 : frame.Add(translated_value);
3137 : return translated_value.GetChildrenCount();
3138 : }
3139 61 : intptr_t value = registers->GetRegister(input_reg);
3140 61 : if (trace_file != nullptr) {
3141 : PrintF(trace_file, "%" V8PRIdPTR " ; %s (bool)", value,
3142 0 : converter.NameOfCPURegister(input_reg));
3143 : }
3144 : TranslatedValue translated_value =
3145 61 : TranslatedValue::NewBool(this, static_cast<uint32_t>(value));
3146 : frame.Add(translated_value);
3147 61 : return translated_value.GetChildrenCount();
3148 : }
3149 :
3150 : case Translation::FLOAT_REGISTER: {
3151 91 : int input_reg = iterator->Next();
3152 91 : if (registers == nullptr) {
3153 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3154 : frame.Add(translated_value);
3155 : return translated_value.GetChildrenCount();
3156 : }
3157 182 : Float32 value = registers->GetFloatRegister(input_reg);
3158 91 : if (trace_file != nullptr) {
3159 : PrintF(trace_file, "%e ; %s (float)", value.get_scalar(),
3160 0 : RegisterName(FloatRegister::from_code(input_reg)));
3161 : }
3162 : TranslatedValue translated_value = TranslatedValue::NewFloat(this, value);
3163 : frame.Add(translated_value);
3164 91 : return translated_value.GetChildrenCount();
3165 : }
3166 :
3167 : case Translation::DOUBLE_REGISTER: {
3168 464 : int input_reg = iterator->Next();
3169 464 : if (registers == nullptr) {
3170 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3171 : frame.Add(translated_value);
3172 : return translated_value.GetChildrenCount();
3173 : }
3174 928 : Float64 value = registers->GetDoubleRegister(input_reg);
3175 464 : if (trace_file != nullptr) {
3176 : PrintF(trace_file, "%e ; %s (double)", value.get_scalar(),
3177 0 : RegisterName(DoubleRegister::from_code(input_reg)));
3178 : }
3179 : TranslatedValue translated_value =
3180 : TranslatedValue::NewDouble(this, value);
3181 : frame.Add(translated_value);
3182 464 : return translated_value.GetChildrenCount();
3183 : }
3184 :
3185 : case Translation::STACK_SLOT: {
3186 : int slot_offset =
3187 7828807 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3188 7828807 : intptr_t value = *(reinterpret_cast<intptr_t*>(fp + slot_offset));
3189 7828807 : if (trace_file != nullptr) {
3190 : PrintF(trace_file, V8PRIxPTR_FMT " ; [fp %c %3d] ", value,
3191 1200 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3192 2400 : Object(value)->ShortPrint(trace_file);
3193 : }
3194 : TranslatedValue translated_value =
3195 7828807 : TranslatedValue::NewTagged(this, Object(value));
3196 : frame.Add(translated_value);
3197 : return translated_value.GetChildrenCount();
3198 : }
3199 :
3200 : case Translation::INT32_STACK_SLOT: {
3201 : int slot_offset =
3202 321576 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3203 : uint32_t value = GetUInt32Slot(fp, slot_offset);
3204 321576 : if (trace_file != nullptr) {
3205 : PrintF(trace_file, "%d ; (int32) [fp %c %3d] ",
3206 : static_cast<int32_t>(value), slot_offset < 0 ? '-' : '+',
3207 0 : std::abs(slot_offset));
3208 : }
3209 321576 : TranslatedValue translated_value = TranslatedValue::NewInt32(this, value);
3210 : frame.Add(translated_value);
3211 : return translated_value.GetChildrenCount();
3212 : }
3213 :
3214 : case Translation::INT64_STACK_SLOT: {
3215 : int slot_offset =
3216 9 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3217 : uint64_t value = GetUInt64Slot(fp, slot_offset);
3218 9 : if (trace_file != nullptr) {
3219 : PrintF(trace_file, "%" V8PRIdPTR " ; (int64) [fp %c %3d] ",
3220 : static_cast<intptr_t>(value), slot_offset < 0 ? '-' : '+',
3221 0 : std::abs(slot_offset));
3222 : }
3223 9 : TranslatedValue translated_value = TranslatedValue::NewInt64(this, value);
3224 : frame.Add(translated_value);
3225 : return translated_value.GetChildrenCount();
3226 : }
3227 :
3228 : case Translation::UINT32_STACK_SLOT: {
3229 : int slot_offset =
3230 57160 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3231 : uint32_t value = GetUInt32Slot(fp, slot_offset);
3232 57160 : if (trace_file != nullptr) {
3233 : PrintF(trace_file, "%u ; (uint32) [fp %c %3d] ", value,
3234 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3235 : }
3236 : TranslatedValue translated_value =
3237 : TranslatedValue::NewUInt32(this, value);
3238 : frame.Add(translated_value);
3239 : return translated_value.GetChildrenCount();
3240 : }
3241 :
3242 : case Translation::BOOL_STACK_SLOT: {
3243 : int slot_offset =
3244 373 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3245 : uint32_t value = GetUInt32Slot(fp, slot_offset);
3246 373 : if (trace_file != nullptr) {
3247 : PrintF(trace_file, "%u ; (bool) [fp %c %3d] ", value,
3248 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3249 : }
3250 : TranslatedValue translated_value = TranslatedValue::NewBool(this, value);
3251 : frame.Add(translated_value);
3252 : return translated_value.GetChildrenCount();
3253 : }
3254 :
3255 : case Translation::FLOAT_STACK_SLOT: {
3256 : int slot_offset =
3257 197 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3258 197 : Float32 value = GetFloatSlot(fp, slot_offset);
3259 197 : if (trace_file != nullptr) {
3260 : PrintF(trace_file, "%e ; (float) [fp %c %3d] ", value.get_scalar(),
3261 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3262 : }
3263 : TranslatedValue translated_value = TranslatedValue::NewFloat(this, value);
3264 : frame.Add(translated_value);
3265 : return translated_value.GetChildrenCount();
3266 : }
3267 :
3268 : case Translation::DOUBLE_STACK_SLOT: {
3269 : int slot_offset =
3270 522337 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3271 522337 : Float64 value = GetDoubleSlot(fp, slot_offset);
3272 522337 : if (trace_file != nullptr) {
3273 : PrintF(trace_file, "%e ; (double) [fp %c %d] ", value.get_scalar(),
3274 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3275 : }
3276 : TranslatedValue translated_value =
3277 : TranslatedValue::NewDouble(this, value);
3278 : frame.Add(translated_value);
3279 : return translated_value.GetChildrenCount();
3280 : }
3281 :
3282 : case Translation::LITERAL: {
3283 11893845 : int literal_index = iterator->Next();
3284 11893845 : Object value = literal_array->get(literal_index);
3285 11893845 : if (trace_file != nullptr) {
3286 : PrintF(trace_file, V8PRIxPTR_FMT " ; (literal %2d) ", value->ptr(),
3287 1350 : literal_index);
3288 1350 : value->ShortPrint(trace_file);
3289 : }
3290 :
3291 : TranslatedValue translated_value =
3292 : TranslatedValue::NewTagged(this, value);
3293 : frame.Add(translated_value);
3294 : return translated_value.GetChildrenCount();
3295 : }
3296 : }
3297 :
3298 0 : FATAL("We should never get here - unexpected deopt info.");
3299 : }
3300 :
3301 3986511 : TranslatedState::TranslatedState(const JavaScriptFrame* frame) {
3302 1328837 : int deopt_index = Safepoint::kNoDeoptimizationIndex;
3303 : DeoptimizationData data =
3304 : static_cast<const OptimizedFrame*>(frame)->GetDeoptimizationData(
3305 1328837 : &deopt_index);
3306 : DCHECK(!data.is_null() && deopt_index != Safepoint::kNoDeoptimizationIndex);
3307 : TranslationIterator it(data->TranslationByteArray(),
3308 3986511 : data->TranslationIndex(deopt_index)->value());
3309 : Init(frame->isolate(), frame->fp(), &it, data->LiteralArray(),
3310 : nullptr /* registers */, nullptr /* trace file */,
3311 2657674 : frame->function()->shared()->internal_formal_parameter_count());
3312 1328837 : }
3313 :
3314 1470068 : void TranslatedState::Init(Isolate* isolate, Address input_frame_pointer,
3315 : TranslationIterator* iterator,
3316 : FixedArray literal_array, RegisterValues* registers,
3317 : FILE* trace_file, int formal_parameter_count) {
3318 : DCHECK(frames_.empty());
3319 :
3320 1470068 : formal_parameter_count_ = formal_parameter_count;
3321 1470068 : isolate_ = isolate;
3322 :
3323 : // Read out the 'header' translation.
3324 : Translation::Opcode opcode =
3325 1470068 : static_cast<Translation::Opcode>(iterator->Next());
3326 1470068 : CHECK(opcode == Translation::BEGIN);
3327 :
3328 1470068 : int count = iterator->Next();
3329 1470068 : frames_.reserve(count);
3330 1470068 : iterator->Next(); // Drop JS frames count.
3331 1470068 : int update_feedback_count = iterator->Next();
3332 1470068 : CHECK_GE(update_feedback_count, 0);
3333 1470068 : CHECK_LE(update_feedback_count, 1);
3334 :
3335 1470068 : if (update_feedback_count == 1) {
3336 1996 : ReadUpdateFeedback(iterator, literal_array, trace_file);
3337 : }
3338 :
3339 1470068 : std::stack<int> nested_counts;
3340 :
3341 : // Read the frames
3342 3474131 : for (int frame_index = 0; frame_index < count; frame_index++) {
3343 : // Read the frame descriptor.
3344 : frames_.push_back(CreateNextTranslatedFrame(
3345 4008126 : iterator, literal_array, input_frame_pointer, trace_file));
3346 2004063 : TranslatedFrame& frame = frames_.back();
3347 :
3348 : // Read the values.
3349 2004063 : int values_to_process = frame.GetValueCount();
3350 26895465 : while (values_to_process > 0 || !nested_counts.empty()) {
3351 20883276 : if (trace_file != nullptr) {
3352 3000 : if (nested_counts.empty()) {
3353 : // For top level values, print the value number.
3354 : PrintF(trace_file, " %3i: ",
3355 3000 : frame.GetValueCount() - values_to_process);
3356 : } else {
3357 : // Take care of indenting for nested values.
3358 0 : PrintF(trace_file, " ");
3359 0 : for (size_t j = 0; j < nested_counts.size(); j++) {
3360 0 : PrintF(trace_file, " ");
3361 : }
3362 : }
3363 : }
3364 :
3365 : int nested_count =
3366 : CreateNextTranslatedValue(frame_index, iterator, literal_array,
3367 20883276 : input_frame_pointer, registers, trace_file);
3368 :
3369 20883276 : if (trace_file != nullptr) {
3370 3000 : PrintF(trace_file, "\n");
3371 : }
3372 :
3373 : // Update the value count and resolve the nesting.
3374 20883276 : values_to_process--;
3375 20883276 : if (nested_count > 0) {
3376 : nested_counts.push(values_to_process);
3377 150928 : values_to_process = nested_count;
3378 : } else {
3379 23038267 : while (values_to_process == 0 && !nested_counts.empty()) {
3380 150928 : values_to_process = nested_counts.top();
3381 : nested_counts.pop();
3382 : }
3383 : }
3384 : }
3385 : }
3386 :
3387 1470068 : CHECK(!iterator->HasNext() ||
3388 : static_cast<Translation::Opcode>(iterator->Next()) ==
3389 : Translation::BEGIN);
3390 1470068 : }
3391 :
3392 1472064 : void TranslatedState::Prepare(Address stack_frame_pointer) {
3393 4944199 : for (auto& frame : frames_) frame.Handlify();
3394 :
3395 1470068 : if (!feedback_vector_.is_null()) {
3396 : feedback_vector_handle_ =
3397 1996 : Handle<FeedbackVector>(feedback_vector_, isolate());
3398 1996 : feedback_vector_ = FeedbackVector();
3399 : }
3400 1470068 : stack_frame_pointer_ = stack_frame_pointer;
3401 :
3402 1470068 : UpdateFromPreviouslyMaterializedObjects();
3403 1470068 : }
3404 :
3405 57473 : TranslatedValue* TranslatedState::GetValueByObjectIndex(int object_index) {
3406 57473 : CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
3407 57473 : TranslatedState::ObjectPosition pos = object_positions_[object_index];
3408 114946 : return &(frames_[pos.frame_index_].values_[pos.value_index_]);
3409 : }
3410 :
3411 88154 : Handle<Object> TranslatedState::InitializeObjectAt(TranslatedValue* slot) {
3412 44077 : slot = ResolveCapturedObject(slot);
3413 :
3414 : DisallowHeapAllocation no_allocation;
3415 44077 : if (slot->materialization_state() != TranslatedValue::kFinished) {
3416 43019 : std::stack<int> worklist;
3417 86038 : worklist.push(slot->object_index());
3418 : slot->mark_finished();
3419 :
3420 86890 : while (!worklist.empty()) {
3421 43871 : int index = worklist.top();
3422 : worklist.pop();
3423 43871 : InitializeCapturedObjectAt(index, &worklist, no_allocation);
3424 : }
3425 : }
3426 44077 : return slot->GetStorage();
3427 : }
3428 :
3429 43871 : void TranslatedState::InitializeCapturedObjectAt(
3430 : int object_index, std::stack<int>* worklist,
3431 : const DisallowHeapAllocation& no_allocation) {
3432 43871 : CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
3433 43871 : TranslatedState::ObjectPosition pos = object_positions_[object_index];
3434 43871 : int value_index = pos.value_index_;
3435 :
3436 43871 : TranslatedFrame* frame = &(frames_[pos.frame_index_]);
3437 87742 : TranslatedValue* slot = &(frame->values_[value_index]);
3438 43871 : value_index++;
3439 :
3440 43871 : CHECK_EQ(TranslatedValue::kFinished, slot->materialization_state());
3441 43871 : CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3442 :
3443 : // Ensure all fields are initialized.
3444 43871 : int children_init_index = value_index;
3445 1566586 : for (int i = 0; i < slot->GetChildrenCount(); i++) {
3446 : // If the field is an object that has not been initialized yet, queue it
3447 : // for initialization (and mark it as such).
3448 1479936 : TranslatedValue* child_slot = frame->ValueAt(children_init_index);
3449 739422 : if (child_slot->kind() == TranslatedValue::kCapturedObject ||
3450 : child_slot->kind() == TranslatedValue::kDuplicatedObject) {
3451 1092 : child_slot = ResolveCapturedObject(child_slot);
3452 1092 : if (child_slot->materialization_state() != TranslatedValue::kFinished) {
3453 : DCHECK_EQ(TranslatedValue::kAllocated,
3454 : child_slot->materialization_state());
3455 1704 : worklist->push(child_slot->object_index());
3456 : child_slot->mark_finished();
3457 : }
3458 : }
3459 739422 : SkipSlots(1, frame, &children_init_index);
3460 : }
3461 :
3462 : // Read the map.
3463 : // The map should never be materialized, so let us check we already have
3464 : // an existing object here.
3465 87742 : CHECK_EQ(frame->values_[value_index].kind(), TranslatedValue::kTagged);
3466 87742 : Handle<Map> map = Handle<Map>::cast(frame->values_[value_index].GetValue());
3467 87742 : CHECK(map->IsMap());
3468 43871 : value_index++;
3469 :
3470 : // Handle the special cases.
3471 43871 : switch (map->instance_type()) {
3472 : case MUTABLE_HEAP_NUMBER_TYPE:
3473 : case FIXED_DOUBLE_ARRAY_TYPE:
3474 39 : return;
3475 :
3476 : case FIXED_ARRAY_TYPE:
3477 : case AWAIT_CONTEXT_TYPE:
3478 : case BLOCK_CONTEXT_TYPE:
3479 : case CATCH_CONTEXT_TYPE:
3480 : case DEBUG_EVALUATE_CONTEXT_TYPE:
3481 : case EVAL_CONTEXT_TYPE:
3482 : case FUNCTION_CONTEXT_TYPE:
3483 : case MODULE_CONTEXT_TYPE:
3484 : case NATIVE_CONTEXT_TYPE:
3485 : case SCRIPT_CONTEXT_TYPE:
3486 : case WITH_CONTEXT_TYPE:
3487 : case OBJECT_BOILERPLATE_DESCRIPTION_TYPE:
3488 : case HASH_TABLE_TYPE:
3489 : case ORDERED_HASH_MAP_TYPE:
3490 : case ORDERED_HASH_SET_TYPE:
3491 : case NAME_DICTIONARY_TYPE:
3492 : case GLOBAL_DICTIONARY_TYPE:
3493 : case NUMBER_DICTIONARY_TYPE:
3494 : case SIMPLE_NUMBER_DICTIONARY_TYPE:
3495 : case STRING_TABLE_TYPE:
3496 : case PROPERTY_ARRAY_TYPE:
3497 : case SCRIPT_CONTEXT_TABLE_TYPE:
3498 : InitializeObjectWithTaggedFieldsAt(frame, &value_index, slot, map,
3499 869 : no_allocation);
3500 869 : break;
3501 :
3502 : default:
3503 42963 : CHECK(map->IsJSObjectMap());
3504 42963 : InitializeJSObjectAt(frame, &value_index, slot, map, no_allocation);
3505 42963 : break;
3506 : }
3507 43832 : CHECK_EQ(value_index, children_init_index);
3508 : }
3509 :
3510 88154 : void TranslatedState::EnsureObjectAllocatedAt(TranslatedValue* slot) {
3511 44077 : slot = ResolveCapturedObject(slot);
3512 :
3513 44077 : if (slot->materialization_state() == TranslatedValue::kUninitialized) {
3514 43019 : std::stack<int> worklist;
3515 86038 : worklist.push(slot->object_index());
3516 : slot->mark_allocated();
3517 :
3518 86851 : while (!worklist.empty()) {
3519 43832 : int index = worklist.top();
3520 : worklist.pop();
3521 43832 : EnsureCapturedObjectAllocatedAt(index, &worklist);
3522 : }
3523 : }
3524 44077 : }
3525 :
3526 31 : void TranslatedState::MaterializeFixedDoubleArray(TranslatedFrame* frame,
3527 : int* value_index,
3528 : TranslatedValue* slot,
3529 46 : Handle<Map> map) {
3530 62 : int length = Smi::cast(frame->values_[*value_index].GetRawValue())->value();
3531 31 : (*value_index)++;
3532 : Handle<FixedDoubleArray> array = Handle<FixedDoubleArray>::cast(
3533 31 : isolate()->factory()->NewFixedDoubleArray(length));
3534 31 : CHECK_GT(length, 0);
3535 86 : for (int i = 0; i < length; i++) {
3536 172 : CHECK_NE(TranslatedValue::kCapturedObject,
3537 : frame->values_[*value_index].kind());
3538 172 : Handle<Object> value = frame->values_[*value_index].GetValue();
3539 172 : if (value->IsNumber()) {
3540 213 : array->set(i, value->Number());
3541 : } else {
3542 15 : CHECK(value.is_identical_to(isolate()->factory()->the_hole_value()));
3543 : array->set_the_hole(isolate(), i);
3544 : }
3545 86 : (*value_index)++;
3546 : }
3547 : slot->set_storage(array);
3548 31 : }
3549 :
3550 8 : void TranslatedState::MaterializeMutableHeapNumber(TranslatedFrame* frame,
3551 : int* value_index,
3552 8 : TranslatedValue* slot) {
3553 16 : CHECK_NE(TranslatedValue::kCapturedObject,
3554 : frame->values_[*value_index].kind());
3555 16 : Handle<Object> value = frame->values_[*value_index].GetValue();
3556 16 : CHECK(value->IsNumber());
3557 : Handle<MutableHeapNumber> box =
3558 16 : isolate()->factory()->NewMutableHeapNumber(value->Number());
3559 8 : (*value_index)++;
3560 : slot->set_storage(box);
3561 8 : }
3562 :
3563 : namespace {
3564 :
3565 : enum DoubleStorageKind : uint8_t {
3566 : kStoreTagged,
3567 : kStoreUnboxedDouble,
3568 : kStoreMutableHeapNumber,
3569 : };
3570 :
3571 : } // namespace
3572 :
3573 2087350 : void TranslatedState::SkipSlots(int slots_to_skip, TranslatedFrame* frame,
3574 : int* value_index) {
3575 7992634 : while (slots_to_skip > 0) {
3576 7635868 : TranslatedValue* slot = &(frame->values_[*value_index]);
3577 3817934 : (*value_index)++;
3578 3817934 : slots_to_skip--;
3579 :
3580 3817934 : if (slot->kind() == TranslatedValue::kCapturedObject) {
3581 2132 : slots_to_skip += slot->GetChildrenCount();
3582 : }
3583 : }
3584 2087350 : }
3585 :
3586 43832 : void TranslatedState::EnsureCapturedObjectAllocatedAt(
3587 860 : int object_index, std::stack<int>* worklist) {
3588 43832 : CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
3589 43832 : TranslatedState::ObjectPosition pos = object_positions_[object_index];
3590 43832 : int value_index = pos.value_index_;
3591 :
3592 43832 : TranslatedFrame* frame = &(frames_[pos.frame_index_]);
3593 87664 : TranslatedValue* slot = &(frame->values_[value_index]);
3594 43832 : value_index++;
3595 :
3596 43832 : CHECK_EQ(TranslatedValue::kAllocated, slot->materialization_state());
3597 43832 : CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3598 :
3599 : // Read the map.
3600 : // The map should never be materialized, so let us check we already have
3601 : // an existing object here.
3602 43832 : CHECK_EQ(frame->values_[value_index].kind(), TranslatedValue::kTagged);
3603 87664 : Handle<Map> map = Handle<Map>::cast(frame->values_[value_index].GetValue());
3604 87664 : CHECK(map->IsMap());
3605 43832 : value_index++;
3606 :
3607 : // Handle the special cases.
3608 43832 : switch (map->instance_type()) {
3609 : case FIXED_DOUBLE_ARRAY_TYPE:
3610 : // Materialize (i.e. allocate&initialize) the array and return since
3611 : // there is no need to process the children.
3612 31 : return MaterializeFixedDoubleArray(frame, &value_index, slot, map);
3613 :
3614 : case MUTABLE_HEAP_NUMBER_TYPE:
3615 : // Materialize (i.e. allocate&initialize) the heap number and return.
3616 : // There is no need to process the children.
3617 8 : return MaterializeMutableHeapNumber(frame, &value_index, slot);
3618 :
3619 : case FIXED_ARRAY_TYPE:
3620 : case SCRIPT_CONTEXT_TABLE_TYPE:
3621 : case AWAIT_CONTEXT_TYPE:
3622 : case BLOCK_CONTEXT_TYPE:
3623 : case CATCH_CONTEXT_TYPE:
3624 : case DEBUG_EVALUATE_CONTEXT_TYPE:
3625 : case EVAL_CONTEXT_TYPE:
3626 : case FUNCTION_CONTEXT_TYPE:
3627 : case MODULE_CONTEXT_TYPE:
3628 : case NATIVE_CONTEXT_TYPE:
3629 : case SCRIPT_CONTEXT_TYPE:
3630 : case WITH_CONTEXT_TYPE:
3631 : case HASH_TABLE_TYPE:
3632 : case ORDERED_HASH_MAP_TYPE:
3633 : case ORDERED_HASH_SET_TYPE:
3634 : case NAME_DICTIONARY_TYPE:
3635 : case GLOBAL_DICTIONARY_TYPE:
3636 : case NUMBER_DICTIONARY_TYPE:
3637 : case SIMPLE_NUMBER_DICTIONARY_TYPE:
3638 : case STRING_TABLE_TYPE: {
3639 : // Check we have the right size.
3640 : int array_length =
3641 1660 : Smi::cast(frame->values_[value_index].GetRawValue())->value();
3642 :
3643 : int instance_size = FixedArray::SizeFor(array_length);
3644 830 : CHECK_EQ(instance_size, slot->GetChildrenCount() * kTaggedSize);
3645 :
3646 : // Canonicalize empty fixed array.
3647 830 : if (*map == ReadOnlyRoots(isolate()).empty_fixed_array()->map() &&
3648 : array_length == 0) {
3649 : slot->set_storage(isolate()->factory()->empty_fixed_array());
3650 : } else {
3651 800 : slot->set_storage(AllocateStorageFor(slot));
3652 : }
3653 :
3654 : // Make sure all the remaining children (after the map) are allocated.
3655 : return EnsureChildrenAllocated(slot->GetChildrenCount() - 1, frame,
3656 830 : &value_index, worklist);
3657 : }
3658 :
3659 : case PROPERTY_ARRAY_TYPE: {
3660 : // Check we have the right size.
3661 : int length_or_hash =
3662 0 : Smi::cast(frame->values_[value_index].GetRawValue())->value();
3663 : int array_length = PropertyArray::LengthField::decode(length_or_hash);
3664 : int instance_size = PropertyArray::SizeFor(array_length);
3665 0 : CHECK_EQ(instance_size, slot->GetChildrenCount() * kTaggedSize);
3666 :
3667 0 : slot->set_storage(AllocateStorageFor(slot));
3668 : // Make sure all the remaining children (after the map) are allocated.
3669 : return EnsureChildrenAllocated(slot->GetChildrenCount() - 1, frame,
3670 0 : &value_index, worklist);
3671 : }
3672 :
3673 : default:
3674 42963 : CHECK(map->IsJSObjectMap());
3675 42963 : EnsureJSObjectAllocated(slot, map);
3676 85926 : TranslatedValue* properties_slot = &(frame->values_[value_index]);
3677 42963 : value_index++;
3678 42963 : if (properties_slot->kind() == TranslatedValue::kCapturedObject) {
3679 : // If we are materializing the property array, make sure we put
3680 : // the mutable heap numbers at the right places.
3681 39 : EnsurePropertiesAllocatedAndMarked(properties_slot, map);
3682 : EnsureChildrenAllocated(properties_slot->GetChildrenCount(), frame,
3683 39 : &value_index, worklist);
3684 : }
3685 : // Make sure all the remaining children (after the map and properties) are
3686 : // allocated.
3687 : return EnsureChildrenAllocated(slot->GetChildrenCount() - 2, frame,
3688 42963 : &value_index, worklist);
3689 : }
3690 : UNREACHABLE();
3691 : }
3692 :
3693 43832 : void TranslatedState::EnsureChildrenAllocated(int count, TranslatedFrame* frame,
3694 : int* value_index,
3695 : std::stack<int>* worklist) {
3696 : // Ensure all children are allocated.
3697 696334 : for (int i = 0; i < count; i++) {
3698 : // If the field is an object that has not been allocated yet, queue it
3699 : // for initialization (and mark it as such).
3700 1306057 : TranslatedValue* child_slot = frame->ValueAt(*value_index);
3701 652502 : if (child_slot->kind() == TranslatedValue::kCapturedObject ||
3702 : child_slot->kind() == TranslatedValue::kDuplicatedObject) {
3703 1053 : child_slot = ResolveCapturedObject(child_slot);
3704 1053 : if (child_slot->materialization_state() ==
3705 : TranslatedValue::kUninitialized) {
3706 1626 : worklist->push(child_slot->object_index());
3707 : child_slot->mark_allocated();
3708 : }
3709 : } else {
3710 : // Make sure the simple values (heap numbers, etc.) are properly
3711 : // initialized.
3712 651449 : child_slot->MaterializeSimple();
3713 : }
3714 652502 : SkipSlots(1, frame, value_index);
3715 : }
3716 43832 : }
3717 :
3718 39 : void TranslatedState::EnsurePropertiesAllocatedAndMarked(
3719 78 : TranslatedValue* properties_slot, Handle<Map> map) {
3720 39 : CHECK_EQ(TranslatedValue::kUninitialized,
3721 : properties_slot->materialization_state());
3722 :
3723 39 : Handle<ByteArray> object_storage = AllocateStorageFor(properties_slot);
3724 : properties_slot->mark_allocated();
3725 : properties_slot->set_storage(object_storage);
3726 :
3727 : // Set markers for the double properties.
3728 78 : Handle<DescriptorArray> descriptors(map->instance_descriptors(), isolate());
3729 : int field_count = map->NumberOfOwnDescriptors();
3730 86 : for (int i = 0; i < field_count; i++) {
3731 47 : FieldIndex index = FieldIndex::ForDescriptor(*map, i);
3732 102 : if (descriptors->GetDetails(i).representation().IsDouble() &&
3733 : !index.is_inobject()) {
3734 8 : CHECK(!map->IsUnboxedDoubleField(index));
3735 : int outobject_index = index.outobject_array_index();
3736 8 : int array_index = outobject_index * kTaggedSize;
3737 : object_storage->set(array_index, kStoreMutableHeapNumber);
3738 : }
3739 : }
3740 39 : }
3741 :
3742 87604 : Handle<ByteArray> TranslatedState::AllocateStorageFor(TranslatedValue* slot) {
3743 : int allocate_size =
3744 43802 : ByteArray::LengthFor(slot->GetChildrenCount() * kTaggedSize);
3745 : // It is important to allocate all the objects tenured so that the marker
3746 : // does not visit them.
3747 : Handle<ByteArray> object_storage =
3748 43802 : isolate()->factory()->NewByteArray(allocate_size, TENURED);
3749 10513108 : for (int i = 0; i < object_storage->length(); i++) {
3750 : object_storage->set(i, kStoreTagged);
3751 : }
3752 43802 : return object_storage;
3753 : }
3754 :
3755 42963 : void TranslatedState::EnsureJSObjectAllocated(TranslatedValue* slot,
3756 42963 : Handle<Map> map) {
3757 85926 : CHECK_EQ(map->instance_size(), slot->GetChildrenCount() * kTaggedSize);
3758 :
3759 42963 : Handle<ByteArray> object_storage = AllocateStorageFor(slot);
3760 : // Now we handle the interesting (JSObject) case.
3761 85926 : Handle<DescriptorArray> descriptors(map->instance_descriptors(), isolate());
3762 : int field_count = map->NumberOfOwnDescriptors();
3763 :
3764 : // Set markers for the double properties.
3765 72884 : for (int i = 0; i < field_count; i++) {
3766 29921 : FieldIndex index = FieldIndex::ForDescriptor(*map, i);
3767 59873 : if (descriptors->GetDetails(i).representation().IsDouble() &&
3768 : index.is_inobject()) {
3769 15 : CHECK_GE(index.index(), FixedArray::kHeaderSize / kTaggedSize);
3770 15 : int array_index = index.index() * kTaggedSize - FixedArray::kHeaderSize;
3771 30 : uint8_t marker = map->IsUnboxedDoubleField(index)
3772 : ? kStoreUnboxedDouble
3773 15 : : kStoreMutableHeapNumber;
3774 : object_storage->set(array_index, marker);
3775 : }
3776 : }
3777 : slot->set_storage(object_storage);
3778 42963 : }
3779 :
3780 695426 : Handle<Object> TranslatedState::GetValueAndAdvance(TranslatedFrame* frame,
3781 : int* value_index) {
3782 2086278 : TranslatedValue* slot = frame->ValueAt(*value_index);
3783 695426 : SkipSlots(1, frame, value_index);
3784 695426 : if (slot->kind() == TranslatedValue::kDuplicatedObject) {
3785 459 : slot = ResolveCapturedObject(slot);
3786 : }
3787 695426 : CHECK_NE(TranslatedValue::kUninitialized, slot->materialization_state());
3788 695426 : return slot->GetStorage();
3789 : }
3790 :
3791 42963 : void TranslatedState::InitializeJSObjectAt(
3792 : TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
3793 42963 : Handle<Map> map, const DisallowHeapAllocation& no_allocation) {
3794 42963 : Handle<HeapObject> object_storage = Handle<HeapObject>::cast(slot->storage_);
3795 : DCHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3796 :
3797 : // The object should have at least a map and some payload.
3798 42963 : CHECK_GE(slot->GetChildrenCount(), 2);
3799 :
3800 : // Notify the concurrent marker about the layout change.
3801 : isolate()->heap()->NotifyObjectLayoutChange(
3802 85926 : *object_storage, slot->GetChildrenCount() * kTaggedSize, no_allocation);
3803 :
3804 : // Fill the property array field.
3805 : {
3806 42963 : Handle<Object> properties = GetValueAndAdvance(frame, value_index);
3807 42963 : WRITE_FIELD(*object_storage, JSObject::kPropertiesOrHashOffset,
3808 : *properties);
3809 85926 : WRITE_BARRIER(*object_storage, JSObject::kPropertiesOrHashOffset,
3810 : *properties);
3811 : }
3812 :
3813 : // For all the other fields we first look at the fixed array and check the
3814 : // marker to see if we store an unboxed double.
3815 : DCHECK_EQ(kTaggedSize, JSObject::kPropertiesOrHashOffset);
3816 235282 : for (int i = 2; i < slot->GetChildrenCount(); i++) {
3817 : // Initialize and extract the value from its slot.
3818 74678 : Handle<Object> field_value = GetValueAndAdvance(frame, value_index);
3819 :
3820 : // Read out the marker and ensure the field is consistent with
3821 : // what the markers in the storage say (note that all heap numbers
3822 : // should be fully initialized by now).
3823 74678 : int offset = i * kTaggedSize;
3824 74678 : uint8_t marker = READ_UINT8_FIELD(*object_storage, offset);
3825 74678 : if (marker == kStoreUnboxedDouble) {
3826 : double double_field_value;
3827 30 : if (field_value->IsSmi()) {
3828 4 : double_field_value = Smi::cast(*field_value)->value();
3829 : } else {
3830 22 : CHECK(field_value->IsHeapNumber());
3831 : double_field_value = HeapNumber::cast(*field_value)->value();
3832 : }
3833 15 : WRITE_DOUBLE_FIELD(*object_storage, offset, double_field_value);
3834 74663 : } else if (marker == kStoreMutableHeapNumber) {
3835 0 : CHECK(field_value->IsMutableHeapNumber());
3836 0 : WRITE_FIELD(*object_storage, offset, *field_value);
3837 0 : WRITE_BARRIER(*object_storage, offset, *field_value);
3838 : } else {
3839 74663 : CHECK_EQ(kStoreTagged, marker);
3840 74663 : WRITE_FIELD(*object_storage, offset, *field_value);
3841 149326 : WRITE_BARRIER(*object_storage, offset, *field_value);
3842 : }
3843 : }
3844 42963 : object_storage->synchronized_set_map(*map);
3845 42963 : }
3846 :
3847 869 : void TranslatedState::InitializeObjectWithTaggedFieldsAt(
3848 : TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
3849 1708 : Handle<Map> map, const DisallowHeapAllocation& no_allocation) {
3850 869 : Handle<HeapObject> object_storage = Handle<HeapObject>::cast(slot->storage_);
3851 :
3852 : // Skip the writes if we already have the canonical empty fixed array.
3853 869 : if (*object_storage == ReadOnlyRoots(isolate()).empty_fixed_array()) {
3854 30 : CHECK_EQ(2, slot->GetChildrenCount());
3855 30 : Handle<Object> length_value = GetValueAndAdvance(frame, value_index);
3856 90 : CHECK_EQ(*length_value, Smi::FromInt(0));
3857 869 : return;
3858 : }
3859 :
3860 : // Notify the concurrent marker about the layout change.
3861 : isolate()->heap()->NotifyObjectLayoutChange(
3862 1678 : *object_storage, slot->GetChildrenCount() * kTaggedSize, no_allocation);
3863 :
3864 : // Write the fields to the object.
3865 1157188 : for (int i = 1; i < slot->GetChildrenCount(); i++) {
3866 577755 : Handle<Object> field_value = GetValueAndAdvance(frame, value_index);
3867 577755 : int offset = i * kTaggedSize;
3868 577755 : uint8_t marker = READ_UINT8_FIELD(*object_storage, offset);
3869 577755 : if (i > 1 && marker == kStoreMutableHeapNumber) {
3870 16 : CHECK(field_value->IsMutableHeapNumber());
3871 : } else {
3872 577747 : CHECK(marker == kStoreTagged || i == 1);
3873 1155494 : CHECK(!field_value->IsMutableHeapNumber());
3874 : }
3875 :
3876 577755 : WRITE_FIELD(*object_storage, offset, *field_value);
3877 1155510 : WRITE_BARRIER(*object_storage, offset, *field_value);
3878 : }
3879 :
3880 839 : object_storage->synchronized_set_map(*map);
3881 : }
3882 :
3883 238989 : TranslatedValue* TranslatedState::ResolveCapturedObject(TranslatedValue* slot) {
3884 238989 : while (slot->kind() == TranslatedValue::kDuplicatedObject) {
3885 57473 : slot = GetValueByObjectIndex(slot->object_index());
3886 : }
3887 90758 : CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3888 90758 : return slot;
3889 : }
3890 :
3891 45 : TranslatedFrame* TranslatedState::GetFrameFromJSFrameIndex(int jsframe_index) {
3892 180 : for (size_t i = 0; i < frames_.size(); i++) {
3893 287 : if (frames_[i].kind() == TranslatedFrame::kInterpretedFunction ||
3894 90 : frames_[i].kind() == TranslatedFrame::kJavaScriptBuiltinContinuation ||
3895 : frames_[i].kind() ==
3896 : TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch) {
3897 90 : if (jsframe_index > 0) {
3898 45 : jsframe_index--;
3899 : } else {
3900 : return &(frames_[i]);
3901 : }
3902 : }
3903 : }
3904 : return nullptr;
3905 : }
3906 :
3907 2592 : TranslatedFrame* TranslatedState::GetArgumentsInfoFromJSFrameIndex(
3908 : int jsframe_index, int* args_count) {
3909 27990 : for (size_t i = 0; i < frames_.size(); i++) {
3910 49961 : if (frames_[i].kind() == TranslatedFrame::kInterpretedFunction ||
3911 19372 : frames_[i].kind() == TranslatedFrame::kJavaScriptBuiltinContinuation ||
3912 : frames_[i].kind() ==
3913 : TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch) {
3914 8618 : if (jsframe_index > 0) {
3915 6026 : jsframe_index--;
3916 : } else {
3917 : // We have the JS function frame, now check if it has arguments
3918 : // adaptor.
3919 5184 : if (i > 0 &&
3920 5184 : frames_[i - 1].kind() == TranslatedFrame::kArgumentsAdaptor) {
3921 2186 : *args_count = frames_[i - 1].height();
3922 1093 : return &(frames_[i - 1]);
3923 : }
3924 : *args_count =
3925 1499 : frames_[i].shared_info()->internal_formal_parameter_count() + 1;
3926 1499 : return &(frames_[i]);
3927 : }
3928 : }
3929 : }
3930 : return nullptr;
3931 : }
3932 :
3933 171 : void TranslatedState::StoreMaterializedValuesAndDeopt(JavaScriptFrame* frame) {
3934 : MaterializedObjectStore* materialized_store =
3935 171 : isolate_->materialized_object_store();
3936 : Handle<FixedArray> previously_materialized_objects =
3937 171 : materialized_store->Get(stack_frame_pointer_);
3938 :
3939 171 : Handle<Object> marker = isolate_->factory()->arguments_marker();
3940 :
3941 171 : int length = static_cast<int>(object_positions_.size());
3942 : bool new_store = false;
3943 171 : if (previously_materialized_objects.is_null()) {
3944 : previously_materialized_objects =
3945 143 : isolate_->factory()->NewFixedArray(length, TENURED);
3946 531 : for (int i = 0; i < length; i++) {
3947 388 : previously_materialized_objects->set(i, *marker);
3948 : }
3949 : new_store = true;
3950 : }
3951 :
3952 171 : CHECK_EQ(length, previously_materialized_objects->length());
3953 :
3954 : bool value_changed = false;
3955 458 : for (int i = 0; i < length; i++) {
3956 458 : TranslatedState::ObjectPosition pos = object_positions_[i];
3957 : TranslatedValue* value_info =
3958 601 : &(frames_[pos.frame_index_].values_[pos.value_index_]);
3959 :
3960 458 : CHECK(value_info->IsMaterializedObject());
3961 :
3962 : // Skip duplicate objects (i.e., those that point to some
3963 : // other object id).
3964 458 : if (value_info->object_index() != i) continue;
3965 :
3966 304 : Handle<Object> value(value_info->GetRawValue(), isolate_);
3967 :
3968 304 : if (!value.is_identical_to(marker)) {
3969 269 : if (previously_materialized_objects->get(i) == *marker) {
3970 227 : previously_materialized_objects->set(i, *value);
3971 : value_changed = true;
3972 : } else {
3973 42 : CHECK(previously_materialized_objects->get(i) == *value);
3974 : }
3975 : }
3976 : }
3977 171 : if (new_store && value_changed) {
3978 : materialized_store->Set(stack_frame_pointer_,
3979 143 : previously_materialized_objects);
3980 143 : CHECK_EQ(frames_[0].kind(), TranslatedFrame::kInterpretedFunction);
3981 286 : CHECK_EQ(frame->function(), frames_[0].front().GetRawValue());
3982 143 : Deoptimizer::DeoptimizeFunction(frame->function(), frame->LookupCode());
3983 : }
3984 171 : }
3985 :
3986 1470068 : void TranslatedState::UpdateFromPreviouslyMaterializedObjects() {
3987 : MaterializedObjectStore* materialized_store =
3988 1470068 : isolate_->materialized_object_store();
3989 : Handle<FixedArray> previously_materialized_objects =
3990 1470068 : materialized_store->Get(stack_frame_pointer_);
3991 :
3992 : // If we have no previously materialized objects, there is nothing to do.
3993 2940136 : if (previously_materialized_objects.is_null()) return;
3994 :
3995 199 : Handle<Object> marker = isolate_->factory()->arguments_marker();
3996 :
3997 199 : int length = static_cast<int>(object_positions_.size());
3998 199 : CHECK_EQ(length, previously_materialized_objects->length());
3999 :
4000 528 : for (int i = 0; i < length; i++) {
4001 : // For a previously materialized objects, inject their value into the
4002 : // translated values.
4003 528 : if (previously_materialized_objects->get(i) != *marker) {
4004 311 : TranslatedState::ObjectPosition pos = object_positions_[i];
4005 : TranslatedValue* value_info =
4006 311 : &(frames_[pos.frame_index_].values_[pos.value_index_]);
4007 311 : CHECK(value_info->IsMaterializedObject());
4008 :
4009 311 : if (value_info->kind() == TranslatedValue::kCapturedObject) {
4010 : value_info->set_initialized_storage(
4011 311 : Handle<Object>(previously_materialized_objects->get(i), isolate_));
4012 : }
4013 : }
4014 : }
4015 : }
4016 :
4017 0 : void TranslatedState::VerifyMaterializedObjects() {
4018 : #if VERIFY_HEAP
4019 : int length = static_cast<int>(object_positions_.size());
4020 : for (int i = 0; i < length; i++) {
4021 : TranslatedValue* slot = GetValueByObjectIndex(i);
4022 : if (slot->kind() == TranslatedValue::kCapturedObject) {
4023 : CHECK_EQ(slot, GetValueByObjectIndex(slot->object_index()));
4024 : if (slot->materialization_state() == TranslatedValue::kFinished) {
4025 : slot->GetStorage()->ObjectVerify(isolate());
4026 : } else {
4027 : CHECK_EQ(slot->materialization_state(),
4028 : TranslatedValue::kUninitialized);
4029 : }
4030 : }
4031 : }
4032 : #endif
4033 0 : }
4034 :
4035 143227 : bool TranslatedState::DoUpdateFeedback() {
4036 141231 : if (!feedback_vector_handle_.is_null()) {
4037 1996 : CHECK(!feedback_slot_.IsInvalid());
4038 1996 : isolate()->CountUsage(v8::Isolate::kDeoptimizerDisableSpeculation);
4039 1996 : FeedbackNexus nexus(feedback_vector_handle_, feedback_slot_);
4040 1996 : nexus.SetSpeculationMode(SpeculationMode::kDisallowSpeculation);
4041 : return true;
4042 : }
4043 : return false;
4044 : }
4045 :
4046 1996 : void TranslatedState::ReadUpdateFeedback(TranslationIterator* iterator,
4047 : FixedArray literal_array,
4048 : FILE* trace_file) {
4049 3992 : CHECK_EQ(Translation::UPDATE_FEEDBACK, iterator->Next());
4050 3992 : feedback_vector_ = FeedbackVector::cast(literal_array->get(iterator->Next()));
4051 1996 : feedback_slot_ = FeedbackSlot(iterator->Next());
4052 1996 : if (trace_file != nullptr) {
4053 : PrintF(trace_file, " reading FeedbackVector (slot %d)\n",
4054 0 : feedback_slot_.ToInt());
4055 : }
4056 1996 : }
4057 :
4058 : } // namespace internal
4059 183867 : } // namespace v8
4060 :
4061 : // Undefine the heap manipulation macros.
4062 : #include "src/objects/object-macros-undef.h"
|
