Line data Source code
1 : // Copyright 2013 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/deoptimizer.h"
6 :
7 : #include <memory>
8 :
9 : #include "src/accessors.h"
10 : #include "src/assembler-inl.h"
11 : #include "src/ast/prettyprinter.h"
12 : #include "src/callable.h"
13 : #include "src/counters.h"
14 : #include "src/disasm.h"
15 : #include "src/frames-inl.h"
16 : #include "src/global-handles.h"
17 : #include "src/heap/heap-inl.h"
18 : #include "src/interpreter/interpreter.h"
19 : #include "src/log.h"
20 : #include "src/macro-assembler.h"
21 : #include "src/objects/debug-objects-inl.h"
22 : #include "src/objects/heap-number-inl.h"
23 : #include "src/objects/smi.h"
24 : #include "src/register-configuration.h"
25 : #include "src/tracing/trace-event.h"
26 : #include "src/v8.h"
27 : #include "src/v8threads.h"
28 :
29 : // Has to be the last include (doesn't have include guards)
30 : #include "src/objects/object-macros.h"
31 :
32 : namespace v8 {
33 : namespace internal {
34 :
35 : // {FrameWriter} offers a stack writer abstraction for writing
36 : // FrameDescriptions. The main service the class provides is managing
37 : // {top_offset_}, i.e. the offset of the next slot to write to.
38 : class FrameWriter {
39 : public:
40 : static const int NO_INPUT_INDEX = -1;
41 : FrameWriter(Deoptimizer* deoptimizer, FrameDescription* frame,
42 : CodeTracer::Scope* trace_scope)
43 : : deoptimizer_(deoptimizer),
44 : frame_(frame),
45 : trace_scope_(trace_scope),
46 101930 : top_offset_(frame->GetFrameSize()) {}
47 :
48 17754 : void PushRawValue(intptr_t value, const char* debug_hint) {
49 : PushValue(value);
50 :
51 17754 : if (trace_scope_ != nullptr) {
52 0 : DebugPrintOutputValue(value, debug_hint);
53 : }
54 17754 : }
55 :
56 679472 : void PushRawObject(Object obj, const char* debug_hint) {
57 679472 : intptr_t value = obj->ptr();
58 : PushValue(value);
59 679472 : if (trace_scope_ != nullptr) {
60 2880 : DebugPrintOutputObject(obj, top_offset_, debug_hint);
61 : }
62 679472 : }
63 :
64 50965 : void PushCallerPc(intptr_t pc) {
65 50965 : top_offset_ -= kPCOnStackSize;
66 50965 : frame_->SetCallerPc(top_offset_, pc);
67 50965 : DebugPrintOutputValue(pc, "caller's pc\n");
68 50965 : }
69 :
70 50965 : void PushCallerFp(intptr_t fp) {
71 50965 : top_offset_ -= kFPOnStackSize;
72 50965 : frame_->SetCallerFp(top_offset_, fp);
73 50965 : DebugPrintOutputValue(fp, "caller's fp\n");
74 50965 : }
75 :
76 : void PushCallerConstantPool(intptr_t cp) {
77 : top_offset_ -= kSystemPointerSize;
78 : frame_->SetCallerConstantPool(top_offset_, cp);
79 : DebugPrintOutputValue(cp, "caller's constant_pool\n");
80 : }
81 :
82 571534 : void PushTranslatedValue(const TranslatedFrame::iterator& iterator,
83 : const char* debug_hint = "") {
84 571534 : Object obj = iterator->GetRawValue();
85 :
86 571534 : PushRawObject(obj, debug_hint);
87 :
88 571534 : if (trace_scope_) {
89 2400 : PrintF(trace_scope_->file(), " (input #%d)\n", iterator.input_index());
90 : }
91 :
92 1143068 : deoptimizer_->QueueValueForMaterialization(output_address(top_offset_), obj,
93 571534 : iterator);
94 571534 : }
95 :
96 : unsigned top_offset() const { return top_offset_; }
97 :
98 : private:
99 : void PushValue(intptr_t value) {
100 : CHECK_GE(top_offset_, 0);
101 697226 : top_offset_ -= kSystemPointerSize;
102 : frame_->SetFrameSlot(top_offset_, value);
103 : }
104 :
105 : Address output_address(unsigned output_offset) {
106 : Address output_address =
107 574894 : static_cast<Address>(frame_->GetTop()) + output_offset;
108 : return output_address;
109 : }
110 :
111 101930 : void DebugPrintOutputValue(intptr_t value, const char* debug_hint = "") {
112 101930 : if (trace_scope_ != nullptr) {
113 480 : PrintF(trace_scope_->file(),
114 : " " V8PRIxPTR_FMT ": [top + %3d] <- " V8PRIxPTR_FMT " ; %s",
115 480 : output_address(top_offset_), top_offset_, value, debug_hint);
116 : }
117 101930 : }
118 :
119 2880 : void DebugPrintOutputObject(Object obj, unsigned output_offset,
120 : const char* debug_hint = "") {
121 2880 : if (trace_scope_ != nullptr) {
122 : PrintF(trace_scope_->file(), " " V8PRIxPTR_FMT ": [top + %3d] <- ",
123 2880 : output_address(output_offset), output_offset);
124 2880 : if (obj->IsSmi()) {
125 312 : PrintF(V8PRIxPTR_FMT " <Smi %d>", obj->ptr(), Smi::cast(obj)->value());
126 : } else {
127 5136 : obj->ShortPrint(trace_scope_->file());
128 : }
129 5760 : PrintF(trace_scope_->file(), " ; %s", debug_hint);
130 : }
131 2880 : }
132 :
133 : Deoptimizer* deoptimizer_;
134 : FrameDescription* frame_;
135 : CodeTracer::Scope* trace_scope_;
136 : unsigned top_offset_;
137 : };
138 :
139 61534 : DeoptimizerData::DeoptimizerData(Heap* heap) : heap_(heap), current_(nullptr) {
140 : Code* start = &deopt_entry_code_[0];
141 : Code* end = &deopt_entry_code_[DeoptimizerData::kLastDeoptimizeKind + 1];
142 61534 : heap_->RegisterStrongRoots(FullObjectSlot(start), FullObjectSlot(end));
143 61534 : }
144 :
145 :
146 123037 : DeoptimizerData::~DeoptimizerData() {
147 : Code* start = &deopt_entry_code_[0];
148 61518 : heap_->UnregisterStrongRoots(FullObjectSlot(start));
149 61519 : }
150 :
151 0 : Code DeoptimizerData::deopt_entry_code(DeoptimizeKind kind) {
152 4790309 : return deopt_entry_code_[static_cast<int>(kind)];
153 : }
154 :
155 0 : void DeoptimizerData::set_deopt_entry_code(DeoptimizeKind kind, Code code) {
156 44028 : deopt_entry_code_[static_cast<int>(kind)] = code;
157 0 : }
158 :
159 45770 : Code Deoptimizer::FindDeoptimizingCode(Address addr) {
160 45770 : if (function_->IsHeapObject()) {
161 : // Search all deoptimizing code in the native context of the function.
162 45770 : Isolate* isolate = isolate_;
163 45770 : Context native_context = function_->context()->native_context();
164 45770 : Object element = native_context->DeoptimizedCodeListHead();
165 16802206 : while (!element->IsUndefined(isolate)) {
166 8398984 : Code code = Code::cast(element);
167 8398984 : CHECK(code->kind() == Code::OPTIMIZED_FUNCTION);
168 8398984 : if (code->contains(addr)) return code;
169 8378218 : element = code->next_code_link();
170 : }
171 : }
172 25004 : return Code();
173 : }
174 :
175 :
176 : // We rely on this function not causing a GC. It is called from generated code
177 : // without having a real stack frame in place.
178 45770 : Deoptimizer* Deoptimizer::New(Address raw_function, DeoptimizeKind kind,
179 : unsigned bailout_id, Address from,
180 : int fp_to_sp_delta, Isolate* isolate) {
181 : JSFunction function = JSFunction::cast(Object(raw_function));
182 : Deoptimizer* deoptimizer = new Deoptimizer(isolate, function, kind,
183 91540 : bailout_id, from, fp_to_sp_delta);
184 45770 : CHECK_NULL(isolate->deoptimizer_data()->current_);
185 45770 : isolate->deoptimizer_data()->current_ = deoptimizer;
186 45770 : return deoptimizer;
187 : }
188 :
189 45770 : Deoptimizer* Deoptimizer::Grab(Isolate* isolate) {
190 45770 : Deoptimizer* result = isolate->deoptimizer_data()->current_;
191 45770 : CHECK_NOT_NULL(result);
192 45770 : result->DeleteFrameDescriptions();
193 45770 : isolate->deoptimizer_data()->current_ = nullptr;
194 45770 : return result;
195 : }
196 :
197 24974 : DeoptimizedFrameInfo* Deoptimizer::DebuggerInspectableFrame(
198 : JavaScriptFrame* frame,
199 : int jsframe_index,
200 : Isolate* isolate) {
201 49948 : CHECK(frame->is_optimized());
202 :
203 24974 : TranslatedState translated_values(frame);
204 24974 : translated_values.Prepare(frame->fp());
205 :
206 : TranslatedState::iterator frame_it = translated_values.end();
207 : int counter = jsframe_index;
208 83076 : for (auto it = translated_values.begin(); it != translated_values.end();
209 : it++) {
210 111244 : if (it->kind() == TranslatedFrame::kInterpretedFunction ||
211 111216 : it->kind() == TranslatedFrame::kJavaScriptBuiltinContinuation ||
212 : it->kind() ==
213 : TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch) {
214 54936 : if (counter == 0) {
215 : frame_it = it;
216 : break;
217 : }
218 29962 : counter--;
219 : }
220 : }
221 24974 : CHECK(frame_it != translated_values.end());
222 : // We only include kJavaScriptBuiltinContinuation frames above to get the
223 : // counting right.
224 24974 : CHECK_EQ(frame_it->kind(), TranslatedFrame::kInterpretedFunction);
225 :
226 : DeoptimizedFrameInfo* info =
227 49948 : new DeoptimizedFrameInfo(&translated_values, frame_it, isolate);
228 :
229 24974 : return info;
230 : }
231 :
232 : namespace {
233 140274 : class ActivationsFinder : public ThreadVisitor {
234 : public:
235 : explicit ActivationsFinder(std::set<Code>* codes, Code topmost_optimized_code,
236 : bool safe_to_deopt_topmost_optimized_code)
237 70137 : : codes_(codes) {
238 : #ifdef DEBUG
239 : topmost_ = topmost_optimized_code;
240 : safe_to_deopt_ = safe_to_deopt_topmost_optimized_code;
241 : #endif
242 : }
243 :
244 : // Find the frames with activations of codes marked for deoptimization, search
245 : // for the trampoline to the deoptimizer call respective to each code, and use
246 : // it to replace the current pc on the stack.
247 154136 : void VisitThread(Isolate* isolate, ThreadLocalTop* top) override {
248 707876 : for (StackFrameIterator it(isolate, top); !it.done(); it.Advance()) {
249 553740 : if (it.frame()->type() == StackFrame::OPTIMIZED) {
250 69817 : Code code = it.frame()->LookupCode();
251 122280 : if (code->kind() == Code::OPTIMIZED_FUNCTION &&
252 52463 : code->marked_for_deoptimization()) {
253 22409 : codes_->erase(code);
254 : // Obtain the trampoline to the deoptimizer call.
255 22409 : SafepointEntry safepoint = code->GetSafepointEntry(it.frame()->pc());
256 : int trampoline_pc = safepoint.trampoline_pc();
257 : DCHECK_IMPLIES(code == topmost_, safe_to_deopt_);
258 : // Replace the current pc on the stack with the trampoline.
259 22409 : it.frame()->set_pc(code->raw_instruction_start() + trampoline_pc);
260 : }
261 : }
262 : }
263 154136 : }
264 :
265 : private:
266 : std::set<Code>* codes_;
267 :
268 : #ifdef DEBUG
269 : Code topmost_;
270 : bool safe_to_deopt_;
271 : #endif
272 : };
273 : } // namespace
274 :
275 : // Move marked code from the optimized code list to the deoptimized code list,
276 : // and replace pc on the stack for codes marked for deoptimization.
277 70137 : void Deoptimizer::DeoptimizeMarkedCodeForContext(Context context) {
278 : DisallowHeapAllocation no_allocation;
279 :
280 : Isolate* isolate = context->GetIsolate();
281 : Code topmost_optimized_code;
282 : bool safe_to_deopt_topmost_optimized_code = false;
283 : #ifdef DEBUG
284 : // Make sure all activations of optimized code can deopt at their current PC.
285 : // The topmost optimized code has special handling because it cannot be
286 : // deoptimized due to weak object dependency.
287 : for (StackFrameIterator it(isolate, isolate->thread_local_top());
288 : !it.done(); it.Advance()) {
289 : StackFrame::Type type = it.frame()->type();
290 : if (type == StackFrame::OPTIMIZED) {
291 : Code code = it.frame()->LookupCode();
292 : JSFunction function =
293 : static_cast<OptimizedFrame*>(it.frame())->function();
294 : if (FLAG_trace_deopt) {
295 : CodeTracer::Scope scope(isolate->GetCodeTracer());
296 : PrintF(scope.file(), "[deoptimizer found activation of function: ");
297 : function->PrintName(scope.file());
298 : PrintF(scope.file(), " / %" V8PRIxPTR "]\n", function.ptr());
299 : }
300 : SafepointEntry safepoint = code->GetSafepointEntry(it.frame()->pc());
301 :
302 : // Turbofan deopt is checked when we are patching addresses on stack.
303 : bool safe_if_deopt_triggered = safepoint.has_deoptimization_index();
304 : bool is_builtin_code = code->kind() == Code::BUILTIN;
305 : DCHECK(topmost_optimized_code.is_null() || safe_if_deopt_triggered ||
306 : is_builtin_code);
307 : if (topmost_optimized_code.is_null()) {
308 : topmost_optimized_code = code;
309 : safe_to_deopt_topmost_optimized_code = safe_if_deopt_triggered;
310 : }
311 : }
312 : }
313 : #endif
314 :
315 : // We will use this set to mark those Code objects that are marked for
316 : // deoptimization and have not been found in stack frames.
317 : std::set<Code> codes;
318 :
319 : // Move marked code from the optimized code list to the deoptimized code list.
320 : // Walk over all optimized code objects in this native context.
321 70137 : Code prev;
322 70137 : Object element = context->OptimizedCodeListHead();
323 1937069 : while (!element->IsUndefined(isolate)) {
324 933466 : Code code = Code::cast(element);
325 933466 : CHECK_EQ(code->kind(), Code::OPTIMIZED_FUNCTION);
326 933466 : Object next = code->next_code_link();
327 :
328 933466 : if (code->marked_for_deoptimization()) {
329 : codes.insert(code);
330 :
331 277341 : if (!prev.is_null()) {
332 : // Skip this code in the optimized code list.
333 2209 : prev->set_next_code_link(next);
334 : } else {
335 : // There was no previous node, the next node is the new head.
336 275132 : context->SetOptimizedCodeListHead(next);
337 : }
338 :
339 : // Move the code to the _deoptimized_ code list.
340 277341 : code->set_next_code_link(context->DeoptimizedCodeListHead());
341 277341 : context->SetDeoptimizedCodeListHead(code);
342 : } else {
343 : // Not marked; preserve this element.
344 656125 : prev = code;
345 : }
346 933466 : element = next;
347 : }
348 :
349 : ActivationsFinder visitor(&codes, topmost_optimized_code,
350 : safe_to_deopt_topmost_optimized_code);
351 : // Iterate over the stack of this thread.
352 70137 : visitor.VisitThread(isolate, isolate->thread_local_top());
353 : // In addition to iterate over the stack of this thread, we also
354 : // need to consider all the other threads as they may also use
355 : // the code currently beings deoptimized.
356 70137 : isolate->thread_manager()->IterateArchivedThreads(&visitor);
357 :
358 : // If there's no activation of a code in any stack then we can remove its
359 : // deoptimization data. We do this to ensure that code objects that are
360 : // unlinked don't transitively keep objects alive unnecessarily.
361 344547 : for (Code code : codes) {
362 274410 : isolate->heap()->InvalidateCodeDeoptimizationData(code);
363 : }
364 70137 : }
365 :
366 :
367 6844 : void Deoptimizer::DeoptimizeAll(Isolate* isolate) {
368 : RuntimeCallTimerScope runtimeTimer(isolate,
369 6844 : RuntimeCallCounterId::kDeoptimizeCode);
370 : TimerEventScope<TimerEventDeoptimizeCode> timer(isolate);
371 20532 : TRACE_EVENT0("v8", "V8.DeoptimizeCode");
372 6844 : if (FLAG_trace_deopt) {
373 0 : CodeTracer::Scope scope(isolate->GetCodeTracer());
374 0 : PrintF(scope.file(), "[deoptimize all code in all contexts]\n");
375 : }
376 6844 : isolate->AbortConcurrentOptimization(BlockingBehavior::kBlock);
377 : DisallowHeapAllocation no_allocation;
378 : // For all contexts, mark all code, then deoptimize.
379 : Object context = isolate->heap()->native_contexts_list();
380 59878 : while (!context->IsUndefined(isolate)) {
381 26517 : Context native_context = Context::cast(context);
382 26517 : MarkAllCodeForContext(native_context);
383 26517 : DeoptimizeMarkedCodeForContext(native_context);
384 26517 : context = native_context->next_context_link();
385 : }
386 6844 : }
387 :
388 :
389 1913 : void Deoptimizer::DeoptimizeMarkedCode(Isolate* isolate) {
390 : RuntimeCallTimerScope runtimeTimer(isolate,
391 1913 : RuntimeCallCounterId::kDeoptimizeCode);
392 : TimerEventScope<TimerEventDeoptimizeCode> timer(isolate);
393 5739 : TRACE_EVENT0("v8", "V8.DeoptimizeCode");
394 1913 : if (FLAG_trace_deopt) {
395 0 : CodeTracer::Scope scope(isolate->GetCodeTracer());
396 0 : PrintF(scope.file(), "[deoptimize marked code in all contexts]\n");
397 : }
398 : DisallowHeapAllocation no_allocation;
399 : // For all contexts, deoptimize code already marked.
400 : Object context = isolate->heap()->native_contexts_list();
401 10553 : while (!context->IsUndefined(isolate)) {
402 4320 : Context native_context = Context::cast(context);
403 4320 : DeoptimizeMarkedCodeForContext(native_context);
404 4320 : context = native_context->next_context_link();
405 : }
406 1913 : }
407 :
408 26517 : void Deoptimizer::MarkAllCodeForContext(Context context) {
409 26517 : Object element = context->OptimizedCodeListHead();
410 : Isolate* isolate = context->GetIsolate();
411 499159 : while (!element->IsUndefined(isolate)) {
412 236321 : Code code = Code::cast(element);
413 236321 : CHECK_EQ(code->kind(), Code::OPTIMIZED_FUNCTION);
414 236321 : code->set_marked_for_deoptimization(true);
415 236321 : element = code->next_code_link();
416 : }
417 26517 : }
418 :
419 589481 : void Deoptimizer::DeoptimizeFunction(JSFunction function, Code code) {
420 : Isolate* isolate = function->GetIsolate();
421 : RuntimeCallTimerScope runtimeTimer(isolate,
422 589481 : RuntimeCallCounterId::kDeoptimizeCode);
423 : TimerEventScope<TimerEventDeoptimizeCode> timer(isolate);
424 1768443 : TRACE_EVENT0("v8", "V8.DeoptimizeCode");
425 589481 : function->ResetIfBytecodeFlushed();
426 1178812 : if (code.is_null()) code = function->code();
427 :
428 589481 : if (code->kind() == Code::OPTIMIZED_FUNCTION) {
429 : // Mark the code for deoptimization and unlink any functions that also
430 : // refer to that code. The code cannot be shared across native contexts,
431 : // so we only need to search one.
432 39300 : code->set_marked_for_deoptimization(true);
433 : // The code in the function's optimized code feedback vector slot might
434 : // be different from the code on the function - evict it if necessary.
435 78600 : function->feedback_vector()->EvictOptimizedCodeMarkedForDeoptimization(
436 39300 : function->shared(), "unlinking code marked for deopt");
437 39300 : if (!code->deopt_already_counted()) {
438 29534 : function->feedback_vector()->increment_deopt_count();
439 14767 : code->set_deopt_already_counted(true);
440 : }
441 39300 : DeoptimizeMarkedCodeForContext(function->context()->native_context());
442 : }
443 589481 : }
444 :
445 45770 : void Deoptimizer::ComputeOutputFrames(Deoptimizer* deoptimizer) {
446 45770 : deoptimizer->DoComputeOutputFrames();
447 45770 : }
448 :
449 486 : const char* Deoptimizer::MessageFor(DeoptimizeKind kind) {
450 486 : switch (kind) {
451 : case DeoptimizeKind::kEager:
452 : return "eager";
453 : case DeoptimizeKind::kSoft:
454 3 : return "soft";
455 : case DeoptimizeKind::kLazy:
456 0 : return "lazy";
457 : }
458 0 : FATAL("Unsupported deopt kind");
459 : return nullptr;
460 : }
461 :
462 45770 : Deoptimizer::Deoptimizer(Isolate* isolate, JSFunction function,
463 : DeoptimizeKind kind, unsigned bailout_id, Address from,
464 : int fp_to_sp_delta)
465 : : isolate_(isolate),
466 : function_(function),
467 : bailout_id_(bailout_id),
468 : deopt_kind_(kind),
469 : from_(from),
470 : fp_to_sp_delta_(fp_to_sp_delta),
471 : deoptimizing_throw_(false),
472 : catch_handler_data_(-1),
473 : catch_handler_pc_offset_(-1),
474 : input_(nullptr),
475 : output_count_(0),
476 : jsframe_count_(0),
477 : output_(nullptr),
478 : caller_frame_top_(0),
479 : caller_fp_(0),
480 : caller_pc_(0),
481 : caller_constant_pool_(0),
482 : input_frame_context_(0),
483 : stack_fp_(0),
484 137310 : trace_scope_(nullptr) {
485 45770 : if (isolate->deoptimizer_lazy_throw()) {
486 : isolate->set_deoptimizer_lazy_throw(false);
487 5368 : deoptimizing_throw_ = true;
488 : }
489 :
490 : DCHECK_NE(from, kNullAddress);
491 45770 : compiled_code_ = FindOptimizedCode();
492 : DCHECK(!compiled_code_.is_null());
493 :
494 : DCHECK(function->IsJSFunction());
495 : trace_scope_ = FLAG_trace_deopt
496 : ? new CodeTracer::Scope(isolate->GetCodeTracer())
497 91540 : : nullptr;
498 : #ifdef DEBUG
499 : DCHECK(AllowHeapAllocation::IsAllowed());
500 : disallow_heap_allocation_ = new DisallowHeapAllocation();
501 : #endif // DEBUG
502 91540 : if (compiled_code_->kind() != Code::OPTIMIZED_FUNCTION ||
503 45770 : !compiled_code_->deopt_already_counted()) {
504 : // If the function is optimized, and we haven't counted that deopt yet, then
505 : // increment the function's deopt count so that we can avoid optimising
506 : // functions that deopt too often.
507 :
508 25349 : if (deopt_kind_ == DeoptimizeKind::kSoft) {
509 : // Soft deopts shouldn't count against the overall deoptimization count
510 : // that can eventually lead to disabling optimization for a function.
511 4528 : isolate->counters()->soft_deopts_executed()->Increment();
512 20821 : } else if (!function.is_null()) {
513 41642 : function->feedback_vector()->increment_deopt_count();
514 : }
515 : }
516 45770 : if (compiled_code_->kind() == Code::OPTIMIZED_FUNCTION) {
517 45770 : compiled_code_->set_deopt_already_counted(true);
518 91540 : PROFILE(isolate_,
519 : CodeDeoptEvent(compiled_code_, kind, from_, fp_to_sp_delta_));
520 : }
521 45770 : unsigned size = ComputeInputFrameSize();
522 : int parameter_count =
523 45770 : function->shared()->internal_formal_parameter_count() + 1;
524 45770 : input_ = new (size) FrameDescription(size, parameter_count);
525 45770 : }
526 :
527 45770 : Code Deoptimizer::FindOptimizedCode() {
528 45770 : Code compiled_code = FindDeoptimizingCode(from_);
529 : return !compiled_code.is_null() ? compiled_code
530 45770 : : isolate_->FindCodeObject(from_);
531 : }
532 :
533 :
534 480 : void Deoptimizer::PrintFunctionName() {
535 960 : if (function_->IsHeapObject() && function_->IsJSFunction()) {
536 960 : function_->ShortPrint(trace_scope_->file());
537 : } else {
538 0 : PrintF(trace_scope_->file(),
539 0 : "%s", Code::Kind2String(compiled_code_->kind()));
540 : }
541 480 : }
542 :
543 91540 : Handle<JSFunction> Deoptimizer::function() const {
544 91540 : return Handle<JSFunction>(function_, isolate());
545 : }
546 0 : Handle<Code> Deoptimizer::compiled_code() const {
547 0 : return Handle<Code>(compiled_code_, isolate());
548 : }
549 :
550 91540 : Deoptimizer::~Deoptimizer() {
551 : DCHECK(input_ == nullptr && output_ == nullptr);
552 : DCHECK_NULL(disallow_heap_allocation_);
553 45770 : delete trace_scope_;
554 45770 : }
555 :
556 :
557 45770 : void Deoptimizer::DeleteFrameDescriptions() {
558 45770 : delete input_;
559 147700 : for (int i = 0; i < output_count_; ++i) {
560 50965 : if (output_[i] != input_) delete output_[i];
561 : }
562 45770 : delete[] output_;
563 45770 : input_ = nullptr;
564 45770 : output_ = nullptr;
565 : #ifdef DEBUG
566 : DCHECK(!AllowHeapAllocation::IsAllowed());
567 : DCHECK_NOT_NULL(disallow_heap_allocation_);
568 : delete disallow_heap_allocation_;
569 : disallow_heap_allocation_ = nullptr;
570 : #endif // DEBUG
571 45770 : }
572 :
573 3354665 : Address Deoptimizer::GetDeoptimizationEntry(Isolate* isolate,
574 : DeoptimizeKind kind) {
575 : DeoptimizerData* data = isolate->deoptimizer_data();
576 3354665 : CHECK_LE(kind, DeoptimizerData::kLastDeoptimizeKind);
577 3354665 : CHECK(!data->deopt_entry_code(kind).is_null());
578 3354665 : return data->deopt_entry_code(kind)->raw_instruction_start();
579 : }
580 :
581 0 : bool Deoptimizer::IsDeoptimizationEntry(Isolate* isolate, Address addr,
582 : DeoptimizeKind type) {
583 : DeoptimizerData* data = isolate->deoptimizer_data();
584 0 : CHECK_LE(type, DeoptimizerData::kLastDeoptimizeKind);
585 : Code code = data->deopt_entry_code(type);
586 0 : if (code.is_null()) return false;
587 0 : return addr == code->raw_instruction_start();
588 : }
589 :
590 0 : bool Deoptimizer::IsDeoptimizationEntry(Isolate* isolate, Address addr,
591 : DeoptimizeKind* type) {
592 0 : if (IsDeoptimizationEntry(isolate, addr, DeoptimizeKind::kEager)) {
593 0 : *type = DeoptimizeKind::kEager;
594 0 : return true;
595 : }
596 0 : if (IsDeoptimizationEntry(isolate, addr, DeoptimizeKind::kSoft)) {
597 0 : *type = DeoptimizeKind::kSoft;
598 0 : return true;
599 : }
600 0 : if (IsDeoptimizationEntry(isolate, addr, DeoptimizeKind::kLazy)) {
601 0 : *type = DeoptimizeKind::kLazy;
602 0 : return true;
603 : }
604 : return false;
605 : }
606 :
607 85 : int Deoptimizer::GetDeoptimizedCodeCount(Isolate* isolate) {
608 : int length = 0;
609 : // Count all entries in the deoptimizing code list of every context.
610 : Object context = isolate->heap()->native_contexts_list();
611 255 : while (!context->IsUndefined(isolate)) {
612 85 : Context native_context = Context::cast(context);
613 85 : Object element = native_context->DeoptimizedCodeListHead();
614 269 : while (!element->IsUndefined(isolate)) {
615 92 : Code code = Code::cast(element);
616 : DCHECK(code->kind() == Code::OPTIMIZED_FUNCTION);
617 92 : if (!code->marked_for_deoptimization()) {
618 0 : length++;
619 : }
620 92 : element = code->next_code_link();
621 : }
622 85 : context = Context::cast(context)->next_context_link();
623 : }
624 85 : return length;
625 : }
626 :
627 : namespace {
628 :
629 5435 : int LookupCatchHandler(TranslatedFrame* translated_frame, int* data_out) {
630 5435 : switch (translated_frame->kind()) {
631 : case TranslatedFrame::kInterpretedFunction: {
632 : int bytecode_offset = translated_frame->node_id().ToInt();
633 : HandlerTable table(
634 5395 : translated_frame->raw_shared_info()->GetBytecodeArray());
635 5395 : return table.LookupRange(bytecode_offset, data_out, nullptr);
636 : }
637 : case TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch: {
638 : return 0;
639 : }
640 : default:
641 : break;
642 : }
643 32 : return -1;
644 : }
645 :
646 : bool ShouldPadArguments(int arg_count) {
647 : return kPadArguments && (arg_count % 2 != 0);
648 : }
649 :
650 : } // namespace
651 :
652 : // We rely on this function not causing a GC. It is called from generated code
653 : // without having a real stack frame in place.
654 45770 : void Deoptimizer::DoComputeOutputFrames() {
655 : base::ElapsedTimer timer;
656 :
657 : // Determine basic deoptimization information. The optimized frame is
658 : // described by the input data.
659 : DeoptimizationData input_data =
660 45770 : DeoptimizationData::cast(compiled_code_->deoptimization_data());
661 :
662 : {
663 : // Read caller's PC, caller's FP and caller's constant pool values
664 : // from input frame. Compute caller's frame top address.
665 :
666 45770 : Register fp_reg = JavaScriptFrame::fp_register();
667 91540 : stack_fp_ = input_->GetRegister(fp_reg.code());
668 :
669 45770 : caller_frame_top_ = stack_fp_ + ComputeInputFrameAboveFpFixedSize();
670 :
671 45770 : Address fp_address = input_->GetFramePointerAddress();
672 45770 : caller_fp_ = Memory<intptr_t>(fp_address);
673 : caller_pc_ =
674 91540 : Memory<intptr_t>(fp_address + CommonFrameConstants::kCallerPCOffset);
675 45770 : input_frame_context_ = Memory<intptr_t>(
676 91540 : fp_address + CommonFrameConstants::kContextOrFrameTypeOffset);
677 :
678 : if (FLAG_enable_embedded_constant_pool) {
679 : caller_constant_pool_ = Memory<intptr_t>(
680 : fp_address + CommonFrameConstants::kConstantPoolOffset);
681 : }
682 : }
683 :
684 45770 : if (trace_scope_ != nullptr) {
685 : timer.Start();
686 240 : PrintF(trace_scope_->file(), "[deoptimizing (DEOPT %s): begin ",
687 240 : MessageFor(deopt_kind_));
688 240 : PrintFunctionName();
689 480 : PrintF(trace_scope_->file(),
690 : " (opt #%d) @%d, FP to SP delta: %d, caller sp: " V8PRIxPTR_FMT
691 : "]\n",
692 : input_data->OptimizationId()->value(), bailout_id_, fp_to_sp_delta_,
693 240 : caller_frame_top_);
694 240 : if (deopt_kind_ == DeoptimizeKind::kEager ||
695 : deopt_kind_ == DeoptimizeKind::kSoft) {
696 480 : compiled_code_->PrintDeoptLocation(
697 480 : trace_scope_->file(), " ;;; deoptimize at ", from_);
698 : }
699 : }
700 :
701 45770 : BailoutId node_id = input_data->BytecodeOffset(bailout_id_);
702 : ByteArray translations = input_data->TranslationByteArray();
703 : unsigned translation_index =
704 91540 : input_data->TranslationIndex(bailout_id_)->value();
705 :
706 : TranslationIterator state_iterator(translations, translation_index);
707 183080 : translated_state_.Init(
708 45770 : isolate_, input_->GetFramePointerAddress(), &state_iterator,
709 45770 : input_data->LiteralArray(), input_->GetRegisterValues(),
710 45770 : trace_scope_ == nullptr ? nullptr : trace_scope_->file(),
711 : function_->IsHeapObject()
712 : ? function_->shared()->internal_formal_parameter_count()
713 45770 : : 0);
714 :
715 : // Do the input frame to output frame(s) translation.
716 : size_t count = translated_state_.frames().size();
717 : // If we are supposed to go to the catch handler, find the catching frame
718 : // for the catch and make sure we only deoptimize upto that frame.
719 45770 : if (deoptimizing_throw_) {
720 : size_t catch_handler_frame_index = count;
721 5435 : for (size_t i = count; i-- > 0;) {
722 10870 : catch_handler_pc_offset_ = LookupCatchHandler(
723 5435 : &(translated_state_.frames()[i]), &catch_handler_data_);
724 5435 : if (catch_handler_pc_offset_ >= 0) {
725 : catch_handler_frame_index = i;
726 : break;
727 : }
728 : }
729 5368 : CHECK_LT(catch_handler_frame_index, count);
730 5368 : count = catch_handler_frame_index + 1;
731 : }
732 :
733 : DCHECK_NULL(output_);
734 45770 : output_ = new FrameDescription*[count];
735 147700 : for (size_t i = 0; i < count; ++i) {
736 50965 : output_[i] = nullptr;
737 : }
738 45770 : output_count_ = static_cast<int>(count);
739 :
740 : // Translate each output frame.
741 : int frame_index = 0; // output_frame_index
742 147700 : for (size_t i = 0; i < count; ++i, ++frame_index) {
743 : // Read the ast node id, function, and frame height for this output frame.
744 : TranslatedFrame* translated_frame = &(translated_state_.frames()[i]);
745 50965 : bool handle_exception = deoptimizing_throw_ && i == count - 1;
746 50965 : switch (translated_frame->kind()) {
747 : case TranslatedFrame::kInterpretedFunction:
748 48974 : DoComputeInterpretedFrame(translated_frame, frame_index,
749 48974 : handle_exception);
750 48974 : jsframe_count_++;
751 48974 : break;
752 : case TranslatedFrame::kArgumentsAdaptor:
753 839 : DoComputeArgumentsAdaptorFrame(translated_frame, frame_index);
754 839 : break;
755 : case TranslatedFrame::kConstructStub:
756 536 : DoComputeConstructStubFrame(translated_frame, frame_index);
757 536 : break;
758 : case TranslatedFrame::kBuiltinContinuation:
759 : DoComputeBuiltinContinuation(translated_frame, frame_index,
760 16 : BuiltinContinuationMode::STUB);
761 16 : break;
762 : case TranslatedFrame::kJavaScriptBuiltinContinuation:
763 : DoComputeBuiltinContinuation(translated_frame, frame_index,
764 568 : BuiltinContinuationMode::JAVASCRIPT);
765 568 : break;
766 : case TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch:
767 32 : DoComputeBuiltinContinuation(
768 : translated_frame, frame_index,
769 : handle_exception
770 : ? BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION
771 32 : : BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH);
772 32 : break;
773 : case TranslatedFrame::kInvalid:
774 0 : FATAL("invalid frame");
775 : break;
776 : }
777 : }
778 :
779 45770 : FrameDescription* topmost = output_[count - 1];
780 45770 : topmost->GetRegisterValues()->SetRegister(kRootRegister.code(),
781 : isolate()->isolate_root());
782 :
783 : // Print some helpful diagnostic information.
784 45770 : if (trace_scope_ != nullptr) {
785 240 : double ms = timer.Elapsed().InMillisecondsF();
786 240 : int index = output_count_ - 1; // Index of the topmost frame.
787 240 : PrintF(trace_scope_->file(), "[deoptimizing (%s): end ",
788 240 : MessageFor(deopt_kind_));
789 240 : PrintFunctionName();
790 480 : PrintF(trace_scope_->file(),
791 : " @%d => node=%d, pc=" V8PRIxPTR_FMT ", caller sp=" V8PRIxPTR_FMT
792 : ", took %0.3f ms]\n",
793 240 : bailout_id_, node_id.ToInt(), output_[index]->GetPc(),
794 240 : caller_frame_top_, ms);
795 : }
796 45770 : }
797 :
798 48974 : void Deoptimizer::DoComputeInterpretedFrame(TranslatedFrame* translated_frame,
799 : int frame_index,
800 : bool goto_catch_handler) {
801 48974 : SharedFunctionInfo shared = translated_frame->raw_shared_info();
802 :
803 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
804 : bool is_bottommost = (0 == frame_index);
805 48974 : bool is_topmost = (output_count_ - 1 == frame_index);
806 :
807 : int bytecode_offset = translated_frame->node_id().ToInt();
808 : int height = translated_frame->height();
809 48974 : int register_count = height - 1; // Exclude accumulator.
810 : int register_stack_slot_count =
811 48974 : InterpreterFrameConstants::RegisterStackSlotCount(register_count);
812 48974 : int height_in_bytes = register_stack_slot_count * kSystemPointerSize;
813 :
814 : // The topmost frame will contain the accumulator.
815 48974 : if (is_topmost) {
816 45508 : height_in_bytes += kSystemPointerSize;
817 45508 : if (PadTopOfStackRegister()) height_in_bytes += kSystemPointerSize;
818 : }
819 :
820 : TranslatedFrame::iterator function_iterator = value_iterator++;
821 48974 : if (trace_scope_ != nullptr) {
822 240 : PrintF(trace_scope_->file(), " translating interpreted frame ");
823 240 : std::unique_ptr<char[]> name = shared->DebugName()->ToCString();
824 480 : PrintF(trace_scope_->file(), "%s", name.get());
825 240 : PrintF(trace_scope_->file(), " => bytecode_offset=%d, height=%d%s\n",
826 : bytecode_offset, height_in_bytes,
827 240 : goto_catch_handler ? " (throw)" : "");
828 : }
829 48974 : if (goto_catch_handler) {
830 5360 : bytecode_offset = catch_handler_pc_offset_;
831 : }
832 :
833 : // The 'fixed' part of the frame consists of the incoming parameters and
834 : // the part described by InterpreterFrameConstants. This will include
835 : // argument padding, when needed.
836 : unsigned fixed_frame_size = ComputeInterpretedFixedSize(shared);
837 48974 : unsigned output_frame_size = height_in_bytes + fixed_frame_size;
838 :
839 : // Allocate and store the output frame description.
840 : int parameter_count = shared->internal_formal_parameter_count() + 1;
841 : FrameDescription* output_frame = new (output_frame_size)
842 48974 : FrameDescription(output_frame_size, parameter_count);
843 48974 : FrameWriter frame_writer(this, output_frame, trace_scope_);
844 :
845 48974 : CHECK(frame_index >= 0 && frame_index < output_count_);
846 48974 : CHECK_NULL(output_[frame_index]);
847 48974 : output_[frame_index] = output_frame;
848 :
849 : // The top address of the frame is computed from the previous frame's top and
850 : // this frame's size.
851 : intptr_t top_address;
852 48974 : if (is_bottommost) {
853 45770 : top_address = caller_frame_top_ - output_frame_size;
854 : } else {
855 3204 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
856 : }
857 : output_frame->SetTop(top_address);
858 :
859 : // Compute the incoming parameter translation.
860 :
861 : ReadOnlyRoots roots(isolate());
862 : if (ShouldPadArguments(parameter_count)) {
863 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
864 : }
865 :
866 248606 : for (int i = 0; i < parameter_count; ++i, ++value_iterator) {
867 99816 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
868 : }
869 :
870 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
871 : frame_writer.top_offset());
872 48974 : if (trace_scope_ != nullptr) {
873 240 : PrintF(trace_scope_->file(), " -------------------------\n");
874 : }
875 :
876 : // There are no translation commands for the caller's pc and fp, the
877 : // context, the function and the bytecode offset. Synthesize
878 : // their values and set them up
879 : // explicitly.
880 : //
881 : // The caller's pc for the bottommost output frame is the same as in the
882 : // input frame. For all subsequent output frames, it can be read from the
883 : // previous one. This frame's pc can be computed from the non-optimized
884 : // function code and AST id of the bailout.
885 : const intptr_t caller_pc =
886 48974 : is_bottommost ? caller_pc_ : output_[frame_index - 1]->GetPc();
887 48974 : frame_writer.PushCallerPc(caller_pc);
888 :
889 : // The caller's frame pointer for the bottommost output frame is the same
890 : // as in the input frame. For all subsequent output frames, it can be
891 : // read from the previous one. Also compute and set this frame's frame
892 : // pointer.
893 : const intptr_t caller_fp =
894 48974 : is_bottommost ? caller_fp_ : output_[frame_index - 1]->GetFp();
895 48974 : frame_writer.PushCallerFp(caller_fp);
896 :
897 48974 : intptr_t fp_value = top_address + frame_writer.top_offset();
898 : output_frame->SetFp(fp_value);
899 48974 : if (is_topmost) {
900 45508 : Register fp_reg = InterpretedFrame::fp_register();
901 45508 : output_frame->SetRegister(fp_reg.code(), fp_value);
902 : }
903 :
904 : if (FLAG_enable_embedded_constant_pool) {
905 : // For the bottommost output frame the constant pool pointer can be gotten
906 : // from the input frame. For subsequent output frames, it can be read from
907 : // the previous frame.
908 : const intptr_t caller_cp =
909 : is_bottommost ? caller_constant_pool_
910 : : output_[frame_index - 1]->GetConstantPool();
911 : frame_writer.PushCallerConstantPool(caller_cp);
912 : }
913 :
914 : // For the bottommost output frame the context can be gotten from the input
915 : // frame. For all subsequent output frames it can be gotten from the function
916 : // so long as we don't inline functions that need local contexts.
917 :
918 : // When deoptimizing into a catch block, we need to take the context
919 : // from a register that was specified in the handler table.
920 : TranslatedFrame::iterator context_pos = value_iterator++;
921 48974 : if (goto_catch_handler) {
922 : // Skip to the translated value of the register specified
923 : // in the handler table.
924 152926 : for (int i = 0; i < catch_handler_data_ + 1; ++i) {
925 : context_pos++;
926 : }
927 : }
928 : // Read the context from the translations.
929 48974 : Object context = context_pos->GetRawValue();
930 48974 : output_frame->SetContext(static_cast<intptr_t>(context->ptr()));
931 48974 : frame_writer.PushTranslatedValue(context_pos, "context");
932 :
933 : // The function was mentioned explicitly in the BEGIN_FRAME.
934 48974 : frame_writer.PushTranslatedValue(function_iterator, "function");
935 :
936 : // Set the bytecode array pointer.
937 48974 : Object bytecode_array = shared->HasBreakInfo()
938 : ? shared->GetDebugInfo()->DebugBytecodeArray()
939 97706 : : shared->GetBytecodeArray();
940 48974 : frame_writer.PushRawObject(bytecode_array, "bytecode array\n");
941 :
942 : // The bytecode offset was mentioned explicitly in the BEGIN_FRAME.
943 : int raw_bytecode_offset =
944 48974 : BytecodeArray::kHeaderSize - kHeapObjectTag + bytecode_offset;
945 48974 : Smi smi_bytecode_offset = Smi::FromInt(raw_bytecode_offset);
946 48974 : frame_writer.PushRawObject(smi_bytecode_offset, "bytecode offset\n");
947 :
948 48974 : if (trace_scope_ != nullptr) {
949 240 : PrintF(trace_scope_->file(), " -------------------------\n");
950 : }
951 :
952 : // Translate the rest of the interpreter registers in the frame.
953 : // The return_value_offset is counted from the top. Here, we compute the
954 : // register index (counted from the start).
955 : int return_value_first_reg =
956 48974 : register_count - translated_frame->return_value_offset();
957 : int return_value_count = translated_frame->return_value_count();
958 711304 : for (int i = 0; i < register_count; ++i, ++value_iterator) {
959 : // Ensure we write the return value if we have one and we are returning
960 : // normally to a lazy deopt point.
961 518711 : if (is_topmost && !goto_catch_handler &&
962 270196 : deopt_kind_ == DeoptimizeKind::kLazy && i >= return_value_first_reg &&
963 36 : i < return_value_first_reg + return_value_count) {
964 27 : int return_index = i - return_value_first_reg;
965 27 : if (return_index == 0) {
966 27 : frame_writer.PushRawValue(input_->GetRegister(kReturnRegister0.code()),
967 27 : "return value 0\n");
968 : // We do not handle the situation when one return value should go into
969 : // the accumulator and another one into an ordinary register. Since
970 : // the interpreter should never create such situation, just assert
971 : // this does not happen.
972 27 : CHECK_LE(return_value_first_reg + return_value_count, register_count);
973 : } else {
974 0 : CHECK_EQ(return_index, 1);
975 0 : frame_writer.PushRawValue(input_->GetRegister(kReturnRegister1.code()),
976 0 : "return value 1\n");
977 : }
978 : } else {
979 : // This is not return value, just write the value from the translations.
980 331138 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
981 : }
982 : }
983 :
984 : int register_slots_written = register_count;
985 : DCHECK_LE(register_slots_written, register_stack_slot_count);
986 : // Some architectures must pad the stack frame with extra stack slots
987 : // to ensure the stack frame is aligned. Do this now.
988 48974 : while (register_slots_written < register_stack_slot_count) {
989 0 : register_slots_written++;
990 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
991 : }
992 :
993 : // Translate the accumulator register (depending on frame position).
994 48974 : if (is_topmost) {
995 45508 : if (PadTopOfStackRegister()) {
996 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
997 : }
998 : // For topmost frame, put the accumulator on the stack. The
999 : // {NotifyDeoptimized} builtin pops it off the topmost frame (possibly
1000 : // after materialization).
1001 45508 : if (goto_catch_handler) {
1002 : // If we are lazy deopting to a catch handler, we set the accumulator to
1003 : // the exception (which lives in the result register).
1004 : intptr_t accumulator_value =
1005 5360 : input_->GetRegister(kInterpreterAccumulatorRegister.code());
1006 10720 : frame_writer.PushRawObject(Object(accumulator_value), "accumulator\n");
1007 : } else {
1008 : // If we are lazily deoptimizing make sure we store the deopt
1009 : // return value into the appropriate slot.
1010 95636 : if (deopt_kind_ == DeoptimizeKind::kLazy &&
1011 55461 : translated_frame->return_value_offset() == 0 &&
1012 : translated_frame->return_value_count() > 0) {
1013 15038 : CHECK_EQ(translated_frame->return_value_count(), 1);
1014 15038 : frame_writer.PushRawValue(input_->GetRegister(kReturnRegister0.code()),
1015 15038 : "return value 0\n");
1016 : } else {
1017 25110 : frame_writer.PushTranslatedValue(value_iterator, "accumulator");
1018 : }
1019 : }
1020 : ++value_iterator; // Move over the accumulator.
1021 : } else {
1022 : // For non-topmost frames, skip the accumulator translation. For those
1023 : // frames, the return value from the callee will become the accumulator.
1024 : ++value_iterator;
1025 : }
1026 48974 : CHECK_EQ(translated_frame->end(), value_iterator);
1027 48974 : CHECK_EQ(0u, frame_writer.top_offset());
1028 :
1029 : // Compute this frame's PC and state. The PC will be a special builtin that
1030 : // continues the bytecode dispatch. Note that non-topmost and lazy-style
1031 : // bailout handlers also advance the bytecode offset before dispatch, hence
1032 : // simulating what normal handlers do upon completion of the operation.
1033 48974 : Builtins* builtins = isolate_->builtins();
1034 : Code dispatch_builtin =
1035 45508 : (!is_topmost || (deopt_kind_ == DeoptimizeKind::kLazy)) &&
1036 : !goto_catch_handler
1037 : ? builtins->builtin(Builtins::kInterpreterEnterBytecodeAdvance)
1038 67780 : : builtins->builtin(Builtins::kInterpreterEnterBytecodeDispatch);
1039 48974 : output_frame->SetPc(
1040 : static_cast<intptr_t>(dispatch_builtin->InstructionStart()));
1041 :
1042 : // Update constant pool.
1043 : if (FLAG_enable_embedded_constant_pool) {
1044 : intptr_t constant_pool_value =
1045 : static_cast<intptr_t>(dispatch_builtin->constant_pool());
1046 : output_frame->SetConstantPool(constant_pool_value);
1047 : if (is_topmost) {
1048 : Register constant_pool_reg =
1049 : InterpretedFrame::constant_pool_pointer_register();
1050 : output_frame->SetRegister(constant_pool_reg.code(), constant_pool_value);
1051 : }
1052 : }
1053 :
1054 : // Clear the context register. The context might be a de-materialized object
1055 : // and will be materialized by {Runtime_NotifyDeoptimized}. For additional
1056 : // safety we use Smi(0) instead of the potential {arguments_marker} here.
1057 48974 : if (is_topmost) {
1058 : intptr_t context_value = static_cast<intptr_t>(Smi::zero().ptr());
1059 45508 : Register context_reg = JavaScriptFrame::context_register();
1060 45508 : output_frame->SetRegister(context_reg.code(), context_value);
1061 : // Set the continuation for the topmost frame.
1062 45508 : Code continuation = builtins->builtin(Builtins::kNotifyDeoptimized);
1063 45508 : output_frame->SetContinuation(
1064 : static_cast<intptr_t>(continuation->InstructionStart()));
1065 : }
1066 48974 : }
1067 :
1068 839 : void Deoptimizer::DoComputeArgumentsAdaptorFrame(
1069 : TranslatedFrame* translated_frame, int frame_index) {
1070 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
1071 : bool is_bottommost = (0 == frame_index);
1072 :
1073 839 : unsigned height = translated_frame->height();
1074 839 : unsigned height_in_bytes = height * kSystemPointerSize;
1075 : int parameter_count = height;
1076 : if (ShouldPadArguments(parameter_count))
1077 : height_in_bytes += kSystemPointerSize;
1078 :
1079 : TranslatedFrame::iterator function_iterator = value_iterator++;
1080 839 : if (trace_scope_ != nullptr) {
1081 : PrintF(trace_scope_->file(),
1082 0 : " translating arguments adaptor => height=%d\n", height_in_bytes);
1083 : }
1084 :
1085 : unsigned fixed_frame_size = ArgumentsAdaptorFrameConstants::kFixedFrameSize;
1086 839 : unsigned output_frame_size = height_in_bytes + fixed_frame_size;
1087 :
1088 : // Allocate and store the output frame description.
1089 : FrameDescription* output_frame = new (output_frame_size)
1090 839 : FrameDescription(output_frame_size, parameter_count);
1091 839 : FrameWriter frame_writer(this, output_frame, trace_scope_);
1092 :
1093 : // Arguments adaptor can not be topmost.
1094 839 : CHECK(frame_index < output_count_ - 1);
1095 839 : CHECK_NULL(output_[frame_index]);
1096 839 : output_[frame_index] = output_frame;
1097 :
1098 : // The top address of the frame is computed from the previous frame's top and
1099 : // this frame's size.
1100 : intptr_t top_address;
1101 839 : if (is_bottommost) {
1102 0 : top_address = caller_frame_top_ - output_frame_size;
1103 : } else {
1104 839 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
1105 : }
1106 : output_frame->SetTop(top_address);
1107 :
1108 : ReadOnlyRoots roots(isolate());
1109 : if (ShouldPadArguments(parameter_count)) {
1110 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1111 : }
1112 :
1113 : // Compute the incoming parameter translation.
1114 5793 : for (int i = 0; i < parameter_count; ++i, ++value_iterator) {
1115 2477 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
1116 : }
1117 :
1118 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
1119 : frame_writer.top_offset());
1120 :
1121 : // Read caller's PC from the previous frame.
1122 : const intptr_t caller_pc =
1123 839 : is_bottommost ? caller_pc_ : output_[frame_index - 1]->GetPc();
1124 839 : frame_writer.PushCallerPc(caller_pc);
1125 :
1126 : // Read caller's FP from the previous frame, and set this frame's FP.
1127 : const intptr_t caller_fp =
1128 839 : is_bottommost ? caller_fp_ : output_[frame_index - 1]->GetFp();
1129 839 : frame_writer.PushCallerFp(caller_fp);
1130 :
1131 839 : intptr_t fp_value = top_address + frame_writer.top_offset();
1132 : output_frame->SetFp(fp_value);
1133 :
1134 : if (FLAG_enable_embedded_constant_pool) {
1135 : // Read the caller's constant pool from the previous frame.
1136 : const intptr_t caller_cp =
1137 : is_bottommost ? caller_constant_pool_
1138 : : output_[frame_index - 1]->GetConstantPool();
1139 : frame_writer.PushCallerConstantPool(caller_cp);
1140 : }
1141 :
1142 : // A marker value is used in place of the context.
1143 : intptr_t marker = StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR);
1144 839 : frame_writer.PushRawValue(marker, "context (adaptor sentinel)\n");
1145 :
1146 : // The function was mentioned explicitly in the ARGUMENTS_ADAPTOR_FRAME.
1147 839 : frame_writer.PushTranslatedValue(function_iterator, "function\n");
1148 :
1149 : // Number of incoming arguments.
1150 1678 : frame_writer.PushRawObject(Smi::FromInt(height - 1), "argc\n");
1151 :
1152 839 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1153 :
1154 839 : CHECK_EQ(translated_frame->end(), value_iterator);
1155 : DCHECK_EQ(0, frame_writer.top_offset());
1156 :
1157 839 : Builtins* builtins = isolate_->builtins();
1158 : Code adaptor_trampoline =
1159 839 : builtins->builtin(Builtins::kArgumentsAdaptorTrampoline);
1160 : intptr_t pc_value = static_cast<intptr_t>(
1161 839 : adaptor_trampoline->InstructionStart() +
1162 1678 : isolate_->heap()->arguments_adaptor_deopt_pc_offset()->value());
1163 : output_frame->SetPc(pc_value);
1164 : if (FLAG_enable_embedded_constant_pool) {
1165 : intptr_t constant_pool_value =
1166 : static_cast<intptr_t>(adaptor_trampoline->constant_pool());
1167 : output_frame->SetConstantPool(constant_pool_value);
1168 : }
1169 839 : }
1170 :
1171 536 : void Deoptimizer::DoComputeConstructStubFrame(TranslatedFrame* translated_frame,
1172 : int frame_index) {
1173 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
1174 536 : bool is_topmost = (output_count_ - 1 == frame_index);
1175 : // The construct frame could become topmost only if we inlined a constructor
1176 : // call which does a tail call (otherwise the tail callee's frame would be
1177 : // the topmost one). So it could only be the DeoptimizeKind::kLazy case.
1178 536 : CHECK(!is_topmost || deopt_kind_ == DeoptimizeKind::kLazy);
1179 :
1180 536 : Builtins* builtins = isolate_->builtins();
1181 536 : Code construct_stub = builtins->builtin(Builtins::kJSConstructStubGeneric);
1182 : BailoutId bailout_id = translated_frame->node_id();
1183 536 : unsigned height = translated_frame->height();
1184 536 : unsigned parameter_count = height - 1; // Exclude the context.
1185 536 : unsigned height_in_bytes = parameter_count * kSystemPointerSize;
1186 :
1187 : // If the construct frame appears to be topmost we should ensure that the
1188 : // value of result register is preserved during continuation execution.
1189 : // We do this here by "pushing" the result of the constructor function to the
1190 : // top of the reconstructed stack and popping it in
1191 : // {Builtins::kNotifyDeoptimized}.
1192 536 : if (is_topmost) {
1193 40 : height_in_bytes += kSystemPointerSize;
1194 40 : if (PadTopOfStackRegister()) height_in_bytes += kSystemPointerSize;
1195 : }
1196 :
1197 : if (ShouldPadArguments(parameter_count))
1198 : height_in_bytes += kSystemPointerSize;
1199 :
1200 : TranslatedFrame::iterator function_iterator = value_iterator++;
1201 536 : if (trace_scope_ != nullptr) {
1202 0 : PrintF(trace_scope_->file(),
1203 : " translating construct stub => bailout_id=%d (%s), height=%d\n",
1204 : bailout_id.ToInt(),
1205 : bailout_id == BailoutId::ConstructStubCreate() ? "create" : "invoke",
1206 0 : height_in_bytes);
1207 : }
1208 :
1209 : unsigned fixed_frame_size = ConstructFrameConstants::kFixedFrameSize;
1210 536 : unsigned output_frame_size = height_in_bytes + fixed_frame_size;
1211 :
1212 : // Allocate and store the output frame description.
1213 : FrameDescription* output_frame = new (output_frame_size)
1214 536 : FrameDescription(output_frame_size, parameter_count);
1215 536 : FrameWriter frame_writer(this, output_frame, trace_scope_);
1216 :
1217 : // Construct stub can not be topmost.
1218 : DCHECK(frame_index > 0 && frame_index < output_count_);
1219 : DCHECK_NULL(output_[frame_index]);
1220 536 : output_[frame_index] = output_frame;
1221 :
1222 : // The top address of the frame is computed from the previous frame's top and
1223 : // this frame's size.
1224 : intptr_t top_address;
1225 536 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
1226 : output_frame->SetTop(top_address);
1227 :
1228 : ReadOnlyRoots roots(isolate());
1229 : if (ShouldPadArguments(parameter_count)) {
1230 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1231 : }
1232 :
1233 : // The allocated receiver of a construct stub frame is passed as the
1234 : // receiver parameter through the translation. It might be encoding
1235 : // a captured object, so we need save it for later.
1236 : TranslatedFrame::iterator receiver_iterator = value_iterator;
1237 :
1238 : // Compute the incoming parameter translation.
1239 3794 : for (unsigned i = 0; i < parameter_count; ++i, ++value_iterator) {
1240 1629 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
1241 : }
1242 :
1243 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
1244 : frame_writer.top_offset());
1245 :
1246 : // Read caller's PC from the previous frame.
1247 536 : const intptr_t caller_pc = output_[frame_index - 1]->GetPc();
1248 536 : frame_writer.PushCallerPc(caller_pc);
1249 :
1250 : // Read caller's FP from the previous frame, and set this frame's FP.
1251 536 : const intptr_t caller_fp = output_[frame_index - 1]->GetFp();
1252 536 : frame_writer.PushCallerFp(caller_fp);
1253 :
1254 536 : intptr_t fp_value = top_address + frame_writer.top_offset();
1255 : output_frame->SetFp(fp_value);
1256 536 : if (is_topmost) {
1257 40 : Register fp_reg = JavaScriptFrame::fp_register();
1258 40 : output_frame->SetRegister(fp_reg.code(), fp_value);
1259 : }
1260 :
1261 : if (FLAG_enable_embedded_constant_pool) {
1262 : // Read the caller's constant pool from the previous frame.
1263 : const intptr_t caller_cp = output_[frame_index - 1]->GetConstantPool();
1264 : frame_writer.PushCallerConstantPool(caller_cp);
1265 : }
1266 :
1267 : // A marker value is used to mark the frame.
1268 : intptr_t marker = StackFrame::TypeToMarker(StackFrame::CONSTRUCT);
1269 536 : frame_writer.PushRawValue(marker, "context (construct stub sentinel)\n");
1270 :
1271 536 : frame_writer.PushTranslatedValue(value_iterator++, "context");
1272 :
1273 : // Number of incoming arguments.
1274 1072 : frame_writer.PushRawObject(Smi::FromInt(parameter_count - 1), "argc\n");
1275 :
1276 : // The constructor function was mentioned explicitly in the
1277 : // CONSTRUCT_STUB_FRAME.
1278 536 : frame_writer.PushTranslatedValue(function_iterator, "constructor function\n");
1279 :
1280 : // The deopt info contains the implicit receiver or the new target at the
1281 : // position of the receiver. Copy it to the top of stack, with the hole value
1282 : // as padding to maintain alignment.
1283 :
1284 536 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1285 :
1286 536 : CHECK(bailout_id == BailoutId::ConstructStubCreate() ||
1287 : bailout_id == BailoutId::ConstructStubInvoke());
1288 : const char* debug_hint = bailout_id == BailoutId::ConstructStubCreate()
1289 : ? "new target\n"
1290 536 : : "allocated receiver\n";
1291 536 : frame_writer.PushTranslatedValue(receiver_iterator, debug_hint);
1292 :
1293 536 : if (is_topmost) {
1294 40 : if (PadTopOfStackRegister()) {
1295 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1296 : }
1297 : // Ensure the result is restored back when we return to the stub.
1298 : Register result_reg = kReturnRegister0;
1299 40 : intptr_t result = input_->GetRegister(result_reg.code());
1300 40 : frame_writer.PushRawValue(result, "subcall result\n");
1301 : }
1302 :
1303 536 : CHECK_EQ(translated_frame->end(), value_iterator);
1304 536 : CHECK_EQ(0u, frame_writer.top_offset());
1305 :
1306 : // Compute this frame's PC.
1307 : DCHECK(bailout_id.IsValidForConstructStub());
1308 : Address start = construct_stub->InstructionStart();
1309 : int pc_offset =
1310 : bailout_id == BailoutId::ConstructStubCreate()
1311 40 : ? isolate_->heap()->construct_stub_create_deopt_pc_offset()->value()
1312 1032 : : isolate_->heap()->construct_stub_invoke_deopt_pc_offset()->value();
1313 536 : intptr_t pc_value = static_cast<intptr_t>(start + pc_offset);
1314 : output_frame->SetPc(pc_value);
1315 :
1316 : // Update constant pool.
1317 : if (FLAG_enable_embedded_constant_pool) {
1318 : intptr_t constant_pool_value =
1319 : static_cast<intptr_t>(construct_stub->constant_pool());
1320 : output_frame->SetConstantPool(constant_pool_value);
1321 : if (is_topmost) {
1322 : Register constant_pool_reg =
1323 : JavaScriptFrame::constant_pool_pointer_register();
1324 : output_frame->SetRegister(constant_pool_reg.code(), constant_pool_value);
1325 : }
1326 : }
1327 :
1328 : // Clear the context register. The context might be a de-materialized object
1329 : // and will be materialized by {Runtime_NotifyDeoptimized}. For additional
1330 : // safety we use Smi(0) instead of the potential {arguments_marker} here.
1331 536 : if (is_topmost) {
1332 : intptr_t context_value = static_cast<intptr_t>(Smi::zero().ptr());
1333 40 : Register context_reg = JavaScriptFrame::context_register();
1334 40 : output_frame->SetRegister(context_reg.code(), context_value);
1335 : }
1336 :
1337 : // Set the continuation for the topmost frame.
1338 536 : if (is_topmost) {
1339 40 : Builtins* builtins = isolate_->builtins();
1340 : DCHECK_EQ(DeoptimizeKind::kLazy, deopt_kind_);
1341 40 : Code continuation = builtins->builtin(Builtins::kNotifyDeoptimized);
1342 40 : output_frame->SetContinuation(
1343 : static_cast<intptr_t>(continuation->InstructionStart()));
1344 : }
1345 536 : }
1346 :
1347 0 : bool Deoptimizer::BuiltinContinuationModeIsJavaScript(
1348 : BuiltinContinuationMode mode) {
1349 1232 : switch (mode) {
1350 : case BuiltinContinuationMode::STUB:
1351 : return false;
1352 : case BuiltinContinuationMode::JAVASCRIPT:
1353 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1354 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1355 0 : return true;
1356 : }
1357 0 : UNREACHABLE();
1358 : }
1359 :
1360 0 : bool Deoptimizer::BuiltinContinuationModeIsWithCatch(
1361 : BuiltinContinuationMode mode) {
1362 616 : switch (mode) {
1363 : case BuiltinContinuationMode::STUB:
1364 : case BuiltinContinuationMode::JAVASCRIPT:
1365 : return false;
1366 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1367 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1368 0 : return true;
1369 : }
1370 0 : UNREACHABLE();
1371 : }
1372 :
1373 616 : StackFrame::Type Deoptimizer::BuiltinContinuationModeToFrameType(
1374 : BuiltinContinuationMode mode) {
1375 616 : switch (mode) {
1376 : case BuiltinContinuationMode::STUB:
1377 : return StackFrame::BUILTIN_CONTINUATION;
1378 : case BuiltinContinuationMode::JAVASCRIPT:
1379 568 : return StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION;
1380 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1381 24 : return StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH;
1382 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1383 8 : return StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH;
1384 : }
1385 0 : UNREACHABLE();
1386 : }
1387 :
1388 616 : Builtins::Name Deoptimizer::TrampolineForBuiltinContinuation(
1389 : BuiltinContinuationMode mode, bool must_handle_result) {
1390 616 : switch (mode) {
1391 : case BuiltinContinuationMode::STUB:
1392 : return must_handle_result ? Builtins::kContinueToCodeStubBuiltinWithResult
1393 16 : : Builtins::kContinueToCodeStubBuiltin;
1394 : case BuiltinContinuationMode::JAVASCRIPT:
1395 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH:
1396 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION:
1397 : return must_handle_result
1398 : ? Builtins::kContinueToJavaScriptBuiltinWithResult
1399 600 : : Builtins::kContinueToJavaScriptBuiltin;
1400 : }
1401 0 : UNREACHABLE();
1402 : }
1403 :
1404 : // BuiltinContinuationFrames capture the machine state that is expected as input
1405 : // to a builtin, including both input register values and stack parameters. When
1406 : // the frame is reactivated (i.e. the frame below it returns), a
1407 : // ContinueToBuiltin stub restores the register state from the frame and tail
1408 : // calls to the actual target builtin, making it appear that the stub had been
1409 : // directly called by the frame above it. The input values to populate the frame
1410 : // are taken from the deopt's FrameState.
1411 : //
1412 : // Frame translation happens in two modes, EAGER and LAZY. In EAGER mode, all of
1413 : // the parameters to the Builtin are explicitly specified in the TurboFan
1414 : // FrameState node. In LAZY mode, there is always one fewer parameters specified
1415 : // in the FrameState than expected by the Builtin. In that case, construction of
1416 : // BuiltinContinuationFrame adds the final missing parameter during
1417 : // deoptimization, and that parameter is always on the stack and contains the
1418 : // value returned from the callee of the call site triggering the LAZY deopt
1419 : // (e.g. rax on x64). This requires that continuation Builtins for LAZY deopts
1420 : // must have at least one stack parameter.
1421 : //
1422 : // TO
1423 : // | .... |
1424 : // +-------------------------+
1425 : // | arg padding (arch dept) |<- at most 1*kSystemPointerSize
1426 : // +-------------------------+
1427 : // | builtin param 0 |<- FrameState input value n becomes
1428 : // +-------------------------+
1429 : // | ... |
1430 : // +-------------------------+
1431 : // | builtin param m |<- FrameState input value n+m-1, or in
1432 : // +-----needs-alignment-----+ the LAZY case, return LAZY result value
1433 : // | ContinueToBuiltin entry |
1434 : // +-------------------------+
1435 : // | | saved frame (FP) |
1436 : // | +=====needs=alignment=====+<- fpreg
1437 : // | |constant pool (if ool_cp)|
1438 : // v +-------------------------+
1439 : // |BUILTIN_CONTINUATION mark|
1440 : // +-------------------------+
1441 : // | JSFunction (or zero) |<- only if JavaScript builtin
1442 : // +-------------------------+
1443 : // | frame height above FP |
1444 : // +-------------------------+
1445 : // | context |<- this non-standard context slot contains
1446 : // +-------------------------+ the context, even for non-JS builtins.
1447 : // | builtin address |
1448 : // +-------------------------+
1449 : // | builtin input GPR reg0 |<- populated from deopt FrameState using
1450 : // +-------------------------+ the builtin's CallInterfaceDescriptor
1451 : // | ... | to map a FrameState's 0..n-1 inputs to
1452 : // +-------------------------+ the builtin's n input register params.
1453 : // | builtin input GPR regn |
1454 : // +-------------------------+
1455 : // | reg padding (arch dept) |
1456 : // +-----needs--alignment----+
1457 : // | res padding (arch dept) |<- only if {is_topmost}; result is pop'd by
1458 : // +-------------------------+<- kNotifyDeopt ASM stub and moved to acc
1459 : // | result value |<- reg, as ContinueToBuiltin stub expects.
1460 : // +-----needs-alignment-----+<- spreg
1461 : //
1462 616 : void Deoptimizer::DoComputeBuiltinContinuation(
1463 : TranslatedFrame* translated_frame, int frame_index,
1464 : BuiltinContinuationMode mode) {
1465 : TranslatedFrame::iterator value_iterator = translated_frame->begin();
1466 :
1467 : // The output frame must have room for all of the parameters that need to be
1468 : // passed to the builtin continuation.
1469 : const int height_in_words = translated_frame->height();
1470 :
1471 616 : BailoutId bailout_id = translated_frame->node_id();
1472 616 : Builtins::Name builtin_name = Builtins::GetBuiltinFromBailoutId(bailout_id);
1473 616 : Code builtin = isolate()->builtins()->builtin(builtin_name);
1474 : Callable continuation_callable =
1475 616 : Builtins::CallableFor(isolate(), builtin_name);
1476 : CallInterfaceDescriptor continuation_descriptor =
1477 : continuation_callable.descriptor();
1478 :
1479 : const bool is_bottommost = (0 == frame_index);
1480 616 : const bool is_topmost = (output_count_ - 1 == frame_index);
1481 : const bool must_handle_result =
1482 616 : !is_topmost || deopt_kind_ == DeoptimizeKind::kLazy;
1483 :
1484 616 : const RegisterConfiguration* config(RegisterConfiguration::Default());
1485 : const int allocatable_register_count =
1486 : config->num_allocatable_general_registers();
1487 : const int padding_slot_count =
1488 : BuiltinContinuationFrameConstants::PaddingSlotCount(
1489 616 : allocatable_register_count);
1490 :
1491 : const int register_parameter_count =
1492 : continuation_descriptor.GetRegisterParameterCount();
1493 : // Make sure to account for the context by removing it from the register
1494 : // parameter count.
1495 : const int translated_stack_parameters =
1496 616 : height_in_words - register_parameter_count - 1;
1497 : const int stack_param_count =
1498 1232 : translated_stack_parameters + (must_handle_result ? 1 : 0) +
1499 616 : (BuiltinContinuationModeIsWithCatch(mode) ? 1 : 0);
1500 : const int stack_param_pad_count =
1501 : ShouldPadArguments(stack_param_count) ? 1 : 0;
1502 :
1503 : // If the builtins frame appears to be topmost we should ensure that the
1504 : // value of result register is preserved during continuation execution.
1505 : // We do this here by "pushing" the result of callback function to the
1506 : // top of the reconstructed stack and popping it in
1507 : // {Builtins::kNotifyDeoptimized}.
1508 : const int push_result_count =
1509 616 : is_topmost ? (PadTopOfStackRegister() ? 2 : 1) : 0;
1510 :
1511 : const unsigned output_frame_size =
1512 616 : kSystemPointerSize * (stack_param_count + stack_param_pad_count +
1513 616 : allocatable_register_count + padding_slot_count +
1514 616 : push_result_count) +
1515 616 : BuiltinContinuationFrameConstants::kFixedFrameSize;
1516 :
1517 : const unsigned output_frame_size_above_fp =
1518 616 : kSystemPointerSize * (allocatable_register_count + padding_slot_count +
1519 616 : push_result_count) +
1520 : (BuiltinContinuationFrameConstants::kFixedFrameSize -
1521 : BuiltinContinuationFrameConstants::kFixedFrameSizeAboveFp);
1522 :
1523 : // Validate types of parameters. They must all be tagged except for argc for
1524 : // JS builtins.
1525 : bool has_argc = false;
1526 4216 : for (int i = 0; i < register_parameter_count; ++i) {
1527 : MachineType type = continuation_descriptor.GetParameterType(i);
1528 : int code = continuation_descriptor.GetRegisterParameter(i).code();
1529 : // Only tagged and int32 arguments are supported, and int32 only for the
1530 : // arguments count on JavaScript builtins.
1531 1800 : if (type == MachineType::Int32()) {
1532 600 : CHECK_EQ(code, kJavaScriptCallArgCountRegister.code());
1533 : has_argc = true;
1534 : } else {
1535 : // Any other argument must be a tagged value.
1536 1200 : CHECK(IsAnyTagged(type.representation()));
1537 : }
1538 : }
1539 616 : CHECK_EQ(BuiltinContinuationModeIsJavaScript(mode), has_argc);
1540 :
1541 616 : if (trace_scope_ != nullptr) {
1542 0 : PrintF(trace_scope_->file(),
1543 : " translating BuiltinContinuation to %s,"
1544 : " register param count %d,"
1545 : " stack param count %d\n",
1546 : Builtins::name(builtin_name), register_parameter_count,
1547 0 : stack_param_count);
1548 : }
1549 :
1550 : FrameDescription* output_frame = new (output_frame_size)
1551 616 : FrameDescription(output_frame_size, stack_param_count);
1552 616 : output_[frame_index] = output_frame;
1553 616 : FrameWriter frame_writer(this, output_frame, trace_scope_);
1554 :
1555 : // The top address of the frame is computed from the previous frame's top and
1556 : // this frame's size.
1557 : intptr_t top_address;
1558 616 : if (is_bottommost) {
1559 0 : top_address = caller_frame_top_ - output_frame_size;
1560 : } else {
1561 616 : top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
1562 : }
1563 : output_frame->SetTop(top_address);
1564 :
1565 : // Get the possible JSFunction for the case that this is a
1566 : // JavaScriptBuiltinContinuationFrame, which needs the JSFunction pointer
1567 : // like a normal JavaScriptFrame.
1568 1232 : const intptr_t maybe_function = value_iterator->GetRawValue()->ptr();
1569 : ++value_iterator;
1570 :
1571 : ReadOnlyRoots roots(isolate());
1572 : if (ShouldPadArguments(stack_param_count)) {
1573 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1574 : }
1575 :
1576 6538 : for (int i = 0; i < translated_stack_parameters; ++i, ++value_iterator) {
1577 2961 : frame_writer.PushTranslatedValue(value_iterator, "stack parameter");
1578 : }
1579 :
1580 616 : switch (mode) {
1581 : case BuiltinContinuationMode::STUB:
1582 : break;
1583 : case BuiltinContinuationMode::JAVASCRIPT:
1584 : break;
1585 : case BuiltinContinuationMode::JAVASCRIPT_WITH_CATCH: {
1586 48 : frame_writer.PushRawObject(roots.the_hole_value(),
1587 24 : "placeholder for exception on lazy deopt\n");
1588 24 : } break;
1589 : case BuiltinContinuationMode::JAVASCRIPT_HANDLE_EXCEPTION: {
1590 : intptr_t accumulator_value =
1591 8 : input_->GetRegister(kInterpreterAccumulatorRegister.code());
1592 16 : frame_writer.PushRawObject(Object(accumulator_value),
1593 8 : "exception (from accumulator)\n");
1594 8 : } break;
1595 : }
1596 :
1597 616 : if (must_handle_result) {
1598 872 : frame_writer.PushRawObject(roots.the_hole_value(),
1599 436 : "placeholder for return result on lazy deopt\n");
1600 : }
1601 :
1602 : DCHECK_EQ(output_frame->GetLastArgumentSlotOffset(),
1603 : frame_writer.top_offset());
1604 :
1605 : std::vector<TranslatedFrame::iterator> register_values;
1606 : int total_registers = config->num_general_registers();
1607 616 : register_values.resize(total_registers, {value_iterator});
1608 :
1609 4216 : for (int i = 0; i < register_parameter_count; ++i, ++value_iterator) {
1610 : int code = continuation_descriptor.GetRegisterParameter(i).code();
1611 3600 : register_values[code] = value_iterator;
1612 : }
1613 :
1614 : // The context register is always implicit in the CallInterfaceDescriptor but
1615 : // its register must be explicitly set when continuing to the builtin. Make
1616 : // sure that it's harvested from the translation and copied into the register
1617 : // set (it was automatically added at the end of the FrameState by the
1618 : // instruction selector).
1619 616 : Object context = value_iterator->GetRawValue();
1620 616 : const intptr_t value = context->ptr();
1621 : TranslatedFrame::iterator context_register_value = value_iterator++;
1622 616 : register_values[kContextRegister.code()] = context_register_value;
1623 : output_frame->SetContext(value);
1624 : output_frame->SetRegister(kContextRegister.code(), value);
1625 :
1626 : // Set caller's PC (JSFunction continuation).
1627 : const intptr_t caller_pc =
1628 616 : is_bottommost ? caller_pc_ : output_[frame_index - 1]->GetPc();
1629 616 : frame_writer.PushCallerPc(caller_pc);
1630 :
1631 : // Read caller's FP from the previous frame, and set this frame's FP.
1632 : const intptr_t caller_fp =
1633 616 : is_bottommost ? caller_fp_ : output_[frame_index - 1]->GetFp();
1634 616 : frame_writer.PushCallerFp(caller_fp);
1635 :
1636 616 : const intptr_t fp_value = top_address + frame_writer.top_offset();
1637 : output_frame->SetFp(fp_value);
1638 :
1639 : DCHECK_EQ(output_frame_size_above_fp, frame_writer.top_offset());
1640 :
1641 : if (FLAG_enable_embedded_constant_pool) {
1642 : // Read the caller's constant pool from the previous frame.
1643 : const intptr_t caller_cp =
1644 : is_bottommost ? caller_constant_pool_
1645 : : output_[frame_index - 1]->GetConstantPool();
1646 : frame_writer.PushCallerConstantPool(caller_cp);
1647 : }
1648 :
1649 : // A marker value is used in place of the context.
1650 : const intptr_t marker =
1651 1232 : StackFrame::TypeToMarker(BuiltinContinuationModeToFrameType(mode));
1652 : frame_writer.PushRawValue(marker,
1653 616 : "context (builtin continuation sentinel)\n");
1654 :
1655 616 : if (BuiltinContinuationModeIsJavaScript(mode)) {
1656 600 : frame_writer.PushRawValue(maybe_function, "JSFunction\n");
1657 : } else {
1658 16 : frame_writer.PushRawValue(0, "unused\n");
1659 : }
1660 :
1661 : // The delta from the SP to the FP; used to reconstruct SP in
1662 : // Isolate::UnwindAndFindHandler.
1663 1232 : frame_writer.PushRawObject(Smi::FromInt(output_frame_size_above_fp),
1664 616 : "frame height at deoptimization\n");
1665 :
1666 : // The context even if this is a stub contininuation frame. We can't use the
1667 : // usual context slot, because we must store the frame marker there.
1668 : frame_writer.PushTranslatedValue(context_register_value,
1669 616 : "builtin JavaScript context\n");
1670 :
1671 : // The builtin to continue to.
1672 616 : frame_writer.PushRawObject(builtin, "builtin address\n");
1673 :
1674 15400 : for (int i = 0; i < allocatable_register_count; ++i) {
1675 : int code = config->GetAllocatableGeneralCode(i);
1676 : ScopedVector<char> str(128);
1677 7392 : if (trace_scope_ != nullptr) {
1678 0 : if (BuiltinContinuationModeIsJavaScript(mode) &&
1679 : code == kJavaScriptCallArgCountRegister.code()) {
1680 : SNPrintF(
1681 : str,
1682 : "tagged argument count %s (will be untagged by continuation)\n",
1683 0 : RegisterName(Register::from_code(code)));
1684 : } else {
1685 : SNPrintF(str, "builtin register argument %s\n",
1686 0 : RegisterName(Register::from_code(code)));
1687 : }
1688 : }
1689 7392 : frame_writer.PushTranslatedValue(
1690 22176 : register_values[code], trace_scope_ != nullptr ? str.start() : "");
1691 : }
1692 :
1693 : // Some architectures must pad the stack frame with extra stack slots
1694 : // to ensure the stack frame is aligned.
1695 616 : for (int i = 0; i < padding_slot_count; ++i) {
1696 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1697 : }
1698 :
1699 616 : if (is_topmost) {
1700 222 : if (PadTopOfStackRegister()) {
1701 0 : frame_writer.PushRawObject(roots.the_hole_value(), "padding\n");
1702 : }
1703 : // Ensure the result is restored back when we return to the stub.
1704 :
1705 222 : if (must_handle_result) {
1706 : Register result_reg = kReturnRegister0;
1707 42 : frame_writer.PushRawValue(input_->GetRegister(result_reg.code()),
1708 42 : "callback result\n");
1709 : } else {
1710 180 : frame_writer.PushRawObject(roots.undefined_value(), "callback result\n");
1711 : }
1712 : }
1713 :
1714 616 : CHECK_EQ(translated_frame->end(), value_iterator);
1715 616 : CHECK_EQ(0u, frame_writer.top_offset());
1716 :
1717 : // Clear the context register. The context might be a de-materialized object
1718 : // and will be materialized by {Runtime_NotifyDeoptimized}. For additional
1719 : // safety we use Smi(0) instead of the potential {arguments_marker} here.
1720 616 : if (is_topmost) {
1721 : intptr_t context_value = static_cast<intptr_t>(Smi::zero().ptr());
1722 222 : Register context_reg = JavaScriptFrame::context_register();
1723 222 : output_frame->SetRegister(context_reg.code(), context_value);
1724 : }
1725 :
1726 : // Ensure the frame pointer register points to the callee's frame. The builtin
1727 : // will build its own frame once we continue to it.
1728 616 : Register fp_reg = JavaScriptFrame::fp_register();
1729 616 : output_frame->SetRegister(fp_reg.code(), fp_value);
1730 :
1731 : Code continue_to_builtin = isolate()->builtins()->builtin(
1732 1232 : TrampolineForBuiltinContinuation(mode, must_handle_result));
1733 616 : output_frame->SetPc(
1734 : static_cast<intptr_t>(continue_to_builtin->InstructionStart()));
1735 :
1736 : Code continuation =
1737 616 : isolate()->builtins()->builtin(Builtins::kNotifyDeoptimized);
1738 616 : output_frame->SetContinuation(
1739 : static_cast<intptr_t>(continuation->InstructionStart()));
1740 616 : }
1741 :
1742 45770 : void Deoptimizer::MaterializeHeapObjects() {
1743 45770 : translated_state_.Prepare(static_cast<Address>(stack_fp_));
1744 45770 : if (FLAG_deopt_every_n_times > 0) {
1745 : // Doing a GC here will find problems with the deoptimized frames.
1746 20 : isolate_->heap()->CollectAllGarbage(Heap::kNoGCFlags,
1747 20 : GarbageCollectionReason::kTesting);
1748 : }
1749 :
1750 62600 : for (auto& materialization : values_to_materialize_) {
1751 16830 : Handle<Object> value = materialization.value_->GetValue();
1752 :
1753 16830 : if (trace_scope_ != nullptr) {
1754 0 : PrintF("Materialization [" V8PRIxPTR_FMT "] <- " V8PRIxPTR_FMT " ; ",
1755 0 : static_cast<intptr_t>(materialization.output_slot_address_),
1756 0 : value->ptr());
1757 0 : value->ShortPrint(trace_scope_->file());
1758 0 : PrintF(trace_scope_->file(), "\n");
1759 : }
1760 :
1761 16830 : *(reinterpret_cast<Address*>(materialization.output_slot_address_)) =
1762 33660 : value->ptr();
1763 : }
1764 :
1765 : translated_state_.VerifyMaterializedObjects();
1766 :
1767 45770 : bool feedback_updated = translated_state_.DoUpdateFeedback();
1768 45770 : if (trace_scope_ != nullptr && feedback_updated) {
1769 0 : PrintF(trace_scope_->file(), "Feedback updated");
1770 0 : compiled_code_->PrintDeoptLocation(trace_scope_->file(),
1771 0 : " from deoptimization at ", from_);
1772 : }
1773 :
1774 45770 : isolate_->materialized_object_store()->Remove(
1775 91540 : static_cast<Address>(stack_fp_));
1776 45770 : }
1777 :
1778 571534 : void Deoptimizer::QueueValueForMaterialization(
1779 : Address output_address, Object obj,
1780 : const TranslatedFrame::iterator& iterator) {
1781 1143068 : if (obj == ReadOnlyRoots(isolate_).arguments_marker()) {
1782 50490 : values_to_materialize_.push_back({output_address, iterator});
1783 : }
1784 571534 : }
1785 :
1786 91540 : unsigned Deoptimizer::ComputeInputFrameAboveFpFixedSize() const {
1787 : unsigned fixed_size = CommonFrameConstants::kFixedFrameSizeAboveFp;
1788 : // TODO(jkummerow): If {function_->IsSmi()} can indeed be true, then
1789 : // {function_} should not have type {JSFunction}.
1790 91540 : if (!function_->IsSmi()) {
1791 91540 : fixed_size += ComputeIncomingArgumentSize(function_->shared());
1792 : }
1793 91540 : return fixed_size;
1794 : }
1795 :
1796 45770 : unsigned Deoptimizer::ComputeInputFrameSize() const {
1797 : // The fp-to-sp delta already takes the context, constant pool pointer and the
1798 : // function into account so we have to avoid double counting them.
1799 45770 : unsigned fixed_size_above_fp = ComputeInputFrameAboveFpFixedSize();
1800 45770 : unsigned result = fixed_size_above_fp + fp_to_sp_delta_;
1801 45770 : if (compiled_code_->kind() == Code::OPTIMIZED_FUNCTION) {
1802 : unsigned stack_slots = compiled_code_->stack_slots();
1803 : unsigned outgoing_size = 0;
1804 : // ComputeOutgoingArgumentSize(compiled_code_, bailout_id_);
1805 45770 : CHECK_EQ(fixed_size_above_fp + (stack_slots * kSystemPointerSize) -
1806 : CommonFrameConstants::kFixedFrameSizeAboveFp + outgoing_size,
1807 : result);
1808 : }
1809 45770 : return result;
1810 : }
1811 :
1812 : // static
1813 0 : unsigned Deoptimizer::ComputeInterpretedFixedSize(SharedFunctionInfo shared) {
1814 : // The fixed part of the frame consists of the return address, frame
1815 : // pointer, function, context, bytecode offset and all the incoming arguments.
1816 : return ComputeIncomingArgumentSize(shared) +
1817 48974 : InterpreterFrameConstants::kFixedFrameSize;
1818 : }
1819 :
1820 : // static
1821 0 : unsigned Deoptimizer::ComputeIncomingArgumentSize(SharedFunctionInfo shared) {
1822 140514 : int parameter_slots = shared->internal_formal_parameter_count() + 1;
1823 : if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
1824 140514 : return parameter_slots * kSystemPointerSize;
1825 : }
1826 :
1827 1391616 : void Deoptimizer::EnsureCodeForDeoptimizationEntry(Isolate* isolate,
1828 : DeoptimizeKind kind) {
1829 1391616 : CHECK(kind == DeoptimizeKind::kEager || kind == DeoptimizeKind::kSoft ||
1830 : kind == DeoptimizeKind::kLazy);
1831 : DeoptimizerData* data = isolate->deoptimizer_data();
1832 2739204 : if (!data->deopt_entry_code(kind).is_null()) return;
1833 :
1834 : MacroAssembler masm(isolate, CodeObjectRequired::kYes,
1835 88056 : NewAssemblerBuffer(16 * KB));
1836 : masm.set_emit_debug_code(false);
1837 44028 : GenerateDeoptimizationEntries(&masm, masm.isolate(), kind);
1838 44028 : CodeDesc desc;
1839 : masm.GetCode(isolate, &desc);
1840 : DCHECK(!RelocInfo::RequiresRelocationAfterCodegen(desc));
1841 :
1842 : // Allocate the code as immovable since the entry addresses will be used
1843 : // directly and there is no support for relocating them.
1844 : Handle<Code> code = isolate->factory()->NewCode(
1845 : desc, Code::STUB, Handle<Object>(), Builtins::kNoBuiltinId,
1846 88052 : MaybeHandle<ByteArray>(), MaybeHandle<DeoptimizationData>(), kImmovable);
1847 88056 : CHECK(isolate->heap()->IsImmovable(*code));
1848 :
1849 44028 : CHECK(data->deopt_entry_code(kind).is_null());
1850 : data->set_deopt_entry_code(kind, *code);
1851 : }
1852 :
1853 463862 : void Deoptimizer::EnsureCodeForDeoptimizationEntries(Isolate* isolate) {
1854 463862 : EnsureCodeForDeoptimizationEntry(isolate, DeoptimizeKind::kEager);
1855 463873 : EnsureCodeForDeoptimizationEntry(isolate, DeoptimizeKind::kLazy);
1856 463875 : EnsureCodeForDeoptimizationEntry(isolate, DeoptimizeKind::kSoft);
1857 463876 : }
1858 :
1859 96735 : FrameDescription::FrameDescription(uint32_t frame_size, int parameter_count)
1860 : : frame_size_(frame_size),
1861 : parameter_count_(parameter_count),
1862 : top_(kZapUint32),
1863 : pc_(kZapUint32),
1864 : fp_(kZapUint32),
1865 : context_(kZapUint32),
1866 193470 : constant_pool_(kZapUint32) {
1867 : // Zap all the registers.
1868 3192255 : for (int r = 0; r < Register::kNumRegisters; r++) {
1869 : // TODO(jbramley): It isn't safe to use kZapUint32 here. If the register
1870 : // isn't used before the next safepoint, the GC will try to scan it as a
1871 : // tagged value. kZapUint32 looks like a valid tagged pointer, but it isn't.
1872 : #if defined(V8_OS_WIN) && defined(V8_TARGET_ARCH_ARM64)
1873 : // x18 is reserved as platform register on Windows arm64 platform
1874 : const int kPlatformRegister = 18;
1875 : if (r != kPlatformRegister) {
1876 : SetRegister(r, kZapUint32);
1877 : }
1878 : #else
1879 1547760 : SetRegister(r, kZapUint32);
1880 : #endif
1881 : }
1882 :
1883 : // Zap all the slots.
1884 2587555 : for (unsigned o = 0; o < frame_size; o += kSystemPointerSize) {
1885 : SetFrameSlot(o, kZapUint32);
1886 : }
1887 96735 : }
1888 :
1889 85911549 : void TranslationBuffer::Add(int32_t value) {
1890 : // This wouldn't handle kMinInt correctly if it ever encountered it.
1891 : DCHECK_NE(value, kMinInt);
1892 : // Encode the sign bit in the least significant bit.
1893 85911549 : bool is_negative = (value < 0);
1894 85911549 : uint32_t bits = (static_cast<uint32_t>(is_negative ? -value : value) << 1) |
1895 85911549 : static_cast<uint32_t>(is_negative);
1896 : // Encode the individual bytes using the least significant bit of
1897 : // each byte to indicate whether or not more bytes follow.
1898 : do {
1899 90323527 : uint32_t next = bits >> 7;
1900 146306953 : contents_.push_back(((bits << 1) & 0xFF) | (next != 0));
1901 : bits = next;
1902 90323668 : } while (bits != 0);
1903 85911690 : }
1904 :
1905 43066 : TranslationIterator::TranslationIterator(ByteArray buffer, int index)
1906 1408276 : : buffer_(buffer), index_(index) {
1907 : DCHECK(index >= 0 && index < buffer->length());
1908 43066 : }
1909 :
1910 59205752 : int32_t TranslationIterator::Next() {
1911 : // Run through the bytes until we reach one with a least significant
1912 : // bit of zero (marks the end).
1913 : uint32_t bits = 0;
1914 543264 : for (int i = 0; true; i += 7) {
1915 : DCHECK(HasNext());
1916 59749016 : uint8_t next = buffer_->get(index_++);
1917 59749016 : bits |= (next >> 1) << i;
1918 59749016 : if ((next & 1) == 0) break;
1919 : }
1920 : // The bits encode the sign in the least significant bit.
1921 59205752 : bool is_negative = (bits & 1) == 1;
1922 59205752 : int32_t result = bits >> 1;
1923 59205752 : return is_negative ? -result : result;
1924 : }
1925 :
1926 2730420 : bool TranslationIterator::HasNext() const { return index_ < buffer_->length(); }
1927 :
1928 463631 : Handle<ByteArray> TranslationBuffer::CreateByteArray(Factory* factory) {
1929 : Handle<ByteArray> result =
1930 463631 : factory->NewByteArray(CurrentIndex(), AllocationType::kOld);
1931 463633 : contents_.CopyTo(result->GetDataStartAddress());
1932 463633 : return result;
1933 : }
1934 :
1935 2086 : void Translation::BeginBuiltinContinuationFrame(BailoutId bailout_id,
1936 : int literal_id,
1937 : unsigned height) {
1938 2086 : buffer_->Add(BUILTIN_CONTINUATION_FRAME);
1939 2086 : buffer_->Add(bailout_id.ToInt());
1940 2086 : buffer_->Add(literal_id);
1941 2086 : buffer_->Add(height);
1942 2086 : }
1943 :
1944 12285 : void Translation::BeginJavaScriptBuiltinContinuationFrame(BailoutId bailout_id,
1945 : int literal_id,
1946 : unsigned height) {
1947 12285 : buffer_->Add(JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME);
1948 12285 : buffer_->Add(bailout_id.ToInt());
1949 12285 : buffer_->Add(literal_id);
1950 12285 : buffer_->Add(height);
1951 12285 : }
1952 :
1953 274 : void Translation::BeginJavaScriptBuiltinContinuationWithCatchFrame(
1954 : BailoutId bailout_id, int literal_id, unsigned height) {
1955 274 : buffer_->Add(JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME);
1956 274 : buffer_->Add(bailout_id.ToInt());
1957 274 : buffer_->Add(literal_id);
1958 274 : buffer_->Add(height);
1959 274 : }
1960 :
1961 26390 : void Translation::BeginConstructStubFrame(BailoutId bailout_id, int literal_id,
1962 : unsigned height) {
1963 26390 : buffer_->Add(CONSTRUCT_STUB_FRAME);
1964 26390 : buffer_->Add(bailout_id.ToInt());
1965 26390 : buffer_->Add(literal_id);
1966 26390 : buffer_->Add(height);
1967 26390 : }
1968 :
1969 :
1970 84306 : void Translation::BeginArgumentsAdaptorFrame(int literal_id, unsigned height) {
1971 84306 : buffer_->Add(ARGUMENTS_ADAPTOR_FRAME);
1972 84306 : buffer_->Add(literal_id);
1973 84306 : buffer_->Add(height);
1974 84306 : }
1975 :
1976 3830514 : void Translation::BeginInterpretedFrame(BailoutId bytecode_offset,
1977 : int literal_id, unsigned height,
1978 : int return_value_offset,
1979 : int return_value_count) {
1980 3830514 : buffer_->Add(INTERPRETED_FRAME);
1981 3830518 : buffer_->Add(bytecode_offset.ToInt());
1982 3830521 : buffer_->Add(literal_id);
1983 3830521 : buffer_->Add(height);
1984 3830531 : buffer_->Add(return_value_offset);
1985 3830535 : buffer_->Add(return_value_count);
1986 3830531 : }
1987 :
1988 5954 : void Translation::ArgumentsElements(CreateArgumentsType type) {
1989 5954 : buffer_->Add(ARGUMENTS_ELEMENTS);
1990 5954 : buffer_->Add(static_cast<uint8_t>(type));
1991 5954 : }
1992 :
1993 6262 : void Translation::ArgumentsLength(CreateArgumentsType type) {
1994 6262 : buffer_->Add(ARGUMENTS_LENGTH);
1995 6262 : buffer_->Add(static_cast<uint8_t>(type));
1996 6262 : }
1997 :
1998 90086 : void Translation::BeginCapturedObject(int length) {
1999 90086 : buffer_->Add(CAPTURED_OBJECT);
2000 90086 : buffer_->Add(length);
2001 90086 : }
2002 :
2003 :
2004 19797 : void Translation::DuplicateObject(int object_index) {
2005 19797 : buffer_->Add(DUPLICATED_OBJECT);
2006 19797 : buffer_->Add(object_index);
2007 19797 : }
2008 :
2009 :
2010 461834 : void Translation::StoreRegister(Register reg) {
2011 461834 : buffer_->Add(REGISTER);
2012 461831 : buffer_->Add(reg.code());
2013 461835 : }
2014 :
2015 :
2016 35070 : void Translation::StoreInt32Register(Register reg) {
2017 35070 : buffer_->Add(INT32_REGISTER);
2018 35070 : buffer_->Add(reg.code());
2019 35070 : }
2020 :
2021 14 : void Translation::StoreInt64Register(Register reg) {
2022 14 : buffer_->Add(INT64_REGISTER);
2023 14 : buffer_->Add(reg.code());
2024 14 : }
2025 :
2026 1104 : void Translation::StoreUint32Register(Register reg) {
2027 1104 : buffer_->Add(UINT32_REGISTER);
2028 1104 : buffer_->Add(reg.code());
2029 1104 : }
2030 :
2031 :
2032 3205 : void Translation::StoreBoolRegister(Register reg) {
2033 3205 : buffer_->Add(BOOL_REGISTER);
2034 3205 : buffer_->Add(reg.code());
2035 3205 : }
2036 :
2037 118 : void Translation::StoreFloatRegister(FloatRegister reg) {
2038 118 : buffer_->Add(FLOAT_REGISTER);
2039 118 : buffer_->Add(reg.code());
2040 118 : }
2041 :
2042 64071 : void Translation::StoreDoubleRegister(DoubleRegister reg) {
2043 64071 : buffer_->Add(DOUBLE_REGISTER);
2044 64073 : buffer_->Add(reg.code());
2045 64073 : }
2046 :
2047 :
2048 18961964 : void Translation::StoreStackSlot(int index) {
2049 18961964 : buffer_->Add(STACK_SLOT);
2050 18961991 : buffer_->Add(index);
2051 18961949 : }
2052 :
2053 :
2054 481409 : void Translation::StoreInt32StackSlot(int index) {
2055 481409 : buffer_->Add(INT32_STACK_SLOT);
2056 481408 : buffer_->Add(index);
2057 481408 : }
2058 :
2059 237 : void Translation::StoreInt64StackSlot(int index) {
2060 237 : buffer_->Add(INT64_STACK_SLOT);
2061 237 : buffer_->Add(index);
2062 237 : }
2063 :
2064 7150 : void Translation::StoreUint32StackSlot(int index) {
2065 7150 : buffer_->Add(UINT32_STACK_SLOT);
2066 7150 : buffer_->Add(index);
2067 7150 : }
2068 :
2069 :
2070 21489 : void Translation::StoreBoolStackSlot(int index) {
2071 21489 : buffer_->Add(BOOL_STACK_SLOT);
2072 21489 : buffer_->Add(index);
2073 21489 : }
2074 :
2075 897 : void Translation::StoreFloatStackSlot(int index) {
2076 897 : buffer_->Add(FLOAT_STACK_SLOT);
2077 897 : buffer_->Add(index);
2078 897 : }
2079 :
2080 258735 : void Translation::StoreDoubleStackSlot(int index) {
2081 258735 : buffer_->Add(DOUBLE_STACK_SLOT);
2082 258735 : buffer_->Add(index);
2083 258735 : }
2084 :
2085 :
2086 31567368 : void Translation::StoreLiteral(int literal_id) {
2087 31567368 : buffer_->Add(LITERAL);
2088 31567211 : buffer_->Add(literal_id);
2089 31567275 : }
2090 :
2091 40807 : void Translation::AddUpdateFeedback(int vector_literal, int slot) {
2092 40807 : buffer_->Add(UPDATE_FEEDBACK);
2093 40807 : buffer_->Add(vector_literal);
2094 40808 : buffer_->Add(slot);
2095 40808 : }
2096 :
2097 3630538 : void Translation::StoreJSFrameFunction() {
2098 : StoreStackSlot((StandardFrameConstants::kCallerPCOffset -
2099 : StandardFrameConstants::kFunctionOffset) /
2100 3630538 : kSystemPointerSize);
2101 3630537 : }
2102 :
2103 182018 : int Translation::NumberOfOperandsFor(Opcode opcode) {
2104 182018 : switch (opcode) {
2105 : case DUPLICATED_OBJECT:
2106 : case ARGUMENTS_ELEMENTS:
2107 : case ARGUMENTS_LENGTH:
2108 : case CAPTURED_OBJECT:
2109 : case REGISTER:
2110 : case INT32_REGISTER:
2111 : case INT64_REGISTER:
2112 : case UINT32_REGISTER:
2113 : case BOOL_REGISTER:
2114 : case FLOAT_REGISTER:
2115 : case DOUBLE_REGISTER:
2116 : case STACK_SLOT:
2117 : case INT32_STACK_SLOT:
2118 : case INT64_STACK_SLOT:
2119 : case UINT32_STACK_SLOT:
2120 : case BOOL_STACK_SLOT:
2121 : case FLOAT_STACK_SLOT:
2122 : case DOUBLE_STACK_SLOT:
2123 : case LITERAL:
2124 : return 1;
2125 : case ARGUMENTS_ADAPTOR_FRAME:
2126 : case UPDATE_FEEDBACK:
2127 0 : return 2;
2128 : case BEGIN:
2129 : case CONSTRUCT_STUB_FRAME:
2130 : case BUILTIN_CONTINUATION_FRAME:
2131 : case JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME:
2132 : case JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME:
2133 0 : return 3;
2134 : case INTERPRETED_FRAME:
2135 53355 : return 5;
2136 : }
2137 0 : FATAL("Unexpected translation type");
2138 : return -1;
2139 : }
2140 :
2141 :
2142 : #if defined(OBJECT_PRINT) || defined(ENABLE_DISASSEMBLER)
2143 :
2144 : const char* Translation::StringFor(Opcode opcode) {
2145 : #define TRANSLATION_OPCODE_CASE(item) case item: return #item;
2146 : switch (opcode) {
2147 : TRANSLATION_OPCODE_LIST(TRANSLATION_OPCODE_CASE)
2148 : }
2149 : #undef TRANSLATION_OPCODE_CASE
2150 : UNREACHABLE();
2151 : }
2152 :
2153 : #endif
2154 :
2155 :
2156 1365392 : Handle<FixedArray> MaterializedObjectStore::Get(Address fp) {
2157 1365392 : int index = StackIdToIndex(fp);
2158 1365392 : if (index == -1) {
2159 : return Handle<FixedArray>::null();
2160 : }
2161 246 : Handle<FixedArray> array = GetStackEntries();
2162 246 : CHECK_GT(array->length(), index);
2163 : return Handle<FixedArray>::cast(Handle<Object>(array->get(index), isolate()));
2164 : }
2165 :
2166 :
2167 150 : void MaterializedObjectStore::Set(Address fp,
2168 : Handle<FixedArray> materialized_objects) {
2169 150 : int index = StackIdToIndex(fp);
2170 150 : if (index == -1) {
2171 150 : index = static_cast<int>(frame_fps_.size());
2172 150 : frame_fps_.push_back(fp);
2173 : }
2174 :
2175 150 : Handle<FixedArray> array = EnsureStackEntries(index + 1);
2176 300 : array->set(index, *materialized_objects);
2177 150 : }
2178 :
2179 :
2180 3051820 : bool MaterializedObjectStore::Remove(Address fp) {
2181 3051820 : auto it = std::find(frame_fps_.begin(), frame_fps_.end(), fp);
2182 3051820 : if (it == frame_fps_.end()) return false;
2183 150 : int index = static_cast<int>(std::distance(frame_fps_.begin(), it));
2184 :
2185 150 : frame_fps_.erase(it);
2186 150 : FixedArray array = isolate()->heap()->materialized_objects();
2187 :
2188 150 : CHECK_LT(index, array->length());
2189 150 : int fps_size = static_cast<int>(frame_fps_.size());
2190 150 : for (int i = index; i < fps_size; i++) {
2191 0 : array->set(i, array->get(i + 1));
2192 : }
2193 150 : array->set(fps_size, ReadOnlyRoots(isolate()).undefined_value());
2194 150 : return true;
2195 : }
2196 :
2197 :
2198 1365542 : int MaterializedObjectStore::StackIdToIndex(Address fp) {
2199 1365542 : auto it = std::find(frame_fps_.begin(), frame_fps_.end(), fp);
2200 : return it == frame_fps_.end()
2201 : ? -1
2202 1365788 : : static_cast<int>(std::distance(frame_fps_.begin(), it));
2203 : }
2204 :
2205 :
2206 396 : Handle<FixedArray> MaterializedObjectStore::GetStackEntries() {
2207 : return Handle<FixedArray>(isolate()->heap()->materialized_objects(),
2208 396 : isolate());
2209 : }
2210 :
2211 :
2212 150 : Handle<FixedArray> MaterializedObjectStore::EnsureStackEntries(int length) {
2213 150 : Handle<FixedArray> array = GetStackEntries();
2214 150 : if (array->length() >= length) {
2215 102 : return array;
2216 : }
2217 :
2218 48 : int new_length = length > 10 ? length : 10;
2219 48 : if (new_length < 2 * array->length()) {
2220 : new_length = 2 * array->length();
2221 : }
2222 :
2223 : Handle<FixedArray> new_array =
2224 48 : isolate()->factory()->NewFixedArray(new_length, AllocationType::kOld);
2225 48 : for (int i = 0; i < array->length(); i++) {
2226 0 : new_array->set(i, array->get(i));
2227 : }
2228 48 : HeapObject undefined_value = ReadOnlyRoots(isolate()).undefined_value();
2229 144 : for (int i = array->length(); i < length; i++) {
2230 48 : new_array->set(i, undefined_value);
2231 : }
2232 : isolate()->heap()->SetRootMaterializedObjects(*new_array);
2233 48 : return new_array;
2234 : }
2235 :
2236 : namespace {
2237 :
2238 181542 : Handle<Object> GetValueForDebugger(TranslatedFrame::iterator it,
2239 : Isolate* isolate) {
2240 363084 : if (it->GetRawValue() == ReadOnlyRoots(isolate).arguments_marker()) {
2241 21261 : if (!it->IsMaterializableByDebugger()) {
2242 236 : return isolate->factory()->optimized_out();
2243 : }
2244 : }
2245 181306 : return it->GetValue();
2246 : }
2247 :
2248 : } // namespace
2249 :
2250 24974 : DeoptimizedFrameInfo::DeoptimizedFrameInfo(TranslatedState* state,
2251 : TranslatedState::iterator frame_it,
2252 : Isolate* isolate) {
2253 : int parameter_count =
2254 24974 : frame_it->shared_info()->internal_formal_parameter_count();
2255 : TranslatedFrame::iterator stack_it = frame_it->begin();
2256 :
2257 : // Get the function. Note that this might materialize the function.
2258 : // In case the debugger mutates this value, we should deoptimize
2259 : // the function and remember the value in the materialized value store.
2260 24974 : function_ = Handle<JSFunction>::cast(stack_it->GetValue());
2261 : stack_it++; // Skip the function.
2262 : stack_it++; // Skip the receiver.
2263 :
2264 : DCHECK_EQ(TranslatedFrame::kInterpretedFunction, frame_it->kind());
2265 24974 : source_position_ = Deoptimizer::ComputeSourcePositionFromBytecodeArray(
2266 24974 : *frame_it->shared_info(), frame_it->node_id());
2267 :
2268 : DCHECK_EQ(parameter_count,
2269 : function_->shared()->internal_formal_parameter_count());
2270 :
2271 24974 : parameters_.resize(static_cast<size_t>(parameter_count));
2272 107500 : for (int i = 0; i < parameter_count; i++) {
2273 41263 : Handle<Object> parameter = GetValueForDebugger(stack_it, isolate);
2274 : SetParameter(i, parameter);
2275 : stack_it++;
2276 : }
2277 :
2278 : // Get the context.
2279 24974 : context_ = GetValueForDebugger(stack_it, isolate);
2280 : stack_it++;
2281 :
2282 : // Get the expression stack.
2283 : int stack_height = frame_it->height();
2284 24974 : if (frame_it->kind() == TranslatedFrame::kInterpretedFunction) {
2285 : // For interpreter frames, we should not count the accumulator.
2286 : // TODO(jarin): Clean up the indexing in translated frames.
2287 24974 : stack_height--;
2288 : }
2289 24974 : expression_stack_.resize(static_cast<size_t>(stack_height));
2290 255584 : for (int i = 0; i < stack_height; i++) {
2291 115305 : Handle<Object> expression = GetValueForDebugger(stack_it, isolate);
2292 : SetExpression(i, expression);
2293 : stack_it++;
2294 : }
2295 :
2296 : // For interpreter frame, skip the accumulator.
2297 24974 : if (frame_it->kind() == TranslatedFrame::kInterpretedFunction) {
2298 : stack_it++;
2299 : }
2300 24974 : CHECK(stack_it == frame_it->end());
2301 24974 : }
2302 :
2303 255 : Deoptimizer::DeoptInfo Deoptimizer::GetDeoptInfo(Code code, Address pc) {
2304 255 : CHECK(code->InstructionStart() <= pc && pc <= code->InstructionEnd());
2305 : SourcePosition last_position = SourcePosition::Unknown();
2306 : DeoptimizeReason last_reason = DeoptimizeReason::kUnknown;
2307 : int last_deopt_id = kNoDeoptimizationId;
2308 : int mask = RelocInfo::ModeMask(RelocInfo::DEOPT_REASON) |
2309 : RelocInfo::ModeMask(RelocInfo::DEOPT_ID) |
2310 : RelocInfo::ModeMask(RelocInfo::DEOPT_SCRIPT_OFFSET) |
2311 : RelocInfo::ModeMask(RelocInfo::DEOPT_INLINING_ID);
2312 2091 : for (RelocIterator it(code, mask); !it.done(); it.next()) {
2313 : RelocInfo* info = it.rinfo();
2314 2088 : if (info->pc() >= pc) break;
2315 1836 : if (info->rmode() == RelocInfo::DEOPT_SCRIPT_OFFSET) {
2316 612 : int script_offset = static_cast<int>(info->data());
2317 612 : it.next();
2318 : DCHECK(it.rinfo()->rmode() == RelocInfo::DEOPT_INLINING_ID);
2319 612 : int inlining_id = static_cast<int>(it.rinfo()->data());
2320 : last_position = SourcePosition(script_offset, inlining_id);
2321 1224 : } else if (info->rmode() == RelocInfo::DEOPT_ID) {
2322 612 : last_deopt_id = static_cast<int>(info->data());
2323 612 : } else if (info->rmode() == RelocInfo::DEOPT_REASON) {
2324 612 : last_reason = static_cast<DeoptimizeReason>(info->data());
2325 : }
2326 : }
2327 255 : return DeoptInfo(last_position, last_reason, last_deopt_id);
2328 : }
2329 :
2330 :
2331 : // static
2332 24974 : int Deoptimizer::ComputeSourcePositionFromBytecodeArray(
2333 : SharedFunctionInfo shared, BailoutId node_id) {
2334 : DCHECK(shared->HasBytecodeArray());
2335 49948 : return AbstractCode::cast(shared->GetBytecodeArray())
2336 49948 : ->SourcePosition(node_id.ToInt());
2337 : }
2338 :
2339 : // static
2340 0 : TranslatedValue TranslatedValue::NewDeferredObject(TranslatedState* container,
2341 : int length,
2342 : int object_index) {
2343 : TranslatedValue slot(container, kCapturedObject);
2344 146828 : slot.materialization_info_ = {object_index, length};
2345 0 : return slot;
2346 : }
2347 :
2348 :
2349 : // static
2350 0 : TranslatedValue TranslatedValue::NewDuplicateObject(TranslatedState* container,
2351 : int id) {
2352 : TranslatedValue slot(container, kDuplicatedObject);
2353 83254 : slot.materialization_info_ = {id, -1};
2354 0 : return slot;
2355 : }
2356 :
2357 :
2358 : // static
2359 0 : TranslatedValue TranslatedValue::NewFloat(TranslatedState* container,
2360 : Float32 value) {
2361 : TranslatedValue slot(container, kFloat);
2362 288 : slot.float_value_ = value;
2363 0 : return slot;
2364 : }
2365 :
2366 : // static
2367 0 : TranslatedValue TranslatedValue::NewDouble(TranslatedState* container,
2368 : Float64 value) {
2369 : TranslatedValue slot(container, kDouble);
2370 521480 : slot.double_value_ = value;
2371 0 : return slot;
2372 : }
2373 :
2374 :
2375 : // static
2376 0 : TranslatedValue TranslatedValue::NewInt32(TranslatedState* container,
2377 : int32_t value) {
2378 : TranslatedValue slot(container, kInt32);
2379 314315 : slot.int32_value_ = value;
2380 0 : return slot;
2381 : }
2382 :
2383 : // static
2384 0 : TranslatedValue TranslatedValue::NewInt64(TranslatedState* container,
2385 : int64_t value) {
2386 : TranslatedValue slot(container, kInt64);
2387 8 : slot.int64_value_ = value;
2388 0 : return slot;
2389 : }
2390 :
2391 : // static
2392 0 : TranslatedValue TranslatedValue::NewUInt32(TranslatedState* container,
2393 : uint32_t value) {
2394 : TranslatedValue slot(container, kUInt32);
2395 53523 : slot.uint32_value_ = value;
2396 0 : return slot;
2397 : }
2398 :
2399 :
2400 : // static
2401 0 : TranslatedValue TranslatedValue::NewBool(TranslatedState* container,
2402 : uint32_t value) {
2403 : TranslatedValue slot(container, kBoolBit);
2404 437 : slot.uint32_value_ = value;
2405 0 : return slot;
2406 : }
2407 :
2408 :
2409 : // static
2410 0 : TranslatedValue TranslatedValue::NewTagged(TranslatedState* container,
2411 : Object literal) {
2412 : TranslatedValue slot(container, kTagged);
2413 19753199 : slot.raw_literal_ = literal;
2414 0 : return slot;
2415 : }
2416 :
2417 : // static
2418 0 : TranslatedValue TranslatedValue::NewInvalid(TranslatedState* container) {
2419 0 : return TranslatedValue(container, kInvalid);
2420 : }
2421 :
2422 :
2423 19853844 : Isolate* TranslatedValue::isolate() const { return container_->isolate(); }
2424 :
2425 0 : Object TranslatedValue::raw_literal() const {
2426 : DCHECK_EQ(kTagged, kind());
2427 20355464 : return raw_literal_;
2428 : }
2429 :
2430 0 : int32_t TranslatedValue::int32_value() const {
2431 : DCHECK_EQ(kInt32, kind());
2432 27214 : return int32_value_;
2433 : }
2434 :
2435 0 : int64_t TranslatedValue::int64_value() const {
2436 : DCHECK_EQ(kInt64, kind());
2437 8 : return int64_value_;
2438 : }
2439 :
2440 0 : uint32_t TranslatedValue::uint32_value() const {
2441 : DCHECK(kind() == kUInt32 || kind() == kBoolBit);
2442 158 : return uint32_value_;
2443 : }
2444 :
2445 0 : Float32 TranslatedValue::float_value() const {
2446 : DCHECK_EQ(kFloat, kind());
2447 280 : return float_value_;
2448 : }
2449 :
2450 0 : Float64 TranslatedValue::double_value() const {
2451 : DCHECK_EQ(kDouble, kind());
2452 23814 : return double_value_;
2453 : }
2454 :
2455 :
2456 0 : int TranslatedValue::object_length() const {
2457 : DCHECK_EQ(kind(), kCapturedObject);
2458 1852538 : return materialization_info_.length_;
2459 : }
2460 :
2461 :
2462 0 : int TranslatedValue::object_index() const {
2463 : DCHECK(kind() == kCapturedObject || kind() == kDuplicatedObject);
2464 143917 : return materialization_info_.id_;
2465 : }
2466 :
2467 842503 : Object TranslatedValue::GetRawValue() const {
2468 : // If we have a value, return it.
2469 842503 : if (materialization_state() == kFinished) {
2470 : return *storage_;
2471 : }
2472 :
2473 : // Otherwise, do a best effort to get the value without allocation.
2474 692097 : switch (kind()) {
2475 : case kTagged:
2476 : return raw_literal();
2477 :
2478 : case kInt32: {
2479 : bool is_smi = Smi::IsValid(int32_value());
2480 27163 : if (is_smi) {
2481 27061 : return Smi::FromInt(int32_value());
2482 : }
2483 : break;
2484 : }
2485 :
2486 : case kInt64: {
2487 8 : bool is_smi = (int64_value() >= static_cast<int64_t>(Smi::kMinValue) &&
2488 : int64_value() <= static_cast<int64_t>(Smi::kMaxValue));
2489 8 : if (is_smi) {
2490 8 : return Smi::FromIntptr(static_cast<intptr_t>(int64_value()));
2491 : }
2492 : break;
2493 : }
2494 :
2495 : case kUInt32: {
2496 : bool is_smi = (uint32_value() <= static_cast<uintptr_t>(Smi::kMaxValue));
2497 77 : if (is_smi) {
2498 126 : return Smi::FromInt(static_cast<int32_t>(uint32_value()));
2499 : }
2500 : break;
2501 : }
2502 :
2503 : case kBoolBit: {
2504 74 : if (uint32_value() == 0) {
2505 57 : return ReadOnlyRoots(isolate()).false_value();
2506 : } else {
2507 17 : CHECK_EQ(1U, uint32_value());
2508 17 : return ReadOnlyRoots(isolate()).true_value();
2509 : }
2510 : }
2511 :
2512 : default:
2513 : break;
2514 : }
2515 :
2516 : // If we could not get the value without allocation, return the arguments
2517 : // marker.
2518 62626 : return ReadOnlyRoots(isolate()).arguments_marker();
2519 : }
2520 :
2521 0 : void TranslatedValue::set_initialized_storage(Handle<Object> storage) {
2522 : DCHECK_EQ(kUninitialized, materialization_state());
2523 19791490 : storage_ = storage;
2524 19791490 : materialization_state_ = kFinished;
2525 0 : }
2526 :
2527 3490280 : Handle<Object> TranslatedValue::GetValue() {
2528 : // If we already have a value, then get it.
2529 3490280 : if (materialization_state() == kFinished) return storage_;
2530 :
2531 : // Otherwise we have to materialize.
2532 80675 : switch (kind()) {
2533 : case TranslatedValue::kTagged:
2534 : case TranslatedValue::kInt32:
2535 : case TranslatedValue::kInt64:
2536 : case TranslatedValue::kUInt32:
2537 : case TranslatedValue::kBoolBit:
2538 : case TranslatedValue::kFloat:
2539 : case TranslatedValue::kDouble: {
2540 37404 : MaterializeSimple();
2541 37404 : return storage_;
2542 : }
2543 :
2544 : case TranslatedValue::kCapturedObject:
2545 : case TranslatedValue::kDuplicatedObject: {
2546 : // We need to materialize the object (or possibly even object graphs).
2547 : // To make the object verifier happy, we materialize in two steps.
2548 :
2549 : // 1. Allocate storage for reachable objects. This makes sure that for
2550 : // each object we have allocated space on heap. The space will be
2551 : // a byte array that will be later initialized, or a fully
2552 : // initialized object if it is safe to allocate one that will
2553 : // pass the verifier.
2554 43271 : container_->EnsureObjectAllocatedAt(this);
2555 :
2556 : // 2. Initialize the objects. If we have allocated only byte arrays
2557 : // for some objects, we now overwrite the byte arrays with the
2558 : // correct object fields. Note that this phase does not allocate
2559 : // any new objects, so it does not trigger the object verifier.
2560 43271 : return container_->InitializeObjectAt(this);
2561 : }
2562 :
2563 : case TranslatedValue::kInvalid:
2564 0 : FATAL("unexpected case");
2565 : return Handle<Object>::null();
2566 : }
2567 :
2568 0 : FATAL("internal error: value missing");
2569 : return Handle<Object>::null();
2570 : }
2571 :
2572 686745 : void TranslatedValue::MaterializeSimple() {
2573 : // If we already have materialized, return.
2574 1373490 : if (materialization_state() == kFinished) return;
2575 :
2576 37945 : Object raw_value = GetRawValue();
2577 37945 : if (raw_value != ReadOnlyRoots(isolate()).arguments_marker()) {
2578 : // We can get the value without allocation, just return it here.
2579 : set_initialized_storage(Handle<Object>(raw_value, isolate()));
2580 13793 : return;
2581 : }
2582 :
2583 24152 : switch (kind()) {
2584 : case kInt32:
2585 51 : set_initialized_storage(
2586 : Handle<Object>(isolate()->factory()->NewNumber(int32_value())));
2587 : return;
2588 :
2589 : case kInt64:
2590 0 : set_initialized_storage(Handle<Object>(
2591 : isolate()->factory()->NewNumber(static_cast<double>(int64_value()))));
2592 : return;
2593 :
2594 : case kUInt32:
2595 7 : set_initialized_storage(
2596 : Handle<Object>(isolate()->factory()->NewNumber(uint32_value())));
2597 : return;
2598 :
2599 : case kFloat: {
2600 280 : double scalar_value = float_value().get_scalar();
2601 280 : set_initialized_storage(
2602 : Handle<Object>(isolate()->factory()->NewNumber(scalar_value)));
2603 : return;
2604 : }
2605 :
2606 : case kDouble: {
2607 : double scalar_value = double_value().get_scalar();
2608 23814 : set_initialized_storage(
2609 : Handle<Object>(isolate()->factory()->NewNumber(scalar_value)));
2610 : return;
2611 : }
2612 :
2613 : case kCapturedObject:
2614 : case kDuplicatedObject:
2615 : case kInvalid:
2616 : case kTagged:
2617 : case kBoolBit:
2618 0 : FATAL("internal error: unexpected materialization.");
2619 : break;
2620 : }
2621 : }
2622 :
2623 :
2624 10106 : bool TranslatedValue::IsMaterializedObject() const {
2625 10950 : switch (kind()) {
2626 : case kCapturedObject:
2627 : case kDuplicatedObject:
2628 : return true;
2629 : default:
2630 9924 : return false;
2631 : }
2632 : }
2633 :
2634 0 : bool TranslatedValue::IsMaterializableByDebugger() const {
2635 : // At the moment, we only allow materialization of doubles.
2636 0 : return (kind() == kDouble);
2637 : }
2638 :
2639 0 : int TranslatedValue::GetChildrenCount() const {
2640 26738670 : if (kind() == kCapturedObject) {
2641 0 : return object_length();
2642 : } else {
2643 : return 0;
2644 : }
2645 : }
2646 :
2647 0 : uint64_t TranslatedState::GetUInt64Slot(Address fp, int slot_offset) {
2648 : #if V8_TARGET_ARCH_32_BIT
2649 : return ReadUnalignedValue<uint64_t>(fp + slot_offset);
2650 : #else
2651 1042130 : return Memory<uint64_t>(fp + slot_offset);
2652 : #endif
2653 : }
2654 :
2655 0 : uint32_t TranslatedState::GetUInt32Slot(Address fp, int slot_offset) {
2656 366984 : Address address = fp + slot_offset;
2657 : #if V8_TARGET_BIG_ENDIAN && V8_HOST_ARCH_64_BIT
2658 : return Memory<uint32_t>(address + kIntSize);
2659 : #else
2660 366984 : return Memory<uint32_t>(address);
2661 : #endif
2662 : }
2663 :
2664 0 : Float32 TranslatedState::GetFloatSlot(Address fp, int slot_offset) {
2665 : #if !V8_TARGET_ARCH_S390X && !V8_TARGET_ARCH_PPC64
2666 0 : return Float32::FromBits(GetUInt32Slot(fp, slot_offset));
2667 : #else
2668 : return Float32::FromBits(Memory<uint32_t>(fp + slot_offset));
2669 : #endif
2670 : }
2671 :
2672 0 : Float64 TranslatedState::GetDoubleSlot(Address fp, int slot_offset) {
2673 0 : return Float64::FromBits(GetUInt64Slot(fp, slot_offset));
2674 : }
2675 :
2676 20873332 : void TranslatedValue::Handlify() {
2677 20873332 : if (kind() == kTagged) {
2678 : set_initialized_storage(Handle<Object>(raw_literal(), isolate()));
2679 19753199 : raw_literal_ = Object();
2680 : }
2681 20873332 : }
2682 :
2683 0 : TranslatedFrame TranslatedFrame::InterpretedFrame(
2684 : BailoutId bytecode_offset, SharedFunctionInfo shared_info, int height,
2685 : int return_value_offset, int return_value_count) {
2686 : TranslatedFrame frame(kInterpretedFunction, shared_info, height,
2687 : return_value_offset, return_value_count);
2688 1719746 : frame.node_id_ = bytecode_offset;
2689 0 : return frame;
2690 : }
2691 :
2692 0 : TranslatedFrame TranslatedFrame::ArgumentsAdaptorFrame(
2693 : SharedFunctionInfo shared_info, int height) {
2694 0 : return TranslatedFrame(kArgumentsAdaptor, shared_info, height);
2695 : }
2696 :
2697 0 : TranslatedFrame TranslatedFrame::ConstructStubFrame(
2698 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2699 : TranslatedFrame frame(kConstructStub, shared_info, height);
2700 40126 : frame.node_id_ = bailout_id;
2701 0 : return frame;
2702 : }
2703 :
2704 0 : TranslatedFrame TranslatedFrame::BuiltinContinuationFrame(
2705 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2706 : TranslatedFrame frame(kBuiltinContinuation, shared_info, height);
2707 40764 : frame.node_id_ = bailout_id;
2708 0 : return frame;
2709 : }
2710 :
2711 0 : TranslatedFrame TranslatedFrame::JavaScriptBuiltinContinuationFrame(
2712 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2713 : TranslatedFrame frame(kJavaScriptBuiltinContinuation, shared_info, height);
2714 2095 : frame.node_id_ = bailout_id;
2715 0 : return frame;
2716 : }
2717 :
2718 0 : TranslatedFrame TranslatedFrame::JavaScriptBuiltinContinuationWithCatchFrame(
2719 : BailoutId bailout_id, SharedFunctionInfo shared_info, int height) {
2720 : TranslatedFrame frame(kJavaScriptBuiltinContinuationWithCatch, shared_info,
2721 : height);
2722 86 : frame.node_id_ = bailout_id;
2723 0 : return frame;
2724 : }
2725 :
2726 1941782 : int TranslatedFrame::GetValueCount() {
2727 1941782 : switch (kind()) {
2728 : case kInterpretedFunction: {
2729 : int parameter_count =
2730 1722146 : raw_shared_info_->internal_formal_parameter_count() + 1;
2731 : // + 2 for function and context.
2732 1722146 : return height_ + parameter_count + 2;
2733 : }
2734 :
2735 : case kArgumentsAdaptor:
2736 : case kConstructStub:
2737 : case kBuiltinContinuation:
2738 : case kJavaScriptBuiltinContinuation:
2739 : case kJavaScriptBuiltinContinuationWithCatch:
2740 219636 : return 1 + height_;
2741 :
2742 : case kInvalid:
2743 0 : UNREACHABLE();
2744 : break;
2745 : }
2746 0 : UNREACHABLE();
2747 : }
2748 :
2749 :
2750 1939382 : void TranslatedFrame::Handlify() {
2751 1939382 : if (!raw_shared_info_.is_null()) {
2752 : shared_info_ = Handle<SharedFunctionInfo>(raw_shared_info_,
2753 1939382 : raw_shared_info_->GetIsolate());
2754 1939382 : raw_shared_info_ = SharedFunctionInfo();
2755 : }
2756 22812714 : for (auto& value : values_) {
2757 20873332 : value.Handlify();
2758 : }
2759 1939382 : }
2760 :
2761 1939382 : TranslatedFrame TranslatedState::CreateNextTranslatedFrame(
2762 : TranslationIterator* iterator, FixedArray literal_array, Address fp,
2763 : FILE* trace_file) {
2764 : Translation::Opcode opcode =
2765 1939382 : static_cast<Translation::Opcode>(iterator->Next());
2766 1939382 : switch (opcode) {
2767 : case Translation::INTERPRETED_FRAME: {
2768 1719746 : BailoutId bytecode_offset = BailoutId(iterator->Next());
2769 : SharedFunctionInfo shared_info =
2770 3439492 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2771 1719746 : int height = iterator->Next();
2772 1719746 : int return_value_offset = iterator->Next();
2773 1719746 : int return_value_count = iterator->Next();
2774 1719746 : if (trace_file != nullptr) {
2775 240 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2776 240 : PrintF(trace_file, " reading input frame %s", name.get());
2777 240 : int arg_count = shared_info->internal_formal_parameter_count() + 1;
2778 : PrintF(trace_file,
2779 : " => bytecode_offset=%d, args=%d, height=%d, retval=%i(#%i); "
2780 : "inputs:\n",
2781 : bytecode_offset.ToInt(), arg_count, height, return_value_offset,
2782 240 : return_value_count);
2783 : }
2784 : return TranslatedFrame::InterpretedFrame(bytecode_offset, shared_info,
2785 : height, return_value_offset,
2786 : return_value_count);
2787 : }
2788 :
2789 : case Translation::ARGUMENTS_ADAPTOR_FRAME: {
2790 : SharedFunctionInfo shared_info =
2791 273130 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2792 136565 : int height = iterator->Next();
2793 136565 : if (trace_file != nullptr) {
2794 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2795 0 : PrintF(trace_file, " reading arguments adaptor frame %s", name.get());
2796 0 : PrintF(trace_file, " => height=%d; inputs:\n", height);
2797 : }
2798 : return TranslatedFrame::ArgumentsAdaptorFrame(shared_info, height);
2799 : }
2800 :
2801 : case Translation::CONSTRUCT_STUB_FRAME: {
2802 40126 : BailoutId bailout_id = BailoutId(iterator->Next());
2803 : SharedFunctionInfo shared_info =
2804 80252 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2805 40126 : int height = iterator->Next();
2806 40126 : if (trace_file != nullptr) {
2807 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2808 0 : PrintF(trace_file, " reading construct stub frame %s", name.get());
2809 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2810 0 : bailout_id.ToInt(), height);
2811 : }
2812 : return TranslatedFrame::ConstructStubFrame(bailout_id, shared_info,
2813 : height);
2814 : }
2815 :
2816 : case Translation::BUILTIN_CONTINUATION_FRAME: {
2817 40764 : BailoutId bailout_id = BailoutId(iterator->Next());
2818 : SharedFunctionInfo shared_info =
2819 81528 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2820 40764 : int height = iterator->Next();
2821 40764 : if (trace_file != nullptr) {
2822 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2823 : PrintF(trace_file, " reading builtin continuation frame %s",
2824 0 : name.get());
2825 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2826 0 : bailout_id.ToInt(), height);
2827 : }
2828 : // Add one to the height to account for the context which was implicitly
2829 : // added to the translation during code generation.
2830 40764 : int height_with_context = height + 1;
2831 : return TranslatedFrame::BuiltinContinuationFrame(bailout_id, shared_info,
2832 : height_with_context);
2833 : }
2834 :
2835 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME: {
2836 2095 : BailoutId bailout_id = BailoutId(iterator->Next());
2837 : SharedFunctionInfo shared_info =
2838 4190 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2839 2095 : int height = iterator->Next();
2840 2095 : if (trace_file != nullptr) {
2841 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2842 : PrintF(trace_file, " reading JavaScript builtin continuation frame %s",
2843 0 : name.get());
2844 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2845 0 : bailout_id.ToInt(), height);
2846 : }
2847 : // Add one to the height to account for the context which was implicitly
2848 : // added to the translation during code generation.
2849 2095 : int height_with_context = height + 1;
2850 : return TranslatedFrame::JavaScriptBuiltinContinuationFrame(
2851 : bailout_id, shared_info, height_with_context);
2852 : }
2853 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME: {
2854 86 : BailoutId bailout_id = BailoutId(iterator->Next());
2855 : SharedFunctionInfo shared_info =
2856 172 : SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
2857 86 : int height = iterator->Next();
2858 86 : if (trace_file != nullptr) {
2859 0 : std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
2860 : PrintF(trace_file,
2861 : " reading JavaScript builtin continuation frame with catch %s",
2862 0 : name.get());
2863 : PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
2864 0 : bailout_id.ToInt(), height);
2865 : }
2866 : // Add one to the height to account for the context which was implicitly
2867 : // added to the translation during code generation.
2868 86 : int height_with_context = height + 1;
2869 : return TranslatedFrame::JavaScriptBuiltinContinuationWithCatchFrame(
2870 : bailout_id, shared_info, height_with_context);
2871 : }
2872 : case Translation::UPDATE_FEEDBACK:
2873 : case Translation::BEGIN:
2874 : case Translation::DUPLICATED_OBJECT:
2875 : case Translation::ARGUMENTS_ELEMENTS:
2876 : case Translation::ARGUMENTS_LENGTH:
2877 : case Translation::CAPTURED_OBJECT:
2878 : case Translation::REGISTER:
2879 : case Translation::INT32_REGISTER:
2880 : case Translation::INT64_REGISTER:
2881 : case Translation::UINT32_REGISTER:
2882 : case Translation::BOOL_REGISTER:
2883 : case Translation::FLOAT_REGISTER:
2884 : case Translation::DOUBLE_REGISTER:
2885 : case Translation::STACK_SLOT:
2886 : case Translation::INT32_STACK_SLOT:
2887 : case Translation::INT64_STACK_SLOT:
2888 : case Translation::UINT32_STACK_SLOT:
2889 : case Translation::BOOL_STACK_SLOT:
2890 : case Translation::FLOAT_STACK_SLOT:
2891 : case Translation::DOUBLE_STACK_SLOT:
2892 : case Translation::LITERAL:
2893 : break;
2894 : }
2895 0 : FATAL("We should never get here - unexpected deopt info.");
2896 : return TranslatedFrame::InvalidFrame();
2897 : }
2898 :
2899 : // static
2900 4103704 : void TranslatedFrame::AdvanceIterator(
2901 : std::deque<TranslatedValue>::iterator* iter) {
2902 : int values_to_skip = 1;
2903 8854880 : while (values_to_skip > 0) {
2904 : // Consume the current element.
2905 4751176 : values_to_skip--;
2906 : // Add all the children.
2907 4751176 : values_to_skip += (*iter)->GetChildrenCount();
2908 :
2909 : (*iter)++;
2910 : }
2911 4103704 : }
2912 :
2913 0 : Address TranslatedState::ComputeArgumentsPosition(Address input_frame_pointer,
2914 : CreateArgumentsType type,
2915 : int* length) {
2916 : Address parent_frame_pointer = *reinterpret_cast<Address*>(
2917 487 : input_frame_pointer + StandardFrameConstants::kCallerFPOffset);
2918 : intptr_t parent_frame_type = Memory<intptr_t>(
2919 974 : parent_frame_pointer + CommonFrameConstants::kContextOrFrameTypeOffset);
2920 :
2921 : Address arguments_frame;
2922 487 : if (parent_frame_type ==
2923 : StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)) {
2924 0 : if (length)
2925 : *length = Smi::cast(*FullObjectSlot(
2926 : parent_frame_pointer +
2927 327 : ArgumentsAdaptorFrameConstants::kLengthOffset))
2928 0 : ->value();
2929 : arguments_frame = parent_frame_pointer;
2930 : } else {
2931 160 : if (length) *length = formal_parameter_count_;
2932 : arguments_frame = input_frame_pointer;
2933 : }
2934 :
2935 487 : if (type == CreateArgumentsType::kRestParameter) {
2936 : // If the actual number of arguments is less than the number of formal
2937 : // parameters, we have zero rest parameters.
2938 72 : if (length) *length = std::max(0, *length - formal_parameter_count_);
2939 : }
2940 :
2941 0 : return arguments_frame;
2942 : }
2943 :
2944 : // Creates translated values for an arguments backing store, or the backing
2945 : // store for rest parameters depending on the given {type}. The TranslatedValue
2946 : // objects for the fields are not read from the TranslationIterator, but instead
2947 : // created on-the-fly based on dynamic information in the optimized frame.
2948 240 : void TranslatedState::CreateArgumentsElementsTranslatedValues(
2949 : int frame_index, Address input_frame_pointer, CreateArgumentsType type,
2950 : FILE* trace_file) {
2951 240 : TranslatedFrame& frame = frames_[frame_index];
2952 :
2953 : int length;
2954 : Address arguments_frame =
2955 : ComputeArgumentsPosition(input_frame_pointer, type, &length);
2956 :
2957 240 : int object_index = static_cast<int>(object_positions_.size());
2958 240 : int value_index = static_cast<int>(frame.values_.size());
2959 240 : if (trace_file != nullptr) {
2960 0 : PrintF(trace_file, "arguments elements object #%d (type = %d, length = %d)",
2961 0 : object_index, static_cast<uint8_t>(type), length);
2962 : }
2963 :
2964 480 : object_positions_.push_back({frame_index, value_index});
2965 240 : frame.Add(TranslatedValue::NewDeferredObject(
2966 240 : this, length + FixedArray::kHeaderSize / kTaggedSize, object_index));
2967 :
2968 240 : ReadOnlyRoots roots(isolate_);
2969 240 : frame.Add(TranslatedValue::NewTagged(this, roots.fixed_array_map()));
2970 240 : frame.Add(TranslatedValue::NewInt32(this, length));
2971 :
2972 : int number_of_holes = 0;
2973 240 : if (type == CreateArgumentsType::kMappedArguments) {
2974 : // If the actual number of arguments is less than the number of formal
2975 : // parameters, we have fewer holes to fill to not overshoot the length.
2976 43 : number_of_holes = Min(formal_parameter_count_, length);
2977 : }
2978 356 : for (int i = 0; i < number_of_holes; ++i) {
2979 58 : frame.Add(TranslatedValue::NewTagged(this, roots.the_hole_value()));
2980 : }
2981 573921 : for (int i = length - number_of_holes - 1; i >= 0; --i) {
2982 : Address argument_slot = arguments_frame +
2983 573681 : CommonFrameConstants::kFixedFrameSizeAboveFp +
2984 1147362 : i * kSystemPointerSize;
2985 573681 : frame.Add(TranslatedValue::NewTagged(this, *FullObjectSlot(argument_slot)));
2986 : }
2987 240 : }
2988 :
2989 : // We can't intermix stack decoding and allocations because the deoptimization
2990 : // infrastracture is not GC safe.
2991 : // Thus we build a temporary structure in malloced space.
2992 : // The TranslatedValue objects created correspond to the static translation
2993 : // instructions from the TranslationIterator, except for
2994 : // Translation::ARGUMENTS_ELEMENTS, where the number and values of the
2995 : // FixedArray elements depend on dynamic information from the optimized frame.
2996 : // Returns the number of expected nested translations from the
2997 : // TranslationIterator.
2998 20299113 : int TranslatedState::CreateNextTranslatedValue(
2999 : int frame_index, TranslationIterator* iterator, FixedArray literal_array,
3000 : Address fp, RegisterValues* registers, FILE* trace_file) {
3001 : disasm::NameConverter converter;
3002 :
3003 20299113 : TranslatedFrame& frame = frames_[frame_index];
3004 20299113 : int value_index = static_cast<int>(frame.values_.size());
3005 :
3006 : Translation::Opcode opcode =
3007 20299113 : static_cast<Translation::Opcode>(iterator->Next());
3008 20299113 : switch (opcode) {
3009 : case Translation::BEGIN:
3010 : case Translation::INTERPRETED_FRAME:
3011 : case Translation::ARGUMENTS_ADAPTOR_FRAME:
3012 : case Translation::CONSTRUCT_STUB_FRAME:
3013 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME:
3014 : case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_WITH_CATCH_FRAME:
3015 : case Translation::BUILTIN_CONTINUATION_FRAME:
3016 : case Translation::UPDATE_FEEDBACK:
3017 : // Peeled off before getting here.
3018 : break;
3019 :
3020 : case Translation::DUPLICATED_OBJECT: {
3021 83254 : int object_id = iterator->Next();
3022 83254 : if (trace_file != nullptr) {
3023 0 : PrintF(trace_file, "duplicated object #%d", object_id);
3024 : }
3025 83254 : object_positions_.push_back(object_positions_[object_id]);
3026 : TranslatedValue translated_value =
3027 : TranslatedValue::NewDuplicateObject(this, object_id);
3028 : frame.Add(translated_value);
3029 : return translated_value.GetChildrenCount();
3030 : }
3031 :
3032 : case Translation::ARGUMENTS_ELEMENTS: {
3033 : CreateArgumentsType arguments_type =
3034 240 : static_cast<CreateArgumentsType>(iterator->Next());
3035 : CreateArgumentsElementsTranslatedValues(frame_index, fp, arguments_type,
3036 240 : trace_file);
3037 240 : return 0;
3038 : }
3039 :
3040 : case Translation::ARGUMENTS_LENGTH: {
3041 : CreateArgumentsType arguments_type =
3042 247 : static_cast<CreateArgumentsType>(iterator->Next());
3043 : int length;
3044 : ComputeArgumentsPosition(fp, arguments_type, &length);
3045 247 : if (trace_file != nullptr) {
3046 : PrintF(trace_file, "arguments length field (type = %d, length = %d)",
3047 0 : static_cast<uint8_t>(arguments_type), length);
3048 : }
3049 247 : frame.Add(TranslatedValue::NewInt32(this, length));
3050 : return 0;
3051 : }
3052 :
3053 : case Translation::CAPTURED_OBJECT: {
3054 146588 : int field_count = iterator->Next();
3055 146588 : int object_index = static_cast<int>(object_positions_.size());
3056 146588 : if (trace_file != nullptr) {
3057 : PrintF(trace_file, "captured object #%d (length = %d)", object_index,
3058 0 : field_count);
3059 : }
3060 293176 : object_positions_.push_back({frame_index, value_index});
3061 : TranslatedValue translated_value =
3062 : TranslatedValue::NewDeferredObject(this, field_count, object_index);
3063 : frame.Add(translated_value);
3064 : return translated_value.GetChildrenCount();
3065 : }
3066 :
3067 : case Translation::REGISTER: {
3068 26455 : int input_reg = iterator->Next();
3069 26455 : if (registers == nullptr) {
3070 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3071 : frame.Add(translated_value);
3072 : return translated_value.GetChildrenCount();
3073 : }
3074 26455 : intptr_t value = registers->GetRegister(input_reg);
3075 26455 : if (trace_file != nullptr) {
3076 360 : PrintF(trace_file, V8PRIxPTR_FMT " ; %s ", value,
3077 360 : converter.NameOfCPURegister(input_reg));
3078 720 : Object(value)->ShortPrint(trace_file);
3079 : }
3080 : TranslatedValue translated_value =
3081 26455 : TranslatedValue::NewTagged(this, Object(value));
3082 : frame.Add(translated_value);
3083 26455 : return translated_value.GetChildrenCount();
3084 : }
3085 :
3086 : case Translation::INT32_REGISTER: {
3087 930 : int input_reg = iterator->Next();
3088 930 : if (registers == nullptr) {
3089 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3090 : frame.Add(translated_value);
3091 : return translated_value.GetChildrenCount();
3092 : }
3093 930 : intptr_t value = registers->GetRegister(input_reg);
3094 930 : if (trace_file != nullptr) {
3095 0 : PrintF(trace_file, "%" V8PRIdPTR " ; %s (int32)", value,
3096 0 : converter.NameOfCPURegister(input_reg));
3097 : }
3098 : TranslatedValue translated_value =
3099 930 : TranslatedValue::NewInt32(this, static_cast<int32_t>(value));
3100 : frame.Add(translated_value);
3101 930 : return translated_value.GetChildrenCount();
3102 : }
3103 :
3104 : case Translation::INT64_REGISTER: {
3105 0 : int input_reg = iterator->Next();
3106 0 : if (registers == nullptr) {
3107 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3108 : frame.Add(translated_value);
3109 : return translated_value.GetChildrenCount();
3110 : }
3111 0 : intptr_t value = registers->GetRegister(input_reg);
3112 0 : if (trace_file != nullptr) {
3113 0 : PrintF(trace_file, "%" V8PRIdPTR " ; %s (int64)", value,
3114 0 : converter.NameOfCPURegister(input_reg));
3115 : }
3116 : TranslatedValue translated_value =
3117 : TranslatedValue::NewInt64(this, static_cast<int64_t>(value));
3118 : frame.Add(translated_value);
3119 0 : return translated_value.GetChildrenCount();
3120 : }
3121 :
3122 : case Translation::UINT32_REGISTER: {
3123 14 : int input_reg = iterator->Next();
3124 14 : if (registers == nullptr) {
3125 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3126 : frame.Add(translated_value);
3127 : return translated_value.GetChildrenCount();
3128 : }
3129 14 : intptr_t value = registers->GetRegister(input_reg);
3130 14 : if (trace_file != nullptr) {
3131 0 : PrintF(trace_file, "%" V8PRIuPTR " ; %s (uint32)", value,
3132 0 : converter.NameOfCPURegister(input_reg));
3133 : }
3134 : TranslatedValue translated_value =
3135 14 : TranslatedValue::NewUInt32(this, static_cast<uint32_t>(value));
3136 : frame.Add(translated_value);
3137 14 : return translated_value.GetChildrenCount();
3138 : }
3139 :
3140 : case Translation::BOOL_REGISTER: {
3141 57 : int input_reg = iterator->Next();
3142 57 : if (registers == nullptr) {
3143 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3144 : frame.Add(translated_value);
3145 : return translated_value.GetChildrenCount();
3146 : }
3147 57 : intptr_t value = registers->GetRegister(input_reg);
3148 57 : if (trace_file != nullptr) {
3149 0 : PrintF(trace_file, "%" V8PRIdPTR " ; %s (bool)", value,
3150 0 : converter.NameOfCPURegister(input_reg));
3151 : }
3152 : TranslatedValue translated_value =
3153 57 : TranslatedValue::NewBool(this, static_cast<uint32_t>(value));
3154 : frame.Add(translated_value);
3155 57 : return translated_value.GetChildrenCount();
3156 : }
3157 :
3158 : case Translation::FLOAT_REGISTER: {
3159 91 : int input_reg = iterator->Next();
3160 91 : if (registers == nullptr) {
3161 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3162 : frame.Add(translated_value);
3163 : return translated_value.GetChildrenCount();
3164 : }
3165 91 : Float32 value = registers->GetFloatRegister(input_reg);
3166 91 : if (trace_file != nullptr) {
3167 0 : PrintF(trace_file, "%e ; %s (float)", value.get_scalar(),
3168 0 : RegisterName(FloatRegister::from_code(input_reg)));
3169 : }
3170 : TranslatedValue translated_value = TranslatedValue::NewFloat(this, value);
3171 : frame.Add(translated_value);
3172 91 : return translated_value.GetChildrenCount();
3173 : }
3174 :
3175 : case Translation::DOUBLE_REGISTER: {
3176 423 : int input_reg = iterator->Next();
3177 423 : if (registers == nullptr) {
3178 : TranslatedValue translated_value = TranslatedValue::NewInvalid(this);
3179 : frame.Add(translated_value);
3180 : return translated_value.GetChildrenCount();
3181 : }
3182 423 : Float64 value = registers->GetDoubleRegister(input_reg);
3183 423 : if (trace_file != nullptr) {
3184 : PrintF(trace_file, "%e ; %s (double)", value.get_scalar(),
3185 0 : RegisterName(DoubleRegister::from_code(input_reg)));
3186 : }
3187 : TranslatedValue translated_value =
3188 : TranslatedValue::NewDouble(this, value);
3189 : frame.Add(translated_value);
3190 423 : return translated_value.GetChildrenCount();
3191 : }
3192 :
3193 : case Translation::STACK_SLOT: {
3194 : int slot_offset =
3195 7301200 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3196 7301200 : intptr_t value = *(reinterpret_cast<intptr_t*>(fp + slot_offset));
3197 7301200 : if (trace_file != nullptr) {
3198 960 : PrintF(trace_file, V8PRIxPTR_FMT " ; [fp %c %3d] ", value,
3199 1920 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3200 1920 : Object(value)->ShortPrint(trace_file);
3201 : }
3202 : TranslatedValue translated_value =
3203 7301200 : TranslatedValue::NewTagged(this, Object(value));
3204 : frame.Add(translated_value);
3205 : return translated_value.GetChildrenCount();
3206 : }
3207 :
3208 : case Translation::INT32_STACK_SLOT: {
3209 : int slot_offset =
3210 312898 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3211 : uint32_t value = GetUInt32Slot(fp, slot_offset);
3212 312898 : if (trace_file != nullptr) {
3213 0 : PrintF(trace_file, "%d ; (int32) [fp %c %3d] ",
3214 : static_cast<int32_t>(value), slot_offset < 0 ? '-' : '+',
3215 0 : std::abs(slot_offset));
3216 : }
3217 312898 : TranslatedValue translated_value = TranslatedValue::NewInt32(this, value);
3218 : frame.Add(translated_value);
3219 : return translated_value.GetChildrenCount();
3220 : }
3221 :
3222 : case Translation::INT64_STACK_SLOT: {
3223 : int slot_offset =
3224 8 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3225 : uint64_t value = GetUInt64Slot(fp, slot_offset);
3226 8 : if (trace_file != nullptr) {
3227 0 : PrintF(trace_file, "%" V8PRIdPTR " ; (int64) [fp %c %3d] ",
3228 : static_cast<intptr_t>(value), slot_offset < 0 ? '-' : '+',
3229 0 : std::abs(slot_offset));
3230 : }
3231 8 : TranslatedValue translated_value = TranslatedValue::NewInt64(this, value);
3232 : frame.Add(translated_value);
3233 : return translated_value.GetChildrenCount();
3234 : }
3235 :
3236 : case Translation::UINT32_STACK_SLOT: {
3237 : int slot_offset =
3238 53509 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3239 : uint32_t value = GetUInt32Slot(fp, slot_offset);
3240 53509 : if (trace_file != nullptr) {
3241 0 : PrintF(trace_file, "%u ; (uint32) [fp %c %3d] ", value,
3242 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3243 : }
3244 : TranslatedValue translated_value =
3245 : TranslatedValue::NewUInt32(this, value);
3246 : frame.Add(translated_value);
3247 : return translated_value.GetChildrenCount();
3248 : }
3249 :
3250 : case Translation::BOOL_STACK_SLOT: {
3251 : int slot_offset =
3252 380 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3253 : uint32_t value = GetUInt32Slot(fp, slot_offset);
3254 380 : if (trace_file != nullptr) {
3255 0 : PrintF(trace_file, "%u ; (bool) [fp %c %3d] ", value,
3256 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3257 : }
3258 : TranslatedValue translated_value = TranslatedValue::NewBool(this, value);
3259 : frame.Add(translated_value);
3260 : return translated_value.GetChildrenCount();
3261 : }
3262 :
3263 : case Translation::FLOAT_STACK_SLOT: {
3264 : int slot_offset =
3265 197 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3266 : Float32 value = GetFloatSlot(fp, slot_offset);
3267 197 : if (trace_file != nullptr) {
3268 0 : PrintF(trace_file, "%e ; (float) [fp %c %3d] ", value.get_scalar(),
3269 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3270 : }
3271 : TranslatedValue translated_value = TranslatedValue::NewFloat(this, value);
3272 : frame.Add(translated_value);
3273 : return translated_value.GetChildrenCount();
3274 : }
3275 :
3276 : case Translation::DOUBLE_STACK_SLOT: {
3277 : int slot_offset =
3278 521057 : OptimizedFrame::StackSlotOffsetRelativeToFp(iterator->Next());
3279 : Float64 value = GetDoubleSlot(fp, slot_offset);
3280 521057 : if (trace_file != nullptr) {
3281 0 : PrintF(trace_file, "%e ; (double) [fp %c %d] ", value.get_scalar(),
3282 0 : slot_offset < 0 ? '-' : '+', std::abs(slot_offset));
3283 : }
3284 : TranslatedValue translated_value =
3285 : TranslatedValue::NewDouble(this, value);
3286 : frame.Add(translated_value);
3287 : return translated_value.GetChildrenCount();
3288 : }
3289 :
3290 : case Translation::LITERAL: {
3291 11851565 : int literal_index = iterator->Next();
3292 11851565 : Object value = literal_array->get(literal_index);
3293 11851565 : if (trace_file != nullptr) {
3294 : PrintF(trace_file, V8PRIxPTR_FMT " ; (literal %2d) ", value->ptr(),
3295 1080 : literal_index);
3296 1080 : value->ShortPrint(trace_file);
3297 : }
3298 :
3299 : TranslatedValue translated_value =
3300 : TranslatedValue::NewTagged(this, value);
3301 : frame.Add(translated_value);
3302 : return translated_value.GetChildrenCount();
3303 : }
3304 : }
3305 :
3306 0 : FATAL("We should never get here - unexpected deopt info.");
3307 : }
3308 :
3309 2638880 : TranslatedState::TranslatedState(const JavaScriptFrame* frame) {
3310 1319440 : int deopt_index = Safepoint::kNoDeoptimizationIndex;
3311 : DeoptimizationData data =
3312 : static_cast<const OptimizedFrame*>(frame)->GetDeoptimizationData(
3313 1319440 : &deopt_index);
3314 : DCHECK(!data.is_null() && deopt_index != Safepoint::kNoDeoptimizationIndex);
3315 : TranslationIterator it(data->TranslationByteArray(),
3316 2638880 : data->TranslationIndex(deopt_index)->value());
3317 1319440 : Init(frame->isolate(), frame->fp(), &it, data->LiteralArray(),
3318 : nullptr /* registers */, nullptr /* trace file */,
3319 2638880 : frame->function()->shared()->internal_formal_parameter_count());
3320 1319440 : }
3321 :
3322 1365210 : void TranslatedState::Init(Isolate* isolate, Address input_frame_pointer,
3323 : TranslationIterator* iterator,
3324 : FixedArray literal_array, RegisterValues* registers,
3325 : FILE* trace_file, int formal_parameter_count) {
3326 : DCHECK(frames_.empty());
3327 :
3328 1365210 : formal_parameter_count_ = formal_parameter_count;
3329 1365210 : isolate_ = isolate;
3330 :
3331 : // Read out the 'header' translation.
3332 : Translation::Opcode opcode =
3333 1365210 : static_cast<Translation::Opcode>(iterator->Next());
3334 1365210 : CHECK(opcode == Translation::BEGIN);
3335 :
3336 1365210 : int count = iterator->Next();
3337 1365210 : frames_.reserve(count);
3338 1365210 : iterator->Next(); // Drop JS frames count.
3339 1365210 : int update_feedback_count = iterator->Next();
3340 1365210 : CHECK_GE(update_feedback_count, 0);
3341 1365210 : CHECK_LE(update_feedback_count, 1);
3342 :
3343 1365210 : if (update_feedback_count == 1) {
3344 1940 : ReadUpdateFeedback(iterator, literal_array, trace_file);
3345 : }
3346 :
3347 1365210 : std::stack<int> nested_counts;
3348 :
3349 : // Read the frames
3350 5243974 : for (int frame_index = 0; frame_index < count; frame_index++) {
3351 : // Read the frame descriptor.
3352 3878764 : frames_.push_back(CreateNextTranslatedFrame(
3353 : iterator, literal_array, input_frame_pointer, trace_file));
3354 : TranslatedFrame& frame = frames_.back();
3355 :
3356 : // Read the values.
3357 1939382 : int values_to_process = frame.GetValueCount();
3358 22238495 : while (values_to_process > 0 || !nested_counts.empty()) {
3359 20299113 : if (trace_file != nullptr) {
3360 2400 : if (nested_counts.empty()) {
3361 : // For top level values, print the value number.
3362 2400 : PrintF(trace_file, " %3i: ",
3363 4800 : frame.GetValueCount() - values_to_process);
3364 : } else {
3365 : // Take care of indenting for nested values.
3366 0 : PrintF(trace_file, " ");
3367 0 : for (size_t j = 0; j < nested_counts.size(); j++) {
3368 0 : PrintF(trace_file, " ");
3369 : }
3370 : }
3371 : }
3372 :
3373 : int nested_count =
3374 : CreateNextTranslatedValue(frame_index, iterator, literal_array,
3375 20299113 : input_frame_pointer, registers, trace_file);
3376 :
3377 20299113 : if (trace_file != nullptr) {
3378 2400 : PrintF(trace_file, "\n");
3379 : }
3380 :
3381 : // Update the value count and resolve the nesting.
3382 20299113 : values_to_process--;
3383 20299113 : if (nested_count > 0) {
3384 : nested_counts.push(values_to_process);
3385 146588 : values_to_process = nested_count;
3386 : } else {
3387 20299113 : while (values_to_process == 0 && !nested_counts.empty()) {
3388 146588 : values_to_process = nested_counts.top();
3389 : nested_counts.pop();
3390 : }
3391 : }
3392 : }
3393 : }
3394 :
3395 1365210 : CHECK(!iterator->HasNext() ||
3396 : static_cast<Translation::Opcode>(iterator->Next()) ==
3397 : Translation::BEGIN);
3398 1365210 : }
3399 :
3400 1365210 : void TranslatedState::Prepare(Address stack_frame_pointer) {
3401 3304592 : for (auto& frame : frames_) frame.Handlify();
3402 :
3403 1365210 : if (!feedback_vector_.is_null()) {
3404 : feedback_vector_handle_ =
3405 1940 : Handle<FeedbackVector>(feedback_vector_, isolate());
3406 1940 : feedback_vector_ = FeedbackVector();
3407 : }
3408 1365210 : stack_frame_pointer_ = stack_frame_pointer;
3409 :
3410 1365210 : UpdateFromPreviouslyMaterializedObjects();
3411 1365210 : }
3412 :
3413 57218 : TranslatedValue* TranslatedState::GetValueByObjectIndex(int object_index) {
3414 57218 : CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
3415 57218 : TranslatedState::ObjectPosition pos = object_positions_[object_index];
3416 114436 : return &(frames_[pos.frame_index_].values_[pos.value_index_]);
3417 : }
3418 :
3419 43271 : Handle<Object> TranslatedState::InitializeObjectAt(TranslatedValue* slot) {
3420 43271 : slot = ResolveCapturedObject(slot);
3421 :
3422 : DisallowHeapAllocation no_allocation;
3423 43271 : if (slot->materialization_state() != TranslatedValue::kFinished) {
3424 42292 : std::stack<int> worklist;
3425 84584 : worklist.push(slot->object_index());
3426 : slot->mark_finished();
3427 :
3428 128532 : while (!worklist.empty()) {
3429 43120 : int index = worklist.top();
3430 : worklist.pop();
3431 43120 : InitializeCapturedObjectAt(index, &worklist, no_allocation);
3432 : }
3433 : }
3434 43271 : return slot->GetStorage();
3435 : }
3436 :
3437 43120 : void TranslatedState::InitializeCapturedObjectAt(
3438 : int object_index, std::stack<int>* worklist,
3439 : const DisallowHeapAllocation& no_allocation) {
3440 43120 : CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
3441 43120 : TranslatedState::ObjectPosition pos = object_positions_[object_index];
3442 43120 : int value_index = pos.value_index_;
3443 :
3444 43120 : TranslatedFrame* frame = &(frames_[pos.frame_index_]);
3445 : TranslatedValue* slot = &(frame->values_[value_index]);
3446 43120 : value_index++;
3447 :
3448 43120 : CHECK_EQ(TranslatedValue::kFinished, slot->materialization_state());
3449 43120 : CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3450 :
3451 : // Ensure all fields are initialized.
3452 43120 : int children_init_index = value_index;
3453 1514804 : for (int i = 0; i < slot->GetChildrenCount(); i++) {
3454 : // If the field is an object that has not been initialized yet, queue it
3455 : // for initialization (and mark it as such).
3456 735842 : TranslatedValue* child_slot = frame->ValueAt(children_init_index);
3457 735842 : if (child_slot->kind() == TranslatedValue::kCapturedObject ||
3458 : child_slot->kind() == TranslatedValue::kDuplicatedObject) {
3459 1057 : child_slot = ResolveCapturedObject(child_slot);
3460 1057 : if (child_slot->materialization_state() != TranslatedValue::kFinished) {
3461 : DCHECK_EQ(TranslatedValue::kAllocated,
3462 : child_slot->materialization_state());
3463 1656 : worklist->push(child_slot->object_index());
3464 : child_slot->mark_finished();
3465 : }
3466 : }
3467 735842 : SkipSlots(1, frame, &children_init_index);
3468 : }
3469 :
3470 : // Read the map.
3471 : // The map should never be materialized, so let us check we already have
3472 : // an existing object here.
3473 86240 : CHECK_EQ(frame->values_[value_index].kind(), TranslatedValue::kTagged);
3474 86240 : Handle<Map> map = Handle<Map>::cast(frame->values_[value_index].GetValue());
3475 43120 : CHECK(map->IsMap());
3476 43120 : value_index++;
3477 :
3478 : // Handle the special cases.
3479 43120 : switch (map->instance_type()) {
3480 : case MUTABLE_HEAP_NUMBER_TYPE:
3481 : case FIXED_DOUBLE_ARRAY_TYPE:
3482 54 : return;
3483 :
3484 : case FIXED_ARRAY_TYPE:
3485 : case AWAIT_CONTEXT_TYPE:
3486 : case BLOCK_CONTEXT_TYPE:
3487 : case CATCH_CONTEXT_TYPE:
3488 : case DEBUG_EVALUATE_CONTEXT_TYPE:
3489 : case EVAL_CONTEXT_TYPE:
3490 : case FUNCTION_CONTEXT_TYPE:
3491 : case MODULE_CONTEXT_TYPE:
3492 : case NATIVE_CONTEXT_TYPE:
3493 : case SCRIPT_CONTEXT_TYPE:
3494 : case WITH_CONTEXT_TYPE:
3495 : case OBJECT_BOILERPLATE_DESCRIPTION_TYPE:
3496 : case HASH_TABLE_TYPE:
3497 : case ORDERED_HASH_MAP_TYPE:
3498 : case ORDERED_HASH_SET_TYPE:
3499 : case NAME_DICTIONARY_TYPE:
3500 : case GLOBAL_DICTIONARY_TYPE:
3501 : case NUMBER_DICTIONARY_TYPE:
3502 : case SIMPLE_NUMBER_DICTIONARY_TYPE:
3503 : case STRING_TABLE_TYPE:
3504 : case PROPERTY_ARRAY_TYPE:
3505 : case SCRIPT_CONTEXT_TABLE_TYPE:
3506 : InitializeObjectWithTaggedFieldsAt(frame, &value_index, slot, map,
3507 804 : no_allocation);
3508 804 : break;
3509 :
3510 : default:
3511 42262 : CHECK(map->IsJSObjectMap());
3512 42262 : InitializeJSObjectAt(frame, &value_index, slot, map, no_allocation);
3513 42262 : break;
3514 : }
3515 43066 : CHECK_EQ(value_index, children_init_index);
3516 : }
3517 :
3518 43271 : void TranslatedState::EnsureObjectAllocatedAt(TranslatedValue* slot) {
3519 43271 : slot = ResolveCapturedObject(slot);
3520 :
3521 43271 : if (slot->materialization_state() == TranslatedValue::kUninitialized) {
3522 42292 : std::stack<int> worklist;
3523 84584 : worklist.push(slot->object_index());
3524 : slot->mark_allocated();
3525 :
3526 128454 : while (!worklist.empty()) {
3527 43081 : int index = worklist.top();
3528 : worklist.pop();
3529 43081 : EnsureCapturedObjectAllocatedAt(index, &worklist);
3530 : }
3531 : }
3532 43271 : }
3533 :
3534 31 : void TranslatedState::MaterializeFixedDoubleArray(TranslatedFrame* frame,
3535 : int* value_index,
3536 : TranslatedValue* slot,
3537 : Handle<Map> map) {
3538 62 : int length = Smi::cast(frame->values_[*value_index].GetRawValue())->value();
3539 31 : (*value_index)++;
3540 : Handle<FixedDoubleArray> array = Handle<FixedDoubleArray>::cast(
3541 31 : isolate()->factory()->NewFixedDoubleArray(length));
3542 31 : CHECK_GT(length, 0);
3543 203 : for (int i = 0; i < length; i++) {
3544 172 : CHECK_NE(TranslatedValue::kCapturedObject,
3545 : frame->values_[*value_index].kind());
3546 172 : Handle<Object> value = frame->values_[*value_index].GetValue();
3547 86 : if (value->IsNumber()) {
3548 71 : array->set(i, value->Number());
3549 : } else {
3550 15 : CHECK(value.is_identical_to(isolate()->factory()->the_hole_value()));
3551 30 : array->set_the_hole(isolate(), i);
3552 : }
3553 86 : (*value_index)++;
3554 : }
3555 : slot->set_storage(array);
3556 31 : }
3557 :
3558 23 : void TranslatedState::MaterializeMutableHeapNumber(TranslatedFrame* frame,
3559 : int* value_index,
3560 : TranslatedValue* slot) {
3561 46 : CHECK_NE(TranslatedValue::kCapturedObject,
3562 : frame->values_[*value_index].kind());
3563 46 : Handle<Object> value = frame->values_[*value_index].GetValue();
3564 23 : CHECK(value->IsNumber());
3565 : Handle<MutableHeapNumber> box =
3566 23 : isolate()->factory()->NewMutableHeapNumber(value->Number());
3567 23 : (*value_index)++;
3568 : slot->set_storage(box);
3569 23 : }
3570 :
3571 : namespace {
3572 :
3573 : enum DoubleStorageKind : uint8_t {
3574 : kStoreTagged,
3575 : kStoreUnboxedDouble,
3576 : kStoreMutableHeapNumber,
3577 : };
3578 :
3579 : } // namespace
3580 :
3581 2078783 : void TranslatedState::SkipSlots(int slots_to_skip, TranslatedFrame* frame,
3582 : int* value_index) {
3583 5887895 : while (slots_to_skip > 0) {
3584 3809112 : TranslatedValue* slot = &(frame->values_[*value_index]);
3585 3809112 : (*value_index)++;
3586 3809112 : slots_to_skip--;
3587 :
3588 3809112 : if (slot->kind() == TranslatedValue::kCapturedObject) {
3589 2120 : slots_to_skip += slot->GetChildrenCount();
3590 : }
3591 : }
3592 2078783 : }
3593 :
3594 43081 : void TranslatedState::EnsureCapturedObjectAllocatedAt(
3595 : int object_index, std::stack<int>* worklist) {
3596 43081 : CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
3597 43081 : TranslatedState::ObjectPosition pos = object_positions_[object_index];
3598 43081 : int value_index = pos.value_index_;
3599 :
3600 43081 : TranslatedFrame* frame = &(frames_[pos.frame_index_]);
3601 : TranslatedValue* slot = &(frame->values_[value_index]);
3602 43081 : value_index++;
3603 :
3604 43081 : CHECK_EQ(TranslatedValue::kAllocated, slot->materialization_state());
3605 43081 : CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3606 :
3607 : // Read the map.
3608 : // The map should never be materialized, so let us check we already have
3609 : // an existing object here.
3610 43081 : CHECK_EQ(frame->values_[value_index].kind(), TranslatedValue::kTagged);
3611 86162 : Handle<Map> map = Handle<Map>::cast(frame->values_[value_index].GetValue());
3612 43081 : CHECK(map->IsMap());
3613 43081 : value_index++;
3614 :
3615 : // Handle the special cases.
3616 43081 : switch (map->instance_type()) {
3617 : case FIXED_DOUBLE_ARRAY_TYPE:
3618 : // Materialize (i.e. allocate&initialize) the array and return since
3619 : // there is no need to process the children.
3620 31 : return MaterializeFixedDoubleArray(frame, &value_index, slot, map);
3621 :
3622 : case MUTABLE_HEAP_NUMBER_TYPE:
3623 : // Materialize (i.e. allocate&initialize) the heap number and return.
3624 : // There is no need to process the children.
3625 23 : return MaterializeMutableHeapNumber(frame, &value_index, slot);
3626 :
3627 : case FIXED_ARRAY_TYPE:
3628 : case SCRIPT_CONTEXT_TABLE_TYPE:
3629 : case AWAIT_CONTEXT_TYPE:
3630 : case BLOCK_CONTEXT_TYPE:
3631 : case CATCH_CONTEXT_TYPE:
3632 : case DEBUG_EVALUATE_CONTEXT_TYPE:
3633 : case EVAL_CONTEXT_TYPE:
3634 : case FUNCTION_CONTEXT_TYPE:
3635 : case MODULE_CONTEXT_TYPE:
3636 : case NATIVE_CONTEXT_TYPE:
3637 : case SCRIPT_CONTEXT_TYPE:
3638 : case WITH_CONTEXT_TYPE:
3639 : case HASH_TABLE_TYPE:
3640 : case ORDERED_HASH_MAP_TYPE:
3641 : case ORDERED_HASH_SET_TYPE:
3642 : case NAME_DICTIONARY_TYPE:
3643 : case GLOBAL_DICTIONARY_TYPE:
3644 : case NUMBER_DICTIONARY_TYPE:
3645 : case SIMPLE_NUMBER_DICTIONARY_TYPE:
3646 : case STRING_TABLE_TYPE: {
3647 : // Check we have the right size.
3648 : int array_length =
3649 765 : Smi::cast(frame->values_[value_index].GetRawValue())->value();
3650 :
3651 : int instance_size = FixedArray::SizeFor(array_length);
3652 765 : CHECK_EQ(instance_size, slot->GetChildrenCount() * kTaggedSize);
3653 :
3654 : // Canonicalize empty fixed array.
3655 765 : if (*map == ReadOnlyRoots(isolate()).empty_fixed_array()->map() &&
3656 : array_length == 0) {
3657 : slot->set_storage(isolate()->factory()->empty_fixed_array());
3658 : } else {
3659 736 : slot->set_storage(AllocateStorageFor(slot));
3660 : }
3661 :
3662 : // Make sure all the remaining children (after the map) are allocated.
3663 765 : return EnsureChildrenAllocated(slot->GetChildrenCount() - 1, frame,
3664 765 : &value_index, worklist);
3665 : }
3666 :
3667 : case PROPERTY_ARRAY_TYPE: {
3668 : // Check we have the right size.
3669 : int length_or_hash =
3670 0 : Smi::cast(frame->values_[value_index].GetRawValue())->value();
3671 : int array_length = PropertyArray::LengthField::decode(length_or_hash);
3672 : int instance_size = PropertyArray::SizeFor(array_length);
3673 0 : CHECK_EQ(instance_size, slot->GetChildrenCount() * kTaggedSize);
3674 :
3675 0 : slot->set_storage(AllocateStorageFor(slot));
3676 : // Make sure all the remaining children (after the map) are allocated.
3677 0 : return EnsureChildrenAllocated(slot->GetChildrenCount() - 1, frame,
3678 0 : &value_index, worklist);
3679 : }
3680 :
3681 : default:
3682 42262 : CHECK(map->IsJSObjectMap());
3683 42262 : EnsureJSObjectAllocated(slot, map);
3684 42262 : TranslatedValue* properties_slot = &(frame->values_[value_index]);
3685 42262 : value_index++;
3686 42262 : if (properties_slot->kind() == TranslatedValue::kCapturedObject) {
3687 : // If we are materializing the property array, make sure we put
3688 : // the mutable heap numbers at the right places.
3689 39 : EnsurePropertiesAllocatedAndMarked(properties_slot, map);
3690 : EnsureChildrenAllocated(properties_slot->GetChildrenCount(), frame,
3691 39 : &value_index, worklist);
3692 : }
3693 : // Make sure all the remaining children (after the map and properties) are
3694 : // allocated.
3695 42262 : return EnsureChildrenAllocated(slot->GetChildrenCount() - 2, frame,
3696 42262 : &value_index, worklist);
3697 : }
3698 : UNREACHABLE();
3699 : }
3700 :
3701 43066 : void TranslatedState::EnsureChildrenAllocated(int count, TranslatedFrame* frame,
3702 : int* value_index,
3703 : std::stack<int>* worklist) {
3704 : // Ensure all children are allocated.
3705 1343784 : for (int i = 0; i < count; i++) {
3706 : // If the field is an object that has not been allocated yet, queue it
3707 : // for initialization (and mark it as such).
3708 650359 : TranslatedValue* child_slot = frame->ValueAt(*value_index);
3709 650359 : if (child_slot->kind() == TranslatedValue::kCapturedObject ||
3710 : child_slot->kind() == TranslatedValue::kDuplicatedObject) {
3711 1018 : child_slot = ResolveCapturedObject(child_slot);
3712 1018 : if (child_slot->materialization_state() ==
3713 : TranslatedValue::kUninitialized) {
3714 1578 : worklist->push(child_slot->object_index());
3715 : child_slot->mark_allocated();
3716 : }
3717 : } else {
3718 : // Make sure the simple values (heap numbers, etc.) are properly
3719 : // initialized.
3720 649341 : child_slot->MaterializeSimple();
3721 : }
3722 650359 : SkipSlots(1, frame, value_index);
3723 : }
3724 43066 : }
3725 :
3726 39 : void TranslatedState::EnsurePropertiesAllocatedAndMarked(
3727 : TranslatedValue* properties_slot, Handle<Map> map) {
3728 39 : CHECK_EQ(TranslatedValue::kUninitialized,
3729 : properties_slot->materialization_state());
3730 :
3731 39 : Handle<ByteArray> object_storage = AllocateStorageFor(properties_slot);
3732 : properties_slot->mark_allocated();
3733 : properties_slot->set_storage(object_storage);
3734 :
3735 : // Set markers for the double properties.
3736 : Handle<DescriptorArray> descriptors(map->instance_descriptors(), isolate());
3737 : int field_count = map->NumberOfOwnDescriptors();
3738 133 : for (int i = 0; i < field_count; i++) {
3739 47 : FieldIndex index = FieldIndex::ForDescriptor(*map, i);
3740 102 : if (descriptors->GetDetails(i).representation().IsDouble() &&
3741 : !index.is_inobject()) {
3742 : CHECK(!map->IsUnboxedDoubleField(index));
3743 : int outobject_index = index.outobject_array_index();
3744 8 : int array_index = outobject_index * kTaggedSize;
3745 : object_storage->set(array_index, kStoreMutableHeapNumber);
3746 : }
3747 : }
3748 39 : }
3749 :
3750 43037 : Handle<ByteArray> TranslatedState::AllocateStorageFor(TranslatedValue* slot) {
3751 : int allocate_size =
3752 43037 : ByteArray::LengthFor(slot->GetChildrenCount() * kTaggedSize);
3753 : // It is important to allocate all the objects tenured so that the marker
3754 : // does not visit them.
3755 : Handle<ByteArray> object_storage =
3756 43037 : isolate()->factory()->NewByteArray(allocate_size, AllocationType::kOld);
3757 5239165 : for (int i = 0; i < object_storage->length(); i++) {
3758 : object_storage->set(i, kStoreTagged);
3759 : }
3760 43037 : return object_storage;
3761 : }
3762 :
3763 42262 : void TranslatedState::EnsureJSObjectAllocated(TranslatedValue* slot,
3764 : Handle<Map> map) {
3765 84524 : CHECK_EQ(map->instance_size(), slot->GetChildrenCount() * kTaggedSize);
3766 :
3767 42262 : Handle<ByteArray> object_storage = AllocateStorageFor(slot);
3768 : // Now we handle the interesting (JSObject) case.
3769 : Handle<DescriptorArray> descriptors(map->instance_descriptors(), isolate());
3770 : int field_count = map->NumberOfOwnDescriptors();
3771 :
3772 : // Set markers for the double properties.
3773 100238 : for (int i = 0; i < field_count; i++) {
3774 28988 : FieldIndex index = FieldIndex::ForDescriptor(*map, i);
3775 58007 : if (descriptors->GetDetails(i).representation().IsDouble() &&
3776 : index.is_inobject()) {
3777 15 : CHECK_GE(index.index(), FixedArray::kHeaderSize / kTaggedSize);
3778 15 : int array_index = index.index() * kTaggedSize - FixedArray::kHeaderSize;
3779 : uint8_t marker = map->IsUnboxedDoubleField(index)
3780 : ? kStoreUnboxedDouble
3781 : : kStoreMutableHeapNumber;
3782 : object_storage->set(array_index, marker);
3783 : }
3784 : }
3785 : slot->set_storage(object_storage);
3786 42262 : }
3787 :
3788 692582 : Handle<Object> TranslatedState::GetValueAndAdvance(TranslatedFrame* frame,
3789 : int* value_index) {
3790 692582 : TranslatedValue* slot = frame->ValueAt(*value_index);
3791 692582 : SkipSlots(1, frame, value_index);
3792 692582 : if (slot->kind() == TranslatedValue::kDuplicatedObject) {
3793 428 : slot = ResolveCapturedObject(slot);
3794 : }
3795 692582 : CHECK_NE(TranslatedValue::kUninitialized, slot->materialization_state());
3796 692582 : return slot->GetStorage();
3797 : }
3798 :
3799 42262 : void TranslatedState::InitializeJSObjectAt(
3800 : TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
3801 : Handle<Map> map, const DisallowHeapAllocation& no_allocation) {
3802 : Handle<HeapObject> object_storage = Handle<HeapObject>::cast(slot->storage_);
3803 : DCHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3804 :
3805 : // The object should have at least a map and some payload.
3806 42262 : CHECK_GE(slot->GetChildrenCount(), 2);
3807 :
3808 : // Notify the concurrent marker about the layout change.
3809 42262 : isolate()->heap()->NotifyObjectLayoutChange(
3810 42262 : *object_storage, slot->GetChildrenCount() * kTaggedSize, no_allocation);
3811 :
3812 : // Fill the property array field.
3813 : {
3814 42262 : Handle<Object> properties = GetValueAndAdvance(frame, value_index);
3815 42262 : WRITE_FIELD(*object_storage, JSObject::kPropertiesOrHashOffset,
3816 : *properties);
3817 42262 : WRITE_BARRIER(*object_storage, JSObject::kPropertiesOrHashOffset,
3818 : *properties);
3819 : }
3820 :
3821 : // For all the other fields we first look at the fixed array and check the
3822 : // marker to see if we store an unboxed double.
3823 : DCHECK_EQ(kTaggedSize, JSObject::kPropertiesOrHashOffset);
3824 188044 : for (int i = 2; i < slot->GetChildrenCount(); i++) {
3825 : // Initialize and extract the value from its slot.
3826 72891 : Handle<Object> field_value = GetValueAndAdvance(frame, value_index);
3827 :
3828 : // Read out the marker and ensure the field is consistent with
3829 : // what the markers in the storage say (note that all heap numbers
3830 : // should be fully initialized by now).
3831 72891 : int offset = i * kTaggedSize;
3832 72891 : uint8_t marker = READ_UINT8_FIELD(*object_storage, offset);
3833 72891 : if (marker == kStoreUnboxedDouble) {
3834 : double double_field_value;
3835 0 : if (field_value->IsSmi()) {
3836 0 : double_field_value = Smi::cast(*field_value)->value();
3837 : } else {
3838 0 : CHECK(field_value->IsHeapNumber());
3839 : double_field_value = HeapNumber::cast(*field_value)->value();
3840 : }
3841 0 : WRITE_DOUBLE_FIELD(*object_storage, offset, double_field_value);
3842 72891 : } else if (marker == kStoreMutableHeapNumber) {
3843 15 : CHECK(field_value->IsMutableHeapNumber());
3844 15 : WRITE_FIELD(*object_storage, offset, *field_value);
3845 15 : WRITE_BARRIER(*object_storage, offset, *field_value);
3846 : } else {
3847 72876 : CHECK_EQ(kStoreTagged, marker);
3848 : WRITE_FIELD(*object_storage, offset, *field_value);
3849 72876 : WRITE_BARRIER(*object_storage, offset, *field_value);
3850 : }
3851 : }
3852 42262 : object_storage->synchronized_set_map(*map);
3853 42262 : }
3854 :
3855 804 : void TranslatedState::InitializeObjectWithTaggedFieldsAt(
3856 : TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
3857 : Handle<Map> map, const DisallowHeapAllocation& no_allocation) {
3858 : Handle<HeapObject> object_storage = Handle<HeapObject>::cast(slot->storage_);
3859 :
3860 : // Skip the writes if we already have the canonical empty fixed array.
3861 804 : if (*object_storage == ReadOnlyRoots(isolate()).empty_fixed_array()) {
3862 29 : CHECK_EQ(2, slot->GetChildrenCount());
3863 29 : Handle<Object> length_value = GetValueAndAdvance(frame, value_index);
3864 29 : CHECK_EQ(*length_value, Smi::FromInt(0));
3865 : return;
3866 : }
3867 :
3868 : // Notify the concurrent marker about the layout change.
3869 775 : isolate()->heap()->NotifyObjectLayoutChange(
3870 775 : *object_storage, slot->GetChildrenCount() * kTaggedSize, no_allocation);
3871 :
3872 : // Write the fields to the object.
3873 1155575 : for (int i = 1; i < slot->GetChildrenCount(); i++) {
3874 577400 : Handle<Object> field_value = GetValueAndAdvance(frame, value_index);
3875 577400 : int offset = i * kTaggedSize;
3876 577400 : uint8_t marker = READ_UINT8_FIELD(*object_storage, offset);
3877 577400 : if (i > 1 && marker == kStoreMutableHeapNumber) {
3878 8 : CHECK(field_value->IsMutableHeapNumber());
3879 : } else {
3880 577392 : CHECK(marker == kStoreTagged || i == 1);
3881 577392 : CHECK(!field_value->IsMutableHeapNumber());
3882 : }
3883 :
3884 577400 : WRITE_FIELD(*object_storage, offset, *field_value);
3885 577400 : WRITE_BARRIER(*object_storage, offset, *field_value);
3886 : }
3887 :
3888 775 : object_storage->synchronized_set_map(*map);
3889 : }
3890 :
3891 89045 : TranslatedValue* TranslatedState::ResolveCapturedObject(TranslatedValue* slot) {
3892 203481 : while (slot->kind() == TranslatedValue::kDuplicatedObject) {
3893 57218 : slot = GetValueByObjectIndex(slot->object_index());
3894 : }
3895 89045 : CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
3896 89045 : return slot;
3897 : }
3898 :
3899 45 : TranslatedFrame* TranslatedState::GetFrameFromJSFrameIndex(int jsframe_index) {
3900 135 : for (size_t i = 0; i < frames_.size(); i++) {
3901 104 : if (frames_[i].kind() == TranslatedFrame::kInterpretedFunction ||
3902 90 : frames_[i].kind() == TranslatedFrame::kJavaScriptBuiltinContinuation ||
3903 : frames_[i].kind() ==
3904 : TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch) {
3905 90 : if (jsframe_index > 0) {
3906 45 : jsframe_index--;
3907 : } else {
3908 : return &(frames_[i]);
3909 : }
3910 : }
3911 : }
3912 : return nullptr;
3913 : }
3914 :
3915 2729 : TranslatedFrame* TranslatedState::GetArgumentsInfoFromJSFrameIndex(
3916 : int jsframe_index, int* args_count) {
3917 25903 : for (size_t i = 0; i < frames_.size(); i++) {
3918 19713 : if (frames_[i].kind() == TranslatedFrame::kInterpretedFunction ||
3919 19706 : frames_[i].kind() == TranslatedFrame::kJavaScriptBuiltinContinuation ||
3920 : frames_[i].kind() ==
3921 : TranslatedFrame::kJavaScriptBuiltinContinuationWithCatch) {
3922 8926 : if (jsframe_index > 0) {
3923 6197 : jsframe_index--;
3924 : } else {
3925 : // We have the JS function frame, now check if it has arguments
3926 : // adaptor.
3927 5458 : if (i > 0 &&
3928 2729 : frames_[i - 1].kind() == TranslatedFrame::kArgumentsAdaptor) {
3929 2198 : *args_count = frames_[i - 1].height();
3930 1099 : return &(frames_[i - 1]);
3931 : }
3932 : *args_count =
3933 1630 : frames_[i].shared_info()->internal_formal_parameter_count() + 1;
3934 1630 : return &(frames_[i]);
3935 : }
3936 : }
3937 : }
3938 : return nullptr;
3939 : }
3940 :
3941 182 : void TranslatedState::StoreMaterializedValuesAndDeopt(JavaScriptFrame* frame) {
3942 : MaterializedObjectStore* materialized_store =
3943 182 : isolate_->materialized_object_store();
3944 : Handle<FixedArray> previously_materialized_objects =
3945 182 : materialized_store->Get(stack_frame_pointer_);
3946 :
3947 182 : Handle<Object> marker = isolate_->factory()->arguments_marker();
3948 :
3949 182 : int length = static_cast<int>(object_positions_.size());
3950 : bool new_store = false;
3951 182 : if (previously_materialized_objects.is_null()) {
3952 : previously_materialized_objects =
3953 150 : isolate_->factory()->NewFixedArray(length, AllocationType::kOld);
3954 978 : for (int i = 0; i < length; i++) {
3955 414 : previously_materialized_objects->set(i, *marker);
3956 : }
3957 : new_store = true;
3958 : }
3959 :
3960 182 : CHECK_EQ(length, previously_materialized_objects->length());
3961 :
3962 : bool value_changed = false;
3963 1178 : for (int i = 0; i < length; i++) {
3964 498 : TranslatedState::ObjectPosition pos = object_positions_[i];
3965 : TranslatedValue* value_info =
3966 498 : &(frames_[pos.frame_index_].values_[pos.value_index_]);
3967 :
3968 498 : CHECK(value_info->IsMaterializedObject());
3969 :
3970 : // Skip duplicate objects (i.e., those that point to some
3971 : // other object id).
3972 498 : if (value_info->object_index() != i) continue;
3973 :
3974 330 : Handle<Object> value(value_info->GetRawValue(), isolate_);
3975 :
3976 330 : if (!value.is_identical_to(marker)) {
3977 294 : if (previously_materialized_objects->get(i) == *marker) {
3978 242 : previously_materialized_objects->set(i, *value);
3979 : value_changed = true;
3980 : } else {
3981 52 : CHECK(previously_materialized_objects->get(i) == *value);
3982 : }
3983 : }
3984 : }
3985 182 : if (new_store && value_changed) {
3986 150 : materialized_store->Set(stack_frame_pointer_,
3987 150 : previously_materialized_objects);
3988 150 : CHECK_EQ(frames_[0].kind(), TranslatedFrame::kInterpretedFunction);
3989 300 : CHECK_EQ(frame->function(), frames_[0].front().GetRawValue());
3990 150 : Deoptimizer::DeoptimizeFunction(frame->function(), frame->LookupCode());
3991 : }
3992 182 : }
3993 :
3994 1365210 : void TranslatedState::UpdateFromPreviouslyMaterializedObjects() {
3995 : MaterializedObjectStore* materialized_store =
3996 1365210 : isolate_->materialized_object_store();
3997 : Handle<FixedArray> previously_materialized_objects =
3998 1365210 : materialized_store->Get(stack_frame_pointer_);
3999 :
4000 : // If we have no previously materialized objects, there is nothing to do.
4001 1365210 : if (previously_materialized_objects.is_null()) return;
4002 :
4003 214 : Handle<Object> marker = isolate_->factory()->arguments_marker();
4004 :
4005 214 : int length = static_cast<int>(object_positions_.size());
4006 214 : CHECK_EQ(length, previously_materialized_objects->length());
4007 :
4008 1378 : for (int i = 0; i < length; i++) {
4009 : // For a previously materialized objects, inject their value into the
4010 : // translated values.
4011 582 : if (previously_materialized_objects->get(i) != *marker) {
4012 346 : TranslatedState::ObjectPosition pos = object_positions_[i];
4013 : TranslatedValue* value_info =
4014 346 : &(frames_[pos.frame_index_].values_[pos.value_index_]);
4015 346 : CHECK(value_info->IsMaterializedObject());
4016 :
4017 346 : if (value_info->kind() == TranslatedValue::kCapturedObject) {
4018 346 : value_info->set_initialized_storage(
4019 : Handle<Object>(previously_materialized_objects->get(i), isolate_));
4020 : }
4021 : }
4022 : }
4023 : }
4024 :
4025 0 : void TranslatedState::VerifyMaterializedObjects() {
4026 : #if VERIFY_HEAP
4027 : int length = static_cast<int>(object_positions_.size());
4028 : for (int i = 0; i < length; i++) {
4029 : TranslatedValue* slot = GetValueByObjectIndex(i);
4030 : if (slot->kind() == TranslatedValue::kCapturedObject) {
4031 : CHECK_EQ(slot, GetValueByObjectIndex(slot->object_index()));
4032 : if (slot->materialization_state() == TranslatedValue::kFinished) {
4033 : slot->GetStorage()->ObjectVerify(isolate());
4034 : } else {
4035 : CHECK_EQ(slot->materialization_state(),
4036 : TranslatedValue::kUninitialized);
4037 : }
4038 : }
4039 : }
4040 : #endif
4041 0 : }
4042 :
4043 45770 : bool TranslatedState::DoUpdateFeedback() {
4044 45770 : if (!feedback_vector_handle_.is_null()) {
4045 1940 : CHECK(!feedback_slot_.IsInvalid());
4046 1940 : isolate()->CountUsage(v8::Isolate::kDeoptimizerDisableSpeculation);
4047 : FeedbackNexus nexus(feedback_vector_handle_, feedback_slot_);
4048 1940 : nexus.SetSpeculationMode(SpeculationMode::kDisallowSpeculation);
4049 : return true;
4050 : }
4051 : return false;
4052 : }
4053 :
4054 1940 : void TranslatedState::ReadUpdateFeedback(TranslationIterator* iterator,
4055 : FixedArray literal_array,
4056 : FILE* trace_file) {
4057 3880 : CHECK_EQ(Translation::UPDATE_FEEDBACK, iterator->Next());
4058 3880 : feedback_vector_ = FeedbackVector::cast(literal_array->get(iterator->Next()));
4059 1940 : feedback_slot_ = FeedbackSlot(iterator->Next());
4060 1940 : if (trace_file != nullptr) {
4061 : PrintF(trace_file, " reading FeedbackVector (slot %d)\n",
4062 0 : feedback_slot_.ToInt());
4063 : }
4064 1940 : }
4065 :
4066 : } // namespace internal
4067 120216 : } // namespace v8
4068 :
4069 : // Undefine the heap manipulation macros.
4070 : #include "src/objects/object-macros-undef.h"
|
