Line data Source code
1 : // Copyright 2016 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/code-stub-assembler.h"
6 :
7 : #include "src/code-factory.h"
8 : #include "src/counters.h"
9 : #include "src/frames-inl.h"
10 : #include "src/frames.h"
11 : #include "src/function-kind.h"
12 : #include "src/heap/heap-inl.h" // For Page/MemoryChunk. TODO(jkummerow): Drop.
13 : #include "src/objects/api-callbacks.h"
14 : #include "src/objects/cell.h"
15 : #include "src/objects/descriptor-array.h"
16 : #include "src/objects/heap-number.h"
17 : #include "src/objects/oddball.h"
18 : #include "src/objects/ordered-hash-table-inl.h"
19 : #include "src/objects/property-cell.h"
20 : #include "src/wasm/wasm-objects.h"
21 :
22 : namespace v8 {
23 : namespace internal {
24 :
25 : using compiler::Node;
26 : template <class T>
27 : using TNode = compiler::TNode<T>;
28 : template <class T>
29 : using SloppyTNode = compiler::SloppyTNode<T>;
30 :
31 532433 : CodeStubAssembler::CodeStubAssembler(compiler::CodeAssemblerState* state)
32 532433 : : compiler::CodeAssembler(state), BaseBuiltinsFromDSLAssembler(state) {
33 : if (DEBUG_BOOL && FLAG_csa_trap_on_node != nullptr) {
34 : HandleBreakOnNode();
35 : }
36 532433 : }
37 :
38 0 : void CodeStubAssembler::HandleBreakOnNode() {
39 : // FLAG_csa_trap_on_node should be in a form "STUB,NODE" where STUB is a
40 : // string specifying the name of a stub and NODE is number specifying node id.
41 0 : const char* name = state()->name();
42 0 : size_t name_length = strlen(name);
43 0 : if (strncmp(FLAG_csa_trap_on_node, name, name_length) != 0) {
44 : // Different name.
45 0 : return;
46 : }
47 0 : size_t option_length = strlen(FLAG_csa_trap_on_node);
48 0 : if (option_length < name_length + 2 ||
49 0 : FLAG_csa_trap_on_node[name_length] != ',') {
50 : // Option is too short.
51 0 : return;
52 : }
53 0 : const char* start = &FLAG_csa_trap_on_node[name_length + 1];
54 : char* end;
55 0 : int node_id = static_cast<int>(strtol(start, &end, 10));
56 0 : if (start == end) {
57 : // Bad node id.
58 0 : return;
59 : }
60 0 : BreakOnNode(node_id);
61 : }
62 :
63 0 : void CodeStubAssembler::Assert(const BranchGenerator& branch,
64 : const char* message, const char* file, int line,
65 : Node* extra_node1, const char* extra_node1_name,
66 : Node* extra_node2, const char* extra_node2_name,
67 : Node* extra_node3, const char* extra_node3_name,
68 : Node* extra_node4, const char* extra_node4_name,
69 : Node* extra_node5,
70 : const char* extra_node5_name) {
71 : #if defined(DEBUG)
72 : if (FLAG_debug_code) {
73 : Check(branch, message, file, line, extra_node1, extra_node1_name,
74 : extra_node2, extra_node2_name, extra_node3, extra_node3_name,
75 : extra_node4, extra_node4_name, extra_node5, extra_node5_name);
76 : }
77 : #endif
78 0 : }
79 :
80 0 : void CodeStubAssembler::Assert(const NodeGenerator& condition_body,
81 : const char* message, const char* file, int line,
82 : Node* extra_node1, const char* extra_node1_name,
83 : Node* extra_node2, const char* extra_node2_name,
84 : Node* extra_node3, const char* extra_node3_name,
85 : Node* extra_node4, const char* extra_node4_name,
86 : Node* extra_node5,
87 : const char* extra_node5_name) {
88 : #if defined(DEBUG)
89 : if (FLAG_debug_code) {
90 : Check(condition_body, message, file, line, extra_node1, extra_node1_name,
91 : extra_node2, extra_node2_name, extra_node3, extra_node3_name,
92 : extra_node4, extra_node4_name, extra_node5, extra_node5_name);
93 : }
94 : #endif
95 0 : }
96 :
97 : #ifdef DEBUG
98 : namespace {
99 : void MaybePrintNodeWithName(CodeStubAssembler* csa, Node* node,
100 : const char* node_name) {
101 : if (node != nullptr) {
102 : csa->CallRuntime(Runtime::kPrintWithNameForAssert, csa->SmiConstant(0),
103 : csa->StringConstant(node_name), node);
104 : }
105 : }
106 : } // namespace
107 : #endif
108 :
109 0 : void CodeStubAssembler::Check(const BranchGenerator& branch,
110 : const char* message, const char* file, int line,
111 : Node* extra_node1, const char* extra_node1_name,
112 : Node* extra_node2, const char* extra_node2_name,
113 : Node* extra_node3, const char* extra_node3_name,
114 : Node* extra_node4, const char* extra_node4_name,
115 : Node* extra_node5, const char* extra_node5_name) {
116 0 : Label ok(this);
117 0 : Label not_ok(this, Label::kDeferred);
118 0 : if (message != nullptr && FLAG_code_comments) {
119 0 : Comment("[ Assert: ", message);
120 : } else {
121 0 : Comment("[ Assert");
122 : }
123 0 : branch(&ok, ¬_ok);
124 :
125 0 : BIND(¬_ok);
126 : FailAssert(message, file, line, extra_node1, extra_node1_name, extra_node2,
127 : extra_node2_name, extra_node3, extra_node3_name, extra_node4,
128 0 : extra_node4_name, extra_node5, extra_node5_name);
129 :
130 0 : BIND(&ok);
131 0 : Comment("] Assert");
132 0 : }
133 :
134 0 : void CodeStubAssembler::Check(const NodeGenerator& condition_body,
135 : const char* message, const char* file, int line,
136 : Node* extra_node1, const char* extra_node1_name,
137 : Node* extra_node2, const char* extra_node2_name,
138 : Node* extra_node3, const char* extra_node3_name,
139 : Node* extra_node4, const char* extra_node4_name,
140 : Node* extra_node5, const char* extra_node5_name) {
141 0 : BranchGenerator branch = [=](Label* ok, Label* not_ok) {
142 0 : Node* condition = condition_body();
143 : DCHECK_NOT_NULL(condition);
144 0 : Branch(condition, ok, not_ok);
145 0 : };
146 :
147 : Check(branch, message, file, line, extra_node1, extra_node1_name, extra_node2,
148 : extra_node2_name, extra_node3, extra_node3_name, extra_node4,
149 0 : extra_node4_name, extra_node5, extra_node5_name);
150 0 : }
151 :
152 74632 : void CodeStubAssembler::FastCheck(TNode<BoolT> condition) {
153 149264 : Label ok(this), not_ok(this, Label::kDeferred);
154 74632 : Branch(condition, &ok, ¬_ok);
155 74632 : BIND(¬_ok);
156 : {
157 74632 : DebugBreak();
158 74632 : Goto(&ok);
159 : }
160 74632 : BIND(&ok);
161 74632 : }
162 :
163 0 : void CodeStubAssembler::FailAssert(
164 : const char* message, const char* file, int line, Node* extra_node1,
165 : const char* extra_node1_name, Node* extra_node2,
166 : const char* extra_node2_name, Node* extra_node3,
167 : const char* extra_node3_name, Node* extra_node4,
168 : const char* extra_node4_name, Node* extra_node5,
169 : const char* extra_node5_name) {
170 : DCHECK_NOT_NULL(message);
171 : char chars[1024];
172 0 : Vector<char> buffer(chars);
173 0 : if (file != nullptr) {
174 0 : SNPrintF(buffer, "CSA_ASSERT failed: %s [%s:%d]\n", message, file, line);
175 : } else {
176 0 : SNPrintF(buffer, "CSA_ASSERT failed: %s\n", message);
177 : }
178 0 : Node* message_node = StringConstant(&(buffer[0]));
179 :
180 : #ifdef DEBUG
181 : // Only print the extra nodes in debug builds.
182 : MaybePrintNodeWithName(this, extra_node1, extra_node1_name);
183 : MaybePrintNodeWithName(this, extra_node2, extra_node2_name);
184 : MaybePrintNodeWithName(this, extra_node3, extra_node3_name);
185 : MaybePrintNodeWithName(this, extra_node4, extra_node4_name);
186 : MaybePrintNodeWithName(this, extra_node5, extra_node5_name);
187 : #endif
188 :
189 0 : DebugAbort(message_node);
190 0 : Unreachable();
191 0 : }
192 :
193 47260 : Node* CodeStubAssembler::SelectImpl(TNode<BoolT> condition,
194 : const NodeGenerator& true_body,
195 : const NodeGenerator& false_body,
196 : MachineRepresentation rep) {
197 94520 : VARIABLE(value, rep);
198 94520 : Label vtrue(this), vfalse(this), end(this);
199 47260 : Branch(condition, &vtrue, &vfalse);
200 :
201 47260 : BIND(&vtrue);
202 : {
203 47260 : value.Bind(true_body());
204 47260 : Goto(&end);
205 : }
206 47260 : BIND(&vfalse);
207 : {
208 47260 : value.Bind(false_body());
209 47260 : Goto(&end);
210 : }
211 :
212 47260 : BIND(&end);
213 94520 : return value.value();
214 : }
215 :
216 224 : TNode<Int32T> CodeStubAssembler::SelectInt32Constant(
217 : SloppyTNode<BoolT> condition, int true_value, int false_value) {
218 : return SelectConstant<Int32T>(condition, Int32Constant(true_value),
219 224 : Int32Constant(false_value));
220 : }
221 :
222 0 : TNode<IntPtrT> CodeStubAssembler::SelectIntPtrConstant(
223 : SloppyTNode<BoolT> condition, int true_value, int false_value) {
224 : return SelectConstant<IntPtrT>(condition, IntPtrConstant(true_value),
225 0 : IntPtrConstant(false_value));
226 : }
227 :
228 3084 : TNode<Oddball> CodeStubAssembler::SelectBooleanConstant(
229 : SloppyTNode<BoolT> condition) {
230 3084 : return SelectConstant<Oddball>(condition, TrueConstant(), FalseConstant());
231 : }
232 :
233 4144 : TNode<Smi> CodeStubAssembler::SelectSmiConstant(SloppyTNode<BoolT> condition,
234 : Smi true_value,
235 : Smi false_value) {
236 : return SelectConstant<Smi>(condition, SmiConstant(true_value),
237 4144 : SmiConstant(false_value));
238 : }
239 :
240 55196 : TNode<Object> CodeStubAssembler::NoContextConstant() {
241 55196 : return SmiConstant(Context::kNoContext);
242 : }
243 :
244 : #define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \
245 : compiler::TNode<std::remove_pointer<std::remove_reference<decltype( \
246 : std::declval<Heap>().rootAccessorName())>::type>::type> \
247 : CodeStubAssembler::name##Constant() { \
248 : return UncheckedCast<std::remove_pointer<std::remove_reference<decltype( \
249 : std::declval<Heap>().rootAccessorName())>::type>::type>( \
250 : LoadRoot(RootIndex::k##rootIndexName)); \
251 : }
252 0 : HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR)
253 : #undef HEAP_CONSTANT_ACCESSOR
254 :
255 : #define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \
256 : compiler::TNode<std::remove_pointer<std::remove_reference<decltype( \
257 : std::declval<ReadOnlyRoots>().rootAccessorName())>::type>::type> \
258 : CodeStubAssembler::name##Constant() { \
259 : return UncheckedCast<std::remove_pointer<std::remove_reference<decltype( \
260 : std::declval<ReadOnlyRoots>().rootAccessorName())>::type>::type>( \
261 : LoadRoot(RootIndex::k##rootIndexName)); \
262 : }
263 305300 : HEAP_IMMUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR)
264 : #undef HEAP_CONSTANT_ACCESSOR
265 :
266 : #define HEAP_CONSTANT_TEST(rootIndexName, rootAccessorName, name) \
267 : compiler::TNode<BoolT> CodeStubAssembler::Is##name( \
268 : SloppyTNode<Object> value) { \
269 : return WordEqual(value, name##Constant()); \
270 : } \
271 : compiler::TNode<BoolT> CodeStubAssembler::IsNot##name( \
272 : SloppyTNode<Object> value) { \
273 : return WordNotEqual(value, name##Constant()); \
274 : }
275 159092 : HEAP_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_TEST)
276 : #undef HEAP_CONSTANT_TEST
277 :
278 112456 : Node* CodeStubAssembler::IntPtrOrSmiConstant(int value, ParameterMode mode) {
279 112456 : if (mode == SMI_PARAMETERS) {
280 3408 : return SmiConstant(value);
281 : } else {
282 : DCHECK_EQ(INTPTR_PARAMETERS, mode);
283 109048 : return IntPtrConstant(value);
284 : }
285 : }
286 :
287 2912 : bool CodeStubAssembler::IsIntPtrOrSmiConstantZero(Node* test,
288 : ParameterMode mode) {
289 : int32_t constant_test;
290 2912 : Smi smi_test;
291 2912 : if (mode == INTPTR_PARAMETERS) {
292 1680 : if (ToInt32Constant(test, constant_test) && constant_test == 0) {
293 448 : return true;
294 : }
295 : } else {
296 : DCHECK_EQ(mode, SMI_PARAMETERS);
297 1232 : if (ToSmiConstant(test, &smi_test) && smi_test->value() == 0) {
298 504 : return true;
299 : }
300 : }
301 1960 : return false;
302 : }
303 :
304 0 : bool CodeStubAssembler::TryGetIntPtrOrSmiConstantValue(Node* maybe_constant,
305 : int* value,
306 : ParameterMode mode) {
307 : int32_t int32_constant;
308 0 : if (mode == INTPTR_PARAMETERS) {
309 0 : if (ToInt32Constant(maybe_constant, int32_constant)) {
310 0 : *value = int32_constant;
311 0 : return true;
312 : }
313 : } else {
314 : DCHECK_EQ(mode, SMI_PARAMETERS);
315 0 : Smi smi_constant;
316 0 : if (ToSmiConstant(maybe_constant, &smi_constant)) {
317 0 : *value = Smi::ToInt(smi_constant);
318 0 : return true;
319 : }
320 : }
321 0 : return false;
322 : }
323 :
324 956 : TNode<IntPtrT> CodeStubAssembler::IntPtrRoundUpToPowerOfTwo32(
325 : TNode<IntPtrT> value) {
326 956 : Comment("IntPtrRoundUpToPowerOfTwo32");
327 : CSA_ASSERT(this, UintPtrLessThanOrEqual(value, IntPtrConstant(0x80000000u)));
328 956 : value = Signed(IntPtrSub(value, IntPtrConstant(1)));
329 5736 : for (int i = 1; i <= 16; i *= 2) {
330 4780 : value = Signed(WordOr(value, WordShr(value, IntPtrConstant(i))));
331 : }
332 956 : return Signed(IntPtrAdd(value, IntPtrConstant(1)));
333 : }
334 :
335 0 : Node* CodeStubAssembler::MatchesParameterMode(Node* value, ParameterMode mode) {
336 0 : if (mode == SMI_PARAMETERS) {
337 0 : return TaggedIsSmi(value);
338 : } else {
339 0 : return Int32Constant(1);
340 : }
341 : }
342 :
343 0 : TNode<BoolT> CodeStubAssembler::WordIsPowerOfTwo(SloppyTNode<IntPtrT> value) {
344 : // value && !(value & (value - 1))
345 : return WordEqual(
346 0 : Select<IntPtrT>(
347 0 : WordEqual(value, IntPtrConstant(0)),
348 0 : [=] { return IntPtrConstant(1); },
349 0 : [=] { return WordAnd(value, IntPtrSub(value, IntPtrConstant(1))); }),
350 0 : IntPtrConstant(0));
351 : }
352 :
353 56 : TNode<Float64T> CodeStubAssembler::Float64Round(SloppyTNode<Float64T> x) {
354 56 : Node* one = Float64Constant(1.0);
355 56 : Node* one_half = Float64Constant(0.5);
356 :
357 112 : Label return_x(this);
358 :
359 : // Round up {x} towards Infinity.
360 112 : VARIABLE(var_x, MachineRepresentation::kFloat64, Float64Ceil(x));
361 :
362 112 : GotoIf(Float64LessThanOrEqual(Float64Sub(var_x.value(), one_half), x),
363 56 : &return_x);
364 56 : var_x.Bind(Float64Sub(var_x.value(), one));
365 56 : Goto(&return_x);
366 :
367 56 : BIND(&return_x);
368 112 : return TNode<Float64T>::UncheckedCast(var_x.value());
369 : }
370 :
371 112 : TNode<Float64T> CodeStubAssembler::Float64Ceil(SloppyTNode<Float64T> x) {
372 112 : if (IsFloat64RoundUpSupported()) {
373 110 : return Float64RoundUp(x);
374 : }
375 :
376 2 : Node* one = Float64Constant(1.0);
377 2 : Node* zero = Float64Constant(0.0);
378 2 : Node* two_52 = Float64Constant(4503599627370496.0E0);
379 2 : Node* minus_two_52 = Float64Constant(-4503599627370496.0E0);
380 :
381 4 : VARIABLE(var_x, MachineRepresentation::kFloat64, x);
382 4 : Label return_x(this), return_minus_x(this);
383 :
384 : // Check if {x} is greater than zero.
385 4 : Label if_xgreaterthanzero(this), if_xnotgreaterthanzero(this);
386 4 : Branch(Float64GreaterThan(x, zero), &if_xgreaterthanzero,
387 2 : &if_xnotgreaterthanzero);
388 :
389 2 : BIND(&if_xgreaterthanzero);
390 : {
391 : // Just return {x} unless it's in the range ]0,2^52[.
392 2 : GotoIf(Float64GreaterThanOrEqual(x, two_52), &return_x);
393 :
394 : // Round positive {x} towards Infinity.
395 2 : var_x.Bind(Float64Sub(Float64Add(two_52, x), two_52));
396 2 : GotoIfNot(Float64LessThan(var_x.value(), x), &return_x);
397 2 : var_x.Bind(Float64Add(var_x.value(), one));
398 2 : Goto(&return_x);
399 : }
400 :
401 2 : BIND(&if_xnotgreaterthanzero);
402 : {
403 : // Just return {x} unless it's in the range ]-2^52,0[
404 2 : GotoIf(Float64LessThanOrEqual(x, minus_two_52), &return_x);
405 2 : GotoIfNot(Float64LessThan(x, zero), &return_x);
406 :
407 : // Round negated {x} towards Infinity and return the result negated.
408 2 : Node* minus_x = Float64Neg(x);
409 2 : var_x.Bind(Float64Sub(Float64Add(two_52, minus_x), two_52));
410 2 : GotoIfNot(Float64GreaterThan(var_x.value(), minus_x), &return_minus_x);
411 2 : var_x.Bind(Float64Sub(var_x.value(), one));
412 2 : Goto(&return_minus_x);
413 : }
414 :
415 2 : BIND(&return_minus_x);
416 2 : var_x.Bind(Float64Neg(var_x.value()));
417 2 : Goto(&return_x);
418 :
419 2 : BIND(&return_x);
420 2 : return TNode<Float64T>::UncheckedCast(var_x.value());
421 : }
422 :
423 119 : TNode<Float64T> CodeStubAssembler::Float64Floor(SloppyTNode<Float64T> x) {
424 119 : if (IsFloat64RoundDownSupported()) {
425 110 : return Float64RoundDown(x);
426 : }
427 :
428 9 : Node* one = Float64Constant(1.0);
429 9 : Node* zero = Float64Constant(0.0);
430 9 : Node* two_52 = Float64Constant(4503599627370496.0E0);
431 9 : Node* minus_two_52 = Float64Constant(-4503599627370496.0E0);
432 :
433 18 : VARIABLE(var_x, MachineRepresentation::kFloat64, x);
434 18 : Label return_x(this), return_minus_x(this);
435 :
436 : // Check if {x} is greater than zero.
437 18 : Label if_xgreaterthanzero(this), if_xnotgreaterthanzero(this);
438 18 : Branch(Float64GreaterThan(x, zero), &if_xgreaterthanzero,
439 9 : &if_xnotgreaterthanzero);
440 :
441 9 : BIND(&if_xgreaterthanzero);
442 : {
443 : // Just return {x} unless it's in the range ]0,2^52[.
444 9 : GotoIf(Float64GreaterThanOrEqual(x, two_52), &return_x);
445 :
446 : // Round positive {x} towards -Infinity.
447 9 : var_x.Bind(Float64Sub(Float64Add(two_52, x), two_52));
448 9 : GotoIfNot(Float64GreaterThan(var_x.value(), x), &return_x);
449 9 : var_x.Bind(Float64Sub(var_x.value(), one));
450 9 : Goto(&return_x);
451 : }
452 :
453 9 : BIND(&if_xnotgreaterthanzero);
454 : {
455 : // Just return {x} unless it's in the range ]-2^52,0[
456 9 : GotoIf(Float64LessThanOrEqual(x, minus_two_52), &return_x);
457 9 : GotoIfNot(Float64LessThan(x, zero), &return_x);
458 :
459 : // Round negated {x} towards -Infinity and return the result negated.
460 9 : Node* minus_x = Float64Neg(x);
461 9 : var_x.Bind(Float64Sub(Float64Add(two_52, minus_x), two_52));
462 9 : GotoIfNot(Float64LessThan(var_x.value(), minus_x), &return_minus_x);
463 9 : var_x.Bind(Float64Add(var_x.value(), one));
464 9 : Goto(&return_minus_x);
465 : }
466 :
467 9 : BIND(&return_minus_x);
468 9 : var_x.Bind(Float64Neg(var_x.value()));
469 9 : Goto(&return_x);
470 :
471 9 : BIND(&return_x);
472 9 : return TNode<Float64T>::UncheckedCast(var_x.value());
473 : }
474 :
475 392 : TNode<Float64T> CodeStubAssembler::Float64RoundToEven(SloppyTNode<Float64T> x) {
476 392 : if (IsFloat64RoundTiesEvenSupported()) {
477 385 : return Float64RoundTiesEven(x);
478 : }
479 : // See ES#sec-touint8clamp for details.
480 7 : Node* f = Float64Floor(x);
481 7 : Node* f_and_half = Float64Add(f, Float64Constant(0.5));
482 :
483 14 : VARIABLE(var_result, MachineRepresentation::kFloat64);
484 14 : Label return_f(this), return_f_plus_one(this), done(this);
485 :
486 7 : GotoIf(Float64LessThan(f_and_half, x), &return_f_plus_one);
487 7 : GotoIf(Float64LessThan(x, f_and_half), &return_f);
488 : {
489 7 : Node* f_mod_2 = Float64Mod(f, Float64Constant(2.0));
490 14 : Branch(Float64Equal(f_mod_2, Float64Constant(0.0)), &return_f,
491 7 : &return_f_plus_one);
492 : }
493 :
494 7 : BIND(&return_f);
495 7 : var_result.Bind(f);
496 7 : Goto(&done);
497 :
498 7 : BIND(&return_f_plus_one);
499 7 : var_result.Bind(Float64Add(f, Float64Constant(1.0)));
500 7 : Goto(&done);
501 :
502 7 : BIND(&done);
503 7 : return TNode<Float64T>::UncheckedCast(var_result.value());
504 : }
505 :
506 340 : TNode<Float64T> CodeStubAssembler::Float64Trunc(SloppyTNode<Float64T> x) {
507 340 : if (IsFloat64RoundTruncateSupported()) {
508 334 : return Float64RoundTruncate(x);
509 : }
510 :
511 6 : Node* one = Float64Constant(1.0);
512 6 : Node* zero = Float64Constant(0.0);
513 6 : Node* two_52 = Float64Constant(4503599627370496.0E0);
514 6 : Node* minus_two_52 = Float64Constant(-4503599627370496.0E0);
515 :
516 12 : VARIABLE(var_x, MachineRepresentation::kFloat64, x);
517 12 : Label return_x(this), return_minus_x(this);
518 :
519 : // Check if {x} is greater than 0.
520 12 : Label if_xgreaterthanzero(this), if_xnotgreaterthanzero(this);
521 12 : Branch(Float64GreaterThan(x, zero), &if_xgreaterthanzero,
522 6 : &if_xnotgreaterthanzero);
523 :
524 6 : BIND(&if_xgreaterthanzero);
525 : {
526 6 : if (IsFloat64RoundDownSupported()) {
527 0 : var_x.Bind(Float64RoundDown(x));
528 : } else {
529 : // Just return {x} unless it's in the range ]0,2^52[.
530 6 : GotoIf(Float64GreaterThanOrEqual(x, two_52), &return_x);
531 :
532 : // Round positive {x} towards -Infinity.
533 6 : var_x.Bind(Float64Sub(Float64Add(two_52, x), two_52));
534 6 : GotoIfNot(Float64GreaterThan(var_x.value(), x), &return_x);
535 6 : var_x.Bind(Float64Sub(var_x.value(), one));
536 : }
537 6 : Goto(&return_x);
538 : }
539 :
540 6 : BIND(&if_xnotgreaterthanzero);
541 : {
542 6 : if (IsFloat64RoundUpSupported()) {
543 0 : var_x.Bind(Float64RoundUp(x));
544 0 : Goto(&return_x);
545 : } else {
546 : // Just return {x} unless its in the range ]-2^52,0[.
547 6 : GotoIf(Float64LessThanOrEqual(x, minus_two_52), &return_x);
548 6 : GotoIfNot(Float64LessThan(x, zero), &return_x);
549 :
550 : // Round negated {x} towards -Infinity and return result negated.
551 6 : Node* minus_x = Float64Neg(x);
552 6 : var_x.Bind(Float64Sub(Float64Add(two_52, minus_x), two_52));
553 6 : GotoIfNot(Float64GreaterThan(var_x.value(), minus_x), &return_minus_x);
554 6 : var_x.Bind(Float64Sub(var_x.value(), one));
555 6 : Goto(&return_minus_x);
556 : }
557 : }
558 :
559 6 : BIND(&return_minus_x);
560 6 : var_x.Bind(Float64Neg(var_x.value()));
561 6 : Goto(&return_x);
562 :
563 6 : BIND(&return_x);
564 6 : return TNode<Float64T>::UncheckedCast(var_x.value());
565 : }
566 :
567 0 : TNode<BoolT> CodeStubAssembler::IsValidSmi(TNode<Smi> smi) {
568 0 : if (SmiValuesAre31Bits() && kSystemPointerSize == kInt64Size) {
569 : // Check that the Smi value is properly sign-extended.
570 0 : TNode<IntPtrT> value = Signed(BitcastTaggedToWord(smi));
571 0 : return WordEqual(value, ChangeInt32ToIntPtr(TruncateIntPtrToInt32(value)));
572 : }
573 0 : return Int32TrueConstant();
574 : }
575 :
576 355117 : Node* CodeStubAssembler::SmiShiftBitsConstant() {
577 355117 : return IntPtrConstant(kSmiShiftSize + kSmiTagSize);
578 : }
579 :
580 10768 : TNode<Smi> CodeStubAssembler::SmiFromInt32(SloppyTNode<Int32T> value) {
581 10768 : TNode<IntPtrT> value_intptr = ChangeInt32ToIntPtr(value);
582 : TNode<Smi> smi =
583 10768 : BitcastWordToTaggedSigned(WordShl(value_intptr, SmiShiftBitsConstant()));
584 10768 : return smi;
585 : }
586 :
587 7160 : TNode<BoolT> CodeStubAssembler::IsValidPositiveSmi(TNode<IntPtrT> value) {
588 : intptr_t constant_value;
589 7160 : if (ToIntPtrConstant(value, constant_value)) {
590 44 : return (static_cast<uintptr_t>(constant_value) <=
591 : static_cast<uintptr_t>(Smi::kMaxValue))
592 12 : ? Int32TrueConstant()
593 56 : : Int32FalseConstant();
594 : }
595 :
596 7116 : return UintPtrLessThanOrEqual(value, IntPtrConstant(Smi::kMaxValue));
597 : }
598 :
599 122114 : TNode<Smi> CodeStubAssembler::SmiTag(SloppyTNode<IntPtrT> value) {
600 : int32_t constant_value;
601 122114 : if (ToInt32Constant(value, constant_value) && Smi::IsValid(constant_value)) {
602 6861 : return SmiConstant(constant_value);
603 : }
604 : TNode<Smi> smi =
605 115253 : BitcastWordToTaggedSigned(WordShl(value, SmiShiftBitsConstant()));
606 115253 : return smi;
607 : }
608 :
609 229173 : TNode<IntPtrT> CodeStubAssembler::SmiUntag(SloppyTNode<Smi> value) {
610 : intptr_t constant_value;
611 229173 : if (ToIntPtrConstant(value, constant_value)) {
612 417 : return IntPtrConstant(constant_value >> (kSmiShiftSize + kSmiTagSize));
613 : }
614 228756 : return Signed(WordSar(BitcastTaggedToWord(value), SmiShiftBitsConstant()));
615 : }
616 :
617 85580 : TNode<Int32T> CodeStubAssembler::SmiToInt32(SloppyTNode<Smi> value) {
618 85580 : TNode<IntPtrT> result = SmiUntag(value);
619 85580 : return TruncateIntPtrToInt32(result);
620 : }
621 :
622 47612 : TNode<Float64T> CodeStubAssembler::SmiToFloat64(SloppyTNode<Smi> value) {
623 47612 : return ChangeInt32ToFloat64(SmiToInt32(value));
624 : }
625 :
626 2016 : TNode<Smi> CodeStubAssembler::SmiMax(TNode<Smi> a, TNode<Smi> b) {
627 2016 : return SelectConstant<Smi>(SmiLessThan(a, b), b, a);
628 : }
629 :
630 224 : TNode<Smi> CodeStubAssembler::SmiMin(TNode<Smi> a, TNode<Smi> b) {
631 224 : return SelectConstant<Smi>(SmiLessThan(a, b), a, b);
632 : }
633 :
634 672 : TNode<IntPtrT> CodeStubAssembler::TryIntPtrAdd(TNode<IntPtrT> a,
635 : TNode<IntPtrT> b,
636 : Label* if_overflow) {
637 672 : TNode<PairT<IntPtrT, BoolT>> pair = IntPtrAddWithOverflow(a, b);
638 672 : TNode<BoolT> overflow = Projection<1>(pair);
639 672 : GotoIf(overflow, if_overflow);
640 672 : return Projection<0>(pair);
641 : }
642 :
643 4148 : TNode<Smi> CodeStubAssembler::TrySmiAdd(TNode<Smi> lhs, TNode<Smi> rhs,
644 : Label* if_overflow) {
645 4148 : if (SmiValuesAre32Bits()) {
646 0 : return BitcastWordToTaggedSigned(TryIntPtrAdd(
647 0 : BitcastTaggedToWord(lhs), BitcastTaggedToWord(rhs), if_overflow));
648 : } else {
649 : DCHECK(SmiValuesAre31Bits());
650 : TNode<PairT<Int32T, BoolT>> pair =
651 8296 : Int32AddWithOverflow(TruncateIntPtrToInt32(BitcastTaggedToWord(lhs)),
652 12444 : TruncateIntPtrToInt32(BitcastTaggedToWord(rhs)));
653 4148 : TNode<BoolT> overflow = Projection<1>(pair);
654 4148 : GotoIf(overflow, if_overflow);
655 4148 : TNode<Int32T> result = Projection<0>(pair);
656 4148 : return BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(result));
657 : }
658 : }
659 :
660 2916 : TNode<Smi> CodeStubAssembler::TrySmiSub(TNode<Smi> lhs, TNode<Smi> rhs,
661 : Label* if_overflow) {
662 2916 : if (SmiValuesAre32Bits()) {
663 : TNode<PairT<IntPtrT, BoolT>> pair = IntPtrSubWithOverflow(
664 0 : BitcastTaggedToWord(lhs), BitcastTaggedToWord(rhs));
665 0 : TNode<BoolT> overflow = Projection<1>(pair);
666 0 : GotoIf(overflow, if_overflow);
667 0 : TNode<IntPtrT> result = Projection<0>(pair);
668 0 : return BitcastWordToTaggedSigned(result);
669 : } else {
670 : DCHECK(SmiValuesAre31Bits());
671 : TNode<PairT<Int32T, BoolT>> pair =
672 5832 : Int32SubWithOverflow(TruncateIntPtrToInt32(BitcastTaggedToWord(lhs)),
673 8748 : TruncateIntPtrToInt32(BitcastTaggedToWord(rhs)));
674 2916 : TNode<BoolT> overflow = Projection<1>(pair);
675 2916 : GotoIf(overflow, if_overflow);
676 2916 : TNode<Int32T> result = Projection<0>(pair);
677 2916 : return BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(result));
678 : }
679 : }
680 :
681 564 : TNode<Number> CodeStubAssembler::NumberMax(SloppyTNode<Number> a,
682 : SloppyTNode<Number> b) {
683 : // TODO(danno): This could be optimized by specifically handling smi cases.
684 1128 : TVARIABLE(Number, result);
685 1128 : Label done(this), greater_than_equal_a(this), greater_than_equal_b(this);
686 564 : GotoIfNumberGreaterThanOrEqual(a, b, &greater_than_equal_a);
687 564 : GotoIfNumberGreaterThanOrEqual(b, a, &greater_than_equal_b);
688 564 : result = NanConstant();
689 564 : Goto(&done);
690 564 : BIND(&greater_than_equal_a);
691 564 : result = a;
692 564 : Goto(&done);
693 564 : BIND(&greater_than_equal_b);
694 564 : result = b;
695 564 : Goto(&done);
696 564 : BIND(&done);
697 1128 : return result.value();
698 : }
699 :
700 620 : TNode<Number> CodeStubAssembler::NumberMin(SloppyTNode<Number> a,
701 : SloppyTNode<Number> b) {
702 : // TODO(danno): This could be optimized by specifically handling smi cases.
703 1240 : TVARIABLE(Number, result);
704 1240 : Label done(this), greater_than_equal_a(this), greater_than_equal_b(this);
705 620 : GotoIfNumberGreaterThanOrEqual(a, b, &greater_than_equal_a);
706 620 : GotoIfNumberGreaterThanOrEqual(b, a, &greater_than_equal_b);
707 620 : result = NanConstant();
708 620 : Goto(&done);
709 620 : BIND(&greater_than_equal_a);
710 620 : result = b;
711 620 : Goto(&done);
712 620 : BIND(&greater_than_equal_b);
713 620 : result = a;
714 620 : Goto(&done);
715 620 : BIND(&done);
716 1240 : return result.value();
717 : }
718 :
719 392 : TNode<IntPtrT> CodeStubAssembler::ConvertToRelativeIndex(
720 : TNode<Context> context, TNode<Object> index, TNode<IntPtrT> length) {
721 784 : TVARIABLE(IntPtrT, result);
722 :
723 : TNode<Number> const index_int =
724 392 : ToInteger_Inline(context, index, CodeStubAssembler::kTruncateMinusZero);
725 392 : TNode<IntPtrT> zero = IntPtrConstant(0);
726 :
727 784 : Label done(this);
728 784 : Label if_issmi(this), if_isheapnumber(this, Label::kDeferred);
729 392 : Branch(TaggedIsSmi(index_int), &if_issmi, &if_isheapnumber);
730 :
731 392 : BIND(&if_issmi);
732 : {
733 392 : TNode<Smi> const index_smi = CAST(index_int);
734 1568 : result = Select<IntPtrT>(
735 784 : IntPtrLessThan(SmiUntag(index_smi), zero),
736 392 : [=] { return IntPtrMax(IntPtrAdd(length, SmiUntag(index_smi)), zero); },
737 784 : [=] { return IntPtrMin(SmiUntag(index_smi), length); });
738 392 : Goto(&done);
739 : }
740 :
741 392 : BIND(&if_isheapnumber);
742 : {
743 : // If {index} is a heap number, it is definitely out of bounds. If it is
744 : // negative, {index} = max({length} + {index}),0) = 0'. If it is positive,
745 : // set {index} to {length}.
746 392 : TNode<HeapNumber> const index_hn = CAST(index_int);
747 392 : TNode<Float64T> const float_zero = Float64Constant(0.);
748 392 : TNode<Float64T> const index_float = LoadHeapNumberValue(index_hn);
749 784 : result = SelectConstant<IntPtrT>(Float64LessThan(index_float, float_zero),
750 392 : zero, length);
751 392 : Goto(&done);
752 : }
753 392 : BIND(&done);
754 784 : return result.value();
755 : }
756 :
757 392 : TNode<Number> CodeStubAssembler::SmiMod(TNode<Smi> a, TNode<Smi> b) {
758 784 : TVARIABLE(Number, var_result);
759 784 : Label return_result(this, &var_result),
760 784 : return_minuszero(this, Label::kDeferred),
761 784 : return_nan(this, Label::kDeferred);
762 :
763 : // Untag {a} and {b}.
764 392 : TNode<Int32T> int_a = SmiToInt32(a);
765 392 : TNode<Int32T> int_b = SmiToInt32(b);
766 :
767 : // Return NaN if {b} is zero.
768 392 : GotoIf(Word32Equal(int_b, Int32Constant(0)), &return_nan);
769 :
770 : // Check if {a} is non-negative.
771 784 : Label if_aisnotnegative(this), if_aisnegative(this, Label::kDeferred);
772 784 : Branch(Int32LessThanOrEqual(Int32Constant(0), int_a), &if_aisnotnegative,
773 392 : &if_aisnegative);
774 :
775 392 : BIND(&if_aisnotnegative);
776 : {
777 : // Fast case, don't need to check any other edge cases.
778 392 : TNode<Int32T> r = Int32Mod(int_a, int_b);
779 392 : var_result = SmiFromInt32(r);
780 392 : Goto(&return_result);
781 : }
782 :
783 392 : BIND(&if_aisnegative);
784 : {
785 392 : if (SmiValuesAre32Bits()) {
786 : // Check if {a} is kMinInt and {b} is -1 (only relevant if the
787 : // kMinInt is actually representable as a Smi).
788 0 : Label join(this);
789 0 : GotoIfNot(Word32Equal(int_a, Int32Constant(kMinInt)), &join);
790 0 : GotoIf(Word32Equal(int_b, Int32Constant(-1)), &return_minuszero);
791 0 : Goto(&join);
792 0 : BIND(&join);
793 : }
794 :
795 : // Perform the integer modulus operation.
796 392 : TNode<Int32T> r = Int32Mod(int_a, int_b);
797 :
798 : // Check if {r} is zero, and if so return -0, because we have to
799 : // take the sign of the left hand side {a}, which is negative.
800 392 : GotoIf(Word32Equal(r, Int32Constant(0)), &return_minuszero);
801 :
802 : // The remainder {r} can be outside the valid Smi range on 32bit
803 : // architectures, so we cannot just say SmiFromInt32(r) here.
804 392 : var_result = ChangeInt32ToTagged(r);
805 392 : Goto(&return_result);
806 : }
807 :
808 392 : BIND(&return_minuszero);
809 392 : var_result = MinusZeroConstant();
810 392 : Goto(&return_result);
811 :
812 392 : BIND(&return_nan);
813 392 : var_result = NanConstant();
814 392 : Goto(&return_result);
815 :
816 392 : BIND(&return_result);
817 784 : return var_result.value();
818 : }
819 :
820 448 : TNode<Number> CodeStubAssembler::SmiMul(TNode<Smi> a, TNode<Smi> b) {
821 896 : TVARIABLE(Number, var_result);
822 896 : VARIABLE(var_lhs_float64, MachineRepresentation::kFloat64);
823 896 : VARIABLE(var_rhs_float64, MachineRepresentation::kFloat64);
824 896 : Label return_result(this, &var_result);
825 :
826 : // Both {a} and {b} are Smis. Convert them to integers and multiply.
827 448 : Node* lhs32 = SmiToInt32(a);
828 448 : Node* rhs32 = SmiToInt32(b);
829 448 : Node* pair = Int32MulWithOverflow(lhs32, rhs32);
830 :
831 448 : Node* overflow = Projection(1, pair);
832 :
833 : // Check if the multiplication overflowed.
834 896 : Label if_overflow(this, Label::kDeferred), if_notoverflow(this);
835 448 : Branch(overflow, &if_overflow, &if_notoverflow);
836 448 : BIND(&if_notoverflow);
837 : {
838 : // If the answer is zero, we may need to return -0.0, depending on the
839 : // input.
840 896 : Label answer_zero(this), answer_not_zero(this);
841 448 : Node* answer = Projection(0, pair);
842 448 : Node* zero = Int32Constant(0);
843 448 : Branch(Word32Equal(answer, zero), &answer_zero, &answer_not_zero);
844 448 : BIND(&answer_not_zero);
845 : {
846 448 : var_result = ChangeInt32ToTagged(answer);
847 448 : Goto(&return_result);
848 : }
849 448 : BIND(&answer_zero);
850 : {
851 448 : Node* or_result = Word32Or(lhs32, rhs32);
852 896 : Label if_should_be_negative_zero(this), if_should_be_zero(this);
853 896 : Branch(Int32LessThan(or_result, zero), &if_should_be_negative_zero,
854 448 : &if_should_be_zero);
855 448 : BIND(&if_should_be_negative_zero);
856 : {
857 448 : var_result = MinusZeroConstant();
858 448 : Goto(&return_result);
859 : }
860 448 : BIND(&if_should_be_zero);
861 : {
862 448 : var_result = SmiConstant(0);
863 448 : Goto(&return_result);
864 : }
865 : }
866 : }
867 448 : BIND(&if_overflow);
868 : {
869 448 : var_lhs_float64.Bind(SmiToFloat64(a));
870 448 : var_rhs_float64.Bind(SmiToFloat64(b));
871 448 : Node* value = Float64Mul(var_lhs_float64.value(), var_rhs_float64.value());
872 448 : var_result = AllocateHeapNumberWithValue(value);
873 448 : Goto(&return_result);
874 : }
875 :
876 448 : BIND(&return_result);
877 896 : return var_result.value();
878 : }
879 :
880 336 : TNode<Smi> CodeStubAssembler::TrySmiDiv(TNode<Smi> dividend, TNode<Smi> divisor,
881 : Label* bailout) {
882 : // Both {a} and {b} are Smis. Bailout to floating point division if {divisor}
883 : // is zero.
884 336 : GotoIf(WordEqual(divisor, SmiConstant(0)), bailout);
885 :
886 : // Do floating point division if {dividend} is zero and {divisor} is
887 : // negative.
888 672 : Label dividend_is_zero(this), dividend_is_not_zero(this);
889 672 : Branch(WordEqual(dividend, SmiConstant(0)), ÷nd_is_zero,
890 336 : ÷nd_is_not_zero);
891 :
892 336 : BIND(÷nd_is_zero);
893 : {
894 336 : GotoIf(SmiLessThan(divisor, SmiConstant(0)), bailout);
895 336 : Goto(÷nd_is_not_zero);
896 : }
897 336 : BIND(÷nd_is_not_zero);
898 :
899 336 : TNode<Int32T> untagged_divisor = SmiToInt32(divisor);
900 336 : TNode<Int32T> untagged_dividend = SmiToInt32(dividend);
901 :
902 : // Do floating point division if {dividend} is kMinInt (or kMinInt - 1
903 : // if the Smi size is 31) and {divisor} is -1.
904 672 : Label divisor_is_minus_one(this), divisor_is_not_minus_one(this);
905 672 : Branch(Word32Equal(untagged_divisor, Int32Constant(-1)),
906 336 : &divisor_is_minus_one, &divisor_is_not_minus_one);
907 :
908 336 : BIND(&divisor_is_minus_one);
909 : {
910 672 : GotoIf(Word32Equal(
911 : untagged_dividend,
912 672 : Int32Constant(kSmiValueSize == 32 ? kMinInt : (kMinInt >> 1))),
913 336 : bailout);
914 336 : Goto(&divisor_is_not_minus_one);
915 : }
916 336 : BIND(&divisor_is_not_minus_one);
917 :
918 336 : TNode<Int32T> untagged_result = Int32Div(untagged_dividend, untagged_divisor);
919 336 : TNode<Int32T> truncated = Signed(Int32Mul(untagged_result, untagged_divisor));
920 :
921 : // Do floating point division if the remainder is not 0.
922 336 : GotoIf(Word32NotEqual(untagged_dividend, truncated), bailout);
923 :
924 672 : return SmiFromInt32(untagged_result);
925 : }
926 :
927 56 : TNode<Smi> CodeStubAssembler::SmiLexicographicCompare(TNode<Smi> x,
928 : TNode<Smi> y) {
929 : TNode<ExternalReference> smi_lexicographic_compare =
930 56 : ExternalConstant(ExternalReference::smi_lexicographic_compare_function());
931 : TNode<ExternalReference> isolate_ptr =
932 56 : ExternalConstant(ExternalReference::isolate_address(isolate()));
933 56 : return CAST(CallCFunction3(MachineType::AnyTagged(), MachineType::Pointer(),
934 : MachineType::AnyTagged(), MachineType::AnyTagged(),
935 : smi_lexicographic_compare, isolate_ptr, x, y));
936 : }
937 :
938 375988 : TNode<Int32T> CodeStubAssembler::TruncateIntPtrToInt32(
939 : SloppyTNode<IntPtrT> value) {
940 375988 : if (Is64()) {
941 375988 : return TruncateInt64ToInt32(ReinterpretCast<Int64T>(value));
942 : }
943 0 : return ReinterpretCast<Int32T>(value);
944 : }
945 :
946 164524 : TNode<BoolT> CodeStubAssembler::TaggedIsSmi(SloppyTNode<Object> a) {
947 329048 : return WordEqual(WordAnd(BitcastTaggedToWord(a), IntPtrConstant(kSmiTagMask)),
948 493572 : IntPtrConstant(0));
949 : }
950 :
951 1680 : TNode<BoolT> CodeStubAssembler::TaggedIsSmi(TNode<MaybeObject> a) {
952 : return WordEqual(
953 3360 : WordAnd(BitcastMaybeObjectToWord(a), IntPtrConstant(kSmiTagMask)),
954 5040 : IntPtrConstant(0));
955 : }
956 :
957 24464 : TNode<BoolT> CodeStubAssembler::TaggedIsNotSmi(SloppyTNode<Object> a) {
958 : return WordNotEqual(
959 48928 : WordAnd(BitcastTaggedToWord(a), IntPtrConstant(kSmiTagMask)),
960 73392 : IntPtrConstant(0));
961 : }
962 :
963 1972 : TNode<BoolT> CodeStubAssembler::TaggedIsPositiveSmi(SloppyTNode<Object> a) {
964 3944 : return WordEqual(WordAnd(BitcastTaggedToWord(a),
965 1972 : IntPtrConstant(kSmiTagMask | kSmiSignMask)),
966 5916 : IntPtrConstant(0));
967 : }
968 :
969 0 : TNode<BoolT> CodeStubAssembler::WordIsAligned(SloppyTNode<WordT> word,
970 : size_t alignment) {
971 : DCHECK(base::bits::IsPowerOfTwo(alignment));
972 0 : return WordEqual(IntPtrConstant(0),
973 0 : WordAnd(word, IntPtrConstant(alignment - 1)));
974 : }
975 :
976 : #if DEBUG
977 : void CodeStubAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
978 : CodeAssembler::Bind(label, debug_info);
979 : }
980 : #endif // DEBUG
981 :
982 1474621 : void CodeStubAssembler::Bind(Label* label) { CodeAssembler::Bind(label); }
983 :
984 1512 : TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck(
985 : TNode<FixedDoubleArray> array, TNode<Smi> index, Label* if_hole) {
986 : return LoadFixedDoubleArrayElement(array, index, MachineType::Float64(), 0,
987 1512 : SMI_PARAMETERS, if_hole);
988 : }
989 :
990 0 : TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck(
991 : TNode<FixedDoubleArray> array, TNode<IntPtrT> index, Label* if_hole) {
992 : return LoadFixedDoubleArrayElement(array, index, MachineType::Float64(), 0,
993 0 : INTPTR_PARAMETERS, if_hole);
994 : }
995 :
996 168 : void CodeStubAssembler::BranchIfPrototypesHaveNoElements(
997 : Node* receiver_map, Label* definitely_no_elements,
998 : Label* possibly_elements) {
999 : CSA_SLOW_ASSERT(this, IsMap(receiver_map));
1000 336 : VARIABLE(var_map, MachineRepresentation::kTagged, receiver_map);
1001 336 : Label loop_body(this, &var_map);
1002 168 : Node* empty_fixed_array = LoadRoot(RootIndex::kEmptyFixedArray);
1003 : Node* empty_slow_element_dictionary =
1004 168 : LoadRoot(RootIndex::kEmptySlowElementDictionary);
1005 168 : Goto(&loop_body);
1006 :
1007 168 : BIND(&loop_body);
1008 : {
1009 168 : Node* map = var_map.value();
1010 168 : Node* prototype = LoadMapPrototype(map);
1011 168 : GotoIf(IsNull(prototype), definitely_no_elements);
1012 168 : Node* prototype_map = LoadMap(prototype);
1013 168 : TNode<Int32T> prototype_instance_type = LoadMapInstanceType(prototype_map);
1014 :
1015 : // Pessimistically assume elements if a Proxy, Special API Object,
1016 : // or JSValue wrapper is found on the prototype chain. After this
1017 : // instance type check, it's not necessary to check for interceptors or
1018 : // access checks.
1019 336 : Label if_custom(this, Label::kDeferred), if_notcustom(this);
1020 336 : Branch(IsCustomElementsReceiverInstanceType(prototype_instance_type),
1021 168 : &if_custom, &if_notcustom);
1022 :
1023 168 : BIND(&if_custom);
1024 : {
1025 : // For string JSValue wrappers we still support the checks as long
1026 : // as they wrap the empty string.
1027 336 : GotoIfNot(InstanceTypeEqual(prototype_instance_type, JS_VALUE_TYPE),
1028 168 : possibly_elements);
1029 168 : Node* prototype_value = LoadJSValueValue(prototype);
1030 168 : Branch(IsEmptyString(prototype_value), &if_notcustom, possibly_elements);
1031 : }
1032 :
1033 168 : BIND(&if_notcustom);
1034 : {
1035 168 : Node* prototype_elements = LoadElements(prototype);
1036 168 : var_map.Bind(prototype_map);
1037 168 : GotoIf(WordEqual(prototype_elements, empty_fixed_array), &loop_body);
1038 336 : Branch(WordEqual(prototype_elements, empty_slow_element_dictionary),
1039 168 : &loop_body, possibly_elements);
1040 : }
1041 : }
1042 168 : }
1043 :
1044 1904 : void CodeStubAssembler::BranchIfJSReceiver(Node* object, Label* if_true,
1045 : Label* if_false) {
1046 1904 : GotoIf(TaggedIsSmi(object), if_false);
1047 : STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
1048 1904 : Branch(IsJSReceiver(object), if_true, if_false);
1049 1904 : }
1050 :
1051 2296 : void CodeStubAssembler::GotoIfForceSlowPath(Label* if_true) {
1052 : #ifdef V8_ENABLE_FORCE_SLOW_PATH
1053 : Node* const force_slow_path_addr =
1054 : ExternalConstant(ExternalReference::force_slow_path(isolate()));
1055 : Node* const force_slow = Load(MachineType::Uint8(), force_slow_path_addr);
1056 :
1057 : GotoIf(force_slow, if_true);
1058 : #endif
1059 2296 : }
1060 :
1061 4 : void CodeStubAssembler::GotoIfDebugExecutionModeChecksSideEffects(
1062 : Label* if_true) {
1063 : STATIC_ASSERT(sizeof(DebugInfo::ExecutionMode) >= sizeof(int32_t));
1064 :
1065 : TNode<ExternalReference> execution_mode_address = ExternalConstant(
1066 4 : ExternalReference::debug_execution_mode_address(isolate()));
1067 : TNode<Int32T> execution_mode =
1068 4 : UncheckedCast<Int32T>(Load(MachineType::Int32(), execution_mode_address));
1069 :
1070 8 : GotoIf(Word32Equal(execution_mode, Int32Constant(DebugInfo::kSideEffects)),
1071 4 : if_true);
1072 4 : }
1073 :
1074 7536 : TNode<HeapObject> CodeStubAssembler::AllocateRaw(TNode<IntPtrT> size_in_bytes,
1075 : AllocationFlags flags,
1076 : TNode<RawPtrT> top_address,
1077 : TNode<RawPtrT> limit_address) {
1078 15072 : Label if_out_of_memory(this, Label::kDeferred);
1079 :
1080 : // TODO(jgruber,jkummerow): Extract the slow paths (= probably everything
1081 : // but bump pointer allocation) into a builtin to save code space. The
1082 : // size_in_bytes check may be moved there as well since a non-smi
1083 : // size_in_bytes probably doesn't fit into the bump pointer region
1084 : // (double-check that).
1085 :
1086 : intptr_t size_in_bytes_constant;
1087 7536 : bool size_in_bytes_is_constant = false;
1088 7536 : if (ToIntPtrConstant(size_in_bytes, size_in_bytes_constant)) {
1089 588 : size_in_bytes_is_constant = true;
1090 1176 : CHECK(Internals::IsValidSmi(size_in_bytes_constant));
1091 1176 : CHECK_GT(size_in_bytes_constant, 0);
1092 : } else {
1093 6948 : GotoIfNot(IsValidPositiveSmi(size_in_bytes), &if_out_of_memory);
1094 : }
1095 :
1096 : TNode<RawPtrT> top =
1097 7536 : UncheckedCast<RawPtrT>(Load(MachineType::Pointer(), top_address));
1098 : TNode<RawPtrT> limit =
1099 7536 : UncheckedCast<RawPtrT>(Load(MachineType::Pointer(), limit_address));
1100 :
1101 : // If there's not enough space, call the runtime.
1102 15072 : TVARIABLE(Object, result);
1103 15072 : Label runtime_call(this, Label::kDeferred), no_runtime_call(this), out(this);
1104 :
1105 7536 : bool needs_double_alignment = flags & kDoubleAlignment;
1106 :
1107 7536 : if (flags & kAllowLargeObjectAllocation) {
1108 11128 : Label next(this);
1109 5564 : GotoIf(IsRegularHeapObjectSize(size_in_bytes), &next);
1110 :
1111 5564 : if (FLAG_young_generation_large_objects) {
1112 22256 : result = CallRuntime(Runtime::kAllocateInYoungGeneration,
1113 16692 : NoContextConstant(), SmiTag(size_in_bytes));
1114 : } else {
1115 : TNode<Smi> alignment_flag = SmiConstant(Smi::FromInt(
1116 0 : AllocateDoubleAlignFlag::encode(needs_double_alignment)));
1117 0 : result =
1118 0 : CallRuntime(Runtime::kAllocateInOldGeneration, NoContextConstant(),
1119 0 : SmiTag(size_in_bytes), alignment_flag);
1120 : }
1121 5564 : Goto(&out);
1122 :
1123 5564 : BIND(&next);
1124 : }
1125 :
1126 15072 : TVARIABLE(IntPtrT, adjusted_size, size_in_bytes);
1127 :
1128 7536 : if (needs_double_alignment) {
1129 0 : Label next(this);
1130 0 : GotoIfNot(WordAnd(top, IntPtrConstant(kDoubleAlignmentMask)), &next);
1131 :
1132 0 : adjusted_size = IntPtrAdd(size_in_bytes, IntPtrConstant(4));
1133 0 : Goto(&next);
1134 :
1135 0 : BIND(&next);
1136 : }
1137 :
1138 : TNode<IntPtrT> new_top =
1139 7536 : IntPtrAdd(UncheckedCast<IntPtrT>(top), adjusted_size.value());
1140 :
1141 15072 : Branch(UintPtrGreaterThanOrEqual(new_top, limit), &runtime_call,
1142 7536 : &no_runtime_call);
1143 :
1144 7536 : BIND(&runtime_call);
1145 : {
1146 7536 : if (flags & kPretenured) {
1147 : TNode<Smi> runtime_flags = SmiConstant(Smi::FromInt(
1148 0 : AllocateDoubleAlignFlag::encode(needs_double_alignment)));
1149 0 : result =
1150 0 : CallRuntime(Runtime::kAllocateInOldGeneration, NoContextConstant(),
1151 0 : SmiTag(size_in_bytes), runtime_flags);
1152 : } else {
1153 30144 : result = CallRuntime(Runtime::kAllocateInYoungGeneration,
1154 22608 : NoContextConstant(), SmiTag(size_in_bytes));
1155 : }
1156 7536 : Goto(&out);
1157 : }
1158 :
1159 : // When there is enough space, return `top' and bump it up.
1160 7536 : BIND(&no_runtime_call);
1161 : {
1162 7536 : StoreNoWriteBarrier(MachineType::PointerRepresentation(), top_address,
1163 7536 : new_top);
1164 :
1165 15072 : TVARIABLE(IntPtrT, address, UncheckedCast<IntPtrT>(top));
1166 :
1167 7536 : if (needs_double_alignment) {
1168 0 : Label next(this);
1169 0 : GotoIf(IntPtrEqual(adjusted_size.value(), size_in_bytes), &next);
1170 :
1171 : // Store a filler and increase the address by 4.
1172 0 : StoreNoWriteBarrier(MachineRepresentation::kTagged, top,
1173 0 : LoadRoot(RootIndex::kOnePointerFillerMap));
1174 0 : address = IntPtrAdd(UncheckedCast<IntPtrT>(top), IntPtrConstant(4));
1175 0 : Goto(&next);
1176 :
1177 0 : BIND(&next);
1178 : }
1179 :
1180 15072 : result = BitcastWordToTagged(
1181 22608 : IntPtrAdd(address.value(), IntPtrConstant(kHeapObjectTag)));
1182 7536 : Goto(&out);
1183 : }
1184 :
1185 7536 : if (!size_in_bytes_is_constant) {
1186 6948 : BIND(&if_out_of_memory);
1187 : CallRuntime(Runtime::kFatalProcessOutOfMemoryInAllocateRaw,
1188 6948 : NoContextConstant());
1189 6948 : Unreachable();
1190 : }
1191 :
1192 7536 : BIND(&out);
1193 15072 : return UncheckedCast<HeapObject>(result.value());
1194 : }
1195 :
1196 4748 : TNode<HeapObject> CodeStubAssembler::AllocateRawUnaligned(
1197 : TNode<IntPtrT> size_in_bytes, AllocationFlags flags,
1198 : TNode<RawPtrT> top_address, TNode<RawPtrT> limit_address) {
1199 : DCHECK_EQ(flags & kDoubleAlignment, 0);
1200 4748 : return AllocateRaw(size_in_bytes, flags, top_address, limit_address);
1201 : }
1202 :
1203 2788 : TNode<HeapObject> CodeStubAssembler::AllocateRawDoubleAligned(
1204 : TNode<IntPtrT> size_in_bytes, AllocationFlags flags,
1205 : TNode<RawPtrT> top_address, TNode<RawPtrT> limit_address) {
1206 : #if defined(V8_HOST_ARCH_32_BIT)
1207 : return AllocateRaw(size_in_bytes, flags | kDoubleAlignment, top_address,
1208 : limit_address);
1209 : #elif defined(V8_HOST_ARCH_64_BIT)
1210 : #ifdef V8_COMPRESS_POINTERS
1211 : // TODO(ishell, v8:8875): Consider using aligned allocations once the
1212 : // allocation alignment inconsistency is fixed. For now we keep using
1213 : // unaligned access since both x64 and arm64 architectures (where pointer
1214 : // compression is supported) allow unaligned access to doubles and full words.
1215 : #endif // V8_COMPRESS_POINTERS
1216 : // Allocation on 64 bit machine is naturally double aligned
1217 : return AllocateRaw(size_in_bytes, flags & ~kDoubleAlignment, top_address,
1218 2788 : limit_address);
1219 : #else
1220 : #error Architecture not supported
1221 : #endif
1222 : }
1223 :
1224 13064 : TNode<HeapObject> CodeStubAssembler::AllocateInNewSpace(
1225 : TNode<IntPtrT> size_in_bytes, AllocationFlags flags) {
1226 : DCHECK(flags == kNone || flags == kDoubleAlignment);
1227 : CSA_ASSERT(this, IsRegularHeapObjectSize(size_in_bytes));
1228 13064 : return Allocate(size_in_bytes, flags);
1229 : }
1230 :
1231 71980 : TNode<HeapObject> CodeStubAssembler::Allocate(TNode<IntPtrT> size_in_bytes,
1232 : AllocationFlags flags) {
1233 71980 : Comment("Allocate");
1234 71980 : bool const new_space = !(flags & kPretenured);
1235 71980 : if (!(flags & kAllowLargeObjectAllocation)) {
1236 : intptr_t size_constant;
1237 66416 : if (ToIntPtrConstant(size_in_bytes, size_constant)) {
1238 101800 : CHECK_LE(size_constant, kMaxRegularHeapObjectSize);
1239 : }
1240 : }
1241 71980 : if (!(flags & kDoubleAlignment) && !(flags & kAllowLargeObjectAllocation)) {
1242 : return OptimizedAllocate(size_in_bytes, new_space ? AllocationType::kYoung
1243 64444 : : AllocationType::kOld);
1244 : }
1245 : TNode<ExternalReference> top_address = ExternalConstant(
1246 : new_space
1247 7536 : ? ExternalReference::new_space_allocation_top_address(isolate())
1248 15072 : : ExternalReference::old_space_allocation_top_address(isolate()));
1249 : DCHECK_EQ(kSystemPointerSize,
1250 : ExternalReference::new_space_allocation_limit_address(isolate())
1251 : .address() -
1252 : ExternalReference::new_space_allocation_top_address(isolate())
1253 : .address());
1254 : DCHECK_EQ(kSystemPointerSize,
1255 : ExternalReference::old_space_allocation_limit_address(isolate())
1256 : .address() -
1257 : ExternalReference::old_space_allocation_top_address(isolate())
1258 : .address());
1259 : TNode<IntPtrT> limit_address =
1260 : IntPtrAdd(ReinterpretCast<IntPtrT>(top_address),
1261 7536 : IntPtrConstant(kSystemPointerSize));
1262 :
1263 7536 : if (flags & kDoubleAlignment) {
1264 : return AllocateRawDoubleAligned(size_in_bytes, flags,
1265 : ReinterpretCast<RawPtrT>(top_address),
1266 2788 : ReinterpretCast<RawPtrT>(limit_address));
1267 : } else {
1268 : return AllocateRawUnaligned(size_in_bytes, flags,
1269 : ReinterpretCast<RawPtrT>(top_address),
1270 4748 : ReinterpretCast<RawPtrT>(limit_address));
1271 : }
1272 : }
1273 :
1274 1976 : TNode<HeapObject> CodeStubAssembler::AllocateInNewSpace(int size_in_bytes,
1275 : AllocationFlags flags) {
1276 1976 : CHECK(flags == kNone || flags == kDoubleAlignment);
1277 : DCHECK_LE(size_in_bytes, kMaxRegularHeapObjectSize);
1278 1976 : return CodeStubAssembler::Allocate(IntPtrConstant(size_in_bytes), flags);
1279 : }
1280 :
1281 39568 : TNode<HeapObject> CodeStubAssembler::Allocate(int size_in_bytes,
1282 : AllocationFlags flags) {
1283 39568 : return CodeStubAssembler::Allocate(IntPtrConstant(size_in_bytes), flags);
1284 : }
1285 :
1286 5432 : TNode<HeapObject> CodeStubAssembler::InnerAllocate(TNode<HeapObject> previous,
1287 : TNode<IntPtrT> offset) {
1288 : return UncheckedCast<HeapObject>(
1289 5432 : BitcastWordToTagged(IntPtrAdd(BitcastTaggedToWord(previous), offset)));
1290 : }
1291 :
1292 4312 : TNode<HeapObject> CodeStubAssembler::InnerAllocate(TNode<HeapObject> previous,
1293 : int offset) {
1294 4312 : return InnerAllocate(previous, IntPtrConstant(offset));
1295 : }
1296 :
1297 5732 : TNode<BoolT> CodeStubAssembler::IsRegularHeapObjectSize(TNode<IntPtrT> size) {
1298 : return UintPtrLessThanOrEqual(size,
1299 5732 : IntPtrConstant(kMaxRegularHeapObjectSize));
1300 : }
1301 :
1302 5040 : void CodeStubAssembler::BranchIfToBooleanIsTrue(Node* value, Label* if_true,
1303 : Label* if_false) {
1304 10080 : Label if_smi(this), if_notsmi(this), if_heapnumber(this, Label::kDeferred),
1305 10080 : if_bigint(this, Label::kDeferred);
1306 : // Rule out false {value}.
1307 5040 : GotoIf(WordEqual(value, FalseConstant()), if_false);
1308 :
1309 : // Check if {value} is a Smi or a HeapObject.
1310 5040 : Branch(TaggedIsSmi(value), &if_smi, &if_notsmi);
1311 :
1312 5040 : BIND(&if_smi);
1313 : {
1314 : // The {value} is a Smi, only need to check against zero.
1315 5040 : BranchIfSmiEqual(CAST(value), SmiConstant(0), if_false, if_true);
1316 : }
1317 :
1318 5040 : BIND(&if_notsmi);
1319 : {
1320 : // Check if {value} is the empty string.
1321 5040 : GotoIf(IsEmptyString(value), if_false);
1322 :
1323 : // The {value} is a HeapObject, load its map.
1324 5040 : Node* value_map = LoadMap(value);
1325 :
1326 : // Only null, undefined and document.all have the undetectable bit set,
1327 : // so we can return false immediately when that bit is set.
1328 5040 : GotoIf(IsUndetectableMap(value_map), if_false);
1329 :
1330 : // We still need to handle numbers specially, but all other {value}s
1331 : // that make it here yield true.
1332 5040 : GotoIf(IsHeapNumberMap(value_map), &if_heapnumber);
1333 5040 : Branch(IsBigInt(value), &if_bigint, if_true);
1334 :
1335 5040 : BIND(&if_heapnumber);
1336 : {
1337 : // Load the floating point value of {value}.
1338 10080 : Node* value_value = LoadObjectField(value, HeapNumber::kValueOffset,
1339 5040 : MachineType::Float64());
1340 :
1341 : // Check if the floating point {value} is neither 0.0, -0.0 nor NaN.
1342 10080 : Branch(Float64LessThan(Float64Constant(0.0), Float64Abs(value_value)),
1343 5040 : if_true, if_false);
1344 : }
1345 :
1346 5040 : BIND(&if_bigint);
1347 : {
1348 : Node* result =
1349 5040 : CallRuntime(Runtime::kBigIntToBoolean, NoContextConstant(), value);
1350 : CSA_ASSERT(this, IsBoolean(result));
1351 5040 : Branch(WordEqual(result, TrueConstant()), if_true, if_false);
1352 : }
1353 : }
1354 5040 : }
1355 :
1356 2184 : Node* CodeStubAssembler::LoadFromParentFrame(int offset, MachineType rep) {
1357 2184 : Node* frame_pointer = LoadParentFramePointer();
1358 2184 : return Load(rep, frame_pointer, IntPtrConstant(offset));
1359 : }
1360 :
1361 3192 : Node* CodeStubAssembler::LoadBufferObject(Node* buffer, int offset,
1362 : MachineType rep) {
1363 3192 : return Load(rep, buffer, IntPtrConstant(offset));
1364 : }
1365 :
1366 956752 : Node* CodeStubAssembler::LoadObjectField(SloppyTNode<HeapObject> object,
1367 : int offset, MachineType rep) {
1368 : CSA_ASSERT(this, IsStrong(object));
1369 956752 : return Load(rep, object, IntPtrConstant(offset - kHeapObjectTag));
1370 : }
1371 :
1372 9416 : Node* CodeStubAssembler::LoadObjectField(SloppyTNode<HeapObject> object,
1373 : SloppyTNode<IntPtrT> offset,
1374 : MachineType rep) {
1375 : CSA_ASSERT(this, IsStrong(object));
1376 9416 : return Load(rep, object, IntPtrSub(offset, IntPtrConstant(kHeapObjectTag)));
1377 : }
1378 :
1379 76100 : TNode<IntPtrT> CodeStubAssembler::LoadAndUntagObjectField(
1380 : SloppyTNode<HeapObject> object, int offset) {
1381 76100 : if (SmiValuesAre32Bits()) {
1382 : #if V8_TARGET_LITTLE_ENDIAN
1383 0 : offset += 4;
1384 : #endif
1385 : return ChangeInt32ToIntPtr(
1386 0 : LoadObjectField(object, offset, MachineType::Int32()));
1387 : } else {
1388 : return SmiToIntPtr(
1389 76100 : LoadObjectField(object, offset, MachineType::AnyTagged()));
1390 : }
1391 : }
1392 :
1393 4264 : TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32ObjectField(Node* object,
1394 : int offset) {
1395 4264 : if (SmiValuesAre32Bits()) {
1396 : #if V8_TARGET_LITTLE_ENDIAN
1397 0 : offset += 4;
1398 : #endif
1399 : return UncheckedCast<Int32T>(
1400 0 : LoadObjectField(object, offset, MachineType::Int32()));
1401 : } else {
1402 : return SmiToInt32(
1403 4264 : LoadObjectField(object, offset, MachineType::AnyTagged()));
1404 : }
1405 : }
1406 :
1407 1512 : TNode<IntPtrT> CodeStubAssembler::LoadAndUntagSmi(Node* base, int index) {
1408 1512 : if (SmiValuesAre32Bits()) {
1409 : #if V8_TARGET_LITTLE_ENDIAN
1410 0 : index += 4;
1411 : #endif
1412 : return ChangeInt32ToIntPtr(
1413 0 : Load(MachineType::Int32(), base, IntPtrConstant(index)));
1414 : } else {
1415 : return SmiToIntPtr(
1416 1512 : Load(MachineType::AnyTagged(), base, IntPtrConstant(index)));
1417 : }
1418 : }
1419 :
1420 57077 : void CodeStubAssembler::StoreAndTagSmi(Node* base, int offset, Node* value) {
1421 57077 : if (SmiValuesAre32Bits()) {
1422 0 : int zero_offset = offset + 4;
1423 0 : int payload_offset = offset;
1424 : #if V8_TARGET_LITTLE_ENDIAN
1425 0 : std::swap(zero_offset, payload_offset);
1426 : #endif
1427 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, base,
1428 0 : IntPtrConstant(zero_offset), Int32Constant(0));
1429 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, base,
1430 0 : IntPtrConstant(payload_offset),
1431 0 : TruncateInt64ToInt32(value));
1432 : } else {
1433 114154 : StoreNoWriteBarrier(MachineRepresentation::kTaggedSigned, base,
1434 171231 : IntPtrConstant(offset), SmiTag(value));
1435 : }
1436 57077 : }
1437 :
1438 89020 : TNode<Float64T> CodeStubAssembler::LoadHeapNumberValue(
1439 : SloppyTNode<HeapNumber> object) {
1440 : return TNode<Float64T>::UncheckedCast(LoadObjectField(
1441 89020 : object, HeapNumber::kValueOffset, MachineType::Float64()));
1442 : }
1443 :
1444 217764 : TNode<Map> CodeStubAssembler::LoadMap(SloppyTNode<HeapObject> object) {
1445 : return UncheckedCast<Map>(LoadObjectField(object, HeapObject::kMapOffset,
1446 217764 : MachineType::TaggedPointer()));
1447 : }
1448 :
1449 66668 : TNode<Int32T> CodeStubAssembler::LoadInstanceType(
1450 : SloppyTNode<HeapObject> object) {
1451 66668 : return LoadMapInstanceType(LoadMap(object));
1452 : }
1453 :
1454 10528 : TNode<BoolT> CodeStubAssembler::HasInstanceType(SloppyTNode<HeapObject> object,
1455 : InstanceType instance_type) {
1456 10528 : return InstanceTypeEqual(LoadInstanceType(object), instance_type);
1457 : }
1458 :
1459 504 : TNode<BoolT> CodeStubAssembler::DoesntHaveInstanceType(
1460 : SloppyTNode<HeapObject> object, InstanceType instance_type) {
1461 504 : return Word32NotEqual(LoadInstanceType(object), Int32Constant(instance_type));
1462 : }
1463 :
1464 0 : TNode<BoolT> CodeStubAssembler::TaggedDoesntHaveInstanceType(
1465 : SloppyTNode<HeapObject> any_tagged, InstanceType type) {
1466 : /* return Phi <TaggedIsSmi(val), DoesntHaveInstanceType(val, type)> */
1467 0 : TNode<BoolT> tagged_is_smi = TaggedIsSmi(any_tagged);
1468 : return Select<BoolT>(
1469 0 : tagged_is_smi, [=]() { return tagged_is_smi; },
1470 0 : [=]() { return DoesntHaveInstanceType(any_tagged, type); });
1471 : }
1472 :
1473 3756 : TNode<HeapObject> CodeStubAssembler::LoadFastProperties(
1474 : SloppyTNode<JSObject> object) {
1475 : CSA_SLOW_ASSERT(this, Word32BinaryNot(IsDictionaryMap(LoadMap(object))));
1476 3756 : TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object);
1477 7512 : return Select<HeapObject>(TaggedIsSmi(properties),
1478 3756 : [=] { return EmptyFixedArrayConstant(); },
1479 15024 : [=] { return CAST(properties); });
1480 : }
1481 :
1482 6344 : TNode<HeapObject> CodeStubAssembler::LoadSlowProperties(
1483 : SloppyTNode<JSObject> object) {
1484 : CSA_SLOW_ASSERT(this, IsDictionaryMap(LoadMap(object)));
1485 6344 : TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object);
1486 12688 : return Select<HeapObject>(TaggedIsSmi(properties),
1487 6344 : [=] { return EmptyPropertyDictionaryConstant(); },
1488 25376 : [=] { return CAST(properties); });
1489 : }
1490 :
1491 13628 : TNode<Number> CodeStubAssembler::LoadJSArrayLength(SloppyTNode<JSArray> array) {
1492 : CSA_ASSERT(this, IsJSArray(array));
1493 13628 : return CAST(LoadObjectField(array, JSArray::kLengthOffset));
1494 : }
1495 :
1496 0 : TNode<Object> CodeStubAssembler::LoadJSArgumentsObjectWithLength(
1497 : SloppyTNode<JSArgumentsObjectWithLength> array) {
1498 0 : return LoadObjectField(array, JSArgumentsObjectWithLength::kLengthOffset);
1499 : }
1500 :
1501 4608 : TNode<Smi> CodeStubAssembler::LoadFastJSArrayLength(
1502 : SloppyTNode<JSArray> array) {
1503 4608 : TNode<Object> length = LoadJSArrayLength(array);
1504 : CSA_ASSERT(this, IsFastElementsKind(LoadElementsKind(array)));
1505 : // JSArray length is always a positive Smi for fast arrays.
1506 : CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
1507 4608 : return UncheckedCast<Smi>(length);
1508 : }
1509 :
1510 20444 : TNode<Smi> CodeStubAssembler::LoadFixedArrayBaseLength(
1511 : SloppyTNode<FixedArrayBase> array) {
1512 : CSA_SLOW_ASSERT(this, IsNotWeakFixedArraySubclass(array));
1513 20444 : return CAST(LoadObjectField(array, FixedArrayBase::kLengthOffset));
1514 : }
1515 :
1516 67748 : TNode<IntPtrT> CodeStubAssembler::LoadAndUntagFixedArrayBaseLength(
1517 : SloppyTNode<FixedArrayBase> array) {
1518 67748 : return LoadAndUntagObjectField(array, FixedArrayBase::kLengthOffset);
1519 : }
1520 :
1521 0 : TNode<IntPtrT> CodeStubAssembler::LoadFeedbackVectorLength(
1522 : TNode<FeedbackVector> vector) {
1523 : return ChangeInt32ToIntPtr(
1524 0 : LoadObjectField<Int32T>(vector, FeedbackVector::kLengthOffset));
1525 : }
1526 :
1527 0 : TNode<Smi> CodeStubAssembler::LoadWeakFixedArrayLength(
1528 : TNode<WeakFixedArray> array) {
1529 0 : return CAST(LoadObjectField(array, WeakFixedArray::kLengthOffset));
1530 : }
1531 :
1532 1296 : TNode<IntPtrT> CodeStubAssembler::LoadAndUntagWeakFixedArrayLength(
1533 : SloppyTNode<WeakFixedArray> array) {
1534 1296 : return LoadAndUntagObjectField(array, WeakFixedArray::kLengthOffset);
1535 : }
1536 :
1537 2024 : TNode<Int32T> CodeStubAssembler::LoadNumberOfDescriptors(
1538 : TNode<DescriptorArray> array) {
1539 : return UncheckedCast<Int32T>(
1540 : LoadObjectField(array, DescriptorArray::kNumberOfDescriptorsOffset,
1541 2024 : MachineType::Int16()));
1542 : }
1543 :
1544 26900 : TNode<Int32T> CodeStubAssembler::LoadMapBitField(SloppyTNode<Map> map) {
1545 : CSA_SLOW_ASSERT(this, IsMap(map));
1546 : return UncheckedCast<Int32T>(
1547 26900 : LoadObjectField(map, Map::kBitFieldOffset, MachineType::Uint8()));
1548 : }
1549 :
1550 9916 : TNode<Int32T> CodeStubAssembler::LoadMapBitField2(SloppyTNode<Map> map) {
1551 : CSA_SLOW_ASSERT(this, IsMap(map));
1552 : return UncheckedCast<Int32T>(
1553 9916 : LoadObjectField(map, Map::kBitField2Offset, MachineType::Uint8()));
1554 : }
1555 :
1556 6952 : TNode<Uint32T> CodeStubAssembler::LoadMapBitField3(SloppyTNode<Map> map) {
1557 : CSA_SLOW_ASSERT(this, IsMap(map));
1558 : return UncheckedCast<Uint32T>(
1559 6952 : LoadObjectField(map, Map::kBitField3Offset, MachineType::Uint32()));
1560 : }
1561 :
1562 122356 : TNode<Int32T> CodeStubAssembler::LoadMapInstanceType(SloppyTNode<Map> map) {
1563 : return UncheckedCast<Int32T>(
1564 122356 : LoadObjectField(map, Map::kInstanceTypeOffset, MachineType::Uint16()));
1565 : }
1566 :
1567 9020 : TNode<Int32T> CodeStubAssembler::LoadMapElementsKind(SloppyTNode<Map> map) {
1568 : CSA_SLOW_ASSERT(this, IsMap(map));
1569 9020 : Node* bit_field2 = LoadMapBitField2(map);
1570 9020 : return Signed(DecodeWord32<Map::ElementsKindBits>(bit_field2));
1571 : }
1572 :
1573 2576 : TNode<Int32T> CodeStubAssembler::LoadElementsKind(
1574 : SloppyTNode<HeapObject> object) {
1575 2576 : return LoadMapElementsKind(LoadMap(object));
1576 : }
1577 :
1578 9528 : TNode<DescriptorArray> CodeStubAssembler::LoadMapDescriptors(
1579 : SloppyTNode<Map> map) {
1580 : CSA_SLOW_ASSERT(this, IsMap(map));
1581 9528 : return CAST(LoadObjectField(map, Map::kDescriptorsOffset));
1582 : }
1583 :
1584 16076 : TNode<HeapObject> CodeStubAssembler::LoadMapPrototype(SloppyTNode<Map> map) {
1585 : CSA_SLOW_ASSERT(this, IsMap(map));
1586 16076 : return CAST(LoadObjectField(map, Map::kPrototypeOffset));
1587 : }
1588 :
1589 168 : TNode<PrototypeInfo> CodeStubAssembler::LoadMapPrototypeInfo(
1590 : SloppyTNode<Map> map, Label* if_no_proto_info) {
1591 336 : Label if_strong_heap_object(this);
1592 : CSA_ASSERT(this, IsMap(map));
1593 : TNode<MaybeObject> maybe_prototype_info =
1594 168 : LoadMaybeWeakObjectField(map, Map::kTransitionsOrPrototypeInfoOffset);
1595 336 : TVARIABLE(Object, prototype_info);
1596 : DispatchMaybeObject(maybe_prototype_info, if_no_proto_info, if_no_proto_info,
1597 : if_no_proto_info, &if_strong_heap_object,
1598 168 : &prototype_info);
1599 :
1600 168 : BIND(&if_strong_heap_object);
1601 504 : GotoIfNot(WordEqual(LoadMap(CAST(prototype_info.value())),
1602 336 : LoadRoot(RootIndex::kPrototypeInfoMap)),
1603 168 : if_no_proto_info);
1604 336 : return CAST(prototype_info.value());
1605 : }
1606 :
1607 4608 : TNode<IntPtrT> CodeStubAssembler::LoadMapInstanceSizeInWords(
1608 : SloppyTNode<Map> map) {
1609 : CSA_SLOW_ASSERT(this, IsMap(map));
1610 : return ChangeInt32ToIntPtr(LoadObjectField(
1611 4608 : map, Map::kInstanceSizeInWordsOffset, MachineType::Uint8()));
1612 : }
1613 :
1614 2132 : TNode<IntPtrT> CodeStubAssembler::LoadMapInobjectPropertiesStartInWords(
1615 : SloppyTNode<Map> map) {
1616 : CSA_SLOW_ASSERT(this, IsMap(map));
1617 : // See Map::GetInObjectPropertiesStartInWords() for details.
1618 : CSA_ASSERT(this, IsJSObjectMap(map));
1619 : return ChangeInt32ToIntPtr(LoadObjectField(
1620 : map, Map::kInObjectPropertiesStartOrConstructorFunctionIndexOffset,
1621 2132 : MachineType::Uint8()));
1622 : }
1623 :
1624 56 : TNode<IntPtrT> CodeStubAssembler::LoadMapConstructorFunctionIndex(
1625 : SloppyTNode<Map> map) {
1626 : CSA_SLOW_ASSERT(this, IsMap(map));
1627 : // See Map::GetConstructorFunctionIndex() for details.
1628 : CSA_ASSERT(this, IsPrimitiveInstanceType(LoadMapInstanceType(map)));
1629 : return ChangeInt32ToIntPtr(LoadObjectField(
1630 : map, Map::kInObjectPropertiesStartOrConstructorFunctionIndexOffset,
1631 56 : MachineType::Uint8()));
1632 : }
1633 :
1634 0 : TNode<Object> CodeStubAssembler::LoadMapConstructor(SloppyTNode<Map> map) {
1635 : CSA_SLOW_ASSERT(this, IsMap(map));
1636 0 : TVARIABLE(Object, result,
1637 : LoadObjectField(map, Map::kConstructorOrBackPointerOffset));
1638 :
1639 0 : Label done(this), loop(this, &result);
1640 0 : Goto(&loop);
1641 0 : BIND(&loop);
1642 : {
1643 0 : GotoIf(TaggedIsSmi(result.value()), &done);
1644 : Node* is_map_type =
1645 0 : InstanceTypeEqual(LoadInstanceType(CAST(result.value())), MAP_TYPE);
1646 0 : GotoIfNot(is_map_type, &done);
1647 0 : result = LoadObjectField(CAST(result.value()),
1648 0 : Map::kConstructorOrBackPointerOffset);
1649 0 : Goto(&loop);
1650 : }
1651 0 : BIND(&done);
1652 0 : return result.value();
1653 : }
1654 :
1655 840 : Node* CodeStubAssembler::LoadMapEnumLength(SloppyTNode<Map> map) {
1656 : CSA_SLOW_ASSERT(this, IsMap(map));
1657 840 : Node* bit_field3 = LoadMapBitField3(map);
1658 840 : return DecodeWordFromWord32<Map::EnumLengthBits>(bit_field3);
1659 : }
1660 :
1661 0 : TNode<Object> CodeStubAssembler::LoadMapBackPointer(SloppyTNode<Map> map) {
1662 : TNode<HeapObject> object =
1663 0 : CAST(LoadObjectField(map, Map::kConstructorOrBackPointerOffset));
1664 0 : return Select<Object>(IsMap(object), [=] { return object; },
1665 0 : [=] { return UndefinedConstant(); });
1666 : }
1667 :
1668 112 : TNode<Uint32T> CodeStubAssembler::EnsureOnlyHasSimpleProperties(
1669 : TNode<Map> map, TNode<Int32T> instance_type, Label* bailout) {
1670 : // This check can have false positives, since it applies to any JSValueType.
1671 112 : GotoIf(IsCustomElementsReceiverInstanceType(instance_type), bailout);
1672 :
1673 112 : TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
1674 224 : GotoIf(IsSetWord32(bit_field3, Map::IsDictionaryMapBit::kMask |
1675 224 : Map::HasHiddenPrototypeBit::kMask),
1676 112 : bailout);
1677 :
1678 112 : return bit_field3;
1679 : }
1680 :
1681 560 : TNode<IntPtrT> CodeStubAssembler::LoadJSReceiverIdentityHash(
1682 : SloppyTNode<Object> receiver, Label* if_no_hash) {
1683 1120 : TVARIABLE(IntPtrT, var_hash);
1684 1120 : Label done(this), if_smi(this), if_property_array(this),
1685 1120 : if_property_dictionary(this), if_fixed_array(this);
1686 :
1687 : TNode<Object> properties_or_hash =
1688 1120 : LoadObjectField(TNode<HeapObject>::UncheckedCast(receiver),
1689 560 : JSReceiver::kPropertiesOrHashOffset);
1690 560 : GotoIf(TaggedIsSmi(properties_or_hash), &if_smi);
1691 :
1692 : TNode<HeapObject> properties =
1693 560 : TNode<HeapObject>::UncheckedCast(properties_or_hash);
1694 560 : TNode<Int32T> properties_instance_type = LoadInstanceType(properties);
1695 :
1696 1120 : GotoIf(InstanceTypeEqual(properties_instance_type, PROPERTY_ARRAY_TYPE),
1697 560 : &if_property_array);
1698 1120 : Branch(InstanceTypeEqual(properties_instance_type, NAME_DICTIONARY_TYPE),
1699 560 : &if_property_dictionary, &if_fixed_array);
1700 :
1701 560 : BIND(&if_fixed_array);
1702 : {
1703 560 : var_hash = IntPtrConstant(PropertyArray::kNoHashSentinel);
1704 560 : Goto(&done);
1705 : }
1706 :
1707 560 : BIND(&if_smi);
1708 : {
1709 560 : var_hash = SmiUntag(TNode<Smi>::UncheckedCast(properties_or_hash));
1710 560 : Goto(&done);
1711 : }
1712 :
1713 560 : BIND(&if_property_array);
1714 : {
1715 : TNode<IntPtrT> length_and_hash = LoadAndUntagObjectField(
1716 560 : properties, PropertyArray::kLengthAndHashOffset);
1717 560 : var_hash = TNode<IntPtrT>::UncheckedCast(
1718 1120 : DecodeWord<PropertyArray::HashField>(length_and_hash));
1719 560 : Goto(&done);
1720 : }
1721 :
1722 560 : BIND(&if_property_dictionary);
1723 : {
1724 1120 : var_hash = SmiUntag(CAST(LoadFixedArrayElement(
1725 560 : CAST(properties), NameDictionary::kObjectHashIndex)));
1726 560 : Goto(&done);
1727 : }
1728 :
1729 560 : BIND(&done);
1730 560 : if (if_no_hash != nullptr) {
1731 896 : GotoIf(IntPtrEqual(var_hash.value(),
1732 896 : IntPtrConstant(PropertyArray::kNoHashSentinel)),
1733 224 : if_no_hash);
1734 : }
1735 1120 : return var_hash.value();
1736 : }
1737 :
1738 16768 : TNode<Uint32T> CodeStubAssembler::LoadNameHashField(SloppyTNode<Name> name) {
1739 : CSA_ASSERT(this, IsName(name));
1740 16768 : return LoadObjectField<Uint32T>(name, Name::kHashFieldOffset);
1741 : }
1742 :
1743 7080 : TNode<Uint32T> CodeStubAssembler::LoadNameHash(SloppyTNode<Name> name,
1744 : Label* if_hash_not_computed) {
1745 7080 : TNode<Uint32T> hash_field = LoadNameHashField(name);
1746 7080 : if (if_hash_not_computed != nullptr) {
1747 672 : GotoIf(IsSetWord32(hash_field, Name::kHashNotComputedMask),
1748 336 : if_hash_not_computed);
1749 : }
1750 7080 : return Unsigned(Word32Shr(hash_field, Int32Constant(Name::kHashShift)));
1751 : }
1752 :
1753 6108 : TNode<Smi> CodeStubAssembler::LoadStringLengthAsSmi(
1754 : SloppyTNode<String> string) {
1755 6108 : return SmiFromIntPtr(LoadStringLengthAsWord(string));
1756 : }
1757 :
1758 11716 : TNode<IntPtrT> CodeStubAssembler::LoadStringLengthAsWord(
1759 : SloppyTNode<String> string) {
1760 11716 : return Signed(ChangeUint32ToWord(LoadStringLengthAsWord32(string)));
1761 : }
1762 :
1763 12724 : TNode<Uint32T> CodeStubAssembler::LoadStringLengthAsWord32(
1764 : SloppyTNode<String> string) {
1765 : CSA_ASSERT(this, IsString(string));
1766 12724 : return LoadObjectField<Uint32T>(string, String::kLengthOffset);
1767 : }
1768 :
1769 56 : Node* CodeStubAssembler::PointerToSeqStringData(Node* seq_string) {
1770 : CSA_ASSERT(this, IsString(seq_string));
1771 : CSA_ASSERT(this,
1772 : IsSequentialStringInstanceType(LoadInstanceType(seq_string)));
1773 : STATIC_ASSERT(SeqOneByteString::kHeaderSize == SeqTwoByteString::kHeaderSize);
1774 112 : return IntPtrAdd(
1775 : BitcastTaggedToWord(seq_string),
1776 168 : IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag));
1777 : }
1778 :
1779 5556 : Node* CodeStubAssembler::LoadJSValueValue(Node* object) {
1780 : CSA_ASSERT(this, IsJSValue(object));
1781 5556 : return LoadObjectField(object, JSValue::kValueOffset);
1782 : }
1783 :
1784 448 : void CodeStubAssembler::DispatchMaybeObject(TNode<MaybeObject> maybe_object,
1785 : Label* if_smi, Label* if_cleared,
1786 : Label* if_weak, Label* if_strong,
1787 : TVariable<Object>* extracted) {
1788 896 : Label inner_if_smi(this), inner_if_strong(this);
1789 :
1790 448 : GotoIf(TaggedIsSmi(maybe_object), &inner_if_smi);
1791 :
1792 448 : GotoIf(IsCleared(maybe_object), if_cleared);
1793 :
1794 2688 : GotoIf(Word32Equal(Word32And(TruncateIntPtrToInt32(
1795 896 : BitcastMaybeObjectToWord(maybe_object)),
1796 1792 : Int32Constant(kHeapObjectTagMask)),
1797 1792 : Int32Constant(kHeapObjectTag)),
1798 448 : &inner_if_strong);
1799 :
1800 896 : *extracted =
1801 896 : BitcastWordToTagged(WordAnd(BitcastMaybeObjectToWord(maybe_object),
1802 1344 : IntPtrConstant(~kWeakHeapObjectMask)));
1803 448 : Goto(if_weak);
1804 :
1805 448 : BIND(&inner_if_smi);
1806 448 : *extracted = CAST(maybe_object);
1807 448 : Goto(if_smi);
1808 :
1809 448 : BIND(&inner_if_strong);
1810 448 : *extracted = CAST(maybe_object);
1811 448 : Goto(if_strong);
1812 448 : }
1813 :
1814 560 : TNode<BoolT> CodeStubAssembler::IsStrong(TNode<MaybeObject> value) {
1815 1120 : return WordEqual(WordAnd(BitcastMaybeObjectToWord(value),
1816 1120 : IntPtrConstant(kHeapObjectTagMask)),
1817 1680 : IntPtrConstant(kHeapObjectTag));
1818 : }
1819 :
1820 560 : TNode<HeapObject> CodeStubAssembler::GetHeapObjectIfStrong(
1821 : TNode<MaybeObject> value, Label* if_not_strong) {
1822 560 : GotoIfNot(IsStrong(value), if_not_strong);
1823 560 : return CAST(value);
1824 : }
1825 :
1826 504 : TNode<BoolT> CodeStubAssembler::IsWeakOrCleared(TNode<MaybeObject> value) {
1827 : return Word32Equal(
1828 2016 : Word32And(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)),
1829 2016 : Int32Constant(kHeapObjectTagMask)),
1830 1512 : Int32Constant(kWeakHeapObjectTag));
1831 : }
1832 :
1833 7280 : TNode<BoolT> CodeStubAssembler::IsCleared(TNode<MaybeObject> value) {
1834 14560 : return Word32Equal(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)),
1835 21840 : Int32Constant(kClearedWeakHeapObjectLower32));
1836 : }
1837 :
1838 1288 : TNode<BoolT> CodeStubAssembler::IsNotCleared(TNode<MaybeObject> value) {
1839 2576 : return Word32NotEqual(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)),
1840 3864 : Int32Constant(kClearedWeakHeapObjectLower32));
1841 : }
1842 :
1843 6216 : TNode<HeapObject> CodeStubAssembler::GetHeapObjectAssumeWeak(
1844 : TNode<MaybeObject> value) {
1845 : CSA_ASSERT(this, IsWeakOrCleared(value));
1846 : CSA_ASSERT(this, IsNotCleared(value));
1847 12432 : return UncheckedCast<HeapObject>(BitcastWordToTagged(WordAnd(
1848 18648 : BitcastMaybeObjectToWord(value), IntPtrConstant(~kWeakHeapObjectMask))));
1849 : }
1850 :
1851 4480 : TNode<HeapObject> CodeStubAssembler::GetHeapObjectAssumeWeak(
1852 : TNode<MaybeObject> value, Label* if_cleared) {
1853 4480 : GotoIf(IsCleared(value), if_cleared);
1854 4480 : return GetHeapObjectAssumeWeak(value);
1855 : }
1856 :
1857 2184 : TNode<BoolT> CodeStubAssembler::IsWeakReferenceTo(TNode<MaybeObject> object,
1858 : TNode<Object> value) {
1859 4368 : return WordEqual(WordAnd(BitcastMaybeObjectToWord(object),
1860 4368 : IntPtrConstant(~kWeakHeapObjectMask)),
1861 6552 : BitcastTaggedToWord(value));
1862 : }
1863 :
1864 1512 : TNode<BoolT> CodeStubAssembler::IsStrongReferenceTo(TNode<MaybeObject> object,
1865 : TNode<Object> value) {
1866 3024 : return WordEqual(BitcastMaybeObjectToWord(object),
1867 4536 : BitcastTaggedToWord(value));
1868 : }
1869 :
1870 1288 : TNode<BoolT> CodeStubAssembler::IsNotWeakReferenceTo(TNode<MaybeObject> object,
1871 : TNode<Object> value) {
1872 2576 : return WordNotEqual(WordAnd(BitcastMaybeObjectToWord(object),
1873 2576 : IntPtrConstant(~kWeakHeapObjectMask)),
1874 3864 : BitcastTaggedToWord(value));
1875 : }
1876 :
1877 2240 : TNode<MaybeObject> CodeStubAssembler::MakeWeak(TNode<HeapObject> value) {
1878 4480 : return ReinterpretCast<MaybeObject>(BitcastWordToTagged(
1879 6720 : WordOr(BitcastTaggedToWord(value), IntPtrConstant(kWeakHeapObjectTag))));
1880 : }
1881 :
1882 : template <>
1883 0 : TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(TNode<FixedArray> array) {
1884 0 : return LoadAndUntagFixedArrayBaseLength(array);
1885 : }
1886 :
1887 : template <>
1888 0 : TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(TNode<WeakFixedArray> array) {
1889 0 : return LoadAndUntagWeakFixedArrayLength(array);
1890 : }
1891 :
1892 : template <>
1893 0 : TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(TNode<PropertyArray> array) {
1894 0 : return LoadPropertyArrayLength(array);
1895 : }
1896 :
1897 : template <>
1898 0 : TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(
1899 : TNode<DescriptorArray> array) {
1900 0 : return IntPtrMul(ChangeInt32ToIntPtr(LoadNumberOfDescriptors(array)),
1901 0 : IntPtrConstant(DescriptorArray::kEntrySize));
1902 : }
1903 :
1904 : template <>
1905 0 : TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(
1906 : TNode<TransitionArray> array) {
1907 0 : return LoadAndUntagWeakFixedArrayLength(array);
1908 : }
1909 :
1910 : template <typename Array>
1911 117288 : TNode<MaybeObject> CodeStubAssembler::LoadArrayElement(
1912 : TNode<Array> array, int array_header_size, Node* index_node,
1913 : int additional_offset, ParameterMode parameter_mode,
1914 : LoadSensitivity needs_poisoning) {
1915 : CSA_ASSERT(this, IntPtrGreaterThanOrEqual(
1916 : ParameterToIntPtr(index_node, parameter_mode),
1917 : IntPtrConstant(0)));
1918 : DCHECK(IsAligned(additional_offset, kTaggedSize));
1919 117288 : int32_t header_size = array_header_size + additional_offset - kHeapObjectTag;
1920 : TNode<IntPtrT> offset = ElementOffsetFromIndex(index_node, HOLEY_ELEMENTS,
1921 117288 : parameter_mode, header_size);
1922 : CSA_ASSERT(this, IsOffsetInBounds(offset, LoadArrayLength(array),
1923 : array_header_size));
1924 : return UncheckedCast<MaybeObject>(
1925 117288 : Load(MachineType::AnyTagged(), array, offset, needs_poisoning));
1926 : }
1927 :
1928 : template TNode<MaybeObject>
1929 : CodeStubAssembler::LoadArrayElement<TransitionArray>(TNode<TransitionArray>,
1930 : int, Node*, int,
1931 : ParameterMode,
1932 : LoadSensitivity);
1933 :
1934 : template TNode<MaybeObject>
1935 : CodeStubAssembler::LoadArrayElement<DescriptorArray>(TNode<DescriptorArray>,
1936 : int, Node*, int,
1937 : ParameterMode,
1938 : LoadSensitivity);
1939 :
1940 72616 : void CodeStubAssembler::FixedArrayBoundsCheck(TNode<FixedArrayBase> array,
1941 : Node* index,
1942 : int additional_offset,
1943 : ParameterMode parameter_mode) {
1944 : if (!FLAG_fixed_array_bounds_checks) return;
1945 : DCHECK(IsAligned(additional_offset, kTaggedSize));
1946 72616 : if (parameter_mode == ParameterMode::SMI_PARAMETERS) {
1947 8684 : TNode<Smi> effective_index;
1948 8684 : Smi constant_index;
1949 8684 : bool index_is_constant = ToSmiConstant(index, &constant_index);
1950 8684 : if (index_is_constant) {
1951 8 : effective_index = SmiConstant(Smi::ToInt(constant_index) +
1952 8 : additional_offset / kTaggedSize);
1953 8680 : } else if (additional_offset != 0) {
1954 0 : effective_index =
1955 0 : SmiAdd(CAST(index), SmiConstant(additional_offset / kTaggedSize));
1956 : } else {
1957 8680 : effective_index = CAST(index);
1958 : }
1959 8684 : CSA_CHECK(this, SmiBelow(effective_index, LoadFixedArrayBaseLength(array)));
1960 : } else {
1961 : // IntPtrAdd does constant-folding automatically.
1962 : TNode<IntPtrT> effective_index =
1963 : IntPtrAdd(UncheckedCast<IntPtrT>(index),
1964 63932 : IntPtrConstant(additional_offset / kTaggedSize));
1965 127864 : CSA_CHECK(this, UintPtrLessThan(effective_index,
1966 63932 : LoadAndUntagFixedArrayBaseLength(array)));
1967 : }
1968 : }
1969 :
1970 98764 : TNode<Object> CodeStubAssembler::LoadFixedArrayElement(
1971 : TNode<FixedArray> object, Node* index_node, int additional_offset,
1972 : ParameterMode parameter_mode, LoadSensitivity needs_poisoning,
1973 : CheckBounds check_bounds) {
1974 : CSA_ASSERT(this, IsFixedArraySubclass(object));
1975 : CSA_ASSERT(this, IsNotWeakFixedArraySubclass(object));
1976 98764 : if (NeedsBoundsCheck(check_bounds)) {
1977 74640 : FixedArrayBoundsCheck(object, index_node, additional_offset,
1978 37320 : parameter_mode);
1979 : }
1980 : TNode<MaybeObject> element =
1981 : LoadArrayElement(object, FixedArray::kHeaderSize, index_node,
1982 98764 : additional_offset, parameter_mode, needs_poisoning);
1983 98764 : return CAST(element);
1984 : }
1985 :
1986 1852 : TNode<Object> CodeStubAssembler::LoadPropertyArrayElement(
1987 : TNode<PropertyArray> object, SloppyTNode<IntPtrT> index) {
1988 1852 : int additional_offset = 0;
1989 1852 : ParameterMode parameter_mode = INTPTR_PARAMETERS;
1990 1852 : LoadSensitivity needs_poisoning = LoadSensitivity::kSafe;
1991 1852 : return CAST(LoadArrayElement(object, PropertyArray::kHeaderSize, index,
1992 : additional_offset, parameter_mode,
1993 : needs_poisoning));
1994 : }
1995 :
1996 56 : TNode<IntPtrT> CodeStubAssembler::LoadPropertyArrayLength(
1997 : TNode<PropertyArray> object) {
1998 : TNode<IntPtrT> value =
1999 56 : LoadAndUntagObjectField(object, PropertyArray::kLengthAndHashOffset);
2000 56 : return Signed(DecodeWord<PropertyArray::LengthField>(value));
2001 : }
2002 :
2003 6608 : TNode<RawPtrT> CodeStubAssembler::LoadFixedTypedArrayBackingStore(
2004 : TNode<FixedTypedArrayBase> typed_array) {
2005 : // Backing store = external_pointer + base_pointer.
2006 : Node* external_pointer =
2007 13216 : LoadObjectField(typed_array, FixedTypedArrayBase::kExternalPointerOffset,
2008 6608 : MachineType::Pointer());
2009 : Node* base_pointer =
2010 6608 : LoadObjectField(typed_array, FixedTypedArrayBase::kBasePointerOffset);
2011 : return UncheckedCast<RawPtrT>(
2012 6608 : IntPtrAdd(external_pointer, BitcastTaggedToWord(base_pointer)));
2013 : }
2014 :
2015 56 : TNode<RawPtrT> CodeStubAssembler::LoadFixedTypedArrayOnHeapBackingStore(
2016 : TNode<FixedTypedArrayBase> typed_array) {
2017 : // This is specialized method of retrieving the backing store pointer for on
2018 : // heap allocated typed array buffer. On heap allocated buffer's backing
2019 : // stores are a fixed offset from the pointer to a typed array's elements. See
2020 : // TypedArrayBuiltinsAssembler::AllocateOnHeapElements().
2021 : TNode<WordT> backing_store =
2022 112 : IntPtrAdd(BitcastTaggedToWord(typed_array),
2023 : IntPtrConstant(
2024 168 : FixedTypedArrayBase::ExternalPointerValueForOnHeapArray()));
2025 :
2026 : #ifdef DEBUG
2027 : // Verify that this is an on heap backing store.
2028 : TNode<RawPtrT> expected_backing_store_pointer =
2029 : LoadFixedTypedArrayBackingStore(typed_array);
2030 : CSA_ASSERT(this, WordEqual(backing_store, expected_backing_store_pointer));
2031 : #endif
2032 :
2033 56 : return UncheckedCast<RawPtrT>(backing_store);
2034 : }
2035 :
2036 336 : Node* CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged(
2037 : Node* data_pointer, Node* offset) {
2038 336 : if (Is64()) {
2039 : TNode<IntPtrT> value = UncheckedCast<IntPtrT>(
2040 336 : Load(MachineType::IntPtr(), data_pointer, offset));
2041 336 : return BigIntFromInt64(value);
2042 : } else {
2043 : DCHECK(!Is64());
2044 : #if defined(V8_TARGET_BIG_ENDIAN)
2045 : TNode<IntPtrT> high = UncheckedCast<IntPtrT>(
2046 : Load(MachineType::UintPtr(), data_pointer, offset));
2047 : TNode<IntPtrT> low = UncheckedCast<IntPtrT>(
2048 : Load(MachineType::UintPtr(), data_pointer,
2049 : Int32Add(offset, Int32Constant(kSystemPointerSize))));
2050 : #else
2051 : TNode<IntPtrT> low = UncheckedCast<IntPtrT>(
2052 0 : Load(MachineType::UintPtr(), data_pointer, offset));
2053 : TNode<IntPtrT> high = UncheckedCast<IntPtrT>(
2054 : Load(MachineType::UintPtr(), data_pointer,
2055 0 : Int32Add(offset, Int32Constant(kSystemPointerSize))));
2056 : #endif
2057 0 : return BigIntFromInt32Pair(low, high);
2058 : }
2059 : }
2060 :
2061 0 : TNode<BigInt> CodeStubAssembler::BigIntFromInt32Pair(TNode<IntPtrT> low,
2062 : TNode<IntPtrT> high) {
2063 : DCHECK(!Is64());
2064 0 : TVARIABLE(BigInt, var_result);
2065 0 : TVARIABLE(Word32T, var_sign, Int32Constant(BigInt::SignBits::encode(false)));
2066 0 : TVARIABLE(IntPtrT, var_high, high);
2067 0 : TVARIABLE(IntPtrT, var_low, low);
2068 0 : Label high_zero(this), negative(this), allocate_one_digit(this),
2069 0 : allocate_two_digits(this), if_zero(this), done(this);
2070 :
2071 0 : GotoIf(WordEqual(var_high.value(), IntPtrConstant(0)), &high_zero);
2072 0 : Branch(IntPtrLessThan(var_high.value(), IntPtrConstant(0)), &negative,
2073 0 : &allocate_two_digits);
2074 :
2075 0 : BIND(&high_zero);
2076 0 : Branch(WordEqual(var_low.value(), IntPtrConstant(0)), &if_zero,
2077 0 : &allocate_one_digit);
2078 :
2079 0 : BIND(&negative);
2080 : {
2081 0 : var_sign = Int32Constant(BigInt::SignBits::encode(true));
2082 : // We must negate the value by computing "0 - (high|low)", performing
2083 : // both parts of the subtraction separately and manually taking care
2084 : // of the carry bit (which is 1 iff low != 0).
2085 0 : var_high = IntPtrSub(IntPtrConstant(0), var_high.value());
2086 0 : Label carry(this), no_carry(this);
2087 0 : Branch(WordEqual(var_low.value(), IntPtrConstant(0)), &no_carry, &carry);
2088 0 : BIND(&carry);
2089 0 : var_high = IntPtrSub(var_high.value(), IntPtrConstant(1));
2090 0 : Goto(&no_carry);
2091 0 : BIND(&no_carry);
2092 0 : var_low = IntPtrSub(IntPtrConstant(0), var_low.value());
2093 : // var_high was non-zero going into this block, but subtracting the
2094 : // carry bit from it could bring us back onto the "one digit" path.
2095 0 : Branch(WordEqual(var_high.value(), IntPtrConstant(0)), &allocate_one_digit,
2096 0 : &allocate_two_digits);
2097 : }
2098 :
2099 0 : BIND(&allocate_one_digit);
2100 : {
2101 0 : var_result = AllocateRawBigInt(IntPtrConstant(1));
2102 0 : StoreBigIntBitfield(var_result.value(),
2103 0 : Word32Or(var_sign.value(),
2104 0 : Int32Constant(BigInt::LengthBits::encode(1))));
2105 0 : StoreBigIntDigit(var_result.value(), 0, Unsigned(var_low.value()));
2106 0 : Goto(&done);
2107 : }
2108 :
2109 0 : BIND(&allocate_two_digits);
2110 : {
2111 0 : var_result = AllocateRawBigInt(IntPtrConstant(2));
2112 0 : StoreBigIntBitfield(var_result.value(),
2113 0 : Word32Or(var_sign.value(),
2114 0 : Int32Constant(BigInt::LengthBits::encode(2))));
2115 0 : StoreBigIntDigit(var_result.value(), 0, Unsigned(var_low.value()));
2116 0 : StoreBigIntDigit(var_result.value(), 1, Unsigned(var_high.value()));
2117 0 : Goto(&done);
2118 : }
2119 :
2120 0 : BIND(&if_zero);
2121 0 : var_result = AllocateBigInt(IntPtrConstant(0));
2122 0 : Goto(&done);
2123 :
2124 0 : BIND(&done);
2125 0 : return var_result.value();
2126 : }
2127 :
2128 840 : TNode<BigInt> CodeStubAssembler::BigIntFromInt64(TNode<IntPtrT> value) {
2129 : DCHECK(Is64());
2130 1680 : TVARIABLE(BigInt, var_result);
2131 1680 : Label done(this), if_positive(this), if_negative(this), if_zero(this);
2132 840 : GotoIf(WordEqual(value, IntPtrConstant(0)), &if_zero);
2133 840 : var_result = AllocateRawBigInt(IntPtrConstant(1));
2134 1680 : Branch(IntPtrGreaterThan(value, IntPtrConstant(0)), &if_positive,
2135 840 : &if_negative);
2136 :
2137 840 : BIND(&if_positive);
2138 : {
2139 1680 : StoreBigIntBitfield(var_result.value(),
2140 3360 : Int32Constant(BigInt::SignBits::encode(false) |
2141 2520 : BigInt::LengthBits::encode(1)));
2142 840 : StoreBigIntDigit(var_result.value(), 0, Unsigned(value));
2143 840 : Goto(&done);
2144 : }
2145 :
2146 840 : BIND(&if_negative);
2147 : {
2148 1680 : StoreBigIntBitfield(var_result.value(),
2149 3360 : Int32Constant(BigInt::SignBits::encode(true) |
2150 2520 : BigInt::LengthBits::encode(1)));
2151 1680 : StoreBigIntDigit(var_result.value(), 0,
2152 2520 : Unsigned(IntPtrSub(IntPtrConstant(0), value)));
2153 840 : Goto(&done);
2154 : }
2155 :
2156 840 : BIND(&if_zero);
2157 : {
2158 840 : var_result = AllocateBigInt(IntPtrConstant(0));
2159 840 : Goto(&done);
2160 : }
2161 :
2162 840 : BIND(&done);
2163 1680 : return var_result.value();
2164 : }
2165 :
2166 336 : Node* CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged(
2167 : Node* data_pointer, Node* offset) {
2168 672 : Label if_zero(this), done(this);
2169 336 : if (Is64()) {
2170 : TNode<UintPtrT> value = UncheckedCast<UintPtrT>(
2171 336 : Load(MachineType::UintPtr(), data_pointer, offset));
2172 336 : return BigIntFromUint64(value);
2173 : } else {
2174 : DCHECK(!Is64());
2175 : #if defined(V8_TARGET_BIG_ENDIAN)
2176 : TNode<UintPtrT> high = UncheckedCast<UintPtrT>(
2177 : Load(MachineType::UintPtr(), data_pointer, offset));
2178 : TNode<UintPtrT> low = UncheckedCast<UintPtrT>(
2179 : Load(MachineType::UintPtr(), data_pointer,
2180 : Int32Add(offset, Int32Constant(kSystemPointerSize))));
2181 : #else
2182 : TNode<UintPtrT> low = UncheckedCast<UintPtrT>(
2183 0 : Load(MachineType::UintPtr(), data_pointer, offset));
2184 : TNode<UintPtrT> high = UncheckedCast<UintPtrT>(
2185 : Load(MachineType::UintPtr(), data_pointer,
2186 0 : Int32Add(offset, Int32Constant(kSystemPointerSize))));
2187 : #endif
2188 0 : return BigIntFromUint32Pair(low, high);
2189 : }
2190 : }
2191 :
2192 0 : TNode<BigInt> CodeStubAssembler::BigIntFromUint32Pair(TNode<UintPtrT> low,
2193 : TNode<UintPtrT> high) {
2194 : DCHECK(!Is64());
2195 0 : TVARIABLE(BigInt, var_result);
2196 0 : Label high_zero(this), if_zero(this), done(this);
2197 :
2198 0 : GotoIf(WordEqual(high, IntPtrConstant(0)), &high_zero);
2199 0 : var_result = AllocateBigInt(IntPtrConstant(2));
2200 0 : StoreBigIntDigit(var_result.value(), 0, low);
2201 0 : StoreBigIntDigit(var_result.value(), 1, high);
2202 0 : Goto(&done);
2203 :
2204 0 : BIND(&high_zero);
2205 0 : GotoIf(WordEqual(low, IntPtrConstant(0)), &if_zero);
2206 0 : var_result = AllocateBigInt(IntPtrConstant(1));
2207 0 : StoreBigIntDigit(var_result.value(), 0, low);
2208 0 : Goto(&done);
2209 :
2210 0 : BIND(&if_zero);
2211 0 : var_result = AllocateBigInt(IntPtrConstant(0));
2212 0 : Goto(&done);
2213 :
2214 0 : BIND(&done);
2215 0 : return var_result.value();
2216 : }
2217 :
2218 784 : TNode<BigInt> CodeStubAssembler::BigIntFromUint64(TNode<UintPtrT> value) {
2219 : DCHECK(Is64());
2220 1568 : TVARIABLE(BigInt, var_result);
2221 1568 : Label done(this), if_zero(this);
2222 784 : GotoIf(WordEqual(value, IntPtrConstant(0)), &if_zero);
2223 784 : var_result = AllocateBigInt(IntPtrConstant(1));
2224 784 : StoreBigIntDigit(var_result.value(), 0, value);
2225 784 : Goto(&done);
2226 :
2227 784 : BIND(&if_zero);
2228 784 : var_result = AllocateBigInt(IntPtrConstant(0));
2229 784 : Goto(&done);
2230 784 : BIND(&done);
2231 1568 : return var_result.value();
2232 : }
2233 :
2234 2688 : Node* CodeStubAssembler::LoadFixedTypedArrayElementAsTagged(
2235 : Node* data_pointer, Node* index_node, ElementsKind elements_kind,
2236 : ParameterMode parameter_mode) {
2237 : Node* offset =
2238 2688 : ElementOffsetFromIndex(index_node, elements_kind, parameter_mode, 0);
2239 2688 : switch (elements_kind) {
2240 : case UINT8_ELEMENTS: /* fall through */
2241 : case UINT8_CLAMPED_ELEMENTS:
2242 448 : return SmiFromInt32(Load(MachineType::Uint8(), data_pointer, offset));
2243 : case INT8_ELEMENTS:
2244 224 : return SmiFromInt32(Load(MachineType::Int8(), data_pointer, offset));
2245 : case UINT16_ELEMENTS:
2246 224 : return SmiFromInt32(Load(MachineType::Uint16(), data_pointer, offset));
2247 : case INT16_ELEMENTS:
2248 224 : return SmiFromInt32(Load(MachineType::Int16(), data_pointer, offset));
2249 : case UINT32_ELEMENTS:
2250 448 : return ChangeUint32ToTagged(
2251 672 : Load(MachineType::Uint32(), data_pointer, offset));
2252 : case INT32_ELEMENTS:
2253 448 : return ChangeInt32ToTagged(
2254 672 : Load(MachineType::Int32(), data_pointer, offset));
2255 : case FLOAT32_ELEMENTS:
2256 896 : return AllocateHeapNumberWithValue(ChangeFloat32ToFloat64(
2257 1120 : Load(MachineType::Float32(), data_pointer, offset)));
2258 : case FLOAT64_ELEMENTS:
2259 448 : return AllocateHeapNumberWithValue(
2260 672 : Load(MachineType::Float64(), data_pointer, offset));
2261 : case BIGINT64_ELEMENTS:
2262 336 : return LoadFixedBigInt64ArrayElementAsTagged(data_pointer, offset);
2263 : case BIGUINT64_ELEMENTS:
2264 336 : return LoadFixedBigUint64ArrayElementAsTagged(data_pointer, offset);
2265 : default:
2266 0 : UNREACHABLE();
2267 : }
2268 : }
2269 :
2270 56 : TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged(
2271 : TNode<WordT> data_pointer, TNode<Smi> index, TNode<Int32T> elements_kind) {
2272 112 : TVARIABLE(Numeric, var_result);
2273 112 : Label done(this), if_unknown_type(this, Label::kDeferred);
2274 : int32_t elements_kinds[] = {
2275 : #define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) TYPE##_ELEMENTS,
2276 : TYPED_ARRAYS(TYPED_ARRAY_CASE)
2277 : #undef TYPED_ARRAY_CASE
2278 56 : };
2279 :
2280 : #define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) Label if_##type##array(this);
2281 112 : TYPED_ARRAYS(TYPED_ARRAY_CASE)
2282 : #undef TYPED_ARRAY_CASE
2283 :
2284 : Label* elements_kind_labels[] = {
2285 : #define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) &if_##type##array,
2286 : TYPED_ARRAYS(TYPED_ARRAY_CASE)
2287 : #undef TYPED_ARRAY_CASE
2288 56 : };
2289 : STATIC_ASSERT(arraysize(elements_kinds) == arraysize(elements_kind_labels));
2290 :
2291 56 : Switch(elements_kind, &if_unknown_type, elements_kinds, elements_kind_labels,
2292 56 : arraysize(elements_kinds));
2293 :
2294 56 : BIND(&if_unknown_type);
2295 56 : Unreachable();
2296 :
2297 : #define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) \
2298 : BIND(&if_##type##array); \
2299 : { \
2300 : var_result = CAST(LoadFixedTypedArrayElementAsTagged( \
2301 : data_pointer, index, TYPE##_ELEMENTS, SMI_PARAMETERS)); \
2302 : Goto(&done); \
2303 : }
2304 56 : TYPED_ARRAYS(TYPED_ARRAY_CASE)
2305 : #undef TYPED_ARRAY_CASE
2306 :
2307 56 : BIND(&done);
2308 112 : return var_result.value();
2309 : }
2310 :
2311 616 : void CodeStubAssembler::StoreFixedTypedArrayElementFromTagged(
2312 : TNode<Context> context, TNode<FixedTypedArrayBase> elements,
2313 : TNode<Object> index_node, TNode<Object> value, ElementsKind elements_kind,
2314 : ParameterMode parameter_mode) {
2315 616 : TNode<RawPtrT> data_pointer = LoadFixedTypedArrayBackingStore(elements);
2316 616 : switch (elements_kind) {
2317 : case UINT8_ELEMENTS:
2318 : case UINT8_CLAMPED_ELEMENTS:
2319 : case INT8_ELEMENTS:
2320 : case UINT16_ELEMENTS:
2321 : case INT16_ELEMENTS:
2322 280 : StoreElement(data_pointer, elements_kind, index_node,
2323 560 : SmiToInt32(CAST(value)), parameter_mode);
2324 280 : break;
2325 : case UINT32_ELEMENTS:
2326 : case INT32_ELEMENTS:
2327 112 : StoreElement(data_pointer, elements_kind, index_node,
2328 112 : TruncateTaggedToWord32(context, value), parameter_mode);
2329 112 : break;
2330 : case FLOAT32_ELEMENTS:
2331 56 : StoreElement(data_pointer, elements_kind, index_node,
2332 112 : TruncateFloat64ToFloat32(LoadHeapNumberValue(CAST(value))),
2333 56 : parameter_mode);
2334 56 : break;
2335 : case FLOAT64_ELEMENTS:
2336 56 : StoreElement(data_pointer, elements_kind, index_node,
2337 112 : LoadHeapNumberValue(CAST(value)), parameter_mode);
2338 56 : break;
2339 : case BIGUINT64_ELEMENTS:
2340 : case BIGINT64_ELEMENTS: {
2341 : TNode<IntPtrT> offset =
2342 112 : ElementOffsetFromIndex(index_node, elements_kind, parameter_mode, 0);
2343 112 : EmitBigTypedArrayElementStore(elements, data_pointer, offset,
2344 224 : CAST(value));
2345 112 : break;
2346 : }
2347 : default:
2348 0 : UNREACHABLE();
2349 : }
2350 616 : }
2351 :
2352 15904 : TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot(
2353 : Node* object, Node* slot_index_node, int additional_offset,
2354 : ParameterMode parameter_mode) {
2355 : CSA_SLOW_ASSERT(this, IsFeedbackVector(object));
2356 : CSA_SLOW_ASSERT(this, MatchesParameterMode(slot_index_node, parameter_mode));
2357 : int32_t header_size =
2358 15904 : FeedbackVector::kFeedbackSlotsOffset + additional_offset - kHeapObjectTag;
2359 31808 : Node* offset = ElementOffsetFromIndex(slot_index_node, HOLEY_ELEMENTS,
2360 15904 : parameter_mode, header_size);
2361 : CSA_SLOW_ASSERT(
2362 : this, IsOffsetInBounds(offset, LoadFeedbackVectorLength(CAST(object)),
2363 : FeedbackVector::kHeaderSize));
2364 : return UncheckedCast<MaybeObject>(
2365 15904 : Load(MachineType::AnyTagged(), object, offset));
2366 : }
2367 :
2368 : template <typename Array>
2369 15032 : TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32ArrayElement(
2370 : TNode<Array> object, int array_header_size, Node* index_node,
2371 : int additional_offset, ParameterMode parameter_mode) {
2372 : CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, parameter_mode));
2373 : DCHECK(IsAligned(additional_offset, kTaggedSize));
2374 15032 : int endian_correction = 0;
2375 : #if V8_TARGET_LITTLE_ENDIAN
2376 15032 : if (SmiValuesAre32Bits()) endian_correction = 4;
2377 : #endif
2378 : int32_t header_size = array_header_size + additional_offset - kHeapObjectTag +
2379 15032 : endian_correction;
2380 : Node* offset = ElementOffsetFromIndex(index_node, HOLEY_ELEMENTS,
2381 15032 : parameter_mode, header_size);
2382 : CSA_ASSERT(this, IsOffsetInBounds(offset, LoadArrayLength(object),
2383 : array_header_size + endian_correction));
2384 15032 : if (SmiValuesAre32Bits()) {
2385 0 : return UncheckedCast<Int32T>(Load(MachineType::Int32(), object, offset));
2386 : } else {
2387 15032 : return SmiToInt32(Load(MachineType::AnyTagged(), object, offset));
2388 : }
2389 : }
2390 :
2391 6044 : TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32FixedArrayElement(
2392 : TNode<FixedArray> object, Node* index_node, int additional_offset,
2393 : ParameterMode parameter_mode) {
2394 : CSA_SLOW_ASSERT(this, IsFixedArraySubclass(object));
2395 : return LoadAndUntagToWord32ArrayElement(object, FixedArray::kHeaderSize,
2396 : index_node, additional_offset,
2397 6044 : parameter_mode);
2398 : }
2399 :
2400 1456 : TNode<MaybeObject> CodeStubAssembler::LoadWeakFixedArrayElement(
2401 : TNode<WeakFixedArray> object, Node* index, int additional_offset,
2402 : ParameterMode parameter_mode, LoadSensitivity needs_poisoning) {
2403 : return LoadArrayElement(object, WeakFixedArray::kHeaderSize, index,
2404 1456 : additional_offset, parameter_mode, needs_poisoning);
2405 : }
2406 :
2407 3364 : TNode<Float64T> CodeStubAssembler::LoadFixedDoubleArrayElement(
2408 : SloppyTNode<FixedDoubleArray> object, Node* index_node,
2409 : MachineType machine_type, int additional_offset,
2410 : ParameterMode parameter_mode, Label* if_hole) {
2411 : CSA_ASSERT(this, IsFixedDoubleArray(object));
2412 : DCHECK(IsAligned(additional_offset, kTaggedSize));
2413 : CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, parameter_mode));
2414 : int32_t header_size =
2415 3364 : FixedDoubleArray::kHeaderSize + additional_offset - kHeapObjectTag;
2416 : TNode<IntPtrT> offset = ElementOffsetFromIndex(
2417 3364 : index_node, HOLEY_DOUBLE_ELEMENTS, parameter_mode, header_size);
2418 : CSA_ASSERT(this, IsOffsetInBounds(
2419 : offset, LoadAndUntagFixedArrayBaseLength(object),
2420 : FixedDoubleArray::kHeaderSize, HOLEY_DOUBLE_ELEMENTS));
2421 3364 : return LoadDoubleWithHoleCheck(object, offset, if_hole, machine_type);
2422 : }
2423 :
2424 56 : TNode<Object> CodeStubAssembler::LoadFixedArrayBaseElementAsTagged(
2425 : TNode<FixedArrayBase> elements, TNode<IntPtrT> index,
2426 : TNode<Int32T> elements_kind, Label* if_accessor, Label* if_hole) {
2427 112 : TVARIABLE(Object, var_result);
2428 112 : Label done(this), if_packed(this), if_holey(this), if_packed_double(this),
2429 112 : if_holey_double(this), if_dictionary(this, Label::kDeferred);
2430 :
2431 : int32_t kinds[] = {// Handled by if_packed.
2432 : PACKED_SMI_ELEMENTS, PACKED_ELEMENTS,
2433 : // Handled by if_holey.
2434 : HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS,
2435 : // Handled by if_packed_double.
2436 : PACKED_DOUBLE_ELEMENTS,
2437 : // Handled by if_holey_double.
2438 56 : HOLEY_DOUBLE_ELEMENTS};
2439 : Label* labels[] = {// PACKED_{SMI,}_ELEMENTS
2440 : &if_packed, &if_packed,
2441 : // HOLEY_{SMI,}_ELEMENTS
2442 : &if_holey, &if_holey,
2443 : // PACKED_DOUBLE_ELEMENTS
2444 : &if_packed_double,
2445 : // HOLEY_DOUBLE_ELEMENTS
2446 56 : &if_holey_double};
2447 56 : Switch(elements_kind, &if_dictionary, kinds, labels, arraysize(kinds));
2448 :
2449 56 : BIND(&if_packed);
2450 : {
2451 56 : var_result = LoadFixedArrayElement(CAST(elements), index, 0);
2452 56 : Goto(&done);
2453 : }
2454 :
2455 56 : BIND(&if_holey);
2456 : {
2457 56 : var_result = LoadFixedArrayElement(CAST(elements), index);
2458 56 : Branch(WordEqual(var_result.value(), TheHoleConstant()), if_hole, &done);
2459 : }
2460 :
2461 56 : BIND(&if_packed_double);
2462 : {
2463 224 : var_result = AllocateHeapNumberWithValue(LoadFixedDoubleArrayElement(
2464 280 : CAST(elements), index, MachineType::Float64()));
2465 56 : Goto(&done);
2466 : }
2467 :
2468 56 : BIND(&if_holey_double);
2469 : {
2470 224 : var_result = AllocateHeapNumberWithValue(LoadFixedDoubleArrayElement(
2471 112 : CAST(elements), index, MachineType::Float64(), 0, INTPTR_PARAMETERS,
2472 280 : if_hole));
2473 56 : Goto(&done);
2474 : }
2475 :
2476 56 : BIND(&if_dictionary);
2477 : {
2478 : CSA_ASSERT(this, IsDictionaryElementsKind(elements_kind));
2479 112 : var_result = BasicLoadNumberDictionaryElement(CAST(elements), index,
2480 56 : if_accessor, if_hole);
2481 56 : Goto(&done);
2482 : }
2483 :
2484 56 : BIND(&done);
2485 112 : return var_result.value();
2486 : }
2487 :
2488 5616 : TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck(
2489 : SloppyTNode<Object> base, SloppyTNode<IntPtrT> offset, Label* if_hole,
2490 : MachineType machine_type) {
2491 5616 : if (if_hole) {
2492 : // TODO(ishell): Compare only the upper part for the hole once the
2493 : // compiler is able to fold addition of already complex |offset| with
2494 : // |kIeeeDoubleExponentWordOffset| into one addressing mode.
2495 5112 : if (Is64()) {
2496 5112 : Node* element = Load(MachineType::Uint64(), base, offset);
2497 5112 : GotoIf(Word64Equal(element, Int64Constant(kHoleNanInt64)), if_hole);
2498 : } else {
2499 0 : Node* element_upper = Load(
2500 : MachineType::Uint32(), base,
2501 0 : IntPtrAdd(offset, IntPtrConstant(kIeeeDoubleExponentWordOffset)));
2502 0 : GotoIf(Word32Equal(element_upper, Int32Constant(kHoleNanUpper32)),
2503 0 : if_hole);
2504 : }
2505 : }
2506 5616 : if (machine_type.IsNone()) {
2507 : // This means the actual value is not needed.
2508 956 : return TNode<Float64T>();
2509 : }
2510 4660 : return UncheckedCast<Float64T>(Load(machine_type, base, offset));
2511 : }
2512 :
2513 76720 : TNode<Object> CodeStubAssembler::LoadContextElement(
2514 : SloppyTNode<Context> context, int slot_index) {
2515 76720 : int offset = Context::SlotOffset(slot_index);
2516 : return UncheckedCast<Object>(
2517 76720 : Load(MachineType::AnyTagged(), context, IntPtrConstant(offset)));
2518 : }
2519 :
2520 5772 : TNode<Object> CodeStubAssembler::LoadContextElement(
2521 : SloppyTNode<Context> context, SloppyTNode<IntPtrT> slot_index) {
2522 11544 : Node* offset = ElementOffsetFromIndex(
2523 11544 : slot_index, PACKED_ELEMENTS, INTPTR_PARAMETERS, Context::SlotOffset(0));
2524 5772 : return UncheckedCast<Object>(Load(MachineType::AnyTagged(), context, offset));
2525 : }
2526 :
2527 56 : TNode<Object> CodeStubAssembler::LoadContextElement(TNode<Context> context,
2528 : TNode<Smi> slot_index) {
2529 112 : Node* offset = ElementOffsetFromIndex(slot_index, PACKED_ELEMENTS,
2530 112 : SMI_PARAMETERS, Context::SlotOffset(0));
2531 56 : return UncheckedCast<Object>(Load(MachineType::AnyTagged(), context, offset));
2532 : }
2533 :
2534 112 : void CodeStubAssembler::StoreContextElement(SloppyTNode<Context> context,
2535 : int slot_index,
2536 : SloppyTNode<Object> value) {
2537 112 : int offset = Context::SlotOffset(slot_index);
2538 112 : Store(context, IntPtrConstant(offset), value);
2539 112 : }
2540 :
2541 1008 : void CodeStubAssembler::StoreContextElement(SloppyTNode<Context> context,
2542 : SloppyTNode<IntPtrT> slot_index,
2543 : SloppyTNode<Object> value) {
2544 2016 : Node* offset = IntPtrAdd(TimesTaggedSize(slot_index),
2545 2016 : IntPtrConstant(Context::SlotOffset(0)));
2546 1008 : Store(context, offset, value);
2547 1008 : }
2548 :
2549 6880 : void CodeStubAssembler::StoreContextElementNoWriteBarrier(
2550 : SloppyTNode<Context> context, int slot_index, SloppyTNode<Object> value) {
2551 6880 : int offset = Context::SlotOffset(slot_index);
2552 13760 : StoreNoWriteBarrier(MachineRepresentation::kTagged, context,
2553 13760 : IntPtrConstant(offset), value);
2554 6880 : }
2555 :
2556 31576 : TNode<Context> CodeStubAssembler::LoadNativeContext(
2557 : SloppyTNode<Context> context) {
2558 : return UncheckedCast<Context>(
2559 31576 : LoadContextElement(context, Context::NATIVE_CONTEXT_INDEX));
2560 : }
2561 :
2562 168 : TNode<Context> CodeStubAssembler::LoadModuleContext(
2563 : SloppyTNode<Context> context) {
2564 168 : Node* module_map = LoadRoot(RootIndex::kModuleContextMap);
2565 336 : Variable cur_context(this, MachineRepresentation::kTaggedPointer);
2566 168 : cur_context.Bind(context);
2567 :
2568 336 : Label context_found(this);
2569 :
2570 168 : Variable* context_search_loop_variables[1] = {&cur_context};
2571 336 : Label context_search(this, 1, context_search_loop_variables);
2572 :
2573 : // Loop until cur_context->map() is module_map.
2574 168 : Goto(&context_search);
2575 168 : BIND(&context_search);
2576 : {
2577 : CSA_ASSERT(this, Word32BinaryNot(IsNativeContext(cur_context.value())));
2578 168 : GotoIf(WordEqual(LoadMap(cur_context.value()), module_map), &context_found);
2579 :
2580 168 : cur_context.Bind(
2581 336 : LoadContextElement(cur_context.value(), Context::PREVIOUS_INDEX));
2582 168 : Goto(&context_search);
2583 : }
2584 :
2585 168 : BIND(&context_found);
2586 336 : return UncheckedCast<Context>(cur_context.value());
2587 : }
2588 :
2589 788 : TNode<Map> CodeStubAssembler::LoadJSArrayElementsMap(
2590 : SloppyTNode<Int32T> kind, SloppyTNode<Context> native_context) {
2591 : CSA_ASSERT(this, IsFastElementsKind(kind));
2592 : CSA_ASSERT(this, IsNativeContext(native_context));
2593 1576 : Node* offset = IntPtrAdd(IntPtrConstant(Context::FIRST_JS_ARRAY_MAP_SLOT),
2594 2364 : ChangeInt32ToIntPtr(kind));
2595 788 : return UncheckedCast<Map>(LoadContextElement(native_context, offset));
2596 : }
2597 :
2598 4760 : TNode<Map> CodeStubAssembler::LoadJSArrayElementsMap(
2599 : ElementsKind kind, SloppyTNode<Context> native_context) {
2600 : CSA_ASSERT(this, IsNativeContext(native_context));
2601 : return UncheckedCast<Map>(
2602 4760 : LoadContextElement(native_context, Context::ArrayMapIndex(kind)));
2603 : }
2604 :
2605 3980 : TNode<BoolT> CodeStubAssembler::IsGeneratorFunction(
2606 : TNode<JSFunction> function) {
2607 : TNode<SharedFunctionInfo> const shared_function_info =
2608 3980 : CAST(LoadObjectField(function, JSFunction::kSharedFunctionInfoOffset));
2609 :
2610 : TNode<Uint32T> const function_kind =
2611 : DecodeWord32<SharedFunctionInfo::FunctionKindBits>(LoadObjectField(
2612 : shared_function_info, SharedFunctionInfo::kFlagsOffset,
2613 3980 : MachineType::Uint32()));
2614 :
2615 7960 : return TNode<BoolT>::UncheckedCast(Word32Or(
2616 7960 : Word32Or(
2617 7960 : Word32Or(
2618 7960 : Word32Equal(function_kind,
2619 7960 : Int32Constant(FunctionKind::kAsyncGeneratorFunction)),
2620 7960 : Word32Equal(
2621 : function_kind,
2622 23880 : Int32Constant(FunctionKind::kAsyncConciseGeneratorMethod))),
2623 7960 : Word32Equal(function_kind,
2624 23880 : Int32Constant(FunctionKind::kGeneratorFunction))),
2625 7960 : Word32Equal(function_kind,
2626 27860 : Int32Constant(FunctionKind::kConciseGeneratorMethod))));
2627 : }
2628 :
2629 3980 : TNode<BoolT> CodeStubAssembler::HasPrototypeProperty(TNode<JSFunction> function,
2630 : TNode<Map> map) {
2631 : // (has_prototype_slot() && IsConstructor()) ||
2632 : // IsGeneratorFunction(shared()->kind())
2633 : uint32_t mask =
2634 3980 : Map::HasPrototypeSlotBit::kMask | Map::IsConstructorBit::kMask;
2635 : return TNode<BoolT>::UncheckedCast(
2636 15920 : Word32Or(IsAllSetWord32(LoadMapBitField(map), mask),
2637 19900 : IsGeneratorFunction(function)));
2638 : }
2639 :
2640 3980 : void CodeStubAssembler::GotoIfPrototypeRequiresRuntimeLookup(
2641 : TNode<JSFunction> function, TNode<Map> map, Label* runtime) {
2642 : // !has_prototype_property() || has_non_instance_prototype()
2643 3980 : GotoIfNot(HasPrototypeProperty(function, map), runtime);
2644 7960 : GotoIf(IsSetWord32<Map::HasNonInstancePrototypeBit>(LoadMapBitField(map)),
2645 3980 : runtime);
2646 3980 : }
2647 :
2648 3924 : Node* CodeStubAssembler::LoadJSFunctionPrototype(Node* function,
2649 : Label* if_bailout) {
2650 : CSA_ASSERT(this, TaggedIsNotSmi(function));
2651 : CSA_ASSERT(this, IsJSFunction(function));
2652 : CSA_ASSERT(this, IsFunctionWithPrototypeSlotMap(LoadMap(function)));
2653 : CSA_ASSERT(this, IsClearWord32<Map::HasNonInstancePrototypeBit>(
2654 : LoadMapBitField(LoadMap(function))));
2655 : Node* proto_or_map =
2656 3924 : LoadObjectField(function, JSFunction::kPrototypeOrInitialMapOffset);
2657 3924 : GotoIf(IsTheHole(proto_or_map), if_bailout);
2658 :
2659 7848 : VARIABLE(var_result, MachineRepresentation::kTagged, proto_or_map);
2660 7848 : Label done(this, &var_result);
2661 3924 : GotoIfNot(IsMap(proto_or_map), &done);
2662 :
2663 3924 : var_result.Bind(LoadMapPrototype(proto_or_map));
2664 3924 : Goto(&done);
2665 :
2666 3924 : BIND(&done);
2667 7848 : return var_result.value();
2668 : }
2669 :
2670 112 : TNode<BytecodeArray> CodeStubAssembler::LoadSharedFunctionInfoBytecodeArray(
2671 : SloppyTNode<SharedFunctionInfo> shared) {
2672 : Node* function_data =
2673 112 : LoadObjectField(shared, SharedFunctionInfo::kFunctionDataOffset);
2674 :
2675 224 : VARIABLE(var_result, MachineRepresentation::kTagged, function_data);
2676 224 : Label done(this, &var_result);
2677 :
2678 112 : GotoIfNot(HasInstanceType(function_data, INTERPRETER_DATA_TYPE), &done);
2679 : Node* bytecode_array =
2680 112 : LoadObjectField(function_data, InterpreterData::kBytecodeArrayOffset);
2681 112 : var_result.Bind(bytecode_array);
2682 112 : Goto(&done);
2683 :
2684 112 : BIND(&done);
2685 224 : return CAST(var_result.value());
2686 : }
2687 :
2688 24 : void CodeStubAssembler::StoreObjectByteNoWriteBarrier(TNode<HeapObject> object,
2689 : int offset,
2690 : TNode<Word32T> value) {
2691 48 : StoreNoWriteBarrier(MachineRepresentation::kWord8, object,
2692 48 : IntPtrConstant(offset - kHeapObjectTag), value);
2693 24 : }
2694 :
2695 29588 : void CodeStubAssembler::StoreHeapNumberValue(SloppyTNode<HeapNumber> object,
2696 : SloppyTNode<Float64T> value) {
2697 29588 : StoreObjectFieldNoWriteBarrier(object, HeapNumber::kValueOffset, value,
2698 29588 : MachineRepresentation::kFloat64);
2699 29588 : }
2700 :
2701 1232 : void CodeStubAssembler::StoreMutableHeapNumberValue(
2702 : SloppyTNode<MutableHeapNumber> object, SloppyTNode<Float64T> value) {
2703 1232 : StoreObjectFieldNoWriteBarrier(object, MutableHeapNumber::kValueOffset, value,
2704 1232 : MachineRepresentation::kFloat64);
2705 1232 : }
2706 :
2707 16668 : void CodeStubAssembler::StoreObjectField(Node* object, int offset,
2708 : Node* value) {
2709 : DCHECK_NE(HeapObject::kMapOffset, offset); // Use StoreMap instead.
2710 :
2711 16668 : OptimizedStoreField(MachineRepresentation::kTagged,
2712 : UncheckedCast<HeapObject>(object), offset, value,
2713 16668 : WriteBarrierKind::kFullWriteBarrier);
2714 16668 : }
2715 :
2716 2128 : void CodeStubAssembler::StoreObjectField(Node* object, Node* offset,
2717 : Node* value) {
2718 : int const_offset;
2719 2128 : if (ToInt32Constant(offset, const_offset)) {
2720 0 : StoreObjectField(object, const_offset, value);
2721 : } else {
2722 2128 : Store(object, IntPtrSub(offset, IntPtrConstant(kHeapObjectTag)), value);
2723 : }
2724 2128 : }
2725 :
2726 159376 : void CodeStubAssembler::StoreObjectFieldNoWriteBarrier(
2727 : Node* object, int offset, Node* value, MachineRepresentation rep) {
2728 159376 : OptimizedStoreField(rep, UncheckedCast<HeapObject>(object), offset, value,
2729 159376 : WriteBarrierKind::kNoWriteBarrier);
2730 159376 : }
2731 :
2732 1568 : void CodeStubAssembler::StoreObjectFieldNoWriteBarrier(
2733 : Node* object, Node* offset, Node* value, MachineRepresentation rep) {
2734 : int const_offset;
2735 1568 : if (ToInt32Constant(offset, const_offset)) {
2736 336 : return StoreObjectFieldNoWriteBarrier(object, const_offset, value, rep);
2737 : }
2738 1232 : StoreNoWriteBarrier(rep, object,
2739 2464 : IntPtrSub(offset, IntPtrConstant(kHeapObjectTag)), value);
2740 : }
2741 :
2742 10764 : void CodeStubAssembler::StoreMap(Node* object, Node* map) {
2743 10764 : OptimizedStoreMap(UncheckedCast<HeapObject>(object), CAST(map));
2744 10764 : }
2745 :
2746 58304 : void CodeStubAssembler::StoreMapNoWriteBarrier(Node* object,
2747 : RootIndex map_root_index) {
2748 58304 : StoreMapNoWriteBarrier(object, LoadRoot(map_root_index));
2749 58304 : }
2750 :
2751 73312 : void CodeStubAssembler::StoreMapNoWriteBarrier(Node* object, Node* map) {
2752 : CSA_SLOW_ASSERT(this, IsMap(map));
2753 73312 : OptimizedStoreField(MachineRepresentation::kTaggedPointer,
2754 : UncheckedCast<HeapObject>(object), HeapObject::kMapOffset,
2755 73312 : map, WriteBarrierKind::kNoWriteBarrier);
2756 73312 : }
2757 :
2758 22260 : void CodeStubAssembler::StoreObjectFieldRoot(Node* object, int offset,
2759 : RootIndex root_index) {
2760 22260 : if (RootsTable::IsImmortalImmovable(root_index)) {
2761 22260 : return StoreObjectFieldNoWriteBarrier(object, offset, LoadRoot(root_index));
2762 : } else {
2763 0 : return StoreObjectField(object, offset, LoadRoot(root_index));
2764 : }
2765 : }
2766 :
2767 0 : void CodeStubAssembler::StoreJSArrayLength(TNode<JSArray> array,
2768 : TNode<Smi> length) {
2769 0 : StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
2770 0 : }
2771 :
2772 0 : void CodeStubAssembler::StoreElements(TNode<Object> object,
2773 : TNode<FixedArrayBase> elements) {
2774 0 : StoreObjectField(object, JSObject::kElementsOffset, elements);
2775 0 : }
2776 :
2777 39428 : void CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement(
2778 : Node* object, Node* index_node, Node* value, WriteBarrierMode barrier_mode,
2779 : int additional_offset, ParameterMode parameter_mode) {
2780 : CSA_SLOW_ASSERT(
2781 : this, Word32Or(IsFixedArraySubclass(object), IsPropertyArray(object)));
2782 : CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, parameter_mode));
2783 : DCHECK(barrier_mode == SKIP_WRITE_BARRIER ||
2784 : barrier_mode == UPDATE_WRITE_BARRIER);
2785 : DCHECK(IsAligned(additional_offset, kTaggedSize));
2786 : STATIC_ASSERT(static_cast<int>(FixedArray::kHeaderSize) ==
2787 : static_cast<int>(PropertyArray::kHeaderSize));
2788 : int header_size =
2789 39428 : FixedArray::kHeaderSize + additional_offset - kHeapObjectTag;
2790 78856 : Node* offset = ElementOffsetFromIndex(index_node, HOLEY_ELEMENTS,
2791 39428 : parameter_mode, header_size);
2792 : STATIC_ASSERT(static_cast<int>(FixedArrayBase::kLengthOffset) ==
2793 : static_cast<int>(WeakFixedArray::kLengthOffset));
2794 : STATIC_ASSERT(static_cast<int>(FixedArrayBase::kLengthOffset) ==
2795 : static_cast<int>(PropertyArray::kLengthAndHashOffset));
2796 : // Check that index_node + additional_offset <= object.length.
2797 : // TODO(cbruni): Use proper LoadXXLength helpers
2798 : CSA_ASSERT(
2799 : this,
2800 : IsOffsetInBounds(
2801 : offset,
2802 : Select<IntPtrT>(
2803 : IsPropertyArray(object),
2804 : [=] {
2805 : TNode<IntPtrT> length_and_hash = LoadAndUntagObjectField(
2806 : object, PropertyArray::kLengthAndHashOffset);
2807 : return TNode<IntPtrT>::UncheckedCast(
2808 : DecodeWord<PropertyArray::LengthField>(length_and_hash));
2809 : },
2810 : [=] {
2811 : return LoadAndUntagObjectField(object,
2812 : FixedArrayBase::kLengthOffset);
2813 : }),
2814 : FixedArray::kHeaderSize));
2815 39428 : if (barrier_mode == SKIP_WRITE_BARRIER) {
2816 21644 : StoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, value);
2817 : } else {
2818 17784 : Store(object, offset, value);
2819 : }
2820 39428 : }
2821 :
2822 1916 : void CodeStubAssembler::StoreFixedDoubleArrayElement(
2823 : TNode<FixedDoubleArray> object, Node* index_node, TNode<Float64T> value,
2824 : ParameterMode parameter_mode, CheckBounds check_bounds) {
2825 : CSA_ASSERT(this, IsFixedDoubleArray(object));
2826 : CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, parameter_mode));
2827 1916 : if (NeedsBoundsCheck(check_bounds)) {
2828 1916 : FixedArrayBoundsCheck(object, index_node, 0, parameter_mode);
2829 : }
2830 : Node* offset =
2831 3832 : ElementOffsetFromIndex(index_node, PACKED_DOUBLE_ELEMENTS, parameter_mode,
2832 1916 : FixedArray::kHeaderSize - kHeapObjectTag);
2833 1916 : MachineRepresentation rep = MachineRepresentation::kFloat64;
2834 1916 : StoreNoWriteBarrier(rep, object, offset, value);
2835 1916 : }
2836 :
2837 15960 : void CodeStubAssembler::StoreFeedbackVectorSlot(Node* object,
2838 : Node* slot_index_node,
2839 : Node* value,
2840 : WriteBarrierMode barrier_mode,
2841 : int additional_offset,
2842 : ParameterMode parameter_mode) {
2843 : CSA_SLOW_ASSERT(this, IsFeedbackVector(object));
2844 : CSA_SLOW_ASSERT(this, MatchesParameterMode(slot_index_node, parameter_mode));
2845 : DCHECK(IsAligned(additional_offset, kTaggedSize));
2846 : DCHECK(barrier_mode == SKIP_WRITE_BARRIER ||
2847 : barrier_mode == UPDATE_WRITE_BARRIER);
2848 : int header_size =
2849 15960 : FeedbackVector::kFeedbackSlotsOffset + additional_offset - kHeapObjectTag;
2850 31920 : Node* offset = ElementOffsetFromIndex(slot_index_node, HOLEY_ELEMENTS,
2851 15960 : parameter_mode, header_size);
2852 : // Check that slot_index_node <= object.length.
2853 : CSA_ASSERT(this,
2854 : IsOffsetInBounds(offset, LoadFeedbackVectorLength(CAST(object)),
2855 : FeedbackVector::kHeaderSize));
2856 15960 : if (barrier_mode == SKIP_WRITE_BARRIER) {
2857 13160 : StoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, value);
2858 : } else {
2859 2800 : Store(object, offset, value);
2860 : }
2861 15960 : }
2862 :
2863 336 : void CodeStubAssembler::EnsureArrayLengthWritable(TNode<Map> map,
2864 : Label* bailout) {
2865 : // Don't support arrays in dictionary named property mode.
2866 336 : GotoIf(IsDictionaryMap(map), bailout);
2867 :
2868 : // Check whether the length property is writable. The length property is the
2869 : // only default named property on arrays. It's nonconfigurable, hence is
2870 : // guaranteed to stay the first property.
2871 336 : TNode<DescriptorArray> descriptors = LoadMapDescriptors(map);
2872 :
2873 336 : int length_index = JSArray::kLengthDescriptorIndex;
2874 : #ifdef DEBUG
2875 : TNode<Name> maybe_length =
2876 : LoadKeyByDescriptorEntry(descriptors, length_index);
2877 : CSA_ASSERT(this,
2878 : WordEqual(maybe_length, LoadRoot(RootIndex::klength_string)));
2879 : #endif
2880 :
2881 : TNode<Uint32T> details =
2882 336 : LoadDetailsByDescriptorEntry(descriptors, length_index);
2883 672 : GotoIf(IsSetWord32(details, PropertyDetails::kAttributesReadOnlyMask),
2884 336 : bailout);
2885 336 : }
2886 :
2887 224 : TNode<Int32T> CodeStubAssembler::EnsureArrayPushable(TNode<Map> map,
2888 : Label* bailout) {
2889 : // Disallow pushing onto prototypes. It might be the JSArray prototype.
2890 : // Disallow pushing onto non-extensible objects.
2891 224 : Comment("Disallow pushing onto prototypes");
2892 224 : Node* bit_field2 = LoadMapBitField2(map);
2893 224 : int mask = Map::IsPrototypeMapBit::kMask | Map::IsExtensibleBit::kMask;
2894 224 : Node* test = Word32And(bit_field2, Int32Constant(mask));
2895 448 : GotoIf(Word32NotEqual(test, Int32Constant(Map::IsExtensibleBit::kMask)),
2896 224 : bailout);
2897 :
2898 224 : EnsureArrayLengthWritable(map, bailout);
2899 :
2900 224 : TNode<Uint32T> kind = DecodeWord32<Map::ElementsKindBits>(bit_field2);
2901 224 : return Signed(kind);
2902 : }
2903 :
2904 540 : void CodeStubAssembler::PossiblyGrowElementsCapacity(
2905 : ParameterMode mode, ElementsKind kind, Node* array, Node* length,
2906 : Variable* var_elements, Node* growth, Label* bailout) {
2907 1080 : Label fits(this, var_elements);
2908 : Node* capacity =
2909 540 : TaggedToParameter(LoadFixedArrayBaseLength(var_elements->value()), mode);
2910 : // length and growth nodes are already in a ParameterMode appropriate
2911 : // representation.
2912 540 : Node* new_length = IntPtrOrSmiAdd(growth, length, mode);
2913 540 : GotoIfNot(IntPtrOrSmiGreaterThan(new_length, capacity, mode), &fits);
2914 540 : Node* new_capacity = CalculateNewElementsCapacity(new_length, mode);
2915 540 : var_elements->Bind(GrowElementsCapacity(array, var_elements->value(), kind,
2916 : kind, capacity, new_capacity, mode,
2917 540 : bailout));
2918 540 : Goto(&fits);
2919 540 : BIND(&fits);
2920 540 : }
2921 :
2922 204 : TNode<Smi> CodeStubAssembler::BuildAppendJSArray(ElementsKind kind,
2923 : SloppyTNode<JSArray> array,
2924 : CodeStubArguments* args,
2925 : TVariable<IntPtrT>* arg_index,
2926 : Label* bailout) {
2927 : CSA_SLOW_ASSERT(this, IsJSArray(array));
2928 204 : Comment("BuildAppendJSArray: ", ElementsKindToString(kind));
2929 408 : Label pre_bailout(this);
2930 408 : Label success(this);
2931 408 : TVARIABLE(Smi, var_tagged_length);
2932 204 : ParameterMode mode = OptimalParameterMode();
2933 408 : VARIABLE(var_length, OptimalParameterRepresentation(),
2934 : TaggedToParameter(LoadFastJSArrayLength(array), mode));
2935 408 : VARIABLE(var_elements, MachineRepresentation::kTagged, LoadElements(array));
2936 :
2937 : // Resize the capacity of the fixed array if it doesn't fit.
2938 204 : TNode<IntPtrT> first = arg_index->value();
2939 408 : Node* growth = IntPtrToParameter(
2940 408 : IntPtrSub(UncheckedCast<IntPtrT>(args->GetLength(INTPTR_PARAMETERS)),
2941 204 : first),
2942 204 : mode);
2943 204 : PossiblyGrowElementsCapacity(mode, kind, array, var_length.value(),
2944 204 : &var_elements, growth, &pre_bailout);
2945 :
2946 : // Push each argument onto the end of the array now that there is enough
2947 : // capacity.
2948 408 : CodeStubAssembler::VariableList push_vars({&var_length}, zone());
2949 204 : Node* elements = var_elements.value();
2950 408 : args->ForEach(
2951 : push_vars,
2952 816 : [this, kind, mode, elements, &var_length, &pre_bailout](Node* arg) {
2953 408 : TryStoreArrayElement(kind, mode, &pre_bailout, elements,
2954 204 : var_length.value(), arg);
2955 204 : Increment(&var_length, 1, mode);
2956 204 : },
2957 204 : first, nullptr);
2958 : {
2959 204 : TNode<Smi> length = ParameterToTagged(var_length.value(), mode);
2960 204 : var_tagged_length = length;
2961 204 : StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
2962 204 : Goto(&success);
2963 : }
2964 :
2965 204 : BIND(&pre_bailout);
2966 : {
2967 204 : TNode<Smi> length = ParameterToTagged(var_length.value(), mode);
2968 204 : var_tagged_length = length;
2969 204 : Node* diff = SmiSub(length, LoadFastJSArrayLength(array));
2970 204 : StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
2971 204 : *arg_index = IntPtrAdd(arg_index->value(), SmiUntag(diff));
2972 204 : Goto(bailout);
2973 : }
2974 :
2975 204 : BIND(&success);
2976 408 : return var_tagged_length.value();
2977 : }
2978 :
2979 540 : void CodeStubAssembler::TryStoreArrayElement(ElementsKind kind,
2980 : ParameterMode mode, Label* bailout,
2981 : Node* elements, Node* index,
2982 : Node* value) {
2983 540 : if (IsSmiElementsKind(kind)) {
2984 184 : GotoIf(TaggedIsNotSmi(value), bailout);
2985 356 : } else if (IsDoubleElementsKind(kind)) {
2986 180 : GotoIfNotNumber(value, bailout);
2987 : }
2988 540 : if (IsDoubleElementsKind(kind)) value = ChangeNumberToFloat64(value);
2989 540 : StoreElement(elements, kind, index, value, mode);
2990 540 : }
2991 :
2992 336 : void CodeStubAssembler::BuildAppendJSArray(ElementsKind kind, Node* array,
2993 : Node* value, Label* bailout) {
2994 : CSA_SLOW_ASSERT(this, IsJSArray(array));
2995 336 : Comment("BuildAppendJSArray: ", ElementsKindToString(kind));
2996 336 : ParameterMode mode = OptimalParameterMode();
2997 672 : VARIABLE(var_length, OptimalParameterRepresentation(),
2998 : TaggedToParameter(LoadFastJSArrayLength(array), mode));
2999 672 : VARIABLE(var_elements, MachineRepresentation::kTagged, LoadElements(array));
3000 :
3001 : // Resize the capacity of the fixed array if it doesn't fit.
3002 336 : Node* growth = IntPtrOrSmiConstant(1, mode);
3003 336 : PossiblyGrowElementsCapacity(mode, kind, array, var_length.value(),
3004 336 : &var_elements, growth, bailout);
3005 :
3006 : // Push each argument onto the end of the array now that there is enough
3007 : // capacity.
3008 336 : TryStoreArrayElement(kind, mode, bailout, var_elements.value(),
3009 336 : var_length.value(), value);
3010 336 : Increment(&var_length, 1, mode);
3011 :
3012 336 : Node* length = ParameterToTagged(var_length.value(), mode);
3013 336 : StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
3014 336 : }
3015 :
3016 0 : Node* CodeStubAssembler::AllocateCellWithValue(Node* value,
3017 : WriteBarrierMode mode) {
3018 0 : Node* result = Allocate(Cell::kSize, kNone);
3019 0 : StoreMapNoWriteBarrier(result, RootIndex::kCellMap);
3020 0 : StoreCellValue(result, value, mode);
3021 0 : return result;
3022 : }
3023 :
3024 1288 : Node* CodeStubAssembler::LoadCellValue(Node* cell) {
3025 : CSA_SLOW_ASSERT(this, HasInstanceType(cell, CELL_TYPE));
3026 1288 : return LoadObjectField(cell, Cell::kValueOffset);
3027 : }
3028 :
3029 0 : void CodeStubAssembler::StoreCellValue(Node* cell, Node* value,
3030 : WriteBarrierMode mode) {
3031 : CSA_SLOW_ASSERT(this, HasInstanceType(cell, CELL_TYPE));
3032 : DCHECK(mode == SKIP_WRITE_BARRIER || mode == UPDATE_WRITE_BARRIER);
3033 :
3034 0 : if (mode == UPDATE_WRITE_BARRIER) {
3035 0 : StoreObjectField(cell, Cell::kValueOffset, value);
3036 : } else {
3037 0 : StoreObjectFieldNoWriteBarrier(cell, Cell::kValueOffset, value);
3038 : }
3039 0 : }
3040 :
3041 31748 : TNode<HeapNumber> CodeStubAssembler::AllocateHeapNumber() {
3042 31748 : Node* result = Allocate(HeapNumber::kSize, kNone);
3043 31748 : RootIndex heap_map_index = RootIndex::kHeapNumberMap;
3044 31748 : StoreMapNoWriteBarrier(result, heap_map_index);
3045 31748 : return UncheckedCast<HeapNumber>(result);
3046 : }
3047 :
3048 28524 : TNode<HeapNumber> CodeStubAssembler::AllocateHeapNumberWithValue(
3049 : SloppyTNode<Float64T> value) {
3050 28524 : TNode<HeapNumber> result = AllocateHeapNumber();
3051 28524 : StoreHeapNumberValue(result, value);
3052 28524 : return result;
3053 : }
3054 :
3055 1232 : TNode<MutableHeapNumber> CodeStubAssembler::AllocateMutableHeapNumber() {
3056 1232 : Node* result = Allocate(MutableHeapNumber::kSize, kNone);
3057 1232 : RootIndex heap_map_index = RootIndex::kMutableHeapNumberMap;
3058 1232 : StoreMapNoWriteBarrier(result, heap_map_index);
3059 1232 : return UncheckedCast<MutableHeapNumber>(result);
3060 : }
3061 :
3062 112 : TNode<Object> CodeStubAssembler::CloneIfMutablePrimitive(TNode<Object> object) {
3063 224 : TVARIABLE(Object, result, object);
3064 224 : Label done(this);
3065 :
3066 112 : GotoIf(TaggedIsSmi(object), &done);
3067 112 : GotoIfNot(IsMutableHeapNumber(UncheckedCast<HeapObject>(object)), &done);
3068 : {
3069 : // Mutable heap number found --- allocate a clone.
3070 : TNode<Float64T> value =
3071 112 : LoadHeapNumberValue(UncheckedCast<HeapNumber>(object));
3072 112 : result = AllocateMutableHeapNumberWithValue(value);
3073 112 : Goto(&done);
3074 : }
3075 :
3076 112 : BIND(&done);
3077 224 : return result.value();
3078 : }
3079 :
3080 1232 : TNode<MutableHeapNumber> CodeStubAssembler::AllocateMutableHeapNumberWithValue(
3081 : SloppyTNode<Float64T> value) {
3082 1232 : TNode<MutableHeapNumber> result = AllocateMutableHeapNumber();
3083 1232 : StoreMutableHeapNumberValue(result, value);
3084 1232 : return result;
3085 : }
3086 :
3087 2632 : TNode<BigInt> CodeStubAssembler::AllocateBigInt(TNode<IntPtrT> length) {
3088 2632 : TNode<BigInt> result = AllocateRawBigInt(length);
3089 10528 : StoreBigIntBitfield(result,
3090 5264 : Word32Shl(TruncateIntPtrToInt32(length),
3091 7896 : Int32Constant(BigInt::LengthBits::kShift)));
3092 2632 : return result;
3093 : }
3094 :
3095 3472 : TNode<BigInt> CodeStubAssembler::AllocateRawBigInt(TNode<IntPtrT> length) {
3096 : // This is currently used only for 64-bit wide BigInts. If more general
3097 : // applicability is required, a large-object check must be added.
3098 : CSA_ASSERT(this, UintPtrLessThan(length, IntPtrConstant(3)));
3099 :
3100 : TNode<IntPtrT> size =
3101 : IntPtrAdd(IntPtrConstant(BigInt::kHeaderSize),
3102 3472 : Signed(WordShl(length, kSystemPointerSizeLog2)));
3103 3472 : Node* raw_result = Allocate(size, kNone);
3104 3472 : StoreMapNoWriteBarrier(raw_result, RootIndex::kBigIntMap);
3105 : if (FIELD_SIZE(BigInt::kOptionalPaddingOffset) != 0) {
3106 : DCHECK_EQ(4, FIELD_SIZE(BigInt::kOptionalPaddingOffset));
3107 : StoreObjectFieldNoWriteBarrier(raw_result, BigInt::kOptionalPaddingOffset,
3108 : Int32Constant(0),
3109 : MachineRepresentation::kWord32);
3110 : }
3111 3472 : return UncheckedCast<BigInt>(raw_result);
3112 : }
3113 :
3114 4536 : void CodeStubAssembler::StoreBigIntBitfield(TNode<BigInt> bigint,
3115 : TNode<Word32T> bitfield) {
3116 4536 : StoreObjectFieldNoWriteBarrier(bigint, BigInt::kBitfieldOffset, bitfield,
3117 4536 : MachineRepresentation::kWord32);
3118 4536 : }
3119 :
3120 2576 : void CodeStubAssembler::StoreBigIntDigit(TNode<BigInt> bigint, int digit_index,
3121 : TNode<UintPtrT> digit) {
3122 2576 : StoreObjectFieldNoWriteBarrier(
3123 : bigint, BigInt::kDigitsOffset + digit_index * kSystemPointerSize, digit,
3124 2576 : UintPtrT::kMachineRepresentation);
3125 2576 : }
3126 :
3127 1680 : TNode<Word32T> CodeStubAssembler::LoadBigIntBitfield(TNode<BigInt> bigint) {
3128 : return UncheckedCast<Word32T>(
3129 1680 : LoadObjectField(bigint, BigInt::kBitfieldOffset, MachineType::Uint32()));
3130 : }
3131 :
3132 1568 : TNode<UintPtrT> CodeStubAssembler::LoadBigIntDigit(TNode<BigInt> bigint,
3133 : int digit_index) {
3134 : return UncheckedCast<UintPtrT>(LoadObjectField(
3135 : bigint, BigInt::kDigitsOffset + digit_index * kSystemPointerSize,
3136 1568 : MachineType::UintPtr()));
3137 : }
3138 :
3139 784 : TNode<String> CodeStubAssembler::AllocateSeqOneByteString(
3140 : uint32_t length, AllocationFlags flags) {
3141 784 : Comment("AllocateSeqOneByteString");
3142 784 : if (length == 0) {
3143 0 : return CAST(LoadRoot(RootIndex::kempty_string));
3144 : }
3145 784 : Node* result = Allocate(SeqOneByteString::SizeFor(length), flags);
3146 : DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kOneByteStringMap));
3147 784 : StoreMapNoWriteBarrier(result, RootIndex::kOneByteStringMap);
3148 784 : StoreObjectFieldNoWriteBarrier(result, SeqOneByteString::kLengthOffset,
3149 1568 : Uint32Constant(length),
3150 784 : MachineRepresentation::kWord32);
3151 784 : StoreObjectFieldNoWriteBarrier(result, SeqOneByteString::kHashFieldOffset,
3152 1568 : Int32Constant(String::kEmptyHashField),
3153 784 : MachineRepresentation::kWord32);
3154 784 : return CAST(result);
3155 : }
3156 :
3157 0 : TNode<BoolT> CodeStubAssembler::IsZeroOrContext(SloppyTNode<Object> object) {
3158 0 : return Select<BoolT>(WordEqual(object, SmiConstant(0)),
3159 0 : [=] { return Int32TrueConstant(); },
3160 0 : [=] { return IsContext(CAST(object)); });
3161 : }
3162 :
3163 1960 : TNode<String> CodeStubAssembler::AllocateSeqOneByteString(
3164 : Node* context, TNode<Uint32T> length, AllocationFlags flags) {
3165 1960 : Comment("AllocateSeqOneByteString");
3166 : CSA_SLOW_ASSERT(this, IsZeroOrContext(context));
3167 3920 : VARIABLE(var_result, MachineRepresentation::kTagged);
3168 :
3169 : // Compute the SeqOneByteString size and check if it fits into new space.
3170 3920 : Label if_lengthiszero(this), if_sizeissmall(this),
3171 3920 : if_notsizeissmall(this, Label::kDeferred), if_join(this);
3172 1960 : GotoIf(Word32Equal(length, Uint32Constant(0)), &if_lengthiszero);
3173 :
3174 3920 : Node* raw_size = GetArrayAllocationSize(
3175 3920 : Signed(ChangeUint32ToWord(length)), UINT8_ELEMENTS, INTPTR_PARAMETERS,
3176 3920 : SeqOneByteString::kHeaderSize + kObjectAlignmentMask);
3177 1960 : TNode<WordT> size = WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask));
3178 3920 : Branch(IntPtrLessThanOrEqual(size, IntPtrConstant(kMaxRegularHeapObjectSize)),
3179 1960 : &if_sizeissmall, &if_notsizeissmall);
3180 :
3181 1960 : BIND(&if_sizeissmall);
3182 : {
3183 : // Just allocate the SeqOneByteString in new space.
3184 : TNode<Object> result =
3185 1960 : AllocateInNewSpace(UncheckedCast<IntPtrT>(size), flags);
3186 : DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kOneByteStringMap));
3187 1960 : StoreMapNoWriteBarrier(result, RootIndex::kOneByteStringMap);
3188 1960 : StoreObjectFieldNoWriteBarrier(result, SeqOneByteString::kLengthOffset,
3189 1960 : length, MachineRepresentation::kWord32);
3190 1960 : StoreObjectFieldNoWriteBarrier(result, SeqOneByteString::kHashFieldOffset,
3191 3920 : Int32Constant(String::kEmptyHashField),
3192 1960 : MachineRepresentation::kWord32);
3193 1960 : var_result.Bind(result);
3194 1960 : Goto(&if_join);
3195 : }
3196 :
3197 1960 : BIND(&if_notsizeissmall);
3198 : {
3199 : // We might need to allocate in large object space, go to the runtime.
3200 3920 : Node* result = CallRuntime(Runtime::kAllocateSeqOneByteString, context,
3201 5880 : ChangeUint32ToTagged(length));
3202 1960 : var_result.Bind(result);
3203 1960 : Goto(&if_join);
3204 : }
3205 :
3206 1960 : BIND(&if_lengthiszero);
3207 : {
3208 1960 : var_result.Bind(LoadRoot(RootIndex::kempty_string));
3209 1960 : Goto(&if_join);
3210 : }
3211 :
3212 1960 : BIND(&if_join);
3213 3920 : return CAST(var_result.value());
3214 : }
3215 :
3216 896 : TNode<String> CodeStubAssembler::AllocateSeqTwoByteString(
3217 : uint32_t length, AllocationFlags flags) {
3218 896 : Comment("AllocateSeqTwoByteString");
3219 896 : if (length == 0) {
3220 0 : return CAST(LoadRoot(RootIndex::kempty_string));
3221 : }
3222 896 : Node* result = Allocate(SeqTwoByteString::SizeFor(length), flags);
3223 : DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kStringMap));
3224 896 : StoreMapNoWriteBarrier(result, RootIndex::kStringMap);
3225 896 : StoreObjectFieldNoWriteBarrier(result, SeqTwoByteString::kLengthOffset,
3226 1792 : Uint32Constant(length),
3227 896 : MachineRepresentation::kWord32);
3228 896 : StoreObjectFieldNoWriteBarrier(result, SeqTwoByteString::kHashFieldOffset,
3229 1792 : Int32Constant(String::kEmptyHashField),
3230 896 : MachineRepresentation::kWord32);
3231 896 : return CAST(result);
3232 : }
3233 :
3234 1568 : TNode<String> CodeStubAssembler::AllocateSeqTwoByteString(
3235 : Node* context, TNode<Uint32T> length, AllocationFlags flags) {
3236 : CSA_SLOW_ASSERT(this, IsZeroOrContext(context));
3237 1568 : Comment("AllocateSeqTwoByteString");
3238 3136 : VARIABLE(var_result, MachineRepresentation::kTagged);
3239 :
3240 : // Compute the SeqTwoByteString size and check if it fits into new space.
3241 3136 : Label if_lengthiszero(this), if_sizeissmall(this),
3242 3136 : if_notsizeissmall(this, Label::kDeferred), if_join(this);
3243 1568 : GotoIf(Word32Equal(length, Uint32Constant(0)), &if_lengthiszero);
3244 :
3245 3136 : Node* raw_size = GetArrayAllocationSize(
3246 3136 : Signed(ChangeUint32ToWord(length)), UINT16_ELEMENTS, INTPTR_PARAMETERS,
3247 3136 : SeqOneByteString::kHeaderSize + kObjectAlignmentMask);
3248 1568 : TNode<WordT> size = WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask));
3249 3136 : Branch(IntPtrLessThanOrEqual(size, IntPtrConstant(kMaxRegularHeapObjectSize)),
3250 1568 : &if_sizeissmall, &if_notsizeissmall);
3251 :
3252 1568 : BIND(&if_sizeissmall);
3253 : {
3254 : // Just allocate the SeqTwoByteString in new space.
3255 : TNode<Object> result =
3256 1568 : AllocateInNewSpace(UncheckedCast<IntPtrT>(size), flags);
3257 : DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kStringMap));
3258 1568 : StoreMapNoWriteBarrier(result, RootIndex::kStringMap);
3259 1568 : StoreObjectFieldNoWriteBarrier(result, SeqTwoByteString::kLengthOffset,
3260 1568 : length, MachineRepresentation::kWord32);
3261 1568 : StoreObjectFieldNoWriteBarrier(result, SeqTwoByteString::kHashFieldOffset,
3262 3136 : Int32Constant(String::kEmptyHashField),
3263 1568 : MachineRepresentation::kWord32);
3264 1568 : var_result.Bind(result);
3265 1568 : Goto(&if_join);
3266 : }
3267 :
3268 1568 : BIND(&if_notsizeissmall);
3269 : {
3270 : // We might need to allocate in large object space, go to the runtime.
3271 3136 : Node* result = CallRuntime(Runtime::kAllocateSeqTwoByteString, context,
3272 4704 : ChangeUint32ToTagged(length));
3273 1568 : var_result.Bind(result);
3274 1568 : Goto(&if_join);
3275 : }
3276 :
3277 1568 : BIND(&if_lengthiszero);
3278 : {
3279 1568 : var_result.Bind(LoadRoot(RootIndex::kempty_string));
3280 1568 : Goto(&if_join);
3281 : }
3282 :
3283 1568 : BIND(&if_join);
3284 3136 : return CAST(var_result.value());
3285 : }
3286 :
3287 896 : TNode<String> CodeStubAssembler::AllocateSlicedString(RootIndex map_root_index,
3288 : TNode<Uint32T> length,
3289 : TNode<String> parent,
3290 : TNode<Smi> offset) {
3291 : DCHECK(map_root_index == RootIndex::kSlicedOneByteStringMap ||
3292 : map_root_index == RootIndex::kSlicedStringMap);
3293 896 : Node* result = Allocate(SlicedString::kSize);
3294 : DCHECK(RootsTable::IsImmortalImmovable(map_root_index));
3295 896 : StoreMapNoWriteBarrier(result, map_root_index);
3296 896 : StoreObjectFieldNoWriteBarrier(result, SlicedString::kHashFieldOffset,
3297 1792 : Int32Constant(String::kEmptyHashField),
3298 896 : MachineRepresentation::kWord32);
3299 896 : StoreObjectFieldNoWriteBarrier(result, SlicedString::kLengthOffset, length,
3300 896 : MachineRepresentation::kWord32);
3301 896 : StoreObjectFieldNoWriteBarrier(result, SlicedString::kParentOffset, parent,
3302 896 : MachineRepresentation::kTagged);
3303 896 : StoreObjectFieldNoWriteBarrier(result, SlicedString::kOffsetOffset, offset,
3304 896 : MachineRepresentation::kTagged);
3305 896 : return CAST(result);
3306 : }
3307 :
3308 448 : TNode<String> CodeStubAssembler::AllocateSlicedOneByteString(
3309 : TNode<Uint32T> length, TNode<String> parent, TNode<Smi> offset) {
3310 : return AllocateSlicedString(RootIndex::kSlicedOneByteStringMap, length,
3311 448 : parent, offset);
3312 : }
3313 :
3314 448 : TNode<String> CodeStubAssembler::AllocateSlicedTwoByteString(
3315 : TNode<Uint32T> length, TNode<String> parent, TNode<Smi> offset) {
3316 : return AllocateSlicedString(RootIndex::kSlicedStringMap, length, parent,
3317 448 : offset);
3318 : }
3319 :
3320 392 : TNode<String> CodeStubAssembler::AllocateConsString(TNode<Uint32T> length,
3321 : TNode<String> left,
3322 : TNode<String> right,
3323 : Variable* var_feedback) {
3324 : // Added string can be a cons string.
3325 392 : Comment("Allocating ConsString");
3326 392 : Node* left_instance_type = LoadInstanceType(left);
3327 392 : Node* right_instance_type = LoadInstanceType(right);
3328 :
3329 : // Determine the resulting ConsString map to use depending on whether
3330 : // any of {left} or {right} has two byte encoding.
3331 : STATIC_ASSERT(kOneByteStringTag != 0);
3332 : STATIC_ASSERT(kTwoByteStringTag == 0);
3333 : Node* combined_instance_type =
3334 392 : Word32And(left_instance_type, right_instance_type);
3335 1176 : TNode<Map> result_map = CAST(Select<Object>(
3336 : IsSetWord32(combined_instance_type, kStringEncodingMask),
3337 : [=] {
3338 : if (var_feedback != nullptr) {
3339 : var_feedback->Bind(
3340 : SmiConstant(BinaryOperationFeedback::kConsOneByteString));
3341 : }
3342 : return LoadRoot(RootIndex::kConsOneByteStringMap);
3343 : },
3344 : [=] {
3345 : if (var_feedback != nullptr) {
3346 : var_feedback->Bind(
3347 : SmiConstant(BinaryOperationFeedback::kConsTwoByteString));
3348 : }
3349 : return LoadRoot(RootIndex::kConsStringMap);
3350 : }));
3351 392 : Node* result = AllocateInNewSpace(ConsString::kSize);
3352 392 : StoreMapNoWriteBarrier(result, result_map);
3353 392 : StoreObjectFieldNoWriteBarrier(result, ConsString::kLengthOffset, length,
3354 392 : MachineRepresentation::kWord32);
3355 392 : StoreObjectFieldNoWriteBarrier(result, ConsString::kHashFieldOffset,
3356 784 : Int32Constant(String::kEmptyHashField),
3357 392 : MachineRepresentation::kWord32);
3358 392 : StoreObjectFieldNoWriteBarrier(result, ConsString::kFirstOffset, left);
3359 392 : StoreObjectFieldNoWriteBarrier(result, ConsString::kSecondOffset, right);
3360 392 : return CAST(result);
3361 : }
3362 :
3363 616 : TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary(
3364 : int at_least_space_for) {
3365 616 : return AllocateNameDictionary(IntPtrConstant(at_least_space_for));
3366 : }
3367 :
3368 844 : TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary(
3369 : TNode<IntPtrT> at_least_space_for) {
3370 : CSA_ASSERT(this, UintPtrLessThanOrEqual(
3371 : at_least_space_for,
3372 : IntPtrConstant(NameDictionary::kMaxCapacity)));
3373 844 : TNode<IntPtrT> capacity = HashTableComputeCapacity(at_least_space_for);
3374 844 : return AllocateNameDictionaryWithCapacity(capacity);
3375 : }
3376 :
3377 1068 : TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionaryWithCapacity(
3378 : TNode<IntPtrT> capacity) {
3379 : CSA_ASSERT(this, WordIsPowerOfTwo(capacity));
3380 : CSA_ASSERT(this, IntPtrGreaterThan(capacity, IntPtrConstant(0)));
3381 1068 : TNode<IntPtrT> length = EntryToIndex<NameDictionary>(capacity);
3382 : TNode<IntPtrT> store_size = IntPtrAdd(
3383 1068 : TimesTaggedSize(length), IntPtrConstant(NameDictionary::kHeaderSize));
3384 :
3385 : TNode<NameDictionary> result =
3386 1068 : UncheckedCast<NameDictionary>(AllocateInNewSpace(store_size));
3387 1068 : Comment("Initialize NameDictionary");
3388 : // Initialize FixedArray fields.
3389 : DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kNameDictionaryMap));
3390 1068 : StoreMapNoWriteBarrier(result, RootIndex::kNameDictionaryMap);
3391 2136 : StoreObjectFieldNoWriteBarrier(result, FixedArray::kLengthOffset,
3392 1068 : SmiFromIntPtr(length));
3393 : // Initialized HashTable fields.
3394 1068 : TNode<Smi> zero = SmiConstant(0);
3395 2136 : StoreFixedArrayElement(result, NameDictionary::kNumberOfElementsIndex, zero,
3396 1068 : SKIP_WRITE_BARRIER);
3397 2136 : StoreFixedArrayElement(result, NameDictionary::kNumberOfDeletedElementsIndex,
3398 1068 : zero, SKIP_WRITE_BARRIER);
3399 2136 : StoreFixedArrayElement(result, NameDictionary::kCapacityIndex,
3400 3204 : SmiTag(capacity), SKIP_WRITE_BARRIER);
3401 : // Initialize Dictionary fields.
3402 1068 : TNode<HeapObject> filler = UndefinedConstant();
3403 2136 : StoreFixedArrayElement(result, NameDictionary::kNextEnumerationIndexIndex,
3404 2136 : SmiConstant(PropertyDetails::kInitialIndex),
3405 1068 : SKIP_WRITE_BARRIER);
3406 2136 : StoreFixedArrayElement(result, NameDictionary::kObjectHashIndex,
3407 2136 : SmiConstant(PropertyArray::kNoHashSentinel),
3408 1068 : SKIP_WRITE_BARRIER);
3409 :
3410 : // Initialize NameDictionary elements.
3411 1068 : TNode<WordT> result_word = BitcastTaggedToWord(result);
3412 : TNode<WordT> start_address = IntPtrAdd(
3413 3204 : result_word, IntPtrConstant(NameDictionary::OffsetOfElementAt(
3414 1068 : NameDictionary::kElementsStartIndex) -
3415 2136 : kHeapObjectTag));
3416 : TNode<WordT> end_address = IntPtrAdd(
3417 1068 : result_word, IntPtrSub(store_size, IntPtrConstant(kHeapObjectTag)));
3418 1068 : StoreFieldsNoWriteBarrier(start_address, end_address, filler);
3419 1068 : return result;
3420 : }
3421 :
3422 224 : TNode<NameDictionary> CodeStubAssembler::CopyNameDictionary(
3423 : TNode<NameDictionary> dictionary, Label* large_object_fallback) {
3424 224 : Comment("Copy boilerplate property dict");
3425 224 : TNode<IntPtrT> capacity = SmiUntag(GetCapacity<NameDictionary>(dictionary));
3426 : CSA_ASSERT(this, IntPtrGreaterThanOrEqual(capacity, IntPtrConstant(0)));
3427 448 : GotoIf(UintPtrGreaterThan(
3428 448 : capacity, IntPtrConstant(NameDictionary::kMaxRegularCapacity)),
3429 224 : large_object_fallback);
3430 : TNode<NameDictionary> properties =
3431 224 : AllocateNameDictionaryWithCapacity(capacity);
3432 224 : TNode<IntPtrT> length = SmiUntag(LoadFixedArrayBaseLength(dictionary));
3433 224 : CopyFixedArrayElements(PACKED_ELEMENTS, dictionary, properties, length,
3434 224 : SKIP_WRITE_BARRIER, INTPTR_PARAMETERS);
3435 224 : return properties;
3436 : }
3437 :
3438 : template <typename CollectionType>
3439 112 : Node* CodeStubAssembler::AllocateOrderedHashTable() {
3440 : static const int kCapacity = CollectionType::kMinCapacity;
3441 : static const int kBucketCount = kCapacity / CollectionType::kLoadFactor;
3442 : static const int kDataTableLength = kCapacity * CollectionType::kEntrySize;
3443 : static const int kFixedArrayLength =
3444 : CollectionType::HashTableStartIndex() + kBucketCount + kDataTableLength;
3445 : static const int kDataTableStartIndex =
3446 : CollectionType::HashTableStartIndex() + kBucketCount;
3447 :
3448 : STATIC_ASSERT(base::bits::IsPowerOfTwo(kCapacity));
3449 : STATIC_ASSERT(kCapacity <= CollectionType::MaxCapacity());
3450 :
3451 : // Allocate the table and add the proper map.
3452 112 : const ElementsKind elements_kind = HOLEY_ELEMENTS;
3453 112 : TNode<IntPtrT> length_intptr = IntPtrConstant(kFixedArrayLength);
3454 : TNode<Map> fixed_array_map =
3455 112 : CAST(LoadRoot(CollectionType::GetMapRootIndex()));
3456 : TNode<FixedArray> table =
3457 112 : CAST(AllocateFixedArray(elements_kind, length_intptr,
3458 : kAllowLargeObjectAllocation, fixed_array_map));
3459 :
3460 : // Initialize the OrderedHashTable fields.
3461 112 : const WriteBarrierMode barrier_mode = SKIP_WRITE_BARRIER;
3462 112 : StoreFixedArrayElement(table, CollectionType::NumberOfElementsIndex(),
3463 112 : SmiConstant(0), barrier_mode);
3464 112 : StoreFixedArrayElement(table, CollectionType::NumberOfDeletedElementsIndex(),
3465 112 : SmiConstant(0), barrier_mode);
3466 112 : StoreFixedArrayElement(table, CollectionType::NumberOfBucketsIndex(),
3467 112 : SmiConstant(kBucketCount), barrier_mode);
3468 :
3469 : // Fill the buckets with kNotFound.
3470 112 : TNode<Smi> not_found = SmiConstant(CollectionType::kNotFound);
3471 : STATIC_ASSERT(CollectionType::HashTableStartIndex() ==
3472 : CollectionType::NumberOfBucketsIndex() + 1);
3473 : STATIC_ASSERT((CollectionType::HashTableStartIndex() + kBucketCount) ==
3474 : kDataTableStartIndex);
3475 336 : for (int i = 0; i < kBucketCount; i++) {
3476 224 : StoreFixedArrayElement(table, CollectionType::HashTableStartIndex() + i,
3477 : not_found, barrier_mode);
3478 : }
3479 :
3480 : // Fill the data table with undefined.
3481 : STATIC_ASSERT(kDataTableStartIndex + kDataTableLength == kFixedArrayLength);
3482 1232 : for (int i = 0; i < kDataTableLength; i++) {
3483 1120 : StoreFixedArrayElement(table, kDataTableStartIndex + i, UndefinedConstant(),
3484 : barrier_mode);
3485 : }
3486 :
3487 112 : return table;
3488 : }
3489 :
3490 : template Node* CodeStubAssembler::AllocateOrderedHashTable<OrderedHashMap>();
3491 : template Node* CodeStubAssembler::AllocateOrderedHashTable<OrderedHashSet>();
3492 :
3493 : template <typename CollectionType>
3494 8 : TNode<CollectionType> CodeStubAssembler::AllocateSmallOrderedHashTable(
3495 : TNode<IntPtrT> capacity) {
3496 : CSA_ASSERT(this, WordIsPowerOfTwo(capacity));
3497 : CSA_ASSERT(this, IntPtrLessThan(
3498 : capacity, IntPtrConstant(CollectionType::kMaxCapacity)));
3499 :
3500 : TNode<IntPtrT> data_table_start_offset =
3501 8 : IntPtrConstant(CollectionType::DataTableStartOffset());
3502 :
3503 : TNode<IntPtrT> data_table_size = IntPtrMul(
3504 8 : capacity, IntPtrConstant(CollectionType::kEntrySize * kTaggedSize));
3505 :
3506 : TNode<Int32T> hash_table_size =
3507 16 : Int32Div(TruncateIntPtrToInt32(capacity),
3508 24 : Int32Constant(CollectionType::kLoadFactor));
3509 :
3510 : TNode<IntPtrT> hash_table_start_offset =
3511 8 : IntPtrAdd(data_table_start_offset, data_table_size);
3512 :
3513 : TNode<IntPtrT> hash_table_and_chain_table_size =
3514 8 : IntPtrAdd(ChangeInt32ToIntPtr(hash_table_size), capacity);
3515 :
3516 : TNode<IntPtrT> total_size =
3517 8 : IntPtrAdd(hash_table_start_offset, hash_table_and_chain_table_size);
3518 :
3519 : TNode<IntPtrT> total_size_word_aligned =
3520 8 : IntPtrAdd(total_size, IntPtrConstant(kTaggedSize - 1));
3521 8 : total_size_word_aligned = ChangeInt32ToIntPtr(
3522 16 : Int32Div(TruncateIntPtrToInt32(total_size_word_aligned),
3523 8 : Int32Constant(kTaggedSize)));
3524 8 : total_size_word_aligned =
3525 : UncheckedCast<IntPtrT>(TimesTaggedSize(total_size_word_aligned));
3526 :
3527 : // Allocate the table and add the proper map.
3528 : TNode<Map> small_ordered_hash_map =
3529 8 : CAST(LoadRoot(CollectionType::GetMapRootIndex()));
3530 8 : TNode<Object> table_obj = AllocateInNewSpace(total_size_word_aligned);
3531 8 : StoreMapNoWriteBarrier(table_obj, small_ordered_hash_map);
3532 8 : TNode<CollectionType> table = UncheckedCast<CollectionType>(table_obj);
3533 :
3534 : // Initialize the SmallOrderedHashTable fields.
3535 16 : StoreObjectByteNoWriteBarrier(
3536 : table, CollectionType::NumberOfBucketsOffset(),
3537 8 : Word32And(Int32Constant(0xFF), hash_table_size));
3538 8 : StoreObjectByteNoWriteBarrier(table, CollectionType::NumberOfElementsOffset(),
3539 8 : Int32Constant(0));
3540 8 : StoreObjectByteNoWriteBarrier(
3541 8 : table, CollectionType::NumberOfDeletedElementsOffset(), Int32Constant(0));
3542 :
3543 : TNode<IntPtrT> table_address =
3544 8 : IntPtrSub(BitcastTaggedToWord(table), IntPtrConstant(kHeapObjectTag));
3545 : TNode<IntPtrT> hash_table_start_address =
3546 8 : IntPtrAdd(table_address, hash_table_start_offset);
3547 :
3548 : // Initialize the HashTable part.
3549 8 : Node* memset = ExternalConstant(ExternalReference::libc_memset_function());
3550 16 : CallCFunction3(MachineType::AnyTagged(), MachineType::Pointer(),
3551 : MachineType::IntPtr(), MachineType::UintPtr(), memset,
3552 8 : hash_table_start_address, IntPtrConstant(0xFF),
3553 : hash_table_and_chain_table_size);
3554 :
3555 : // Initialize the DataTable part.
3556 8 : TNode<HeapObject> filler = TheHoleConstant();
3557 : TNode<WordT> data_table_start_address =
3558 8 : IntPtrAdd(table_address, data_table_start_offset);
3559 : TNode<WordT> data_table_end_address =
3560 8 : IntPtrAdd(data_table_start_address, data_table_size);
3561 8 : StoreFieldsNoWriteBarrier(data_table_start_address, data_table_end_address,
3562 : filler);
3563 :
3564 8 : return table;
3565 : }
3566 :
3567 : template TNode<SmallOrderedHashMap>
3568 : CodeStubAssembler::AllocateSmallOrderedHashTable<SmallOrderedHashMap>(
3569 : TNode<IntPtrT> capacity);
3570 : template TNode<SmallOrderedHashSet>
3571 : CodeStubAssembler::AllocateSmallOrderedHashTable<SmallOrderedHashSet>(
3572 : TNode<IntPtrT> capacity);
3573 :
3574 : template <typename CollectionType>
3575 1680 : void CodeStubAssembler::FindOrderedHashTableEntry(
3576 : Node* table, Node* hash,
3577 : const std::function<void(Node*, Label*, Label*)>& key_compare,
3578 : Variable* entry_start_position, Label* entry_found, Label* not_found) {
3579 : // Get the index of the bucket.
3580 1680 : Node* const number_of_buckets = SmiUntag(CAST(UnsafeLoadFixedArrayElement(
3581 3360 : CAST(table), CollectionType::NumberOfBucketsIndex())));
3582 : Node* const bucket =
3583 1680 : WordAnd(hash, IntPtrSub(number_of_buckets, IntPtrConstant(1)));
3584 3360 : Node* const first_entry = SmiUntag(CAST(UnsafeLoadFixedArrayElement(
3585 : CAST(table), bucket,
3586 3360 : CollectionType::HashTableStartIndex() * kTaggedSize)));
3587 :
3588 : // Walk the bucket chain.
3589 : Node* entry_start;
3590 3360 : Label if_key_found(this);
3591 : {
3592 3360 : VARIABLE(var_entry, MachineType::PointerRepresentation(), first_entry);
3593 3360 : Label loop(this, {&var_entry, entry_start_position}),
3594 3360 : continue_next_entry(this);
3595 1680 : Goto(&loop);
3596 1680 : BIND(&loop);
3597 :
3598 : // If the entry index is the not-found sentinel, we are done.
3599 1680 : GotoIf(
3600 : WordEqual(var_entry.value(), IntPtrConstant(CollectionType::kNotFound)),
3601 : not_found);
3602 :
3603 : // Make sure the entry index is within range.
3604 : CSA_ASSERT(
3605 : this,
3606 : UintPtrLessThan(
3607 : var_entry.value(),
3608 : SmiUntag(SmiAdd(
3609 : CAST(UnsafeLoadFixedArrayElement(
3610 : CAST(table), CollectionType::NumberOfElementsIndex())),
3611 : CAST(UnsafeLoadFixedArrayElement(
3612 : CAST(table),
3613 : CollectionType::NumberOfDeletedElementsIndex()))))));
3614 :
3615 : // Compute the index of the entry relative to kHashTableStartIndex.
3616 1680 : entry_start =
3617 : IntPtrAdd(IntPtrMul(var_entry.value(),
3618 : IntPtrConstant(CollectionType::kEntrySize)),
3619 : number_of_buckets);
3620 :
3621 : // Load the key from the entry.
3622 : Node* const candidate_key = UnsafeLoadFixedArrayElement(
3623 1680 : CAST(table), entry_start,
3624 3360 : CollectionType::HashTableStartIndex() * kTaggedSize);
3625 :
3626 1680 : key_compare(candidate_key, &if_key_found, &continue_next_entry);
3627 :
3628 1680 : BIND(&continue_next_entry);
3629 : // Load the index of the next entry in the bucket chain.
3630 1680 : var_entry.Bind(SmiUntag(CAST(UnsafeLoadFixedArrayElement(
3631 : CAST(table), entry_start,
3632 : (CollectionType::HashTableStartIndex() + CollectionType::kChainOffset) *
3633 : kTaggedSize))));
3634 :
3635 1680 : Goto(&loop);
3636 : }
3637 :
3638 1680 : BIND(&if_key_found);
3639 1680 : entry_start_position->Bind(entry_start);
3640 1680 : Goto(entry_found);
3641 1680 : }
3642 :
3643 : template void CodeStubAssembler::FindOrderedHashTableEntry<OrderedHashMap>(
3644 : Node* table, Node* hash,
3645 : const std::function<void(Node*, Label*, Label*)>& key_compare,
3646 : Variable* entry_start_position, Label* entry_found, Label* not_found);
3647 : template void CodeStubAssembler::FindOrderedHashTableEntry<OrderedHashSet>(
3648 : Node* table, Node* hash,
3649 : const std::function<void(Node*, Label*, Label*)>& key_compare,
3650 : Variable* entry_start_position, Label* entry_found, Label* not_found);
3651 :
3652 8 : Node* CodeStubAssembler::AllocateStruct(Node* map, AllocationFlags flags) {
3653 8 : Comment("AllocateStruct");
3654 : CSA_ASSERT(this, IsMap(map));
3655 8 : TNode<IntPtrT> size = TimesTaggedSize(LoadMapInstanceSizeInWords(map));
3656 8 : TNode<Object> object = Allocate(size, flags);
3657 8 : StoreMapNoWriteBarrier(object, map);
3658 8 : InitializeStructBody(object, map, size, Struct::kHeaderSize);
3659 8 : return object;
3660 : }
3661 :
3662 8 : void CodeStubAssembler::InitializeStructBody(Node* object, Node* map,
3663 : Node* size, int start_offset) {
3664 : CSA_SLOW_ASSERT(this, IsMap(map));
3665 8 : Comment("InitializeStructBody");
3666 8 : Node* filler = UndefinedConstant();
3667 : // Calculate the untagged field addresses.
3668 8 : object = BitcastTaggedToWord(object);
3669 : Node* start_address =
3670 8 : IntPtrAdd(object, IntPtrConstant(start_offset - kHeapObjectTag));
3671 : Node* end_address =
3672 8 : IntPtrSub(IntPtrAdd(object, size), IntPtrConstant(kHeapObjectTag));
3673 8 : StoreFieldsNoWriteBarrier(start_address, end_address, filler);
3674 8 : }
3675 :
3676 2132 : Node* CodeStubAssembler::AllocateJSObjectFromMap(
3677 : Node* map, Node* properties, Node* elements, AllocationFlags flags,
3678 : SlackTrackingMode slack_tracking_mode) {
3679 : CSA_ASSERT(this, IsMap(map));
3680 : CSA_ASSERT(this, Word32BinaryNot(IsJSFunctionMap(map)));
3681 : CSA_ASSERT(this, Word32BinaryNot(InstanceTypeEqual(LoadMapInstanceType(map),
3682 : JS_GLOBAL_OBJECT_TYPE)));
3683 : TNode<IntPtrT> instance_size =
3684 2132 : TimesTaggedSize(LoadMapInstanceSizeInWords(map));
3685 2132 : TNode<Object> object = AllocateInNewSpace(instance_size, flags);
3686 2132 : StoreMapNoWriteBarrier(object, map);
3687 2132 : InitializeJSObjectFromMap(object, map, instance_size, properties, elements,
3688 2132 : slack_tracking_mode);
3689 2132 : return object;
3690 : }
3691 :
3692 2132 : void CodeStubAssembler::InitializeJSObjectFromMap(
3693 : Node* object, Node* map, Node* instance_size, Node* properties,
3694 : Node* elements, SlackTrackingMode slack_tracking_mode) {
3695 : CSA_SLOW_ASSERT(this, IsMap(map));
3696 : // This helper assumes that the object is in new-space, as guarded by the
3697 : // check in AllocatedJSObjectFromMap.
3698 2132 : if (properties == nullptr) {
3699 : CSA_ASSERT(this, Word32BinaryNot(IsDictionaryMap((map))));
3700 : StoreObjectFieldRoot(object, JSObject::kPropertiesOrHashOffset,
3701 1120 : RootIndex::kEmptyFixedArray);
3702 : } else {
3703 : CSA_ASSERT(this, Word32Or(Word32Or(IsPropertyArray(properties),
3704 : IsNameDictionary(properties)),
3705 : IsEmptyFixedArray(properties)));
3706 : StoreObjectFieldNoWriteBarrier(object, JSObject::kPropertiesOrHashOffset,
3707 1012 : properties);
3708 : }
3709 2132 : if (elements == nullptr) {
3710 : StoreObjectFieldRoot(object, JSObject::kElementsOffset,
3711 1960 : RootIndex::kEmptyFixedArray);
3712 : } else {
3713 : CSA_ASSERT(this, IsFixedArray(elements));
3714 172 : StoreObjectFieldNoWriteBarrier(object, JSObject::kElementsOffset, elements);
3715 : }
3716 2132 : if (slack_tracking_mode == kNoSlackTracking) {
3717 1628 : InitializeJSObjectBodyNoSlackTracking(object, map, instance_size);
3718 : } else {
3719 : DCHECK_EQ(slack_tracking_mode, kWithSlackTracking);
3720 504 : InitializeJSObjectBodyWithSlackTracking(object, map, instance_size);
3721 : }
3722 2132 : }
3723 :
3724 2188 : void CodeStubAssembler::InitializeJSObjectBodyNoSlackTracking(
3725 : Node* object, Node* map, Node* instance_size, int start_offset) {
3726 : STATIC_ASSERT(Map::kNoSlackTracking == 0);
3727 : CSA_ASSERT(
3728 : this, IsClearWord32<Map::ConstructionCounterBits>(LoadMapBitField3(map)));
3729 4376 : InitializeFieldsWithRoot(object, IntPtrConstant(start_offset), instance_size,
3730 2188 : RootIndex::kUndefinedValue);
3731 2188 : }
3732 :
3733 504 : void CodeStubAssembler::InitializeJSObjectBodyWithSlackTracking(
3734 : Node* object, Node* map, Node* instance_size) {
3735 : CSA_SLOW_ASSERT(this, IsMap(map));
3736 504 : Comment("InitializeJSObjectBodyNoSlackTracking");
3737 :
3738 : // Perform in-object slack tracking if requested.
3739 504 : int start_offset = JSObject::kHeaderSize;
3740 504 : Node* bit_field3 = LoadMapBitField3(map);
3741 1008 : Label end(this), slack_tracking(this), complete(this, Label::kDeferred);
3742 : STATIC_ASSERT(Map::kNoSlackTracking == 0);
3743 1008 : GotoIf(IsSetWord32<Map::ConstructionCounterBits>(bit_field3),
3744 504 : &slack_tracking);
3745 504 : Comment("No slack tracking");
3746 504 : InitializeJSObjectBodyNoSlackTracking(object, map, instance_size);
3747 504 : Goto(&end);
3748 :
3749 504 : BIND(&slack_tracking);
3750 : {
3751 504 : Comment("Decrease construction counter");
3752 : // Slack tracking is only done on initial maps.
3753 : CSA_ASSERT(this, IsUndefined(LoadMapBackPointer(map)));
3754 : STATIC_ASSERT(Map::ConstructionCounterBits::kNext == 32);
3755 1008 : Node* new_bit_field3 = Int32Sub(
3756 1512 : bit_field3, Int32Constant(1 << Map::ConstructionCounterBits::kShift));
3757 : StoreObjectFieldNoWriteBarrier(map, Map::kBitField3Offset, new_bit_field3,
3758 504 : MachineRepresentation::kWord32);
3759 : STATIC_ASSERT(Map::kSlackTrackingCounterEnd == 1);
3760 :
3761 : // The object still has in-object slack therefore the |unsed_or_unused|
3762 : // field contain the "used" value.
3763 1008 : Node* used_size = TimesTaggedSize(ChangeUint32ToWord(
3764 : LoadObjectField(map, Map::kUsedOrUnusedInstanceSizeInWordsOffset,
3765 1512 : MachineType::Uint8())));
3766 :
3767 504 : Comment("iInitialize filler fields");
3768 : InitializeFieldsWithRoot(object, used_size, instance_size,
3769 504 : RootIndex::kOnePointerFillerMap);
3770 :
3771 504 : Comment("Initialize undefined fields");
3772 1008 : InitializeFieldsWithRoot(object, IntPtrConstant(start_offset), used_size,
3773 504 : RootIndex::kUndefinedValue);
3774 :
3775 : STATIC_ASSERT(Map::kNoSlackTracking == 0);
3776 1008 : GotoIf(IsClearWord32<Map::ConstructionCounterBits>(new_bit_field3),
3777 504 : &complete);
3778 504 : Goto(&end);
3779 : }
3780 :
3781 : // Finalize the instance size.
3782 504 : BIND(&complete);
3783 : {
3784 : // ComplextInobjectSlackTracking doesn't allocate and thus doesn't need a
3785 : // context.
3786 : CallRuntime(Runtime::kCompleteInobjectSlackTrackingForMap,
3787 504 : NoContextConstant(), map);
3788 504 : Goto(&end);
3789 : }
3790 :
3791 504 : BIND(&end);
3792 504 : }
3793 :
3794 1084 : void CodeStubAssembler::StoreFieldsNoWriteBarrier(Node* start_address,
3795 : Node* end_address,
3796 : Node* value) {
3797 1084 : Comment("StoreFieldsNoWriteBarrier");
3798 : CSA_ASSERT(this, WordIsAligned(start_address, kTaggedSize));
3799 : CSA_ASSERT(this, WordIsAligned(end_address, kTaggedSize));
3800 2168 : BuildFastLoop(
3801 : start_address, end_address,
3802 1084 : [this, value](Node* current) {
3803 1084 : StoreNoWriteBarrier(MachineRepresentation::kTagged, current, value);
3804 1084 : },
3805 1084 : kTaggedSize, INTPTR_PARAMETERS, IndexAdvanceMode::kPost);
3806 1084 : }
3807 :
3808 168 : TNode<BoolT> CodeStubAssembler::IsValidFastJSArrayCapacity(
3809 : Node* capacity, ParameterMode capacity_mode) {
3810 : return UncheckedCast<BoolT>(
3811 336 : UintPtrLessThanOrEqual(ParameterToIntPtr(capacity, capacity_mode),
3812 504 : IntPtrConstant(JSArray::kMaxFastArrayLength)));
3813 : }
3814 :
3815 3752 : TNode<JSArray> CodeStubAssembler::AllocateJSArray(
3816 : TNode<Map> array_map, TNode<FixedArrayBase> elements, TNode<Smi> length,
3817 : Node* allocation_site) {
3818 3752 : Comment("begin allocation of JSArray passing in elements");
3819 : CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
3820 :
3821 3752 : int base_size = JSArray::kSize;
3822 3752 : if (allocation_site != nullptr) {
3823 616 : base_size += AllocationMemento::kSize;
3824 : }
3825 :
3826 3752 : TNode<IntPtrT> size = IntPtrConstant(base_size);
3827 : TNode<JSArray> result =
3828 3752 : AllocateUninitializedJSArray(array_map, length, allocation_site, size);
3829 3752 : StoreObjectFieldNoWriteBarrier(result, JSArray::kElementsOffset, elements);
3830 3752 : return result;
3831 : }
3832 :
3833 : std::pair<TNode<JSArray>, TNode<FixedArrayBase>>
3834 2520 : CodeStubAssembler::AllocateUninitializedJSArrayWithElements(
3835 : ElementsKind kind, TNode<Map> array_map, TNode<Smi> length,
3836 : Node* allocation_site, Node* capacity, ParameterMode capacity_mode,
3837 : AllocationFlags allocation_flags) {
3838 2520 : Comment("begin allocation of JSArray with elements");
3839 5040 : CHECK_EQ(allocation_flags & ~kAllowLargeObjectAllocation, 0);
3840 : CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
3841 :
3842 5040 : TVARIABLE(JSArray, array);
3843 5040 : TVARIABLE(FixedArrayBase, elements);
3844 :
3845 2520 : if (IsIntPtrOrSmiConstantZero(capacity, capacity_mode)) {
3846 784 : TNode<FixedArrayBase> empty_array = EmptyFixedArrayConstant();
3847 784 : array = AllocateJSArray(array_map, empty_array, length, allocation_site);
3848 784 : return {array.value(), empty_array};
3849 : }
3850 :
3851 3472 : Label out(this), empty(this), nonempty(this);
3852 :
3853 3472 : Branch(SmiEqual(ParameterToTagged(capacity, capacity_mode), SmiConstant(0)),
3854 1736 : &empty, &nonempty);
3855 :
3856 1736 : BIND(&empty);
3857 : {
3858 1736 : TNode<FixedArrayBase> empty_array = EmptyFixedArrayConstant();
3859 1736 : array = AllocateJSArray(array_map, empty_array, length, allocation_site);
3860 1736 : elements = empty_array;
3861 1736 : Goto(&out);
3862 : }
3863 :
3864 1736 : BIND(&nonempty);
3865 : {
3866 1736 : int base_size = JSArray::kSize;
3867 1736 : if (allocation_site != nullptr) base_size += AllocationMemento::kSize;
3868 :
3869 1736 : const int elements_offset = base_size;
3870 :
3871 : // Compute space for elements
3872 1736 : base_size += FixedArray::kHeaderSize;
3873 : TNode<IntPtrT> size =
3874 1736 : ElementOffsetFromIndex(capacity, kind, capacity_mode, base_size);
3875 :
3876 : // For very large arrays in which the requested allocation exceeds the
3877 : // maximal size of a regular heap object, we cannot use the allocation
3878 : // folding trick. Instead, we first allocate the elements in large object
3879 : // space, and then allocate the JSArray (and possibly the allocation
3880 : // memento) in new space.
3881 1736 : if (allocation_flags & kAllowLargeObjectAllocation) {
3882 336 : Label next(this);
3883 168 : GotoIf(IsRegularHeapObjectSize(size), &next);
3884 :
3885 168 : CSA_CHECK(this, IsValidFastJSArrayCapacity(capacity, capacity_mode));
3886 :
3887 : // Allocate and initialize the elements first. Full initialization is
3888 : // needed because the upcoming JSArray allocation could trigger GC.
3889 336 : elements =
3890 168 : AllocateFixedArray(kind, capacity, capacity_mode, allocation_flags);
3891 :
3892 168 : if (IsDoubleElementsKind(kind)) {
3893 0 : FillFixedDoubleArrayWithZero(CAST(elements.value()),
3894 0 : ParameterToIntPtr(capacity, capacity_mode));
3895 : } else {
3896 336 : FillFixedArrayWithSmiZero(CAST(elements.value()),
3897 168 : ParameterToIntPtr(capacity, capacity_mode));
3898 : }
3899 :
3900 : // The JSArray and possibly allocation memento next. Note that
3901 : // allocation_flags are *not* passed on here and the resulting JSArray
3902 : // will always be in new space.
3903 168 : array =
3904 168 : AllocateJSArray(array_map, elements.value(), length, allocation_site);
3905 :
3906 168 : Goto(&out);
3907 :
3908 168 : BIND(&next);
3909 : }
3910 :
3911 : // Fold all objects into a single new space allocation.
3912 1736 : array =
3913 1736 : AllocateUninitializedJSArray(array_map, length, allocation_site, size);
3914 3472 : elements = UncheckedCast<FixedArrayBase>(
3915 5208 : InnerAllocate(array.value(), elements_offset));
3916 :
3917 3472 : StoreObjectFieldNoWriteBarrier(array.value(), JSObject::kElementsOffset,
3918 1736 : elements.value());
3919 :
3920 : // Setup elements object.
3921 : STATIC_ASSERT(FixedArrayBase::kHeaderSize == 2 * kTaggedSize);
3922 1736 : RootIndex elements_map_index = IsDoubleElementsKind(kind)
3923 : ? RootIndex::kFixedDoubleArrayMap
3924 1736 : : RootIndex::kFixedArrayMap;
3925 : DCHECK(RootsTable::IsImmortalImmovable(elements_map_index));
3926 1736 : StoreMapNoWriteBarrier(elements.value(), elements_map_index);
3927 :
3928 1736 : TNode<Smi> capacity_smi = ParameterToTagged(capacity, capacity_mode);
3929 : CSA_ASSERT(this, SmiGreaterThan(capacity_smi, SmiConstant(0)));
3930 3472 : StoreObjectFieldNoWriteBarrier(elements.value(), FixedArray::kLengthOffset,
3931 1736 : capacity_smi);
3932 1736 : Goto(&out);
3933 : }
3934 :
3935 1736 : BIND(&out);
3936 1736 : return {array.value(), elements.value()};
3937 : }
3938 :
3939 5488 : TNode<JSArray> CodeStubAssembler::AllocateUninitializedJSArray(
3940 : TNode<Map> array_map, TNode<Smi> length, Node* allocation_site,
3941 : TNode<IntPtrT> size_in_bytes) {
3942 : CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
3943 :
3944 : // Allocate space for the JSArray and the elements FixedArray in one go.
3945 5488 : TNode<Object> array = AllocateInNewSpace(size_in_bytes);
3946 :
3947 5488 : StoreMapNoWriteBarrier(array, array_map);
3948 5488 : StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
3949 5488 : StoreObjectFieldRoot(array, JSArray::kPropertiesOrHashOffset,
3950 5488 : RootIndex::kEmptyFixedArray);
3951 :
3952 5488 : if (allocation_site != nullptr) {
3953 1680 : InitializeAllocationMemento(array, IntPtrConstant(JSArray::kSize),
3954 840 : allocation_site);
3955 : }
3956 :
3957 5488 : return CAST(array);
3958 : }
3959 :
3960 2352 : TNode<JSArray> CodeStubAssembler::AllocateJSArray(
3961 : ElementsKind kind, TNode<Map> array_map, Node* capacity, TNode<Smi> length,
3962 : Node* allocation_site, ParameterMode capacity_mode,
3963 : AllocationFlags allocation_flags) {
3964 : CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
3965 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, capacity_mode));
3966 :
3967 2352 : TNode<JSArray> array;
3968 2352 : TNode<FixedArrayBase> elements;
3969 :
3970 4704 : std::tie(array, elements) = AllocateUninitializedJSArrayWithElements(
3971 : kind, array_map, length, allocation_site, capacity, capacity_mode,
3972 4704 : allocation_flags);
3973 :
3974 4704 : Label out(this), nonempty(this);
3975 :
3976 4704 : Branch(SmiEqual(ParameterToTagged(capacity, capacity_mode), SmiConstant(0)),
3977 2352 : &out, &nonempty);
3978 :
3979 2352 : BIND(&nonempty);
3980 : {
3981 2352 : FillFixedArrayWithValue(kind, elements,
3982 : IntPtrOrSmiConstant(0, capacity_mode), capacity,
3983 2352 : RootIndex::kTheHoleValue, capacity_mode);
3984 2352 : Goto(&out);
3985 : }
3986 :
3987 2352 : BIND(&out);
3988 4704 : return array;
3989 : }
3990 :
3991 56 : Node* CodeStubAssembler::ExtractFastJSArray(Node* context, Node* array,
3992 : Node* begin, Node* count,
3993 : ParameterMode mode, Node* capacity,
3994 : Node* allocation_site) {
3995 56 : Node* original_array_map = LoadMap(array);
3996 56 : Node* elements_kind = LoadMapElementsKind(original_array_map);
3997 :
3998 : // Use the cannonical map for the Array's ElementsKind
3999 56 : Node* native_context = LoadNativeContext(context);
4000 56 : TNode<Map> array_map = LoadJSArrayElementsMap(elements_kind, native_context);
4001 :
4002 : TNode<FixedArrayBase> new_elements = ExtractFixedArray(
4003 112 : LoadElements(array), begin, count, capacity,
4004 168 : ExtractFixedArrayFlag::kAllFixedArrays, mode, nullptr, elements_kind);
4005 :
4006 112 : TNode<Object> result = AllocateJSArray(
4007 112 : array_map, new_elements, ParameterToTagged(count, mode), allocation_site);
4008 56 : return result;
4009 : }
4010 :
4011 336 : Node* CodeStubAssembler::CloneFastJSArray(Node* context, Node* array,
4012 : ParameterMode mode,
4013 : Node* allocation_site,
4014 : HoleConversionMode convert_holes) {
4015 : // TODO(dhai): we should be able to assert IsFastJSArray(array) here, but this
4016 : // function is also used to copy boilerplates even when the no-elements
4017 : // protector is invalid. This function should be renamed to reflect its uses.
4018 : CSA_ASSERT(this, IsJSArray(array));
4019 :
4020 336 : Node* length = LoadJSArrayLength(array);
4021 336 : Node* new_elements = nullptr;
4022 672 : VARIABLE(var_new_elements, MachineRepresentation::kTagged);
4023 672 : TVARIABLE(Int32T, var_elements_kind, LoadMapElementsKind(LoadMap(array)));
4024 :
4025 672 : Label allocate_jsarray(this), holey_extract(this);
4026 :
4027 : bool need_conversion =
4028 336 : convert_holes == HoleConversionMode::kConvertToUndefined;
4029 336 : if (need_conversion) {
4030 : // We need to take care of holes, if the array is of holey elements kind.
4031 56 : GotoIf(IsHoleyFastElementsKind(var_elements_kind.value()), &holey_extract);
4032 : }
4033 :
4034 : // Simple extraction that preserves holes.
4035 : new_elements =
4036 1008 : ExtractFixedArray(LoadElements(array), IntPtrOrSmiConstant(0, mode),
4037 : TaggedToParameter(length, mode), nullptr,
4038 : ExtractFixedArrayFlag::kAllFixedArraysDontCopyCOW, mode,
4039 1344 : nullptr, var_elements_kind.value());
4040 336 : var_new_elements.Bind(new_elements);
4041 336 : Goto(&allocate_jsarray);
4042 :
4043 336 : if (need_conversion) {
4044 56 : BIND(&holey_extract);
4045 : // Convert holes to undefined.
4046 112 : TVARIABLE(BoolT, var_holes_converted, Int32FalseConstant());
4047 : // Copy |array|'s elements store. The copy will be compatible with the
4048 : // original elements kind unless there are holes in the source. Any holes
4049 : // get converted to undefined, hence in that case the copy is compatible
4050 : // only with PACKED_ELEMENTS and HOLEY_ELEMENTS, and we will choose
4051 : // PACKED_ELEMENTS. Also, if we want to replace holes, we must not use
4052 : // ExtractFixedArrayFlag::kDontCopyCOW.
4053 112 : new_elements = ExtractFixedArray(
4054 112 : LoadElements(array), IntPtrOrSmiConstant(0, mode),
4055 : TaggedToParameter(length, mode), nullptr,
4056 224 : ExtractFixedArrayFlag::kAllFixedArrays, mode, &var_holes_converted);
4057 56 : var_new_elements.Bind(new_elements);
4058 : // If the array type didn't change, use the original elements kind.
4059 56 : GotoIfNot(var_holes_converted.value(), &allocate_jsarray);
4060 : // Otherwise use PACKED_ELEMENTS for the target's elements kind.
4061 56 : var_elements_kind = Int32Constant(PACKED_ELEMENTS);
4062 56 : Goto(&allocate_jsarray);
4063 : }
4064 :
4065 336 : BIND(&allocate_jsarray);
4066 : // Use the cannonical map for the chosen elements kind.
4067 336 : Node* native_context = LoadNativeContext(context);
4068 : TNode<Map> array_map =
4069 336 : LoadJSArrayElementsMap(var_elements_kind.value(), native_context);
4070 :
4071 672 : TNode<Object> result = AllocateJSArray(
4072 1008 : array_map, CAST(var_new_elements.value()), CAST(length), allocation_site);
4073 672 : return result;
4074 : }
4075 :
4076 11820 : TNode<FixedArrayBase> CodeStubAssembler::AllocateFixedArray(
4077 : ElementsKind kind, Node* capacity, ParameterMode mode,
4078 : AllocationFlags flags, SloppyTNode<Map> fixed_array_map) {
4079 11820 : Comment("AllocateFixedArray");
4080 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, mode));
4081 : CSA_ASSERT(this, IntPtrOrSmiGreaterThan(capacity,
4082 : IntPtrOrSmiConstant(0, mode), mode));
4083 :
4084 11820 : const intptr_t kMaxLength = IsDoubleElementsKind(kind)
4085 : ? FixedDoubleArray::kMaxLength
4086 11820 : : FixedArray::kMaxLength;
4087 : intptr_t capacity_constant;
4088 11820 : if (ToParameterConstant(capacity, &capacity_constant, mode)) {
4089 1296 : CHECK_LE(capacity_constant, kMaxLength);
4090 : } else {
4091 22344 : Label if_out_of_memory(this, Label::kDeferred), next(this);
4092 22344 : Branch(IntPtrOrSmiGreaterThan(
4093 : capacity,
4094 : IntPtrOrSmiConstant(static_cast<int>(kMaxLength), mode), mode),
4095 11172 : &if_out_of_memory, &next);
4096 :
4097 11172 : BIND(&if_out_of_memory);
4098 : CallRuntime(Runtime::kFatalProcessOutOfMemoryInvalidArrayLength,
4099 11172 : NoContextConstant());
4100 11172 : Unreachable();
4101 :
4102 11172 : BIND(&next);
4103 : }
4104 :
4105 11820 : TNode<IntPtrT> total_size = GetFixedArrayAllocationSize(capacity, kind, mode);
4106 :
4107 11820 : if (IsDoubleElementsKind(kind)) flags |= kDoubleAlignment;
4108 : // Allocate both array and elements object, and initialize the JSArray.
4109 11820 : Node* array = Allocate(total_size, flags);
4110 11820 : if (fixed_array_map != nullptr) {
4111 : // Conservatively only skip the write barrier if there are no allocation
4112 : // flags, this ensures that the object hasn't ended up in LOS. Note that the
4113 : // fixed array map is currently always immortal and technically wouldn't
4114 : // need the write barrier even in LOS, but it's better to not take chances
4115 : // in case this invariant changes later, since it's difficult to enforce
4116 : // locally here.
4117 3820 : if (flags == CodeStubAssembler::kNone) {
4118 224 : StoreMapNoWriteBarrier(array, fixed_array_map);
4119 : } else {
4120 3596 : StoreMap(array, fixed_array_map);
4121 : }
4122 : } else {
4123 8000 : RootIndex map_index = IsDoubleElementsKind(kind)
4124 : ? RootIndex::kFixedDoubleArrayMap
4125 8000 : : RootIndex::kFixedArrayMap;
4126 : DCHECK(RootsTable::IsImmortalImmovable(map_index));
4127 8000 : StoreMapNoWriteBarrier(array, map_index);
4128 : }
4129 11820 : StoreObjectFieldNoWriteBarrier(array, FixedArray::kLengthOffset,
4130 23640 : ParameterToTagged(capacity, mode));
4131 11820 : return UncheckedCast<FixedArray>(array);
4132 : }
4133 :
4134 2836 : TNode<FixedArray> CodeStubAssembler::ExtractToFixedArray(
4135 : Node* source, Node* first, Node* count, Node* capacity, Node* source_map,
4136 : ElementsKind from_kind, AllocationFlags allocation_flags,
4137 : ExtractFixedArrayFlags extract_flags, ParameterMode parameter_mode,
4138 : HoleConversionMode convert_holes, TVariable<BoolT>* var_holes_converted,
4139 : Node* source_elements_kind) {
4140 : DCHECK_NE(first, nullptr);
4141 : DCHECK_NE(count, nullptr);
4142 : DCHECK_NE(capacity, nullptr);
4143 : DCHECK(extract_flags & ExtractFixedArrayFlag::kFixedArrays);
4144 : CSA_ASSERT(this,
4145 : WordNotEqual(IntPtrOrSmiConstant(0, parameter_mode), capacity));
4146 : CSA_ASSERT(this, WordEqual(source_map, LoadMap(source)));
4147 :
4148 5672 : VARIABLE(var_result, MachineRepresentation::kTagged);
4149 5672 : VARIABLE(var_target_map, MachineRepresentation::kTagged, source_map);
4150 :
4151 5672 : Label done(this, {&var_result}), is_cow(this),
4152 5672 : new_space_check(this, {&var_target_map});
4153 :
4154 : // If source_map is either FixedDoubleArrayMap, or FixedCOWArrayMap but
4155 : // we can't just use COW, use FixedArrayMap as the target map. Otherwise, use
4156 : // source_map as the target map.
4157 2836 : if (IsDoubleElementsKind(from_kind)) {
4158 : CSA_ASSERT(this, IsFixedDoubleArrayMap(source_map));
4159 56 : var_target_map.Bind(LoadRoot(RootIndex::kFixedArrayMap));
4160 56 : Goto(&new_space_check);
4161 : } else {
4162 : CSA_ASSERT(this, Word32BinaryNot(IsFixedDoubleArrayMap(source_map)));
4163 5560 : Branch(WordEqual(var_target_map.value(),
4164 2780 : LoadRoot(RootIndex::kFixedCOWArrayMap)),
4165 2780 : &is_cow, &new_space_check);
4166 :
4167 2780 : BIND(&is_cow);
4168 : {
4169 : // |source| is a COW array, so we don't actually need to allocate a new
4170 : // array unless:
4171 : // 1) |extract_flags| forces us to, or
4172 : // 2) we're asked to extract only part of the |source| (|first| != 0).
4173 2780 : if (extract_flags & ExtractFixedArrayFlag::kDontCopyCOW) {
4174 1280 : Branch(WordNotEqual(IntPtrOrSmiConstant(0, parameter_mode), first),
4175 632 : &new_space_check, [&] {
4176 632 : var_result.Bind(source);
4177 632 : Goto(&done);
4178 1272 : });
4179 : } else {
4180 2140 : var_target_map.Bind(LoadRoot(RootIndex::kFixedArrayMap));
4181 2140 : Goto(&new_space_check);
4182 : }
4183 : }
4184 : }
4185 :
4186 2836 : BIND(&new_space_check);
4187 : {
4188 2836 : bool handle_old_space = !FLAG_young_generation_large_objects;
4189 2836 : if (handle_old_space) {
4190 0 : if (extract_flags & ExtractFixedArrayFlag::kNewSpaceAllocationOnly) {
4191 0 : handle_old_space = false;
4192 : CSA_ASSERT(this, Word32BinaryNot(FixedArraySizeDoesntFitInNewSpace(
4193 : count, FixedArray::kHeaderSize, parameter_mode)));
4194 : } else {
4195 : int constant_count;
4196 : handle_old_space =
4197 0 : !TryGetIntPtrOrSmiConstantValue(count, &constant_count,
4198 0 : parameter_mode) ||
4199 0 : (constant_count >
4200 0 : FixedArray::GetMaxLengthForNewSpaceAllocation(PACKED_ELEMENTS));
4201 : }
4202 : }
4203 :
4204 5672 : Label old_space(this, Label::kDeferred);
4205 2836 : if (handle_old_space) {
4206 : GotoIfFixedArraySizeDoesntFitInNewSpace(
4207 0 : capacity, &old_space, FixedArray::kHeaderSize, parameter_mode);
4208 : }
4209 :
4210 2836 : Comment("Copy FixedArray in young generation");
4211 : // We use PACKED_ELEMENTS to tell AllocateFixedArray and
4212 : // CopyFixedArrayElements that we want a FixedArray.
4213 2836 : const ElementsKind to_kind = PACKED_ELEMENTS;
4214 : TNode<FixedArrayBase> to_elements =
4215 : AllocateFixedArray(to_kind, capacity, parameter_mode, allocation_flags,
4216 2836 : var_target_map.value());
4217 2836 : var_result.Bind(to_elements);
4218 :
4219 : #ifdef DEBUG
4220 : TNode<IntPtrT> object_word = BitcastTaggedToWord(to_elements);
4221 : TNode<IntPtrT> object_page = PageFromAddress(object_word);
4222 : TNode<IntPtrT> page_flags =
4223 : UncheckedCast<IntPtrT>(Load(MachineType::IntPtr(), object_page,
4224 : IntPtrConstant(Page::kFlagsOffset)));
4225 : CSA_ASSERT(
4226 : this,
4227 : WordNotEqual(
4228 : WordAnd(page_flags,
4229 : IntPtrConstant(MemoryChunk::kIsInYoungGenerationMask)),
4230 : IntPtrConstant(0)));
4231 : #endif
4232 :
4233 5560 : if (convert_holes == HoleConversionMode::kDontConvert &&
4234 2724 : !IsDoubleElementsKind(from_kind)) {
4235 : // We can use CopyElements (memcpy) because we don't need to replace or
4236 : // convert any values. Since {to_elements} is in new-space, CopyElements
4237 : // will efficiently use memcpy.
4238 2724 : FillFixedArrayWithValue(to_kind, to_elements, count, capacity,
4239 2724 : RootIndex::kTheHoleValue, parameter_mode);
4240 5448 : CopyElements(to_kind, to_elements, IntPtrConstant(0), CAST(source),
4241 : ParameterToIntPtr(first, parameter_mode),
4242 : ParameterToIntPtr(count, parameter_mode),
4243 2724 : SKIP_WRITE_BARRIER);
4244 : } else {
4245 224 : CopyFixedArrayElements(from_kind, source, to_kind, to_elements, first,
4246 : count, capacity, SKIP_WRITE_BARRIER,
4247 : parameter_mode, convert_holes,
4248 224 : var_holes_converted);
4249 : }
4250 2836 : Goto(&done);
4251 :
4252 2836 : if (handle_old_space) {
4253 0 : BIND(&old_space);
4254 : {
4255 0 : Comment("Copy FixedArray in old generation");
4256 0 : Label copy_one_by_one(this);
4257 :
4258 : // Try to use memcpy if we don't need to convert holes to undefined.
4259 0 : if (convert_holes == HoleConversionMode::kDontConvert &&
4260 : source_elements_kind != nullptr) {
4261 : // Only try memcpy if we're not copying object pointers.
4262 0 : GotoIfNot(IsFastSmiElementsKind(source_elements_kind),
4263 0 : ©_one_by_one);
4264 :
4265 0 : const ElementsKind to_smi_kind = PACKED_SMI_ELEMENTS;
4266 0 : to_elements =
4267 : AllocateFixedArray(to_smi_kind, capacity, parameter_mode,
4268 0 : allocation_flags, var_target_map.value());
4269 0 : var_result.Bind(to_elements);
4270 :
4271 0 : FillFixedArrayWithValue(to_smi_kind, to_elements, count, capacity,
4272 0 : RootIndex::kTheHoleValue, parameter_mode);
4273 : // CopyElements will try to use memcpy if it's not conflicting with
4274 : // GC. Otherwise it will copy elements by elements, but skip write
4275 : // barriers (since we're copying smis to smis).
4276 0 : CopyElements(to_smi_kind, to_elements, IntPtrConstant(0),
4277 0 : CAST(source), ParameterToIntPtr(first, parameter_mode),
4278 : ParameterToIntPtr(count, parameter_mode),
4279 0 : SKIP_WRITE_BARRIER);
4280 0 : Goto(&done);
4281 : } else {
4282 0 : Goto(©_one_by_one);
4283 : }
4284 :
4285 0 : BIND(©_one_by_one);
4286 : {
4287 0 : to_elements =
4288 : AllocateFixedArray(to_kind, capacity, parameter_mode,
4289 0 : allocation_flags, var_target_map.value());
4290 0 : var_result.Bind(to_elements);
4291 0 : CopyFixedArrayElements(from_kind, source, to_kind, to_elements, first,
4292 : count, capacity, UPDATE_WRITE_BARRIER,
4293 : parameter_mode, convert_holes,
4294 0 : var_holes_converted);
4295 0 : Goto(&done);
4296 : }
4297 : }
4298 : }
4299 : }
4300 :
4301 2836 : BIND(&done);
4302 5672 : return UncheckedCast<FixedArray>(var_result.value());
4303 : }
4304 :
4305 56 : TNode<FixedArrayBase> CodeStubAssembler::ExtractFixedDoubleArrayFillingHoles(
4306 : Node* from_array, Node* first, Node* count, Node* capacity,
4307 : Node* fixed_array_map, TVariable<BoolT>* var_holes_converted,
4308 : AllocationFlags allocation_flags, ExtractFixedArrayFlags extract_flags,
4309 : ParameterMode mode) {
4310 : DCHECK_NE(first, nullptr);
4311 : DCHECK_NE(count, nullptr);
4312 : DCHECK_NE(capacity, nullptr);
4313 : DCHECK_NE(var_holes_converted, nullptr);
4314 : CSA_ASSERT(this, IsFixedDoubleArrayMap(fixed_array_map));
4315 :
4316 112 : VARIABLE(var_result, MachineRepresentation::kTagged);
4317 56 : const ElementsKind kind = PACKED_DOUBLE_ELEMENTS;
4318 112 : Node* to_elements = AllocateFixedArray(kind, capacity, mode, allocation_flags,
4319 168 : fixed_array_map);
4320 56 : var_result.Bind(to_elements);
4321 : // We first try to copy the FixedDoubleArray to a new FixedDoubleArray.
4322 : // |var_holes_converted| is set to False preliminarily.
4323 56 : *var_holes_converted = Int32FalseConstant();
4324 :
4325 : // The construction of the loop and the offsets for double elements is
4326 : // extracted from CopyFixedArrayElements.
4327 : CSA_SLOW_ASSERT(this, MatchesParameterMode(count, mode));
4328 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, mode));
4329 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(from_array, kind));
4330 : STATIC_ASSERT(FixedArray::kHeaderSize == FixedDoubleArray::kHeaderSize);
4331 :
4332 56 : Comment("[ ExtractFixedDoubleArrayFillingHoles");
4333 :
4334 : // This copy can trigger GC, so we pre-initialize the array with holes.
4335 56 : FillFixedArrayWithValue(kind, to_elements, IntPtrOrSmiConstant(0, mode),
4336 56 : capacity, RootIndex::kTheHoleValue, mode);
4337 :
4338 56 : const int first_element_offset = FixedArray::kHeaderSize - kHeapObjectTag;
4339 : Node* first_from_element_offset =
4340 56 : ElementOffsetFromIndex(first, kind, mode, 0);
4341 112 : Node* limit_offset = IntPtrAdd(first_from_element_offset,
4342 168 : IntPtrConstant(first_element_offset));
4343 112 : VARIABLE(var_from_offset, MachineType::PointerRepresentation(),
4344 : ElementOffsetFromIndex(IntPtrOrSmiAdd(first, count, mode), kind,
4345 : mode, first_element_offset));
4346 :
4347 112 : Label decrement(this, {&var_from_offset}), done(this);
4348 : Node* to_array_adjusted =
4349 56 : IntPtrSub(BitcastTaggedToWord(to_elements), first_from_element_offset);
4350 :
4351 56 : Branch(WordEqual(var_from_offset.value(), limit_offset), &done, &decrement);
4352 :
4353 56 : BIND(&decrement);
4354 : {
4355 : Node* from_offset =
4356 56 : IntPtrSub(var_from_offset.value(), IntPtrConstant(kDoubleSize));
4357 56 : var_from_offset.Bind(from_offset);
4358 :
4359 56 : Node* to_offset = from_offset;
4360 :
4361 112 : Label if_hole(this);
4362 :
4363 56 : Node* value = LoadElementAndPrepareForStore(
4364 56 : from_array, var_from_offset.value(), kind, kind, &if_hole);
4365 :
4366 56 : StoreNoWriteBarrier(MachineRepresentation::kFloat64, to_array_adjusted,
4367 56 : to_offset, value);
4368 :
4369 56 : Node* compare = WordNotEqual(from_offset, limit_offset);
4370 56 : Branch(compare, &decrement, &done);
4371 :
4372 56 : BIND(&if_hole);
4373 : // We are unlucky: there are holes! We need to restart the copy, this time
4374 : // we will copy the FixedDoubleArray to a new FixedArray with undefined
4375 : // replacing holes. We signal this to the caller through
4376 : // |var_holes_converted|.
4377 56 : *var_holes_converted = Int32TrueConstant();
4378 : to_elements =
4379 112 : ExtractToFixedArray(from_array, first, count, capacity, fixed_array_map,
4380 : kind, allocation_flags, extract_flags, mode,
4381 56 : HoleConversionMode::kConvertToUndefined);
4382 56 : var_result.Bind(to_elements);
4383 56 : Goto(&done);
4384 : }
4385 :
4386 56 : BIND(&done);
4387 56 : Comment("] ExtractFixedDoubleArrayFillingHoles");
4388 112 : return UncheckedCast<FixedArrayBase>(var_result.value());
4389 : }
4390 :
4391 2780 : TNode<FixedArrayBase> CodeStubAssembler::ExtractFixedArray(
4392 : Node* source, Node* first, Node* count, Node* capacity,
4393 : ExtractFixedArrayFlags extract_flags, ParameterMode parameter_mode,
4394 : TVariable<BoolT>* var_holes_converted, Node* source_runtime_kind) {
4395 : DCHECK(extract_flags & ExtractFixedArrayFlag::kFixedArrays ||
4396 : extract_flags & ExtractFixedArrayFlag::kFixedDoubleArrays);
4397 : // If we want to replace holes, ExtractFixedArrayFlag::kDontCopyCOW should not
4398 : // be used, because that disables the iteration which detects holes.
4399 : DCHECK_IMPLIES(var_holes_converted != nullptr,
4400 : !(extract_flags & ExtractFixedArrayFlag::kDontCopyCOW));
4401 : HoleConversionMode convert_holes =
4402 : var_holes_converted != nullptr ? HoleConversionMode::kConvertToUndefined
4403 2780 : : HoleConversionMode::kDontConvert;
4404 5560 : VARIABLE(var_result, MachineRepresentation::kTagged);
4405 : const AllocationFlags allocation_flags =
4406 5560 : (extract_flags & ExtractFixedArrayFlag::kNewSpaceAllocationOnly)
4407 : ? CodeStubAssembler::kNone
4408 2780 : : CodeStubAssembler::kAllowLargeObjectAllocation;
4409 2780 : if (first == nullptr) {
4410 560 : first = IntPtrOrSmiConstant(0, parameter_mode);
4411 : }
4412 2780 : if (count == nullptr) {
4413 704 : count = IntPtrOrSmiSub(
4414 704 : TaggedToParameter(LoadFixedArrayBaseLength(source), parameter_mode),
4415 352 : first, parameter_mode);
4416 :
4417 : CSA_ASSERT(
4418 : this, IntPtrOrSmiLessThanOrEqual(IntPtrOrSmiConstant(0, parameter_mode),
4419 : count, parameter_mode));
4420 : }
4421 2780 : if (capacity == nullptr) {
4422 820 : capacity = count;
4423 : } else {
4424 : CSA_ASSERT(this, Word32BinaryNot(IntPtrOrSmiGreaterThan(
4425 : IntPtrOrSmiAdd(first, count, parameter_mode), capacity,
4426 : parameter_mode)));
4427 : }
4428 :
4429 5560 : Label if_fixed_double_array(this), empty(this), done(this, {&var_result});
4430 2780 : Node* source_map = LoadMap(source);
4431 2780 : GotoIf(WordEqual(IntPtrOrSmiConstant(0, parameter_mode), capacity), &empty);
4432 :
4433 2780 : if (extract_flags & ExtractFixedArrayFlag::kFixedDoubleArrays) {
4434 872 : if (extract_flags & ExtractFixedArrayFlag::kFixedArrays) {
4435 872 : GotoIf(IsFixedDoubleArrayMap(source_map), &if_fixed_double_array);
4436 : } else {
4437 : CSA_ASSERT(this, IsFixedDoubleArrayMap(source_map));
4438 : }
4439 : }
4440 :
4441 2780 : if (extract_flags & ExtractFixedArrayFlag::kFixedArrays) {
4442 : // Here we can only get |source| as FixedArray, never FixedDoubleArray.
4443 : // PACKED_ELEMENTS is used to signify that the source is a FixedArray.
4444 5560 : Node* to_elements = ExtractToFixedArray(
4445 : source, first, count, capacity, source_map, PACKED_ELEMENTS,
4446 : allocation_flags, extract_flags, parameter_mode, convert_holes,
4447 2780 : var_holes_converted, source_runtime_kind);
4448 2780 : var_result.Bind(to_elements);
4449 2780 : Goto(&done);
4450 : }
4451 :
4452 2780 : if (extract_flags & ExtractFixedArrayFlag::kFixedDoubleArrays) {
4453 872 : BIND(&if_fixed_double_array);
4454 872 : Comment("Copy FixedDoubleArray");
4455 :
4456 872 : if (convert_holes == HoleConversionMode::kConvertToUndefined) {
4457 112 : Node* to_elements = ExtractFixedDoubleArrayFillingHoles(
4458 : source, first, count, capacity, source_map, var_holes_converted,
4459 56 : allocation_flags, extract_flags, parameter_mode);
4460 56 : var_result.Bind(to_elements);
4461 : } else {
4462 : // We use PACKED_DOUBLE_ELEMENTS to signify that both the source and
4463 : // the target are FixedDoubleArray. That it is PACKED or HOLEY does not
4464 : // matter.
4465 816 : ElementsKind kind = PACKED_DOUBLE_ELEMENTS;
4466 : TNode<FixedArrayBase> to_elements = AllocateFixedArray(
4467 816 : kind, capacity, parameter_mode, allocation_flags, source_map);
4468 816 : FillFixedArrayWithValue(kind, to_elements, count, capacity,
4469 816 : RootIndex::kTheHoleValue, parameter_mode);
4470 1632 : CopyElements(kind, to_elements, IntPtrConstant(0), CAST(source),
4471 : ParameterToIntPtr(first, parameter_mode),
4472 816 : ParameterToIntPtr(count, parameter_mode));
4473 816 : var_result.Bind(to_elements);
4474 : }
4475 :
4476 872 : Goto(&done);
4477 : }
4478 :
4479 2780 : BIND(&empty);
4480 : {
4481 2780 : Comment("Copy empty array");
4482 :
4483 2780 : var_result.Bind(EmptyFixedArrayConstant());
4484 2780 : Goto(&done);
4485 : }
4486 :
4487 2780 : BIND(&done);
4488 5560 : return UncheckedCast<FixedArray>(var_result.value());
4489 : }
4490 :
4491 504 : void CodeStubAssembler::InitializePropertyArrayLength(Node* property_array,
4492 : Node* length,
4493 : ParameterMode mode) {
4494 : CSA_SLOW_ASSERT(this, IsPropertyArray(property_array));
4495 : CSA_ASSERT(
4496 : this, IntPtrOrSmiGreaterThan(length, IntPtrOrSmiConstant(0, mode), mode));
4497 : CSA_ASSERT(
4498 : this,
4499 : IntPtrOrSmiLessThanOrEqual(
4500 : length, IntPtrOrSmiConstant(PropertyArray::LengthField::kMax, mode),
4501 : mode));
4502 504 : StoreObjectFieldNoWriteBarrier(
4503 : property_array, PropertyArray::kLengthAndHashOffset,
4504 1008 : ParameterToTagged(length, mode), MachineRepresentation::kTaggedSigned);
4505 504 : }
4506 :
4507 504 : Node* CodeStubAssembler::AllocatePropertyArray(Node* capacity_node,
4508 : ParameterMode mode,
4509 : AllocationFlags flags) {
4510 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity_node, mode));
4511 : CSA_ASSERT(this, IntPtrOrSmiGreaterThan(capacity_node,
4512 : IntPtrOrSmiConstant(0, mode), mode));
4513 : TNode<IntPtrT> total_size =
4514 504 : GetPropertyArrayAllocationSize(capacity_node, mode);
4515 :
4516 504 : TNode<Object> array = Allocate(total_size, flags);
4517 504 : RootIndex map_index = RootIndex::kPropertyArrayMap;
4518 : DCHECK(RootsTable::IsImmortalImmovable(map_index));
4519 504 : StoreMapNoWriteBarrier(array, map_index);
4520 504 : InitializePropertyArrayLength(array, capacity_node, mode);
4521 504 : return array;
4522 : }
4523 :
4524 504 : void CodeStubAssembler::FillPropertyArrayWithUndefined(Node* array,
4525 : Node* from_node,
4526 : Node* to_node,
4527 : ParameterMode mode) {
4528 : CSA_SLOW_ASSERT(this, MatchesParameterMode(from_node, mode));
4529 : CSA_SLOW_ASSERT(this, MatchesParameterMode(to_node, mode));
4530 : CSA_SLOW_ASSERT(this, IsPropertyArray(array));
4531 504 : ElementsKind kind = PACKED_ELEMENTS;
4532 504 : Node* value = UndefinedConstant();
4533 1008 : BuildFastFixedArrayForEach(array, kind, from_node, to_node,
4534 504 : [this, value](Node* array, Node* offset) {
4535 504 : StoreNoWriteBarrier(
4536 : MachineRepresentation::kTagged, array,
4537 504 : offset, value);
4538 504 : },
4539 504 : mode);
4540 504 : }
4541 :
4542 12424 : void CodeStubAssembler::FillFixedArrayWithValue(ElementsKind kind, Node* array,
4543 : Node* from_node, Node* to_node,
4544 : RootIndex value_root_index,
4545 : ParameterMode mode) {
4546 : CSA_SLOW_ASSERT(this, MatchesParameterMode(from_node, mode));
4547 : CSA_SLOW_ASSERT(this, MatchesParameterMode(to_node, mode));
4548 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKind(array, kind));
4549 : DCHECK(value_root_index == RootIndex::kTheHoleValue ||
4550 : value_root_index == RootIndex::kUndefinedValue);
4551 :
4552 : // Determine the value to initialize the {array} based
4553 : // on the {value_root_index} and the elements {kind}.
4554 12424 : Node* value = LoadRoot(value_root_index);
4555 12424 : if (IsDoubleElementsKind(kind)) {
4556 2452 : value = LoadHeapNumberValue(value);
4557 : }
4558 :
4559 24848 : BuildFastFixedArrayForEach(
4560 : array, kind, from_node, to_node,
4561 39396 : [this, value, kind](Node* array, Node* offset) {
4562 26264 : if (IsDoubleElementsKind(kind)) {
4563 2780 : StoreNoWriteBarrier(MachineRepresentation::kFloat64, array, offset,
4564 2780 : value);
4565 : } else {
4566 10352 : StoreNoWriteBarrier(MachineRepresentation::kTagged, array, offset,
4567 10352 : value);
4568 : }
4569 13132 : },
4570 12424 : mode);
4571 12424 : }
4572 :
4573 112 : void CodeStubAssembler::StoreFixedDoubleArrayHole(
4574 : TNode<FixedDoubleArray> array, Node* index, ParameterMode parameter_mode) {
4575 : CSA_SLOW_ASSERT(this, MatchesParameterMode(index, parameter_mode));
4576 : Node* offset =
4577 224 : ElementOffsetFromIndex(index, PACKED_DOUBLE_ELEMENTS, parameter_mode,
4578 112 : FixedArray::kHeaderSize - kHeapObjectTag);
4579 : CSA_ASSERT(this, IsOffsetInBounds(
4580 : offset, LoadAndUntagFixedArrayBaseLength(array),
4581 : FixedDoubleArray::kHeaderSize, PACKED_DOUBLE_ELEMENTS));
4582 : Node* double_hole =
4583 560 : Is64() ? ReinterpretCast<UintPtrT>(Int64Constant(kHoleNanInt64))
4584 336 : : ReinterpretCast<UintPtrT>(Int32Constant(kHoleNanLower32));
4585 : // TODO(danno): When we have a Float32/Float64 wrapper class that
4586 : // preserves double bits during manipulation, remove this code/change
4587 : // this to an indexed Float64 store.
4588 112 : if (Is64()) {
4589 112 : StoreNoWriteBarrier(MachineRepresentation::kWord64, array, offset,
4590 112 : double_hole);
4591 : } else {
4592 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, array, offset,
4593 0 : double_hole);
4594 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, array,
4595 0 : IntPtrAdd(offset, IntPtrConstant(kInt32Size)),
4596 0 : double_hole);
4597 : }
4598 112 : }
4599 :
4600 1184 : void CodeStubAssembler::FillFixedArrayWithSmiZero(TNode<FixedArray> array,
4601 : TNode<IntPtrT> length) {
4602 : CSA_ASSERT(this, WordEqual(length, LoadAndUntagFixedArrayBaseLength(array)));
4603 :
4604 1184 : TNode<IntPtrT> byte_length = TimesTaggedSize(length);
4605 : CSA_ASSERT(this, UintPtrLessThan(length, byte_length));
4606 :
4607 : static const int32_t fa_base_data_offset =
4608 : FixedArray::kHeaderSize - kHeapObjectTag;
4609 : TNode<IntPtrT> backing_store = IntPtrAdd(BitcastTaggedToWord(array),
4610 1184 : IntPtrConstant(fa_base_data_offset));
4611 :
4612 : // Call out to memset to perform initialization.
4613 : TNode<ExternalReference> memset =
4614 1184 : ExternalConstant(ExternalReference::libc_memset_function());
4615 : STATIC_ASSERT(kSizetSize == kIntptrSize);
4616 2368 : CallCFunction3(MachineType::Pointer(), MachineType::Pointer(),
4617 : MachineType::IntPtr(), MachineType::UintPtr(), memset,
4618 2368 : backing_store, IntPtrConstant(0), byte_length);
4619 1184 : }
4620 :
4621 56 : void CodeStubAssembler::FillFixedDoubleArrayWithZero(
4622 : TNode<FixedDoubleArray> array, TNode<IntPtrT> length) {
4623 : CSA_ASSERT(this, WordEqual(length, LoadAndUntagFixedArrayBaseLength(array)));
4624 :
4625 56 : TNode<IntPtrT> byte_length = TimesDoubleSize(length);
4626 : CSA_ASSERT(this, UintPtrLessThan(length, byte_length));
4627 :
4628 : static const int32_t fa_base_data_offset =
4629 : FixedDoubleArray::kHeaderSize - kHeapObjectTag;
4630 : TNode<IntPtrT> backing_store = IntPtrAdd(BitcastTaggedToWord(array),
4631 56 : IntPtrConstant(fa_base_data_offset));
4632 :
4633 : // Call out to memset to perform initialization.
4634 : TNode<ExternalReference> memset =
4635 56 : ExternalConstant(ExternalReference::libc_memset_function());
4636 : STATIC_ASSERT(kSizetSize == kIntptrSize);
4637 112 : CallCFunction3(MachineType::Pointer(), MachineType::Pointer(),
4638 : MachineType::IntPtr(), MachineType::UintPtr(), memset,
4639 112 : backing_store, IntPtrConstant(0), byte_length);
4640 56 : }
4641 :
4642 3116 : void CodeStubAssembler::JumpIfPointersFromHereAreInteresting(
4643 : TNode<Object> object, Label* interesting) {
4644 6232 : Label finished(this);
4645 3116 : TNode<IntPtrT> object_word = BitcastTaggedToWord(object);
4646 3116 : TNode<IntPtrT> object_page = PageFromAddress(object_word);
4647 : TNode<IntPtrT> page_flags = UncheckedCast<IntPtrT>(Load(
4648 3116 : MachineType::IntPtr(), object_page, IntPtrConstant(Page::kFlagsOffset)));
4649 6232 : Branch(
4650 12464 : WordEqual(WordAnd(page_flags,
4651 : IntPtrConstant(
4652 3116 : MemoryChunk::kPointersFromHereAreInterestingMask)),
4653 12464 : IntPtrConstant(0)),
4654 3116 : &finished, interesting);
4655 3116 : BIND(&finished);
4656 3116 : }
4657 :
4658 392 : void CodeStubAssembler::MoveElements(ElementsKind kind,
4659 : TNode<FixedArrayBase> elements,
4660 : TNode<IntPtrT> dst_index,
4661 : TNode<IntPtrT> src_index,
4662 : TNode<IntPtrT> length) {
4663 784 : Label finished(this);
4664 784 : Label needs_barrier(this);
4665 392 : const bool needs_barrier_check = !IsDoubleElementsKind(kind);
4666 :
4667 : DCHECK(IsFastElementsKind(kind));
4668 : CSA_ASSERT(this, IsFixedArrayWithKind(elements, kind));
4669 : CSA_ASSERT(this,
4670 : IntPtrLessThanOrEqual(IntPtrAdd(dst_index, length),
4671 : LoadAndUntagFixedArrayBaseLength(elements)));
4672 : CSA_ASSERT(this,
4673 : IntPtrLessThanOrEqual(IntPtrAdd(src_index, length),
4674 : LoadAndUntagFixedArrayBaseLength(elements)));
4675 :
4676 : // The write barrier can be ignored if {dst_elements} is in new space, or if
4677 : // the elements pointer is FixedDoubleArray.
4678 392 : if (needs_barrier_check) {
4679 224 : JumpIfPointersFromHereAreInteresting(elements, &needs_barrier);
4680 : }
4681 :
4682 : const TNode<IntPtrT> source_byte_length =
4683 392 : IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind)));
4684 : static const int32_t fa_base_data_offset =
4685 : FixedArrayBase::kHeaderSize - kHeapObjectTag;
4686 392 : TNode<IntPtrT> elements_intptr = BitcastTaggedToWord(elements);
4687 : TNode<IntPtrT> target_data_ptr =
4688 : IntPtrAdd(elements_intptr,
4689 : ElementOffsetFromIndex(dst_index, kind, INTPTR_PARAMETERS,
4690 392 : fa_base_data_offset));
4691 : TNode<IntPtrT> source_data_ptr =
4692 : IntPtrAdd(elements_intptr,
4693 : ElementOffsetFromIndex(src_index, kind, INTPTR_PARAMETERS,
4694 392 : fa_base_data_offset));
4695 : TNode<ExternalReference> memmove =
4696 392 : ExternalConstant(ExternalReference::libc_memmove_function());
4697 392 : CallCFunction3(MachineType::Pointer(), MachineType::Pointer(),
4698 : MachineType::Pointer(), MachineType::UintPtr(), memmove,
4699 392 : target_data_ptr, source_data_ptr, source_byte_length);
4700 :
4701 392 : if (needs_barrier_check) {
4702 224 : Goto(&finished);
4703 :
4704 224 : BIND(&needs_barrier);
4705 : {
4706 224 : const TNode<IntPtrT> begin = src_index;
4707 224 : const TNode<IntPtrT> end = IntPtrAdd(begin, length);
4708 :
4709 : // If dst_index is less than src_index, then walk forward.
4710 : const TNode<IntPtrT> delta =
4711 : IntPtrMul(IntPtrSub(dst_index, begin),
4712 224 : IntPtrConstant(ElementsKindToByteSize(kind)));
4713 448 : auto loop_body = [&](Node* array, Node* offset) {
4714 1344 : Node* const element = Load(MachineType::AnyTagged(), array, offset);
4715 896 : Node* const delta_offset = IntPtrAdd(offset, delta);
4716 448 : Store(array, delta_offset, element);
4717 672 : };
4718 :
4719 448 : Label iterate_forward(this);
4720 448 : Label iterate_backward(this);
4721 448 : Branch(IntPtrLessThan(delta, IntPtrConstant(0)), &iterate_forward,
4722 224 : &iterate_backward);
4723 224 : BIND(&iterate_forward);
4724 : {
4725 : // Make a loop for the stores.
4726 448 : BuildFastFixedArrayForEach(elements, kind, begin, end, loop_body,
4727 : INTPTR_PARAMETERS,
4728 224 : ForEachDirection::kForward);
4729 224 : Goto(&finished);
4730 : }
4731 :
4732 224 : BIND(&iterate_backward);
4733 : {
4734 448 : BuildFastFixedArrayForEach(elements, kind, begin, end, loop_body,
4735 : INTPTR_PARAMETERS,
4736 224 : ForEachDirection::kReverse);
4737 224 : Goto(&finished);
4738 : }
4739 : }
4740 224 : BIND(&finished);
4741 : }
4742 392 : }
4743 :
4744 3764 : void CodeStubAssembler::CopyElements(ElementsKind kind,
4745 : TNode<FixedArrayBase> dst_elements,
4746 : TNode<IntPtrT> dst_index,
4747 : TNode<FixedArrayBase> src_elements,
4748 : TNode<IntPtrT> src_index,
4749 : TNode<IntPtrT> length,
4750 : WriteBarrierMode write_barrier) {
4751 7528 : Label finished(this);
4752 7528 : Label needs_barrier(this);
4753 3764 : const bool needs_barrier_check = !IsDoubleElementsKind(kind);
4754 :
4755 : DCHECK(IsFastElementsKind(kind));
4756 : CSA_ASSERT(this, IsFixedArrayWithKind(dst_elements, kind));
4757 : CSA_ASSERT(this, IsFixedArrayWithKind(src_elements, kind));
4758 : CSA_ASSERT(this, IntPtrLessThanOrEqual(
4759 : IntPtrAdd(dst_index, length),
4760 : LoadAndUntagFixedArrayBaseLength(dst_elements)));
4761 : CSA_ASSERT(this, IntPtrLessThanOrEqual(
4762 : IntPtrAdd(src_index, length),
4763 : LoadAndUntagFixedArrayBaseLength(src_elements)));
4764 : CSA_ASSERT(this, Word32Or(WordNotEqual(dst_elements, src_elements),
4765 : WordEqual(length, IntPtrConstant(0))));
4766 :
4767 : // The write barrier can be ignored if {dst_elements} is in new space, or if
4768 : // the elements pointer is FixedDoubleArray.
4769 3764 : if (needs_barrier_check) {
4770 2892 : JumpIfPointersFromHereAreInteresting(dst_elements, &needs_barrier);
4771 : }
4772 :
4773 : TNode<IntPtrT> source_byte_length =
4774 3764 : IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind)));
4775 : static const int32_t fa_base_data_offset =
4776 : FixedArrayBase::kHeaderSize - kHeapObjectTag;
4777 : TNode<IntPtrT> src_offset_start = ElementOffsetFromIndex(
4778 3764 : src_index, kind, INTPTR_PARAMETERS, fa_base_data_offset);
4779 : TNode<IntPtrT> dst_offset_start = ElementOffsetFromIndex(
4780 3764 : dst_index, kind, INTPTR_PARAMETERS, fa_base_data_offset);
4781 3764 : TNode<IntPtrT> src_elements_intptr = BitcastTaggedToWord(src_elements);
4782 : TNode<IntPtrT> source_data_ptr =
4783 3764 : IntPtrAdd(src_elements_intptr, src_offset_start);
4784 3764 : TNode<IntPtrT> dst_elements_intptr = BitcastTaggedToWord(dst_elements);
4785 : TNode<IntPtrT> dst_data_ptr =
4786 3764 : IntPtrAdd(dst_elements_intptr, dst_offset_start);
4787 : TNode<ExternalReference> memcpy =
4788 3764 : ExternalConstant(ExternalReference::libc_memcpy_function());
4789 3764 : CallCFunction3(MachineType::Pointer(), MachineType::Pointer(),
4790 : MachineType::Pointer(), MachineType::UintPtr(), memcpy,
4791 3764 : dst_data_ptr, source_data_ptr, source_byte_length);
4792 :
4793 3764 : if (needs_barrier_check) {
4794 2892 : Goto(&finished);
4795 :
4796 2892 : BIND(&needs_barrier);
4797 : {
4798 2892 : const TNode<IntPtrT> begin = src_index;
4799 2892 : const TNode<IntPtrT> end = IntPtrAdd(begin, length);
4800 : const TNode<IntPtrT> delta =
4801 : IntPtrMul(IntPtrSub(dst_index, src_index),
4802 2892 : IntPtrConstant(ElementsKindToByteSize(kind)));
4803 5784 : BuildFastFixedArrayForEach(
4804 : src_elements, kind, begin, end,
4805 2904 : [&](Node* array, Node* offset) {
4806 8712 : Node* const element = Load(MachineType::AnyTagged(), array, offset);
4807 5808 : Node* const delta_offset = IntPtrAdd(offset, delta);
4808 2904 : if (write_barrier == SKIP_WRITE_BARRIER) {
4809 5640 : StoreNoWriteBarrier(MachineRepresentation::kTagged, dst_elements,
4810 2736 : delta_offset, element);
4811 : } else {
4812 336 : Store(dst_elements, delta_offset, element);
4813 : }
4814 2904 : },
4815 2892 : INTPTR_PARAMETERS, ForEachDirection::kForward);
4816 2892 : Goto(&finished);
4817 : }
4818 2892 : BIND(&finished);
4819 : }
4820 3764 : }
4821 :
4822 6140 : void CodeStubAssembler::CopyFixedArrayElements(
4823 : ElementsKind from_kind, Node* from_array, ElementsKind to_kind,
4824 : Node* to_array, Node* first_element, Node* element_count, Node* capacity,
4825 : WriteBarrierMode barrier_mode, ParameterMode mode,
4826 : HoleConversionMode convert_holes, TVariable<BoolT>* var_holes_converted) {
4827 : DCHECK_IMPLIES(var_holes_converted != nullptr,
4828 : convert_holes == HoleConversionMode::kConvertToUndefined);
4829 : CSA_SLOW_ASSERT(this, MatchesParameterMode(element_count, mode));
4830 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, mode));
4831 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(from_array, from_kind));
4832 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(to_array, to_kind));
4833 : STATIC_ASSERT(FixedArray::kHeaderSize == FixedDoubleArray::kHeaderSize);
4834 6140 : const int first_element_offset = FixedArray::kHeaderSize - kHeapObjectTag;
4835 6140 : Comment("[ CopyFixedArrayElements");
4836 :
4837 : // Typed array elements are not supported.
4838 : DCHECK(!IsFixedTypedArrayElementsKind(from_kind));
4839 : DCHECK(!IsFixedTypedArrayElementsKind(to_kind));
4840 :
4841 12280 : Label done(this);
4842 6140 : bool from_double_elements = IsDoubleElementsKind(from_kind);
4843 6140 : bool to_double_elements = IsDoubleElementsKind(to_kind);
4844 : bool doubles_to_objects_conversion =
4845 6140 : IsDoubleElementsKind(from_kind) && IsObjectElementsKind(to_kind);
4846 : bool needs_write_barrier =
4847 7708 : doubles_to_objects_conversion ||
4848 6252 : (barrier_mode == UPDATE_WRITE_BARRIER && IsObjectElementsKind(to_kind));
4849 : bool element_offset_matches =
4850 10712 : !needs_write_barrier &&
4851 : (kTaggedSize == kDoubleSize ||
4852 10712 : IsDoubleElementsKind(from_kind) == IsDoubleElementsKind(to_kind));
4853 : Node* double_hole =
4854 30700 : Is64() ? ReinterpretCast<UintPtrT>(Int64Constant(kHoleNanInt64))
4855 18420 : : ReinterpretCast<UintPtrT>(Int32Constant(kHoleNanLower32));
4856 :
4857 : // If copying might trigger a GC, we pre-initialize the FixedArray such that
4858 : // it's always in a consistent state.
4859 6140 : if (convert_holes == HoleConversionMode::kConvertToUndefined) {
4860 : DCHECK(IsObjectElementsKind(to_kind));
4861 : // Use undefined for the part that we copy and holes for the rest.
4862 : // Later if we run into a hole in the source we can just skip the writing
4863 : // to the target and are still guaranteed that we get an undefined.
4864 112 : FillFixedArrayWithValue(to_kind, to_array, IntPtrOrSmiConstant(0, mode),
4865 112 : element_count, RootIndex::kUndefinedValue, mode);
4866 112 : FillFixedArrayWithValue(to_kind, to_array, element_count, capacity,
4867 112 : RootIndex::kTheHoleValue, mode);
4868 6028 : } else if (doubles_to_objects_conversion) {
4869 : // Pre-initialized the target with holes so later if we run into a hole in
4870 : // the source we can just skip the writing to the target.
4871 1400 : FillFixedArrayWithValue(to_kind, to_array, IntPtrOrSmiConstant(0, mode),
4872 1400 : capacity, RootIndex::kTheHoleValue, mode);
4873 4628 : } else if (element_count != capacity) {
4874 3788 : FillFixedArrayWithValue(to_kind, to_array, element_count, capacity,
4875 3788 : RootIndex::kTheHoleValue, mode);
4876 : }
4877 :
4878 : Node* first_from_element_offset =
4879 6140 : ElementOffsetFromIndex(first_element, from_kind, mode, 0);
4880 12280 : Node* limit_offset = IntPtrAdd(first_from_element_offset,
4881 18420 : IntPtrConstant(first_element_offset));
4882 12280 : VARIABLE(
4883 : var_from_offset, MachineType::PointerRepresentation(),
4884 : ElementOffsetFromIndex(IntPtrOrSmiAdd(first_element, element_count, mode),
4885 : from_kind, mode, first_element_offset));
4886 : // This second variable is used only when the element sizes of source and
4887 : // destination arrays do not match.
4888 12280 : VARIABLE(var_to_offset, MachineType::PointerRepresentation());
4889 6140 : if (element_offset_matches) {
4890 3396 : var_to_offset.Bind(var_from_offset.value());
4891 : } else {
4892 5488 : var_to_offset.Bind(ElementOffsetFromIndex(element_count, to_kind, mode,
4893 5488 : first_element_offset));
4894 : }
4895 :
4896 6140 : Variable* vars[] = {&var_from_offset, &var_to_offset, var_holes_converted};
4897 : int num_vars =
4898 6140 : var_holes_converted != nullptr ? arraysize(vars) : arraysize(vars) - 1;
4899 12280 : Label decrement(this, num_vars, vars);
4900 :
4901 : Node* to_array_adjusted =
4902 : element_offset_matches
4903 12932 : ? IntPtrSub(BitcastTaggedToWord(to_array), first_from_element_offset)
4904 12280 : : to_array;
4905 :
4906 6140 : Branch(WordEqual(var_from_offset.value(), limit_offset), &done, &decrement);
4907 :
4908 6140 : BIND(&decrement);
4909 : {
4910 12280 : Node* from_offset = IntPtrSub(
4911 : var_from_offset.value(),
4912 18420 : IntPtrConstant(from_double_elements ? kDoubleSize : kTaggedSize));
4913 6140 : var_from_offset.Bind(from_offset);
4914 :
4915 : Node* to_offset;
4916 6140 : if (element_offset_matches) {
4917 3396 : to_offset = from_offset;
4918 : } else {
4919 5488 : to_offset = IntPtrSub(
4920 : var_to_offset.value(),
4921 8232 : IntPtrConstant(to_double_elements ? kDoubleSize : kTaggedSize));
4922 2744 : var_to_offset.Bind(to_offset);
4923 : }
4924 :
4925 12280 : Label next_iter(this), store_double_hole(this), signal_hole(this);
4926 : Label* if_hole;
4927 6140 : if (convert_holes == HoleConversionMode::kConvertToUndefined) {
4928 : // The target elements array is already preinitialized with undefined
4929 : // so we only need to signal that a hole was found and continue the loop.
4930 112 : if_hole = &signal_hole;
4931 6028 : } else if (doubles_to_objects_conversion) {
4932 : // The target elements array is already preinitialized with holes, so we
4933 : // can just proceed with the next iteration.
4934 1400 : if_hole = &next_iter;
4935 4628 : } else if (IsDoubleElementsKind(to_kind)) {
4936 1748 : if_hole = &store_double_hole;
4937 : } else {
4938 : // In all the other cases don't check for holes and copy the data as is.
4939 2880 : if_hole = nullptr;
4940 : }
4941 :
4942 6140 : Node* value = LoadElementAndPrepareForStore(
4943 6140 : from_array, var_from_offset.value(), from_kind, to_kind, if_hole);
4944 :
4945 6140 : if (needs_write_barrier) {
4946 1568 : CHECK_EQ(to_array, to_array_adjusted);
4947 1568 : Store(to_array_adjusted, to_offset, value);
4948 4572 : } else if (to_double_elements) {
4949 1748 : StoreNoWriteBarrier(MachineRepresentation::kFloat64, to_array_adjusted,
4950 1748 : to_offset, value);
4951 : } else {
4952 2824 : StoreNoWriteBarrier(MachineRepresentation::kTagged, to_array_adjusted,
4953 2824 : to_offset, value);
4954 : }
4955 6140 : Goto(&next_iter);
4956 :
4957 6140 : if (if_hole == &store_double_hole) {
4958 1748 : BIND(&store_double_hole);
4959 : // Don't use doubles to store the hole double, since manipulating the
4960 : // signaling NaN used for the hole in C++, e.g. with bit_cast, will
4961 : // change its value on ia32 (the x87 stack is used to return values
4962 : // and stores to the stack silently clear the signalling bit).
4963 : //
4964 : // TODO(danno): When we have a Float32/Float64 wrapper class that
4965 : // preserves double bits during manipulation, remove this code/change
4966 : // this to an indexed Float64 store.
4967 1748 : if (Is64()) {
4968 1748 : StoreNoWriteBarrier(MachineRepresentation::kWord64, to_array_adjusted,
4969 1748 : to_offset, double_hole);
4970 : } else {
4971 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, to_array_adjusted,
4972 0 : to_offset, double_hole);
4973 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, to_array_adjusted,
4974 0 : IntPtrAdd(to_offset, IntPtrConstant(kInt32Size)),
4975 0 : double_hole);
4976 : }
4977 1748 : Goto(&next_iter);
4978 4392 : } else if (if_hole == &signal_hole) {
4979 : // This case happens only when IsObjectElementsKind(to_kind).
4980 112 : BIND(&signal_hole);
4981 112 : if (var_holes_converted != nullptr) {
4982 56 : *var_holes_converted = Int32TrueConstant();
4983 : }
4984 112 : Goto(&next_iter);
4985 : }
4986 :
4987 6140 : BIND(&next_iter);
4988 6140 : Node* compare = WordNotEqual(from_offset, limit_offset);
4989 6140 : Branch(compare, &decrement, &done);
4990 : }
4991 :
4992 6140 : BIND(&done);
4993 6140 : Comment("] CopyFixedArrayElements");
4994 6140 : }
4995 :
4996 1288 : TNode<FixedArray> CodeStubAssembler::HeapObjectToFixedArray(
4997 : TNode<HeapObject> base, Label* cast_fail) {
4998 2576 : Label fixed_array(this);
4999 1288 : TNode<Map> map = LoadMap(base);
5000 1288 : GotoIf(WordEqual(map, LoadRoot(RootIndex::kFixedArrayMap)), &fixed_array);
5001 1288 : GotoIf(WordNotEqual(map, LoadRoot(RootIndex::kFixedCOWArrayMap)), cast_fail);
5002 1288 : Goto(&fixed_array);
5003 1288 : BIND(&fixed_array);
5004 2576 : return UncheckedCast<FixedArray>(base);
5005 : }
5006 :
5007 504 : void CodeStubAssembler::CopyPropertyArrayValues(Node* from_array,
5008 : Node* to_array,
5009 : Node* property_count,
5010 : WriteBarrierMode barrier_mode,
5011 : ParameterMode mode,
5012 : DestroySource destroy_source) {
5013 : CSA_SLOW_ASSERT(this, MatchesParameterMode(property_count, mode));
5014 : CSA_SLOW_ASSERT(this, Word32Or(IsPropertyArray(from_array),
5015 : IsEmptyFixedArray(from_array)));
5016 : CSA_SLOW_ASSERT(this, IsPropertyArray(to_array));
5017 504 : Comment("[ CopyPropertyArrayValues");
5018 :
5019 504 : bool needs_write_barrier = barrier_mode == UPDATE_WRITE_BARRIER;
5020 :
5021 504 : if (destroy_source == DestroySource::kNo) {
5022 : // PropertyArray may contain MutableHeapNumbers, which will be cloned on the
5023 : // heap, requiring a write barrier.
5024 56 : needs_write_barrier = true;
5025 : }
5026 :
5027 504 : Node* start = IntPtrOrSmiConstant(0, mode);
5028 504 : ElementsKind kind = PACKED_ELEMENTS;
5029 1008 : BuildFastFixedArrayForEach(
5030 : from_array, kind, start, property_count,
5031 : [this, to_array, needs_write_barrier, destroy_source](Node* array,
5032 2632 : Node* offset) {
5033 1008 : Node* value = Load(MachineType::AnyTagged(), array, offset);
5034 :
5035 504 : if (destroy_source == DestroySource::kNo) {
5036 112 : value = CloneIfMutablePrimitive(CAST(value));
5037 : }
5038 :
5039 504 : if (needs_write_barrier) {
5040 56 : Store(to_array, offset, value);
5041 : } else {
5042 448 : StoreNoWriteBarrier(MachineRepresentation::kTagged, to_array, offset,
5043 448 : value);
5044 : }
5045 504 : },
5046 504 : mode);
5047 :
5048 : #ifdef DEBUG
5049 : // Zap {from_array} if the copying above has made it invalid.
5050 : if (destroy_source == DestroySource::kYes) {
5051 : Label did_zap(this);
5052 : GotoIf(IsEmptyFixedArray(from_array), &did_zap);
5053 : FillPropertyArrayWithUndefined(from_array, start, property_count, mode);
5054 :
5055 : Goto(&did_zap);
5056 : BIND(&did_zap);
5057 : }
5058 : #endif
5059 504 : Comment("] CopyPropertyArrayValues");
5060 504 : }
5061 :
5062 3432 : void CodeStubAssembler::CopyStringCharacters(Node* from_string, Node* to_string,
5063 : TNode<IntPtrT> from_index,
5064 : TNode<IntPtrT> to_index,
5065 : TNode<IntPtrT> character_count,
5066 : String::Encoding from_encoding,
5067 : String::Encoding to_encoding) {
5068 : // Cannot assert IsString(from_string) and IsString(to_string) here because
5069 : // CSA::SubString can pass in faked sequential strings when handling external
5070 : // subject strings.
5071 3432 : bool from_one_byte = from_encoding == String::ONE_BYTE_ENCODING;
5072 3432 : bool to_one_byte = to_encoding == String::ONE_BYTE_ENCODING;
5073 : DCHECK_IMPLIES(to_one_byte, from_one_byte);
5074 3432 : Comment("CopyStringCharacters ",
5075 : from_one_byte ? "ONE_BYTE_ENCODING" : "TWO_BYTE_ENCODING", " -> ",
5076 3432 : to_one_byte ? "ONE_BYTE_ENCODING" : "TWO_BYTE_ENCODING");
5077 :
5078 3432 : ElementsKind from_kind = from_one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS;
5079 3432 : ElementsKind to_kind = to_one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS;
5080 : STATIC_ASSERT(SeqOneByteString::kHeaderSize == SeqTwoByteString::kHeaderSize);
5081 3432 : int header_size = SeqOneByteString::kHeaderSize - kHeapObjectTag;
5082 6864 : Node* from_offset = ElementOffsetFromIndex(from_index, from_kind,
5083 6864 : INTPTR_PARAMETERS, header_size);
5084 : Node* to_offset =
5085 3432 : ElementOffsetFromIndex(to_index, to_kind, INTPTR_PARAMETERS, header_size);
5086 : Node* byte_count =
5087 3432 : ElementOffsetFromIndex(character_count, from_kind, INTPTR_PARAMETERS);
5088 3432 : Node* limit_offset = IntPtrAdd(from_offset, byte_count);
5089 :
5090 : // Prepare the fast loop
5091 : MachineType type =
5092 3432 : from_one_byte ? MachineType::Uint8() : MachineType::Uint16();
5093 : MachineRepresentation rep = to_one_byte ? MachineRepresentation::kWord8
5094 3432 : : MachineRepresentation::kWord16;
5095 3432 : int from_increment = 1 << ElementsKindToShiftSize(from_kind);
5096 3432 : int to_increment = 1 << ElementsKindToShiftSize(to_kind);
5097 :
5098 6864 : VARIABLE(current_to_offset, MachineType::PointerRepresentation(), to_offset);
5099 6864 : VariableList vars({¤t_to_offset}, zone());
5100 3432 : int to_index_constant = 0, from_index_constant = 0;
5101 7596 : bool index_same = (from_encoding == to_encoding) &&
5102 6744 : (from_index == to_index ||
5103 4952 : (ToInt32Constant(from_index, from_index_constant) &&
5104 2376 : ToInt32Constant(to_index, to_index_constant) &&
5105 4228 : from_index_constant == to_index_constant));
5106 6864 : BuildFastLoop(vars, from_offset, limit_offset,
5107 : [this, from_string, to_string, ¤t_to_offset, to_increment,
5108 19008 : type, rep, index_same](Node* offset) {
5109 3432 : Node* value = Load(type, from_string, offset);
5110 9504 : StoreNoWriteBarrier(
5111 : rep, to_string,
5112 3432 : index_same ? offset : current_to_offset.value(), value);
5113 3432 : if (!index_same) {
5114 2640 : Increment(¤t_to_offset, to_increment);
5115 : }
5116 3432 : },
5117 3432 : from_increment, INTPTR_PARAMETERS, IndexAdvanceMode::kPost);
5118 3432 : }
5119 :
5120 6196 : Node* CodeStubAssembler::LoadElementAndPrepareForStore(Node* array,
5121 : Node* offset,
5122 : ElementsKind from_kind,
5123 : ElementsKind to_kind,
5124 : Label* if_hole) {
5125 : CSA_ASSERT(this, IsFixedArrayWithKind(array, from_kind));
5126 6196 : if (IsDoubleElementsKind(from_kind)) {
5127 : Node* value =
5128 2084 : LoadDoubleWithHoleCheck(array, offset, if_hole, MachineType::Float64());
5129 2084 : if (!IsDoubleElementsKind(to_kind)) {
5130 1456 : value = AllocateHeapNumberWithValue(value);
5131 : }
5132 2084 : return value;
5133 :
5134 : } else {
5135 4112 : Node* value = Load(MachineType::AnyTagged(), array, offset);
5136 4112 : if (if_hole) {
5137 1232 : GotoIf(WordEqual(value, TheHoleConstant()), if_hole);
5138 : }
5139 4112 : if (IsDoubleElementsKind(to_kind)) {
5140 1176 : if (IsSmiElementsKind(from_kind)) {
5141 1176 : value = SmiToFloat64(value);
5142 : } else {
5143 0 : value = LoadHeapNumberValue(value);
5144 : }
5145 : }
5146 4112 : return value;
5147 : }
5148 : }
5149 :
5150 2564 : Node* CodeStubAssembler::CalculateNewElementsCapacity(Node* old_capacity,
5151 : ParameterMode mode) {
5152 : CSA_SLOW_ASSERT(this, MatchesParameterMode(old_capacity, mode));
5153 2564 : Node* half_old_capacity = WordOrSmiShr(old_capacity, 1, mode);
5154 2564 : Node* new_capacity = IntPtrOrSmiAdd(half_old_capacity, old_capacity, mode);
5155 : Node* padding =
5156 2564 : IntPtrOrSmiConstant(JSObject::kMinAddedElementsCapacity, mode);
5157 2564 : return IntPtrOrSmiAdd(new_capacity, padding, mode);
5158 : }
5159 :
5160 112 : Node* CodeStubAssembler::TryGrowElementsCapacity(Node* object, Node* elements,
5161 : ElementsKind kind, Node* key,
5162 : Label* bailout) {
5163 : CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object));
5164 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, kind));
5165 : CSA_SLOW_ASSERT(this, TaggedIsSmi(key));
5166 112 : Node* capacity = LoadFixedArrayBaseLength(elements);
5167 :
5168 112 : ParameterMode mode = OptimalParameterMode();
5169 112 : capacity = TaggedToParameter(capacity, mode);
5170 112 : key = TaggedToParameter(key, mode);
5171 :
5172 112 : return TryGrowElementsCapacity(object, elements, kind, key, capacity, mode,
5173 112 : bailout);
5174 : }
5175 :
5176 1120 : Node* CodeStubAssembler::TryGrowElementsCapacity(Node* object, Node* elements,
5177 : ElementsKind kind, Node* key,
5178 : Node* capacity,
5179 : ParameterMode mode,
5180 : Label* bailout) {
5181 1120 : Comment("TryGrowElementsCapacity");
5182 : CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object));
5183 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, kind));
5184 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, mode));
5185 : CSA_SLOW_ASSERT(this, MatchesParameterMode(key, mode));
5186 :
5187 : // If the gap growth is too big, fall back to the runtime.
5188 1120 : Node* max_gap = IntPtrOrSmiConstant(JSObject::kMaxGap, mode);
5189 1120 : Node* max_capacity = IntPtrOrSmiAdd(capacity, max_gap, mode);
5190 1120 : GotoIf(UintPtrOrSmiGreaterThanOrEqual(key, max_capacity, mode), bailout);
5191 :
5192 : // Calculate the capacity of the new backing store.
5193 1120 : Node* new_capacity = CalculateNewElementsCapacity(
5194 1120 : IntPtrOrSmiAdd(key, IntPtrOrSmiConstant(1, mode), mode), mode);
5195 1120 : return GrowElementsCapacity(object, elements, kind, kind, capacity,
5196 1120 : new_capacity, mode, bailout);
5197 : }
5198 :
5199 5356 : Node* CodeStubAssembler::GrowElementsCapacity(
5200 : Node* object, Node* elements, ElementsKind from_kind, ElementsKind to_kind,
5201 : Node* capacity, Node* new_capacity, ParameterMode mode, Label* bailout) {
5202 5356 : Comment("[ GrowElementsCapacity");
5203 : CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object));
5204 : CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, from_kind));
5205 : CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, mode));
5206 : CSA_SLOW_ASSERT(this, MatchesParameterMode(new_capacity, mode));
5207 :
5208 : // If size of the allocation for the new capacity doesn't fit in a page
5209 : // that we can bump-pointer allocate from, fall back to the runtime.
5210 5356 : int max_size = FixedArrayBase::GetMaxLengthForNewSpaceAllocation(to_kind);
5211 10712 : GotoIf(UintPtrOrSmiGreaterThanOrEqual(
5212 : new_capacity, IntPtrOrSmiConstant(max_size, mode), mode),
5213 5356 : bailout);
5214 :
5215 : // Allocate the new backing store.
5216 5356 : Node* new_elements = AllocateFixedArray(to_kind, new_capacity, mode);
5217 :
5218 : // Copy the elements from the old elements store to the new.
5219 : // The size-check above guarantees that the |new_elements| is allocated
5220 : // in new space so we can skip the write barrier.
5221 5356 : CopyFixedArrayElements(from_kind, elements, to_kind, new_elements, capacity,
5222 5356 : new_capacity, SKIP_WRITE_BARRIER, mode);
5223 :
5224 5356 : StoreObjectField(object, JSObject::kElementsOffset, new_elements);
5225 5356 : Comment("] GrowElementsCapacity");
5226 5356 : return new_elements;
5227 : }
5228 :
5229 1064 : void CodeStubAssembler::InitializeAllocationMemento(Node* base,
5230 : Node* base_allocation_size,
5231 : Node* allocation_site) {
5232 1064 : Comment("[Initialize AllocationMemento");
5233 : TNode<Object> memento =
5234 1064 : InnerAllocate(CAST(base), UncheckedCast<IntPtrT>(base_allocation_size));
5235 1064 : StoreMapNoWriteBarrier(memento, RootIndex::kAllocationMementoMap);
5236 1064 : StoreObjectFieldNoWriteBarrier(
5237 1064 : memento, AllocationMemento::kAllocationSiteOffset, allocation_site);
5238 1064 : if (FLAG_allocation_site_pretenuring) {
5239 : TNode<Int32T> count = UncheckedCast<Int32T>(LoadObjectField(
5240 : allocation_site, AllocationSite::kPretenureCreateCountOffset,
5241 1064 : MachineType::Int32()));
5242 :
5243 1064 : TNode<Int32T> incremented_count = Int32Add(count, Int32Constant(1));
5244 1064 : StoreObjectFieldNoWriteBarrier(
5245 : allocation_site, AllocationSite::kPretenureCreateCountOffset,
5246 1064 : incremented_count, MachineRepresentation::kWord32);
5247 : }
5248 1064 : Comment("]");
5249 1064 : }
5250 :
5251 3696 : Node* CodeStubAssembler::TryTaggedToFloat64(Node* value,
5252 : Label* if_valueisnotnumber) {
5253 7392 : Label out(this);
5254 7392 : VARIABLE(var_result, MachineRepresentation::kFloat64);
5255 :
5256 : // Check if the {value} is a Smi or a HeapObject.
5257 7392 : Label if_valueissmi(this), if_valueisnotsmi(this);
5258 3696 : Branch(TaggedIsSmi(value), &if_valueissmi, &if_valueisnotsmi);
5259 :
5260 3696 : BIND(&if_valueissmi);
5261 : {
5262 : // Convert the Smi {value}.
5263 3696 : var_result.Bind(SmiToFloat64(value));
5264 3696 : Goto(&out);
5265 : }
5266 :
5267 3696 : BIND(&if_valueisnotsmi);
5268 : {
5269 : // Check if {value} is a HeapNumber.
5270 7392 : Label if_valueisheapnumber(this);
5271 3696 : Branch(IsHeapNumber(value), &if_valueisheapnumber, if_valueisnotnumber);
5272 :
5273 3696 : BIND(&if_valueisheapnumber);
5274 : {
5275 : // Load the floating point value.
5276 3696 : var_result.Bind(LoadHeapNumberValue(value));
5277 3696 : Goto(&out);
5278 : }
5279 : }
5280 3696 : BIND(&out);
5281 7392 : return var_result.value();
5282 : }
5283 :
5284 1680 : Node* CodeStubAssembler::TruncateTaggedToFloat64(Node* context, Node* value) {
5285 : // We might need to loop once due to ToNumber conversion.
5286 3360 : VARIABLE(var_value, MachineRepresentation::kTagged);
5287 3360 : VARIABLE(var_result, MachineRepresentation::kFloat64);
5288 3360 : Label loop(this, &var_value), done_loop(this, &var_result);
5289 1680 : var_value.Bind(value);
5290 1680 : Goto(&loop);
5291 1680 : BIND(&loop);
5292 : {
5293 3360 : Label if_valueisnotnumber(this, Label::kDeferred);
5294 :
5295 : // Load the current {value}.
5296 1680 : value = var_value.value();
5297 :
5298 : // Convert {value} to Float64 if it is a number and convert it to a number
5299 : // otherwise.
5300 1680 : Node* const result = TryTaggedToFloat64(value, &if_valueisnotnumber);
5301 1680 : var_result.Bind(result);
5302 1680 : Goto(&done_loop);
5303 :
5304 1680 : BIND(&if_valueisnotnumber);
5305 : {
5306 : // Convert the {value} to a Number first.
5307 1680 : var_value.Bind(CallBuiltin(Builtins::kNonNumberToNumber, context, value));
5308 1680 : Goto(&loop);
5309 : }
5310 : }
5311 1680 : BIND(&done_loop);
5312 3360 : return var_result.value();
5313 : }
5314 :
5315 1400 : Node* CodeStubAssembler::TruncateTaggedToWord32(Node* context, Node* value) {
5316 2800 : VARIABLE(var_result, MachineRepresentation::kWord32);
5317 2800 : Label done(this);
5318 : TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumber>(context, value,
5319 1400 : &done, &var_result);
5320 1400 : BIND(&done);
5321 2800 : return var_result.value();
5322 : }
5323 :
5324 : // Truncate {value} to word32 and jump to {if_number} if it is a Number,
5325 : // or find that it is a BigInt and jump to {if_bigint}.
5326 672 : void CodeStubAssembler::TaggedToWord32OrBigInt(Node* context, Node* value,
5327 : Label* if_number,
5328 : Variable* var_word32,
5329 : Label* if_bigint,
5330 : Variable* var_bigint) {
5331 : TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumeric>(
5332 672 : context, value, if_number, var_word32, if_bigint, var_bigint);
5333 672 : }
5334 :
5335 : // Truncate {value} to word32 and jump to {if_number} if it is a Number,
5336 : // or find that it is a BigInt and jump to {if_bigint}. In either case,
5337 : // store the type feedback in {var_feedback}.
5338 3192 : void CodeStubAssembler::TaggedToWord32OrBigIntWithFeedback(
5339 : Node* context, Node* value, Label* if_number, Variable* var_word32,
5340 : Label* if_bigint, Variable* var_bigint, Variable* var_feedback) {
5341 : TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumeric>(
5342 : context, value, if_number, var_word32, if_bigint, var_bigint,
5343 3192 : var_feedback);
5344 3192 : }
5345 :
5346 : template <Object::Conversion conversion>
5347 5264 : void CodeStubAssembler::TaggedToWord32OrBigIntImpl(
5348 : Node* context, Node* value, Label* if_number, Variable* var_word32,
5349 : Label* if_bigint, Variable* var_bigint, Variable* var_feedback) {
5350 : DCHECK(var_word32->rep() == MachineRepresentation::kWord32);
5351 : DCHECK(var_bigint == nullptr ||
5352 : var_bigint->rep() == MachineRepresentation::kTagged);
5353 : DCHECK(var_feedback == nullptr ||
5354 : var_feedback->rep() == MachineRepresentation::kTaggedSigned);
5355 :
5356 : // We might need to loop after conversion.
5357 10528 : VARIABLE(var_value, MachineRepresentation::kTagged, value);
5358 5264 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNone);
5359 5264 : Variable* loop_vars[] = {&var_value, var_feedback};
5360 : int num_vars =
5361 5264 : var_feedback != nullptr ? arraysize(loop_vars) : arraysize(loop_vars) - 1;
5362 10528 : Label loop(this, num_vars, loop_vars);
5363 5264 : Goto(&loop);
5364 5264 : BIND(&loop);
5365 : {
5366 5264 : value = var_value.value();
5367 10528 : Label not_smi(this), is_heap_number(this), is_oddball(this),
5368 10528 : is_bigint(this);
5369 5264 : GotoIf(TaggedIsNotSmi(value), ¬_smi);
5370 :
5371 : // {value} is a Smi.
5372 5264 : var_word32->Bind(SmiToInt32(value));
5373 5264 : CombineFeedback(var_feedback, BinaryOperationFeedback::kSignedSmall);
5374 5264 : Goto(if_number);
5375 :
5376 5264 : BIND(¬_smi);
5377 5264 : Node* map = LoadMap(value);
5378 5264 : GotoIf(IsHeapNumberMap(map), &is_heap_number);
5379 5264 : Node* instance_type = LoadMapInstanceType(map);
5380 : if (conversion == Object::Conversion::kToNumeric) {
5381 3864 : GotoIf(IsBigIntInstanceType(instance_type), &is_bigint);
5382 : }
5383 :
5384 : // Not HeapNumber (or BigInt if conversion == kToNumeric).
5385 : {
5386 : if (var_feedback != nullptr) {
5387 : // We do not require an Or with earlier feedback here because once we
5388 : // convert the value to a Numeric, we cannot reach this path. We can
5389 : // only reach this path on the first pass when the feedback is kNone.
5390 : CSA_ASSERT(this, SmiEqual(CAST(var_feedback->value()),
5391 : SmiConstant(BinaryOperationFeedback::kNone)));
5392 : }
5393 5264 : GotoIf(InstanceTypeEqual(instance_type, ODDBALL_TYPE), &is_oddball);
5394 : // Not an oddball either -> convert.
5395 : auto builtin = conversion == Object::Conversion::kToNumeric
5396 : ? Builtins::kNonNumberToNumeric
5397 5264 : : Builtins::kNonNumberToNumber;
5398 5264 : var_value.Bind(CallBuiltin(builtin, context, value));
5399 5264 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kAny);
5400 5264 : Goto(&loop);
5401 :
5402 5264 : BIND(&is_oddball);
5403 5264 : var_value.Bind(LoadObjectField(value, Oddball::kToNumberOffset));
5404 5264 : OverwriteFeedback(var_feedback,
5405 : BinaryOperationFeedback::kNumberOrOddball);
5406 5264 : Goto(&loop);
5407 : }
5408 :
5409 5264 : BIND(&is_heap_number);
5410 5264 : var_word32->Bind(TruncateHeapNumberValueToWord32(value));
5411 5264 : CombineFeedback(var_feedback, BinaryOperationFeedback::kNumber);
5412 5264 : Goto(if_number);
5413 :
5414 : if (conversion == Object::Conversion::kToNumeric) {
5415 3864 : BIND(&is_bigint);
5416 3864 : var_bigint->Bind(value);
5417 3864 : CombineFeedback(var_feedback, BinaryOperationFeedback::kBigInt);
5418 3864 : Goto(if_bigint);
5419 : }
5420 : }
5421 5264 : }
5422 :
5423 5320 : Node* CodeStubAssembler::TruncateHeapNumberValueToWord32(Node* object) {
5424 5320 : Node* value = LoadHeapNumberValue(object);
5425 5320 : return TruncateFloat64ToWord32(value);
5426 : }
5427 :
5428 340 : void CodeStubAssembler::TryHeapNumberToSmi(TNode<HeapNumber> number,
5429 : TVariable<Smi>& var_result_smi,
5430 : Label* if_smi) {
5431 340 : TNode<Float64T> value = LoadHeapNumberValue(number);
5432 340 : TryFloat64ToSmi(value, var_result_smi, if_smi);
5433 340 : }
5434 :
5435 5672 : void CodeStubAssembler::TryFloat64ToSmi(TNode<Float64T> value,
5436 : TVariable<Smi>& var_result_smi,
5437 : Label* if_smi) {
5438 5672 : TNode<Int32T> value32 = RoundFloat64ToInt32(value);
5439 5672 : TNode<Float64T> value64 = ChangeInt32ToFloat64(value32);
5440 :
5441 11344 : Label if_int32(this), if_heap_number(this, Label::kDeferred);
5442 :
5443 5672 : GotoIfNot(Float64Equal(value, value64), &if_heap_number);
5444 5672 : GotoIfNot(Word32Equal(value32, Int32Constant(0)), &if_int32);
5445 22688 : Branch(Int32LessThan(UncheckedCast<Int32T>(Float64ExtractHighWord32(value)),
5446 22688 : Int32Constant(0)),
5447 5672 : &if_heap_number, &if_int32);
5448 :
5449 11344 : TVARIABLE(Number, var_result);
5450 5672 : BIND(&if_int32);
5451 : {
5452 5672 : if (SmiValuesAre32Bits()) {
5453 0 : var_result_smi = SmiTag(ChangeInt32ToIntPtr(value32));
5454 : } else {
5455 : DCHECK(SmiValuesAre31Bits());
5456 5672 : TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(value32, value32);
5457 5672 : TNode<BoolT> overflow = Projection<1>(pair);
5458 5672 : GotoIf(overflow, &if_heap_number);
5459 11344 : var_result_smi =
5460 17016 : BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Projection<0>(pair)));
5461 : }
5462 5672 : Goto(if_smi);
5463 : }
5464 5672 : BIND(&if_heap_number);
5465 5672 : }
5466 :
5467 5332 : TNode<Number> CodeStubAssembler::ChangeFloat64ToTagged(
5468 : SloppyTNode<Float64T> value) {
5469 10664 : Label if_smi(this), done(this);
5470 10664 : TVARIABLE(Smi, var_smi_result);
5471 10664 : TVARIABLE(Number, var_result);
5472 5332 : TryFloat64ToSmi(value, var_smi_result, &if_smi);
5473 :
5474 5332 : var_result = AllocateHeapNumberWithValue(value);
5475 5332 : Goto(&done);
5476 :
5477 5332 : BIND(&if_smi);
5478 : {
5479 5332 : var_result = var_smi_result.value();
5480 5332 : Goto(&done);
5481 : }
5482 5332 : BIND(&done);
5483 10664 : return var_result.value();
5484 : }
5485 :
5486 4984 : TNode<Number> CodeStubAssembler::ChangeInt32ToTagged(
5487 : SloppyTNode<Int32T> value) {
5488 4984 : if (SmiValuesAre32Bits()) {
5489 0 : return SmiTag(ChangeInt32ToIntPtr(value));
5490 : }
5491 : DCHECK(SmiValuesAre31Bits());
5492 9968 : TVARIABLE(Number, var_result);
5493 4984 : TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(value, value);
5494 4984 : TNode<BoolT> overflow = Projection<1>(pair);
5495 9968 : Label if_overflow(this, Label::kDeferred), if_notoverflow(this),
5496 9968 : if_join(this);
5497 4984 : Branch(overflow, &if_overflow, &if_notoverflow);
5498 4984 : BIND(&if_overflow);
5499 : {
5500 4984 : TNode<Float64T> value64 = ChangeInt32ToFloat64(value);
5501 4984 : TNode<HeapNumber> result = AllocateHeapNumberWithValue(value64);
5502 4984 : var_result = result;
5503 4984 : Goto(&if_join);
5504 : }
5505 4984 : BIND(&if_notoverflow);
5506 : {
5507 : TNode<IntPtrT> almost_tagged_value =
5508 4984 : ChangeInt32ToIntPtr(Projection<0>(pair));
5509 4984 : TNode<Smi> result = BitcastWordToTaggedSigned(almost_tagged_value);
5510 4984 : var_result = result;
5511 4984 : Goto(&if_join);
5512 : }
5513 4984 : BIND(&if_join);
5514 4984 : return var_result.value();
5515 : }
5516 :
5517 4816 : TNode<Number> CodeStubAssembler::ChangeUint32ToTagged(
5518 : SloppyTNode<Uint32T> value) {
5519 9632 : Label if_overflow(this, Label::kDeferred), if_not_overflow(this),
5520 9632 : if_join(this);
5521 9632 : TVARIABLE(Number, var_result);
5522 : // If {value} > 2^31 - 1, we need to store it in a HeapNumber.
5523 9632 : Branch(Uint32LessThan(Uint32Constant(Smi::kMaxValue), value), &if_overflow,
5524 4816 : &if_not_overflow);
5525 :
5526 4816 : BIND(&if_not_overflow);
5527 : {
5528 : // The {value} is definitely in valid Smi range.
5529 4816 : var_result = SmiTag(Signed(ChangeUint32ToWord(value)));
5530 : }
5531 4816 : Goto(&if_join);
5532 :
5533 4816 : BIND(&if_overflow);
5534 : {
5535 4816 : TNode<Float64T> float64_value = ChangeUint32ToFloat64(value);
5536 4816 : var_result = AllocateHeapNumberWithValue(float64_value);
5537 : }
5538 4816 : Goto(&if_join);
5539 :
5540 4816 : BIND(&if_join);
5541 9632 : return var_result.value();
5542 : }
5543 :
5544 616 : TNode<Number> CodeStubAssembler::ChangeUintPtrToTagged(TNode<UintPtrT> value) {
5545 1232 : Label if_overflow(this, Label::kDeferred), if_not_overflow(this),
5546 1232 : if_join(this);
5547 1232 : TVARIABLE(Number, var_result);
5548 : // If {value} > 2^31 - 1, we need to store it in a HeapNumber.
5549 1232 : Branch(UintPtrLessThan(UintPtrConstant(Smi::kMaxValue), value), &if_overflow,
5550 616 : &if_not_overflow);
5551 :
5552 616 : BIND(&if_not_overflow);
5553 : {
5554 : // The {value} is definitely in valid Smi range.
5555 616 : var_result = SmiTag(Signed(value));
5556 : }
5557 616 : Goto(&if_join);
5558 :
5559 616 : BIND(&if_overflow);
5560 : {
5561 616 : TNode<Float64T> float64_value = ChangeUintPtrToFloat64(value);
5562 616 : var_result = AllocateHeapNumberWithValue(float64_value);
5563 : }
5564 616 : Goto(&if_join);
5565 :
5566 616 : BIND(&if_join);
5567 1232 : return var_result.value();
5568 : }
5569 :
5570 840 : TNode<String> CodeStubAssembler::ToThisString(TNode<Context> context,
5571 : TNode<Object> value,
5572 : TNode<String> method_name) {
5573 1680 : VARIABLE(var_value, MachineRepresentation::kTagged, value);
5574 :
5575 : // Check if the {value} is a Smi or a HeapObject.
5576 1680 : Label if_valueissmi(this, Label::kDeferred), if_valueisnotsmi(this),
5577 1680 : if_valueisstring(this);
5578 840 : Branch(TaggedIsSmi(value), &if_valueissmi, &if_valueisnotsmi);
5579 840 : BIND(&if_valueisnotsmi);
5580 : {
5581 : // Load the instance type of the {value}.
5582 840 : Node* value_instance_type = LoadInstanceType(CAST(value));
5583 :
5584 : // Check if the {value} is already String.
5585 1680 : Label if_valueisnotstring(this, Label::kDeferred);
5586 1680 : Branch(IsStringInstanceType(value_instance_type), &if_valueisstring,
5587 840 : &if_valueisnotstring);
5588 840 : BIND(&if_valueisnotstring);
5589 : {
5590 : // Check if the {value} is null.
5591 1680 : Label if_valueisnullorundefined(this, Label::kDeferred);
5592 840 : GotoIf(IsNullOrUndefined(value), &if_valueisnullorundefined);
5593 : // Convert the {value} to a String.
5594 840 : var_value.Bind(CallBuiltin(Builtins::kToString, context, value));
5595 840 : Goto(&if_valueisstring);
5596 :
5597 840 : BIND(&if_valueisnullorundefined);
5598 : {
5599 : // The {value} is either null or undefined.
5600 840 : ThrowTypeError(context, MessageTemplate::kCalledOnNullOrUndefined,
5601 840 : method_name);
5602 : }
5603 : }
5604 : }
5605 840 : BIND(&if_valueissmi);
5606 : {
5607 : // The {value} is a Smi, convert it to a String.
5608 840 : var_value.Bind(CallBuiltin(Builtins::kNumberToString, context, value));
5609 840 : Goto(&if_valueisstring);
5610 : }
5611 840 : BIND(&if_valueisstring);
5612 1680 : return CAST(var_value.value());
5613 : }
5614 :
5615 112 : TNode<Uint32T> CodeStubAssembler::ChangeNumberToUint32(TNode<Number> value) {
5616 224 : TVARIABLE(Uint32T, var_result);
5617 224 : Label if_smi(this), if_heapnumber(this, Label::kDeferred), done(this);
5618 112 : Branch(TaggedIsSmi(value), &if_smi, &if_heapnumber);
5619 112 : BIND(&if_smi);
5620 : {
5621 112 : var_result = Unsigned(SmiToInt32(CAST(value)));
5622 112 : Goto(&done);
5623 : }
5624 112 : BIND(&if_heapnumber);
5625 : {
5626 112 : var_result = ChangeFloat64ToUint32(LoadHeapNumberValue(CAST(value)));
5627 112 : Goto(&done);
5628 : }
5629 112 : BIND(&done);
5630 224 : return var_result.value();
5631 : }
5632 :
5633 12628 : TNode<Float64T> CodeStubAssembler::ChangeNumberToFloat64(
5634 : SloppyTNode<Number> value) {
5635 : // TODO(tebbi): Remove assert once argument is TNode instead of SloppyTNode.
5636 : CSA_SLOW_ASSERT(this, IsNumber(value));
5637 25256 : TVARIABLE(Float64T, result);
5638 25256 : Label smi(this);
5639 25256 : Label done(this, &result);
5640 12628 : GotoIf(TaggedIsSmi(value), &smi);
5641 12628 : result = LoadHeapNumberValue(CAST(value));
5642 12628 : Goto(&done);
5643 :
5644 12628 : BIND(&smi);
5645 : {
5646 12628 : result = SmiToFloat64(CAST(value));
5647 12628 : Goto(&done);
5648 : }
5649 :
5650 12628 : BIND(&done);
5651 25256 : return result.value();
5652 : }
5653 :
5654 336 : TNode<UintPtrT> CodeStubAssembler::TryNumberToUintPtr(TNode<Number> value,
5655 : Label* if_negative) {
5656 672 : TVARIABLE(UintPtrT, result);
5657 672 : Label done(this, &result);
5658 672 : Branch(TaggedIsSmi(value),
5659 336 : [&] {
5660 1512 : TNode<Smi> value_smi = CAST(value);
5661 336 : if (if_negative == nullptr) {
5662 : CSA_SLOW_ASSERT(this, SmiLessThan(SmiConstant(-1), value_smi));
5663 : } else {
5664 224 : GotoIfNot(TaggedIsPositiveSmi(value), if_negative);
5665 : }
5666 1008 : result = UncheckedCast<UintPtrT>(SmiToIntPtr(value_smi));
5667 672 : Goto(&done);
5668 336 : },
5669 336 : [&] {
5670 1512 : TNode<HeapNumber> value_hn = CAST(value);
5671 672 : TNode<Float64T> value = LoadHeapNumberValue(value_hn);
5672 336 : if (if_negative != nullptr) {
5673 224 : GotoIf(Float64LessThan(value, Float64Constant(0.0)), if_negative);
5674 : }
5675 672 : result = ChangeFloat64ToUintPtr(value);
5676 672 : Goto(&done);
5677 672 : });
5678 :
5679 336 : BIND(&done);
5680 672 : return result.value();
5681 : }
5682 :
5683 71736 : TNode<WordT> CodeStubAssembler::TimesSystemPointerSize(
5684 : SloppyTNode<WordT> value) {
5685 71736 : return WordShl(value, kSystemPointerSizeLog2);
5686 : }
5687 :
5688 8436 : TNode<WordT> CodeStubAssembler::TimesTaggedSize(SloppyTNode<WordT> value) {
5689 8436 : return WordShl(value, kTaggedSizeLog2);
5690 : }
5691 :
5692 56 : TNode<WordT> CodeStubAssembler::TimesDoubleSize(SloppyTNode<WordT> value) {
5693 56 : return WordShl(value, kDoubleSizeLog2);
5694 : }
5695 :
5696 504 : Node* CodeStubAssembler::ToThisValue(Node* context, Node* value,
5697 : PrimitiveType primitive_type,
5698 : char const* method_name) {
5699 : // We might need to loop once due to JSValue unboxing.
5700 1008 : VARIABLE(var_value, MachineRepresentation::kTagged, value);
5701 1008 : Label loop(this, &var_value), done_loop(this),
5702 1008 : done_throw(this, Label::kDeferred);
5703 504 : Goto(&loop);
5704 504 : BIND(&loop);
5705 : {
5706 : // Load the current {value}.
5707 504 : value = var_value.value();
5708 :
5709 : // Check if the {value} is a Smi or a HeapObject.
5710 1008 : GotoIf(TaggedIsSmi(value), (primitive_type == PrimitiveType::kNumber)
5711 : ? &done_loop
5712 504 : : &done_throw);
5713 :
5714 : // Load the map of the {value}.
5715 504 : Node* value_map = LoadMap(value);
5716 :
5717 : // Load the instance type of the {value}.
5718 504 : Node* value_instance_type = LoadMapInstanceType(value_map);
5719 :
5720 : // Check if {value} is a JSValue.
5721 1008 : Label if_valueisvalue(this, Label::kDeferred), if_valueisnotvalue(this);
5722 1008 : Branch(InstanceTypeEqual(value_instance_type, JS_VALUE_TYPE),
5723 504 : &if_valueisvalue, &if_valueisnotvalue);
5724 :
5725 504 : BIND(&if_valueisvalue);
5726 : {
5727 : // Load the actual value from the {value}.
5728 504 : var_value.Bind(LoadObjectField(value, JSValue::kValueOffset));
5729 504 : Goto(&loop);
5730 : }
5731 :
5732 504 : BIND(&if_valueisnotvalue);
5733 : {
5734 504 : switch (primitive_type) {
5735 : case PrimitiveType::kBoolean:
5736 112 : GotoIf(WordEqual(value_map, BooleanMapConstant()), &done_loop);
5737 112 : break;
5738 : case PrimitiveType::kNumber:
5739 56 : GotoIf(WordEqual(value_map, HeapNumberMapConstant()), &done_loop);
5740 56 : break;
5741 : case PrimitiveType::kString:
5742 112 : GotoIf(IsStringInstanceType(value_instance_type), &done_loop);
5743 112 : break;
5744 : case PrimitiveType::kSymbol:
5745 224 : GotoIf(WordEqual(value_map, SymbolMapConstant()), &done_loop);
5746 224 : break;
5747 : }
5748 504 : Goto(&done_throw);
5749 : }
5750 : }
5751 :
5752 504 : BIND(&done_throw);
5753 : {
5754 504 : const char* primitive_name = nullptr;
5755 504 : switch (primitive_type) {
5756 : case PrimitiveType::kBoolean:
5757 112 : primitive_name = "Boolean";
5758 112 : break;
5759 : case PrimitiveType::kNumber:
5760 56 : primitive_name = "Number";
5761 56 : break;
5762 : case PrimitiveType::kString:
5763 112 : primitive_name = "String";
5764 112 : break;
5765 : case PrimitiveType::kSymbol:
5766 224 : primitive_name = "Symbol";
5767 224 : break;
5768 : }
5769 504 : CHECK_NOT_NULL(primitive_name);
5770 :
5771 : // The {value} is not a compatible receiver for this method.
5772 : ThrowTypeError(context, MessageTemplate::kNotGeneric, method_name,
5773 504 : primitive_name);
5774 : }
5775 :
5776 504 : BIND(&done_loop);
5777 1008 : return var_value.value();
5778 : }
5779 :
5780 2912 : Node* CodeStubAssembler::ThrowIfNotInstanceType(Node* context, Node* value,
5781 : InstanceType instance_type,
5782 : char const* method_name) {
5783 5824 : Label out(this), throw_exception(this, Label::kDeferred);
5784 5824 : VARIABLE(var_value_map, MachineRepresentation::kTagged);
5785 :
5786 2912 : GotoIf(TaggedIsSmi(value), &throw_exception);
5787 :
5788 : // Load the instance type of the {value}.
5789 2912 : var_value_map.Bind(LoadMap(value));
5790 2912 : Node* const value_instance_type = LoadMapInstanceType(var_value_map.value());
5791 :
5792 5824 : Branch(Word32Equal(value_instance_type, Int32Constant(instance_type)), &out,
5793 2912 : &throw_exception);
5794 :
5795 : // The {value} is not a compatible receiver for this method.
5796 2912 : BIND(&throw_exception);
5797 2912 : ThrowTypeError(context, MessageTemplate::kIncompatibleMethodReceiver,
5798 5824 : StringConstant(method_name), value);
5799 :
5800 2912 : BIND(&out);
5801 5824 : return var_value_map.value();
5802 : }
5803 :
5804 840 : Node* CodeStubAssembler::ThrowIfNotJSReceiver(Node* context, Node* value,
5805 : MessageTemplate msg_template,
5806 : const char* method_name) {
5807 1680 : Label out(this), throw_exception(this, Label::kDeferred);
5808 1680 : VARIABLE(var_value_map, MachineRepresentation::kTagged);
5809 :
5810 840 : GotoIf(TaggedIsSmi(value), &throw_exception);
5811 :
5812 : // Load the instance type of the {value}.
5813 840 : var_value_map.Bind(LoadMap(value));
5814 840 : Node* const value_instance_type = LoadMapInstanceType(var_value_map.value());
5815 :
5816 840 : Branch(IsJSReceiverInstanceType(value_instance_type), &out, &throw_exception);
5817 :
5818 : // The {value} is not a compatible receiver for this method.
5819 840 : BIND(&throw_exception);
5820 840 : ThrowTypeError(context, msg_template, method_name);
5821 :
5822 840 : BIND(&out);
5823 1680 : return var_value_map.value();
5824 : }
5825 :
5826 4032 : void CodeStubAssembler::ThrowRangeError(Node* context, MessageTemplate message,
5827 : Node* arg0, Node* arg1, Node* arg2) {
5828 4032 : Node* template_index = SmiConstant(static_cast<int>(message));
5829 4032 : if (arg0 == nullptr) {
5830 3584 : CallRuntime(Runtime::kThrowRangeError, context, template_index);
5831 448 : } else if (arg1 == nullptr) {
5832 448 : CallRuntime(Runtime::kThrowRangeError, context, template_index, arg0);
5833 0 : } else if (arg2 == nullptr) {
5834 0 : CallRuntime(Runtime::kThrowRangeError, context, template_index, arg0, arg1);
5835 : } else {
5836 : CallRuntime(Runtime::kThrowRangeError, context, template_index, arg0, arg1,
5837 0 : arg2);
5838 : }
5839 4032 : Unreachable();
5840 4032 : }
5841 :
5842 11820 : void CodeStubAssembler::ThrowTypeError(Node* context, MessageTemplate message,
5843 : char const* arg0, char const* arg1) {
5844 11820 : Node* arg0_node = nullptr;
5845 11820 : if (arg0) arg0_node = StringConstant(arg0);
5846 11820 : Node* arg1_node = nullptr;
5847 11820 : if (arg1) arg1_node = StringConstant(arg1);
5848 11820 : ThrowTypeError(context, message, arg0_node, arg1_node);
5849 11820 : }
5850 :
5851 24532 : void CodeStubAssembler::ThrowTypeError(Node* context, MessageTemplate message,
5852 : Node* arg0, Node* arg1, Node* arg2) {
5853 24532 : Node* template_index = SmiConstant(static_cast<int>(message));
5854 24532 : if (arg0 == nullptr) {
5855 5660 : CallRuntime(Runtime::kThrowTypeError, context, template_index);
5856 18872 : } else if (arg1 == nullptr) {
5857 13832 : CallRuntime(Runtime::kThrowTypeError, context, template_index, arg0);
5858 5040 : } else if (arg2 == nullptr) {
5859 4536 : CallRuntime(Runtime::kThrowTypeError, context, template_index, arg0, arg1);
5860 : } else {
5861 : CallRuntime(Runtime::kThrowTypeError, context, template_index, arg0, arg1,
5862 504 : arg2);
5863 : }
5864 24532 : Unreachable();
5865 24532 : }
5866 :
5867 99288 : TNode<BoolT> CodeStubAssembler::InstanceTypeEqual(
5868 : SloppyTNode<Int32T> instance_type, int type) {
5869 99288 : return Word32Equal(instance_type, Int32Constant(type));
5870 : }
5871 :
5872 1568 : TNode<BoolT> CodeStubAssembler::IsDictionaryMap(SloppyTNode<Map> map) {
5873 : CSA_SLOW_ASSERT(this, IsMap(map));
5874 1568 : Node* bit_field3 = LoadMapBitField3(map);
5875 1568 : return IsSetWord32<Map::IsDictionaryMapBit>(bit_field3);
5876 : }
5877 :
5878 168 : TNode<BoolT> CodeStubAssembler::IsExtensibleMap(SloppyTNode<Map> map) {
5879 : CSA_ASSERT(this, IsMap(map));
5880 168 : return IsSetWord32<Map::IsExtensibleBit>(LoadMapBitField2(map));
5881 : }
5882 :
5883 0 : TNode<BoolT> CodeStubAssembler::IsExtensibleNonPrototypeMap(TNode<Map> map) {
5884 0 : int kMask = Map::IsExtensibleBit::kMask | Map::IsPrototypeMapBit::kMask;
5885 0 : int kExpected = Map::IsExtensibleBit::kMask;
5886 0 : return Word32Equal(Word32And(LoadMapBitField2(map), Int32Constant(kMask)),
5887 0 : Int32Constant(kExpected));
5888 : }
5889 :
5890 9076 : TNode<BoolT> CodeStubAssembler::IsCallableMap(SloppyTNode<Map> map) {
5891 : CSA_ASSERT(this, IsMap(map));
5892 9076 : return IsSetWord32<Map::IsCallableBit>(LoadMapBitField(map));
5893 : }
5894 :
5895 728 : TNode<BoolT> CodeStubAssembler::IsDeprecatedMap(SloppyTNode<Map> map) {
5896 : CSA_ASSERT(this, IsMap(map));
5897 728 : return IsSetWord32<Map::IsDeprecatedBit>(LoadMapBitField3(map));
5898 : }
5899 :
5900 5824 : TNode<BoolT> CodeStubAssembler::IsUndetectableMap(SloppyTNode<Map> map) {
5901 : CSA_ASSERT(this, IsMap(map));
5902 5824 : return IsSetWord32<Map::IsUndetectableBit>(LoadMapBitField(map));
5903 : }
5904 :
5905 3528 : TNode<BoolT> CodeStubAssembler::IsNoElementsProtectorCellInvalid() {
5906 3528 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5907 3528 : Node* cell = LoadRoot(RootIndex::kNoElementsProtector);
5908 3528 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5909 3528 : return WordEqual(cell_value, invalid);
5910 : }
5911 :
5912 448 : TNode<BoolT> CodeStubAssembler::IsArrayIteratorProtectorCellInvalid() {
5913 448 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5914 448 : Node* cell = LoadRoot(RootIndex::kArrayIteratorProtector);
5915 448 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5916 448 : return WordEqual(cell_value, invalid);
5917 : }
5918 :
5919 168 : TNode<BoolT> CodeStubAssembler::IsPromiseResolveProtectorCellInvalid() {
5920 168 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5921 168 : Node* cell = LoadRoot(RootIndex::kPromiseResolveProtector);
5922 168 : Node* cell_value = LoadObjectField(cell, Cell::kValueOffset);
5923 168 : return WordEqual(cell_value, invalid);
5924 : }
5925 :
5926 392 : TNode<BoolT> CodeStubAssembler::IsPromiseThenProtectorCellInvalid() {
5927 392 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5928 392 : Node* cell = LoadRoot(RootIndex::kPromiseThenProtector);
5929 392 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5930 392 : return WordEqual(cell_value, invalid);
5931 : }
5932 :
5933 280 : TNode<BoolT> CodeStubAssembler::IsArraySpeciesProtectorCellInvalid() {
5934 280 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5935 280 : Node* cell = LoadRoot(RootIndex::kArraySpeciesProtector);
5936 280 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5937 280 : return WordEqual(cell_value, invalid);
5938 : }
5939 :
5940 224 : TNode<BoolT> CodeStubAssembler::IsTypedArraySpeciesProtectorCellInvalid() {
5941 224 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5942 224 : Node* cell = LoadRoot(RootIndex::kTypedArraySpeciesProtector);
5943 224 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5944 224 : return WordEqual(cell_value, invalid);
5945 : }
5946 :
5947 952 : TNode<BoolT> CodeStubAssembler::IsRegExpSpeciesProtectorCellInvalid() {
5948 952 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5949 952 : Node* cell = LoadRoot(RootIndex::kRegExpSpeciesProtector);
5950 952 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5951 952 : return WordEqual(cell_value, invalid);
5952 : }
5953 :
5954 616 : TNode<BoolT> CodeStubAssembler::IsPromiseSpeciesProtectorCellInvalid() {
5955 616 : Node* invalid = SmiConstant(Isolate::kProtectorInvalid);
5956 616 : Node* cell = LoadRoot(RootIndex::kPromiseSpeciesProtector);
5957 616 : Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
5958 616 : return WordEqual(cell_value, invalid);
5959 : }
5960 :
5961 2688 : TNode<BoolT> CodeStubAssembler::IsPrototypeInitialArrayPrototype(
5962 : SloppyTNode<Context> context, SloppyTNode<Map> map) {
5963 2688 : Node* const native_context = LoadNativeContext(context);
5964 5376 : Node* const initial_array_prototype = LoadContextElement(
5965 8064 : native_context, Context::INITIAL_ARRAY_PROTOTYPE_INDEX);
5966 2688 : Node* proto = LoadMapPrototype(map);
5967 2688 : return WordEqual(proto, initial_array_prototype);
5968 : }
5969 :
5970 224 : TNode<BoolT> CodeStubAssembler::IsPrototypeTypedArrayPrototype(
5971 : SloppyTNode<Context> context, SloppyTNode<Map> map) {
5972 224 : TNode<Context> const native_context = LoadNativeContext(context);
5973 : TNode<Object> const typed_array_prototype =
5974 224 : LoadContextElement(native_context, Context::TYPED_ARRAY_PROTOTYPE_INDEX);
5975 224 : TNode<HeapObject> proto = LoadMapPrototype(map);
5976 : TNode<HeapObject> proto_of_proto = Select<HeapObject>(
5977 672 : IsJSObject(proto), [=] { return LoadMapPrototype(LoadMap(proto)); },
5978 896 : [=] { return NullConstant(); });
5979 224 : return WordEqual(proto_of_proto, typed_array_prototype);
5980 : }
5981 :
5982 1176 : TNode<BoolT> CodeStubAssembler::IsFastAliasedArgumentsMap(
5983 : TNode<Context> context, TNode<Map> map) {
5984 1176 : TNode<Context> const native_context = LoadNativeContext(context);
5985 : TNode<Object> const arguments_map = LoadContextElement(
5986 1176 : native_context, Context::FAST_ALIASED_ARGUMENTS_MAP_INDEX);
5987 1176 : return WordEqual(arguments_map, map);
5988 : }
5989 :
5990 1120 : TNode<BoolT> CodeStubAssembler::IsSlowAliasedArgumentsMap(
5991 : TNode<Context> context, TNode<Map> map) {
5992 1120 : TNode<Context> const native_context = LoadNativeContext(context);
5993 : TNode<Object> const arguments_map = LoadContextElement(
5994 1120 : native_context, Context::SLOW_ALIASED_ARGUMENTS_MAP_INDEX);
5995 1120 : return WordEqual(arguments_map, map);
5996 : }
5997 :
5998 1176 : TNode<BoolT> CodeStubAssembler::IsSloppyArgumentsMap(TNode<Context> context,
5999 : TNode<Map> map) {
6000 1176 : TNode<Context> const native_context = LoadNativeContext(context);
6001 : TNode<Object> const arguments_map =
6002 1176 : LoadContextElement(native_context, Context::SLOPPY_ARGUMENTS_MAP_INDEX);
6003 1176 : return WordEqual(arguments_map, map);
6004 : }
6005 :
6006 1176 : TNode<BoolT> CodeStubAssembler::IsStrictArgumentsMap(TNode<Context> context,
6007 : TNode<Map> map) {
6008 1176 : TNode<Context> const native_context = LoadNativeContext(context);
6009 : TNode<Object> const arguments_map =
6010 1176 : LoadContextElement(native_context, Context::STRICT_ARGUMENTS_MAP_INDEX);
6011 1176 : return WordEqual(arguments_map, map);
6012 : }
6013 :
6014 168 : TNode<BoolT> CodeStubAssembler::TaggedIsCallable(TNode<Object> object) {
6015 : return Select<BoolT>(
6016 504 : TaggedIsSmi(object), [=] { return Int32FalseConstant(); },
6017 168 : [=] {
6018 336 : return IsCallableMap(LoadMap(UncheckedCast<HeapObject>(object)));
6019 840 : });
6020 : }
6021 :
6022 4200 : TNode<BoolT> CodeStubAssembler::IsCallable(SloppyTNode<HeapObject> object) {
6023 4200 : return IsCallableMap(LoadMap(object));
6024 : }
6025 :
6026 0 : TNode<BoolT> CodeStubAssembler::IsCell(SloppyTNode<HeapObject> object) {
6027 0 : return WordEqual(LoadMap(object), LoadRoot(RootIndex::kCellMap));
6028 : }
6029 :
6030 616 : TNode<BoolT> CodeStubAssembler::IsCode(SloppyTNode<HeapObject> object) {
6031 616 : return HasInstanceType(object, CODE_TYPE);
6032 : }
6033 :
6034 1568 : TNode<BoolT> CodeStubAssembler::IsConstructorMap(SloppyTNode<Map> map) {
6035 : CSA_ASSERT(this, IsMap(map));
6036 1568 : return IsSetWord32<Map::IsConstructorBit>(LoadMapBitField(map));
6037 : }
6038 :
6039 728 : TNode<BoolT> CodeStubAssembler::IsConstructor(SloppyTNode<HeapObject> object) {
6040 728 : return IsConstructorMap(LoadMap(object));
6041 : }
6042 :
6043 112 : TNode<BoolT> CodeStubAssembler::IsFunctionWithPrototypeSlotMap(
6044 : SloppyTNode<Map> map) {
6045 : CSA_ASSERT(this, IsMap(map));
6046 112 : return IsSetWord32<Map::HasPrototypeSlotBit>(LoadMapBitField(map));
6047 : }
6048 :
6049 2868 : TNode<BoolT> CodeStubAssembler::IsSpecialReceiverInstanceType(
6050 : TNode<Int32T> instance_type) {
6051 : STATIC_ASSERT(JS_GLOBAL_OBJECT_TYPE <= LAST_SPECIAL_RECEIVER_TYPE);
6052 : return Int32LessThanOrEqual(instance_type,
6053 2868 : Int32Constant(LAST_SPECIAL_RECEIVER_TYPE));
6054 : }
6055 :
6056 1624 : TNode<BoolT> CodeStubAssembler::IsCustomElementsReceiverInstanceType(
6057 : TNode<Int32T> instance_type) {
6058 : return Int32LessThanOrEqual(instance_type,
6059 1624 : Int32Constant(LAST_CUSTOM_ELEMENTS_RECEIVER));
6060 : }
6061 :
6062 20232 : TNode<BoolT> CodeStubAssembler::IsStringInstanceType(
6063 : SloppyTNode<Int32T> instance_type) {
6064 : STATIC_ASSERT(INTERNALIZED_STRING_TYPE == FIRST_TYPE);
6065 20232 : return Int32LessThan(instance_type, Int32Constant(FIRST_NONSTRING_TYPE));
6066 : }
6067 :
6068 5488 : TNode<BoolT> CodeStubAssembler::IsOneByteStringInstanceType(
6069 : SloppyTNode<Int32T> instance_type) {
6070 : CSA_ASSERT(this, IsStringInstanceType(instance_type));
6071 : return Word32Equal(
6072 10976 : Word32And(instance_type, Int32Constant(kStringEncodingMask)),
6073 16464 : Int32Constant(kOneByteStringTag));
6074 : }
6075 :
6076 4704 : TNode<BoolT> CodeStubAssembler::IsSequentialStringInstanceType(
6077 : SloppyTNode<Int32T> instance_type) {
6078 : CSA_ASSERT(this, IsStringInstanceType(instance_type));
6079 : return Word32Equal(
6080 9408 : Word32And(instance_type, Int32Constant(kStringRepresentationMask)),
6081 14112 : Int32Constant(kSeqStringTag));
6082 : }
6083 :
6084 56 : TNode<BoolT> CodeStubAssembler::IsConsStringInstanceType(
6085 : SloppyTNode<Int32T> instance_type) {
6086 : CSA_ASSERT(this, IsStringInstanceType(instance_type));
6087 : return Word32Equal(
6088 112 : Word32And(instance_type, Int32Constant(kStringRepresentationMask)),
6089 168 : Int32Constant(kConsStringTag));
6090 : }
6091 :
6092 0 : TNode<BoolT> CodeStubAssembler::IsIndirectStringInstanceType(
6093 : SloppyTNode<Int32T> instance_type) {
6094 : CSA_ASSERT(this, IsStringInstanceType(instance_type));
6095 : STATIC_ASSERT(kIsIndirectStringMask == 0x1);
6096 : STATIC_ASSERT(kIsIndirectStringTag == 0x1);
6097 : return UncheckedCast<BoolT>(
6098 0 : Word32And(instance_type, Int32Constant(kIsIndirectStringMask)));
6099 : }
6100 :
6101 0 : TNode<BoolT> CodeStubAssembler::IsExternalStringInstanceType(
6102 : SloppyTNode<Int32T> instance_type) {
6103 : CSA_ASSERT(this, IsStringInstanceType(instance_type));
6104 : return Word32Equal(
6105 0 : Word32And(instance_type, Int32Constant(kStringRepresentationMask)),
6106 0 : Int32Constant(kExternalStringTag));
6107 : }
6108 :
6109 4704 : TNode<BoolT> CodeStubAssembler::IsUncachedExternalStringInstanceType(
6110 : SloppyTNode<Int32T> instance_type) {
6111 : CSA_ASSERT(this, IsStringInstanceType(instance_type));
6112 : STATIC_ASSERT(kUncachedExternalStringTag != 0);
6113 4704 : return IsSetWord32(instance_type, kUncachedExternalStringMask);
6114 : }
6115 :
6116 15576 : TNode<BoolT> CodeStubAssembler::IsJSReceiverInstanceType(
6117 : SloppyTNode<Int32T> instance_type) {
6118 : STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
6119 : return Int32GreaterThanOrEqual(instance_type,
6120 15576 : Int32Constant(FIRST_JS_RECEIVER_TYPE));
6121 : }
6122 :
6123 7616 : TNode<BoolT> CodeStubAssembler::IsJSReceiverMap(SloppyTNode<Map> map) {
6124 7616 : return IsJSReceiverInstanceType(LoadMapInstanceType(map));
6125 : }
6126 :
6127 6720 : TNode<BoolT> CodeStubAssembler::IsJSReceiver(SloppyTNode<HeapObject> object) {
6128 6720 : return IsJSReceiverMap(LoadMap(object));
6129 : }
6130 :
6131 0 : TNode<BoolT> CodeStubAssembler::IsNullOrJSReceiver(
6132 : SloppyTNode<HeapObject> object) {
6133 0 : return UncheckedCast<BoolT>(Word32Or(IsJSReceiver(object), IsNull(object)));
6134 : }
6135 :
6136 3584 : TNode<BoolT> CodeStubAssembler::IsNullOrUndefined(SloppyTNode<Object> value) {
6137 3584 : return UncheckedCast<BoolT>(Word32Or(IsUndefined(value), IsNull(value)));
6138 : }
6139 :
6140 0 : TNode<BoolT> CodeStubAssembler::IsJSGlobalProxyInstanceType(
6141 : SloppyTNode<Int32T> instance_type) {
6142 0 : return InstanceTypeEqual(instance_type, JS_GLOBAL_PROXY_TYPE);
6143 : }
6144 :
6145 448 : TNode<BoolT> CodeStubAssembler::IsJSObjectInstanceType(
6146 : SloppyTNode<Int32T> instance_type) {
6147 : STATIC_ASSERT(LAST_JS_OBJECT_TYPE == LAST_TYPE);
6148 : return Int32GreaterThanOrEqual(instance_type,
6149 448 : Int32Constant(FIRST_JS_OBJECT_TYPE));
6150 : }
6151 :
6152 336 : TNode<BoolT> CodeStubAssembler::IsJSObjectMap(SloppyTNode<Map> map) {
6153 : CSA_ASSERT(this, IsMap(map));
6154 336 : return IsJSObjectInstanceType(LoadMapInstanceType(map));
6155 : }
6156 :
6157 224 : TNode<BoolT> CodeStubAssembler::IsJSObject(SloppyTNode<HeapObject> object) {
6158 224 : return IsJSObjectMap(LoadMap(object));
6159 : }
6160 :
6161 840 : TNode<BoolT> CodeStubAssembler::IsJSPromiseMap(SloppyTNode<Map> map) {
6162 : CSA_ASSERT(this, IsMap(map));
6163 840 : return InstanceTypeEqual(LoadMapInstanceType(map), JS_PROMISE_TYPE);
6164 : }
6165 :
6166 0 : TNode<BoolT> CodeStubAssembler::IsJSPromise(SloppyTNode<HeapObject> object) {
6167 0 : return IsJSPromiseMap(LoadMap(object));
6168 : }
6169 :
6170 336 : TNode<BoolT> CodeStubAssembler::IsJSProxy(SloppyTNode<HeapObject> object) {
6171 336 : return HasInstanceType(object, JS_PROXY_TYPE);
6172 : }
6173 :
6174 728 : TNode<BoolT> CodeStubAssembler::IsJSGlobalProxy(
6175 : SloppyTNode<HeapObject> object) {
6176 728 : return HasInstanceType(object, JS_GLOBAL_PROXY_TYPE);
6177 : }
6178 :
6179 4484 : TNode<BoolT> CodeStubAssembler::IsMap(SloppyTNode<HeapObject> map) {
6180 4484 : return IsMetaMap(LoadMap(map));
6181 : }
6182 :
6183 3812 : TNode<BoolT> CodeStubAssembler::IsJSValueInstanceType(
6184 : SloppyTNode<Int32T> instance_type) {
6185 3812 : return InstanceTypeEqual(instance_type, JS_VALUE_TYPE);
6186 : }
6187 :
6188 0 : TNode<BoolT> CodeStubAssembler::IsJSValue(SloppyTNode<HeapObject> object) {
6189 0 : return IsJSValueMap(LoadMap(object));
6190 : }
6191 :
6192 0 : TNode<BoolT> CodeStubAssembler::IsJSValueMap(SloppyTNode<Map> map) {
6193 0 : return IsJSValueInstanceType(LoadMapInstanceType(map));
6194 : }
6195 :
6196 14508 : TNode<BoolT> CodeStubAssembler::IsJSArrayInstanceType(
6197 : SloppyTNode<Int32T> instance_type) {
6198 14508 : return InstanceTypeEqual(instance_type, JS_ARRAY_TYPE);
6199 : }
6200 :
6201 7784 : TNode<BoolT> CodeStubAssembler::IsJSArray(SloppyTNode<HeapObject> object) {
6202 7784 : return IsJSArrayMap(LoadMap(object));
6203 : }
6204 :
6205 10696 : TNode<BoolT> CodeStubAssembler::IsJSArrayMap(SloppyTNode<Map> map) {
6206 10696 : return IsJSArrayInstanceType(LoadMapInstanceType(map));
6207 : }
6208 :
6209 0 : TNode<BoolT> CodeStubAssembler::IsJSArrayIterator(
6210 : SloppyTNode<HeapObject> object) {
6211 0 : return HasInstanceType(object, JS_ARRAY_ITERATOR_TYPE);
6212 : }
6213 :
6214 0 : TNode<BoolT> CodeStubAssembler::IsJSAsyncGeneratorObject(
6215 : SloppyTNode<HeapObject> object) {
6216 0 : return HasInstanceType(object, JS_ASYNC_GENERATOR_OBJECT_TYPE);
6217 : }
6218 :
6219 0 : TNode<BoolT> CodeStubAssembler::IsContext(SloppyTNode<HeapObject> object) {
6220 0 : Node* instance_type = LoadInstanceType(object);
6221 : return UncheckedCast<BoolT>(Word32And(
6222 0 : Int32GreaterThanOrEqual(instance_type, Int32Constant(FIRST_CONTEXT_TYPE)),
6223 0 : Int32LessThanOrEqual(instance_type, Int32Constant(LAST_CONTEXT_TYPE))));
6224 : }
6225 :
6226 224 : TNode<BoolT> CodeStubAssembler::IsFixedArray(SloppyTNode<HeapObject> object) {
6227 224 : return HasInstanceType(object, FIXED_ARRAY_TYPE);
6228 : }
6229 :
6230 0 : TNode<BoolT> CodeStubAssembler::IsFixedArraySubclass(
6231 : SloppyTNode<HeapObject> object) {
6232 0 : Node* instance_type = LoadInstanceType(object);
6233 : return UncheckedCast<BoolT>(
6234 0 : Word32And(Int32GreaterThanOrEqual(instance_type,
6235 0 : Int32Constant(FIRST_FIXED_ARRAY_TYPE)),
6236 0 : Int32LessThanOrEqual(instance_type,
6237 0 : Int32Constant(LAST_FIXED_ARRAY_TYPE))));
6238 : }
6239 :
6240 0 : TNode<BoolT> CodeStubAssembler::IsNotWeakFixedArraySubclass(
6241 : SloppyTNode<HeapObject> object) {
6242 0 : Node* instance_type = LoadInstanceType(object);
6243 : return UncheckedCast<BoolT>(Word32Or(
6244 0 : Int32LessThan(instance_type, Int32Constant(FIRST_WEAK_FIXED_ARRAY_TYPE)),
6245 0 : Int32GreaterThan(instance_type,
6246 0 : Int32Constant(LAST_WEAK_FIXED_ARRAY_TYPE))));
6247 : }
6248 :
6249 392 : TNode<BoolT> CodeStubAssembler::IsPromiseCapability(
6250 : SloppyTNode<HeapObject> object) {
6251 392 : return HasInstanceType(object, PROMISE_CAPABILITY_TYPE);
6252 : }
6253 :
6254 0 : TNode<BoolT> CodeStubAssembler::IsPropertyArray(
6255 : SloppyTNode<HeapObject> object) {
6256 0 : return HasInstanceType(object, PROPERTY_ARRAY_TYPE);
6257 : }
6258 :
6259 : // This complicated check is due to elements oddities. If a smi array is empty
6260 : // after Array.p.shift, it is replaced by the empty array constant. If it is
6261 : // later filled with a double element, we try to grow it but pass in a double
6262 : // elements kind. Usually this would cause a size mismatch (since the source
6263 : // fixed array has HOLEY_ELEMENTS and destination has
6264 : // HOLEY_DOUBLE_ELEMENTS), but we don't have to worry about it when the
6265 : // source array is empty.
6266 : // TODO(jgruber): It might we worth creating an empty_double_array constant to
6267 : // simplify this case.
6268 0 : TNode<BoolT> CodeStubAssembler::IsFixedArrayWithKindOrEmpty(
6269 : SloppyTNode<HeapObject> object, ElementsKind kind) {
6270 0 : Label out(this);
6271 0 : TVARIABLE(BoolT, var_result, Int32TrueConstant());
6272 :
6273 0 : GotoIf(IsFixedArrayWithKind(object, kind), &out);
6274 :
6275 0 : TNode<Smi> const length = LoadFixedArrayBaseLength(CAST(object));
6276 0 : GotoIf(SmiEqual(length, SmiConstant(0)), &out);
6277 :
6278 0 : var_result = Int32FalseConstant();
6279 0 : Goto(&out);
6280 :
6281 0 : BIND(&out);
6282 0 : return var_result.value();
6283 : }
6284 :
6285 0 : TNode<BoolT> CodeStubAssembler::IsFixedArrayWithKind(
6286 : SloppyTNode<HeapObject> object, ElementsKind kind) {
6287 0 : if (IsDoubleElementsKind(kind)) {
6288 0 : return IsFixedDoubleArray(object);
6289 : } else {
6290 : DCHECK(IsSmiOrObjectElementsKind(kind));
6291 0 : return IsFixedArraySubclass(object);
6292 : }
6293 : }
6294 :
6295 168 : TNode<BoolT> CodeStubAssembler::IsBoolean(SloppyTNode<HeapObject> object) {
6296 168 : return IsBooleanMap(LoadMap(object));
6297 : }
6298 :
6299 0 : TNode<BoolT> CodeStubAssembler::IsPropertyCell(SloppyTNode<HeapObject> object) {
6300 0 : return IsPropertyCellMap(LoadMap(object));
6301 : }
6302 :
6303 336 : TNode<BoolT> CodeStubAssembler::IsAccessorInfo(SloppyTNode<HeapObject> object) {
6304 336 : return IsAccessorInfoMap(LoadMap(object));
6305 : }
6306 :
6307 3980 : TNode<BoolT> CodeStubAssembler::IsAccessorPair(SloppyTNode<HeapObject> object) {
6308 3980 : return IsAccessorPairMap(LoadMap(object));
6309 : }
6310 :
6311 168 : TNode<BoolT> CodeStubAssembler::IsAllocationSite(
6312 : SloppyTNode<HeapObject> object) {
6313 168 : return IsAllocationSiteInstanceType(LoadInstanceType(object));
6314 : }
6315 :
6316 0 : TNode<BoolT> CodeStubAssembler::IsAnyHeapNumber(
6317 : SloppyTNode<HeapObject> object) {
6318 : return UncheckedCast<BoolT>(
6319 0 : Word32Or(IsMutableHeapNumber(object), IsHeapNumber(object)));
6320 : }
6321 :
6322 31556 : TNode<BoolT> CodeStubAssembler::IsHeapNumber(SloppyTNode<HeapObject> object) {
6323 31556 : return IsHeapNumberMap(LoadMap(object));
6324 : }
6325 :
6326 452 : TNode<BoolT> CodeStubAssembler::IsHeapNumberInstanceType(
6327 : SloppyTNode<Int32T> instance_type) {
6328 452 : return InstanceTypeEqual(instance_type, HEAP_NUMBER_TYPE);
6329 : }
6330 :
6331 0 : TNode<BoolT> CodeStubAssembler::IsOddball(SloppyTNode<HeapObject> object) {
6332 0 : return IsOddballInstanceType(LoadInstanceType(object));
6333 : }
6334 :
6335 1068 : TNode<BoolT> CodeStubAssembler::IsOddballInstanceType(
6336 : SloppyTNode<Int32T> instance_type) {
6337 1068 : return InstanceTypeEqual(instance_type, ODDBALL_TYPE);
6338 : }
6339 :
6340 560 : TNode<BoolT> CodeStubAssembler::IsMutableHeapNumber(
6341 : SloppyTNode<HeapObject> object) {
6342 560 : return IsMutableHeapNumberMap(LoadMap(object));
6343 : }
6344 :
6345 0 : TNode<BoolT> CodeStubAssembler::IsFeedbackCell(SloppyTNode<HeapObject> object) {
6346 0 : return HasInstanceType(object, FEEDBACK_CELL_TYPE);
6347 : }
6348 :
6349 12612 : TNode<BoolT> CodeStubAssembler::IsFeedbackVector(
6350 : SloppyTNode<HeapObject> object) {
6351 12612 : return IsFeedbackVectorMap(LoadMap(object));
6352 : }
6353 :
6354 56 : TNode<BoolT> CodeStubAssembler::IsName(SloppyTNode<HeapObject> object) {
6355 56 : return IsNameInstanceType(LoadInstanceType(object));
6356 : }
6357 :
6358 112 : TNode<BoolT> CodeStubAssembler::IsNameInstanceType(
6359 : SloppyTNode<Int32T> instance_type) {
6360 112 : return Int32LessThanOrEqual(instance_type, Int32Constant(LAST_NAME_TYPE));
6361 : }
6362 :
6363 9300 : TNode<BoolT> CodeStubAssembler::IsString(SloppyTNode<HeapObject> object) {
6364 9300 : return IsStringInstanceType(LoadInstanceType(object));
6365 : }
6366 :
6367 1068 : TNode<BoolT> CodeStubAssembler::IsSymbolInstanceType(
6368 : SloppyTNode<Int32T> instance_type) {
6369 1068 : return InstanceTypeEqual(instance_type, SYMBOL_TYPE);
6370 : }
6371 :
6372 2920 : TNode<BoolT> CodeStubAssembler::IsSymbol(SloppyTNode<HeapObject> object) {
6373 2920 : return IsSymbolMap(LoadMap(object));
6374 : }
6375 :
6376 168 : TNode<BoolT> CodeStubAssembler::IsInternalizedStringInstanceType(
6377 : TNode<Int32T> instance_type) {
6378 : STATIC_ASSERT(kNotInternalizedTag != 0);
6379 : return Word32Equal(
6380 336 : Word32And(instance_type,
6381 336 : Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
6382 504 : Int32Constant(kStringTag | kInternalizedTag));
6383 : }
6384 :
6385 0 : TNode<BoolT> CodeStubAssembler::IsUniqueName(TNode<HeapObject> object) {
6386 0 : TNode<Int32T> instance_type = LoadInstanceType(object);
6387 : return Select<BoolT>(
6388 0 : IsInternalizedStringInstanceType(instance_type),
6389 0 : [=] { return Int32TrueConstant(); },
6390 0 : [=] { return IsSymbolInstanceType(instance_type); });
6391 : }
6392 :
6393 168 : TNode<BoolT> CodeStubAssembler::IsUniqueNameNoIndex(TNode<HeapObject> object) {
6394 168 : TNode<Int32T> instance_type = LoadInstanceType(object);
6395 : return Select<BoolT>(
6396 336 : IsInternalizedStringInstanceType(instance_type),
6397 168 : [=] {
6398 504 : return IsSetWord32(LoadNameHashField(CAST(object)),
6399 336 : Name::kIsNotArrayIndexMask);
6400 336 : },
6401 672 : [=] { return IsSymbolInstanceType(instance_type); });
6402 : }
6403 :
6404 29404 : TNode<BoolT> CodeStubAssembler::IsBigIntInstanceType(
6405 : SloppyTNode<Int32T> instance_type) {
6406 29404 : return InstanceTypeEqual(instance_type, BIGINT_TYPE);
6407 : }
6408 :
6409 10472 : TNode<BoolT> CodeStubAssembler::IsBigInt(SloppyTNode<HeapObject> object) {
6410 10472 : return IsBigIntInstanceType(LoadInstanceType(object));
6411 : }
6412 :
6413 448 : TNode<BoolT> CodeStubAssembler::IsPrimitiveInstanceType(
6414 : SloppyTNode<Int32T> instance_type) {
6415 : return Int32LessThanOrEqual(instance_type,
6416 448 : Int32Constant(LAST_PRIMITIVE_TYPE));
6417 : }
6418 :
6419 2636 : TNode<BoolT> CodeStubAssembler::IsPrivateSymbol(
6420 : SloppyTNode<HeapObject> object) {
6421 5272 : return Select<BoolT>(IsSymbol(object),
6422 2636 : [=] {
6423 7908 : TNode<Symbol> symbol = CAST(object);
6424 : TNode<Uint32T> flags = LoadObjectField<Uint32T>(
6425 5272 : symbol, Symbol::kFlagsOffset);
6426 5272 : return IsSetWord32<Symbol::IsPrivateBit>(flags);
6427 : },
6428 10544 : [=] { return Int32FalseConstant(); });
6429 : }
6430 :
6431 56 : TNode<BoolT> CodeStubAssembler::IsNativeContext(
6432 : SloppyTNode<HeapObject> object) {
6433 56 : return WordEqual(LoadMap(object), LoadRoot(RootIndex::kNativeContextMap));
6434 : }
6435 :
6436 112 : TNode<BoolT> CodeStubAssembler::IsFixedDoubleArray(
6437 : SloppyTNode<HeapObject> object) {
6438 112 : return WordEqual(LoadMap(object), FixedDoubleArrayMapConstant());
6439 : }
6440 :
6441 0 : TNode<BoolT> CodeStubAssembler::IsHashTable(SloppyTNode<HeapObject> object) {
6442 0 : Node* instance_type = LoadInstanceType(object);
6443 : return UncheckedCast<BoolT>(
6444 0 : Word32And(Int32GreaterThanOrEqual(instance_type,
6445 0 : Int32Constant(FIRST_HASH_TABLE_TYPE)),
6446 0 : Int32LessThanOrEqual(instance_type,
6447 0 : Int32Constant(LAST_HASH_TABLE_TYPE))));
6448 : }
6449 :
6450 0 : TNode<BoolT> CodeStubAssembler::IsEphemeronHashTable(
6451 : SloppyTNode<HeapObject> object) {
6452 0 : return HasInstanceType(object, EPHEMERON_HASH_TABLE_TYPE);
6453 : }
6454 :
6455 0 : TNode<BoolT> CodeStubAssembler::IsNameDictionary(
6456 : SloppyTNode<HeapObject> object) {
6457 0 : return HasInstanceType(object, NAME_DICTIONARY_TYPE);
6458 : }
6459 :
6460 0 : TNode<BoolT> CodeStubAssembler::IsGlobalDictionary(
6461 : SloppyTNode<HeapObject> object) {
6462 0 : return HasInstanceType(object, GLOBAL_DICTIONARY_TYPE);
6463 : }
6464 :
6465 0 : TNode<BoolT> CodeStubAssembler::IsNumberDictionary(
6466 : SloppyTNode<HeapObject> object) {
6467 0 : return HasInstanceType(object, NUMBER_DICTIONARY_TYPE);
6468 : }
6469 :
6470 0 : TNode<BoolT> CodeStubAssembler::IsJSGeneratorObject(
6471 : SloppyTNode<HeapObject> object) {
6472 0 : return HasInstanceType(object, JS_GENERATOR_OBJECT_TYPE);
6473 : }
6474 :
6475 3868 : TNode<BoolT> CodeStubAssembler::IsJSFunctionInstanceType(
6476 : SloppyTNode<Int32T> instance_type) {
6477 3868 : return InstanceTypeEqual(instance_type, JS_FUNCTION_TYPE);
6478 : }
6479 :
6480 168 : TNode<BoolT> CodeStubAssembler::IsAllocationSiteInstanceType(
6481 : SloppyTNode<Int32T> instance_type) {
6482 168 : return InstanceTypeEqual(instance_type, ALLOCATION_SITE_TYPE);
6483 : }
6484 :
6485 56 : TNode<BoolT> CodeStubAssembler::IsJSFunction(SloppyTNode<HeapObject> object) {
6486 56 : return IsJSFunctionMap(LoadMap(object));
6487 : }
6488 :
6489 56 : TNode<BoolT> CodeStubAssembler::IsJSFunctionMap(SloppyTNode<Map> map) {
6490 56 : return IsJSFunctionInstanceType(LoadMapInstanceType(map));
6491 : }
6492 :
6493 1064 : TNode<BoolT> CodeStubAssembler::IsJSTypedArray(SloppyTNode<HeapObject> object) {
6494 1064 : return HasInstanceType(object, JS_TYPED_ARRAY_TYPE);
6495 : }
6496 :
6497 168 : TNode<BoolT> CodeStubAssembler::IsJSArrayBuffer(
6498 : SloppyTNode<HeapObject> object) {
6499 168 : return HasInstanceType(object, JS_ARRAY_BUFFER_TYPE);
6500 : }
6501 :
6502 1288 : TNode<BoolT> CodeStubAssembler::IsJSDataView(TNode<HeapObject> object) {
6503 1288 : return HasInstanceType(object, JS_DATA_VIEW_TYPE);
6504 : }
6505 :
6506 0 : TNode<BoolT> CodeStubAssembler::IsFixedTypedArray(
6507 : SloppyTNode<HeapObject> object) {
6508 0 : TNode<Int32T> instance_type = LoadInstanceType(object);
6509 : return UncheckedCast<BoolT>(Word32And(
6510 0 : Int32GreaterThanOrEqual(instance_type,
6511 0 : Int32Constant(FIRST_FIXED_TYPED_ARRAY_TYPE)),
6512 0 : Int32LessThanOrEqual(instance_type,
6513 0 : Int32Constant(LAST_FIXED_TYPED_ARRAY_TYPE))));
6514 : }
6515 :
6516 1008 : TNode<BoolT> CodeStubAssembler::IsJSRegExp(SloppyTNode<HeapObject> object) {
6517 1008 : return HasInstanceType(object, JS_REGEXP_TYPE);
6518 : }
6519 :
6520 3644 : TNode<BoolT> CodeStubAssembler::IsNumber(SloppyTNode<Object> object) {
6521 10932 : return Select<BoolT>(TaggedIsSmi(object), [=] { return Int32TrueConstant(); },
6522 14576 : [=] { return IsHeapNumber(CAST(object)); });
6523 : }
6524 :
6525 112 : TNode<BoolT> CodeStubAssembler::IsNumeric(SloppyTNode<Object> object) {
6526 : return Select<BoolT>(
6527 336 : TaggedIsSmi(object), [=] { return Int32TrueConstant(); },
6528 112 : [=] {
6529 : return UncheckedCast<BoolT>(
6530 224 : Word32Or(IsHeapNumber(CAST(object)), IsBigInt(CAST(object))));
6531 560 : });
6532 : }
6533 :
6534 0 : TNode<BoolT> CodeStubAssembler::IsNumberNormalized(SloppyTNode<Number> number) {
6535 0 : TVARIABLE(BoolT, var_result, Int32TrueConstant());
6536 0 : Label out(this);
6537 :
6538 0 : GotoIf(TaggedIsSmi(number), &out);
6539 :
6540 0 : TNode<Float64T> value = LoadHeapNumberValue(CAST(number));
6541 : TNode<Float64T> smi_min =
6542 0 : Float64Constant(static_cast<double>(Smi::kMinValue));
6543 : TNode<Float64T> smi_max =
6544 0 : Float64Constant(static_cast<double>(Smi::kMaxValue));
6545 :
6546 0 : GotoIf(Float64LessThan(value, smi_min), &out);
6547 0 : GotoIf(Float64GreaterThan(value, smi_max), &out);
6548 0 : GotoIfNot(Float64Equal(value, value), &out); // NaN.
6549 :
6550 0 : var_result = Int32FalseConstant();
6551 0 : Goto(&out);
6552 :
6553 0 : BIND(&out);
6554 0 : return var_result.value();
6555 : }
6556 :
6557 0 : TNode<BoolT> CodeStubAssembler::IsNumberPositive(SloppyTNode<Number> number) {
6558 0 : return Select<BoolT>(TaggedIsSmi(number),
6559 0 : [=] { return TaggedIsPositiveSmi(number); },
6560 0 : [=] { return IsHeapNumberPositive(CAST(number)); });
6561 : }
6562 :
6563 : // TODO(cbruni): Use TNode<HeapNumber> instead of custom name.
6564 4 : TNode<BoolT> CodeStubAssembler::IsHeapNumberPositive(TNode<HeapNumber> number) {
6565 4 : TNode<Float64T> value = LoadHeapNumberValue(number);
6566 4 : TNode<Float64T> float_zero = Float64Constant(0.);
6567 4 : return Float64GreaterThanOrEqual(value, float_zero);
6568 : }
6569 :
6570 0 : TNode<BoolT> CodeStubAssembler::IsNumberNonNegativeSafeInteger(
6571 : TNode<Number> number) {
6572 : return Select<BoolT>(
6573 : // TODO(cbruni): Introduce TaggedIsNonNegateSmi to avoid confusion.
6574 0 : TaggedIsSmi(number), [=] { return TaggedIsPositiveSmi(number); },
6575 0 : [=] {
6576 0 : TNode<HeapNumber> heap_number = CAST(number);
6577 0 : return Select<BoolT>(IsInteger(heap_number),
6578 0 : [=] { return IsHeapNumberPositive(heap_number); },
6579 0 : [=] { return Int32FalseConstant(); });
6580 0 : });
6581 : }
6582 :
6583 56 : TNode<BoolT> CodeStubAssembler::IsSafeInteger(TNode<Object> number) {
6584 : return Select<BoolT>(
6585 168 : TaggedIsSmi(number), [=] { return Int32TrueConstant(); },
6586 56 : [=] {
6587 : return Select<BoolT>(
6588 280 : IsHeapNumber(CAST(number)),
6589 56 : [=] { return IsSafeInteger(UncheckedCast<HeapNumber>(number)); },
6590 336 : [=] { return Int32FalseConstant(); });
6591 280 : });
6592 : }
6593 :
6594 56 : TNode<BoolT> CodeStubAssembler::IsSafeInteger(TNode<HeapNumber> number) {
6595 : // Load the actual value of {number}.
6596 56 : TNode<Float64T> number_value = LoadHeapNumberValue(number);
6597 : // Truncate the value of {number} to an integer (or an infinity).
6598 56 : TNode<Float64T> integer = Float64Trunc(number_value);
6599 :
6600 : return Select<BoolT>(
6601 : // Check if {number}s value matches the integer (ruling out the
6602 : // infinities).
6603 112 : Float64Equal(Float64Sub(number_value, integer), Float64Constant(0.0)),
6604 56 : [=] {
6605 : // Check if the {integer} value is in safe integer range.
6606 224 : return Float64LessThanOrEqual(Float64Abs(integer),
6607 280 : Float64Constant(kMaxSafeInteger));
6608 112 : },
6609 224 : [=] { return Int32FalseConstant(); });
6610 : }
6611 :
6612 56 : TNode<BoolT> CodeStubAssembler::IsInteger(TNode<Object> number) {
6613 : return Select<BoolT>(
6614 168 : TaggedIsSmi(number), [=] { return Int32TrueConstant(); },
6615 56 : [=] {
6616 : return Select<BoolT>(
6617 280 : IsHeapNumber(CAST(number)),
6618 56 : [=] { return IsInteger(UncheckedCast<HeapNumber>(number)); },
6619 336 : [=] { return Int32FalseConstant(); });
6620 280 : });
6621 : }
6622 :
6623 56 : TNode<BoolT> CodeStubAssembler::IsInteger(TNode<HeapNumber> number) {
6624 56 : TNode<Float64T> number_value = LoadHeapNumberValue(number);
6625 : // Truncate the value of {number} to an integer (or an infinity).
6626 56 : TNode<Float64T> integer = Float64Trunc(number_value);
6627 : // Check if {number}s value matches the integer (ruling out the infinities).
6628 56 : return Float64Equal(Float64Sub(number_value, integer), Float64Constant(0.0));
6629 : }
6630 :
6631 4 : TNode<BoolT> CodeStubAssembler::IsHeapNumberUint32(TNode<HeapNumber> number) {
6632 : // Check that the HeapNumber is a valid uint32
6633 : return Select<BoolT>(
6634 8 : IsHeapNumberPositive(number),
6635 4 : [=] {
6636 20 : TNode<Float64T> value = LoadHeapNumberValue(number);
6637 12 : TNode<Uint32T> int_value = Unsigned(TruncateFloat64ToWord32(value));
6638 12 : return Float64Equal(value, ChangeUint32ToFloat64(int_value));
6639 : },
6640 16 : [=] { return Int32FalseConstant(); });
6641 : }
6642 :
6643 4 : TNode<BoolT> CodeStubAssembler::IsNumberArrayIndex(TNode<Number> number) {
6644 8 : return Select<BoolT>(TaggedIsSmi(number),
6645 4 : [=] { return TaggedIsPositiveSmi(number); },
6646 16 : [=] { return IsHeapNumberUint32(CAST(number)); });
6647 : }
6648 :
6649 224 : Node* CodeStubAssembler::FixedArraySizeDoesntFitInNewSpace(Node* element_count,
6650 : int base_size,
6651 : ParameterMode mode) {
6652 : int max_newspace_elements =
6653 224 : (kMaxRegularHeapObjectSize - base_size) / kTaggedSize;
6654 224 : return IntPtrOrSmiGreaterThan(
6655 224 : element_count, IntPtrOrSmiConstant(max_newspace_elements, mode), mode);
6656 : }
6657 :
6658 2856 : TNode<Int32T> CodeStubAssembler::StringCharCodeAt(SloppyTNode<String> string,
6659 : SloppyTNode<IntPtrT> index) {
6660 : CSA_ASSERT(this, IsString(string));
6661 :
6662 : CSA_ASSERT(this, IntPtrGreaterThanOrEqual(index, IntPtrConstant(0)));
6663 : CSA_ASSERT(this, IntPtrLessThan(index, LoadStringLengthAsWord(string)));
6664 :
6665 5712 : TVARIABLE(Int32T, var_result);
6666 :
6667 5712 : Label return_result(this), if_runtime(this, Label::kDeferred),
6668 5712 : if_stringistwobyte(this), if_stringisonebyte(this);
6669 :
6670 5712 : ToDirectStringAssembler to_direct(state(), string);
6671 2856 : to_direct.TryToDirect(&if_runtime);
6672 2856 : Node* const offset = IntPtrAdd(index, to_direct.offset());
6673 2856 : Node* const instance_type = to_direct.instance_type();
6674 :
6675 2856 : Node* const string_data = to_direct.PointerToData(&if_runtime);
6676 :
6677 : // Check if the {string} is a TwoByteSeqString or a OneByteSeqString.
6678 5712 : Branch(IsOneByteStringInstanceType(instance_type), &if_stringisonebyte,
6679 2856 : &if_stringistwobyte);
6680 :
6681 2856 : BIND(&if_stringisonebyte);
6682 : {
6683 2856 : var_result =
6684 2856 : UncheckedCast<Int32T>(Load(MachineType::Uint8(), string_data, offset));
6685 2856 : Goto(&return_result);
6686 : }
6687 :
6688 2856 : BIND(&if_stringistwobyte);
6689 : {
6690 2856 : var_result =
6691 : UncheckedCast<Int32T>(Load(MachineType::Uint16(), string_data,
6692 5712 : WordShl(offset, IntPtrConstant(1))));
6693 2856 : Goto(&return_result);
6694 : }
6695 :
6696 2856 : BIND(&if_runtime);
6697 : {
6698 11424 : Node* result = CallRuntime(Runtime::kStringCharCodeAt, NoContextConstant(),
6699 11424 : string, SmiTag(index));
6700 2856 : var_result = SmiToInt32(result);
6701 2856 : Goto(&return_result);
6702 : }
6703 :
6704 2856 : BIND(&return_result);
6705 5712 : return var_result.value();
6706 : }
6707 :
6708 784 : TNode<String> CodeStubAssembler::StringFromSingleCharCode(TNode<Int32T> code) {
6709 1568 : VARIABLE(var_result, MachineRepresentation::kTagged);
6710 :
6711 : // Check if the {code} is a one-byte char code.
6712 1568 : Label if_codeisonebyte(this), if_codeistwobyte(this, Label::kDeferred),
6713 1568 : if_done(this);
6714 1568 : Branch(Int32LessThanOrEqual(code, Int32Constant(String::kMaxOneByteCharCode)),
6715 784 : &if_codeisonebyte, &if_codeistwobyte);
6716 784 : BIND(&if_codeisonebyte);
6717 : {
6718 : // Load the isolate wide single character string cache.
6719 : TNode<FixedArray> cache =
6720 784 : CAST(LoadRoot(RootIndex::kSingleCharacterStringCache));
6721 784 : TNode<IntPtrT> code_index = Signed(ChangeUint32ToWord(code));
6722 :
6723 : // Check if we have an entry for the {code} in the single character string
6724 : // cache already.
6725 1568 : Label if_entryisundefined(this, Label::kDeferred),
6726 1568 : if_entryisnotundefined(this);
6727 784 : Node* entry = UnsafeLoadFixedArrayElement(cache, code_index);
6728 784 : Branch(IsUndefined(entry), &if_entryisundefined, &if_entryisnotundefined);
6729 :
6730 784 : BIND(&if_entryisundefined);
6731 : {
6732 : // Allocate a new SeqOneByteString for {code} and store it in the {cache}.
6733 784 : TNode<String> result = AllocateSeqOneByteString(1);
6734 1568 : StoreNoWriteBarrier(
6735 : MachineRepresentation::kWord8, result,
6736 1568 : IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag), code);
6737 784 : StoreFixedArrayElement(cache, code_index, result);
6738 784 : var_result.Bind(result);
6739 784 : Goto(&if_done);
6740 : }
6741 :
6742 784 : BIND(&if_entryisnotundefined);
6743 : {
6744 : // Return the entry from the {cache}.
6745 784 : var_result.Bind(entry);
6746 784 : Goto(&if_done);
6747 : }
6748 : }
6749 :
6750 784 : BIND(&if_codeistwobyte);
6751 : {
6752 : // Allocate a new SeqTwoByteString for {code}.
6753 784 : Node* result = AllocateSeqTwoByteString(1);
6754 1568 : StoreNoWriteBarrier(
6755 : MachineRepresentation::kWord16, result,
6756 1568 : IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag), code);
6757 784 : var_result.Bind(result);
6758 784 : Goto(&if_done);
6759 : }
6760 :
6761 784 : BIND(&if_done);
6762 : CSA_ASSERT(this, IsString(var_result.value()));
6763 1568 : return CAST(var_result.value());
6764 : }
6765 :
6766 : // A wrapper around CopyStringCharacters which determines the correct string
6767 : // encoding, allocates a corresponding sequential string, and then copies the
6768 : // given character range using CopyStringCharacters.
6769 : // |from_string| must be a sequential string.
6770 : // 0 <= |from_index| <= |from_index| + |character_count| < from_string.length.
6771 896 : TNode<String> CodeStubAssembler::AllocAndCopyStringCharacters(
6772 : Node* from, Node* from_instance_type, TNode<IntPtrT> from_index,
6773 : TNode<IntPtrT> character_count) {
6774 1792 : Label end(this), one_byte_sequential(this), two_byte_sequential(this);
6775 1792 : TVARIABLE(String, var_result);
6776 :
6777 1792 : Branch(IsOneByteStringInstanceType(from_instance_type), &one_byte_sequential,
6778 896 : &two_byte_sequential);
6779 :
6780 : // The subject string is a sequential one-byte string.
6781 896 : BIND(&one_byte_sequential);
6782 : {
6783 : TNode<String> result = AllocateSeqOneByteString(
6784 896 : NoContextConstant(), Unsigned(TruncateIntPtrToInt32(character_count)));
6785 896 : CopyStringCharacters(from, result, from_index, IntPtrConstant(0),
6786 : character_count, String::ONE_BYTE_ENCODING,
6787 896 : String::ONE_BYTE_ENCODING);
6788 896 : var_result = result;
6789 896 : Goto(&end);
6790 : }
6791 :
6792 : // The subject string is a sequential two-byte string.
6793 896 : BIND(&two_byte_sequential);
6794 : {
6795 : TNode<String> result = AllocateSeqTwoByteString(
6796 896 : NoContextConstant(), Unsigned(TruncateIntPtrToInt32(character_count)));
6797 896 : CopyStringCharacters(from, result, from_index, IntPtrConstant(0),
6798 : character_count, String::TWO_BYTE_ENCODING,
6799 896 : String::TWO_BYTE_ENCODING);
6800 896 : var_result = result;
6801 896 : Goto(&end);
6802 : }
6803 :
6804 896 : BIND(&end);
6805 1792 : return var_result.value();
6806 : }
6807 :
6808 448 : TNode<String> CodeStubAssembler::SubString(TNode<String> string,
6809 : TNode<IntPtrT> from,
6810 : TNode<IntPtrT> to) {
6811 896 : TVARIABLE(String, var_result);
6812 896 : ToDirectStringAssembler to_direct(state(), string);
6813 896 : Label end(this), runtime(this);
6814 :
6815 448 : TNode<IntPtrT> const substr_length = IntPtrSub(to, from);
6816 448 : TNode<IntPtrT> const string_length = LoadStringLengthAsWord(string);
6817 :
6818 : // Begin dispatching based on substring length.
6819 :
6820 896 : Label original_string_or_invalid_length(this);
6821 896 : GotoIf(UintPtrGreaterThanOrEqual(substr_length, string_length),
6822 448 : &original_string_or_invalid_length);
6823 :
6824 : // A real substring (substr_length < string_length).
6825 896 : Label empty(this);
6826 448 : GotoIf(IntPtrEqual(substr_length, IntPtrConstant(0)), &empty);
6827 :
6828 896 : Label single_char(this);
6829 448 : GotoIf(IntPtrEqual(substr_length, IntPtrConstant(1)), &single_char);
6830 :
6831 : // Deal with different string types: update the index if necessary
6832 : // and extract the underlying string.
6833 :
6834 448 : TNode<String> direct_string = to_direct.TryToDirect(&runtime);
6835 448 : TNode<IntPtrT> offset = IntPtrAdd(from, to_direct.offset());
6836 448 : Node* const instance_type = to_direct.instance_type();
6837 :
6838 : // The subject string can only be external or sequential string of either
6839 : // encoding at this point.
6840 896 : Label external_string(this);
6841 : {
6842 : if (FLAG_string_slices) {
6843 896 : Label next(this);
6844 :
6845 : // Short slice. Copy instead of slicing.
6846 896 : GotoIf(IntPtrLessThan(substr_length,
6847 896 : IntPtrConstant(SlicedString::kMinLength)),
6848 448 : &next);
6849 :
6850 : // Allocate new sliced string.
6851 :
6852 448 : Counters* counters = isolate()->counters();
6853 448 : IncrementCounter(counters->sub_string_native(), 1);
6854 :
6855 896 : Label one_byte_slice(this), two_byte_slice(this);
6856 896 : Branch(IsOneByteStringInstanceType(to_direct.instance_type()),
6857 448 : &one_byte_slice, &two_byte_slice);
6858 :
6859 448 : BIND(&one_byte_slice);
6860 : {
6861 1792 : var_result = AllocateSlicedOneByteString(
6862 896 : Unsigned(TruncateIntPtrToInt32(substr_length)), direct_string,
6863 448 : SmiTag(offset));
6864 448 : Goto(&end);
6865 : }
6866 :
6867 448 : BIND(&two_byte_slice);
6868 : {
6869 1792 : var_result = AllocateSlicedTwoByteString(
6870 896 : Unsigned(TruncateIntPtrToInt32(substr_length)), direct_string,
6871 448 : SmiTag(offset));
6872 448 : Goto(&end);
6873 : }
6874 :
6875 448 : BIND(&next);
6876 : }
6877 :
6878 : // The subject string can only be external or sequential string of either
6879 : // encoding at this point.
6880 448 : GotoIf(to_direct.is_external(), &external_string);
6881 :
6882 448 : var_result = AllocAndCopyStringCharacters(direct_string, instance_type,
6883 448 : offset, substr_length);
6884 :
6885 448 : Counters* counters = isolate()->counters();
6886 448 : IncrementCounter(counters->sub_string_native(), 1);
6887 :
6888 448 : Goto(&end);
6889 : }
6890 :
6891 : // Handle external string.
6892 448 : BIND(&external_string);
6893 : {
6894 448 : Node* const fake_sequential_string = to_direct.PointerToString(&runtime);
6895 :
6896 448 : var_result = AllocAndCopyStringCharacters(
6897 448 : fake_sequential_string, instance_type, offset, substr_length);
6898 :
6899 448 : Counters* counters = isolate()->counters();
6900 448 : IncrementCounter(counters->sub_string_native(), 1);
6901 :
6902 448 : Goto(&end);
6903 : }
6904 :
6905 448 : BIND(&empty);
6906 : {
6907 448 : var_result = EmptyStringConstant();
6908 448 : Goto(&end);
6909 : }
6910 :
6911 : // Substrings of length 1 are generated through CharCodeAt and FromCharCode.
6912 448 : BIND(&single_char);
6913 : {
6914 448 : TNode<Int32T> char_code = StringCharCodeAt(string, from);
6915 448 : var_result = StringFromSingleCharCode(char_code);
6916 448 : Goto(&end);
6917 : }
6918 :
6919 448 : BIND(&original_string_or_invalid_length);
6920 : {
6921 : CSA_ASSERT(this, IntPtrEqual(substr_length, string_length));
6922 :
6923 : // Equal length - check if {from, to} == {0, str.length}.
6924 448 : GotoIf(UintPtrGreaterThan(from, IntPtrConstant(0)), &runtime);
6925 :
6926 : // Return the original string (substr_length == string_length).
6927 :
6928 448 : Counters* counters = isolate()->counters();
6929 448 : IncrementCounter(counters->sub_string_native(), 1);
6930 :
6931 448 : var_result = string;
6932 448 : Goto(&end);
6933 : }
6934 :
6935 : // Fall back to a runtime call.
6936 448 : BIND(&runtime);
6937 : {
6938 448 : var_result =
6939 896 : CAST(CallRuntime(Runtime::kStringSubstring, NoContextConstant(), string,
6940 448 : SmiTag(from), SmiTag(to)));
6941 448 : Goto(&end);
6942 : }
6943 :
6944 448 : BIND(&end);
6945 896 : return var_result.value();
6946 : }
6947 :
6948 4704 : ToDirectStringAssembler::ToDirectStringAssembler(
6949 : compiler::CodeAssemblerState* state, Node* string, Flags flags)
6950 : : CodeStubAssembler(state),
6951 : var_string_(this, MachineRepresentation::kTagged, string),
6952 : var_instance_type_(this, MachineRepresentation::kWord32),
6953 : var_offset_(this, MachineType::PointerRepresentation()),
6954 : var_is_external_(this, MachineRepresentation::kWord32),
6955 4704 : flags_(flags) {
6956 : CSA_ASSERT(this, TaggedIsNotSmi(string));
6957 : CSA_ASSERT(this, IsString(string));
6958 :
6959 4704 : var_string_.Bind(string);
6960 4704 : var_offset_.Bind(IntPtrConstant(0));
6961 4704 : var_instance_type_.Bind(LoadInstanceType(string));
6962 4704 : var_is_external_.Bind(Int32Constant(0));
6963 4704 : }
6964 :
6965 4704 : TNode<String> ToDirectStringAssembler::TryToDirect(Label* if_bailout) {
6966 9408 : VariableList vars({&var_string_, &var_offset_, &var_instance_type_}, zone());
6967 9408 : Label dispatch(this, vars);
6968 9408 : Label if_iscons(this);
6969 9408 : Label if_isexternal(this);
6970 9408 : Label if_issliced(this);
6971 9408 : Label if_isthin(this);
6972 9408 : Label out(this);
6973 :
6974 9408 : Branch(IsSequentialStringInstanceType(var_instance_type_.value()), &out,
6975 4704 : &dispatch);
6976 :
6977 : // Dispatch based on string representation.
6978 4704 : BIND(&dispatch);
6979 : {
6980 : int32_t values[] = {
6981 : kSeqStringTag, kConsStringTag, kExternalStringTag,
6982 : kSlicedStringTag, kThinStringTag,
6983 4704 : };
6984 : Label* labels[] = {
6985 : &out, &if_iscons, &if_isexternal, &if_issliced, &if_isthin,
6986 4704 : };
6987 : STATIC_ASSERT(arraysize(values) == arraysize(labels));
6988 :
6989 9408 : Node* const representation = Word32And(
6990 14112 : var_instance_type_.value(), Int32Constant(kStringRepresentationMask));
6991 4704 : Switch(representation, if_bailout, values, labels, arraysize(values));
6992 : }
6993 :
6994 : // Cons string. Check whether it is flat, then fetch first part.
6995 : // Flat cons strings have an empty second part.
6996 4704 : BIND(&if_iscons);
6997 : {
6998 4704 : Node* const string = var_string_.value();
6999 9408 : GotoIfNot(IsEmptyString(LoadObjectField(string, ConsString::kSecondOffset)),
7000 4704 : if_bailout);
7001 :
7002 4704 : Node* const lhs = LoadObjectField(string, ConsString::kFirstOffset);
7003 4704 : var_string_.Bind(lhs);
7004 4704 : var_instance_type_.Bind(LoadInstanceType(lhs));
7005 :
7006 4704 : Goto(&dispatch);
7007 : }
7008 :
7009 : // Sliced string. Fetch parent and correct start index by offset.
7010 4704 : BIND(&if_issliced);
7011 : {
7012 4704 : if (!FLAG_string_slices || (flags_ & kDontUnpackSlicedStrings)) {
7013 56 : Goto(if_bailout);
7014 : } else {
7015 4648 : Node* const string = var_string_.value();
7016 : Node* const sliced_offset =
7017 4648 : LoadAndUntagObjectField(string, SlicedString::kOffsetOffset);
7018 4648 : var_offset_.Bind(IntPtrAdd(var_offset_.value(), sliced_offset));
7019 :
7020 4648 : Node* const parent = LoadObjectField(string, SlicedString::kParentOffset);
7021 4648 : var_string_.Bind(parent);
7022 4648 : var_instance_type_.Bind(LoadInstanceType(parent));
7023 :
7024 4648 : Goto(&dispatch);
7025 : }
7026 : }
7027 :
7028 : // Thin string. Fetch the actual string.
7029 4704 : BIND(&if_isthin);
7030 : {
7031 4704 : Node* const string = var_string_.value();
7032 : Node* const actual_string =
7033 4704 : LoadObjectField(string, ThinString::kActualOffset);
7034 4704 : Node* const actual_instance_type = LoadInstanceType(actual_string);
7035 :
7036 4704 : var_string_.Bind(actual_string);
7037 4704 : var_instance_type_.Bind(actual_instance_type);
7038 :
7039 4704 : Goto(&dispatch);
7040 : }
7041 :
7042 : // External string.
7043 4704 : BIND(&if_isexternal);
7044 4704 : var_is_external_.Bind(Int32Constant(1));
7045 4704 : Goto(&out);
7046 :
7047 4704 : BIND(&out);
7048 9408 : return CAST(var_string_.value());
7049 : }
7050 :
7051 4704 : TNode<RawPtrT> ToDirectStringAssembler::TryToSequential(
7052 : StringPointerKind ptr_kind, Label* if_bailout) {
7053 4704 : CHECK(ptr_kind == PTR_TO_DATA || ptr_kind == PTR_TO_STRING);
7054 :
7055 9408 : TVARIABLE(RawPtrT, var_result);
7056 9408 : Label out(this), if_issequential(this), if_isexternal(this, Label::kDeferred);
7057 4704 : Branch(is_external(), &if_isexternal, &if_issequential);
7058 :
7059 4704 : BIND(&if_issequential);
7060 : {
7061 : STATIC_ASSERT(SeqOneByteString::kHeaderSize ==
7062 : SeqTwoByteString::kHeaderSize);
7063 4704 : TNode<IntPtrT> result = BitcastTaggedToWord(var_string_.value());
7064 4704 : if (ptr_kind == PTR_TO_DATA) {
7065 4256 : result = IntPtrAdd(result, IntPtrConstant(SeqOneByteString::kHeaderSize -
7066 4256 : kHeapObjectTag));
7067 : }
7068 4704 : var_result = ReinterpretCast<RawPtrT>(result);
7069 4704 : Goto(&out);
7070 : }
7071 :
7072 4704 : BIND(&if_isexternal);
7073 : {
7074 9408 : GotoIf(IsUncachedExternalStringInstanceType(var_instance_type_.value()),
7075 4704 : if_bailout);
7076 :
7077 4704 : TNode<String> string = CAST(var_string_.value());
7078 : TNode<IntPtrT> result =
7079 4704 : LoadObjectField<IntPtrT>(string, ExternalString::kResourceDataOffset);
7080 4704 : if (ptr_kind == PTR_TO_STRING) {
7081 448 : result = IntPtrSub(result, IntPtrConstant(SeqOneByteString::kHeaderSize -
7082 448 : kHeapObjectTag));
7083 : }
7084 4704 : var_result = ReinterpretCast<RawPtrT>(result);
7085 4704 : Goto(&out);
7086 : }
7087 :
7088 4704 : BIND(&out);
7089 9408 : return var_result.value();
7090 : }
7091 :
7092 2016 : void CodeStubAssembler::BranchIfCanDerefIndirectString(Node* string,
7093 : Node* instance_type,
7094 : Label* can_deref,
7095 : Label* cannot_deref) {
7096 : CSA_ASSERT(this, IsString(string));
7097 : Node* representation =
7098 2016 : Word32And(instance_type, Int32Constant(kStringRepresentationMask));
7099 2016 : GotoIf(Word32Equal(representation, Int32Constant(kThinStringTag)), can_deref);
7100 4032 : GotoIf(Word32NotEqual(representation, Int32Constant(kConsStringTag)),
7101 2016 : cannot_deref);
7102 : // Cons string.
7103 2016 : Node* rhs = LoadObjectField(string, ConsString::kSecondOffset);
7104 2016 : GotoIf(IsEmptyString(rhs), can_deref);
7105 2016 : Goto(cannot_deref);
7106 2016 : }
7107 :
7108 0 : Node* CodeStubAssembler::DerefIndirectString(TNode<String> string,
7109 : TNode<Int32T> instance_type,
7110 : Label* cannot_deref) {
7111 0 : Label deref(this);
7112 0 : BranchIfCanDerefIndirectString(string, instance_type, &deref, cannot_deref);
7113 0 : BIND(&deref);
7114 : STATIC_ASSERT(static_cast<int>(ThinString::kActualOffset) ==
7115 : static_cast<int>(ConsString::kFirstOffset));
7116 0 : return LoadObjectField(string, ThinString::kActualOffset);
7117 : }
7118 :
7119 2016 : void CodeStubAssembler::DerefIndirectString(Variable* var_string,
7120 : Node* instance_type) {
7121 : #ifdef DEBUG
7122 : Label can_deref(this), cannot_deref(this);
7123 : BranchIfCanDerefIndirectString(var_string->value(), instance_type, &can_deref,
7124 : &cannot_deref);
7125 : BIND(&cannot_deref);
7126 : DebugBreak(); // Should be able to dereference string.
7127 : Goto(&can_deref);
7128 : BIND(&can_deref);
7129 : #endif // DEBUG
7130 :
7131 : STATIC_ASSERT(static_cast<int>(ThinString::kActualOffset) ==
7132 : static_cast<int>(ConsString::kFirstOffset));
7133 2016 : var_string->Bind(
7134 4032 : LoadObjectField(var_string->value(), ThinString::kActualOffset));
7135 2016 : }
7136 :
7137 2016 : void CodeStubAssembler::MaybeDerefIndirectString(Variable* var_string,
7138 : Node* instance_type,
7139 : Label* did_deref,
7140 : Label* cannot_deref) {
7141 4032 : Label deref(this);
7142 2016 : BranchIfCanDerefIndirectString(var_string->value(), instance_type, &deref,
7143 2016 : cannot_deref);
7144 :
7145 2016 : BIND(&deref);
7146 : {
7147 2016 : DerefIndirectString(var_string, instance_type);
7148 2016 : Goto(did_deref);
7149 : }
7150 2016 : }
7151 :
7152 672 : void CodeStubAssembler::MaybeDerefIndirectStrings(Variable* var_left,
7153 : Node* left_instance_type,
7154 : Variable* var_right,
7155 : Node* right_instance_type,
7156 : Label* did_something) {
7157 1344 : Label did_nothing_left(this), did_something_left(this),
7158 1344 : didnt_do_anything(this);
7159 : MaybeDerefIndirectString(var_left, left_instance_type, &did_something_left,
7160 672 : &did_nothing_left);
7161 :
7162 672 : BIND(&did_something_left);
7163 : {
7164 : MaybeDerefIndirectString(var_right, right_instance_type, did_something,
7165 672 : did_something);
7166 : }
7167 :
7168 672 : BIND(&did_nothing_left);
7169 : {
7170 : MaybeDerefIndirectString(var_right, right_instance_type, did_something,
7171 672 : &didnt_do_anything);
7172 : }
7173 :
7174 672 : BIND(&didnt_do_anything);
7175 : // Fall through if neither string was an indirect string.
7176 672 : }
7177 :
7178 392 : TNode<String> CodeStubAssembler::StringAdd(Node* context, TNode<String> left,
7179 : TNode<String> right,
7180 : Variable* var_feedback) {
7181 784 : TVARIABLE(String, result);
7182 784 : Label check_right(this), runtime(this, Label::kDeferred), cons(this),
7183 784 : done(this, &result), done_native(this, &result);
7184 392 : Counters* counters = isolate()->counters();
7185 :
7186 : // Default to "String" feedback if we don't learn anything else below.
7187 392 : if (var_feedback != nullptr) {
7188 336 : var_feedback->Bind(SmiConstant(BinaryOperationFeedback::kString));
7189 : }
7190 :
7191 392 : TNode<Uint32T> left_length = LoadStringLengthAsWord32(left);
7192 392 : GotoIfNot(Word32Equal(left_length, Uint32Constant(0)), &check_right);
7193 392 : result = right;
7194 392 : Goto(&done_native);
7195 :
7196 392 : BIND(&check_right);
7197 392 : TNode<Uint32T> right_length = LoadStringLengthAsWord32(right);
7198 392 : GotoIfNot(Word32Equal(right_length, Uint32Constant(0)), &cons);
7199 392 : result = left;
7200 392 : Goto(&done_native);
7201 :
7202 392 : BIND(&cons);
7203 : {
7204 392 : TNode<Uint32T> new_length = Uint32Add(left_length, right_length);
7205 :
7206 : // If new length is greater than String::kMaxLength, goto runtime to
7207 : // throw. Note: we also need to invalidate the string length protector, so
7208 : // can't just throw here directly.
7209 784 : GotoIf(Uint32GreaterThan(new_length, Uint32Constant(String::kMaxLength)),
7210 392 : &runtime);
7211 :
7212 784 : TVARIABLE(String, var_left, left);
7213 784 : TVARIABLE(String, var_right, right);
7214 392 : Variable* input_vars[2] = {&var_left, &var_right};
7215 784 : Label non_cons(this, 2, input_vars);
7216 784 : Label slow(this, Label::kDeferred);
7217 784 : GotoIf(Uint32LessThan(new_length, Uint32Constant(ConsString::kMinLength)),
7218 392 : &non_cons);
7219 :
7220 392 : result = AllocateConsString(new_length, var_left.value(), var_right.value(),
7221 392 : var_feedback);
7222 392 : Goto(&done_native);
7223 :
7224 392 : BIND(&non_cons);
7225 :
7226 392 : Comment("Full string concatenate");
7227 392 : Node* left_instance_type = LoadInstanceType(var_left.value());
7228 392 : Node* right_instance_type = LoadInstanceType(var_right.value());
7229 : // Compute intersection and difference of instance types.
7230 :
7231 : Node* ored_instance_types =
7232 392 : Word32Or(left_instance_type, right_instance_type);
7233 : Node* xored_instance_types =
7234 392 : Word32Xor(left_instance_type, right_instance_type);
7235 :
7236 : // Check if both strings have the same encoding and both are sequential.
7237 392 : GotoIf(IsSetWord32(xored_instance_types, kStringEncodingMask), &runtime);
7238 392 : GotoIf(IsSetWord32(ored_instance_types, kStringRepresentationMask), &slow);
7239 :
7240 392 : TNode<IntPtrT> word_left_length = Signed(ChangeUint32ToWord(left_length));
7241 392 : TNode<IntPtrT> word_right_length = Signed(ChangeUint32ToWord(right_length));
7242 :
7243 784 : Label two_byte(this);
7244 1568 : GotoIf(Word32Equal(Word32And(ored_instance_types,
7245 784 : Int32Constant(kStringEncodingMask)),
7246 1568 : Int32Constant(kTwoByteStringTag)),
7247 392 : &two_byte);
7248 : // One-byte sequential string case
7249 392 : result = AllocateSeqOneByteString(context, new_length);
7250 784 : CopyStringCharacters(var_left.value(), result.value(), IntPtrConstant(0),
7251 : IntPtrConstant(0), word_left_length,
7252 392 : String::ONE_BYTE_ENCODING, String::ONE_BYTE_ENCODING);
7253 784 : CopyStringCharacters(var_right.value(), result.value(), IntPtrConstant(0),
7254 : word_left_length, word_right_length,
7255 392 : String::ONE_BYTE_ENCODING, String::ONE_BYTE_ENCODING);
7256 392 : Goto(&done_native);
7257 :
7258 392 : BIND(&two_byte);
7259 : {
7260 : // Two-byte sequential string case
7261 392 : result = AllocateSeqTwoByteString(context, new_length);
7262 784 : CopyStringCharacters(var_left.value(), result.value(), IntPtrConstant(0),
7263 : IntPtrConstant(0), word_left_length,
7264 : String::TWO_BYTE_ENCODING,
7265 392 : String::TWO_BYTE_ENCODING);
7266 784 : CopyStringCharacters(var_right.value(), result.value(), IntPtrConstant(0),
7267 : word_left_length, word_right_length,
7268 : String::TWO_BYTE_ENCODING,
7269 392 : String::TWO_BYTE_ENCODING);
7270 392 : Goto(&done_native);
7271 : }
7272 :
7273 392 : BIND(&slow);
7274 : {
7275 : // Try to unwrap indirect strings, restart the above attempt on success.
7276 : MaybeDerefIndirectStrings(&var_left, left_instance_type, &var_right,
7277 392 : right_instance_type, &non_cons);
7278 392 : Goto(&runtime);
7279 : }
7280 : }
7281 392 : BIND(&runtime);
7282 : {
7283 392 : result = CAST(CallRuntime(Runtime::kStringAdd, context, left, right));
7284 392 : Goto(&done);
7285 : }
7286 :
7287 392 : BIND(&done_native);
7288 : {
7289 392 : IncrementCounter(counters->string_add_native(), 1);
7290 392 : Goto(&done);
7291 : }
7292 :
7293 392 : BIND(&done);
7294 784 : return result.value();
7295 : }
7296 :
7297 112 : TNode<String> CodeStubAssembler::StringFromSingleCodePoint(
7298 : TNode<Int32T> codepoint, UnicodeEncoding encoding) {
7299 224 : VARIABLE(var_result, MachineRepresentation::kTagged, EmptyStringConstant());
7300 :
7301 224 : Label if_isword16(this), if_isword32(this), return_result(this);
7302 :
7303 224 : Branch(Uint32LessThan(codepoint, Int32Constant(0x10000)), &if_isword16,
7304 112 : &if_isword32);
7305 :
7306 112 : BIND(&if_isword16);
7307 : {
7308 112 : var_result.Bind(StringFromSingleCharCode(codepoint));
7309 112 : Goto(&return_result);
7310 : }
7311 :
7312 112 : BIND(&if_isword32);
7313 : {
7314 112 : switch (encoding) {
7315 : case UnicodeEncoding::UTF16:
7316 112 : break;
7317 : case UnicodeEncoding::UTF32: {
7318 : // Convert UTF32 to UTF16 code units, and store as a 32 bit word.
7319 0 : Node* lead_offset = Int32Constant(0xD800 - (0x10000 >> 10));
7320 :
7321 : // lead = (codepoint >> 10) + LEAD_OFFSET
7322 : Node* lead =
7323 0 : Int32Add(Word32Shr(codepoint, Int32Constant(10)), lead_offset);
7324 :
7325 : // trail = (codepoint & 0x3FF) + 0xDC00;
7326 0 : Node* trail = Int32Add(Word32And(codepoint, Int32Constant(0x3FF)),
7327 0 : Int32Constant(0xDC00));
7328 :
7329 : // codpoint = (trail << 16) | lead;
7330 0 : codepoint = Signed(Word32Or(Word32Shl(trail, Int32Constant(16)), lead));
7331 0 : break;
7332 : }
7333 : }
7334 :
7335 112 : Node* value = AllocateSeqTwoByteString(2);
7336 224 : StoreNoWriteBarrier(
7337 : MachineRepresentation::kWord32, value,
7338 224 : IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
7339 112 : codepoint);
7340 112 : var_result.Bind(value);
7341 112 : Goto(&return_result);
7342 : }
7343 :
7344 112 : BIND(&return_result);
7345 224 : return CAST(var_result.value());
7346 : }
7347 :
7348 900 : TNode<Number> CodeStubAssembler::StringToNumber(TNode<String> input) {
7349 1800 : Label runtime(this, Label::kDeferred);
7350 1800 : Label end(this);
7351 :
7352 1800 : TVARIABLE(Number, var_result);
7353 :
7354 : // Check if string has a cached array index.
7355 900 : TNode<Uint32T> hash = LoadNameHashField(input);
7356 1800 : GotoIf(IsSetWord32(hash, Name::kDoesNotContainCachedArrayIndexMask),
7357 900 : &runtime);
7358 :
7359 1800 : var_result =
7360 2700 : SmiTag(Signed(DecodeWordFromWord32<String::ArrayIndexValueBits>(hash)));
7361 900 : Goto(&end);
7362 :
7363 900 : BIND(&runtime);
7364 : {
7365 900 : var_result =
7366 1800 : CAST(CallRuntime(Runtime::kStringToNumber, NoContextConstant(), input));
7367 900 : Goto(&end);
7368 : }
7369 :
7370 900 : BIND(&end);
7371 1800 : return var_result.value();
7372 : }
7373 :
7374 340 : TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) {
7375 680 : TVARIABLE(String, result);
7376 680 : TVARIABLE(Smi, smi_input);
7377 680 : Label runtime(this, Label::kDeferred), if_smi(this), if_heap_number(this),
7378 680 : done(this, &result);
7379 :
7380 : // Load the number string cache.
7381 340 : Node* number_string_cache = LoadRoot(RootIndex::kNumberStringCache);
7382 :
7383 : // Make the hash mask from the length of the number string cache. It
7384 : // contains two elements (number and string) for each cache entry.
7385 : // TODO(ishell): cleanup mask handling.
7386 : Node* mask =
7387 340 : BitcastTaggedToWord(LoadFixedArrayBaseLength(number_string_cache));
7388 340 : TNode<IntPtrT> one = IntPtrConstant(1);
7389 340 : mask = IntPtrSub(mask, one);
7390 :
7391 340 : GotoIfNot(TaggedIsSmi(input), &if_heap_number);
7392 340 : smi_input = CAST(input);
7393 340 : Goto(&if_smi);
7394 :
7395 340 : BIND(&if_heap_number);
7396 : {
7397 340 : TNode<HeapNumber> heap_number_input = CAST(input);
7398 : // Try normalizing the HeapNumber.
7399 340 : TryHeapNumberToSmi(heap_number_input, smi_input, &if_smi);
7400 :
7401 : // Make a hash from the two 32-bit values of the double.
7402 : TNode<Int32T> low =
7403 340 : LoadObjectField<Int32T>(heap_number_input, HeapNumber::kValueOffset);
7404 : TNode<Int32T> high = LoadObjectField<Int32T>(
7405 340 : heap_number_input, HeapNumber::kValueOffset + kIntSize);
7406 340 : TNode<Word32T> hash = Word32Xor(low, high);
7407 340 : TNode<WordT> word_hash = WordShl(ChangeInt32ToIntPtr(hash), one);
7408 : TNode<WordT> index =
7409 340 : WordAnd(word_hash, WordSar(mask, SmiShiftBitsConstant()));
7410 :
7411 : // Cache entry's key must be a heap number
7412 : Node* number_key =
7413 340 : UnsafeLoadFixedArrayElement(CAST(number_string_cache), index);
7414 340 : GotoIf(TaggedIsSmi(number_key), &runtime);
7415 340 : GotoIfNot(IsHeapNumber(number_key), &runtime);
7416 :
7417 : // Cache entry's key must match the heap number value we're looking for.
7418 680 : Node* low_compare = LoadObjectField(number_key, HeapNumber::kValueOffset,
7419 340 : MachineType::Int32());
7420 680 : Node* high_compare = LoadObjectField(
7421 340 : number_key, HeapNumber::kValueOffset + kIntSize, MachineType::Int32());
7422 340 : GotoIfNot(Word32Equal(low, low_compare), &runtime);
7423 340 : GotoIfNot(Word32Equal(high, high_compare), &runtime);
7424 :
7425 : // Heap number match, return value from cache entry.
7426 680 : result = CAST(UnsafeLoadFixedArrayElement(CAST(number_string_cache), index,
7427 340 : kTaggedSize));
7428 340 : Goto(&done);
7429 : }
7430 :
7431 340 : BIND(&if_smi);
7432 : {
7433 : // Load the smi key, make sure it matches the smi we're looking for.
7434 680 : Node* smi_index = BitcastWordToTagged(
7435 1020 : WordAnd(WordShl(BitcastTaggedToWord(smi_input.value()), one), mask));
7436 1020 : Node* smi_key = UnsafeLoadFixedArrayElement(CAST(number_string_cache),
7437 680 : smi_index, 0, SMI_PARAMETERS);
7438 340 : GotoIf(WordNotEqual(smi_key, smi_input.value()), &runtime);
7439 :
7440 : // Smi match, return value from cache entry.
7441 680 : result = CAST(UnsafeLoadFixedArrayElement(
7442 340 : CAST(number_string_cache), smi_index, kTaggedSize, SMI_PARAMETERS));
7443 340 : Goto(&done);
7444 : }
7445 :
7446 340 : BIND(&runtime);
7447 : {
7448 : // No cache entry, go to the runtime.
7449 340 : result =
7450 680 : CAST(CallRuntime(Runtime::kNumberToString, NoContextConstant(), input));
7451 340 : Goto(&done);
7452 : }
7453 340 : BIND(&done);
7454 680 : return result.value();
7455 : }
7456 :
7457 844 : Node* CodeStubAssembler::NonNumberToNumberOrNumeric(
7458 : Node* context, Node* input, Object::Conversion mode,
7459 : BigIntHandling bigint_handling) {
7460 : CSA_ASSERT(this, Word32BinaryNot(TaggedIsSmi(input)));
7461 : CSA_ASSERT(this, Word32BinaryNot(IsHeapNumber(input)));
7462 :
7463 : // We might need to loop once here due to ToPrimitive conversions.
7464 1688 : VARIABLE(var_input, MachineRepresentation::kTagged, input);
7465 1688 : VARIABLE(var_result, MachineRepresentation::kTagged);
7466 1688 : Label loop(this, &var_input);
7467 1688 : Label end(this);
7468 844 : Goto(&loop);
7469 844 : BIND(&loop);
7470 : {
7471 : // Load the current {input} value (known to be a HeapObject).
7472 844 : Node* input = var_input.value();
7473 :
7474 : // Dispatch on the {input} instance type.
7475 844 : Node* input_instance_type = LoadInstanceType(input);
7476 1688 : Label if_inputisstring(this), if_inputisoddball(this),
7477 1688 : if_inputisbigint(this), if_inputisreceiver(this, Label::kDeferred),
7478 1688 : if_inputisother(this, Label::kDeferred);
7479 844 : GotoIf(IsStringInstanceType(input_instance_type), &if_inputisstring);
7480 844 : GotoIf(IsBigIntInstanceType(input_instance_type), &if_inputisbigint);
7481 1688 : GotoIf(InstanceTypeEqual(input_instance_type, ODDBALL_TYPE),
7482 844 : &if_inputisoddball);
7483 1688 : Branch(IsJSReceiverInstanceType(input_instance_type), &if_inputisreceiver,
7484 844 : &if_inputisother);
7485 :
7486 844 : BIND(&if_inputisstring);
7487 : {
7488 : // The {input} is a String, use the fast stub to convert it to a Number.
7489 844 : TNode<String> string_input = CAST(input);
7490 844 : var_result.Bind(StringToNumber(string_input));
7491 844 : Goto(&end);
7492 : }
7493 :
7494 844 : BIND(&if_inputisbigint);
7495 844 : if (mode == Object::Conversion::kToNumeric) {
7496 112 : var_result.Bind(input);
7497 112 : Goto(&end);
7498 : } else {
7499 : DCHECK_EQ(mode, Object::Conversion::kToNumber);
7500 732 : if (bigint_handling == BigIntHandling::kThrow) {
7501 620 : Goto(&if_inputisother);
7502 : } else {
7503 : DCHECK_EQ(bigint_handling, BigIntHandling::kConvertToNumber);
7504 112 : var_result.Bind(CallRuntime(Runtime::kBigIntToNumber, context, input));
7505 112 : Goto(&end);
7506 : }
7507 : }
7508 :
7509 844 : BIND(&if_inputisoddball);
7510 : {
7511 : // The {input} is an Oddball, we just need to load the Number value of it.
7512 844 : var_result.Bind(LoadObjectField(input, Oddball::kToNumberOffset));
7513 844 : Goto(&end);
7514 : }
7515 :
7516 844 : BIND(&if_inputisreceiver);
7517 : {
7518 : // The {input} is a JSReceiver, we need to convert it to a Primitive first
7519 : // using the ToPrimitive type conversion, preferably yielding a Number.
7520 : Callable callable = CodeFactory::NonPrimitiveToPrimitive(
7521 1688 : isolate(), ToPrimitiveHint::kNumber);
7522 844 : Node* result = CallStub(callable, context, input);
7523 :
7524 : // Check if the {result} is already a Number/Numeric.
7525 1688 : Label if_done(this), if_notdone(this);
7526 3376 : Branch(mode == Object::Conversion::kToNumber ? IsNumber(result)
7527 956 : : IsNumeric(result),
7528 956 : &if_done, &if_notdone);
7529 :
7530 844 : BIND(&if_done);
7531 : {
7532 : // The ToPrimitive conversion already gave us a Number/Numeric, so we're
7533 : // done.
7534 844 : var_result.Bind(result);
7535 844 : Goto(&end);
7536 : }
7537 :
7538 844 : BIND(&if_notdone);
7539 : {
7540 : // We now have a Primitive {result}, but it's not yet a Number/Numeric.
7541 844 : var_input.Bind(result);
7542 844 : Goto(&loop);
7543 : }
7544 : }
7545 :
7546 844 : BIND(&if_inputisother);
7547 : {
7548 : // The {input} is something else (e.g. Symbol), let the runtime figure
7549 : // out the correct exception.
7550 : // Note: We cannot tail call to the runtime here, as js-to-wasm
7551 : // trampolines also use this code currently, and they declare all
7552 : // outgoing parameters as untagged, while we would push a tagged
7553 : // object here.
7554 : auto function_id = mode == Object::Conversion::kToNumber
7555 : ? Runtime::kToNumber
7556 844 : : Runtime::kToNumeric;
7557 844 : var_result.Bind(CallRuntime(function_id, context, input));
7558 844 : Goto(&end);
7559 : }
7560 : }
7561 :
7562 844 : BIND(&end);
7563 : if (mode == Object::Conversion::kToNumeric) {
7564 : CSA_ASSERT(this, IsNumeric(var_result.value()));
7565 : } else {
7566 : DCHECK_EQ(mode, Object::Conversion::kToNumber);
7567 : CSA_ASSERT(this, IsNumber(var_result.value()));
7568 : }
7569 1688 : return var_result.value();
7570 : }
7571 :
7572 732 : TNode<Number> CodeStubAssembler::NonNumberToNumber(
7573 : SloppyTNode<Context> context, SloppyTNode<HeapObject> input,
7574 : BigIntHandling bigint_handling) {
7575 732 : return CAST(NonNumberToNumberOrNumeric(
7576 : context, input, Object::Conversion::kToNumber, bigint_handling));
7577 : }
7578 :
7579 112 : TNode<Numeric> CodeStubAssembler::NonNumberToNumeric(
7580 : SloppyTNode<Context> context, SloppyTNode<HeapObject> input) {
7581 112 : Node* result = NonNumberToNumberOrNumeric(context, input,
7582 112 : Object::Conversion::kToNumeric);
7583 : CSA_SLOW_ASSERT(this, IsNumeric(result));
7584 112 : return UncheckedCast<Numeric>(result);
7585 : }
7586 :
7587 616 : TNode<Number> CodeStubAssembler::ToNumber_Inline(SloppyTNode<Context> context,
7588 : SloppyTNode<Object> input) {
7589 1232 : TVARIABLE(Number, var_result);
7590 1232 : Label end(this), not_smi(this, Label::kDeferred);
7591 :
7592 616 : GotoIfNot(TaggedIsSmi(input), ¬_smi);
7593 616 : var_result = CAST(input);
7594 616 : Goto(&end);
7595 :
7596 616 : BIND(¬_smi);
7597 : {
7598 2464 : var_result =
7599 1848 : Select<Number>(IsHeapNumber(CAST(input)), [=] { return CAST(input); },
7600 616 : [=] {
7601 1232 : return CAST(CallBuiltin(Builtins::kNonNumberToNumber,
7602 : context, input));
7603 1848 : });
7604 616 : Goto(&end);
7605 : }
7606 :
7607 616 : BIND(&end);
7608 1232 : return var_result.value();
7609 : }
7610 :
7611 676 : TNode<Number> CodeStubAssembler::ToNumber(SloppyTNode<Context> context,
7612 : SloppyTNode<Object> input,
7613 : BigIntHandling bigint_handling) {
7614 1352 : TVARIABLE(Number, var_result);
7615 1352 : Label end(this);
7616 :
7617 1352 : Label not_smi(this, Label::kDeferred);
7618 676 : GotoIfNot(TaggedIsSmi(input), ¬_smi);
7619 676 : TNode<Smi> input_smi = CAST(input);
7620 676 : var_result = input_smi;
7621 676 : Goto(&end);
7622 :
7623 676 : BIND(¬_smi);
7624 : {
7625 1352 : Label not_heap_number(this, Label::kDeferred);
7626 676 : TNode<HeapObject> input_ho = CAST(input);
7627 676 : GotoIfNot(IsHeapNumber(input_ho), ¬_heap_number);
7628 :
7629 676 : TNode<HeapNumber> input_hn = CAST(input_ho);
7630 676 : var_result = input_hn;
7631 676 : Goto(&end);
7632 :
7633 676 : BIND(¬_heap_number);
7634 : {
7635 676 : var_result = NonNumberToNumber(context, input_ho, bigint_handling);
7636 676 : Goto(&end);
7637 : }
7638 : }
7639 :
7640 676 : BIND(&end);
7641 1352 : return var_result.value();
7642 : }
7643 :
7644 1568 : TNode<BigInt> CodeStubAssembler::ToBigInt(SloppyTNode<Context> context,
7645 : SloppyTNode<Object> input) {
7646 3136 : TVARIABLE(BigInt, var_result);
7647 3136 : Label if_bigint(this), done(this), if_throw(this);
7648 :
7649 1568 : GotoIf(TaggedIsSmi(input), &if_throw);
7650 1568 : GotoIf(IsBigInt(CAST(input)), &if_bigint);
7651 1568 : var_result = CAST(CallRuntime(Runtime::kToBigInt, context, input));
7652 1568 : Goto(&done);
7653 :
7654 1568 : BIND(&if_bigint);
7655 1568 : var_result = CAST(input);
7656 1568 : Goto(&done);
7657 :
7658 1568 : BIND(&if_throw);
7659 1568 : ThrowTypeError(context, MessageTemplate::kBigIntFromObject, input);
7660 :
7661 1568 : BIND(&done);
7662 3136 : return var_result.value();
7663 : }
7664 :
7665 336 : void CodeStubAssembler::TaggedToNumeric(Node* context, Node* value, Label* done,
7666 : Variable* var_numeric) {
7667 336 : TaggedToNumeric(context, value, done, var_numeric, nullptr);
7668 336 : }
7669 :
7670 1008 : void CodeStubAssembler::TaggedToNumericWithFeedback(Node* context, Node* value,
7671 : Label* done,
7672 : Variable* var_numeric,
7673 : Variable* var_feedback) {
7674 : DCHECK_NOT_NULL(var_feedback);
7675 1008 : TaggedToNumeric(context, value, done, var_numeric, var_feedback);
7676 1008 : }
7677 :
7678 1344 : void CodeStubAssembler::TaggedToNumeric(Node* context, Node* value, Label* done,
7679 : Variable* var_numeric,
7680 : Variable* var_feedback) {
7681 1344 : var_numeric->Bind(value);
7682 2688 : Label if_smi(this), if_heapnumber(this), if_bigint(this), if_oddball(this);
7683 1344 : GotoIf(TaggedIsSmi(value), &if_smi);
7684 1344 : Node* map = LoadMap(value);
7685 1344 : GotoIf(IsHeapNumberMap(map), &if_heapnumber);
7686 1344 : Node* instance_type = LoadMapInstanceType(map);
7687 1344 : GotoIf(IsBigIntInstanceType(instance_type), &if_bigint);
7688 :
7689 : // {value} is not a Numeric yet.
7690 1344 : GotoIf(Word32Equal(instance_type, Int32Constant(ODDBALL_TYPE)), &if_oddball);
7691 1344 : var_numeric->Bind(CallBuiltin(Builtins::kNonNumberToNumeric, context, value));
7692 1344 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kAny);
7693 1344 : Goto(done);
7694 :
7695 1344 : BIND(&if_smi);
7696 1344 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kSignedSmall);
7697 1344 : Goto(done);
7698 :
7699 1344 : BIND(&if_heapnumber);
7700 1344 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNumber);
7701 1344 : Goto(done);
7702 :
7703 1344 : BIND(&if_bigint);
7704 1344 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kBigInt);
7705 1344 : Goto(done);
7706 :
7707 1344 : BIND(&if_oddball);
7708 1344 : OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNumberOrOddball);
7709 1344 : var_numeric->Bind(LoadObjectField(value, Oddball::kToNumberOffset));
7710 1344 : Goto(done);
7711 1344 : }
7712 :
7713 : // ES#sec-touint32
7714 60 : TNode<Number> CodeStubAssembler::ToUint32(SloppyTNode<Context> context,
7715 : SloppyTNode<Object> input) {
7716 60 : Node* const float_zero = Float64Constant(0.0);
7717 60 : Node* const float_two_32 = Float64Constant(static_cast<double>(1ULL << 32));
7718 :
7719 120 : Label out(this);
7720 :
7721 120 : VARIABLE(var_result, MachineRepresentation::kTagged, input);
7722 :
7723 : // Early exit for positive smis.
7724 : {
7725 : // TODO(jgruber): This branch and the recheck below can be removed once we
7726 : // have a ToNumber with multiple exits.
7727 120 : Label next(this, Label::kDeferred);
7728 60 : Branch(TaggedIsPositiveSmi(input), &out, &next);
7729 60 : BIND(&next);
7730 : }
7731 :
7732 60 : Node* const number = ToNumber(context, input);
7733 60 : var_result.Bind(number);
7734 :
7735 : // Perhaps we have a positive smi now.
7736 : {
7737 120 : Label next(this, Label::kDeferred);
7738 60 : Branch(TaggedIsPositiveSmi(number), &out, &next);
7739 60 : BIND(&next);
7740 : }
7741 :
7742 120 : Label if_isnegativesmi(this), if_isheapnumber(this);
7743 60 : Branch(TaggedIsSmi(number), &if_isnegativesmi, &if_isheapnumber);
7744 :
7745 60 : BIND(&if_isnegativesmi);
7746 : {
7747 60 : Node* const uint32_value = SmiToInt32(number);
7748 60 : Node* float64_value = ChangeUint32ToFloat64(uint32_value);
7749 60 : var_result.Bind(AllocateHeapNumberWithValue(float64_value));
7750 60 : Goto(&out);
7751 : }
7752 :
7753 60 : BIND(&if_isheapnumber);
7754 : {
7755 120 : Label return_zero(this);
7756 60 : Node* const value = LoadHeapNumberValue(number);
7757 :
7758 : {
7759 : // +-0.
7760 120 : Label next(this);
7761 60 : Branch(Float64Equal(value, float_zero), &return_zero, &next);
7762 60 : BIND(&next);
7763 : }
7764 :
7765 : {
7766 : // NaN.
7767 120 : Label next(this);
7768 60 : Branch(Float64Equal(value, value), &next, &return_zero);
7769 60 : BIND(&next);
7770 : }
7771 :
7772 : {
7773 : // +Infinity.
7774 120 : Label next(this);
7775 : Node* const positive_infinity =
7776 60 : Float64Constant(std::numeric_limits<double>::infinity());
7777 60 : Branch(Float64Equal(value, positive_infinity), &return_zero, &next);
7778 60 : BIND(&next);
7779 : }
7780 :
7781 : {
7782 : // -Infinity.
7783 120 : Label next(this);
7784 : Node* const negative_infinity =
7785 60 : Float64Constant(-1.0 * std::numeric_limits<double>::infinity());
7786 60 : Branch(Float64Equal(value, negative_infinity), &return_zero, &next);
7787 60 : BIND(&next);
7788 : }
7789 :
7790 : // * Let int be the mathematical value that is the same sign as number and
7791 : // whose magnitude is floor(abs(number)).
7792 : // * Let int32bit be int modulo 2^32.
7793 : // * Return int32bit.
7794 : {
7795 60 : Node* x = Float64Trunc(value);
7796 60 : x = Float64Mod(x, float_two_32);
7797 60 : x = Float64Add(x, float_two_32);
7798 60 : x = Float64Mod(x, float_two_32);
7799 :
7800 60 : Node* const result = ChangeFloat64ToTagged(x);
7801 60 : var_result.Bind(result);
7802 60 : Goto(&out);
7803 : }
7804 :
7805 60 : BIND(&return_zero);
7806 : {
7807 60 : var_result.Bind(SmiConstant(0));
7808 60 : Goto(&out);
7809 : }
7810 : }
7811 :
7812 60 : BIND(&out);
7813 120 : return CAST(var_result.value());
7814 : }
7815 :
7816 172 : TNode<String> CodeStubAssembler::ToString(SloppyTNode<Context> context,
7817 : SloppyTNode<Object> input) {
7818 344 : TVARIABLE(Object, result, input);
7819 344 : Label loop(this, &result), done(this);
7820 172 : Goto(&loop);
7821 172 : BIND(&loop);
7822 : {
7823 : // Load the current {input} value.
7824 172 : TNode<Object> input = result.value();
7825 :
7826 : // Dispatch based on the type of the {input.}
7827 344 : Label if_inputisnumber(this), if_inputisoddball(this),
7828 344 : if_inputissymbol(this), if_inputisreceiver(this, Label::kDeferred),
7829 344 : runtime(this, Label::kDeferred);
7830 172 : GotoIf(TaggedIsSmi(input), &if_inputisnumber);
7831 172 : TNode<Int32T> input_instance_type = LoadInstanceType(CAST(input));
7832 172 : GotoIf(IsStringInstanceType(input_instance_type), &done);
7833 172 : GotoIf(IsJSReceiverInstanceType(input_instance_type), &if_inputisreceiver);
7834 172 : GotoIf(IsHeapNumberInstanceType(input_instance_type), &if_inputisnumber);
7835 172 : GotoIf(IsOddballInstanceType(input_instance_type), &if_inputisoddball);
7836 344 : Branch(IsSymbolInstanceType(input_instance_type), &if_inputissymbol,
7837 172 : &runtime);
7838 :
7839 172 : BIND(&if_inputisnumber);
7840 : {
7841 : // Convert the Number {input} to a String.
7842 172 : TNode<Number> number_input = CAST(input);
7843 172 : result = NumberToString(number_input);
7844 172 : Goto(&done);
7845 : }
7846 :
7847 172 : BIND(&if_inputisoddball);
7848 : {
7849 : // Just return the {input}'s string representation.
7850 172 : result = LoadObjectField(CAST(input), Oddball::kToStringOffset);
7851 172 : Goto(&done);
7852 : }
7853 :
7854 172 : BIND(&if_inputissymbol);
7855 : {
7856 : // Throw a type error when {input} is a Symbol.
7857 172 : ThrowTypeError(context, MessageTemplate::kSymbolToString);
7858 : }
7859 :
7860 172 : BIND(&if_inputisreceiver);
7861 : {
7862 : // Convert the JSReceiver {input} to a primitive first,
7863 : // and then run the loop again with the new {input},
7864 : // which is then a primitive value.
7865 344 : result = CallBuiltin(Builtins::kNonPrimitiveToPrimitive_String, context,
7866 172 : input);
7867 172 : Goto(&loop);
7868 : }
7869 :
7870 172 : BIND(&runtime);
7871 : {
7872 172 : result = CallRuntime(Runtime::kToString, context, input);
7873 172 : Goto(&done);
7874 : }
7875 : }
7876 :
7877 172 : BIND(&done);
7878 344 : return CAST(result.value());
7879 : }
7880 :
7881 2800 : TNode<String> CodeStubAssembler::ToString_Inline(SloppyTNode<Context> context,
7882 : SloppyTNode<Object> input) {
7883 5600 : VARIABLE(var_result, MachineRepresentation::kTagged, input);
7884 5600 : Label stub_call(this, Label::kDeferred), out(this);
7885 :
7886 2800 : GotoIf(TaggedIsSmi(input), &stub_call);
7887 2800 : Branch(IsString(CAST(input)), &out, &stub_call);
7888 :
7889 2800 : BIND(&stub_call);
7890 2800 : var_result.Bind(CallBuiltin(Builtins::kToString, context, input));
7891 2800 : Goto(&out);
7892 :
7893 2800 : BIND(&out);
7894 5600 : return CAST(var_result.value());
7895 : }
7896 :
7897 112 : Node* CodeStubAssembler::JSReceiverToPrimitive(Node* context, Node* input) {
7898 224 : Label if_isreceiver(this, Label::kDeferred), if_isnotreceiver(this);
7899 224 : VARIABLE(result, MachineRepresentation::kTagged);
7900 224 : Label done(this, &result);
7901 :
7902 112 : BranchIfJSReceiver(input, &if_isreceiver, &if_isnotreceiver);
7903 :
7904 112 : BIND(&if_isreceiver);
7905 : {
7906 : // Convert {input} to a primitive first passing Number hint.
7907 224 : Callable callable = CodeFactory::NonPrimitiveToPrimitive(isolate());
7908 112 : result.Bind(CallStub(callable, context, input));
7909 112 : Goto(&done);
7910 : }
7911 :
7912 112 : BIND(&if_isnotreceiver);
7913 : {
7914 112 : result.Bind(input);
7915 112 : Goto(&done);
7916 : }
7917 :
7918 112 : BIND(&done);
7919 224 : return result.value();
7920 : }
7921 :
7922 1904 : TNode<JSReceiver> CodeStubAssembler::ToObject(SloppyTNode<Context> context,
7923 : SloppyTNode<Object> input) {
7924 1904 : return CAST(CallBuiltin(Builtins::kToObject, context, input));
7925 : }
7926 :
7927 1680 : TNode<JSReceiver> CodeStubAssembler::ToObject_Inline(TNode<Context> context,
7928 : TNode<Object> input) {
7929 3360 : TVARIABLE(JSReceiver, result);
7930 3360 : Label if_isreceiver(this), if_isnotreceiver(this, Label::kDeferred);
7931 3360 : Label done(this);
7932 :
7933 1680 : BranchIfJSReceiver(input, &if_isreceiver, &if_isnotreceiver);
7934 :
7935 1680 : BIND(&if_isreceiver);
7936 : {
7937 1680 : result = CAST(input);
7938 1680 : Goto(&done);
7939 : }
7940 :
7941 1680 : BIND(&if_isnotreceiver);
7942 : {
7943 1680 : result = ToObject(context, input);
7944 1680 : Goto(&done);
7945 : }
7946 :
7947 1680 : BIND(&done);
7948 3360 : return result.value();
7949 : }
7950 :
7951 560 : TNode<Smi> CodeStubAssembler::ToSmiIndex(TNode<Context> context,
7952 : TNode<Object> input,
7953 : Label* range_error) {
7954 1120 : TVARIABLE(Smi, result);
7955 1120 : Label check_undefined(this), return_zero(this), defined(this),
7956 1120 : negative_check(this), done(this);
7957 :
7958 560 : GotoIfNot(TaggedIsSmi(input), &check_undefined);
7959 560 : result = CAST(input);
7960 560 : Goto(&negative_check);
7961 :
7962 560 : BIND(&check_undefined);
7963 560 : Branch(IsUndefined(input), &return_zero, &defined);
7964 :
7965 560 : BIND(&defined);
7966 : TNode<Number> integer_input =
7967 560 : CAST(CallBuiltin(Builtins::kToInteger_TruncateMinusZero, context, input));
7968 560 : GotoIfNot(TaggedIsSmi(integer_input), range_error);
7969 560 : result = CAST(integer_input);
7970 560 : Goto(&negative_check);
7971 :
7972 560 : BIND(&negative_check);
7973 560 : Branch(SmiLessThan(result.value(), SmiConstant(0)), range_error, &done);
7974 :
7975 560 : BIND(&return_zero);
7976 560 : result = SmiConstant(0);
7977 560 : Goto(&done);
7978 :
7979 560 : BIND(&done);
7980 1120 : return result.value();
7981 : }
7982 :
7983 168 : TNode<Smi> CodeStubAssembler::ToSmiLength(TNode<Context> context,
7984 : TNode<Object> input,
7985 : Label* range_error) {
7986 336 : TVARIABLE(Smi, result);
7987 336 : Label to_integer(this), negative_check(this),
7988 336 : heap_number_negative_check(this), return_zero(this), done(this);
7989 :
7990 168 : GotoIfNot(TaggedIsSmi(input), &to_integer);
7991 168 : result = CAST(input);
7992 168 : Goto(&negative_check);
7993 :
7994 168 : BIND(&to_integer);
7995 : {
7996 168 : TNode<Number> integer_input = CAST(
7997 : CallBuiltin(Builtins::kToInteger_TruncateMinusZero, context, input));
7998 168 : GotoIfNot(TaggedIsSmi(integer_input), &heap_number_negative_check);
7999 168 : result = CAST(integer_input);
8000 168 : Goto(&negative_check);
8001 :
8002 : // integer_input can still be a negative HeapNumber here.
8003 168 : BIND(&heap_number_negative_check);
8004 168 : TNode<HeapNumber> heap_number_input = CAST(integer_input);
8005 672 : Branch(IsTrue(CallBuiltin(Builtins::kLessThan, context, heap_number_input,
8006 672 : SmiConstant(0))),
8007 168 : &return_zero, range_error);
8008 : }
8009 :
8010 168 : BIND(&negative_check);
8011 168 : Branch(SmiLessThan(result.value(), SmiConstant(0)), &return_zero, &done);
8012 :
8013 168 : BIND(&return_zero);
8014 168 : result = SmiConstant(0);
8015 168 : Goto(&done);
8016 :
8017 168 : BIND(&done);
8018 336 : return result.value();
8019 : }
8020 :
8021 1736 : TNode<Number> CodeStubAssembler::ToLength_Inline(SloppyTNode<Context> context,
8022 : SloppyTNode<Object> input) {
8023 1736 : TNode<Smi> smi_zero = SmiConstant(0);
8024 : return Select<Number>(
8025 5208 : TaggedIsSmi(input), [=] { return SmiMax(CAST(input), smi_zero); },
8026 6944 : [=] { return CAST(CallBuiltin(Builtins::kToLength, context, input)); });
8027 : }
8028 :
8029 3192 : TNode<Number> CodeStubAssembler::ToInteger_Inline(
8030 : SloppyTNode<Context> context, SloppyTNode<Object> input,
8031 : ToIntegerTruncationMode mode) {
8032 : Builtins::Name builtin = (mode == kNoTruncation)
8033 : ? Builtins::kToInteger
8034 3192 : : Builtins::kToInteger_TruncateMinusZero;
8035 : return Select<Number>(
8036 9576 : TaggedIsSmi(input), [=] { return CAST(input); },
8037 12768 : [=] { return CAST(CallBuiltin(builtin, context, input)); });
8038 : }
8039 :
8040 112 : TNode<Number> CodeStubAssembler::ToInteger(SloppyTNode<Context> context,
8041 : SloppyTNode<Object> input,
8042 : ToIntegerTruncationMode mode) {
8043 : // We might need to loop once for ToNumber conversion.
8044 224 : TVARIABLE(Object, var_arg, input);
8045 224 : Label loop(this, &var_arg), out(this);
8046 112 : Goto(&loop);
8047 112 : BIND(&loop);
8048 : {
8049 : // Shared entry points.
8050 224 : Label return_zero(this, Label::kDeferred);
8051 :
8052 : // Load the current {arg} value.
8053 112 : TNode<Object> arg = var_arg.value();
8054 :
8055 : // Check if {arg} is a Smi.
8056 112 : GotoIf(TaggedIsSmi(arg), &out);
8057 :
8058 : // Check if {arg} is a HeapNumber.
8059 224 : Label if_argisheapnumber(this),
8060 224 : if_argisnotheapnumber(this, Label::kDeferred);
8061 224 : Branch(IsHeapNumber(CAST(arg)), &if_argisheapnumber,
8062 112 : &if_argisnotheapnumber);
8063 :
8064 112 : BIND(&if_argisheapnumber);
8065 : {
8066 112 : TNode<HeapNumber> arg_hn = CAST(arg);
8067 : // Load the floating-point value of {arg}.
8068 112 : Node* arg_value = LoadHeapNumberValue(arg_hn);
8069 :
8070 : // Check if {arg} is NaN.
8071 112 : GotoIfNot(Float64Equal(arg_value, arg_value), &return_zero);
8072 :
8073 : // Truncate {arg} towards zero.
8074 112 : TNode<Float64T> value = Float64Trunc(arg_value);
8075 :
8076 112 : if (mode == kTruncateMinusZero) {
8077 : // Truncate -0.0 to 0.
8078 56 : GotoIf(Float64Equal(value, Float64Constant(0.0)), &return_zero);
8079 : }
8080 :
8081 112 : var_arg = ChangeFloat64ToTagged(value);
8082 112 : Goto(&out);
8083 : }
8084 :
8085 112 : BIND(&if_argisnotheapnumber);
8086 : {
8087 : // Need to convert {arg} to a Number first.
8088 224 : var_arg = UncheckedCast<Object>(
8089 112 : CallBuiltin(Builtins::kNonNumberToNumber, context, arg));
8090 112 : Goto(&loop);
8091 : }
8092 :
8093 112 : BIND(&return_zero);
8094 112 : var_arg = SmiConstant(0);
8095 112 : Goto(&out);
8096 : }
8097 :
8098 112 : BIND(&out);
8099 : if (mode == kTruncateMinusZero) {
8100 : CSA_ASSERT(this, IsNumberNormalized(CAST(var_arg.value())));
8101 : }
8102 224 : return CAST(var_arg.value());
8103 : }
8104 :
8105 35212 : TNode<Uint32T> CodeStubAssembler::DecodeWord32(SloppyTNode<Word32T> word32,
8106 : uint32_t shift, uint32_t mask) {
8107 : return UncheckedCast<Uint32T>(Word32Shr(
8108 35212 : Word32And(word32, Int32Constant(mask)), static_cast<int>(shift)));
8109 : }
8110 :
8111 21240 : TNode<UintPtrT> CodeStubAssembler::DecodeWord(SloppyTNode<WordT> word,
8112 : uint32_t shift, uint32_t mask) {
8113 : return Unsigned(
8114 21240 : WordShr(WordAnd(word, IntPtrConstant(mask)), static_cast<int>(shift)));
8115 : }
8116 :
8117 392 : TNode<WordT> CodeStubAssembler::UpdateWord(TNode<WordT> word,
8118 : TNode<WordT> value, uint32_t shift,
8119 : uint32_t mask) {
8120 392 : TNode<WordT> encoded_value = WordShl(value, static_cast<int>(shift));
8121 392 : TNode<IntPtrT> inverted_mask = IntPtrConstant(~static_cast<intptr_t>(mask));
8122 : // Ensure the {value} fits fully in the mask.
8123 : CSA_ASSERT(this, WordEqual(WordAnd(encoded_value, inverted_mask),
8124 : IntPtrConstant(0)));
8125 392 : return WordOr(WordAnd(word, inverted_mask), encoded_value);
8126 : }
8127 :
8128 0 : void CodeStubAssembler::SetCounter(StatsCounter* counter, int value) {
8129 0 : if (FLAG_native_code_counters && counter->Enabled()) {
8130 : Node* counter_address =
8131 0 : ExternalConstant(ExternalReference::Create(counter));
8132 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, counter_address,
8133 0 : Int32Constant(value));
8134 : }
8135 0 : }
8136 :
8137 3704 : void CodeStubAssembler::IncrementCounter(StatsCounter* counter, int delta) {
8138 : DCHECK_GT(delta, 0);
8139 3704 : if (FLAG_native_code_counters && counter->Enabled()) {
8140 : Node* counter_address =
8141 0 : ExternalConstant(ExternalReference::Create(counter));
8142 0 : Node* value = Load(MachineType::Int32(), counter_address);
8143 0 : value = Int32Add(value, Int32Constant(delta));
8144 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, counter_address, value);
8145 : }
8146 3704 : }
8147 :
8148 0 : void CodeStubAssembler::DecrementCounter(StatsCounter* counter, int delta) {
8149 : DCHECK_GT(delta, 0);
8150 0 : if (FLAG_native_code_counters && counter->Enabled()) {
8151 : Node* counter_address =
8152 0 : ExternalConstant(ExternalReference::Create(counter));
8153 0 : Node* value = Load(MachineType::Int32(), counter_address);
8154 0 : value = Int32Sub(value, Int32Constant(delta));
8155 0 : StoreNoWriteBarrier(MachineRepresentation::kWord32, counter_address, value);
8156 : }
8157 0 : }
8158 :
8159 44684 : void CodeStubAssembler::Increment(Variable* variable, int value,
8160 : ParameterMode mode) {
8161 : DCHECK_IMPLIES(mode == INTPTR_PARAMETERS,
8162 : variable->rep() == MachineType::PointerRepresentation());
8163 : DCHECK_IMPLIES(mode == SMI_PARAMETERS,
8164 : variable->rep() == MachineRepresentation::kTagged ||
8165 : variable->rep() == MachineRepresentation::kTaggedSigned);
8166 44684 : variable->Bind(IntPtrOrSmiAdd(variable->value(),
8167 44684 : IntPtrOrSmiConstant(value, mode), mode));
8168 44684 : }
8169 :
8170 56 : void CodeStubAssembler::Use(Label* label) {
8171 56 : GotoIf(Word32Equal(Int32Constant(0), Int32Constant(1)), label);
8172 56 : }
8173 :
8174 1460 : void CodeStubAssembler::TryToName(Node* key, Label* if_keyisindex,
8175 : Variable* var_index, Label* if_keyisunique,
8176 : Variable* var_unique, Label* if_bailout,
8177 : Label* if_notinternalized) {
8178 : DCHECK_EQ(MachineType::PointerRepresentation(), var_index->rep());
8179 : DCHECK_EQ(MachineRepresentation::kTagged, var_unique->rep());
8180 1460 : Comment("TryToName");
8181 :
8182 2920 : Label if_hascachedindex(this), if_keyisnotindex(this), if_thinstring(this),
8183 2920 : if_keyisother(this, Label::kDeferred);
8184 : // Handle Smi and HeapNumber keys.
8185 1460 : var_index->Bind(TryToIntptr(key, &if_keyisnotindex));
8186 1460 : Goto(if_keyisindex);
8187 :
8188 1460 : BIND(&if_keyisnotindex);
8189 1460 : Node* key_map = LoadMap(key);
8190 1460 : var_unique->Bind(key);
8191 : // Symbols are unique.
8192 1460 : GotoIf(IsSymbolMap(key_map), if_keyisunique);
8193 1460 : Node* key_instance_type = LoadMapInstanceType(key_map);
8194 : // Miss if |key| is not a String.
8195 : STATIC_ASSERT(FIRST_NAME_TYPE == FIRST_TYPE);
8196 1460 : GotoIfNot(IsStringInstanceType(key_instance_type), &if_keyisother);
8197 :
8198 : // |key| is a String. Check if it has a cached array index.
8199 1460 : Node* hash = LoadNameHashField(key);
8200 2920 : GotoIf(IsClearWord32(hash, Name::kDoesNotContainCachedArrayIndexMask),
8201 1460 : &if_hascachedindex);
8202 : // No cached array index. If the string knows that it contains an index,
8203 : // then it must be an uncacheable index. Handle this case in the runtime.
8204 1460 : GotoIf(IsClearWord32(hash, Name::kIsNotArrayIndexMask), if_bailout);
8205 : // Check if we have a ThinString.
8206 2920 : GotoIf(InstanceTypeEqual(key_instance_type, THIN_STRING_TYPE),
8207 1460 : &if_thinstring);
8208 2920 : GotoIf(InstanceTypeEqual(key_instance_type, THIN_ONE_BYTE_STRING_TYPE),
8209 1460 : &if_thinstring);
8210 : // Finally, check if |key| is internalized.
8211 : STATIC_ASSERT(kNotInternalizedTag != 0);
8212 2920 : GotoIf(IsSetWord32(key_instance_type, kIsNotInternalizedMask),
8213 1460 : if_notinternalized != nullptr ? if_notinternalized : if_bailout);
8214 1460 : Goto(if_keyisunique);
8215 :
8216 1460 : BIND(&if_thinstring);
8217 1460 : var_unique->Bind(LoadObjectField(key, ThinString::kActualOffset));
8218 1460 : Goto(if_keyisunique);
8219 :
8220 1460 : BIND(&if_hascachedindex);
8221 1460 : var_index->Bind(DecodeWordFromWord32<Name::ArrayIndexValueBits>(hash));
8222 1460 : Goto(if_keyisindex);
8223 :
8224 1460 : BIND(&if_keyisother);
8225 1460 : GotoIfNot(InstanceTypeEqual(key_instance_type, ODDBALL_TYPE), if_bailout);
8226 1460 : var_unique->Bind(LoadObjectField(key, Oddball::kToStringOffset));
8227 1460 : Goto(if_keyisunique);
8228 1460 : }
8229 :
8230 392 : void CodeStubAssembler::TryInternalizeString(
8231 : Node* string, Label* if_index, Variable* var_index, Label* if_internalized,
8232 : Variable* var_internalized, Label* if_not_internalized, Label* if_bailout) {
8233 : DCHECK(var_index->rep() == MachineType::PointerRepresentation());
8234 : DCHECK_EQ(var_internalized->rep(), MachineRepresentation::kTagged);
8235 : CSA_SLOW_ASSERT(this, IsString(string));
8236 : Node* function =
8237 392 : ExternalConstant(ExternalReference::try_internalize_string_function());
8238 : Node* const isolate_ptr =
8239 392 : ExternalConstant(ExternalReference::isolate_address(isolate()));
8240 : Node* result =
8241 392 : CallCFunction2(MachineType::AnyTagged(), MachineType::Pointer(),
8242 392 : MachineType::AnyTagged(), function, isolate_ptr, string);
8243 784 : Label internalized(this);
8244 392 : GotoIf(TaggedIsNotSmi(result), &internalized);
8245 392 : Node* word_result = SmiUntag(result);
8246 784 : GotoIf(WordEqual(word_result, IntPtrConstant(ResultSentinel::kNotFound)),
8247 392 : if_not_internalized);
8248 784 : GotoIf(WordEqual(word_result, IntPtrConstant(ResultSentinel::kUnsupported)),
8249 392 : if_bailout);
8250 392 : var_index->Bind(word_result);
8251 392 : Goto(if_index);
8252 :
8253 392 : BIND(&internalized);
8254 392 : var_internalized->Bind(result);
8255 392 : Goto(if_internalized);
8256 392 : }
8257 :
8258 : template <typename Dictionary>
8259 32284 : TNode<IntPtrT> CodeStubAssembler::EntryToIndex(TNode<IntPtrT> entry,
8260 : int field_index) {
8261 : TNode<IntPtrT> entry_index =
8262 32284 : IntPtrMul(entry, IntPtrConstant(Dictionary::kEntrySize));
8263 : return IntPtrAdd(entry_index, IntPtrConstant(Dictionary::kElementsStartIndex +
8264 32284 : field_index));
8265 : }
8266 :
8267 8180 : TNode<MaybeObject> CodeStubAssembler::LoadDescriptorArrayElement(
8268 : TNode<DescriptorArray> object, Node* index, int additional_offset) {
8269 : return LoadArrayElement(object, DescriptorArray::kHeaderSize, index,
8270 8180 : additional_offset);
8271 : }
8272 :
8273 392 : TNode<Name> CodeStubAssembler::LoadKeyByKeyIndex(
8274 : TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
8275 392 : return CAST(LoadDescriptorArrayElement(container, key_index, 0));
8276 : }
8277 :
8278 2132 : TNode<Uint32T> CodeStubAssembler::LoadDetailsByKeyIndex(
8279 : TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
8280 : const int kKeyToDetails =
8281 2132 : DescriptorArray::ToDetailsIndex(0) - DescriptorArray::ToKeyIndex(0);
8282 : return Unsigned(
8283 4264 : LoadAndUntagToWord32ArrayElement(container, DescriptorArray::kHeaderSize,
8284 6396 : key_index, kKeyToDetails * kTaggedSize));
8285 : }
8286 :
8287 2020 : TNode<Object> CodeStubAssembler::LoadValueByKeyIndex(
8288 : TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
8289 : const int kKeyToValue =
8290 2020 : DescriptorArray::ToValueIndex(0) - DescriptorArray::ToKeyIndex(0);
8291 2020 : return CAST(LoadDescriptorArrayElement(container, key_index,
8292 : kKeyToValue * kTaggedSize));
8293 : }
8294 :
8295 728 : TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByKeyIndex(
8296 : TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
8297 : const int kKeyToValue =
8298 728 : DescriptorArray::ToValueIndex(0) - DescriptorArray::ToKeyIndex(0);
8299 : return LoadDescriptorArrayElement(container, key_index,
8300 728 : kKeyToValue * kTaggedSize);
8301 : }
8302 :
8303 4928 : TNode<IntPtrT> CodeStubAssembler::DescriptorEntryToIndex(
8304 : TNode<IntPtrT> descriptor_entry) {
8305 : return IntPtrMul(descriptor_entry,
8306 4928 : IntPtrConstant(DescriptorArray::kEntrySize));
8307 : }
8308 :
8309 112 : TNode<Name> CodeStubAssembler::LoadKeyByDescriptorEntry(
8310 : TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) {
8311 112 : return CAST(LoadDescriptorArrayElement(
8312 : container, DescriptorEntryToIndex(descriptor_entry),
8313 : DescriptorArray::ToKeyIndex(0) * kTaggedSize));
8314 : }
8315 :
8316 112 : TNode<Name> CodeStubAssembler::LoadKeyByDescriptorEntry(
8317 : TNode<DescriptorArray> container, int descriptor_entry) {
8318 112 : return CAST(LoadDescriptorArrayElement(
8319 : container, IntPtrConstant(0),
8320 : DescriptorArray::ToKeyIndex(descriptor_entry) * kTaggedSize));
8321 : }
8322 :
8323 112 : TNode<Uint32T> CodeStubAssembler::LoadDetailsByDescriptorEntry(
8324 : TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) {
8325 224 : return Unsigned(LoadAndUntagToWord32ArrayElement(
8326 : container, DescriptorArray::kHeaderSize,
8327 224 : DescriptorEntryToIndex(descriptor_entry),
8328 448 : DescriptorArray::ToDetailsIndex(0) * kTaggedSize));
8329 : }
8330 :
8331 672 : TNode<Uint32T> CodeStubAssembler::LoadDetailsByDescriptorEntry(
8332 : TNode<DescriptorArray> container, int descriptor_entry) {
8333 1344 : return Unsigned(LoadAndUntagToWord32ArrayElement(
8334 1344 : container, DescriptorArray::kHeaderSize, IntPtrConstant(0),
8335 2688 : DescriptorArray::ToDetailsIndex(descriptor_entry) * kTaggedSize));
8336 : }
8337 :
8338 112 : TNode<Object> CodeStubAssembler::LoadValueByDescriptorEntry(
8339 : TNode<DescriptorArray> container, int descriptor_entry) {
8340 112 : return CAST(LoadDescriptorArrayElement(
8341 : container, IntPtrConstant(0),
8342 : DescriptorArray::ToValueIndex(descriptor_entry) * kTaggedSize));
8343 : }
8344 :
8345 4704 : TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByDescriptorEntry(
8346 : TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) {
8347 : return LoadDescriptorArrayElement(
8348 9408 : container, DescriptorEntryToIndex(descriptor_entry),
8349 14112 : DescriptorArray::ToValueIndex(0) * kTaggedSize);
8350 : }
8351 :
8352 : template TNode<IntPtrT> CodeStubAssembler::EntryToIndex<NameDictionary>(
8353 : TNode<IntPtrT>, int);
8354 : template TNode<IntPtrT> CodeStubAssembler::EntryToIndex<GlobalDictionary>(
8355 : TNode<IntPtrT>, int);
8356 : template TNode<IntPtrT> CodeStubAssembler::EntryToIndex<NumberDictionary>(
8357 : TNode<IntPtrT>, int);
8358 :
8359 : // This must be kept in sync with HashTableBase::ComputeCapacity().
8360 956 : TNode<IntPtrT> CodeStubAssembler::HashTableComputeCapacity(
8361 : TNode<IntPtrT> at_least_space_for) {
8362 : TNode<IntPtrT> capacity = IntPtrRoundUpToPowerOfTwo32(
8363 956 : IntPtrAdd(at_least_space_for, WordShr(at_least_space_for, 1)));
8364 956 : return IntPtrMax(capacity, IntPtrConstant(HashTableBase::kMinCapacity));
8365 : }
8366 :
8367 1685 : TNode<IntPtrT> CodeStubAssembler::IntPtrMax(SloppyTNode<IntPtrT> left,
8368 : SloppyTNode<IntPtrT> right) {
8369 : intptr_t left_constant;
8370 : intptr_t right_constant;
8371 2302 : if (ToIntPtrConstant(left, left_constant) &&
8372 617 : ToIntPtrConstant(right, right_constant)) {
8373 617 : return IntPtrConstant(std::max(left_constant, right_constant));
8374 : }
8375 : return SelectConstant<IntPtrT>(IntPtrGreaterThanOrEqual(left, right), left,
8376 1068 : right);
8377 : }
8378 :
8379 1009 : TNode<IntPtrT> CodeStubAssembler::IntPtrMin(SloppyTNode<IntPtrT> left,
8380 : SloppyTNode<IntPtrT> right) {
8381 : intptr_t left_constant;
8382 : intptr_t right_constant;
8383 1010 : if (ToIntPtrConstant(left, left_constant) &&
8384 1 : ToIntPtrConstant(right, right_constant)) {
8385 1 : return IntPtrConstant(std::min(left_constant, right_constant));
8386 : }
8387 : return SelectConstant<IntPtrT>(IntPtrLessThanOrEqual(left, right), left,
8388 1008 : right);
8389 : }
8390 :
8391 : template <>
8392 21620 : TNode<HeapObject> CodeStubAssembler::LoadName<NameDictionary>(
8393 : TNode<HeapObject> key) {
8394 : CSA_ASSERT(this, Word32Or(IsTheHole(key), IsName(key)));
8395 21620 : return key;
8396 : }
8397 :
8398 : template <>
8399 7060 : TNode<HeapObject> CodeStubAssembler::LoadName<GlobalDictionary>(
8400 : TNode<HeapObject> key) {
8401 7060 : TNode<PropertyCell> property_cell = CAST(key);
8402 7060 : return CAST(LoadObjectField(property_cell, PropertyCell::kNameOffset));
8403 : }
8404 :
8405 : template <typename Dictionary>
8406 6744 : void CodeStubAssembler::NameDictionaryLookup(
8407 : TNode<Dictionary> dictionary, TNode<Name> unique_name, Label* if_found,
8408 : TVariable<IntPtrT>* var_name_index, Label* if_not_found, int inlined_probes,
8409 : LookupMode mode) {
8410 : static_assert(std::is_same<Dictionary, NameDictionary>::value ||
8411 : std::is_same<Dictionary, GlobalDictionary>::value,
8412 : "Unexpected NameDictionary");
8413 : DCHECK_EQ(MachineType::PointerRepresentation(), var_name_index->rep());
8414 : DCHECK_IMPLIES(mode == kFindInsertionIndex,
8415 : inlined_probes == 0 && if_found == nullptr);
8416 6744 : Comment("NameDictionaryLookup");
8417 : CSA_ASSERT(this, IsUniqueName(unique_name));
8418 :
8419 6744 : TNode<IntPtrT> capacity = SmiUntag(GetCapacity<Dictionary>(dictionary));
8420 6744 : TNode<WordT> mask = IntPtrSub(capacity, IntPtrConstant(1));
8421 6744 : TNode<WordT> hash = ChangeUint32ToWord(LoadNameHash(unique_name));
8422 :
8423 : // See Dictionary::FirstProbe().
8424 6744 : TNode<IntPtrT> count = IntPtrConstant(0);
8425 6744 : TNode<IntPtrT> entry = Signed(WordAnd(hash, mask));
8426 6744 : Node* undefined = UndefinedConstant();
8427 :
8428 29688 : for (int i = 0; i < inlined_probes; i++) {
8429 22944 : TNode<IntPtrT> index = EntryToIndex<Dictionary>(entry);
8430 22944 : *var_name_index = index;
8431 :
8432 : TNode<HeapObject> current =
8433 22944 : CAST(UnsafeLoadFixedArrayElement(dictionary, index));
8434 22944 : GotoIf(WordEqual(current, undefined), if_not_found);
8435 22944 : current = LoadName<Dictionary>(current);
8436 22944 : GotoIf(WordEqual(current, unique_name), if_found);
8437 :
8438 : // See Dictionary::NextProbe().
8439 22944 : count = IntPtrConstant(i + 1);
8440 22944 : entry = Signed(WordAnd(IntPtrAdd(entry, count), mask));
8441 : }
8442 6744 : if (mode == kFindInsertionIndex) {
8443 : // Appease the variable merging algorithm for "Goto(&loop)" below.
8444 1008 : *var_name_index = IntPtrConstant(0);
8445 : }
8446 :
8447 13488 : TVARIABLE(IntPtrT, var_count, count);
8448 13488 : TVARIABLE(IntPtrT, var_entry, entry);
8449 6744 : Variable* loop_vars[] = {&var_count, &var_entry, var_name_index};
8450 13488 : Label loop(this, 3, loop_vars);
8451 6744 : Goto(&loop);
8452 6744 : BIND(&loop);
8453 : {
8454 6744 : TNode<IntPtrT> entry = var_entry.value();
8455 :
8456 6744 : TNode<IntPtrT> index = EntryToIndex<Dictionary>(entry);
8457 6744 : *var_name_index = index;
8458 :
8459 6744 : TNode<HeapObject> current = CAST(LoadFixedArrayElement(dictionary, index));
8460 6744 : GotoIf(WordEqual(current, undefined), if_not_found);
8461 6744 : if (mode == kFindExisting) {
8462 5736 : current = LoadName<Dictionary>(current);
8463 5736 : GotoIf(WordEqual(current, unique_name), if_found);
8464 : } else {
8465 : DCHECK_EQ(kFindInsertionIndex, mode);
8466 1008 : GotoIf(WordEqual(current, TheHoleConstant()), if_not_found);
8467 : }
8468 :
8469 : // See Dictionary::NextProbe().
8470 6744 : Increment(&var_count);
8471 6744 : entry = Signed(WordAnd(IntPtrAdd(entry, var_count.value()), mask));
8472 :
8473 6744 : var_entry = entry;
8474 6744 : Goto(&loop);
8475 : }
8476 6744 : }
8477 :
8478 : // Instantiate template methods to workaround GCC compilation issue.
8479 : template void CodeStubAssembler::NameDictionaryLookup<NameDictionary>(
8480 : TNode<NameDictionary>, TNode<Name>, Label*, TVariable<IntPtrT>*, Label*,
8481 : int, LookupMode);
8482 : template void CodeStubAssembler::NameDictionaryLookup<GlobalDictionary>(
8483 : TNode<GlobalDictionary>, TNode<Name>, Label*, TVariable<IntPtrT>*, Label*,
8484 : int, LookupMode);
8485 :
8486 336 : Node* CodeStubAssembler::ComputeUnseededHash(Node* key) {
8487 : // See v8::internal::ComputeUnseededHash()
8488 336 : Node* hash = TruncateIntPtrToInt32(key);
8489 1344 : hash = Int32Add(Word32Xor(hash, Int32Constant(0xFFFFFFFF)),
8490 1680 : Word32Shl(hash, Int32Constant(15)));
8491 336 : hash = Word32Xor(hash, Word32Shr(hash, Int32Constant(12)));
8492 336 : hash = Int32Add(hash, Word32Shl(hash, Int32Constant(2)));
8493 336 : hash = Word32Xor(hash, Word32Shr(hash, Int32Constant(4)));
8494 336 : hash = Int32Mul(hash, Int32Constant(2057));
8495 336 : hash = Word32Xor(hash, Word32Shr(hash, Int32Constant(16)));
8496 336 : return Word32And(hash, Int32Constant(0x3FFFFFFF));
8497 : }
8498 :
8499 1132 : Node* CodeStubAssembler::ComputeSeededHash(Node* key) {
8500 : Node* const function_addr =
8501 1132 : ExternalConstant(ExternalReference::compute_integer_hash());
8502 : Node* const isolate_ptr =
8503 1132 : ExternalConstant(ExternalReference::isolate_address(isolate()));
8504 :
8505 1132 : MachineType type_ptr = MachineType::Pointer();
8506 1132 : MachineType type_uint32 = MachineType::Uint32();
8507 :
8508 : Node* const result =
8509 1132 : CallCFunction2(type_uint32, type_ptr, type_uint32, function_addr,
8510 2264 : isolate_ptr, TruncateIntPtrToInt32(key));
8511 1132 : return result;
8512 : }
8513 :
8514 1128 : void CodeStubAssembler::NumberDictionaryLookup(
8515 : TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index,
8516 : Label* if_found, TVariable<IntPtrT>* var_entry, Label* if_not_found) {
8517 : CSA_ASSERT(this, IsNumberDictionary(dictionary));
8518 : DCHECK_EQ(MachineType::PointerRepresentation(), var_entry->rep());
8519 1128 : Comment("NumberDictionaryLookup");
8520 :
8521 1128 : TNode<IntPtrT> capacity = SmiUntag(GetCapacity<NumberDictionary>(dictionary));
8522 1128 : TNode<WordT> mask = IntPtrSub(capacity, IntPtrConstant(1));
8523 :
8524 1128 : TNode<WordT> hash = ChangeUint32ToWord(ComputeSeededHash(intptr_index));
8525 1128 : Node* key_as_float64 = RoundIntPtrToFloat64(intptr_index);
8526 :
8527 : // See Dictionary::FirstProbe().
8528 1128 : TNode<IntPtrT> count = IntPtrConstant(0);
8529 1128 : TNode<IntPtrT> entry = Signed(WordAnd(hash, mask));
8530 :
8531 1128 : Node* undefined = UndefinedConstant();
8532 1128 : Node* the_hole = TheHoleConstant();
8533 :
8534 2256 : TVARIABLE(IntPtrT, var_count, count);
8535 1128 : Variable* loop_vars[] = {&var_count, var_entry};
8536 2256 : Label loop(this, 2, loop_vars);
8537 1128 : *var_entry = entry;
8538 1128 : Goto(&loop);
8539 1128 : BIND(&loop);
8540 : {
8541 1128 : TNode<IntPtrT> entry = var_entry->value();
8542 :
8543 1128 : TNode<IntPtrT> index = EntryToIndex<NumberDictionary>(entry);
8544 1128 : Node* current = UnsafeLoadFixedArrayElement(dictionary, index);
8545 1128 : GotoIf(WordEqual(current, undefined), if_not_found);
8546 2256 : Label next_probe(this);
8547 : {
8548 2256 : Label if_currentissmi(this), if_currentisnotsmi(this);
8549 1128 : Branch(TaggedIsSmi(current), &if_currentissmi, &if_currentisnotsmi);
8550 1128 : BIND(&if_currentissmi);
8551 : {
8552 1128 : Node* current_value = SmiUntag(current);
8553 1128 : Branch(WordEqual(current_value, intptr_index), if_found, &next_probe);
8554 : }
8555 1128 : BIND(&if_currentisnotsmi);
8556 : {
8557 1128 : GotoIf(WordEqual(current, the_hole), &next_probe);
8558 : // Current must be the Number.
8559 1128 : Node* current_value = LoadHeapNumberValue(current);
8560 2256 : Branch(Float64Equal(current_value, key_as_float64), if_found,
8561 1128 : &next_probe);
8562 : }
8563 : }
8564 :
8565 1128 : BIND(&next_probe);
8566 : // See Dictionary::NextProbe().
8567 1128 : Increment(&var_count);
8568 1128 : entry = Signed(WordAnd(IntPtrAdd(entry, var_count.value()), mask));
8569 :
8570 1128 : *var_entry = entry;
8571 1128 : Goto(&loop);
8572 : }
8573 1128 : }
8574 :
8575 336 : TNode<Object> CodeStubAssembler::BasicLoadNumberDictionaryElement(
8576 : TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index,
8577 : Label* not_data, Label* if_hole) {
8578 672 : TVARIABLE(IntPtrT, var_entry);
8579 672 : Label if_found(this);
8580 : NumberDictionaryLookup(dictionary, intptr_index, &if_found, &var_entry,
8581 336 : if_hole);
8582 336 : BIND(&if_found);
8583 :
8584 : // Check that the value is a data property.
8585 336 : TNode<IntPtrT> index = EntryToIndex<NumberDictionary>(var_entry.value());
8586 : TNode<Uint32T> details =
8587 336 : LoadDetailsByKeyIndex<NumberDictionary>(dictionary, index);
8588 336 : TNode<Uint32T> kind = DecodeWord32<PropertyDetails::KindField>(details);
8589 : // TODO(jkummerow): Support accessors without missing?
8590 336 : GotoIfNot(Word32Equal(kind, Int32Constant(kData)), not_data);
8591 : // Finally, load the value.
8592 672 : return LoadValueByKeyIndex<NumberDictionary>(dictionary, index);
8593 : }
8594 :
8595 56 : void CodeStubAssembler::BasicStoreNumberDictionaryElement(
8596 : TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index,
8597 : TNode<Object> value, Label* not_data, Label* if_hole, Label* read_only) {
8598 112 : TVARIABLE(IntPtrT, var_entry);
8599 112 : Label if_found(this);
8600 : NumberDictionaryLookup(dictionary, intptr_index, &if_found, &var_entry,
8601 56 : if_hole);
8602 56 : BIND(&if_found);
8603 :
8604 : // Check that the value is a data property.
8605 56 : TNode<IntPtrT> index = EntryToIndex<NumberDictionary>(var_entry.value());
8606 : TNode<Uint32T> details =
8607 56 : LoadDetailsByKeyIndex<NumberDictionary>(dictionary, index);
8608 56 : TNode<Uint32T> kind = DecodeWord32<PropertyDetails::KindField>(details);
8609 : // TODO(jkummerow): Support accessors without missing?
8610 56 : GotoIfNot(Word32Equal(kind, Int32Constant(kData)), not_data);
8611 :
8612 : // Check that the property is writeable.
8613 112 : GotoIf(IsSetWord32(details, PropertyDetails::kAttributesReadOnlyMask),
8614 56 : read_only);
8615 :
8616 : // Finally, store the value.
8617 56 : StoreValueByKeyIndex<NumberDictionary>(dictionary, index, value);
8618 56 : }
8619 :
8620 : template <class Dictionary>
8621 : void CodeStubAssembler::FindInsertionEntry(TNode<Dictionary> dictionary,
8622 : TNode<Name> key,
8623 : TVariable<IntPtrT>* var_key_index) {
8624 : UNREACHABLE();
8625 : }
8626 :
8627 : template <>
8628 1008 : void CodeStubAssembler::FindInsertionEntry<NameDictionary>(
8629 : TNode<NameDictionary> dictionary, TNode<Name> key,
8630 : TVariable<IntPtrT>* var_key_index) {
8631 2016 : Label done(this);
8632 : NameDictionaryLookup<NameDictionary>(dictionary, key, nullptr, var_key_index,
8633 1008 : &done, 0, kFindInsertionIndex);
8634 1008 : BIND(&done);
8635 1008 : }
8636 :
8637 : template <class Dictionary>
8638 : void CodeStubAssembler::InsertEntry(TNode<Dictionary> dictionary,
8639 : TNode<Name> key, TNode<Object> value,
8640 : TNode<IntPtrT> index,
8641 : TNode<Smi> enum_index) {
8642 : UNREACHABLE(); // Use specializations instead.
8643 : }
8644 :
8645 : template <>
8646 1008 : void CodeStubAssembler::InsertEntry<NameDictionary>(
8647 : TNode<NameDictionary> dictionary, TNode<Name> name, TNode<Object> value,
8648 : TNode<IntPtrT> index, TNode<Smi> enum_index) {
8649 : // Store name and value.
8650 1008 : StoreFixedArrayElement(dictionary, index, name);
8651 1008 : StoreValueByKeyIndex<NameDictionary>(dictionary, index, value);
8652 :
8653 : // Prepare details of the new property.
8654 1008 : PropertyDetails d(kData, NONE, PropertyCellType::kNoCell);
8655 1008 : enum_index =
8656 1008 : SmiShl(enum_index, PropertyDetails::DictionaryStorageField::kShift);
8657 : // We OR over the actual index below, so we expect the initial value to be 0.
8658 : DCHECK_EQ(0, d.dictionary_index());
8659 2016 : TVARIABLE(Smi, var_details, SmiOr(SmiConstant(d.AsSmi()), enum_index));
8660 :
8661 : // Private names must be marked non-enumerable.
8662 2016 : Label not_private(this, &var_details);
8663 1008 : GotoIfNot(IsPrivateSymbol(name), ¬_private);
8664 : TNode<Smi> dont_enum =
8665 1008 : SmiShl(SmiConstant(DONT_ENUM), PropertyDetails::AttributesField::kShift);
8666 1008 : var_details = SmiOr(var_details.value(), dont_enum);
8667 1008 : Goto(¬_private);
8668 1008 : BIND(¬_private);
8669 :
8670 : // Finally, store the details.
8671 1008 : StoreDetailsByKeyIndex<NameDictionary>(dictionary, index,
8672 1008 : var_details.value());
8673 1008 : }
8674 :
8675 : template <>
8676 0 : void CodeStubAssembler::InsertEntry<GlobalDictionary>(
8677 : TNode<GlobalDictionary> dictionary, TNode<Name> key, TNode<Object> value,
8678 : TNode<IntPtrT> index, TNode<Smi> enum_index) {
8679 0 : UNIMPLEMENTED();
8680 : }
8681 :
8682 : template <class Dictionary>
8683 1008 : void CodeStubAssembler::Add(TNode<Dictionary> dictionary, TNode<Name> key,
8684 : TNode<Object> value, Label* bailout) {
8685 : CSA_ASSERT(this, Word32BinaryNot(IsEmptyPropertyDictionary(dictionary)));
8686 1008 : TNode<Smi> capacity = GetCapacity<Dictionary>(dictionary);
8687 1008 : TNode<Smi> nof = GetNumberOfElements<Dictionary>(dictionary);
8688 1008 : TNode<Smi> new_nof = SmiAdd(nof, SmiConstant(1));
8689 : // Require 33% to still be free after adding additional_elements.
8690 : // Computing "x + (x >> 1)" on a Smi x does not return a valid Smi!
8691 : // But that's OK here because it's only used for a comparison.
8692 1008 : TNode<Smi> required_capacity_pseudo_smi = SmiAdd(new_nof, SmiShr(new_nof, 1));
8693 1008 : GotoIf(SmiBelow(capacity, required_capacity_pseudo_smi), bailout);
8694 : // Require rehashing if more than 50% of free elements are deleted elements.
8695 1008 : TNode<Smi> deleted = GetNumberOfDeletedElements<Dictionary>(dictionary);
8696 : CSA_ASSERT(this, SmiAbove(capacity, new_nof));
8697 1008 : TNode<Smi> half_of_free_elements = SmiShr(SmiSub(capacity, new_nof), 1);
8698 1008 : GotoIf(SmiAbove(deleted, half_of_free_elements), bailout);
8699 :
8700 1008 : TNode<Smi> enum_index = GetNextEnumerationIndex<Dictionary>(dictionary);
8701 1008 : TNode<Smi> new_enum_index = SmiAdd(enum_index, SmiConstant(1));
8702 : TNode<Smi> max_enum_index =
8703 1008 : SmiConstant(PropertyDetails::DictionaryStorageField::kMax);
8704 1008 : GotoIf(SmiAbove(new_enum_index, max_enum_index), bailout);
8705 :
8706 : // No more bailouts after this point.
8707 : // Operations from here on can have side effects.
8708 :
8709 1008 : SetNextEnumerationIndex<Dictionary>(dictionary, new_enum_index);
8710 1008 : SetNumberOfElements<Dictionary>(dictionary, new_nof);
8711 :
8712 2016 : TVARIABLE(IntPtrT, var_key_index);
8713 1008 : FindInsertionEntry<Dictionary>(dictionary, key, &var_key_index);
8714 1008 : InsertEntry<Dictionary>(dictionary, key, value, var_key_index.value(),
8715 : enum_index);
8716 1008 : }
8717 :
8718 : template void CodeStubAssembler::Add<NameDictionary>(TNode<NameDictionary>,
8719 : TNode<Name>, TNode<Object>,
8720 : Label*);
8721 :
8722 : template <typename Array>
8723 2252 : void CodeStubAssembler::LookupLinear(TNode<Name> unique_name,
8724 : TNode<Array> array,
8725 : TNode<Uint32T> number_of_valid_entries,
8726 : Label* if_found,
8727 : TVariable<IntPtrT>* var_name_index,
8728 : Label* if_not_found) {
8729 : static_assert(std::is_base_of<FixedArray, Array>::value ||
8730 : std::is_base_of<WeakFixedArray, Array>::value ||
8731 : std::is_base_of<DescriptorArray, Array>::value,
8732 : "T must be a descendant of FixedArray or a WeakFixedArray");
8733 2252 : Comment("LookupLinear");
8734 : CSA_ASSERT(this, IsUniqueName(unique_name));
8735 2252 : TNode<IntPtrT> first_inclusive = IntPtrConstant(Array::ToKeyIndex(0));
8736 2252 : TNode<IntPtrT> factor = IntPtrConstant(Array::kEntrySize);
8737 4504 : TNode<IntPtrT> last_exclusive = IntPtrAdd(
8738 : first_inclusive,
8739 2252 : IntPtrMul(ChangeInt32ToIntPtr(number_of_valid_entries), factor));
8740 :
8741 2252 : BuildFastLoop(last_exclusive, first_inclusive,
8742 2252 : [=](SloppyTNode<IntPtrT> name_index) {
8743 : TNode<MaybeObject> element =
8744 4504 : LoadArrayElement(array, Array::kHeaderSize, name_index);
8745 9008 : TNode<Name> candidate_name = CAST(element);
8746 2252 : *var_name_index = name_index;
8747 6756 : GotoIf(WordEqual(candidate_name, unique_name), if_found);
8748 2252 : },
8749 : -Array::kEntrySize, INTPTR_PARAMETERS, IndexAdvanceMode::kPre);
8750 2252 : Goto(if_not_found);
8751 2252 : }
8752 :
8753 : template <>
8754 1968 : TNode<Uint32T> CodeStubAssembler::NumberOfEntries<DescriptorArray>(
8755 : TNode<DescriptorArray> descriptors) {
8756 1968 : return Unsigned(LoadNumberOfDescriptors(descriptors));
8757 : }
8758 :
8759 : template <>
8760 568 : TNode<Uint32T> CodeStubAssembler::NumberOfEntries<TransitionArray>(
8761 : TNode<TransitionArray> transitions) {
8762 568 : TNode<IntPtrT> length = LoadAndUntagWeakFixedArrayLength(transitions);
8763 : return Select<Uint32T>(
8764 1136 : UintPtrLessThan(length, IntPtrConstant(TransitionArray::kFirstIndex)),
8765 568 : [=] { return Unsigned(Int32Constant(0)); },
8766 568 : [=] {
8767 1136 : return Unsigned(LoadAndUntagToWord32ArrayElement(
8768 : transitions, WeakFixedArray::kHeaderSize,
8769 2272 : IntPtrConstant(TransitionArray::kTransitionLengthIndex)));
8770 2840 : });
8771 : }
8772 :
8773 : template <typename Array>
8774 12596 : TNode<IntPtrT> CodeStubAssembler::EntryIndexToIndex(
8775 : TNode<Uint32T> entry_index) {
8776 12596 : TNode<Int32T> entry_size = Int32Constant(Array::kEntrySize);
8777 12596 : TNode<Word32T> index = Int32Mul(entry_index, entry_size);
8778 12596 : return ChangeInt32ToIntPtr(index);
8779 : }
8780 :
8781 : template <typename Array>
8782 2588 : TNode<IntPtrT> CodeStubAssembler::ToKeyIndex(TNode<Uint32T> entry_index) {
8783 2588 : return IntPtrAdd(IntPtrConstant(Array::ToKeyIndex(0)),
8784 5176 : EntryIndexToIndex<Array>(entry_index));
8785 : }
8786 :
8787 : template TNode<IntPtrT> CodeStubAssembler::ToKeyIndex<DescriptorArray>(
8788 : TNode<Uint32T>);
8789 : template TNode<IntPtrT> CodeStubAssembler::ToKeyIndex<TransitionArray>(
8790 : TNode<Uint32T>);
8791 :
8792 : template <>
8793 3936 : TNode<Uint32T> CodeStubAssembler::GetSortedKeyIndex<DescriptorArray>(
8794 : TNode<DescriptorArray> descriptors, TNode<Uint32T> descriptor_number) {
8795 : TNode<Uint32T> details =
8796 3936 : DescriptorArrayGetDetails(descriptors, descriptor_number);
8797 3936 : return DecodeWord32<PropertyDetails::DescriptorPointer>(details);
8798 : }
8799 :
8800 : template <>
8801 568 : TNode<Uint32T> CodeStubAssembler::GetSortedKeyIndex<TransitionArray>(
8802 : TNode<TransitionArray> transitions, TNode<Uint32T> transition_number) {
8803 568 : return transition_number;
8804 : }
8805 :
8806 : template <typename Array>
8807 4504 : TNode<Name> CodeStubAssembler::GetKey(TNode<Array> array,
8808 : TNode<Uint32T> entry_index) {
8809 : static_assert(std::is_base_of<TransitionArray, Array>::value ||
8810 : std::is_base_of<DescriptorArray, Array>::value,
8811 : "T must be a descendant of DescriptorArray or TransitionArray");
8812 4504 : const int key_offset = Array::ToKeyIndex(0) * kTaggedSize;
8813 : TNode<MaybeObject> element =
8814 : LoadArrayElement(array, Array::kHeaderSize,
8815 4504 : EntryIndexToIndex<Array>(entry_index), key_offset);
8816 4504 : return CAST(element);
8817 : }
8818 :
8819 : template TNode<Name> CodeStubAssembler::GetKey<DescriptorArray>(
8820 : TNode<DescriptorArray>, TNode<Uint32T>);
8821 : template TNode<Name> CodeStubAssembler::GetKey<TransitionArray>(
8822 : TNode<TransitionArray>, TNode<Uint32T>);
8823 :
8824 5504 : TNode<Uint32T> CodeStubAssembler::DescriptorArrayGetDetails(
8825 : TNode<DescriptorArray> descriptors, TNode<Uint32T> descriptor_number) {
8826 5504 : const int details_offset = DescriptorArray::ToDetailsIndex(0) * kTaggedSize;
8827 11008 : return Unsigned(LoadAndUntagToWord32ArrayElement(
8828 : descriptors, DescriptorArray::kHeaderSize,
8829 16512 : EntryIndexToIndex<DescriptorArray>(descriptor_number), details_offset));
8830 : }
8831 :
8832 : template <typename Array>
8833 2252 : void CodeStubAssembler::LookupBinary(TNode<Name> unique_name,
8834 : TNode<Array> array,
8835 : TNode<Uint32T> number_of_valid_entries,
8836 : Label* if_found,
8837 : TVariable<IntPtrT>* var_name_index,
8838 : Label* if_not_found) {
8839 2252 : Comment("LookupBinary");
8840 4504 : TVARIABLE(Uint32T, var_low, Unsigned(Int32Constant(0)));
8841 : TNode<Uint32T> limit =
8842 2252 : Unsigned(Int32Sub(NumberOfEntries<Array>(array), Int32Constant(1)));
8843 4504 : TVARIABLE(Uint32T, var_high, limit);
8844 2252 : TNode<Uint32T> hash = LoadNameHashField(unique_name);
8845 : CSA_ASSERT(this, Word32NotEqual(hash, Int32Constant(0)));
8846 :
8847 : // Assume non-empty array.
8848 : CSA_ASSERT(this, Uint32LessThanOrEqual(var_low.value(), var_high.value()));
8849 :
8850 4504 : Label binary_loop(this, {&var_high, &var_low});
8851 2252 : Goto(&binary_loop);
8852 2252 : BIND(&binary_loop);
8853 : {
8854 : // mid = low + (high - low) / 2 (to avoid overflow in "(low + high) / 2").
8855 4504 : TNode<Uint32T> mid = Unsigned(
8856 : Int32Add(var_low.value(),
8857 6756 : Word32Shr(Int32Sub(var_high.value(), var_low.value()), 1)));
8858 : // mid_name = array->GetSortedKey(mid).
8859 2252 : TNode<Uint32T> sorted_key_index = GetSortedKeyIndex<Array>(array, mid);
8860 2252 : TNode<Name> mid_name = GetKey<Array>(array, sorted_key_index);
8861 :
8862 2252 : TNode<Uint32T> mid_hash = LoadNameHashField(mid_name);
8863 :
8864 4504 : Label mid_greater(this), mid_less(this), merge(this);
8865 2252 : Branch(Uint32GreaterThanOrEqual(mid_hash, hash), &mid_greater, &mid_less);
8866 2252 : BIND(&mid_greater);
8867 : {
8868 2252 : var_high = mid;
8869 2252 : Goto(&merge);
8870 : }
8871 2252 : BIND(&mid_less);
8872 : {
8873 2252 : var_low = Unsigned(Int32Add(mid, Int32Constant(1)));
8874 2252 : Goto(&merge);
8875 : }
8876 2252 : BIND(&merge);
8877 2252 : GotoIf(Word32NotEqual(var_low.value(), var_high.value()), &binary_loop);
8878 : }
8879 :
8880 4504 : Label scan_loop(this, &var_low);
8881 2252 : Goto(&scan_loop);
8882 2252 : BIND(&scan_loop);
8883 : {
8884 2252 : GotoIf(Int32GreaterThan(var_low.value(), limit), if_not_found);
8885 :
8886 : TNode<Uint32T> sort_index =
8887 2252 : GetSortedKeyIndex<Array>(array, var_low.value());
8888 2252 : TNode<Name> current_name = GetKey<Array>(array, sort_index);
8889 2252 : TNode<Uint32T> current_hash = LoadNameHashField(current_name);
8890 2252 : GotoIf(Word32NotEqual(current_hash, hash), if_not_found);
8891 4504 : Label next(this);
8892 2252 : GotoIf(WordNotEqual(current_name, unique_name), &next);
8893 2252 : GotoIf(Uint32GreaterThanOrEqual(sort_index, number_of_valid_entries),
8894 : if_not_found);
8895 2252 : *var_name_index = ToKeyIndex<Array>(sort_index);
8896 2252 : Goto(if_found);
8897 :
8898 2252 : BIND(&next);
8899 2252 : var_low = Unsigned(Int32Add(var_low.value(), Int32Constant(1)));
8900 2252 : Goto(&scan_loop);
8901 : }
8902 2252 : }
8903 :
8904 112 : void CodeStubAssembler::ForEachEnumerableOwnProperty(
8905 : TNode<Context> context, TNode<Map> map, TNode<JSObject> object,
8906 : ForEachEnumerationMode mode, const ForEachKeyValueFunction& body,
8907 : Label* bailout) {
8908 112 : TNode<Int32T> type = LoadMapInstanceType(map);
8909 112 : TNode<Uint32T> bit_field3 = EnsureOnlyHasSimpleProperties(map, type, bailout);
8910 :
8911 112 : TNode<DescriptorArray> descriptors = LoadMapDescriptors(map);
8912 : TNode<Uint32T> nof_descriptors =
8913 112 : DecodeWord32<Map::NumberOfOwnDescriptorsBits>(bit_field3);
8914 :
8915 224 : TVARIABLE(BoolT, var_stable, Int32TrueConstant());
8916 :
8917 224 : TVARIABLE(BoolT, var_has_symbol, Int32FalseConstant());
8918 : // false - iterate only string properties, true - iterate only symbol
8919 : // properties
8920 224 : TVARIABLE(BoolT, var_is_symbol_processing_loop, Int32FalseConstant());
8921 224 : TVARIABLE(IntPtrT, var_start_key_index,
8922 : ToKeyIndex<DescriptorArray>(Unsigned(Int32Constant(0))));
8923 : // Note: var_end_key_index is exclusive for the loop
8924 224 : TVARIABLE(IntPtrT, var_end_key_index,
8925 : ToKeyIndex<DescriptorArray>(nof_descriptors));
8926 : VariableList list(
8927 : {&var_stable, &var_has_symbol, &var_is_symbol_processing_loop,
8928 : &var_start_key_index, &var_end_key_index},
8929 224 : zone());
8930 : Label descriptor_array_loop(
8931 : this, {&var_stable, &var_has_symbol, &var_is_symbol_processing_loop,
8932 224 : &var_start_key_index, &var_end_key_index});
8933 :
8934 112 : Goto(&descriptor_array_loop);
8935 112 : BIND(&descriptor_array_loop);
8936 :
8937 448 : BuildFastLoop(
8938 336 : list, var_start_key_index.value(), var_end_key_index.value(),
8939 : [=, &var_stable, &var_has_symbol, &var_is_symbol_processing_loop,
8940 1120 : &var_start_key_index, &var_end_key_index](Node* index) {
8941 : TNode<IntPtrT> descriptor_key_index =
8942 112 : TNode<IntPtrT>::UncheckedCast(index);
8943 : TNode<Name> next_key =
8944 5992 : LoadKeyByKeyIndex(descriptors, descriptor_key_index);
8945 :
8946 224 : TVARIABLE(Object, var_value, SmiConstant(0));
8947 336 : Label callback(this), next_iteration(this);
8948 :
8949 112 : if (mode == kEnumerationOrder) {
8950 : // |next_key| is either a string or a symbol
8951 : // Skip strings or symbols depending on
8952 : // |var_is_symbol_processing_loop|.
8953 224 : Label if_string(this), if_symbol(this), if_name_ok(this);
8954 168 : Branch(IsSymbol(next_key), &if_symbol, &if_string);
8955 56 : BIND(&if_symbol);
8956 : {
8957 : // Process symbol property when |var_is_symbol_processing_loop| is
8958 : // true.
8959 112 : GotoIf(var_is_symbol_processing_loop.value(), &if_name_ok);
8960 : // First iteration need to calculate smaller range for processing
8961 : // symbols
8962 112 : Label if_first_symbol(this);
8963 : // var_end_key_index is still inclusive at this point.
8964 56 : var_end_key_index = descriptor_key_index;
8965 112 : Branch(var_has_symbol.value(), &next_iteration, &if_first_symbol);
8966 56 : BIND(&if_first_symbol);
8967 : {
8968 56 : var_start_key_index = descriptor_key_index;
8969 112 : var_has_symbol = Int32TrueConstant();
8970 56 : Goto(&next_iteration);
8971 : }
8972 : }
8973 56 : BIND(&if_string);
8974 : {
8975 : CSA_ASSERT(this, IsString(next_key));
8976 : // Process string property when |var_is_symbol_processing_loop| is
8977 : // false.
8978 168 : Branch(var_is_symbol_processing_loop.value(), &next_iteration,
8979 56 : &if_name_ok);
8980 : }
8981 56 : BIND(&if_name_ok);
8982 : }
8983 : {
8984 224 : TVARIABLE(Map, var_map);
8985 224 : TVARIABLE(HeapObject, var_meta_storage);
8986 224 : TVARIABLE(IntPtrT, var_entry);
8987 224 : TVARIABLE(Uint32T, var_details);
8988 224 : Label if_found(this);
8989 :
8990 336 : Label if_found_fast(this), if_found_dict(this);
8991 :
8992 336 : Label if_stable(this), if_not_stable(this);
8993 224 : Branch(var_stable.value(), &if_stable, &if_not_stable);
8994 112 : BIND(&if_stable);
8995 : {
8996 : // Directly decode from the descriptor array if |object| did not
8997 : // change shape.
8998 112 : var_map = map;
8999 112 : var_meta_storage = descriptors;
9000 224 : var_entry = Signed(descriptor_key_index);
9001 112 : Goto(&if_found_fast);
9002 : }
9003 112 : BIND(&if_not_stable);
9004 : {
9005 : // If the map did change, do a slower lookup. We are still
9006 : // guaranteed that the object has a simple shape, and that the key
9007 : // is a name.
9008 224 : var_map = LoadMap(object);
9009 112 : TryLookupPropertyInSimpleObject(
9010 : object, var_map.value(), next_key, &if_found_fast,
9011 112 : &if_found_dict, &var_meta_storage, &var_entry, &next_iteration);
9012 : }
9013 :
9014 112 : BIND(&if_found_fast);
9015 : {
9016 224 : TNode<DescriptorArray> descriptors = CAST(var_meta_storage.value());
9017 112 : TNode<IntPtrT> name_index = var_entry.value();
9018 :
9019 : // Skip non-enumerable properties.
9020 112 : var_details = LoadDetailsByKeyIndex(descriptors, name_index);
9021 560 : GotoIf(IsSetWord32(var_details.value(),
9022 224 : PropertyDetails::kAttributesDontEnumMask),
9023 112 : &next_iteration);
9024 :
9025 224 : LoadPropertyFromFastObject(object, var_map.value(), descriptors,
9026 224 : name_index, var_details.value(),
9027 112 : &var_value);
9028 112 : Goto(&if_found);
9029 : }
9030 112 : BIND(&if_found_dict);
9031 : {
9032 224 : TNode<NameDictionary> dictionary = CAST(var_meta_storage.value());
9033 112 : TNode<IntPtrT> entry = var_entry.value();
9034 :
9035 : TNode<Uint32T> details =
9036 224 : LoadDetailsByKeyIndex<NameDictionary>(dictionary, entry);
9037 : // Skip non-enumerable properties.
9038 336 : GotoIf(
9039 336 : IsSetWord32(details, PropertyDetails::kAttributesDontEnumMask),
9040 112 : &next_iteration);
9041 :
9042 112 : var_details = details;
9043 224 : var_value = LoadValueByKeyIndex<NameDictionary>(dictionary, entry);
9044 112 : Goto(&if_found);
9045 : }
9046 :
9047 : // Here we have details and value which could be an accessor.
9048 112 : BIND(&if_found);
9049 : {
9050 224 : Label slow_load(this, Label::kDeferred);
9051 :
9052 448 : var_value = CallGetterIfAccessor(var_value.value(),
9053 224 : var_details.value(), context,
9054 112 : object, &slow_load, kCallJSGetter);
9055 112 : Goto(&callback);
9056 :
9057 112 : BIND(&slow_load);
9058 336 : var_value =
9059 112 : CallRuntime(Runtime::kGetProperty, context, object, next_key);
9060 112 : Goto(&callback);
9061 :
9062 112 : BIND(&callback);
9063 112 : body(next_key, var_value.value());
9064 :
9065 : // Check if |object| is still stable, i.e. we can proceed using
9066 : // property details from preloaded |descriptors|.
9067 672 : var_stable =
9068 224 : Select<BoolT>(var_stable.value(),
9069 112 : [=] { return WordEqual(LoadMap(object), map); },
9070 224 : [=] { return Int32FalseConstant(); });
9071 :
9072 112 : Goto(&next_iteration);
9073 : }
9074 : }
9075 112 : BIND(&next_iteration);
9076 112 : },
9077 112 : DescriptorArray::kEntrySize, INTPTR_PARAMETERS, IndexAdvanceMode::kPost);
9078 :
9079 112 : if (mode == kEnumerationOrder) {
9080 112 : Label done(this);
9081 56 : GotoIf(var_is_symbol_processing_loop.value(), &done);
9082 56 : GotoIfNot(var_has_symbol.value(), &done);
9083 : // All string properties are processed, now process symbol properties.
9084 56 : var_is_symbol_processing_loop = Int32TrueConstant();
9085 : // Add DescriptorArray::kEntrySize to make the var_end_key_index exclusive
9086 : // as BuildFastLoop() expects.
9087 : Increment(&var_end_key_index, DescriptorArray::kEntrySize,
9088 56 : INTPTR_PARAMETERS);
9089 56 : Goto(&descriptor_array_loop);
9090 :
9091 56 : BIND(&done);
9092 : }
9093 112 : }
9094 :
9095 1968 : void CodeStubAssembler::DescriptorLookup(
9096 : SloppyTNode<Name> unique_name, SloppyTNode<DescriptorArray> descriptors,
9097 : SloppyTNode<Uint32T> bitfield3, Label* if_found,
9098 : TVariable<IntPtrT>* var_name_index, Label* if_not_found) {
9099 1968 : Comment("DescriptorArrayLookup");
9100 1968 : TNode<Uint32T> nof = DecodeWord32<Map::NumberOfOwnDescriptorsBits>(bitfield3);
9101 : Lookup<DescriptorArray>(unique_name, descriptors, nof, if_found,
9102 1968 : var_name_index, if_not_found);
9103 1968 : }
9104 :
9105 284 : void CodeStubAssembler::TransitionLookup(
9106 : SloppyTNode<Name> unique_name, SloppyTNode<TransitionArray> transitions,
9107 : Label* if_found, TVariable<IntPtrT>* var_name_index, Label* if_not_found) {
9108 284 : Comment("TransitionArrayLookup");
9109 : TNode<Uint32T> number_of_valid_transitions =
9110 284 : NumberOfEntries<TransitionArray>(transitions);
9111 : Lookup<TransitionArray>(unique_name, transitions, number_of_valid_transitions,
9112 284 : if_found, var_name_index, if_not_found);
9113 284 : }
9114 :
9115 : template <typename Array>
9116 2252 : void CodeStubAssembler::Lookup(TNode<Name> unique_name, TNode<Array> array,
9117 : TNode<Uint32T> number_of_valid_entries,
9118 : Label* if_found,
9119 : TVariable<IntPtrT>* var_name_index,
9120 : Label* if_not_found) {
9121 2252 : Comment("ArrayLookup");
9122 2252 : if (!number_of_valid_entries) {
9123 0 : number_of_valid_entries = NumberOfEntries(array);
9124 : }
9125 2252 : GotoIf(Word32Equal(number_of_valid_entries, Int32Constant(0)), if_not_found);
9126 4504 : Label linear_search(this), binary_search(this);
9127 2252 : const int kMaxElementsForLinearSearch = 32;
9128 2252 : Branch(Uint32LessThanOrEqual(number_of_valid_entries,
9129 2252 : Int32Constant(kMaxElementsForLinearSearch)),
9130 : &linear_search, &binary_search);
9131 2252 : BIND(&linear_search);
9132 : {
9133 2252 : LookupLinear<Array>(unique_name, array, number_of_valid_entries, if_found,
9134 : var_name_index, if_not_found);
9135 : }
9136 2252 : BIND(&binary_search);
9137 : {
9138 2252 : LookupBinary<Array>(unique_name, array, number_of_valid_entries, if_found,
9139 : var_name_index, if_not_found);
9140 : }
9141 2252 : }
9142 :
9143 56 : TNode<BoolT> CodeStubAssembler::IsSimpleObjectMap(TNode<Map> map) {
9144 : uint32_t mask =
9145 56 : Map::HasNamedInterceptorBit::kMask | Map::IsAccessCheckNeededBit::kMask;
9146 : // !IsSpecialReceiverType && !IsNamedInterceptor && !IsAccessCheckNeeded
9147 : return Select<BoolT>(
9148 112 : IsSpecialReceiverInstanceType(LoadMapInstanceType(map)),
9149 56 : [=] { return Int32FalseConstant(); },
9150 224 : [=] { return IsClearWord32(LoadMapBitField(map), mask); });
9151 : }
9152 :
9153 1520 : void CodeStubAssembler::TryLookupPropertyInSimpleObject(
9154 : TNode<JSObject> object, TNode<Map> map, TNode<Name> unique_name,
9155 : Label* if_found_fast, Label* if_found_dict,
9156 : TVariable<HeapObject>* var_meta_storage, TVariable<IntPtrT>* var_name_index,
9157 : Label* if_not_found) {
9158 : CSA_ASSERT(this, IsSimpleObjectMap(map));
9159 : CSA_ASSERT(this, IsUniqueNameNoIndex(unique_name));
9160 :
9161 1520 : TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
9162 3040 : Label if_isfastmap(this), if_isslowmap(this);
9163 3040 : Branch(IsSetWord32<Map::IsDictionaryMapBit>(bit_field3), &if_isslowmap,
9164 1520 : &if_isfastmap);
9165 1520 : BIND(&if_isfastmap);
9166 : {
9167 1520 : TNode<DescriptorArray> descriptors = LoadMapDescriptors(map);
9168 1520 : *var_meta_storage = descriptors;
9169 :
9170 3040 : DescriptorLookup(unique_name, descriptors, bit_field3, if_found_fast,
9171 1520 : var_name_index, if_not_found);
9172 : }
9173 1520 : BIND(&if_isslowmap);
9174 : {
9175 1520 : TNode<NameDictionary> dictionary = CAST(LoadSlowProperties(object));
9176 1520 : *var_meta_storage = dictionary;
9177 :
9178 : NameDictionaryLookup<NameDictionary>(dictionary, unique_name, if_found_dict,
9179 1520 : var_name_index, if_not_found);
9180 : }
9181 1520 : }
9182 :
9183 1408 : void CodeStubAssembler::TryLookupProperty(
9184 : SloppyTNode<JSObject> object, SloppyTNode<Map> map,
9185 : SloppyTNode<Int32T> instance_type, SloppyTNode<Name> unique_name,
9186 : Label* if_found_fast, Label* if_found_dict, Label* if_found_global,
9187 : TVariable<HeapObject>* var_meta_storage, TVariable<IntPtrT>* var_name_index,
9188 : Label* if_not_found, Label* if_bailout) {
9189 2816 : Label if_objectisspecial(this);
9190 1408 : GotoIf(IsSpecialReceiverInstanceType(instance_type), &if_objectisspecial);
9191 :
9192 : TryLookupPropertyInSimpleObject(object, map, unique_name, if_found_fast,
9193 : if_found_dict, var_meta_storage,
9194 1408 : var_name_index, if_not_found);
9195 :
9196 1408 : BIND(&if_objectisspecial);
9197 : {
9198 : // Handle global object here and bailout for other special objects.
9199 2816 : GotoIfNot(InstanceTypeEqual(instance_type, JS_GLOBAL_OBJECT_TYPE),
9200 1408 : if_bailout);
9201 :
9202 : // Handle interceptors and access checks in runtime.
9203 1408 : TNode<Int32T> bit_field = LoadMapBitField(map);
9204 : int mask =
9205 1408 : Map::HasNamedInterceptorBit::kMask | Map::IsAccessCheckNeededBit::kMask;
9206 1408 : GotoIf(IsSetWord32(bit_field, mask), if_bailout);
9207 :
9208 1408 : TNode<GlobalDictionary> dictionary = CAST(LoadSlowProperties(object));
9209 1408 : *var_meta_storage = dictionary;
9210 :
9211 : NameDictionaryLookup<GlobalDictionary>(
9212 1408 : dictionary, unique_name, if_found_global, var_name_index, if_not_found);
9213 : }
9214 1408 : }
9215 :
9216 732 : void CodeStubAssembler::TryHasOwnProperty(Node* object, Node* map,
9217 : Node* instance_type,
9218 : Node* unique_name, Label* if_found,
9219 : Label* if_not_found,
9220 : Label* if_bailout) {
9221 732 : Comment("TryHasOwnProperty");
9222 : CSA_ASSERT(this, IsUniqueNameNoIndex(CAST(unique_name)));
9223 1464 : TVARIABLE(HeapObject, var_meta_storage);
9224 1464 : TVARIABLE(IntPtrT, var_name_index);
9225 :
9226 1464 : Label if_found_global(this);
9227 1464 : TryLookupProperty(object, map, instance_type, unique_name, if_found, if_found,
9228 : &if_found_global, &var_meta_storage, &var_name_index,
9229 732 : if_not_found, if_bailout);
9230 :
9231 732 : BIND(&if_found_global);
9232 : {
9233 1464 : VARIABLE(var_value, MachineRepresentation::kTagged);
9234 1464 : VARIABLE(var_details, MachineRepresentation::kWord32);
9235 : // Check if the property cell is not deleted.
9236 1464 : LoadPropertyFromGlobalDictionary(var_meta_storage.value(),
9237 1464 : var_name_index.value(), &var_value,
9238 732 : &var_details, if_not_found);
9239 732 : Goto(if_found);
9240 : }
9241 732 : }
9242 :
9243 392 : Node* CodeStubAssembler::GetMethod(Node* context, Node* object,
9244 : Handle<Name> name,
9245 : Label* if_null_or_undefined) {
9246 392 : Node* method = GetProperty(context, object, name);
9247 :
9248 392 : GotoIf(IsUndefined(method), if_null_or_undefined);
9249 392 : GotoIf(IsNull(method), if_null_or_undefined);
9250 :
9251 392 : return method;
9252 : }
9253 :
9254 56 : TNode<Object> CodeStubAssembler::GetIteratorMethod(
9255 : TNode<Context> context, TNode<HeapObject> heap_obj,
9256 : Label* if_iteratorundefined) {
9257 112 : return CAST(GetMethod(context, heap_obj,
9258 : isolate()->factory()->iterator_symbol(),
9259 : if_iteratorundefined));
9260 : }
9261 :
9262 1068 : void CodeStubAssembler::LoadPropertyFromFastObject(
9263 : Node* object, Node* map, TNode<DescriptorArray> descriptors,
9264 : Node* name_index, Variable* var_details, Variable* var_value) {
9265 : DCHECK_EQ(MachineRepresentation::kWord32, var_details->rep());
9266 : DCHECK_EQ(MachineRepresentation::kTagged, var_value->rep());
9267 :
9268 : Node* details =
9269 1068 : LoadDetailsByKeyIndex(descriptors, UncheckedCast<IntPtrT>(name_index));
9270 1068 : var_details->Bind(details);
9271 :
9272 : LoadPropertyFromFastObject(object, map, descriptors, name_index, details,
9273 1068 : var_value);
9274 1068 : }
9275 :
9276 1292 : void CodeStubAssembler::LoadPropertyFromFastObject(
9277 : Node* object, Node* map, TNode<DescriptorArray> descriptors,
9278 : Node* name_index, Node* details, Variable* var_value) {
9279 1292 : Comment("[ LoadPropertyFromFastObject");
9280 :
9281 1292 : Node* location = DecodeWord32<PropertyDetails::LocationField>(details);
9282 :
9283 2584 : Label if_in_field(this), if_in_descriptor(this), done(this);
9284 2584 : Branch(Word32Equal(location, Int32Constant(kField)), &if_in_field,
9285 1292 : &if_in_descriptor);
9286 1292 : BIND(&if_in_field);
9287 : {
9288 : Node* field_index =
9289 1292 : DecodeWordFromWord32<PropertyDetails::FieldIndexField>(details);
9290 : Node* representation =
9291 1292 : DecodeWord32<PropertyDetails::RepresentationField>(details);
9292 :
9293 : field_index =
9294 1292 : IntPtrAdd(field_index, LoadMapInobjectPropertiesStartInWords(map));
9295 1292 : Node* instance_size_in_words = LoadMapInstanceSizeInWords(map);
9296 :
9297 2584 : Label if_inobject(this), if_backing_store(this);
9298 2584 : VARIABLE(var_double_value, MachineRepresentation::kFloat64);
9299 2584 : Label rebox_double(this, &var_double_value);
9300 2584 : Branch(UintPtrLessThan(field_index, instance_size_in_words), &if_inobject,
9301 1292 : &if_backing_store);
9302 1292 : BIND(&if_inobject);
9303 : {
9304 1292 : Comment("if_inobject");
9305 1292 : Node* field_offset = TimesTaggedSize(field_index);
9306 :
9307 2584 : Label if_double(this), if_tagged(this);
9308 2584 : Branch(Word32NotEqual(representation,
9309 2584 : Int32Constant(Representation::kDouble)),
9310 1292 : &if_tagged, &if_double);
9311 1292 : BIND(&if_tagged);
9312 : {
9313 1292 : var_value->Bind(LoadObjectField(object, field_offset));
9314 1292 : Goto(&done);
9315 : }
9316 1292 : BIND(&if_double);
9317 : {
9318 : if (FLAG_unbox_double_fields) {
9319 : var_double_value.Bind(
9320 : LoadObjectField(object, field_offset, MachineType::Float64()));
9321 : } else {
9322 1292 : Node* mutable_heap_number = LoadObjectField(object, field_offset);
9323 1292 : var_double_value.Bind(LoadHeapNumberValue(mutable_heap_number));
9324 : }
9325 1292 : Goto(&rebox_double);
9326 : }
9327 : }
9328 1292 : BIND(&if_backing_store);
9329 : {
9330 1292 : Comment("if_backing_store");
9331 1292 : TNode<HeapObject> properties = LoadFastProperties(object);
9332 1292 : field_index = IntPtrSub(field_index, instance_size_in_words);
9333 1292 : Node* value = LoadPropertyArrayElement(CAST(properties), field_index);
9334 :
9335 2584 : Label if_double(this), if_tagged(this);
9336 2584 : Branch(Word32NotEqual(representation,
9337 2584 : Int32Constant(Representation::kDouble)),
9338 1292 : &if_tagged, &if_double);
9339 1292 : BIND(&if_tagged);
9340 : {
9341 1292 : var_value->Bind(value);
9342 1292 : Goto(&done);
9343 : }
9344 1292 : BIND(&if_double);
9345 : {
9346 1292 : var_double_value.Bind(LoadHeapNumberValue(value));
9347 1292 : Goto(&rebox_double);
9348 : }
9349 : }
9350 1292 : BIND(&rebox_double);
9351 : {
9352 1292 : Comment("rebox_double");
9353 1292 : Node* heap_number = AllocateHeapNumberWithValue(var_double_value.value());
9354 1292 : var_value->Bind(heap_number);
9355 1292 : Goto(&done);
9356 : }
9357 : }
9358 1292 : BIND(&if_in_descriptor);
9359 : {
9360 1292 : var_value->Bind(
9361 2584 : LoadValueByKeyIndex(descriptors, UncheckedCast<IntPtrT>(name_index)));
9362 1292 : Goto(&done);
9363 : }
9364 1292 : BIND(&done);
9365 :
9366 1292 : Comment("] LoadPropertyFromFastObject");
9367 1292 : }
9368 :
9369 2412 : void CodeStubAssembler::LoadPropertyFromNameDictionary(Node* dictionary,
9370 : Node* name_index,
9371 : Variable* var_details,
9372 : Variable* var_value) {
9373 2412 : Comment("LoadPropertyFromNameDictionary");
9374 : CSA_ASSERT(this, IsNameDictionary(dictionary));
9375 :
9376 2412 : var_details->Bind(
9377 4824 : LoadDetailsByKeyIndex<NameDictionary>(dictionary, name_index));
9378 2412 : var_value->Bind(LoadValueByKeyIndex<NameDictionary>(dictionary, name_index));
9379 :
9380 2412 : Comment("] LoadPropertyFromNameDictionary");
9381 2412 : }
9382 :
9383 1184 : void CodeStubAssembler::LoadPropertyFromGlobalDictionary(Node* dictionary,
9384 : Node* name_index,
9385 : Variable* var_details,
9386 : Variable* var_value,
9387 : Label* if_deleted) {
9388 1184 : Comment("[ LoadPropertyFromGlobalDictionary");
9389 : CSA_ASSERT(this, IsGlobalDictionary(dictionary));
9390 :
9391 1184 : Node* property_cell = LoadFixedArrayElement(CAST(dictionary), name_index);
9392 : CSA_ASSERT(this, IsPropertyCell(property_cell));
9393 :
9394 1184 : Node* value = LoadObjectField(property_cell, PropertyCell::kValueOffset);
9395 1184 : GotoIf(WordEqual(value, TheHoleConstant()), if_deleted);
9396 :
9397 1184 : var_value->Bind(value);
9398 :
9399 2368 : Node* details = LoadAndUntagToWord32ObjectField(property_cell,
9400 1184 : PropertyCell::kDetailsOffset);
9401 1184 : var_details->Bind(details);
9402 :
9403 1184 : Comment("] LoadPropertyFromGlobalDictionary");
9404 1184 : }
9405 :
9406 : // |value| is the property backing store's contents, which is either a value
9407 : // or an accessor pair, as specified by |details|.
9408 : // Returns either the original value, or the result of the getter call.
9409 3812 : TNode<Object> CodeStubAssembler::CallGetterIfAccessor(
9410 : Node* value, Node* details, Node* context, Node* receiver,
9411 : Label* if_bailout, GetOwnPropertyMode mode) {
9412 7624 : VARIABLE(var_value, MachineRepresentation::kTagged, value);
9413 7624 : Label done(this), if_accessor_info(this, Label::kDeferred);
9414 :
9415 3812 : Node* kind = DecodeWord32<PropertyDetails::KindField>(details);
9416 3812 : GotoIf(Word32Equal(kind, Int32Constant(kData)), &done);
9417 :
9418 : // Accessor case.
9419 3812 : GotoIfNot(IsAccessorPair(value), &if_accessor_info);
9420 :
9421 : // AccessorPair case.
9422 : {
9423 3812 : if (mode == kCallJSGetter) {
9424 3588 : Node* accessor_pair = value;
9425 : Node* getter =
9426 3588 : LoadObjectField(accessor_pair, AccessorPair::kGetterOffset);
9427 3588 : Node* getter_map = LoadMap(getter);
9428 3588 : Node* instance_type = LoadMapInstanceType(getter_map);
9429 : // FunctionTemplateInfo getters are not supported yet.
9430 7176 : GotoIf(InstanceTypeEqual(instance_type, FUNCTION_TEMPLATE_INFO_TYPE),
9431 3588 : if_bailout);
9432 :
9433 : // Return undefined if the {getter} is not callable.
9434 3588 : var_value.Bind(UndefinedConstant());
9435 3588 : GotoIfNot(IsCallableMap(getter_map), &done);
9436 :
9437 : // Call the accessor.
9438 7176 : Callable callable = CodeFactory::Call(isolate());
9439 3588 : Node* result = CallJS(callable, context, getter, receiver);
9440 3588 : var_value.Bind(result);
9441 : }
9442 3812 : Goto(&done);
9443 : }
9444 :
9445 : // AccessorInfo case.
9446 3812 : BIND(&if_accessor_info);
9447 : {
9448 3812 : Node* accessor_info = value;
9449 : CSA_ASSERT(this, IsAccessorInfo(value));
9450 : CSA_ASSERT(this, TaggedIsNotSmi(receiver));
9451 7624 : Label if_array(this), if_function(this), if_value(this);
9452 :
9453 : // Dispatch based on {receiver} instance type.
9454 3812 : Node* receiver_map = LoadMap(receiver);
9455 3812 : Node* receiver_instance_type = LoadMapInstanceType(receiver_map);
9456 3812 : GotoIf(IsJSArrayInstanceType(receiver_instance_type), &if_array);
9457 3812 : GotoIf(IsJSFunctionInstanceType(receiver_instance_type), &if_function);
9458 7624 : Branch(IsJSValueInstanceType(receiver_instance_type), &if_value,
9459 3812 : if_bailout);
9460 :
9461 : // JSArray AccessorInfo case.
9462 3812 : BIND(&if_array);
9463 : {
9464 : // We only deal with the "length" accessor on JSArray.
9465 7624 : GotoIfNot(IsLengthString(
9466 7624 : LoadObjectField(accessor_info, AccessorInfo::kNameOffset)),
9467 3812 : if_bailout);
9468 3812 : var_value.Bind(LoadJSArrayLength(receiver));
9469 3812 : Goto(&done);
9470 : }
9471 :
9472 : // JSFunction AccessorInfo case.
9473 3812 : BIND(&if_function);
9474 : {
9475 : // We only deal with the "prototype" accessor on JSFunction here.
9476 7624 : GotoIfNot(IsPrototypeString(
9477 7624 : LoadObjectField(accessor_info, AccessorInfo::kNameOffset)),
9478 3812 : if_bailout);
9479 :
9480 7624 : GotoIfPrototypeRequiresRuntimeLookup(CAST(receiver), CAST(receiver_map),
9481 3812 : if_bailout);
9482 3812 : var_value.Bind(LoadJSFunctionPrototype(receiver, if_bailout));
9483 3812 : Goto(&done);
9484 : }
9485 :
9486 : // JSValue AccessorInfo case.
9487 3812 : BIND(&if_value);
9488 : {
9489 : // We only deal with the "length" accessor on JSValue string wrappers.
9490 7624 : GotoIfNot(IsLengthString(
9491 7624 : LoadObjectField(accessor_info, AccessorInfo::kNameOffset)),
9492 3812 : if_bailout);
9493 3812 : Node* receiver_value = LoadJSValueValue(receiver);
9494 3812 : GotoIfNot(TaggedIsNotSmi(receiver_value), if_bailout);
9495 3812 : GotoIfNot(IsString(receiver_value), if_bailout);
9496 3812 : var_value.Bind(LoadStringLengthAsSmi(receiver_value));
9497 3812 : Goto(&done);
9498 : }
9499 : }
9500 :
9501 3812 : BIND(&done);
9502 7624 : return UncheckedCast<Object>(var_value.value());
9503 : }
9504 :
9505 228 : void CodeStubAssembler::TryGetOwnProperty(
9506 : Node* context, Node* receiver, Node* object, Node* map, Node* instance_type,
9507 : Node* unique_name, Label* if_found_value, Variable* var_value,
9508 : Label* if_not_found, Label* if_bailout) {
9509 : TryGetOwnProperty(context, receiver, object, map, instance_type, unique_name,
9510 : if_found_value, var_value, nullptr, nullptr, if_not_found,
9511 228 : if_bailout, kCallJSGetter);
9512 228 : }
9513 :
9514 452 : void CodeStubAssembler::TryGetOwnProperty(
9515 : Node* context, Node* receiver, Node* object, Node* map, Node* instance_type,
9516 : Node* unique_name, Label* if_found_value, Variable* var_value,
9517 : Variable* var_details, Variable* var_raw_value, Label* if_not_found,
9518 : Label* if_bailout, GetOwnPropertyMode mode) {
9519 : DCHECK_EQ(MachineRepresentation::kTagged, var_value->rep());
9520 452 : Comment("TryGetOwnProperty");
9521 : CSA_ASSERT(this, IsUniqueNameNoIndex(CAST(unique_name)));
9522 :
9523 904 : TVARIABLE(HeapObject, var_meta_storage);
9524 904 : TVARIABLE(IntPtrT, var_entry);
9525 :
9526 904 : Label if_found_fast(this), if_found_dict(this), if_found_global(this);
9527 :
9528 904 : VARIABLE(local_var_details, MachineRepresentation::kWord32);
9529 452 : if (!var_details) {
9530 228 : var_details = &local_var_details;
9531 : }
9532 904 : Label if_found(this);
9533 :
9534 904 : TryLookupProperty(object, map, instance_type, unique_name, &if_found_fast,
9535 : &if_found_dict, &if_found_global, &var_meta_storage,
9536 452 : &var_entry, if_not_found, if_bailout);
9537 452 : BIND(&if_found_fast);
9538 : {
9539 452 : TNode<DescriptorArray> descriptors = CAST(var_meta_storage.value());
9540 452 : Node* name_index = var_entry.value();
9541 :
9542 : LoadPropertyFromFastObject(object, map, descriptors, name_index,
9543 452 : var_details, var_value);
9544 452 : Goto(&if_found);
9545 : }
9546 452 : BIND(&if_found_dict);
9547 : {
9548 452 : Node* dictionary = var_meta_storage.value();
9549 452 : Node* entry = var_entry.value();
9550 452 : LoadPropertyFromNameDictionary(dictionary, entry, var_details, var_value);
9551 452 : Goto(&if_found);
9552 : }
9553 452 : BIND(&if_found_global);
9554 : {
9555 452 : Node* dictionary = var_meta_storage.value();
9556 452 : Node* entry = var_entry.value();
9557 :
9558 : LoadPropertyFromGlobalDictionary(dictionary, entry, var_details, var_value,
9559 452 : if_not_found);
9560 452 : Goto(&if_found);
9561 : }
9562 : // Here we have details and value which could be an accessor.
9563 452 : BIND(&if_found);
9564 : {
9565 : // TODO(ishell): Execute C++ accessor in case of accessor info
9566 452 : if (var_raw_value) {
9567 224 : var_raw_value->Bind(var_value->value());
9568 : }
9569 904 : Node* value = CallGetterIfAccessor(var_value->value(), var_details->value(),
9570 904 : context, receiver, if_bailout, mode);
9571 452 : var_value->Bind(value);
9572 452 : Goto(if_found_value);
9573 : }
9574 452 : }
9575 :
9576 732 : void CodeStubAssembler::TryLookupElement(Node* object, Node* map,
9577 : SloppyTNode<Int32T> instance_type,
9578 : SloppyTNode<IntPtrT> intptr_index,
9579 : Label* if_found, Label* if_absent,
9580 : Label* if_not_found,
9581 : Label* if_bailout) {
9582 : // Handle special objects in runtime.
9583 732 : GotoIf(IsSpecialReceiverInstanceType(instance_type), if_bailout);
9584 :
9585 732 : Node* elements_kind = LoadMapElementsKind(map);
9586 :
9587 : // TODO(verwaest): Support other elements kinds as well.
9588 1464 : Label if_isobjectorsmi(this), if_isdouble(this), if_isdictionary(this),
9589 1464 : if_isfaststringwrapper(this), if_isslowstringwrapper(this), if_oob(this),
9590 1464 : if_typedarray(this);
9591 : // clang-format off
9592 : int32_t values[] = {
9593 : // Handled by {if_isobjectorsmi}.
9594 : PACKED_SMI_ELEMENTS, HOLEY_SMI_ELEMENTS, PACKED_ELEMENTS,
9595 : HOLEY_ELEMENTS,
9596 : // Handled by {if_isdouble}.
9597 : PACKED_DOUBLE_ELEMENTS, HOLEY_DOUBLE_ELEMENTS,
9598 : // Handled by {if_isdictionary}.
9599 : DICTIONARY_ELEMENTS,
9600 : // Handled by {if_isfaststringwrapper}.
9601 : FAST_STRING_WRAPPER_ELEMENTS,
9602 : // Handled by {if_isslowstringwrapper}.
9603 : SLOW_STRING_WRAPPER_ELEMENTS,
9604 : // Handled by {if_not_found}.
9605 : NO_ELEMENTS,
9606 : // Handled by {if_typed_array}.
9607 : UINT8_ELEMENTS,
9608 : INT8_ELEMENTS,
9609 : UINT16_ELEMENTS,
9610 : INT16_ELEMENTS,
9611 : UINT32_ELEMENTS,
9612 : INT32_ELEMENTS,
9613 : FLOAT32_ELEMENTS,
9614 : FLOAT64_ELEMENTS,
9615 : UINT8_CLAMPED_ELEMENTS,
9616 : BIGUINT64_ELEMENTS,
9617 : BIGINT64_ELEMENTS,
9618 732 : };
9619 : Label* labels[] = {
9620 : &if_isobjectorsmi, &if_isobjectorsmi, &if_isobjectorsmi,
9621 : &if_isobjectorsmi,
9622 : &if_isdouble, &if_isdouble,
9623 : &if_isdictionary,
9624 : &if_isfaststringwrapper,
9625 : &if_isslowstringwrapper,
9626 : if_not_found,
9627 : &if_typedarray,
9628 : &if_typedarray,
9629 : &if_typedarray,
9630 : &if_typedarray,
9631 : &if_typedarray,
9632 : &if_typedarray,
9633 : &if_typedarray,
9634 : &if_typedarray,
9635 : &if_typedarray,
9636 : &if_typedarray,
9637 : &if_typedarray,
9638 732 : };
9639 : // clang-format on
9640 : STATIC_ASSERT(arraysize(values) == arraysize(labels));
9641 732 : Switch(elements_kind, if_bailout, values, labels, arraysize(values));
9642 :
9643 732 : BIND(&if_isobjectorsmi);
9644 : {
9645 732 : TNode<FixedArray> elements = CAST(LoadElements(object));
9646 732 : TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(elements);
9647 :
9648 732 : GotoIfNot(UintPtrLessThan(intptr_index, length), &if_oob);
9649 :
9650 732 : TNode<Object> element = UnsafeLoadFixedArrayElement(elements, intptr_index);
9651 732 : TNode<Oddball> the_hole = TheHoleConstant();
9652 732 : Branch(WordEqual(element, the_hole), if_not_found, if_found);
9653 : }
9654 732 : BIND(&if_isdouble);
9655 : {
9656 732 : TNode<FixedArrayBase> elements = LoadElements(object);
9657 732 : TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(elements);
9658 :
9659 732 : GotoIfNot(UintPtrLessThan(intptr_index, length), &if_oob);
9660 :
9661 : // Check if the element is a double hole, but don't load it.
9662 1464 : LoadFixedDoubleArrayElement(CAST(elements), intptr_index,
9663 : MachineType::None(), 0, INTPTR_PARAMETERS,
9664 1464 : if_not_found);
9665 732 : Goto(if_found);
9666 : }
9667 732 : BIND(&if_isdictionary);
9668 : {
9669 : // Negative keys must be converted to property names.
9670 732 : GotoIf(IntPtrLessThan(intptr_index, IntPtrConstant(0)), if_bailout);
9671 :
9672 1464 : TVARIABLE(IntPtrT, var_entry);
9673 732 : TNode<NumberDictionary> elements = CAST(LoadElements(object));
9674 : NumberDictionaryLookup(elements, intptr_index, if_found, &var_entry,
9675 732 : if_not_found);
9676 : }
9677 732 : BIND(&if_isfaststringwrapper);
9678 : {
9679 : CSA_ASSERT(this, HasInstanceType(object, JS_VALUE_TYPE));
9680 732 : Node* string = LoadJSValueValue(object);
9681 : CSA_ASSERT(this, IsString(string));
9682 732 : Node* length = LoadStringLengthAsWord(string);
9683 732 : GotoIf(UintPtrLessThan(intptr_index, length), if_found);
9684 732 : Goto(&if_isobjectorsmi);
9685 : }
9686 732 : BIND(&if_isslowstringwrapper);
9687 : {
9688 : CSA_ASSERT(this, HasInstanceType(object, JS_VALUE_TYPE));
9689 732 : Node* string = LoadJSValueValue(object);
9690 : CSA_ASSERT(this, IsString(string));
9691 732 : Node* length = LoadStringLengthAsWord(string);
9692 732 : GotoIf(UintPtrLessThan(intptr_index, length), if_found);
9693 732 : Goto(&if_isdictionary);
9694 : }
9695 732 : BIND(&if_typedarray);
9696 : {
9697 732 : Node* buffer = LoadObjectField(object, JSArrayBufferView::kBufferOffset);
9698 732 : GotoIf(IsDetachedBuffer(buffer), if_absent);
9699 :
9700 732 : Node* length = SmiUntag(LoadJSTypedArrayLength(CAST(object)));
9701 732 : Branch(UintPtrLessThan(intptr_index, length), if_found, if_absent);
9702 : }
9703 732 : BIND(&if_oob);
9704 : {
9705 : // Positive OOB indices mean "not found", negative indices must be
9706 : // converted to property names.
9707 732 : GotoIf(IntPtrLessThan(intptr_index, IntPtrConstant(0)), if_bailout);
9708 732 : Goto(if_not_found);
9709 : }
9710 732 : }
9711 :
9712 728 : void CodeStubAssembler::BranchIfMaybeSpecialIndex(TNode<String> name_string,
9713 : Label* if_maybe_special_index,
9714 : Label* if_not_special_index) {
9715 : // TODO(cwhan.tunz): Implement fast cases more.
9716 :
9717 : // If a name is empty or too long, it's not a special index
9718 : // Max length of canonical double: -X.XXXXXXXXXXXXXXXXX-eXXX
9719 728 : const int kBufferSize = 24;
9720 728 : TNode<Smi> string_length = LoadStringLengthAsSmi(name_string);
9721 728 : GotoIf(SmiEqual(string_length, SmiConstant(0)), if_not_special_index);
9722 1456 : GotoIf(SmiGreaterThan(string_length, SmiConstant(kBufferSize)),
9723 728 : if_not_special_index);
9724 :
9725 : // If the first character of name is not a digit or '-', or we can't match it
9726 : // to Infinity or NaN, then this is not a special index.
9727 728 : TNode<Int32T> first_char = StringCharCodeAt(name_string, IntPtrConstant(0));
9728 : // If the name starts with '-', it can be a negative index.
9729 728 : GotoIf(Word32Equal(first_char, Int32Constant('-')), if_maybe_special_index);
9730 : // If the name starts with 'I', it can be "Infinity".
9731 728 : GotoIf(Word32Equal(first_char, Int32Constant('I')), if_maybe_special_index);
9732 : // If the name starts with 'N', it can be "NaN".
9733 728 : GotoIf(Word32Equal(first_char, Int32Constant('N')), if_maybe_special_index);
9734 : // Finally, if the first character is not a digit either, then we are sure
9735 : // that the name is not a special index.
9736 728 : GotoIf(Uint32LessThan(first_char, Int32Constant('0')), if_not_special_index);
9737 728 : GotoIf(Uint32LessThan(Int32Constant('9'), first_char), if_not_special_index);
9738 728 : Goto(if_maybe_special_index);
9739 728 : }
9740 :
9741 728 : void CodeStubAssembler::TryPrototypeChainLookup(
9742 : Node* receiver, Node* key, const LookupInHolder& lookup_property_in_holder,
9743 : const LookupInHolder& lookup_element_in_holder, Label* if_end,
9744 : Label* if_bailout, Label* if_proxy) {
9745 : // Ensure receiver is JSReceiver, otherwise bailout.
9746 1456 : Label if_objectisnotsmi(this);
9747 728 : Branch(TaggedIsSmi(receiver), if_bailout, &if_objectisnotsmi);
9748 728 : BIND(&if_objectisnotsmi);
9749 :
9750 728 : Node* map = LoadMap(receiver);
9751 728 : Node* instance_type = LoadMapInstanceType(map);
9752 : {
9753 1456 : Label if_objectisreceiver(this);
9754 : STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
9755 : STATIC_ASSERT(FIRST_JS_RECEIVER_TYPE == JS_PROXY_TYPE);
9756 1456 : Branch(IsJSReceiverInstanceType(instance_type), &if_objectisreceiver,
9757 728 : if_bailout);
9758 728 : BIND(&if_objectisreceiver);
9759 :
9760 728 : if (if_proxy) {
9761 728 : GotoIf(InstanceTypeEqual(instance_type, JS_PROXY_TYPE), if_proxy);
9762 : }
9763 : }
9764 :
9765 1456 : VARIABLE(var_index, MachineType::PointerRepresentation());
9766 1456 : VARIABLE(var_unique, MachineRepresentation::kTagged);
9767 :
9768 1456 : Label if_keyisindex(this), if_iskeyunique(this);
9769 : TryToName(key, &if_keyisindex, &var_index, &if_iskeyunique, &var_unique,
9770 728 : if_bailout);
9771 :
9772 728 : BIND(&if_iskeyunique);
9773 : {
9774 1456 : VARIABLE(var_holder, MachineRepresentation::kTagged, receiver);
9775 1456 : VARIABLE(var_holder_map, MachineRepresentation::kTagged, map);
9776 1456 : VARIABLE(var_holder_instance_type, MachineRepresentation::kWord32,
9777 : instance_type);
9778 :
9779 : Variable* merged_variables[] = {&var_holder, &var_holder_map,
9780 728 : &var_holder_instance_type};
9781 1456 : Label loop(this, arraysize(merged_variables), merged_variables);
9782 728 : Goto(&loop);
9783 728 : BIND(&loop);
9784 : {
9785 728 : Node* holder_map = var_holder_map.value();
9786 728 : Node* holder_instance_type = var_holder_instance_type.value();
9787 :
9788 1456 : Label next_proto(this), check_integer_indexed_exotic(this);
9789 728 : lookup_property_in_holder(receiver, var_holder.value(), holder_map,
9790 : holder_instance_type, var_unique.value(),
9791 728 : &check_integer_indexed_exotic, if_bailout);
9792 :
9793 728 : BIND(&check_integer_indexed_exotic);
9794 : {
9795 : // Bailout if it can be an integer indexed exotic case.
9796 1456 : GotoIfNot(InstanceTypeEqual(holder_instance_type, JS_TYPED_ARRAY_TYPE),
9797 728 : &next_proto);
9798 728 : GotoIfNot(IsString(var_unique.value()), &next_proto);
9799 1456 : BranchIfMaybeSpecialIndex(CAST(var_unique.value()), if_bailout,
9800 728 : &next_proto);
9801 : }
9802 :
9803 728 : BIND(&next_proto);
9804 :
9805 728 : Node* proto = LoadMapPrototype(holder_map);
9806 :
9807 728 : GotoIf(IsNull(proto), if_end);
9808 :
9809 728 : Node* map = LoadMap(proto);
9810 728 : Node* instance_type = LoadMapInstanceType(map);
9811 :
9812 728 : var_holder.Bind(proto);
9813 728 : var_holder_map.Bind(map);
9814 728 : var_holder_instance_type.Bind(instance_type);
9815 728 : Goto(&loop);
9816 : }
9817 : }
9818 728 : BIND(&if_keyisindex);
9819 : {
9820 1456 : VARIABLE(var_holder, MachineRepresentation::kTagged, receiver);
9821 1456 : VARIABLE(var_holder_map, MachineRepresentation::kTagged, map);
9822 1456 : VARIABLE(var_holder_instance_type, MachineRepresentation::kWord32,
9823 : instance_type);
9824 :
9825 : Variable* merged_variables[] = {&var_holder, &var_holder_map,
9826 728 : &var_holder_instance_type};
9827 1456 : Label loop(this, arraysize(merged_variables), merged_variables);
9828 728 : Goto(&loop);
9829 728 : BIND(&loop);
9830 : {
9831 1456 : Label next_proto(this);
9832 728 : lookup_element_in_holder(receiver, var_holder.value(),
9833 : var_holder_map.value(),
9834 : var_holder_instance_type.value(),
9835 728 : var_index.value(), &next_proto, if_bailout);
9836 728 : BIND(&next_proto);
9837 :
9838 728 : Node* proto = LoadMapPrototype(var_holder_map.value());
9839 :
9840 728 : GotoIf(IsNull(proto), if_end);
9841 :
9842 728 : Node* map = LoadMap(proto);
9843 728 : Node* instance_type = LoadMapInstanceType(map);
9844 :
9845 728 : var_holder.Bind(proto);
9846 728 : var_holder_map.Bind(map);
9847 728 : var_holder_instance_type.Bind(instance_type);
9848 728 : Goto(&loop);
9849 : }
9850 : }
9851 728 : }
9852 :
9853 168 : Node* CodeStubAssembler::HasInPrototypeChain(Node* context, Node* object,
9854 : Node* prototype) {
9855 : CSA_ASSERT(this, TaggedIsNotSmi(object));
9856 336 : VARIABLE(var_result, MachineRepresentation::kTagged);
9857 336 : Label return_false(this), return_true(this),
9858 336 : return_runtime(this, Label::kDeferred), return_result(this);
9859 :
9860 : // Loop through the prototype chain looking for the {prototype}.
9861 336 : VARIABLE(var_object_map, MachineRepresentation::kTagged, LoadMap(object));
9862 336 : Label loop(this, &var_object_map);
9863 168 : Goto(&loop);
9864 168 : BIND(&loop);
9865 : {
9866 : // Check if we can determine the prototype directly from the {object_map}.
9867 336 : Label if_objectisdirect(this), if_objectisspecial(this, Label::kDeferred);
9868 168 : Node* object_map = var_object_map.value();
9869 168 : TNode<Int32T> object_instance_type = LoadMapInstanceType(object_map);
9870 336 : Branch(IsSpecialReceiverInstanceType(object_instance_type),
9871 168 : &if_objectisspecial, &if_objectisdirect);
9872 168 : BIND(&if_objectisspecial);
9873 : {
9874 : // The {object_map} is a special receiver map or a primitive map, check
9875 : // if we need to use the if_objectisspecial path in the runtime.
9876 336 : GotoIf(InstanceTypeEqual(object_instance_type, JS_PROXY_TYPE),
9877 168 : &return_runtime);
9878 168 : Node* object_bitfield = LoadMapBitField(object_map);
9879 : int mask = Map::HasNamedInterceptorBit::kMask |
9880 168 : Map::IsAccessCheckNeededBit::kMask;
9881 336 : Branch(IsSetWord32(object_bitfield, mask), &return_runtime,
9882 168 : &if_objectisdirect);
9883 : }
9884 168 : BIND(&if_objectisdirect);
9885 :
9886 : // Check the current {object} prototype.
9887 168 : Node* object_prototype = LoadMapPrototype(object_map);
9888 168 : GotoIf(IsNull(object_prototype), &return_false);
9889 168 : GotoIf(WordEqual(object_prototype, prototype), &return_true);
9890 :
9891 : // Continue with the prototype.
9892 : CSA_ASSERT(this, TaggedIsNotSmi(object_prototype));
9893 168 : var_object_map.Bind(LoadMap(object_prototype));
9894 168 : Goto(&loop);
9895 : }
9896 :
9897 168 : BIND(&return_true);
9898 168 : var_result.Bind(TrueConstant());
9899 168 : Goto(&return_result);
9900 :
9901 168 : BIND(&return_false);
9902 168 : var_result.Bind(FalseConstant());
9903 168 : Goto(&return_result);
9904 :
9905 168 : BIND(&return_runtime);
9906 : {
9907 : // Fallback to the runtime implementation.
9908 168 : var_result.Bind(
9909 336 : CallRuntime(Runtime::kHasInPrototypeChain, context, object, prototype));
9910 : }
9911 168 : Goto(&return_result);
9912 :
9913 168 : BIND(&return_result);
9914 336 : return var_result.value();
9915 : }
9916 :
9917 112 : Node* CodeStubAssembler::OrdinaryHasInstance(Node* context, Node* callable,
9918 : Node* object) {
9919 224 : VARIABLE(var_result, MachineRepresentation::kTagged);
9920 224 : Label return_runtime(this, Label::kDeferred), return_result(this);
9921 :
9922 : // Goto runtime if {object} is a Smi.
9923 112 : GotoIf(TaggedIsSmi(object), &return_runtime);
9924 :
9925 : // Goto runtime if {callable} is a Smi.
9926 112 : GotoIf(TaggedIsSmi(callable), &return_runtime);
9927 :
9928 : // Load map of {callable}.
9929 112 : Node* callable_map = LoadMap(callable);
9930 :
9931 : // Goto runtime if {callable} is not a JSFunction.
9932 112 : Node* callable_instance_type = LoadMapInstanceType(callable_map);
9933 224 : GotoIfNot(InstanceTypeEqual(callable_instance_type, JS_FUNCTION_TYPE),
9934 112 : &return_runtime);
9935 :
9936 224 : GotoIfPrototypeRequiresRuntimeLookup(CAST(callable), CAST(callable_map),
9937 112 : &return_runtime);
9938 :
9939 : // Get the "prototype" (or initial map) of the {callable}.
9940 : Node* callable_prototype =
9941 112 : LoadObjectField(callable, JSFunction::kPrototypeOrInitialMapOffset);
9942 : {
9943 224 : Label callable_prototype_valid(this);
9944 224 : VARIABLE(var_callable_prototype, MachineRepresentation::kTagged,
9945 : callable_prototype);
9946 :
9947 : // Resolve the "prototype" if the {callable} has an initial map. Afterwards
9948 : // the {callable_prototype} will be either the JSReceiver prototype object
9949 : // or the hole value, which means that no instances of the {callable} were
9950 : // created so far and hence we should return false.
9951 : Node* callable_prototype_instance_type =
9952 112 : LoadInstanceType(callable_prototype);
9953 224 : GotoIfNot(InstanceTypeEqual(callable_prototype_instance_type, MAP_TYPE),
9954 112 : &callable_prototype_valid);
9955 112 : var_callable_prototype.Bind(
9956 224 : LoadObjectField(callable_prototype, Map::kPrototypeOffset));
9957 112 : Goto(&callable_prototype_valid);
9958 112 : BIND(&callable_prototype_valid);
9959 112 : callable_prototype = var_callable_prototype.value();
9960 : }
9961 :
9962 : // Loop through the prototype chain looking for the {callable} prototype.
9963 112 : var_result.Bind(HasInPrototypeChain(context, object, callable_prototype));
9964 112 : Goto(&return_result);
9965 :
9966 112 : BIND(&return_runtime);
9967 : {
9968 : // Fallback to the runtime implementation.
9969 112 : var_result.Bind(
9970 224 : CallRuntime(Runtime::kOrdinaryHasInstance, context, callable, object));
9971 : }
9972 112 : Goto(&return_result);
9973 :
9974 112 : BIND(&return_result);
9975 224 : return var_result.value();
9976 : }
9977 :
9978 338680 : TNode<IntPtrT> CodeStubAssembler::ElementOffsetFromIndex(Node* index_node,
9979 : ElementsKind kind,
9980 : ParameterMode mode,
9981 : int base_size) {
9982 : CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, mode));
9983 338680 : int element_size_shift = ElementsKindToShiftSize(kind);
9984 338680 : int element_size = 1 << element_size_shift;
9985 338680 : int const kSmiShiftBits = kSmiShiftSize + kSmiTagSize;
9986 338680 : intptr_t index = 0;
9987 338680 : bool constant_index = false;
9988 338680 : if (mode == SMI_PARAMETERS) {
9989 22604 : element_size_shift -= kSmiShiftBits;
9990 22604 : Smi smi_index;
9991 22604 : constant_index = ToSmiConstant(index_node, &smi_index);
9992 22604 : if (constant_index) index = smi_index->value();
9993 22604 : index_node = BitcastTaggedToWord(index_node);
9994 : } else {
9995 : DCHECK(mode == INTPTR_PARAMETERS);
9996 316076 : constant_index = ToIntPtrConstant(index_node, index);
9997 : }
9998 338680 : if (constant_index) {
9999 94740 : return IntPtrConstant(base_size + element_size * index);
10000 : }
10001 :
10002 : TNode<WordT> shifted_index =
10003 : (element_size_shift == 0)
10004 8512 : ? UncheckedCast<WordT>(index_node)
10005 : : ((element_size_shift > 0)
10006 1181620 : ? WordShl(index_node, IntPtrConstant(element_size_shift))
10007 965232 : : WordSar(index_node, IntPtrConstant(-element_size_shift)));
10008 243940 : return IntPtrAdd(IntPtrConstant(base_size), Signed(shifted_index));
10009 : }
10010 :
10011 0 : TNode<BoolT> CodeStubAssembler::IsOffsetInBounds(SloppyTNode<IntPtrT> offset,
10012 : SloppyTNode<IntPtrT> length,
10013 : int header_size,
10014 : ElementsKind kind) {
10015 : // Make sure we point to the last field.
10016 0 : int element_size = 1 << ElementsKindToShiftSize(kind);
10017 0 : int correction = header_size - kHeapObjectTag - element_size;
10018 : TNode<IntPtrT> last_offset =
10019 0 : ElementOffsetFromIndex(length, kind, INTPTR_PARAMETERS, correction);
10020 0 : return IntPtrLessThanOrEqual(offset, last_offset);
10021 : }
10022 :
10023 12836 : TNode<HeapObject> CodeStubAssembler::LoadFeedbackCellValue(
10024 : SloppyTNode<JSFunction> closure) {
10025 : TNode<FeedbackCell> feedback_cell =
10026 12836 : CAST(LoadObjectField(closure, JSFunction::kFeedbackCellOffset));
10027 12836 : return CAST(LoadObjectField(feedback_cell, FeedbackCell::kValueOffset));
10028 : }
10029 :
10030 12612 : TNode<HeapObject> CodeStubAssembler::LoadFeedbackVector(
10031 : SloppyTNode<JSFunction> closure) {
10032 25224 : TVARIABLE(HeapObject, maybe_vector, LoadFeedbackCellValue(closure));
10033 25224 : Label done(this);
10034 :
10035 : // If the closure doesn't have a feedback vector allocated yet, return
10036 : // undefined. FeedbackCell can contain Undefined / FixedArray (for lazy
10037 : // allocations) / FeedbackVector.
10038 12612 : GotoIf(IsFeedbackVector(maybe_vector.value()), &done);
10039 :
10040 : // In all other cases return Undefined.
10041 12612 : maybe_vector = UndefinedConstant();
10042 12612 : Goto(&done);
10043 :
10044 12612 : BIND(&done);
10045 25224 : return maybe_vector.value();
10046 : }
10047 :
10048 168 : TNode<FixedArray> CodeStubAssembler::LoadClosureFeedbackArray(
10049 : SloppyTNode<JSFunction> closure) {
10050 336 : TVARIABLE(HeapObject, feedback_cell_array, LoadFeedbackCellValue(closure));
10051 336 : Label end(this);
10052 :
10053 : // When feedback vectors are not yet allocated feedback cell contains a
10054 : // an array of feedback cells used by create closures.
10055 168 : GotoIf(IsFixedArray(feedback_cell_array.value()), &end);
10056 :
10057 : // Load FeedbackCellArray from feedback vector.
10058 168 : TNode<FeedbackVector> vector = CAST(feedback_cell_array.value());
10059 336 : feedback_cell_array = CAST(
10060 168 : LoadObjectField(vector, FeedbackVector::kClosureFeedbackCellArrayOffset));
10061 168 : Goto(&end);
10062 :
10063 168 : BIND(&end);
10064 336 : return CAST(feedback_cell_array.value());
10065 : }
10066 :
10067 504 : TNode<FeedbackVector> CodeStubAssembler::LoadFeedbackVectorForStub() {
10068 : TNode<JSFunction> function =
10069 504 : CAST(LoadFromParentFrame(JavaScriptFrameConstants::kFunctionOffset));
10070 504 : return CAST(LoadFeedbackVector(function));
10071 : }
10072 :
10073 8736 : void CodeStubAssembler::UpdateFeedback(Node* feedback, Node* maybe_vector,
10074 : Node* slot_id) {
10075 17472 : Label end(this);
10076 : // If feedback_vector is not valid, then nothing to do.
10077 8736 : GotoIf(IsUndefined(maybe_vector), &end);
10078 :
10079 : // This method is used for binary op and compare feedback. These
10080 : // vector nodes are initialized with a smi 0, so we can simply OR
10081 : // our new feedback in place.
10082 8736 : TNode<FeedbackVector> feedback_vector = CAST(maybe_vector);
10083 : TNode<MaybeObject> feedback_element =
10084 8736 : LoadFeedbackVectorSlot(feedback_vector, slot_id);
10085 8736 : TNode<Smi> previous_feedback = CAST(feedback_element);
10086 8736 : TNode<Smi> combined_feedback = SmiOr(previous_feedback, CAST(feedback));
10087 :
10088 8736 : GotoIf(SmiEqual(previous_feedback, combined_feedback), &end);
10089 : {
10090 8736 : StoreFeedbackVectorSlot(feedback_vector, slot_id, combined_feedback,
10091 8736 : SKIP_WRITE_BARRIER);
10092 8736 : ReportFeedbackUpdate(feedback_vector, slot_id, "UpdateFeedback");
10093 8736 : Goto(&end);
10094 : }
10095 :
10096 8736 : BIND(&end);
10097 8736 : }
10098 :
10099 13272 : void CodeStubAssembler::ReportFeedbackUpdate(
10100 : SloppyTNode<FeedbackVector> feedback_vector, SloppyTNode<IntPtrT> slot_id,
10101 : const char* reason) {
10102 : // Reset profiler ticks.
10103 13272 : StoreObjectFieldNoWriteBarrier(
10104 26544 : feedback_vector, FeedbackVector::kProfilerTicksOffset, Int32Constant(0),
10105 13272 : MachineRepresentation::kWord32);
10106 :
10107 : #ifdef V8_TRACE_FEEDBACK_UPDATES
10108 : // Trace the update.
10109 : CallRuntime(Runtime::kInterpreterTraceUpdateFeedback, NoContextConstant(),
10110 : LoadFromParentFrame(JavaScriptFrameConstants::kFunctionOffset),
10111 : SmiTag(slot_id), StringConstant(reason));
10112 : #endif // V8_TRACE_FEEDBACK_UPDATES
10113 13272 : }
10114 :
10115 33320 : void CodeStubAssembler::OverwriteFeedback(Variable* existing_feedback,
10116 : int new_feedback) {
10117 33320 : if (existing_feedback == nullptr) return;
10118 23184 : existing_feedback->Bind(SmiConstant(new_feedback));
10119 : }
10120 :
10121 26320 : void CodeStubAssembler::CombineFeedback(Variable* existing_feedback,
10122 : int feedback) {
10123 26320 : if (existing_feedback == nullptr) return;
10124 19320 : existing_feedback->Bind(
10125 38640 : SmiOr(CAST(existing_feedback->value()), SmiConstant(feedback)));
10126 : }
10127 :
10128 560 : void CodeStubAssembler::CombineFeedback(Variable* existing_feedback,
10129 : Node* feedback) {
10130 560 : if (existing_feedback == nullptr) return;
10131 504 : existing_feedback->Bind(
10132 1008 : SmiOr(CAST(existing_feedback->value()), CAST(feedback)));
10133 : }
10134 :
10135 896 : void CodeStubAssembler::CheckForAssociatedProtector(Node* name,
10136 : Label* if_protector) {
10137 : // This list must be kept in sync with LookupIterator::UpdateProtector!
10138 : // TODO(jkummerow): Would it be faster to have a bit in Symbol::flags()?
10139 1792 : GotoIf(WordEqual(name, LoadRoot(RootIndex::kconstructor_string)),
10140 896 : if_protector);
10141 896 : GotoIf(WordEqual(name, LoadRoot(RootIndex::kiterator_symbol)), if_protector);
10142 896 : GotoIf(WordEqual(name, LoadRoot(RootIndex::knext_string)), if_protector);
10143 896 : GotoIf(WordEqual(name, LoadRoot(RootIndex::kspecies_symbol)), if_protector);
10144 1792 : GotoIf(WordEqual(name, LoadRoot(RootIndex::kis_concat_spreadable_symbol)),
10145 896 : if_protector);
10146 896 : GotoIf(WordEqual(name, LoadRoot(RootIndex::kresolve_string)), if_protector);
10147 896 : GotoIf(WordEqual(name, LoadRoot(RootIndex::kthen_string)), if_protector);
10148 : // Fall through if no case matched.
10149 896 : }
10150 :
10151 728 : TNode<Map> CodeStubAssembler::LoadReceiverMap(SloppyTNode<Object> receiver) {
10152 : return Select<Map>(
10153 1456 : TaggedIsSmi(receiver),
10154 728 : [=] { return CAST(LoadRoot(RootIndex::kHeapNumberMap)); },
10155 2912 : [=] { return LoadMap(UncheckedCast<HeapObject>(receiver)); });
10156 : }
10157 :
10158 8740 : TNode<IntPtrT> CodeStubAssembler::TryToIntptr(Node* key, Label* miss) {
10159 17480 : TVARIABLE(IntPtrT, var_intptr_key);
10160 17480 : Label done(this, &var_intptr_key), key_is_smi(this);
10161 8740 : GotoIf(TaggedIsSmi(key), &key_is_smi);
10162 : // Try to convert a heap number to a Smi.
10163 8740 : GotoIfNot(IsHeapNumber(key), miss);
10164 : {
10165 8740 : TNode<Float64T> value = LoadHeapNumberValue(key);
10166 8740 : TNode<Int32T> int_value = RoundFloat64ToInt32(value);
10167 8740 : GotoIfNot(Float64Equal(value, ChangeInt32ToFloat64(int_value)), miss);
10168 8740 : var_intptr_key = ChangeInt32ToIntPtr(int_value);
10169 8740 : Goto(&done);
10170 : }
10171 :
10172 8740 : BIND(&key_is_smi);
10173 : {
10174 8740 : var_intptr_key = SmiUntag(key);
10175 8740 : Goto(&done);
10176 : }
10177 :
10178 8740 : BIND(&done);
10179 17480 : return var_intptr_key.value();
10180 : }
10181 :
10182 336 : Node* CodeStubAssembler::EmitKeyedSloppyArguments(
10183 : Node* receiver, Node* key, Node* value, Label* bailout,
10184 : ArgumentsAccessMode access_mode) {
10185 : // Mapped arguments are actual arguments. Unmapped arguments are values added
10186 : // to the arguments object after it was created for the call. Mapped arguments
10187 : // are stored in the context at indexes given by elements[key + 2]. Unmapped
10188 : // arguments are stored as regular indexed properties in the arguments array,
10189 : // held at elements[1]. See NewSloppyArguments() in runtime.cc for a detailed
10190 : // look at argument object construction.
10191 : //
10192 : // The sloppy arguments elements array has a special format:
10193 : //
10194 : // 0: context
10195 : // 1: unmapped arguments array
10196 : // 2: mapped_index0,
10197 : // 3: mapped_index1,
10198 : // ...
10199 : //
10200 : // length is 2 + min(number_of_actual_arguments, number_of_formal_arguments).
10201 : // If key + 2 >= elements.length then attempt to look in the unmapped
10202 : // arguments array (given by elements[1]) and return the value at key, missing
10203 : // to the runtime if the unmapped arguments array is not a fixed array or if
10204 : // key >= unmapped_arguments_array.length.
10205 : //
10206 : // Otherwise, t = elements[key + 2]. If t is the hole, then look up the value
10207 : // in the unmapped arguments array, as described above. Otherwise, t is a Smi
10208 : // index into the context array given at elements[0]. Return the value at
10209 : // context[t].
10210 :
10211 336 : GotoIfNot(TaggedIsSmi(key), bailout);
10212 336 : key = SmiUntag(key);
10213 336 : GotoIf(IntPtrLessThan(key, IntPtrConstant(0)), bailout);
10214 :
10215 336 : TNode<FixedArray> elements = CAST(LoadElements(receiver));
10216 336 : TNode<IntPtrT> elements_length = LoadAndUntagFixedArrayBaseLength(elements);
10217 :
10218 672 : VARIABLE(var_result, MachineRepresentation::kTagged);
10219 336 : if (access_mode == ArgumentsAccessMode::kStore) {
10220 224 : var_result.Bind(value);
10221 : } else {
10222 : DCHECK(access_mode == ArgumentsAccessMode::kLoad ||
10223 : access_mode == ArgumentsAccessMode::kHas);
10224 : }
10225 672 : Label if_mapped(this), if_unmapped(this), end(this, &var_result);
10226 336 : Node* intptr_two = IntPtrConstant(2);
10227 336 : Node* adjusted_length = IntPtrSub(elements_length, intptr_two);
10228 :
10229 336 : GotoIf(UintPtrGreaterThanOrEqual(key, adjusted_length), &if_unmapped);
10230 :
10231 : TNode<Object> mapped_index =
10232 336 : LoadFixedArrayElement(elements, IntPtrAdd(key, intptr_two));
10233 336 : Branch(WordEqual(mapped_index, TheHoleConstant()), &if_unmapped, &if_mapped);
10234 :
10235 336 : BIND(&if_mapped);
10236 : {
10237 336 : TNode<IntPtrT> mapped_index_intptr = SmiUntag(CAST(mapped_index));
10238 336 : TNode<Context> the_context = CAST(LoadFixedArrayElement(elements, 0));
10239 336 : if (access_mode == ArgumentsAccessMode::kLoad) {
10240 56 : Node* result = LoadContextElement(the_context, mapped_index_intptr);
10241 : CSA_ASSERT(this, WordNotEqual(result, TheHoleConstant()));
10242 56 : var_result.Bind(result);
10243 280 : } else if (access_mode == ArgumentsAccessMode::kHas) {
10244 : CSA_ASSERT(this, Word32BinaryNot(IsTheHole(LoadContextElement(
10245 : the_context, mapped_index_intptr))));
10246 56 : var_result.Bind(TrueConstant());
10247 : } else {
10248 224 : StoreContextElement(the_context, mapped_index_intptr, value);
10249 : }
10250 336 : Goto(&end);
10251 : }
10252 :
10253 336 : BIND(&if_unmapped);
10254 : {
10255 : TNode<HeapObject> backing_store_ho =
10256 336 : CAST(LoadFixedArrayElement(elements, 1));
10257 672 : GotoIf(WordNotEqual(LoadMap(backing_store_ho), FixedArrayMapConstant()),
10258 336 : bailout);
10259 336 : TNode<FixedArray> backing_store = CAST(backing_store_ho);
10260 :
10261 : TNode<IntPtrT> backing_store_length =
10262 336 : LoadAndUntagFixedArrayBaseLength(backing_store);
10263 336 : if (access_mode == ArgumentsAccessMode::kHas) {
10264 112 : Label out_of_bounds(this);
10265 112 : GotoIf(UintPtrGreaterThanOrEqual(key, backing_store_length),
10266 56 : &out_of_bounds);
10267 56 : Node* result = LoadFixedArrayElement(backing_store, key);
10268 56 : var_result.Bind(
10269 112 : SelectBooleanConstant(WordNotEqual(result, TheHoleConstant())));
10270 56 : Goto(&end);
10271 :
10272 56 : BIND(&out_of_bounds);
10273 56 : var_result.Bind(FalseConstant());
10274 56 : Goto(&end);
10275 : } else {
10276 280 : GotoIf(UintPtrGreaterThanOrEqual(key, backing_store_length), bailout);
10277 :
10278 : // The key falls into unmapped range.
10279 280 : if (access_mode == ArgumentsAccessMode::kLoad) {
10280 56 : Node* result = LoadFixedArrayElement(backing_store, key);
10281 56 : GotoIf(WordEqual(result, TheHoleConstant()), bailout);
10282 56 : var_result.Bind(result);
10283 : } else {
10284 224 : StoreFixedArrayElement(backing_store, key, value);
10285 : }
10286 280 : Goto(&end);
10287 : }
10288 : }
10289 :
10290 336 : BIND(&end);
10291 672 : return var_result.value();
10292 : }
10293 :
10294 840 : TNode<Context> CodeStubAssembler::LoadScriptContext(
10295 : TNode<Context> context, TNode<IntPtrT> context_index) {
10296 840 : TNode<Context> native_context = LoadNativeContext(context);
10297 840 : TNode<ScriptContextTable> script_context_table = CAST(
10298 : LoadContextElement(native_context, Context::SCRIPT_CONTEXT_TABLE_INDEX));
10299 :
10300 840 : TNode<Context> script_context = CAST(LoadFixedArrayElement(
10301 : script_context_table, context_index,
10302 : ScriptContextTable::kFirstContextSlotIndex * kTaggedSize));
10303 840 : return script_context;
10304 : }
10305 :
10306 : namespace {
10307 :
10308 : // Converts typed array elements kind to a machine representations.
10309 4032 : MachineRepresentation ElementsKindToMachineRepresentation(ElementsKind kind) {
10310 4032 : switch (kind) {
10311 : case UINT8_CLAMPED_ELEMENTS:
10312 : case UINT8_ELEMENTS:
10313 : case INT8_ELEMENTS:
10314 1344 : return MachineRepresentation::kWord8;
10315 : case UINT16_ELEMENTS:
10316 : case INT16_ELEMENTS:
10317 896 : return MachineRepresentation::kWord16;
10318 : case UINT32_ELEMENTS:
10319 : case INT32_ELEMENTS:
10320 896 : return MachineRepresentation::kWord32;
10321 : case FLOAT32_ELEMENTS:
10322 448 : return MachineRepresentation::kFloat32;
10323 : case FLOAT64_ELEMENTS:
10324 448 : return MachineRepresentation::kFloat64;
10325 : default:
10326 0 : UNREACHABLE();
10327 : }
10328 : }
10329 :
10330 : } // namespace
10331 :
10332 8604 : void CodeStubAssembler::StoreElement(Node* elements, ElementsKind kind,
10333 : Node* index, Node* value,
10334 : ParameterMode mode) {
10335 8604 : if (IsFixedTypedArrayElementsKind(kind)) {
10336 : if (kind == UINT8_CLAMPED_ELEMENTS) {
10337 : CSA_ASSERT(this,
10338 : Word32Equal(value, Word32And(Int32Constant(0xFF), value)));
10339 : }
10340 4032 : Node* offset = ElementOffsetFromIndex(index, kind, mode, 0);
10341 : // TODO(cbruni): Add OOB check once typed.
10342 4032 : MachineRepresentation rep = ElementsKindToMachineRepresentation(kind);
10343 4032 : StoreNoWriteBarrier(rep, elements, offset, value);
10344 4032 : return;
10345 4572 : } else if (IsDoubleElementsKind(kind)) {
10346 : // Make sure we do not store signalling NaNs into double arrays.
10347 1524 : TNode<Float64T> value_silenced = Float64SilenceNaN(value);
10348 1524 : StoreFixedDoubleArrayElement(CAST(elements), index, value_silenced, mode);
10349 : } else {
10350 : WriteBarrierMode barrier_mode =
10351 3048 : IsSmiElementsKind(kind) ? SKIP_WRITE_BARRIER : UPDATE_WRITE_BARRIER;
10352 3048 : StoreFixedArrayElement(CAST(elements), index, value, barrier_mode, 0, mode);
10353 : }
10354 : }
10355 :
10356 392 : Node* CodeStubAssembler::Int32ToUint8Clamped(Node* int32_value) {
10357 784 : Label done(this);
10358 392 : Node* int32_zero = Int32Constant(0);
10359 392 : Node* int32_255 = Int32Constant(255);
10360 784 : VARIABLE(var_value, MachineRepresentation::kWord32, int32_value);
10361 392 : GotoIf(Uint32LessThanOrEqual(int32_value, int32_255), &done);
10362 392 : var_value.Bind(int32_zero);
10363 392 : GotoIf(Int32LessThan(int32_value, int32_zero), &done);
10364 392 : var_value.Bind(int32_255);
10365 392 : Goto(&done);
10366 392 : BIND(&done);
10367 784 : return var_value.value();
10368 : }
10369 :
10370 392 : Node* CodeStubAssembler::Float64ToUint8Clamped(Node* float64_value) {
10371 784 : Label done(this);
10372 784 : VARIABLE(var_value, MachineRepresentation::kWord32, Int32Constant(0));
10373 392 : GotoIf(Float64LessThanOrEqual(float64_value, Float64Constant(0.0)), &done);
10374 392 : var_value.Bind(Int32Constant(255));
10375 392 : GotoIf(Float64LessThanOrEqual(Float64Constant(255.0), float64_value), &done);
10376 : {
10377 392 : Node* rounded_value = Float64RoundToEven(float64_value);
10378 392 : var_value.Bind(TruncateFloat64ToWord32(rounded_value));
10379 392 : Goto(&done);
10380 : }
10381 392 : BIND(&done);
10382 784 : return var_value.value();
10383 : }
10384 :
10385 4088 : Node* CodeStubAssembler::PrepareValueForWriteToTypedArray(
10386 : TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) {
10387 : DCHECK(IsFixedTypedArrayElementsKind(elements_kind));
10388 :
10389 : MachineRepresentation rep;
10390 4088 : switch (elements_kind) {
10391 : case UINT8_ELEMENTS:
10392 : case INT8_ELEMENTS:
10393 : case UINT16_ELEMENTS:
10394 : case INT16_ELEMENTS:
10395 : case UINT32_ELEMENTS:
10396 : case INT32_ELEMENTS:
10397 : case UINT8_CLAMPED_ELEMENTS:
10398 2744 : rep = MachineRepresentation::kWord32;
10399 2744 : break;
10400 : case FLOAT32_ELEMENTS:
10401 392 : rep = MachineRepresentation::kFloat32;
10402 392 : break;
10403 : case FLOAT64_ELEMENTS:
10404 392 : rep = MachineRepresentation::kFloat64;
10405 392 : break;
10406 : case BIGINT64_ELEMENTS:
10407 : case BIGUINT64_ELEMENTS:
10408 560 : return ToBigInt(context, input);
10409 : default:
10410 0 : UNREACHABLE();
10411 : }
10412 :
10413 7056 : VARIABLE(var_result, rep);
10414 7056 : VARIABLE(var_input, MachineRepresentation::kTagged, input);
10415 7056 : Label done(this, &var_result), if_smi(this), if_heapnumber_or_oddball(this),
10416 7056 : convert(this), loop(this, &var_input);
10417 3528 : Goto(&loop);
10418 3528 : BIND(&loop);
10419 3528 : GotoIf(TaggedIsSmi(var_input.value()), &if_smi);
10420 : // We can handle both HeapNumber and Oddball here, since Oddball has the
10421 : // same layout as the HeapNumber for the HeapNumber::value field. This
10422 : // way we can also properly optimize stores of oddballs to typed arrays.
10423 3528 : GotoIf(IsHeapNumber(var_input.value()), &if_heapnumber_or_oddball);
10424 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
10425 7056 : Branch(HasInstanceType(var_input.value(), ODDBALL_TYPE),
10426 3528 : &if_heapnumber_or_oddball, &convert);
10427 :
10428 3528 : BIND(&if_heapnumber_or_oddball);
10429 : {
10430 7056 : Node* value = UncheckedCast<Float64T>(LoadObjectField(
10431 10584 : var_input.value(), HeapNumber::kValueOffset, MachineType::Float64()));
10432 3528 : if (rep == MachineRepresentation::kWord32) {
10433 2744 : if (elements_kind == UINT8_CLAMPED_ELEMENTS) {
10434 392 : value = Float64ToUint8Clamped(value);
10435 : } else {
10436 2352 : value = TruncateFloat64ToWord32(value);
10437 : }
10438 784 : } else if (rep == MachineRepresentation::kFloat32) {
10439 392 : value = TruncateFloat64ToFloat32(value);
10440 : } else {
10441 : DCHECK_EQ(MachineRepresentation::kFloat64, rep);
10442 : }
10443 3528 : var_result.Bind(value);
10444 3528 : Goto(&done);
10445 : }
10446 :
10447 3528 : BIND(&if_smi);
10448 : {
10449 3528 : Node* value = SmiToInt32(var_input.value());
10450 3528 : if (rep == MachineRepresentation::kFloat32) {
10451 392 : value = RoundInt32ToFloat32(value);
10452 3136 : } else if (rep == MachineRepresentation::kFloat64) {
10453 392 : value = ChangeInt32ToFloat64(value);
10454 : } else {
10455 : DCHECK_EQ(MachineRepresentation::kWord32, rep);
10456 2744 : if (elements_kind == UINT8_CLAMPED_ELEMENTS) {
10457 392 : value = Int32ToUint8Clamped(value);
10458 : }
10459 : }
10460 3528 : var_result.Bind(value);
10461 3528 : Goto(&done);
10462 : }
10463 :
10464 3528 : BIND(&convert);
10465 : {
10466 3528 : var_input.Bind(CallBuiltin(Builtins::kNonNumberToNumber, context, input));
10467 3528 : Goto(&loop);
10468 : }
10469 :
10470 3528 : BIND(&done);
10471 3528 : return var_result.value();
10472 : }
10473 :
10474 224 : void CodeStubAssembler::EmitBigTypedArrayElementStore(
10475 : TNode<JSTypedArray> object, TNode<FixedTypedArrayBase> elements,
10476 : TNode<IntPtrT> intptr_key, TNode<Object> value, TNode<Context> context,
10477 : Label* opt_if_detached) {
10478 224 : TNode<BigInt> bigint_value = ToBigInt(context, value);
10479 :
10480 224 : if (opt_if_detached != nullptr) {
10481 : // Check if buffer has been detached. Must happen after {ToBigInt}!
10482 224 : Node* buffer = LoadObjectField(object, JSArrayBufferView::kBufferOffset);
10483 224 : GotoIf(IsDetachedBuffer(buffer), opt_if_detached);
10484 : }
10485 :
10486 224 : TNode<RawPtrT> backing_store = LoadFixedTypedArrayBackingStore(elements);
10487 : TNode<IntPtrT> offset = ElementOffsetFromIndex(intptr_key, BIGINT64_ELEMENTS,
10488 224 : INTPTR_PARAMETERS, 0);
10489 224 : EmitBigTypedArrayElementStore(elements, backing_store, offset, bigint_value);
10490 224 : }
10491 :
10492 1456 : void CodeStubAssembler::BigIntToRawBytes(TNode<BigInt> bigint,
10493 : TVariable<UintPtrT>* var_low,
10494 : TVariable<UintPtrT>* var_high) {
10495 2912 : Label done(this);
10496 1456 : *var_low = Unsigned(IntPtrConstant(0));
10497 1456 : *var_high = Unsigned(IntPtrConstant(0));
10498 1456 : TNode<Word32T> bitfield = LoadBigIntBitfield(bigint);
10499 1456 : TNode<Uint32T> length = DecodeWord32<BigIntBase::LengthBits>(bitfield);
10500 1456 : TNode<Uint32T> sign = DecodeWord32<BigIntBase::SignBits>(bitfield);
10501 1456 : GotoIf(Word32Equal(length, Int32Constant(0)), &done);
10502 1456 : *var_low = LoadBigIntDigit(bigint, 0);
10503 1456 : if (!Is64()) {
10504 0 : Label load_done(this);
10505 0 : GotoIf(Word32Equal(length, Int32Constant(1)), &load_done);
10506 0 : *var_high = LoadBigIntDigit(bigint, 1);
10507 0 : Goto(&load_done);
10508 0 : BIND(&load_done);
10509 : }
10510 1456 : GotoIf(Word32Equal(sign, Int32Constant(0)), &done);
10511 : // Negative value. Simulate two's complement.
10512 1456 : if (!Is64()) {
10513 0 : *var_high = Unsigned(IntPtrSub(IntPtrConstant(0), var_high->value()));
10514 0 : Label no_carry(this);
10515 0 : GotoIf(WordEqual(var_low->value(), IntPtrConstant(0)), &no_carry);
10516 0 : *var_high = Unsigned(IntPtrSub(var_high->value(), IntPtrConstant(1)));
10517 0 : Goto(&no_carry);
10518 0 : BIND(&no_carry);
10519 : }
10520 1456 : *var_low = Unsigned(IntPtrSub(IntPtrConstant(0), var_low->value()));
10521 1456 : Goto(&done);
10522 1456 : BIND(&done);
10523 1456 : }
10524 :
10525 896 : void CodeStubAssembler::EmitBigTypedArrayElementStore(
10526 : TNode<FixedTypedArrayBase> elements, TNode<RawPtrT> backing_store,
10527 : TNode<IntPtrT> offset, TNode<BigInt> bigint_value) {
10528 1792 : TVARIABLE(UintPtrT, var_low);
10529 : // Only used on 32-bit platforms.
10530 1792 : TVARIABLE(UintPtrT, var_high);
10531 896 : BigIntToRawBytes(bigint_value, &var_low, &var_high);
10532 :
10533 : // Assert that offset < elements.length. Given that it's an offset for a raw
10534 : // pointer we correct it by the usual kHeapObjectTag offset.
10535 : CSA_ASSERT(
10536 : this, IsOffsetInBounds(offset, LoadAndUntagFixedArrayBaseLength(elements),
10537 : kHeapObjectTag, BIGINT64_ELEMENTS));
10538 :
10539 896 : MachineRepresentation rep = WordT::kMachineRepresentation;
10540 : #if defined(V8_TARGET_BIG_ENDIAN)
10541 : if (!Is64()) {
10542 : StoreNoWriteBarrier(rep, backing_store, offset, var_high.value());
10543 : StoreNoWriteBarrier(rep, backing_store,
10544 : IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)),
10545 : var_low.value());
10546 : } else {
10547 : StoreNoWriteBarrier(rep, backing_store, offset, var_low.value());
10548 : }
10549 : #else
10550 896 : StoreNoWriteBarrier(rep, backing_store, offset, var_low.value());
10551 896 : if (!Is64()) {
10552 0 : StoreNoWriteBarrier(rep, backing_store,
10553 0 : IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)),
10554 0 : var_high.value());
10555 : }
10556 : #endif
10557 896 : }
10558 :
10559 7112 : void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value,
10560 : ElementsKind elements_kind,
10561 : KeyedAccessStoreMode store_mode,
10562 : Label* bailout, Node* context) {
10563 : CSA_ASSERT(this, Word32BinaryNot(IsJSProxy(object)));
10564 :
10565 7112 : Node* elements = LoadElements(object);
10566 7112 : if (!IsSmiOrObjectElementsKind(elements_kind)) {
10567 : CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements))));
10568 2688 : } else if (!IsCOWHandlingStoreMode(store_mode)) {
10569 1344 : GotoIf(IsFixedCOWArrayMap(LoadMap(elements)), bailout);
10570 : }
10571 :
10572 : // TODO(ishell): introduce TryToIntPtrOrSmi() and use OptimalParameterMode().
10573 7112 : ParameterMode parameter_mode = INTPTR_PARAMETERS;
10574 7112 : TNode<IntPtrT> intptr_key = TryToIntptr(key, bailout);
10575 :
10576 7112 : if (IsFixedTypedArrayElementsKind(elements_kind)) {
10577 6160 : Label done(this);
10578 :
10579 : // IntegerIndexedElementSet converts value to a Number/BigInt prior to the
10580 : // bounds check.
10581 6160 : value = PrepareValueForWriteToTypedArray(CAST(value), elements_kind,
10582 9240 : CAST(context));
10583 :
10584 : // There must be no allocations between the buffer load and
10585 : // and the actual store to backing store, because GC may decide that
10586 : // the buffer is not alive or move the elements.
10587 : // TODO(ishell): introduce DisallowHeapAllocationCode scope here.
10588 :
10589 : // Check if buffer has been detached.
10590 3080 : Node* buffer = LoadObjectField(object, JSArrayBufferView::kBufferOffset);
10591 3080 : GotoIf(IsDetachedBuffer(buffer), bailout);
10592 :
10593 : // Bounds check.
10594 : Node* length =
10595 3080 : TaggedToParameter(LoadJSTypedArrayLength(CAST(object)), parameter_mode);
10596 :
10597 3080 : if (store_mode == STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS) {
10598 : // Skip the store if we write beyond the length or
10599 : // to a property with a negative integer index.
10600 616 : GotoIfNot(UintPtrLessThan(intptr_key, length), &done);
10601 2464 : } else if (store_mode == STANDARD_STORE) {
10602 1232 : GotoIfNot(UintPtrLessThan(intptr_key, length), bailout);
10603 : } else {
10604 : // This case is produced due to the dispatched call in
10605 : // ElementsTransitionAndStore and StoreFastElement.
10606 : // TODO(jgruber): Avoid generating unsupported combinations to save code
10607 : // size.
10608 1232 : DebugBreak();
10609 : }
10610 :
10611 3080 : if (elements_kind == BIGINT64_ELEMENTS ||
10612 : elements_kind == BIGUINT64_ELEMENTS) {
10613 560 : TNode<BigInt> bigint_value = UncheckedCast<BigInt>(value);
10614 :
10615 : TNode<RawPtrT> backing_store =
10616 560 : LoadFixedTypedArrayBackingStore(CAST(elements));
10617 : TNode<IntPtrT> offset = ElementOffsetFromIndex(
10618 560 : intptr_key, BIGINT64_ELEMENTS, INTPTR_PARAMETERS, 0);
10619 1120 : EmitBigTypedArrayElementStore(CAST(elements), backing_store, offset,
10620 1120 : bigint_value);
10621 : } else {
10622 2520 : Node* backing_store = LoadFixedTypedArrayBackingStore(CAST(elements));
10623 2520 : StoreElement(backing_store, elements_kind, intptr_key, value,
10624 2520 : parameter_mode);
10625 : }
10626 3080 : Goto(&done);
10627 :
10628 3080 : BIND(&done);
10629 3080 : return;
10630 : }
10631 : DCHECK(IsFastElementsKind(elements_kind));
10632 :
10633 : Node* length =
10634 12096 : SelectImpl(IsJSArray(object), [=]() { return LoadJSArrayLength(object); },
10635 4032 : [=]() { return LoadFixedArrayBaseLength(elements); },
10636 4032 : MachineRepresentation::kTagged);
10637 4032 : length = TaggedToParameter(length, parameter_mode);
10638 :
10639 : // In case value is stored into a fast smi array, assure that the value is
10640 : // a smi before manipulating the backing store. Otherwise the backing store
10641 : // may be left in an invalid state.
10642 4032 : if (IsSmiElementsKind(elements_kind)) {
10643 672 : GotoIfNot(TaggedIsSmi(value), bailout);
10644 3360 : } else if (IsDoubleElementsKind(elements_kind)) {
10645 1344 : value = TryTaggedToFloat64(value, bailout);
10646 : }
10647 :
10648 4032 : if (IsGrowStoreMode(store_mode)) {
10649 1008 : elements = CheckForCapacityGrow(object, elements, elements_kind, length,
10650 1008 : intptr_key, parameter_mode, bailout);
10651 : } else {
10652 3024 : GotoIfNot(UintPtrLessThan(intptr_key, length), bailout);
10653 : }
10654 :
10655 : // If we didn't grow {elements}, it might still be COW, in which case we
10656 : // copy it now.
10657 4032 : if (!IsSmiOrObjectElementsKind(elements_kind)) {
10658 : CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements))));
10659 2688 : } else if (IsCOWHandlingStoreMode(store_mode)) {
10660 1344 : elements = CopyElementsOnWrite(object, elements, elements_kind, length,
10661 1344 : parameter_mode, bailout);
10662 : }
10663 :
10664 : CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements))));
10665 4032 : StoreElement(elements, elements_kind, intptr_key, value, parameter_mode);
10666 : }
10667 :
10668 1008 : Node* CodeStubAssembler::CheckForCapacityGrow(Node* object, Node* elements,
10669 : ElementsKind kind, Node* length,
10670 : Node* key, ParameterMode mode,
10671 : Label* bailout) {
10672 : DCHECK(IsFastElementsKind(kind));
10673 2016 : VARIABLE(checked_elements, MachineRepresentation::kTagged);
10674 2016 : Label grow_case(this), no_grow_case(this), done(this),
10675 2016 : grow_bailout(this, Label::kDeferred);
10676 :
10677 : Node* condition;
10678 1008 : if (IsHoleyElementsKind(kind)) {
10679 672 : condition = UintPtrGreaterThanOrEqual(key, length);
10680 : } else {
10681 : // We don't support growing here unless the value is being appended.
10682 336 : condition = WordEqual(key, length);
10683 : }
10684 1008 : Branch(condition, &grow_case, &no_grow_case);
10685 :
10686 1008 : BIND(&grow_case);
10687 : {
10688 : Node* current_capacity =
10689 1008 : TaggedToParameter(LoadFixedArrayBaseLength(elements), mode);
10690 1008 : checked_elements.Bind(elements);
10691 2016 : Label fits_capacity(this);
10692 : // If key is negative, we will notice in Runtime::kGrowArrayElements.
10693 1008 : GotoIf(UintPtrLessThan(key, current_capacity), &fits_capacity);
10694 :
10695 : {
10696 1008 : Node* new_elements = TryGrowElementsCapacity(
10697 1008 : object, elements, kind, key, current_capacity, mode, &grow_bailout);
10698 1008 : checked_elements.Bind(new_elements);
10699 1008 : Goto(&fits_capacity);
10700 : }
10701 :
10702 1008 : BIND(&grow_bailout);
10703 : {
10704 : Node* tagged_key = mode == SMI_PARAMETERS
10705 : ? key
10706 1008 : : ChangeInt32ToTagged(TruncateIntPtrToInt32(key));
10707 2016 : Node* maybe_elements = CallRuntime(
10708 3024 : Runtime::kGrowArrayElements, NoContextConstant(), object, tagged_key);
10709 1008 : GotoIf(TaggedIsSmi(maybe_elements), bailout);
10710 : CSA_ASSERT(this, IsFixedArrayWithKind(maybe_elements, kind));
10711 1008 : checked_elements.Bind(maybe_elements);
10712 1008 : Goto(&fits_capacity);
10713 : }
10714 :
10715 1008 : BIND(&fits_capacity);
10716 1008 : GotoIfNot(IsJSArray(object), &done);
10717 :
10718 1008 : Node* new_length = IntPtrAdd(key, IntPtrOrSmiConstant(1, mode));
10719 1008 : StoreObjectFieldNoWriteBarrier(object, JSArray::kLengthOffset,
10720 2016 : ParameterToTagged(new_length, mode));
10721 1008 : Goto(&done);
10722 : }
10723 :
10724 1008 : BIND(&no_grow_case);
10725 : {
10726 1008 : GotoIfNot(UintPtrLessThan(key, length), bailout);
10727 1008 : checked_elements.Bind(elements);
10728 1008 : Goto(&done);
10729 : }
10730 :
10731 1008 : BIND(&done);
10732 2016 : return checked_elements.value();
10733 : }
10734 :
10735 1344 : Node* CodeStubAssembler::CopyElementsOnWrite(Node* object, Node* elements,
10736 : ElementsKind kind, Node* length,
10737 : ParameterMode mode,
10738 : Label* bailout) {
10739 2688 : VARIABLE(new_elements_var, MachineRepresentation::kTagged, elements);
10740 2688 : Label done(this);
10741 :
10742 1344 : GotoIfNot(IsFixedCOWArrayMap(LoadMap(elements)), &done);
10743 : {
10744 : Node* capacity =
10745 1344 : TaggedToParameter(LoadFixedArrayBaseLength(elements), mode);
10746 1344 : Node* new_elements = GrowElementsCapacity(object, elements, kind, kind,
10747 1344 : length, capacity, mode, bailout);
10748 1344 : new_elements_var.Bind(new_elements);
10749 1344 : Goto(&done);
10750 : }
10751 :
10752 1344 : BIND(&done);
10753 2688 : return new_elements_var.value();
10754 : }
10755 :
10756 2688 : void CodeStubAssembler::TransitionElementsKind(Node* object, Node* map,
10757 : ElementsKind from_kind,
10758 : ElementsKind to_kind,
10759 : Label* bailout) {
10760 : DCHECK(!IsHoleyElementsKind(from_kind) || IsHoleyElementsKind(to_kind));
10761 2688 : if (AllocationSite::ShouldTrack(from_kind, to_kind)) {
10762 1568 : TrapAllocationMemento(object, bailout);
10763 : }
10764 :
10765 2688 : if (!IsSimpleMapChangeTransition(from_kind, to_kind)) {
10766 1344 : Comment("Non-simple map transition");
10767 1344 : Node* elements = LoadElements(object);
10768 :
10769 2688 : Label done(this);
10770 1344 : GotoIf(WordEqual(elements, EmptyFixedArrayConstant()), &done);
10771 :
10772 : // TODO(ishell): Use OptimalParameterMode().
10773 1344 : ParameterMode mode = INTPTR_PARAMETERS;
10774 1344 : Node* elements_length = SmiUntag(LoadFixedArrayBaseLength(elements));
10775 2688 : Node* array_length = SelectImpl(
10776 : IsJSArray(object),
10777 1344 : [=]() {
10778 : CSA_ASSERT(this, IsFastElementsKind(LoadElementsKind(object)));
10779 1344 : return SmiUntag(LoadFastJSArrayLength(object));
10780 : },
10781 1344 : [=]() { return elements_length; },
10782 1344 : MachineType::PointerRepresentation());
10783 :
10784 : CSA_ASSERT(this, WordNotEqual(elements_length, IntPtrConstant(0)));
10785 :
10786 1344 : GrowElementsCapacity(object, elements, from_kind, to_kind, array_length,
10787 1344 : elements_length, mode, bailout);
10788 1344 : Goto(&done);
10789 1344 : BIND(&done);
10790 : }
10791 :
10792 2688 : StoreMap(object, map);
10793 2688 : }
10794 :
10795 2744 : void CodeStubAssembler::TrapAllocationMemento(Node* object,
10796 : Label* memento_found) {
10797 2744 : Comment("[ TrapAllocationMemento");
10798 5488 : Label no_memento_found(this);
10799 5488 : Label top_check(this), map_check(this);
10800 :
10801 : TNode<ExternalReference> new_space_top_address = ExternalConstant(
10802 2744 : ExternalReference::new_space_allocation_top_address(isolate()));
10803 2744 : const int kMementoMapOffset = JSArray::kSize;
10804 : const int kMementoLastWordOffset =
10805 2744 : kMementoMapOffset + AllocationMemento::kSize - kTaggedSize;
10806 :
10807 : // Bail out if the object is not in new space.
10808 2744 : TNode<IntPtrT> object_word = BitcastTaggedToWord(object);
10809 2744 : TNode<IntPtrT> object_page = PageFromAddress(object_word);
10810 : {
10811 : TNode<IntPtrT> page_flags =
10812 : UncheckedCast<IntPtrT>(Load(MachineType::IntPtr(), object_page,
10813 2744 : IntPtrConstant(Page::kFlagsOffset)));
10814 5488 : GotoIf(WordEqual(
10815 5488 : WordAnd(page_flags,
10816 2744 : IntPtrConstant(MemoryChunk::kIsInYoungGenerationMask)),
10817 10976 : IntPtrConstant(0)),
10818 2744 : &no_memento_found);
10819 : // TODO(ulan): Support allocation memento for a large object by allocating
10820 : // additional word for the memento after the large object.
10821 10976 : GotoIf(WordNotEqual(WordAnd(page_flags,
10822 2744 : IntPtrConstant(MemoryChunk::kIsLargePageMask)),
10823 10976 : IntPtrConstant(0)),
10824 2744 : &no_memento_found);
10825 : }
10826 :
10827 : TNode<IntPtrT> memento_last_word = IntPtrAdd(
10828 2744 : object_word, IntPtrConstant(kMementoLastWordOffset - kHeapObjectTag));
10829 2744 : TNode<IntPtrT> memento_last_word_page = PageFromAddress(memento_last_word);
10830 :
10831 : TNode<IntPtrT> new_space_top = UncheckedCast<IntPtrT>(
10832 2744 : Load(MachineType::Pointer(), new_space_top_address));
10833 2744 : TNode<IntPtrT> new_space_top_page = PageFromAddress(new_space_top);
10834 :
10835 : // If the object is in new space, we need to check whether respective
10836 : // potential memento object is on the same page as the current top.
10837 2744 : GotoIf(WordEqual(memento_last_word_page, new_space_top_page), &top_check);
10838 :
10839 : // The object is on a different page than allocation top. Bail out if the
10840 : // object sits on the page boundary as no memento can follow and we cannot
10841 : // touch the memory following it.
10842 5488 : Branch(WordEqual(object_page, memento_last_word_page), &map_check,
10843 2744 : &no_memento_found);
10844 :
10845 : // If top is on the same page as the current object, we need to check whether
10846 : // we are below top.
10847 2744 : BIND(&top_check);
10848 : {
10849 5488 : Branch(UintPtrGreaterThanOrEqual(memento_last_word, new_space_top),
10850 2744 : &no_memento_found, &map_check);
10851 : }
10852 :
10853 : // Memento map check.
10854 2744 : BIND(&map_check);
10855 : {
10856 2744 : TNode<Object> memento_map = LoadObjectField(object, kMementoMapOffset);
10857 5488 : Branch(WordEqual(memento_map, LoadRoot(RootIndex::kAllocationMementoMap)),
10858 2744 : memento_found, &no_memento_found);
10859 : }
10860 2744 : BIND(&no_memento_found);
10861 2744 : Comment("] TrapAllocationMemento");
10862 2744 : }
10863 :
10864 11628 : TNode<IntPtrT> CodeStubAssembler::PageFromAddress(TNode<IntPtrT> address) {
10865 11628 : return WordAnd(address, IntPtrConstant(~kPageAlignmentMask));
10866 : }
10867 :
10868 392 : TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector(
10869 : SloppyTNode<FeedbackVector> feedback_vector, TNode<Smi> slot) {
10870 392 : TNode<IntPtrT> size = IntPtrConstant(AllocationSite::kSizeWithWeakNext);
10871 392 : Node* site = Allocate(size, CodeStubAssembler::kPretenured);
10872 392 : StoreMapNoWriteBarrier(site, RootIndex::kAllocationSiteWithWeakNextMap);
10873 : // Should match AllocationSite::Initialize.
10874 : TNode<WordT> field = UpdateWord<AllocationSite::ElementsKindBits>(
10875 392 : IntPtrConstant(0), IntPtrConstant(GetInitialFastElementsKind()));
10876 392 : StoreObjectFieldNoWriteBarrier(
10877 : site, AllocationSite::kTransitionInfoOrBoilerplateOffset,
10878 784 : SmiTag(Signed(field)));
10879 :
10880 : // Unlike literals, constructed arrays don't have nested sites
10881 392 : TNode<Smi> zero = SmiConstant(0);
10882 392 : StoreObjectFieldNoWriteBarrier(site, AllocationSite::kNestedSiteOffset, zero);
10883 :
10884 : // Pretenuring calculation field.
10885 392 : StoreObjectFieldNoWriteBarrier(site, AllocationSite::kPretenureDataOffset,
10886 784 : Int32Constant(0),
10887 392 : MachineRepresentation::kWord32);
10888 :
10889 : // Pretenuring memento creation count field.
10890 392 : StoreObjectFieldNoWriteBarrier(
10891 784 : site, AllocationSite::kPretenureCreateCountOffset, Int32Constant(0),
10892 392 : MachineRepresentation::kWord32);
10893 :
10894 : // Store an empty fixed array for the code dependency.
10895 : StoreObjectFieldRoot(site, AllocationSite::kDependentCodeOffset,
10896 392 : RootIndex::kEmptyWeakFixedArray);
10897 :
10898 : // Link the object to the allocation site list
10899 : TNode<ExternalReference> site_list = ExternalConstant(
10900 392 : ExternalReference::allocation_sites_list_address(isolate()));
10901 392 : TNode<Object> next_site = CAST(LoadBufferObject(site_list, 0));
10902 :
10903 : // TODO(mvstanton): This is a store to a weak pointer, which we may want to
10904 : // mark as such in order to skip the write barrier, once we have a unified
10905 : // system for weakness. For now we decided to keep it like this because having
10906 : // an initial write barrier backed store makes this pointer strong until the
10907 : // next GC, and allocation sites are designed to survive several GCs anyway.
10908 392 : StoreObjectField(site, AllocationSite::kWeakNextOffset, next_site);
10909 392 : StoreFullTaggedNoWriteBarrier(site_list, site);
10910 :
10911 392 : StoreFeedbackVectorSlot(feedback_vector, slot, site, UPDATE_WRITE_BARRIER, 0,
10912 392 : SMI_PARAMETERS);
10913 392 : return CAST(site);
10914 : }
10915 :
10916 2240 : TNode<MaybeObject> CodeStubAssembler::StoreWeakReferenceInFeedbackVector(
10917 : SloppyTNode<FeedbackVector> feedback_vector, Node* slot,
10918 : SloppyTNode<HeapObject> value, int additional_offset,
10919 : ParameterMode parameter_mode) {
10920 2240 : TNode<MaybeObject> weak_value = MakeWeak(value);
10921 2240 : StoreFeedbackVectorSlot(feedback_vector, slot, weak_value,
10922 : UPDATE_WRITE_BARRIER, additional_offset,
10923 2240 : parameter_mode);
10924 2240 : return weak_value;
10925 : }
10926 :
10927 672 : TNode<BoolT> CodeStubAssembler::NotHasBoilerplate(
10928 : TNode<Object> maybe_literal_site) {
10929 672 : return TaggedIsSmi(maybe_literal_site);
10930 : }
10931 :
10932 336 : TNode<Smi> CodeStubAssembler::LoadTransitionInfo(
10933 : TNode<AllocationSite> allocation_site) {
10934 336 : TNode<Smi> transition_info = CAST(LoadObjectField(
10935 : allocation_site, AllocationSite::kTransitionInfoOrBoilerplateOffset));
10936 336 : return transition_info;
10937 : }
10938 :
10939 448 : TNode<JSObject> CodeStubAssembler::LoadBoilerplate(
10940 : TNode<AllocationSite> allocation_site) {
10941 448 : TNode<JSObject> boilerplate = CAST(LoadObjectField(
10942 : allocation_site, AllocationSite::kTransitionInfoOrBoilerplateOffset));
10943 448 : return boilerplate;
10944 : }
10945 :
10946 280 : TNode<Int32T> CodeStubAssembler::LoadElementsKind(
10947 : TNode<AllocationSite> allocation_site) {
10948 280 : TNode<Smi> transition_info = LoadTransitionInfo(allocation_site);
10949 : TNode<Int32T> elements_kind =
10950 560 : Signed(DecodeWord32<AllocationSite::ElementsKindBits>(
10951 840 : SmiToInt32(transition_info)));
10952 : CSA_ASSERT(this, IsFastElementsKind(elements_kind));
10953 280 : return elements_kind;
10954 : }
10955 :
10956 30896 : Node* CodeStubAssembler::BuildFastLoop(
10957 : const CodeStubAssembler::VariableList& vars, Node* start_index,
10958 : Node* end_index, const FastLoopBody& body, int increment,
10959 : ParameterMode parameter_mode, IndexAdvanceMode advance_mode) {
10960 : CSA_SLOW_ASSERT(this, MatchesParameterMode(start_index, parameter_mode));
10961 : CSA_SLOW_ASSERT(this, MatchesParameterMode(end_index, parameter_mode));
10962 : MachineRepresentation index_rep = (parameter_mode == INTPTR_PARAMETERS)
10963 : ? MachineType::PointerRepresentation()
10964 30896 : : MachineRepresentation::kTaggedSigned;
10965 61792 : VARIABLE(var, index_rep, start_index);
10966 61792 : VariableList vars_copy(vars.begin(), vars.end(), zone());
10967 30896 : vars_copy.push_back(&var);
10968 61792 : Label loop(this, vars_copy);
10969 61792 : Label after_loop(this);
10970 : // Introduce an explicit second check of the termination condition before the
10971 : // loop that helps turbofan generate better code. If there's only a single
10972 : // check, then the CodeStubAssembler forces it to be at the beginning of the
10973 : // loop requiring a backwards branch at the end of the loop (it's not possible
10974 : // to force the loop header check at the end of the loop and branch forward to
10975 : // it from the pre-header). The extra branch is slower in the case that the
10976 : // loop actually iterates.
10977 30896 : Node* first_check = WordEqual(var.value(), end_index);
10978 : int32_t first_check_val;
10979 30896 : if (ToInt32Constant(first_check, first_check_val)) {
10980 976 : if (first_check_val) return var.value();
10981 72 : Goto(&loop);
10982 : } else {
10983 29920 : Branch(first_check, &after_loop, &loop);
10984 : }
10985 :
10986 29992 : BIND(&loop);
10987 : {
10988 29992 : if (advance_mode == IndexAdvanceMode::kPre) {
10989 17908 : Increment(&var, increment, parameter_mode);
10990 : }
10991 29992 : body(var.value());
10992 29992 : if (advance_mode == IndexAdvanceMode::kPost) {
10993 12084 : Increment(&var, increment, parameter_mode);
10994 : }
10995 29992 : Branch(WordNotEqual(var.value(), end_index), &loop, &after_loop);
10996 : }
10997 29992 : BIND(&after_loop);
10998 29992 : return var.value();
10999 : }
11000 :
11001 16772 : void CodeStubAssembler::BuildFastFixedArrayForEach(
11002 : const CodeStubAssembler::VariableList& vars, Node* fixed_array,
11003 : ElementsKind kind, Node* first_element_inclusive,
11004 : Node* last_element_exclusive, const FastFixedArrayForEachBody& body,
11005 : ParameterMode mode, ForEachDirection direction) {
11006 : STATIC_ASSERT(FixedArray::kHeaderSize == FixedDoubleArray::kHeaderSize);
11007 : CSA_SLOW_ASSERT(this, MatchesParameterMode(first_element_inclusive, mode));
11008 : CSA_SLOW_ASSERT(this, MatchesParameterMode(last_element_exclusive, mode));
11009 : CSA_SLOW_ASSERT(this, Word32Or(IsFixedArrayWithKind(fixed_array, kind),
11010 : IsPropertyArray(fixed_array)));
11011 : int32_t first_val;
11012 16772 : bool constant_first = ToInt32Constant(first_element_inclusive, first_val);
11013 : int32_t last_val;
11014 16772 : bool constent_last = ToInt32Constant(last_element_exclusive, last_val);
11015 16772 : if (constant_first && constent_last) {
11016 976 : int delta = last_val - first_val;
11017 : DCHECK_GE(delta, 0);
11018 976 : if (delta <= kElementLoopUnrollThreshold) {
11019 920 : if (direction == ForEachDirection::kForward) {
11020 36 : for (int i = first_val; i < last_val; ++i) {
11021 24 : Node* index = IntPtrConstant(i);
11022 : Node* offset =
11023 48 : ElementOffsetFromIndex(index, kind, INTPTR_PARAMETERS,
11024 48 : FixedArray::kHeaderSize - kHeapObjectTag);
11025 24 : body(fixed_array, offset);
11026 : }
11027 : } else {
11028 3036 : for (int i = last_val - 1; i >= first_val; --i) {
11029 2128 : Node* index = IntPtrConstant(i);
11030 : Node* offset =
11031 4256 : ElementOffsetFromIndex(index, kind, INTPTR_PARAMETERS,
11032 4256 : FixedArray::kHeaderSize - kHeapObjectTag);
11033 2128 : body(fixed_array, offset);
11034 : }
11035 : }
11036 1840 : return;
11037 : }
11038 : }
11039 :
11040 : Node* start =
11041 31704 : ElementOffsetFromIndex(first_element_inclusive, kind, mode,
11042 31704 : FixedArray::kHeaderSize - kHeapObjectTag);
11043 : Node* limit =
11044 31704 : ElementOffsetFromIndex(last_element_exclusive, kind, mode,
11045 31704 : FixedArray::kHeaderSize - kHeapObjectTag);
11046 15852 : if (direction == ForEachDirection::kReverse) std::swap(start, limit);
11047 :
11048 15852 : int increment = IsDoubleElementsKind(kind) ? kDoubleSize : kTaggedSize;
11049 31704 : BuildFastLoop(
11050 : vars, start, limit,
11051 15340 : [fixed_array, &body](Node* offset) { body(fixed_array, offset); },
11052 : direction == ForEachDirection::kReverse ? -increment : increment,
11053 : INTPTR_PARAMETERS,
11054 : direction == ForEachDirection::kReverse ? IndexAdvanceMode::kPre
11055 15852 : : IndexAdvanceMode::kPost);
11056 : }
11057 :
11058 224 : void CodeStubAssembler::GotoIfFixedArraySizeDoesntFitInNewSpace(
11059 : Node* element_count, Label* doesnt_fit, int base_size, ParameterMode mode) {
11060 448 : GotoIf(FixedArraySizeDoesntFitInNewSpace(element_count, base_size, mode),
11061 224 : doesnt_fit);
11062 224 : }
11063 :
11064 3700 : void CodeStubAssembler::InitializeFieldsWithRoot(Node* object,
11065 : Node* start_offset,
11066 : Node* end_offset,
11067 : RootIndex root_index) {
11068 : CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object));
11069 3700 : start_offset = IntPtrAdd(start_offset, IntPtrConstant(-kHeapObjectTag));
11070 3700 : end_offset = IntPtrAdd(end_offset, IntPtrConstant(-kHeapObjectTag));
11071 3700 : Node* root_value = LoadRoot(root_index);
11072 7400 : BuildFastLoop(
11073 : end_offset, start_offset,
11074 3308 : [this, object, root_value](Node* current) {
11075 3308 : StoreNoWriteBarrier(MachineRepresentation::kTagged, object, current,
11076 3308 : root_value);
11077 3308 : },
11078 : -kTaggedSize, INTPTR_PARAMETERS,
11079 3700 : CodeStubAssembler::IndexAdvanceMode::kPre);
11080 3700 : }
11081 :
11082 8700 : void CodeStubAssembler::BranchIfNumberRelationalComparison(
11083 : Operation op, Node* left, Node* right, Label* if_true, Label* if_false) {
11084 : CSA_SLOW_ASSERT(this, IsNumber(left));
11085 : CSA_SLOW_ASSERT(this, IsNumber(right));
11086 :
11087 17400 : Label do_float_comparison(this);
11088 17400 : TVARIABLE(Float64T, var_left_float);
11089 17400 : TVARIABLE(Float64T, var_right_float);
11090 :
11091 17400 : Branch(
11092 : TaggedIsSmi(left),
11093 8700 : [&] {
11094 43500 : TNode<Smi> smi_left = CAST(left);
11095 :
11096 121800 : Branch(
11097 26100 : TaggedIsSmi(right),
11098 8700 : [&] {
11099 17400 : TNode<Smi> smi_right = CAST(right);
11100 :
11101 : // Both {left} and {right} are Smi, so just perform a fast
11102 : // Smi comparison.
11103 17400 : switch (op) {
11104 : case Operation::kEqual:
11105 39972 : BranchIfSmiEqual(smi_left, smi_right, if_true, if_false);
11106 336 : break;
11107 : case Operation::kLessThan:
11108 9576 : BranchIfSmiLessThan(smi_left, smi_right, if_true, if_false);
11109 3192 : break;
11110 : case Operation::kLessThanOrEqual:
11111 112 : BranchIfSmiLessThanOrEqual(smi_left, smi_right, if_true,
11112 56 : if_false);
11113 56 : break;
11114 : case Operation::kGreaterThan:
11115 5712 : BranchIfSmiLessThan(smi_right, smi_left, if_true, if_false);
11116 1904 : break;
11117 : case Operation::kGreaterThanOrEqual:
11118 6424 : BranchIfSmiLessThanOrEqual(smi_right, smi_left, if_true,
11119 3212 : if_false);
11120 3212 : break;
11121 : default:
11122 0 : UNREACHABLE();
11123 : }
11124 8700 : },
11125 8700 : [&] {
11126 : CSA_ASSERT(this, IsHeapNumber(right));
11127 34800 : var_left_float = SmiToFloat64(smi_left);
11128 26100 : var_right_float = LoadHeapNumberValue(right);
11129 26100 : Goto(&do_float_comparison);
11130 17400 : });
11131 8700 : },
11132 8700 : [&] {
11133 : CSA_ASSERT(this, IsHeapNumber(left));
11134 43500 : var_left_float = LoadHeapNumberValue(left);
11135 :
11136 104400 : Branch(
11137 26100 : TaggedIsSmi(right),
11138 8700 : [&] {
11139 34800 : var_right_float = SmiToFloat64(right);
11140 34800 : Goto(&do_float_comparison);
11141 8700 : },
11142 8700 : [&] {
11143 : CSA_ASSERT(this, IsHeapNumber(right));
11144 17400 : var_right_float = LoadHeapNumberValue(right);
11145 17400 : Goto(&do_float_comparison);
11146 17400 : });
11147 17400 : });
11148 :
11149 8700 : BIND(&do_float_comparison);
11150 : {
11151 8700 : switch (op) {
11152 : case Operation::kEqual:
11153 672 : Branch(Float64Equal(var_left_float.value(), var_right_float.value()),
11154 672 : if_true, if_false);
11155 336 : break;
11156 : case Operation::kLessThan:
11157 6384 : Branch(Float64LessThan(var_left_float.value(), var_right_float.value()),
11158 6384 : if_true, if_false);
11159 3192 : break;
11160 : case Operation::kLessThanOrEqual:
11161 224 : Branch(Float64LessThanOrEqual(var_left_float.value(),
11162 224 : var_right_float.value()),
11163 112 : if_true, if_false);
11164 56 : break;
11165 : case Operation::kGreaterThan:
11166 3808 : Branch(
11167 3808 : Float64GreaterThan(var_left_float.value(), var_right_float.value()),
11168 3808 : if_true, if_false);
11169 1904 : break;
11170 : case Operation::kGreaterThanOrEqual:
11171 12848 : Branch(Float64GreaterThanOrEqual(var_left_float.value(),
11172 12848 : var_right_float.value()),
11173 6424 : if_true, if_false);
11174 3212 : break;
11175 : default:
11176 0 : UNREACHABLE();
11177 : }
11178 : }
11179 8700 : }
11180 :
11181 2760 : void CodeStubAssembler::GotoIfNumberGreaterThanOrEqual(Node* left, Node* right,
11182 : Label* if_true) {
11183 5520 : Label if_false(this);
11184 : BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, left,
11185 2760 : right, if_true, &if_false);
11186 2760 : BIND(&if_false);
11187 2760 : }
11188 :
11189 : namespace {
11190 2688 : Operation Reverse(Operation op) {
11191 2688 : switch (op) {
11192 : case Operation::kLessThan:
11193 672 : return Operation::kGreaterThan;
11194 : case Operation::kLessThanOrEqual:
11195 672 : return Operation::kGreaterThanOrEqual;
11196 : case Operation::kGreaterThan:
11197 672 : return Operation::kLessThan;
11198 : case Operation::kGreaterThanOrEqual:
11199 672 : return Operation::kLessThanOrEqual;
11200 : default:
11201 0 : break;
11202 : }
11203 0 : UNREACHABLE();
11204 : }
11205 : } // anonymous namespace
11206 :
11207 896 : Node* CodeStubAssembler::RelationalComparison(Operation op, Node* left,
11208 : Node* right, Node* context,
11209 : Variable* var_type_feedback) {
11210 1792 : Label return_true(this), return_false(this), do_float_comparison(this),
11211 1792 : end(this);
11212 1792 : TVARIABLE(Oddball, var_result); // Actually only "true" or "false".
11213 1792 : TVARIABLE(Float64T, var_left_float);
11214 1792 : TVARIABLE(Float64T, var_right_float);
11215 :
11216 : // We might need to loop several times due to ToPrimitive and/or ToNumeric
11217 : // conversions.
11218 1792 : VARIABLE(var_left, MachineRepresentation::kTagged, left);
11219 1792 : VARIABLE(var_right, MachineRepresentation::kTagged, right);
11220 1792 : VariableList loop_variable_list({&var_left, &var_right}, zone());
11221 896 : if (var_type_feedback != nullptr) {
11222 : // Initialize the type feedback to None. The current feedback is combined
11223 : // with the previous feedback.
11224 672 : var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kNone));
11225 672 : loop_variable_list.push_back(var_type_feedback);
11226 : }
11227 1792 : Label loop(this, loop_variable_list);
11228 896 : Goto(&loop);
11229 896 : BIND(&loop);
11230 : {
11231 896 : left = var_left.value();
11232 896 : right = var_right.value();
11233 :
11234 1792 : Label if_left_smi(this), if_left_not_smi(this);
11235 896 : Branch(TaggedIsSmi(left), &if_left_smi, &if_left_not_smi);
11236 :
11237 896 : BIND(&if_left_smi);
11238 : {
11239 896 : TNode<Smi> smi_left = CAST(left);
11240 1792 : Label if_right_smi(this), if_right_heapnumber(this),
11241 1792 : if_right_bigint(this, Label::kDeferred),
11242 1792 : if_right_not_numeric(this, Label::kDeferred);
11243 896 : GotoIf(TaggedIsSmi(right), &if_right_smi);
11244 896 : Node* right_map = LoadMap(right);
11245 896 : GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
11246 896 : Node* right_instance_type = LoadMapInstanceType(right_map);
11247 1792 : Branch(IsBigIntInstanceType(right_instance_type), &if_right_bigint,
11248 896 : &if_right_not_numeric);
11249 :
11250 896 : BIND(&if_right_smi);
11251 : {
11252 896 : TNode<Smi> smi_right = CAST(right);
11253 : CombineFeedback(var_type_feedback,
11254 896 : CompareOperationFeedback::kSignedSmall);
11255 896 : switch (op) {
11256 : case Operation::kLessThan:
11257 : BranchIfSmiLessThan(smi_left, smi_right, &return_true,
11258 224 : &return_false);
11259 224 : break;
11260 : case Operation::kLessThanOrEqual:
11261 : BranchIfSmiLessThanOrEqual(smi_left, smi_right, &return_true,
11262 224 : &return_false);
11263 224 : break;
11264 : case Operation::kGreaterThan:
11265 : BranchIfSmiLessThan(smi_right, smi_left, &return_true,
11266 224 : &return_false);
11267 224 : break;
11268 : case Operation::kGreaterThanOrEqual:
11269 : BranchIfSmiLessThanOrEqual(smi_right, smi_left, &return_true,
11270 224 : &return_false);
11271 224 : break;
11272 : default:
11273 0 : UNREACHABLE();
11274 : }
11275 : }
11276 :
11277 896 : BIND(&if_right_heapnumber);
11278 : {
11279 896 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
11280 896 : var_left_float = SmiToFloat64(smi_left);
11281 896 : var_right_float = LoadHeapNumberValue(right);
11282 896 : Goto(&do_float_comparison);
11283 : }
11284 :
11285 896 : BIND(&if_right_bigint);
11286 : {
11287 896 : OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
11288 1792 : var_result = CAST(CallRuntime(Runtime::kBigIntCompareToNumber,
11289 : NoContextConstant(),
11290 896 : SmiConstant(Reverse(op)), right, left));
11291 896 : Goto(&end);
11292 : }
11293 :
11294 896 : BIND(&if_right_not_numeric);
11295 : {
11296 896 : OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
11297 : // Convert {right} to a Numeric; we don't need to perform the
11298 : // dedicated ToPrimitive(right, hint Number) operation, as the
11299 : // ToNumeric(right) will by itself already invoke ToPrimitive with
11300 : // a Number hint.
11301 896 : var_right.Bind(
11302 1792 : CallBuiltin(Builtins::kNonNumberToNumeric, context, right));
11303 896 : Goto(&loop);
11304 : }
11305 : }
11306 :
11307 896 : BIND(&if_left_not_smi);
11308 : {
11309 896 : Node* left_map = LoadMap(left);
11310 :
11311 1792 : Label if_right_smi(this), if_right_not_smi(this);
11312 896 : Branch(TaggedIsSmi(right), &if_right_smi, &if_right_not_smi);
11313 :
11314 896 : BIND(&if_right_smi);
11315 : {
11316 1792 : Label if_left_heapnumber(this), if_left_bigint(this, Label::kDeferred),
11317 1792 : if_left_not_numeric(this, Label::kDeferred);
11318 896 : GotoIf(IsHeapNumberMap(left_map), &if_left_heapnumber);
11319 896 : Node* left_instance_type = LoadMapInstanceType(left_map);
11320 1792 : Branch(IsBigIntInstanceType(left_instance_type), &if_left_bigint,
11321 896 : &if_left_not_numeric);
11322 :
11323 896 : BIND(&if_left_heapnumber);
11324 : {
11325 896 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
11326 896 : var_left_float = LoadHeapNumberValue(left);
11327 896 : var_right_float = SmiToFloat64(right);
11328 896 : Goto(&do_float_comparison);
11329 : }
11330 :
11331 896 : BIND(&if_left_bigint);
11332 : {
11333 896 : OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
11334 1792 : var_result = CAST(CallRuntime(Runtime::kBigIntCompareToNumber,
11335 : NoContextConstant(), SmiConstant(op),
11336 896 : left, right));
11337 896 : Goto(&end);
11338 : }
11339 :
11340 896 : BIND(&if_left_not_numeric);
11341 : {
11342 896 : OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
11343 : // Convert {left} to a Numeric; we don't need to perform the
11344 : // dedicated ToPrimitive(left, hint Number) operation, as the
11345 : // ToNumeric(left) will by itself already invoke ToPrimitive with
11346 : // a Number hint.
11347 896 : var_left.Bind(
11348 1792 : CallBuiltin(Builtins::kNonNumberToNumeric, context, left));
11349 896 : Goto(&loop);
11350 : }
11351 : }
11352 :
11353 896 : BIND(&if_right_not_smi);
11354 : {
11355 896 : Node* right_map = LoadMap(right);
11356 :
11357 1792 : Label if_left_heapnumber(this), if_left_bigint(this, Label::kDeferred),
11358 1792 : if_left_string(this), if_left_other(this, Label::kDeferred);
11359 896 : GotoIf(IsHeapNumberMap(left_map), &if_left_heapnumber);
11360 896 : Node* left_instance_type = LoadMapInstanceType(left_map);
11361 896 : GotoIf(IsBigIntInstanceType(left_instance_type), &if_left_bigint);
11362 1792 : Branch(IsStringInstanceType(left_instance_type), &if_left_string,
11363 896 : &if_left_other);
11364 :
11365 896 : BIND(&if_left_heapnumber);
11366 : {
11367 1792 : Label if_right_heapnumber(this),
11368 1792 : if_right_bigint(this, Label::kDeferred),
11369 1792 : if_right_not_numeric(this, Label::kDeferred);
11370 896 : GotoIf(WordEqual(right_map, left_map), &if_right_heapnumber);
11371 896 : Node* right_instance_type = LoadMapInstanceType(right_map);
11372 1792 : Branch(IsBigIntInstanceType(right_instance_type), &if_right_bigint,
11373 896 : &if_right_not_numeric);
11374 :
11375 896 : BIND(&if_right_heapnumber);
11376 : {
11377 : CombineFeedback(var_type_feedback,
11378 896 : CompareOperationFeedback::kNumber);
11379 896 : var_left_float = LoadHeapNumberValue(left);
11380 896 : var_right_float = LoadHeapNumberValue(right);
11381 896 : Goto(&do_float_comparison);
11382 : }
11383 :
11384 896 : BIND(&if_right_bigint);
11385 : {
11386 : OverwriteFeedback(var_type_feedback,
11387 896 : CompareOperationFeedback::kAny);
11388 1792 : var_result = CAST(CallRuntime(
11389 : Runtime::kBigIntCompareToNumber, NoContextConstant(),
11390 896 : SmiConstant(Reverse(op)), right, left));
11391 896 : Goto(&end);
11392 : }
11393 :
11394 896 : BIND(&if_right_not_numeric);
11395 : {
11396 : OverwriteFeedback(var_type_feedback,
11397 896 : CompareOperationFeedback::kAny);
11398 : // Convert {right} to a Numeric; we don't need to perform
11399 : // dedicated ToPrimitive(right, hint Number) operation, as the
11400 : // ToNumeric(right) will by itself already invoke ToPrimitive with
11401 : // a Number hint.
11402 896 : var_right.Bind(
11403 1792 : CallBuiltin(Builtins::kNonNumberToNumeric, context, right));
11404 896 : Goto(&loop);
11405 : }
11406 : }
11407 :
11408 896 : BIND(&if_left_bigint);
11409 : {
11410 1792 : Label if_right_heapnumber(this), if_right_bigint(this),
11411 1792 : if_right_string(this), if_right_other(this);
11412 896 : GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
11413 896 : Node* right_instance_type = LoadMapInstanceType(right_map);
11414 896 : GotoIf(IsBigIntInstanceType(right_instance_type), &if_right_bigint);
11415 1792 : Branch(IsStringInstanceType(right_instance_type), &if_right_string,
11416 896 : &if_right_other);
11417 :
11418 896 : BIND(&if_right_heapnumber);
11419 : {
11420 : OverwriteFeedback(var_type_feedback,
11421 896 : CompareOperationFeedback::kAny);
11422 1792 : var_result = CAST(CallRuntime(Runtime::kBigIntCompareToNumber,
11423 : NoContextConstant(), SmiConstant(op),
11424 896 : left, right));
11425 896 : Goto(&end);
11426 : }
11427 :
11428 896 : BIND(&if_right_bigint);
11429 : {
11430 : CombineFeedback(var_type_feedback,
11431 896 : CompareOperationFeedback::kBigInt);
11432 1792 : var_result = CAST(CallRuntime(Runtime::kBigIntCompareToBigInt,
11433 : NoContextConstant(), SmiConstant(op),
11434 896 : left, right));
11435 896 : Goto(&end);
11436 : }
11437 :
11438 896 : BIND(&if_right_string);
11439 : {
11440 : OverwriteFeedback(var_type_feedback,
11441 896 : CompareOperationFeedback::kAny);
11442 1792 : var_result = CAST(CallRuntime(Runtime::kBigIntCompareToString,
11443 : NoContextConstant(), SmiConstant(op),
11444 896 : left, right));
11445 896 : Goto(&end);
11446 : }
11447 :
11448 : // {right} is not a Number, BigInt, or String.
11449 896 : BIND(&if_right_other);
11450 : {
11451 : OverwriteFeedback(var_type_feedback,
11452 896 : CompareOperationFeedback::kAny);
11453 : // Convert {right} to a Numeric; we don't need to perform
11454 : // dedicated ToPrimitive(right, hint Number) operation, as the
11455 : // ToNumeric(right) will by itself already invoke ToPrimitive with
11456 : // a Number hint.
11457 896 : var_right.Bind(
11458 1792 : CallBuiltin(Builtins::kNonNumberToNumeric, context, right));
11459 896 : Goto(&loop);
11460 : }
11461 : }
11462 :
11463 896 : BIND(&if_left_string);
11464 : {
11465 896 : Node* right_instance_type = LoadMapInstanceType(right_map);
11466 :
11467 1792 : Label if_right_not_string(this, Label::kDeferred);
11468 1792 : GotoIfNot(IsStringInstanceType(right_instance_type),
11469 896 : &if_right_not_string);
11470 :
11471 : // Both {left} and {right} are strings.
11472 896 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kString);
11473 : Builtins::Name builtin;
11474 896 : switch (op) {
11475 : case Operation::kLessThan:
11476 224 : builtin = Builtins::kStringLessThan;
11477 224 : break;
11478 : case Operation::kLessThanOrEqual:
11479 224 : builtin = Builtins::kStringLessThanOrEqual;
11480 224 : break;
11481 : case Operation::kGreaterThan:
11482 224 : builtin = Builtins::kStringGreaterThan;
11483 224 : break;
11484 : case Operation::kGreaterThanOrEqual:
11485 224 : builtin = Builtins::kStringGreaterThanOrEqual;
11486 224 : break;
11487 : default:
11488 0 : UNREACHABLE();
11489 : }
11490 896 : var_result = CAST(CallBuiltin(builtin, context, left, right));
11491 896 : Goto(&end);
11492 :
11493 896 : BIND(&if_right_not_string);
11494 : {
11495 : OverwriteFeedback(var_type_feedback,
11496 896 : CompareOperationFeedback::kAny);
11497 : // {left} is a String, while {right} isn't. Check if {right} is
11498 : // a BigInt, otherwise call ToPrimitive(right, hint Number) if
11499 : // {right} is a receiver, or ToNumeric(left) and then
11500 : // ToNumeric(right) in the other cases.
11501 : STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
11502 1792 : Label if_right_bigint(this),
11503 1792 : if_right_receiver(this, Label::kDeferred);
11504 896 : GotoIf(IsBigIntInstanceType(right_instance_type), &if_right_bigint);
11505 1792 : GotoIf(IsJSReceiverInstanceType(right_instance_type),
11506 896 : &if_right_receiver);
11507 :
11508 896 : var_left.Bind(
11509 1792 : CallBuiltin(Builtins::kNonNumberToNumeric, context, left));
11510 896 : var_right.Bind(CallBuiltin(Builtins::kToNumeric, context, right));
11511 896 : Goto(&loop);
11512 :
11513 896 : BIND(&if_right_bigint);
11514 : {
11515 1792 : var_result = CAST(CallRuntime(
11516 : Runtime::kBigIntCompareToString, NoContextConstant(),
11517 896 : SmiConstant(Reverse(op)), right, left));
11518 896 : Goto(&end);
11519 : }
11520 :
11521 896 : BIND(&if_right_receiver);
11522 : {
11523 : Callable callable = CodeFactory::NonPrimitiveToPrimitive(
11524 1792 : isolate(), ToPrimitiveHint::kNumber);
11525 896 : var_right.Bind(CallStub(callable, context, right));
11526 896 : Goto(&loop);
11527 : }
11528 : }
11529 : }
11530 :
11531 896 : BIND(&if_left_other);
11532 : {
11533 : // {left} is neither a Numeric nor a String, and {right} is not a Smi.
11534 896 : if (var_type_feedback != nullptr) {
11535 : // Collect NumberOrOddball feedback if {left} is an Oddball
11536 : // and {right} is either a HeapNumber or Oddball. Otherwise collect
11537 : // Any feedback.
11538 1344 : Label collect_any_feedback(this), collect_oddball_feedback(this),
11539 1344 : collect_feedback_done(this);
11540 1344 : GotoIfNot(InstanceTypeEqual(left_instance_type, ODDBALL_TYPE),
11541 672 : &collect_any_feedback);
11542 :
11543 672 : GotoIf(IsHeapNumberMap(right_map), &collect_oddball_feedback);
11544 672 : Node* right_instance_type = LoadMapInstanceType(right_map);
11545 1344 : Branch(InstanceTypeEqual(right_instance_type, ODDBALL_TYPE),
11546 672 : &collect_oddball_feedback, &collect_any_feedback);
11547 :
11548 672 : BIND(&collect_oddball_feedback);
11549 : {
11550 : CombineFeedback(var_type_feedback,
11551 672 : CompareOperationFeedback::kNumberOrOddball);
11552 672 : Goto(&collect_feedback_done);
11553 : }
11554 :
11555 672 : BIND(&collect_any_feedback);
11556 : {
11557 : OverwriteFeedback(var_type_feedback,
11558 672 : CompareOperationFeedback::kAny);
11559 672 : Goto(&collect_feedback_done);
11560 : }
11561 :
11562 672 : BIND(&collect_feedback_done);
11563 : }
11564 :
11565 : // If {left} is a receiver, call ToPrimitive(left, hint Number).
11566 : // Otherwise call ToNumeric(right) and then ToNumeric(left), the
11567 : // order here is important as it's observable by user code.
11568 : STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
11569 1792 : Label if_left_receiver(this, Label::kDeferred);
11570 1792 : GotoIf(IsJSReceiverInstanceType(left_instance_type),
11571 896 : &if_left_receiver);
11572 :
11573 896 : var_right.Bind(CallBuiltin(Builtins::kToNumeric, context, right));
11574 896 : var_left.Bind(
11575 1792 : CallBuiltin(Builtins::kNonNumberToNumeric, context, left));
11576 896 : Goto(&loop);
11577 :
11578 896 : BIND(&if_left_receiver);
11579 : {
11580 : Callable callable = CodeFactory::NonPrimitiveToPrimitive(
11581 1792 : isolate(), ToPrimitiveHint::kNumber);
11582 896 : var_left.Bind(CallStub(callable, context, left));
11583 896 : Goto(&loop);
11584 : }
11585 : }
11586 : }
11587 : }
11588 : }
11589 :
11590 896 : BIND(&do_float_comparison);
11591 : {
11592 896 : switch (op) {
11593 : case Operation::kLessThan:
11594 448 : Branch(Float64LessThan(var_left_float.value(), var_right_float.value()),
11595 224 : &return_true, &return_false);
11596 224 : break;
11597 : case Operation::kLessThanOrEqual:
11598 896 : Branch(Float64LessThanOrEqual(var_left_float.value(),
11599 896 : var_right_float.value()),
11600 224 : &return_true, &return_false);
11601 224 : break;
11602 : case Operation::kGreaterThan:
11603 448 : Branch(
11604 448 : Float64GreaterThan(var_left_float.value(), var_right_float.value()),
11605 224 : &return_true, &return_false);
11606 224 : break;
11607 : case Operation::kGreaterThanOrEqual:
11608 896 : Branch(Float64GreaterThanOrEqual(var_left_float.value(),
11609 896 : var_right_float.value()),
11610 224 : &return_true, &return_false);
11611 224 : break;
11612 : default:
11613 0 : UNREACHABLE();
11614 : }
11615 : }
11616 :
11617 896 : BIND(&return_true);
11618 : {
11619 896 : var_result = TrueConstant();
11620 896 : Goto(&end);
11621 : }
11622 :
11623 896 : BIND(&return_false);
11624 : {
11625 896 : var_result = FalseConstant();
11626 896 : Goto(&end);
11627 : }
11628 :
11629 896 : BIND(&end);
11630 1792 : return var_result.value();
11631 : }
11632 :
11633 1120 : TNode<Smi> CodeStubAssembler::CollectFeedbackForString(
11634 : SloppyTNode<Int32T> instance_type) {
11635 : TNode<Smi> feedback = SelectSmiConstant(
11636 2240 : Word32Equal(
11637 2240 : Word32And(instance_type, Int32Constant(kIsNotInternalizedMask)),
11638 4480 : Int32Constant(kInternalizedTag)),
11639 : CompareOperationFeedback::kInternalizedString,
11640 1120 : CompareOperationFeedback::kString);
11641 1120 : return feedback;
11642 : }
11643 :
11644 616 : void CodeStubAssembler::GenerateEqual_Same(Node* value, Label* if_equal,
11645 : Label* if_notequal,
11646 : Variable* var_type_feedback) {
11647 : // In case of abstract or strict equality checks, we need additional checks
11648 : // for NaN values because they are not considered equal, even if both the
11649 : // left and the right hand side reference exactly the same value.
11650 :
11651 1232 : Label if_smi(this), if_heapnumber(this);
11652 616 : GotoIf(TaggedIsSmi(value), &if_smi);
11653 :
11654 616 : Node* value_map = LoadMap(value);
11655 616 : GotoIf(IsHeapNumberMap(value_map), &if_heapnumber);
11656 :
11657 : // For non-HeapNumbers, all we do is collect type feedback.
11658 616 : if (var_type_feedback != nullptr) {
11659 336 : Node* instance_type = LoadMapInstanceType(value_map);
11660 :
11661 672 : Label if_string(this), if_receiver(this), if_oddball(this), if_symbol(this),
11662 672 : if_bigint(this);
11663 336 : GotoIf(IsStringInstanceType(instance_type), &if_string);
11664 336 : GotoIf(IsJSReceiverInstanceType(instance_type), &if_receiver);
11665 336 : GotoIf(IsOddballInstanceType(instance_type), &if_oddball);
11666 336 : Branch(IsBigIntInstanceType(instance_type), &if_bigint, &if_symbol);
11667 :
11668 336 : BIND(&if_string);
11669 : {
11670 : CSA_ASSERT(this, IsString(value));
11671 336 : CombineFeedback(var_type_feedback,
11672 672 : CollectFeedbackForString(instance_type));
11673 336 : Goto(if_equal);
11674 : }
11675 :
11676 336 : BIND(&if_symbol);
11677 : {
11678 : CSA_ASSERT(this, IsSymbol(value));
11679 336 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kSymbol);
11680 336 : Goto(if_equal);
11681 : }
11682 :
11683 336 : BIND(&if_receiver);
11684 : {
11685 : CSA_ASSERT(this, IsJSReceiver(value));
11686 336 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kReceiver);
11687 336 : Goto(if_equal);
11688 : }
11689 :
11690 336 : BIND(&if_bigint);
11691 : {
11692 : CSA_ASSERT(this, IsBigInt(value));
11693 336 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt);
11694 336 : Goto(if_equal);
11695 : }
11696 :
11697 336 : BIND(&if_oddball);
11698 : {
11699 : CSA_ASSERT(this, IsOddball(value));
11700 672 : Label if_boolean(this), if_not_boolean(this);
11701 336 : Branch(IsBooleanMap(value_map), &if_boolean, &if_not_boolean);
11702 :
11703 336 : BIND(&if_boolean);
11704 : {
11705 336 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kAny);
11706 336 : Goto(if_equal);
11707 : }
11708 :
11709 336 : BIND(&if_not_boolean);
11710 : {
11711 : CSA_ASSERT(this, IsNullOrUndefined(value));
11712 : CombineFeedback(var_type_feedback,
11713 336 : CompareOperationFeedback::kReceiverOrNullOrUndefined);
11714 336 : Goto(if_equal);
11715 : }
11716 : }
11717 : } else {
11718 280 : Goto(if_equal);
11719 : }
11720 :
11721 616 : BIND(&if_heapnumber);
11722 : {
11723 616 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
11724 616 : Node* number_value = LoadHeapNumberValue(value);
11725 616 : BranchIfFloat64IsNaN(number_value, if_notequal, if_equal);
11726 : }
11727 :
11728 616 : BIND(&if_smi);
11729 : {
11730 616 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kSignedSmall);
11731 616 : Goto(if_equal);
11732 : }
11733 616 : }
11734 :
11735 : // ES6 section 7.2.12 Abstract Equality Comparison
11736 224 : Node* CodeStubAssembler::Equal(Node* left, Node* right, Node* context,
11737 : Variable* var_type_feedback) {
11738 : // This is a slightly optimized version of Object::Equals. Whenever you
11739 : // change something functionality wise in here, remember to update the
11740 : // Object::Equals method as well.
11741 :
11742 448 : Label if_equal(this), if_notequal(this), do_float_comparison(this),
11743 448 : do_right_stringtonumber(this, Label::kDeferred), end(this);
11744 448 : VARIABLE(result, MachineRepresentation::kTagged);
11745 448 : TVARIABLE(Float64T, var_left_float);
11746 448 : TVARIABLE(Float64T, var_right_float);
11747 :
11748 : // We can avoid code duplication by exploiting the fact that abstract equality
11749 : // is symmetric.
11750 448 : Label use_symmetry(this);
11751 :
11752 : // We might need to loop several times due to ToPrimitive and/or ToNumber
11753 : // conversions.
11754 448 : VARIABLE(var_left, MachineRepresentation::kTagged, left);
11755 448 : VARIABLE(var_right, MachineRepresentation::kTagged, right);
11756 448 : VariableList loop_variable_list({&var_left, &var_right}, zone());
11757 224 : if (var_type_feedback != nullptr) {
11758 : // Initialize the type feedback to None. The current feedback will be
11759 : // combined with the previous feedback.
11760 168 : OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kNone);
11761 168 : loop_variable_list.push_back(var_type_feedback);
11762 : }
11763 448 : Label loop(this, loop_variable_list);
11764 224 : Goto(&loop);
11765 224 : BIND(&loop);
11766 : {
11767 224 : left = var_left.value();
11768 224 : right = var_right.value();
11769 :
11770 448 : Label if_notsame(this);
11771 224 : GotoIf(WordNotEqual(left, right), &if_notsame);
11772 : {
11773 : // {left} and {right} reference the exact same value, yet we need special
11774 : // treatment for HeapNumber, as NaN is not equal to NaN.
11775 224 : GenerateEqual_Same(left, &if_equal, &if_notequal, var_type_feedback);
11776 : }
11777 :
11778 224 : BIND(&if_notsame);
11779 448 : Label if_left_smi(this), if_left_not_smi(this);
11780 224 : Branch(TaggedIsSmi(left), &if_left_smi, &if_left_not_smi);
11781 :
11782 224 : BIND(&if_left_smi);
11783 : {
11784 448 : Label if_right_smi(this), if_right_not_smi(this);
11785 224 : Branch(TaggedIsSmi(right), &if_right_smi, &if_right_not_smi);
11786 :
11787 224 : BIND(&if_right_smi);
11788 : {
11789 : // We have already checked for {left} and {right} being the same value,
11790 : // so when we get here they must be different Smis.
11791 : CombineFeedback(var_type_feedback,
11792 224 : CompareOperationFeedback::kSignedSmall);
11793 224 : Goto(&if_notequal);
11794 : }
11795 :
11796 224 : BIND(&if_right_not_smi);
11797 224 : Node* right_map = LoadMap(right);
11798 448 : Label if_right_heapnumber(this), if_right_boolean(this),
11799 448 : if_right_bigint(this, Label::kDeferred),
11800 448 : if_right_receiver(this, Label::kDeferred);
11801 224 : GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
11802 : // {left} is Smi and {right} is not HeapNumber or Smi.
11803 224 : if (var_type_feedback != nullptr) {
11804 168 : var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kAny));
11805 : }
11806 224 : GotoIf(IsBooleanMap(right_map), &if_right_boolean);
11807 224 : Node* right_type = LoadMapInstanceType(right_map);
11808 224 : GotoIf(IsStringInstanceType(right_type), &do_right_stringtonumber);
11809 224 : GotoIf(IsBigIntInstanceType(right_type), &if_right_bigint);
11810 448 : Branch(IsJSReceiverInstanceType(right_type), &if_right_receiver,
11811 224 : &if_notequal);
11812 :
11813 224 : BIND(&if_right_heapnumber);
11814 : {
11815 224 : var_left_float = SmiToFloat64(left);
11816 224 : var_right_float = LoadHeapNumberValue(right);
11817 224 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
11818 224 : Goto(&do_float_comparison);
11819 : }
11820 :
11821 224 : BIND(&if_right_boolean);
11822 : {
11823 224 : var_right.Bind(LoadObjectField(right, Oddball::kToNumberOffset));
11824 224 : Goto(&loop);
11825 : }
11826 :
11827 224 : BIND(&if_right_bigint);
11828 : {
11829 448 : result.Bind(CallRuntime(Runtime::kBigIntEqualToNumber,
11830 672 : NoContextConstant(), right, left));
11831 224 : Goto(&end);
11832 : }
11833 :
11834 224 : BIND(&if_right_receiver);
11835 : {
11836 448 : Callable callable = CodeFactory::NonPrimitiveToPrimitive(isolate());
11837 224 : var_right.Bind(CallStub(callable, context, right));
11838 224 : Goto(&loop);
11839 : }
11840 : }
11841 :
11842 224 : BIND(&if_left_not_smi);
11843 : {
11844 224 : GotoIf(TaggedIsSmi(right), &use_symmetry);
11845 :
11846 448 : Label if_left_symbol(this), if_left_number(this), if_left_string(this),
11847 448 : if_left_bigint(this, Label::kDeferred), if_left_oddball(this),
11848 448 : if_left_receiver(this);
11849 :
11850 224 : Node* left_map = LoadMap(left);
11851 224 : Node* right_map = LoadMap(right);
11852 224 : Node* left_type = LoadMapInstanceType(left_map);
11853 224 : Node* right_type = LoadMapInstanceType(right_map);
11854 :
11855 224 : GotoIf(IsStringInstanceType(left_type), &if_left_string);
11856 224 : GotoIf(IsSymbolInstanceType(left_type), &if_left_symbol);
11857 224 : GotoIf(IsHeapNumberInstanceType(left_type), &if_left_number);
11858 224 : GotoIf(IsOddballInstanceType(left_type), &if_left_oddball);
11859 448 : Branch(IsBigIntInstanceType(left_type), &if_left_bigint,
11860 224 : &if_left_receiver);
11861 :
11862 224 : BIND(&if_left_string);
11863 : {
11864 224 : GotoIfNot(IsStringInstanceType(right_type), &use_symmetry);
11865 224 : result.Bind(CallBuiltin(Builtins::kStringEqual, context, left, right));
11866 224 : CombineFeedback(var_type_feedback,
11867 448 : SmiOr(CollectFeedbackForString(left_type),
11868 672 : CollectFeedbackForString(right_type)));
11869 224 : Goto(&end);
11870 : }
11871 :
11872 224 : BIND(&if_left_number);
11873 : {
11874 448 : Label if_right_not_number(this);
11875 224 : GotoIf(Word32NotEqual(left_type, right_type), &if_right_not_number);
11876 :
11877 224 : var_left_float = LoadHeapNumberValue(left);
11878 224 : var_right_float = LoadHeapNumberValue(right);
11879 224 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
11880 224 : Goto(&do_float_comparison);
11881 :
11882 224 : BIND(&if_right_not_number);
11883 : {
11884 448 : Label if_right_boolean(this);
11885 224 : if (var_type_feedback != nullptr) {
11886 168 : var_type_feedback->Bind(
11887 336 : SmiConstant(CompareOperationFeedback::kAny));
11888 : }
11889 224 : GotoIf(IsStringInstanceType(right_type), &do_right_stringtonumber);
11890 224 : GotoIf(IsBooleanMap(right_map), &if_right_boolean);
11891 224 : GotoIf(IsBigIntInstanceType(right_type), &use_symmetry);
11892 448 : Branch(IsJSReceiverInstanceType(right_type), &use_symmetry,
11893 224 : &if_notequal);
11894 :
11895 224 : BIND(&if_right_boolean);
11896 : {
11897 224 : var_right.Bind(LoadObjectField(right, Oddball::kToNumberOffset));
11898 224 : Goto(&loop);
11899 : }
11900 : }
11901 : }
11902 :
11903 224 : BIND(&if_left_bigint);
11904 : {
11905 448 : Label if_right_heapnumber(this), if_right_bigint(this),
11906 448 : if_right_string(this), if_right_boolean(this);
11907 224 : GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
11908 224 : GotoIf(IsBigIntInstanceType(right_type), &if_right_bigint);
11909 224 : GotoIf(IsStringInstanceType(right_type), &if_right_string);
11910 224 : GotoIf(IsBooleanMap(right_map), &if_right_boolean);
11911 448 : Branch(IsJSReceiverInstanceType(right_type), &use_symmetry,
11912 224 : &if_notequal);
11913 :
11914 224 : BIND(&if_right_heapnumber);
11915 : {
11916 224 : if (var_type_feedback != nullptr) {
11917 168 : var_type_feedback->Bind(
11918 336 : SmiConstant(CompareOperationFeedback::kAny));
11919 : }
11920 448 : result.Bind(CallRuntime(Runtime::kBigIntEqualToNumber,
11921 672 : NoContextConstant(), left, right));
11922 224 : Goto(&end);
11923 : }
11924 :
11925 224 : BIND(&if_right_bigint);
11926 : {
11927 224 : CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt);
11928 448 : result.Bind(CallRuntime(Runtime::kBigIntEqualToBigInt,
11929 672 : NoContextConstant(), left, right));
11930 224 : Goto(&end);
11931 : }
11932 :
11933 224 : BIND(&if_right_string);
11934 : {
11935 224 : if (var_type_feedback != nullptr) {
11936 168 : var_type_feedback->Bind(
11937 336 : SmiConstant(CompareOperationFeedback::kAny));
11938 : }
11939 448 : result.Bind(CallRuntime(Runtime::kBigIntEqualToString,
11940 672 : NoContextConstant(), left, right));
11941 224 : Goto(&end);
11942 : }
11943 :
11944 224 : BIND(&if_right_boolean);
11945 : {
11946 224 : if (var_type_feedback != nullptr) {
11947 168 : var_type_feedback->Bind(
11948 336 : SmiConstant(CompareOperationFeedback::kAny));
11949 : }
11950 224 : var_right.Bind(LoadObjectField(right, Oddball::kToNumberOffset));
11951 224 : Goto(&loop);
11952 : }
11953 : }
11954 :
11955 224 : BIND(&if_left_oddball);
11956 : {
11957 448 : Label if_left_boolean(this), if_left_not_boolean(this);
11958 224 : Branch(IsBooleanMap(left_map), &if_left_boolean, &if_left_not_boolean);
11959 :
11960 224 : BIND(&if_left_not_boolean);
11961 : {
11962 : // {left} is either Null or Undefined. Check if {right} is
11963 : // undetectable (which includes Null and Undefined).
11964 448 : Label if_right_undetectable(this), if_right_not_undetectable(this);
11965 448 : Branch(IsUndetectableMap(right_map), &if_right_undetectable,
11966 224 : &if_right_not_undetectable);
11967 :
11968 224 : BIND(&if_right_undetectable);
11969 : {
11970 224 : if (var_type_feedback != nullptr) {
11971 : // If {right} is undetectable, it must be either also
11972 : // Null or Undefined, or a Receiver (aka document.all).
11973 336 : var_type_feedback->Bind(SmiConstant(
11974 336 : CompareOperationFeedback::kReceiverOrNullOrUndefined));
11975 : }
11976 224 : Goto(&if_equal);
11977 : }
11978 :
11979 224 : BIND(&if_right_not_undetectable);
11980 : {
11981 224 : if (var_type_feedback != nullptr) {
11982 : // Track whether {right} is Null, Undefined or Receiver.
11983 336 : var_type_feedback->Bind(SmiConstant(
11984 336 : CompareOperationFeedback::kReceiverOrNullOrUndefined));
11985 168 : GotoIf(IsJSReceiverInstanceType(right_type), &if_notequal);
11986 168 : GotoIfNot(IsBooleanMap(right_map), &if_notequal);
11987 168 : var_type_feedback->Bind(
11988 336 : SmiConstant(CompareOperationFeedback::kAny));
11989 : }
11990 224 : Goto(&if_notequal);
11991 : }
11992 : }
11993 :
11994 224 : BIND(&if_left_boolean);
11995 : {
11996 224 : if (var_type_feedback != nullptr) {
11997 168 : var_type_feedback->Bind(
11998 336 : SmiConstant(CompareOperationFeedback::kAny));
11999 : }
12000 :
12001 : // If {right} is a Boolean too, it must be a different Boolean.
12002 224 : GotoIf(WordEqual(right_map, left_map), &if_notequal);
12003 :
12004 : // Otherwise, convert {left} to number and try again.
12005 224 : var_left.Bind(LoadObjectField(left, Oddball::kToNumberOffset));
12006 224 : Goto(&loop);
12007 : }
12008 : }
12009 :
12010 224 : BIND(&if_left_symbol);
12011 : {
12012 448 : Label if_right_receiver(this);
12013 224 : GotoIf(IsJSReceiverInstanceType(right_type), &if_right_receiver);
12014 : // {right} is not a JSReceiver and also not the same Symbol as {left},
12015 : // so the result is "not equal".
12016 224 : if (var_type_feedback != nullptr) {
12017 336 : Label if_right_symbol(this);
12018 168 : GotoIf(IsSymbolInstanceType(right_type), &if_right_symbol);
12019 168 : var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kAny));
12020 168 : Goto(&if_notequal);
12021 :
12022 168 : BIND(&if_right_symbol);
12023 : {
12024 : CombineFeedback(var_type_feedback,
12025 168 : CompareOperationFeedback::kSymbol);
12026 168 : Goto(&if_notequal);
12027 : }
12028 : } else {
12029 56 : Goto(&if_notequal);
12030 : }
12031 :
12032 224 : BIND(&if_right_receiver);
12033 : {
12034 : // {left} is a Primitive and {right} is a JSReceiver, so swapping
12035 : // the order is not observable.
12036 224 : if (var_type_feedback != nullptr) {
12037 168 : var_type_feedback->Bind(
12038 336 : SmiConstant(CompareOperationFeedback::kAny));
12039 : }
12040 224 : Goto(&use_symmetry);
12041 : }
12042 : }
12043 :
12044 224 : BIND(&if_left_receiver);
12045 : {
12046 : CSA_ASSERT(this, IsJSReceiverInstanceType(left_type));
12047 448 : Label if_right_receiver(this), if_right_not_receiver(this);
12048 448 : Branch(IsJSReceiverInstanceType(right_type), &if_right_receiver,
12049 224 : &if_right_not_receiver);
12050 :
12051 224 : BIND(&if_right_receiver);
12052 : {
12053 : // {left} and {right} are different JSReceiver references.
12054 : CombineFeedback(var_type_feedback,
12055 224 : CompareOperationFeedback::kReceiver);
12056 224 : Goto(&if_notequal);
12057 : }
12058 :
12059 224 : BIND(&if_right_not_receiver);
12060 : {
12061 : // Check if {right} is undetectable, which means it must be Null
12062 : // or Undefined, since we already ruled out Receiver for {right}.
12063 448 : Label if_right_undetectable(this),
12064 448 : if_right_not_undetectable(this, Label::kDeferred);
12065 448 : Branch(IsUndetectableMap(right_map), &if_right_undetectable,
12066 224 : &if_right_not_undetectable);
12067 :
12068 224 : BIND(&if_right_undetectable);
12069 : {
12070 : // When we get here, {right} must be either Null or Undefined.
12071 : CSA_ASSERT(this, IsNullOrUndefined(right));
12072 224 : if (var_type_feedback != nullptr) {
12073 336 : var_type_feedback->Bind(SmiConstant(
12074 336 : CompareOperationFeedback::kReceiverOrNullOrUndefined));
12075 : }
12076 224 : Branch(IsUndetectableMap(left_map), &if_equal, &if_notequal);
12077 : }
12078 :
12079 224 : BIND(&if_right_not_undetectable);
12080 : {
12081 : // {right} is a Primitive, and neither Null or Undefined;
12082 : // convert {left} to Primitive too.
12083 224 : if (var_type_feedback != nullptr) {
12084 168 : var_type_feedback->Bind(
12085 336 : SmiConstant(CompareOperationFeedback::kAny));
12086 : }
12087 448 : Callable callable = CodeFactory::NonPrimitiveToPrimitive(isolate());
12088 224 : var_left.Bind(CallStub(callable, context, left));
12089 224 : Goto(&loop);
12090 : }
12091 : }
12092 : }
12093 : }
12094 :
12095 224 : BIND(&do_right_stringtonumber);
12096 : {
12097 224 : var_right.Bind(CallBuiltin(Builtins::kStringToNumber, context, right));
12098 224 : Goto(&loop);
12099 : }
12100 :
12101 224 : BIND(&use_symmetry);
12102 : {
12103 224 : var_left.Bind(right);
12104 224 : var_right.Bind(left);
12105 224 : Goto(&loop);
12106 : }
12107 : }
12108 :
12109 224 : BIND(&do_float_comparison);
12110 : {
12111 448 : Branch(Float64Equal(var_left_float.value(), var_right_float.value()),
12112 224 : &if_equal, &if_notequal);
12113 : }
12114 :
12115 224 : BIND(&if_equal);
12116 : {
12117 224 : result.Bind(TrueConstant());
12118 224 : Goto(&end);
12119 : }
12120 :
12121 224 : BIND(&if_notequal);
12122 : {
12123 224 : result.Bind(FalseConstant());
12124 224 : Goto(&end);
12125 : }
12126 :
12127 224 : BIND(&end);
12128 448 : return result.value();
12129 : }
12130 :
12131 392 : Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs,
12132 : Variable* var_type_feedback) {
12133 : // Pseudo-code for the algorithm below:
12134 : //
12135 : // if (lhs == rhs) {
12136 : // if (lhs->IsHeapNumber()) return HeapNumber::cast(lhs)->value() != NaN;
12137 : // return true;
12138 : // }
12139 : // if (!lhs->IsSmi()) {
12140 : // if (lhs->IsHeapNumber()) {
12141 : // if (rhs->IsSmi()) {
12142 : // return Smi::ToInt(rhs) == HeapNumber::cast(lhs)->value();
12143 : // } else if (rhs->IsHeapNumber()) {
12144 : // return HeapNumber::cast(rhs)->value() ==
12145 : // HeapNumber::cast(lhs)->value();
12146 : // } else {
12147 : // return false;
12148 : // }
12149 : // } else {
12150 : // if (rhs->IsSmi()) {
12151 : // return false;
12152 : // } else {
12153 : // if (lhs->IsString()) {
12154 : // if (rhs->IsString()) {
12155 : // return %StringEqual(lhs, rhs);
12156 : // } else {
12157 : // return false;
12158 : // }
12159 : // } else if (lhs->IsBigInt()) {
12160 : // if (rhs->IsBigInt()) {
12161 : // return %BigIntEqualToBigInt(lhs, rhs);
12162 : // } else {
12163 : // return false;
12164 : // }
12165 : // } else {
12166 : // return false;
12167 : // }
12168 : // }
12169 : // }
12170 : // } else {
12171 : // if (rhs->IsSmi()) {
12172 : // return false;
12173 : // } else {
12174 : // if (rhs->IsHeapNumber()) {
12175 : // return Smi::ToInt(lhs) == HeapNumber::cast(rhs)->value();
12176 : // } else {
12177 : // return false;
12178 : // }
12179 : // }
12180 : // }
12181 :
12182 784 : Label if_equal(this), if_notequal(this), end(this);
12183 784 : VARIABLE(result, MachineRepresentation::kTagged);
12184 :
12185 : // Check if {lhs} and {rhs} refer to the same object.
12186 784 : Label if_same(this), if_notsame(this);
12187 392 : Branch(WordEqual(lhs, rhs), &if_same, &if_notsame);
12188 :
12189 392 : BIND(&if_same);
12190 : {
12191 : // The {lhs} and {rhs} reference the exact same value, yet we need special
12192 : // treatment for HeapNumber, as NaN is not equal to NaN.
12193 392 : if (var_type_feedback != nullptr) {
12194 168 : var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kNone));
12195 : }
12196 392 : GenerateEqual_Same(lhs, &if_equal, &if_notequal, var_type_feedback);
12197 : }
12198 :
12199 392 : BIND(&if_notsame);
12200 : {
12201 : // The {lhs} and {rhs} reference different objects, yet for Smi, HeapNumber,
12202 : // BigInt and String they can still be considered equal.
12203 :
12204 392 : if (var_type_feedback != nullptr) {
12205 168 : var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kAny));
12206 : }
12207 :
12208 : // Check if {lhs} is a Smi or a HeapObject.
12209 784 : Label if_lhsissmi(this), if_lhsisnotsmi(this);
12210 392 : Branch(TaggedIsSmi(lhs), &if_lhsissmi, &if_lhsisnotsmi);
12211 :
12212 392 : BIND(&if_lhsisnotsmi);
12213 : {
12214 : // Load the map of {lhs}.
12215 392 : Node* lhs_map = LoadMap(lhs);
12216 :
12217 : // Check if {lhs} is a HeapNumber.
12218 784 : Label if_lhsisnumber(this), if_lhsisnotnumber(this);
12219 392 : Branch(IsHeapNumberMap(lhs_map), &if_lhsisnumber, &if_lhsisnotnumber);
12220 :
12221 392 : BIND(&if_lhsisnumber);
12222 : {
12223 : // Check if {rhs} is a Smi or a HeapObject.
12224 784 : Label if_rhsissmi(this), if_rhsisnotsmi(this);
12225 392 : Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisnotsmi);
12226 :
12227 392 : BIND(&if_rhsissmi);
12228 : {
12229 : // Convert {lhs} and {rhs} to floating point values.
12230 392 : Node* lhs_value = LoadHeapNumberValue(lhs);
12231 392 : Node* rhs_value = SmiToFloat64(rhs);
12232 :
12233 392 : if (var_type_feedback != nullptr) {
12234 168 : var_type_feedback->Bind(
12235 336 : SmiConstant(CompareOperationFeedback::kNumber));
12236 : }
12237 :
12238 : // Perform a floating point comparison of {lhs} and {rhs}.
12239 392 : Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
12240 : }
12241 :
12242 392 : BIND(&if_rhsisnotsmi);
12243 : {
12244 : // Load the map of {rhs}.
12245 392 : Node* rhs_map = LoadMap(rhs);
12246 :
12247 : // Check if {rhs} is also a HeapNumber.
12248 784 : Label if_rhsisnumber(this), if_rhsisnotnumber(this);
12249 392 : Branch(IsHeapNumberMap(rhs_map), &if_rhsisnumber, &if_rhsisnotnumber);
12250 :
12251 392 : BIND(&if_rhsisnumber);
12252 : {
12253 : // Convert {lhs} and {rhs} to floating point values.
12254 392 : Node* lhs_value = LoadHeapNumberValue(lhs);
12255 392 : Node* rhs_value = LoadHeapNumberValue(rhs);
12256 :
12257 392 : if (var_type_feedback != nullptr) {
12258 168 : var_type_feedback->Bind(
12259 336 : SmiConstant(CompareOperationFeedback::kNumber));
12260 : }
12261 :
12262 : // Perform a floating point comparison of {lhs} and {rhs}.
12263 392 : Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
12264 : }
12265 :
12266 392 : BIND(&if_rhsisnotnumber);
12267 392 : Goto(&if_notequal);
12268 : }
12269 : }
12270 :
12271 392 : BIND(&if_lhsisnotnumber);
12272 : {
12273 : // Check if {rhs} is a Smi or a HeapObject.
12274 784 : Label if_rhsissmi(this), if_rhsisnotsmi(this);
12275 392 : Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisnotsmi);
12276 :
12277 392 : BIND(&if_rhsissmi);
12278 392 : Goto(&if_notequal);
12279 :
12280 392 : BIND(&if_rhsisnotsmi);
12281 : {
12282 : // Load the instance type of {lhs}.
12283 392 : Node* lhs_instance_type = LoadMapInstanceType(lhs_map);
12284 :
12285 : // Check if {lhs} is a String.
12286 784 : Label if_lhsisstring(this), if_lhsisnotstring(this);
12287 784 : Branch(IsStringInstanceType(lhs_instance_type), &if_lhsisstring,
12288 392 : &if_lhsisnotstring);
12289 :
12290 392 : BIND(&if_lhsisstring);
12291 : {
12292 : // Load the instance type of {rhs}.
12293 392 : Node* rhs_instance_type = LoadInstanceType(rhs);
12294 :
12295 : // Check if {rhs} is also a String.
12296 784 : Label if_rhsisstring(this, Label::kDeferred),
12297 784 : if_rhsisnotstring(this);
12298 784 : Branch(IsStringInstanceType(rhs_instance_type), &if_rhsisstring,
12299 392 : &if_rhsisnotstring);
12300 :
12301 392 : BIND(&if_rhsisstring);
12302 : {
12303 392 : if (var_type_feedback != nullptr) {
12304 : TNode<Smi> lhs_feedback =
12305 168 : CollectFeedbackForString(lhs_instance_type);
12306 : TNode<Smi> rhs_feedback =
12307 168 : CollectFeedbackForString(rhs_instance_type);
12308 168 : var_type_feedback->Bind(SmiOr(lhs_feedback, rhs_feedback));
12309 : }
12310 784 : result.Bind(CallBuiltin(Builtins::kStringEqual,
12311 1176 : NoContextConstant(), lhs, rhs));
12312 392 : Goto(&end);
12313 : }
12314 :
12315 392 : BIND(&if_rhsisnotstring);
12316 392 : Goto(&if_notequal);
12317 : }
12318 :
12319 392 : BIND(&if_lhsisnotstring);
12320 :
12321 : // Check if {lhs} is a BigInt.
12322 784 : Label if_lhsisbigint(this), if_lhsisnotbigint(this);
12323 784 : Branch(IsBigIntInstanceType(lhs_instance_type), &if_lhsisbigint,
12324 392 : &if_lhsisnotbigint);
12325 :
12326 392 : BIND(&if_lhsisbigint);
12327 : {
12328 : // Load the instance type of {rhs}.
12329 392 : Node* rhs_instance_type = LoadInstanceType(rhs);
12330 :
12331 : // Check if {rhs} is also a BigInt.
12332 784 : Label if_rhsisbigint(this, Label::kDeferred),
12333 784 : if_rhsisnotbigint(this);
12334 784 : Branch(IsBigIntInstanceType(rhs_instance_type), &if_rhsisbigint,
12335 392 : &if_rhsisnotbigint);
12336 :
12337 392 : BIND(&if_rhsisbigint);
12338 : {
12339 392 : if (var_type_feedback != nullptr) {
12340 168 : var_type_feedback->Bind(
12341 336 : SmiConstant(CompareOperationFeedback::kBigInt));
12342 : }
12343 784 : result.Bind(CallRuntime(Runtime::kBigIntEqualToBigInt,
12344 1176 : NoContextConstant(), lhs, rhs));
12345 392 : Goto(&end);
12346 : }
12347 :
12348 392 : BIND(&if_rhsisnotbigint);
12349 392 : Goto(&if_notequal);
12350 : }
12351 :
12352 392 : BIND(&if_lhsisnotbigint);
12353 392 : if (var_type_feedback != nullptr) {
12354 : // Load the instance type of {rhs}.
12355 168 : Node* rhs_map = LoadMap(rhs);
12356 168 : Node* rhs_instance_type = LoadMapInstanceType(rhs_map);
12357 :
12358 336 : Label if_lhsissymbol(this), if_lhsisreceiver(this),
12359 336 : if_lhsisoddball(this);
12360 336 : GotoIf(IsJSReceiverInstanceType(lhs_instance_type),
12361 168 : &if_lhsisreceiver);
12362 168 : GotoIf(IsBooleanMap(lhs_map), &if_notequal);
12363 168 : GotoIf(IsOddballInstanceType(lhs_instance_type), &if_lhsisoddball);
12364 336 : Branch(IsSymbolInstanceType(lhs_instance_type), &if_lhsissymbol,
12365 168 : &if_notequal);
12366 :
12367 168 : BIND(&if_lhsisreceiver);
12368 : {
12369 168 : GotoIf(IsBooleanMap(rhs_map), &if_notequal);
12370 168 : var_type_feedback->Bind(
12371 336 : SmiConstant(CompareOperationFeedback::kReceiver));
12372 168 : GotoIf(IsJSReceiverInstanceType(rhs_instance_type), &if_notequal);
12373 336 : var_type_feedback->Bind(SmiConstant(
12374 336 : CompareOperationFeedback::kReceiverOrNullOrUndefined));
12375 168 : GotoIf(IsOddballInstanceType(rhs_instance_type), &if_notequal);
12376 168 : var_type_feedback->Bind(
12377 336 : SmiConstant(CompareOperationFeedback::kAny));
12378 168 : Goto(&if_notequal);
12379 : }
12380 :
12381 168 : BIND(&if_lhsisoddball);
12382 : {
12383 : STATIC_ASSERT(LAST_PRIMITIVE_TYPE == ODDBALL_TYPE);
12384 168 : GotoIf(IsBooleanMap(rhs_map), &if_notequal);
12385 336 : GotoIf(
12386 336 : Int32LessThan(rhs_instance_type, Int32Constant(ODDBALL_TYPE)),
12387 168 : &if_notequal);
12388 336 : var_type_feedback->Bind(SmiConstant(
12389 336 : CompareOperationFeedback::kReceiverOrNullOrUndefined));
12390 168 : Goto(&if_notequal);
12391 : }
12392 :
12393 168 : BIND(&if_lhsissymbol);
12394 : {
12395 168 : GotoIfNot(IsSymbolInstanceType(rhs_instance_type), &if_notequal);
12396 168 : var_type_feedback->Bind(
12397 336 : SmiConstant(CompareOperationFeedback::kSymbol));
12398 168 : Goto(&if_notequal);
12399 : }
12400 : } else {
12401 224 : Goto(&if_notequal);
12402 : }
12403 : }
12404 : }
12405 : }
12406 :
12407 392 : BIND(&if_lhsissmi);
12408 : {
12409 : // We already know that {lhs} and {rhs} are not reference equal, and {lhs}
12410 : // is a Smi; so {lhs} and {rhs} can only be strictly equal if {rhs} is a
12411 : // HeapNumber with an equal floating point value.
12412 :
12413 : // Check if {rhs} is a Smi or a HeapObject.
12414 784 : Label if_rhsissmi(this), if_rhsisnotsmi(this);
12415 392 : Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisnotsmi);
12416 :
12417 392 : BIND(&if_rhsissmi);
12418 392 : if (var_type_feedback != nullptr) {
12419 168 : var_type_feedback->Bind(
12420 336 : SmiConstant(CompareOperationFeedback::kSignedSmall));
12421 : }
12422 392 : Goto(&if_notequal);
12423 :
12424 392 : BIND(&if_rhsisnotsmi);
12425 : {
12426 : // Load the map of the {rhs}.
12427 392 : Node* rhs_map = LoadMap(rhs);
12428 :
12429 : // The {rhs} could be a HeapNumber with the same value as {lhs}.
12430 784 : Label if_rhsisnumber(this), if_rhsisnotnumber(this);
12431 392 : Branch(IsHeapNumberMap(rhs_map), &if_rhsisnumber, &if_rhsisnotnumber);
12432 :
12433 392 : BIND(&if_rhsisnumber);
12434 : {
12435 : // Convert {lhs} and {rhs} to floating point values.
12436 392 : Node* lhs_value = SmiToFloat64(lhs);
12437 392 : Node* rhs_value = LoadHeapNumberValue(rhs);
12438 :
12439 392 : if (var_type_feedback != nullptr) {
12440 168 : var_type_feedback->Bind(
12441 336 : SmiConstant(CompareOperationFeedback::kNumber));
12442 : }
12443 :
12444 : // Perform a floating point comparison of {lhs} and {rhs}.
12445 392 : Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
12446 : }
12447 :
12448 392 : BIND(&if_rhsisnotnumber);
12449 392 : Goto(&if_notequal);
12450 : }
12451 : }
12452 : }
12453 :
12454 392 : BIND(&if_equal);
12455 : {
12456 392 : result.Bind(TrueConstant());
12457 392 : Goto(&end);
12458 : }
12459 :
12460 392 : BIND(&if_notequal);
12461 : {
12462 392 : result.Bind(FalseConstant());
12463 392 : Goto(&end);
12464 : }
12465 :
12466 392 : BIND(&end);
12467 784 : return result.value();
12468 : }
12469 :
12470 : // ECMA#sec-samevalue
12471 : // This algorithm differs from the Strict Equality Comparison Algorithm in its
12472 : // treatment of signed zeroes and NaNs.
12473 336 : void CodeStubAssembler::BranchIfSameValue(Node* lhs, Node* rhs, Label* if_true,
12474 : Label* if_false) {
12475 672 : VARIABLE(var_lhs_value, MachineRepresentation::kFloat64);
12476 672 : VARIABLE(var_rhs_value, MachineRepresentation::kFloat64);
12477 672 : Label do_fcmp(this);
12478 :
12479 : // Immediately jump to {if_true} if {lhs} == {rhs}, because - unlike
12480 : // StrictEqual - SameValue considers two NaNs to be equal.
12481 336 : GotoIf(WordEqual(lhs, rhs), if_true);
12482 :
12483 : // Check if the {lhs} is a Smi.
12484 672 : Label if_lhsissmi(this), if_lhsisheapobject(this);
12485 336 : Branch(TaggedIsSmi(lhs), &if_lhsissmi, &if_lhsisheapobject);
12486 :
12487 336 : BIND(&if_lhsissmi);
12488 : {
12489 : // Since {lhs} is a Smi, the comparison can only yield true
12490 : // iff the {rhs} is a HeapNumber with the same float64 value.
12491 1344 : Branch(TaggedIsSmi(rhs), if_false, [&] {
12492 1680 : GotoIfNot(IsHeapNumber(rhs), if_false);
12493 672 : var_lhs_value.Bind(SmiToFloat64(lhs));
12494 672 : var_rhs_value.Bind(LoadHeapNumberValue(rhs));
12495 672 : Goto(&do_fcmp);
12496 1008 : });
12497 : }
12498 :
12499 336 : BIND(&if_lhsisheapobject);
12500 : {
12501 : // Check if the {rhs} is a Smi.
12502 672 : Branch(TaggedIsSmi(rhs),
12503 336 : [&] {
12504 : // Since {rhs} is a Smi, the comparison can only yield true
12505 : // iff the {lhs} is a HeapNumber with the same float64 value.
12506 1344 : GotoIfNot(IsHeapNumber(lhs), if_false);
12507 672 : var_lhs_value.Bind(LoadHeapNumberValue(lhs));
12508 672 : var_rhs_value.Bind(SmiToFloat64(rhs));
12509 672 : Goto(&do_fcmp);
12510 336 : },
12511 336 : [&] {
12512 : // Now this can only yield true if either both {lhs} and {rhs} are
12513 : // HeapNumbers with the same value, or both are Strings with the
12514 : // same character sequence, or both are BigInts with the same
12515 : // value.
12516 10080 : Label if_lhsisheapnumber(this), if_lhsisstring(this),
12517 672 : if_lhsisbigint(this);
12518 1680 : Node* const lhs_map = LoadMap(lhs);
12519 1008 : GotoIf(IsHeapNumberMap(lhs_map), &if_lhsisheapnumber);
12520 672 : Node* const lhs_instance_type = LoadMapInstanceType(lhs_map);
12521 1008 : GotoIf(IsStringInstanceType(lhs_instance_type), &if_lhsisstring);
12522 1344 : Branch(IsBigIntInstanceType(lhs_instance_type), &if_lhsisbigint,
12523 2352 : if_false);
12524 :
12525 336 : BIND(&if_lhsisheapnumber);
12526 : {
12527 2688 : GotoIfNot(IsHeapNumber(rhs), if_false);
12528 672 : var_lhs_value.Bind(LoadHeapNumberValue(lhs));
12529 672 : var_rhs_value.Bind(LoadHeapNumberValue(rhs));
12530 672 : Goto(&do_fcmp);
12531 : }
12532 :
12533 336 : BIND(&if_lhsisstring);
12534 : {
12535 : // Now we can only yield true if {rhs} is also a String
12536 : // with the same sequence of characters.
12537 1344 : GotoIfNot(IsString(rhs), if_false);
12538 672 : Node* const result = CallBuiltin(Builtins::kStringEqual,
12539 1680 : NoContextConstant(), lhs, rhs);
12540 1344 : Branch(IsTrue(result), if_true, if_false);
12541 : }
12542 :
12543 336 : BIND(&if_lhsisbigint);
12544 : {
12545 1344 : GotoIfNot(IsBigInt(rhs), if_false);
12546 672 : Node* const result = CallRuntime(Runtime::kBigIntEqualToBigInt,
12547 2016 : NoContextConstant(), lhs, rhs);
12548 1344 : Branch(IsTrue(result), if_true, if_false);
12549 : }
12550 672 : });
12551 : }
12552 :
12553 336 : BIND(&do_fcmp);
12554 : {
12555 336 : Node* const lhs_value = var_lhs_value.value();
12556 336 : Node* const rhs_value = var_rhs_value.value();
12557 :
12558 672 : Label if_equal(this), if_notequal(this);
12559 336 : Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
12560 :
12561 336 : BIND(&if_equal);
12562 : {
12563 : // We still need to handle the case when {lhs} and {rhs} are -0.0 and
12564 : // 0.0 (or vice versa). Compare the high word to
12565 : // distinguish between the two.
12566 336 : Node* const lhs_hi_word = Float64ExtractHighWord32(lhs_value);
12567 336 : Node* const rhs_hi_word = Float64ExtractHighWord32(rhs_value);
12568 :
12569 : // If x is +0 and y is -0, return false.
12570 : // If x is -0 and y is +0, return false.
12571 336 : Branch(Word32Equal(lhs_hi_word, rhs_hi_word), if_true, if_false);
12572 : }
12573 :
12574 336 : BIND(&if_notequal);
12575 : {
12576 : // Return true iff both {rhs} and {lhs} are NaN.
12577 336 : GotoIf(Float64Equal(lhs_value, lhs_value), if_false);
12578 336 : Branch(Float64Equal(rhs_value, rhs_value), if_false, if_true);
12579 : }
12580 : }
12581 336 : }
12582 :
12583 672 : TNode<Oddball> CodeStubAssembler::HasProperty(SloppyTNode<Context> context,
12584 : SloppyTNode<Object> object,
12585 : SloppyTNode<Object> key,
12586 : HasPropertyLookupMode mode) {
12587 1344 : Label call_runtime(this, Label::kDeferred), return_true(this),
12588 1344 : return_false(this), end(this), if_proxy(this, Label::kDeferred);
12589 :
12590 : CodeStubAssembler::LookupInHolder lookup_property_in_holder =
12591 : [this, &return_true](Node* receiver, Node* holder, Node* holder_map,
12592 : Node* holder_instance_type, Node* unique_name,
12593 672 : Label* next_holder, Label* if_bailout) {
12594 : TryHasOwnProperty(holder, holder_map, holder_instance_type, unique_name,
12595 672 : &return_true, next_holder, if_bailout);
12596 2016 : };
12597 :
12598 : CodeStubAssembler::LookupInHolder lookup_element_in_holder =
12599 : [this, &return_true, &return_false](
12600 : Node* receiver, Node* holder, Node* holder_map,
12601 : Node* holder_instance_type, Node* index, Label* next_holder,
12602 1344 : Label* if_bailout) {
12603 1344 : TryLookupElement(holder, holder_map, holder_instance_type, index,
12604 672 : &return_true, &return_false, next_holder, if_bailout);
12605 2016 : };
12606 :
12607 672 : TryPrototypeChainLookup(object, key, lookup_property_in_holder,
12608 : lookup_element_in_holder, &return_false,
12609 672 : &call_runtime, &if_proxy);
12610 :
12611 1344 : TVARIABLE(Oddball, result);
12612 :
12613 672 : BIND(&if_proxy);
12614 : {
12615 672 : TNode<Name> name = CAST(CallBuiltin(Builtins::kToName, context, key));
12616 672 : switch (mode) {
12617 : case kHasProperty:
12618 616 : GotoIf(IsPrivateSymbol(name), &return_false);
12619 :
12620 1232 : result = CAST(
12621 616 : CallBuiltin(Builtins::kProxyHasProperty, context, object, name));
12622 616 : Goto(&end);
12623 616 : break;
12624 : case kForInHasProperty:
12625 56 : Goto(&call_runtime);
12626 56 : break;
12627 : }
12628 : }
12629 :
12630 672 : BIND(&return_true);
12631 : {
12632 672 : result = TrueConstant();
12633 672 : Goto(&end);
12634 : }
12635 :
12636 672 : BIND(&return_false);
12637 : {
12638 672 : result = FalseConstant();
12639 672 : Goto(&end);
12640 : }
12641 :
12642 672 : BIND(&call_runtime);
12643 : {
12644 : Runtime::FunctionId fallback_runtime_function_id;
12645 672 : switch (mode) {
12646 : case kHasProperty:
12647 616 : fallback_runtime_function_id = Runtime::kHasProperty;
12648 616 : break;
12649 : case kForInHasProperty:
12650 56 : fallback_runtime_function_id = Runtime::kForInHasProperty;
12651 56 : break;
12652 : }
12653 :
12654 672 : result =
12655 1344 : CAST(CallRuntime(fallback_runtime_function_id, context, object, key));
12656 672 : Goto(&end);
12657 : }
12658 :
12659 672 : BIND(&end);
12660 : CSA_ASSERT(this, IsBoolean(result.value()));
12661 1344 : return result.value();
12662 : }
12663 :
12664 392 : Node* CodeStubAssembler::Typeof(Node* value) {
12665 784 : VARIABLE(result_var, MachineRepresentation::kTagged);
12666 :
12667 784 : Label return_number(this, Label::kDeferred), if_oddball(this),
12668 784 : return_function(this), return_undefined(this), return_object(this),
12669 784 : return_string(this), return_bigint(this), return_result(this);
12670 :
12671 392 : GotoIf(TaggedIsSmi(value), &return_number);
12672 :
12673 392 : Node* map = LoadMap(value);
12674 :
12675 392 : GotoIf(IsHeapNumberMap(map), &return_number);
12676 :
12677 392 : Node* instance_type = LoadMapInstanceType(map);
12678 :
12679 392 : GotoIf(InstanceTypeEqual(instance_type, ODDBALL_TYPE), &if_oddball);
12680 :
12681 784 : Node* callable_or_undetectable_mask = Word32And(
12682 784 : LoadMapBitField(map),
12683 1960 : Int32Constant(Map::IsCallableBit::kMask | Map::IsUndetectableBit::kMask));
12684 :
12685 784 : GotoIf(Word32Equal(callable_or_undetectable_mask,
12686 784 : Int32Constant(Map::IsCallableBit::kMask)),
12687 392 : &return_function);
12688 :
12689 784 : GotoIfNot(Word32Equal(callable_or_undetectable_mask, Int32Constant(0)),
12690 392 : &return_undefined);
12691 :
12692 392 : GotoIf(IsJSReceiverInstanceType(instance_type), &return_object);
12693 :
12694 392 : GotoIf(IsStringInstanceType(instance_type), &return_string);
12695 :
12696 392 : GotoIf(IsBigIntInstanceType(instance_type), &return_bigint);
12697 :
12698 : CSA_ASSERT(this, InstanceTypeEqual(instance_type, SYMBOL_TYPE));
12699 392 : result_var.Bind(HeapConstant(isolate()->factory()->symbol_string()));
12700 392 : Goto(&return_result);
12701 :
12702 392 : BIND(&return_number);
12703 : {
12704 392 : result_var.Bind(HeapConstant(isolate()->factory()->number_string()));
12705 392 : Goto(&return_result);
12706 : }
12707 :
12708 392 : BIND(&if_oddball);
12709 : {
12710 392 : Node* type = LoadObjectField(value, Oddball::kTypeOfOffset);
12711 392 : result_var.Bind(type);
12712 392 : Goto(&return_result);
12713 : }
12714 :
12715 392 : BIND(&return_function);
12716 : {
12717 392 : result_var.Bind(HeapConstant(isolate()->factory()->function_string()));
12718 392 : Goto(&return_result);
12719 : }
12720 :
12721 392 : BIND(&return_undefined);
12722 : {
12723 392 : result_var.Bind(HeapConstant(isolate()->factory()->undefined_string()));
12724 392 : Goto(&return_result);
12725 : }
12726 :
12727 392 : BIND(&return_object);
12728 : {
12729 392 : result_var.Bind(HeapConstant(isolate()->factory()->object_string()));
12730 392 : Goto(&return_result);
12731 : }
12732 :
12733 392 : BIND(&return_string);
12734 : {
12735 392 : result_var.Bind(HeapConstant(isolate()->factory()->string_string()));
12736 392 : Goto(&return_result);
12737 : }
12738 :
12739 392 : BIND(&return_bigint);
12740 : {
12741 392 : result_var.Bind(HeapConstant(isolate()->factory()->bigint_string()));
12742 392 : Goto(&return_result);
12743 : }
12744 :
12745 392 : BIND(&return_result);
12746 784 : return result_var.value();
12747 : }
12748 :
12749 224 : TNode<Object> CodeStubAssembler::GetSuperConstructor(
12750 : SloppyTNode<Context> context, SloppyTNode<JSFunction> active_function) {
12751 448 : Label is_not_constructor(this, Label::kDeferred), out(this);
12752 448 : TVARIABLE(Object, result);
12753 :
12754 224 : TNode<Map> map = LoadMap(active_function);
12755 224 : TNode<Object> prototype = LoadMapPrototype(map);
12756 224 : TNode<Map> prototype_map = LoadMap(CAST(prototype));
12757 224 : GotoIfNot(IsConstructorMap(prototype_map), &is_not_constructor);
12758 :
12759 224 : result = prototype;
12760 224 : Goto(&out);
12761 :
12762 224 : BIND(&is_not_constructor);
12763 : {
12764 : CallRuntime(Runtime::kThrowNotSuperConstructor, context, prototype,
12765 224 : active_function);
12766 224 : Unreachable();
12767 : }
12768 :
12769 224 : BIND(&out);
12770 448 : return result.value();
12771 : }
12772 :
12773 504 : TNode<JSReceiver> CodeStubAssembler::SpeciesConstructor(
12774 : SloppyTNode<Context> context, SloppyTNode<Object> object,
12775 : SloppyTNode<JSReceiver> default_constructor) {
12776 504 : Isolate* isolate = this->isolate();
12777 1008 : TVARIABLE(JSReceiver, var_result, default_constructor);
12778 :
12779 : // 2. Let C be ? Get(O, "constructor").
12780 : TNode<Object> constructor =
12781 1008 : GetProperty(context, object, isolate->factory()->constructor_string());
12782 :
12783 : // 3. If C is undefined, return defaultConstructor.
12784 1008 : Label out(this);
12785 504 : GotoIf(IsUndefined(constructor), &out);
12786 :
12787 : // 4. If Type(C) is not Object, throw a TypeError exception.
12788 504 : ThrowIfNotJSReceiver(context, constructor,
12789 504 : MessageTemplate::kConstructorNotReceiver);
12790 :
12791 : // 5. Let S be ? Get(C, @@species).
12792 : TNode<Object> species =
12793 1008 : GetProperty(context, constructor, isolate->factory()->species_symbol());
12794 :
12795 : // 6. If S is either undefined or null, return defaultConstructor.
12796 504 : GotoIf(IsNullOrUndefined(species), &out);
12797 :
12798 : // 7. If IsConstructor(S) is true, return S.
12799 1008 : Label throw_error(this);
12800 504 : GotoIf(TaggedIsSmi(species), &throw_error);
12801 504 : GotoIfNot(IsConstructorMap(LoadMap(CAST(species))), &throw_error);
12802 504 : var_result = CAST(species);
12803 504 : Goto(&out);
12804 :
12805 : // 8. Throw a TypeError exception.
12806 504 : BIND(&throw_error);
12807 504 : ThrowTypeError(context, MessageTemplate::kSpeciesNotConstructor);
12808 :
12809 504 : BIND(&out);
12810 1008 : return var_result.value();
12811 : }
12812 :
12813 224 : Node* CodeStubAssembler::InstanceOf(Node* object, Node* callable,
12814 : Node* context) {
12815 448 : VARIABLE(var_result, MachineRepresentation::kTagged);
12816 448 : Label if_notcallable(this, Label::kDeferred),
12817 448 : if_notreceiver(this, Label::kDeferred), if_otherhandler(this),
12818 448 : if_nohandler(this, Label::kDeferred), return_true(this),
12819 448 : return_false(this), return_result(this, &var_result);
12820 :
12821 : // Ensure that the {callable} is actually a JSReceiver.
12822 224 : GotoIf(TaggedIsSmi(callable), &if_notreceiver);
12823 224 : GotoIfNot(IsJSReceiver(callable), &if_notreceiver);
12824 :
12825 : // Load the @@hasInstance property from {callable}.
12826 : Node* inst_of_handler =
12827 224 : GetProperty(context, callable, HasInstanceSymbolConstant());
12828 :
12829 : // Optimize for the likely case where {inst_of_handler} is the builtin
12830 : // Function.prototype[@@hasInstance] method, and emit a direct call in
12831 : // that case without any additional checking.
12832 224 : Node* native_context = LoadNativeContext(context);
12833 : Node* function_has_instance =
12834 224 : LoadContextElement(native_context, Context::FUNCTION_HAS_INSTANCE_INDEX);
12835 448 : GotoIfNot(WordEqual(inst_of_handler, function_has_instance),
12836 224 : &if_otherhandler);
12837 : {
12838 : // Call to Function.prototype[@@hasInstance] directly.
12839 : Callable builtin(BUILTIN_CODE(isolate(), FunctionPrototypeHasInstance),
12840 448 : CallTrampolineDescriptor{});
12841 224 : Node* result = CallJS(builtin, context, inst_of_handler, callable, object);
12842 224 : var_result.Bind(result);
12843 224 : Goto(&return_result);
12844 : }
12845 :
12846 224 : BIND(&if_otherhandler);
12847 : {
12848 : // Check if there's actually an {inst_of_handler}.
12849 224 : GotoIf(IsNull(inst_of_handler), &if_nohandler);
12850 224 : GotoIf(IsUndefined(inst_of_handler), &if_nohandler);
12851 :
12852 : // Call the {inst_of_handler} for {callable} and {object}.
12853 224 : Node* result = CallJS(
12854 448 : CodeFactory::Call(isolate(), ConvertReceiverMode::kNotNullOrUndefined),
12855 224 : context, inst_of_handler, callable, object);
12856 :
12857 : // Convert the {result} to a Boolean.
12858 224 : BranchIfToBooleanIsTrue(result, &return_true, &return_false);
12859 : }
12860 :
12861 224 : BIND(&if_nohandler);
12862 : {
12863 : // Ensure that the {callable} is actually Callable.
12864 224 : GotoIfNot(IsCallable(callable), &if_notcallable);
12865 :
12866 : // Use the OrdinaryHasInstance algorithm.
12867 : Node* result =
12868 224 : CallBuiltin(Builtins::kOrdinaryHasInstance, context, callable, object);
12869 224 : var_result.Bind(result);
12870 224 : Goto(&return_result);
12871 : }
12872 :
12873 224 : BIND(&if_notcallable);
12874 224 : { ThrowTypeError(context, MessageTemplate::kNonCallableInInstanceOfCheck); }
12875 :
12876 224 : BIND(&if_notreceiver);
12877 224 : { ThrowTypeError(context, MessageTemplate::kNonObjectInInstanceOfCheck); }
12878 :
12879 224 : BIND(&return_true);
12880 224 : var_result.Bind(TrueConstant());
12881 224 : Goto(&return_result);
12882 :
12883 224 : BIND(&return_false);
12884 224 : var_result.Bind(FalseConstant());
12885 224 : Goto(&return_result);
12886 :
12887 224 : BIND(&return_result);
12888 448 : return var_result.value();
12889 : }
12890 :
12891 1064 : TNode<Number> CodeStubAssembler::NumberInc(SloppyTNode<Number> value) {
12892 2128 : TVARIABLE(Number, var_result);
12893 2128 : TVARIABLE(Float64T, var_finc_value);
12894 2128 : Label if_issmi(this), if_isnotsmi(this), do_finc(this), end(this);
12895 1064 : Branch(TaggedIsSmi(value), &if_issmi, &if_isnotsmi);
12896 :
12897 1064 : BIND(&if_issmi);
12898 : {
12899 2128 : Label if_overflow(this);
12900 1064 : TNode<Smi> smi_value = CAST(value);
12901 1064 : TNode<Smi> one = SmiConstant(1);
12902 1064 : var_result = TrySmiAdd(smi_value, one, &if_overflow);
12903 1064 : Goto(&end);
12904 :
12905 1064 : BIND(&if_overflow);
12906 : {
12907 1064 : var_finc_value = SmiToFloat64(smi_value);
12908 1064 : Goto(&do_finc);
12909 : }
12910 : }
12911 :
12912 1064 : BIND(&if_isnotsmi);
12913 : {
12914 1064 : TNode<HeapNumber> heap_number_value = CAST(value);
12915 :
12916 : // Load the HeapNumber value.
12917 1064 : var_finc_value = LoadHeapNumberValue(heap_number_value);
12918 1064 : Goto(&do_finc);
12919 : }
12920 :
12921 1064 : BIND(&do_finc);
12922 : {
12923 1064 : TNode<Float64T> finc_value = var_finc_value.value();
12924 1064 : TNode<Float64T> one = Float64Constant(1.0);
12925 1064 : TNode<Float64T> finc_result = Float64Add(finc_value, one);
12926 1064 : var_result = AllocateHeapNumberWithValue(finc_result);
12927 1064 : Goto(&end);
12928 : }
12929 :
12930 1064 : BIND(&end);
12931 2128 : return var_result.value();
12932 : }
12933 :
12934 224 : TNode<Number> CodeStubAssembler::NumberDec(SloppyTNode<Number> value) {
12935 448 : TVARIABLE(Number, var_result);
12936 448 : TVARIABLE(Float64T, var_fdec_value);
12937 448 : Label if_issmi(this), if_isnotsmi(this), do_fdec(this), end(this);
12938 224 : Branch(TaggedIsSmi(value), &if_issmi, &if_isnotsmi);
12939 :
12940 224 : BIND(&if_issmi);
12941 : {
12942 224 : TNode<Smi> smi_value = CAST(value);
12943 224 : TNode<Smi> one = SmiConstant(1);
12944 448 : Label if_overflow(this);
12945 224 : var_result = TrySmiSub(smi_value, one, &if_overflow);
12946 224 : Goto(&end);
12947 :
12948 224 : BIND(&if_overflow);
12949 : {
12950 224 : var_fdec_value = SmiToFloat64(smi_value);
12951 224 : Goto(&do_fdec);
12952 : }
12953 : }
12954 :
12955 224 : BIND(&if_isnotsmi);
12956 : {
12957 224 : TNode<HeapNumber> heap_number_value = CAST(value);
12958 :
12959 : // Load the HeapNumber value.
12960 224 : var_fdec_value = LoadHeapNumberValue(heap_number_value);
12961 224 : Goto(&do_fdec);
12962 : }
12963 :
12964 224 : BIND(&do_fdec);
12965 : {
12966 224 : TNode<Float64T> fdec_value = var_fdec_value.value();
12967 224 : TNode<Float64T> minus_one = Float64Constant(-1.0);
12968 224 : TNode<Float64T> fdec_result = Float64Add(fdec_value, minus_one);
12969 224 : var_result = AllocateHeapNumberWithValue(fdec_result);
12970 224 : Goto(&end);
12971 : }
12972 :
12973 224 : BIND(&end);
12974 448 : return var_result.value();
12975 : }
12976 :
12977 2524 : TNode<Number> CodeStubAssembler::NumberAdd(SloppyTNode<Number> a,
12978 : SloppyTNode<Number> b) {
12979 5048 : TVARIABLE(Number, var_result);
12980 5048 : Label float_add(this, Label::kDeferred), end(this);
12981 2524 : GotoIf(TaggedIsNotSmi(a), &float_add);
12982 2524 : GotoIf(TaggedIsNotSmi(b), &float_add);
12983 :
12984 : // Try fast Smi addition first.
12985 2524 : var_result = TrySmiAdd(CAST(a), CAST(b), &float_add);
12986 2524 : Goto(&end);
12987 :
12988 2524 : BIND(&float_add);
12989 : {
12990 5048 : var_result = ChangeFloat64ToTagged(
12991 7572 : Float64Add(ChangeNumberToFloat64(a), ChangeNumberToFloat64(b)));
12992 2524 : Goto(&end);
12993 : }
12994 :
12995 2524 : BIND(&end);
12996 5048 : return var_result.value();
12997 : }
12998 :
12999 2076 : TNode<Number> CodeStubAssembler::NumberSub(SloppyTNode<Number> a,
13000 : SloppyTNode<Number> b) {
13001 4152 : TVARIABLE(Number, var_result);
13002 4152 : Label float_sub(this, Label::kDeferred), end(this);
13003 2076 : GotoIf(TaggedIsNotSmi(a), &float_sub);
13004 2076 : GotoIf(TaggedIsNotSmi(b), &float_sub);
13005 :
13006 : // Try fast Smi subtraction first.
13007 2076 : var_result = TrySmiSub(CAST(a), CAST(b), &float_sub);
13008 2076 : Goto(&end);
13009 :
13010 2076 : BIND(&float_sub);
13011 : {
13012 4152 : var_result = ChangeFloat64ToTagged(
13013 6228 : Float64Sub(ChangeNumberToFloat64(a), ChangeNumberToFloat64(b)));
13014 2076 : Goto(&end);
13015 : }
13016 :
13017 2076 : BIND(&end);
13018 4152 : return var_result.value();
13019 : }
13020 :
13021 236 : void CodeStubAssembler::GotoIfNotNumber(Node* input, Label* is_not_number) {
13022 472 : Label is_number(this);
13023 236 : GotoIf(TaggedIsSmi(input), &is_number);
13024 236 : Branch(IsHeapNumber(input), &is_number, is_not_number);
13025 236 : BIND(&is_number);
13026 236 : }
13027 :
13028 112 : void CodeStubAssembler::GotoIfNumber(Node* input, Label* is_number) {
13029 112 : GotoIf(TaggedIsSmi(input), is_number);
13030 112 : GotoIf(IsHeapNumber(input), is_number);
13031 112 : }
13032 :
13033 2352 : TNode<Number> CodeStubAssembler::BitwiseOp(Node* left32, Node* right32,
13034 : Operation bitwise_op) {
13035 2352 : switch (bitwise_op) {
13036 : case Operation::kBitwiseAnd:
13037 392 : return ChangeInt32ToTagged(Signed(Word32And(left32, right32)));
13038 : case Operation::kBitwiseOr:
13039 392 : return ChangeInt32ToTagged(Signed(Word32Or(left32, right32)));
13040 : case Operation::kBitwiseXor:
13041 392 : return ChangeInt32ToTagged(Signed(Word32Xor(left32, right32)));
13042 : case Operation::kShiftLeft:
13043 392 : if (!Word32ShiftIsSafe()) {
13044 0 : right32 = Word32And(right32, Int32Constant(0x1F));
13045 : }
13046 392 : return ChangeInt32ToTagged(Signed(Word32Shl(left32, right32)));
13047 : case Operation::kShiftRight:
13048 392 : if (!Word32ShiftIsSafe()) {
13049 0 : right32 = Word32And(right32, Int32Constant(0x1F));
13050 : }
13051 392 : return ChangeInt32ToTagged(Signed(Word32Sar(left32, right32)));
13052 : case Operation::kShiftRightLogical:
13053 392 : if (!Word32ShiftIsSafe()) {
13054 0 : right32 = Word32And(right32, Int32Constant(0x1F));
13055 : }
13056 392 : return ChangeUint32ToTagged(Unsigned(Word32Shr(left32, right32)));
13057 : default:
13058 0 : break;
13059 : }
13060 0 : UNREACHABLE();
13061 : }
13062 :
13063 : // ES #sec-createarrayiterator
13064 336 : TNode<JSArrayIterator> CodeStubAssembler::CreateArrayIterator(
13065 : TNode<Context> context, TNode<Object> object, IterationKind kind) {
13066 336 : TNode<Context> native_context = LoadNativeContext(context);
13067 336 : TNode<Map> iterator_map = CAST(LoadContextElement(
13068 : native_context, Context::INITIAL_ARRAY_ITERATOR_MAP_INDEX));
13069 336 : Node* iterator = Allocate(JSArrayIterator::kSize);
13070 336 : StoreMapNoWriteBarrier(iterator, iterator_map);
13071 : StoreObjectFieldRoot(iterator, JSArrayIterator::kPropertiesOrHashOffset,
13072 336 : RootIndex::kEmptyFixedArray);
13073 : StoreObjectFieldRoot(iterator, JSArrayIterator::kElementsOffset,
13074 336 : RootIndex::kEmptyFixedArray);
13075 336 : StoreObjectFieldNoWriteBarrier(
13076 336 : iterator, JSArrayIterator::kIteratedObjectOffset, object);
13077 336 : StoreObjectFieldNoWriteBarrier(iterator, JSArrayIterator::kNextIndexOffset,
13078 672 : SmiConstant(0));
13079 336 : StoreObjectFieldNoWriteBarrier(
13080 : iterator, JSArrayIterator::kKindOffset,
13081 672 : SmiConstant(Smi::FromInt(static_cast<int>(kind))));
13082 336 : return CAST(iterator);
13083 : }
13084 :
13085 336 : Node* CodeStubAssembler::AllocateJSIteratorResult(Node* context, Node* value,
13086 : Node* done) {
13087 : CSA_ASSERT(this, IsBoolean(done));
13088 336 : Node* native_context = LoadNativeContext(context);
13089 : Node* map =
13090 336 : LoadContextElement(native_context, Context::ITERATOR_RESULT_MAP_INDEX);
13091 336 : Node* result = Allocate(JSIteratorResult::kSize);
13092 336 : StoreMapNoWriteBarrier(result, map);
13093 : StoreObjectFieldRoot(result, JSIteratorResult::kPropertiesOrHashOffset,
13094 336 : RootIndex::kEmptyFixedArray);
13095 : StoreObjectFieldRoot(result, JSIteratorResult::kElementsOffset,
13096 336 : RootIndex::kEmptyFixedArray);
13097 336 : StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kValueOffset, value);
13098 336 : StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kDoneOffset, done);
13099 336 : return result;
13100 : }
13101 :
13102 168 : Node* CodeStubAssembler::AllocateJSIteratorResultForEntry(Node* context,
13103 : Node* key,
13104 : Node* value) {
13105 168 : Node* native_context = LoadNativeContext(context);
13106 168 : Node* length = SmiConstant(2);
13107 168 : int const elements_size = FixedArray::SizeFor(2);
13108 : TNode<FixedArray> elements = UncheckedCast<FixedArray>(
13109 168 : Allocate(elements_size + JSArray::kSize + JSIteratorResult::kSize));
13110 168 : StoreObjectFieldRoot(elements, FixedArray::kMapOffset,
13111 168 : RootIndex::kFixedArrayMap);
13112 168 : StoreObjectFieldNoWriteBarrier(elements, FixedArray::kLengthOffset, length);
13113 168 : StoreFixedArrayElement(elements, 0, key);
13114 168 : StoreFixedArrayElement(elements, 1, value);
13115 336 : Node* array_map = LoadContextElement(
13116 504 : native_context, Context::JS_ARRAY_PACKED_ELEMENTS_MAP_INDEX);
13117 168 : TNode<HeapObject> array = InnerAllocate(elements, elements_size);
13118 168 : StoreMapNoWriteBarrier(array, array_map);
13119 168 : StoreObjectFieldRoot(array, JSArray::kPropertiesOrHashOffset,
13120 168 : RootIndex::kEmptyFixedArray);
13121 168 : StoreObjectFieldNoWriteBarrier(array, JSArray::kElementsOffset, elements);
13122 168 : StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
13123 : Node* iterator_map =
13124 168 : LoadContextElement(native_context, Context::ITERATOR_RESULT_MAP_INDEX);
13125 168 : TNode<HeapObject> result = InnerAllocate(array, JSArray::kSize);
13126 168 : StoreMapNoWriteBarrier(result, iterator_map);
13127 168 : StoreObjectFieldRoot(result, JSIteratorResult::kPropertiesOrHashOffset,
13128 168 : RootIndex::kEmptyFixedArray);
13129 168 : StoreObjectFieldRoot(result, JSIteratorResult::kElementsOffset,
13130 168 : RootIndex::kEmptyFixedArray);
13131 168 : StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kValueOffset, array);
13132 168 : StoreObjectFieldRoot(result, JSIteratorResult::kDoneOffset,
13133 168 : RootIndex::kFalseValue);
13134 168 : return result;
13135 : }
13136 :
13137 224 : TNode<JSReceiver> CodeStubAssembler::ArraySpeciesCreate(TNode<Context> context,
13138 : TNode<Object> o,
13139 : TNode<Number> len) {
13140 : TNode<JSReceiver> constructor =
13141 224 : CAST(CallRuntime(Runtime::kArraySpeciesConstructor, context, o));
13142 224 : return Construct(context, constructor, len);
13143 : }
13144 :
13145 9636 : Node* CodeStubAssembler::IsDetachedBuffer(Node* buffer) {
13146 : CSA_ASSERT(this, HasInstanceType(buffer, JS_ARRAY_BUFFER_TYPE));
13147 9636 : TNode<Uint32T> buffer_bit_field = LoadJSArrayBufferBitField(CAST(buffer));
13148 9636 : return IsSetWord32<JSArrayBuffer::WasDetachedBit>(buffer_bit_field);
13149 : }
13150 :
13151 952 : void CodeStubAssembler::ThrowIfArrayBufferIsDetached(
13152 : SloppyTNode<Context> context, TNode<JSArrayBuffer> array_buffer,
13153 : const char* method_name) {
13154 1904 : Label if_detached(this, Label::kDeferred), if_not_detached(this);
13155 952 : Branch(IsDetachedBuffer(array_buffer), &if_detached, &if_not_detached);
13156 952 : BIND(&if_detached);
13157 952 : ThrowTypeError(context, MessageTemplate::kDetachedOperation, method_name);
13158 952 : BIND(&if_not_detached);
13159 952 : }
13160 :
13161 896 : void CodeStubAssembler::ThrowIfArrayBufferViewBufferIsDetached(
13162 : SloppyTNode<Context> context, TNode<JSArrayBufferView> array_buffer_view,
13163 : const char* method_name) {
13164 896 : TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(array_buffer_view);
13165 896 : ThrowIfArrayBufferIsDetached(context, buffer, method_name);
13166 896 : }
13167 :
13168 10140 : TNode<Uint32T> CodeStubAssembler::LoadJSArrayBufferBitField(
13169 : TNode<JSArrayBuffer> array_buffer) {
13170 10140 : return LoadObjectField<Uint32T>(array_buffer, JSArrayBuffer::kBitFieldOffset);
13171 : }
13172 :
13173 504 : TNode<RawPtrT> CodeStubAssembler::LoadJSArrayBufferBackingStore(
13174 : TNode<JSArrayBuffer> array_buffer) {
13175 : return LoadObjectField<RawPtrT>(array_buffer,
13176 504 : JSArrayBuffer::kBackingStoreOffset);
13177 : }
13178 :
13179 1680 : TNode<JSArrayBuffer> CodeStubAssembler::LoadJSArrayBufferViewBuffer(
13180 : TNode<JSArrayBufferView> array_buffer_view) {
13181 : return LoadObjectField<JSArrayBuffer>(array_buffer_view,
13182 1680 : JSArrayBufferView::kBufferOffset);
13183 : }
13184 :
13185 56 : TNode<UintPtrT> CodeStubAssembler::LoadJSArrayBufferViewByteLength(
13186 : TNode<JSArrayBufferView> array_buffer_view) {
13187 : return LoadObjectField<UintPtrT>(array_buffer_view,
13188 56 : JSArrayBufferView::kByteLengthOffset);
13189 : }
13190 :
13191 560 : TNode<UintPtrT> CodeStubAssembler::LoadJSArrayBufferViewByteOffset(
13192 : TNode<JSArrayBufferView> array_buffer_view) {
13193 : return LoadObjectField<UintPtrT>(array_buffer_view,
13194 560 : JSArrayBufferView::kByteOffsetOffset);
13195 : }
13196 :
13197 5044 : TNode<Smi> CodeStubAssembler::LoadJSTypedArrayLength(
13198 : TNode<JSTypedArray> typed_array) {
13199 5044 : return LoadObjectField<Smi>(typed_array, JSTypedArray::kLengthOffset);
13200 : }
13201 :
13202 7780 : CodeStubArguments::CodeStubArguments(
13203 : CodeStubAssembler* assembler, Node* argc, Node* fp,
13204 : CodeStubAssembler::ParameterMode param_mode, ReceiverMode receiver_mode)
13205 : : assembler_(assembler),
13206 : argc_mode_(param_mode),
13207 : receiver_mode_(receiver_mode),
13208 : argc_(argc),
13209 : arguments_(),
13210 7780 : fp_(fp != nullptr ? fp : assembler_->LoadFramePointer()) {
13211 15560 : Node* offset = assembler_->ElementOffsetFromIndex(
13212 : argc_, SYSTEM_POINTER_ELEMENTS, param_mode,
13213 : (StandardFrameConstants::kFixedSlotCountAboveFp - 1) *
13214 15560 : kSystemPointerSize);
13215 31120 : arguments_ =
13216 23340 : assembler_->UncheckedCast<WordT>(assembler_->IntPtrAdd(fp_, offset));
13217 7780 : }
13218 :
13219 6508 : TNode<Object> CodeStubArguments::GetReceiver() const {
13220 : DCHECK_EQ(receiver_mode_, ReceiverMode::kHasReceiver);
13221 13016 : return assembler_->UncheckedCast<Object>(assembler_->LoadFullTagged(
13222 19524 : arguments_, assembler_->IntPtrConstant(kSystemPointerSize)));
13223 : }
13224 :
13225 224 : void CodeStubArguments::SetReceiver(TNode<Object> object) const {
13226 : DCHECK_EQ(receiver_mode_, ReceiverMode::kHasReceiver);
13227 448 : assembler_->StoreFullTaggedNoWriteBarrier(
13228 448 : arguments_, assembler_->IntPtrConstant(kSystemPointerSize), object);
13229 224 : }
13230 :
13231 12432 : TNode<WordT> CodeStubArguments::AtIndexPtr(
13232 : Node* index, CodeStubAssembler::ParameterMode mode) const {
13233 : typedef compiler::Node Node;
13234 12432 : Node* negated_index = assembler_->IntPtrOrSmiSub(
13235 24864 : assembler_->IntPtrOrSmiConstant(0, mode), index, mode);
13236 24864 : Node* offset = assembler_->ElementOffsetFromIndex(
13237 12432 : negated_index, SYSTEM_POINTER_ELEMENTS, mode, 0);
13238 37296 : return assembler_->IntPtrAdd(assembler_->UncheckedCast<IntPtrT>(arguments_),
13239 49728 : offset);
13240 : }
13241 :
13242 12376 : TNode<Object> CodeStubArguments::AtIndex(
13243 : Node* index, CodeStubAssembler::ParameterMode mode) const {
13244 : DCHECK_EQ(argc_mode_, mode);
13245 : CSA_ASSERT(assembler_,
13246 : assembler_->UintPtrOrSmiLessThan(index, GetLength(mode), mode));
13247 12376 : return assembler_->UncheckedCast<Object>(
13248 24752 : assembler_->LoadFullTagged(AtIndexPtr(index, mode)));
13249 : }
13250 :
13251 3920 : TNode<Object> CodeStubArguments::AtIndex(int index) const {
13252 3920 : return AtIndex(assembler_->IntPtrConstant(index));
13253 : }
13254 :
13255 2968 : TNode<Object> CodeStubArguments::GetOptionalArgumentValue(
13256 : int index, TNode<Object> default_value) {
13257 5936 : CodeStubAssembler::TVariable<Object> result(assembler_);
13258 5936 : CodeStubAssembler::Label argument_missing(assembler_),
13259 5936 : argument_done(assembler_, &result);
13260 :
13261 8904 : assembler_->GotoIf(assembler_->UintPtrOrSmiGreaterThanOrEqual(
13262 2968 : assembler_->IntPtrOrSmiConstant(index, argc_mode_),
13263 : argc_, argc_mode_),
13264 2968 : &argument_missing);
13265 2968 : result = AtIndex(index);
13266 2968 : assembler_->Goto(&argument_done);
13267 :
13268 2968 : assembler_->BIND(&argument_missing);
13269 2968 : result = default_value;
13270 2968 : assembler_->Goto(&argument_done);
13271 :
13272 2968 : assembler_->BIND(&argument_done);
13273 5936 : return result.value();
13274 : }
13275 :
13276 7672 : TNode<Object> CodeStubArguments::GetOptionalArgumentValue(
13277 : TNode<IntPtrT> index, TNode<Object> default_value) {
13278 15344 : CodeStubAssembler::TVariable<Object> result(assembler_);
13279 15344 : CodeStubAssembler::Label argument_missing(assembler_),
13280 15344 : argument_done(assembler_, &result);
13281 :
13282 30688 : assembler_->GotoIf(
13283 7672 : assembler_->UintPtrOrSmiGreaterThanOrEqual(
13284 7672 : assembler_->IntPtrToParameter(index, argc_mode_), argc_, argc_mode_),
13285 7672 : &argument_missing);
13286 7672 : result = AtIndex(index);
13287 7672 : assembler_->Goto(&argument_done);
13288 :
13289 7672 : assembler_->BIND(&argument_missing);
13290 7672 : result = default_value;
13291 7672 : assembler_->Goto(&argument_done);
13292 :
13293 7672 : assembler_->BIND(&argument_done);
13294 15344 : return result.value();
13295 : }
13296 :
13297 1048 : void CodeStubArguments::ForEach(
13298 : const CodeStubAssembler::VariableList& vars,
13299 : const CodeStubArguments::ForEachBodyFunction& body, Node* first, Node* last,
13300 : CodeStubAssembler::ParameterMode mode) {
13301 1048 : assembler_->Comment("CodeStubArguments::ForEach");
13302 1048 : if (first == nullptr) {
13303 452 : first = assembler_->IntPtrOrSmiConstant(0, mode);
13304 : }
13305 1048 : if (last == nullptr) {
13306 : DCHECK_EQ(mode, argc_mode_);
13307 1048 : last = argc_;
13308 : }
13309 3144 : Node* start = assembler_->IntPtrSub(
13310 1048 : assembler_->UncheckedCast<IntPtrT>(arguments_),
13311 4192 : assembler_->ElementOffsetFromIndex(first, SYSTEM_POINTER_ELEMENTS, mode));
13312 3144 : Node* end = assembler_->IntPtrSub(
13313 1048 : assembler_->UncheckedCast<IntPtrT>(arguments_),
13314 4192 : assembler_->ElementOffsetFromIndex(last, SYSTEM_POINTER_ELEMENTS, mode));
13315 2096 : assembler_->BuildFastLoop(
13316 : vars, start, end,
13317 3144 : [this, &body](Node* current) {
13318 2096 : Node* arg = assembler_->Load(MachineType::AnyTagged(), current);
13319 1048 : body(arg);
13320 1048 : },
13321 : -kSystemPointerSize, CodeStubAssembler::INTPTR_PARAMETERS,
13322 1048 : CodeStubAssembler::IndexAdvanceMode::kPost);
13323 1048 : }
13324 :
13325 12728 : void CodeStubArguments::PopAndReturn(Node* value) {
13326 : Node* pop_count;
13327 12728 : if (receiver_mode_ == ReceiverMode::kHasReceiver) {
13328 25456 : pop_count = assembler_->IntPtrOrSmiAdd(
13329 25456 : argc_, assembler_->IntPtrOrSmiConstant(1, argc_mode_), argc_mode_);
13330 : } else {
13331 0 : pop_count = argc_;
13332 : }
13333 :
13334 25456 : assembler_->PopAndReturn(assembler_->ParameterToIntPtr(pop_count, argc_mode_),
13335 12728 : value);
13336 12728 : }
13337 :
13338 3472 : Node* CodeStubAssembler::IsFastElementsKind(Node* elements_kind) {
13339 : STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
13340 6944 : return Uint32LessThanOrEqual(elements_kind,
13341 10416 : Int32Constant(LAST_FAST_ELEMENTS_KIND));
13342 : }
13343 :
13344 284 : TNode<BoolT> CodeStubAssembler::IsDoubleElementsKind(
13345 : TNode<Int32T> elements_kind) {
13346 : STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
13347 : STATIC_ASSERT((PACKED_DOUBLE_ELEMENTS & 1) == 0);
13348 : STATIC_ASSERT(PACKED_DOUBLE_ELEMENTS + 1 == HOLEY_DOUBLE_ELEMENTS);
13349 568 : return Word32Equal(Word32Shr(elements_kind, Int32Constant(1)),
13350 852 : Int32Constant(PACKED_DOUBLE_ELEMENTS / 2));
13351 : }
13352 :
13353 336 : Node* CodeStubAssembler::IsFastSmiOrTaggedElementsKind(Node* elements_kind) {
13354 : STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
13355 : STATIC_ASSERT(PACKED_DOUBLE_ELEMENTS > TERMINAL_FAST_ELEMENTS_KIND);
13356 : STATIC_ASSERT(HOLEY_DOUBLE_ELEMENTS > TERMINAL_FAST_ELEMENTS_KIND);
13357 672 : return Uint32LessThanOrEqual(elements_kind,
13358 1008 : Int32Constant(TERMINAL_FAST_ELEMENTS_KIND));
13359 : }
13360 :
13361 112 : Node* CodeStubAssembler::IsFastSmiElementsKind(Node* elements_kind) {
13362 224 : return Uint32LessThanOrEqual(elements_kind,
13363 336 : Int32Constant(HOLEY_SMI_ELEMENTS));
13364 : }
13365 :
13366 56 : Node* CodeStubAssembler::IsHoleyFastElementsKind(Node* elements_kind) {
13367 : CSA_ASSERT(this, IsFastElementsKind(elements_kind));
13368 :
13369 : STATIC_ASSERT(HOLEY_SMI_ELEMENTS == (PACKED_SMI_ELEMENTS | 1));
13370 : STATIC_ASSERT(HOLEY_ELEMENTS == (PACKED_ELEMENTS | 1));
13371 : STATIC_ASSERT(HOLEY_DOUBLE_ELEMENTS == (PACKED_DOUBLE_ELEMENTS | 1));
13372 56 : return IsSetWord32(elements_kind, 1);
13373 : }
13374 :
13375 728 : Node* CodeStubAssembler::IsElementsKindGreaterThan(
13376 : Node* target_kind, ElementsKind reference_kind) {
13377 728 : return Int32GreaterThan(target_kind, Int32Constant(reference_kind));
13378 : }
13379 :
13380 1456 : TNode<BoolT> CodeStubAssembler::IsElementsKindLessThanOrEqual(
13381 : TNode<Int32T> target_kind, ElementsKind reference_kind) {
13382 1456 : return Int32LessThanOrEqual(target_kind, Int32Constant(reference_kind));
13383 : }
13384 :
13385 396 : Node* CodeStubAssembler::IsDebugActive() {
13386 396 : Node* is_debug_active = Load(
13387 : MachineType::Uint8(),
13388 792 : ExternalConstant(ExternalReference::debug_is_active_address(isolate())));
13389 396 : return Word32NotEqual(is_debug_active, Int32Constant(0));
13390 : }
13391 :
13392 2576 : TNode<BoolT> CodeStubAssembler::IsRuntimeCallStatsEnabled() {
13393 : STATIC_ASSERT(sizeof(TracingFlags::runtime_stats) == kInt32Size);
13394 : TNode<Word32T> flag_value = UncheckedCast<Word32T>(Load(
13395 : MachineType::Int32(),
13396 2576 : ExternalConstant(ExternalReference::address_of_runtime_stats_flag())));
13397 2576 : return Word32NotEqual(flag_value, Int32Constant(0));
13398 : }
13399 :
13400 56 : Node* CodeStubAssembler::IsPromiseHookEnabled() {
13401 56 : Node* const promise_hook = Load(
13402 : MachineType::Pointer(),
13403 112 : ExternalConstant(ExternalReference::promise_hook_address(isolate())));
13404 56 : return WordNotEqual(promise_hook, IntPtrConstant(0));
13405 : }
13406 :
13407 224 : Node* CodeStubAssembler::HasAsyncEventDelegate() {
13408 : Node* const async_event_delegate =
13409 224 : Load(MachineType::Pointer(),
13410 448 : ExternalConstant(
13411 448 : ExternalReference::async_event_delegate_address(isolate())));
13412 224 : return WordNotEqual(async_event_delegate, IntPtrConstant(0));
13413 : }
13414 :
13415 756 : Node* CodeStubAssembler::IsPromiseHookEnabledOrHasAsyncEventDelegate() {
13416 : Node* const promise_hook_or_async_event_delegate =
13417 756 : Load(MachineType::Uint8(),
13418 1512 : ExternalConstant(
13419 : ExternalReference::promise_hook_or_async_event_delegate_address(
13420 1512 : isolate())));
13421 756 : return Word32NotEqual(promise_hook_or_async_event_delegate, Int32Constant(0));
13422 : }
13423 :
13424 1176 : Node* CodeStubAssembler::
13425 : IsPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate() {
13426 1176 : Node* const promise_hook_or_debug_is_active_or_async_event_delegate = Load(
13427 : MachineType::Uint8(),
13428 2352 : ExternalConstant(
13429 : ExternalReference::
13430 : promise_hook_or_debug_is_active_or_async_event_delegate_address(
13431 2352 : isolate())));
13432 2352 : return Word32NotEqual(promise_hook_or_debug_is_active_or_async_event_delegate,
13433 3528 : Int32Constant(0));
13434 : }
13435 :
13436 2364 : TNode<Code> CodeStubAssembler::LoadBuiltin(TNode<Smi> builtin_id) {
13437 : CSA_ASSERT(this, SmiGreaterThanOrEqual(builtin_id, SmiConstant(0)));
13438 : CSA_ASSERT(this,
13439 : SmiLessThan(builtin_id, SmiConstant(Builtins::builtin_count)));
13440 :
13441 2364 : int const kSmiShiftBits = kSmiShiftSize + kSmiTagSize;
13442 2364 : int index_shift = kSystemPointerSizeLog2 - kSmiShiftBits;
13443 : TNode<WordT> table_index =
13444 9456 : index_shift >= 0 ? WordShl(BitcastTaggedToWord(builtin_id), index_shift)
13445 7092 : : WordSar(BitcastTaggedToWord(builtin_id), -index_shift);
13446 :
13447 2364 : return CAST(
13448 : Load(MachineType::TaggedPointer(),
13449 : ExternalConstant(ExternalReference::builtins_address(isolate())),
13450 : table_index));
13451 : }
13452 :
13453 1020 : TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode(
13454 : SloppyTNode<SharedFunctionInfo> shared_info, Label* if_compile_lazy) {
13455 : TNode<Object> sfi_data =
13456 1020 : LoadObjectField(shared_info, SharedFunctionInfo::kFunctionDataOffset);
13457 :
13458 2040 : TVARIABLE(Code, sfi_code);
13459 :
13460 2040 : Label done(this);
13461 2040 : Label check_instance_type(this);
13462 :
13463 : // IsSmi: Is builtin
13464 1020 : GotoIf(TaggedIsNotSmi(sfi_data), &check_instance_type);
13465 1020 : if (if_compile_lazy) {
13466 112 : GotoIf(SmiEqual(CAST(sfi_data), SmiConstant(Builtins::kCompileLazy)),
13467 56 : if_compile_lazy);
13468 : }
13469 1020 : sfi_code = LoadBuiltin(CAST(sfi_data));
13470 1020 : Goto(&done);
13471 :
13472 : // Switch on data's instance type.
13473 1020 : BIND(&check_instance_type);
13474 1020 : TNode<Int32T> data_type = LoadInstanceType(CAST(sfi_data));
13475 :
13476 : int32_t case_values[] = {BYTECODE_ARRAY_TYPE,
13477 : WASM_EXPORTED_FUNCTION_DATA_TYPE,
13478 : ASM_WASM_DATA_TYPE,
13479 : UNCOMPILED_DATA_WITHOUT_PREPARSE_DATA_TYPE,
13480 : UNCOMPILED_DATA_WITH_PREPARSE_DATA_TYPE,
13481 1020 : FUNCTION_TEMPLATE_INFO_TYPE};
13482 2040 : Label check_is_bytecode_array(this);
13483 2040 : Label check_is_exported_function_data(this);
13484 2040 : Label check_is_asm_wasm_data(this);
13485 2040 : Label check_is_uncompiled_data_without_preparse_data(this);
13486 2040 : Label check_is_uncompiled_data_with_preparse_data(this);
13487 2040 : Label check_is_function_template_info(this);
13488 2040 : Label check_is_interpreter_data(this);
13489 : Label* case_labels[] = {&check_is_bytecode_array,
13490 : &check_is_exported_function_data,
13491 : &check_is_asm_wasm_data,
13492 : &check_is_uncompiled_data_without_preparse_data,
13493 : &check_is_uncompiled_data_with_preparse_data,
13494 1020 : &check_is_function_template_info};
13495 : STATIC_ASSERT(arraysize(case_values) == arraysize(case_labels));
13496 1020 : Switch(data_type, &check_is_interpreter_data, case_values, case_labels,
13497 1020 : arraysize(case_labels));
13498 :
13499 : // IsBytecodeArray: Interpret bytecode
13500 1020 : BIND(&check_is_bytecode_array);
13501 1020 : sfi_code = HeapConstant(BUILTIN_CODE(isolate(), InterpreterEntryTrampoline));
13502 1020 : Goto(&done);
13503 :
13504 : // IsWasmExportedFunctionData: Use the wrapper code
13505 1020 : BIND(&check_is_exported_function_data);
13506 2040 : sfi_code = CAST(LoadObjectField(
13507 1020 : CAST(sfi_data), WasmExportedFunctionData::kWrapperCodeOffset));
13508 1020 : Goto(&done);
13509 :
13510 : // IsAsmWasmData: Instantiate using AsmWasmData
13511 1020 : BIND(&check_is_asm_wasm_data);
13512 1020 : sfi_code = HeapConstant(BUILTIN_CODE(isolate(), InstantiateAsmJs));
13513 1020 : Goto(&done);
13514 :
13515 : // IsUncompiledDataWithPreparseData | IsUncompiledDataWithoutPreparseData:
13516 : // Compile lazy
13517 1020 : BIND(&check_is_uncompiled_data_with_preparse_data);
13518 1020 : Goto(&check_is_uncompiled_data_without_preparse_data);
13519 1020 : BIND(&check_is_uncompiled_data_without_preparse_data);
13520 1020 : sfi_code = HeapConstant(BUILTIN_CODE(isolate(), CompileLazy));
13521 1020 : Goto(if_compile_lazy ? if_compile_lazy : &done);
13522 :
13523 : // IsFunctionTemplateInfo: API call
13524 1020 : BIND(&check_is_function_template_info);
13525 1020 : sfi_code = HeapConstant(BUILTIN_CODE(isolate(), HandleApiCall));
13526 1020 : Goto(&done);
13527 :
13528 : // IsInterpreterData: Interpret bytecode
13529 1020 : BIND(&check_is_interpreter_data);
13530 : // This is the default branch, so assert that we have the expected data type.
13531 : CSA_ASSERT(this,
13532 : Word32Equal(data_type, Int32Constant(INTERPRETER_DATA_TYPE)));
13533 2040 : sfi_code = CAST(LoadObjectField(
13534 1020 : CAST(sfi_data), InterpreterData::kInterpreterTrampolineOffset));
13535 1020 : Goto(&done);
13536 :
13537 1020 : BIND(&done);
13538 2040 : return sfi_code.value();
13539 : }
13540 :
13541 908 : Node* CodeStubAssembler::AllocateFunctionWithMapAndContext(Node* map,
13542 : Node* shared_info,
13543 : Node* context) {
13544 : CSA_SLOW_ASSERT(this, IsMap(map));
13545 :
13546 908 : Node* const code = GetSharedFunctionInfoCode(shared_info);
13547 :
13548 : // TODO(ishell): All the callers of this function pass map loaded from
13549 : // Context::STRICT_FUNCTION_WITHOUT_PROTOTYPE_MAP_INDEX. So we can remove
13550 : // map parameter.
13551 : CSA_ASSERT(this, Word32BinaryNot(IsConstructorMap(map)));
13552 : CSA_ASSERT(this, Word32BinaryNot(IsFunctionWithPrototypeSlotMap(map)));
13553 908 : Node* const fun = Allocate(JSFunction::kSizeWithoutPrototype);
13554 : STATIC_ASSERT(JSFunction::kSizeWithoutPrototype == 7 * kTaggedSize);
13555 908 : StoreMapNoWriteBarrier(fun, map);
13556 : StoreObjectFieldRoot(fun, JSObject::kPropertiesOrHashOffset,
13557 908 : RootIndex::kEmptyFixedArray);
13558 : StoreObjectFieldRoot(fun, JSObject::kElementsOffset,
13559 908 : RootIndex::kEmptyFixedArray);
13560 : StoreObjectFieldRoot(fun, JSFunction::kFeedbackCellOffset,
13561 908 : RootIndex::kManyClosuresCell);
13562 : StoreObjectFieldNoWriteBarrier(fun, JSFunction::kSharedFunctionInfoOffset,
13563 908 : shared_info);
13564 908 : StoreObjectFieldNoWriteBarrier(fun, JSFunction::kContextOffset, context);
13565 908 : StoreObjectFieldNoWriteBarrier(fun, JSFunction::kCodeOffset, code);
13566 908 : return fun;
13567 : }
13568 :
13569 0 : Node* CodeStubAssembler::MarkerIsFrameType(Node* marker_or_function,
13570 : StackFrame::Type frame_type) {
13571 0 : return WordEqual(marker_or_function,
13572 0 : IntPtrConstant(StackFrame::TypeToMarker(frame_type)));
13573 : }
13574 :
13575 0 : Node* CodeStubAssembler::MarkerIsNotFrameType(Node* marker_or_function,
13576 : StackFrame::Type frame_type) {
13577 0 : return WordNotEqual(marker_or_function,
13578 0 : IntPtrConstant(StackFrame::TypeToMarker(frame_type)));
13579 : }
13580 :
13581 448 : void CodeStubAssembler::CheckPrototypeEnumCache(Node* receiver,
13582 : Node* receiver_map,
13583 : Label* if_fast,
13584 : Label* if_slow) {
13585 896 : VARIABLE(var_object, MachineRepresentation::kTagged, receiver);
13586 896 : VARIABLE(var_object_map, MachineRepresentation::kTagged, receiver_map);
13587 :
13588 896 : Label loop(this, {&var_object, &var_object_map}), done_loop(this);
13589 448 : Goto(&loop);
13590 448 : BIND(&loop);
13591 : {
13592 : // Check that there are no elements on the current {object}.
13593 896 : Label if_no_elements(this);
13594 448 : Node* object = var_object.value();
13595 448 : Node* object_map = var_object_map.value();
13596 :
13597 : // The following relies on the elements only aliasing with JSProxy::target,
13598 : // which is a Javascript value and hence cannot be confused with an elements
13599 : // backing store.
13600 : STATIC_ASSERT(static_cast<int>(JSObject::kElementsOffset) ==
13601 : static_cast<int>(JSProxy::kTargetOffset));
13602 448 : Node* object_elements = LoadObjectField(object, JSObject::kElementsOffset);
13603 448 : GotoIf(IsEmptyFixedArray(object_elements), &if_no_elements);
13604 448 : GotoIf(IsEmptySlowElementDictionary(object_elements), &if_no_elements);
13605 :
13606 : // It might still be an empty JSArray.
13607 448 : GotoIfNot(IsJSArrayMap(object_map), if_slow);
13608 448 : Node* object_length = LoadJSArrayLength(object);
13609 448 : Branch(WordEqual(object_length, SmiConstant(0)), &if_no_elements, if_slow);
13610 :
13611 : // Continue with the {object}s prototype.
13612 448 : BIND(&if_no_elements);
13613 448 : object = LoadMapPrototype(object_map);
13614 448 : GotoIf(IsNull(object), if_fast);
13615 :
13616 : // For all {object}s but the {receiver}, check that the cache is empty.
13617 448 : var_object.Bind(object);
13618 448 : object_map = LoadMap(object);
13619 448 : var_object_map.Bind(object_map);
13620 448 : Node* object_enum_length = LoadMapEnumLength(object_map);
13621 448 : Branch(WordEqual(object_enum_length, IntPtrConstant(0)), &loop, if_slow);
13622 : }
13623 448 : }
13624 :
13625 224 : Node* CodeStubAssembler::CheckEnumCache(Node* receiver, Label* if_empty,
13626 : Label* if_runtime) {
13627 448 : Label if_fast(this), if_cache(this), if_no_cache(this, Label::kDeferred);
13628 224 : Node* receiver_map = LoadMap(receiver);
13629 :
13630 : // Check if the enum length field of the {receiver} is properly initialized,
13631 : // indicating that there is an enum cache.
13632 224 : Node* receiver_enum_length = LoadMapEnumLength(receiver_map);
13633 448 : Branch(WordEqual(receiver_enum_length,
13634 448 : IntPtrConstant(kInvalidEnumCacheSentinel)),
13635 224 : &if_no_cache, &if_cache);
13636 :
13637 224 : BIND(&if_no_cache);
13638 : {
13639 : // Avoid runtime-call for empty dictionary receivers.
13640 224 : GotoIfNot(IsDictionaryMap(receiver_map), if_runtime);
13641 224 : TNode<NameDictionary> properties = CAST(LoadSlowProperties(receiver));
13642 224 : TNode<Smi> length = GetNumberOfElements(properties);
13643 224 : GotoIfNot(WordEqual(length, SmiConstant(0)), if_runtime);
13644 : // Check that there are no elements on the {receiver} and its prototype
13645 : // chain. Given that we do not create an EnumCache for dict-mode objects,
13646 : // directly jump to {if_empty} if there are no elements and no properties
13647 : // on the {receiver}.
13648 224 : CheckPrototypeEnumCache(receiver, receiver_map, if_empty, if_runtime);
13649 : }
13650 :
13651 : // Check that there are no elements on the fast {receiver} and its
13652 : // prototype chain.
13653 224 : BIND(&if_cache);
13654 224 : CheckPrototypeEnumCache(receiver, receiver_map, &if_fast, if_runtime);
13655 :
13656 224 : BIND(&if_fast);
13657 448 : return receiver_map;
13658 : }
13659 :
13660 4536 : TNode<IntPtrT> CodeStubAssembler::GetArgumentsLength(CodeStubArguments* args) {
13661 4536 : return args->GetLength();
13662 : }
13663 :
13664 7672 : TNode<Object> CodeStubAssembler::GetArgumentValue(CodeStubArguments* args,
13665 : TNode<IntPtrT> index) {
13666 7672 : return args->GetOptionalArgumentValue(index);
13667 : }
13668 :
13669 0 : void CodeStubAssembler::Print(const char* s) {
13670 0 : std::string formatted(s);
13671 0 : formatted += "\n";
13672 0 : CallRuntime(Runtime::kGlobalPrint, NoContextConstant(),
13673 0 : StringConstant(formatted.c_str()));
13674 0 : }
13675 :
13676 0 : void CodeStubAssembler::Print(const char* prefix, Node* tagged_value) {
13677 0 : if (prefix != nullptr) {
13678 0 : std::string formatted(prefix);
13679 0 : formatted += ": ";
13680 : Handle<String> string = isolate()->factory()->NewStringFromAsciiChecked(
13681 0 : formatted.c_str(), AllocationType::kOld);
13682 0 : CallRuntime(Runtime::kGlobalPrint, NoContextConstant(),
13683 0 : HeapConstant(string));
13684 : }
13685 0 : CallRuntime(Runtime::kDebugPrint, NoContextConstant(), tagged_value);
13686 0 : }
13687 :
13688 19264 : void CodeStubAssembler::PerformStackCheck(TNode<Context> context) {
13689 38528 : Label ok(this), stack_check_interrupt(this, Label::kDeferred);
13690 :
13691 : // The instruction sequence below is carefully crafted to hit our pattern
13692 : // matcher for stack checks within instruction selection.
13693 : // See StackCheckMatcher::Matched and JSGenericLowering::LowerJSStackCheck.
13694 :
13695 19264 : TNode<UintPtrT> sp = UncheckedCast<UintPtrT>(LoadStackPointer());
13696 : TNode<UintPtrT> stack_limit = UncheckedCast<UintPtrT>(Load(
13697 : MachineType::Pointer(),
13698 19264 : ExternalConstant(ExternalReference::address_of_stack_limit(isolate()))));
13699 19264 : TNode<BoolT> sp_within_limit = UintPtrLessThan(stack_limit, sp);
13700 :
13701 19264 : Branch(sp_within_limit, &ok, &stack_check_interrupt);
13702 :
13703 19264 : BIND(&stack_check_interrupt);
13704 19264 : CallRuntime(Runtime::kStackGuard, context);
13705 19264 : Goto(&ok);
13706 :
13707 19264 : BIND(&ok);
13708 19264 : }
13709 :
13710 688 : void CodeStubAssembler::InitializeFunctionContext(Node* native_context,
13711 : Node* context, int slots) {
13712 : DCHECK_GE(slots, Context::MIN_CONTEXT_SLOTS);
13713 688 : StoreMapNoWriteBarrier(context, RootIndex::kFunctionContextMap);
13714 688 : StoreObjectFieldNoWriteBarrier(context, FixedArray::kLengthOffset,
13715 1376 : SmiConstant(slots));
13716 :
13717 : Node* const empty_scope_info =
13718 688 : LoadContextElement(native_context, Context::SCOPE_INFO_INDEX);
13719 1376 : StoreContextElementNoWriteBarrier(context, Context::SCOPE_INFO_INDEX,
13720 688 : empty_scope_info);
13721 1376 : StoreContextElementNoWriteBarrier(context, Context::PREVIOUS_INDEX,
13722 2064 : UndefinedConstant());
13723 1376 : StoreContextElementNoWriteBarrier(context, Context::EXTENSION_INDEX,
13724 2064 : TheHoleConstant());
13725 1376 : StoreContextElementNoWriteBarrier(context, Context::NATIVE_CONTEXT_INDEX,
13726 688 : native_context);
13727 688 : }
13728 :
13729 112 : TNode<JSArray> CodeStubAssembler::ArrayCreate(TNode<Context> context,
13730 : TNode<Number> length) {
13731 224 : TVARIABLE(JSArray, array);
13732 224 : Label allocate_js_array(this);
13733 :
13734 224 : Label done(this), next(this), runtime(this, Label::kDeferred);
13735 112 : TNode<Smi> limit = SmiConstant(JSArray::kInitialMaxFastElementArray);
13736 : CSA_ASSERT_BRANCH(this, [=](Label* ok, Label* not_ok) {
13737 : BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, length,
13738 : SmiConstant(0), ok, not_ok);
13739 : });
13740 : // This check also transitively covers the case where length is too big
13741 : // to be representable by a SMI and so is not usable with
13742 : // AllocateJSArray.
13743 112 : BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, length,
13744 112 : limit, &runtime, &next);
13745 :
13746 112 : BIND(&runtime);
13747 : {
13748 112 : TNode<Context> native_context = LoadNativeContext(context);
13749 : TNode<JSFunction> array_function =
13750 112 : CAST(LoadContextElement(native_context, Context::ARRAY_FUNCTION_INDEX));
13751 224 : array = CAST(CallRuntime(Runtime::kNewArray, context, array_function,
13752 112 : length, array_function, UndefinedConstant()));
13753 112 : Goto(&done);
13754 : }
13755 :
13756 112 : BIND(&next);
13757 : CSA_ASSERT(this, TaggedIsSmi(length));
13758 :
13759 112 : TNode<Map> array_map = CAST(LoadContextElement(
13760 : context, Context::JS_ARRAY_PACKED_SMI_ELEMENTS_MAP_INDEX));
13761 :
13762 : // TODO(delphick): Consider using
13763 : // AllocateUninitializedJSArrayWithElements to avoid initializing an
13764 : // array and then writing over it.
13765 224 : array =
13766 : AllocateJSArray(PACKED_SMI_ELEMENTS, array_map, length, SmiConstant(0),
13767 112 : nullptr, ParameterMode::SMI_PARAMETERS);
13768 112 : Goto(&done);
13769 :
13770 112 : BIND(&done);
13771 224 : return array.value();
13772 : }
13773 :
13774 112 : void CodeStubAssembler::SetPropertyLength(TNode<Context> context,
13775 : TNode<Object> array,
13776 : TNode<Number> length) {
13777 224 : Label fast(this), runtime(this), done(this);
13778 : // There's no need to set the length, if
13779 : // 1) the array is a fast JS array and
13780 : // 2) the new length is equal to the old length.
13781 : // as the set is not observable. Otherwise fall back to the run-time.
13782 :
13783 : // 1) Check that the array has fast elements.
13784 : // TODO(delphick): Consider changing this since it does an an unnecessary
13785 : // check for SMIs.
13786 : // TODO(delphick): Also we could hoist this to after the array construction
13787 : // and copy the args into array in the same way as the Array constructor.
13788 112 : BranchIfFastJSArray(array, context, &fast, &runtime);
13789 :
13790 112 : BIND(&fast);
13791 : {
13792 112 : TNode<JSArray> fast_array = CAST(array);
13793 :
13794 112 : TNode<Smi> length_smi = CAST(length);
13795 112 : TNode<Smi> old_length = LoadFastJSArrayLength(fast_array);
13796 : CSA_ASSERT(this, TaggedIsPositiveSmi(old_length));
13797 :
13798 : // 2) If the created array's length matches the required length, then
13799 : // there's nothing else to do. Otherwise use the runtime to set the
13800 : // property as that will insert holes into excess elements or shrink
13801 : // the backing store as appropriate.
13802 112 : Branch(SmiNotEqual(length_smi, old_length), &runtime, &done);
13803 : }
13804 :
13805 112 : BIND(&runtime);
13806 : {
13807 224 : SetPropertyStrict(context, array, CodeStubAssembler::LengthStringConstant(),
13808 336 : length);
13809 112 : Goto(&done);
13810 : }
13811 :
13812 112 : BIND(&done);
13813 112 : }
13814 :
13815 224 : void CodeStubAssembler::GotoIfInitialPrototypePropertyModified(
13816 : TNode<Map> object_map, TNode<Map> initial_prototype_map, int descriptor,
13817 : RootIndex field_name_root_index, Label* if_modified) {
13818 224 : DescriptorIndexAndName index_name{descriptor, field_name_root_index};
13819 448 : GotoIfInitialPrototypePropertiesModified(
13820 : object_map, initial_prototype_map,
13821 224 : Vector<DescriptorIndexAndName>(&index_name, 1), if_modified);
13822 224 : }
13823 :
13824 1176 : void CodeStubAssembler::GotoIfInitialPrototypePropertiesModified(
13825 : TNode<Map> object_map, TNode<Map> initial_prototype_map,
13826 : Vector<DescriptorIndexAndName> properties, Label* if_modified) {
13827 1176 : TNode<Map> prototype_map = LoadMap(LoadMapPrototype(object_map));
13828 1176 : GotoIfNot(WordEqual(prototype_map, initial_prototype_map), if_modified);
13829 :
13830 : if (FLAG_track_constant_fields) {
13831 : // With constant field tracking, we need to make sure that important
13832 : // properties in the prototype has not been tampered with. We do this by
13833 : // checking that their slots in the prototype's descriptor array are still
13834 : // marked as const.
13835 1176 : TNode<DescriptorArray> descriptors = LoadMapDescriptors(prototype_map);
13836 :
13837 1176 : TNode<Uint32T> combined_details;
13838 2744 : for (int i = 0; i < properties.length(); i++) {
13839 : // Assert the descriptor index is in-bounds.
13840 1568 : int descriptor = properties[i].descriptor_index;
13841 : CSA_ASSERT(this, Int32LessThan(Int32Constant(descriptor),
13842 : LoadNumberOfDescriptors(descriptors)));
13843 : // Assert that the name is correct. This essentially checks that
13844 : // the descriptor index corresponds to the insertion order in
13845 : // the bootstrapper.
13846 : CSA_ASSERT(this,
13847 : WordEqual(LoadKeyByDescriptorEntry(descriptors, descriptor),
13848 : LoadRoot(properties[i].name_root_index)));
13849 :
13850 : TNode<Uint32T> details =
13851 1568 : DescriptorArrayGetDetails(descriptors, Uint32Constant(descriptor));
13852 1568 : if (i == 0) {
13853 1176 : combined_details = details;
13854 : } else {
13855 392 : combined_details = Unsigned(Word32And(combined_details, details));
13856 : }
13857 : }
13858 :
13859 : TNode<Uint32T> constness =
13860 1176 : DecodeWord32<PropertyDetails::ConstnessField>(combined_details);
13861 :
13862 2352 : GotoIfNot(
13863 2352 : Word32Equal(constness,
13864 2352 : Int32Constant(static_cast<int>(PropertyConstness::kConst))),
13865 1176 : if_modified);
13866 : }
13867 1176 : }
13868 :
13869 224 : TNode<String> CodeStubAssembler::TaggedToDirectString(TNode<Object> value,
13870 : Label* fail) {
13871 448 : ToDirectStringAssembler to_direct(state(), value);
13872 224 : to_direct.TryToDirect(fail);
13873 224 : to_direct.PointerToData(fail);
13874 448 : return CAST(value);
13875 : }
13876 :
13877 : } // namespace internal
13878 87414 : } // namespace v8
|
