Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/wasm-compiler.h"
6 :
7 : #include <memory>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/assembler.h"
11 : #include "src/base/optional.h"
12 : #include "src/base/platform/elapsed-timer.h"
13 : #include "src/base/platform/platform.h"
14 : #include "src/base/v8-fallthrough.h"
15 : #include "src/builtins/builtins.h"
16 : #include "src/code-factory.h"
17 : #include "src/compiler/backend/code-generator.h"
18 : #include "src/compiler/backend/instruction-selector.h"
19 : #include "src/compiler/common-operator.h"
20 : #include "src/compiler/compiler-source-position-table.h"
21 : #include "src/compiler/diamond.h"
22 : #include "src/compiler/graph-visualizer.h"
23 : #include "src/compiler/graph.h"
24 : #include "src/compiler/int64-lowering.h"
25 : #include "src/compiler/js-graph.h"
26 : #include "src/compiler/js-operator.h"
27 : #include "src/compiler/linkage.h"
28 : #include "src/compiler/machine-operator.h"
29 : #include "src/compiler/node-matchers.h"
30 : #include "src/compiler/node-origin-table.h"
31 : #include "src/compiler/pipeline.h"
32 : #include "src/compiler/simd-scalar-lowering.h"
33 : #include "src/compiler/zone-stats.h"
34 : #include "src/counters.h"
35 : #include "src/heap/factory.h"
36 : #include "src/interface-descriptors.h"
37 : #include "src/isolate-inl.h"
38 : #include "src/log.h"
39 : #include "src/objects/heap-number.h"
40 : #include "src/optimized-compilation-info.h"
41 : #include "src/tracing/trace-event.h"
42 : #include "src/trap-handler/trap-handler.h"
43 : #include "src/vector.h"
44 : #include "src/wasm/function-body-decoder-impl.h"
45 : #include "src/wasm/function-compiler.h"
46 : #include "src/wasm/graph-builder-interface.h"
47 : #include "src/wasm/jump-table-assembler.h"
48 : #include "src/wasm/memory-tracing.h"
49 : #include "src/wasm/object-access.h"
50 : #include "src/wasm/wasm-code-manager.h"
51 : #include "src/wasm/wasm-limits.h"
52 : #include "src/wasm/wasm-linkage.h"
53 : #include "src/wasm/wasm-module.h"
54 : #include "src/wasm/wasm-objects-inl.h"
55 : #include "src/wasm/wasm-opcodes.h"
56 : #include "src/wasm/wasm-text.h"
57 :
58 : namespace v8 {
59 : namespace internal {
60 : namespace compiler {
61 :
62 : namespace {
63 :
64 : // TODO(titzer): pull WASM_64 up to a common header.
65 : #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
66 : #define WASM_64 1
67 : #else
68 : #define WASM_64 0
69 : #endif
70 :
71 : #define FATAL_UNSUPPORTED_OPCODE(opcode) \
72 : FATAL("Unsupported opcode 0x%x:%s", (opcode), \
73 : wasm::WasmOpcodes::OpcodeName(opcode));
74 :
75 : MachineType assert_size(int expected_size, MachineType type) {
76 : DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()));
77 : return type;
78 : }
79 :
80 : #define WASM_INSTANCE_OBJECT_SIZE(name) \
81 : (WasmInstanceObject::k##name##OffsetEnd - \
82 : WasmInstanceObject::k##name##Offset + 1) // NOLINT(whitespace/indent)
83 :
84 : #define WASM_INSTANCE_OBJECT_OFFSET(name) \
85 : wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
86 :
87 : #define LOAD_RAW(base_pointer, byte_offset, type) \
88 : SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), base_pointer, \
89 : mcgraph()->Int32Constant(byte_offset), Effect(), \
90 : Control()))
91 :
92 : #define LOAD_INSTANCE_FIELD(name, type) \
93 : LOAD_RAW(instance_node_.get(), WASM_INSTANCE_OBJECT_OFFSET(name), \
94 : assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type))
95 :
96 : #define LOAD_TAGGED_POINTER(base_pointer, byte_offset) \
97 : LOAD_RAW(base_pointer, byte_offset, MachineType::TaggedPointer())
98 :
99 : #define LOAD_TAGGED_ANY(base_pointer, byte_offset) \
100 : LOAD_RAW(base_pointer, byte_offset, MachineType::AnyTagged())
101 :
102 : #define LOAD_FIXED_ARRAY_SLOT(array_node, index, type) \
103 : LOAD_RAW(array_node, \
104 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), type)
105 :
106 : #define LOAD_FIXED_ARRAY_SLOT_SMI(array_node, index) \
107 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedSigned())
108 :
109 : #define LOAD_FIXED_ARRAY_SLOT_PTR(array_node, index) \
110 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedPointer())
111 :
112 : #define LOAD_FIXED_ARRAY_SLOT_ANY(array_node, index) \
113 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::AnyTagged())
114 :
115 : // This can be used to store tagged Smi values only.
116 : #define STORE_FIXED_ARRAY_SLOT_SMI(array_node, index, value) \
117 : SetEffect(graph()->NewNode( \
118 : mcgraph()->machine()->Store(StoreRepresentation( \
119 : MachineRepresentation::kTaggedSigned, kNoWriteBarrier)), \
120 : array_node, \
121 : mcgraph()->Int32Constant( \
122 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
123 : value, Effect(), Control()))
124 :
125 : // This can be used to store any tagged (Smi and HeapObject) value.
126 : #define STORE_FIXED_ARRAY_SLOT_ANY(array_node, index, value) \
127 : SetEffect(graph()->NewNode( \
128 : mcgraph()->machine()->Store(StoreRepresentation( \
129 : MachineRepresentation::kTagged, kFullWriteBarrier)), \
130 : array_node, \
131 : mcgraph()->Int32Constant( \
132 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
133 : value, Effect(), Control()))
134 :
135 1280045 : void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
136 : Graph* g = mcgraph->graph();
137 1280045 : if (g->end()) {
138 141823 : NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
139 : } else {
140 1138222 : g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
141 : }
142 1281269 : }
143 :
144 : bool ContainsSimd(wasm::FunctionSig* sig) {
145 3727733 : for (auto type : sig->all()) {
146 1289361 : if (type == wasm::kWasmS128) return true;
147 : }
148 : return false;
149 : }
150 :
151 : bool ContainsInt64(wasm::FunctionSig* sig) {
152 1148636 : for (auto type : sig->all()) {
153 391913 : if (type == wasm::kWasmI64) return true;
154 : }
155 : return false;
156 : }
157 : } // namespace
158 :
159 492 : WasmGraphBuilder::WasmGraphBuilder(
160 : wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
161 : wasm::FunctionSig* sig,
162 : compiler::SourcePositionTable* source_position_table)
163 : : zone_(zone),
164 : mcgraph_(mcgraph),
165 : env_(env),
166 : cur_buffer_(def_buffer_),
167 : cur_bufsize_(kDefaultBufferSize),
168 : has_simd_(ContainsSimd(sig)),
169 : untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
170 : sig_(sig),
171 3444165 : source_position_table_(source_position_table) {
172 : DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
173 : DCHECK_NOT_NULL(mcgraph_);
174 492 : }
175 :
176 0 : Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
177 :
178 1147781 : Node* WasmGraphBuilder::Start(unsigned params) {
179 1147781 : Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
180 : graph()->SetStart(start);
181 1147122 : return start;
182 : }
183 :
184 1413075 : Node* WasmGraphBuilder::Param(unsigned index) {
185 1413075 : return graph()->NewNode(mcgraph()->common()->Parameter(index),
186 1413964 : graph()->start());
187 : }
188 :
189 9611 : Node* WasmGraphBuilder::Loop(Node* entry) {
190 19222 : return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
191 : }
192 :
193 9611 : Node* WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
194 : Node* terminate =
195 9611 : graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
196 9609 : MergeControlToEnd(mcgraph(), terminate);
197 9609 : return terminate;
198 : }
199 :
200 955 : Node* WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
201 : Node* terminate =
202 955 : graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
203 958 : MergeControlToEnd(mcgraph(), terminate);
204 959 : return terminate;
205 : }
206 :
207 1207359 : bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
208 2641723 : return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
209 1434357 : NodeProperties::GetControlInput(phi) == merge;
210 : }
211 :
212 1192 : bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
213 : Node** if_exception) {
214 1192 : if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
215 : return false;
216 : }
217 :
218 1219 : *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
219 : *if_exception =
220 1220 : graph()->NewNode(mcgraph()->common()->IfException(), node, node);
221 :
222 610 : return true;
223 : }
224 :
225 213697 : void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
226 : DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
227 213697 : merge->AppendInput(mcgraph()->zone(), from);
228 : int new_size = merge->InputCount();
229 213701 : NodeProperties::ChangeOp(
230 213699 : merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
231 213698 : }
232 :
233 108145 : void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
234 : DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
235 : int new_size = phi->InputCount();
236 108145 : phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
237 108147 : NodeProperties::ChangeOp(
238 108144 : phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
239 108146 : }
240 :
241 35540 : Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
242 35540 : return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
243 : }
244 :
245 98336 : Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
246 : Node* control) {
247 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
248 98336 : Node** buf = Realloc(vals, count, count + 1);
249 98332 : buf[count] = control;
250 98332 : return graph()->NewNode(
251 : mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
252 : count),
253 98331 : count + 1, buf);
254 : }
255 :
256 31844 : Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
257 : Node* control) {
258 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
259 31844 : Node** buf = Realloc(effects, count, count + 1);
260 31848 : buf[count] = control;
261 31848 : return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
262 31844 : buf);
263 : }
264 :
265 296 : Node* WasmGraphBuilder::RefNull() {
266 888 : return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
267 : }
268 :
269 0 : Node* WasmGraphBuilder::NoContextConstant() {
270 : // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
271 371285 : return mcgraph()->IntPtrConstant(0);
272 : }
273 :
274 0 : Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
275 0 : return mcgraph()->Uint32Constant(value);
276 : }
277 :
278 1959851 : Node* WasmGraphBuilder::Int32Constant(int32_t value) {
279 2487599 : return mcgraph()->Int32Constant(value);
280 : }
281 :
282 57624 : Node* WasmGraphBuilder::Int64Constant(int64_t value) {
283 58325 : return mcgraph()->Int64Constant(value);
284 : }
285 :
286 0 : Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
287 38786 : return mcgraph()->IntPtrConstant(value);
288 : }
289 :
290 133249 : void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
291 : Node** effect, Node** control) {
292 : DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks.
293 133249 : if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
294 3200 : return;
295 : }
296 130049 : if (effect == nullptr) effect = effect_;
297 130049 : if (control == nullptr) control = control_;
298 :
299 : // This instruction sequence is matched in the instruction selector to
300 : // load the stack pointer directly on some platforms. Hence, when modifying
301 : // please also fix WasmStackCheckMatcher in node-matchers.h
302 :
303 260254 : Node* limit_address = graph()->NewNode(
304 : mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
305 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
306 : *effect, *control);
307 130228 : Node* limit = graph()->NewNode(
308 : mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
309 : mcgraph()->IntPtrConstant(0), limit_address, *control);
310 130134 : *effect = limit;
311 130134 : Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
312 :
313 : Node* check =
314 130061 : graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
315 :
316 130112 : Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
317 130128 : stack_check.Chain(*control);
318 :
319 130152 : if (stack_check_call_operator_ == nullptr) {
320 : // Build and cache the stack check call operator and the constant
321 : // representing the stack check code.
322 122418 : auto call_descriptor = Linkage::GetStubCallDescriptor(
323 : mcgraph()->zone(), // zone
324 : NoContextDescriptor{}, // descriptor
325 : 0, // stack parameter count
326 : CallDescriptor::kNoFlags, // flags
327 : Operator::kNoProperties, // properties
328 122539 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
329 : // A direct call to a wasm runtime stub defined in this module.
330 : // Just encode the stub index. This will be patched at relocation.
331 122418 : stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
332 : wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
333 122560 : stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
334 : }
335 :
336 130209 : Node* call = graph()->NewNode(stack_check_call_operator_.get(),
337 : stack_check_code_node_.get(), *effect,
338 : stack_check.if_false);
339 :
340 : SetSourcePosition(call, position);
341 :
342 130241 : Node* ephi = stack_check.EffectPhi(*effect, call);
343 :
344 130069 : *control = stack_check.merge;
345 130069 : *effect = ephi;
346 : }
347 :
348 617960 : void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
349 1115403 : if (!needs_stack_check_) return;
350 :
351 : Node* start = graph()->start();
352 : // Place a stack check which uses a dummy node as control and effect.
353 123567 : Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
354 123696 : Node* control = dummy;
355 123696 : Node* effect = dummy;
356 : // The function-prologue stack check is associated with position 0, which
357 : // is never a position of any instruction in the function.
358 123696 : StackCheck(0, &effect, &control);
359 :
360 : // In testing, no steck checks were emitted. Nothing to rewire then.
361 123672 : if (effect == dummy) return;
362 :
363 : // Now patch all control uses of {start} to use {control} and all effect uses
364 : // to use {effect} instead. Then rewire the dummy node to use start instead.
365 120622 : NodeProperties::ReplaceUses(start, start, effect, control);
366 120788 : NodeProperties::ReplaceUses(dummy, nullptr, start, start);
367 : }
368 :
369 1154043 : Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
370 : wasm::WasmCodePosition position) {
371 : const Operator* op;
372 : MachineOperatorBuilder* m = mcgraph()->machine();
373 1154043 : switch (opcode) {
374 : case wasm::kExprI32Add:
375 368547 : op = m->Int32Add();
376 368548 : break;
377 : case wasm::kExprI32Sub:
378 27870 : op = m->Int32Sub();
379 27841 : break;
380 : case wasm::kExprI32Mul:
381 18124 : op = m->Int32Mul();
382 18124 : break;
383 : case wasm::kExprI32DivS:
384 14196 : return BuildI32DivS(left, right, position);
385 : case wasm::kExprI32DivU:
386 14176 : return BuildI32DivU(left, right, position);
387 : case wasm::kExprI32RemS:
388 14112 : return BuildI32RemS(left, right, position);
389 : case wasm::kExprI32RemU:
390 14108 : return BuildI32RemU(left, right, position);
391 : case wasm::kExprI32And:
392 119093 : op = m->Word32And();
393 119094 : break;
394 : case wasm::kExprI32Ior:
395 19025 : op = m->Word32Or();
396 19025 : break;
397 : case wasm::kExprI32Xor:
398 14367 : op = m->Word32Xor();
399 14367 : break;
400 : case wasm::kExprI32Shl:
401 25757 : op = m->Word32Shl();
402 25757 : right = MaskShiftCount32(right);
403 25757 : break;
404 : case wasm::kExprI32ShrU:
405 29477 : op = m->Word32Shr();
406 29477 : right = MaskShiftCount32(right);
407 29477 : break;
408 : case wasm::kExprI32ShrS:
409 15639 : op = m->Word32Sar();
410 15639 : right = MaskShiftCount32(right);
411 15639 : break;
412 : case wasm::kExprI32Ror:
413 26959 : op = m->Word32Ror();
414 26959 : right = MaskShiftCount32(right);
415 26959 : break;
416 : case wasm::kExprI32Rol:
417 13487 : right = MaskShiftCount32(right);
418 13487 : return BuildI32Rol(left, right);
419 : case wasm::kExprI32Eq:
420 139018 : op = m->Word32Equal();
421 139022 : break;
422 : case wasm::kExprI32Ne:
423 20921 : return Invert(Binop(wasm::kExprI32Eq, left, right));
424 : case wasm::kExprI32LtS:
425 15517 : op = m->Int32LessThan();
426 15517 : break;
427 : case wasm::kExprI32LeS:
428 14398 : op = m->Int32LessThanOrEqual();
429 14399 : break;
430 : case wasm::kExprI32LtU:
431 17851 : op = m->Uint32LessThan();
432 17851 : break;
433 : case wasm::kExprI32LeU:
434 13627 : op = m->Uint32LessThanOrEqual();
435 13627 : break;
436 : case wasm::kExprI32GtS:
437 15693 : op = m->Int32LessThan();
438 : std::swap(left, right);
439 : break;
440 : case wasm::kExprI32GeS:
441 14068 : op = m->Int32LessThanOrEqual();
442 : std::swap(left, right);
443 : break;
444 : case wasm::kExprI32GtU:
445 15198 : op = m->Uint32LessThan();
446 : std::swap(left, right);
447 : break;
448 : case wasm::kExprI32GeU:
449 13648 : op = m->Uint32LessThanOrEqual();
450 : std::swap(left, right);
451 : break;
452 : case wasm::kExprI64And:
453 7731 : op = m->Word64And();
454 7731 : break;
455 : case wasm::kExprI64Add:
456 684 : op = m->Int64Add();
457 684 : break;
458 : case wasm::kExprI64Sub:
459 848 : op = m->Int64Sub();
460 848 : break;
461 : case wasm::kExprI64Mul:
462 823 : op = m->Int64Mul();
463 821 : break;
464 : case wasm::kExprI64DivS:
465 720 : return BuildI64DivS(left, right, position);
466 : case wasm::kExprI64DivU:
467 660 : return BuildI64DivU(left, right, position);
468 : case wasm::kExprI64RemS:
469 644 : return BuildI64RemS(left, right, position);
470 : case wasm::kExprI64RemU:
471 647 : return BuildI64RemU(left, right, position);
472 : case wasm::kExprI64Ior:
473 13390 : op = m->Word64Or();
474 13381 : break;
475 : case wasm::kExprI64Xor:
476 56 : op = m->Word64Xor();
477 56 : break;
478 : case wasm::kExprI64Shl:
479 13856 : op = m->Word64Shl();
480 13851 : right = MaskShiftCount64(right);
481 13861 : break;
482 : case wasm::kExprI64ShrU:
483 807 : op = m->Word64Shr();
484 807 : right = MaskShiftCount64(right);
485 807 : break;
486 : case wasm::kExprI64ShrS:
487 636 : op = m->Word64Sar();
488 636 : right = MaskShiftCount64(right);
489 636 : break;
490 : case wasm::kExprI64Eq:
491 34754 : op = m->Word64Equal();
492 34755 : break;
493 : case wasm::kExprI64Ne:
494 84 : return Invert(Binop(wasm::kExprI64Eq, left, right));
495 : case wasm::kExprI64LtS:
496 64 : op = m->Int64LessThan();
497 64 : break;
498 : case wasm::kExprI64LeS:
499 32 : op = m->Int64LessThanOrEqual();
500 32 : break;
501 : case wasm::kExprI64LtU:
502 40 : op = m->Uint64LessThan();
503 40 : break;
504 : case wasm::kExprI64LeU:
505 48 : op = m->Uint64LessThanOrEqual();
506 48 : break;
507 : case wasm::kExprI64GtS:
508 40 : op = m->Int64LessThan();
509 : std::swap(left, right);
510 : break;
511 : case wasm::kExprI64GeS:
512 32 : op = m->Int64LessThanOrEqual();
513 : std::swap(left, right);
514 : break;
515 : case wasm::kExprI64GtU:
516 40 : op = m->Uint64LessThan();
517 : std::swap(left, right);
518 : break;
519 : case wasm::kExprI64GeU:
520 32 : op = m->Uint64LessThanOrEqual();
521 : std::swap(left, right);
522 : break;
523 : case wasm::kExprI64Ror:
524 110 : op = m->Word64Ror();
525 110 : right = MaskShiftCount64(right);
526 110 : break;
527 : case wasm::kExprI64Rol:
528 55 : return BuildI64Rol(left, right);
529 : case wasm::kExprF32CopySign:
530 32 : return BuildF32CopySign(left, right);
531 : case wasm::kExprF64CopySign:
532 32 : return BuildF64CopySign(left, right);
533 : case wasm::kExprF32Add:
534 477 : op = m->Float32Add();
535 477 : break;
536 : case wasm::kExprF32Sub:
537 369 : op = m->Float32Sub();
538 369 : break;
539 : case wasm::kExprF32Mul:
540 456 : op = m->Float32Mul();
541 456 : break;
542 : case wasm::kExprF32Div:
543 389 : op = m->Float32Div();
544 389 : break;
545 : case wasm::kExprF32Eq:
546 494 : op = m->Float32Equal();
547 494 : break;
548 : case wasm::kExprF32Ne:
549 333 : return Invert(Binop(wasm::kExprF32Eq, left, right));
550 : case wasm::kExprF32Lt:
551 153 : op = m->Float32LessThan();
552 153 : break;
553 : case wasm::kExprF32Ge:
554 113 : op = m->Float32LessThanOrEqual();
555 : std::swap(left, right);
556 : break;
557 : case wasm::kExprF32Gt:
558 129 : op = m->Float32LessThan();
559 : std::swap(left, right);
560 : break;
561 : case wasm::kExprF32Le:
562 121 : op = m->Float32LessThanOrEqual();
563 121 : break;
564 : case wasm::kExprF64Add:
565 4249 : op = m->Float64Add();
566 4249 : break;
567 : case wasm::kExprF64Sub:
568 2982 : op = m->Float64Sub();
569 2982 : break;
570 : case wasm::kExprF64Mul:
571 6569 : op = m->Float64Mul();
572 6569 : break;
573 : case wasm::kExprF64Div:
574 739 : op = m->Float64Div();
575 739 : break;
576 : case wasm::kExprF64Eq:
577 738 : op = m->Float64Equal();
578 738 : break;
579 : case wasm::kExprF64Ne:
580 413 : return Invert(Binop(wasm::kExprF64Eq, left, right));
581 : case wasm::kExprF64Lt:
582 758 : op = m->Float64LessThan();
583 758 : break;
584 : case wasm::kExprF64Le:
585 303 : op = m->Float64LessThanOrEqual();
586 303 : break;
587 : case wasm::kExprF64Gt:
588 696 : op = m->Float64LessThan();
589 : std::swap(left, right);
590 : break;
591 : case wasm::kExprF64Ge:
592 301 : op = m->Float64LessThanOrEqual();
593 : std::swap(left, right);
594 : break;
595 : case wasm::kExprF32Min:
596 70 : op = m->Float32Min();
597 70 : break;
598 : case wasm::kExprF64Min:
599 83 : op = m->Float64Min();
600 83 : break;
601 : case wasm::kExprF32Max:
602 70 : op = m->Float32Max();
603 70 : break;
604 : case wasm::kExprF64Max:
605 73 : op = m->Float64Max();
606 73 : break;
607 : case wasm::kExprF64Pow:
608 9 : return BuildF64Pow(left, right);
609 : case wasm::kExprF64Atan2:
610 18 : op = m->Float64Atan2();
611 18 : break;
612 : case wasm::kExprF64Mod:
613 20 : return BuildF64Mod(left, right);
614 : case wasm::kExprI32AsmjsDivS:
615 385 : return BuildI32AsmjsDivS(left, right);
616 : case wasm::kExprI32AsmjsDivU:
617 137 : return BuildI32AsmjsDivU(left, right);
618 : case wasm::kExprI32AsmjsRemS:
619 421 : return BuildI32AsmjsRemS(left, right);
620 : case wasm::kExprI32AsmjsRemU:
621 186 : return BuildI32AsmjsRemU(left, right);
622 : case wasm::kExprI32AsmjsStoreMem8:
623 3041 : return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
624 : case wasm::kExprI32AsmjsStoreMem16:
625 1385 : return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
626 : case wasm::kExprI32AsmjsStoreMem:
627 31532 : return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
628 : case wasm::kExprF32AsmjsStoreMem:
629 3299 : return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
630 : case wasm::kExprF64AsmjsStoreMem:
631 759 : return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
632 : default:
633 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
634 : }
635 1018209 : return graph()->NewNode(op, left, right);
636 : }
637 :
638 339773 : Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
639 : wasm::WasmCodePosition position) {
640 : const Operator* op;
641 : MachineOperatorBuilder* m = mcgraph()->machine();
642 339773 : switch (opcode) {
643 : case wasm::kExprI32Eqz:
644 136420 : op = m->Word32Equal();
645 272862 : return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
646 : case wasm::kExprF32Abs:
647 62 : op = m->Float32Abs();
648 62 : break;
649 : case wasm::kExprF32Neg: {
650 237 : op = m->Float32Neg();
651 237 : break;
652 : }
653 : case wasm::kExprF32Sqrt:
654 182 : op = m->Float32Sqrt();
655 182 : break;
656 : case wasm::kExprF64Abs:
657 87 : op = m->Float64Abs();
658 87 : break;
659 : case wasm::kExprF64Neg: {
660 1066 : op = m->Float64Neg();
661 1066 : break;
662 : }
663 : case wasm::kExprF64Sqrt:
664 257 : op = m->Float64Sqrt();
665 257 : break;
666 : case wasm::kExprI32SConvertF32:
667 : case wasm::kExprI32UConvertF32:
668 : case wasm::kExprI32SConvertF64:
669 : case wasm::kExprI32UConvertF64:
670 : case wasm::kExprI32SConvertSatF64:
671 : case wasm::kExprI32UConvertSatF64:
672 : case wasm::kExprI32SConvertSatF32:
673 : case wasm::kExprI32UConvertSatF32:
674 355 : return BuildIntConvertFloat(input, position, opcode);
675 : case wasm::kExprI32AsmjsSConvertF64:
676 108 : return BuildI32AsmjsSConvertF64(input);
677 : case wasm::kExprI32AsmjsUConvertF64:
678 8 : return BuildI32AsmjsUConvertF64(input);
679 : case wasm::kExprF32ConvertF64:
680 3467 : op = m->TruncateFloat64ToFloat32();
681 3467 : break;
682 : case wasm::kExprF64SConvertI32:
683 595 : op = m->ChangeInt32ToFloat64();
684 595 : break;
685 : case wasm::kExprF64UConvertI32:
686 287 : op = m->ChangeUint32ToFloat64();
687 287 : break;
688 : case wasm::kExprF32SConvertI32:
689 212 : op = m->RoundInt32ToFloat32();
690 212 : break;
691 : case wasm::kExprF32UConvertI32:
692 96 : op = m->RoundUint32ToFloat32();
693 96 : break;
694 : case wasm::kExprI32AsmjsSConvertF32:
695 8 : return BuildI32AsmjsSConvertF32(input);
696 : case wasm::kExprI32AsmjsUConvertF32:
697 8 : return BuildI32AsmjsUConvertF32(input);
698 : case wasm::kExprF64ConvertF32:
699 5579 : op = m->ChangeFloat32ToFloat64();
700 5579 : break;
701 : case wasm::kExprF32ReinterpretI32:
702 202 : op = m->BitcastInt32ToFloat32();
703 202 : break;
704 : case wasm::kExprI32ReinterpretF32:
705 50957 : op = m->BitcastFloat32ToInt32();
706 50958 : break;
707 : case wasm::kExprI32Clz:
708 1156 : op = m->Word32Clz();
709 1156 : break;
710 : case wasm::kExprI32Ctz: {
711 340 : if (m->Word32Ctz().IsSupported()) {
712 340 : op = m->Word32Ctz().op();
713 340 : break;
714 0 : } else if (m->Word32ReverseBits().IsSupported()) {
715 0 : Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
716 0 : Node* result = graph()->NewNode(m->Word32Clz(), reversed);
717 0 : return result;
718 : } else {
719 0 : return BuildI32Ctz(input);
720 : }
721 : }
722 : case wasm::kExprI32Popcnt: {
723 80 : if (m->Word32Popcnt().IsSupported()) {
724 80 : op = m->Word32Popcnt().op();
725 80 : break;
726 : } else {
727 0 : return BuildI32Popcnt(input);
728 : }
729 : }
730 : case wasm::kExprF32Floor: {
731 50 : if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
732 50 : op = m->Float32RoundDown().op();
733 50 : break;
734 : }
735 : case wasm::kExprF32Ceil: {
736 50 : if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
737 50 : op = m->Float32RoundUp().op();
738 50 : break;
739 : }
740 : case wasm::kExprF32Trunc: {
741 204 : if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
742 204 : op = m->Float32RoundTruncate().op();
743 204 : break;
744 : }
745 : case wasm::kExprF32NearestInt: {
746 24 : if (!m->Float32RoundTiesEven().IsSupported())
747 0 : return BuildF32NearestInt(input);
748 24 : op = m->Float32RoundTiesEven().op();
749 24 : break;
750 : }
751 : case wasm::kExprF64Floor: {
752 99 : if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
753 99 : op = m->Float64RoundDown().op();
754 99 : break;
755 : }
756 : case wasm::kExprF64Ceil: {
757 66 : if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
758 66 : op = m->Float64RoundUp().op();
759 66 : break;
760 : }
761 : case wasm::kExprF64Trunc: {
762 199 : if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
763 199 : op = m->Float64RoundTruncate().op();
764 199 : break;
765 : }
766 : case wasm::kExprF64NearestInt: {
767 24 : if (!m->Float64RoundTiesEven().IsSupported())
768 0 : return BuildF64NearestInt(input);
769 24 : op = m->Float64RoundTiesEven().op();
770 24 : break;
771 : }
772 : case wasm::kExprF64Acos: {
773 21 : return BuildF64Acos(input);
774 : }
775 : case wasm::kExprF64Asin: {
776 22 : return BuildF64Asin(input);
777 : }
778 : case wasm::kExprF64Atan:
779 21 : op = m->Float64Atan();
780 21 : break;
781 : case wasm::kExprF64Cos: {
782 147 : op = m->Float64Cos();
783 147 : break;
784 : }
785 : case wasm::kExprF64Sin: {
786 146 : op = m->Float64Sin();
787 146 : break;
788 : }
789 : case wasm::kExprF64Tan: {
790 21 : op = m->Float64Tan();
791 21 : break;
792 : }
793 : case wasm::kExprF64Exp: {
794 22 : op = m->Float64Exp();
795 22 : break;
796 : }
797 : case wasm::kExprF64Log:
798 22 : op = m->Float64Log();
799 22 : break;
800 : case wasm::kExprI32ConvertI64:
801 966 : op = m->TruncateInt64ToInt32();
802 966 : break;
803 : case wasm::kExprI64SConvertI32:
804 76 : op = m->ChangeInt32ToInt64();
805 76 : break;
806 : case wasm::kExprI64UConvertI32:
807 26511 : op = m->ChangeUint32ToUint64();
808 26519 : break;
809 : case wasm::kExprF64ReinterpretI64:
810 218 : op = m->BitcastInt64ToFloat64();
811 218 : break;
812 : case wasm::kExprI64ReinterpretF64:
813 50597 : op = m->BitcastFloat64ToInt64();
814 50595 : break;
815 : case wasm::kExprI64Clz:
816 34 : op = m->Word64Clz();
817 34 : break;
818 : case wasm::kExprI64Ctz: {
819 218 : OptionalOperator ctz64 = m->Word64Ctz();
820 218 : if (ctz64.IsSupported()) {
821 : op = ctz64.op();
822 218 : break;
823 0 : } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
824 : op = ctz64.placeholder();
825 : break;
826 0 : } else if (m->Word64ReverseBits().IsSupported()) {
827 0 : Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
828 0 : Node* result = graph()->NewNode(m->Word64Clz(), reversed);
829 0 : return result;
830 : } else {
831 0 : return BuildI64Ctz(input);
832 : }
833 : }
834 : case wasm::kExprI64Popcnt: {
835 42 : OptionalOperator popcnt64 = m->Word64Popcnt();
836 41 : if (popcnt64.IsSupported()) {
837 : op = popcnt64.op();
838 0 : } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
839 : op = popcnt64.placeholder();
840 : } else {
841 0 : return BuildI64Popcnt(input);
842 : }
843 41 : break;
844 : }
845 : case wasm::kExprI64Eqz:
846 168 : op = m->Word64Equal();
847 336 : return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
848 : case wasm::kExprF32SConvertI64:
849 90 : if (m->Is32()) {
850 0 : return BuildF32SConvertI64(input);
851 : }
852 90 : op = m->RoundInt64ToFloat32();
853 90 : break;
854 : case wasm::kExprF32UConvertI64:
855 32 : if (m->Is32()) {
856 0 : return BuildF32UConvertI64(input);
857 : }
858 32 : op = m->RoundUint64ToFloat32();
859 32 : break;
860 : case wasm::kExprF64SConvertI64:
861 80 : if (m->Is32()) {
862 0 : return BuildF64SConvertI64(input);
863 : }
864 80 : op = m->RoundInt64ToFloat64();
865 80 : break;
866 : case wasm::kExprF64UConvertI64:
867 135 : if (m->Is32()) {
868 0 : return BuildF64UConvertI64(input);
869 : }
870 135 : op = m->RoundUint64ToFloat64();
871 135 : break;
872 : case wasm::kExprI32SExtendI8:
873 12 : op = m->SignExtendWord8ToInt32();
874 12 : break;
875 : case wasm::kExprI32SExtendI16:
876 4 : op = m->SignExtendWord16ToInt32();
877 4 : break;
878 : case wasm::kExprI64SExtendI8:
879 12 : op = m->SignExtendWord8ToInt64();
880 12 : break;
881 : case wasm::kExprI64SExtendI16:
882 4 : op = m->SignExtendWord16ToInt64();
883 4 : break;
884 : case wasm::kExprI64SExtendI32:
885 4 : op = m->SignExtendWord32ToInt64();
886 4 : break;
887 : case wasm::kExprI64SConvertF32:
888 : case wasm::kExprI64UConvertF32:
889 : case wasm::kExprI64SConvertF64:
890 : case wasm::kExprI64UConvertF64:
891 : case wasm::kExprI64SConvertSatF32:
892 : case wasm::kExprI64UConvertSatF32:
893 : case wasm::kExprI64SConvertSatF64:
894 : case wasm::kExprI64UConvertSatF64:
895 : return mcgraph()->machine()->Is32()
896 : ? BuildCcallConvertFloat(input, position, opcode)
897 1225 : : BuildIntConvertFloat(input, position, opcode);
898 : case wasm::kExprRefIsNull:
899 88 : return graph()->NewNode(m->WordEqual(), input, RefNull());
900 : case wasm::kExprI32AsmjsLoadMem8S:
901 2804 : return BuildAsmjsLoadMem(MachineType::Int8(), input);
902 : case wasm::kExprI32AsmjsLoadMem8U:
903 2082 : return BuildAsmjsLoadMem(MachineType::Uint8(), input);
904 : case wasm::kExprI32AsmjsLoadMem16S:
905 958 : return BuildAsmjsLoadMem(MachineType::Int16(), input);
906 : case wasm::kExprI32AsmjsLoadMem16U:
907 531 : return BuildAsmjsLoadMem(MachineType::Uint16(), input);
908 : case wasm::kExprI32AsmjsLoadMem:
909 43778 : return BuildAsmjsLoadMem(MachineType::Int32(), input);
910 : case wasm::kExprF32AsmjsLoadMem:
911 5396 : return BuildAsmjsLoadMem(MachineType::Float32(), input);
912 : case wasm::kExprF64AsmjsLoadMem:
913 845 : return BuildAsmjsLoadMem(MachineType::Float64(), input);
914 : default:
915 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
916 : }
917 144984 : return graph()->NewNode(op, input);
918 : }
919 :
920 118919 : Node* WasmGraphBuilder::Float32Constant(float value) {
921 118951 : return mcgraph()->Float32Constant(value);
922 : }
923 :
924 126700 : Node* WasmGraphBuilder::Float64Constant(double value) {
925 126732 : return mcgraph()->Float64Constant(value);
926 : }
927 :
928 : namespace {
929 186084 : Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
930 : Node** false_node, Node* control, BranchHint hint) {
931 : DCHECK_NOT_NULL(cond);
932 : DCHECK_NOT_NULL(control);
933 : Node* branch =
934 186084 : mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
935 372299 : *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
936 372331 : *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
937 186169 : return branch;
938 : }
939 : } // namespace
940 :
941 171167 : Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
942 : Node** false_node) {
943 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
944 171167 : BranchHint::kNone);
945 : }
946 :
947 0 : Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
948 : Node** false_node) {
949 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
950 0 : BranchHint::kTrue);
951 : }
952 :
953 0 : Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
954 : Node** false_node) {
955 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
956 14916 : BranchHint::kFalse);
957 : }
958 :
959 142866 : TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
960 : // TODO(wasm): "!env_" should not happen when compiling an actual wasm
961 : // function.
962 142866 : if (!env_ || !env_->runtime_exception_support) {
963 : // We use TrapId::kInvalid as a marker to tell the code generator
964 : // to generate a call to a testing c-function instead of a runtime
965 : // stub. This code should only be called from a cctest.
966 : return TrapId::kInvalid;
967 : }
968 :
969 131936 : switch (reason) {
970 : #define TRAPREASON_TO_TRAPID(name) \
971 : case wasm::k##name: \
972 : static_assert( \
973 : static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
974 : "trap id mismatch"); \
975 : return TrapId::k##name;
976 11691 : FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
977 : #undef TRAPREASON_TO_TRAPID
978 : default:
979 0 : UNREACHABLE();
980 : }
981 : }
982 :
983 6313 : Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
984 : wasm::WasmCodePosition position) {
985 6313 : TrapId trap_id = GetTrapIdForTrap(reason);
986 6316 : Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
987 : cond, Effect(), Control()));
988 : SetSourcePosition(node, position);
989 6319 : return node;
990 : }
991 :
992 136550 : Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
993 : wasm::WasmCodePosition position) {
994 136550 : TrapId trap_id = GetTrapIdForTrap(reason);
995 136588 : Node* node = SetControl(graph()->NewNode(
996 : mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
997 : SetSourcePosition(node, position);
998 136590 : return node;
999 : }
1000 :
1001 : // Add a check that traps if {node} is equal to {val}.
1002 70807 : Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
1003 : int32_t val,
1004 : wasm::WasmCodePosition position) {
1005 : Int32Matcher m(node);
1006 138366 : if (m.HasValue() && !m.Is(val)) return graph()->start();
1007 4483 : if (val == 0) {
1008 3519 : return TrapIfFalse(reason, node, position);
1009 : } else {
1010 964 : return TrapIfTrue(reason,
1011 : graph()->NewNode(mcgraph()->machine()->Word32Equal(),
1012 : node, mcgraph()->Int32Constant(val)),
1013 964 : position);
1014 : }
1015 : }
1016 :
1017 : // Add a check that traps if {node} is zero.
1018 0 : Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
1019 : wasm::WasmCodePosition position) {
1020 56592 : return TrapIfEq32(reason, node, 0, position);
1021 : }
1022 :
1023 : // Add a check that traps if {node} is equal to {val}.
1024 4585 : Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
1025 : int64_t val,
1026 : wasm::WasmCodePosition position) {
1027 : Int64Matcher m(node);
1028 4823 : if (m.HasValue() && !m.Is(val)) return graph()->start();
1029 4367 : return TrapIfTrue(reason,
1030 : graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
1031 : mcgraph()->Int64Constant(val)),
1032 4370 : position);
1033 : }
1034 :
1035 : // Add a check that traps if {node} is zero.
1036 0 : Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
1037 : wasm::WasmCodePosition position) {
1038 3866 : return TrapIfEq64(reason, node, 0, position);
1039 : }
1040 :
1041 4858 : Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
1042 9717 : return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
1043 : }
1044 :
1045 210714 : Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1046 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1047 421427 : return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1048 : }
1049 :
1050 4858 : Node* WasmGraphBuilder::IfDefault(Node* sw) {
1051 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1052 9716 : return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1053 : }
1054 :
1055 1269984 : Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1056 : static const int kStackAllocatedNodeBufferSize = 8;
1057 : Node* stack_buffer[kStackAllocatedNodeBufferSize];
1058 : std::vector<Node*> heap_buffer;
1059 :
1060 : Node** buf = stack_buffer;
1061 1269984 : if (count + 3 > kStackAllocatedNodeBufferSize) {
1062 0 : heap_buffer.resize(count + 3);
1063 : buf = heap_buffer.data();
1064 : }
1065 :
1066 1269984 : buf[0] = mcgraph()->Int32Constant(0);
1067 1270944 : if (count > 0) {
1068 1028219 : memcpy(buf + 1, vals, sizeof(void*) * count);
1069 : }
1070 2541888 : buf[count + 1] = Effect();
1071 2541888 : buf[count + 2] = Control();
1072 : Node* ret =
1073 1270944 : graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1074 :
1075 1269962 : MergeControlToEnd(mcgraph(), ret);
1076 1270500 : return ret;
1077 : }
1078 :
1079 106305 : Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1080 :
1081 106152 : Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1082 106169 : TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1083 : ReturnVoid();
1084 106202 : return nullptr;
1085 : }
1086 :
1087 111318 : Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1088 : static const int32_t kMask32 = 0x1F;
1089 111318 : if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1090 : // Shifts by constants are so common we pattern-match them here.
1091 : Int32Matcher match(node);
1092 24 : if (match.HasValue()) {
1093 0 : int32_t masked = (match.Value() & kMask32);
1094 0 : if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1095 : } else {
1096 24 : node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1097 : mcgraph()->Int32Constant(kMask32));
1098 : }
1099 : }
1100 111318 : return node;
1101 : }
1102 :
1103 15405 : Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1104 : static const int64_t kMask64 = 0x3F;
1105 15405 : if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1106 : // Shifts by constants are so common we pattern-match them here.
1107 : Int64Matcher match(node);
1108 20 : if (match.HasValue()) {
1109 0 : int64_t masked = (match.Value() & kMask64);
1110 0 : if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1111 : } else {
1112 20 : node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1113 : mcgraph()->Int64Constant(kMask64));
1114 : }
1115 : }
1116 15405 : return node;
1117 : }
1118 :
1119 : static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1120 : size_t size_in_bytes) {
1121 0 : switch (size_in_bytes) {
1122 : case 4:
1123 : case 16:
1124 : return true;
1125 : case 8:
1126 : return m->Is64();
1127 : default:
1128 : break;
1129 : }
1130 : return false;
1131 : }
1132 :
1133 0 : Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1134 : Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1135 : Node* result;
1136 : Node* value = node;
1137 : MachineOperatorBuilder* m = mcgraph()->machine();
1138 0 : int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1139 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1140 : bool isFloat = false;
1141 :
1142 0 : switch (wasmtype) {
1143 : case wasm::kWasmF64:
1144 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1145 : isFloat = true;
1146 : V8_FALLTHROUGH;
1147 : case wasm::kWasmI64:
1148 0 : result = mcgraph()->Int64Constant(0);
1149 0 : break;
1150 : case wasm::kWasmF32:
1151 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1152 : isFloat = true;
1153 : V8_FALLTHROUGH;
1154 : case wasm::kWasmI32:
1155 0 : result = mcgraph()->Int32Constant(0);
1156 0 : break;
1157 : case wasm::kWasmS128:
1158 : DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1159 : break;
1160 : default:
1161 0 : UNREACHABLE();
1162 : break;
1163 : }
1164 :
1165 0 : if (mem_rep == MachineRepresentation::kWord8) {
1166 : // No need to change endianness for byte size, return original node
1167 : return node;
1168 : }
1169 0 : if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1170 : // In case we store lower part of WasmI64 expression, we can truncate
1171 : // upper 32bits
1172 0 : value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1173 : valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1174 : valueSizeInBits = 8 * valueSizeInBytes;
1175 0 : if (mem_rep == MachineRepresentation::kWord16) {
1176 : value =
1177 0 : graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1178 : }
1179 0 : } else if (wasmtype == wasm::kWasmI32 &&
1180 0 : mem_rep == MachineRepresentation::kWord16) {
1181 : value =
1182 0 : graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1183 : }
1184 :
1185 : int i;
1186 : uint32_t shiftCount;
1187 :
1188 0 : if (ReverseBytesSupported(m, valueSizeInBytes)) {
1189 0 : switch (valueSizeInBytes) {
1190 : case 4:
1191 0 : result = graph()->NewNode(m->Word32ReverseBytes(), value);
1192 0 : break;
1193 : case 8:
1194 0 : result = graph()->NewNode(m->Word64ReverseBytes(), value);
1195 0 : break;
1196 : case 16: {
1197 : Node* byte_reversed_lanes[4];
1198 0 : for (int lane = 0; lane < 4; lane++) {
1199 0 : byte_reversed_lanes[lane] = graph()->NewNode(
1200 : m->Word32ReverseBytes(),
1201 : graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1202 0 : value));
1203 : }
1204 :
1205 : // This is making a copy of the value.
1206 : result =
1207 0 : graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1208 :
1209 0 : for (int lane = 0; lane < 4; lane++) {
1210 : result =
1211 0 : graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1212 : result, byte_reversed_lanes[lane]);
1213 : }
1214 :
1215 : break;
1216 : }
1217 : default:
1218 0 : UNREACHABLE();
1219 : break;
1220 : }
1221 : } else {
1222 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1223 : i += 8, shiftCount -= 16) {
1224 : Node* shiftLower;
1225 : Node* shiftHigher;
1226 : Node* lowerByte;
1227 : Node* higherByte;
1228 :
1229 : DCHECK_LT(0, shiftCount);
1230 : DCHECK_EQ(0, (shiftCount + 8) % 16);
1231 :
1232 0 : if (valueSizeInBits > 32) {
1233 0 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1234 : mcgraph()->Int64Constant(shiftCount));
1235 0 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1236 : mcgraph()->Int64Constant(shiftCount));
1237 0 : lowerByte = graph()->NewNode(
1238 : m->Word64And(), shiftLower,
1239 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1240 0 : << (valueSizeInBits - 8 - i)));
1241 0 : higherByte = graph()->NewNode(
1242 : m->Word64And(), shiftHigher,
1243 0 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1244 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1245 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1246 : } else {
1247 0 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1248 : mcgraph()->Int32Constant(shiftCount));
1249 0 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1250 : mcgraph()->Int32Constant(shiftCount));
1251 0 : lowerByte = graph()->NewNode(
1252 : m->Word32And(), shiftLower,
1253 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1254 0 : << (valueSizeInBits - 8 - i)));
1255 0 : higherByte = graph()->NewNode(
1256 : m->Word32And(), shiftHigher,
1257 0 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1258 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1259 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1260 : }
1261 : }
1262 : }
1263 :
1264 0 : if (isFloat) {
1265 0 : switch (wasmtype) {
1266 : case wasm::kWasmF64:
1267 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1268 0 : break;
1269 : case wasm::kWasmF32:
1270 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1271 0 : break;
1272 : default:
1273 0 : UNREACHABLE();
1274 : break;
1275 : }
1276 : }
1277 :
1278 : return result;
1279 : }
1280 :
1281 0 : Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1282 : MachineType memtype,
1283 : wasm::ValueType wasmtype) {
1284 : Node* result;
1285 : Node* value = node;
1286 : MachineOperatorBuilder* m = mcgraph()->machine();
1287 : int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1288 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1289 : bool isFloat = false;
1290 :
1291 0 : switch (memtype.representation()) {
1292 : case MachineRepresentation::kFloat64:
1293 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1294 : isFloat = true;
1295 : V8_FALLTHROUGH;
1296 : case MachineRepresentation::kWord64:
1297 0 : result = mcgraph()->Int64Constant(0);
1298 0 : break;
1299 : case MachineRepresentation::kFloat32:
1300 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1301 : isFloat = true;
1302 : V8_FALLTHROUGH;
1303 : case MachineRepresentation::kWord32:
1304 : case MachineRepresentation::kWord16:
1305 0 : result = mcgraph()->Int32Constant(0);
1306 0 : break;
1307 : case MachineRepresentation::kWord8:
1308 : // No need to change endianness for byte size, return original node
1309 : return node;
1310 : break;
1311 : case MachineRepresentation::kSimd128:
1312 : DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1313 : break;
1314 : default:
1315 0 : UNREACHABLE();
1316 : break;
1317 : }
1318 :
1319 : int i;
1320 : uint32_t shiftCount;
1321 :
1322 0 : if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1323 0 : switch (valueSizeInBytes) {
1324 : case 2:
1325 : result =
1326 0 : graph()->NewNode(m->Word32ReverseBytes(),
1327 : graph()->NewNode(m->Word32Shl(), value,
1328 : mcgraph()->Int32Constant(16)));
1329 0 : break;
1330 : case 4:
1331 0 : result = graph()->NewNode(m->Word32ReverseBytes(), value);
1332 0 : break;
1333 : case 8:
1334 0 : result = graph()->NewNode(m->Word64ReverseBytes(), value);
1335 0 : break;
1336 : case 16: {
1337 : Node* byte_reversed_lanes[4];
1338 0 : for (int lane = 0; lane < 4; lane++) {
1339 0 : byte_reversed_lanes[lane] = graph()->NewNode(
1340 : m->Word32ReverseBytes(),
1341 : graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1342 0 : value));
1343 : }
1344 :
1345 : // This is making a copy of the value.
1346 : result =
1347 0 : graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1348 :
1349 0 : for (int lane = 0; lane < 4; lane++) {
1350 : result =
1351 0 : graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1352 : result, byte_reversed_lanes[lane]);
1353 : }
1354 :
1355 : break;
1356 : }
1357 : default:
1358 0 : UNREACHABLE();
1359 : }
1360 : } else {
1361 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1362 : i += 8, shiftCount -= 16) {
1363 : Node* shiftLower;
1364 : Node* shiftHigher;
1365 : Node* lowerByte;
1366 : Node* higherByte;
1367 :
1368 : DCHECK_LT(0, shiftCount);
1369 : DCHECK_EQ(0, (shiftCount + 8) % 16);
1370 :
1371 0 : if (valueSizeInBits > 32) {
1372 0 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1373 : mcgraph()->Int64Constant(shiftCount));
1374 0 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1375 : mcgraph()->Int64Constant(shiftCount));
1376 0 : lowerByte = graph()->NewNode(
1377 : m->Word64And(), shiftLower,
1378 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1379 0 : << (valueSizeInBits - 8 - i)));
1380 0 : higherByte = graph()->NewNode(
1381 : m->Word64And(), shiftHigher,
1382 0 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1383 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1384 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1385 : } else {
1386 0 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1387 : mcgraph()->Int32Constant(shiftCount));
1388 0 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1389 : mcgraph()->Int32Constant(shiftCount));
1390 0 : lowerByte = graph()->NewNode(
1391 : m->Word32And(), shiftLower,
1392 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1393 0 : << (valueSizeInBits - 8 - i)));
1394 0 : higherByte = graph()->NewNode(
1395 : m->Word32And(), shiftHigher,
1396 0 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1397 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1398 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1399 : }
1400 : }
1401 : }
1402 :
1403 0 : if (isFloat) {
1404 0 : switch (memtype.representation()) {
1405 : case MachineRepresentation::kFloat64:
1406 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1407 0 : break;
1408 : case MachineRepresentation::kFloat32:
1409 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1410 0 : break;
1411 : default:
1412 0 : UNREACHABLE();
1413 : break;
1414 : }
1415 : }
1416 :
1417 : // We need to sign extend the value
1418 0 : if (memtype.IsSigned()) {
1419 : DCHECK(!isFloat);
1420 0 : if (valueSizeInBits < 32) {
1421 : Node* shiftBitCount;
1422 : // Perform sign extension using following trick
1423 : // result = (x << machine_width - type_width) >> (machine_width -
1424 : // type_width)
1425 0 : if (wasmtype == wasm::kWasmI64) {
1426 0 : shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1427 0 : result = graph()->NewNode(
1428 : m->Word64Sar(),
1429 : graph()->NewNode(m->Word64Shl(),
1430 : graph()->NewNode(m->ChangeInt32ToInt64(), result),
1431 : shiftBitCount),
1432 : shiftBitCount);
1433 0 : } else if (wasmtype == wasm::kWasmI32) {
1434 0 : shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1435 0 : result = graph()->NewNode(
1436 : m->Word32Sar(),
1437 : graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1438 : shiftBitCount);
1439 : }
1440 : }
1441 : }
1442 :
1443 : return result;
1444 : }
1445 :
1446 32 : Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1447 32 : Node* result = Unop(
1448 : wasm::kExprF32ReinterpretI32,
1449 : Binop(wasm::kExprI32Ior,
1450 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1451 : mcgraph()->Int32Constant(0x7FFFFFFF)),
1452 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1453 32 : mcgraph()->Int32Constant(0x80000000))));
1454 :
1455 32 : return result;
1456 : }
1457 :
1458 32 : Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1459 : #if WASM_64
1460 32 : Node* result = Unop(
1461 : wasm::kExprF64ReinterpretI64,
1462 : Binop(wasm::kExprI64Ior,
1463 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1464 : mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1465 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1466 32 : mcgraph()->Int64Constant(0x8000000000000000))));
1467 :
1468 32 : return result;
1469 : #else
1470 : MachineOperatorBuilder* m = mcgraph()->machine();
1471 :
1472 : Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1473 : Node* high_word_right =
1474 : graph()->NewNode(m->Float64ExtractHighWord32(), right);
1475 :
1476 : Node* new_high_word = Binop(wasm::kExprI32Ior,
1477 : Binop(wasm::kExprI32And, high_word_left,
1478 : mcgraph()->Int32Constant(0x7FFFFFFF)),
1479 : Binop(wasm::kExprI32And, high_word_right,
1480 : mcgraph()->Int32Constant(0x80000000)));
1481 :
1482 : return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1483 : #endif
1484 : }
1485 :
1486 : namespace {
1487 :
1488 1582 : MachineType IntConvertType(wasm::WasmOpcode opcode) {
1489 1582 : switch (opcode) {
1490 : case wasm::kExprI32SConvertF32:
1491 : case wasm::kExprI32SConvertF64:
1492 : case wasm::kExprI32SConvertSatF32:
1493 : case wasm::kExprI32SConvertSatF64:
1494 : return MachineType::Int32();
1495 : case wasm::kExprI32UConvertF32:
1496 : case wasm::kExprI32UConvertF64:
1497 : case wasm::kExprI32UConvertSatF32:
1498 : case wasm::kExprI32UConvertSatF64:
1499 : return MachineType::Uint32();
1500 : case wasm::kExprI64SConvertF32:
1501 : case wasm::kExprI64SConvertF64:
1502 : case wasm::kExprI64SConvertSatF32:
1503 : case wasm::kExprI64SConvertSatF64:
1504 : return MachineType::Int64();
1505 : case wasm::kExprI64UConvertF32:
1506 : case wasm::kExprI64UConvertF64:
1507 : case wasm::kExprI64UConvertSatF32:
1508 : case wasm::kExprI64UConvertSatF64:
1509 : return MachineType::Uint64();
1510 : default:
1511 0 : UNREACHABLE();
1512 : }
1513 : }
1514 :
1515 1580 : MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1516 1580 : switch (opcode) {
1517 : case wasm::kExprI32SConvertF32:
1518 : case wasm::kExprI32UConvertF32:
1519 : case wasm::kExprI32SConvertSatF32:
1520 : case wasm::kExprI64SConvertF32:
1521 : case wasm::kExprI64UConvertF32:
1522 : case wasm::kExprI32UConvertSatF32:
1523 : case wasm::kExprI64SConvertSatF32:
1524 : case wasm::kExprI64UConvertSatF32:
1525 : return MachineType::Float32();
1526 : case wasm::kExprI32SConvertF64:
1527 : case wasm::kExprI32UConvertF64:
1528 : case wasm::kExprI64SConvertF64:
1529 : case wasm::kExprI64UConvertF64:
1530 : case wasm::kExprI32SConvertSatF64:
1531 : case wasm::kExprI32UConvertSatF64:
1532 : case wasm::kExprI64SConvertSatF64:
1533 : case wasm::kExprI64UConvertSatF64:
1534 : return MachineType::Float64();
1535 : default:
1536 0 : UNREACHABLE();
1537 : }
1538 : }
1539 :
1540 1582 : const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1541 1582 : switch (opcode) {
1542 : case wasm::kExprI32SConvertF32:
1543 : case wasm::kExprI32SConvertSatF32:
1544 124 : return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1545 : case wasm::kExprI32UConvertF32:
1546 : case wasm::kExprI32UConvertSatF32:
1547 56 : return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1548 : case wasm::kExprI32SConvertF64:
1549 : case wasm::kExprI32SConvertSatF64:
1550 127 : return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1551 : case wasm::kExprI32UConvertF64:
1552 : case wasm::kExprI32UConvertSatF64:
1553 48 : return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1554 : case wasm::kExprI64SConvertF32:
1555 : case wasm::kExprI64SConvertSatF32:
1556 48 : return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1557 : case wasm::kExprI64UConvertF32:
1558 : case wasm::kExprI64UConvertSatF32:
1559 48 : return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1560 : case wasm::kExprI64SConvertF64:
1561 : case wasm::kExprI64SConvertSatF64:
1562 1075 : return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1563 : case wasm::kExprI64UConvertF64:
1564 : case wasm::kExprI64UConvertSatF64:
1565 56 : return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1566 : default:
1567 0 : UNREACHABLE();
1568 : }
1569 : }
1570 :
1571 353 : wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1572 353 : switch (opcode) {
1573 : case wasm::kExprI32SConvertF32:
1574 : case wasm::kExprI32SConvertSatF32:
1575 : return wasm::kExprF32SConvertI32;
1576 : case wasm::kExprI32UConvertF32:
1577 : case wasm::kExprI32UConvertSatF32:
1578 56 : return wasm::kExprF32UConvertI32;
1579 : case wasm::kExprI32SConvertF64:
1580 : case wasm::kExprI32SConvertSatF64:
1581 127 : return wasm::kExprF64SConvertI32;
1582 : case wasm::kExprI32UConvertF64:
1583 : case wasm::kExprI32UConvertSatF64:
1584 48 : return wasm::kExprF64UConvertI32;
1585 : default:
1586 0 : UNREACHABLE();
1587 : }
1588 : }
1589 :
1590 1580 : bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1591 1580 : switch (opcode) {
1592 : case wasm::kExprI32SConvertF32:
1593 : case wasm::kExprI32UConvertF32:
1594 : case wasm::kExprI32SConvertF64:
1595 : case wasm::kExprI32UConvertF64:
1596 : case wasm::kExprI64SConvertF32:
1597 : case wasm::kExprI64UConvertF32:
1598 : case wasm::kExprI64SConvertF64:
1599 : case wasm::kExprI64UConvertF64:
1600 : return true;
1601 : case wasm::kExprI32SConvertSatF64:
1602 : case wasm::kExprI32UConvertSatF64:
1603 : case wasm::kExprI32SConvertSatF32:
1604 : case wasm::kExprI32UConvertSatF32:
1605 : case wasm::kExprI64SConvertSatF32:
1606 : case wasm::kExprI64UConvertSatF32:
1607 : case wasm::kExprI64SConvertSatF64:
1608 : case wasm::kExprI64UConvertSatF64:
1609 64 : return false;
1610 : default:
1611 0 : UNREACHABLE();
1612 : }
1613 : }
1614 :
1615 128 : Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1616 128 : switch (ty.representation()) {
1617 : case MachineRepresentation::kWord32:
1618 32 : return builder->Int32Constant(0);
1619 : case MachineRepresentation::kWord64:
1620 32 : return builder->Int64Constant(0);
1621 : case MachineRepresentation::kFloat32:
1622 32 : return builder->Float32Constant(0.0);
1623 : case MachineRepresentation::kFloat64:
1624 32 : return builder->Float64Constant(0.0);
1625 : default:
1626 0 : UNREACHABLE();
1627 : }
1628 : }
1629 :
1630 64 : Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1631 64 : switch (ty.semantic()) {
1632 : case MachineSemantic::kInt32:
1633 16 : return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1634 : case MachineSemantic::kUint32:
1635 16 : return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1636 : case MachineSemantic::kInt64:
1637 16 : return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1638 : case MachineSemantic::kUint64:
1639 16 : return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1640 : default:
1641 0 : UNREACHABLE();
1642 : }
1643 : }
1644 :
1645 64 : Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1646 64 : switch (ty.semantic()) {
1647 : case MachineSemantic::kInt32:
1648 16 : return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1649 : case MachineSemantic::kUint32:
1650 16 : return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1651 : case MachineSemantic::kInt64:
1652 16 : return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1653 : case MachineSemantic::kUint64:
1654 16 : return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1655 : default:
1656 0 : UNREACHABLE();
1657 : }
1658 : }
1659 :
1660 354 : wasm::WasmOpcode TruncOp(const MachineType& ty) {
1661 354 : switch (ty.representation()) {
1662 : case MachineRepresentation::kFloat32:
1663 : return wasm::kExprF32Trunc;
1664 : case MachineRepresentation::kFloat64:
1665 175 : return wasm::kExprF64Trunc;
1666 : default:
1667 0 : UNREACHABLE();
1668 : }
1669 : }
1670 :
1671 419 : wasm::WasmOpcode NeOp(const MachineType& ty) {
1672 419 : switch (ty.representation()) {
1673 : case MachineRepresentation::kFloat32:
1674 : return wasm::kExprF32Ne;
1675 : case MachineRepresentation::kFloat64:
1676 207 : return wasm::kExprF64Ne;
1677 : default:
1678 0 : UNREACHABLE();
1679 : }
1680 : }
1681 :
1682 64 : wasm::WasmOpcode LtOp(const MachineType& ty) {
1683 64 : switch (ty.representation()) {
1684 : case MachineRepresentation::kFloat32:
1685 : return wasm::kExprF32Lt;
1686 : case MachineRepresentation::kFloat64:
1687 32 : return wasm::kExprF64Lt;
1688 : default:
1689 0 : UNREACHABLE();
1690 : }
1691 : }
1692 :
1693 1580 : Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1694 : const MachineType& int_ty, const MachineType& float_ty,
1695 : Node* trunc, Node* converted_value) {
1696 1580 : if (int_ty.representation() == MachineRepresentation::kWord32) {
1697 353 : Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1698 355 : return builder->Binop(NeOp(float_ty), trunc, check);
1699 : }
1700 1227 : return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1701 1229 : trunc, builder->graph()->start());
1702 : }
1703 :
1704 64 : Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1705 : const MachineType& int_ty,
1706 : const MachineType& float_ty, Node* trunc,
1707 : Node* converted_value) {
1708 : Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1709 64 : converted_value);
1710 64 : if (int_ty.representation() == MachineRepresentation::kWord64) {
1711 32 : test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1712 : }
1713 64 : return test;
1714 : }
1715 :
1716 : } // namespace
1717 :
1718 1579 : Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1719 : wasm::WasmCodePosition position,
1720 : wasm::WasmOpcode opcode) {
1721 1579 : const MachineType int_ty = IntConvertType(opcode);
1722 1580 : const MachineType float_ty = FloatConvertType(opcode);
1723 1580 : const Operator* conv_op = ConvertOp(this, opcode);
1724 : Node* trunc = nullptr;
1725 : Node* converted_value = nullptr;
1726 : const bool is_int32 =
1727 : int_ty.representation() == MachineRepresentation::kWord32;
1728 1578 : if (is_int32) {
1729 354 : trunc = Unop(TruncOp(float_ty), input);
1730 : converted_value = graph()->NewNode(conv_op, trunc);
1731 : } else {
1732 : trunc = graph()->NewNode(conv_op, input);
1733 1230 : converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1734 : trunc, graph()->start());
1735 : }
1736 1583 : if (IsTrappingConvertOp(opcode)) {
1737 : Node* test =
1738 1516 : ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1739 1518 : if (is_int32) {
1740 323 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1741 : } else {
1742 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1743 : }
1744 : return converted_value;
1745 : }
1746 : Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1747 64 : converted_value);
1748 64 : Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1749 : tl_d.Chain(Control());
1750 64 : Node* nan_test = Binop(NeOp(float_ty), input, input);
1751 64 : Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1752 64 : nan_d.Nest(tl_d, true);
1753 64 : Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1754 64 : Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1755 64 : sat_d.Nest(nan_d, false);
1756 : Node* sat_val =
1757 64 : sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1758 : Node* nan_val =
1759 64 : nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1760 64 : return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1761 : }
1762 :
1763 8 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1764 : MachineOperatorBuilder* m = mcgraph()->machine();
1765 : // asm.js must use the wacky JS semantics.
1766 8 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1767 16 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1768 : }
1769 :
1770 108 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1771 : MachineOperatorBuilder* m = mcgraph()->machine();
1772 : // asm.js must use the wacky JS semantics.
1773 216 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1774 : }
1775 :
1776 8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1777 : MachineOperatorBuilder* m = mcgraph()->machine();
1778 : // asm.js must use the wacky JS semantics.
1779 8 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1780 16 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1781 : }
1782 :
1783 8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1784 : MachineOperatorBuilder* m = mcgraph()->machine();
1785 : // asm.js must use the wacky JS semantics.
1786 16 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1787 : }
1788 :
1789 0 : Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1790 : MachineRepresentation input_type) {
1791 : Node* stack_slot_param =
1792 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1793 :
1794 0 : const Operator* store_op = mcgraph()->machine()->Store(
1795 0 : StoreRepresentation(input_type, kNoWriteBarrier));
1796 0 : SetEffect(graph()->NewNode(store_op, stack_slot_param,
1797 : mcgraph()->Int32Constant(0), input, Effect(),
1798 : Control()));
1799 :
1800 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1801 : MachineSignature sig(1, 1, sig_types);
1802 :
1803 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1804 :
1805 0 : return BuildCCall(&sig, function, stack_slot_param);
1806 : }
1807 :
1808 0 : Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1809 0 : return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1810 0 : MachineRepresentation::kWord32);
1811 : }
1812 :
1813 0 : Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1814 0 : return Unop(wasm::kExprI64UConvertI32,
1815 : BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1816 0 : MachineRepresentation::kWord64));
1817 : }
1818 :
1819 0 : Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1820 0 : return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1821 0 : MachineRepresentation::kWord32);
1822 : }
1823 :
1824 0 : Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1825 0 : return Unop(
1826 : wasm::kExprI64UConvertI32,
1827 : BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1828 0 : MachineRepresentation::kWord64));
1829 : }
1830 :
1831 0 : Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1832 0 : MachineType type = MachineType::Float32();
1833 0 : ExternalReference ref = ExternalReference::wasm_f32_trunc();
1834 :
1835 0 : return BuildCFuncInstruction(ref, type, input);
1836 : }
1837 :
1838 0 : Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1839 0 : MachineType type = MachineType::Float32();
1840 0 : ExternalReference ref = ExternalReference::wasm_f32_floor();
1841 0 : return BuildCFuncInstruction(ref, type, input);
1842 : }
1843 :
1844 0 : Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1845 0 : MachineType type = MachineType::Float32();
1846 0 : ExternalReference ref = ExternalReference::wasm_f32_ceil();
1847 0 : return BuildCFuncInstruction(ref, type, input);
1848 : }
1849 :
1850 0 : Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1851 0 : MachineType type = MachineType::Float32();
1852 0 : ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1853 0 : return BuildCFuncInstruction(ref, type, input);
1854 : }
1855 :
1856 0 : Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1857 0 : MachineType type = MachineType::Float64();
1858 0 : ExternalReference ref = ExternalReference::wasm_f64_trunc();
1859 0 : return BuildCFuncInstruction(ref, type, input);
1860 : }
1861 :
1862 0 : Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1863 0 : MachineType type = MachineType::Float64();
1864 0 : ExternalReference ref = ExternalReference::wasm_f64_floor();
1865 0 : return BuildCFuncInstruction(ref, type, input);
1866 : }
1867 :
1868 0 : Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1869 0 : MachineType type = MachineType::Float64();
1870 0 : ExternalReference ref = ExternalReference::wasm_f64_ceil();
1871 0 : return BuildCFuncInstruction(ref, type, input);
1872 : }
1873 :
1874 0 : Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1875 0 : MachineType type = MachineType::Float64();
1876 0 : ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1877 0 : return BuildCFuncInstruction(ref, type, input);
1878 : }
1879 :
1880 21 : Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1881 21 : MachineType type = MachineType::Float64();
1882 21 : ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1883 21 : return BuildCFuncInstruction(ref, type, input);
1884 : }
1885 :
1886 22 : Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1887 22 : MachineType type = MachineType::Float64();
1888 22 : ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1889 22 : return BuildCFuncInstruction(ref, type, input);
1890 : }
1891 :
1892 9 : Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1893 9 : MachineType type = MachineType::Float64();
1894 9 : ExternalReference ref = ExternalReference::wasm_float64_pow();
1895 9 : return BuildCFuncInstruction(ref, type, left, right);
1896 : }
1897 :
1898 20 : Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1899 20 : MachineType type = MachineType::Float64();
1900 20 : ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1901 20 : return BuildCFuncInstruction(ref, type, left, right);
1902 : }
1903 :
1904 72 : Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1905 : MachineType type, Node* input0,
1906 : Node* input1) {
1907 : // We do truncation by calling a C function which calculates the result.
1908 : // The input is passed to the C function as a byte buffer holding the two
1909 : // input doubles. We reserve this byte buffer as a stack slot, store the
1910 : // parameters in this buffer slots, pass a pointer to the buffer to the C
1911 : // function, and after calling the C function we collect the return value from
1912 : // the buffer.
1913 :
1914 : const int type_size = ElementSizeInBytes(type.representation());
1915 72 : const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1916 : Node* stack_slot =
1917 72 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1918 :
1919 72 : const Operator* store_op = mcgraph()->machine()->Store(
1920 72 : StoreRepresentation(type.representation(), kNoWriteBarrier));
1921 72 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1922 : input0, Effect(), Control()));
1923 :
1924 72 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1925 :
1926 72 : if (input1 != nullptr) {
1927 29 : SetEffect(graph()->NewNode(store_op, stack_slot,
1928 : mcgraph()->Int32Constant(type_size), input1,
1929 : Effect(), Control()));
1930 : }
1931 :
1932 72 : MachineType sig_types[] = {MachineType::Pointer()};
1933 : MachineSignature sig(0, 1, sig_types);
1934 72 : BuildCCall(&sig, function, stack_slot);
1935 :
1936 72 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
1937 : stack_slot, mcgraph()->Int32Constant(0),
1938 72 : Effect(), Control()));
1939 : }
1940 :
1941 0 : Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1942 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1943 0 : return BuildIntToFloatConversionInstruction(
1944 : input, ExternalReference::wasm_int64_to_float32(),
1945 0 : MachineRepresentation::kWord64, MachineType::Float32());
1946 : }
1947 0 : Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1948 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1949 0 : return BuildIntToFloatConversionInstruction(
1950 : input, ExternalReference::wasm_uint64_to_float32(),
1951 0 : MachineRepresentation::kWord64, MachineType::Float32());
1952 : }
1953 0 : Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1954 0 : return BuildIntToFloatConversionInstruction(
1955 : input, ExternalReference::wasm_int64_to_float64(),
1956 0 : MachineRepresentation::kWord64, MachineType::Float64());
1957 : }
1958 0 : Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1959 0 : return BuildIntToFloatConversionInstruction(
1960 : input, ExternalReference::wasm_uint64_to_float64(),
1961 0 : MachineRepresentation::kWord64, MachineType::Float64());
1962 : }
1963 :
1964 0 : Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1965 : Node* input, ExternalReference ref,
1966 : MachineRepresentation parameter_representation,
1967 : const MachineType result_type) {
1968 : int stack_slot_size =
1969 : std::max(ElementSizeInBytes(parameter_representation),
1970 0 : ElementSizeInBytes(result_type.representation()));
1971 : Node* stack_slot =
1972 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1973 0 : const Operator* store_op = mcgraph()->machine()->Store(
1974 0 : StoreRepresentation(parameter_representation, kNoWriteBarrier));
1975 0 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1976 : input, Effect(), Control()));
1977 0 : MachineType sig_types[] = {MachineType::Pointer()};
1978 : MachineSignature sig(0, 1, sig_types);
1979 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1980 0 : BuildCCall(&sig, function, stack_slot);
1981 0 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
1982 : stack_slot, mcgraph()->Int32Constant(0),
1983 0 : Effect(), Control()));
1984 : }
1985 :
1986 : namespace {
1987 :
1988 0 : ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1989 : wasm::WasmOpcode opcode) {
1990 0 : switch (opcode) {
1991 : case wasm::kExprI64SConvertF32:
1992 : case wasm::kExprI64SConvertSatF32:
1993 0 : return ExternalReference::wasm_float32_to_int64();
1994 : case wasm::kExprI64UConvertF32:
1995 : case wasm::kExprI64UConvertSatF32:
1996 0 : return ExternalReference::wasm_float32_to_uint64();
1997 : case wasm::kExprI64SConvertF64:
1998 : case wasm::kExprI64SConvertSatF64:
1999 0 : return ExternalReference::wasm_float64_to_int64();
2000 : case wasm::kExprI64UConvertF64:
2001 : case wasm::kExprI64UConvertSatF64:
2002 0 : return ExternalReference::wasm_float64_to_uint64();
2003 : default:
2004 0 : UNREACHABLE();
2005 : }
2006 : }
2007 :
2008 : } // namespace
2009 :
2010 0 : Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
2011 : wasm::WasmCodePosition position,
2012 : wasm::WasmOpcode opcode) {
2013 0 : const MachineType int_ty = IntConvertType(opcode);
2014 0 : const MachineType float_ty = FloatConvertType(opcode);
2015 0 : ExternalReference call_ref = convert_ccall_ref(this, opcode);
2016 : int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
2017 0 : ElementSizeInBytes(float_ty.representation()));
2018 : Node* stack_slot =
2019 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
2020 0 : const Operator* store_op = mcgraph()->machine()->Store(
2021 0 : StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
2022 : SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
2023 : Effect(), Control()));
2024 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2025 : MachineSignature sig(1, 1, sig_types);
2026 : Node* function =
2027 0 : graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
2028 0 : Node* overflow = BuildCCall(&sig, function, stack_slot);
2029 0 : if (IsTrappingConvertOp(opcode)) {
2030 : ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
2031 0 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
2032 : stack_slot, Int32Constant(0), Effect(),
2033 0 : Control()));
2034 : }
2035 0 : Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
2036 0 : Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
2037 : tl_d.Chain(Control());
2038 0 : Node* nan_test = Binop(NeOp(float_ty), input, input);
2039 0 : Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2040 0 : nan_d.Nest(tl_d, true);
2041 0 : Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2042 0 : Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2043 0 : sat_d.Nest(nan_d, false);
2044 : Node* sat_val =
2045 0 : sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2046 : Node* load =
2047 0 : SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
2048 : Int32Constant(0), Effect(), Control()));
2049 : Node* nan_val =
2050 0 : nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2051 0 : return tl_d.Phi(int_ty.representation(), nan_val, load);
2052 : }
2053 :
2054 1307 : Node* WasmGraphBuilder::MemoryGrow(Node* input) {
2055 1307 : needs_stack_check_ = true;
2056 :
2057 : WasmMemoryGrowDescriptor interface_descriptor;
2058 1308 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2059 : mcgraph()->zone(), // zone
2060 : interface_descriptor, // descriptor
2061 : interface_descriptor.GetStackParameterCount(), // stack parameter count
2062 : CallDescriptor::kNoFlags, // flags
2063 : Operator::kNoProperties, // properties
2064 1307 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
2065 : // A direct call to a wasm runtime stub defined in this module.
2066 : // Just encode the stub index. This will be patched at relocation.
2067 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2068 1308 : wasm::WasmCode::kWasmMemoryGrow, RelocInfo::WASM_STUB_CALL);
2069 1308 : return SetEffect(
2070 : SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
2071 1306 : call_target, input, Effect(), Control())));
2072 : }
2073 :
2074 551 : Node* WasmGraphBuilder::Throw(uint32_t exception_index,
2075 : const wasm::WasmException* exception,
2076 : const Vector<Node*> values) {
2077 551 : needs_stack_check_ = true;
2078 551 : uint32_t encoded_size = WasmExceptionPackage::GetEncodedSize(exception);
2079 : Node* create_parameters[] = {
2080 551 : LoadExceptionTagFromTable(exception_index),
2081 1104 : BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2082 : Node* except_obj =
2083 550 : BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2084 552 : arraysize(create_parameters));
2085 : Node* values_array =
2086 552 : BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2087 552 : uint32_t index = 0;
2088 552 : const wasm::WasmExceptionSig* sig = exception->sig;
2089 : MachineOperatorBuilder* m = mcgraph()->machine();
2090 1124 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2091 286 : Node* value = values[i];
2092 286 : switch (sig->GetParam(i)) {
2093 : case wasm::kWasmF32:
2094 22 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2095 : V8_FALLTHROUGH;
2096 : case wasm::kWasmI32:
2097 177 : BuildEncodeException32BitValue(values_array, &index, value);
2098 176 : break;
2099 : case wasm::kWasmF64:
2100 21 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2101 : V8_FALLTHROUGH;
2102 : case wasm::kWasmI64: {
2103 43 : Node* upper32 = graph()->NewNode(
2104 : m->TruncateInt64ToInt32(),
2105 : Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2106 43 : BuildEncodeException32BitValue(values_array, &index, upper32);
2107 44 : Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2108 44 : BuildEncodeException32BitValue(values_array, &index, lower32);
2109 44 : break;
2110 : }
2111 : case wasm::kWasmS128:
2112 22 : BuildEncodeException32BitValue(
2113 : values_array, &index,
2114 22 : graph()->NewNode(m->I32x4ExtractLane(0), value));
2115 22 : BuildEncodeException32BitValue(
2116 : values_array, &index,
2117 22 : graph()->NewNode(m->I32x4ExtractLane(1), value));
2118 22 : BuildEncodeException32BitValue(
2119 : values_array, &index,
2120 21 : graph()->NewNode(m->I32x4ExtractLane(2), value));
2121 22 : BuildEncodeException32BitValue(
2122 : values_array, &index,
2123 22 : graph()->NewNode(m->I32x4ExtractLane(3), value));
2124 22 : break;
2125 : case wasm::kWasmAnyRef:
2126 176 : STORE_FIXED_ARRAY_SLOT_ANY(values_array, index, value);
2127 44 : ++index;
2128 44 : break;
2129 : default:
2130 0 : UNREACHABLE();
2131 : }
2132 : }
2133 : DCHECK_EQ(encoded_size, index);
2134 : WasmThrowDescriptor interface_descriptor;
2135 548 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2136 : mcgraph()->zone(), interface_descriptor,
2137 : interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2138 552 : Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2139 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2140 548 : wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2141 549 : return SetEffect(SetControl(
2142 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2143 552 : except_obj, Effect(), Control())));
2144 : }
2145 :
2146 350 : void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
2147 : uint32_t* index,
2148 : Node* value) {
2149 : MachineOperatorBuilder* machine = mcgraph()->machine();
2150 351 : Node* upper_halfword_as_smi = BuildChangeUint31ToSmi(
2151 352 : graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16)));
2152 1407 : STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, upper_halfword_as_smi);
2153 352 : ++(*index);
2154 352 : Node* lower_halfword_as_smi = BuildChangeUint31ToSmi(
2155 352 : graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu)));
2156 1408 : STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, lower_halfword_as_smi);
2157 352 : ++(*index);
2158 352 : }
2159 :
2160 240 : Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
2161 : uint32_t* index) {
2162 : MachineOperatorBuilder* machine = mcgraph()->machine();
2163 : Node* upper =
2164 722 : BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2165 241 : (*index)++;
2166 241 : upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2167 : Node* lower =
2168 723 : BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2169 242 : (*index)++;
2170 242 : Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2171 242 : return value;
2172 : }
2173 :
2174 21 : Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
2175 : uint32_t* index) {
2176 22 : Node* upper = Binop(wasm::kExprI64Shl,
2177 : Unop(wasm::kExprI64UConvertI32,
2178 : BuildDecodeException32BitValue(values_array, index)),
2179 21 : Int64Constant(32));
2180 21 : Node* lower = Unop(wasm::kExprI64UConvertI32,
2181 22 : BuildDecodeException32BitValue(values_array, index));
2182 22 : return Binop(wasm::kExprI64Ior, upper, lower);
2183 : }
2184 :
2185 405 : Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
2186 405 : needs_stack_check_ = true;
2187 : WasmThrowDescriptor interface_descriptor;
2188 406 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2189 : mcgraph()->zone(), interface_descriptor,
2190 : interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2191 405 : Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2192 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2193 406 : wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2194 406 : return SetEffect(SetControl(
2195 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2196 406 : except_obj, Effect(), Control())));
2197 : }
2198 :
2199 328 : Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
2200 : Node* expected_tag) {
2201 : MachineOperatorBuilder* machine = mcgraph()->machine();
2202 658 : return graph()->NewNode(machine->WordEqual(), caught_tag, expected_tag);
2203 : }
2204 :
2205 881 : Node* WasmGraphBuilder::LoadExceptionTagFromTable(uint32_t exception_index) {
2206 : Node* exceptions_table =
2207 1763 : LOAD_INSTANCE_FIELD(ExceptionsTable, MachineType::TaggedPointer());
2208 1764 : Node* tag = LOAD_FIXED_ARRAY_SLOT_PTR(exceptions_table, exception_index);
2209 881 : return tag;
2210 : }
2211 :
2212 328 : Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
2213 328 : needs_stack_check_ = true;
2214 328 : return BuildCallToRuntime(Runtime::kWasmExceptionGetTag, &except_obj, 1);
2215 : }
2216 :
2217 330 : Node** WasmGraphBuilder::GetExceptionValues(
2218 : Node* except_obj, const wasm::WasmException* exception) {
2219 : Node* values_array =
2220 330 : BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2221 329 : uint32_t index = 0;
2222 329 : const wasm::WasmExceptionSig* sig = exception->sig;
2223 329 : Node** values = Buffer(sig->parameter_count());
2224 746 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2225 : Node* value;
2226 207 : switch (sig->GetParam(i)) {
2227 : case wasm::kWasmI32:
2228 143 : value = BuildDecodeException32BitValue(values_array, &index);
2229 143 : break;
2230 : case wasm::kWasmI64:
2231 11 : value = BuildDecodeException64BitValue(values_array, &index);
2232 11 : break;
2233 : case wasm::kWasmF32: {
2234 11 : value = Unop(wasm::kExprF32ReinterpretI32,
2235 11 : BuildDecodeException32BitValue(values_array, &index));
2236 11 : break;
2237 : }
2238 : case wasm::kWasmF64: {
2239 10 : value = Unop(wasm::kExprF64ReinterpretI64,
2240 11 : BuildDecodeException64BitValue(values_array, &index));
2241 11 : break;
2242 : }
2243 : case wasm::kWasmS128:
2244 10 : value = graph()->NewNode(
2245 : mcgraph()->machine()->I32x4Splat(),
2246 : BuildDecodeException32BitValue(values_array, &index));
2247 11 : value = graph()->NewNode(
2248 : mcgraph()->machine()->I32x4ReplaceLane(1), value,
2249 : BuildDecodeException32BitValue(values_array, &index));
2250 11 : value = graph()->NewNode(
2251 : mcgraph()->machine()->I32x4ReplaceLane(2), value,
2252 : BuildDecodeException32BitValue(values_array, &index));
2253 11 : value = graph()->NewNode(
2254 : mcgraph()->machine()->I32x4ReplaceLane(3), value,
2255 : BuildDecodeException32BitValue(values_array, &index));
2256 11 : break;
2257 : case wasm::kWasmAnyRef:
2258 44 : value = LOAD_FIXED_ARRAY_SLOT_ANY(values_array, index);
2259 22 : ++index;
2260 22 : break;
2261 : default:
2262 0 : UNREACHABLE();
2263 : }
2264 209 : values[i] = value;
2265 : }
2266 : DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(exception));
2267 330 : return values;
2268 : }
2269 :
2270 14196 : Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2271 : wasm::WasmCodePosition position) {
2272 : MachineOperatorBuilder* m = mcgraph()->machine();
2273 : ZeroCheck32(wasm::kTrapDivByZero, right, position);
2274 : Node* before = Control();
2275 : Node* denom_is_m1;
2276 : Node* denom_is_not_m1;
2277 14196 : BranchExpectFalse(
2278 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2279 : &denom_is_m1, &denom_is_not_m1);
2280 14196 : SetControl(denom_is_m1);
2281 14196 : TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2282 14196 : if (Control() != denom_is_m1) {
2283 964 : SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2284 : Control()));
2285 : } else {
2286 : SetControl(before);
2287 : }
2288 28392 : return graph()->NewNode(m->Int32Div(), left, right, Control());
2289 : }
2290 :
2291 14112 : Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2292 : wasm::WasmCodePosition position) {
2293 : MachineOperatorBuilder* m = mcgraph()->machine();
2294 :
2295 : ZeroCheck32(wasm::kTrapRemByZero, right, position);
2296 :
2297 : Diamond d(
2298 : graph(), mcgraph()->common(),
2299 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2300 28224 : BranchHint::kFalse);
2301 : d.Chain(Control());
2302 :
2303 28224 : return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2304 14112 : graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2305 : }
2306 :
2307 14176 : Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2308 : wasm::WasmCodePosition position) {
2309 : MachineOperatorBuilder* m = mcgraph()->machine();
2310 14176 : return graph()->NewNode(m->Uint32Div(), left, right,
2311 14176 : ZeroCheck32(wasm::kTrapDivByZero, right, position));
2312 : }
2313 :
2314 14108 : Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2315 : wasm::WasmCodePosition position) {
2316 : MachineOperatorBuilder* m = mcgraph()->machine();
2317 14108 : return graph()->NewNode(m->Uint32Mod(), left, right,
2318 14108 : ZeroCheck32(wasm::kTrapRemByZero, right, position));
2319 : }
2320 :
2321 385 : Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2322 : MachineOperatorBuilder* m = mcgraph()->machine();
2323 :
2324 : Int32Matcher mr(right);
2325 385 : if (mr.HasValue()) {
2326 317 : if (mr.Value() == 0) {
2327 13 : return mcgraph()->Int32Constant(0);
2328 304 : } else if (mr.Value() == -1) {
2329 : // The result is the negation of the left input.
2330 26 : return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2331 : }
2332 582 : return graph()->NewNode(m->Int32Div(), left, right, Control());
2333 : }
2334 :
2335 : // asm.js semantics return 0 on divide or mod by zero.
2336 68 : if (m->Int32DivIsSafe()) {
2337 : // The hardware instruction does the right thing (e.g. arm).
2338 0 : return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2339 : }
2340 :
2341 : // Check denominator for zero.
2342 : Diamond z(
2343 : graph(), mcgraph()->common(),
2344 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2345 136 : BranchHint::kFalse);
2346 :
2347 : // Check numerator for -1. (avoid minint / -1 case).
2348 : Diamond n(
2349 : graph(), mcgraph()->common(),
2350 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2351 136 : BranchHint::kFalse);
2352 :
2353 68 : Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2354 : Node* neg =
2355 68 : graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2356 :
2357 68 : return n.Phi(
2358 : MachineRepresentation::kWord32, neg,
2359 68 : z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2360 : }
2361 :
2362 421 : Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2363 : CommonOperatorBuilder* c = mcgraph()->common();
2364 : MachineOperatorBuilder* m = mcgraph()->machine();
2365 421 : Node* const zero = mcgraph()->Int32Constant(0);
2366 :
2367 : Int32Matcher mr(right);
2368 421 : if (mr.HasValue()) {
2369 360 : if (mr.Value() == 0 || mr.Value() == -1) {
2370 : return zero;
2371 : }
2372 668 : return graph()->NewNode(m->Int32Mod(), left, right, Control());
2373 : }
2374 :
2375 : // General case for signed integer modulus, with optimization for (unknown)
2376 : // power of 2 right hand side.
2377 : //
2378 : // if 0 < right then
2379 : // msk = right - 1
2380 : // if right & msk != 0 then
2381 : // left % right
2382 : // else
2383 : // if left < 0 then
2384 : // -(-left & msk)
2385 : // else
2386 : // left & msk
2387 : // else
2388 : // if right < -1 then
2389 : // left % right
2390 : // else
2391 : // zero
2392 : //
2393 : // Note: We do not use the Diamond helper class here, because it really hurts
2394 : // readability with nested diamonds.
2395 61 : Node* const minus_one = mcgraph()->Int32Constant(-1);
2396 :
2397 61 : const Operator* const merge_op = c->Merge(2);
2398 61 : const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2399 :
2400 61 : Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2401 : Node* branch0 =
2402 61 : graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2403 :
2404 61 : Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2405 : Node* true0;
2406 : {
2407 61 : Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2408 :
2409 61 : Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2410 61 : Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2411 :
2412 61 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2413 61 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2414 :
2415 61 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2416 : Node* false1;
2417 : {
2418 61 : Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2419 : Node* branch2 =
2420 61 : graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2421 :
2422 61 : Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2423 183 : Node* true2 = graph()->NewNode(
2424 : m->Int32Sub(), zero,
2425 : graph()->NewNode(m->Word32And(),
2426 : graph()->NewNode(m->Int32Sub(), zero, left), msk));
2427 :
2428 61 : Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2429 61 : Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2430 :
2431 : if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2432 : false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2433 : }
2434 :
2435 : if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2436 : true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2437 : }
2438 :
2439 61 : Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2440 : Node* false0;
2441 : {
2442 61 : Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2443 : Node* branch1 =
2444 61 : graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2445 :
2446 61 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2447 61 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2448 :
2449 61 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2450 : Node* false1 = zero;
2451 :
2452 : if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2453 : false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2454 : }
2455 :
2456 : Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2457 61 : return graph()->NewNode(phi_op, true0, false0, merge0);
2458 : }
2459 :
2460 137 : Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2461 : MachineOperatorBuilder* m = mcgraph()->machine();
2462 : // asm.js semantics return 0 on divide or mod by zero.
2463 137 : if (m->Uint32DivIsSafe()) {
2464 : // The hardware instruction does the right thing (e.g. arm).
2465 0 : return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2466 : }
2467 :
2468 : // Explicit check for x % 0.
2469 : Diamond z(
2470 : graph(), mcgraph()->common(),
2471 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2472 274 : BranchHint::kFalse);
2473 :
2474 274 : return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2475 : graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2476 137 : z.if_false));
2477 : }
2478 :
2479 186 : Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2480 : MachineOperatorBuilder* m = mcgraph()->machine();
2481 : // asm.js semantics return 0 on divide or mod by zero.
2482 : // Explicit check for x % 0.
2483 : Diamond z(
2484 : graph(), mcgraph()->common(),
2485 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2486 372 : BranchHint::kFalse);
2487 :
2488 186 : Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2489 : z.if_false);
2490 186 : return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2491 186 : rem);
2492 : }
2493 :
2494 720 : Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2495 : wasm::WasmCodePosition position) {
2496 720 : if (mcgraph()->machine()->Is32()) {
2497 0 : return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2498 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2499 : }
2500 : ZeroCheck64(wasm::kTrapDivByZero, right, position);
2501 : Node* before = Control();
2502 : Node* denom_is_m1;
2503 : Node* denom_is_not_m1;
2504 720 : BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2505 : mcgraph()->Int64Constant(-1)),
2506 : &denom_is_m1, &denom_is_not_m1);
2507 720 : SetControl(denom_is_m1);
2508 : TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2509 720 : std::numeric_limits<int64_t>::min(), position);
2510 720 : if (Control() != denom_is_m1) {
2511 716 : SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2512 : Control()));
2513 : } else {
2514 : SetControl(before);
2515 : }
2516 720 : return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2517 720 : Control());
2518 : }
2519 :
2520 644 : Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2521 : wasm::WasmCodePosition position) {
2522 644 : if (mcgraph()->machine()->Is32()) {
2523 0 : return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2524 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2525 : }
2526 : ZeroCheck64(wasm::kTrapRemByZero, right, position);
2527 : Diamond d(mcgraph()->graph(), mcgraph()->common(),
2528 : graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2529 1288 : mcgraph()->Int64Constant(-1)));
2530 :
2531 : d.Chain(Control());
2532 :
2533 644 : Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2534 : d.if_false);
2535 :
2536 644 : return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2537 644 : rem);
2538 : }
2539 :
2540 660 : Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2541 : wasm::WasmCodePosition position) {
2542 660 : if (mcgraph()->machine()->Is32()) {
2543 0 : return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2544 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2545 : }
2546 660 : return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2547 660 : ZeroCheck64(wasm::kTrapDivByZero, right, position));
2548 : }
2549 647 : Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2550 : wasm::WasmCodePosition position) {
2551 647 : if (mcgraph()->machine()->Is32()) {
2552 0 : return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2553 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2554 : }
2555 647 : return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2556 647 : ZeroCheck64(wasm::kTrapRemByZero, right, position));
2557 : }
2558 :
2559 0 : Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2560 : ExternalReference ref,
2561 : MachineType result_type,
2562 : wasm::TrapReason trap_zero,
2563 : wasm::WasmCodePosition position) {
2564 : Node* stack_slot =
2565 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2566 :
2567 0 : const Operator* store_op = mcgraph()->machine()->Store(
2568 0 : StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2569 0 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2570 : left, Effect(), Control()));
2571 0 : SetEffect(graph()->NewNode(store_op, stack_slot,
2572 : mcgraph()->Int32Constant(sizeof(double)), right,
2573 : Effect(), Control()));
2574 :
2575 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2576 : MachineSignature sig(1, 1, sig_types);
2577 :
2578 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2579 0 : Node* call = BuildCCall(&sig, function, stack_slot);
2580 :
2581 : ZeroCheck32(trap_zero, call, position);
2582 0 : TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2583 0 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
2584 : stack_slot, mcgraph()->Int32Constant(0),
2585 0 : Effect(), Control()));
2586 : }
2587 :
2588 : template <typename... Args>
2589 234 : Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2590 : Args... args) {
2591 : DCHECK_LE(sig->return_count(), 1);
2592 : DCHECK_EQ(sizeof...(args), sig->parameter_count());
2593 702 : Node* const call_args[] = {function, args..., Effect(), Control()};
2594 :
2595 : auto call_descriptor =
2596 234 : Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2597 :
2598 234 : const Operator* op = mcgraph()->common()->Call(call_descriptor);
2599 468 : return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
2600 : }
2601 :
2602 285218 : Node* WasmGraphBuilder::BuildCallNode(wasm::FunctionSig* sig, Node** args,
2603 : wasm::WasmCodePosition position,
2604 : Node* instance_node, const Operator* op) {
2605 285218 : if (instance_node == nullptr) {
2606 : DCHECK_NOT_NULL(instance_node_);
2607 : instance_node = instance_node_.get();
2608 : }
2609 285218 : needs_stack_check_ = true;
2610 : const size_t params = sig->parameter_count();
2611 : const size_t extra = 3; // instance_node, effect, and control.
2612 285218 : const size_t count = 1 + params + extra;
2613 :
2614 : // Reallocate the buffer to make space for extra inputs.
2615 285218 : args = Realloc(args, 1 + params, count);
2616 :
2617 : // Make room for the instance_node parameter at index 1, just after code.
2618 285299 : memmove(&args[2], &args[1], params * sizeof(Node*));
2619 285299 : args[1] = instance_node;
2620 :
2621 : // Add effect and control inputs.
2622 570598 : args[params + 2] = Effect();
2623 570598 : args[params + 3] = Control();
2624 :
2625 285299 : Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
2626 : DCHECK(position == wasm::kNoCodePosition || position > 0);
2627 285391 : if (position > 0) SetSourcePosition(call, position);
2628 :
2629 285469 : return call;
2630 : }
2631 :
2632 285061 : Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2633 : Node*** rets,
2634 : wasm::WasmCodePosition position,
2635 : Node* instance_node,
2636 : UseRetpoline use_retpoline) {
2637 : auto call_descriptor =
2638 285061 : GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2639 285153 : const Operator* op = mcgraph()->common()->Call(call_descriptor);
2640 285078 : Node* call = BuildCallNode(sig, args, position, instance_node, op);
2641 :
2642 : size_t ret_count = sig->return_count();
2643 285266 : if (ret_count == 0) return call; // No return value.
2644 :
2645 147786 : *rets = Buffer(ret_count);
2646 147785 : if (ret_count == 1) {
2647 : // Only a single return value.
2648 146679 : (*rets)[0] = call;
2649 : } else {
2650 : // Create projections for all return values.
2651 5552 : for (size_t i = 0; i < ret_count; i++) {
2652 2223 : (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2653 2223 : graph()->start());
2654 : }
2655 : }
2656 : return call;
2657 : }
2658 :
2659 218 : Node* WasmGraphBuilder::BuildWasmReturnCall(wasm::FunctionSig* sig, Node** args,
2660 : wasm::WasmCodePosition position,
2661 : Node* instance_node,
2662 : UseRetpoline use_retpoline) {
2663 : auto call_descriptor =
2664 218 : GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2665 219 : const Operator* op = mcgraph()->common()->TailCall(call_descriptor);
2666 218 : Node* call = BuildCallNode(sig, args, position, instance_node, op);
2667 :
2668 219 : MergeControlToEnd(mcgraph(), call);
2669 :
2670 218 : return call;
2671 : }
2672 :
2673 118752 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2674 : Node*** rets,
2675 : wasm::WasmCodePosition position,
2676 : int func_index,
2677 : IsReturnCall continuation) {
2678 : // Load the imported function refs array from the instance.
2679 : Node* imported_function_refs =
2680 237522 : LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2681 : Node* ref_node =
2682 118772 : LOAD_FIXED_ARRAY_SLOT_PTR(imported_function_refs, func_index);
2683 :
2684 : // Load the target from the imported_targets array at a known offset.
2685 : Node* imported_targets =
2686 237539 : LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2687 118780 : Node* target_node = SetEffect(graph()->NewNode(
2688 : mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2689 : mcgraph()->Int32Constant(func_index * kSystemPointerSize), Effect(),
2690 : Control()));
2691 118783 : args[0] = target_node;
2692 : const UseRetpoline use_retpoline =
2693 118783 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
2694 :
2695 118783 : switch (continuation) {
2696 : case kCallContinues:
2697 118764 : return BuildWasmCall(sig, args, rets, position, ref_node, use_retpoline);
2698 : case kReturnCall:
2699 : DCHECK_NULL(rets);
2700 19 : return BuildWasmReturnCall(sig, args, position, ref_node, use_retpoline);
2701 : }
2702 : }
2703 :
2704 428 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2705 : Node*** rets,
2706 : wasm::WasmCodePosition position,
2707 : Node* func_index,
2708 : IsReturnCall continuation) {
2709 : // Load the imported function refs array from the instance.
2710 : Node* imported_function_refs =
2711 856 : LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2712 : // Access fixed array at {header_size - tag + func_index * kTaggedSize}.
2713 428 : Node* imported_instances_data = graph()->NewNode(
2714 : mcgraph()->machine()->IntAdd(), imported_function_refs,
2715 : mcgraph()->IntPtrConstant(
2716 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
2717 428 : Node* func_index_times_tagged_size = graph()->NewNode(
2718 : mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
2719 : mcgraph()->Int32Constant(kTaggedSize));
2720 428 : Node* ref_node = SetEffect(
2721 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
2722 : imported_instances_data, func_index_times_tagged_size,
2723 : Effect(), Control()));
2724 :
2725 : // Load the target from the imported_targets array at the offset of
2726 : // {func_index}.
2727 : Node* func_index_times_pointersize;
2728 : if (kSystemPointerSize == kTaggedSize) {
2729 : func_index_times_pointersize = func_index_times_tagged_size;
2730 :
2731 : } else {
2732 : DCHECK_EQ(kSystemPointerSize, kTaggedSize + kTaggedSize);
2733 : func_index_times_pointersize = graph()->NewNode(
2734 : mcgraph()->machine()->Int32Add(), func_index_times_tagged_size,
2735 : func_index_times_tagged_size);
2736 : }
2737 : Node* imported_targets =
2738 856 : LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2739 428 : Node* target_node = SetEffect(graph()->NewNode(
2740 : mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2741 : func_index_times_pointersize, Effect(), Control()));
2742 428 : args[0] = target_node;
2743 : const UseRetpoline use_retpoline =
2744 428 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
2745 :
2746 428 : switch (continuation) {
2747 : case kCallContinues:
2748 428 : return BuildWasmCall(sig, args, rets, position, ref_node, use_retpoline);
2749 : case kReturnCall:
2750 : DCHECK_NULL(rets);
2751 0 : return BuildWasmReturnCall(sig, args, position, ref_node, use_retpoline);
2752 : }
2753 : }
2754 :
2755 140967 : Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2756 : wasm::WasmCodePosition position) {
2757 : DCHECK_NULL(args[0]);
2758 281934 : wasm::FunctionSig* sig = env_->module->functions[index].sig;
2759 :
2760 140967 : if (env_ && index < env_->module->num_imported_functions) {
2761 : // Call to an imported function.
2762 118744 : return BuildImportCall(sig, args, rets, position, index, kCallContinues);
2763 : }
2764 :
2765 : // A direct call to a wasm function defined in this module.
2766 : // Just encode the function index. This will be patched at instantiation.
2767 : Address code = static_cast<Address>(index);
2768 22223 : args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2769 :
2770 22291 : return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2771 : }
2772 :
2773 3870 : Node* WasmGraphBuilder::CallIndirect(uint32_t table_index, uint32_t sig_index,
2774 : Node** args, Node*** rets,
2775 : wasm::WasmCodePosition position) {
2776 3870 : if (table_index == 0) {
2777 3804 : return BuildIndirectCall(sig_index, args, rets, position, kCallContinues);
2778 : }
2779 : return BuildIndirectCall(table_index, sig_index, args, rets, position,
2780 66 : kCallContinues);
2781 : }
2782 :
2783 3840 : Node* WasmGraphBuilder::BuildIndirectCall(uint32_t sig_index, Node** args,
2784 : Node*** rets,
2785 : wasm::WasmCodePosition position,
2786 : IsReturnCall continuation) {
2787 : DCHECK_NOT_NULL(args[0]);
2788 : DCHECK_NOT_NULL(env_);
2789 :
2790 : // Assume only one table for now.
2791 7680 : wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2792 :
2793 : Node* ift_size =
2794 7684 : LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2795 :
2796 : MachineOperatorBuilder* machine = mcgraph()->machine();
2797 3841 : Node* key = args[0];
2798 :
2799 : // Bounds check against the table size.
2800 3841 : Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2801 3840 : TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2802 :
2803 : // Mask the key to prevent SSCA.
2804 3843 : if (untrusted_code_mitigations_) {
2805 : // mask = ((key - size) & ~key) >> 31
2806 : Node* neg_key =
2807 0 : graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2808 0 : Node* masked_diff = graph()->NewNode(
2809 : machine->Word32And(),
2810 : graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2811 : Node* mask =
2812 0 : graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2813 0 : key = graph()->NewNode(machine->Word32And(), key, mask);
2814 : }
2815 :
2816 : // Load signature from the table and check.
2817 : Node* ift_sig_ids =
2818 7676 : LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2819 :
2820 7682 : int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2821 3843 : Node* int32_scaled_key = Uint32ToUintptr(
2822 3842 : graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
2823 :
2824 3842 : Node* loaded_sig = SetEffect(
2825 : graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
2826 : int32_scaled_key, Effect(), Control()));
2827 3840 : Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2828 : Int32Constant(expected_sig_id));
2829 :
2830 3838 : TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2831 :
2832 : Node* ift_targets =
2833 7683 : LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2834 7682 : Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableRefs,
2835 : MachineType::TaggedPointer());
2836 :
2837 : Node* tagged_scaled_key;
2838 : if (kTaggedSize == kInt32Size) {
2839 : tagged_scaled_key = int32_scaled_key;
2840 : } else {
2841 : DCHECK_EQ(kTaggedSize, kInt32Size * 2);
2842 3835 : tagged_scaled_key = graph()->NewNode(machine->Int32Add(), int32_scaled_key,
2843 : int32_scaled_key);
2844 : }
2845 :
2846 7687 : Node* target_instance = SetEffect(graph()->NewNode(
2847 : machine->Load(MachineType::TaggedPointer()),
2848 : graph()->NewNode(machine->IntAdd(), ift_instances, tagged_scaled_key),
2849 : Int32Constant(wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)),
2850 : Effect(), Control()));
2851 :
2852 : Node* intptr_scaled_key;
2853 : if (kSystemPointerSize == kTaggedSize) {
2854 : intptr_scaled_key = tagged_scaled_key;
2855 : } else {
2856 : DCHECK_EQ(kSystemPointerSize, kTaggedSize + kTaggedSize);
2857 : intptr_scaled_key = graph()->NewNode(machine->Int32Add(), tagged_scaled_key,
2858 : tagged_scaled_key);
2859 : }
2860 :
2861 3839 : Node* target = SetEffect(
2862 : graph()->NewNode(machine->Load(MachineType::Pointer()), ift_targets,
2863 : intptr_scaled_key, Effect(), Control()));
2864 :
2865 3845 : args[0] = target;
2866 : const UseRetpoline use_retpoline =
2867 3845 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
2868 :
2869 3845 : switch (continuation) {
2870 : case kCallContinues:
2871 3807 : return BuildWasmCall(sig, args, rets, position, target_instance,
2872 3807 : use_retpoline);
2873 : case kReturnCall:
2874 38 : return BuildWasmReturnCall(sig, args, position, target_instance,
2875 38 : use_retpoline);
2876 : }
2877 : }
2878 :
2879 99 : Node* WasmGraphBuilder::BuildIndirectCall(uint32_t table_index,
2880 : uint32_t sig_index, Node** args,
2881 : Node*** rets,
2882 : wasm::WasmCodePosition position,
2883 : IsReturnCall continuation) {
2884 : DCHECK_NOT_NULL(args[0]);
2885 99 : Node* entry_index = args[0];
2886 : DCHECK_NOT_NULL(env_);
2887 : BoundsCheckTable(table_index, entry_index, position, wasm::kTrapFuncInvalid,
2888 99 : nullptr);
2889 :
2890 : DCHECK(Smi::IsValid(table_index));
2891 : DCHECK(Smi::IsValid(sig_index));
2892 : Node* runtime_args[]{
2893 99 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
2894 99 : BuildChangeUint31ToSmi(entry_index),
2895 297 : graph()->NewNode(mcgraph()->common()->NumberConstant(sig_index))};
2896 :
2897 : Node* target_instance = BuildCallToRuntime(
2898 : Runtime::kWasmIndirectCallCheckSignatureAndGetTargetInstance,
2899 99 : runtime_args, arraysize(runtime_args));
2900 :
2901 : // We reuse the runtime_args array here, even though we only need the first
2902 : // two arguments.
2903 : Node* call_target = BuildCallToRuntime(
2904 99 : Runtime::kWasmIndirectCallGetTargetAddress, runtime_args, 2);
2905 :
2906 198 : wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2907 99 : args[0] = call_target;
2908 : const UseRetpoline use_retpoline =
2909 99 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
2910 :
2911 99 : switch (continuation) {
2912 : case kCallContinues:
2913 66 : return BuildWasmCall(sig, args, rets, position, target_instance,
2914 66 : use_retpoline);
2915 : case kReturnCall:
2916 33 : return BuildWasmReturnCall(sig, args, position, target_instance,
2917 33 : use_retpoline);
2918 : }
2919 : }
2920 :
2921 147 : Node* WasmGraphBuilder::ReturnCall(uint32_t index, Node** args,
2922 : wasm::WasmCodePosition position) {
2923 : DCHECK_NULL(args[0]);
2924 294 : wasm::FunctionSig* sig = env_->module->functions[index].sig;
2925 :
2926 147 : if (env_ && index < env_->module->num_imported_functions) {
2927 : // Return Call to an imported function.
2928 19 : return BuildImportCall(sig, args, nullptr, position, index, kReturnCall);
2929 : }
2930 :
2931 : // A direct tail call to a wasm function defined in this module.
2932 : // Just encode the function index. This will be patched during code
2933 : // generation.
2934 : Address code = static_cast<Address>(index);
2935 128 : args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2936 :
2937 129 : return BuildWasmReturnCall(sig, args, position, nullptr, kNoRetpoline);
2938 : }
2939 :
2940 71 : Node* WasmGraphBuilder::ReturnCallIndirect(uint32_t table_index,
2941 : uint32_t sig_index, Node** args,
2942 : wasm::WasmCodePosition position) {
2943 71 : if (table_index == 0) {
2944 38 : return BuildIndirectCall(sig_index, args, nullptr, position, kReturnCall);
2945 : }
2946 : return BuildIndirectCall(table_index, sig_index, args, nullptr, position,
2947 33 : kReturnCall);
2948 : }
2949 :
2950 13487 : Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2951 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2952 : // TODO(weiliang): support Word32Rol opcode in TurboFan.
2953 : Int32Matcher m(right);
2954 13487 : if (m.HasValue()) {
2955 13456 : return Binop(wasm::kExprI32Ror, left,
2956 26912 : mcgraph()->Int32Constant(32 - (m.Value() & 0x1F)));
2957 : } else {
2958 31 : return Binop(wasm::kExprI32Ror, left,
2959 31 : Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2960 : }
2961 : }
2962 :
2963 55 : Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2964 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2965 : // TODO(weiliang): support Word64Rol opcode in TurboFan.
2966 : Int64Matcher m(right);
2967 55 : if (m.HasValue()) {
2968 16 : return Binop(wasm::kExprI64Ror, left,
2969 32 : mcgraph()->Int64Constant(64 - (m.Value() & 0x3F)));
2970 : } else {
2971 39 : return Binop(wasm::kExprI64Ror, left,
2972 39 : Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2973 : }
2974 : }
2975 :
2976 21751 : Node* WasmGraphBuilder::Invert(Node* node) {
2977 21751 : return Unop(wasm::kExprI32Eqz, node);
2978 : }
2979 :
2980 0 : bool CanCover(Node* value, IrOpcode::Value opcode) {
2981 0 : if (value->opcode() != opcode) return false;
2982 : bool first = true;
2983 0 : for (Edge const edge : value->use_edges()) {
2984 0 : if (NodeProperties::IsControlEdge(edge)) continue;
2985 0 : if (NodeProperties::IsEffectEdge(edge)) continue;
2986 : DCHECK(NodeProperties::IsValueEdge(edge));
2987 0 : if (!first) return false;
2988 : first = false;
2989 : }
2990 0 : return true;
2991 : }
2992 :
2993 64775 : Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
2994 64775 : if (mcgraph()->machine()->Is64()) {
2995 64775 : value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2996 : }
2997 64775 : return value;
2998 : }
2999 :
3000 64775 : Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
3001 64775 : value = BuildChangeInt32ToIntPtr(value);
3002 64775 : return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
3003 64775 : BuildSmiShiftBitsConstant());
3004 : }
3005 :
3006 1579 : Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
3007 1580 : return graph()->NewNode(mcgraph()->machine()->WordShl(),
3008 1576 : Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
3009 : }
3010 :
3011 0 : Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
3012 290683 : return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
3013 : }
3014 :
3015 224329 : Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
3016 224332 : value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
3017 : BuildSmiShiftBitsConstant());
3018 224332 : if (mcgraph()->machine()->Is64()) {
3019 : value =
3020 224332 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
3021 : }
3022 224330 : return value;
3023 : }
3024 :
3025 225 : Node* WasmGraphBuilder::BuildConvertUint32ToSmiWithSaturation(Node* value,
3026 : uint32_t maxval) {
3027 : DCHECK(Smi::IsValid(maxval));
3028 : Node* max = Uint32Constant(maxval);
3029 225 : Node* check = graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(),
3030 : value, max);
3031 225 : Node* valsmi = BuildChangeUint31ToSmi(value);
3032 225 : Node* maxsmi = graph()->NewNode(mcgraph()->common()->NumberConstant(maxval));
3033 225 : Diamond d(graph(), mcgraph()->common(), check, BranchHint::kTrue);
3034 : d.Chain(Control());
3035 225 : return d.Phi(MachineRepresentation::kTagged, valsmi, maxsmi);
3036 : }
3037 :
3038 774042 : void WasmGraphBuilder::InitInstanceCache(
3039 : WasmInstanceCacheNodes* instance_cache) {
3040 : DCHECK_NOT_NULL(instance_node_);
3041 :
3042 : // Load the memory start.
3043 : instance_cache->mem_start =
3044 2323914 : LOAD_INSTANCE_FIELD(MemoryStart, MachineType::UintPtr());
3045 :
3046 : // Load the memory size.
3047 : instance_cache->mem_size =
3048 2323345 : LOAD_INSTANCE_FIELD(MemorySize, MachineType::UintPtr());
3049 :
3050 774417 : if (untrusted_code_mitigations_) {
3051 : // Load the memory mask.
3052 : instance_cache->mem_mask =
3053 0 : LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr());
3054 : } else {
3055 : // Explicitly set to nullptr to ensure a SEGV when we try to use it.
3056 774417 : instance_cache->mem_mask = nullptr;
3057 : }
3058 774417 : }
3059 :
3060 4381 : void WasmGraphBuilder::PrepareInstanceCacheForLoop(
3061 : WasmInstanceCacheNodes* instance_cache, Node* control) {
3062 : #define INTRODUCE_PHI(field, rep) \
3063 : instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
3064 : instance_cache->field, control);
3065 :
3066 8763 : INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
3067 8764 : INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
3068 4382 : if (untrusted_code_mitigations_) {
3069 0 : INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation());
3070 : }
3071 :
3072 : #undef INTRODUCE_PHI
3073 4382 : }
3074 :
3075 34696 : void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
3076 : WasmInstanceCacheNodes* from,
3077 : Node* merge) {
3078 : #define INTRODUCE_PHI(field, rep) \
3079 : if (to->field != from->field) { \
3080 : Node* vals[] = {to->field, from->field, merge}; \
3081 : to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
3082 : }
3083 :
3084 34696 : INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
3085 34697 : INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
3086 34697 : if (untrusted_code_mitigations_) {
3087 0 : INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
3088 : }
3089 :
3090 : #undef INTRODUCE_PHI
3091 34697 : }
3092 :
3093 213698 : void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
3094 : WasmInstanceCacheNodes* from,
3095 : Node* merge) {
3096 213698 : to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
3097 213700 : merge, to->mem_size, from->mem_size);
3098 213700 : to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
3099 213701 : merge, to->mem_start, from->mem_start);
3100 213701 : if (untrusted_code_mitigations_) {
3101 0 : to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
3102 0 : merge, to->mem_mask, from->mem_mask);
3103 : }
3104 213701 : }
3105 :
3106 993667 : Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
3107 : Node* merge, Node* tnode,
3108 : Node* fnode) {
3109 993667 : if (IsPhiWithMerge(tnode, merge)) {
3110 94525 : AppendToPhi(tnode, fnode);
3111 899157 : } else if (tnode != fnode) {
3112 13360 : uint32_t count = merge->InputCount();
3113 : // + 1 for the merge node.
3114 13360 : Node** vals = Buffer(count + 1);
3115 40113 : for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
3116 13360 : vals[count - 1] = fnode;
3117 13360 : vals[count] = merge;
3118 13360 : return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
3119 13359 : vals);
3120 : }
3121 : return tnode;
3122 : }
3123 :
3124 213697 : Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
3125 : Node* fnode) {
3126 213697 : if (IsPhiWithMerge(tnode, merge)) {
3127 13620 : AppendToPhi(tnode, fnode);
3128 200082 : } else if (tnode != fnode) {
3129 128 : uint32_t count = merge->InputCount();
3130 128 : Node** effects = Buffer(count);
3131 858 : for (uint32_t j = 0; j < count - 1; j++) {
3132 365 : effects[j] = tnode;
3133 : }
3134 128 : effects[count - 1] = fnode;
3135 128 : tnode = EffectPhi(count, effects, merge);
3136 : }
3137 213702 : return tnode;
3138 : }
3139 :
3140 328 : Node* WasmGraphBuilder::GetImportedMutableGlobals() {
3141 328 : if (imported_mutable_globals_ == nullptr) {
3142 : // Load imported_mutable_globals_ from the instance object at runtime.
3143 612 : imported_mutable_globals_ = graph()->NewNode(
3144 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3145 : instance_node_.get(),
3146 : mcgraph()->Int32Constant(
3147 : WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
3148 : graph()->start(), graph()->start());
3149 : }
3150 327 : return imported_mutable_globals_.get();
3151 : }
3152 :
3153 53606 : void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
3154 : const wasm::WasmGlobal& global,
3155 : Node** base_node,
3156 : Node** offset_node) {
3157 : DCHECK_NOT_NULL(instance_node_);
3158 53606 : if (global.mutability && global.imported) {
3159 152 : *base_node = SetEffect(graph()->NewNode(
3160 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3161 : GetImportedMutableGlobals(),
3162 152 : mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
3163 152 : Control()));
3164 152 : *offset_node = mcgraph()->Int32Constant(0);
3165 : } else {
3166 53454 : if (globals_start_ == nullptr) {
3167 : // Load globals_start from the instance object at runtime.
3168 : // TODO(wasm): we currently generate only one load of the {globals_start}
3169 : // start per graph, which means it can be placed anywhere by the
3170 : // scheduler. This is legal because the globals_start should never change.
3171 : // However, in some cases (e.g. if the instance object is already in a
3172 : // register), it is slightly more efficient to reload this value from the
3173 : // instance object. Since this depends on register allocation, it is not
3174 : // possible to express in the graph, and would essentially constitute a
3175 : // "mem2reg" optimization in TurboFan.
3176 24221 : globals_start_ = graph()->NewNode(
3177 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3178 : instance_node_.get(),
3179 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
3180 : graph()->start(), graph()->start());
3181 : }
3182 53455 : *base_node = globals_start_.get();
3183 53455 : *offset_node = mcgraph()->Int32Constant(global.offset);
3184 :
3185 53457 : if (mem_type == MachineType::Simd128() && global.offset != 0) {
3186 : // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
3187 10392 : *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
3188 10392 : *offset_node);
3189 10392 : *offset_node = mcgraph()->Int32Constant(0);
3190 : }
3191 : }
3192 53609 : }
3193 :
3194 176 : void WasmGraphBuilder::GetBaseAndOffsetForImportedMutableAnyRefGlobal(
3195 : const wasm::WasmGlobal& global, Node** base, Node** offset) {
3196 : // Load the base from the ImportedMutableGlobalsBuffer of the instance.
3197 352 : Node* buffers = LOAD_INSTANCE_FIELD(ImportedMutableGlobalsBuffers,
3198 : MachineType::TaggedPointer());
3199 528 : *base = LOAD_FIXED_ARRAY_SLOT_ANY(buffers, global.index);
3200 :
3201 : // For the offset we need the index of the global in the buffer, and then
3202 : // calculate the actual offset from the index. Load the index from the
3203 : // ImportedMutableGlobals array of the instance.
3204 176 : Node* index = SetEffect(
3205 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::UintPtr()),
3206 : GetImportedMutableGlobals(),
3207 176 : mcgraph()->Int32Constant(global.index * sizeof(Address)),
3208 : Effect(), Control()));
3209 :
3210 : // From the index, calculate the actual offset in the FixeArray. This
3211 : // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
3212 : // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
3213 : Node* index_times_tagged_size =
3214 176 : graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
3215 : mcgraph()->Int32Constant(kTaggedSize));
3216 176 : *offset = graph()->NewNode(
3217 : mcgraph()->machine()->IntAdd(), index_times_tagged_size,
3218 : mcgraph()->IntPtrConstant(
3219 176 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
3220 176 : }
3221 :
3222 277812 : Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
3223 : DCHECK_NOT_NULL(instance_cache_);
3224 278017 : Node* mem_start = instance_cache_->mem_start;
3225 : DCHECK_NOT_NULL(mem_start);
3226 277812 : if (offset == 0) return mem_start;
3227 45119 : return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
3228 45138 : mcgraph()->IntPtrConstant(offset));
3229 : }
3230 :
3231 491 : Node* WasmGraphBuilder::CurrentMemoryPages() {
3232 : // CurrentMemoryPages can not be called from asm.js.
3233 : DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
3234 : DCHECK_NOT_NULL(instance_cache_);
3235 491 : Node* mem_size = instance_cache_->mem_size;
3236 : DCHECK_NOT_NULL(mem_size);
3237 : Node* result =
3238 491 : graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
3239 : mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
3240 491 : if (mcgraph()->machine()->Is64()) {
3241 : result =
3242 491 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
3243 : }
3244 491 : return result;
3245 : }
3246 :
3247 798 : Node* WasmGraphBuilder::BuildLoadBuiltinFromInstance(int builtin_index) {
3248 : DCHECK(Builtins::IsBuiltinId(builtin_index));
3249 1596 : Node* isolate_root = LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
3250 1596 : return LOAD_TAGGED_POINTER(isolate_root,
3251 : IsolateData::builtin_slot_offset(builtin_index));
3252 : }
3253 :
3254 : // Only call this function for code which is not reused across instantiations,
3255 : // as we do not patch the embedded js_context.
3256 372557 : Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(
3257 : Runtime::FunctionId f, Node* js_context, Node** parameters,
3258 : int parameter_count, Node** effect, Node* control) {
3259 372557 : const Runtime::Function* fun = Runtime::FunctionForId(f);
3260 745117 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3261 372558 : mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
3262 372558 : CallDescriptor::kNoFlags);
3263 : // The CEntryStub is loaded from the instance_node so that generated code is
3264 : // Isolate independent. At the moment this is only done for CEntryStub(1).
3265 : DCHECK_EQ(1, fun->result_size);
3266 : Node* centry_stub =
3267 745121 : LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
3268 : // TODO(titzer): allow arbitrary number of runtime arguments
3269 : // At the moment we only allow 5 parameters. If more parameters are needed,
3270 : // increase this constant accordingly.
3271 : static const int kMaxParams = 5;
3272 : DCHECK_GE(kMaxParams, parameter_count);
3273 : Node* inputs[kMaxParams + 6];
3274 : int count = 0;
3275 372566 : inputs[count++] = centry_stub;
3276 1856338 : for (int i = 0; i < parameter_count; i++) {
3277 741886 : inputs[count++] = parameters[i];
3278 : }
3279 372566 : inputs[count++] =
3280 372566 : mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
3281 372552 : inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity
3282 372561 : inputs[count++] = js_context; // js_context
3283 372561 : inputs[count++] = *effect;
3284 372561 : inputs[count++] = control;
3285 :
3286 372561 : Node* call = mcgraph()->graph()->NewNode(
3287 372561 : mcgraph()->common()->Call(call_descriptor), count, inputs);
3288 372568 : *effect = call;
3289 372568 : return call;
3290 : }
3291 :
3292 371269 : Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
3293 : Node** parameters,
3294 : int parameter_count) {
3295 371269 : return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3296 371272 : parameter_count, effect_, Control());
3297 : }
3298 :
3299 36595 : Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3300 36595 : const wasm::WasmGlobal& global = env_->module->globals[index];
3301 73190 : if (wasm::ValueTypes::IsReferenceType(global.type)) {
3302 393 : if (global.mutability && global.imported) {
3303 132 : Node* base = nullptr;
3304 132 : Node* offset = nullptr;
3305 132 : GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
3306 132 : return SetEffect(
3307 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
3308 : base, offset, Effect(), Control()));
3309 : }
3310 : Node* globals_buffer =
3311 523 : LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3312 791 : return LOAD_FIXED_ARRAY_SLOT_ANY(globals_buffer, global.offset);
3313 : }
3314 :
3315 : MachineType mem_type =
3316 36202 : wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3317 36203 : Node* base = nullptr;
3318 36203 : Node* offset = nullptr;
3319 36203 : GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3320 36203 : &offset);
3321 36203 : Node* result = SetEffect(graph()->NewNode(
3322 : mcgraph()->machine()->Load(mem_type), base, offset, Effect(), Control()));
3323 : #if defined(V8_TARGET_BIG_ENDIAN)
3324 : result = BuildChangeEndiannessLoad(result, mem_type,
3325 : env_->module->globals[index].type);
3326 : #endif
3327 36204 : return result;
3328 : }
3329 :
3330 17623 : Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3331 17623 : const wasm::WasmGlobal& global = env_->module->globals[index];
3332 35246 : if (wasm::ValueTypes::IsReferenceType(global.type)) {
3333 217 : if (global.mutability && global.imported) {
3334 44 : Node* base = nullptr;
3335 44 : Node* offset = nullptr;
3336 44 : GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
3337 :
3338 132 : return SetEffect(graph()->NewNode(
3339 : mcgraph()->machine()->Store(StoreRepresentation(
3340 : MachineRepresentation::kTagged, kFullWriteBarrier)),
3341 : base, offset, val, Effect(), Control()));
3342 : }
3343 : Node* globals_buffer =
3344 348 : LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3345 876 : return STORE_FIXED_ARRAY_SLOT_ANY(globals_buffer,
3346 : env_->module->globals[index].offset, val);
3347 : }
3348 :
3349 : MachineType mem_type =
3350 17406 : wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3351 17406 : Node* base = nullptr;
3352 17406 : Node* offset = nullptr;
3353 17406 : GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3354 17406 : &offset);
3355 17406 : const Operator* op = mcgraph()->machine()->Store(
3356 17406 : StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3357 : #if defined(V8_TARGET_BIG_ENDIAN)
3358 : val = BuildChangeEndiannessStore(val, mem_type.representation(),
3359 : env_->module->globals[index].type);
3360 : #endif
3361 17406 : return SetEffect(
3362 17406 : graph()->NewNode(op, base, offset, val, Effect(), Control()));
3363 : }
3364 :
3365 142 : void WasmGraphBuilder::BoundsCheckTable(uint32_t table_index, Node* entry_index,
3366 : wasm::WasmCodePosition position,
3367 : wasm::TrapReason trap_reason,
3368 : Node** base_node) {
3369 285 : Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
3370 284 : Node* table = LOAD_FIXED_ARRAY_SLOT_ANY(tables, table_index);
3371 :
3372 : int storage_field_size = WasmTableObject::kElementsOffsetEnd -
3373 : WasmTableObject::kElementsOffset + 1;
3374 143 : Node* storage = LOAD_RAW(
3375 : table, wasm::ObjectAccess::ToTagged(WasmTableObject::kElementsOffset),
3376 : assert_size(storage_field_size, MachineType::TaggedPointer()));
3377 :
3378 : int length_field_size =
3379 : FixedArray::kLengthOffsetEnd - FixedArray::kLengthOffset + 1;
3380 : Node* storage_size =
3381 142 : LOAD_RAW(storage, wasm::ObjectAccess::ToTagged(FixedArray::kLengthOffset),
3382 : assert_size(length_field_size, MachineType::TaggedSigned()));
3383 :
3384 143 : storage_size = BuildChangeSmiToInt32(storage_size);
3385 : // Bounds check against the table size.
3386 143 : Node* in_bounds = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
3387 : entry_index, storage_size);
3388 143 : TrapIfFalse(trap_reason, in_bounds, position);
3389 :
3390 143 : if (base_node) {
3391 44 : *base_node = storage;
3392 : }
3393 143 : }
3394 :
3395 43 : void WasmGraphBuilder::GetTableBaseAndOffset(uint32_t table_index,
3396 : Node* entry_index,
3397 : wasm::WasmCodePosition position,
3398 : Node** base_node,
3399 : Node** offset_node) {
3400 : BoundsCheckTable(table_index, entry_index, position,
3401 43 : wasm::kTrapTableOutOfBounds, base_node);
3402 : // From the index, calculate the actual offset in the FixeArray. This
3403 : // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
3404 : // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
3405 44 : Node* index_times_tagged_size = graph()->NewNode(
3406 : mcgraph()->machine()->IntMul(), Uint32ToUintptr(entry_index),
3407 : mcgraph()->Int32Constant(kTaggedSize));
3408 :
3409 43 : *offset_node = graph()->NewNode(
3410 : mcgraph()->machine()->IntAdd(), index_times_tagged_size,
3411 : mcgraph()->IntPtrConstant(
3412 44 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
3413 44 : }
3414 :
3415 65 : Node* WasmGraphBuilder::GetTable(uint32_t table_index, Node* index,
3416 : wasm::WasmCodePosition position) {
3417 130 : if (env_->module->tables[table_index].type == wasm::kWasmAnyRef) {
3418 21 : Node* base = nullptr;
3419 21 : Node* offset = nullptr;
3420 21 : GetTableBaseAndOffset(table_index, index, position, &base, &offset);
3421 22 : return SetEffect(
3422 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
3423 : base, offset, Effect(), Control()));
3424 : }
3425 : // We access anyfunc tables through runtime calls.
3426 : WasmTableGetDescriptor interface_descriptor;
3427 44 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3428 : mcgraph()->zone(), // zone
3429 : interface_descriptor, // descriptor
3430 : interface_descriptor.GetStackParameterCount(), // stack parameter count
3431 : CallDescriptor::kNoFlags, // flags
3432 : Operator::kNoProperties, // properties
3433 44 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
3434 : // A direct call to a wasm runtime stub defined in this module.
3435 : // Just encode the stub index. This will be patched at relocation.
3436 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
3437 44 : wasm::WasmCode::kWasmTableGet, RelocInfo::WASM_STUB_CALL);
3438 :
3439 88 : return SetEffect(SetControl(graph()->NewNode(
3440 : mcgraph()->common()->Call(call_descriptor), call_target,
3441 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)), index,
3442 : Effect(), Control())));
3443 : }
3444 :
3445 44 : Node* WasmGraphBuilder::SetTable(uint32_t table_index, Node* index, Node* val,
3446 : wasm::WasmCodePosition position) {
3447 88 : if (env_->module->tables[table_index].type == wasm::kWasmAnyRef) {
3448 22 : Node* base = nullptr;
3449 22 : Node* offset = nullptr;
3450 22 : GetTableBaseAndOffset(table_index, index, position, &base, &offset);
3451 :
3452 22 : const Operator* op = mcgraph()->machine()->Store(
3453 22 : StoreRepresentation(MachineRepresentation::kTagged, kFullWriteBarrier));
3454 :
3455 22 : Node* store = graph()->NewNode(op, base, offset, val, Effect(), Control());
3456 : return SetEffect(store);
3457 : } else {
3458 : // We access anyfunc tables through runtime calls.
3459 : WasmTableSetDescriptor interface_descriptor;
3460 22 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3461 : mcgraph()->zone(), // zone
3462 : interface_descriptor, // descriptor
3463 : interface_descriptor.GetStackParameterCount(), // stack parameter count
3464 : CallDescriptor::kNoFlags, // flags
3465 : Operator::kNoProperties, // properties
3466 22 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
3467 : // A direct call to a wasm runtime stub defined in this module.
3468 : // Just encode the stub index. This will be patched at relocation.
3469 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
3470 22 : wasm::WasmCode::kWasmTableSet, RelocInfo::WASM_STUB_CALL);
3471 :
3472 44 : return SetEffect(SetControl(graph()->NewNode(
3473 : mcgraph()->common()->Call(call_descriptor), call_target,
3474 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
3475 : index, val, Effect(), Control())));
3476 : }
3477 : }
3478 :
3479 32662 : Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3480 : uint8_t access_size, Node* index, uint32_t offset,
3481 : wasm::WasmCodePosition position) {
3482 : // Atomic operations need bounds checks until the backend can emit protected
3483 : // loads.
3484 : index =
3485 32662 : BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
3486 :
3487 32817 : const uintptr_t align_mask = access_size - 1;
3488 :
3489 : // Don't emit an alignment check if the index is a constant.
3490 : // TODO(wasm): a constant match is also done above in {BoundsCheckMem}.
3491 : UintPtrMatcher match(index);
3492 32817 : if (match.HasValue()) {
3493 26940 : uintptr_t effective_offset = match.Value() + offset;
3494 26940 : if ((effective_offset & align_mask) != 0) {
3495 : // statically known to be unaligned; trap.
3496 0 : TrapIfEq32(wasm::kTrapUnalignedAccess, Int32Constant(0), 0, position);
3497 : }
3498 : return index;
3499 : }
3500 :
3501 : // Unlike regular memory accesses, atomic memory accesses should trap if
3502 : // the effective offset is misaligned.
3503 : // TODO(wasm): this addition is redundant with one inserted by {MemBuffer}.
3504 5877 : Node* effective_offset = graph()->NewNode(mcgraph()->machine()->IntAdd(),
3505 : MemBuffer(offset), index);
3506 :
3507 11752 : Node* cond = graph()->NewNode(mcgraph()->machine()->WordAnd(),
3508 : effective_offset, IntPtrConstant(align_mask));
3509 5876 : TrapIfFalse(wasm::kTrapUnalignedAccess,
3510 : graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
3511 : mcgraph()->Int32Constant(0)),
3512 5875 : position);
3513 5879 : return index;
3514 : }
3515 :
3516 : // Insert code to bounds check a memory access if necessary. Return the
3517 : // bounds-checked index, which is guaranteed to have (the equivalent of)
3518 : // {uintptr_t} representation.
3519 272805 : Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3520 : uint32_t offset,
3521 : wasm::WasmCodePosition position,
3522 : EnforceBoundsCheck enforce_check) {
3523 : DCHECK_LE(1, access_size);
3524 272805 : index = Uint32ToUintptr(index);
3525 273167 : if (FLAG_wasm_no_bounds_checks) return index;
3526 :
3527 273266 : if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3528 : return index;
3529 : }
3530 :
3531 65856 : if (!IsInBounds(offset, access_size, env_->max_memory_size)) {
3532 : // The access will be out of bounds, even for the largest memory.
3533 19 : TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3534 19 : return mcgraph()->IntPtrConstant(0);
3535 : }
3536 32909 : uint64_t end_offset = uint64_t{offset} + access_size - 1u;
3537 32909 : Node* end_offset_node = IntPtrConstant(end_offset);
3538 :
3539 : // The accessed memory is [index + offset, index + end_offset].
3540 : // Check that the last read byte (at {index + end_offset}) is in bounds.
3541 : // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3542 : // compute {effective_size} as {mem_size - end_offset)}.
3543 : // {effective_size} is >= 1 if condition 1) holds.
3544 : // 2) Check that {index + end_offset < mem_size} by
3545 : // - computing {effective_size} as {mem_size - end_offset} and
3546 : // - checking that {index < effective_size}.
3547 :
3548 : auto m = mcgraph()->machine();
3549 32796 : Node* mem_size = instance_cache_->mem_size;
3550 32796 : if (end_offset >= env_->min_memory_size) {
3551 : // The end offset is larger than the smallest memory.
3552 : // Dynamically check the end offset against the dynamic memory size.
3553 6580 : Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
3554 6595 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3555 : } else {
3556 : // The end offset is smaller than the smallest memory, so only one check is
3557 : // required. Check to see if the index is also a constant.
3558 : UintPtrMatcher match(index);
3559 26216 : if (match.HasValue()) {
3560 : uintptr_t index_val = match.Value();
3561 26258 : if (index_val < env_->min_memory_size - end_offset) {
3562 : // The input index is a constant and everything is statically within
3563 : // bounds of the smallest possible memory.
3564 : return index;
3565 : }
3566 : }
3567 : }
3568 :
3569 : // This produces a positive number, since {end_offset < min_size <= mem_size}.
3570 : Node* effective_size =
3571 6659 : graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
3572 :
3573 : // Introduce the actual bounds check.
3574 6654 : Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
3575 6662 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3576 :
3577 6664 : if (untrusted_code_mitigations_) {
3578 : // In the fallthrough case, condition the index with the memory mask.
3579 0 : Node* mem_mask = instance_cache_->mem_mask;
3580 : DCHECK_NOT_NULL(mem_mask);
3581 0 : index = graph()->NewNode(m->WordAnd(), index, mem_mask);
3582 : }
3583 : return index;
3584 : }
3585 :
3586 281 : Node* WasmGraphBuilder::BoundsCheckRange(Node* start, Node** size, Node* max,
3587 : wasm::WasmCodePosition position) {
3588 : auto m = mcgraph()->machine();
3589 : // The region we are trying to access is [start, start+size). If
3590 : // {start} > {max}, none of this region is valid, so we trap. Otherwise,
3591 : // there may be a subset of the region that is valid. {max - start} is the
3592 : // maximum valid size, so if {max - start < size}, then the region is
3593 : // partially out-of-bounds.
3594 281 : TrapIfTrue(wasm::kTrapMemOutOfBounds,
3595 281 : graph()->NewNode(m->Uint32LessThan(), max, start), position);
3596 281 : Node* sub = graph()->NewNode(m->Int32Sub(), max, start);
3597 281 : Node* fail = graph()->NewNode(m->Uint32LessThan(), sub, *size);
3598 281 : Diamond d(graph(), mcgraph()->common(), fail, BranchHint::kFalse);
3599 : d.Chain(Control());
3600 281 : *size = d.Phi(MachineRepresentation::kWord32, sub, *size);
3601 281 : return fail;
3602 : }
3603 :
3604 205 : Node* WasmGraphBuilder::BoundsCheckMemRange(Node** start, Node** size,
3605 : wasm::WasmCodePosition position) {
3606 : // TODO(binji): Support trap handler and no bounds check mode.
3607 : Node* fail =
3608 205 : BoundsCheckRange(*start, size, instance_cache_->mem_size, position);
3609 410 : *start = graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(0),
3610 205 : Uint32ToUintptr(*start));
3611 205 : return fail;
3612 : }
3613 :
3614 372371 : const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3615 : wasm::ValueType type) {
3616 372371 : int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3617 372372 : MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3618 : if (COMPRESS_POINTERS_BOOL && mach_type.IsTagged()) {
3619 : // We are loading tagged value from off-heap location, so we need to load
3620 : // it as a full word otherwise we will not be able to decompress it.
3621 : mach_type = MachineType::Pointer();
3622 : }
3623 372375 : if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3624 : wasm::ValueTypes::MachineRepresentationFor(type))) {
3625 372371 : return mcgraph()->machine()->Load(mach_type);
3626 : }
3627 0 : return mcgraph()->machine()->UnalignedLoad(mach_type);
3628 : }
3629 :
3630 32922 : const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3631 : wasm::ValueType type) {
3632 32922 : int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3633 32922 : MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3634 : if (COMPRESS_POINTERS_BOOL && IsAnyTagged(rep)) {
3635 : // We are storing tagged value to off-heap location, so we need to store
3636 : // it as a full word otherwise we will not be able to decompress it.
3637 : rep = MachineType::PointerRepresentation();
3638 : }
3639 33666 : if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3640 : StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3641 32922 : return mcgraph()->machine()->Store(store_rep);
3642 : }
3643 : UnalignedStoreRepresentation store_rep(rep);
3644 0 : return mcgraph()->machine()->UnalignedStore(store_rep);
3645 : }
3646 :
3647 18 : Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3648 : MachineRepresentation rep,
3649 : Node* index, uint32_t offset,
3650 : wasm::WasmCodePosition position) {
3651 : int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3652 18 : Node* info = graph()->NewNode(
3653 20 : mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3654 :
3655 40 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3656 : Int32Constant(offset), index);
3657 60 : auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3658 356 : SetEffect(graph()->NewNode(
3659 : mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3660 59 : info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
3661 79 : };
3662 : // Store address, is_store, and mem_rep.
3663 : store(offsetof(wasm::MemoryTracingInfo, address),
3664 20 : MachineRepresentation::kWord32, address);
3665 19 : store(offsetof(wasm::MemoryTracingInfo, is_store),
3666 : MachineRepresentation::kWord8,
3667 20 : mcgraph()->Int32Constant(is_store ? 1 : 0));
3668 20 : store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3669 : MachineRepresentation::kWord8,
3670 20 : mcgraph()->Int32Constant(static_cast<int>(rep)));
3671 :
3672 20 : Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3673 : SetSourcePosition(call, position);
3674 19 : return call;
3675 : }
3676 :
3677 97203 : Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3678 : Node* index, uint32_t offset,
3679 : uint32_t alignment,
3680 : wasm::WasmCodePosition position) {
3681 : Node* load;
3682 :
3683 : // Wasm semantics throw on OOB. Introduce explicit bounds check and
3684 : // conditioning when not using the trap handler.
3685 : index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3686 97217 : position, kCanOmitBoundsCheck);
3687 :
3688 190876 : if (memtype.representation() == MachineRepresentation::kWord8 ||
3689 : mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3690 97354 : if (use_trap_handler()) {
3691 97314 : load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3692 : MemBuffer(offset), index, Effect(), Control());
3693 : SetSourcePosition(load, position);
3694 : } else {
3695 40 : load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3696 : MemBuffer(offset), index, Effect(), Control());
3697 : }
3698 : } else {
3699 : // TODO(eholk): Support unaligned loads with trap handlers.
3700 : DCHECK(!use_trap_handler());
3701 0 : load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3702 : MemBuffer(offset), index, Effect(), Control());
3703 : }
3704 :
3705 : SetEffect(load);
3706 :
3707 : #if defined(V8_TARGET_BIG_ENDIAN)
3708 : load = BuildChangeEndiannessLoad(load, memtype, type);
3709 : #endif
3710 :
3711 138680 : if (type == wasm::kWasmI64 &&
3712 : ElementSizeInBytes(memtype.representation()) < 8) {
3713 : // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3714 756 : if (memtype.IsSigned()) {
3715 : // sign extend
3716 381 : load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3717 : } else {
3718 : // zero extend
3719 : load =
3720 375 : graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3721 : }
3722 : }
3723 :
3724 97341 : if (FLAG_trace_wasm_memory) {
3725 : TraceMemoryOperation(false, memtype.representation(), index, offset,
3726 11 : position);
3727 : }
3728 :
3729 97342 : return load;
3730 : }
3731 :
3732 143082 : Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3733 : uint32_t offset, uint32_t alignment, Node* val,
3734 : wasm::WasmCodePosition position,
3735 : wasm::ValueType type) {
3736 : Node* store;
3737 :
3738 143154 : index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3739 143154 : position, kCanOmitBoundsCheck);
3740 :
3741 : #if defined(V8_TARGET_BIG_ENDIAN)
3742 : val = BuildChangeEndiannessStore(val, mem_rep, type);
3743 : #endif
3744 :
3745 285363 : if (mem_rep == MachineRepresentation::kWord8 ||
3746 : mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3747 143225 : if (use_trap_handler()) {
3748 : store =
3749 143225 : graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3750 : MemBuffer(offset), index, val, Effect(), Control());
3751 : SetSourcePosition(store, position);
3752 : } else {
3753 : StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3754 : store =
3755 0 : graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3756 : index, val, Effect(), Control());
3757 : }
3758 : } else {
3759 : // TODO(eholk): Support unaligned stores with trap handlers.
3760 : DCHECK(!use_trap_handler());
3761 : UnalignedStoreRepresentation rep(mem_rep);
3762 : store =
3763 0 : graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3764 : MemBuffer(offset), index, val, Effect(), Control());
3765 : }
3766 :
3767 : SetEffect(store);
3768 :
3769 143483 : if (FLAG_trace_wasm_memory) {
3770 8 : TraceMemoryOperation(true, mem_rep, index, offset, position);
3771 : }
3772 :
3773 143483 : return store;
3774 : }
3775 :
3776 : namespace {
3777 56394 : Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3778 56394 : switch (rep) {
3779 : case MachineRepresentation::kWord8:
3780 : case MachineRepresentation::kWord16:
3781 : case MachineRepresentation::kWord32:
3782 50153 : return mcgraph->Int32Constant(0);
3783 : case MachineRepresentation::kWord64:
3784 0 : return mcgraph->Int64Constant(0);
3785 : case MachineRepresentation::kFloat32:
3786 5396 : return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3787 : case MachineRepresentation::kFloat64:
3788 845 : return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3789 : default:
3790 0 : UNREACHABLE();
3791 : }
3792 : }
3793 : } // namespace
3794 :
3795 56394 : Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3796 : DCHECK_NOT_NULL(instance_cache_);
3797 56394 : Node* mem_start = instance_cache_->mem_start;
3798 56394 : Node* mem_size = instance_cache_->mem_size;
3799 : DCHECK_NOT_NULL(mem_start);
3800 : DCHECK_NOT_NULL(mem_size);
3801 :
3802 : // Asm.js semantics are defined in terms of typed arrays, hence OOB
3803 : // reads return {undefined} coerced to the result type (0 for integers, NaN
3804 : // for float and double).
3805 : // Note that we check against the memory size ignoring the size of the
3806 : // stored value, which is conservative if misaligned. Technically, asm.js
3807 : // should never have misaligned accesses.
3808 56394 : index = Uint32ToUintptr(index);
3809 : Diamond bounds_check(
3810 : graph(), mcgraph()->common(),
3811 : graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
3812 112788 : BranchHint::kTrue);
3813 : bounds_check.Chain(Control());
3814 :
3815 56394 : if (untrusted_code_mitigations_) {
3816 : // Condition the index with the memory mask.
3817 0 : Node* mem_mask = instance_cache_->mem_mask;
3818 : DCHECK_NOT_NULL(mem_mask);
3819 0 : index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
3820 : }
3821 :
3822 112788 : Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3823 : index, Effect(), bounds_check.if_true);
3824 56394 : SetEffect(bounds_check.EffectPhi(load, Effect()));
3825 56394 : SetControl(bounds_check.merge);
3826 56394 : return bounds_check.Phi(type.representation(), load,
3827 56394 : GetAsmJsOOBValue(type.representation(), mcgraph()));
3828 : }
3829 :
3830 375636 : Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3831 375636 : if (mcgraph()->machine()->Is32()) return node;
3832 : // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3833 : Uint32Matcher matcher(node);
3834 375841 : if (matcher.HasValue()) {
3835 : uintptr_t value = matcher.Value();
3836 256212 : return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3837 : }
3838 239264 : return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3839 : }
3840 :
3841 40016 : Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3842 : Node* val) {
3843 : DCHECK_NOT_NULL(instance_cache_);
3844 40016 : Node* mem_start = instance_cache_->mem_start;
3845 40016 : Node* mem_size = instance_cache_->mem_size;
3846 : DCHECK_NOT_NULL(mem_start);
3847 : DCHECK_NOT_NULL(mem_size);
3848 :
3849 : // Asm.js semantics are to ignore OOB writes.
3850 : // Note that we check against the memory size ignoring the size of the
3851 : // stored value, which is conservative if misaligned. Technically, asm.js
3852 : // should never have misaligned accesses.
3853 : Diamond bounds_check(
3854 : graph(), mcgraph()->common(),
3855 : graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3856 80032 : BranchHint::kTrue);
3857 : bounds_check.Chain(Control());
3858 :
3859 40016 : if (untrusted_code_mitigations_) {
3860 : // Condition the index with the memory mask.
3861 0 : Node* mem_mask = instance_cache_->mem_mask;
3862 : DCHECK_NOT_NULL(mem_mask);
3863 : index =
3864 0 : graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3865 : }
3866 :
3867 40016 : index = Uint32ToUintptr(index);
3868 40016 : const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3869 40016 : type.representation(), WriteBarrierKind::kNoWriteBarrier));
3870 40016 : Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
3871 : bounds_check.if_true);
3872 40016 : SetEffect(bounds_check.EffectPhi(store, Effect()));
3873 40016 : SetControl(bounds_check.merge);
3874 40016 : return val;
3875 : }
3876 :
3877 0 : void WasmGraphBuilder::PrintDebugName(Node* node) {
3878 0 : PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3879 0 : }
3880 :
3881 0 : Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3882 :
3883 : namespace {
3884 4084 : Signature<MachineRepresentation>* CreateMachineSignature(
3885 : Zone* zone, wasm::FunctionSig* sig) {
3886 : Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3887 : sig->parameter_count());
3888 12252 : for (auto ret : sig->returns()) {
3889 4084 : builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3890 : }
3891 :
3892 6756 : for (auto param : sig->parameters()) {
3893 1336 : builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3894 : }
3895 4084 : return builder.Build();
3896 : }
3897 : } // namespace
3898 :
3899 620400 : void WasmGraphBuilder::LowerInt64() {
3900 1240800 : if (mcgraph()->machine()->Is64()) return;
3901 : Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3902 : mcgraph()->zone(),
3903 0 : CreateMachineSignature(mcgraph()->zone(), sig_));
3904 0 : r.LowerGraph();
3905 : }
3906 :
3907 0 : void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3908 0 : SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3909 0 : .LowerGraph();
3910 0 : }
3911 :
3912 0 : void WasmGraphBuilder::SetSourcePosition(Node* node,
3913 : wasm::WasmCodePosition position) {
3914 : DCHECK_NE(position, wasm::kNoCodePosition);
3915 748761 : if (source_position_table_)
3916 659474 : source_position_table_->SetSourcePosition(node, SourcePosition(position));
3917 0 : }
3918 :
3919 2178 : Node* WasmGraphBuilder::S128Zero() {
3920 2178 : has_simd_ = true;
3921 4356 : return graph()->NewNode(mcgraph()->machine()->S128Zero());
3922 : }
3923 :
3924 4320 : Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3925 4320 : has_simd_ = true;
3926 4320 : switch (opcode) {
3927 : case wasm::kExprF32x4Splat:
3928 560 : return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3929 : case wasm::kExprF32x4SConvertI32x4:
3930 8 : return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3931 8 : inputs[0]);
3932 : case wasm::kExprF32x4UConvertI32x4:
3933 8 : return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3934 8 : inputs[0]);
3935 : case wasm::kExprF32x4Abs:
3936 16 : return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3937 : case wasm::kExprF32x4Neg:
3938 16 : return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3939 : case wasm::kExprF32x4RecipApprox:
3940 8 : return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3941 8 : inputs[0]);
3942 : case wasm::kExprF32x4RecipSqrtApprox:
3943 8 : return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3944 8 : inputs[0]);
3945 : case wasm::kExprF32x4Add:
3946 24 : return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3947 24 : inputs[1]);
3948 : case wasm::kExprF32x4AddHoriz:
3949 8 : return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3950 8 : inputs[1]);
3951 : case wasm::kExprF32x4Sub:
3952 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3953 8 : inputs[1]);
3954 : case wasm::kExprF32x4Mul:
3955 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3956 8 : inputs[1]);
3957 : case wasm::kExprF32x4Min:
3958 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3959 8 : inputs[1]);
3960 : case wasm::kExprF32x4Max:
3961 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3962 8 : inputs[1]);
3963 : case wasm::kExprF32x4Eq:
3964 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3965 8 : inputs[1]);
3966 : case wasm::kExprF32x4Ne:
3967 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3968 8 : inputs[1]);
3969 : case wasm::kExprF32x4Lt:
3970 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3971 8 : inputs[1]);
3972 : case wasm::kExprF32x4Le:
3973 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3974 8 : inputs[1]);
3975 : case wasm::kExprF32x4Gt:
3976 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3977 8 : inputs[0]);
3978 : case wasm::kExprF32x4Ge:
3979 8 : return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3980 8 : inputs[0]);
3981 : case wasm::kExprI32x4Splat:
3982 2552 : return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3983 : case wasm::kExprI32x4SConvertF32x4:
3984 8 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3985 8 : inputs[0]);
3986 : case wasm::kExprI32x4UConvertF32x4:
3987 8 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3988 8 : inputs[0]);
3989 : case wasm::kExprI32x4SConvertI16x8Low:
3990 8 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3991 8 : inputs[0]);
3992 : case wasm::kExprI32x4SConvertI16x8High:
3993 8 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3994 8 : inputs[0]);
3995 : case wasm::kExprI32x4Neg:
3996 16 : return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3997 : case wasm::kExprI32x4Add:
3998 24 : return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3999 24 : inputs[1]);
4000 : case wasm::kExprI32x4AddHoriz:
4001 8 : return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
4002 8 : inputs[1]);
4003 : case wasm::kExprI32x4Sub:
4004 8 : return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
4005 8 : inputs[1]);
4006 : case wasm::kExprI32x4Mul:
4007 8 : return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
4008 8 : inputs[1]);
4009 : case wasm::kExprI32x4MinS:
4010 8 : return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
4011 8 : inputs[1]);
4012 : case wasm::kExprI32x4MaxS:
4013 8 : return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
4014 8 : inputs[1]);
4015 : case wasm::kExprI32x4Eq:
4016 40 : return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
4017 40 : inputs[1]);
4018 : case wasm::kExprI32x4Ne:
4019 48 : return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
4020 48 : inputs[1]);
4021 : case wasm::kExprI32x4LtS:
4022 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
4023 8 : inputs[0]);
4024 : case wasm::kExprI32x4LeS:
4025 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
4026 8 : inputs[0]);
4027 : case wasm::kExprI32x4GtS:
4028 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
4029 8 : inputs[1]);
4030 : case wasm::kExprI32x4GeS:
4031 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
4032 8 : inputs[1]);
4033 : case wasm::kExprI32x4UConvertI16x8Low:
4034 8 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
4035 8 : inputs[0]);
4036 : case wasm::kExprI32x4UConvertI16x8High:
4037 8 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
4038 8 : inputs[0]);
4039 : case wasm::kExprI32x4MinU:
4040 8 : return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
4041 8 : inputs[1]);
4042 : case wasm::kExprI32x4MaxU:
4043 8 : return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
4044 8 : inputs[1]);
4045 : case wasm::kExprI32x4LtU:
4046 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
4047 8 : inputs[0]);
4048 : case wasm::kExprI32x4LeU:
4049 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
4050 8 : inputs[0]);
4051 : case wasm::kExprI32x4GtU:
4052 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
4053 8 : inputs[1]);
4054 : case wasm::kExprI32x4GeU:
4055 8 : return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
4056 8 : inputs[1]);
4057 : case wasm::kExprI16x8Splat:
4058 1640 : return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
4059 : case wasm::kExprI16x8SConvertI8x16Low:
4060 8 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
4061 8 : inputs[0]);
4062 : case wasm::kExprI16x8SConvertI8x16High:
4063 8 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
4064 8 : inputs[0]);
4065 : case wasm::kExprI16x8Neg:
4066 16 : return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
4067 : case wasm::kExprI16x8SConvertI32x4:
4068 8 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
4069 8 : inputs[0], inputs[1]);
4070 : case wasm::kExprI16x8Add:
4071 8 : return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
4072 8 : inputs[1]);
4073 : case wasm::kExprI16x8AddSaturateS:
4074 8 : return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
4075 8 : inputs[0], inputs[1]);
4076 : case wasm::kExprI16x8AddHoriz:
4077 8 : return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
4078 8 : inputs[1]);
4079 : case wasm::kExprI16x8Sub:
4080 8 : return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
4081 8 : inputs[1]);
4082 : case wasm::kExprI16x8SubSaturateS:
4083 8 : return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
4084 8 : inputs[0], inputs[1]);
4085 : case wasm::kExprI16x8Mul:
4086 8 : return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
4087 8 : inputs[1]);
4088 : case wasm::kExprI16x8MinS:
4089 8 : return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
4090 8 : inputs[1]);
4091 : case wasm::kExprI16x8MaxS:
4092 8 : return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
4093 8 : inputs[1]);
4094 : case wasm::kExprI16x8Eq:
4095 40 : return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
4096 40 : inputs[1]);
4097 : case wasm::kExprI16x8Ne:
4098 48 : return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
4099 48 : inputs[1]);
4100 : case wasm::kExprI16x8LtS:
4101 8 : return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
4102 8 : inputs[0]);
4103 : case wasm::kExprI16x8LeS:
4104 8 : return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
4105 8 : inputs[0]);
4106 : case wasm::kExprI16x8GtS:
4107 8 : return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
4108 8 : inputs[1]);
4109 : case wasm::kExprI16x8GeS:
4110 8 : return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
4111 8 : inputs[1]);
4112 : case wasm::kExprI16x8UConvertI8x16Low:
4113 8 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
4114 8 : inputs[0]);
4115 : case wasm::kExprI16x8UConvertI8x16High:
4116 8 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
4117 8 : inputs[0]);
4118 : case wasm::kExprI16x8UConvertI32x4:
4119 8 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
4120 8 : inputs[0], inputs[1]);
4121 : case wasm::kExprI16x8AddSaturateU:
4122 8 : return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
4123 8 : inputs[0], inputs[1]);
4124 : case wasm::kExprI16x8SubSaturateU:
4125 8 : return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
4126 8 : inputs[0], inputs[1]);
4127 : case wasm::kExprI16x8MinU:
4128 8 : return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
4129 8 : inputs[1]);
4130 : case wasm::kExprI16x8MaxU:
4131 8 : return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
4132 8 : inputs[1]);
4133 : case wasm::kExprI16x8LtU:
4134 12 : return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
4135 12 : inputs[0]);
4136 : case wasm::kExprI16x8LeU:
4137 12 : return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
4138 12 : inputs[0]);
4139 : case wasm::kExprI16x8GtU:
4140 12 : return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
4141 12 : inputs[1]);
4142 : case wasm::kExprI16x8GeU:
4143 12 : return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
4144 12 : inputs[1]);
4145 : case wasm::kExprI8x16Splat:
4146 1208 : return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
4147 : case wasm::kExprI8x16Neg:
4148 16 : return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
4149 : case wasm::kExprI8x16SConvertI16x8:
4150 8 : return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
4151 8 : inputs[0], inputs[1]);
4152 : case wasm::kExprI8x16Add:
4153 8 : return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
4154 8 : inputs[1]);
4155 : case wasm::kExprI8x16AddSaturateS:
4156 8 : return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
4157 8 : inputs[0], inputs[1]);
4158 : case wasm::kExprI8x16Sub:
4159 8 : return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
4160 8 : inputs[1]);
4161 : case wasm::kExprI8x16SubSaturateS:
4162 8 : return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
4163 8 : inputs[0], inputs[1]);
4164 : case wasm::kExprI8x16Mul:
4165 8 : return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
4166 8 : inputs[1]);
4167 : case wasm::kExprI8x16MinS:
4168 8 : return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
4169 8 : inputs[1]);
4170 : case wasm::kExprI8x16MaxS:
4171 8 : return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
4172 8 : inputs[1]);
4173 : case wasm::kExprI8x16Eq:
4174 40 : return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
4175 40 : inputs[1]);
4176 : case wasm::kExprI8x16Ne:
4177 48 : return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
4178 48 : inputs[1]);
4179 : case wasm::kExprI8x16LtS:
4180 8 : return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
4181 8 : inputs[0]);
4182 : case wasm::kExprI8x16LeS:
4183 8 : return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
4184 8 : inputs[0]);
4185 : case wasm::kExprI8x16GtS:
4186 8 : return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
4187 8 : inputs[1]);
4188 : case wasm::kExprI8x16GeS:
4189 8 : return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
4190 8 : inputs[1]);
4191 : case wasm::kExprI8x16UConvertI16x8:
4192 8 : return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
4193 8 : inputs[0], inputs[1]);
4194 : case wasm::kExprI8x16AddSaturateU:
4195 8 : return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
4196 8 : inputs[0], inputs[1]);
4197 : case wasm::kExprI8x16SubSaturateU:
4198 8 : return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
4199 8 : inputs[0], inputs[1]);
4200 : case wasm::kExprI8x16MinU:
4201 8 : return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
4202 8 : inputs[1]);
4203 : case wasm::kExprI8x16MaxU:
4204 8 : return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
4205 8 : inputs[1]);
4206 : case wasm::kExprI8x16LtU:
4207 12 : return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
4208 12 : inputs[0]);
4209 : case wasm::kExprI8x16LeU:
4210 12 : return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
4211 12 : inputs[0]);
4212 : case wasm::kExprI8x16GtU:
4213 12 : return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
4214 12 : inputs[1]);
4215 : case wasm::kExprI8x16GeU:
4216 12 : return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
4217 12 : inputs[1]);
4218 : case wasm::kExprS128And:
4219 8 : return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
4220 8 : inputs[1]);
4221 : case wasm::kExprS128Or:
4222 8 : return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
4223 8 : inputs[1]);
4224 : case wasm::kExprS128Xor:
4225 8 : return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
4226 8 : inputs[1]);
4227 : case wasm::kExprS128Not:
4228 16 : return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
4229 : case wasm::kExprS128Select:
4230 44 : return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[2],
4231 44 : inputs[0], inputs[1]);
4232 : case wasm::kExprS1x4AnyTrue:
4233 72 : return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
4234 : case wasm::kExprS1x4AllTrue:
4235 72 : return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
4236 : case wasm::kExprS1x8AnyTrue:
4237 72 : return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
4238 : case wasm::kExprS1x8AllTrue:
4239 72 : return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
4240 : case wasm::kExprS1x16AnyTrue:
4241 72 : return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
4242 : case wasm::kExprS1x16AllTrue:
4243 72 : return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
4244 : default:
4245 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4246 : }
4247 : }
4248 :
4249 784 : Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
4250 : Node* const* inputs) {
4251 784 : has_simd_ = true;
4252 784 : switch (opcode) {
4253 : case wasm::kExprF32x4ExtractLane:
4254 64 : return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
4255 64 : inputs[0]);
4256 : case wasm::kExprF32x4ReplaceLane:
4257 64 : return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
4258 64 : inputs[0], inputs[1]);
4259 : case wasm::kExprI32x4ExtractLane:
4260 168 : return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
4261 168 : inputs[0]);
4262 : case wasm::kExprI32x4ReplaceLane:
4263 104 : return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
4264 104 : inputs[0], inputs[1]);
4265 : case wasm::kExprI16x8ExtractLane:
4266 64 : return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
4267 64 : inputs[0]);
4268 : case wasm::kExprI16x8ReplaceLane:
4269 96 : return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
4270 96 : inputs[0], inputs[1]);
4271 : case wasm::kExprI8x16ExtractLane:
4272 64 : return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
4273 64 : inputs[0]);
4274 : case wasm::kExprI8x16ReplaceLane:
4275 160 : return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
4276 160 : inputs[0], inputs[1]);
4277 : default:
4278 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4279 : }
4280 : }
4281 :
4282 1272 : Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
4283 : Node* const* inputs) {
4284 1272 : has_simd_ = true;
4285 1272 : switch (opcode) {
4286 : case wasm::kExprI32x4Shl:
4287 496 : return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
4288 : case wasm::kExprI32x4ShrS:
4289 248 : return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
4290 248 : inputs[0]);
4291 : case wasm::kExprI32x4ShrU:
4292 248 : return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
4293 248 : inputs[0]);
4294 : case wasm::kExprI16x8Shl:
4295 240 : return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
4296 : case wasm::kExprI16x8ShrS:
4297 120 : return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
4298 120 : inputs[0]);
4299 : case wasm::kExprI16x8ShrU:
4300 120 : return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
4301 120 : inputs[0]);
4302 : case wasm::kExprI8x16Shl:
4303 112 : return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
4304 : case wasm::kExprI8x16ShrS:
4305 56 : return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
4306 56 : inputs[0]);
4307 : case wasm::kExprI8x16ShrU:
4308 56 : return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
4309 56 : inputs[0]);
4310 : default:
4311 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4312 : }
4313 : }
4314 :
4315 14448 : Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
4316 : Node* const* inputs) {
4317 14448 : has_simd_ = true;
4318 14448 : return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
4319 14448 : inputs[0], inputs[1]);
4320 : }
4321 :
4322 : #define ATOMIC_BINOP_LIST(V) \
4323 : V(I32AtomicAdd, Add, Uint32, Word32) \
4324 : V(I64AtomicAdd, Add, Uint64, Word64) \
4325 : V(I32AtomicAdd8U, Add, Uint8, Word32) \
4326 : V(I32AtomicAdd16U, Add, Uint16, Word32) \
4327 : V(I64AtomicAdd8U, Add, Uint8, Word64) \
4328 : V(I64AtomicAdd16U, Add, Uint16, Word64) \
4329 : V(I64AtomicAdd32U, Add, Uint32, Word64) \
4330 : V(I32AtomicSub, Sub, Uint32, Word32) \
4331 : V(I64AtomicSub, Sub, Uint64, Word64) \
4332 : V(I32AtomicSub8U, Sub, Uint8, Word32) \
4333 : V(I32AtomicSub16U, Sub, Uint16, Word32) \
4334 : V(I64AtomicSub8U, Sub, Uint8, Word64) \
4335 : V(I64AtomicSub16U, Sub, Uint16, Word64) \
4336 : V(I64AtomicSub32U, Sub, Uint32, Word64) \
4337 : V(I32AtomicAnd, And, Uint32, Word32) \
4338 : V(I64AtomicAnd, And, Uint64, Word64) \
4339 : V(I32AtomicAnd8U, And, Uint8, Word32) \
4340 : V(I64AtomicAnd16U, And, Uint16, Word64) \
4341 : V(I32AtomicAnd16U, And, Uint16, Word32) \
4342 : V(I64AtomicAnd8U, And, Uint8, Word64) \
4343 : V(I64AtomicAnd32U, And, Uint32, Word64) \
4344 : V(I32AtomicOr, Or, Uint32, Word32) \
4345 : V(I64AtomicOr, Or, Uint64, Word64) \
4346 : V(I32AtomicOr8U, Or, Uint8, Word32) \
4347 : V(I32AtomicOr16U, Or, Uint16, Word32) \
4348 : V(I64AtomicOr8U, Or, Uint8, Word64) \
4349 : V(I64AtomicOr16U, Or, Uint16, Word64) \
4350 : V(I64AtomicOr32U, Or, Uint32, Word64) \
4351 : V(I32AtomicXor, Xor, Uint32, Word32) \
4352 : V(I64AtomicXor, Xor, Uint64, Word64) \
4353 : V(I32AtomicXor8U, Xor, Uint8, Word32) \
4354 : V(I32AtomicXor16U, Xor, Uint16, Word32) \
4355 : V(I64AtomicXor8U, Xor, Uint8, Word64) \
4356 : V(I64AtomicXor16U, Xor, Uint16, Word64) \
4357 : V(I64AtomicXor32U, Xor, Uint32, Word64) \
4358 : V(I32AtomicExchange, Exchange, Uint32, Word32) \
4359 : V(I64AtomicExchange, Exchange, Uint64, Word64) \
4360 : V(I32AtomicExchange8U, Exchange, Uint8, Word32) \
4361 : V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
4362 : V(I64AtomicExchange8U, Exchange, Uint8, Word64) \
4363 : V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
4364 : V(I64AtomicExchange32U, Exchange, Uint32, Word64)
4365 :
4366 : #define ATOMIC_CMP_EXCHG_LIST(V) \
4367 : V(I32AtomicCompareExchange, Uint32, Word32) \
4368 : V(I64AtomicCompareExchange, Uint64, Word64) \
4369 : V(I32AtomicCompareExchange8U, Uint8, Word32) \
4370 : V(I32AtomicCompareExchange16U, Uint16, Word32) \
4371 : V(I64AtomicCompareExchange8U, Uint8, Word64) \
4372 : V(I64AtomicCompareExchange16U, Uint16, Word64) \
4373 : V(I64AtomicCompareExchange32U, Uint32, Word64)
4374 :
4375 : #define ATOMIC_LOAD_LIST(V) \
4376 : V(I32AtomicLoad, Uint32, Word32) \
4377 : V(I64AtomicLoad, Uint64, Word64) \
4378 : V(I32AtomicLoad8U, Uint8, Word32) \
4379 : V(I32AtomicLoad16U, Uint16, Word32) \
4380 : V(I64AtomicLoad8U, Uint8, Word64) \
4381 : V(I64AtomicLoad16U, Uint16, Word64) \
4382 : V(I64AtomicLoad32U, Uint32, Word64)
4383 :
4384 : #define ATOMIC_STORE_LIST(V) \
4385 : V(I32AtomicStore, Uint32, kWord32, Word32) \
4386 : V(I64AtomicStore, Uint64, kWord64, Word64) \
4387 : V(I32AtomicStore8U, Uint8, kWord8, Word32) \
4388 : V(I32AtomicStore16U, Uint16, kWord16, Word32) \
4389 : V(I64AtomicStore8U, Uint8, kWord8, Word64) \
4390 : V(I64AtomicStore16U, Uint16, kWord16, Word64) \
4391 : V(I64AtomicStore32U, Uint32, kWord32, Word64)
4392 :
4393 32966 : Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
4394 : uint32_t alignment, uint32_t offset,
4395 : wasm::WasmCodePosition position) {
4396 : Node* node;
4397 32966 : switch (opcode) {
4398 : #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \
4399 : case wasm::kExpr##Name: { \
4400 : Node* index = CheckBoundsAndAlignment( \
4401 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4402 : position); \
4403 : node = graph()->NewNode( \
4404 : mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
4405 : MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4406 : break; \
4407 : }
4408 71413 : ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
4409 : #undef BUILD_ATOMIC_BINOP
4410 :
4411 : #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \
4412 : case wasm::kExpr##Name: { \
4413 : Node* index = CheckBoundsAndAlignment( \
4414 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4415 : position); \
4416 : node = graph()->NewNode( \
4417 : mcgraph()->machine()->Prefix##AtomicCompareExchange( \
4418 : MachineType::Type()), \
4419 : MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
4420 : break; \
4421 : }
4422 682 : ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
4423 : #undef BUILD_ATOMIC_CMP_EXCHG
4424 :
4425 : #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \
4426 : case wasm::kExpr##Name: { \
4427 : Node* index = CheckBoundsAndAlignment( \
4428 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4429 : position); \
4430 : node = graph()->NewNode( \
4431 : mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \
4432 : MemBuffer(offset), index, Effect(), Control()); \
4433 : break; \
4434 : }
4435 11458 : ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
4436 : #undef BUILD_ATOMIC_LOAD_OP
4437 :
4438 : #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \
4439 : case wasm::kExpr##Name: { \
4440 : Node* index = CheckBoundsAndAlignment( \
4441 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4442 : position); \
4443 : node = graph()->NewNode( \
4444 : mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
4445 : MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4446 : break; \
4447 : }
4448 12950 : ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
4449 : #undef BUILD_ATOMIC_STORE_OP
4450 : case wasm::kExprAtomicNotify: {
4451 286 : Node* index = CheckBoundsAndAlignment(
4452 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4453 286 : position);
4454 : // Now that we've bounds-checked, compute the effective address.
4455 286 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4456 : Uint32Constant(offset), index);
4457 : WasmAtomicNotifyDescriptor interface_descriptor;
4458 285 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4459 : mcgraph()->zone(), interface_descriptor,
4460 : interface_descriptor.GetStackParameterCount(),
4461 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4462 286 : StubCallMode::kCallWasmRuntimeStub);
4463 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4464 285 : wasm::WasmCode::kWasmAtomicNotify, RelocInfo::WASM_STUB_CALL);
4465 286 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4466 : call_target, address, inputs[1], Effect(),
4467 : Control());
4468 : break;
4469 : }
4470 :
4471 : case wasm::kExprI32AtomicWait: {
4472 421 : Node* index = CheckBoundsAndAlignment(
4473 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4474 421 : position);
4475 : // Now that we've bounds-checked, compute the effective address.
4476 422 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4477 : Uint32Constant(offset), index);
4478 : Node* timeout;
4479 420 : if (mcgraph()->machine()->Is32()) {
4480 0 : timeout = BuildF64SConvertI64(inputs[2]);
4481 : } else {
4482 420 : timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4483 : inputs[2]);
4484 : }
4485 : WasmI32AtomicWaitDescriptor interface_descriptor;
4486 422 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4487 : mcgraph()->zone(), interface_descriptor,
4488 : interface_descriptor.GetStackParameterCount(),
4489 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4490 421 : StubCallMode::kCallWasmRuntimeStub);
4491 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4492 422 : wasm::WasmCode::kWasmI32AtomicWait, RelocInfo::WASM_STUB_CALL);
4493 422 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4494 : call_target, address, inputs[1], timeout,
4495 : Effect(), Control());
4496 : break;
4497 : }
4498 :
4499 : case wasm::kExprI64AtomicWait: {
4500 504 : Node* index = CheckBoundsAndAlignment(
4501 : wasm::ValueTypes::MemSize(MachineType::Uint64()), inputs[0], offset,
4502 503 : position);
4503 : // Now that we've bounds-checked, compute the effective address.
4504 510 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4505 : Uint32Constant(offset), index);
4506 : Node* timeout;
4507 510 : if (mcgraph()->machine()->Is32()) {
4508 0 : timeout = BuildF64SConvertI64(inputs[2]);
4509 : } else {
4510 510 : timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4511 : inputs[2]);
4512 : }
4513 507 : Node* expected_value_low = graph()->NewNode(
4514 : mcgraph()->machine()->TruncateInt64ToInt32(), inputs[1]);
4515 510 : Node* tmp = graph()->NewNode(mcgraph()->machine()->Word64Shr(), inputs[1],
4516 : Int64Constant(32));
4517 : Node* expected_value_high =
4518 509 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), tmp);
4519 : WasmI64AtomicWaitDescriptor interface_descriptor;
4520 506 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4521 : mcgraph()->zone(), interface_descriptor,
4522 : interface_descriptor.GetStackParameterCount(),
4523 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4524 508 : StubCallMode::kCallWasmRuntimeStub);
4525 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4526 506 : wasm::WasmCode::kWasmI64AtomicWait, RelocInfo::WASM_STUB_CALL);
4527 509 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4528 : call_target, address, expected_value_high,
4529 : expected_value_low, timeout, Effect(), Control());
4530 : break;
4531 : }
4532 :
4533 : default:
4534 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4535 : }
4536 32987 : return SetEffect(node);
4537 : }
4538 :
4539 : #undef ATOMIC_BINOP_LIST
4540 : #undef ATOMIC_CMP_EXCHG_LIST
4541 : #undef ATOMIC_LOAD_LIST
4542 : #undef ATOMIC_STORE_LIST
4543 :
4544 125 : Node* WasmGraphBuilder::CheckDataSegmentIsPassiveAndNotDropped(
4545 : uint32_t data_segment_index, wasm::WasmCodePosition position) {
4546 : // The data segment index must be in bounds since it is required by
4547 : // validation.
4548 : DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments);
4549 :
4550 : Node* dropped_data_segments =
4551 250 : LOAD_INSTANCE_FIELD(DroppedDataSegments, MachineType::Pointer());
4552 125 : Node* is_segment_dropped = SetEffect(graph()->NewNode(
4553 : mcgraph()->machine()->Load(MachineType::Uint8()), dropped_data_segments,
4554 : mcgraph()->IntPtrConstant(data_segment_index), Effect(), Control()));
4555 125 : TrapIfTrue(wasm::kTrapDataSegmentDropped, is_segment_dropped, position);
4556 125 : return dropped_data_segments;
4557 : }
4558 :
4559 76 : Node* WasmGraphBuilder::MemoryInit(uint32_t data_segment_index, Node* dst,
4560 : Node* src, Node* size,
4561 : wasm::WasmCodePosition position) {
4562 76 : CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
4563 76 : Node* dst_fail = BoundsCheckMemRange(&dst, &size, position);
4564 : auto m = mcgraph()->machine();
4565 :
4566 : Node* seg_index = Uint32Constant(data_segment_index);
4567 : Node* src_fail;
4568 :
4569 : {
4570 : // Load segment size from WasmInstanceObject::data_segment_sizes.
4571 : Node* seg_size_array =
4572 152 : LOAD_INSTANCE_FIELD(DataSegmentSizes, MachineType::Pointer());
4573 : STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2);
4574 76 : Node* scaled_index = Uint32ToUintptr(
4575 76 : graph()->NewNode(m->Word32Shl(), seg_index, Int32Constant(2)));
4576 76 : Node* seg_size = SetEffect(graph()->NewNode(m->Load(MachineType::Uint32()),
4577 : seg_size_array, scaled_index,
4578 : Effect(), Control()));
4579 :
4580 : // Bounds check the src index against the segment size.
4581 76 : src_fail = BoundsCheckRange(src, &size, seg_size, position);
4582 : }
4583 :
4584 : {
4585 : // Load segment's base pointer from WasmInstanceObject::data_segment_starts.
4586 : Node* seg_start_array =
4587 152 : LOAD_INSTANCE_FIELD(DataSegmentStarts, MachineType::Pointer());
4588 : STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <=
4589 : kMaxUInt32 / kSystemPointerSize);
4590 76 : Node* scaled_index = Uint32ToUintptr(graph()->NewNode(
4591 76 : m->Word32Shl(), seg_index, Int32Constant(kSystemPointerSizeLog2)));
4592 76 : Node* seg_start = SetEffect(
4593 : graph()->NewNode(m->Load(MachineType::Pointer()), seg_start_array,
4594 : scaled_index, Effect(), Control()));
4595 :
4596 : // Convert src index to pointer.
4597 76 : src = graph()->NewNode(m->IntAdd(), seg_start, Uint32ToUintptr(src));
4598 : }
4599 :
4600 76 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4601 152 : ExternalReference::wasm_memory_copy()));
4602 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
4603 76 : MachineType::Uint32()};
4604 : MachineSignature sig(0, 3, sig_types);
4605 76 : BuildCCall(&sig, function, dst, src, size);
4606 76 : return TrapIfTrue(wasm::kTrapMemOutOfBounds,
4607 : graph()->NewNode(m->Word32Or(), dst_fail, src_fail),
4608 76 : position);
4609 : }
4610 :
4611 49 : Node* WasmGraphBuilder::DataDrop(uint32_t data_segment_index,
4612 : wasm::WasmCodePosition position) {
4613 : Node* dropped_data_segments =
4614 49 : CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
4615 49 : const Operator* store_op = mcgraph()->machine()->Store(
4616 49 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
4617 49 : return SetEffect(
4618 : graph()->NewNode(store_op, dropped_data_segments,
4619 : mcgraph()->IntPtrConstant(data_segment_index),
4620 49 : mcgraph()->Int32Constant(1), Effect(), Control()));
4621 : }
4622 :
4623 43 : Node* WasmGraphBuilder::MemoryCopy(Node* dst, Node* src, Node* size,
4624 : wasm::WasmCodePosition position) {
4625 : auto m = mcgraph()->machine();
4626 : // The data must be copied backward if the regions overlap and src < dst. The
4627 : // regions overlap if {src + size > dst && dst + size > src}. Since we already
4628 : // test that {src < dst}, we know that {dst + size > src}, so this simplifies
4629 : // to just {src + size > dst}. That sum can overflow, but if we subtract
4630 : // {size} from both sides of the inequality we get the equivalent test
4631 : // {size > dst - src}.
4632 172 : Node* copy_backward = graph()->NewNode(
4633 : m->Word32And(), graph()->NewNode(m->Uint32LessThan(), src, dst),
4634 : graph()->NewNode(m->Uint32LessThan(),
4635 : graph()->NewNode(m->Int32Sub(), dst, src), size));
4636 :
4637 43 : Node* dst_fail = BoundsCheckMemRange(&dst, &size, position);
4638 :
4639 : // Trap without copying any bytes if we are copying backward and the copy is
4640 : // partially out-of-bounds. We only need to check that the dst region is
4641 : // out-of-bounds, because we know that {src < dst}, so the src region is
4642 : // always out of bounds if the dst region is.
4643 43 : TrapIfTrue(wasm::kTrapMemOutOfBounds,
4644 : graph()->NewNode(m->Word32And(), dst_fail, copy_backward),
4645 43 : position);
4646 :
4647 43 : Node* src_fail = BoundsCheckMemRange(&src, &size, position);
4648 :
4649 43 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4650 86 : ExternalReference::wasm_memory_copy()));
4651 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
4652 43 : MachineType::Uint32()};
4653 : MachineSignature sig(0, 3, sig_types);
4654 43 : BuildCCall(&sig, function, dst, src, size);
4655 43 : return TrapIfTrue(wasm::kTrapMemOutOfBounds,
4656 : graph()->NewNode(m->Word32Or(), dst_fail, src_fail),
4657 43 : position);
4658 : }
4659 :
4660 43 : Node* WasmGraphBuilder::MemoryFill(Node* dst, Node* value, Node* size,
4661 : wasm::WasmCodePosition position) {
4662 43 : Node* fail = BoundsCheckMemRange(&dst, &size, position);
4663 43 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4664 86 : ExternalReference::wasm_memory_fill()));
4665 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Uint32(),
4666 43 : MachineType::Uint32()};
4667 : MachineSignature sig(0, 3, sig_types);
4668 43 : BuildCCall(&sig, function, dst, value, size);
4669 43 : return TrapIfTrue(wasm::kTrapMemOutOfBounds, fail, position);
4670 : }
4671 :
4672 51 : Node* WasmGraphBuilder::CheckElemSegmentIsPassiveAndNotDropped(
4673 : uint32_t elem_segment_index, wasm::WasmCodePosition position) {
4674 : // The elem segment index must be in bounds since it is required by
4675 : // validation.
4676 : DCHECK_LT(elem_segment_index, env_->module->elem_segments.size());
4677 :
4678 : Node* dropped_elem_segments =
4679 102 : LOAD_INSTANCE_FIELD(DroppedElemSegments, MachineType::Pointer());
4680 51 : Node* is_segment_dropped = SetEffect(graph()->NewNode(
4681 : mcgraph()->machine()->Load(MachineType::Uint8()), dropped_elem_segments,
4682 : mcgraph()->IntPtrConstant(elem_segment_index), Effect(), Control()));
4683 51 : TrapIfTrue(wasm::kTrapElemSegmentDropped, is_segment_dropped, position);
4684 51 : return dropped_elem_segments;
4685 : }
4686 :
4687 24 : Node* WasmGraphBuilder::TableInit(uint32_t table_index,
4688 : uint32_t elem_segment_index, Node* dst,
4689 : Node* src, Node* size,
4690 : wasm::WasmCodePosition position) {
4691 24 : CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
4692 : Node* args[] = {
4693 24 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
4694 24 : graph()->NewNode(mcgraph()->common()->NumberConstant(elem_segment_index)),
4695 24 : BuildConvertUint32ToSmiWithSaturation(dst, FLAG_wasm_max_table_size),
4696 24 : BuildConvertUint32ToSmiWithSaturation(src, FLAG_wasm_max_table_size),
4697 96 : BuildConvertUint32ToSmiWithSaturation(size, FLAG_wasm_max_table_size)};
4698 : Node* result =
4699 24 : BuildCallToRuntime(Runtime::kWasmTableInit, args, arraysize(args));
4700 :
4701 24 : return result;
4702 : }
4703 :
4704 27 : Node* WasmGraphBuilder::ElemDrop(uint32_t elem_segment_index,
4705 : wasm::WasmCodePosition position) {
4706 : Node* dropped_elem_segments =
4707 27 : CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
4708 27 : const Operator* store_op = mcgraph()->machine()->Store(
4709 27 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
4710 27 : return SetEffect(
4711 : graph()->NewNode(store_op, dropped_elem_segments,
4712 : mcgraph()->IntPtrConstant(elem_segment_index),
4713 27 : mcgraph()->Int32Constant(1), Effect(), Control()));
4714 : }
4715 :
4716 51 : Node* WasmGraphBuilder::TableCopy(uint32_t table_src_index,
4717 : uint32_t table_dst_index, Node* dst,
4718 : Node* src, Node* size,
4719 : wasm::WasmCodePosition position) {
4720 : Node* args[] = {
4721 51 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_src_index)),
4722 51 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_dst_index)),
4723 51 : BuildConvertUint32ToSmiWithSaturation(dst, FLAG_wasm_max_table_size),
4724 51 : BuildConvertUint32ToSmiWithSaturation(src, FLAG_wasm_max_table_size),
4725 204 : BuildConvertUint32ToSmiWithSaturation(size, FLAG_wasm_max_table_size)};
4726 : Node* result =
4727 51 : BuildCallToRuntime(Runtime::kWasmTableCopy, args, arraysize(args));
4728 :
4729 51 : return result;
4730 : }
4731 :
4732 0 : class WasmDecorator final : public GraphDecorator {
4733 : public:
4734 : explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
4735 8 : : origins_(origins), decoder_(decoder) {}
4736 :
4737 96 : void Decorate(Node* node) final {
4738 192 : origins_->SetNodeOrigin(
4739 : node, NodeOrigin("wasm graph creation", "n/a",
4740 192 : NodeOrigin::kWasmBytecode, decoder_->position()));
4741 96 : }
4742 :
4743 : private:
4744 : compiler::NodeOriginTable* origins_;
4745 : wasm::Decoder* decoder_;
4746 : };
4747 :
4748 8 : void WasmGraphBuilder::AddBytecodePositionDecorator(
4749 : NodeOriginTable* node_origins, wasm::Decoder* decoder) {
4750 : DCHECK_NULL(decorator_);
4751 8 : decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
4752 8 : graph()->AddDecorator(decorator_);
4753 8 : }
4754 :
4755 8 : void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
4756 : DCHECK_NOT_NULL(decorator_);
4757 8 : graph()->RemoveDecorator(decorator_);
4758 8 : decorator_ = nullptr;
4759 8 : }
4760 :
4761 : namespace {
4762 : bool must_record_function_compilation(Isolate* isolate) {
4763 140963 : return isolate->logger()->is_listening_to_code_events() ||
4764 : isolate->is_profiling();
4765 : }
4766 :
4767 : PRINTF_FORMAT(4, 5)
4768 4 : void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
4769 : Isolate* isolate, Handle<Code> code,
4770 : const char* format, ...) {
4771 : DCHECK(must_record_function_compilation(isolate));
4772 :
4773 : ScopedVector<char> buffer(128);
4774 : va_list arguments;
4775 4 : va_start(arguments, format);
4776 4 : int len = VSNPrintF(buffer, format, arguments);
4777 4 : CHECK_LT(0, len);
4778 4 : va_end(arguments);
4779 : Handle<String> name_str =
4780 4 : isolate->factory()->NewStringFromAsciiChecked(buffer.start());
4781 8 : PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
4782 4 : }
4783 :
4784 : class WasmWrapperGraphBuilder : public WasmGraphBuilder {
4785 : public:
4786 518754 : WasmWrapperGraphBuilder(Zone* zone, JSGraph* jsgraph, wasm::FunctionSig* sig,
4787 : compiler::SourcePositionTable* spt,
4788 : StubCallMode stub_mode, wasm::WasmFeatures features)
4789 : : WasmGraphBuilder(nullptr, zone, jsgraph, sig, spt),
4790 : isolate_(jsgraph->isolate()),
4791 : jsgraph_(jsgraph),
4792 : stub_mode_(stub_mode),
4793 1037508 : enabled_features_(features) {}
4794 :
4795 39472 : Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
4796 : MachineOperatorBuilder* machine = mcgraph()->machine();
4797 : CommonOperatorBuilder* common = mcgraph()->common();
4798 39472 : Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4799 : ? mcgraph()->RelocatableIntPtrConstant(
4800 : wasm::WasmCode::kWasmAllocateHeapNumber,
4801 : RelocInfo::WASM_STUB_CALL)
4802 7548 : : jsgraph()->HeapConstant(
4803 47020 : BUILTIN_CODE(isolate_, AllocateHeapNumber));
4804 39472 : if (!allocate_heap_number_operator_.is_set()) {
4805 22100 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4806 : mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
4807 11050 : CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
4808 11050 : allocate_heap_number_operator_.set(common->Call(call_descriptor));
4809 : }
4810 : Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
4811 : target, Effect(), control);
4812 78944 : SetEffect(
4813 : graph()->NewNode(machine->Store(StoreRepresentation(
4814 : MachineRepresentation::kFloat64, kNoWriteBarrier)),
4815 : heap_number, BuildHeapNumberValueIndexConstant(),
4816 : value, heap_number, control));
4817 39472 : return heap_number;
4818 : }
4819 :
4820 83243 : Node* BuildChangeSmiToFloat64(Node* value) {
4821 83243 : return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4822 83243 : BuildChangeSmiToInt32(value));
4823 : }
4824 :
4825 83243 : Node* BuildTestHeapObject(Node* value) {
4826 83243 : return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4827 83243 : mcgraph()->IntPtrConstant(kHeapObjectTag));
4828 : }
4829 :
4830 83243 : Node* BuildLoadHeapNumberValue(Node* value) {
4831 83243 : return SetEffect(graph()->NewNode(
4832 : mcgraph()->machine()->Load(MachineType::Float64()), value,
4833 83243 : BuildHeapNumberValueIndexConstant(), Effect(), Control()));
4834 : }
4835 :
4836 : Node* BuildHeapNumberValueIndexConstant() {
4837 122715 : return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4838 : }
4839 :
4840 : Node* BuildChangeInt32ToTagged(Node* value) {
4841 : MachineOperatorBuilder* machine = mcgraph()->machine();
4842 : CommonOperatorBuilder* common = mcgraph()->common();
4843 :
4844 : if (SmiValuesAre32Bits()) {
4845 25303 : return BuildChangeInt32ToSmi(value);
4846 : }
4847 : DCHECK(SmiValuesAre31Bits());
4848 :
4849 : Node* effect = Effect();
4850 : Node* control = Control();
4851 : Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4852 : graph()->start());
4853 :
4854 : Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4855 : Node* branch =
4856 : graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4857 :
4858 : Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4859 : Node* vtrue = BuildAllocateHeapNumberWithValue(
4860 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4861 : Node* etrue = Effect();
4862 :
4863 : Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4864 : Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4865 : vfalse = BuildChangeInt32ToIntPtr(vfalse);
4866 :
4867 : Node* merge =
4868 : SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
4869 : SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
4870 : return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4871 : vtrue, vfalse, merge);
4872 : }
4873 :
4874 39472 : Node* BuildChangeFloat64ToTagged(Node* value) {
4875 : MachineOperatorBuilder* machine = mcgraph()->machine();
4876 : CommonOperatorBuilder* common = mcgraph()->common();
4877 :
4878 : // Check several conditions:
4879 : // i32?
4880 : // ├─ true: zero?
4881 : // │ ├─ true: negative?
4882 : // │ │ ├─ true: box
4883 : // │ │ └─ false: potentially Smi
4884 : // │ └─ false: potentially Smi
4885 : // └─ false: box
4886 : // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
4887 : // still need to check whether the value fits in a Smi.
4888 :
4889 : Node* effect = Effect();
4890 : Node* control = Control();
4891 39472 : Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4892 78944 : Node* check_i32 = graph()->NewNode(
4893 : machine->Float64Equal(), value,
4894 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4895 39472 : Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
4896 :
4897 39472 : Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
4898 39472 : Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
4899 :
4900 : // We only need to check for -0 if the {value} can potentially contain -0.
4901 39472 : Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4902 : mcgraph()->Int32Constant(0));
4903 39472 : Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4904 : check_zero, if_i32);
4905 :
4906 39472 : Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4907 39472 : Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
4908 :
4909 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4910 78944 : Node* check_negative = graph()->NewNode(
4911 : machine->Int32LessThan(),
4912 : graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4913 : mcgraph()->Int32Constant(0));
4914 39472 : Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4915 : check_negative, if_zero);
4916 :
4917 39472 : Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4918 : Node* if_not_negative =
4919 39472 : graph()->NewNode(common->IfFalse(), branch_negative);
4920 :
4921 : // We need to create a box for negative 0.
4922 : Node* if_smi =
4923 39472 : graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
4924 39472 : Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
4925 :
4926 : // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4927 : // 32-bit machines we need to deal with potential overflow and fallback to
4928 : // boxing.
4929 : Node* vsmi;
4930 : if (SmiValuesAre32Bits()) {
4931 39472 : vsmi = BuildChangeInt32ToSmi(value32);
4932 : } else {
4933 : DCHECK(SmiValuesAre31Bits());
4934 : Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4935 : value32, if_smi);
4936 :
4937 : Node* check_ovf =
4938 : graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4939 : Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4940 : check_ovf, if_smi);
4941 :
4942 : Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4943 : if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4944 :
4945 : if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4946 : vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4947 : vsmi = BuildChangeInt32ToIntPtr(vsmi);
4948 : }
4949 :
4950 : // Allocate the box for the {value}.
4951 39472 : Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4952 : Node* ebox = Effect();
4953 :
4954 : Node* merge =
4955 39472 : SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
4956 39472 : SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
4957 39472 : return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4958 39472 : vsmi, vbox, merge);
4959 : }
4960 :
4961 6782 : int AddArgumentNodes(Node** args, int pos, int param_count,
4962 : wasm::FunctionSig* sig) {
4963 : // Convert wasm numbers to JS values.
4964 93780 : for (int i = 0; i < param_count; ++i) {
4965 : Node* param =
4966 43499 : Param(i + 1); // Start from index 1 to drop the instance_node.
4967 86998 : args[pos++] = ToJS(param, sig->GetParam(i));
4968 : }
4969 6782 : return pos;
4970 : }
4971 :
4972 83243 : Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4973 166486 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4974 : mcgraph()->zone(), TypeConversionDescriptor{}, 0,
4975 83243 : CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
4976 : Node* stub_code =
4977 83243 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4978 : ? mcgraph()->RelocatableIntPtrConstant(
4979 : wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
4980 242647 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
4981 :
4982 83243 : Node* result = SetEffect(
4983 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4984 : node, js_context, Effect(), Control()));
4985 :
4986 : SetSourcePosition(result, 1);
4987 :
4988 83243 : return result;
4989 : }
4990 :
4991 83243 : Node* BuildChangeTaggedToFloat64(Node* value) {
4992 : MachineOperatorBuilder* machine = mcgraph()->machine();
4993 : CommonOperatorBuilder* common = mcgraph()->common();
4994 :
4995 : // Implement the following decision tree:
4996 : // heap object?
4997 : // ├─ true: undefined?
4998 : // │ ├─ true: f64 const
4999 : // │ └─ false: load heap number value
5000 : // └─ false: smi to float64
5001 :
5002 83243 : Node* check_heap_object = BuildTestHeapObject(value);
5003 : Diamond is_heap_object(graph(), common, check_heap_object,
5004 83243 : BranchHint::kFalse);
5005 : is_heap_object.Chain(Control());
5006 :
5007 83243 : SetControl(is_heap_object.if_true);
5008 : Node* orig_effect = Effect();
5009 :
5010 : Node* undefined_node =
5011 166485 : LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
5012 : Node* check_undefined =
5013 83243 : graph()->NewNode(machine->WordEqual(), value, undefined_node);
5014 : Node* effect_tagged = Effect();
5015 :
5016 83243 : Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
5017 83243 : is_undefined.Nest(is_heap_object, true);
5018 :
5019 83243 : SetControl(is_undefined.if_false);
5020 83243 : Node* vheap_number = BuildLoadHeapNumberValue(value);
5021 : Node* effect_undefined = Effect();
5022 :
5023 83243 : SetControl(is_undefined.merge);
5024 : Node* vundefined =
5025 83243 : mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
5026 : Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
5027 83243 : vundefined, vheap_number);
5028 :
5029 83243 : effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
5030 :
5031 : // If input is Smi: just convert to float64.
5032 83243 : Node* vfrom_smi = BuildChangeSmiToFloat64(value);
5033 :
5034 83243 : SetControl(is_heap_object.merge);
5035 83243 : SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
5036 : return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
5037 83243 : vfrom_smi);
5038 : }
5039 :
5040 65887 : Node* ToJS(Node* node, wasm::ValueType type) {
5041 65887 : switch (type) {
5042 : case wasm::kWasmI32:
5043 25303 : return BuildChangeInt32ToTagged(node);
5044 : case wasm::kWasmS128:
5045 0 : UNREACHABLE();
5046 : case wasm::kWasmI64: {
5047 : DCHECK(enabled_features_.bigint);
5048 32 : return BuildChangeInt64ToBigInt(node);
5049 : }
5050 : case wasm::kWasmF32:
5051 11033 : node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
5052 : node);
5053 11033 : return BuildChangeFloat64ToTagged(node);
5054 : case wasm::kWasmF64:
5055 28439 : return BuildChangeFloat64ToTagged(node);
5056 : case wasm::kWasmAnyRef:
5057 : case wasm::kWasmAnyFunc:
5058 : case wasm::kWasmExceptRef:
5059 : return node;
5060 : default:
5061 0 : UNREACHABLE();
5062 : }
5063 : }
5064 :
5065 32 : Node* BuildChangeInt64ToBigInt(Node* input) {
5066 : I64ToBigIntDescriptor interface_descriptor;
5067 :
5068 64 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5069 : mcgraph()->zone(), // zone
5070 : interface_descriptor, // descriptor
5071 : interface_descriptor.GetStackParameterCount(), // stack parameter count
5072 : CallDescriptor::kNoFlags, // flags
5073 : Operator::kNoProperties, // properties
5074 32 : stub_mode_); // stub call mode
5075 :
5076 : Node* target =
5077 32 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
5078 : ? mcgraph()->RelocatableIntPtrConstant(
5079 : wasm::WasmCode::kWasmI64ToBigInt, RelocInfo::WASM_STUB_CALL)
5080 80 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, I64ToBigInt));
5081 :
5082 32 : return SetEffect(
5083 : SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
5084 32 : target, input, Effect(), Control())));
5085 : }
5086 :
5087 16 : Node* BuildChangeBigIntToInt64(Node* input, Node* context) {
5088 : BigIntToI64Descriptor interface_descriptor;
5089 :
5090 32 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5091 : mcgraph()->zone(), // zone
5092 : interface_descriptor, // descriptor
5093 : interface_descriptor.GetStackParameterCount(), // stack parameter count
5094 : CallDescriptor::kNoFlags, // flags
5095 : Operator::kNoProperties, // properties
5096 16 : stub_mode_); // stub call mode
5097 :
5098 : Node* target =
5099 16 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
5100 : ? mcgraph()->RelocatableIntPtrConstant(
5101 : wasm::WasmCode::kWasmBigIntToI64, RelocInfo::WASM_STUB_CALL)
5102 32 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, BigIntToI64));
5103 :
5104 16 : return SetEffect(SetControl(
5105 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), target,
5106 16 : input, context, Effect(), Control())));
5107 : }
5108 :
5109 87419 : Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
5110 : DCHECK_NE(wasm::kWasmStmt, type);
5111 :
5112 : // The parameter is of type anyref or except_ref, we take it as is.
5113 87419 : if (type == wasm::kWasmAnyRef || type == wasm::kWasmExceptRef) {
5114 3928 : return node;
5115 : }
5116 :
5117 83491 : if (type == wasm::kWasmAnyFunc) {
5118 : Node* check =
5119 232 : BuildChangeSmiToInt32(SetEffect(BuildCallToRuntimeWithContext(
5120 : Runtime::kWasmIsValidAnyFuncValue, js_context, &node, 1, effect_,
5121 232 : Control())));
5122 :
5123 : Diamond type_check(graph(), mcgraph()->common(), check,
5124 232 : BranchHint::kTrue);
5125 : type_check.Chain(Control());
5126 :
5127 232 : Node* effect = Effect();
5128 232 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
5129 232 : nullptr, 0, &effect, type_check.if_false);
5130 :
5131 464 : SetEffect(type_check.EffectPhi(Effect(), effect));
5132 :
5133 232 : SetControl(type_check.merge);
5134 :
5135 232 : return node;
5136 : }
5137 : Node* num = nullptr;
5138 :
5139 83259 : if (type != wasm::kWasmI64) {
5140 : // Do a JavaScript ToNumber.
5141 83243 : num = BuildJavaScriptToNumber(node, js_context);
5142 :
5143 : // Change representation.
5144 83243 : num = BuildChangeTaggedToFloat64(num);
5145 : }
5146 :
5147 83259 : switch (type) {
5148 : case wasm::kWasmI32: {
5149 30229 : num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
5150 : num);
5151 30229 : break;
5152 : }
5153 : case wasm::kWasmI64: {
5154 : DCHECK(enabled_features_.bigint);
5155 16 : num = BuildChangeBigIntToInt64(node, js_context);
5156 16 : break;
5157 : }
5158 : case wasm::kWasmF32:
5159 12066 : num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
5160 : num);
5161 12066 : break;
5162 : case wasm::kWasmF64:
5163 : break;
5164 : case wasm::kWasmS128:
5165 0 : UNREACHABLE();
5166 : default:
5167 0 : UNREACHABLE();
5168 : }
5169 : DCHECK_NOT_NULL(num);
5170 :
5171 : return num;
5172 : }
5173 :
5174 294024 : void BuildModifyThreadInWasmFlag(bool new_value) {
5175 294024 : if (!trap_handler::IsTrapHandlerEnabled()) return;
5176 : Node* isolate_root =
5177 587540 : LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
5178 :
5179 : Node* thread_in_wasm_flag_address =
5180 293770 : LOAD_RAW(isolate_root, Isolate::thread_in_wasm_flag_address_offset(),
5181 : MachineType::Pointer());
5182 :
5183 293770 : if (FLAG_debug_code) {
5184 16 : Node* flag_value = SetEffect(
5185 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
5186 : thread_in_wasm_flag_address,
5187 : mcgraph()->Int32Constant(0), Effect(), Control()));
5188 : Node* check =
5189 16 : graph()->NewNode(mcgraph()->machine()->Word32Equal(), flag_value,
5190 : mcgraph()->Int32Constant(new_value ? 0 : 1));
5191 :
5192 : Diamond flag_check(graph(), mcgraph()->common(), check,
5193 16 : BranchHint::kTrue);
5194 : flag_check.Chain(Control());
5195 16 : Node* message_id = jsgraph()->SmiConstant(static_cast<int32_t>(
5196 : new_value ? AbortReason::kUnexpectedThreadInWasmSet
5197 16 : : AbortReason::kUnexpectedThreadInWasmUnset));
5198 :
5199 16 : Node* effect = Effect();
5200 32 : BuildCallToRuntimeWithContext(Runtime::kAbort, NoContextConstant(),
5201 : &message_id, 1, &effect,
5202 16 : flag_check.if_false);
5203 :
5204 32 : SetEffect(flag_check.EffectPhi(Effect(), effect));
5205 :
5206 16 : SetControl(flag_check.merge);
5207 : }
5208 :
5209 881310 : SetEffect(graph()->NewNode(
5210 : mcgraph()->machine()->Store(StoreRepresentation(
5211 : MachineRepresentation::kWord32, kNoWriteBarrier)),
5212 : thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
5213 : mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
5214 : }
5215 :
5216 140966 : Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
5217 140966 : Node* shared = SetEffect(graph()->NewNode(
5218 : jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
5219 : jsgraph()->Int32Constant(
5220 : wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()),
5221 : Effect(), Control()));
5222 140966 : return SetEffect(graph()->NewNode(
5223 : jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
5224 : jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
5225 : kHeapObjectTag),
5226 140966 : Effect(), Control()));
5227 : }
5228 :
5229 140966 : Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
5230 140966 : return SetEffect(graph()->NewNode(
5231 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
5232 : jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
5233 : kHeapObjectTag),
5234 140966 : Effect(), Control()));
5235 : }
5236 :
5237 428 : Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
5238 428 : Node* function_index_smi = SetEffect(graph()->NewNode(
5239 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
5240 : jsgraph()->Int32Constant(
5241 : WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
5242 : Effect(), Control()));
5243 428 : Node* function_index = BuildChangeSmiToInt32(function_index_smi);
5244 428 : return function_index;
5245 : }
5246 :
5247 139802 : Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
5248 139802 : Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
5249 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
5250 : jsgraph()->Int32Constant(
5251 : WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
5252 : Effect(), Control()));
5253 139802 : Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
5254 139802 : return jump_table_offset;
5255 : }
5256 :
5257 140966 : void BuildJSToWasmWrapper(bool is_import) {
5258 140966 : const int wasm_count = static_cast<int>(sig_->parameter_count());
5259 :
5260 : // Build the start and the JS parameter nodes.
5261 140966 : SetEffect(SetControl(Start(wasm_count + 5)));
5262 :
5263 : // Create the js_closure and js_context parameters.
5264 : Node* js_closure =
5265 140966 : graph()->NewNode(jsgraph()->common()->Parameter(
5266 : Linkage::kJSCallClosureParamIndex, "%closure"),
5267 : graph()->start());
5268 140966 : Node* js_context = graph()->NewNode(
5269 : mcgraph()->common()->Parameter(
5270 : Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
5271 : graph()->start());
5272 :
5273 : // Create the instance_node node to pass as parameter. It is loaded from
5274 : // an actual reference to an instance or a placeholder reference,
5275 : // called {WasmExportedFunction} via the {WasmExportedFunctionData}
5276 : // structure.
5277 140966 : Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
5278 140966 : instance_node_.set(
5279 : BuildLoadInstanceFromExportedFunctionData(function_data));
5280 :
5281 140966 : if (!wasm::IsJSCompatibleSignature(sig_, enabled_features_.bigint)) {
5282 : // Throw a TypeError. Use the js_context of the calling javascript
5283 : // function (passed as a parameter), such that the generated code is
5284 : // js_context independent.
5285 736 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
5286 736 : nullptr, 0, effect_, Control());
5287 : Return(jsgraph()->SmiConstant(0));
5288 736 : return;
5289 : }
5290 :
5291 140230 : const int args_count = wasm_count + 1; // +1 for wasm_code.
5292 140230 : Node** args = Buffer(args_count);
5293 : Node** rets;
5294 :
5295 : // Convert JS parameters to wasm numbers.
5296 307970 : for (int i = 0; i < wasm_count; ++i) {
5297 83870 : Node* param = Param(i + 1);
5298 167740 : Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
5299 83870 : args[i + 1] = wasm_param;
5300 : }
5301 :
5302 : // Set the ThreadInWasm flag before we do the actual call.
5303 140230 : BuildModifyThreadInWasmFlag(true);
5304 :
5305 140230 : if (is_import) {
5306 : // Call to an imported function.
5307 : // Load function index from {WasmExportedFunctionData}.
5308 : Node* function_index =
5309 428 : BuildLoadFunctionIndexFromExportedFunctionData(function_data);
5310 428 : BuildImportCall(sig_, args, &rets, wasm::kNoCodePosition, function_index,
5311 428 : kCallContinues);
5312 : } else {
5313 : // Call to a wasm function defined in this module.
5314 : // The call target is the jump table slot for that function.
5315 : Node* jump_table_start =
5316 279604 : LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
5317 : Node* jump_table_offset =
5318 139802 : BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
5319 139802 : Node* jump_table_slot = graph()->NewNode(
5320 : mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
5321 139802 : args[0] = jump_table_slot;
5322 :
5323 139802 : BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
5324 139802 : kNoRetpoline);
5325 : }
5326 :
5327 : // Clear the ThreadInWasm flag.
5328 140230 : BuildModifyThreadInWasmFlag(false);
5329 :
5330 140230 : Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
5331 162618 : : ToJS(rets[0], sig_->GetReturn());
5332 : Return(jsval);
5333 : }
5334 :
5335 6858 : bool BuildWasmImportCallWrapper(WasmImportCallKind kind) {
5336 6858 : int wasm_count = static_cast<int>(sig_->parameter_count());
5337 :
5338 : // Build the start and the parameter nodes.
5339 6858 : SetEffect(SetControl(Start(wasm_count + 4)));
5340 :
5341 6858 : instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
5342 :
5343 : Node* native_context =
5344 13716 : LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
5345 :
5346 6858 : if (kind == WasmImportCallKind::kRuntimeTypeError) {
5347 : // =======================================================================
5348 : // === Runtime TypeError =================================================
5349 : // =======================================================================
5350 76 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
5351 : native_context, nullptr, 0, effect_,
5352 76 : Control());
5353 : // We don't need to return a value here, as the runtime call will not
5354 : // return anyway (the c entry stub will trigger stack unwinding).
5355 : ReturnVoid();
5356 76 : return false;
5357 : }
5358 :
5359 : // The callable is passed as the last parameter, after WASM arguments.
5360 6782 : Node* callable_node = Param(wasm_count + 1);
5361 :
5362 : Node* undefined_node =
5363 13564 : LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
5364 :
5365 : Node* call = nullptr;
5366 : bool sloppy_receiver = true;
5367 :
5368 : // Clear the ThreadInWasm flag.
5369 6782 : BuildModifyThreadInWasmFlag(false);
5370 :
5371 6782 : switch (kind) {
5372 : // =======================================================================
5373 : // === JS Functions with matching arity ==================================
5374 : // =======================================================================
5375 : case WasmImportCallKind::kJSFunctionArityMatch:
5376 : sloppy_receiver = false;
5377 : V8_FALLTHROUGH; // fallthru
5378 : case WasmImportCallKind::kJSFunctionArityMatchSloppy: {
5379 4276 : Node** args = Buffer(wasm_count + 9);
5380 : int pos = 0;
5381 4276 : Node* function_context = SetEffect(graph()->NewNode(
5382 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5383 : callable_node,
5384 : mcgraph()->Int32Constant(
5385 : wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
5386 : Effect(), Control()));
5387 4276 : args[pos++] = callable_node; // target callable.
5388 : // Receiver.
5389 4276 : if (sloppy_receiver) {
5390 4156 : Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
5391 : native_context, Context::GLOBAL_PROXY_INDEX);
5392 4156 : args[pos++] = global_proxy;
5393 : } else {
5394 120 : args[pos++] = undefined_node;
5395 : }
5396 :
5397 4276 : auto call_descriptor = Linkage::GetJSCallDescriptor(
5398 4276 : graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
5399 :
5400 : // Convert wasm numbers to JS values.
5401 4276 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5402 :
5403 4276 : args[pos++] = undefined_node; // new target
5404 4276 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5405 4276 : args[pos++] = function_context;
5406 8552 : args[pos++] = Effect();
5407 8552 : args[pos++] = Control();
5408 :
5409 4276 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5410 4276 : args);
5411 4276 : break;
5412 : }
5413 : // =======================================================================
5414 : // === JS Functions with arguments adapter ===============================
5415 : // =======================================================================
5416 : case WasmImportCallKind::kJSFunctionArityMismatch:
5417 : sloppy_receiver = false;
5418 : V8_FALLTHROUGH; // fallthru
5419 : case WasmImportCallKind::kJSFunctionArityMismatchSloppy: {
5420 798 : Node** args = Buffer(wasm_count + 9);
5421 : int pos = 0;
5422 798 : Node* function_context = SetEffect(graph()->NewNode(
5423 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5424 : callable_node,
5425 : mcgraph()->Int32Constant(
5426 : wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
5427 : Effect(), Control()));
5428 : args[pos++] =
5429 798 : BuildLoadBuiltinFromInstance(Builtins::kArgumentsAdaptorTrampoline);
5430 798 : args[pos++] = callable_node; // target callable
5431 798 : args[pos++] = undefined_node; // new target
5432 798 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5433 :
5434 : // Load shared function info, and then the formal parameter count.
5435 798 : Node* shared_function_info = SetEffect(graph()->NewNode(
5436 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5437 : callable_node,
5438 : mcgraph()->Int32Constant(
5439 : wasm::ObjectAccess::
5440 : SharedFunctionInfoOffsetInTaggedJSFunction()),
5441 : Effect(), Control()));
5442 798 : Node* formal_param_count = SetEffect(graph()->NewNode(
5443 : mcgraph()->machine()->Load(MachineType::Uint16()),
5444 : shared_function_info,
5445 : mcgraph()->Int32Constant(
5446 : wasm::ObjectAccess::
5447 : FormalParameterCountOffsetInSharedFunctionInfo()),
5448 : Effect(), Control()));
5449 798 : args[pos++] = formal_param_count;
5450 :
5451 : // Receiver.
5452 798 : if (sloppy_receiver) {
5453 748 : Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
5454 : native_context, Context::GLOBAL_PROXY_INDEX);
5455 748 : args[pos++] = global_proxy;
5456 : } else {
5457 50 : args[pos++] = undefined_node;
5458 : }
5459 :
5460 : #ifdef V8_TARGET_ARCH_IA32
5461 : // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative
5462 : // call instead once builtins are embedded in every build configuration.
5463 : CallDescriptor::Flags flags = CallDescriptor::kAllowCallThroughSlot;
5464 : #else
5465 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5466 : #endif
5467 798 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5468 : mcgraph()->zone(), ArgumentsAdaptorDescriptor{}, 1 + wasm_count,
5469 798 : flags, Operator::kNoProperties);
5470 :
5471 : // Convert wasm numbers to JS values.
5472 798 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5473 798 : args[pos++] = function_context;
5474 1596 : args[pos++] = Effect();
5475 1596 : args[pos++] = Control();
5476 798 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5477 798 : args);
5478 : break;
5479 : }
5480 : // =======================================================================
5481 : // === General case of unknown callable ==================================
5482 : // =======================================================================
5483 : case WasmImportCallKind::kUseCallBuiltin: {
5484 1708 : Node** args = Buffer(wasm_count + 9);
5485 : int pos = 0;
5486 1708 : args[pos++] = mcgraph()->RelocatableIntPtrConstant(
5487 1708 : wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
5488 1708 : args[pos++] = callable_node;
5489 1708 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5490 1708 : args[pos++] = undefined_node; // receiver
5491 :
5492 1708 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5493 : graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
5494 : CallDescriptor::kNoFlags, Operator::kNoProperties,
5495 1708 : StubCallMode::kCallWasmRuntimeStub);
5496 :
5497 : // Convert wasm numbers to JS values.
5498 1708 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5499 :
5500 : // The native_context is sufficient here, because all kind of callables
5501 : // which depend on the context provide their own context. The context
5502 : // here is only needed if the target is a constructor to throw a
5503 : // TypeError, if the target is a native function, or if the target is a
5504 : // callable JSObject, which can only be constructed by the runtime.
5505 1708 : args[pos++] = native_context;
5506 3416 : args[pos++] = Effect();
5507 3416 : args[pos++] = Control();
5508 :
5509 1708 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5510 1708 : args);
5511 1708 : break;
5512 : }
5513 : default:
5514 0 : UNREACHABLE();
5515 : }
5516 : DCHECK_NOT_NULL(call);
5517 :
5518 : SetEffect(call);
5519 : SetSourcePosition(call, 0);
5520 :
5521 : // Convert the return value back.
5522 6782 : Node* val = sig_->return_count() == 0
5523 : ? mcgraph()->Int32Constant(0)
5524 10331 : : FromJS(call, native_context, sig_->GetReturn());
5525 :
5526 : // Set the ThreadInWasm flag again.
5527 6782 : BuildModifyThreadInWasmFlag(true);
5528 :
5529 : Return(val);
5530 6782 : return true;
5531 : }
5532 :
5533 369217 : void BuildWasmInterpreterEntry(uint32_t func_index) {
5534 369217 : int param_count = static_cast<int>(sig_->parameter_count());
5535 :
5536 : // Build the start and the parameter nodes.
5537 369217 : SetEffect(SetControl(Start(param_count + 3)));
5538 :
5539 : // Create the instance_node from the passed parameter.
5540 369218 : instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
5541 :
5542 : // Compute size for the argument buffer.
5543 369218 : int args_size_bytes = 0;
5544 804276 : for (wasm::ValueType type : sig_->parameters()) {
5545 32920 : args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
5546 : }
5547 :
5548 : // The return value is also passed via this buffer:
5549 369218 : int return_size_bytes = 0;
5550 1104174 : for (wasm::ValueType type : sig_->returns()) {
5551 367478 : return_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
5552 : }
5553 :
5554 : // Get a stack slot for the arguments.
5555 : Node* arg_buffer =
5556 369218 : args_size_bytes == 0 && return_size_bytes == 0
5557 : ? mcgraph()->IntPtrConstant(0)
5558 367504 : : graph()->NewNode(mcgraph()->machine()->StackSlot(
5559 369218 : std::max(args_size_bytes, return_size_bytes), 8));
5560 :
5561 : // Now store all our arguments to the buffer.
5562 : int offset = 0;
5563 :
5564 435061 : for (int i = 0; i < param_count; ++i) {
5565 32921 : wasm::ValueType type = sig_->GetParam(i);
5566 : // Start from the parameter with index 1 to drop the instance_node.
5567 65842 : SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
5568 32921 : Int32Constant(offset), Param(i + 1), Effect(),
5569 : Control()));
5570 32921 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5571 : }
5572 : DCHECK_EQ(args_size_bytes, offset);
5573 :
5574 : // We are passing the raw arg_buffer here. To the GC and other parts, it
5575 : // looks like a Smi (lowest bit not set). In the runtime function however,
5576 : // don't call Smi::value on it, but just cast it to a byte pointer.
5577 738439 : Node* parameters[] = {jsgraph()->SmiConstant(func_index), arg_buffer};
5578 : BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
5579 369220 : arraysize(parameters));
5580 :
5581 : // Read back the return value.
5582 : DCHECK_LT(sig_->return_count(), wasm::kV8MaxWasmFunctionMultiReturns);
5583 369221 : unsigned return_count = static_cast<unsigned>(sig_->return_count());
5584 369221 : if (return_count == 0) {
5585 : Return(Int32Constant(0));
5586 : } else {
5587 366958 : Node** returns = Buffer(return_count);
5588 : offset = 0;
5589 1101912 : for (size_t i = 0; i < return_count; ++i) {
5590 367478 : wasm::ValueType type = sig_->GetReturn(i);
5591 367479 : Node* val = SetEffect(
5592 : graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
5593 : Int32Constant(offset), Effect(), Control()));
5594 367478 : returns[i] = val;
5595 367478 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5596 : }
5597 366957 : Return(return_count, returns);
5598 : }
5599 :
5600 738436 : if (ContainsInt64(sig_)) LowerInt64();
5601 369217 : }
5602 :
5603 1715 : void BuildCWasmEntry() {
5604 : // Build the start and the JS parameter nodes.
5605 1715 : SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
5606 :
5607 : // Create parameter nodes (offset by 1 for the receiver parameter).
5608 1715 : Node* code_entry = Param(CWasmEntryParameters::kCodeEntry + 1);
5609 1715 : Node* object_ref_node = Param(CWasmEntryParameters::kObjectRef + 1);
5610 1715 : Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
5611 :
5612 1715 : int wasm_arg_count = static_cast<int>(sig_->parameter_count());
5613 : int arg_count =
5614 1715 : wasm_arg_count + 4; // code, object_ref_node, control, effect
5615 1715 : Node** args = Buffer(arg_count);
5616 :
5617 : int pos = 0;
5618 1715 : args[pos++] = code_entry;
5619 1715 : args[pos++] = object_ref_node;
5620 :
5621 : int offset = 0;
5622 13218 : for (wasm::ValueType type : sig_->parameters()) {
5623 4894 : Node* arg_load = SetEffect(
5624 : graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
5625 : Int32Constant(offset), Effect(), Control()));
5626 4894 : args[pos++] = arg_load;
5627 4894 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5628 : }
5629 :
5630 3430 : args[pos++] = Effect();
5631 3430 : args[pos++] = Control();
5632 : DCHECK_EQ(arg_count, pos);
5633 :
5634 : // Call the wasm code.
5635 1715 : auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
5636 :
5637 1715 : Node* call = SetEffect(graph()->NewNode(
5638 : mcgraph()->common()->Call(call_descriptor), arg_count, args));
5639 :
5640 : // Store the return value.
5641 : DCHECK_GE(1, sig_->return_count());
5642 1715 : if (sig_->return_count() == 1) {
5643 : StoreRepresentation store_rep(
5644 : wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
5645 1319 : kNoWriteBarrier);
5646 1319 : SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
5647 : arg_buffer, Int32Constant(0), call, Effect(),
5648 : Control()));
5649 : }
5650 : Return(jsgraph()->SmiConstant(0));
5651 :
5652 1715 : if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
5653 : MachineRepresentation sig_reps[] = {
5654 : MachineRepresentation::kWord32, // return value
5655 : MachineRepresentation::kTagged, // receiver
5656 : MachineRepresentation::kTagged, // arg0 (code)
5657 : MachineRepresentation::kTagged // arg1 (buffer)
5658 0 : };
5659 : Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
5660 : Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
5661 0 : mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
5662 0 : r.LowerGraph();
5663 : }
5664 1715 : }
5665 :
5666 : JSGraph* jsgraph() { return jsgraph_; }
5667 :
5668 : private:
5669 : Isolate* const isolate_;
5670 : JSGraph* jsgraph_;
5671 : StubCallMode stub_mode_;
5672 : SetOncePointer<const Operator> allocate_heap_number_operator_;
5673 : wasm::WasmFeatures enabled_features_;
5674 : };
5675 :
5676 142681 : void AppendSignature(char* buffer, size_t max_name_len,
5677 : wasm::FunctionSig* sig) {
5678 142681 : size_t name_len = strlen(buffer);
5679 : auto append_name_char = [&](char c) {
5680 256776 : if (name_len + 1 < max_name_len) buffer[name_len++] = c;
5681 : };
5682 322145 : for (wasm::ValueType t : sig->parameters()) {
5683 : append_name_char(wasm::ValueTypes::ShortNameOf(t));
5684 : }
5685 : append_name_char(':');
5686 191407 : for (wasm::ValueType t : sig->returns()) {
5687 : append_name_char(wasm::ValueTypes::ShortNameOf(t));
5688 : }
5689 142681 : buffer[name_len] = '\0';
5690 142681 : }
5691 :
5692 : } // namespace
5693 :
5694 140966 : MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
5695 : wasm::FunctionSig* sig,
5696 : bool is_import) {
5697 422898 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5698 : "CompileJSToWasmWrapper");
5699 : //----------------------------------------------------------------------------
5700 : // Create the Graph.
5701 : //----------------------------------------------------------------------------
5702 281931 : Zone zone(isolate->allocator(), ZONE_NAME);
5703 140966 : Graph graph(&zone);
5704 140966 : CommonOperatorBuilder common(&zone);
5705 : MachineOperatorBuilder machine(
5706 : &zone, MachineType::PointerRepresentation(),
5707 : InstructionSelector::SupportedMachineOperatorFlags(),
5708 140966 : InstructionSelector::AlignmentRequirements());
5709 140966 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5710 :
5711 140966 : Node* control = nullptr;
5712 140966 : Node* effect = nullptr;
5713 :
5714 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5715 : StubCallMode::kCallCodeObject,
5716 140966 : wasm::WasmFeaturesFromIsolate(isolate));
5717 : builder.set_control_ptr(&control);
5718 : builder.set_effect_ptr(&effect);
5719 140966 : builder.BuildJSToWasmWrapper(is_import);
5720 :
5721 : //----------------------------------------------------------------------------
5722 : // Run the compilation pipeline.
5723 : //----------------------------------------------------------------------------
5724 : static constexpr size_t kMaxNameLen = 128;
5725 140966 : char debug_name[kMaxNameLen] = "js_to_wasm:";
5726 140966 : AppendSignature(debug_name, kMaxNameLen, sig);
5727 :
5728 : // Schedule and compile to machine code.
5729 140966 : int params = static_cast<int>(sig->parameter_count());
5730 281932 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5731 140966 : &zone, false, params + 1, CallDescriptor::kNoFlags);
5732 :
5733 : MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5734 : isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
5735 140966 : WasmAssemblerOptions());
5736 : Handle<Code> code;
5737 140963 : if (!maybe_code.ToHandle(&code)) {
5738 0 : return maybe_code;
5739 : }
5740 : #ifdef ENABLE_DISASSEMBLER
5741 : if (FLAG_print_opt_code) {
5742 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5743 : OFStream os(tracing_scope.file());
5744 : code->Disassemble(debug_name, os);
5745 : }
5746 : #endif
5747 :
5748 140965 : if (must_record_function_compilation(isolate)) {
5749 : RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
5750 4 : debug_name);
5751 : }
5752 :
5753 140965 : return code;
5754 : }
5755 :
5756 120065 : WasmImportCallKind GetWasmImportCallKind(Handle<JSReceiver> target,
5757 : wasm::FunctionSig* expected_sig,
5758 : bool has_bigint_feature) {
5759 120065 : if (WasmExportedFunction::IsWasmExportedFunction(*target)) {
5760 107280 : auto imported_function = WasmExportedFunction::cast(*target);
5761 107280 : auto func_index = imported_function->function_index();
5762 107280 : auto module = imported_function->instance()->module();
5763 214560 : wasm::FunctionSig* imported_sig = module->functions[func_index].sig;
5764 107280 : if (*imported_sig != *expected_sig) {
5765 : return WasmImportCallKind::kLinkError;
5766 : }
5767 107128 : if (static_cast<uint32_t>(func_index) < module->num_imported_functions) {
5768 : // TODO(wasm): this redirects all imported-reexported functions
5769 : // through the call builtin. Fall through to JS function cases below?
5770 : return WasmImportCallKind::kUseCallBuiltin;
5771 : }
5772 107064 : return WasmImportCallKind::kWasmToWasm;
5773 : }
5774 : // Assuming we are calling to JS, check whether this would be a runtime error.
5775 12785 : if (!wasm::IsJSCompatibleSignature(expected_sig, has_bigint_feature)) {
5776 : return WasmImportCallKind::kRuntimeTypeError;
5777 : }
5778 : // For JavaScript calls, determine whether the target has an arity match
5779 : // and whether it has a sloppy receiver.
5780 12709 : if (target->IsJSFunction()) {
5781 : Handle<JSFunction> function = Handle<JSFunction>::cast(target);
5782 : SharedFunctionInfo shared = function->shared();
5783 :
5784 : // Check for math intrinsics.
5785 : #define COMPARE_SIG_FOR_BUILTIN(name) \
5786 : { \
5787 : wasm::FunctionSig* sig = wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
5788 : if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name); \
5789 : DCHECK_NOT_NULL(sig); \
5790 : if (*expected_sig == *sig) return WasmImportCallKind::k##name; \
5791 : }
5792 : #define COMPARE_SIG_FOR_BUILTIN_F64(name) \
5793 : case Builtins::kMath##name: \
5794 : COMPARE_SIG_FOR_BUILTIN(F64##name); \
5795 : break;
5796 : #define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
5797 : case Builtins::kMath##name: \
5798 : COMPARE_SIG_FOR_BUILTIN(F64##name); \
5799 : COMPARE_SIG_FOR_BUILTIN(F32##name); \
5800 : break;
5801 :
5802 21954 : if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
5803 552 : switch (shared->builtin_id()) {
5804 18 : COMPARE_SIG_FOR_BUILTIN_F64(Acos);
5805 18 : COMPARE_SIG_FOR_BUILTIN_F64(Asin);
5806 17 : COMPARE_SIG_FOR_BUILTIN_F64(Atan);
5807 27 : COMPARE_SIG_FOR_BUILTIN_F64(Cos);
5808 27 : COMPARE_SIG_FOR_BUILTIN_F64(Sin);
5809 17 : COMPARE_SIG_FOR_BUILTIN_F64(Tan);
5810 17 : COMPARE_SIG_FOR_BUILTIN_F64(Exp);
5811 18 : COMPARE_SIG_FOR_BUILTIN_F64(Log);
5812 18 : COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
5813 : //===========================================================
5814 : // TODO(8505): Math.pow for wasm does not match JS.
5815 : // COMPARE_SIG_FOR_BUILTIN_F64(Pow);
5816 : //===========================================================
5817 106 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
5818 24 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
5819 34 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
5820 34 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
5821 43 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
5822 52 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
5823 : case Builtins::kMathFround:
5824 8 : COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
5825 : break;
5826 : default:
5827 : break;
5828 : }
5829 : }
5830 :
5831 : #undef COMPARE_SIG_FOR_BUILTIN
5832 : #undef COMPARE_SIG_FOR_BUILTIN_F64
5833 : #undef COMPARE_SIG_FOR_BUILTIN_F32_F64
5834 :
5835 10753 : if (IsClassConstructor(shared->kind())) {
5836 : // Class constructor will throw anyway.
5837 : return WasmImportCallKind::kUseCallBuiltin;
5838 : }
5839 21114 : bool sloppy = is_sloppy(shared->language_mode()) && !shared->native();
5840 10737 : if (shared->internal_formal_parameter_count() ==
5841 : expected_sig->parameter_count()) {
5842 : return sloppy ? WasmImportCallKind::kJSFunctionArityMatchSloppy
5843 7983 : : WasmImportCallKind::kJSFunctionArityMatch;
5844 : }
5845 : return sloppy ? WasmImportCallKind::kJSFunctionArityMismatchSloppy
5846 2754 : : WasmImportCallKind::kJSFunctionArityMismatch;
5847 : }
5848 : // Unknown case. Use the call builtin.
5849 : return WasmImportCallKind::kUseCallBuiltin;
5850 : }
5851 :
5852 224 : wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
5853 : const char** name_ptr) {
5854 : #define CASE(name) \
5855 : case WasmImportCallKind::k##name: \
5856 : *name_ptr = "WasmMathIntrinsic:" #name; \
5857 : return wasm::kExpr##name
5858 224 : switch (kind) {
5859 8 : CASE(F64Acos);
5860 8 : CASE(F64Asin);
5861 8 : CASE(F64Atan);
5862 8 : CASE(F64Cos);
5863 8 : CASE(F64Sin);
5864 8 : CASE(F64Tan);
5865 8 : CASE(F64Exp);
5866 8 : CASE(F64Log);
5867 8 : CASE(F64Atan2);
5868 0 : CASE(F64Pow);
5869 16 : CASE(F64Ceil);
5870 16 : CASE(F64Floor);
5871 16 : CASE(F64Sqrt);
5872 16 : CASE(F64Min);
5873 16 : CASE(F64Max);
5874 16 : CASE(F64Abs);
5875 8 : CASE(F32Min);
5876 8 : CASE(F32Max);
5877 8 : CASE(F32Abs);
5878 8 : CASE(F32Ceil);
5879 8 : CASE(F32Floor);
5880 8 : CASE(F32Sqrt);
5881 8 : CASE(F32ConvertF64);
5882 : default:
5883 0 : UNREACHABLE();
5884 : return wasm::kExprUnreachable;
5885 : }
5886 : #undef CASE
5887 : }
5888 :
5889 224 : wasm::WasmCode* CompileWasmMathIntrinsic(wasm::WasmEngine* wasm_engine,
5890 : wasm::NativeModule* native_module,
5891 : WasmImportCallKind kind,
5892 : wasm::FunctionSig* sig) {
5893 : DCHECK_EQ(1, sig->return_count());
5894 :
5895 672 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5896 : "CompileWasmMathIntrinsic");
5897 :
5898 448 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5899 :
5900 : // Compile a WASM function with a single bytecode and let TurboFan
5901 : // generate either inlined machine code or a call to a helper.
5902 : SourcePositionTable* source_positions = nullptr;
5903 : MachineGraph* mcgraph = new (&zone) MachineGraph(
5904 448 : new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5905 : new (&zone) MachineOperatorBuilder(
5906 : &zone, MachineType::PointerRepresentation(),
5907 448 : InstructionSelector::SupportedMachineOperatorFlags(),
5908 672 : InstructionSelector::AlignmentRequirements()));
5909 :
5910 : wasm::CompilationEnv env(
5911 : native_module->module(), wasm::UseTrapHandler::kNoTrapHandler,
5912 : wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
5913 : wasm::kAllWasmFeatures, wasm::LowerSimd::kNoLowerSimd);
5914 :
5915 : WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
5916 : source_positions);
5917 :
5918 : // Set up the graph start.
5919 224 : Node* start = builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
5920 224 : Node* effect = start;
5921 224 : Node* control = start;
5922 : builder.set_effect_ptr(&effect);
5923 : builder.set_control_ptr(&control);
5924 224 : builder.set_instance_node(builder.Param(wasm::kWasmInstanceParameterIndex));
5925 :
5926 : // Generate either a unop or a binop.
5927 : Node* node = nullptr;
5928 224 : const char* debug_name = "WasmMathIntrinsic";
5929 224 : auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
5930 224 : switch (sig->parameter_count()) {
5931 : case 1:
5932 168 : node = builder.Unop(opcode, builder.Param(1));
5933 168 : break;
5934 : case 2:
5935 56 : node = builder.Binop(opcode, builder.Param(1), builder.Param(2));
5936 56 : break;
5937 : default:
5938 0 : UNREACHABLE();
5939 : break;
5940 : }
5941 :
5942 : builder.Return(node);
5943 :
5944 : // Run the compiler pipeline to generate machine code.
5945 224 : auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
5946 224 : if (mcgraph->machine()->Is32()) {
5947 : call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5948 : }
5949 :
5950 : wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
5951 : wasm_engine, call_descriptor, mcgraph, Code::WASM_FUNCTION,
5952 224 : wasm::WasmCode::kFunction, debug_name, WasmStubAssemblerOptions(),
5953 448 : source_positions);
5954 : std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode(
5955 : wasm::WasmCode::kAnonymousFuncIndex, result.code_desc,
5956 : result.frame_slot_count, result.tagged_parameter_slots,
5957 : std::move(result.protected_instructions),
5958 : std::move(result.source_positions), wasm::WasmCode::kFunction,
5959 896 : wasm::ExecutionTier::kNone);
5960 : // TODO(titzer): add counters for math intrinsic code size / allocation
5961 448 : return native_module->PublishCode(std::move(wasm_code));
5962 : }
5963 :
5964 7082 : wasm::WasmCode* CompileWasmImportCallWrapper(wasm::WasmEngine* wasm_engine,
5965 : wasm::NativeModule* native_module,
5966 : WasmImportCallKind kind,
5967 : wasm::FunctionSig* sig,
5968 : bool source_positions) {
5969 : DCHECK_NE(WasmImportCallKind::kLinkError, kind);
5970 : DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind);
5971 :
5972 : // Check for math intrinsics first.
5973 7082 : if (FLAG_wasm_math_intrinsics &&
5974 7082 : kind >= WasmImportCallKind::kFirstMathIntrinsic &&
5975 : kind <= WasmImportCallKind::kLastMathIntrinsic) {
5976 224 : return CompileWasmMathIntrinsic(wasm_engine, native_module, kind, sig);
5977 : }
5978 :
5979 20574 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5980 : "CompileWasmImportCallWrapper");
5981 : //----------------------------------------------------------------------------
5982 : // Create the Graph
5983 : //----------------------------------------------------------------------------
5984 13716 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5985 6858 : Graph graph(&zone);
5986 6858 : CommonOperatorBuilder common(&zone);
5987 : MachineOperatorBuilder machine(
5988 : &zone, MachineType::PointerRepresentation(),
5989 : InstructionSelector::SupportedMachineOperatorFlags(),
5990 6858 : InstructionSelector::AlignmentRequirements());
5991 6858 : JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
5992 :
5993 6858 : Node* control = nullptr;
5994 6858 : Node* effect = nullptr;
5995 :
5996 : SourcePositionTable* source_position_table =
5997 7352 : source_positions ? new (&zone) SourcePositionTable(&graph) : nullptr;
5998 :
5999 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, source_position_table,
6000 : StubCallMode::kCallWasmRuntimeStub,
6001 6858 : native_module->enabled_features());
6002 : builder.set_control_ptr(&control);
6003 : builder.set_effect_ptr(&effect);
6004 6858 : builder.BuildWasmImportCallWrapper(kind);
6005 :
6006 : const char* func_name = "wasm-to-js";
6007 :
6008 : // Schedule and compile to machine code.
6009 : CallDescriptor* incoming =
6010 : GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline,
6011 6858 : WasmGraphBuilder::kExtraCallableParam);
6012 6858 : if (machine.Is32()) {
6013 : incoming = GetI32WasmCallDescriptor(&zone, incoming);
6014 : }
6015 : wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
6016 : wasm_engine, incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
6017 6858 : wasm::WasmCode::kWasmToJsWrapper, func_name, WasmStubAssemblerOptions(),
6018 13716 : source_position_table);
6019 : std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode(
6020 : wasm::WasmCode::kAnonymousFuncIndex, result.code_desc,
6021 : result.frame_slot_count, result.tagged_parameter_slots,
6022 : std::move(result.protected_instructions),
6023 : std::move(result.source_positions), wasm::WasmCode::kWasmToJsWrapper,
6024 27432 : wasm::ExecutionTier::kNone);
6025 6858 : return native_module->PublishCode(std::move(wasm_code));
6026 : }
6027 :
6028 369214 : wasm::WasmCompilationResult CompileWasmInterpreterEntry(
6029 : wasm::WasmEngine* wasm_engine, const wasm::WasmFeatures& enabled_features,
6030 : uint32_t func_index, wasm::FunctionSig* sig) {
6031 : //----------------------------------------------------------------------------
6032 : // Create the Graph
6033 : //----------------------------------------------------------------------------
6034 738429 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
6035 369213 : Graph graph(&zone);
6036 369214 : CommonOperatorBuilder common(&zone);
6037 : MachineOperatorBuilder machine(
6038 : &zone, MachineType::PointerRepresentation(),
6039 : InstructionSelector::SupportedMachineOperatorFlags(),
6040 369215 : InstructionSelector::AlignmentRequirements());
6041 369217 : JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
6042 :
6043 369216 : Node* control = nullptr;
6044 369216 : Node* effect = nullptr;
6045 :
6046 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
6047 : StubCallMode::kCallWasmRuntimeStub,
6048 369216 : enabled_features);
6049 : builder.set_control_ptr(&control);
6050 : builder.set_effect_ptr(&effect);
6051 369218 : builder.BuildWasmInterpreterEntry(func_index);
6052 :
6053 : // Schedule and compile to machine code.
6054 369217 : CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
6055 369213 : if (machine.Is32()) {
6056 : incoming = GetI32WasmCallDescriptor(&zone, incoming);
6057 : }
6058 :
6059 : EmbeddedVector<char, 32> func_name;
6060 369217 : func_name.Truncate(
6061 369213 : SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
6062 :
6063 : wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
6064 : wasm_engine, incoming, &jsgraph, Code::WASM_INTERPRETER_ENTRY,
6065 : wasm::WasmCode::kInterpreterEntry, func_name.start(),
6066 369217 : WasmStubAssemblerOptions());
6067 369215 : result.result_tier = wasm::ExecutionTier::kInterpreter;
6068 :
6069 369222 : return result;
6070 : }
6071 :
6072 1715 : MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
6073 3430 : Zone zone(isolate->allocator(), ZONE_NAME);
6074 1715 : Graph graph(&zone);
6075 1715 : CommonOperatorBuilder common(&zone);
6076 : MachineOperatorBuilder machine(
6077 : &zone, MachineType::PointerRepresentation(),
6078 : InstructionSelector::SupportedMachineOperatorFlags(),
6079 1715 : InstructionSelector::AlignmentRequirements());
6080 1715 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
6081 :
6082 1715 : Node* control = nullptr;
6083 1715 : Node* effect = nullptr;
6084 :
6085 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
6086 : StubCallMode::kCallCodeObject,
6087 1715 : wasm::WasmFeaturesFromIsolate(isolate));
6088 : builder.set_control_ptr(&control);
6089 : builder.set_effect_ptr(&effect);
6090 1715 : builder.BuildCWasmEntry();
6091 :
6092 : // Schedule and compile to machine code.
6093 1715 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
6094 : &zone, false, CWasmEntryParameters::kNumParameters + 1,
6095 1715 : CallDescriptor::kNoFlags);
6096 :
6097 : // Build a name in the form "c-wasm-entry:<params>:<returns>".
6098 : static constexpr size_t kMaxNameLen = 128;
6099 1715 : char debug_name[kMaxNameLen] = "c-wasm-entry:";
6100 1715 : AppendSignature(debug_name, kMaxNameLen, sig);
6101 :
6102 : MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
6103 : isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
6104 1715 : AssemblerOptions::Default(isolate));
6105 : Handle<Code> code;
6106 1715 : if (!maybe_code.ToHandle(&code)) {
6107 0 : return maybe_code;
6108 : }
6109 : #ifdef ENABLE_DISASSEMBLER
6110 : if (FLAG_print_opt_code) {
6111 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
6112 : OFStream os(tracing_scope.file());
6113 : code->Disassemble(debug_name, os);
6114 : }
6115 : #endif
6116 :
6117 1715 : return code;
6118 : }
6119 :
6120 631163 : TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
6121 : wasm::WasmCompilationUnit* wasm_unit)
6122 631163 : : wasm_unit_(wasm_unit) {}
6123 :
6124 : // Clears unique_ptrs, but (part of) the type is forward declared in the header.
6125 : TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
6126 :
6127 628585 : bool TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
6128 : AccountingAllocator* allocator, wasm::CompilationEnv* env,
6129 : const wasm::FunctionBody& func_body, wasm::WasmFeatures* detected,
6130 : double* decode_ms, MachineGraph* mcgraph, NodeOriginTable* node_origins,
6131 : SourcePositionTable* source_positions) {
6132 : base::ElapsedTimer decode_timer;
6133 : if (FLAG_trace_wasm_decode_time) {
6134 : decode_timer.Start();
6135 : }
6136 :
6137 : // Create a TF graph during decoding.
6138 628585 : WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
6139 : source_positions);
6140 : wasm::VoidResult graph_construction_result =
6141 1256268 : wasm::BuildTFGraph(allocator, env->enabled_features, env->module,
6142 : &builder, detected, func_body, node_origins);
6143 627683 : if (graph_construction_result.failed()) {
6144 : if (FLAG_trace_wasm_compiler) {
6145 : StdoutStream{} << "Compilation failed: "
6146 : << graph_construction_result.error().message()
6147 : << std::endl;
6148 : }
6149 : return false;
6150 : }
6151 :
6152 618084 : builder.LowerInt64();
6153 :
6154 1243964 : if (builder.has_simd() &&
6155 8258 : (!CpuFeatures::SupportsWasmSimd128() || env->lower_simd)) {
6156 : SimdScalarLowering(mcgraph,
6157 4084 : CreateMachineSignature(mcgraph->zone(), func_body.sig))
6158 8168 : .LowerGraph();
6159 : }
6160 :
6161 1235719 : if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
6162 617866 : wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
6163 0 : PrintRawWasmCode(allocator, func_body, env->module, wasm::kPrintLocals);
6164 : }
6165 : if (FLAG_trace_wasm_decode_time) {
6166 : *decode_ms = decode_timer.Elapsed().InMillisecondsF();
6167 : }
6168 : return true;
6169 : }
6170 :
6171 : namespace {
6172 628605 : Vector<const char> GetDebugName(Zone* zone, int index) {
6173 : // TODO(herhut): Use name from module if available.
6174 : constexpr int kBufferLength = 24;
6175 :
6176 : EmbeddedVector<char, kBufferLength> name_vector;
6177 628605 : int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
6178 : DCHECK(name_len > 0 && name_len < name_vector.length());
6179 :
6180 628855 : char* index_name = zone->NewArray<char>(name_len);
6181 : memcpy(index_name, name_vector.start(), name_len);
6182 629065 : return Vector<const char>(index_name, name_len);
6183 : }
6184 : } // namespace
6185 :
6186 628865 : wasm::WasmCompilationResult TurbofanWasmCompilationUnit::ExecuteCompilation(
6187 : wasm::WasmEngine* wasm_engine, wasm::CompilationEnv* env,
6188 : const wasm::FunctionBody& func_body, Counters* counters,
6189 : wasm::WasmFeatures* detected) {
6190 1886763 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
6191 : "ExecuteTurbofanCompilation");
6192 628871 : double decode_ms = 0;
6193 : size_t node_count = 0;
6194 :
6195 1255675 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
6196 : MachineGraph* mcgraph = new (&zone) MachineGraph(
6197 1257566 : new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
6198 : new (&zone) MachineOperatorBuilder(
6199 : &zone, MachineType::PointerRepresentation(),
6200 1257110 : InstructionSelector::SupportedMachineOperatorFlags(),
6201 1886100 : InstructionSelector::AlignmentRequirements()));
6202 :
6203 628581 : OptimizedCompilationInfo info(GetDebugName(&zone, wasm_unit_->func_index_),
6204 1257421 : &zone, Code::WASM_FUNCTION);
6205 628037 : if (env->runtime_exception_support) {
6206 : info.SetWasmRuntimeExceptionSupport();
6207 : }
6208 :
6209 628037 : if (info.trace_turbo_json_enabled()) {
6210 16 : TurboCfgFile tcf;
6211 8 : tcf << AsC1VCompilation(&info);
6212 : }
6213 :
6214 : NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
6215 : ? new (&zone)
6216 8 : NodeOriginTable(mcgraph->graph())
6217 628037 : : nullptr;
6218 : SourcePositionTable* source_positions =
6219 628069 : new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
6220 627991 : if (!BuildGraphForWasmFunction(wasm_engine->allocator(), env, func_body,
6221 : detected, &decode_ms, mcgraph, node_origins,
6222 : source_positions)) {
6223 9130 : return wasm::WasmCompilationResult{};
6224 : }
6225 :
6226 618136 : if (node_origins) {
6227 8 : node_origins->AddDecorator();
6228 : }
6229 :
6230 : base::ElapsedTimer pipeline_timer;
6231 : if (FLAG_trace_wasm_decode_time) {
6232 : node_count = mcgraph->graph()->NodeCount();
6233 : pipeline_timer.Start();
6234 : }
6235 :
6236 : // Run the compiler pipeline to generate machine code.
6237 618136 : auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
6238 618257 : if (mcgraph->machine()->Is32()) {
6239 : call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
6240 : }
6241 :
6242 : Pipeline::GenerateCodeForWasmFunction(
6243 : &info, wasm_engine, mcgraph, call_descriptor, source_positions,
6244 618257 : node_origins, func_body, env->module, wasm_unit_->func_index_);
6245 :
6246 : if (FLAG_trace_wasm_decode_time) {
6247 : double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
6248 : PrintF(
6249 : "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
6250 : "%0.3f ms pipeline\n",
6251 : static_cast<unsigned>(func_body.end - func_body.start), decode_ms,
6252 : node_count, pipeline_ms);
6253 : }
6254 : // TODO(bradnelson): Improve histogram handling of size_t.
6255 619198 : counters->wasm_compile_function_peak_memory_bytes()->AddSample(
6256 619198 : static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
6257 1235889 : auto result = info.ReleaseWasmCompilationResult();
6258 : DCHECK_EQ(wasm::ExecutionTier::kTurbofan, result->result_tier);
6259 : return std::move(*result);
6260 : }
6261 :
6262 1437 : wasm::WasmCompilationResult InterpreterCompilationUnit::ExecuteCompilation(
6263 : wasm::WasmEngine* wasm_engine, wasm::CompilationEnv* env,
6264 : const wasm::FunctionBody& func_body, Counters* counters,
6265 : wasm::WasmFeatures* detected) {
6266 2865 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
6267 1437 : const wasm::WasmModule* module = env ? env->module : nullptr;
6268 : wasm::WasmFullDecoder<wasm::Decoder::kValidate, wasm::EmptyInterface> decoder(
6269 : &zone, module, env->enabled_features, detected, func_body);
6270 1437 : decoder.Decode();
6271 1436 : if (decoder.failed()) return wasm::WasmCompilationResult{};
6272 :
6273 : wasm::WasmCompilationResult result =
6274 : CompileWasmInterpreterEntry(wasm_engine, env->enabled_features,
6275 2869 : wasm_unit_->func_index_, func_body.sig);
6276 : DCHECK(result.succeeded());
6277 : DCHECK_EQ(wasm::ExecutionTier::kInterpreter, result.result_tier);
6278 :
6279 : return result;
6280 : }
6281 :
6282 : namespace {
6283 : // Helper for allocating either an GP or FP reg, or the next stack slot.
6284 : class LinkageLocationAllocator {
6285 : public:
6286 : template <size_t kNumGpRegs, size_t kNumFpRegs>
6287 : constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
6288 : const DoubleRegister (&fp)[kNumFpRegs])
6289 : : allocator_(wasm::LinkageAllocator(gp, fp)) {}
6290 :
6291 5677138 : LinkageLocation Next(MachineRepresentation rep) {
6292 5677138 : MachineType type = MachineType::TypeForRepresentation(rep);
6293 5677286 : if (IsFloatingPoint(rep)) {
6294 616976 : if (allocator_.CanAllocateFP(rep)) {
6295 : int reg_code = allocator_.NextFpReg(rep);
6296 : return LinkageLocation::ForRegister(reg_code, type);
6297 : }
6298 5060310 : } else if (allocator_.CanAllocateGP()) {
6299 : int reg_code = allocator_.NextGpReg();
6300 : return LinkageLocation::ForRegister(reg_code, type);
6301 : }
6302 : // Cannot use register; use stack slot.
6303 174572 : int index = -1 - allocator_.NextStackSlot(rep);
6304 : return LinkageLocation::ForCallerFrameSlot(index, type);
6305 : }
6306 :
6307 : void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
6308 : int NumStackSlots() const { return allocator_.NumStackSlots(); }
6309 :
6310 : private:
6311 : wasm::LinkageAllocator allocator_;
6312 : };
6313 : } // namespace
6314 :
6315 : // General code uses the above configuration data.
6316 2519936 : CallDescriptor* GetWasmCallDescriptor(
6317 : Zone* zone, wasm::FunctionSig* fsig,
6318 : WasmGraphBuilder::UseRetpoline use_retpoline,
6319 : WasmGraphBuilder::ExtraCallableParam extra_callable_param) {
6320 : // The extra here is to accomodate the instance object as first parameter
6321 : // and, in the case of an import wrapper, the additional callable.
6322 2519936 : int extra_params = extra_callable_param ? 2 : 1;
6323 : LocationSignature::Builder locations(zone, fsig->return_count(),
6324 2519936 : fsig->parameter_count() + extra_params);
6325 :
6326 : // Add register and/or stack parameter(s).
6327 : LinkageLocationAllocator params(wasm::kGpParamRegisters,
6328 2520528 : wasm::kFpParamRegisters);
6329 :
6330 : // The instance object.
6331 2520528 : locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
6332 : const size_t param_offset = 1; // Actual params start here.
6333 :
6334 : // Parameters are separated into two groups (first all untagged, then all
6335 : // tagged parameters). This allows for easy iteration of tagged parameters
6336 : // during frame iteration.
6337 : const size_t parameter_count = fsig->parameter_count();
6338 4429315 : for (size_t i = 0; i < parameter_count; i++) {
6339 : MachineRepresentation param =
6340 953809 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
6341 : // Skip tagged parameters (e.g. any-ref).
6342 966969 : if (IsAnyTagged(param)) continue;
6343 945251 : auto l = params.Next(param);
6344 942588 : locations.AddParamAt(i + param_offset, l);
6345 : }
6346 4433299 : for (size_t i = 0; i < parameter_count; i++) {
6347 : MachineRepresentation param =
6348 955782 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
6349 : // Skip untagged parameters.
6350 1902553 : if (!IsAnyTagged(param)) continue;
6351 10847 : auto l = params.Next(param);
6352 9767 : locations.AddParamAt(i + param_offset, l);
6353 : }
6354 :
6355 : // Import call wrappers have an additional (implicit) parameter, the callable.
6356 : // For consistency with JS, we use the JSFunction register.
6357 2521897 : if (extra_callable_param) {
6358 : locations.AddParam(LinkageLocation::ForRegister(
6359 : kJSFunctionRegister.code(), MachineType::TaggedPointer()));
6360 : }
6361 :
6362 : // Add return location(s).
6363 : LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
6364 2521897 : wasm::kFpReturnRegisters);
6365 :
6366 : int parameter_slots = params.NumStackSlots();
6367 : if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
6368 :
6369 : rets.SetStackOffset(parameter_slots);
6370 :
6371 2521897 : const int return_count = static_cast<int>(locations.return_count_);
6372 6924103 : for (int i = 0; i < return_count; i++) {
6373 : MachineRepresentation ret =
6374 4403530 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
6375 2202295 : auto l = rets.Next(ret);
6376 : locations.AddReturn(l);
6377 : }
6378 :
6379 : const RegList kCalleeSaveRegisters = 0;
6380 : const RegList kCalleeSaveFPRegisters = 0;
6381 :
6382 : // The target for wasm calls is always a code object.
6383 : MachineType target_type = MachineType::Pointer();
6384 : LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
6385 :
6386 : CallDescriptor::Kind kind = extra_callable_param
6387 : ? CallDescriptor::kCallWasmImportWrapper
6388 2521235 : : CallDescriptor::kCallWasmFunction;
6389 :
6390 : CallDescriptor::Flags flags =
6391 2521235 : use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
6392 : return new (zone) CallDescriptor( // --
6393 : kind, // kind
6394 : target_type, // target MachineType
6395 : target_loc, // target location
6396 : locations.Build(), // location_sig
6397 : parameter_slots, // stack_parameter_count
6398 : compiler::Operator::kNoProperties, // properties
6399 : kCalleeSaveRegisters, // callee-saved registers
6400 : kCalleeSaveFPRegisters, // callee-saved fp regs
6401 : flags, // flags
6402 : "wasm-call", // debug name
6403 : 0, // allocatable registers
6404 7562625 : rets.NumStackSlots() - parameter_slots); // stack_return_count
6405 : }
6406 :
6407 : namespace {
6408 0 : CallDescriptor* ReplaceTypeInCallDescriptorWith(
6409 : Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
6410 : MachineType input_type, MachineRepresentation output_type) {
6411 : size_t parameter_count = call_descriptor->ParameterCount();
6412 : size_t return_count = call_descriptor->ReturnCount();
6413 0 : for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
6414 0 : if (call_descriptor->GetParameterType(i) == input_type) {
6415 0 : parameter_count += num_replacements - 1;
6416 : }
6417 : }
6418 0 : for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
6419 0 : if (call_descriptor->GetReturnType(i) == input_type) {
6420 0 : return_count += num_replacements - 1;
6421 : }
6422 : }
6423 0 : if (parameter_count == call_descriptor->ParameterCount() &&
6424 : return_count == call_descriptor->ReturnCount()) {
6425 : return call_descriptor;
6426 : }
6427 :
6428 : LocationSignature::Builder locations(zone, return_count, parameter_count);
6429 :
6430 : LinkageLocationAllocator params(wasm::kGpParamRegisters,
6431 0 : wasm::kFpParamRegisters);
6432 0 : for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
6433 0 : if (call_descriptor->GetParameterType(i) == input_type) {
6434 0 : for (size_t j = 0; j < num_replacements; j++) {
6435 0 : locations.AddParam(params.Next(output_type));
6436 : }
6437 : } else {
6438 0 : locations.AddParam(
6439 : params.Next(call_descriptor->GetParameterType(i).representation()));
6440 : }
6441 : }
6442 :
6443 : LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
6444 0 : wasm::kFpReturnRegisters);
6445 : rets.SetStackOffset(params.NumStackSlots());
6446 0 : for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
6447 0 : if (call_descriptor->GetReturnType(i) == input_type) {
6448 0 : for (size_t j = 0; j < num_replacements; j++) {
6449 0 : locations.AddReturn(rets.Next(output_type));
6450 : }
6451 : } else {
6452 0 : locations.AddReturn(
6453 : rets.Next(call_descriptor->GetReturnType(i).representation()));
6454 : }
6455 : }
6456 :
6457 : return new (zone) CallDescriptor( // --
6458 : call_descriptor->kind(), // kind
6459 : call_descriptor->GetInputType(0), // target MachineType
6460 : call_descriptor->GetInputLocation(0), // target location
6461 : locations.Build(), // location_sig
6462 : params.NumStackSlots(), // stack_parameter_count
6463 : call_descriptor->properties(), // properties
6464 : call_descriptor->CalleeSavedRegisters(), // callee-saved registers
6465 : call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
6466 : call_descriptor->flags(), // flags
6467 : call_descriptor->debug_name(), // debug name
6468 : call_descriptor->AllocatableRegisters(), // allocatable registers
6469 0 : rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
6470 : }
6471 : } // namespace
6472 :
6473 0 : CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
6474 : CallDescriptor* call_descriptor) {
6475 : return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
6476 : MachineType::Int64(),
6477 0 : MachineRepresentation::kWord32);
6478 : }
6479 :
6480 0 : CallDescriptor* GetI32WasmCallDescriptorForSimd(
6481 : Zone* zone, CallDescriptor* call_descriptor) {
6482 : return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
6483 : MachineType::Simd128(),
6484 0 : MachineRepresentation::kWord32);
6485 : }
6486 :
6487 618470 : AssemblerOptions WasmAssemblerOptions() {
6488 759436 : AssemblerOptions options;
6489 : // Relocation info required to serialize {WasmCode} for proper functions.
6490 : options.record_reloc_info_for_serialization = true;
6491 : options.enable_root_array_delta_access = false;
6492 618470 : return options;
6493 : }
6494 :
6495 0 : AssemblerOptions WasmStubAssemblerOptions() {
6496 376299 : AssemblerOptions options;
6497 : // Relocation info not necessary because stubs are not serialized.
6498 376299 : options.record_reloc_info_for_serialization = false;
6499 : options.enable_root_array_delta_access = false;
6500 0 : return options;
6501 : }
6502 :
6503 : #undef WASM_64
6504 : #undef FATAL_UNSUPPORTED_OPCODE
6505 : #undef WASM_INSTANCE_OBJECT_SIZE
6506 : #undef WASM_INSTANCE_OBJECT_OFFSET
6507 : #undef LOAD_RAW
6508 : #undef LOAD_INSTANCE_FIELD
6509 : #undef LOAD_TAGGED_POINTER
6510 : #undef LOAD_TAGGED_ANY
6511 : #undef LOAD_FIXED_ARRAY_SLOT
6512 : #undef LOAD_FIXED_ARRAY_SLOT_SMI
6513 : #undef LOAD_FIXED_ARRAY_SLOT_PTR
6514 : #undef LOAD_FIXED_ARRAY_SLOT_ANY
6515 : #undef STORE_FIXED_ARRAY_SLOT_SMI
6516 : #undef STORE_FIXED_ARRAY_SLOT_ANY
6517 :
6518 : } // namespace compiler
6519 : } // namespace internal
6520 122004 : } // namespace v8
|
