Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/wasm-compiler.h"
6 :
7 : #include <memory>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/assembler.h"
11 : #include "src/base/optional.h"
12 : #include "src/base/platform/elapsed-timer.h"
13 : #include "src/base/platform/platform.h"
14 : #include "src/base/v8-fallthrough.h"
15 : #include "src/builtins/builtins.h"
16 : #include "src/code-factory.h"
17 : #include "src/compiler/backend/code-generator.h"
18 : #include "src/compiler/backend/instruction-selector.h"
19 : #include "src/compiler/common-operator.h"
20 : #include "src/compiler/compiler-source-position-table.h"
21 : #include "src/compiler/diamond.h"
22 : #include "src/compiler/graph-visualizer.h"
23 : #include "src/compiler/graph.h"
24 : #include "src/compiler/int64-lowering.h"
25 : #include "src/compiler/js-graph.h"
26 : #include "src/compiler/js-operator.h"
27 : #include "src/compiler/linkage.h"
28 : #include "src/compiler/machine-operator.h"
29 : #include "src/compiler/node-matchers.h"
30 : #include "src/compiler/node-origin-table.h"
31 : #include "src/compiler/pipeline.h"
32 : #include "src/compiler/simd-scalar-lowering.h"
33 : #include "src/compiler/zone-stats.h"
34 : #include "src/counters.h"
35 : #include "src/heap/factory.h"
36 : #include "src/interface-descriptors.h"
37 : #include "src/isolate-inl.h"
38 : #include "src/log-inl.h"
39 : #include "src/objects/heap-number.h"
40 : #include "src/optimized-compilation-info.h"
41 : #include "src/tracing/trace-event.h"
42 : #include "src/trap-handler/trap-handler.h"
43 : #include "src/wasm/function-body-decoder.h"
44 : #include "src/wasm/function-compiler.h"
45 : #include "src/wasm/graph-builder-interface.h"
46 : #include "src/wasm/jump-table-assembler.h"
47 : #include "src/wasm/memory-tracing.h"
48 : #include "src/wasm/object-access.h"
49 : #include "src/wasm/wasm-code-manager.h"
50 : #include "src/wasm/wasm-limits.h"
51 : #include "src/wasm/wasm-linkage.h"
52 : #include "src/wasm/wasm-module.h"
53 : #include "src/wasm/wasm-objects-inl.h"
54 : #include "src/wasm/wasm-opcodes.h"
55 : #include "src/wasm/wasm-text.h"
56 :
57 : namespace v8 {
58 : namespace internal {
59 : namespace compiler {
60 :
61 : namespace {
62 :
63 : // TODO(titzer): pull WASM_64 up to a common header.
64 : #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
65 : #define WASM_64 1
66 : #else
67 : #define WASM_64 0
68 : #endif
69 :
70 : #define FATAL_UNSUPPORTED_OPCODE(opcode) \
71 : FATAL("Unsupported opcode 0x%x:%s", (opcode), \
72 : wasm::WasmOpcodes::OpcodeName(opcode));
73 :
74 : MachineType assert_size(int expected_size, MachineType type) {
75 : DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()));
76 : return type;
77 : }
78 :
79 : #define WASM_INSTANCE_OBJECT_SIZE(name) \
80 : (WasmInstanceObject::k##name##OffsetEnd - \
81 : WasmInstanceObject::k##name##Offset + 1) // NOLINT(whitespace/indent)
82 :
83 : #define WASM_INSTANCE_OBJECT_OFFSET(name) \
84 : wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
85 :
86 : #define LOAD_RAW(base_pointer, byte_offset, type) \
87 : SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), base_pointer, \
88 : mcgraph()->Int32Constant(byte_offset), Effect(), \
89 : Control()))
90 :
91 : #define LOAD_INSTANCE_FIELD(name, type) \
92 : LOAD_RAW(instance_node_.get(), WASM_INSTANCE_OBJECT_OFFSET(name), \
93 : assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type))
94 :
95 : #define LOAD_TAGGED_POINTER(base_pointer, byte_offset) \
96 : LOAD_RAW(base_pointer, byte_offset, MachineType::TaggedPointer())
97 :
98 : #define LOAD_TAGGED_ANY(base_pointer, byte_offset) \
99 : LOAD_RAW(base_pointer, byte_offset, MachineType::AnyTagged())
100 :
101 : #define LOAD_FIXED_ARRAY_SLOT(array_node, index, type) \
102 : LOAD_RAW(array_node, \
103 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), type)
104 :
105 : #define LOAD_FIXED_ARRAY_SLOT_SMI(array_node, index) \
106 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedSigned())
107 :
108 : #define LOAD_FIXED_ARRAY_SLOT_PTR(array_node, index) \
109 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedPointer())
110 :
111 : #define LOAD_FIXED_ARRAY_SLOT_ANY(array_node, index) \
112 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::AnyTagged())
113 :
114 : // This can be used to store tagged Smi values only.
115 : #define STORE_FIXED_ARRAY_SLOT_SMI(array_node, index, value) \
116 : SetEffect(graph()->NewNode( \
117 : mcgraph()->machine()->Store(StoreRepresentation( \
118 : MachineRepresentation::kTaggedSigned, kNoWriteBarrier)), \
119 : array_node, \
120 : mcgraph()->Int32Constant( \
121 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
122 : value, Effect(), Control()))
123 :
124 : // This can be used to store any tagged (Smi and HeapObject) value.
125 : #define STORE_FIXED_ARRAY_SLOT_ANY(array_node, index, value) \
126 : SetEffect(graph()->NewNode( \
127 : mcgraph()->machine()->Store(StoreRepresentation( \
128 : MachineRepresentation::kTagged, kFullWriteBarrier)), \
129 : array_node, \
130 : mcgraph()->Int32Constant( \
131 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
132 : value, Effect(), Control()))
133 :
134 2629972 : void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
135 1314986 : Graph* g = mcgraph->graph();
136 1314986 : if (g->end()) {
137 87013 : NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
138 : } else {
139 1227973 : g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
140 : }
141 1315062 : }
142 :
143 : bool ContainsSimd(wasm::FunctionSig* sig) {
144 2928623 : for (auto type : sig->all()) {
145 1464284 : if (type == wasm::kWasmS128) return true;
146 : }
147 : return false;
148 : }
149 :
150 : bool ContainsInt64(wasm::FunctionSig* sig) {
151 3886 : for (auto type : sig->all()) {
152 1964 : if (type == wasm::kWasmI64) return true;
153 : }
154 : return false;
155 : }
156 : } // namespace
157 :
158 1233795 : WasmGraphBuilder::WasmGraphBuilder(
159 : wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
160 : wasm::FunctionSig* sig,
161 : compiler::SourcePositionTable* source_position_table)
162 : : zone_(zone),
163 : mcgraph_(mcgraph),
164 : env_(env),
165 : cur_buffer_(def_buffer_),
166 : cur_bufsize_(kDefaultBufferSize),
167 : has_simd_(ContainsSimd(sig)),
168 : untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
169 : sig_(sig),
170 2467590 : source_position_table_(source_position_table) {
171 : DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
172 : DCHECK_NOT_NULL(mcgraph_);
173 1233795 : }
174 :
175 0 : Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
176 :
177 2467878 : Node* WasmGraphBuilder::Start(unsigned params) {
178 2467878 : Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
179 : graph()->SetStart(start);
180 1233815 : return start;
181 : }
182 :
183 1477800 : Node* WasmGraphBuilder::Param(unsigned index) {
184 : return graph()->NewNode(mcgraph()->common()->Parameter(index),
185 2955583 : graph()->start());
186 : }
187 :
188 10673 : Node* WasmGraphBuilder::Loop(Node* entry) {
189 21348 : return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
190 : }
191 :
192 21353 : Node* WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
193 : Node* terminate =
194 10677 : graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
195 10676 : MergeControlToEnd(mcgraph(), terminate);
196 10674 : return terminate;
197 : }
198 :
199 2031 : Node* WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
200 : Node* terminate =
201 1015 : graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
202 1016 : MergeControlToEnd(mcgraph(), terminate);
203 1016 : return terminate;
204 : }
205 :
206 2761956 : bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
207 3027842 : return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
208 1646863 : NodeProperties::GetControlInput(phi) == merge;
209 : }
210 :
211 1187 : bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
212 1295 : Node** if_exception) {
213 1187 : if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
214 : return false;
215 : }
216 :
217 1295 : *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
218 : *if_exception =
219 1295 : graph()->NewNode(mcgraph()->common()->IfException(), node, node);
220 :
221 648 : return true;
222 : }
223 :
224 484562 : void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
225 : DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
226 242279 : merge->AppendInput(mcgraph()->zone(), from);
227 : int new_size = merge->InputCount();
228 : NodeProperties::ChangeOp(
229 242283 : merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
230 242279 : }
231 :
232 379687 : void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
233 : DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
234 : int new_size = phi->InputCount();
235 253122 : phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
236 : NodeProperties::ChangeOp(
237 126565 : phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
238 126561 : }
239 :
240 94410 : Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
241 141615 : return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
242 : }
243 :
244 115311 : Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
245 115310 : Node* control) {
246 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
247 115311 : Node** buf = Realloc(vals, count, count + 1);
248 115313 : buf[count] = control;
249 : return graph()->NewNode(
250 : mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
251 : count),
252 345933 : count + 1, buf);
253 : }
254 :
255 36377 : Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
256 36380 : Node* control) {
257 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
258 36377 : Node** buf = Realloc(effects, count, count + 1);
259 36380 : buf[count] = control;
260 : return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
261 109139 : buf);
262 : }
263 :
264 900 : Node* WasmGraphBuilder::RefNull() {
265 540 : return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
266 : }
267 :
268 3285 : Node* WasmGraphBuilder::NoContextConstant() {
269 : // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
270 3285 : return mcgraph()->IntPtrConstant(0);
271 : }
272 :
273 1538 : Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
274 0 : return mcgraph()->Uint32Constant(value);
275 : }
276 :
277 3090024 : Node* WasmGraphBuilder::Int32Constant(int32_t value) {
278 3090374 : return mcgraph()->Int32Constant(value);
279 : }
280 :
281 40391 : Node* WasmGraphBuilder::Int64Constant(int64_t value) {
282 40391 : return mcgraph()->Int64Constant(value);
283 : }
284 :
285 38706 : Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
286 38706 : return mcgraph()->IntPtrConstant(value);
287 : }
288 :
289 28330 : void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
290 195189 : Node** effect, Node** control) {
291 : DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks.
292 28330 : if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
293 6220 : return;
294 : }
295 22110 : if (effect == nullptr) effect = effect_;
296 22110 : if (control == nullptr) control = control_;
297 :
298 : // This instruction sequence is matched in the instruction selector to
299 : // load the stack pointer directly on some platforms. Hence, when modifying
300 : // please also fix WasmStackCheckMatcher in node-matchers.h
301 :
302 : Node* limit_address = graph()->NewNode(
303 : mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
304 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
305 66347 : *effect, *control);
306 : Node* limit = graph()->NewNode(
307 : mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
308 66398 : mcgraph()->IntPtrConstant(0), limit_address, *control);
309 22135 : *effect = limit;
310 22135 : Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
311 :
312 : Node* check =
313 22122 : graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
314 :
315 44234 : Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
316 22119 : stack_check.Chain(*control);
317 :
318 22125 : if (stack_check_call_operator_ == nullptr) {
319 : // Build and cache the stack check call operator and the constant
320 : // representing the stack check code.
321 : auto call_descriptor = Linkage::GetStubCallDescriptor(
322 : mcgraph()->zone(), // zone
323 : NoContextDescriptor{}, // descriptor
324 : 0, // stack parameter count
325 : CallDescriptor::kNoFlags, // flags
326 : Operator::kNoProperties, // properties
327 26871 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
328 : // A direct call to a wasm runtime stub defined in this module.
329 : // Just encode the stub index. This will be patched at relocation.
330 : stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
331 13429 : wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
332 13444 : stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
333 : }
334 :
335 : Node* call = graph()->NewNode(stack_check_call_operator_.get(),
336 : stack_check_code_node_.get(), *effect,
337 22129 : stack_check.if_false);
338 :
339 22132 : SetSourcePosition(call, position);
340 :
341 22134 : Node* ephi = stack_check.EffectPhi(*effect, call);
342 :
343 22133 : *control = stack_check.merge;
344 22133 : *effect = ephi;
345 : }
346 :
347 1060550 : void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
348 2109280 : if (!needs_stack_check_) return;
349 :
350 17664 : Node* start = graph()->start();
351 : // Place a stack check which uses a dummy node as control and effect.
352 17664 : Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
353 17673 : Node* control = dummy;
354 17673 : Node* effect = dummy;
355 : // The function-prologue stack check is associated with position 0, which
356 : // is never a position of any instruction in the function.
357 17673 : StackCheck(0, &effect, &control);
358 :
359 : // In testing, no steck checks were emitted. Nothing to rewire then.
360 17685 : if (effect == dummy) return;
361 :
362 : // Now patch all control uses of {start} to use {control} and all effect uses
363 : // to use {effect} instead. Then rewire the dummy node to use start instead.
364 11841 : NodeProperties::ReplaceUses(start, start, effect, control);
365 11840 : NodeProperties::ReplaceUses(dummy, nullptr, start, start);
366 : }
367 :
368 1464901 : Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
369 1464901 : wasm::WasmCodePosition position) {
370 : const Operator* op;
371 1464901 : MachineOperatorBuilder* m = mcgraph()->machine();
372 1464901 : switch (opcode) {
373 : case wasm::kExprI32Add:
374 142480 : op = m->Int32Add();
375 142474 : break;
376 : case wasm::kExprI32Sub:
377 51123 : op = m->Int32Sub();
378 51123 : break;
379 : case wasm::kExprI32Mul:
380 39770 : op = m->Int32Mul();
381 39770 : break;
382 : case wasm::kExprI32DivS:
383 35303 : return BuildI32DivS(left, right, position);
384 : case wasm::kExprI32DivU:
385 35173 : return BuildI32DivU(left, right, position);
386 : case wasm::kExprI32RemS:
387 35157 : return BuildI32RemS(left, right, position);
388 : case wasm::kExprI32RemU:
389 35128 : return BuildI32RemU(left, right, position);
390 : case wasm::kExprI32And:
391 148494 : op = m->Word32And();
392 148494 : break;
393 : case wasm::kExprI32Ior:
394 40267 : op = m->Word32Or();
395 40267 : break;
396 : case wasm::kExprI32Xor:
397 34723 : op = m->Word32Xor();
398 34723 : break;
399 : case wasm::kExprI32Shl:
400 48881 : op = m->Word32Shl();
401 48881 : right = MaskShiftCount32(right);
402 48881 : break;
403 : case wasm::kExprI32ShrU:
404 53394 : op = m->Word32Shr();
405 53394 : right = MaskShiftCount32(right);
406 53394 : break;
407 : case wasm::kExprI32ShrS:
408 36896 : op = m->Word32Sar();
409 36896 : right = MaskShiftCount32(right);
410 36896 : break;
411 : case wasm::kExprI32Ror:
412 67328 : op = m->Word32Ror();
413 67328 : right = MaskShiftCount32(right);
414 67328 : break;
415 : case wasm::kExprI32Rol:
416 33664 : right = MaskShiftCount32(right);
417 33664 : return BuildI32Rol(left, right);
418 : case wasm::kExprI32Eq:
419 149838 : op = m->Word32Equal();
420 149838 : break;
421 : case wasm::kExprI32Ne:
422 74913 : return Invert(Binop(wasm::kExprI32Eq, left, right));
423 : case wasm::kExprI32LtS:
424 35977 : op = m->Int32LessThan();
425 35977 : break;
426 : case wasm::kExprI32LeS:
427 34679 : op = m->Int32LessThanOrEqual();
428 34679 : break;
429 : case wasm::kExprI32LtU:
430 38823 : op = m->Uint32LessThan();
431 38823 : break;
432 : case wasm::kExprI32LeU:
433 33847 : op = m->Uint32LessThanOrEqual();
434 33847 : break;
435 : case wasm::kExprI32GtS:
436 36251 : op = m->Int32LessThan();
437 : std::swap(left, right);
438 : break;
439 : case wasm::kExprI32GeS:
440 34341 : op = m->Int32LessThanOrEqual();
441 : std::swap(left, right);
442 : break;
443 : case wasm::kExprI32GtU:
444 35711 : op = m->Uint32LessThan();
445 : std::swap(left, right);
446 : break;
447 : case wasm::kExprI32GeU:
448 33835 : op = m->Uint32LessThanOrEqual();
449 : std::swap(left, right);
450 : break;
451 : case wasm::kExprI64And:
452 233 : op = m->Word64And();
453 233 : break;
454 : case wasm::kExprI64Add:
455 1564 : op = m->Int64Add();
456 1564 : break;
457 : case wasm::kExprI64Sub:
458 1758 : op = m->Int64Sub();
459 1758 : break;
460 : case wasm::kExprI64Mul:
461 1718 : op = m->Int64Mul();
462 1718 : break;
463 : case wasm::kExprI64DivS:
464 1663 : return BuildI64DivS(left, right, position);
465 : case wasm::kExprI64DivU:
466 1535 : return BuildI64DivU(left, right, position);
467 : case wasm::kExprI64RemS:
468 1517 : return BuildI64RemS(left, right, position);
469 : case wasm::kExprI64RemU:
470 1517 : return BuildI64RemU(left, right, position);
471 : case wasm::kExprI64Ior:
472 12932 : op = m->Word64Or();
473 12931 : break;
474 : case wasm::kExprI64Xor:
475 113 : op = m->Word64Xor();
476 113 : break;
477 : case wasm::kExprI64Shl:
478 14437 : op = m->Word64Shl();
479 14437 : right = MaskShiftCount64(right);
480 14440 : break;
481 : case wasm::kExprI64ShrU:
482 1584 : op = m->Word64Shr();
483 1584 : right = MaskShiftCount64(right);
484 1584 : break;
485 : case wasm::kExprI64ShrS:
486 1521 : op = m->Word64Sar();
487 1521 : right = MaskShiftCount64(right);
488 1521 : break;
489 : case wasm::kExprI64Eq:
490 308 : op = m->Word64Equal();
491 308 : break;
492 : case wasm::kExprI64Ne:
493 93 : return Invert(Binop(wasm::kExprI64Eq, left, right));
494 : case wasm::kExprI64LtS:
495 80 : op = m->Int64LessThan();
496 80 : break;
497 : case wasm::kExprI64LeS:
498 53 : op = m->Int64LessThanOrEqual();
499 53 : break;
500 : case wasm::kExprI64LtU:
501 62 : op = m->Uint64LessThan();
502 62 : break;
503 : case wasm::kExprI64LeU:
504 71 : op = m->Uint64LessThanOrEqual();
505 71 : break;
506 : case wasm::kExprI64GtS:
507 62 : op = m->Int64LessThan();
508 : std::swap(left, right);
509 : break;
510 : case wasm::kExprI64GeS:
511 53 : op = m->Int64LessThanOrEqual();
512 : std::swap(left, right);
513 : break;
514 : case wasm::kExprI64GtU:
515 62 : op = m->Uint64LessThan();
516 : std::swap(left, right);
517 : break;
518 : case wasm::kExprI64GeU:
519 53 : op = m->Uint64LessThanOrEqual();
520 : std::swap(left, right);
521 : break;
522 : case wasm::kExprI64Ror:
523 196 : op = m->Word64Ror();
524 196 : right = MaskShiftCount64(right);
525 196 : break;
526 : case wasm::kExprI64Rol:
527 98 : return BuildI64Rol(left, right);
528 : case wasm::kExprF32CopySign:
529 47 : return BuildF32CopySign(left, right);
530 : case wasm::kExprF64CopySign:
531 47 : return BuildF64CopySign(left, right);
532 : case wasm::kExprF32Add:
533 560 : op = m->Float32Add();
534 560 : break;
535 : case wasm::kExprF32Sub:
536 366 : op = m->Float32Sub();
537 366 : break;
538 : case wasm::kExprF32Mul:
539 490 : op = m->Float32Mul();
540 490 : break;
541 : case wasm::kExprF32Div:
542 410 : op = m->Float32Div();
543 410 : break;
544 : case wasm::kExprF32Eq:
545 1394 : op = m->Float32Equal();
546 1394 : break;
547 : case wasm::kExprF32Ne:
548 1247 : return Invert(Binop(wasm::kExprF32Eq, left, right));
549 : case wasm::kExprF32Lt:
550 448 : op = m->Float32LessThan();
551 448 : break;
552 : case wasm::kExprF32Ge:
553 130 : op = m->Float32LessThanOrEqual();
554 : std::swap(left, right);
555 : break;
556 : case wasm::kExprF32Gt:
557 397 : op = m->Float32LessThan();
558 : std::swap(left, right);
559 : break;
560 : case wasm::kExprF32Le:
561 148 : op = m->Float32LessThanOrEqual();
562 148 : break;
563 : case wasm::kExprF64Add:
564 4973 : op = m->Float64Add();
565 4973 : break;
566 : case wasm::kExprF64Sub:
567 3466 : op = m->Float64Sub();
568 3466 : break;
569 : case wasm::kExprF64Mul:
570 7713 : op = m->Float64Mul();
571 7712 : break;
572 : case wasm::kExprF64Div:
573 747 : op = m->Float64Div();
574 747 : break;
575 : case wasm::kExprF64Eq:
576 1008 : op = m->Float64Equal();
577 1008 : break;
578 : case wasm::kExprF64Ne:
579 621 : return Invert(Binop(wasm::kExprF64Eq, left, right));
580 : case wasm::kExprF64Lt:
581 909 : op = m->Float64LessThan();
582 909 : break;
583 : case wasm::kExprF64Le:
584 357 : op = m->Float64LessThanOrEqual();
585 357 : break;
586 : case wasm::kExprF64Gt:
587 835 : op = m->Float64LessThan();
588 : std::swap(left, right);
589 : break;
590 : case wasm::kExprF64Ge:
591 365 : op = m->Float64LessThanOrEqual();
592 : std::swap(left, right);
593 : break;
594 : case wasm::kExprF32Min:
595 98 : op = m->Float32Min();
596 98 : break;
597 : case wasm::kExprF64Min:
598 104 : op = m->Float64Min();
599 104 : break;
600 : case wasm::kExprF32Max:
601 93 : op = m->Float32Max();
602 93 : break;
603 : case wasm::kExprF64Max:
604 100 : op = m->Float64Max();
605 100 : break;
606 : case wasm::kExprF64Pow:
607 11 : return BuildF64Pow(left, right);
608 : case wasm::kExprF64Atan2:
609 20 : op = m->Float64Atan2();
610 20 : break;
611 : case wasm::kExprF64Mod:
612 25 : return BuildF64Mod(left, right);
613 : case wasm::kExprI32AsmjsDivS:
614 512 : return BuildI32AsmjsDivS(left, right);
615 : case wasm::kExprI32AsmjsDivU:
616 169 : return BuildI32AsmjsDivU(left, right);
617 : case wasm::kExprI32AsmjsRemS:
618 566 : return BuildI32AsmjsRemS(left, right);
619 : case wasm::kExprI32AsmjsRemU:
620 228 : return BuildI32AsmjsRemU(left, right);
621 : case wasm::kExprI32AsmjsStoreMem8:
622 3555 : return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
623 : case wasm::kExprI32AsmjsStoreMem16:
624 1638 : return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
625 : case wasm::kExprI32AsmjsStoreMem:
626 37038 : return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
627 : case wasm::kExprF32AsmjsStoreMem:
628 3960 : return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
629 : case wasm::kExprF64AsmjsStoreMem:
630 827 : return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
631 : default:
632 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
633 : }
634 1158627 : return graph()->NewNode(op, left, right);
635 : }
636 :
637 204372 : Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
638 293508 : wasm::WasmCodePosition position) {
639 : const Operator* op;
640 204372 : MachineOperatorBuilder* m = mcgraph()->machine();
641 204372 : switch (opcode) {
642 : case wasm::kExprI32Eqz:
643 88948 : op = m->Word32Equal();
644 177899 : return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
645 : case wasm::kExprF32Abs:
646 94 : op = m->Float32Abs();
647 94 : break;
648 : case wasm::kExprF32Neg: {
649 265 : op = m->Float32Neg();
650 265 : break;
651 : }
652 : case wasm::kExprF32Sqrt:
653 209 : op = m->Float32Sqrt();
654 209 : break;
655 : case wasm::kExprF64Abs:
656 124 : op = m->Float64Abs();
657 124 : break;
658 : case wasm::kExprF64Neg: {
659 1259 : op = m->Float64Neg();
660 1259 : break;
661 : }
662 : case wasm::kExprF64Sqrt:
663 308 : op = m->Float64Sqrt();
664 308 : break;
665 : case wasm::kExprI32SConvertF32:
666 : case wasm::kExprI32UConvertF32:
667 : case wasm::kExprI32SConvertF64:
668 : case wasm::kExprI32UConvertF64:
669 : case wasm::kExprI32SConvertSatF64:
670 : case wasm::kExprI32UConvertSatF64:
671 : case wasm::kExprI32SConvertSatF32:
672 : case wasm::kExprI32UConvertSatF32:
673 631 : return BuildIntConvertFloat(input, position, opcode);
674 : case wasm::kExprI32AsmjsSConvertF64:
675 124 : return BuildI32AsmjsSConvertF64(input);
676 : case wasm::kExprI32AsmjsUConvertF64:
677 15 : return BuildI32AsmjsUConvertF64(input);
678 : case wasm::kExprF32ConvertF64:
679 4121 : op = m->TruncateFloat64ToFloat32();
680 4121 : break;
681 : case wasm::kExprF64SConvertI32:
682 783 : op = m->ChangeInt32ToFloat64();
683 783 : break;
684 : case wasm::kExprF64UConvertI32:
685 342 : op = m->ChangeUint32ToFloat64();
686 342 : break;
687 : case wasm::kExprF32SConvertI32:
688 325 : op = m->RoundInt32ToFloat32();
689 325 : break;
690 : case wasm::kExprF32UConvertI32:
691 121 : op = m->RoundUint32ToFloat32();
692 121 : break;
693 : case wasm::kExprI32AsmjsSConvertF32:
694 15 : return BuildI32AsmjsSConvertF32(input);
695 : case wasm::kExprI32AsmjsUConvertF32:
696 15 : return BuildI32AsmjsUConvertF32(input);
697 : case wasm::kExprF64ConvertF32:
698 6646 : op = m->ChangeFloat32ToFloat64();
699 6646 : break;
700 : case wasm::kExprF32ReinterpretI32:
701 242 : op = m->BitcastInt32ToFloat32();
702 242 : break;
703 : case wasm::kExprI32ReinterpretF32:
704 556 : op = m->BitcastFloat32ToInt32();
705 556 : break;
706 : case wasm::kExprI32Clz:
707 702 : op = m->Word32Clz();
708 702 : break;
709 : case wasm::kExprI32Ctz: {
710 746 : if (m->Word32Ctz().IsSupported()) {
711 746 : op = m->Word32Ctz().op();
712 746 : break;
713 0 : } else if (m->Word32ReverseBits().IsSupported()) {
714 0 : Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
715 0 : Node* result = graph()->NewNode(m->Word32Clz(), reversed);
716 0 : return result;
717 : } else {
718 0 : return BuildI32Ctz(input);
719 : }
720 : }
721 : case wasm::kExprI32Popcnt: {
722 133 : if (m->Word32Popcnt().IsSupported()) {
723 133 : op = m->Word32Popcnt().op();
724 133 : break;
725 : } else {
726 0 : return BuildI32Popcnt(input);
727 : }
728 : }
729 : case wasm::kExprF32Floor: {
730 64 : if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
731 64 : op = m->Float32RoundDown().op();
732 64 : break;
733 : }
734 : case wasm::kExprF32Ceil: {
735 64 : if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
736 64 : op = m->Float32RoundUp().op();
737 64 : break;
738 : }
739 : case wasm::kExprF32Trunc: {
740 353 : if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
741 353 : op = m->Float32RoundTruncate().op();
742 353 : break;
743 : }
744 : case wasm::kExprF32NearestInt: {
745 33 : if (!m->Float32RoundTiesEven().IsSupported())
746 0 : return BuildF32NearestInt(input);
747 33 : op = m->Float32RoundTiesEven().op();
748 33 : break;
749 : }
750 : case wasm::kExprF64Floor: {
751 117 : if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
752 117 : op = m->Float64RoundDown().op();
753 117 : break;
754 : }
755 : case wasm::kExprF64Ceil: {
756 82 : if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
757 82 : op = m->Float64RoundUp().op();
758 82 : break;
759 : }
760 : case wasm::kExprF64Trunc: {
761 344 : if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
762 344 : op = m->Float64RoundTruncate().op();
763 344 : break;
764 : }
765 : case wasm::kExprF64NearestInt: {
766 33 : if (!m->Float64RoundTiesEven().IsSupported())
767 0 : return BuildF64NearestInt(input);
768 33 : op = m->Float64RoundTiesEven().op();
769 33 : break;
770 : }
771 : case wasm::kExprF64Acos: {
772 25 : return BuildF64Acos(input);
773 : }
774 : case wasm::kExprF64Asin: {
775 25 : return BuildF64Asin(input);
776 : }
777 : case wasm::kExprF64Atan:
778 25 : op = m->Float64Atan();
779 25 : break;
780 : case wasm::kExprF64Cos: {
781 175 : op = m->Float64Cos();
782 175 : break;
783 : }
784 : case wasm::kExprF64Sin: {
785 175 : op = m->Float64Sin();
786 175 : break;
787 : }
788 : case wasm::kExprF64Tan: {
789 25 : op = m->Float64Tan();
790 25 : break;
791 : }
792 : case wasm::kExprF64Exp: {
793 25 : op = m->Float64Exp();
794 25 : break;
795 : }
796 : case wasm::kExprF64Log:
797 25 : op = m->Float64Log();
798 25 : break;
799 : case wasm::kExprI32ConvertI64:
800 1513 : op = m->TruncateInt64ToInt32();
801 1513 : break;
802 : case wasm::kExprI64SConvertI32:
803 61 : op = m->ChangeInt32ToInt64();
804 61 : break;
805 : case wasm::kExprI64UConvertI32:
806 25703 : op = m->ChangeUint32ToUint64();
807 25703 : break;
808 : case wasm::kExprF64ReinterpretI64:
809 197 : op = m->BitcastInt64ToFloat64();
810 197 : break;
811 : case wasm::kExprI64ReinterpretF64:
812 502 : op = m->BitcastFloat64ToInt64();
813 502 : break;
814 : case wasm::kExprI64Clz:
815 47 : op = m->Word64Clz();
816 47 : break;
817 : case wasm::kExprI64Ctz: {
818 218 : OptionalOperator ctz64 = m->Word64Ctz();
819 218 : if (ctz64.IsSupported()) {
820 : op = ctz64.op();
821 218 : break;
822 0 : } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
823 : op = ctz64.placeholder();
824 : break;
825 0 : } else if (m->Word64ReverseBits().IsSupported()) {
826 0 : Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
827 0 : Node* result = graph()->NewNode(m->Word64Clz(), reversed);
828 0 : return result;
829 : } else {
830 0 : return BuildI64Ctz(input);
831 : }
832 : }
833 : case wasm::kExprI64Popcnt: {
834 62 : OptionalOperator popcnt64 = m->Word64Popcnt();
835 62 : if (popcnt64.IsSupported()) {
836 : op = popcnt64.op();
837 0 : } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
838 : op = popcnt64.placeholder();
839 : } else {
840 0 : return BuildI64Popcnt(input);
841 : }
842 62 : break;
843 : }
844 : case wasm::kExprI64Eqz:
845 188 : op = m->Word64Equal();
846 376 : return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
847 : case wasm::kExprF32SConvertI64:
848 78 : if (m->Is32()) {
849 0 : return BuildF32SConvertI64(input);
850 : }
851 78 : op = m->RoundInt64ToFloat32();
852 78 : break;
853 : case wasm::kExprF32UConvertI64:
854 42 : if (m->Is32()) {
855 0 : return BuildF32UConvertI64(input);
856 : }
857 42 : op = m->RoundUint64ToFloat32();
858 42 : break;
859 : case wasm::kExprF64SConvertI64:
860 108 : if (m->Is32()) {
861 0 : return BuildF64SConvertI64(input);
862 : }
863 108 : op = m->RoundInt64ToFloat64();
864 108 : break;
865 : case wasm::kExprF64UConvertI64:
866 158 : if (m->Is32()) {
867 0 : return BuildF64UConvertI64(input);
868 : }
869 158 : op = m->RoundUint64ToFloat64();
870 158 : break;
871 : case wasm::kExprI32SExtendI8:
872 10 : op = m->SignExtendWord8ToInt32();
873 10 : break;
874 : case wasm::kExprI32SExtendI16:
875 10 : op = m->SignExtendWord16ToInt32();
876 10 : break;
877 : case wasm::kExprI64SExtendI8:
878 10 : op = m->SignExtendWord8ToInt64();
879 10 : break;
880 : case wasm::kExprI64SExtendI16:
881 10 : op = m->SignExtendWord16ToInt64();
882 10 : break;
883 : case wasm::kExprI64SExtendI32:
884 10 : op = m->SignExtendWord32ToInt64();
885 10 : break;
886 : case wasm::kExprI64SConvertF32:
887 : case wasm::kExprI64UConvertF32:
888 : case wasm::kExprI64SConvertF64:
889 : case wasm::kExprI64UConvertF64:
890 : case wasm::kExprI64SConvertSatF32:
891 : case wasm::kExprI64UConvertSatF32:
892 : case wasm::kExprI64SConvertSatF64:
893 : case wasm::kExprI64UConvertSatF64:
894 : return mcgraph()->machine()->Is32()
895 : ? BuildCcallConvertFloat(input, position, opcode)
896 954 : : BuildIntConvertFloat(input, position, opcode);
897 : case wasm::kExprRefIsNull:
898 72 : return graph()->NewNode(m->WordEqual(), input, RefNull());
899 : case wasm::kExprI32AsmjsLoadMem8S:
900 3284 : return BuildAsmjsLoadMem(MachineType::Int8(), input);
901 : case wasm::kExprI32AsmjsLoadMem8U:
902 2459 : return BuildAsmjsLoadMem(MachineType::Uint8(), input);
903 : case wasm::kExprI32AsmjsLoadMem16S:
904 1114 : return BuildAsmjsLoadMem(MachineType::Int16(), input);
905 : case wasm::kExprI32AsmjsLoadMem16U:
906 624 : return BuildAsmjsLoadMem(MachineType::Uint16(), input);
907 : case wasm::kExprI32AsmjsLoadMem:
908 51261 : return BuildAsmjsLoadMem(MachineType::Int32(), input);
909 : case wasm::kExprF32AsmjsLoadMem:
910 6482 : return BuildAsmjsLoadMem(MachineType::Float32(), input);
911 : case wasm::kExprF64AsmjsLoadMem:
912 917 : return BuildAsmjsLoadMem(MachineType::Float64(), input);
913 : default:
914 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
915 : }
916 47248 : return graph()->NewNode(op, input);
917 : }
918 :
919 4625 : Node* WasmGraphBuilder::Float32Constant(float value) {
920 4625 : return mcgraph()->Float32Constant(value);
921 : }
922 :
923 13428 : Node* WasmGraphBuilder::Float64Constant(double value) {
924 13428 : return mcgraph()->Float64Constant(value);
925 : }
926 :
927 : namespace {
928 891475 : Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
929 : Node** false_node, Node* control, BranchHint hint) {
930 : DCHECK_NOT_NULL(cond);
931 : DCHECK_NOT_NULL(control);
932 : Node* branch =
933 148573 : mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
934 297167 : *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
935 297166 : *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
936 148583 : return branch;
937 : }
938 : } // namespace
939 :
940 111607 : Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
941 223214 : Node** false_node) {
942 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
943 111607 : BranchHint::kNone);
944 : }
945 :
946 0 : Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
947 0 : Node** false_node) {
948 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
949 0 : BranchHint::kTrue);
950 : }
951 :
952 0 : Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
953 73932 : Node** false_node) {
954 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
955 36966 : BranchHint::kFalse);
956 : }
957 :
958 47854 : TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
959 : // TODO(wasm): "!env_" should not happen when compiling an actual wasm
960 : // function.
961 47854 : if (!env_ || !env_->runtime_exception_support) {
962 : // We use TrapId::kInvalid as a marker to tell the code generator
963 : // to generate a call to a testing c-function instead of a runtime
964 : // stub. This code should only be called from a cctest.
965 : return TrapId::kInvalid;
966 : }
967 :
968 24977 : switch (reason) {
969 : #define TRAPREASON_TO_TRAPID(name) \
970 : case wasm::k##name: \
971 : static_assert( \
972 : static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
973 : "trap id mismatch"); \
974 : return TrapId::k##name;
975 10659 : FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
976 : #undef TRAPREASON_TO_TRAPID
977 : default:
978 0 : UNREACHABLE();
979 : }
980 : }
981 :
982 11368 : Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
983 45492 : wasm::WasmCodePosition position) {
984 11368 : TrapId trap_id = GetTrapIdForTrap(reason);
985 : Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
986 11370 : cond, Effect(), Control()));
987 11382 : SetSourcePosition(node, position);
988 11386 : return node;
989 : }
990 :
991 36488 : Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
992 146042 : wasm::WasmCodePosition position) {
993 36488 : TrapId trap_id = GetTrapIdForTrap(reason);
994 : Node* node = SetControl(graph()->NewNode(
995 36503 : mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
996 36533 : SetSourcePosition(node, position);
997 36541 : return node;
998 : }
999 :
1000 : // Add a check that traps if {node} is equal to {val}.
1001 176082 : Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
1002 : int32_t val,
1003 4486 : wasm::WasmCodePosition position) {
1004 : Int32Matcher m(node);
1005 510150 : if (m.HasValue() && !m.Is(val)) return graph()->start();
1006 10521 : if (val == 0) {
1007 8278 : return TrapIfFalse(reason, node, position);
1008 : } else {
1009 : return TrapIfTrue(reason,
1010 : graph()->NewNode(mcgraph()->machine()->Word32Equal(),
1011 : node, mcgraph()->Int32Constant(val)),
1012 6729 : position);
1013 : }
1014 : }
1015 :
1016 : // Add a check that traps if {node} is zero.
1017 0 : Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
1018 : wasm::WasmCodePosition position) {
1019 140761 : return TrapIfEq32(reason, node, 0, position);
1020 : }
1021 :
1022 : // Add a check that traps if {node} is equal to {val}.
1023 8786 : Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
1024 : int64_t val,
1025 16938 : wasm::WasmCodePosition position) {
1026 : Int64Matcher m(node);
1027 9454 : if (m.HasValue() && !m.Is(val)) return graph()->start();
1028 : return TrapIfTrue(reason,
1029 : graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
1030 : mcgraph()->Int64Constant(val)),
1031 25415 : position);
1032 : }
1033 :
1034 : // Add a check that traps if {node} is zero.
1035 0 : Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
1036 : wasm::WasmCodePosition position) {
1037 7124 : return TrapIfEq64(reason, node, 0, position);
1038 : }
1039 :
1040 22434 : Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
1041 33651 : return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
1042 : }
1043 :
1044 254235 : Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1045 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1046 508471 : return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1047 : }
1048 :
1049 11216 : Node* WasmGraphBuilder::IfDefault(Node* sw) {
1050 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1051 22433 : return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1052 : }
1053 :
1054 7820661 : Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1055 : static const int kStackAllocatedNodeBufferSize = 8;
1056 : Node* stack_buffer[kStackAllocatedNodeBufferSize];
1057 : std::vector<Node*> heap_buffer;
1058 :
1059 : Node** buf = stack_buffer;
1060 1303299 : if (count + 3 > kStackAllocatedNodeBufferSize) {
1061 0 : heap_buffer.resize(count + 3);
1062 : buf = heap_buffer.data();
1063 : }
1064 :
1065 1303299 : buf[0] = mcgraph()->Int32Constant(0);
1066 1303508 : memcpy(buf + 1, vals, sizeof(void*) * count);
1067 2607016 : buf[count + 1] = Effect();
1068 2607016 : buf[count + 2] = Control();
1069 : Node* ret =
1070 3910276 : graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1071 :
1072 1303539 : MergeControlToEnd(mcgraph(), ret);
1073 1303381 : return ret;
1074 : }
1075 :
1076 1866 : Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1077 :
1078 1794 : Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1079 1794 : TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1080 : ReturnVoid();
1081 1794 : return nullptr;
1082 : }
1083 :
1084 240193 : Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1085 : static const int32_t kMask32 = 0x1F;
1086 480326 : if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1087 : // Shifts by constants are so common we pattern-match them here.
1088 : Int32Matcher match(node);
1089 30 : if (match.HasValue()) {
1090 0 : int32_t masked = (match.Value() & kMask32);
1091 0 : if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1092 : } else {
1093 : node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1094 60 : mcgraph()->Int32Constant(kMask32));
1095 : }
1096 : }
1097 240163 : return node;
1098 : }
1099 :
1100 17763 : Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1101 : static const int64_t kMask64 = 0x3F;
1102 35476 : if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1103 : // Shifts by constants are so common we pattern-match them here.
1104 : Int64Matcher match(node);
1105 25 : if (match.HasValue()) {
1106 0 : int64_t masked = (match.Value() & kMask64);
1107 0 : if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1108 : } else {
1109 : node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1110 50 : mcgraph()->Int64Constant(kMask64));
1111 : }
1112 : }
1113 17738 : return node;
1114 : }
1115 :
1116 : static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1117 : size_t size_in_bytes) {
1118 0 : switch (size_in_bytes) {
1119 : case 4:
1120 : case 16:
1121 : return true;
1122 : case 8:
1123 : return m->Is64();
1124 : default:
1125 : break;
1126 : }
1127 : return false;
1128 : }
1129 :
1130 0 : Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1131 0 : Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1132 : Node* result;
1133 : Node* value = node;
1134 0 : MachineOperatorBuilder* m = mcgraph()->machine();
1135 0 : int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1136 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1137 : bool isFloat = false;
1138 :
1139 0 : switch (wasmtype) {
1140 : case wasm::kWasmF64:
1141 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1142 : isFloat = true;
1143 : V8_FALLTHROUGH;
1144 : case wasm::kWasmI64:
1145 0 : result = mcgraph()->Int64Constant(0);
1146 0 : break;
1147 : case wasm::kWasmF32:
1148 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1149 : isFloat = true;
1150 : V8_FALLTHROUGH;
1151 : case wasm::kWasmI32:
1152 0 : result = mcgraph()->Int32Constant(0);
1153 0 : break;
1154 : case wasm::kWasmS128:
1155 : DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1156 : break;
1157 : default:
1158 0 : UNREACHABLE();
1159 : break;
1160 : }
1161 :
1162 0 : if (mem_rep == MachineRepresentation::kWord8) {
1163 : // No need to change endianness for byte size, return original node
1164 : return node;
1165 : }
1166 0 : if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1167 : // In case we store lower part of WasmI64 expression, we can truncate
1168 : // upper 32bits
1169 0 : value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1170 0 : valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1171 0 : valueSizeInBits = 8 * valueSizeInBytes;
1172 0 : if (mem_rep == MachineRepresentation::kWord16) {
1173 : value =
1174 0 : graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1175 : }
1176 0 : } else if (wasmtype == wasm::kWasmI32 &&
1177 0 : mem_rep == MachineRepresentation::kWord16) {
1178 : value =
1179 0 : graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1180 : }
1181 :
1182 : int i;
1183 : uint32_t shiftCount;
1184 :
1185 0 : if (ReverseBytesSupported(m, valueSizeInBytes)) {
1186 0 : switch (valueSizeInBytes) {
1187 : case 4:
1188 0 : result = graph()->NewNode(m->Word32ReverseBytes(), value);
1189 0 : break;
1190 : case 8:
1191 0 : result = graph()->NewNode(m->Word64ReverseBytes(), value);
1192 0 : break;
1193 : case 16: {
1194 : Node* byte_reversed_lanes[4];
1195 0 : for (int lane = 0; lane < 4; lane++) {
1196 : byte_reversed_lanes[lane] = graph()->NewNode(
1197 : m->Word32ReverseBytes(),
1198 : graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1199 0 : value));
1200 : }
1201 :
1202 : // This is making a copy of the value.
1203 : result =
1204 0 : graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1205 :
1206 0 : for (int lane = 0; lane < 4; lane++) {
1207 : result =
1208 : graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1209 0 : result, byte_reversed_lanes[lane]);
1210 : }
1211 :
1212 : break;
1213 : }
1214 : default:
1215 0 : UNREACHABLE();
1216 : break;
1217 : }
1218 : } else {
1219 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1220 : i += 8, shiftCount -= 16) {
1221 : Node* shiftLower;
1222 : Node* shiftHigher;
1223 : Node* lowerByte;
1224 : Node* higherByte;
1225 :
1226 : DCHECK_LT(0, shiftCount);
1227 : DCHECK_EQ(0, (shiftCount + 8) % 16);
1228 :
1229 0 : if (valueSizeInBits > 32) {
1230 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1231 0 : mcgraph()->Int64Constant(shiftCount));
1232 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1233 0 : mcgraph()->Int64Constant(shiftCount));
1234 : lowerByte = graph()->NewNode(
1235 : m->Word64And(), shiftLower,
1236 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1237 0 : << (valueSizeInBits - 8 - i)));
1238 : higherByte = graph()->NewNode(
1239 : m->Word64And(), shiftHigher,
1240 0 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1241 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1242 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1243 : } else {
1244 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1245 0 : mcgraph()->Int32Constant(shiftCount));
1246 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1247 0 : mcgraph()->Int32Constant(shiftCount));
1248 : lowerByte = graph()->NewNode(
1249 : m->Word32And(), shiftLower,
1250 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1251 0 : << (valueSizeInBits - 8 - i)));
1252 : higherByte = graph()->NewNode(
1253 : m->Word32And(), shiftHigher,
1254 0 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1255 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1256 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1257 : }
1258 : }
1259 : }
1260 :
1261 0 : if (isFloat) {
1262 0 : switch (wasmtype) {
1263 : case wasm::kWasmF64:
1264 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1265 0 : break;
1266 : case wasm::kWasmF32:
1267 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1268 0 : break;
1269 : default:
1270 0 : UNREACHABLE();
1271 : break;
1272 : }
1273 : }
1274 :
1275 0 : return result;
1276 : }
1277 :
1278 0 : Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1279 : MachineType memtype,
1280 0 : wasm::ValueType wasmtype) {
1281 : Node* result;
1282 : Node* value = node;
1283 0 : MachineOperatorBuilder* m = mcgraph()->machine();
1284 : int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1285 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1286 : bool isFloat = false;
1287 :
1288 0 : switch (memtype.representation()) {
1289 : case MachineRepresentation::kFloat64:
1290 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1291 : isFloat = true;
1292 : V8_FALLTHROUGH;
1293 : case MachineRepresentation::kWord64:
1294 0 : result = mcgraph()->Int64Constant(0);
1295 0 : break;
1296 : case MachineRepresentation::kFloat32:
1297 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1298 : isFloat = true;
1299 : V8_FALLTHROUGH;
1300 : case MachineRepresentation::kWord32:
1301 : case MachineRepresentation::kWord16:
1302 0 : result = mcgraph()->Int32Constant(0);
1303 0 : break;
1304 : case MachineRepresentation::kWord8:
1305 : // No need to change endianness for byte size, return original node
1306 : return node;
1307 : break;
1308 : case MachineRepresentation::kSimd128:
1309 : DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1310 : break;
1311 : default:
1312 0 : UNREACHABLE();
1313 : break;
1314 : }
1315 :
1316 : int i;
1317 : uint32_t shiftCount;
1318 :
1319 0 : if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1320 0 : switch (valueSizeInBytes) {
1321 : case 2:
1322 : result =
1323 : graph()->NewNode(m->Word32ReverseBytes(),
1324 : graph()->NewNode(m->Word32Shl(), value,
1325 0 : mcgraph()->Int32Constant(16)));
1326 0 : break;
1327 : case 4:
1328 0 : result = graph()->NewNode(m->Word32ReverseBytes(), value);
1329 0 : break;
1330 : case 8:
1331 0 : result = graph()->NewNode(m->Word64ReverseBytes(), value);
1332 0 : break;
1333 : case 16: {
1334 : Node* byte_reversed_lanes[4];
1335 0 : for (int lane = 0; lane < 4; lane++) {
1336 : byte_reversed_lanes[lane] = graph()->NewNode(
1337 : m->Word32ReverseBytes(),
1338 : graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1339 0 : value));
1340 : }
1341 :
1342 : // This is making a copy of the value.
1343 : result =
1344 0 : graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1345 :
1346 0 : for (int lane = 0; lane < 4; lane++) {
1347 : result =
1348 : graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1349 0 : result, byte_reversed_lanes[lane]);
1350 : }
1351 :
1352 : break;
1353 : }
1354 : default:
1355 0 : UNREACHABLE();
1356 : }
1357 : } else {
1358 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1359 : i += 8, shiftCount -= 16) {
1360 : Node* shiftLower;
1361 : Node* shiftHigher;
1362 : Node* lowerByte;
1363 : Node* higherByte;
1364 :
1365 : DCHECK_LT(0, shiftCount);
1366 : DCHECK_EQ(0, (shiftCount + 8) % 16);
1367 :
1368 0 : if (valueSizeInBits > 32) {
1369 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1370 0 : mcgraph()->Int64Constant(shiftCount));
1371 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1372 0 : mcgraph()->Int64Constant(shiftCount));
1373 : lowerByte = graph()->NewNode(
1374 : m->Word64And(), shiftLower,
1375 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1376 0 : << (valueSizeInBits - 8 - i)));
1377 : higherByte = graph()->NewNode(
1378 : m->Word64And(), shiftHigher,
1379 0 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1380 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1381 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1382 : } else {
1383 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1384 0 : mcgraph()->Int32Constant(shiftCount));
1385 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1386 0 : mcgraph()->Int32Constant(shiftCount));
1387 : lowerByte = graph()->NewNode(
1388 : m->Word32And(), shiftLower,
1389 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1390 0 : << (valueSizeInBits - 8 - i)));
1391 : higherByte = graph()->NewNode(
1392 : m->Word32And(), shiftHigher,
1393 0 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1394 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1395 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1396 : }
1397 : }
1398 : }
1399 :
1400 0 : if (isFloat) {
1401 0 : switch (memtype.representation()) {
1402 : case MachineRepresentation::kFloat64:
1403 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1404 0 : break;
1405 : case MachineRepresentation::kFloat32:
1406 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1407 0 : break;
1408 : default:
1409 0 : UNREACHABLE();
1410 : break;
1411 : }
1412 : }
1413 :
1414 : // We need to sign extend the value
1415 0 : if (memtype.IsSigned()) {
1416 : DCHECK(!isFloat);
1417 0 : if (valueSizeInBits < 32) {
1418 : Node* shiftBitCount;
1419 : // Perform sign extension using following trick
1420 : // result = (x << machine_width - type_width) >> (machine_width -
1421 : // type_width)
1422 0 : if (wasmtype == wasm::kWasmI64) {
1423 0 : shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1424 : result = graph()->NewNode(
1425 : m->Word64Sar(),
1426 : graph()->NewNode(m->Word64Shl(),
1427 : graph()->NewNode(m->ChangeInt32ToInt64(), result),
1428 : shiftBitCount),
1429 0 : shiftBitCount);
1430 0 : } else if (wasmtype == wasm::kWasmI32) {
1431 0 : shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1432 : result = graph()->NewNode(
1433 : m->Word32Sar(),
1434 : graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1435 0 : shiftBitCount);
1436 : }
1437 : }
1438 : }
1439 :
1440 0 : return result;
1441 : }
1442 :
1443 94 : Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1444 : Node* result = Unop(
1445 : wasm::kExprF32ReinterpretI32,
1446 : Binop(wasm::kExprI32Ior,
1447 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1448 : mcgraph()->Int32Constant(0x7FFFFFFF)),
1449 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1450 94 : mcgraph()->Int32Constant(0x80000000))));
1451 :
1452 47 : return result;
1453 : }
1454 :
1455 94 : Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1456 : #if WASM_64
1457 : Node* result = Unop(
1458 : wasm::kExprF64ReinterpretI64,
1459 : Binop(wasm::kExprI64Ior,
1460 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1461 : mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1462 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1463 94 : mcgraph()->Int64Constant(0x8000000000000000))));
1464 :
1465 47 : return result;
1466 : #else
1467 : MachineOperatorBuilder* m = mcgraph()->machine();
1468 :
1469 : Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1470 : Node* high_word_right =
1471 : graph()->NewNode(m->Float64ExtractHighWord32(), right);
1472 :
1473 : Node* new_high_word = Binop(wasm::kExprI32Ior,
1474 : Binop(wasm::kExprI32And, high_word_left,
1475 : mcgraph()->Int32Constant(0x7FFFFFFF)),
1476 : Binop(wasm::kExprI32And, high_word_right,
1477 : mcgraph()->Int32Constant(0x80000000)));
1478 :
1479 : return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1480 : #endif
1481 : }
1482 :
1483 : namespace {
1484 :
1485 1587 : MachineType IntConvertType(wasm::WasmOpcode opcode) {
1486 1587 : switch (opcode) {
1487 : case wasm::kExprI32SConvertF32:
1488 : case wasm::kExprI32SConvertF64:
1489 : case wasm::kExprI32SConvertSatF32:
1490 : case wasm::kExprI32SConvertSatF64:
1491 : return MachineType::Int32();
1492 : case wasm::kExprI32UConvertF32:
1493 : case wasm::kExprI32UConvertF64:
1494 : case wasm::kExprI32UConvertSatF32:
1495 : case wasm::kExprI32UConvertSatF64:
1496 : return MachineType::Uint32();
1497 : case wasm::kExprI64SConvertF32:
1498 : case wasm::kExprI64SConvertF64:
1499 : case wasm::kExprI64SConvertSatF32:
1500 : case wasm::kExprI64SConvertSatF64:
1501 : return MachineType::Int64();
1502 : case wasm::kExprI64UConvertF32:
1503 : case wasm::kExprI64UConvertF64:
1504 : case wasm::kExprI64UConvertSatF32:
1505 : case wasm::kExprI64UConvertSatF64:
1506 : return MachineType::Uint64();
1507 : default:
1508 0 : UNREACHABLE();
1509 : }
1510 : }
1511 :
1512 1587 : MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1513 1587 : switch (opcode) {
1514 : case wasm::kExprI32SConvertF32:
1515 : case wasm::kExprI32UConvertF32:
1516 : case wasm::kExprI32SConvertSatF32:
1517 : case wasm::kExprI64SConvertF32:
1518 : case wasm::kExprI64UConvertF32:
1519 : case wasm::kExprI32UConvertSatF32:
1520 : case wasm::kExprI64SConvertSatF32:
1521 : case wasm::kExprI64UConvertSatF32:
1522 : return MachineType::Float32();
1523 : case wasm::kExprI32SConvertF64:
1524 : case wasm::kExprI32UConvertF64:
1525 : case wasm::kExprI64SConvertF64:
1526 : case wasm::kExprI64UConvertF64:
1527 : case wasm::kExprI32SConvertSatF64:
1528 : case wasm::kExprI32UConvertSatF64:
1529 : case wasm::kExprI64SConvertSatF64:
1530 : case wasm::kExprI64UConvertSatF64:
1531 : return MachineType::Float64();
1532 : default:
1533 0 : UNREACHABLE();
1534 : }
1535 : }
1536 :
1537 3174 : const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1538 1587 : switch (opcode) {
1539 : case wasm::kExprI32SConvertF32:
1540 : case wasm::kExprI32SConvertSatF32:
1541 245 : return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1542 : case wasm::kExprI32UConvertF32:
1543 : case wasm::kExprI32UConvertSatF32:
1544 75 : return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1545 : case wasm::kExprI32SConvertF64:
1546 : case wasm::kExprI32SConvertSatF64:
1547 245 : return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1548 : case wasm::kExprI32UConvertF64:
1549 : case wasm::kExprI32UConvertSatF64:
1550 66 : return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1551 : case wasm::kExprI64SConvertF32:
1552 : case wasm::kExprI64SConvertSatF32:
1553 66 : return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1554 : case wasm::kExprI64UConvertF32:
1555 : case wasm::kExprI64UConvertSatF32:
1556 66 : return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1557 : case wasm::kExprI64SConvertF64:
1558 : case wasm::kExprI64SConvertSatF64:
1559 749 : return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1560 : case wasm::kExprI64UConvertF64:
1561 : case wasm::kExprI64UConvertSatF64:
1562 75 : return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1563 : default:
1564 0 : UNREACHABLE();
1565 : }
1566 : }
1567 :
1568 631 : wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1569 631 : switch (opcode) {
1570 : case wasm::kExprI32SConvertF32:
1571 : case wasm::kExprI32SConvertSatF32:
1572 : return wasm::kExprF32SConvertI32;
1573 : case wasm::kExprI32UConvertF32:
1574 : case wasm::kExprI32UConvertSatF32:
1575 75 : return wasm::kExprF32UConvertI32;
1576 : case wasm::kExprI32SConvertF64:
1577 : case wasm::kExprI32SConvertSatF64:
1578 245 : return wasm::kExprF64SConvertI32;
1579 : case wasm::kExprI32UConvertF64:
1580 : case wasm::kExprI32UConvertSatF64:
1581 66 : return wasm::kExprF64UConvertI32;
1582 : default:
1583 0 : UNREACHABLE();
1584 : }
1585 : }
1586 :
1587 1584 : bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1588 1584 : switch (opcode) {
1589 : case wasm::kExprI32SConvertF32:
1590 : case wasm::kExprI32UConvertF32:
1591 : case wasm::kExprI32SConvertF64:
1592 : case wasm::kExprI32UConvertF64:
1593 : case wasm::kExprI64SConvertF32:
1594 : case wasm::kExprI64UConvertF32:
1595 : case wasm::kExprI64SConvertF64:
1596 : case wasm::kExprI64UConvertF64:
1597 : return true;
1598 : case wasm::kExprI32SConvertSatF64:
1599 : case wasm::kExprI32UConvertSatF64:
1600 : case wasm::kExprI32SConvertSatF32:
1601 : case wasm::kExprI32UConvertSatF32:
1602 : case wasm::kExprI64SConvertSatF32:
1603 : case wasm::kExprI64UConvertSatF32:
1604 : case wasm::kExprI64SConvertSatF64:
1605 : case wasm::kExprI64UConvertSatF64:
1606 120 : return false;
1607 : default:
1608 0 : UNREACHABLE();
1609 : }
1610 : }
1611 :
1612 240 : Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1613 240 : switch (ty.representation()) {
1614 : case MachineRepresentation::kWord32:
1615 60 : return builder->Int32Constant(0);
1616 : case MachineRepresentation::kWord64:
1617 60 : return builder->Int64Constant(0);
1618 : case MachineRepresentation::kFloat32:
1619 60 : return builder->Float32Constant(0.0);
1620 : case MachineRepresentation::kFloat64:
1621 60 : return builder->Float64Constant(0.0);
1622 : default:
1623 0 : UNREACHABLE();
1624 : }
1625 : }
1626 :
1627 120 : Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1628 120 : switch (ty.semantic()) {
1629 : case MachineSemantic::kInt32:
1630 30 : return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1631 : case MachineSemantic::kUint32:
1632 30 : return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1633 : case MachineSemantic::kInt64:
1634 30 : return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1635 : case MachineSemantic::kUint64:
1636 30 : return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1637 : default:
1638 0 : UNREACHABLE();
1639 : }
1640 : }
1641 :
1642 120 : Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1643 120 : switch (ty.semantic()) {
1644 : case MachineSemantic::kInt32:
1645 30 : return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1646 : case MachineSemantic::kUint32:
1647 30 : return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1648 : case MachineSemantic::kInt64:
1649 30 : return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1650 : case MachineSemantic::kUint64:
1651 30 : return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1652 : default:
1653 0 : UNREACHABLE();
1654 : }
1655 : }
1656 :
1657 631 : wasm::WasmOpcode TruncOp(const MachineType& ty) {
1658 631 : switch (ty.representation()) {
1659 : case MachineRepresentation::kFloat32:
1660 : return wasm::kExprF32Trunc;
1661 : case MachineRepresentation::kFloat64:
1662 311 : return wasm::kExprF64Trunc;
1663 : default:
1664 0 : UNREACHABLE();
1665 : }
1666 : }
1667 :
1668 751 : wasm::WasmOpcode NeOp(const MachineType& ty) {
1669 751 : switch (ty.representation()) {
1670 : case MachineRepresentation::kFloat32:
1671 : return wasm::kExprF32Ne;
1672 : case MachineRepresentation::kFloat64:
1673 371 : return wasm::kExprF64Ne;
1674 : default:
1675 0 : UNREACHABLE();
1676 : }
1677 : }
1678 :
1679 120 : wasm::WasmOpcode LtOp(const MachineType& ty) {
1680 120 : switch (ty.representation()) {
1681 : case MachineRepresentation::kFloat32:
1682 : return wasm::kExprF32Lt;
1683 : case MachineRepresentation::kFloat64:
1684 60 : return wasm::kExprF64Lt;
1685 : default:
1686 0 : UNREACHABLE();
1687 : }
1688 : }
1689 :
1690 1587 : Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1691 1587 : const MachineType& int_ty, const MachineType& float_ty,
1692 : Node* trunc, Node* converted_value) {
1693 1587 : if (int_ty.representation() == MachineRepresentation::kWord32) {
1694 631 : Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1695 631 : return builder->Binop(NeOp(float_ty), trunc, check);
1696 : }
1697 : return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1698 1919 : trunc, builder->graph()->start());
1699 : }
1700 :
1701 120 : Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1702 120 : const MachineType& int_ty,
1703 : const MachineType& float_ty, Node* trunc,
1704 : Node* converted_value) {
1705 : Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1706 120 : converted_value);
1707 120 : if (int_ty.representation() == MachineRepresentation::kWord64) {
1708 60 : test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1709 : }
1710 120 : return test;
1711 : }
1712 :
1713 : } // namespace
1714 :
1715 1584 : Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1716 : wasm::WasmCodePosition position,
1717 480 : wasm::WasmOpcode opcode) {
1718 1584 : const MachineType int_ty = IntConvertType(opcode);
1719 1587 : const MachineType float_ty = FloatConvertType(opcode);
1720 1587 : const Operator* conv_op = ConvertOp(this, opcode);
1721 : Node* trunc = nullptr;
1722 : Node* converted_value = nullptr;
1723 : const bool is_int32 =
1724 1586 : int_ty.representation() == MachineRepresentation::kWord32;
1725 1586 : if (is_int32) {
1726 631 : trunc = Unop(TruncOp(float_ty), input);
1727 : converted_value = graph()->NewNode(conv_op, trunc);
1728 : } else {
1729 : trunc = graph()->NewNode(conv_op, input);
1730 : converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1731 958 : trunc, graph()->start());
1732 : }
1733 1593 : if (IsTrappingConvertOp(opcode)) {
1734 : Node* test =
1735 1470 : ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1736 1463 : if (is_int32) {
1737 571 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1738 : } else {
1739 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1740 : }
1741 1478 : return converted_value;
1742 : }
1743 : Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1744 120 : converted_value);
1745 240 : Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1746 120 : tl_d.Chain(Control());
1747 120 : Node* nan_test = Binop(NeOp(float_ty), input, input);
1748 240 : Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1749 120 : nan_d.Nest(tl_d, true);
1750 120 : Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1751 240 : Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1752 120 : sat_d.Nest(nan_d, false);
1753 : Node* sat_val =
1754 120 : sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1755 : Node* nan_val =
1756 120 : nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1757 120 : return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1758 : }
1759 :
1760 15 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1761 15 : MachineOperatorBuilder* m = mcgraph()->machine();
1762 : // asm.js must use the wacky JS semantics.
1763 15 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1764 30 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1765 : }
1766 :
1767 124 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1768 124 : MachineOperatorBuilder* m = mcgraph()->machine();
1769 : // asm.js must use the wacky JS semantics.
1770 248 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1771 : }
1772 :
1773 15 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1774 15 : MachineOperatorBuilder* m = mcgraph()->machine();
1775 : // asm.js must use the wacky JS semantics.
1776 15 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1777 30 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1778 : }
1779 :
1780 15 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1781 15 : MachineOperatorBuilder* m = mcgraph()->machine();
1782 : // asm.js must use the wacky JS semantics.
1783 30 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1784 : }
1785 :
1786 0 : Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1787 0 : MachineRepresentation input_type) {
1788 : Node* stack_slot_param =
1789 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1790 :
1791 : const Operator* store_op = mcgraph()->machine()->Store(
1792 0 : StoreRepresentation(input_type, kNoWriteBarrier));
1793 : SetEffect(graph()->NewNode(store_op, stack_slot_param,
1794 : mcgraph()->Int32Constant(0), input, Effect(),
1795 0 : Control()));
1796 :
1797 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1798 : MachineSignature sig(1, 1, sig_types);
1799 :
1800 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1801 :
1802 0 : return BuildCCall(&sig, function, stack_slot_param);
1803 : }
1804 :
1805 0 : Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1806 : return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1807 0 : MachineRepresentation::kWord32);
1808 : }
1809 :
1810 0 : Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1811 : return Unop(wasm::kExprI64UConvertI32,
1812 : BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1813 0 : MachineRepresentation::kWord64));
1814 : }
1815 :
1816 0 : Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1817 : return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1818 0 : MachineRepresentation::kWord32);
1819 : }
1820 :
1821 0 : Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1822 : return Unop(
1823 : wasm::kExprI64UConvertI32,
1824 : BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1825 0 : MachineRepresentation::kWord64));
1826 : }
1827 :
1828 0 : Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1829 0 : MachineType type = MachineType::Float32();
1830 0 : ExternalReference ref = ExternalReference::wasm_f32_trunc();
1831 :
1832 0 : return BuildCFuncInstruction(ref, type, input);
1833 : }
1834 :
1835 0 : Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1836 0 : MachineType type = MachineType::Float32();
1837 0 : ExternalReference ref = ExternalReference::wasm_f32_floor();
1838 0 : return BuildCFuncInstruction(ref, type, input);
1839 : }
1840 :
1841 0 : Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1842 0 : MachineType type = MachineType::Float32();
1843 0 : ExternalReference ref = ExternalReference::wasm_f32_ceil();
1844 0 : return BuildCFuncInstruction(ref, type, input);
1845 : }
1846 :
1847 0 : Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1848 0 : MachineType type = MachineType::Float32();
1849 0 : ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1850 0 : return BuildCFuncInstruction(ref, type, input);
1851 : }
1852 :
1853 0 : Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1854 0 : MachineType type = MachineType::Float64();
1855 0 : ExternalReference ref = ExternalReference::wasm_f64_trunc();
1856 0 : return BuildCFuncInstruction(ref, type, input);
1857 : }
1858 :
1859 0 : Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1860 0 : MachineType type = MachineType::Float64();
1861 0 : ExternalReference ref = ExternalReference::wasm_f64_floor();
1862 0 : return BuildCFuncInstruction(ref, type, input);
1863 : }
1864 :
1865 0 : Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1866 0 : MachineType type = MachineType::Float64();
1867 0 : ExternalReference ref = ExternalReference::wasm_f64_ceil();
1868 0 : return BuildCFuncInstruction(ref, type, input);
1869 : }
1870 :
1871 0 : Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1872 0 : MachineType type = MachineType::Float64();
1873 0 : ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1874 0 : return BuildCFuncInstruction(ref, type, input);
1875 : }
1876 :
1877 25 : Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1878 25 : MachineType type = MachineType::Float64();
1879 25 : ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1880 25 : return BuildCFuncInstruction(ref, type, input);
1881 : }
1882 :
1883 25 : Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1884 25 : MachineType type = MachineType::Float64();
1885 25 : ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1886 25 : return BuildCFuncInstruction(ref, type, input);
1887 : }
1888 :
1889 11 : Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1890 11 : MachineType type = MachineType::Float64();
1891 11 : ExternalReference ref = ExternalReference::wasm_float64_pow();
1892 11 : return BuildCFuncInstruction(ref, type, left, right);
1893 : }
1894 :
1895 25 : Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1896 25 : MachineType type = MachineType::Float64();
1897 25 : ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1898 25 : return BuildCFuncInstruction(ref, type, left, right);
1899 : }
1900 :
1901 86 : Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1902 : MachineType type, Node* input0,
1903 1176 : Node* input1) {
1904 : // We do truncation by calling a C function which calculates the result.
1905 : // The input is passed to the C function as a byte buffer holding the two
1906 : // input doubles. We reserve this byte buffer as a stack slot, store the
1907 : // parameters in this buffer slots, pass a pointer to the buffer to the C
1908 : // function, and after calling the C function we collect the return value from
1909 : // the buffer.
1910 :
1911 : const int type_size = ElementSizeInBytes(type.representation());
1912 86 : const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1913 : Node* stack_slot =
1914 86 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1915 :
1916 : const Operator* store_op = mcgraph()->machine()->Store(
1917 86 : StoreRepresentation(type.representation(), kNoWriteBarrier));
1918 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1919 86 : input0, Effect(), Control()));
1920 :
1921 86 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1922 :
1923 86 : if (input1 != nullptr) {
1924 : SetEffect(graph()->NewNode(store_op, stack_slot,
1925 : mcgraph()->Int32Constant(type_size), input1,
1926 36 : Effect(), Control()));
1927 : }
1928 :
1929 86 : MachineType sig_types[] = {MachineType::Pointer()};
1930 : MachineSignature sig(0, 1, sig_types);
1931 86 : BuildCCall(&sig, function, stack_slot);
1932 :
1933 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
1934 : stack_slot, mcgraph()->Int32Constant(0),
1935 258 : Effect(), Control()));
1936 : }
1937 :
1938 0 : Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1939 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1940 : return BuildIntToFloatConversionInstruction(
1941 : input, ExternalReference::wasm_int64_to_float32(),
1942 0 : MachineRepresentation::kWord64, MachineType::Float32());
1943 : }
1944 0 : Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1945 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1946 : return BuildIntToFloatConversionInstruction(
1947 : input, ExternalReference::wasm_uint64_to_float32(),
1948 0 : MachineRepresentation::kWord64, MachineType::Float32());
1949 : }
1950 0 : Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1951 : return BuildIntToFloatConversionInstruction(
1952 : input, ExternalReference::wasm_int64_to_float64(),
1953 0 : MachineRepresentation::kWord64, MachineType::Float64());
1954 : }
1955 0 : Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1956 : return BuildIntToFloatConversionInstruction(
1957 : input, ExternalReference::wasm_uint64_to_float64(),
1958 0 : MachineRepresentation::kWord64, MachineType::Float64());
1959 : }
1960 :
1961 0 : Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1962 : Node* input, ExternalReference ref,
1963 : MachineRepresentation parameter_representation,
1964 0 : const MachineType result_type) {
1965 : int stack_slot_size =
1966 : std::max(ElementSizeInBytes(parameter_representation),
1967 0 : ElementSizeInBytes(result_type.representation()));
1968 : Node* stack_slot =
1969 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1970 : const Operator* store_op = mcgraph()->machine()->Store(
1971 0 : StoreRepresentation(parameter_representation, kNoWriteBarrier));
1972 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1973 0 : input, Effect(), Control()));
1974 0 : MachineType sig_types[] = {MachineType::Pointer()};
1975 : MachineSignature sig(0, 1, sig_types);
1976 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1977 0 : BuildCCall(&sig, function, stack_slot);
1978 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
1979 : stack_slot, mcgraph()->Int32Constant(0),
1980 0 : Effect(), Control()));
1981 : }
1982 :
1983 : namespace {
1984 :
1985 0 : ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1986 : wasm::WasmOpcode opcode) {
1987 0 : switch (opcode) {
1988 : case wasm::kExprI64SConvertF32:
1989 : case wasm::kExprI64SConvertSatF32:
1990 0 : return ExternalReference::wasm_float32_to_int64();
1991 : case wasm::kExprI64UConvertF32:
1992 : case wasm::kExprI64UConvertSatF32:
1993 0 : return ExternalReference::wasm_float32_to_uint64();
1994 : case wasm::kExprI64SConvertF64:
1995 : case wasm::kExprI64SConvertSatF64:
1996 0 : return ExternalReference::wasm_float64_to_int64();
1997 : case wasm::kExprI64UConvertF64:
1998 : case wasm::kExprI64UConvertSatF64:
1999 0 : return ExternalReference::wasm_float64_to_uint64();
2000 : default:
2001 0 : UNREACHABLE();
2002 : }
2003 : }
2004 :
2005 : } // namespace
2006 :
2007 0 : Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
2008 : wasm::WasmCodePosition position,
2009 0 : wasm::WasmOpcode opcode) {
2010 0 : const MachineType int_ty = IntConvertType(opcode);
2011 0 : const MachineType float_ty = FloatConvertType(opcode);
2012 0 : ExternalReference call_ref = convert_ccall_ref(this, opcode);
2013 0 : int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
2014 0 : ElementSizeInBytes(float_ty.representation()));
2015 : Node* stack_slot =
2016 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
2017 : const Operator* store_op = mcgraph()->machine()->Store(
2018 0 : StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
2019 : SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
2020 : Effect(), Control()));
2021 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2022 : MachineSignature sig(1, 1, sig_types);
2023 : Node* function =
2024 0 : graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
2025 0 : Node* overflow = BuildCCall(&sig, function, stack_slot);
2026 0 : if (IsTrappingConvertOp(opcode)) {
2027 : ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
2028 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
2029 : stack_slot, Int32Constant(0), Effect(),
2030 0 : Control()));
2031 : }
2032 0 : Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
2033 0 : Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
2034 0 : tl_d.Chain(Control());
2035 0 : Node* nan_test = Binop(NeOp(float_ty), input, input);
2036 0 : Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2037 0 : nan_d.Nest(tl_d, true);
2038 0 : Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2039 0 : Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2040 0 : sat_d.Nest(nan_d, false);
2041 : Node* sat_val =
2042 0 : sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2043 : Node* load =
2044 : SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
2045 0 : Int32Constant(0), Effect(), Control()));
2046 : Node* nan_val =
2047 0 : nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2048 0 : return tl_d.Phi(int_ty.representation(), nan_val, load);
2049 : }
2050 :
2051 10240 : Node* WasmGraphBuilder::MemoryGrow(Node* input) {
2052 1280 : needs_stack_check_ = true;
2053 :
2054 : WasmMemoryGrowDescriptor interface_descriptor;
2055 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2056 : mcgraph()->zone(), // zone
2057 : interface_descriptor, // descriptor
2058 : interface_descriptor.GetStackParameterCount(), // stack parameter count
2059 : CallDescriptor::kNoFlags, // flags
2060 : Operator::kNoProperties, // properties
2061 1280 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
2062 : // A direct call to a wasm runtime stub defined in this module.
2063 : // Just encode the stub index. This will be patched at relocation.
2064 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2065 1280 : wasm::WasmCode::kWasmMemoryGrow, RelocInfo::WASM_STUB_CALL);
2066 : return SetEffect(
2067 : SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
2068 2560 : call_target, input, Effect(), Control())));
2069 : }
2070 :
2071 : #ifdef DEBUG
2072 :
2073 : namespace {
2074 :
2075 : constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2;
2076 :
2077 : size_t ComputeEncodedElementSize(wasm::ValueType type) {
2078 : size_t byte_size =
2079 : static_cast<size_t>(wasm::ValueTypes::ElementSizeInBytes(type));
2080 : DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0);
2081 : DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement);
2082 : return byte_size / kBytesPerExceptionValuesArrayElement;
2083 : }
2084 :
2085 : } // namespace
2086 :
2087 : #endif // DEBUG
2088 :
2089 538 : uint32_t WasmGraphBuilder::GetExceptionEncodedSize(
2090 : const wasm::WasmException* exception) const {
2091 1670 : const wasm::WasmExceptionSig* sig = exception->sig;
2092 : uint32_t encoded_size = 0;
2093 1670 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2094 297 : switch (sig->GetParam(i)) {
2095 : case wasm::kWasmI32:
2096 : case wasm::kWasmF32:
2097 : DCHECK_EQ(2, ComputeEncodedElementSize(sig->GetParam(i)));
2098 207 : encoded_size += 2;
2099 207 : break;
2100 : case wasm::kWasmI64:
2101 : case wasm::kWasmF64:
2102 : DCHECK_EQ(4, ComputeEncodedElementSize(sig->GetParam(i)));
2103 36 : encoded_size += 4;
2104 36 : break;
2105 : case wasm::kWasmS128:
2106 : DCHECK_EQ(8, ComputeEncodedElementSize(sig->GetParam(i)));
2107 18 : encoded_size += 8;
2108 18 : break;
2109 : case wasm::kWasmAnyRef:
2110 36 : encoded_size += 1;
2111 36 : break;
2112 : default:
2113 0 : UNREACHABLE();
2114 : }
2115 : }
2116 538 : return encoded_size;
2117 : }
2118 :
2119 538 : Node* WasmGraphBuilder::Throw(uint32_t exception_index,
2120 : const wasm::WasmException* exception,
2121 4495 : const Vector<Node*> values) {
2122 538 : needs_stack_check_ = true;
2123 538 : uint32_t encoded_size = GetExceptionEncodedSize(exception);
2124 : Node* create_parameters[] = {
2125 538 : LoadExceptionTagFromTable(exception_index),
2126 1078 : BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2127 : Node* except_obj =
2128 : BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2129 538 : arraysize(create_parameters));
2130 : Node* values_array =
2131 540 : BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2132 538 : uint32_t index = 0;
2133 1669 : const wasm::WasmExceptionSig* sig = exception->sig;
2134 538 : MachineOperatorBuilder* m = mcgraph()->machine();
2135 1670 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2136 592 : Node* value = values[i];
2137 296 : switch (sig->GetParam(i)) {
2138 : case wasm::kWasmF32:
2139 18 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2140 : V8_FALLTHROUGH;
2141 : case wasm::kWasmI32:
2142 206 : BuildEncodeException32BitValue(values_array, &index, value);
2143 207 : break;
2144 : case wasm::kWasmF64:
2145 18 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2146 : V8_FALLTHROUGH;
2147 : case wasm::kWasmI64: {
2148 : Node* upper32 = graph()->NewNode(
2149 : m->TruncateInt64ToInt32(),
2150 36 : Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2151 36 : BuildEncodeException32BitValue(values_array, &index, upper32);
2152 36 : Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2153 36 : BuildEncodeException32BitValue(values_array, &index, lower32);
2154 36 : break;
2155 : }
2156 : case wasm::kWasmS128:
2157 : BuildEncodeException32BitValue(
2158 : values_array, &index,
2159 36 : graph()->NewNode(m->I32x4ExtractLane(0), value));
2160 : BuildEncodeException32BitValue(
2161 : values_array, &index,
2162 36 : graph()->NewNode(m->I32x4ExtractLane(1), value));
2163 : BuildEncodeException32BitValue(
2164 : values_array, &index,
2165 36 : graph()->NewNode(m->I32x4ExtractLane(2), value));
2166 : BuildEncodeException32BitValue(
2167 : values_array, &index,
2168 36 : graph()->NewNode(m->I32x4ExtractLane(3), value));
2169 18 : break;
2170 : case wasm::kWasmAnyRef:
2171 144 : STORE_FIXED_ARRAY_SLOT_ANY(values_array, index, value);
2172 36 : ++index;
2173 36 : break;
2174 : default:
2175 0 : UNREACHABLE();
2176 : }
2177 : }
2178 : DCHECK_EQ(encoded_size, index);
2179 : WasmThrowDescriptor interface_descriptor;
2180 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2181 : mcgraph()->zone(), interface_descriptor,
2182 : interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2183 539 : Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2184 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2185 538 : wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2186 : return SetEffect(SetControl(
2187 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2188 1620 : except_obj, Effect(), Control())));
2189 : }
2190 :
2191 350 : void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
2192 : uint32_t* index,
2193 3855 : Node* value) {
2194 350 : MachineOperatorBuilder* machine = mcgraph()->machine();
2195 : Node* upper_halfword_as_smi = BuildChangeUint31ToSmi(
2196 702 : graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16)));
2197 1400 : STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, upper_halfword_as_smi);
2198 350 : ++(*index);
2199 : Node* lower_halfword_as_smi = BuildChangeUint31ToSmi(
2200 702 : graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu)));
2201 1404 : STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, lower_halfword_as_smi);
2202 351 : ++(*index);
2203 351 : }
2204 :
2205 306 : Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
2206 3060 : uint32_t* index) {
2207 306 : MachineOperatorBuilder* machine = mcgraph()->machine();
2208 : Node* upper =
2209 1224 : BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2210 306 : (*index)++;
2211 306 : upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2212 : Node* lower =
2213 1224 : BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2214 306 : (*index)++;
2215 306 : Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2216 306 : return value;
2217 : }
2218 :
2219 18 : Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
2220 : uint32_t* index) {
2221 : Node* upper = Binop(wasm::kExprI64Shl,
2222 : Unop(wasm::kExprI64UConvertI32,
2223 : BuildDecodeException32BitValue(values_array, index)),
2224 18 : Int64Constant(32));
2225 : Node* lower = Unop(wasm::kExprI64UConvertI32,
2226 18 : BuildDecodeException32BitValue(values_array, index));
2227 18 : return Binop(wasm::kExprI64Ior, upper, lower);
2228 : }
2229 :
2230 3816 : Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
2231 477 : needs_stack_check_ = true;
2232 : WasmThrowDescriptor interface_descriptor;
2233 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2234 : mcgraph()->zone(), interface_descriptor,
2235 : interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2236 477 : Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2237 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2238 477 : wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2239 : return SetEffect(SetControl(
2240 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2241 954 : except_obj, Effect(), Control())));
2242 : }
2243 :
2244 423 : Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
2245 423 : Node* expected_tag) {
2246 423 : MachineOperatorBuilder* machine = mcgraph()->machine();
2247 846 : return graph()->NewNode(machine->WordEqual(), caught_tag, expected_tag);
2248 : }
2249 :
2250 9620 : Node* WasmGraphBuilder::LoadExceptionTagFromTable(uint32_t exception_index) {
2251 : Node* exceptions_table =
2252 1922 : LOAD_INSTANCE_FIELD(ExceptionsTable, MachineType::TaggedPointer());
2253 2888 : Node* tag = LOAD_FIXED_ARRAY_SLOT_PTR(exceptions_table, exception_index);
2254 962 : return tag;
2255 : }
2256 :
2257 422 : Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
2258 422 : needs_stack_check_ = true;
2259 422 : return BuildCallToRuntime(Runtime::kWasmExceptionGetTag, &except_obj, 1);
2260 : }
2261 :
2262 423 : Node** WasmGraphBuilder::GetExceptionValues(
2263 126 : Node* except_obj, const wasm::WasmException* exception) {
2264 : Node* values_array =
2265 423 : BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2266 423 : uint32_t index = 0;
2267 1404 : const wasm::WasmExceptionSig* sig = exception->sig;
2268 423 : Node** values = Buffer(sig->parameter_count());
2269 1404 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2270 : Node* value;
2271 279 : switch (sig->GetParam(i)) {
2272 : case wasm::kWasmI32:
2273 225 : value = BuildDecodeException32BitValue(values_array, &index);
2274 225 : break;
2275 : case wasm::kWasmI64:
2276 9 : value = BuildDecodeException64BitValue(values_array, &index);
2277 9 : break;
2278 : case wasm::kWasmF32: {
2279 : value = Unop(wasm::kExprF32ReinterpretI32,
2280 9 : BuildDecodeException32BitValue(values_array, &index));
2281 9 : break;
2282 : }
2283 : case wasm::kWasmF64: {
2284 : value = Unop(wasm::kExprF64ReinterpretI64,
2285 9 : BuildDecodeException64BitValue(values_array, &index));
2286 9 : break;
2287 : }
2288 : case wasm::kWasmS128:
2289 : value = graph()->NewNode(
2290 : mcgraph()->machine()->I32x4Splat(),
2291 18 : BuildDecodeException32BitValue(values_array, &index));
2292 : value = graph()->NewNode(
2293 : mcgraph()->machine()->I32x4ReplaceLane(1), value,
2294 18 : BuildDecodeException32BitValue(values_array, &index));
2295 : value = graph()->NewNode(
2296 : mcgraph()->machine()->I32x4ReplaceLane(2), value,
2297 18 : BuildDecodeException32BitValue(values_array, &index));
2298 : value = graph()->NewNode(
2299 : mcgraph()->machine()->I32x4ReplaceLane(3), value,
2300 18 : BuildDecodeException32BitValue(values_array, &index));
2301 9 : break;
2302 : case wasm::kWasmAnyRef:
2303 54 : value = LOAD_FIXED_ARRAY_SLOT_ANY(values_array, index);
2304 18 : ++index;
2305 18 : break;
2306 : default:
2307 0 : UNREACHABLE();
2308 : }
2309 279 : values[i] = value;
2310 : }
2311 : DCHECK_EQ(index, GetExceptionEncodedSize(exception));
2312 423 : return values;
2313 : }
2314 :
2315 35303 : Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2316 216304 : wasm::WasmCodePosition position) {
2317 35303 : MachineOperatorBuilder* m = mcgraph()->machine();
2318 : ZeroCheck32(wasm::kTrapDivByZero, right, position);
2319 : Node* before = Control();
2320 : Node* denom_is_m1;
2321 : Node* denom_is_not_m1;
2322 : BranchExpectFalse(
2323 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2324 35303 : &denom_is_m1, &denom_is_not_m1);
2325 35303 : SetControl(denom_is_m1);
2326 35303 : TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2327 35303 : if (Control() != denom_is_m1) {
2328 : SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2329 4486 : Control()));
2330 : } else {
2331 : SetControl(before);
2332 : }
2333 70606 : return graph()->NewNode(m->Int32Div(), left, right, Control());
2334 : }
2335 :
2336 35157 : Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2337 175785 : wasm::WasmCodePosition position) {
2338 35157 : MachineOperatorBuilder* m = mcgraph()->machine();
2339 :
2340 : ZeroCheck32(wasm::kTrapRemByZero, right, position);
2341 :
2342 : Diamond d(
2343 : graph(), mcgraph()->common(),
2344 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2345 105471 : BranchHint::kFalse);
2346 35157 : d.Chain(Control());
2347 :
2348 : return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2349 70314 : graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2350 : }
2351 :
2352 35173 : Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2353 35173 : wasm::WasmCodePosition position) {
2354 35173 : MachineOperatorBuilder* m = mcgraph()->machine();
2355 : return graph()->NewNode(m->Uint32Div(), left, right,
2356 70346 : ZeroCheck32(wasm::kTrapDivByZero, right, position));
2357 : }
2358 :
2359 35128 : Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2360 35128 : wasm::WasmCodePosition position) {
2361 35128 : MachineOperatorBuilder* m = mcgraph()->machine();
2362 : return graph()->NewNode(m->Uint32Mod(), left, right,
2363 70256 : ZeroCheck32(wasm::kTrapRemByZero, right, position));
2364 : }
2365 :
2366 1310 : Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2367 512 : MachineOperatorBuilder* m = mcgraph()->machine();
2368 :
2369 : Int32Matcher mr(right);
2370 512 : if (mr.HasValue()) {
2371 430 : if (mr.Value() == 0) {
2372 21 : return mcgraph()->Int32Constant(0);
2373 409 : } else if (mr.Value() == -1) {
2374 : // The result is the negation of the left input.
2375 42 : return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2376 : }
2377 776 : return graph()->NewNode(m->Int32Div(), left, right, Control());
2378 : }
2379 :
2380 : // asm.js semantics return 0 on divide or mod by zero.
2381 82 : if (m->Int32DivIsSafe()) {
2382 : // The hardware instruction does the right thing (e.g. arm).
2383 0 : return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2384 : }
2385 :
2386 : // Check denominator for zero.
2387 : Diamond z(
2388 : graph(), mcgraph()->common(),
2389 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2390 246 : BranchHint::kFalse);
2391 :
2392 : // Check numerator for -1. (avoid minint / -1 case).
2393 : Diamond n(
2394 : graph(), mcgraph()->common(),
2395 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2396 246 : BranchHint::kFalse);
2397 :
2398 82 : Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2399 : Node* neg =
2400 82 : graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2401 :
2402 : return n.Phi(
2403 : MachineRepresentation::kWord32, neg,
2404 82 : z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2405 : }
2406 :
2407 1090 : Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2408 566 : CommonOperatorBuilder* c = mcgraph()->common();
2409 566 : MachineOperatorBuilder* m = mcgraph()->machine();
2410 566 : Node* const zero = mcgraph()->Int32Constant(0);
2411 :
2412 : Int32Matcher mr(right);
2413 566 : if (mr.HasValue()) {
2414 488 : if (mr.Value() == 0 || mr.Value() == -1) {
2415 : return zero;
2416 : }
2417 892 : return graph()->NewNode(m->Int32Mod(), left, right, Control());
2418 : }
2419 :
2420 : // General case for signed integer modulus, with optimization for (unknown)
2421 : // power of 2 right hand side.
2422 : //
2423 : // if 0 < right then
2424 : // msk = right - 1
2425 : // if right & msk != 0 then
2426 : // left % right
2427 : // else
2428 : // if left < 0 then
2429 : // -(-left & msk)
2430 : // else
2431 : // left & msk
2432 : // else
2433 : // if right < -1 then
2434 : // left % right
2435 : // else
2436 : // zero
2437 : //
2438 : // Note: We do not use the Diamond helper class here, because it really hurts
2439 : // readability with nested diamonds.
2440 78 : Node* const minus_one = mcgraph()->Int32Constant(-1);
2441 :
2442 78 : const Operator* const merge_op = c->Merge(2);
2443 78 : const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2444 :
2445 78 : Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2446 : Node* branch0 =
2447 78 : graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2448 :
2449 78 : Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2450 : Node* true0;
2451 : {
2452 78 : Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2453 :
2454 78 : Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2455 78 : Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2456 :
2457 78 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2458 78 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2459 :
2460 78 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2461 : Node* false1;
2462 : {
2463 78 : Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2464 : Node* branch2 =
2465 78 : graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2466 :
2467 78 : Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2468 : Node* true2 = graph()->NewNode(
2469 : m->Int32Sub(), zero,
2470 : graph()->NewNode(m->Word32And(),
2471 234 : graph()->NewNode(m->Int32Sub(), zero, left), msk));
2472 :
2473 78 : Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2474 78 : Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2475 :
2476 : if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2477 : false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2478 : }
2479 :
2480 : if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2481 : true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2482 : }
2483 :
2484 78 : Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2485 : Node* false0;
2486 : {
2487 78 : Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2488 : Node* branch1 =
2489 78 : graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2490 :
2491 78 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2492 78 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2493 :
2494 78 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2495 : Node* false1 = zero;
2496 :
2497 : if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2498 : false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2499 : }
2500 :
2501 : Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2502 78 : return graph()->NewNode(phi_op, true0, false0, merge0);
2503 : }
2504 :
2505 676 : Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2506 169 : MachineOperatorBuilder* m = mcgraph()->machine();
2507 : // asm.js semantics return 0 on divide or mod by zero.
2508 169 : if (m->Uint32DivIsSafe()) {
2509 : // The hardware instruction does the right thing (e.g. arm).
2510 0 : return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2511 : }
2512 :
2513 : // Explicit check for x % 0.
2514 : Diamond z(
2515 : graph(), mcgraph()->common(),
2516 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2517 507 : BranchHint::kFalse);
2518 :
2519 : return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2520 : graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2521 507 : z.if_false));
2522 : }
2523 :
2524 912 : Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2525 228 : MachineOperatorBuilder* m = mcgraph()->machine();
2526 : // asm.js semantics return 0 on divide or mod by zero.
2527 : // Explicit check for x % 0.
2528 : Diamond z(
2529 : graph(), mcgraph()->common(),
2530 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2531 684 : BranchHint::kFalse);
2532 :
2533 : Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2534 456 : z.if_false);
2535 : return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2536 228 : rem);
2537 : }
2538 :
2539 1663 : Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2540 16610 : wasm::WasmCodePosition position) {
2541 3326 : if (mcgraph()->machine()->Is32()) {
2542 : return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2543 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2544 : }
2545 : ZeroCheck64(wasm::kTrapDivByZero, right, position);
2546 : Node* before = Control();
2547 : Node* denom_is_m1;
2548 : Node* denom_is_not_m1;
2549 : BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2550 : mcgraph()->Int64Constant(-1)),
2551 3326 : &denom_is_m1, &denom_is_not_m1);
2552 1663 : SetControl(denom_is_m1);
2553 : TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2554 1663 : std::numeric_limits<int64_t>::min(), position);
2555 1663 : if (Control() != denom_is_m1) {
2556 : SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2557 3306 : Control()));
2558 : } else {
2559 : SetControl(before);
2560 : }
2561 : return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2562 3326 : Control());
2563 : }
2564 :
2565 1517 : Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2566 10619 : wasm::WasmCodePosition position) {
2567 3034 : if (mcgraph()->machine()->Is32()) {
2568 : return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2569 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2570 : }
2571 : ZeroCheck64(wasm::kTrapRemByZero, right, position);
2572 : Diamond d(mcgraph()->graph(), mcgraph()->common(),
2573 : graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2574 4551 : mcgraph()->Int64Constant(-1)));
2575 :
2576 1517 : d.Chain(Control());
2577 :
2578 : Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2579 3034 : d.if_false);
2580 :
2581 : return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2582 1517 : rem);
2583 : }
2584 :
2585 1535 : Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2586 3070 : wasm::WasmCodePosition position) {
2587 3070 : if (mcgraph()->machine()->Is32()) {
2588 : return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2589 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2590 : }
2591 : return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2592 3070 : ZeroCheck64(wasm::kTrapDivByZero, right, position));
2593 : }
2594 1517 : Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2595 3034 : wasm::WasmCodePosition position) {
2596 3034 : if (mcgraph()->machine()->Is32()) {
2597 : return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2598 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2599 : }
2600 : return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2601 3034 : ZeroCheck64(wasm::kTrapRemByZero, right, position));
2602 : }
2603 :
2604 0 : Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2605 : ExternalReference ref,
2606 : MachineType result_type,
2607 : wasm::TrapReason trap_zero,
2608 0 : wasm::WasmCodePosition position) {
2609 : Node* stack_slot =
2610 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2611 :
2612 : const Operator* store_op = mcgraph()->machine()->Store(
2613 0 : StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2614 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2615 0 : left, Effect(), Control()));
2616 : SetEffect(graph()->NewNode(store_op, stack_slot,
2617 : mcgraph()->Int32Constant(sizeof(double)), right,
2618 0 : Effect(), Control()));
2619 :
2620 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2621 : MachineSignature sig(1, 1, sig_types);
2622 :
2623 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2624 0 : Node* call = BuildCCall(&sig, function, stack_slot);
2625 :
2626 : ZeroCheck32(trap_zero, call, position);
2627 0 : TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2628 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
2629 : stack_slot, mcgraph()->Int32Constant(0),
2630 0 : Effect(), Control()));
2631 : }
2632 :
2633 : template <typename... Args>
2634 185 : Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2635 495 : Args... args) {
2636 : DCHECK_LE(sig->return_count(), 1);
2637 : DCHECK_EQ(sizeof...(args), sig->parameter_count());
2638 555 : Node* const call_args[] = {function, args..., Effect(), Control()};
2639 :
2640 : auto call_descriptor =
2641 185 : Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2642 :
2643 185 : const Operator* op = mcgraph()->common()->Call(call_descriptor);
2644 370 : return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
2645 : }
2646 :
2647 586961 : Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2648 : Node*** rets,
2649 : wasm::WasmCodePosition position,
2650 : Node* instance_node,
2651 978278 : UseRetpoline use_retpoline) {
2652 195650 : if (instance_node == nullptr) {
2653 : DCHECK_NOT_NULL(instance_node_);
2654 : instance_node = instance_node_.get();
2655 : }
2656 195650 : needs_stack_check_ = true;
2657 : const size_t params = sig->parameter_count();
2658 : const size_t extra = 3; // instance_node, effect, and control.
2659 195650 : const size_t count = 1 + params + extra;
2660 :
2661 : // Reallocate the buffer to make space for extra inputs.
2662 195650 : args = Realloc(args, 1 + params, count);
2663 :
2664 : // Make room for the instance_node parameter at index 1, just after code.
2665 195653 : memmove(&args[2], &args[1], params * sizeof(Node*));
2666 195653 : args[1] = instance_node;
2667 :
2668 : // Add effect and control inputs.
2669 391306 : args[params + 2] = Effect();
2670 391306 : args[params + 3] = Control();
2671 :
2672 : auto call_descriptor =
2673 391306 : GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2674 195653 : const Operator* op = mcgraph()->common()->Call(call_descriptor);
2675 391332 : Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
2676 : DCHECK(position == wasm::kNoCodePosition || position > 0);
2677 195666 : if (position > 0) SetSourcePosition(call, position);
2678 :
2679 : size_t ret_count = sig->return_count();
2680 195661 : if (ret_count == 0) return call; // No return value.
2681 :
2682 45660 : *rets = Buffer(ret_count);
2683 45660 : if (ret_count == 1) {
2684 : // Only a single return value.
2685 43602 : (*rets)[0] = call;
2686 : } else {
2687 : // Create projections for all return values.
2688 4134 : for (size_t i = 0; i < ret_count; i++) {
2689 4134 : (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2690 8268 : graph()->start());
2691 : }
2692 : }
2693 : return call;
2694 : }
2695 :
2696 8672 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2697 : Node*** rets,
2698 : wasm::WasmCodePosition position,
2699 173438 : int func_index) {
2700 : // Load the imported function refs array from the instance.
2701 : Node* imported_function_refs =
2702 17342 : LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2703 : Node* ref_node =
2704 17344 : LOAD_FIXED_ARRAY_SLOT_PTR(imported_function_refs, func_index);
2705 :
2706 : // Load the target from the imported_targets array at a known offset.
2707 : Node* imported_targets =
2708 17344 : LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2709 : Node* target_node = SetEffect(graph()->NewNode(
2710 : mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2711 : mcgraph()->Int32Constant(func_index * kSystemPointerSize), Effect(),
2712 26016 : Control()));
2713 8672 : args[0] = target_node;
2714 : return BuildWasmCall(sig, args, rets, position, ref_node,
2715 8672 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2716 : }
2717 :
2718 522 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2719 : Node*** rets,
2720 : wasm::WasmCodePosition position,
2721 11484 : Node* func_index) {
2722 : // Load the imported function refs array from the instance.
2723 : Node* imported_function_refs =
2724 1044 : LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2725 : // Access fixed array at {header_size - tag + func_index * kTaggedSize}.
2726 : Node* imported_instances_data = graph()->NewNode(
2727 : mcgraph()->machine()->IntAdd(), imported_function_refs,
2728 : mcgraph()->IntPtrConstant(
2729 1044 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
2730 : Node* func_index_times_tagged_size = graph()->NewNode(
2731 : mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
2732 1044 : mcgraph()->Int32Constant(kTaggedSize));
2733 : Node* ref_node = SetEffect(
2734 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
2735 : imported_instances_data, func_index_times_tagged_size,
2736 522 : Effect(), Control()));
2737 :
2738 : // Load the target from the imported_targets array at the offset of
2739 : // {func_index}.
2740 : STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
2741 : Node* func_index_times_pointersize = func_index_times_tagged_size;
2742 : Node* imported_targets =
2743 1044 : LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2744 : Node* target_node = SetEffect(graph()->NewNode(
2745 : mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2746 522 : func_index_times_pointersize, Effect(), Control()));
2747 522 : args[0] = target_node;
2748 : return BuildWasmCall(sig, args, rets, position, ref_node,
2749 522 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2750 : }
2751 :
2752 35531 : Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2753 26859 : wasm::WasmCodePosition position) {
2754 : DCHECK_NULL(args[0]);
2755 71062 : wasm::FunctionSig* sig = env_->module->functions[index].sig;
2756 :
2757 35531 : if (env_ && index < env_->module->num_imported_functions) {
2758 : // Call to an imported function.
2759 8672 : return BuildImportCall(sig, args, rets, position, index);
2760 : }
2761 :
2762 : // A direct call to a wasm function defined in this module.
2763 : // Just encode the function index. This will be patched at instantiation.
2764 : Address code = static_cast<Address>(index);
2765 53718 : args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2766 :
2767 26864 : return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2768 : }
2769 :
2770 3965 : Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2771 : Node*** rets,
2772 119044 : wasm::WasmCodePosition position) {
2773 : DCHECK_NOT_NULL(args[0]);
2774 : DCHECK_NOT_NULL(env_);
2775 :
2776 : // Assume only one table for now.
2777 7930 : wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2778 :
2779 : Node* ift_size =
2780 7934 : LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2781 :
2782 3968 : MachineOperatorBuilder* machine = mcgraph()->machine();
2783 3968 : Node* key = args[0];
2784 :
2785 : // Bounds check against the table size.
2786 3968 : Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2787 3965 : TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2788 :
2789 : // Mask the key to prevent SSCA.
2790 3969 : if (untrusted_code_mitigations_) {
2791 : // mask = ((key - size) & ~key) >> 31
2792 : Node* neg_key =
2793 0 : graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2794 : Node* masked_diff = graph()->NewNode(
2795 : machine->Word32And(),
2796 0 : graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2797 : Node* mask =
2798 0 : graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2799 0 : key = graph()->NewNode(machine->Word32And(), key, mask);
2800 : }
2801 :
2802 : // Load signature from the table and check.
2803 : Node* ift_sig_ids =
2804 7934 : LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2805 :
2806 7938 : int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2807 : Node* int32_scaled_key = Uint32ToUintptr(
2808 7934 : graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
2809 :
2810 : Node* loaded_sig = SetEffect(
2811 : graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
2812 3969 : int32_scaled_key, Effect(), Control()));
2813 : Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2814 3969 : Int32Constant(expected_sig_id));
2815 :
2816 3969 : TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2817 :
2818 : Node* ift_targets =
2819 7938 : LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2820 7934 : Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableRefs,
2821 : MachineType::TaggedPointer());
2822 :
2823 : Node* intptr_scaled_key = graph()->NewNode(
2824 3969 : machine->Word32Shl(), key, Int32Constant(kSystemPointerSizeLog2));
2825 :
2826 : Node* target = SetEffect(
2827 : graph()->NewNode(machine->Load(MachineType::Pointer()), ift_targets,
2828 3969 : intptr_scaled_key, Effect(), Control()));
2829 :
2830 : STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
2831 : Node* tagged_scaled_key = intptr_scaled_key;
2832 :
2833 : Node* target_instance = SetEffect(graph()->NewNode(
2834 : machine->Load(MachineType::TaggedPointer()),
2835 : graph()->NewNode(machine->IntAdd(), ift_instances, tagged_scaled_key),
2836 : Int32Constant(wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)),
2837 7938 : Effect(), Control()));
2838 :
2839 3969 : args[0] = target;
2840 :
2841 : return BuildWasmCall(sig, args, rets, position, target_instance,
2842 3969 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2843 : }
2844 :
2845 67328 : Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2846 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2847 : // TODO(weiliang): support Word32Rol opcode in TurboFan.
2848 : Int32Matcher m(right);
2849 33664 : if (m.HasValue()) {
2850 : return Binop(wasm::kExprI32Ror, left,
2851 67280 : mcgraph()->Int32Constant(32 - (m.Value() & 0x1F)));
2852 : } else {
2853 : return Binop(wasm::kExprI32Ror, left,
2854 24 : Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2855 : }
2856 : }
2857 :
2858 196 : Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2859 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2860 : // TODO(weiliang): support Word64Rol opcode in TurboFan.
2861 : Int64Matcher m(right);
2862 98 : if (m.HasValue()) {
2863 : return Binop(wasm::kExprI64Ror, left,
2864 60 : mcgraph()->Int64Constant(64 - (m.Value() & 0x3F)));
2865 : } else {
2866 : return Binop(wasm::kExprI64Ror, left,
2867 68 : Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2868 : }
2869 : }
2870 :
2871 76873 : Node* WasmGraphBuilder::Invert(Node* node) {
2872 76873 : return Unop(wasm::kExprI32Eqz, node);
2873 : }
2874 :
2875 0 : bool CanCover(Node* value, IrOpcode::Value opcode) {
2876 0 : if (value->opcode() != opcode) return false;
2877 : bool first = true;
2878 0 : for (Edge const edge : value->use_edges()) {
2879 0 : if (NodeProperties::IsControlEdge(edge)) continue;
2880 0 : if (NodeProperties::IsEffectEdge(edge)) continue;
2881 : DCHECK(NodeProperties::IsValueEdge(edge));
2882 0 : if (!first) return false;
2883 : first = false;
2884 : }
2885 0 : return true;
2886 : }
2887 :
2888 73434 : Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
2889 146868 : if (mcgraph()->machine()->Is64()) {
2890 73436 : value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2891 : }
2892 73434 : return value;
2893 : }
2894 :
2895 146872 : Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2896 73432 : value = BuildChangeInt32ToIntPtr(value);
2897 : return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2898 146879 : BuildSmiShiftBitsConstant());
2899 : }
2900 :
2901 2967 : Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2902 : return graph()->NewNode(mcgraph()->machine()->WordShl(),
2903 4451 : Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2904 : }
2905 :
2906 326641 : Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2907 326641 : return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2908 : }
2909 :
2910 755182 : Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2911 : value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2912 251730 : BuildSmiShiftBitsConstant());
2913 503462 : if (mcgraph()->machine()->Is64()) {
2914 : value =
2915 251726 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2916 : }
2917 251733 : return value;
2918 : }
2919 :
2920 243 : Node* WasmGraphBuilder::BuildConvertUint32ToSmiWithSaturation(Node* value,
2921 972 : uint32_t maxval) {
2922 : DCHECK(Smi::IsValid(maxval));
2923 : Node* max = Uint32Constant(maxval);
2924 : Node* check = graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(),
2925 243 : value, max);
2926 243 : Node* valsmi = BuildChangeUint31ToSmi(value);
2927 486 : Node* maxsmi = graph()->NewNode(mcgraph()->common()->NumberConstant(maxval));
2928 486 : Diamond d(graph(), mcgraph()->common(), check, BranchHint::kTrue);
2929 243 : d.Chain(Control());
2930 243 : return d.Phi(MachineRepresentation::kTagged, valsmi, maxsmi);
2931 : }
2932 :
2933 1108239 : void WasmGraphBuilder::InitInstanceCache(
2934 11083156 : WasmInstanceCacheNodes* instance_cache) {
2935 : DCHECK_NOT_NULL(instance_node_);
2936 :
2937 : // Load the memory start.
2938 : instance_cache->mem_start = SetEffect(graph()->NewNode(
2939 : mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2940 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2941 3324849 : Effect(), Control()));
2942 :
2943 : // Load the memory size.
2944 : instance_cache->mem_size = SetEffect(graph()->NewNode(
2945 : mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2946 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2947 3325055 : Effect(), Control()));
2948 :
2949 1108297 : if (untrusted_code_mitigations_) {
2950 : // Load the memory mask.
2951 : instance_cache->mem_mask =
2952 0 : LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr());
2953 : } else {
2954 : // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2955 1108297 : instance_cache->mem_mask = nullptr;
2956 : }
2957 1108297 : }
2958 :
2959 4911 : void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2960 9822 : WasmInstanceCacheNodes* instance_cache, Node* control) {
2961 : #define INTRODUCE_PHI(field, rep) \
2962 : instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2963 : instance_cache->field, control);
2964 :
2965 14733 : INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2966 14733 : INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
2967 4911 : if (untrusted_code_mitigations_) {
2968 0 : INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation());
2969 : }
2970 :
2971 : #undef INTRODUCE_PHI
2972 4911 : }
2973 :
2974 46261 : void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2975 : WasmInstanceCacheNodes* from,
2976 29869 : Node* merge) {
2977 : #define INTRODUCE_PHI(field, rep) \
2978 : if (to->field != from->field) { \
2979 : Node* vals[] = {to->field, from->field, merge}; \
2980 : to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2981 : }
2982 :
2983 76130 : INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2984 76132 : INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2985 46262 : if (untrusted_code_mitigations_) {
2986 0 : INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2987 : }
2988 :
2989 : #undef INTRODUCE_PHI
2990 46262 : }
2991 :
2992 242282 : void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2993 : WasmInstanceCacheNodes* from,
2994 : Node* merge) {
2995 : to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2996 242282 : merge, to->mem_size, from->mem_size);
2997 : to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2998 242282 : merge, to->mem_start, from->mem_start);
2999 242283 : if (untrusted_code_mitigations_) {
3000 : to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
3001 0 : merge, to->mem_mask, from->mem_mask);
3002 : }
3003 242283 : }
3004 :
3005 1138698 : Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
3006 : Node* merge, Node* tnode,
3007 15608 : Node* fnode) {
3008 1138698 : if (IsPhiWithMerge(tnode, merge)) {
3009 110804 : AppendToPhi(tnode, fnode);
3010 1027905 : } else if (tnode != fnode) {
3011 15607 : uint32_t count = merge->InputCount();
3012 : // + 1 for the merge node.
3013 15607 : Node** vals = Buffer(count + 1);
3014 15608 : for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
3015 15608 : vals[count - 1] = fnode;
3016 15608 : vals[count] = merge;
3017 : return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
3018 46823 : vals);
3019 : }
3020 1123104 : return tnode;
3021 : }
3022 :
3023 242279 : Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
3024 : Node* fnode) {
3025 242279 : if (IsPhiWithMerge(tnode, merge)) {
3026 15757 : AppendToPhi(tnode, fnode);
3027 226524 : } else if (tnode != fnode) {
3028 146 : uint32_t count = merge->InputCount();
3029 146 : Node** effects = Buffer(count);
3030 568 : for (uint32_t j = 0; j < count - 1; j++) {
3031 422 : effects[j] = tnode;
3032 : }
3033 146 : effects[count - 1] = fnode;
3034 146 : tnode = EffectPhi(count, effects, merge);
3035 : }
3036 242281 : return tnode;
3037 : }
3038 :
3039 538 : Node* WasmGraphBuilder::GetImportedMutableGlobals() {
3040 278 : if (imported_mutable_globals_ == nullptr) {
3041 : // Load imported_mutable_globals_ from the instance object at runtime.
3042 520 : imported_mutable_globals_ = graph()->NewNode(
3043 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3044 : instance_node_.get(),
3045 : mcgraph()->Int32Constant(
3046 : WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
3047 : graph()->start(), graph()->start());
3048 : }
3049 279 : return imported_mutable_globals_.get();
3050 : }
3051 :
3052 86022 : void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
3053 : const wasm::WasmGlobal& global,
3054 : Node** base_node,
3055 148975 : Node** offset_node) {
3056 : DCHECK_NOT_NULL(instance_node_);
3057 86022 : if (global.mutability && global.imported) {
3058 : *base_node = SetEffect(graph()->NewNode(
3059 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3060 : GetImportedMutableGlobals(),
3061 : mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
3062 612 : Control()));
3063 153 : *offset_node = mcgraph()->Int32Constant(0);
3064 : } else {
3065 85869 : if (globals_start_ == nullptr) {
3066 : // Load globals_start from the instance object at runtime.
3067 : // TODO(wasm): we currently generate only one load of the {globals_start}
3068 : // start per graph, which means it can be placed anywhere by the
3069 : // scheduler. This is legal because the globals_start should never change.
3070 : // However, in some cases (e.g. if the instance object is already in a
3071 : // register), it is slightly more efficient to reload this value from the
3072 : // instance object. Since this depends on register allocation, it is not
3073 : // possible to express in the graph, and would essentially constitute a
3074 : // "mem2reg" optimization in TurboFan.
3075 32651 : globals_start_ = graph()->NewNode(
3076 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3077 : instance_node_.get(),
3078 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
3079 : graph()->start(), graph()->start());
3080 : }
3081 85871 : *base_node = globals_start_.get();
3082 171742 : *offset_node = mcgraph()->Int32Constant(global.offset);
3083 :
3084 85872 : if (mem_type == MachineType::Simd128() && global.offset != 0) {
3085 : // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
3086 : *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
3087 68790 : *offset_node);
3088 22930 : *offset_node = mcgraph()->Int32Constant(0);
3089 : }
3090 : }
3091 86025 : }
3092 :
3093 126 : void WasmGraphBuilder::GetBaseAndOffsetForImportedMutableAnyRefGlobal(
3094 2394 : const wasm::WasmGlobal& global, Node** base, Node** offset) {
3095 : // Load the base from the ImportedMutableGlobalsBuffer of the instance.
3096 252 : Node* buffers = LOAD_INSTANCE_FIELD(ImportedMutableGlobalsBuffers,
3097 : MachineType::TaggedPointer());
3098 504 : *base = LOAD_FIXED_ARRAY_SLOT_ANY(buffers, global.index);
3099 :
3100 : // For the offset we need the index of the global in the buffer, and then
3101 : // calculate the actual offset from the index. Load the index from the
3102 : // ImportedMutableGlobals array of the instance.
3103 : Node* index = SetEffect(
3104 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::UintPtr()),
3105 : GetImportedMutableGlobals(),
3106 : mcgraph()->Int32Constant(global.index * sizeof(Address)),
3107 378 : Effect(), Control()));
3108 :
3109 : // From the index, calculate the actual offset in the FixeArray. This
3110 : // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
3111 : // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
3112 : Node* index_times_tagged_size =
3113 : graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
3114 252 : mcgraph()->Int32Constant(kTaggedSize));
3115 : *offset = graph()->NewNode(
3116 : mcgraph()->machine()->IntAdd(), index_times_tagged_size,
3117 : mcgraph()->IntPtrConstant(
3118 378 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
3119 126 : }
3120 :
3121 594748 : Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
3122 : DCHECK_NOT_NULL(instance_cache_);
3123 494451 : Node* mem_start = instance_cache_->mem_start;
3124 : DCHECK_NOT_NULL(mem_start);
3125 494451 : if (offset == 0) return mem_start;
3126 : return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
3127 200596 : mcgraph()->IntPtrConstant(offset));
3128 : }
3129 :
3130 1656 : Node* WasmGraphBuilder::CurrentMemoryPages() {
3131 : // CurrentMemoryPages can not be called from asm.js.
3132 : DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
3133 : DCHECK_NOT_NULL(instance_cache_);
3134 414 : Node* mem_size = instance_cache_->mem_size;
3135 : DCHECK_NOT_NULL(mem_size);
3136 : Node* result =
3137 : graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
3138 828 : mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
3139 828 : if (mcgraph()->machine()->Is64()) {
3140 : result =
3141 414 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
3142 : }
3143 414 : return result;
3144 : }
3145 :
3146 5410 : Node* WasmGraphBuilder::BuildLoadBuiltinFromInstance(int builtin_index) {
3147 : DCHECK(Builtins::IsBuiltinId(builtin_index));
3148 1082 : Node* isolate_root = LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
3149 1623 : return LOAD_TAGGED_POINTER(isolate_root,
3150 : IsolateData::builtin_slot_offset(builtin_index));
3151 : }
3152 :
3153 : // Only call this function for code which is not reused across instantiations,
3154 : // as we do not patch the embedded js_context.
3155 4219 : Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(
3156 : Runtime::FunctionId f, Node* js_context, Node** parameters,
3157 42228 : int parameter_count, Node** effect, Node* control) {
3158 4219 : const Runtime::Function* fun = Runtime::FunctionForId(f);
3159 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3160 : mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
3161 8438 : CallDescriptor::kNoFlags);
3162 : // The CEntryStub is loaded from the instance_node so that generated code is
3163 : // Isolate independent. At the moment this is only done for CEntryStub(1).
3164 : DCHECK_EQ(1, fun->result_size);
3165 : Node* centry_stub =
3166 8447 : LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
3167 : // TODO(titzer): allow arbitrary number of runtime arguments
3168 : // At the moment we only allow 5 parameters. If more parameters are needed,
3169 : // increase this constant accordingly.
3170 : static const int kMaxParams = 5;
3171 : DCHECK_GE(kMaxParams, parameter_count);
3172 : Node* inputs[kMaxParams + 6];
3173 : int count = 0;
3174 4225 : inputs[count++] = centry_stub;
3175 9537 : for (int i = 0; i < parameter_count; i++) {
3176 5312 : inputs[count++] = parameters[i];
3177 : }
3178 4225 : inputs[count++] =
3179 8448 : mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
3180 8444 : inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity
3181 4223 : inputs[count++] = js_context; // js_context
3182 4223 : inputs[count++] = *effect;
3183 4223 : inputs[count++] = control;
3184 :
3185 : Node* call = mcgraph()->graph()->NewNode(
3186 8446 : mcgraph()->common()->Call(call_descriptor), count, inputs);
3187 4225 : *effect = call;
3188 4225 : return call;
3189 : }
3190 :
3191 3267 : Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
3192 : Node** parameters,
3193 3267 : int parameter_count) {
3194 : return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3195 6534 : parameter_count, effect_, Control());
3196 : }
3197 :
3198 319234 : Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3199 127114 : const wasm::WasmGlobal& global = env_->module->globals[index];
3200 63674 : if (global.type == wasm::ValueType::kWasmAnyRef) {
3201 234 : if (global.mutability && global.imported) {
3202 90 : Node* base = nullptr;
3203 90 : Node* offset = nullptr;
3204 90 : GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
3205 : return SetEffect(
3206 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
3207 180 : base, offset, Effect(), Control()));
3208 : }
3209 : Node* globals_buffer =
3210 288 : LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3211 576 : return LOAD_FIXED_ARRAY_SLOT_ANY(globals_buffer, global.offset);
3212 : }
3213 :
3214 : MachineType mem_type =
3215 63440 : wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3216 63440 : Node* base = nullptr;
3217 63440 : Node* offset = nullptr;
3218 63440 : GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3219 63440 : &offset);
3220 : Node* result = SetEffect(graph()->NewNode(
3221 126880 : mcgraph()->machine()->Load(mem_type), base, offset, Effect(), Control()));
3222 : #if defined(V8_TARGET_BIG_ENDIAN)
3223 : result = BuildChangeEndiannessLoad(result, mem_type,
3224 : env_->module->globals[index].type);
3225 : #endif
3226 63440 : return result;
3227 : }
3228 :
3229 114194 : Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3230 45404 : const wasm::WasmGlobal& global = env_->module->globals[index];
3231 22720 : if (global.type == wasm::ValueType::kWasmAnyRef) {
3232 135 : if (global.mutability && global.imported) {
3233 36 : Node* base = nullptr;
3234 36 : Node* offset = nullptr;
3235 36 : GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
3236 :
3237 : return SetEffect(graph()->NewNode(
3238 : mcgraph()->machine()->Store(StoreRepresentation(
3239 : MachineRepresentation::kTagged, kFullWriteBarrier)),
3240 108 : base, offset, val, Effect(), Control()));
3241 : }
3242 : Node* globals_buffer =
3243 198 : LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3244 495 : return STORE_FIXED_ARRAY_SLOT_ANY(globals_buffer,
3245 : env_->module->globals[index].offset, val);
3246 : }
3247 :
3248 : MachineType mem_type =
3249 22585 : wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3250 22585 : Node* base = nullptr;
3251 22585 : Node* offset = nullptr;
3252 22585 : GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3253 22585 : &offset);
3254 : const Operator* op = mcgraph()->machine()->Store(
3255 45170 : StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3256 : #if defined(V8_TARGET_BIG_ENDIAN)
3257 : val = BuildChangeEndiannessStore(val, mem_type.representation(),
3258 : env_->module->globals[index].type);
3259 : #endif
3260 : return SetEffect(
3261 45170 : graph()->NewNode(op, base, offset, val, Effect(), Control()));
3262 : }
3263 :
3264 33518 : Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3265 : uint8_t access_size, Node* index, uint32_t offset,
3266 20756 : wasm::WasmCodePosition position) {
3267 : // Atomic operations need bounds checks until the backend can emit protected
3268 : // loads.
3269 : index =
3270 33518 : BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
3271 :
3272 33554 : const uintptr_t align_mask = access_size - 1;
3273 :
3274 : // Don't emit an alignment check if the index is a constant.
3275 : // TODO(wasm): a constant match is also done above in {BoundsCheckMem}.
3276 : UintPtrMatcher match(index);
3277 33554 : if (match.HasValue()) {
3278 28379 : uintptr_t effective_offset = match.Value() + offset;
3279 28379 : if ((effective_offset & align_mask) != 0) {
3280 : // statically known to be unaligned; trap.
3281 0 : TrapIfEq32(wasm::kTrapUnalignedAccess, Int32Constant(0), 0, position);
3282 : }
3283 : return index;
3284 : }
3285 :
3286 : // Unlike regular memory accesses, atomic memory accesses should trap if
3287 : // the effective offset is misaligned.
3288 : // TODO(wasm): this addition is redundant with one inserted by {MemBuffer}.
3289 : Node* effective_offset = graph()->NewNode(mcgraph()->machine()->IntAdd(),
3290 10368 : MemBuffer(offset), index);
3291 :
3292 : Node* cond = graph()->NewNode(mcgraph()->machine()->WordAnd(),
3293 10400 : effective_offset, IntPtrConstant(align_mask));
3294 : TrapIfFalse(wasm::kTrapUnalignedAccess,
3295 : graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
3296 : mcgraph()->Int32Constant(0)),
3297 15541 : position);
3298 5179 : return index;
3299 : }
3300 :
3301 : // Insert code to bounds check a memory access if necessary. Return the
3302 : // bounds-checked index, which is guaranteed to have (the equivalent of)
3303 : // {uintptr_t} representation.
3304 489919 : Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3305 : uint32_t offset,
3306 : wasm::WasmCodePosition position,
3307 523448 : EnforceBoundsCheck enforce_check) {
3308 : DCHECK_LE(1, access_size);
3309 489919 : index = Uint32ToUintptr(index);
3310 489890 : if (FLAG_wasm_no_bounds_checks) return index;
3311 :
3312 489912 : if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3313 : return index;
3314 : }
3315 :
3316 67050 : if (!IsInBounds(offset, access_size, env_->max_memory_size)) {
3317 : // The access will be out of bounds, even for the largest memory.
3318 18 : TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3319 18 : return mcgraph()->IntPtrConstant(0);
3320 : }
3321 33507 : uint64_t end_offset = uint64_t{offset} + access_size - 1u;
3322 33507 : Node* end_offset_node = IntPtrConstant(end_offset);
3323 :
3324 : // The accessed memory is [index + offset, index + end_offset].
3325 : // Check that the last read byte (at {index + end_offset}) is in bounds.
3326 : // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3327 : // compute {effective_size} as {mem_size - end_offset)}.
3328 : // {effective_size} is >= 1 if condition 1) holds.
3329 : // 2) Check that {index + end_offset < mem_size} by
3330 : // - computing {effective_size} as {mem_size - end_offset} and
3331 : // - checking that {index < effective_size}.
3332 :
3333 33518 : auto m = mcgraph()->machine();
3334 33518 : Node* mem_size = instance_cache_->mem_size;
3335 33518 : if (end_offset >= env_->min_memory_size) {
3336 : // The end offset is larger than the smallest memory.
3337 : // Dynamically check the end offset against the dynamic memory size.
3338 6502 : Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
3339 6513 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3340 : } else {
3341 : // The end offset is smaller than the smallest memory, so only one check is
3342 : // required. Check to see if the index is also a constant.
3343 : UintPtrMatcher match(index);
3344 27016 : if (match.HasValue()) {
3345 : uintptr_t index_val = match.Value();
3346 26997 : if (index_val < env_->min_memory_size - end_offset) {
3347 : // The input index is a constant and everything is statically within
3348 : // bounds of the smallest possible memory.
3349 : return index;
3350 : }
3351 : }
3352 : }
3353 :
3354 : // This produces a positive number, since {end_offset < min_size <= mem_size}.
3355 : Node* effective_size =
3356 6537 : graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
3357 :
3358 : // Introduce the actual bounds check.
3359 6537 : Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
3360 6536 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3361 :
3362 6555 : if (untrusted_code_mitigations_) {
3363 : // In the fallthrough case, condition the index with the memory mask.
3364 0 : Node* mem_mask = instance_cache_->mem_mask;
3365 : DCHECK_NOT_NULL(mem_mask);
3366 0 : index = graph()->NewNode(m->WordAnd(), index, mem_mask);
3367 : }
3368 6555 : return index;
3369 : }
3370 :
3371 : // Check that the range [start, start + size) is in the range [0, max).
3372 171 : void WasmGraphBuilder::BoundsCheckRange(Node* start, Node* size, Node* max,
3373 171 : wasm::WasmCodePosition position) {
3374 : // The accessed memory is [start, end), where {end} is {start + size}. We
3375 : // want to check that {start + size <= max}, making sure that {start + size}
3376 : // doesn't overflow. This can be expressed as {start <= max - size} as long
3377 : // as {max - size} isn't negative, which is true if {size <= max}.
3378 171 : auto m = mcgraph()->machine();
3379 171 : Node* cond = graph()->NewNode(m->Uint32LessThanOrEqual(), size, max);
3380 171 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3381 :
3382 : // This produces a positive number, since {size <= max}.
3383 171 : Node* effective_size = graph()->NewNode(m->Int32Sub(), max, size);
3384 :
3385 : // Introduce the actual bounds check.
3386 : Node* check =
3387 171 : graph()->NewNode(m->Uint32LessThanOrEqual(), start, effective_size);
3388 171 : TrapIfFalse(wasm::kTrapMemOutOfBounds, check, position);
3389 :
3390 : // TODO(binji): Does this need addtional untrusted_code_mitigations_ mask
3391 : // like BoundsCheckMem above?
3392 171 : }
3393 :
3394 126 : Node* WasmGraphBuilder::BoundsCheckMemRange(Node* start, Node* size,
3395 126 : wasm::WasmCodePosition position) {
3396 : // TODO(binji): Support trap handler.
3397 126 : if (!FLAG_wasm_no_bounds_checks) {
3398 126 : BoundsCheckRange(start, size, instance_cache_->mem_size, position);
3399 : }
3400 : return graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(0),
3401 378 : Uint32ToUintptr(start));
3402 : }
3403 :
3404 806 : const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3405 811 : wasm::ValueType type) {
3406 806 : int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3407 806 : MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3408 811 : if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3409 10 : wasm::ValueTypes::MachineRepresentationFor(type))) {
3410 806 : return mcgraph()->machine()->Load(mach_type);
3411 : }
3412 0 : return mcgraph()->machine()->UnalignedLoad(mach_type);
3413 : }
3414 :
3415 1002 : const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3416 1016 : wasm::ValueType type) {
3417 1002 : int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3418 1002 : MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3419 1030 : if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3420 : StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3421 1002 : return mcgraph()->machine()->Store(store_rep);
3422 : }
3423 : UnalignedStoreRepresentation store_rep(rep);
3424 0 : return mcgraph()->machine()->UnalignedStore(store_rep);
3425 : }
3426 :
3427 48 : Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3428 : MachineRepresentation rep,
3429 : Node* index, uint32_t offset,
3430 197 : wasm::WasmCodePosition position) {
3431 : int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3432 : Node* info = graph()->NewNode(
3433 96 : mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3434 :
3435 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3436 97 : Int32Constant(offset), index);
3437 146 : auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3438 : SetEffect(graph()->NewNode(
3439 : mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3440 884 : info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
3441 200 : };
3442 : // Store address, is_store, and mem_rep.
3443 : store(offsetof(wasm::MemoryTracingInfo, address),
3444 50 : MachineRepresentation::kWord32, address);
3445 : store(offsetof(wasm::MemoryTracingInfo, is_store),
3446 : MachineRepresentation::kWord8,
3447 100 : mcgraph()->Int32Constant(is_store ? 1 : 0));
3448 : store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3449 : MachineRepresentation::kWord8,
3450 100 : mcgraph()->Int32Constant(static_cast<int>(rep)));
3451 :
3452 50 : Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3453 50 : SetSourcePosition(call, position);
3454 50 : return call;
3455 : }
3456 :
3457 198560 : Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3458 : Node* index, uint32_t offset,
3459 : uint32_t alignment,
3460 1190800 : wasm::WasmCodePosition position) {
3461 : Node* load;
3462 :
3463 : // Wasm semantics throw on OOB. Introduce explicit bounds check and
3464 : // conditioning when not using the trap handler.
3465 : index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3466 198562 : position, kCanOmitBoundsCheck);
3467 :
3468 395743 : if (memtype.representation() == MachineRepresentation::kWord8 ||
3469 197179 : mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3470 198564 : if (use_trap_handler()) {
3471 : load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3472 397123 : MemBuffer(offset), index, Effect(), Control());
3473 198563 : SetSourcePosition(load, position);
3474 : } else {
3475 : load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3476 2 : MemBuffer(offset), index, Effect(), Control());
3477 : }
3478 : } else {
3479 : // TODO(eholk): Support unaligned loads with trap handlers.
3480 : DCHECK(!use_trap_handler());
3481 : load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3482 0 : MemBuffer(offset), index, Effect(), Control());
3483 : }
3484 :
3485 : SetEffect(load);
3486 :
3487 : #if defined(V8_TARGET_BIG_ENDIAN)
3488 : load = BuildChangeEndiannessLoad(load, memtype, type);
3489 : #endif
3490 :
3491 288957 : if (type == wasm::kWasmI64 &&
3492 : ElementSizeInBytes(memtype.representation()) < 8) {
3493 : // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3494 807 : if (memtype.IsSigned()) {
3495 : // sign extend
3496 429 : load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3497 : } else {
3498 : // zero extend
3499 : load =
3500 378 : graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3501 : }
3502 : }
3503 :
3504 198561 : if (FLAG_trace_wasm_memory) {
3505 : TraceMemoryOperation(false, memtype.representation(), index, offset,
3506 30 : position);
3507 : }
3508 :
3509 198561 : return load;
3510 : }
3511 :
3512 257925 : Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3513 : uint32_t offset, uint32_t alignment, Node* val,
3514 : wasm::WasmCodePosition position,
3515 1546405 : wasm::ValueType type) {
3516 : Node* store;
3517 :
3518 : index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3519 257930 : position, kCanOmitBoundsCheck);
3520 :
3521 : #if defined(V8_TARGET_BIG_ENDIAN)
3522 : val = BuildChangeEndiannessStore(val, mem_rep, type);
3523 : #endif
3524 :
3525 514689 : if (mem_rep == MachineRepresentation::kWord8 ||
3526 256767 : mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3527 257922 : if (use_trap_handler()) {
3528 : store =
3529 : graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3530 515837 : MemBuffer(offset), index, val, Effect(), Control());
3531 257958 : SetSourcePosition(store, position);
3532 : } else {
3533 : StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3534 : store =
3535 : graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3536 0 : index, val, Effect(), Control());
3537 : }
3538 : } else {
3539 : // TODO(eholk): Support unaligned stores with trap handlers.
3540 : DCHECK(!use_trap_handler());
3541 : UnalignedStoreRepresentation rep(mem_rep);
3542 : store =
3543 : graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3544 0 : MemBuffer(offset), index, val, Effect(), Control());
3545 : }
3546 :
3547 : SetEffect(store);
3548 :
3549 257957 : if (FLAG_trace_wasm_memory) {
3550 20 : TraceMemoryOperation(true, mem_rep, index, offset, position);
3551 : }
3552 :
3553 257957 : return store;
3554 : }
3555 :
3556 : namespace {
3557 66141 : Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3558 66141 : switch (rep) {
3559 : case MachineRepresentation::kWord8:
3560 : case MachineRepresentation::kWord16:
3561 : case MachineRepresentation::kWord32:
3562 58742 : return mcgraph->Int32Constant(0);
3563 : case MachineRepresentation::kWord64:
3564 0 : return mcgraph->Int64Constant(0);
3565 : case MachineRepresentation::kFloat32:
3566 6482 : return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3567 : case MachineRepresentation::kFloat64:
3568 917 : return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3569 : default:
3570 0 : UNREACHABLE();
3571 : }
3572 : }
3573 : } // namespace
3574 :
3575 661410 : Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3576 : DCHECK_NOT_NULL(instance_cache_);
3577 66141 : Node* mem_start = instance_cache_->mem_start;
3578 66141 : Node* mem_size = instance_cache_->mem_size;
3579 : DCHECK_NOT_NULL(mem_start);
3580 : DCHECK_NOT_NULL(mem_size);
3581 :
3582 : // Asm.js semantics are defined in terms of typed arrays, hence OOB
3583 : // reads return {undefined} coerced to the result type (0 for integers, NaN
3584 : // for float and double).
3585 : // Note that we check against the memory size ignoring the size of the
3586 : // stored value, which is conservative if misaligned. Technically, asm.js
3587 : // should never have misaligned accesses.
3588 66141 : index = Uint32ToUintptr(index);
3589 : Diamond bounds_check(
3590 : graph(), mcgraph()->common(),
3591 : graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
3592 198423 : BranchHint::kTrue);
3593 66141 : bounds_check.Chain(Control());
3594 :
3595 66141 : if (untrusted_code_mitigations_) {
3596 : // Condition the index with the memory mask.
3597 0 : Node* mem_mask = instance_cache_->mem_mask;
3598 : DCHECK_NOT_NULL(mem_mask);
3599 0 : index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
3600 : }
3601 :
3602 : Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3603 132282 : index, Effect(), bounds_check.if_true);
3604 66141 : SetEffect(bounds_check.EffectPhi(load, Effect()));
3605 66141 : SetControl(bounds_check.merge);
3606 : return bounds_check.Phi(type.representation(), load,
3607 66141 : GetAsmJsOOBValue(type.representation(), mcgraph()));
3608 : }
3609 :
3610 609428 : Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3611 1218856 : if (mcgraph()->machine()->Is32()) return node;
3612 : // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3613 : Uint32Matcher matcher(node);
3614 609443 : if (matcher.HasValue()) {
3615 476724 : uintptr_t value = matcher.Value();
3616 476724 : return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3617 : }
3618 265409 : return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3619 : }
3620 :
3621 47018 : Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3622 376144 : Node* val) {
3623 : DCHECK_NOT_NULL(instance_cache_);
3624 47018 : Node* mem_start = instance_cache_->mem_start;
3625 47018 : Node* mem_size = instance_cache_->mem_size;
3626 : DCHECK_NOT_NULL(mem_start);
3627 : DCHECK_NOT_NULL(mem_size);
3628 :
3629 : // Asm.js semantics are to ignore OOB writes.
3630 : // Note that we check against the memory size ignoring the size of the
3631 : // stored value, which is conservative if misaligned. Technically, asm.js
3632 : // should never have misaligned accesses.
3633 : Diamond bounds_check(
3634 : graph(), mcgraph()->common(),
3635 : graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3636 141054 : BranchHint::kTrue);
3637 47018 : bounds_check.Chain(Control());
3638 :
3639 47018 : if (untrusted_code_mitigations_) {
3640 : // Condition the index with the memory mask.
3641 0 : Node* mem_mask = instance_cache_->mem_mask;
3642 : DCHECK_NOT_NULL(mem_mask);
3643 : index =
3644 0 : graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3645 : }
3646 :
3647 47018 : index = Uint32ToUintptr(index);
3648 : const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3649 94036 : type.representation(), WriteBarrierKind::kNoWriteBarrier));
3650 : Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
3651 47018 : bounds_check.if_true);
3652 47018 : SetEffect(bounds_check.EffectPhi(store, Effect()));
3653 47018 : SetControl(bounds_check.merge);
3654 47018 : return val;
3655 : }
3656 :
3657 0 : void WasmGraphBuilder::PrintDebugName(Node* node) {
3658 0 : PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3659 0 : }
3660 :
3661 18118483 : Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3662 :
3663 : namespace {
3664 5085 : Signature<MachineRepresentation>* CreateMachineSignature(
3665 10170 : Zone* zone, wasm::FunctionSig* sig) {
3666 : Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3667 : sig->parameter_count());
3668 10170 : for (auto ret : sig->returns()) {
3669 5085 : builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3670 : }
3671 :
3672 8080 : for (auto param : sig->parameters()) {
3673 2995 : builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3674 : }
3675 5085 : return builder.Build();
3676 : }
3677 : } // namespace
3678 :
3679 1060818 : void WasmGraphBuilder::LowerInt64() {
3680 3182454 : if (mcgraph()->machine()->Is64()) return;
3681 : Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3682 : mcgraph()->zone(),
3683 0 : CreateMachineSignature(mcgraph()->zone(), sig_));
3684 0 : r.LowerGraph();
3685 : }
3686 :
3687 0 : void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3688 : SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3689 0 : .LowerGraph();
3690 0 : }
3691 :
3692 667912 : void WasmGraphBuilder::SetSourcePosition(Node* node,
3693 : wasm::WasmCodePosition position) {
3694 : DCHECK_NE(position, wasm::kNoCodePosition);
3695 667912 : if (source_position_table_)
3696 566761 : source_position_table_->SetSourcePosition(node, SourcePosition(position));
3697 667957 : }
3698 :
3699 7986 : Node* WasmGraphBuilder::S128Zero() {
3700 3993 : has_simd_ = true;
3701 7986 : return graph()->NewNode(mcgraph()->machine()->S128Zero());
3702 : }
3703 :
3704 15370 : Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3705 7685 : has_simd_ = true;
3706 7685 : switch (opcode) {
3707 : case wasm::kExprF32x4Splat:
3708 1575 : return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3709 : case wasm::kExprF32x4SConvertI32x4:
3710 : return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3711 30 : inputs[0]);
3712 : case wasm::kExprF32x4UConvertI32x4:
3713 : return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3714 30 : inputs[0]);
3715 : case wasm::kExprF32x4Abs:
3716 45 : return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3717 : case wasm::kExprF32x4Neg:
3718 45 : return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3719 : case wasm::kExprF32x4RecipApprox:
3720 : return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3721 45 : inputs[0]);
3722 : case wasm::kExprF32x4RecipSqrtApprox:
3723 : return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3724 45 : inputs[0]);
3725 : case wasm::kExprF32x4Add:
3726 : return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3727 135 : inputs[1]);
3728 : case wasm::kExprF32x4AddHoriz:
3729 : return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3730 45 : inputs[1]);
3731 : case wasm::kExprF32x4Sub:
3732 : return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3733 45 : inputs[1]);
3734 : case wasm::kExprF32x4Mul:
3735 : return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3736 45 : inputs[1]);
3737 : case wasm::kExprF32x4Min:
3738 : return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3739 45 : inputs[1]);
3740 : case wasm::kExprF32x4Max:
3741 : return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3742 45 : inputs[1]);
3743 : case wasm::kExprF32x4Eq:
3744 : return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3745 45 : inputs[1]);
3746 : case wasm::kExprF32x4Ne:
3747 : return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3748 45 : inputs[1]);
3749 : case wasm::kExprF32x4Lt:
3750 : return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3751 45 : inputs[1]);
3752 : case wasm::kExprF32x4Le:
3753 : return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3754 45 : inputs[1]);
3755 : case wasm::kExprF32x4Gt:
3756 : return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3757 45 : inputs[0]);
3758 : case wasm::kExprF32x4Ge:
3759 : return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3760 45 : inputs[0]);
3761 : case wasm::kExprI32x4Splat:
3762 6900 : return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3763 : case wasm::kExprI32x4SConvertF32x4:
3764 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3765 45 : inputs[0]);
3766 : case wasm::kExprI32x4UConvertF32x4:
3767 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3768 45 : inputs[0]);
3769 : case wasm::kExprI32x4SConvertI16x8Low:
3770 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3771 45 : inputs[0]);
3772 : case wasm::kExprI32x4SConvertI16x8High:
3773 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3774 45 : inputs[0]);
3775 : case wasm::kExprI32x4Neg:
3776 45 : return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3777 : case wasm::kExprI32x4Add:
3778 : return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3779 135 : inputs[1]);
3780 : case wasm::kExprI32x4AddHoriz:
3781 : return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3782 45 : inputs[1]);
3783 : case wasm::kExprI32x4Sub:
3784 : return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3785 45 : inputs[1]);
3786 : case wasm::kExprI32x4Mul:
3787 : return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3788 45 : inputs[1]);
3789 : case wasm::kExprI32x4MinS:
3790 : return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3791 45 : inputs[1]);
3792 : case wasm::kExprI32x4MaxS:
3793 : return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3794 45 : inputs[1]);
3795 : case wasm::kExprI32x4Eq:
3796 : return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3797 225 : inputs[1]);
3798 : case wasm::kExprI32x4Ne:
3799 : return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3800 240 : inputs[1]);
3801 : case wasm::kExprI32x4LtS:
3802 : return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3803 45 : inputs[0]);
3804 : case wasm::kExprI32x4LeS:
3805 : return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3806 45 : inputs[0]);
3807 : case wasm::kExprI32x4GtS:
3808 : return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3809 45 : inputs[1]);
3810 : case wasm::kExprI32x4GeS:
3811 : return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3812 45 : inputs[1]);
3813 : case wasm::kExprI32x4UConvertI16x8Low:
3814 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3815 45 : inputs[0]);
3816 : case wasm::kExprI32x4UConvertI16x8High:
3817 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3818 45 : inputs[0]);
3819 : case wasm::kExprI32x4MinU:
3820 : return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3821 45 : inputs[1]);
3822 : case wasm::kExprI32x4MaxU:
3823 : return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3824 45 : inputs[1]);
3825 : case wasm::kExprI32x4LtU:
3826 : return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3827 45 : inputs[0]);
3828 : case wasm::kExprI32x4LeU:
3829 : return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3830 45 : inputs[0]);
3831 : case wasm::kExprI32x4GtU:
3832 : return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3833 45 : inputs[1]);
3834 : case wasm::kExprI32x4GeU:
3835 : return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3836 45 : inputs[1]);
3837 : case wasm::kExprI16x8Splat:
3838 4350 : return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3839 : case wasm::kExprI16x8SConvertI8x16Low:
3840 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3841 45 : inputs[0]);
3842 : case wasm::kExprI16x8SConvertI8x16High:
3843 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3844 45 : inputs[0]);
3845 : case wasm::kExprI16x8Neg:
3846 45 : return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3847 : case wasm::kExprI16x8SConvertI32x4:
3848 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3849 45 : inputs[0], inputs[1]);
3850 : case wasm::kExprI16x8Add:
3851 : return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3852 45 : inputs[1]);
3853 : case wasm::kExprI16x8AddSaturateS:
3854 : return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3855 45 : inputs[0], inputs[1]);
3856 : case wasm::kExprI16x8AddHoriz:
3857 : return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3858 45 : inputs[1]);
3859 : case wasm::kExprI16x8Sub:
3860 : return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3861 45 : inputs[1]);
3862 : case wasm::kExprI16x8SubSaturateS:
3863 : return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3864 45 : inputs[0], inputs[1]);
3865 : case wasm::kExprI16x8Mul:
3866 : return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3867 45 : inputs[1]);
3868 : case wasm::kExprI16x8MinS:
3869 : return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3870 45 : inputs[1]);
3871 : case wasm::kExprI16x8MaxS:
3872 : return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3873 45 : inputs[1]);
3874 : case wasm::kExprI16x8Eq:
3875 : return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3876 225 : inputs[1]);
3877 : case wasm::kExprI16x8Ne:
3878 : return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3879 240 : inputs[1]);
3880 : case wasm::kExprI16x8LtS:
3881 : return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3882 45 : inputs[0]);
3883 : case wasm::kExprI16x8LeS:
3884 : return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3885 45 : inputs[0]);
3886 : case wasm::kExprI16x8GtS:
3887 : return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3888 45 : inputs[1]);
3889 : case wasm::kExprI16x8GeS:
3890 : return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3891 45 : inputs[1]);
3892 : case wasm::kExprI16x8UConvertI8x16Low:
3893 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3894 45 : inputs[0]);
3895 : case wasm::kExprI16x8UConvertI8x16High:
3896 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3897 45 : inputs[0]);
3898 : case wasm::kExprI16x8UConvertI32x4:
3899 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3900 45 : inputs[0], inputs[1]);
3901 : case wasm::kExprI16x8AddSaturateU:
3902 : return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3903 45 : inputs[0], inputs[1]);
3904 : case wasm::kExprI16x8SubSaturateU:
3905 : return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3906 45 : inputs[0], inputs[1]);
3907 : case wasm::kExprI16x8MinU:
3908 : return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3909 45 : inputs[1]);
3910 : case wasm::kExprI16x8MaxU:
3911 : return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3912 45 : inputs[1]);
3913 : case wasm::kExprI16x8LtU:
3914 : return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3915 45 : inputs[0]);
3916 : case wasm::kExprI16x8LeU:
3917 : return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3918 45 : inputs[0]);
3919 : case wasm::kExprI16x8GtU:
3920 : return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3921 45 : inputs[1]);
3922 : case wasm::kExprI16x8GeU:
3923 : return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3924 45 : inputs[1]);
3925 : case wasm::kExprI8x16Splat:
3926 3180 : return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3927 : case wasm::kExprI8x16Neg:
3928 45 : return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3929 : case wasm::kExprI8x16SConvertI16x8:
3930 : return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3931 45 : inputs[0], inputs[1]);
3932 : case wasm::kExprI8x16Add:
3933 : return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3934 45 : inputs[1]);
3935 : case wasm::kExprI8x16AddSaturateS:
3936 : return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3937 45 : inputs[0], inputs[1]);
3938 : case wasm::kExprI8x16Sub:
3939 : return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3940 45 : inputs[1]);
3941 : case wasm::kExprI8x16SubSaturateS:
3942 : return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3943 45 : inputs[0], inputs[1]);
3944 : case wasm::kExprI8x16Mul:
3945 : return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3946 45 : inputs[1]);
3947 : case wasm::kExprI8x16MinS:
3948 : return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3949 45 : inputs[1]);
3950 : case wasm::kExprI8x16MaxS:
3951 : return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3952 45 : inputs[1]);
3953 : case wasm::kExprI8x16Eq:
3954 : return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3955 225 : inputs[1]);
3956 : case wasm::kExprI8x16Ne:
3957 : return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3958 240 : inputs[1]);
3959 : case wasm::kExprI8x16LtS:
3960 : return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3961 45 : inputs[0]);
3962 : case wasm::kExprI8x16LeS:
3963 : return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3964 45 : inputs[0]);
3965 : case wasm::kExprI8x16GtS:
3966 : return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3967 45 : inputs[1]);
3968 : case wasm::kExprI8x16GeS:
3969 : return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3970 45 : inputs[1]);
3971 : case wasm::kExprI8x16UConvertI16x8:
3972 : return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3973 45 : inputs[0], inputs[1]);
3974 : case wasm::kExprI8x16AddSaturateU:
3975 : return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3976 45 : inputs[0], inputs[1]);
3977 : case wasm::kExprI8x16SubSaturateU:
3978 : return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3979 45 : inputs[0], inputs[1]);
3980 : case wasm::kExprI8x16MinU:
3981 : return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3982 45 : inputs[1]);
3983 : case wasm::kExprI8x16MaxU:
3984 : return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
3985 45 : inputs[1]);
3986 : case wasm::kExprI8x16LtU:
3987 : return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
3988 45 : inputs[0]);
3989 : case wasm::kExprI8x16LeU:
3990 : return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
3991 45 : inputs[0]);
3992 : case wasm::kExprI8x16GtU:
3993 : return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
3994 45 : inputs[1]);
3995 : case wasm::kExprI8x16GeU:
3996 : return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
3997 45 : inputs[1]);
3998 : case wasm::kExprS128And:
3999 : return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
4000 45 : inputs[1]);
4001 : case wasm::kExprS128Or:
4002 : return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
4003 45 : inputs[1]);
4004 : case wasm::kExprS128Xor:
4005 : return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
4006 45 : inputs[1]);
4007 : case wasm::kExprS128Not:
4008 45 : return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
4009 : case wasm::kExprS128Select:
4010 : return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[2],
4011 105 : inputs[0], inputs[1]);
4012 : case wasm::kExprS1x4AnyTrue:
4013 195 : return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
4014 : case wasm::kExprS1x4AllTrue:
4015 195 : return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
4016 : case wasm::kExprS1x8AnyTrue:
4017 195 : return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
4018 : case wasm::kExprS1x8AllTrue:
4019 195 : return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
4020 : case wasm::kExprS1x16AnyTrue:
4021 195 : return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
4022 : case wasm::kExprS1x16AllTrue:
4023 195 : return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
4024 : default:
4025 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4026 : }
4027 : }
4028 :
4029 34880 : Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
4030 34880 : Node* const* inputs) {
4031 34880 : has_simd_ = true;
4032 34880 : switch (opcode) {
4033 : case wasm::kExprF32x4ExtractLane:
4034 : return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
4035 3840 : inputs[0]);
4036 : case wasm::kExprF32x4ReplaceLane:
4037 : return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
4038 360 : inputs[0], inputs[1]);
4039 : case wasm::kExprI32x4ExtractLane:
4040 : return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
4041 24510 : inputs[0]);
4042 : case wasm::kExprI32x4ReplaceLane:
4043 : return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
4044 510 : inputs[0], inputs[1]);
4045 : case wasm::kExprI16x8ExtractLane:
4046 : return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
4047 29640 : inputs[0]);
4048 : case wasm::kExprI16x8ReplaceLane:
4049 : return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
4050 510 : inputs[0], inputs[1]);
4051 : case wasm::kExprI8x16ExtractLane:
4052 : return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
4053 44400 : inputs[0]);
4054 : case wasm::kExprI8x16ReplaceLane:
4055 : return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
4056 870 : inputs[0], inputs[1]);
4057 : default:
4058 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4059 : }
4060 : }
4061 :
4062 2385 : Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
4063 2385 : Node* const* inputs) {
4064 2385 : has_simd_ = true;
4065 2385 : switch (opcode) {
4066 : case wasm::kExprI32x4Shl:
4067 1395 : return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
4068 : case wasm::kExprI32x4ShrS:
4069 : return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
4070 1395 : inputs[0]);
4071 : case wasm::kExprI32x4ShrU:
4072 : return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
4073 1395 : inputs[0]);
4074 : case wasm::kExprI16x8Shl:
4075 675 : return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
4076 : case wasm::kExprI16x8ShrS:
4077 : return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
4078 675 : inputs[0]);
4079 : case wasm::kExprI16x8ShrU:
4080 : return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
4081 675 : inputs[0]);
4082 : case wasm::kExprI8x16Shl:
4083 315 : return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
4084 : case wasm::kExprI8x16ShrS:
4085 : return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
4086 315 : inputs[0]);
4087 : case wasm::kExprI8x16ShrU:
4088 : return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
4089 315 : inputs[0]);
4090 : default:
4091 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4092 : }
4093 : }
4094 :
4095 33780 : Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
4096 33780 : Node* const* inputs) {
4097 33780 : has_simd_ = true;
4098 : return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
4099 101340 : inputs[0], inputs[1]);
4100 : }
4101 :
4102 : #define ATOMIC_BINOP_LIST(V) \
4103 : V(I32AtomicAdd, Add, Uint32, Word32) \
4104 : V(I64AtomicAdd, Add, Uint64, Word64) \
4105 : V(I32AtomicAdd8U, Add, Uint8, Word32) \
4106 : V(I32AtomicAdd16U, Add, Uint16, Word32) \
4107 : V(I64AtomicAdd8U, Add, Uint8, Word64) \
4108 : V(I64AtomicAdd16U, Add, Uint16, Word64) \
4109 : V(I64AtomicAdd32U, Add, Uint32, Word64) \
4110 : V(I32AtomicSub, Sub, Uint32, Word32) \
4111 : V(I64AtomicSub, Sub, Uint64, Word64) \
4112 : V(I32AtomicSub8U, Sub, Uint8, Word32) \
4113 : V(I32AtomicSub16U, Sub, Uint16, Word32) \
4114 : V(I64AtomicSub8U, Sub, Uint8, Word64) \
4115 : V(I64AtomicSub16U, Sub, Uint16, Word64) \
4116 : V(I64AtomicSub32U, Sub, Uint32, Word64) \
4117 : V(I32AtomicAnd, And, Uint32, Word32) \
4118 : V(I64AtomicAnd, And, Uint64, Word64) \
4119 : V(I32AtomicAnd8U, And, Uint8, Word32) \
4120 : V(I64AtomicAnd16U, And, Uint16, Word64) \
4121 : V(I32AtomicAnd16U, And, Uint16, Word32) \
4122 : V(I64AtomicAnd8U, And, Uint8, Word64) \
4123 : V(I64AtomicAnd32U, And, Uint32, Word64) \
4124 : V(I32AtomicOr, Or, Uint32, Word32) \
4125 : V(I64AtomicOr, Or, Uint64, Word64) \
4126 : V(I32AtomicOr8U, Or, Uint8, Word32) \
4127 : V(I32AtomicOr16U, Or, Uint16, Word32) \
4128 : V(I64AtomicOr8U, Or, Uint8, Word64) \
4129 : V(I64AtomicOr16U, Or, Uint16, Word64) \
4130 : V(I64AtomicOr32U, Or, Uint32, Word64) \
4131 : V(I32AtomicXor, Xor, Uint32, Word32) \
4132 : V(I64AtomicXor, Xor, Uint64, Word64) \
4133 : V(I32AtomicXor8U, Xor, Uint8, Word32) \
4134 : V(I32AtomicXor16U, Xor, Uint16, Word32) \
4135 : V(I64AtomicXor8U, Xor, Uint8, Word64) \
4136 : V(I64AtomicXor16U, Xor, Uint16, Word64) \
4137 : V(I64AtomicXor32U, Xor, Uint32, Word64) \
4138 : V(I32AtomicExchange, Exchange, Uint32, Word32) \
4139 : V(I64AtomicExchange, Exchange, Uint64, Word64) \
4140 : V(I32AtomicExchange8U, Exchange, Uint8, Word32) \
4141 : V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
4142 : V(I64AtomicExchange8U, Exchange, Uint8, Word64) \
4143 : V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
4144 : V(I64AtomicExchange32U, Exchange, Uint32, Word64)
4145 :
4146 : #define ATOMIC_CMP_EXCHG_LIST(V) \
4147 : V(I32AtomicCompareExchange, Uint32, Word32) \
4148 : V(I64AtomicCompareExchange, Uint64, Word64) \
4149 : V(I32AtomicCompareExchange8U, Uint8, Word32) \
4150 : V(I32AtomicCompareExchange16U, Uint16, Word32) \
4151 : V(I64AtomicCompareExchange8U, Uint8, Word64) \
4152 : V(I64AtomicCompareExchange16U, Uint16, Word64) \
4153 : V(I64AtomicCompareExchange32U, Uint32, Word64)
4154 :
4155 : #define ATOMIC_LOAD_LIST(V) \
4156 : V(I32AtomicLoad, Uint32, Word32) \
4157 : V(I64AtomicLoad, Uint64, Word64) \
4158 : V(I32AtomicLoad8U, Uint8, Word32) \
4159 : V(I32AtomicLoad16U, Uint16, Word32) \
4160 : V(I64AtomicLoad8U, Uint8, Word64) \
4161 : V(I64AtomicLoad16U, Uint16, Word64) \
4162 : V(I64AtomicLoad32U, Uint32, Word64)
4163 :
4164 : #define ATOMIC_STORE_LIST(V) \
4165 : V(I32AtomicStore, Uint32, kWord32, Word32) \
4166 : V(I64AtomicStore, Uint64, kWord64, Word64) \
4167 : V(I32AtomicStore8U, Uint8, kWord8, Word32) \
4168 : V(I32AtomicStore16U, Uint16, kWord16, Word32) \
4169 : V(I64AtomicStore8U, Uint8, kWord8, Word64) \
4170 : V(I64AtomicStore16U, Uint16, kWord16, Word64) \
4171 : V(I64AtomicStore32U, Uint32, kWord32, Word64)
4172 :
4173 33583 : Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
4174 : uint32_t alignment, uint32_t offset,
4175 138106 : wasm::WasmCodePosition position) {
4176 : Node* node;
4177 33583 : switch (opcode) {
4178 : #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \
4179 : case wasm::kExpr##Name: { \
4180 : Node* index = CheckBoundsAndAlignment( \
4181 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4182 : position); \
4183 : node = graph()->NewNode( \
4184 : mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
4185 : MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4186 : break; \
4187 : }
4188 96144 : ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
4189 : #undef BUILD_ATOMIC_BINOP
4190 :
4191 : #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \
4192 : case wasm::kExpr##Name: { \
4193 : Node* index = CheckBoundsAndAlignment( \
4194 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4195 : position); \
4196 : node = graph()->NewNode( \
4197 : mcgraph()->machine()->Prefix##AtomicCompareExchange( \
4198 : MachineType::Type()), \
4199 : MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
4200 : break; \
4201 : }
4202 890 : ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
4203 : #undef BUILD_ATOMIC_CMP_EXCHG
4204 :
4205 : #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \
4206 : case wasm::kExpr##Name: { \
4207 : Node* index = CheckBoundsAndAlignment( \
4208 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4209 : position); \
4210 : node = graph()->NewNode( \
4211 : mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \
4212 : MemBuffer(offset), index, Effect(), Control()); \
4213 : break; \
4214 : }
4215 18080 : ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
4216 : #undef BUILD_ATOMIC_LOAD_OP
4217 :
4218 : #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \
4219 : case wasm::kExpr##Name: { \
4220 : Node* index = CheckBoundsAndAlignment( \
4221 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4222 : position); \
4223 : node = graph()->NewNode( \
4224 : mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
4225 : MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4226 : break; \
4227 : }
4228 17771 : ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
4229 : #undef BUILD_ATOMIC_STORE_OP
4230 : case wasm::kExprAtomicWake: {
4231 : Node* index = CheckBoundsAndAlignment(
4232 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4233 360 : position);
4234 : // Now that we've bounds-checked, compute the effective address.
4235 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4236 180 : Uint32Constant(offset), index);
4237 : WasmAtomicWakeDescriptor interface_descriptor;
4238 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4239 : mcgraph()->zone(), interface_descriptor,
4240 : interface_descriptor.GetStackParameterCount(),
4241 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4242 180 : StubCallMode::kCallWasmRuntimeStub);
4243 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4244 180 : wasm::WasmCode::kWasmAtomicWake, RelocInfo::WASM_STUB_CALL);
4245 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4246 : call_target, address, inputs[1], Effect(),
4247 360 : Control());
4248 : break;
4249 : }
4250 :
4251 : case wasm::kExprI32AtomicWait: {
4252 : Node* index = CheckBoundsAndAlignment(
4253 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4254 563 : position);
4255 : // Now that we've bounds-checked, compute the effective address.
4256 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4257 288 : Uint32Constant(offset), index);
4258 : Node* timeout;
4259 574 : if (mcgraph()->machine()->Is32()) {
4260 0 : timeout = BuildF64SConvertI64(inputs[2]);
4261 : } else {
4262 : timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4263 287 : inputs[2]);
4264 : }
4265 : WasmI32AtomicWaitDescriptor interface_descriptor;
4266 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4267 : mcgraph()->zone(), interface_descriptor,
4268 : interface_descriptor.GetStackParameterCount(),
4269 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4270 288 : StubCallMode::kCallWasmRuntimeStub);
4271 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4272 286 : wasm::WasmCode::kWasmI32AtomicWait, RelocInfo::WASM_STUB_CALL);
4273 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4274 : call_target, address, inputs[1], timeout,
4275 576 : Effect(), Control());
4276 : break;
4277 : }
4278 :
4279 : case wasm::kExprI64AtomicWait: {
4280 : Node* index = CheckBoundsAndAlignment(
4281 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4282 561 : position);
4283 : // Now that we've bounds-checked, compute the effective address.
4284 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4285 288 : Uint32Constant(offset), index);
4286 : Node* timeout;
4287 576 : if (mcgraph()->machine()->Is32()) {
4288 0 : timeout = BuildF64SConvertI64(inputs[2]);
4289 : } else {
4290 : timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4291 288 : inputs[2]);
4292 : }
4293 : Node* expected_value_low = graph()->NewNode(
4294 570 : mcgraph()->machine()->TruncateInt64ToInt32(), inputs[1]);
4295 : Node* tmp = graph()->NewNode(mcgraph()->machine()->Word64Shr(), inputs[1],
4296 574 : Int64Constant(32));
4297 : Node* expected_value_high =
4298 288 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), tmp);
4299 : WasmI64AtomicWaitDescriptor interface_descriptor;
4300 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4301 : mcgraph()->zone(), interface_descriptor,
4302 : interface_descriptor.GetStackParameterCount(),
4303 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4304 288 : StubCallMode::kCallWasmRuntimeStub);
4305 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4306 285 : wasm::WasmCode::kWasmI64AtomicWait, RelocInfo::WASM_STUB_CALL);
4307 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4308 : call_target, address, expected_value_high,
4309 288 : expected_value_low, timeout, Effect(), Control());
4310 : break;
4311 : }
4312 :
4313 : default:
4314 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4315 : }
4316 33602 : return SetEffect(node);
4317 : }
4318 :
4319 : #undef ATOMIC_BINOP_LIST
4320 : #undef ATOMIC_CMP_EXCHG_LIST
4321 : #undef ATOMIC_LOAD_LIST
4322 : #undef ATOMIC_STORE_LIST
4323 :
4324 72 : Node* WasmGraphBuilder::CheckDataSegmentIsPassiveAndNotDropped(
4325 720 : uint32_t data_segment_index, wasm::WasmCodePosition position) {
4326 : // The data segment index must be in bounds since it is required by
4327 : // validation.
4328 : DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments);
4329 :
4330 : Node* dropped_data_segments =
4331 144 : LOAD_INSTANCE_FIELD(DroppedDataSegments, MachineType::Pointer());
4332 : Node* is_segment_dropped = SetEffect(graph()->NewNode(
4333 : mcgraph()->machine()->Load(MachineType::Uint8()), dropped_data_segments,
4334 216 : mcgraph()->IntPtrConstant(data_segment_index), Effect(), Control()));
4335 72 : TrapIfTrue(wasm::kTrapDataSegmentDropped, is_segment_dropped, position);
4336 72 : return dropped_data_segments;
4337 : }
4338 :
4339 45 : Node* WasmGraphBuilder::MemoryInit(uint32_t data_segment_index, Node* dst,
4340 : Node* src, Node* size,
4341 810 : wasm::WasmCodePosition position) {
4342 45 : CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
4343 45 : dst = BoundsCheckMemRange(dst, size, position);
4344 45 : MachineOperatorBuilder* m = mcgraph()->machine();
4345 :
4346 : Node* seg_index = Uint32Constant(data_segment_index);
4347 :
4348 : {
4349 : // Load segment size from WasmInstanceObject::data_segment_sizes.
4350 : Node* seg_size_array =
4351 90 : LOAD_INSTANCE_FIELD(DataSegmentSizes, MachineType::Pointer());
4352 : STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2);
4353 : Node* scaled_index = Uint32ToUintptr(
4354 90 : graph()->NewNode(m->Word32Shl(), seg_index, Int32Constant(2)));
4355 : Node* seg_size = SetEffect(graph()->NewNode(m->Load(MachineType::Uint32()),
4356 : seg_size_array, scaled_index,
4357 45 : Effect(), Control()));
4358 :
4359 : // Bounds check the src index against the segment size.
4360 45 : BoundsCheckRange(src, size, seg_size, position);
4361 : }
4362 :
4363 : {
4364 : // Load segment's base pointer from WasmInstanceObject::data_segment_starts.
4365 : Node* seg_start_array =
4366 90 : LOAD_INSTANCE_FIELD(DataSegmentStarts, MachineType::Pointer());
4367 : STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >>
4368 : kPointerSizeLog2);
4369 : Node* scaled_index = Uint32ToUintptr(graph()->NewNode(
4370 90 : m->Word32Shl(), seg_index, Int32Constant(kPointerSizeLog2)));
4371 : Node* seg_start = SetEffect(
4372 : graph()->NewNode(m->Load(MachineType::Pointer()), seg_start_array,
4373 45 : scaled_index, Effect(), Control()));
4374 :
4375 : // Convert src index to pointer.
4376 45 : src = graph()->NewNode(m->IntAdd(), seg_start, Uint32ToUintptr(src));
4377 : }
4378 :
4379 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4380 135 : ExternalReference::wasm_memory_copy()));
4381 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
4382 45 : MachineType::Uint32()};
4383 : MachineSignature sig(0, 3, sig_types);
4384 45 : return BuildCCall(&sig, function, dst, src, size);
4385 : }
4386 :
4387 27 : Node* WasmGraphBuilder::MemoryDrop(uint32_t data_segment_index,
4388 162 : wasm::WasmCodePosition position) {
4389 : Node* dropped_data_segments =
4390 27 : CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
4391 : const Operator* store_op = mcgraph()->machine()->Store(
4392 27 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
4393 : return SetEffect(
4394 : graph()->NewNode(store_op, dropped_data_segments,
4395 : mcgraph()->IntPtrConstant(data_segment_index),
4396 81 : mcgraph()->Int32Constant(1), Effect(), Control()));
4397 : }
4398 :
4399 27 : Node* WasmGraphBuilder::MemoryCopy(Node* dst, Node* src, Node* size,
4400 27 : wasm::WasmCodePosition position) {
4401 27 : dst = BoundsCheckMemRange(dst, size, position);
4402 27 : src = BoundsCheckMemRange(src, size, position);
4403 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4404 81 : ExternalReference::wasm_memory_copy()));
4405 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
4406 27 : MachineType::Uint32()};
4407 : MachineSignature sig(0, 3, sig_types);
4408 27 : return BuildCCall(&sig, function, dst, src, size);
4409 : }
4410 :
4411 27 : Node* WasmGraphBuilder::MemoryFill(Node* dst, Node* value, Node* size,
4412 27 : wasm::WasmCodePosition position) {
4413 27 : dst = BoundsCheckMemRange(dst, size, position);
4414 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4415 81 : ExternalReference::wasm_memory_fill()));
4416 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Uint32(),
4417 27 : MachineType::Uint32()};
4418 : MachineSignature sig(0, 3, sig_types);
4419 27 : return BuildCCall(&sig, function, dst, value, size);
4420 : }
4421 :
4422 18 : Node* WasmGraphBuilder::CheckElemSegmentIsPassiveAndNotDropped(
4423 180 : uint32_t elem_segment_index, wasm::WasmCodePosition position) {
4424 : // The elem segment index must be in bounds since it is required by
4425 : // validation.
4426 : DCHECK_LT(elem_segment_index, env_->module->elem_segments.size());
4427 :
4428 : Node* dropped_elem_segments =
4429 36 : LOAD_INSTANCE_FIELD(DroppedElemSegments, MachineType::Pointer());
4430 : Node* is_segment_dropped = SetEffect(graph()->NewNode(
4431 : mcgraph()->machine()->Load(MachineType::Uint8()), dropped_elem_segments,
4432 54 : mcgraph()->IntPtrConstant(elem_segment_index), Effect(), Control()));
4433 18 : TrapIfTrue(wasm::kTrapElemSegmentDropped, is_segment_dropped, position);
4434 18 : return dropped_elem_segments;
4435 : }
4436 :
4437 9 : Node* WasmGraphBuilder::TableInit(uint32_t table_index,
4438 : uint32_t elem_segment_index, Node* dst,
4439 : Node* src, Node* size,
4440 18 : wasm::WasmCodePosition position) {
4441 : Node* args[] = {
4442 18 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
4443 18 : graph()->NewNode(mcgraph()->common()->NumberConstant(elem_segment_index)),
4444 9 : BuildConvertUint32ToSmiWithSaturation(dst, wasm::kV8MaxWasmTableSize),
4445 9 : BuildConvertUint32ToSmiWithSaturation(src, wasm::kV8MaxWasmTableSize),
4446 36 : BuildConvertUint32ToSmiWithSaturation(size, wasm::kV8MaxWasmTableSize)};
4447 : Node* result =
4448 9 : BuildCallToRuntime(Runtime::kWasmTableInit, args, arraysize(args));
4449 :
4450 9 : return result;
4451 : }
4452 :
4453 18 : Node* WasmGraphBuilder::TableDrop(uint32_t elem_segment_index,
4454 108 : wasm::WasmCodePosition position) {
4455 : Node* dropped_elem_segments =
4456 18 : CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
4457 : const Operator* store_op = mcgraph()->machine()->Store(
4458 18 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
4459 : return SetEffect(
4460 : graph()->NewNode(store_op, dropped_elem_segments,
4461 : mcgraph()->IntPtrConstant(elem_segment_index),
4462 54 : mcgraph()->Int32Constant(1), Effect(), Control()));
4463 : }
4464 :
4465 72 : Node* WasmGraphBuilder::TableCopy(uint32_t table_index, Node* dst, Node* src,
4466 72 : Node* size, wasm::WasmCodePosition position) {
4467 : Node* args[] = {
4468 144 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
4469 72 : BuildConvertUint32ToSmiWithSaturation(dst, wasm::kV8MaxWasmTableSize),
4470 72 : BuildConvertUint32ToSmiWithSaturation(src, wasm::kV8MaxWasmTableSize),
4471 216 : BuildConvertUint32ToSmiWithSaturation(size, wasm::kV8MaxWasmTableSize)};
4472 : Node* result =
4473 72 : BuildCallToRuntime(Runtime::kWasmTableCopy, args, arraysize(args));
4474 :
4475 72 : return result;
4476 : }
4477 :
4478 0 : class WasmDecorator final : public GraphDecorator {
4479 : public:
4480 : explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
4481 0 : : origins_(origins), decoder_(decoder) {}
4482 :
4483 0 : void Decorate(Node* node) final {
4484 : origins_->SetNodeOrigin(
4485 : node, NodeOrigin("wasm graph creation", "n/a",
4486 0 : NodeOrigin::kWasmBytecode, decoder_->position()));
4487 0 : }
4488 :
4489 : private:
4490 : compiler::NodeOriginTable* origins_;
4491 : wasm::Decoder* decoder_;
4492 : };
4493 :
4494 0 : void WasmGraphBuilder::AddBytecodePositionDecorator(
4495 : NodeOriginTable* node_origins, wasm::Decoder* decoder) {
4496 : DCHECK_NULL(decorator_);
4497 0 : decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
4498 0 : graph()->AddDecorator(decorator_);
4499 0 : }
4500 :
4501 0 : void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
4502 : DCHECK_NOT_NULL(decorator_);
4503 0 : graph()->RemoveDecorator(decorator_);
4504 0 : decorator_ = nullptr;
4505 0 : }
4506 :
4507 : namespace {
4508 313944 : bool must_record_function_compilation(Isolate* isolate) {
4509 313944 : return isolate->logger()->is_listening_to_code_events() ||
4510 : isolate->is_profiling();
4511 : }
4512 :
4513 : PRINTF_FORMAT(4, 5)
4514 0 : void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
4515 : Isolate* isolate, Handle<Code> code,
4516 : const char* format, ...) {
4517 : DCHECK(must_record_function_compilation(isolate));
4518 :
4519 : ScopedVector<char> buffer(128);
4520 : va_list arguments;
4521 0 : va_start(arguments, format);
4522 0 : int len = VSNPrintF(buffer, format, arguments);
4523 0 : CHECK_LT(0, len);
4524 0 : va_end(arguments);
4525 : Handle<String> name_str =
4526 0 : isolate->factory()->NewStringFromAsciiChecked(buffer.start());
4527 0 : PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
4528 0 : }
4529 :
4530 : class WasmWrapperGraphBuilder : public WasmGraphBuilder {
4531 : public:
4532 166068 : WasmWrapperGraphBuilder(Zone* zone, JSGraph* jsgraph, wasm::FunctionSig* sig,
4533 : compiler::SourcePositionTable* spt,
4534 : StubCallMode stub_mode, wasm::WasmFeatures features)
4535 : : WasmGraphBuilder(nullptr, zone, jsgraph, sig, spt),
4536 : isolate_(jsgraph->isolate()),
4537 : jsgraph_(jsgraph),
4538 : stub_mode_(stub_mode),
4539 332136 : enabled_features_(features) {}
4540 :
4541 55446 : Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
4542 152474 : MachineOperatorBuilder* machine = mcgraph()->machine();
4543 46422 : CommonOperatorBuilder* common = mcgraph()->common();
4544 46422 : Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4545 : ? mcgraph()->RelocatableIntPtrConstant(
4546 : wasm::WasmCode::kWasmAllocateHeapNumber,
4547 : RelocInfo::WASM_STUB_CALL)
4548 : : jsgraph()->HeapConstant(
4549 55446 : BUILTIN_CODE(isolate_, AllocateHeapNumber));
4550 46422 : if (!allocate_heap_number_operator_.is_set()) {
4551 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4552 : mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
4553 39624 : CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
4554 13208 : allocate_heap_number_operator_.set(common->Call(call_descriptor));
4555 : }
4556 : Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
4557 : target, Effect(), control);
4558 : SetEffect(
4559 : graph()->NewNode(machine->Store(StoreRepresentation(
4560 : MachineRepresentation::kFloat64, kNoWriteBarrier)),
4561 : heap_number, BuildHeapNumberValueIndexConstant(),
4562 92844 : value, heap_number, control));
4563 46422 : return heap_number;
4564 : }
4565 :
4566 94942 : Node* BuildChangeSmiToFloat64(Node* value) {
4567 : return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4568 189886 : BuildChangeSmiToInt32(value));
4569 : }
4570 :
4571 94944 : Node* BuildTestHeapObject(Node* value) {
4572 : return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4573 189893 : mcgraph()->IntPtrConstant(kHeapObjectTag));
4574 : }
4575 :
4576 94947 : Node* BuildLoadHeapNumberValue(Node* value) {
4577 : return SetEffect(graph()->NewNode(
4578 : mcgraph()->machine()->Load(MachineType::Float64()), value,
4579 474737 : BuildHeapNumberValueIndexConstant(), Effect(), Control()));
4580 : }
4581 :
4582 : Node* BuildHeapNumberValueIndexConstant() {
4583 141369 : return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4584 : }
4585 :
4586 : Node* BuildChangeInt32ToTagged(Node* value) {
4587 : MachineOperatorBuilder* machine = mcgraph()->machine();
4588 : CommonOperatorBuilder* common = mcgraph()->common();
4589 :
4590 : if (SmiValuesAre32Bits()) {
4591 27012 : return BuildChangeInt32ToSmi(value);
4592 : }
4593 : DCHECK(SmiValuesAre31Bits());
4594 :
4595 : Node* effect = Effect();
4596 : Node* control = Control();
4597 : Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4598 : graph()->start());
4599 :
4600 : Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4601 : Node* branch =
4602 : graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4603 :
4604 : Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4605 : Node* vtrue = BuildAllocateHeapNumberWithValue(
4606 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4607 : Node* etrue = Effect();
4608 :
4609 : Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4610 : Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4611 : vfalse = BuildChangeInt32ToIntPtr(vfalse);
4612 :
4613 : Node* merge =
4614 : SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
4615 : SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
4616 : return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4617 : vtrue, vfalse, merge);
4618 : }
4619 :
4620 46422 : Node* BuildChangeFloat64ToTagged(Node* value) {
4621 371376 : MachineOperatorBuilder* machine = mcgraph()->machine();
4622 46422 : CommonOperatorBuilder* common = mcgraph()->common();
4623 :
4624 : // Check several conditions:
4625 : // i32?
4626 : // ├─ true: zero?
4627 : // │ ├─ true: negative?
4628 : // │ │ ├─ true: box
4629 : // │ │ └─ false: potentially Smi
4630 : // │ └─ false: potentially Smi
4631 : // └─ false: box
4632 : // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
4633 : // still need to check whether the value fits in a Smi.
4634 :
4635 : Node* effect = Effect();
4636 : Node* control = Control();
4637 46422 : Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4638 : Node* check_i32 = graph()->NewNode(
4639 : machine->Float64Equal(), value,
4640 92844 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4641 46422 : Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
4642 :
4643 46422 : Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
4644 46422 : Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
4645 :
4646 : // We only need to check for -0 if the {value} can potentially contain -0.
4647 : Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4648 46422 : mcgraph()->Int32Constant(0));
4649 : Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4650 46422 : check_zero, if_i32);
4651 :
4652 46422 : Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4653 46422 : Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
4654 :
4655 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4656 : Node* check_negative = graph()->NewNode(
4657 : machine->Int32LessThan(),
4658 : graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4659 92844 : mcgraph()->Int32Constant(0));
4660 : Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4661 46422 : check_negative, if_zero);
4662 :
4663 46422 : Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4664 : Node* if_not_negative =
4665 46422 : graph()->NewNode(common->IfFalse(), branch_negative);
4666 :
4667 : // We need to create a box for negative 0.
4668 : Node* if_smi =
4669 46422 : graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
4670 46422 : Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
4671 :
4672 : // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4673 : // 32-bit machines we need to deal with potential overflow and fallback to
4674 : // boxing.
4675 : Node* vsmi;
4676 : if (SmiValuesAre32Bits()) {
4677 46422 : vsmi = BuildChangeInt32ToSmi(value32);
4678 : } else {
4679 : DCHECK(SmiValuesAre31Bits());
4680 : Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4681 : value32, if_smi);
4682 :
4683 : Node* check_ovf =
4684 : graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4685 : Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4686 : check_ovf, if_smi);
4687 :
4688 : Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4689 : if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4690 :
4691 : if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4692 : vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4693 : vsmi = BuildChangeInt32ToIntPtr(vsmi);
4694 : }
4695 :
4696 : // Allocate the box for the {value}.
4697 46422 : Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4698 : Node* ebox = Effect();
4699 :
4700 : Node* merge =
4701 46422 : SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
4702 46422 : SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
4703 : return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4704 92844 : vsmi, vbox, merge);
4705 : }
4706 :
4707 7015 : int AddArgumentNodes(Node** args, int pos, int param_count,
4708 49721 : wasm::FunctionSig* sig) {
4709 : // Convert wasm numbers to JS values.
4710 63751 : for (int i = 0; i < param_count; ++i) {
4711 : Node* param =
4712 49721 : Param(i + 1); // Start from index 1 to drop the instance_node.
4713 99442 : args[pos++] = ToJS(param, sig->GetParam(i));
4714 : }
4715 7015 : return pos;
4716 : }
4717 :
4718 185860 : Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4719 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4720 : mcgraph()->zone(), TypeConversionDescriptor{}, 0,
4721 763602 : CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
4722 : Node* stub_code =
4723 94947 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4724 : ? mcgraph()->RelocatableIntPtrConstant(
4725 : wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
4726 189895 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
4727 :
4728 : Node* result = SetEffect(
4729 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4730 94947 : node, js_context, Effect(), Control()));
4731 :
4732 94946 : SetSourcePosition(result, 1);
4733 :
4734 94947 : return result;
4735 : }
4736 :
4737 94945 : Node* BuildChangeTaggedToFloat64(Node* value) {
4738 1424172 : MachineOperatorBuilder* machine = mcgraph()->machine();
4739 94945 : CommonOperatorBuilder* common = mcgraph()->common();
4740 :
4741 : // Implement the following decision tree:
4742 : // heap object?
4743 : // ├─ true: undefined?
4744 : // │ ├─ true: f64 const
4745 : // │ └─ false: load heap number value
4746 : // └─ false: smi to float64
4747 :
4748 94945 : Node* check_heap_object = BuildTestHeapObject(value);
4749 : Diamond is_heap_object(graph(), common, check_heap_object,
4750 94949 : BranchHint::kFalse);
4751 94942 : is_heap_object.Chain(Control());
4752 :
4753 94944 : SetControl(is_heap_object.if_true);
4754 : Node* orig_effect = Effect();
4755 :
4756 : Node* undefined_node =
4757 189893 : LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4758 : Node* check_undefined =
4759 94949 : graph()->NewNode(machine->WordEqual(), value, undefined_node);
4760 : Node* effect_tagged = Effect();
4761 :
4762 94941 : Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
4763 94948 : is_undefined.Nest(is_heap_object, true);
4764 :
4765 94947 : SetControl(is_undefined.if_false);
4766 94947 : Node* vheap_number = BuildLoadHeapNumberValue(value);
4767 : Node* effect_undefined = Effect();
4768 :
4769 94946 : SetControl(is_undefined.merge);
4770 : Node* vundefined =
4771 94946 : mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4772 : Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
4773 94949 : vundefined, vheap_number);
4774 :
4775 94944 : effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
4776 :
4777 : // If input is Smi: just convert to float64.
4778 94946 : Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4779 :
4780 94944 : SetControl(is_heap_object.merge);
4781 94944 : SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
4782 : return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
4783 94941 : vfrom_smi);
4784 : }
4785 :
4786 73752 : Node* ToJS(Node* node, wasm::ValueType type) {
4787 73752 : switch (type) {
4788 : case wasm::kWasmI32:
4789 27009 : return BuildChangeInt32ToTagged(node);
4790 : case wasm::kWasmS128:
4791 0 : UNREACHABLE();
4792 : case wasm::kWasmI64: {
4793 : DCHECK(enabled_features_.bigint);
4794 36 : return BuildChangeInt64ToBigInt(node);
4795 : }
4796 : case wasm::kWasmF32:
4797 : node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4798 12419 : node);
4799 12419 : return BuildChangeFloat64ToTagged(node);
4800 : case wasm::kWasmF64:
4801 34003 : return BuildChangeFloat64ToTagged(node);
4802 : case wasm::kWasmAnyRef:
4803 : case wasm::kWasmAnyFunc:
4804 : return node;
4805 : default:
4806 0 : UNREACHABLE();
4807 : }
4808 : }
4809 :
4810 63 : Node* BuildChangeInt64ToBigInt(Node* input) {
4811 : BigIntToWasmI64Descriptor interface_descriptor;
4812 :
4813 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4814 : mcgraph()->zone(), // zone
4815 : interface_descriptor, // descriptor
4816 : interface_descriptor.GetStackParameterCount(), // stack parameter count
4817 : CallDescriptor::kNoFlags, // flags
4818 : Operator::kNoProperties, // properties
4819 297 : stub_mode_); // stub call mode
4820 :
4821 : Node* target =
4822 36 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4823 : ? mcgraph()->RelocatableIntPtrConstant(
4824 : wasm::WasmCode::kBigIntToWasmI64, RelocInfo::WASM_STUB_CALL)
4825 72 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, I64ToBigInt));
4826 :
4827 : return SetEffect(
4828 : SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4829 72 : target, input, Effect(), Control())));
4830 : }
4831 :
4832 27 : Node* BuildChangeBigIntToInt64(Node* input, Node* context) {
4833 : BigIntToI64Descriptor interface_descriptor;
4834 :
4835 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4836 : mcgraph()->zone(), // zone
4837 : interface_descriptor, // descriptor
4838 : interface_descriptor.GetStackParameterCount(), // stack parameter count
4839 : CallDescriptor::kNoFlags, // flags
4840 : Operator::kNoProperties, // properties
4841 153 : stub_mode_); // stub call mode
4842 :
4843 : Node* target =
4844 18 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4845 : ? mcgraph()->RelocatableIntPtrConstant(
4846 : wasm::WasmCode::kWasmBigIntToI64, RelocInfo::WASM_STUB_CALL)
4847 36 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, BigIntToI64));
4848 :
4849 : return SetEffect(SetControl(
4850 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), target,
4851 36 : input, context, Effect(), Control())));
4852 : }
4853 :
4854 95161 : Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4855 : DCHECK_NE(wasm::kWasmStmt, type);
4856 :
4857 : // The parameter is of type AnyRef, we take it as is.
4858 95161 : if (type == wasm::kWasmAnyRef) {
4859 180 : return node;
4860 : }
4861 :
4862 94981 : if (type == wasm::kWasmAnyFunc) {
4863 : Node* check =
4864 : BuildChangeSmiToInt32(SetEffect(BuildCallToRuntimeWithContext(
4865 : Runtime::kWasmIsValidAnyFuncValue, js_context, &node, 1, effect_,
4866 45893 : Control())));
4867 :
4868 : Diamond type_check(graph(), mcgraph()->common(), check,
4869 36 : BranchHint::kTrue);
4870 18 : type_check.Chain(Control());
4871 :
4872 18 : Node* effect = Effect();
4873 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4874 18 : nullptr, 0, &effect, type_check.if_false);
4875 :
4876 36 : SetEffect(type_check.EffectPhi(Effect(), effect));
4877 :
4878 18 : SetControl(type_check.merge);
4879 :
4880 18 : return node;
4881 : }
4882 : Node* num = nullptr;
4883 :
4884 94963 : if (type != wasm::kWasmI64) {
4885 : // Do a JavaScript ToNumber.
4886 94945 : num = BuildJavaScriptToNumber(node, js_context);
4887 :
4888 : // Change representation.
4889 94948 : num = BuildChangeTaggedToFloat64(num);
4890 : }
4891 :
4892 94957 : switch (type) {
4893 : case wasm::kWasmI32: {
4894 : num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4895 32138 : num);
4896 32138 : break;
4897 : }
4898 : case wasm::kWasmI64: {
4899 : DCHECK(enabled_features_.bigint);
4900 18 : num = BuildChangeBigIntToInt64(node, js_context);
4901 18 : break;
4902 : }
4903 : case wasm::kWasmF32:
4904 : num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4905 13611 : num);
4906 13611 : break;
4907 : case wasm::kWasmF64:
4908 : break;
4909 : case wasm::kWasmS128:
4910 0 : UNREACHABLE();
4911 : default:
4912 0 : UNREACHABLE();
4913 : }
4914 : DCHECK_NOT_NULL(num);
4915 :
4916 94957 : return num;
4917 : }
4918 :
4919 326351 : void BuildModifyThreadInWasmFlag(bool new_value) {
4920 652666 : if (!trap_handler::IsTrapHandlerEnabled()) return;
4921 : Node* isolate_root =
4922 5217209 : LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
4923 :
4924 : Node* thread_in_wasm_flag_address =
4925 652122 : LOAD_RAW(isolate_root, Isolate::thread_in_wasm_flag_address_offset(),
4926 : MachineType::Pointer());
4927 :
4928 326062 : if (FLAG_debug_code) {
4929 : Node* flag_value = SetEffect(
4930 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
4931 : thread_in_wasm_flag_address,
4932 36 : mcgraph()->Int32Constant(0), Effect(), Control()));
4933 : Node* check =
4934 : graph()->NewNode(mcgraph()->machine()->Word32Equal(), flag_value,
4935 54 : mcgraph()->Int32Constant(new_value ? 0 : 1));
4936 :
4937 : Diamond flag_check(graph(), mcgraph()->common(), check,
4938 36 : BranchHint::kTrue);
4939 18 : flag_check.Chain(Control());
4940 : Node* message_id = jsgraph()->SmiConstant(static_cast<int32_t>(
4941 : new_value ? AbortReason::kUnexpectedThreadInWasmSet
4942 36 : : AbortReason::kUnexpectedThreadInWasmUnset));
4943 :
4944 18 : Node* effect = Effect();
4945 : BuildCallToRuntimeWithContext(Runtime::kAbort, NoContextConstant(),
4946 : &message_id, 1, &effect,
4947 36 : flag_check.if_false);
4948 :
4949 36 : SetEffect(flag_check.EffectPhi(Effect(), effect));
4950 :
4951 18 : SetControl(flag_check.merge);
4952 : }
4953 :
4954 : SetEffect(graph()->NewNode(
4955 : mcgraph()->machine()->Store(StoreRepresentation(
4956 : MachineRepresentation::kWord32, kNoWriteBarrier)),
4957 : thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
4958 1630307 : mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
4959 : }
4960 :
4961 784904 : Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
4962 : Node* shared = SetEffect(graph()->NewNode(
4963 : jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4964 : jsgraph()->Int32Constant(
4965 : wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()),
4966 1255847 : Effect(), Control()));
4967 : return SetEffect(graph()->NewNode(
4968 : jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4969 : jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4970 : kHeapObjectTag),
4971 470943 : Effect(), Control()));
4972 : }
4973 :
4974 470943 : Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
4975 : return SetEffect(graph()->NewNode(
4976 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4977 : jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4978 : kHeapObjectTag),
4979 941886 : Effect(), Control()));
4980 : }
4981 :
4982 1566 : Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
4983 : Node* function_index_smi = SetEffect(graph()->NewNode(
4984 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4985 : jsgraph()->Int32Constant(
4986 : WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
4987 2610 : Effect(), Control()));
4988 522 : Node* function_index = BuildChangeSmiToInt32(function_index_smi);
4989 522 : return function_index;
4990 : }
4991 :
4992 466888 : Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
4993 : Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
4994 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4995 : jsgraph()->Int32Constant(
4996 : WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
4997 778147 : Effect(), Control()));
4998 155630 : Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
4999 155630 : return jump_table_offset;
5000 : }
5001 :
5002 446908 : void BuildJSToWasmWrapper(bool is_import) {
5003 428282 : const int wasm_count = static_cast<int>(sig_->parameter_count());
5004 :
5005 : // Build the start and the JS parameter nodes.
5006 1248552 : SetEffect(SetControl(Start(wasm_count + 5)));
5007 :
5008 : // Create the js_closure and js_context parameters.
5009 : Node* js_closure =
5010 : graph()->NewNode(jsgraph()->common()->Parameter(
5011 : Linkage::kJSCallClosureParamIndex, "%closure"),
5012 313962 : graph()->start());
5013 : Node* js_context = graph()->NewNode(
5014 : mcgraph()->common()->Parameter(
5015 : Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
5016 313962 : graph()->start());
5017 :
5018 : // Create the instance_node node to pass as parameter. It is loaded from
5019 : // an actual reference to an instance or a placeholder reference,
5020 : // called {WasmExportedFunction} via the {WasmExportedFunctionData}
5021 : // structure.
5022 156981 : Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
5023 : instance_node_.set(
5024 156981 : BuildLoadInstanceFromExportedFunctionData(function_data));
5025 :
5026 156981 : if (!wasm::IsJSCompatibleSignature(sig_, enabled_features_.bigint)) {
5027 : // Throw a TypeError. Use the js_context of the calling javascript
5028 : // function (passed as a parameter), such that the generated code is
5029 : // js_context independent.
5030 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
5031 828 : nullptr, 0, effect_, Control());
5032 : Return(jsgraph()->SmiConstant(0));
5033 157809 : return;
5034 : }
5035 :
5036 156153 : const int args_count = wasm_count + 1; // +1 for wasm_code.
5037 156153 : Node** args = Buffer(args_count);
5038 : Node** rets;
5039 :
5040 : // Convert JS parameters to wasm numbers.
5041 403427 : for (int i = 0; i < wasm_count; ++i) {
5042 91117 : Node* param = Param(i + 1);
5043 182242 : Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
5044 91113 : args[i + 1] = wasm_param;
5045 : }
5046 :
5047 : // Set the ThreadInWasm flag before we do the actual call.
5048 156153 : BuildModifyThreadInWasmFlag(true);
5049 :
5050 156148 : if (is_import) {
5051 : // Call to an imported function.
5052 : // Load function index from {WasmExportedFunctionData}.
5053 : Node* function_index =
5054 522 : BuildLoadFunctionIndexFromExportedFunctionData(function_data);
5055 522 : BuildImportCall(sig_, args, &rets, wasm::kNoCodePosition, function_index);
5056 : } else {
5057 : // Call to a wasm function defined in this module.
5058 : // The call target is the jump table slot for that function.
5059 : Node* jump_table_start =
5060 311250 : LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
5061 : Node* jump_table_offset =
5062 155630 : BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
5063 : Node* jump_table_slot = graph()->NewNode(
5064 155630 : mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
5065 155629 : args[0] = jump_table_slot;
5066 :
5067 : BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
5068 155629 : kNoRetpoline);
5069 : }
5070 :
5071 : // Clear the ThreadInWasm flag.
5072 156153 : BuildModifyThreadInWasmFlag(false);
5073 :
5074 156149 : Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
5075 312298 : : ToJS(rets[0], sig_->GetReturn());
5076 : Return(jsval);
5077 : }
5078 :
5079 7087 : bool BuildWasmImportCallWrapper(WasmImportCallKind kind) {
5080 18144 : int wasm_count = static_cast<int>(sig_->parameter_count());
5081 :
5082 : // Build the start and the parameter nodes.
5083 181211 : SetEffect(SetControl(Start(wasm_count + 4)));
5084 :
5085 7087 : instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
5086 :
5087 : Node* native_context =
5088 14174 : LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
5089 :
5090 7087 : if (kind == WasmImportCallKind::kRuntimeTypeError) {
5091 : // =======================================================================
5092 : // === Runtime TypeError =================================================
5093 : // =======================================================================
5094 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
5095 : native_context, nullptr, 0, effect_,
5096 72 : Control());
5097 : // We don't need to return a value here, as the runtime call will not
5098 : // return anyway (the c entry stub will trigger stack unwinding).
5099 : ReturnVoid();
5100 72 : return false;
5101 : }
5102 :
5103 : // The callable is passed as the last parameter, after WASM arguments.
5104 7015 : Node* callable_node = Param(wasm_count + 1);
5105 :
5106 : Node* undefined_node =
5107 14030 : LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
5108 :
5109 : Node* call = nullptr;
5110 : bool sloppy_receiver = true;
5111 :
5112 : // Clear the ThreadInWasm flag.
5113 7015 : BuildModifyThreadInWasmFlag(false);
5114 :
5115 7015 : switch (kind) {
5116 : // =======================================================================
5117 : // === JS Functions with matching arity ==================================
5118 : // =======================================================================
5119 : case WasmImportCallKind::kJSFunctionArityMatch:
5120 : sloppy_receiver = false;
5121 : V8_FALLTHROUGH; // fallthru
5122 : case WasmImportCallKind::kJSFunctionArityMatchSloppy: {
5123 4593 : Node** args = Buffer(wasm_count + 9);
5124 : int pos = 0;
5125 : Node* function_context = SetEffect(graph()->NewNode(
5126 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5127 : callable_node,
5128 : mcgraph()->Int32Constant(
5129 : wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
5130 9186 : Effect(), Control()));
5131 4593 : args[pos++] = callable_node; // target callable.
5132 : // Receiver.
5133 4593 : if (sloppy_receiver) {
5134 9018 : Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
5135 : native_context, Context::GLOBAL_PROXY_INDEX);
5136 4509 : args[pos++] = global_proxy;
5137 : } else {
5138 84 : args[pos++] = undefined_node;
5139 : }
5140 :
5141 : auto call_descriptor = Linkage::GetJSCallDescriptor(
5142 4593 : graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
5143 :
5144 : // Convert wasm numbers to JS values.
5145 4593 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5146 :
5147 4593 : args[pos++] = undefined_node; // new target
5148 9186 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5149 4593 : args[pos++] = function_context;
5150 9186 : args[pos++] = Effect();
5151 9186 : args[pos++] = Control();
5152 :
5153 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5154 9186 : args);
5155 4593 : break;
5156 : }
5157 : // =======================================================================
5158 : // === JS Functions with arguments adapter ===============================
5159 : // =======================================================================
5160 : case WasmImportCallKind::kJSFunctionArityMismatch:
5161 : sloppy_receiver = false;
5162 : V8_FALLTHROUGH; // fallthru
5163 : case WasmImportCallKind::kJSFunctionArityMismatchSloppy: {
5164 541 : Node** args = Buffer(wasm_count + 9);
5165 : int pos = 0;
5166 : Node* function_context = SetEffect(graph()->NewNode(
5167 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5168 : callable_node,
5169 : mcgraph()->Int32Constant(
5170 : wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
5171 1082 : Effect(), Control()));
5172 : args[pos++] =
5173 541 : BuildLoadBuiltinFromInstance(Builtins::kArgumentsAdaptorTrampoline);
5174 541 : args[pos++] = callable_node; // target callable
5175 541 : args[pos++] = undefined_node; // new target
5176 541 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5177 :
5178 : // Load shared function info, and then the formal parameter count.
5179 : Node* shared_function_info = SetEffect(graph()->NewNode(
5180 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5181 : callable_node,
5182 : mcgraph()->Int32Constant(
5183 : wasm::ObjectAccess::
5184 : SharedFunctionInfoOffsetInTaggedJSFunction()),
5185 1082 : Effect(), Control()));
5186 : Node* formal_param_count = SetEffect(graph()->NewNode(
5187 : mcgraph()->machine()->Load(MachineType::Uint16()),
5188 : shared_function_info,
5189 : mcgraph()->Int32Constant(
5190 : wasm::ObjectAccess::
5191 : FormalParameterCountOffsetInSharedFunctionInfo()),
5192 1082 : Effect(), Control()));
5193 541 : args[pos++] = formal_param_count;
5194 :
5195 : // Receiver.
5196 541 : if (sloppy_receiver) {
5197 944 : Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
5198 : native_context, Context::GLOBAL_PROXY_INDEX);
5199 472 : args[pos++] = global_proxy;
5200 : } else {
5201 69 : args[pos++] = undefined_node;
5202 : }
5203 :
5204 : #ifdef V8_TARGET_ARCH_IA32
5205 : // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative
5206 : // call instead once builtins are embedded in every build configuration.
5207 : CallDescriptor::Flags flags = CallDescriptor::kAllowCallThroughSlot;
5208 : #else
5209 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5210 : #endif
5211 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5212 : mcgraph()->zone(), ArgumentsAdaptorDescriptor{}, 1 + wasm_count,
5213 1082 : flags, Operator::kNoProperties);
5214 :
5215 : // Convert wasm numbers to JS values.
5216 541 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5217 541 : args[pos++] = function_context;
5218 1082 : args[pos++] = Effect();
5219 1082 : args[pos++] = Control();
5220 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5221 1082 : args);
5222 : break;
5223 : }
5224 : // =======================================================================
5225 : // === General case of unknown callable ==================================
5226 : // =======================================================================
5227 : case WasmImportCallKind::kUseCallBuiltin: {
5228 1881 : Node** args = Buffer(wasm_count + 9);
5229 : int pos = 0;
5230 : args[pos++] = mcgraph()->RelocatableIntPtrConstant(
5231 1881 : wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
5232 1881 : args[pos++] = callable_node;
5233 1881 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5234 1881 : args[pos++] = undefined_node; // receiver
5235 :
5236 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5237 : graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
5238 : CallDescriptor::kNoFlags, Operator::kNoProperties,
5239 3762 : StubCallMode::kCallWasmRuntimeStub);
5240 :
5241 : // Convert wasm numbers to JS values.
5242 1881 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5243 :
5244 : // The native_context is sufficient here, because all kind of callables
5245 : // which depend on the context provide their own context. The context
5246 : // here is only needed if the target is a constructor to throw a
5247 : // TypeError, if the target is a native function, or if the target is a
5248 : // callable JSObject, which can only be constructed by the runtime.
5249 1881 : args[pos++] = native_context;
5250 3762 : args[pos++] = Effect();
5251 3762 : args[pos++] = Control();
5252 :
5253 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5254 3762 : args);
5255 1881 : break;
5256 : }
5257 : default:
5258 0 : UNREACHABLE();
5259 : }
5260 : DCHECK_NOT_NULL(call);
5261 :
5262 : SetEffect(call);
5263 7015 : SetSourcePosition(call, 0);
5264 :
5265 : // Convert the return value back.
5266 7015 : Node* val = sig_->return_count() == 0
5267 : ? mcgraph()->Int32Constant(0)
5268 14030 : : FromJS(call, native_context, sig_->GetReturn());
5269 :
5270 : // Set the ThreadInWasm flag again.
5271 7015 : BuildModifyThreadInWasmFlag(true);
5272 :
5273 : Return(val);
5274 7015 : return true;
5275 : }
5276 :
5277 2428 : void BuildWasmInterpreterEntry(uint32_t func_index) {
5278 4407 : int param_count = static_cast<int>(sig_->parameter_count());
5279 :
5280 : // Build the start and the parameter nodes.
5281 10556 : SetEffect(SetControl(Start(param_count + 3)));
5282 :
5283 : // Create the instance_node from the passed parameter.
5284 1214 : instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
5285 :
5286 : // Compute size for the argument buffer.
5287 1214 : int args_size_bytes = 0;
5288 3430 : for (wasm::ValueType type : sig_->parameters()) {
5289 1002 : args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
5290 : }
5291 :
5292 : // The return value is also passed via this buffer:
5293 : DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
5294 : // TODO(wasm): Handle multi-value returns.
5295 : DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
5296 : int return_size_bytes =
5297 : sig_->return_count() == 0
5298 : ? 0
5299 2191 : : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
5300 :
5301 : // Get a stack slot for the arguments.
5302 : Node* arg_buffer =
5303 1214 : args_size_bytes == 0 && return_size_bytes == 0
5304 : ? mcgraph()->IntPtrConstant(0)
5305 : : graph()->NewNode(mcgraph()->machine()->StackSlot(
5306 3528 : std::max(args_size_bytes, return_size_bytes), 8));
5307 :
5308 : // Now store all our arguments to the buffer.
5309 : int offset = 0;
5310 :
5311 3430 : for (int i = 0; i < param_count; ++i) {
5312 1002 : wasm::ValueType type = sig_->GetParam(i);
5313 : // Start from the parameter with index 1 to drop the instance_node.
5314 : SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
5315 1002 : Int32Constant(offset), Param(i + 1), Effect(),
5316 2004 : Control()));
5317 1002 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5318 : }
5319 : DCHECK_EQ(args_size_bytes, offset);
5320 :
5321 : // We are passing the raw arg_buffer here. To the GC and other parts, it
5322 : // looks like a Smi (lowest bit not set). In the runtime function however,
5323 : // don't call Smi::value on it, but just cast it to a byte pointer.
5324 2428 : Node* parameters[] = {jsgraph()->SmiConstant(func_index), arg_buffer};
5325 : BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
5326 1214 : arraysize(parameters));
5327 :
5328 : // Read back the return value.
5329 2428 : if (sig_->return_count() == 0) {
5330 : Return(Int32Constant(0));
5331 : } else {
5332 : // TODO(wasm): Implement multi-return.
5333 : DCHECK_EQ(1, sig_->return_count());
5334 : MachineType load_rep =
5335 977 : wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
5336 : Node* val = SetEffect(
5337 : graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
5338 977 : Int32Constant(0), Effect(), Control()));
5339 : Return(val);
5340 : }
5341 :
5342 2428 : if (ContainsInt64(sig_)) LowerInt64();
5343 1214 : }
5344 :
5345 1572 : void BuildCWasmEntry() {
5346 : // Build the start and the JS parameter nodes.
5347 11634 : SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
5348 :
5349 : // Create parameter nodes (offset by 1 for the receiver parameter).
5350 786 : Node* code_entry = Param(CWasmEntryParameters::kCodeEntry + 1);
5351 786 : Node* object_ref_node = Param(CWasmEntryParameters::kObjectRef + 1);
5352 786 : Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
5353 :
5354 2304 : int wasm_arg_count = static_cast<int>(sig_->parameter_count());
5355 : int arg_count =
5356 786 : wasm_arg_count + 4; // code, object_ref_node, control, effect
5357 786 : Node** args = Buffer(arg_count);
5358 :
5359 : int pos = 0;
5360 786 : args[pos++] = code_entry;
5361 786 : args[pos++] = object_ref_node;
5362 :
5363 : int offset = 0;
5364 2378 : for (wasm::ValueType type : sig_->parameters()) {
5365 : Node* arg_load = SetEffect(
5366 : graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
5367 806 : Int32Constant(offset), Effect(), Control()));
5368 806 : args[pos++] = arg_load;
5369 806 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5370 : }
5371 :
5372 1572 : args[pos++] = Effect();
5373 1572 : args[pos++] = Control();
5374 : DCHECK_EQ(arg_count, pos);
5375 :
5376 : // Call the wasm code.
5377 1572 : auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
5378 :
5379 : Node* call = SetEffect(graph()->NewNode(
5380 1572 : mcgraph()->common()->Call(call_descriptor), arg_count, args));
5381 :
5382 : // Store the return value.
5383 : DCHECK_GE(1, sig_->return_count());
5384 1572 : if (sig_->return_count() == 1) {
5385 : StoreRepresentation store_rep(
5386 : wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
5387 732 : kNoWriteBarrier);
5388 : SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
5389 : arg_buffer, Int32Constant(0), call, Effect(),
5390 732 : Control()));
5391 : }
5392 : Return(jsgraph()->SmiConstant(0));
5393 :
5394 1572 : if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
5395 : MachineRepresentation sig_reps[] = {
5396 : MachineRepresentation::kWord32, // return value
5397 : MachineRepresentation::kTagged, // receiver
5398 : MachineRepresentation::kTagged, // arg0 (code)
5399 : MachineRepresentation::kTagged // arg1 (buffer)
5400 0 : };
5401 : Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
5402 : Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
5403 0 : mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
5404 0 : r.LowerGraph();
5405 : }
5406 786 : }
5407 :
5408 : JSGraph* jsgraph() { return jsgraph_; }
5409 :
5410 : private:
5411 : Isolate* const isolate_;
5412 : JSGraph* jsgraph_;
5413 : StubCallMode stub_mode_;
5414 : SetOncePointer<const Operator> allocate_heap_number_operator_;
5415 : wasm::WasmFeatures enabled_features_;
5416 : };
5417 :
5418 157767 : void AppendSignature(char* buffer, size_t max_name_len,
5419 157767 : wasm::FunctionSig* sig) {
5420 157767 : size_t name_len = strlen(buffer);
5421 : auto append_name_char = [&](char c) {
5422 276264 : if (name_len + 1 < max_name_len) buffer[name_len++] = c;
5423 : };
5424 250760 : for (wasm::ValueType t : sig->parameters()) {
5425 : append_name_char(wasm::ValueTypes::ShortNameOf(t));
5426 : }
5427 : append_name_char(':');
5428 183271 : for (wasm::ValueType t : sig->returns()) {
5429 : append_name_char(wasm::ValueTypes::ShortNameOf(t));
5430 : }
5431 157767 : buffer[name_len] = '\0';
5432 157767 : }
5433 :
5434 : } // namespace
5435 :
5436 313962 : MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
5437 156981 : wasm::FunctionSig* sig,
5438 : bool is_import) {
5439 313962 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5440 : "CompileJSToWasmWrapper");
5441 : //----------------------------------------------------------------------------
5442 : // Create the Graph.
5443 : //----------------------------------------------------------------------------
5444 313959 : Zone zone(isolate->allocator(), ZONE_NAME);
5445 156981 : Graph graph(&zone);
5446 156981 : CommonOperatorBuilder common(&zone);
5447 : MachineOperatorBuilder machine(
5448 : &zone, MachineType::PointerRepresentation(),
5449 : InstructionSelector::SupportedMachineOperatorFlags(),
5450 156981 : InstructionSelector::AlignmentRequirements());
5451 156981 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5452 :
5453 156981 : Node* control = nullptr;
5454 156981 : Node* effect = nullptr;
5455 :
5456 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5457 : StubCallMode::kCallCodeObject,
5458 156981 : wasm::WasmFeaturesFromIsolate(isolate));
5459 : builder.set_control_ptr(&control);
5460 : builder.set_effect_ptr(&effect);
5461 156981 : builder.BuildJSToWasmWrapper(is_import);
5462 :
5463 : //----------------------------------------------------------------------------
5464 : // Run the compilation pipeline.
5465 : //----------------------------------------------------------------------------
5466 : static constexpr size_t kMaxNameLen = 128;
5467 156970 : char debug_name[kMaxNameLen] = "js_to_wasm:";
5468 156970 : AppendSignature(debug_name, kMaxNameLen, sig);
5469 :
5470 : // Schedule and compile to machine code.
5471 156981 : int params = static_cast<int>(sig->parameter_count());
5472 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5473 156981 : &zone, false, params + 1, CallDescriptor::kNoFlags);
5474 :
5475 : MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5476 : isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
5477 156981 : WasmAssemblerOptions());
5478 : Handle<Code> code;
5479 156975 : if (!maybe_code.ToHandle(&code)) {
5480 0 : return maybe_code;
5481 : }
5482 : #ifdef ENABLE_DISASSEMBLER
5483 : if (FLAG_print_opt_code) {
5484 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5485 : OFStream os(tracing_scope.file());
5486 : code->Disassemble(debug_name, os);
5487 : }
5488 : #endif
5489 :
5490 156979 : if (must_record_function_compilation(isolate)) {
5491 : RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
5492 0 : debug_name);
5493 : }
5494 :
5495 313959 : return code;
5496 : }
5497 :
5498 134375 : WasmImportCallKind GetWasmImportCallKind(Handle<JSReceiver> target,
5499 11372 : wasm::FunctionSig* expected_sig,
5500 : bool has_bigint_feature) {
5501 134375 : if (WasmExportedFunction::IsWasmExportedFunction(*target)) {
5502 120699 : auto imported_function = WasmExportedFunction::cast(*target);
5503 : wasm::FunctionSig* imported_sig =
5504 : imported_function->instance()
5505 241398 : ->module()
5506 241398 : ->functions[imported_function->function_index()]
5507 120699 : .sig;
5508 120699 : if (*imported_sig != *expected_sig) {
5509 : return WasmImportCallKind::kLinkError;
5510 : }
5511 120519 : return WasmImportCallKind::kWasmToWasm;
5512 : }
5513 : // Assuming we are calling to JS, check whether this would be a runtime error.
5514 13676 : if (!wasm::IsJSCompatibleSignature(expected_sig, has_bigint_feature)) {
5515 : return WasmImportCallKind::kRuntimeTypeError;
5516 : }
5517 : // For JavaScript calls, determine whether the target has an arity match
5518 : // and whether it has a sloppy receiver.
5519 27208 : if (target->IsJSFunction()) {
5520 11642 : Handle<JSFunction> function = Handle<JSFunction>::cast(target);
5521 11642 : SharedFunctionInfo shared = function->shared();
5522 :
5523 : // Check for math intrinsics.
5524 : #define COMPARE_SIG_FOR_BUILTIN(name) \
5525 : { \
5526 : wasm::FunctionSig* sig = wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
5527 : if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name); \
5528 : DCHECK_NOT_NULL(sig); \
5529 : if (*expected_sig == *sig) return WasmImportCallKind::k##name; \
5530 : }
5531 : #define COMPARE_SIG_FOR_BUILTIN_F64(name) \
5532 : case Builtins::kMath##name: \
5533 : COMPARE_SIG_FOR_BUILTIN(F64##name); \
5534 : break;
5535 : #define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
5536 : case Builtins::kMath##name: \
5537 : COMPARE_SIG_FOR_BUILTIN(F64##name); \
5538 : COMPARE_SIG_FOR_BUILTIN(F32##name); \
5539 : break;
5540 :
5541 11642 : if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
5542 562 : switch (shared->builtin_id()) {
5543 20 : COMPARE_SIG_FOR_BUILTIN_F64(Acos);
5544 20 : COMPARE_SIG_FOR_BUILTIN_F64(Asin);
5545 19 : COMPARE_SIG_FOR_BUILTIN_F64(Atan);
5546 30 : COMPARE_SIG_FOR_BUILTIN_F64(Cos);
5547 30 : COMPARE_SIG_FOR_BUILTIN_F64(Sin);
5548 19 : COMPARE_SIG_FOR_BUILTIN_F64(Tan);
5549 19 : COMPARE_SIG_FOR_BUILTIN_F64(Exp);
5550 20 : COMPARE_SIG_FOR_BUILTIN_F64(Log);
5551 20 : COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
5552 : //===========================================================
5553 : // TODO(8505): Math.pow for wasm does not match JS.
5554 : // COMPARE_SIG_FOR_BUILTIN_F64(Pow);
5555 : //===========================================================
5556 118 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
5557 27 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
5558 38 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
5559 38 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
5560 48 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
5561 58 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
5562 : case Builtins::kMathFround:
5563 9 : COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
5564 : break;
5565 : default:
5566 : break;
5567 : }
5568 : }
5569 :
5570 : #undef COMPARE_SIG_FOR_BUILTIN
5571 : #undef COMPARE_SIG_FOR_BUILTIN_F64
5572 : #undef COMPARE_SIG_FOR_BUILTIN_F32_F64
5573 :
5574 11390 : if (IsClassConstructor(shared->kind())) {
5575 : // Class constructor will throw anyway.
5576 : return WasmImportCallKind::kUseCallBuiltin;
5577 : }
5578 22389 : bool sloppy = is_sloppy(shared->language_mode()) && !shared->native();
5579 22744 : if (shared->internal_formal_parameter_count() ==
5580 : expected_sig->parameter_count()) {
5581 : return sloppy ? WasmImportCallKind::kJSFunctionArityMatchSloppy
5582 8676 : : WasmImportCallKind::kJSFunctionArityMatch;
5583 : }
5584 : return sloppy ? WasmImportCallKind::kJSFunctionArityMismatchSloppy
5585 2696 : : WasmImportCallKind::kJSFunctionArityMismatch;
5586 : }
5587 : // Unknown case. Use the call builtin.
5588 : return WasmImportCallKind::kUseCallBuiltin;
5589 : }
5590 :
5591 252 : wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
5592 : const char** name_ptr) {
5593 : #define CASE(name) \
5594 : case WasmImportCallKind::k##name: \
5595 : *name_ptr = "WasmMathIntrinsic:" #name; \
5596 : return wasm::kExpr##name
5597 252 : switch (kind) {
5598 9 : CASE(F64Acos);
5599 9 : CASE(F64Asin);
5600 9 : CASE(F64Atan);
5601 9 : CASE(F64Cos);
5602 9 : CASE(F64Sin);
5603 9 : CASE(F64Tan);
5604 9 : CASE(F64Exp);
5605 9 : CASE(F64Log);
5606 9 : CASE(F64Atan2);
5607 0 : CASE(F64Pow);
5608 18 : CASE(F64Ceil);
5609 18 : CASE(F64Floor);
5610 18 : CASE(F64Sqrt);
5611 18 : CASE(F64Min);
5612 18 : CASE(F64Max);
5613 18 : CASE(F64Abs);
5614 9 : CASE(F32Min);
5615 9 : CASE(F32Max);
5616 9 : CASE(F32Abs);
5617 9 : CASE(F32Ceil);
5618 9 : CASE(F32Floor);
5619 9 : CASE(F32Sqrt);
5620 9 : CASE(F32ConvertF64);
5621 : default:
5622 0 : UNREACHABLE();
5623 : return wasm::kExprUnreachable;
5624 : }
5625 : #undef CASE
5626 : }
5627 :
5628 252 : wasm::WasmCode* CompileWasmMathIntrinsic(wasm::WasmEngine* wasm_engine,
5629 : wasm::NativeModule* native_module,
5630 : WasmImportCallKind kind,
5631 504 : wasm::FunctionSig* sig) {
5632 : DCHECK_EQ(1, sig->return_count());
5633 :
5634 504 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5635 : "CompileWasmMathIntrinsic");
5636 :
5637 504 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5638 :
5639 : // Compile a WASM function with a single bytecode and let TurboFan
5640 : // generate either inlined machine code or a call to a helper.
5641 : SourcePositionTable* source_positions = nullptr;
5642 252 : MachineGraph* mcgraph = new (&zone) MachineGraph(
5643 504 : new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5644 : new (&zone) MachineOperatorBuilder(
5645 : &zone, MachineType::PointerRepresentation(),
5646 : InstructionSelector::SupportedMachineOperatorFlags(),
5647 756 : InstructionSelector::AlignmentRequirements()));
5648 :
5649 : wasm::CompilationEnv env(
5650 : native_module->module(), wasm::UseTrapHandler::kNoTrapHandler,
5651 : wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
5652 : wasm::kAllWasmFeatures, wasm::LowerSimd::kNoLowerSimd);
5653 :
5654 : WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
5655 252 : source_positions);
5656 :
5657 : // Set up the graph start.
5658 252 : Node* start = builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
5659 252 : Node* effect = start;
5660 252 : Node* control = start;
5661 : builder.set_effect_ptr(&effect);
5662 : builder.set_control_ptr(&control);
5663 252 : builder.set_instance_node(builder.Param(wasm::kWasmInstanceParameterIndex));
5664 :
5665 : // Generate either a unop or a binop.
5666 : Node* result = nullptr;
5667 252 : const char* debug_name = "WasmMathIntrinsic";
5668 252 : auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
5669 252 : switch (sig->parameter_count()) {
5670 : case 1:
5671 189 : result = builder.Unop(opcode, builder.Param(1));
5672 189 : break;
5673 : case 2:
5674 63 : result = builder.Binop(opcode, builder.Param(1), builder.Param(2));
5675 63 : break;
5676 : default:
5677 0 : UNREACHABLE();
5678 : break;
5679 : }
5680 :
5681 : builder.Return(result);
5682 :
5683 : // Run the compiler pipeline to generate machine code.
5684 252 : auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
5685 252 : if (mcgraph->machine()->Is32()) {
5686 : call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5687 : }
5688 :
5689 : wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5690 : wasm_engine, call_descriptor, mcgraph, Code::WASM_FUNCTION,
5691 : wasm::WasmCode::kFunction, debug_name, WasmStubAssemblerOptions(),
5692 252 : native_module, source_positions);
5693 252 : CHECK_NOT_NULL(wasm_code);
5694 : // TODO(titzer): add counters for math intrinsic code size / allocation
5695 :
5696 252 : return wasm_code;
5697 : }
5698 :
5699 7339 : wasm::WasmCode* CompileWasmImportCallWrapper(wasm::WasmEngine* wasm_engine,
5700 : wasm::NativeModule* native_module,
5701 : WasmImportCallKind kind,
5702 : wasm::FunctionSig* sig,
5703 : bool source_positions) {
5704 : DCHECK_NE(WasmImportCallKind::kLinkError, kind);
5705 : DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind);
5706 :
5707 : // Check for math intrinsics first.
5708 7339 : if (FLAG_wasm_math_intrinsics &&
5709 7339 : kind >= WasmImportCallKind::kFirstMathIntrinsic &&
5710 : kind <= WasmImportCallKind::kLastMathIntrinsic) {
5711 252 : return CompileWasmMathIntrinsic(wasm_engine, native_module, kind, sig);
5712 : }
5713 :
5714 14174 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5715 : "CompileWasmImportCallWrapper");
5716 : //----------------------------------------------------------------------------
5717 : // Create the Graph
5718 : //----------------------------------------------------------------------------
5719 14174 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5720 7087 : Graph graph(&zone);
5721 7087 : CommonOperatorBuilder common(&zone);
5722 : MachineOperatorBuilder machine(
5723 : &zone, MachineType::PointerRepresentation(),
5724 : InstructionSelector::SupportedMachineOperatorFlags(),
5725 7087 : InstructionSelector::AlignmentRequirements());
5726 7087 : JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
5727 :
5728 7087 : Node* control = nullptr;
5729 7087 : Node* effect = nullptr;
5730 :
5731 : SourcePositionTable* source_position_table =
5732 7673 : source_positions ? new (&zone) SourcePositionTable(&graph) : nullptr;
5733 :
5734 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, source_position_table,
5735 : StubCallMode::kCallWasmRuntimeStub,
5736 : native_module->enabled_features());
5737 : builder.set_control_ptr(&control);
5738 : builder.set_effect_ptr(&effect);
5739 7087 : builder.BuildWasmImportCallWrapper(kind);
5740 :
5741 : const char* func_name = "wasm-to-js";
5742 :
5743 : // Schedule and compile to machine code.
5744 : CallDescriptor* incoming =
5745 : GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline,
5746 7087 : WasmGraphBuilder::kExtraCallableParam);
5747 7087 : if (machine.Is32()) {
5748 : incoming = GetI32WasmCallDescriptor(&zone, incoming);
5749 : }
5750 : wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5751 : wasm_engine, incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
5752 : wasm::WasmCode::kWasmToJsWrapper, func_name, WasmStubAssemblerOptions(),
5753 7087 : native_module, source_position_table);
5754 7087 : CHECK_NOT_NULL(wasm_code);
5755 :
5756 7087 : return wasm_code;
5757 : }
5758 :
5759 1214 : wasm::WasmCode* CompileWasmInterpreterEntry(wasm::WasmEngine* wasm_engine,
5760 : wasm::NativeModule* native_module,
5761 : uint32_t func_index,
5762 : wasm::FunctionSig* sig) {
5763 : //----------------------------------------------------------------------------
5764 : // Create the Graph
5765 : //----------------------------------------------------------------------------
5766 1214 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5767 1214 : Graph graph(&zone);
5768 1214 : CommonOperatorBuilder common(&zone);
5769 : MachineOperatorBuilder machine(
5770 : &zone, MachineType::PointerRepresentation(),
5771 : InstructionSelector::SupportedMachineOperatorFlags(),
5772 1214 : InstructionSelector::AlignmentRequirements());
5773 1214 : JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
5774 :
5775 1214 : Node* control = nullptr;
5776 1214 : Node* effect = nullptr;
5777 :
5778 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5779 : StubCallMode::kCallWasmRuntimeStub,
5780 : native_module->enabled_features());
5781 : builder.set_control_ptr(&control);
5782 : builder.set_effect_ptr(&effect);
5783 1214 : builder.BuildWasmInterpreterEntry(func_index);
5784 :
5785 : // Schedule and compile to machine code.
5786 1214 : CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
5787 1214 : if (machine.Is32()) {
5788 : incoming = GetI32WasmCallDescriptor(&zone, incoming);
5789 : }
5790 :
5791 : EmbeddedVector<char, 32> func_name;
5792 : func_name.Truncate(
5793 1214 : SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
5794 :
5795 : wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5796 : wasm_engine, incoming, &jsgraph, Code::WASM_INTERPRETER_ENTRY,
5797 1214 : wasm::WasmCode::kInterpreterEntry, func_name.start(),
5798 1214 : WasmStubAssemblerOptions(), native_module);
5799 1214 : CHECK_NOT_NULL(wasm_code);
5800 :
5801 1214 : return wasm_code;
5802 : }
5803 :
5804 786 : MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
5805 786 : Zone zone(isolate->allocator(), ZONE_NAME);
5806 786 : Graph graph(&zone);
5807 786 : CommonOperatorBuilder common(&zone);
5808 : MachineOperatorBuilder machine(
5809 : &zone, MachineType::PointerRepresentation(),
5810 : InstructionSelector::SupportedMachineOperatorFlags(),
5811 786 : InstructionSelector::AlignmentRequirements());
5812 786 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5813 :
5814 786 : Node* control = nullptr;
5815 786 : Node* effect = nullptr;
5816 :
5817 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5818 : StubCallMode::kCallCodeObject,
5819 786 : wasm::WasmFeaturesFromIsolate(isolate));
5820 : builder.set_control_ptr(&control);
5821 : builder.set_effect_ptr(&effect);
5822 786 : builder.BuildCWasmEntry();
5823 :
5824 : // Schedule and compile to machine code.
5825 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5826 : &zone, false, CWasmEntryParameters::kNumParameters + 1,
5827 786 : CallDescriptor::kNoFlags);
5828 :
5829 : // Build a name in the form "c-wasm-entry:<params>:<returns>".
5830 : static constexpr size_t kMaxNameLen = 128;
5831 786 : char debug_name[kMaxNameLen] = "c-wasm-entry:";
5832 786 : AppendSignature(debug_name, kMaxNameLen, sig);
5833 :
5834 : MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5835 : isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
5836 786 : AssemblerOptions::Default(isolate));
5837 : Handle<Code> code;
5838 786 : if (!maybe_code.ToHandle(&code)) {
5839 0 : return maybe_code;
5840 : }
5841 : #ifdef ENABLE_DISASSEMBLER
5842 : if (FLAG_print_opt_code) {
5843 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5844 : OFStream os(tracing_scope.file());
5845 : code->Disassemble(debug_name, os);
5846 : }
5847 : #endif
5848 :
5849 786 : return code;
5850 : }
5851 :
5852 1067642 : TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
5853 : wasm::WasmCompilationUnit* wasm_unit)
5854 1067642 : : wasm_unit_(wasm_unit) {}
5855 :
5856 : // Clears unique_ptrs, but (part of) the type is forward declared in the header.
5857 : TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5858 :
5859 1066843 : bool TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5860 : wasm::CompilationEnv* env, wasm::NativeModule* native_module,
5861 : const wasm::FunctionBody& func_body, wasm::WasmFeatures* detected,
5862 : double* decode_ms, MachineGraph* mcgraph, NodeOriginTable* node_origins,
5863 : SourcePositionTable* source_positions) {
5864 : base::ElapsedTimer decode_timer;
5865 : if (FLAG_trace_wasm_decode_time) {
5866 : decode_timer.Start();
5867 : }
5868 :
5869 : // Create a TF graph during decoding.
5870 : WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
5871 2133686 : source_positions);
5872 : wasm::VoidResult graph_construction_result = wasm::BuildTFGraph(
5873 : wasm_unit_->wasm_engine_->allocator(), env->enabled_features, env->module,
5874 2133907 : &builder, detected, func_body, node_origins);
5875 1066988 : if (graph_construction_result.failed()) {
5876 : if (FLAG_trace_wasm_compiler) {
5877 : StdoutStream{} << "Compilation failed: "
5878 : << graph_construction_result.error().message()
5879 : << std::endl;
5880 : }
5881 : native_module->compilation_state()->SetError(
5882 13478 : wasm_unit_->func_index_, std::move(graph_construction_result).error());
5883 6739 : return false;
5884 : }
5885 :
5886 1060249 : builder.LowerInt64();
5887 :
5888 2136200 : if (builder.has_simd() &&
5889 15828 : (!CpuFeatures::SupportsWasmSimd128() || env->lower_simd)) {
5890 : SimdScalarLowering(mcgraph,
5891 : CreateMachineSignature(mcgraph->zone(), func_body.sig))
5892 15255 : .LowerGraph();
5893 : }
5894 :
5895 2120385 : if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5896 1060199 : wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5897 : PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(), func_body,
5898 0 : env->module, wasm::kPrintLocals);
5899 : }
5900 : if (FLAG_trace_wasm_decode_time) {
5901 : *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5902 : }
5903 : return true;
5904 : }
5905 :
5906 : namespace {
5907 1066967 : Vector<const char> GetDebugName(Zone* zone, int index) {
5908 : // TODO(herhut): Use name from module if available.
5909 : constexpr int kBufferLength = 24;
5910 :
5911 : EmbeddedVector<char, kBufferLength> name_vector;
5912 1066967 : int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
5913 : DCHECK(name_len > 0 && name_len < name_vector.length());
5914 :
5915 1066979 : char* index_name = zone->NewArray<char>(name_len);
5916 1067001 : memcpy(index_name, name_vector.start(), name_len);
5917 1067001 : return Vector<const char>(index_name, name_len);
5918 : }
5919 : } // namespace
5920 :
5921 1067017 : void TurbofanWasmCompilationUnit::ExecuteCompilation(
5922 : wasm::CompilationEnv* env, wasm::NativeModule* native_module,
5923 : const wasm::FunctionBody& func_body, Counters* counters,
5924 : wasm::WasmFeatures* detected) {
5925 2134034 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5926 : "ExecuteTurbofanCompilation");
5927 1067017 : double decode_ms = 0;
5928 : size_t node_count = 0;
5929 :
5930 2127315 : Zone zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5931 3187422 : MachineGraph* mcgraph = new (&zone) MachineGraph(
5932 2134047 : new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5933 : new (&zone) MachineOperatorBuilder(
5934 : &zone, MachineType::PointerRepresentation(),
5935 : InstructionSelector::SupportedMachineOperatorFlags(),
5936 3201028 : InstructionSelector::AlignmentRequirements()));
5937 :
5938 : OptimizedCompilationInfo info(GetDebugName(&zone, wasm_unit_->func_index_),
5939 2127278 : &zone, Code::WASM_FUNCTION);
5940 1066929 : if (env->runtime_exception_support) {
5941 : info.SetWasmRuntimeExceptionSupport();
5942 : }
5943 :
5944 1066929 : if (info.trace_turbo_json_enabled()) {
5945 0 : TurboCfgFile tcf;
5946 0 : tcf << AsC1VCompilation(&info);
5947 : }
5948 :
5949 : NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
5950 : ? new (&zone)
5951 0 : NodeOriginTable(mcgraph->graph())
5952 1066929 : : nullptr;
5953 : SourcePositionTable* source_positions =
5954 1066915 : new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
5955 1066955 : if (!BuildGraphForWasmFunction(env, native_module, func_body, detected,
5956 : &decode_ms, mcgraph, node_origins,
5957 1066853 : source_positions)) {
5958 : // Compilation failed.
5959 6739 : return;
5960 : }
5961 :
5962 1060216 : if (node_origins) {
5963 0 : node_origins->AddDecorator();
5964 : }
5965 :
5966 : base::ElapsedTimer pipeline_timer;
5967 : if (FLAG_trace_wasm_decode_time) {
5968 : node_count = mcgraph->graph()->NodeCount();
5969 : pipeline_timer.Start();
5970 : }
5971 :
5972 : // Run the compiler pipeline to generate machine code.
5973 1060216 : auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
5974 1060194 : if (mcgraph->machine()->Is32()) {
5975 : call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5976 : }
5977 :
5978 1060223 : if (wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmFunction(
5979 : &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
5980 : source_positions, node_origins, func_body, native_module,
5981 1060194 : wasm_unit_->func_index_)) {
5982 1060235 : wasm_unit_->SetResult(wasm_code, counters);
5983 : }
5984 : if (FLAG_trace_wasm_decode_time) {
5985 : double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
5986 : PrintF(
5987 : "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
5988 : "%0.3f ms pipeline\n",
5989 : static_cast<unsigned>(func_body.end - func_body.start), decode_ms,
5990 : node_count, pipeline_ms);
5991 : }
5992 : // TODO(bradnelson): Improve histogram handling of size_t.
5993 : counters->wasm_compile_function_peak_memory_bytes()->AddSample(
5994 3180896 : static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
5995 : }
5996 :
5997 : namespace {
5998 : // Helper for allocating either an GP or FP reg, or the next stack slot.
5999 : class LinkageLocationAllocator {
6000 : public:
6001 : template <size_t kNumGpRegs, size_t kNumFpRegs>
6002 : constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
6003 : const DoubleRegister (&fp)[kNumFpRegs])
6004 : : allocator_(wasm::LinkageAllocator(gp, fp)) {}
6005 :
6006 6758443 : LinkageLocation Next(MachineRepresentation rep) {
6007 6758443 : MachineType type = MachineType::TypeForRepresentation(rep);
6008 6758488 : if (IsFloatingPoint(rep)) {
6009 12828862 : if (allocator_.CanAllocateFP(rep)) {
6010 : int reg_code = allocator_.NextFpReg(rep);
6011 : return LinkageLocation::ForRegister(reg_code, type);
6012 : }
6013 6152584 : } else if (allocator_.CanAllocateGP()) {
6014 : int reg_code = allocator_.NextGpReg();
6015 : return LinkageLocation::ForRegister(reg_code, type);
6016 : }
6017 : // Cannot use register; use stack slot.
6018 188432 : int index = -1 - allocator_.NextStackSlot(rep);
6019 : return LinkageLocation::ForCallerFrameSlot(index, type);
6020 : }
6021 :
6022 : void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
6023 6058372 : int NumStackSlots() const { return allocator_.NumStackSlots(); }
6024 :
6025 : private:
6026 : wasm::LinkageAllocator allocator_;
6027 : };
6028 : } // namespace
6029 :
6030 : // General code uses the above configuration data.
6031 3029095 : CallDescriptor* GetWasmCallDescriptor(
6032 13827960 : Zone* zone, wasm::FunctionSig* fsig,
6033 : WasmGraphBuilder::UseRetpoline use_retpoline,
6034 : WasmGraphBuilder::ExtraCallableParam extra_callable_param) {
6035 : // The extra here is to accomodate the instance object as first parameter
6036 : // and, in the case of an import wrapper, the additional callable.
6037 3029095 : int extra_params = extra_callable_param ? 2 : 1;
6038 : LocationSignature::Builder locations(zone, fsig->return_count(),
6039 3029095 : fsig->parameter_count() + extra_params);
6040 :
6041 : // Add register and/or stack parameter(s).
6042 : LinkageLocationAllocator params(wasm::kGpParamRegisters,
6043 3029225 : wasm::kFpParamRegisters);
6044 :
6045 : // The instance object.
6046 3029225 : locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
6047 : const size_t param_offset = 1; // Actual params start here.
6048 :
6049 : // Parameters are separated into two groups (first all untagged, then all
6050 : // tagged parameters). This allows for easy iteration of tagged parameters
6051 : // during frame iteration.
6052 : const size_t parameter_count = fsig->parameter_count();
6053 4040288 : for (size_t i = 0; i < parameter_count; i++) {
6054 : MachineRepresentation param =
6055 1011124 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
6056 : // Skip tagged parameters (e.g. any-ref).
6057 1011998 : if (IsAnyTagged(param)) continue;
6058 1010132 : auto l = params.Next(param);
6059 1010209 : locations.AddParamAt(i + param_offset, l);
6060 : }
6061 1011171 : for (size_t i = 0; i < parameter_count; i++) {
6062 : MachineRepresentation param =
6063 1011149 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
6064 : // Skip untagged parameters.
6065 2021401 : if (!IsAnyTagged(param)) continue;
6066 933 : auto l = params.Next(param);
6067 937 : locations.AddParamAt(i + param_offset, l);
6068 : }
6069 :
6070 : // Import call wrappers have an additional (implicit) parameter, the callable.
6071 : // For consistency with JS, we use the JSFunction register.
6072 3029186 : if (extra_callable_param) {
6073 : locations.AddParam(LinkageLocation::ForRegister(
6074 : kJSFunctionRegister.code(), MachineType::TaggedPointer()));
6075 : }
6076 :
6077 : // Add return location(s).
6078 : LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
6079 3029186 : wasm::kFpReturnRegisters);
6080 :
6081 : int parameter_slots = params.NumStackSlots();
6082 : if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
6083 :
6084 : rets.SetStackOffset(parameter_slots);
6085 :
6086 3029186 : const int return_count = static_cast<int>(locations.return_count_);
6087 5747559 : for (int i = 0; i < return_count; i++) {
6088 : MachineRepresentation ret =
6089 5436702 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
6090 2718366 : auto l = rets.Next(ret);
6091 2718373 : locations.AddReturn(l);
6092 : }
6093 :
6094 : const RegList kCalleeSaveRegisters = 0;
6095 : const RegList kCalleeSaveFPRegisters = 0;
6096 :
6097 : // The target for wasm calls is always a code object.
6098 : MachineType target_type = MachineType::Pointer();
6099 : LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
6100 :
6101 : CallDescriptor::Kind kind = extra_callable_param
6102 : ? CallDescriptor::kCallWasmImportWrapper
6103 3029208 : : CallDescriptor::kCallWasmFunction;
6104 :
6105 : CallDescriptor::Flags flags =
6106 3029208 : use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
6107 : return new (zone) CallDescriptor( // --
6108 : kind, // kind
6109 : target_type, // target MachineType
6110 : target_loc, // target location
6111 : locations.Build(), // location_sig
6112 : parameter_slots, // stack_parameter_count
6113 : compiler::Operator::kNoProperties, // properties
6114 : kCalleeSaveRegisters, // callee-saved registers
6115 : kCalleeSaveFPRegisters, // callee-saved fp regs
6116 : flags, // flags
6117 : "wasm-call", // debug name
6118 : 0, // allocatable registers
6119 12116742 : rets.NumStackSlots() - parameter_slots); // stack_return_count
6120 : }
6121 :
6122 : namespace {
6123 0 : CallDescriptor* ReplaceTypeInCallDescriptorWith(
6124 0 : Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
6125 : MachineType input_type, MachineRepresentation output_type) {
6126 : size_t parameter_count = call_descriptor->ParameterCount();
6127 : size_t return_count = call_descriptor->ReturnCount();
6128 0 : for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
6129 0 : if (call_descriptor->GetParameterType(i) == input_type) {
6130 0 : parameter_count += num_replacements - 1;
6131 : }
6132 : }
6133 0 : for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
6134 0 : if (call_descriptor->GetReturnType(i) == input_type) {
6135 0 : return_count += num_replacements - 1;
6136 : }
6137 : }
6138 0 : if (parameter_count == call_descriptor->ParameterCount() &&
6139 : return_count == call_descriptor->ReturnCount()) {
6140 : return call_descriptor;
6141 : }
6142 :
6143 : LocationSignature::Builder locations(zone, return_count, parameter_count);
6144 :
6145 : LinkageLocationAllocator params(wasm::kGpParamRegisters,
6146 0 : wasm::kFpParamRegisters);
6147 0 : for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
6148 0 : if (call_descriptor->GetParameterType(i) == input_type) {
6149 0 : for (size_t j = 0; j < num_replacements; j++) {
6150 0 : locations.AddParam(params.Next(output_type));
6151 : }
6152 : } else {
6153 : locations.AddParam(
6154 0 : params.Next(call_descriptor->GetParameterType(i).representation()));
6155 : }
6156 : }
6157 :
6158 : LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
6159 0 : wasm::kFpReturnRegisters);
6160 : rets.SetStackOffset(params.NumStackSlots());
6161 0 : for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
6162 0 : if (call_descriptor->GetReturnType(i) == input_type) {
6163 0 : for (size_t j = 0; j < num_replacements; j++) {
6164 0 : locations.AddReturn(rets.Next(output_type));
6165 : }
6166 : } else {
6167 : locations.AddReturn(
6168 0 : rets.Next(call_descriptor->GetReturnType(i).representation()));
6169 : }
6170 : }
6171 :
6172 : return new (zone) CallDescriptor( // --
6173 : call_descriptor->kind(), // kind
6174 : call_descriptor->GetInputType(0), // target MachineType
6175 : call_descriptor->GetInputLocation(0), // target location
6176 : locations.Build(), // location_sig
6177 : params.NumStackSlots(), // stack_parameter_count
6178 : call_descriptor->properties(), // properties
6179 : call_descriptor->CalleeSavedRegisters(), // callee-saved registers
6180 : call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
6181 : call_descriptor->flags(), // flags
6182 : call_descriptor->debug_name(), // debug name
6183 : call_descriptor->AllocatableRegisters(), // allocatable registers
6184 0 : rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
6185 : }
6186 : } // namespace
6187 :
6188 0 : CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
6189 : CallDescriptor* call_descriptor) {
6190 : return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
6191 : MachineType::Int64(),
6192 0 : MachineRepresentation::kWord32);
6193 : }
6194 :
6195 0 : CallDescriptor* GetI32WasmCallDescriptorForSimd(
6196 : Zone* zone, CallDescriptor* call_descriptor) {
6197 : return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
6198 : MachineType::Simd128(),
6199 0 : MachineRepresentation::kWord32);
6200 : }
6201 :
6202 1059903 : AssemblerOptions WasmAssemblerOptions() {
6203 1216884 : AssemblerOptions options;
6204 : // Relocation info required to serialize {WasmCode} for proper functions.
6205 : options.record_reloc_info_for_serialization = true;
6206 : options.enable_root_array_delta_access = false;
6207 1059903 : return options;
6208 : }
6209 :
6210 0 : AssemblerOptions WasmStubAssemblerOptions() {
6211 8553 : AssemblerOptions options;
6212 : // Relocation info not necessary because stubs are not serialized.
6213 8553 : options.record_reloc_info_for_serialization = false;
6214 : options.enable_root_array_delta_access = false;
6215 0 : return options;
6216 : }
6217 :
6218 : #undef WASM_64
6219 : #undef FATAL_UNSUPPORTED_OPCODE
6220 : #undef WASM_INSTANCE_OBJECT_SIZE
6221 : #undef WASM_INSTANCE_OBJECT_OFFSET
6222 : #undef LOAD_RAW
6223 : #undef LOAD_INSTANCE_FIELD
6224 : #undef LOAD_TAGGED_POINTER
6225 : #undef LOAD_TAGGED_ANY
6226 : #undef LOAD_FIXED_ARRAY_SLOT
6227 : #undef LOAD_FIXED_ARRAY_SLOT_SMI
6228 : #undef LOAD_FIXED_ARRAY_SLOT_PTR
6229 : #undef LOAD_FIXED_ARRAY_SLOT_ANY
6230 : #undef STORE_FIXED_ARRAY_SLOT_SMI
6231 : #undef STORE_FIXED_ARRAY_SLOT_ANY
6232 :
6233 : } // namespace compiler
6234 : } // namespace internal
6235 183867 : } // namespace v8
|
