Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/wasm-compiler.h"
6 :
7 : #include <memory>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/base/platform/elapsed-timer.h"
11 : #include "src/base/platform/platform.h"
12 : #include "src/builtins/builtins.h"
13 : #include "src/code-factory.h"
14 : #include "src/code-stubs.h"
15 : #include "src/compiler/access-builder.h"
16 : #include "src/compiler/common-operator.h"
17 : #include "src/compiler/compiler-source-position-table.h"
18 : #include "src/compiler/diamond.h"
19 : #include "src/compiler/graph-visualizer.h"
20 : #include "src/compiler/graph.h"
21 : #include "src/compiler/instruction-selector.h"
22 : #include "src/compiler/int64-lowering.h"
23 : #include "src/compiler/js-graph.h"
24 : #include "src/compiler/js-operator.h"
25 : #include "src/compiler/linkage.h"
26 : #include "src/compiler/machine-operator.h"
27 : #include "src/compiler/node-matchers.h"
28 : #include "src/compiler/pipeline.h"
29 : #include "src/compiler/simd-scalar-lowering.h"
30 : #include "src/compiler/zone-stats.h"
31 : #include "src/factory.h"
32 : #include "src/isolate-inl.h"
33 : #include "src/log-inl.h"
34 : #include "src/wasm/function-body-decoder.h"
35 : #include "src/wasm/wasm-limits.h"
36 : #include "src/wasm/wasm-module.h"
37 : #include "src/wasm/wasm-objects.h"
38 : #include "src/wasm/wasm-opcodes.h"
39 : #include "src/wasm/wasm-text.h"
40 :
41 : // TODO(titzer): pull WASM_64 up to a common header.
42 : #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
43 : #define WASM_64 1
44 : #else
45 : #define WASM_64 0
46 : #endif
47 :
48 : namespace v8 {
49 : namespace internal {
50 : namespace compiler {
51 :
52 : namespace {
53 0 : const Operator* UnsupportedOpcode(wasm::WasmOpcode opcode) {
54 : V8_Fatal(__FILE__, __LINE__, "Unsupported opcode #%d:%s", opcode,
55 0 : wasm::WasmOpcodes::OpcodeName(opcode));
56 : return nullptr;
57 : }
58 :
59 405540 : void MergeControlToEnd(JSGraph* jsgraph, Node* node) {
60 202770 : Graph* g = jsgraph->graph();
61 202770 : if (g->end()) {
62 63713 : NodeProperties::MergeControlToEnd(g, jsgraph->common(), node);
63 : } else {
64 139057 : g->SetEnd(g->NewNode(jsgraph->common()->End(1), node));
65 : }
66 202775 : }
67 :
68 170689 : Node* BuildModifyThreadInWasmFlag(bool new_value, JSGraph* jsgraph,
69 : Node** effect_ptr, Node* control) {
70 : // TODO(eholk): generate code to modify the thread-local storage directly,
71 : // rather than calling the runtime.
72 138865 : if (!trap_handler::UseTrapHandler()) {
73 : return control;
74 : }
75 :
76 : const Runtime::FunctionId f =
77 10608 : new_value ? Runtime::kSetThreadInWasm : Runtime::kClearThreadInWasm;
78 10608 : const Runtime::Function* fun = Runtime::FunctionForId(f);
79 : DCHECK_EQ(0, fun->nargs);
80 : const CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor(
81 : jsgraph->zone(), f, fun->nargs, Operator::kNoProperties,
82 21216 : CallDescriptor::kNoFlags);
83 : // CEntryStubConstant nodes have to be created and cached in the main
84 : // thread. At the moment this is only done for CEntryStubConstant(1).
85 : DCHECK_EQ(1, fun->result_size);
86 10608 : Node* inputs[] = {jsgraph->CEntryStubConstant(fun->result_size),
87 : jsgraph->ExternalConstant(
88 21216 : ExternalReference(f, jsgraph->isolate())), // ref
89 10608 : jsgraph->Int32Constant(fun->nargs), // arity
90 : jsgraph->NoContextConstant(),
91 : *effect_ptr,
92 42432 : control};
93 :
94 : Node* node = jsgraph->graph()->NewNode(jsgraph->common()->Call(desc),
95 21216 : arraysize(inputs), inputs);
96 10607 : *effect_ptr = node;
97 10607 : return node;
98 : }
99 :
100 : // Only call this function for code which is not reused across instantiations,
101 : // as we do not patch the embedded context.
102 13128 : Node* BuildCallToRuntimeWithContext(Runtime::FunctionId f, JSGraph* jsgraph,
103 : Node* context, Node** parameters,
104 : int parameter_count, Node** effect_ptr,
105 : Node** control) {
106 : // Setting and clearing the thread-in-wasm flag should not be done as a normal
107 : // runtime call.
108 : DCHECK_NE(f, Runtime::kSetThreadInWasm);
109 : DCHECK_NE(f, Runtime::kClearThreadInWasm);
110 : // We're leaving Wasm code, so clear the flag.
111 3282 : *control = BuildModifyThreadInWasmFlag(false, jsgraph, effect_ptr, *control);
112 :
113 3282 : const Runtime::Function* fun = Runtime::FunctionForId(f);
114 : CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor(
115 : jsgraph->zone(), f, fun->nargs, Operator::kNoProperties,
116 6564 : CallDescriptor::kNoFlags);
117 : // CEntryStubConstant nodes have to be created and cached in the main
118 : // thread. At the moment this is only done for CEntryStubConstant(1).
119 : DCHECK_EQ(1, fun->result_size);
120 : // At the moment we only allow 3 parameters. If more parameters are needed,
121 : // increase this constant accordingly.
122 : static const int kMaxParams = 3;
123 : DCHECK_GE(kMaxParams, parameter_count);
124 : Node* inputs[kMaxParams + 6];
125 : int count = 0;
126 3282 : inputs[count++] = jsgraph->CEntryStubConstant(fun->result_size);
127 8745 : for (int i = 0; i < parameter_count; i++) {
128 5463 : inputs[count++] = parameters[i];
129 : }
130 3282 : inputs[count++] = jsgraph->ExternalConstant(
131 3282 : ExternalReference(f, jsgraph->isolate())); // ref
132 3282 : inputs[count++] = jsgraph->Int32Constant(fun->nargs); // arity
133 3282 : inputs[count++] = context; // context
134 3282 : inputs[count++] = *effect_ptr;
135 3282 : inputs[count++] = *control;
136 :
137 : Node* node =
138 6564 : jsgraph->graph()->NewNode(jsgraph->common()->Call(desc), count, inputs);
139 3282 : *effect_ptr = node;
140 :
141 : // Restore the thread-in-wasm flag, since we have returned to Wasm.
142 3282 : *control = BuildModifyThreadInWasmFlag(true, jsgraph, effect_ptr, *control);
143 :
144 3282 : return node;
145 : }
146 :
147 3114 : Node* BuildCallToRuntime(Runtime::FunctionId f, JSGraph* jsgraph,
148 : Node** parameters, int parameter_count,
149 : Node** effect_ptr, Node** control) {
150 : return BuildCallToRuntimeWithContext(f, jsgraph, jsgraph->NoContextConstant(),
151 : parameters, parameter_count, effect_ptr,
152 3114 : control);
153 : }
154 :
155 : } // namespace
156 :
157 139360 : WasmGraphBuilder::WasmGraphBuilder(
158 : wasm::ModuleEnv* module_env, Zone* zone, JSGraph* jsgraph,
159 598306 : wasm::FunctionSig* sig,
160 : compiler::SourcePositionTable* source_position_table)
161 : : zone_(zone),
162 : jsgraph_(jsgraph),
163 : module_(module_env),
164 : signature_tables_(zone),
165 : function_tables_(zone),
166 : function_table_sizes_(zone),
167 : cur_buffer_(def_buffer_),
168 : cur_bufsize_(kDefaultBufferSize),
169 : sig_(sig),
170 418080 : source_position_table_(source_position_table) {
171 483386 : for (size_t i = sig->parameter_count(); i > 0 && !has_simd_; --i) {
172 409332 : if (sig->GetParam(i - 1) == wasm::kWasmS128) has_simd_ = true;
173 : }
174 393640 : for (size_t i = sig->return_count(); i > 0 && !has_simd_; --i) {
175 229840 : if (sig->GetReturn(i - 1) == wasm::kWasmS128) has_simd_ = true;
176 : }
177 : DCHECK_NOT_NULL(jsgraph_);
178 139360 : }
179 :
180 0 : Node* WasmGraphBuilder::Error() { return jsgraph()->Dead(); }
181 :
182 278844 : Node* WasmGraphBuilder::Start(unsigned params) {
183 278844 : Node* start = graph()->NewNode(jsgraph()->common()->Start(params));
184 : graph()->SetStart(start);
185 139418 : return start;
186 : }
187 :
188 204584 : Node* WasmGraphBuilder::Param(unsigned index) {
189 : return graph()->NewNode(jsgraph()->common()->Parameter(index),
190 409166 : graph()->start());
191 : }
192 :
193 8767 : Node* WasmGraphBuilder::Loop(Node* entry) {
194 17534 : return graph()->NewNode(jsgraph()->common()->Loop(1), entry);
195 : }
196 :
197 17534 : Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) {
198 : Node* terminate =
199 8767 : graph()->NewNode(jsgraph()->common()->Terminate(), effect, control);
200 8767 : MergeControlToEnd(jsgraph(), terminate);
201 8767 : return terminate;
202 : }
203 :
204 14199 : unsigned WasmGraphBuilder::InputCount(Node* node) {
205 14199 : return static_cast<unsigned>(node->InputCount());
206 : }
207 :
208 940258 : bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
209 1180148 : return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
210 710019 : NodeProperties::GetControlInput(phi) == merge;
211 : }
212 :
213 644 : bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
214 300 : Node** if_exception) {
215 644 : if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
216 : return false;
217 : }
218 :
219 300 : *if_success = graph()->NewNode(jsgraph()->common()->IfSuccess(), node);
220 : *if_exception =
221 300 : graph()->NewNode(jsgraph()->common()->IfException(), node, node);
222 :
223 150 : return true;
224 : }
225 :
226 48082 : void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
227 : DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
228 24041 : merge->AppendInput(jsgraph()->zone(), from);
229 : int new_size = merge->InputCount();
230 : NodeProperties::ChangeOp(
231 24041 : merge, jsgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
232 24041 : }
233 :
234 325083 : void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
235 : DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
236 : int new_size = phi->InputCount();
237 216722 : phi->InsertInput(jsgraph()->zone(), phi->InputCount() - 1, from);
238 : NodeProperties::ChangeOp(
239 108361 : phi, jsgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
240 108361 : }
241 :
242 100732 : Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
243 151098 : return graph()->NewNode(jsgraph()->common()->Merge(count), count, controls);
244 : }
245 :
246 130271 : Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
247 130271 : Node* control) {
248 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
249 130271 : Node** buf = Realloc(vals, count, count + 1);
250 130271 : buf[count] = control;
251 : return graph()->NewNode(jsgraph()->common()->Phi(type, count), count + 1,
252 390813 : buf);
253 : }
254 :
255 33778 : Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
256 33778 : Node* control) {
257 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
258 33778 : Node** buf = Realloc(effects, count, count + 1);
259 33778 : buf[count] = control;
260 : return graph()->NewNode(jsgraph()->common()->EffectPhi(count), count + 1,
261 101334 : buf);
262 : }
263 :
264 0 : Node* WasmGraphBuilder::NumberConstant(int32_t value) {
265 0 : return jsgraph()->Constant(value);
266 : }
267 :
268 2364 : Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
269 0 : return jsgraph()->Uint32Constant(value);
270 : }
271 :
272 466397 : Node* WasmGraphBuilder::Int32Constant(int32_t value) {
273 466562 : return jsgraph()->Int32Constant(value);
274 : }
275 :
276 5201 : Node* WasmGraphBuilder::Int64Constant(int64_t value) {
277 5201 : return jsgraph()->Int64Constant(value);
278 : }
279 :
280 80551 : void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
281 785129 : Node** effect, Node** control) {
282 105004 : if (FLAG_wasm_no_stack_checks) return;
283 : // We do not generate stack checks for cctests.
284 139429 : if (!module_ || (module_->instance && module_->instance->context.is_null()))
285 : return;
286 56098 : if (effect == nullptr) effect = effect_;
287 56098 : if (control == nullptr) control = control_;
288 :
289 : Node* limit = graph()->NewNode(
290 : jsgraph()->machine()->Load(MachineType::Pointer()),
291 : jsgraph()->ExternalConstant(
292 : ExternalReference::address_of_stack_limit(jsgraph()->isolate())),
293 280452 : jsgraph()->IntPtrConstant(0), *effect, *control);
294 56095 : Node* pointer = graph()->NewNode(jsgraph()->machine()->LoadStackPointer());
295 :
296 : Node* check =
297 56074 : graph()->NewNode(jsgraph()->machine()->UintLessThan(), limit, pointer);
298 :
299 112144 : Diamond stack_check(graph(), jsgraph()->common(), check, BranchHint::kTrue);
300 56058 : stack_check.Chain(*control);
301 56076 : Node* effect_true = *effect;
302 :
303 56076 : Handle<Code> code = jsgraph()->isolate()->builtins()->WasmStackGuard();
304 : CallInterfaceDescriptor idesc =
305 112152 : WasmRuntimeCallDescriptor(jsgraph()->isolate());
306 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
307 : jsgraph()->isolate(), jsgraph()->zone(), idesc, 0,
308 56079 : CallDescriptor::kNoFlags, Operator::kNoProperties);
309 56067 : Node* stub_code = jsgraph()->HeapConstant(code);
310 :
311 : Node* context = jsgraph()->NoContextConstant();
312 : Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), stub_code,
313 112170 : context, *effect, stack_check.if_false);
314 :
315 56083 : SetSourcePosition(call, position);
316 :
317 : Node* ephi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), effect_true,
318 112150 : call, stack_check.merge);
319 :
320 56082 : *control = stack_check.merge;
321 56082 : *effect = ephi;
322 : }
323 :
324 428824 : Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
325 428824 : wasm::WasmCodePosition position) {
326 : const Operator* op;
327 428824 : MachineOperatorBuilder* m = jsgraph()->machine();
328 428824 : switch (opcode) {
329 : case wasm::kExprI32Add:
330 106817 : op = m->Int32Add();
331 106817 : break;
332 : case wasm::kExprI32Sub:
333 20639 : op = m->Int32Sub();
334 20643 : break;
335 : case wasm::kExprI32Mul:
336 4818 : op = m->Int32Mul();
337 4818 : break;
338 : case wasm::kExprI32DivS:
339 151 : return BuildI32DivS(left, right, position);
340 : case wasm::kExprI32DivU:
341 105 : return BuildI32DivU(left, right, position);
342 : case wasm::kExprI32RemS:
343 58 : return BuildI32RemS(left, right, position);
344 : case wasm::kExprI32RemU:
345 74 : return BuildI32RemU(left, right, position);
346 : case wasm::kExprI32And:
347 112220 : op = m->Word32And();
348 112220 : break;
349 : case wasm::kExprI32Ior:
350 7033 : op = m->Word32Or();
351 7033 : break;
352 : case wasm::kExprI32Xor:
353 1169 : op = m->Word32Xor();
354 1169 : break;
355 : case wasm::kExprI32Shl:
356 14756 : op = m->Word32Shl();
357 14756 : right = MaskShiftCount32(right);
358 14756 : break;
359 : case wasm::kExprI32ShrU:
360 7472 : op = m->Word32Shr();
361 7472 : right = MaskShiftCount32(right);
362 7472 : break;
363 : case wasm::kExprI32ShrS:
364 1765 : op = m->Word32Sar();
365 1765 : right = MaskShiftCount32(right);
366 1765 : break;
367 : case wasm::kExprI32Ror:
368 70 : op = m->Word32Ror();
369 70 : right = MaskShiftCount32(right);
370 70 : break;
371 : case wasm::kExprI32Rol:
372 35 : right = MaskShiftCount32(right);
373 35 : return BuildI32Rol(left, right);
374 : case wasm::kExprI32Eq:
375 46591 : op = m->Word32Equal();
376 46591 : break;
377 : case wasm::kExprI32Ne:
378 6387 : return Invert(Binop(wasm::kExprI32Eq, left, right));
379 : case wasm::kExprI32LtS:
380 6253 : op = m->Int32LessThan();
381 6253 : break;
382 : case wasm::kExprI32LeS:
383 1068 : op = m->Int32LessThanOrEqual();
384 1068 : break;
385 : case wasm::kExprI32LtU:
386 5692 : op = m->Uint32LessThan();
387 5692 : break;
388 : case wasm::kExprI32LeU:
389 130 : op = m->Uint32LessThanOrEqual();
390 130 : break;
391 : case wasm::kExprI32GtS:
392 9022 : op = m->Int32LessThan();
393 : std::swap(left, right);
394 : break;
395 : case wasm::kExprI32GeS:
396 653 : op = m->Int32LessThanOrEqual();
397 : std::swap(left, right);
398 : break;
399 : case wasm::kExprI32GtU:
400 2255 : op = m->Uint32LessThan();
401 : std::swap(left, right);
402 : break;
403 : case wasm::kExprI32GeU:
404 88 : op = m->Uint32LessThanOrEqual();
405 : std::swap(left, right);
406 : break;
407 : case wasm::kExprI64And:
408 63 : op = m->Word64And();
409 63 : break;
410 : case wasm::kExprI64Add:
411 86 : op = m->Int64Add();
412 86 : break;
413 : case wasm::kExprI64Sub:
414 138 : op = m->Int64Sub();
415 138 : break;
416 : case wasm::kExprI64Mul:
417 146 : op = m->Int64Mul();
418 146 : break;
419 : case wasm::kExprI64DivS:
420 105 : return BuildI64DivS(left, right, position);
421 : case wasm::kExprI64DivU:
422 35 : return BuildI64DivU(left, right, position);
423 : case wasm::kExprI64RemS:
424 35 : return BuildI64RemS(left, right, position);
425 : case wasm::kExprI64RemU:
426 50 : return BuildI64RemU(left, right, position);
427 : case wasm::kExprI64Ior:
428 71 : op = m->Word64Or();
429 71 : break;
430 : case wasm::kExprI64Xor:
431 70 : op = m->Word64Xor();
432 70 : break;
433 : case wasm::kExprI64Shl:
434 212 : op = m->Word64Shl();
435 212 : right = MaskShiftCount64(right);
436 212 : break;
437 : case wasm::kExprI64ShrU:
438 286 : op = m->Word64Shr();
439 286 : right = MaskShiftCount64(right);
440 286 : break;
441 : case wasm::kExprI64ShrS:
442 63 : op = m->Word64Sar();
443 63 : right = MaskShiftCount64(right);
444 63 : break;
445 : case wasm::kExprI64Eq:
446 56 : op = m->Word64Equal();
447 56 : break;
448 : case wasm::kExprI64Ne:
449 28 : return Invert(Binop(wasm::kExprI64Eq, left, right));
450 : case wasm::kExprI64LtS:
451 43 : op = m->Int64LessThan();
452 43 : break;
453 : case wasm::kExprI64LeS:
454 28 : op = m->Int64LessThanOrEqual();
455 28 : break;
456 : case wasm::kExprI64LtU:
457 28 : op = m->Uint64LessThan();
458 28 : break;
459 : case wasm::kExprI64LeU:
460 28 : op = m->Uint64LessThanOrEqual();
461 28 : break;
462 : case wasm::kExprI64GtS:
463 28 : op = m->Int64LessThan();
464 : std::swap(left, right);
465 : break;
466 : case wasm::kExprI64GeS:
467 28 : op = m->Int64LessThanOrEqual();
468 : std::swap(left, right);
469 : break;
470 : case wasm::kExprI64GtU:
471 28 : op = m->Uint64LessThan();
472 : std::swap(left, right);
473 : break;
474 : case wasm::kExprI64GeU:
475 28 : op = m->Uint64LessThanOrEqual();
476 : std::swap(left, right);
477 : break;
478 : case wasm::kExprI64Ror:
479 84 : op = m->Word64Ror();
480 84 : right = MaskShiftCount64(right);
481 84 : break;
482 : case wasm::kExprI64Rol:
483 42 : return BuildI64Rol(left, right);
484 : case wasm::kExprF32CopySign:
485 14 : return BuildF32CopySign(left, right);
486 : case wasm::kExprF64CopySign:
487 14 : return BuildF64CopySign(left, right);
488 : case wasm::kExprF32Add:
489 109 : op = m->Float32Add();
490 109 : break;
491 : case wasm::kExprF32Sub:
492 110 : op = m->Float32Sub();
493 110 : break;
494 : case wasm::kExprF32Mul:
495 42 : op = m->Float32Mul();
496 42 : break;
497 : case wasm::kExprF32Div:
498 39 : op = m->Float32Div();
499 39 : break;
500 : case wasm::kExprF32Eq:
501 265 : op = m->Float32Equal();
502 265 : break;
503 : case wasm::kExprF32Ne:
504 175 : return Invert(Binop(wasm::kExprF32Eq, left, right));
505 : case wasm::kExprF32Lt:
506 32 : op = m->Float32LessThan();
507 32 : break;
508 : case wasm::kExprF32Ge:
509 32 : op = m->Float32LessThanOrEqual();
510 : std::swap(left, right);
511 : break;
512 : case wasm::kExprF32Gt:
513 32 : op = m->Float32LessThan();
514 : std::swap(left, right);
515 : break;
516 : case wasm::kExprF32Le:
517 32 : op = m->Float32LessThanOrEqual();
518 32 : break;
519 : case wasm::kExprF64Add:
520 4355 : op = m->Float64Add();
521 4355 : break;
522 : case wasm::kExprF64Sub:
523 3682 : op = m->Float64Sub();
524 3682 : break;
525 : case wasm::kExprF64Mul:
526 8697 : op = m->Float64Mul();
527 8697 : break;
528 : case wasm::kExprF64Div:
529 464 : op = m->Float64Div();
530 464 : break;
531 : case wasm::kExprF64Eq:
532 874 : op = m->Float64Equal();
533 874 : break;
534 : case wasm::kExprF64Ne:
535 332 : return Invert(Binop(wasm::kExprF64Eq, left, right));
536 : case wasm::kExprF64Lt:
537 854 : op = m->Float64LessThan();
538 854 : break;
539 : case wasm::kExprF64Le:
540 228 : op = m->Float64LessThanOrEqual();
541 228 : break;
542 : case wasm::kExprF64Gt:
543 841 : op = m->Float64LessThan();
544 : std::swap(left, right);
545 : break;
546 : case wasm::kExprF64Ge:
547 312 : op = m->Float64LessThanOrEqual();
548 : std::swap(left, right);
549 : break;
550 : case wasm::kExprF32Min:
551 32 : op = m->Float32Min();
552 32 : break;
553 : case wasm::kExprF64Min:
554 48 : op = m->Float64Min();
555 48 : break;
556 : case wasm::kExprF32Max:
557 32 : op = m->Float32Max();
558 32 : break;
559 : case wasm::kExprF64Max:
560 32 : op = m->Float64Max();
561 32 : break;
562 : case wasm::kExprF64Pow:
563 14 : return BuildF64Pow(left, right);
564 : case wasm::kExprF64Atan2:
565 16 : op = m->Float64Atan2();
566 16 : break;
567 : case wasm::kExprF64Mod:
568 28 : return BuildF64Mod(left, right);
569 : case wasm::kExprI32AsmjsDivS:
570 262 : return BuildI32AsmjsDivS(left, right);
571 : case wasm::kExprI32AsmjsDivU:
572 88 : return BuildI32AsmjsDivU(left, right);
573 : case wasm::kExprI32AsmjsRemS:
574 394 : return BuildI32AsmjsRemS(left, right);
575 : case wasm::kExprI32AsmjsRemU:
576 158 : return BuildI32AsmjsRemU(left, right);
577 : case wasm::kExprI32AsmjsStoreMem8:
578 3675 : return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
579 : case wasm::kExprI32AsmjsStoreMem16:
580 1778 : return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
581 : case wasm::kExprI32AsmjsStoreMem:
582 38534 : return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
583 : case wasm::kExprF32AsmjsStoreMem:
584 4610 : return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
585 : case wasm::kExprF64AsmjsStoreMem:
586 458 : return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
587 : default:
588 0 : op = UnsupportedOpcode(opcode);
589 : }
590 371227 : return graph()->NewNode(op, left, right);
591 : }
592 :
593 108196 : Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
594 127484 : wasm::WasmCodePosition position) {
595 : const Operator* op;
596 108196 : MachineOperatorBuilder* m = jsgraph()->machine();
597 108196 : switch (opcode) {
598 : case wasm::kExprI32Eqz:
599 19230 : op = m->Word32Equal();
600 38460 : return graph()->NewNode(op, input, jsgraph()->Int32Constant(0));
601 : case wasm::kExprF32Abs:
602 53 : op = m->Float32Abs();
603 53 : break;
604 : case wasm::kExprF32Neg: {
605 29 : op = m->Float32Neg();
606 29 : break;
607 : }
608 : case wasm::kExprF32Sqrt:
609 39 : op = m->Float32Sqrt();
610 39 : break;
611 : case wasm::kExprF64Abs:
612 74 : op = m->Float64Abs();
613 74 : break;
614 : case wasm::kExprF64Neg: {
615 194 : op = m->Float64Neg();
616 194 : break;
617 : }
618 : case wasm::kExprF64Sqrt:
619 165 : op = m->Float64Sqrt();
620 165 : break;
621 : case wasm::kExprI32SConvertF64:
622 149 : return BuildI32SConvertF64(input, position);
623 : case wasm::kExprI32UConvertF64:
624 14 : return BuildI32UConvertF64(input, position);
625 : case wasm::kExprI32AsmjsSConvertF64:
626 70 : return BuildI32AsmjsSConvertF64(input);
627 : case wasm::kExprI32AsmjsUConvertF64:
628 7 : return BuildI32AsmjsUConvertF64(input);
629 : case wasm::kExprF32ConvertF64:
630 4682 : op = m->TruncateFloat64ToFloat32();
631 4682 : break;
632 : case wasm::kExprF64SConvertI32:
633 686 : op = m->ChangeInt32ToFloat64();
634 686 : break;
635 : case wasm::kExprF64UConvertI32:
636 237 : op = m->ChangeUint32ToFloat64();
637 237 : break;
638 : case wasm::kExprF32SConvertI32:
639 172 : op = m->RoundInt32ToFloat32();
640 172 : break;
641 : case wasm::kExprF32UConvertI32:
642 47 : op = m->RoundUint32ToFloat32();
643 47 : break;
644 : case wasm::kExprI32SConvertF32:
645 133 : return BuildI32SConvertF32(input, position);
646 : case wasm::kExprI32UConvertF32:
647 14 : return BuildI32UConvertF32(input, position);
648 : case wasm::kExprI32AsmjsSConvertF32:
649 7 : return BuildI32AsmjsSConvertF32(input);
650 : case wasm::kExprI32AsmjsUConvertF32:
651 7 : return BuildI32AsmjsUConvertF32(input);
652 : case wasm::kExprF64ConvertF32:
653 7643 : op = m->ChangeFloat32ToFloat64();
654 7643 : break;
655 : case wasm::kExprF32ReinterpretI32:
656 60 : op = m->BitcastInt32ToFloat32();
657 60 : break;
658 : case wasm::kExprI32ReinterpretF32:
659 129 : op = m->BitcastFloat32ToInt32();
660 129 : break;
661 : case wasm::kExprI32Clz:
662 2084 : op = m->Word32Clz();
663 2084 : break;
664 : case wasm::kExprI32Ctz: {
665 469 : if (m->Word32Ctz().IsSupported()) {
666 469 : op = m->Word32Ctz().op();
667 469 : break;
668 0 : } else if (m->Word32ReverseBits().IsSupported()) {
669 0 : Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
670 0 : Node* result = graph()->NewNode(m->Word32Clz(), reversed);
671 0 : return result;
672 : } else {
673 0 : return BuildI32Ctz(input);
674 : }
675 : }
676 : case wasm::kExprI32Popcnt: {
677 99 : if (m->Word32Popcnt().IsSupported()) {
678 99 : op = m->Word32Popcnt().op();
679 99 : break;
680 : } else {
681 0 : return BuildI32Popcnt(input);
682 : }
683 : }
684 : case wasm::kExprF32Floor: {
685 43 : if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
686 43 : op = m->Float32RoundDown().op();
687 43 : break;
688 : }
689 : case wasm::kExprF32Ceil: {
690 43 : if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
691 43 : op = m->Float32RoundUp().op();
692 43 : break;
693 : }
694 : case wasm::kExprF32Trunc: {
695 161 : if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
696 161 : op = m->Float32RoundTruncate().op();
697 161 : break;
698 : }
699 : case wasm::kExprF32NearestInt: {
700 14 : if (!m->Float32RoundTiesEven().IsSupported())
701 0 : return BuildF32NearestInt(input);
702 14 : op = m->Float32RoundTiesEven().op();
703 14 : break;
704 : }
705 : case wasm::kExprF64Floor: {
706 65 : if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
707 65 : op = m->Float64RoundDown().op();
708 65 : break;
709 : }
710 : case wasm::kExprF64Ceil: {
711 44 : if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
712 44 : op = m->Float64RoundUp().op();
713 44 : break;
714 : }
715 : case wasm::kExprF64Trunc: {
716 297 : if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
717 297 : op = m->Float64RoundTruncate().op();
718 297 : break;
719 : }
720 : case wasm::kExprF64NearestInt: {
721 14 : if (!m->Float64RoundTiesEven().IsSupported())
722 0 : return BuildF64NearestInt(input);
723 14 : op = m->Float64RoundTiesEven().op();
724 14 : break;
725 : }
726 : case wasm::kExprF64Acos: {
727 23 : return BuildF64Acos(input);
728 : }
729 : case wasm::kExprF64Asin: {
730 23 : return BuildF64Asin(input);
731 : }
732 : case wasm::kExprF64Atan:
733 23 : op = m->Float64Atan();
734 23 : break;
735 : case wasm::kExprF64Cos: {
736 198 : op = m->Float64Cos();
737 198 : break;
738 : }
739 : case wasm::kExprF64Sin: {
740 198 : op = m->Float64Sin();
741 198 : break;
742 : }
743 : case wasm::kExprF64Tan: {
744 23 : op = m->Float64Tan();
745 23 : break;
746 : }
747 : case wasm::kExprF64Exp: {
748 23 : op = m->Float64Exp();
749 23 : break;
750 : }
751 : case wasm::kExprF64Log:
752 23 : op = m->Float64Log();
753 23 : break;
754 : case wasm::kExprI32ConvertI64:
755 1117 : op = m->TruncateInt64ToInt32();
756 1117 : break;
757 : case wasm::kExprI64SConvertI32:
758 51 : op = m->ChangeInt32ToInt64();
759 51 : break;
760 : case wasm::kExprI64UConvertI32:
761 42 : op = m->ChangeUint32ToUint64();
762 42 : break;
763 : case wasm::kExprF64ReinterpretI64:
764 139 : op = m->BitcastInt64ToFloat64();
765 139 : break;
766 : case wasm::kExprI64ReinterpretF64:
767 272 : op = m->BitcastFloat64ToInt64();
768 272 : break;
769 : case wasm::kExprI64Clz:
770 29 : op = m->Word64Clz();
771 29 : break;
772 : case wasm::kExprI64Ctz: {
773 254 : OptionalOperator ctz64 = m->Word64Ctz();
774 254 : if (ctz64.IsSupported()) {
775 : op = ctz64.op();
776 254 : break;
777 0 : } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
778 : op = ctz64.placeholder();
779 : break;
780 0 : } else if (m->Word64ReverseBits().IsSupported()) {
781 0 : Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
782 0 : Node* result = graph()->NewNode(m->Word64Clz(), reversed);
783 0 : return result;
784 : } else {
785 0 : return BuildI64Ctz(input);
786 : }
787 : }
788 : case wasm::kExprI64Popcnt: {
789 36 : OptionalOperator popcnt64 = m->Word64Popcnt();
790 36 : if (popcnt64.IsSupported()) {
791 : op = popcnt64.op();
792 0 : } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
793 : op = popcnt64.placeholder();
794 : } else {
795 0 : return BuildI64Popcnt(input);
796 : }
797 36 : break;
798 : }
799 : case wasm::kExprI64Eqz:
800 58 : op = m->Word64Equal();
801 116 : return graph()->NewNode(op, input, jsgraph()->Int64Constant(0));
802 : case wasm::kExprF32SConvertI64:
803 74 : if (m->Is32()) {
804 0 : return BuildF32SConvertI64(input);
805 : }
806 74 : op = m->RoundInt64ToFloat32();
807 74 : break;
808 : case wasm::kExprF32UConvertI64:
809 14 : if (m->Is32()) {
810 0 : return BuildF32UConvertI64(input);
811 : }
812 14 : op = m->RoundUint64ToFloat32();
813 14 : break;
814 : case wasm::kExprF64SConvertI64:
815 58 : if (m->Is32()) {
816 0 : return BuildF64SConvertI64(input);
817 : }
818 58 : op = m->RoundInt64ToFloat64();
819 58 : break;
820 : case wasm::kExprF64UConvertI64:
821 14 : if (m->Is32()) {
822 0 : return BuildF64UConvertI64(input);
823 : }
824 14 : op = m->RoundUint64ToFloat64();
825 14 : break;
826 : case wasm::kExprI64SConvertF32:
827 21 : return BuildI64SConvertF32(input, position);
828 : case wasm::kExprI64SConvertF64:
829 44 : return BuildI64SConvertF64(input, position);
830 : case wasm::kExprI64UConvertF32:
831 21 : return BuildI64UConvertF32(input, position);
832 : case wasm::kExprI64UConvertF64:
833 21 : return BuildI64UConvertF64(input, position);
834 : case wasm::kExprI32AsmjsLoadMem8S:
835 3392 : return BuildAsmjsLoadMem(MachineType::Int8(), input);
836 : case wasm::kExprI32AsmjsLoadMem8U:
837 2693 : return BuildAsmjsLoadMem(MachineType::Uint8(), input);
838 : case wasm::kExprI32AsmjsLoadMem16S:
839 1087 : return BuildAsmjsLoadMem(MachineType::Int16(), input);
840 : case wasm::kExprI32AsmjsLoadMem16U:
841 664 : return BuildAsmjsLoadMem(MachineType::Uint16(), input);
842 : case wasm::kExprI32AsmjsLoadMem:
843 52399 : return BuildAsmjsLoadMem(MachineType::Int32(), input);
844 : case wasm::kExprF32AsmjsLoadMem:
845 7544 : return BuildAsmjsLoadMem(MachineType::Float32(), input);
846 : case wasm::kExprF64AsmjsLoadMem:
847 444 : return BuildAsmjsLoadMem(MachineType::Float64(), input);
848 : default:
849 0 : op = UnsupportedOpcode(opcode);
850 : }
851 20131 : return graph()->NewNode(op, input);
852 : }
853 :
854 877 : Node* WasmGraphBuilder::Float32Constant(float value) {
855 877 : return jsgraph()->Float32Constant(value);
856 : }
857 :
858 12191 : Node* WasmGraphBuilder::Float64Constant(double value) {
859 12191 : return jsgraph()->Float64Constant(value);
860 : }
861 :
862 176299 : Node* WasmGraphBuilder::HeapConstant(Handle<HeapObject> value) {
863 176299 : return jsgraph()->HeapConstant(value);
864 : }
865 :
866 : namespace {
867 487070 : Node* Branch(JSGraph* jsgraph, Node* cond, Node** true_node, Node** false_node,
868 : Node* control, BranchHint hint) {
869 : DCHECK_NOT_NULL(cond);
870 : DCHECK_NOT_NULL(control);
871 : Node* branch =
872 81178 : jsgraph->graph()->NewNode(jsgraph->common()->Branch(hint), cond, control);
873 162357 : *true_node = jsgraph->graph()->NewNode(jsgraph->common()->IfTrue(), branch);
874 162358 : *false_node = jsgraph->graph()->NewNode(jsgraph->common()->IfFalse(), branch);
875 81179 : return branch;
876 : }
877 : } // namespace
878 :
879 80803 : Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
880 80803 : Node** false_node) {
881 : return Branch(jsgraph(), cond, true_node, false_node, *control_,
882 161606 : BranchHint::kNone);
883 : }
884 :
885 0 : Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
886 0 : Node** false_node) {
887 : return Branch(jsgraph(), cond, true_node, false_node, *control_,
888 0 : BranchHint::kTrue);
889 : }
890 :
891 0 : Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
892 376 : Node** false_node) {
893 : return Branch(jsgraph(), cond, true_node, false_node, *control_,
894 752 : BranchHint::kFalse);
895 : }
896 :
897 : namespace {
898 16600 : Builtins::Name GetBuiltinIdForTrap(bool in_cctest, wasm::TrapReason reason) {
899 16600 : if (in_cctest) {
900 : // We use Builtins::builtin_count as a marker to tell the code generator
901 : // to generate a call to a testing c-function instead of a runtime
902 : // function. This code should only be called from a cctest.
903 : return Builtins::builtin_count;
904 : }
905 :
906 10502 : switch (reason) {
907 : #define TRAPREASON_TO_MESSAGE(name) \
908 : case wasm::k##name: \
909 : return Builtins::kThrowWasm##name;
910 7377 : FOREACH_WASM_TRAPREASON(TRAPREASON_TO_MESSAGE)
911 : #undef TRAPREASON_TO_MESSAGE
912 : default:
913 0 : UNREACHABLE();
914 : return Builtins::builtin_count;
915 : }
916 : }
917 : } // namespace
918 :
919 882 : Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
920 2646 : wasm::WasmCodePosition position) {
921 : // TODO(wasm): Introduce a testing flag instead of trying to infer it here.
922 : bool in_cctest =
923 1666 : !module_ || (module_->instance && module_->instance->context.is_null());
924 882 : int32_t trap_id = GetBuiltinIdForTrap(in_cctest, reason);
925 : Node* node = graph()->NewNode(jsgraph()->common()->TrapIf(trap_id), cond,
926 882 : Effect(), Control());
927 882 : *control_ = node;
928 882 : SetSourcePosition(node, position);
929 882 : return node;
930 : }
931 :
932 15717 : Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
933 47154 : wasm::WasmCodePosition position) {
934 : // TODO(wasm): Introduce a testing flag instead of trying to infer it here.
935 : bool in_cctest =
936 29852 : !module_ || (module_->instance && module_->instance->context.is_null());
937 15717 : int32_t trap_id = GetBuiltinIdForTrap(in_cctest, reason);
938 :
939 : Node* node = graph()->NewNode(jsgraph()->common()->TrapUnless(trap_id), cond,
940 15718 : Effect(), Control());
941 15716 : *control_ = node;
942 15716 : SetSourcePosition(node, position);
943 15716 : return node;
944 : }
945 :
946 : // Add a check that traps if {node} is equal to {val}.
947 545 : Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
948 : int32_t val,
949 256 : wasm::WasmCodePosition position) {
950 : Int32Matcher m(node);
951 862 : if (m.HasValue() && !m.Is(val)) return graph()->start();
952 416 : if (val == 0) {
953 288 : return TrapIfFalse(reason, node, position);
954 : } else {
955 : return TrapIfTrue(reason,
956 : graph()->NewNode(jsgraph()->machine()->Word32Equal(),
957 : node, jsgraph()->Int32Constant(val)),
958 384 : position);
959 : }
960 : }
961 :
962 : // Add a check that traps if {node} is zero.
963 0 : Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
964 : wasm::WasmCodePosition position) {
965 388 : return TrapIfEq32(reason, node, 0, position);
966 : }
967 :
968 : // Add a check that traps if {node} is equal to {val}.
969 437 : Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
970 : int64_t val,
971 648 : wasm::WasmCodePosition position) {
972 : Int64Matcher m(node);
973 670 : if (m.HasValue() && !m.Is(val)) return graph()->start();
974 : return TrapIfTrue(reason,
975 : graph()->NewNode(jsgraph()->machine()->Word64Equal(), node,
976 : jsgraph()->Int64Constant(val)),
977 972 : position);
978 : }
979 :
980 : // Add a check that traps if {node} is zero.
981 0 : Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
982 : wasm::WasmCodePosition position) {
983 332 : return TrapIfEq64(reason, node, 0, position);
984 : }
985 :
986 15278 : Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
987 22917 : return graph()->NewNode(jsgraph()->common()->Switch(count), key, *control_);
988 : }
989 :
990 48425 : Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
991 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
992 96850 : return graph()->NewNode(jsgraph()->common()->IfValue(value), sw);
993 : }
994 :
995 7624 : Node* WasmGraphBuilder::IfDefault(Node* sw) {
996 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
997 15248 : return graph()->NewNode(jsgraph()->common()->IfDefault(), sw);
998 : }
999 :
1000 775879 : Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1001 : DCHECK_NOT_NULL(*control_);
1002 : DCHECK_NOT_NULL(*effect_);
1003 :
1004 : static const int kStackAllocatedNodeBufferSize = 8;
1005 : Node* stack_buffer[kStackAllocatedNodeBufferSize];
1006 : std::vector<Node*> heap_buffer;
1007 :
1008 : Node** buf = stack_buffer;
1009 193904 : if (count + 3 > kStackAllocatedNodeBufferSize) {
1010 0 : heap_buffer.resize(count + 3);
1011 : buf = heap_buffer.data();
1012 : }
1013 :
1014 193904 : buf[0] = jsgraph()->Int32Constant(0);
1015 194024 : memcpy(buf + 1, vals, sizeof(void*) * count);
1016 194024 : buf[count + 1] = *effect_;
1017 194024 : buf[count + 2] = *control_;
1018 : Node* ret =
1019 581984 : graph()->NewNode(jsgraph()->common()->Return(count), count + 3, buf);
1020 :
1021 194047 : MergeControlToEnd(jsgraph(), ret);
1022 193987 : return ret;
1023 : }
1024 :
1025 514 : Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1026 :
1027 439 : Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1028 439 : TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1029 : ReturnVoid();
1030 439 : return nullptr;
1031 : }
1032 :
1033 24140 : Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1034 : static const int32_t kMask32 = 0x1f;
1035 48196 : if (!jsgraph()->machine()->Word32ShiftIsSafe()) {
1036 : // Shifts by constants are so common we pattern-match them here.
1037 : Int32Matcher match(node);
1038 42 : if (match.HasValue()) {
1039 0 : int32_t masked = (match.Value() & kMask32);
1040 0 : if (match.Value() != masked) node = jsgraph()->Int32Constant(masked);
1041 : } else {
1042 : node = graph()->NewNode(jsgraph()->machine()->Word32And(), node,
1043 84 : jsgraph()->Int32Constant(kMask32));
1044 : }
1045 : }
1046 24098 : return node;
1047 : }
1048 :
1049 680 : Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1050 : static const int64_t kMask64 = 0x3f;
1051 1290 : if (!jsgraph()->machine()->Word32ShiftIsSafe()) {
1052 : // Shifts by constants are so common we pattern-match them here.
1053 : Int64Matcher match(node);
1054 35 : if (match.HasValue()) {
1055 0 : int64_t masked = (match.Value() & kMask64);
1056 0 : if (match.Value() != masked) node = jsgraph()->Int64Constant(masked);
1057 : } else {
1058 : node = graph()->NewNode(jsgraph()->machine()->Word64And(), node,
1059 70 : jsgraph()->Int64Constant(kMask64));
1060 : }
1061 : }
1062 645 : return node;
1063 : }
1064 :
1065 0 : static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1066 : size_t size_in_bytes) {
1067 0 : switch (size_in_bytes) {
1068 : case 4:
1069 0 : return m->Word32ReverseBytes().IsSupported();
1070 : case 8:
1071 0 : return m->Word64ReverseBytes().IsSupported();
1072 : default:
1073 : break;
1074 : }
1075 : return false;
1076 : }
1077 :
1078 0 : Node* WasmGraphBuilder::BuildChangeEndianness(Node* node, MachineType memtype,
1079 0 : wasm::ValueType wasmtype) {
1080 : Node* result;
1081 : Node* value = node;
1082 0 : MachineOperatorBuilder* m = jsgraph()->machine();
1083 0 : int valueSizeInBytes = 1 << ElementSizeLog2Of(memtype.representation());
1084 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1085 : bool isFloat = false;
1086 :
1087 0 : switch (memtype.representation()) {
1088 : case MachineRepresentation::kFloat64:
1089 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1090 : isFloat = true;
1091 : case MachineRepresentation::kWord64:
1092 0 : result = jsgraph()->Int64Constant(0);
1093 0 : break;
1094 : case MachineRepresentation::kFloat32:
1095 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1096 : isFloat = true;
1097 : case MachineRepresentation::kWord32:
1098 : case MachineRepresentation::kWord16:
1099 0 : result = jsgraph()->Int32Constant(0);
1100 0 : break;
1101 : case MachineRepresentation::kWord8:
1102 : // No need to change endianness for byte size, return original node
1103 : return node;
1104 : break;
1105 : default:
1106 0 : UNREACHABLE();
1107 : break;
1108 : }
1109 :
1110 : int i;
1111 : uint32_t shiftCount;
1112 :
1113 0 : if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1114 0 : switch (valueSizeInBytes) {
1115 : case 2:
1116 : result =
1117 : graph()->NewNode(m->Word32ReverseBytes().op(),
1118 : graph()->NewNode(m->Word32Shl(), value,
1119 0 : jsgraph()->Int32Constant(16)));
1120 0 : break;
1121 : case 4:
1122 0 : result = graph()->NewNode(m->Word32ReverseBytes().op(), value);
1123 0 : break;
1124 : case 8:
1125 0 : result = graph()->NewNode(m->Word64ReverseBytes().op(), value);
1126 0 : break;
1127 : default:
1128 0 : UNREACHABLE();
1129 : }
1130 : } else {
1131 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1132 : i += 8, shiftCount -= 16) {
1133 : Node* shiftLower;
1134 : Node* shiftHigher;
1135 : Node* lowerByte;
1136 : Node* higherByte;
1137 :
1138 : DCHECK(shiftCount > 0);
1139 : DCHECK((shiftCount + 8) % 16 == 0);
1140 :
1141 0 : if (valueSizeInBits > 32) {
1142 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1143 0 : jsgraph()->Int64Constant(shiftCount));
1144 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1145 0 : jsgraph()->Int64Constant(shiftCount));
1146 : lowerByte = graph()->NewNode(
1147 : m->Word64And(), shiftLower,
1148 : jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1149 0 : << (valueSizeInBits - 8 - i)));
1150 : higherByte = graph()->NewNode(
1151 : m->Word64And(), shiftHigher,
1152 0 : jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1153 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1154 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1155 : } else {
1156 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1157 0 : jsgraph()->Int32Constant(shiftCount));
1158 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1159 0 : jsgraph()->Int32Constant(shiftCount));
1160 : lowerByte = graph()->NewNode(
1161 : m->Word32And(), shiftLower,
1162 : jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1163 0 : << (valueSizeInBits - 8 - i)));
1164 : higherByte = graph()->NewNode(
1165 : m->Word32And(), shiftHigher,
1166 0 : jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1167 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1168 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1169 : }
1170 : }
1171 : }
1172 :
1173 0 : if (isFloat) {
1174 0 : switch (memtype.representation()) {
1175 : case MachineRepresentation::kFloat64:
1176 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1177 0 : break;
1178 : case MachineRepresentation::kFloat32:
1179 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1180 0 : break;
1181 : default:
1182 0 : UNREACHABLE();
1183 : break;
1184 : }
1185 : }
1186 :
1187 : // We need to sign extend the value
1188 0 : if (memtype.IsSigned()) {
1189 : DCHECK(!isFloat);
1190 0 : if (valueSizeInBits < 32) {
1191 : Node* shiftBitCount;
1192 : // Perform sign extension using following trick
1193 : // result = (x << machine_width - type_width) >> (machine_width -
1194 : // type_width)
1195 0 : if (wasmtype == wasm::kWasmI64) {
1196 0 : shiftBitCount = jsgraph()->Int32Constant(64 - valueSizeInBits);
1197 : result = graph()->NewNode(
1198 : m->Word64Sar(),
1199 : graph()->NewNode(m->Word64Shl(),
1200 : graph()->NewNode(m->ChangeInt32ToInt64(), result),
1201 : shiftBitCount),
1202 0 : shiftBitCount);
1203 0 : } else if (wasmtype == wasm::kWasmI32) {
1204 0 : shiftBitCount = jsgraph()->Int32Constant(32 - valueSizeInBits);
1205 : result = graph()->NewNode(
1206 : m->Word32Sar(),
1207 : graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1208 0 : shiftBitCount);
1209 : }
1210 : }
1211 : }
1212 :
1213 0 : return result;
1214 : }
1215 :
1216 28 : Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1217 : Node* result = Unop(
1218 : wasm::kExprF32ReinterpretI32,
1219 : Binop(wasm::kExprI32Ior,
1220 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1221 : jsgraph()->Int32Constant(0x7fffffff)),
1222 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1223 28 : jsgraph()->Int32Constant(0x80000000))));
1224 :
1225 14 : return result;
1226 : }
1227 :
1228 28 : Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1229 : #if WASM_64
1230 : Node* result = Unop(
1231 : wasm::kExprF64ReinterpretI64,
1232 : Binop(wasm::kExprI64Ior,
1233 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1234 : jsgraph()->Int64Constant(0x7fffffffffffffff)),
1235 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1236 28 : jsgraph()->Int64Constant(0x8000000000000000))));
1237 :
1238 14 : return result;
1239 : #else
1240 : MachineOperatorBuilder* m = jsgraph()->machine();
1241 :
1242 : Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1243 : Node* high_word_right =
1244 : graph()->NewNode(m->Float64ExtractHighWord32(), right);
1245 :
1246 : Node* new_high_word =
1247 : Binop(wasm::kExprI32Ior, Binop(wasm::kExprI32And, high_word_left,
1248 : jsgraph()->Int32Constant(0x7fffffff)),
1249 : Binop(wasm::kExprI32And, high_word_right,
1250 : jsgraph()->Int32Constant(0x80000000)));
1251 :
1252 : return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1253 : #endif
1254 : }
1255 :
1256 133 : Node* WasmGraphBuilder::BuildI32SConvertF32(Node* input,
1257 133 : wasm::WasmCodePosition position) {
1258 133 : MachineOperatorBuilder* m = jsgraph()->machine();
1259 : // Truncation of the input value is needed for the overflow check later.
1260 133 : Node* trunc = Unop(wasm::kExprF32Trunc, input);
1261 133 : Node* result = graph()->NewNode(m->TruncateFloat32ToInt32(), trunc);
1262 :
1263 : // Convert the result back to f64. If we end up at a different value than the
1264 : // truncated input value, then there has been an overflow and we trap.
1265 133 : Node* check = Unop(wasm::kExprF32SConvertI32, result);
1266 133 : Node* overflow = Binop(wasm::kExprF32Ne, trunc, check);
1267 133 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, overflow, position);
1268 :
1269 133 : return result;
1270 : }
1271 :
1272 269 : Node* WasmGraphBuilder::BuildI32SConvertF64(Node* input,
1273 269 : wasm::WasmCodePosition position) {
1274 269 : MachineOperatorBuilder* m = jsgraph()->machine();
1275 : // Truncation of the input value is needed for the overflow check later.
1276 269 : Node* trunc = Unop(wasm::kExprF64Trunc, input);
1277 269 : Node* result = graph()->NewNode(m->ChangeFloat64ToInt32(), trunc);
1278 :
1279 : // Convert the result back to f64. If we end up at a different value than the
1280 : // truncated input value, then there has been an overflow and we trap.
1281 269 : Node* check = Unop(wasm::kExprF64SConvertI32, result);
1282 269 : Node* overflow = Binop(wasm::kExprF64Ne, trunc, check);
1283 269 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, overflow, position);
1284 :
1285 269 : return result;
1286 : }
1287 :
1288 14 : Node* WasmGraphBuilder::BuildI32UConvertF32(Node* input,
1289 14 : wasm::WasmCodePosition position) {
1290 14 : MachineOperatorBuilder* m = jsgraph()->machine();
1291 : // Truncation of the input value is needed for the overflow check later.
1292 14 : Node* trunc = Unop(wasm::kExprF32Trunc, input);
1293 14 : Node* result = graph()->NewNode(m->TruncateFloat32ToUint32(), trunc);
1294 :
1295 : // Convert the result back to f32. If we end up at a different value than the
1296 : // truncated input value, then there has been an overflow and we trap.
1297 14 : Node* check = Unop(wasm::kExprF32UConvertI32, result);
1298 14 : Node* overflow = Binop(wasm::kExprF32Ne, trunc, check);
1299 14 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, overflow, position);
1300 :
1301 14 : return result;
1302 : }
1303 :
1304 14 : Node* WasmGraphBuilder::BuildI32UConvertF64(Node* input,
1305 14 : wasm::WasmCodePosition position) {
1306 14 : MachineOperatorBuilder* m = jsgraph()->machine();
1307 : // Truncation of the input value is needed for the overflow check later.
1308 14 : Node* trunc = Unop(wasm::kExprF64Trunc, input);
1309 14 : Node* result = graph()->NewNode(m->TruncateFloat64ToUint32(), trunc);
1310 :
1311 : // Convert the result back to f64. If we end up at a different value than the
1312 : // truncated input value, then there has been an overflow and we trap.
1313 14 : Node* check = Unop(wasm::kExprF64UConvertI32, result);
1314 14 : Node* overflow = Binop(wasm::kExprF64Ne, trunc, check);
1315 14 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, overflow, position);
1316 :
1317 14 : return result;
1318 : }
1319 :
1320 7 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1321 7 : MachineOperatorBuilder* m = jsgraph()->machine();
1322 : // asm.js must use the wacky JS semantics.
1323 7 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1324 14 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1325 : }
1326 :
1327 70 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1328 70 : MachineOperatorBuilder* m = jsgraph()->machine();
1329 : // asm.js must use the wacky JS semantics.
1330 140 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1331 : }
1332 :
1333 7 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1334 7 : MachineOperatorBuilder* m = jsgraph()->machine();
1335 : // asm.js must use the wacky JS semantics.
1336 7 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1337 14 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1338 : }
1339 :
1340 7 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1341 7 : MachineOperatorBuilder* m = jsgraph()->machine();
1342 : // asm.js must use the wacky JS semantics.
1343 14 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1344 : }
1345 :
1346 0 : Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1347 0 : MachineRepresentation input_type) {
1348 : Node* stack_slot_param =
1349 0 : graph()->NewNode(jsgraph()->machine()->StackSlot(input_type));
1350 :
1351 : const Operator* store_op = jsgraph()->machine()->Store(
1352 0 : StoreRepresentation(input_type, kNoWriteBarrier));
1353 : *effect_ =
1354 : graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0),
1355 0 : input, *effect_, *control_);
1356 :
1357 : MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 1);
1358 : sig_builder.AddReturn(MachineType::Int32());
1359 : sig_builder.AddParam(MachineType::Pointer());
1360 :
1361 0 : Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref));
1362 0 : Node* args[] = {function, stack_slot_param};
1363 :
1364 0 : return BuildCCall(sig_builder.Build(), args);
1365 : }
1366 :
1367 0 : Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1368 : return BuildBitCountingCall(
1369 : input, ExternalReference::wasm_word32_ctz(jsgraph()->isolate()),
1370 0 : MachineRepresentation::kWord32);
1371 : }
1372 :
1373 0 : Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1374 : return Unop(wasm::kExprI64UConvertI32,
1375 : BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(
1376 : jsgraph()->isolate()),
1377 0 : MachineRepresentation::kWord64));
1378 : }
1379 :
1380 0 : Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1381 : return BuildBitCountingCall(
1382 : input, ExternalReference::wasm_word32_popcnt(jsgraph()->isolate()),
1383 0 : MachineRepresentation::kWord32);
1384 : }
1385 :
1386 0 : Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1387 : return Unop(wasm::kExprI64UConvertI32,
1388 : BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(
1389 : jsgraph()->isolate()),
1390 0 : MachineRepresentation::kWord64));
1391 : }
1392 :
1393 0 : Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1394 0 : MachineType type = MachineType::Float32();
1395 : ExternalReference ref =
1396 0 : ExternalReference::wasm_f32_trunc(jsgraph()->isolate());
1397 :
1398 0 : return BuildCFuncInstruction(ref, type, input);
1399 : }
1400 :
1401 0 : Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1402 0 : MachineType type = MachineType::Float32();
1403 : ExternalReference ref =
1404 0 : ExternalReference::wasm_f32_floor(jsgraph()->isolate());
1405 0 : return BuildCFuncInstruction(ref, type, input);
1406 : }
1407 :
1408 0 : Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1409 0 : MachineType type = MachineType::Float32();
1410 : ExternalReference ref =
1411 0 : ExternalReference::wasm_f32_ceil(jsgraph()->isolate());
1412 0 : return BuildCFuncInstruction(ref, type, input);
1413 : }
1414 :
1415 0 : Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1416 0 : MachineType type = MachineType::Float32();
1417 : ExternalReference ref =
1418 0 : ExternalReference::wasm_f32_nearest_int(jsgraph()->isolate());
1419 0 : return BuildCFuncInstruction(ref, type, input);
1420 : }
1421 :
1422 0 : Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1423 0 : MachineType type = MachineType::Float64();
1424 : ExternalReference ref =
1425 0 : ExternalReference::wasm_f64_trunc(jsgraph()->isolate());
1426 0 : return BuildCFuncInstruction(ref, type, input);
1427 : }
1428 :
1429 0 : Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1430 0 : MachineType type = MachineType::Float64();
1431 : ExternalReference ref =
1432 0 : ExternalReference::wasm_f64_floor(jsgraph()->isolate());
1433 0 : return BuildCFuncInstruction(ref, type, input);
1434 : }
1435 :
1436 0 : Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1437 0 : MachineType type = MachineType::Float64();
1438 : ExternalReference ref =
1439 0 : ExternalReference::wasm_f64_ceil(jsgraph()->isolate());
1440 0 : return BuildCFuncInstruction(ref, type, input);
1441 : }
1442 :
1443 0 : Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1444 0 : MachineType type = MachineType::Float64();
1445 : ExternalReference ref =
1446 0 : ExternalReference::wasm_f64_nearest_int(jsgraph()->isolate());
1447 0 : return BuildCFuncInstruction(ref, type, input);
1448 : }
1449 :
1450 46 : Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1451 23 : MachineType type = MachineType::Float64();
1452 : ExternalReference ref =
1453 23 : ExternalReference::f64_acos_wrapper_function(jsgraph()->isolate());
1454 23 : return BuildCFuncInstruction(ref, type, input);
1455 : }
1456 :
1457 46 : Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1458 23 : MachineType type = MachineType::Float64();
1459 : ExternalReference ref =
1460 23 : ExternalReference::f64_asin_wrapper_function(jsgraph()->isolate());
1461 23 : return BuildCFuncInstruction(ref, type, input);
1462 : }
1463 :
1464 28 : Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1465 14 : MachineType type = MachineType::Float64();
1466 : ExternalReference ref =
1467 14 : ExternalReference::wasm_float64_pow(jsgraph()->isolate());
1468 14 : return BuildCFuncInstruction(ref, type, left, right);
1469 : }
1470 :
1471 56 : Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1472 28 : MachineType type = MachineType::Float64();
1473 : ExternalReference ref =
1474 28 : ExternalReference::f64_mod_wrapper_function(jsgraph()->isolate());
1475 28 : return BuildCFuncInstruction(ref, type, left, right);
1476 : }
1477 :
1478 88 : Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1479 : MachineType type, Node* input0,
1480 742 : Node* input1) {
1481 : // We do truncation by calling a C function which calculates the result.
1482 : // The input is passed to the C function as a double*'s to avoid double
1483 : // parameters. For this we reserve slots on the stack, store the parameters
1484 : // in those slots, pass pointers to the slot to the C function,
1485 : // and after calling the C function we collect the return value from
1486 : // the stack slot.
1487 :
1488 : Node* stack_slot_param0 =
1489 88 : graph()->NewNode(jsgraph()->machine()->StackSlot(type.representation()));
1490 :
1491 : const Operator* store_op0 = jsgraph()->machine()->Store(
1492 88 : StoreRepresentation(type.representation(), kNoWriteBarrier));
1493 : *effect_ = graph()->NewNode(store_op0, stack_slot_param0,
1494 : jsgraph()->Int32Constant(0), input0, *effect_,
1495 264 : *control_);
1496 :
1497 88 : Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref));
1498 88 : Node** args = Buffer(5);
1499 88 : args[0] = function;
1500 88 : args[1] = stack_slot_param0;
1501 : int input_count = 1;
1502 :
1503 88 : if (input1 != nullptr) {
1504 : Node* stack_slot_param1 = graph()->NewNode(
1505 42 : jsgraph()->machine()->StackSlot(type.representation()));
1506 : const Operator* store_op1 = jsgraph()->machine()->Store(
1507 42 : StoreRepresentation(type.representation(), kNoWriteBarrier));
1508 : *effect_ = graph()->NewNode(store_op1, stack_slot_param1,
1509 : jsgraph()->Int32Constant(0), input1, *effect_,
1510 126 : *control_);
1511 42 : args[2] = stack_slot_param1;
1512 : ++input_count;
1513 : }
1514 :
1515 : Signature<MachineType>::Builder sig_builder(jsgraph()->zone(), 0,
1516 88 : input_count);
1517 : sig_builder.AddParam(MachineType::Pointer());
1518 88 : if (input1 != nullptr) {
1519 : sig_builder.AddParam(MachineType::Pointer());
1520 : }
1521 88 : BuildCCall(sig_builder.Build(), args);
1522 :
1523 88 : const Operator* load_op = jsgraph()->machine()->Load(type);
1524 :
1525 : Node* load =
1526 : graph()->NewNode(load_op, stack_slot_param0, jsgraph()->Int32Constant(0),
1527 176 : *effect_, *control_);
1528 88 : *effect_ = load;
1529 88 : return load;
1530 : }
1531 :
1532 0 : Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1533 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1534 : return BuildIntToFloatConversionInstruction(
1535 : input, ExternalReference::wasm_int64_to_float32(jsgraph()->isolate()),
1536 0 : MachineRepresentation::kWord64, MachineType::Float32());
1537 : }
1538 0 : Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1539 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1540 : return BuildIntToFloatConversionInstruction(
1541 : input, ExternalReference::wasm_uint64_to_float32(jsgraph()->isolate()),
1542 0 : MachineRepresentation::kWord64, MachineType::Float32());
1543 : }
1544 0 : Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1545 : return BuildIntToFloatConversionInstruction(
1546 : input, ExternalReference::wasm_int64_to_float64(jsgraph()->isolate()),
1547 0 : MachineRepresentation::kWord64, MachineType::Float64());
1548 : }
1549 0 : Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1550 : return BuildIntToFloatConversionInstruction(
1551 : input, ExternalReference::wasm_uint64_to_float64(jsgraph()->isolate()),
1552 0 : MachineRepresentation::kWord64, MachineType::Float64());
1553 : }
1554 :
1555 0 : Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1556 : Node* input, ExternalReference ref,
1557 : MachineRepresentation parameter_representation,
1558 0 : const MachineType result_type) {
1559 : Node* stack_slot_param = graph()->NewNode(
1560 0 : jsgraph()->machine()->StackSlot(parameter_representation));
1561 : Node* stack_slot_result = graph()->NewNode(
1562 0 : jsgraph()->machine()->StackSlot(result_type.representation()));
1563 : const Operator* store_op = jsgraph()->machine()->Store(
1564 0 : StoreRepresentation(parameter_representation, kNoWriteBarrier));
1565 : *effect_ =
1566 : graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0),
1567 0 : input, *effect_, *control_);
1568 : MachineSignature::Builder sig_builder(jsgraph()->zone(), 0, 2);
1569 : sig_builder.AddParam(MachineType::Pointer());
1570 : sig_builder.AddParam(MachineType::Pointer());
1571 0 : Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref));
1572 0 : Node* args[] = {function, stack_slot_param, stack_slot_result};
1573 0 : BuildCCall(sig_builder.Build(), args);
1574 0 : const Operator* load_op = jsgraph()->machine()->Load(result_type);
1575 : Node* load =
1576 : graph()->NewNode(load_op, stack_slot_result, jsgraph()->Int32Constant(0),
1577 0 : *effect_, *control_);
1578 0 : *effect_ = load;
1579 0 : return load;
1580 : }
1581 :
1582 21 : Node* WasmGraphBuilder::BuildI64SConvertF32(Node* input,
1583 21 : wasm::WasmCodePosition position) {
1584 42 : if (jsgraph()->machine()->Is32()) {
1585 : return BuildFloatToIntConversionInstruction(
1586 : input, ExternalReference::wasm_float32_to_int64(jsgraph()->isolate()),
1587 0 : MachineRepresentation::kFloat32, MachineType::Int64(), position);
1588 : } else {
1589 : Node* trunc = graph()->NewNode(
1590 21 : jsgraph()->machine()->TryTruncateFloat32ToInt64(), input);
1591 : Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc,
1592 21 : graph()->start());
1593 : Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc,
1594 21 : graph()->start());
1595 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position);
1596 21 : return result;
1597 : }
1598 : }
1599 :
1600 21 : Node* WasmGraphBuilder::BuildI64UConvertF32(Node* input,
1601 21 : wasm::WasmCodePosition position) {
1602 42 : if (jsgraph()->machine()->Is32()) {
1603 : return BuildFloatToIntConversionInstruction(
1604 : input, ExternalReference::wasm_float32_to_uint64(jsgraph()->isolate()),
1605 0 : MachineRepresentation::kFloat32, MachineType::Int64(), position);
1606 : } else {
1607 : Node* trunc = graph()->NewNode(
1608 21 : jsgraph()->machine()->TryTruncateFloat32ToUint64(), input);
1609 : Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc,
1610 21 : graph()->start());
1611 : Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc,
1612 21 : graph()->start());
1613 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position);
1614 21 : return result;
1615 : }
1616 : }
1617 :
1618 44 : Node* WasmGraphBuilder::BuildI64SConvertF64(Node* input,
1619 44 : wasm::WasmCodePosition position) {
1620 88 : if (jsgraph()->machine()->Is32()) {
1621 : return BuildFloatToIntConversionInstruction(
1622 : input, ExternalReference::wasm_float64_to_int64(jsgraph()->isolate()),
1623 0 : MachineRepresentation::kFloat64, MachineType::Int64(), position);
1624 : } else {
1625 : Node* trunc = graph()->NewNode(
1626 44 : jsgraph()->machine()->TryTruncateFloat64ToInt64(), input);
1627 : Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc,
1628 44 : graph()->start());
1629 : Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc,
1630 44 : graph()->start());
1631 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position);
1632 44 : return result;
1633 : }
1634 : }
1635 :
1636 21 : Node* WasmGraphBuilder::BuildI64UConvertF64(Node* input,
1637 21 : wasm::WasmCodePosition position) {
1638 42 : if (jsgraph()->machine()->Is32()) {
1639 : return BuildFloatToIntConversionInstruction(
1640 : input, ExternalReference::wasm_float64_to_uint64(jsgraph()->isolate()),
1641 0 : MachineRepresentation::kFloat64, MachineType::Int64(), position);
1642 : } else {
1643 : Node* trunc = graph()->NewNode(
1644 21 : jsgraph()->machine()->TryTruncateFloat64ToUint64(), input);
1645 : Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc,
1646 21 : graph()->start());
1647 : Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc,
1648 21 : graph()->start());
1649 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position);
1650 21 : return result;
1651 : }
1652 : }
1653 :
1654 0 : Node* WasmGraphBuilder::BuildFloatToIntConversionInstruction(
1655 : Node* input, ExternalReference ref,
1656 : MachineRepresentation parameter_representation,
1657 0 : const MachineType result_type, wasm::WasmCodePosition position) {
1658 : Node* stack_slot_param = graph()->NewNode(
1659 0 : jsgraph()->machine()->StackSlot(parameter_representation));
1660 : Node* stack_slot_result = graph()->NewNode(
1661 0 : jsgraph()->machine()->StackSlot(result_type.representation()));
1662 : const Operator* store_op = jsgraph()->machine()->Store(
1663 0 : StoreRepresentation(parameter_representation, kNoWriteBarrier));
1664 : *effect_ =
1665 : graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0),
1666 0 : input, *effect_, *control_);
1667 : MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 2);
1668 : sig_builder.AddReturn(MachineType::Int32());
1669 : sig_builder.AddParam(MachineType::Pointer());
1670 : sig_builder.AddParam(MachineType::Pointer());
1671 0 : Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref));
1672 0 : Node* args[] = {function, stack_slot_param, stack_slot_result};
1673 : ZeroCheck32(wasm::kTrapFloatUnrepresentable,
1674 0 : BuildCCall(sig_builder.Build(), args), position);
1675 0 : const Operator* load_op = jsgraph()->machine()->Load(result_type);
1676 : Node* load =
1677 : graph()->NewNode(load_op, stack_slot_result, jsgraph()->Int32Constant(0),
1678 0 : *effect_, *control_);
1679 0 : *effect_ = load;
1680 0 : return load;
1681 : }
1682 :
1683 6342 : Node* WasmGraphBuilder::GrowMemory(Node* input) {
1684 : // GrowMemory will not be called from asm.js, hence we cannot be in
1685 : // lazy-compilation mode, hence the instance will be set.
1686 : DCHECK_NOT_NULL(module_);
1687 : DCHECK_NOT_NULL(module_->instance);
1688 :
1689 : Diamond check_input_range(
1690 : graph(), jsgraph()->common(),
1691 : graph()->NewNode(jsgraph()->machine()->Uint32LessThanOrEqual(), input,
1692 : jsgraph()->Uint32Constant(FLAG_wasm_max_mem_pages)),
1693 3624 : BranchHint::kTrue);
1694 :
1695 906 : check_input_range.Chain(*control_);
1696 :
1697 906 : Node* parameters[] = {BuildChangeUint32ToSmi(input)};
1698 906 : Node* old_effect = *effect_;
1699 : Node* call = BuildCallToRuntime(Runtime::kWasmGrowMemory, jsgraph(),
1700 : parameters, arraysize(parameters), effect_,
1701 906 : &check_input_range.if_true);
1702 :
1703 906 : Node* result = BuildChangeSmiToInt32(call);
1704 :
1705 : result = check_input_range.Phi(MachineRepresentation::kWord32, result,
1706 906 : jsgraph()->Int32Constant(-1));
1707 : *effect_ = graph()->NewNode(jsgraph()->common()->EffectPhi(2), *effect_,
1708 2718 : old_effect, check_input_range.merge);
1709 906 : *control_ = check_input_range.merge;
1710 906 : return result;
1711 : }
1712 :
1713 330 : Node* WasmGraphBuilder::Throw(Node* input) {
1714 165 : MachineOperatorBuilder* machine = jsgraph()->machine();
1715 :
1716 : // Pass the thrown value as two SMIs:
1717 : //
1718 : // upper = static_cast<uint32_t>(input) >> 16;
1719 : // lower = input & 0xFFFF;
1720 : //
1721 : // This is needed because we can't safely call BuildChangeInt32ToTagged from
1722 : // this method.
1723 : //
1724 : // TODO(wasm): figure out how to properly pass this to the runtime function.
1725 : Node* upper = BuildChangeInt32ToSmi(
1726 330 : graph()->NewNode(machine->Word32Shr(), input, Int32Constant(16)));
1727 : Node* lower = BuildChangeInt32ToSmi(
1728 330 : graph()->NewNode(machine->Word32And(), input, Int32Constant(0xFFFFu)));
1729 :
1730 165 : Node* parameters[] = {lower, upper}; // thrown value
1731 : return BuildCallToRuntime(Runtime::kWasmThrow, jsgraph(), parameters,
1732 330 : arraysize(parameters), effect_, control_);
1733 : }
1734 :
1735 120 : Node* WasmGraphBuilder::Catch(Node* input, wasm::WasmCodePosition position) {
1736 120 : CommonOperatorBuilder* common = jsgraph()->common();
1737 :
1738 120 : Node* parameters[] = {input}; // caught value
1739 : Node* value =
1740 : BuildCallToRuntime(Runtime::kWasmGetCaughtExceptionValue, jsgraph(),
1741 120 : parameters, arraysize(parameters), effect_, control_);
1742 :
1743 : Node* is_smi;
1744 : Node* is_heap;
1745 120 : BranchExpectFalse(BuildTestNotSmi(value), &is_heap, &is_smi);
1746 :
1747 : // is_smi
1748 120 : Node* smi_i32 = BuildChangeSmiToInt32(value);
1749 120 : Node* is_smi_effect = *effect_;
1750 :
1751 : // is_heap
1752 120 : *control_ = is_heap;
1753 120 : Node* heap_f64 = BuildLoadHeapNumberValue(value, is_heap);
1754 :
1755 : // *control_ needs to point to the current control dependency (is_heap) in
1756 : // case BuildI32SConvertF64 needs to insert nodes that depend on the "current"
1757 : // control node.
1758 120 : Node* heap_i32 = BuildI32SConvertF64(heap_f64, position);
1759 : // *control_ contains the control node that should be used when merging the
1760 : // result for the catch clause. It may be different than *control_ because
1761 : // BuildI32SConvertF64 may introduce a new control node (used for trapping if
1762 : // heap_f64 cannot be converted to an i32.
1763 120 : is_heap = *control_;
1764 120 : Node* is_heap_effect = *effect_;
1765 :
1766 120 : Node* merge = graph()->NewNode(common->Merge(2), is_heap, is_smi);
1767 : Node* effect_merge = graph()->NewNode(common->EffectPhi(2), is_heap_effect,
1768 120 : is_smi_effect, merge);
1769 :
1770 : Node* value_i32 = graph()->NewNode(
1771 120 : common->Phi(MachineRepresentation::kWord32, 2), heap_i32, smi_i32, merge);
1772 :
1773 120 : *control_ = merge;
1774 120 : *effect_ = effect_merge;
1775 120 : return value_i32;
1776 : }
1777 :
1778 151 : Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
1779 430 : wasm::WasmCodePosition position) {
1780 151 : MachineOperatorBuilder* m = jsgraph()->machine();
1781 : ZeroCheck32(wasm::kTrapDivByZero, right, position);
1782 151 : Node* before = *control_;
1783 : Node* denom_is_m1;
1784 : Node* denom_is_not_m1;
1785 : BranchExpectFalse(
1786 : graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)),
1787 151 : &denom_is_m1, &denom_is_not_m1);
1788 151 : *control_ = denom_is_m1;
1789 151 : TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
1790 151 : if (*control_ != denom_is_m1) {
1791 : *control_ = graph()->NewNode(jsgraph()->common()->Merge(2), denom_is_not_m1,
1792 384 : *control_);
1793 : } else {
1794 23 : *control_ = before;
1795 : }
1796 302 : return graph()->NewNode(m->Int32Div(), left, right, *control_);
1797 : }
1798 :
1799 58 : Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
1800 232 : wasm::WasmCodePosition position) {
1801 58 : MachineOperatorBuilder* m = jsgraph()->machine();
1802 :
1803 : ZeroCheck32(wasm::kTrapRemByZero, right, position);
1804 :
1805 : Diamond d(
1806 : graph(), jsgraph()->common(),
1807 : graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)),
1808 174 : BranchHint::kFalse);
1809 58 : d.Chain(*control_);
1810 :
1811 : return d.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0),
1812 116 : graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
1813 : }
1814 :
1815 105 : Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
1816 105 : wasm::WasmCodePosition position) {
1817 105 : MachineOperatorBuilder* m = jsgraph()->machine();
1818 : return graph()->NewNode(m->Uint32Div(), left, right,
1819 210 : ZeroCheck32(wasm::kTrapDivByZero, right, position));
1820 : }
1821 :
1822 74 : Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
1823 74 : wasm::WasmCodePosition position) {
1824 74 : MachineOperatorBuilder* m = jsgraph()->machine();
1825 : return graph()->NewNode(m->Uint32Mod(), left, right,
1826 148 : ZeroCheck32(wasm::kTrapRemByZero, right, position));
1827 : }
1828 :
1829 587 : Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
1830 262 : MachineOperatorBuilder* m = jsgraph()->machine();
1831 :
1832 : Int32Matcher mr(right);
1833 262 : if (mr.HasValue()) {
1834 197 : if (mr.Value() == 0) {
1835 8 : return jsgraph()->Int32Constant(0);
1836 189 : } else if (mr.Value() == -1) {
1837 : // The result is the negation of the left input.
1838 16 : return graph()->NewNode(m->Int32Sub(), jsgraph()->Int32Constant(0), left);
1839 : }
1840 362 : return graph()->NewNode(m->Int32Div(), left, right, *control_);
1841 : }
1842 :
1843 : // asm.js semantics return 0 on divide or mod by zero.
1844 65 : if (m->Int32DivIsSafe()) {
1845 : // The hardware instruction does the right thing (e.g. arm).
1846 0 : return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
1847 : }
1848 :
1849 : // Check denominator for zero.
1850 : Diamond z(
1851 : graph(), jsgraph()->common(),
1852 : graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)),
1853 195 : BranchHint::kFalse);
1854 :
1855 : // Check numerator for -1. (avoid minint / -1 case).
1856 : Diamond n(
1857 : graph(), jsgraph()->common(),
1858 : graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)),
1859 195 : BranchHint::kFalse);
1860 :
1861 65 : Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
1862 : Node* neg =
1863 65 : graph()->NewNode(m->Int32Sub(), jsgraph()->Int32Constant(0), left);
1864 :
1865 : return n.Phi(
1866 : MachineRepresentation::kWord32, neg,
1867 65 : z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), div));
1868 : }
1869 :
1870 468 : Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
1871 394 : CommonOperatorBuilder* c = jsgraph()->common();
1872 394 : MachineOperatorBuilder* m = jsgraph()->machine();
1873 394 : Node* const zero = jsgraph()->Int32Constant(0);
1874 :
1875 : Int32Matcher mr(right);
1876 394 : if (mr.HasValue()) {
1877 320 : if (mr.Value() == 0 || mr.Value() == -1) {
1878 : return zero;
1879 : }
1880 608 : return graph()->NewNode(m->Int32Mod(), left, right, *control_);
1881 : }
1882 :
1883 : // General case for signed integer modulus, with optimization for (unknown)
1884 : // power of 2 right hand side.
1885 : //
1886 : // if 0 < right then
1887 : // msk = right - 1
1888 : // if right & msk != 0 then
1889 : // left % right
1890 : // else
1891 : // if left < 0 then
1892 : // -(-left & msk)
1893 : // else
1894 : // left & msk
1895 : // else
1896 : // if right < -1 then
1897 : // left % right
1898 : // else
1899 : // zero
1900 : //
1901 : // Note: We do not use the Diamond helper class here, because it really hurts
1902 : // readability with nested diamonds.
1903 74 : Node* const minus_one = jsgraph()->Int32Constant(-1);
1904 :
1905 74 : const Operator* const merge_op = c->Merge(2);
1906 74 : const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
1907 :
1908 74 : Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
1909 : Node* branch0 =
1910 74 : graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
1911 :
1912 74 : Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
1913 : Node* true0;
1914 : {
1915 74 : Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
1916 :
1917 74 : Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
1918 74 : Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
1919 :
1920 74 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
1921 74 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
1922 :
1923 74 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
1924 : Node* false1;
1925 : {
1926 74 : Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
1927 : Node* branch2 =
1928 74 : graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
1929 :
1930 74 : Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
1931 : Node* true2 = graph()->NewNode(
1932 : m->Int32Sub(), zero,
1933 : graph()->NewNode(m->Word32And(),
1934 222 : graph()->NewNode(m->Int32Sub(), zero, left), msk));
1935 :
1936 74 : Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
1937 74 : Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
1938 :
1939 : if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
1940 : false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
1941 : }
1942 :
1943 : if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
1944 : true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
1945 : }
1946 :
1947 74 : Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
1948 : Node* false0;
1949 : {
1950 74 : Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
1951 : Node* branch1 =
1952 74 : graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
1953 :
1954 74 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
1955 74 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
1956 :
1957 74 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
1958 : Node* false1 = zero;
1959 :
1960 : if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
1961 : false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
1962 : }
1963 :
1964 : Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
1965 74 : return graph()->NewNode(phi_op, true0, false0, merge0);
1966 : }
1967 :
1968 352 : Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
1969 88 : MachineOperatorBuilder* m = jsgraph()->machine();
1970 : // asm.js semantics return 0 on divide or mod by zero.
1971 88 : if (m->Uint32DivIsSafe()) {
1972 : // The hardware instruction does the right thing (e.g. arm).
1973 0 : return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
1974 : }
1975 :
1976 : // Explicit check for x % 0.
1977 : Diamond z(
1978 : graph(), jsgraph()->common(),
1979 : graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)),
1980 264 : BranchHint::kFalse);
1981 :
1982 : return z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0),
1983 : graph()->NewNode(jsgraph()->machine()->Uint32Div(), left, right,
1984 264 : z.if_false));
1985 : }
1986 :
1987 632 : Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
1988 158 : MachineOperatorBuilder* m = jsgraph()->machine();
1989 : // asm.js semantics return 0 on divide or mod by zero.
1990 : // Explicit check for x % 0.
1991 : Diamond z(
1992 : graph(), jsgraph()->common(),
1993 : graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)),
1994 474 : BranchHint::kFalse);
1995 :
1996 : Node* rem = graph()->NewNode(jsgraph()->machine()->Uint32Mod(), left, right,
1997 316 : z.if_false);
1998 : return z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0),
1999 158 : rem);
2000 : }
2001 :
2002 105 : Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2003 518 : wasm::WasmCodePosition position) {
2004 210 : if (jsgraph()->machine()->Is32()) {
2005 : return BuildDiv64Call(
2006 : left, right, ExternalReference::wasm_int64_div(jsgraph()->isolate()),
2007 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2008 : }
2009 : ZeroCheck64(wasm::kTrapDivByZero, right, position);
2010 105 : Node* before = *control_;
2011 : Node* denom_is_m1;
2012 : Node* denom_is_not_m1;
2013 : BranchExpectFalse(graph()->NewNode(jsgraph()->machine()->Word64Equal(), right,
2014 : jsgraph()->Int64Constant(-1)),
2015 210 : &denom_is_m1, &denom_is_not_m1);
2016 105 : *control_ = denom_is_m1;
2017 : TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2018 105 : std::numeric_limits<int64_t>::min(), position);
2019 105 : if (*control_ != denom_is_m1) {
2020 : *control_ = graph()->NewNode(jsgraph()->common()->Merge(2), denom_is_not_m1,
2021 294 : *control_);
2022 : } else {
2023 7 : *control_ = before;
2024 : }
2025 : return graph()->NewNode(jsgraph()->machine()->Int64Div(), left, right,
2026 315 : *control_);
2027 : }
2028 :
2029 35 : Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2030 210 : wasm::WasmCodePosition position) {
2031 70 : if (jsgraph()->machine()->Is32()) {
2032 : return BuildDiv64Call(
2033 : left, right, ExternalReference::wasm_int64_mod(jsgraph()->isolate()),
2034 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2035 : }
2036 : ZeroCheck64(wasm::kTrapRemByZero, right, position);
2037 : Diamond d(jsgraph()->graph(), jsgraph()->common(),
2038 : graph()->NewNode(jsgraph()->machine()->Word64Equal(), right,
2039 105 : jsgraph()->Int64Constant(-1)));
2040 :
2041 35 : d.Chain(*control_);
2042 :
2043 : Node* rem = graph()->NewNode(jsgraph()->machine()->Int64Mod(), left, right,
2044 70 : d.if_false);
2045 :
2046 : return d.Phi(MachineRepresentation::kWord64, jsgraph()->Int64Constant(0),
2047 35 : rem);
2048 : }
2049 :
2050 35 : Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2051 70 : wasm::WasmCodePosition position) {
2052 70 : if (jsgraph()->machine()->Is32()) {
2053 : return BuildDiv64Call(
2054 : left, right, ExternalReference::wasm_uint64_div(jsgraph()->isolate()),
2055 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2056 : }
2057 : return graph()->NewNode(jsgraph()->machine()->Uint64Div(), left, right,
2058 70 : ZeroCheck64(wasm::kTrapDivByZero, right, position));
2059 : }
2060 50 : Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2061 100 : wasm::WasmCodePosition position) {
2062 100 : if (jsgraph()->machine()->Is32()) {
2063 : return BuildDiv64Call(
2064 : left, right, ExternalReference::wasm_uint64_mod(jsgraph()->isolate()),
2065 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2066 : }
2067 : return graph()->NewNode(jsgraph()->machine()->Uint64Mod(), left, right,
2068 100 : ZeroCheck64(wasm::kTrapRemByZero, right, position));
2069 : }
2070 :
2071 0 : Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2072 : ExternalReference ref,
2073 : MachineType result_type, int trap_zero,
2074 0 : wasm::WasmCodePosition position) {
2075 : Node* stack_slot_dst = graph()->NewNode(
2076 0 : jsgraph()->machine()->StackSlot(MachineRepresentation::kWord64));
2077 : Node* stack_slot_src = graph()->NewNode(
2078 0 : jsgraph()->machine()->StackSlot(MachineRepresentation::kWord64));
2079 :
2080 : const Operator* store_op = jsgraph()->machine()->Store(
2081 0 : StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2082 : *effect_ =
2083 : graph()->NewNode(store_op, stack_slot_dst, jsgraph()->Int32Constant(0),
2084 0 : left, *effect_, *control_);
2085 : *effect_ =
2086 : graph()->NewNode(store_op, stack_slot_src, jsgraph()->Int32Constant(0),
2087 0 : right, *effect_, *control_);
2088 :
2089 : MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 2);
2090 : sig_builder.AddReturn(MachineType::Int32());
2091 : sig_builder.AddParam(MachineType::Pointer());
2092 : sig_builder.AddParam(MachineType::Pointer());
2093 :
2094 0 : Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref));
2095 0 : Node* args[] = {function, stack_slot_dst, stack_slot_src};
2096 :
2097 0 : Node* call = BuildCCall(sig_builder.Build(), args);
2098 :
2099 : // TODO(wasm): This can get simpler if we have a specialized runtime call to
2100 : // throw WASM exceptions by trap code instead of by string.
2101 0 : ZeroCheck32(static_cast<wasm::TrapReason>(trap_zero), call, position);
2102 0 : TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2103 0 : const Operator* load_op = jsgraph()->machine()->Load(result_type);
2104 : Node* load =
2105 : graph()->NewNode(load_op, stack_slot_dst, jsgraph()->Int32Constant(0),
2106 0 : *effect_, *control_);
2107 0 : *effect_ = load;
2108 0 : return load;
2109 : }
2110 :
2111 264 : Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node** args) {
2112 : const size_t params = sig->parameter_count();
2113 : const size_t extra = 2; // effect and control inputs.
2114 88 : const size_t count = 1 + params + extra;
2115 :
2116 : // Reallocate the buffer to make space for extra inputs.
2117 88 : args = Realloc(args, 1 + params, count);
2118 :
2119 : // Add effect and control inputs.
2120 88 : args[params + 1] = *effect_;
2121 88 : args[params + 2] = *control_;
2122 :
2123 : CallDescriptor* desc =
2124 88 : Linkage::GetSimplifiedCDescriptor(jsgraph()->zone(), sig);
2125 :
2126 88 : const Operator* op = jsgraph()->common()->Call(desc);
2127 176 : Node* call = graph()->NewNode(op, static_cast<int>(count), args);
2128 88 : *effect_ = call;
2129 88 : return call;
2130 : }
2131 :
2132 71119 : Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2133 : Node*** rets,
2134 71112 : wasm::WasmCodePosition position) {
2135 : const size_t params = sig->parameter_count();
2136 : const size_t extra = 2; // effect and control inputs.
2137 35557 : const size_t count = 1 + params + extra;
2138 :
2139 : // Reallocate the buffer to make space for extra inputs.
2140 35557 : args = Realloc(args, 1 + params, count);
2141 :
2142 : // Add effect and control inputs.
2143 35557 : args[params + 1] = *effect_;
2144 35557 : args[params + 2] = *control_;
2145 :
2146 : CallDescriptor* descriptor =
2147 35557 : wasm::ModuleEnv::GetWasmCallDescriptor(jsgraph()->zone(), sig);
2148 35555 : const Operator* op = jsgraph()->common()->Call(descriptor);
2149 71110 : Node* call = graph()->NewNode(op, static_cast<int>(count), args);
2150 35558 : SetSourcePosition(call, position);
2151 :
2152 35562 : *effect_ = call;
2153 : size_t ret_count = sig->return_count();
2154 35562 : if (ret_count == 0) return call; // No return value.
2155 :
2156 20646 : *rets = Buffer(ret_count);
2157 20646 : if (ret_count == 1) {
2158 : // Only a single return value.
2159 19743 : (*rets)[0] = call;
2160 : } else {
2161 : // Create projections for all return values.
2162 1806 : for (size_t i = 0; i < ret_count; i++) {
2163 1806 : (*rets)[i] = graph()->NewNode(jsgraph()->common()->Projection(i), call,
2164 3612 : graph()->start());
2165 : }
2166 : }
2167 : return call;
2168 : }
2169 :
2170 33175 : Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2171 20202 : wasm::WasmCodePosition position) {
2172 : DCHECK_NULL(args[0]);
2173 :
2174 : // Add code object as constant.
2175 : // TODO(wasm): Always use the illegal builtin, except for testing.
2176 : Handle<Code> code = module_->instance
2177 : ? module_->GetFunctionCode(index)
2178 86570 : : jsgraph()->isolate()->builtins()->Illegal();
2179 : DCHECK(!code.is_null());
2180 33193 : args[0] = HeapConstant(code);
2181 33193 : wasm::FunctionSig* sig = module_->GetFunctionSignature(index);
2182 :
2183 33193 : return BuildWasmCall(sig, args, rets, position);
2184 : }
2185 :
2186 2364 : Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2187 : Node*** rets,
2188 4728 : wasm::WasmCodePosition position) {
2189 : DCHECK_NOT_NULL(args[0]);
2190 : DCHECK_NOT_NULL(module_);
2191 :
2192 : // Assume only one table for now.
2193 : uint32_t table_index = 0;
2194 2364 : wasm::FunctionSig* sig = module_->GetSignature(sig_index);
2195 :
2196 : DCHECK(module_->IsValidTable(table_index));
2197 :
2198 2364 : EnsureFunctionTableNodes();
2199 2364 : MachineOperatorBuilder* machine = jsgraph()->machine();
2200 2364 : Node* key = args[0];
2201 :
2202 : // Bounds check against the table size.
2203 2364 : Node* size = function_table_sizes_[table_index];
2204 2364 : Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, size);
2205 2364 : TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2206 2364 : Node* table = function_tables_[table_index];
2207 2364 : Node* signatures = signature_tables_[table_index];
2208 :
2209 : // Load signature from the table and check.
2210 : // The table is a FixedArray; signatures are encoded as SMIs.
2211 : // [sig1, sig2, sig3, ...., code1, code2, code3 ...]
2212 2364 : ElementAccess access = AccessBuilder::ForFixedArrayElement();
2213 4728 : const int fixed_offset = access.header_size - access.tag();
2214 : {
2215 : Node* load_sig = graph()->NewNode(
2216 : machine->Load(MachineType::AnyTagged()), signatures,
2217 : graph()->NewNode(machine->Int32Add(),
2218 : graph()->NewNode(machine->Word32Shl(), key,
2219 : Int32Constant(kPointerSizeLog2)),
2220 : Int32Constant(fixed_offset)),
2221 9456 : *effect_, *control_);
2222 : auto map = const_cast<wasm::SignatureMap&>(
2223 2364 : module_->module->function_tables[0].map);
2224 : Node* sig_match = graph()->NewNode(
2225 : machine->WordEqual(), load_sig,
2226 4728 : jsgraph()->SmiConstant(static_cast<int>(map.FindOrInsert(sig))));
2227 2364 : TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2228 : }
2229 :
2230 : // Load code object from the table.
2231 : Node* load_code = graph()->NewNode(
2232 : machine->Load(MachineType::AnyTagged()), table,
2233 : graph()->NewNode(machine->Int32Add(),
2234 : graph()->NewNode(machine->Word32Shl(), key,
2235 : Int32Constant(kPointerSizeLog2)),
2236 : Uint32Constant(fixed_offset)),
2237 9456 : *effect_, *control_);
2238 :
2239 2364 : args[0] = load_code;
2240 2364 : return BuildWasmCall(sig, args, rets, position);
2241 : }
2242 :
2243 70 : Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2244 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2245 : // TODO(weiliang): support Word32Rol opcode in TurboFan.
2246 : Int32Matcher m(right);
2247 35 : if (m.HasValue()) {
2248 : return Binop(wasm::kExprI32Ror, left,
2249 28 : jsgraph()->Int32Constant(32 - m.Value()));
2250 : } else {
2251 : return Binop(wasm::kExprI32Ror, left,
2252 21 : Binop(wasm::kExprI32Sub, jsgraph()->Int32Constant(32), right));
2253 : }
2254 : }
2255 :
2256 84 : Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2257 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2258 : // TODO(weiliang): support Word64Rol opcode in TurboFan.
2259 : Int64Matcher m(right);
2260 42 : if (m.HasValue()) {
2261 : return Binop(wasm::kExprI64Ror, left,
2262 28 : jsgraph()->Int64Constant(64 - m.Value()));
2263 : } else {
2264 : return Binop(wasm::kExprI64Ror, left,
2265 28 : Binop(wasm::kExprI64Sub, jsgraph()->Int64Constant(64), right));
2266 : }
2267 : }
2268 :
2269 6922 : Node* WasmGraphBuilder::Invert(Node* node) {
2270 6922 : return Unop(wasm::kExprI32Eqz, node);
2271 : }
2272 :
2273 48043 : Node* WasmGraphBuilder::BuildChangeInt32ToTagged(Node* value) {
2274 48043 : MachineOperatorBuilder* machine = jsgraph()->machine();
2275 48043 : CommonOperatorBuilder* common = jsgraph()->common();
2276 :
2277 48043 : if (machine->Is64()) {
2278 48043 : return BuildChangeInt32ToSmi(value);
2279 : }
2280 :
2281 : Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
2282 0 : graph()->start());
2283 :
2284 0 : Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
2285 : Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), ovf,
2286 0 : graph()->start());
2287 :
2288 0 : Node* if_true = graph()->NewNode(common->IfTrue(), branch);
2289 : Node* vtrue = BuildAllocateHeapNumberWithValue(
2290 0 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
2291 :
2292 0 : Node* if_false = graph()->NewNode(common->IfFalse(), branch);
2293 0 : Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
2294 :
2295 0 : Node* merge = graph()->NewNode(common->Merge(2), if_true, if_false);
2296 : Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
2297 0 : vtrue, vfalse, merge);
2298 0 : return phi;
2299 : }
2300 :
2301 38832 : Node* WasmGraphBuilder::BuildChangeFloat64ToTagged(Node* value) {
2302 12944 : MachineOperatorBuilder* machine = jsgraph()->machine();
2303 12944 : CommonOperatorBuilder* common = jsgraph()->common();
2304 :
2305 12944 : Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
2306 : Node* check_same = graph()->NewNode(
2307 : machine->Float64Equal(), value,
2308 25888 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
2309 : Node* branch_same =
2310 12944 : graph()->NewNode(common->Branch(), check_same, graph()->start());
2311 :
2312 12944 : Node* if_smi = graph()->NewNode(common->IfTrue(), branch_same);
2313 : Node* vsmi;
2314 12944 : Node* if_box = graph()->NewNode(common->IfFalse(), branch_same);
2315 : Node* vbox;
2316 :
2317 : // We only need to check for -0 if the {value} can potentially contain -0.
2318 : Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
2319 12944 : jsgraph()->Int32Constant(0));
2320 : Node* branch_zero =
2321 12944 : graph()->NewNode(common->Branch(BranchHint::kFalse), check_zero, if_smi);
2322 :
2323 12944 : Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
2324 12944 : Node* if_notzero = graph()->NewNode(common->IfFalse(), branch_zero);
2325 :
2326 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2327 : Node* check_negative = graph()->NewNode(
2328 : machine->Int32LessThan(),
2329 : graph()->NewNode(machine->Float64ExtractHighWord32(), value),
2330 25888 : jsgraph()->Int32Constant(0));
2331 : Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
2332 12944 : check_negative, if_zero);
2333 :
2334 12944 : Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
2335 12944 : Node* if_notnegative = graph()->NewNode(common->IfFalse(), branch_negative);
2336 :
2337 : // We need to create a box for negative 0.
2338 12944 : if_smi = graph()->NewNode(common->Merge(2), if_notzero, if_notnegative);
2339 12944 : if_box = graph()->NewNode(common->Merge(2), if_box, if_negative);
2340 :
2341 : // On 64-bit machines we can just wrap the 32-bit integer in a smi, for 32-bit
2342 : // machines we need to deal with potential overflow and fallback to boxing.
2343 12944 : if (machine->Is64()) {
2344 12944 : vsmi = BuildChangeInt32ToSmi(value32);
2345 : } else {
2346 : Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
2347 0 : value32, if_smi);
2348 :
2349 0 : Node* check_ovf = graph()->NewNode(common->Projection(1), smi_tag, if_smi);
2350 : Node* branch_ovf =
2351 0 : graph()->NewNode(common->Branch(BranchHint::kFalse), check_ovf, if_smi);
2352 :
2353 0 : Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
2354 0 : if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
2355 :
2356 0 : if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
2357 0 : vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
2358 : }
2359 :
2360 : // Allocate the box for the {value}.
2361 12944 : vbox = BuildAllocateHeapNumberWithValue(value, if_box);
2362 :
2363 12944 : Node* control = graph()->NewNode(common->Merge(2), if_smi, if_box);
2364 : value = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), vsmi,
2365 12944 : vbox, control);
2366 12944 : return value;
2367 : }
2368 :
2369 85840 : Node* WasmGraphBuilder::ToJS(Node* node, wasm::ValueType type) {
2370 73019 : switch (type) {
2371 : case wasm::kWasmI32:
2372 48043 : return BuildChangeInt32ToTagged(node);
2373 : case wasm::kWasmS128:
2374 : case wasm::kWasmI64:
2375 0 : UNREACHABLE();
2376 : case wasm::kWasmF32:
2377 : node = graph()->NewNode(jsgraph()->machine()->ChangeFloat32ToFloat64(),
2378 789 : node);
2379 789 : return BuildChangeFloat64ToTagged(node);
2380 : case wasm::kWasmF64:
2381 12155 : return BuildChangeFloat64ToTagged(node);
2382 : case wasm::kWasmStmt:
2383 12032 : return jsgraph()->UndefinedConstant();
2384 : default:
2385 0 : UNREACHABLE();
2386 : return nullptr;
2387 : }
2388 : }
2389 :
2390 289964 : Node* WasmGraphBuilder::BuildJavaScriptToNumber(Node* node, Node* context) {
2391 72491 : Callable callable = CodeFactory::ToNumber(jsgraph()->isolate());
2392 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
2393 : jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0,
2394 144982 : CallDescriptor::kNoFlags, Operator::kNoProperties);
2395 72491 : Node* stub_code = jsgraph()->HeapConstant(callable.code());
2396 :
2397 : Node* result = graph()->NewNode(jsgraph()->common()->Call(desc), stub_code,
2398 144982 : node, context, *effect_, *control_);
2399 :
2400 72491 : SetSourcePosition(result, 1);
2401 :
2402 72491 : *effect_ = result;
2403 :
2404 72491 : return result;
2405 : }
2406 :
2407 0 : bool CanCover(Node* value, IrOpcode::Value opcode) {
2408 0 : if (value->opcode() != opcode) return false;
2409 : bool first = true;
2410 0 : for (Edge const edge : value->use_edges()) {
2411 0 : if (NodeProperties::IsControlEdge(edge)) continue;
2412 0 : if (NodeProperties::IsEffectEdge(edge)) continue;
2413 : DCHECK(NodeProperties::IsValueEdge(edge));
2414 0 : if (!first) return false;
2415 : first = false;
2416 : }
2417 0 : return true;
2418 : }
2419 :
2420 217473 : Node* WasmGraphBuilder::BuildChangeTaggedToFloat64(Node* value) {
2421 72491 : MachineOperatorBuilder* machine = jsgraph()->machine();
2422 72491 : CommonOperatorBuilder* common = jsgraph()->common();
2423 :
2424 72491 : Node* check = BuildTestNotSmi(value);
2425 : Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), check,
2426 72491 : graph()->start());
2427 :
2428 72491 : Node* if_not_smi = graph()->NewNode(common->IfTrue(), branch);
2429 :
2430 : Node* vnot_smi;
2431 : Node* check_undefined = graph()->NewNode(machine->WordEqual(), value,
2432 72491 : jsgraph()->UndefinedConstant());
2433 : Node* branch_undefined = graph()->NewNode(common->Branch(BranchHint::kFalse),
2434 72491 : check_undefined, if_not_smi);
2435 :
2436 72491 : Node* if_undefined = graph()->NewNode(common->IfTrue(), branch_undefined);
2437 : Node* vundefined =
2438 72491 : jsgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
2439 :
2440 : Node* if_not_undefined =
2441 72491 : graph()->NewNode(common->IfFalse(), branch_undefined);
2442 72491 : Node* vheap_number = BuildLoadHeapNumberValue(value, if_not_undefined);
2443 :
2444 : if_not_smi =
2445 72491 : graph()->NewNode(common->Merge(2), if_undefined, if_not_undefined);
2446 : vnot_smi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2),
2447 72491 : vundefined, vheap_number, if_not_smi);
2448 :
2449 72491 : Node* if_smi = graph()->NewNode(common->IfFalse(), branch);
2450 72491 : Node* vfrom_smi = BuildChangeSmiToFloat64(value);
2451 :
2452 72491 : Node* merge = graph()->NewNode(common->Merge(2), if_not_smi, if_smi);
2453 : Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2),
2454 72491 : vnot_smi, vfrom_smi, merge);
2455 :
2456 72491 : return phi;
2457 : }
2458 :
2459 72491 : Node* WasmGraphBuilder::FromJS(Node* node, Node* context,
2460 123990 : wasm::ValueType type) {
2461 : DCHECK_NE(wasm::kWasmStmt, type);
2462 :
2463 : // Do a JavaScript ToNumber.
2464 72491 : Node* num = BuildJavaScriptToNumber(node, context);
2465 :
2466 : // Change representation.
2467 72491 : SimplifiedOperatorBuilder simplified(jsgraph()->zone());
2468 72491 : num = BuildChangeTaggedToFloat64(num);
2469 :
2470 72491 : switch (type) {
2471 : case wasm::kWasmI32: {
2472 : num = graph()->NewNode(jsgraph()->machine()->TruncateFloat64ToWord32(),
2473 48118 : num);
2474 48118 : break;
2475 : }
2476 : case wasm::kWasmS128:
2477 : case wasm::kWasmI64:
2478 0 : UNREACHABLE();
2479 : case wasm::kWasmF32:
2480 : num = graph()->NewNode(jsgraph()->machine()->TruncateFloat64ToFloat32(),
2481 3381 : num);
2482 3381 : break;
2483 : case wasm::kWasmF64:
2484 : break;
2485 : default:
2486 0 : UNREACHABLE();
2487 : return nullptr;
2488 : }
2489 : return num;
2490 : }
2491 :
2492 122634 : Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2493 122634 : if (jsgraph()->machine()->Is64()) {
2494 61317 : value = graph()->NewNode(jsgraph()->machine()->ChangeInt32ToInt64(), value);
2495 : }
2496 : return graph()->NewNode(jsgraph()->machine()->WordShl(), value,
2497 122634 : BuildSmiShiftBitsConstant());
2498 : }
2499 :
2500 222213 : Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2501 : value = graph()->NewNode(jsgraph()->machine()->WordSar(), value,
2502 74071 : BuildSmiShiftBitsConstant());
2503 148142 : if (jsgraph()->machine()->Is64()) {
2504 : value =
2505 74071 : graph()->NewNode(jsgraph()->machine()->TruncateInt64ToInt32(), value);
2506 : }
2507 74071 : return value;
2508 : }
2509 :
2510 1812 : Node* WasmGraphBuilder::BuildChangeUint32ToSmi(Node* value) {
2511 1812 : if (jsgraph()->machine()->Is64()) {
2512 : value =
2513 906 : graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), value);
2514 : }
2515 : return graph()->NewNode(jsgraph()->machine()->WordShl(), value,
2516 1812 : BuildSmiShiftBitsConstant());
2517 : }
2518 :
2519 144982 : Node* WasmGraphBuilder::BuildChangeSmiToFloat64(Node* value) {
2520 : return graph()->NewNode(jsgraph()->machine()->ChangeInt32ToFloat64(),
2521 217473 : BuildChangeSmiToInt32(value));
2522 : }
2523 :
2524 145222 : Node* WasmGraphBuilder::BuildTestNotSmi(Node* value) {
2525 : STATIC_ASSERT(kSmiTag == 0);
2526 : STATIC_ASSERT(kSmiTagMask == 1);
2527 : return graph()->NewNode(jsgraph()->machine()->WordAnd(), value,
2528 217833 : jsgraph()->IntPtrConstant(kSmiTagMask));
2529 : }
2530 :
2531 136294 : Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2532 136294 : return jsgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2533 : }
2534 :
2535 12944 : Node* WasmGraphBuilder::BuildAllocateHeapNumberWithValue(Node* value,
2536 44914 : Node* control) {
2537 12944 : MachineOperatorBuilder* machine = jsgraph()->machine();
2538 12944 : CommonOperatorBuilder* common = jsgraph()->common();
2539 : // The AllocateHeapNumberStub does not use the context, so we can safely pass
2540 : // in Smi zero here.
2541 12944 : Callable callable = CodeFactory::AllocateHeapNumber(jsgraph()->isolate());
2542 12944 : Node* target = jsgraph()->HeapConstant(callable.code());
2543 : Node* context = jsgraph()->NoContextConstant();
2544 : Node* effect =
2545 : graph()->NewNode(common->BeginRegion(RegionObservability::kNotObservable),
2546 12944 : graph()->start());
2547 12944 : if (!allocate_heap_number_operator_.is_set()) {
2548 : CallDescriptor* descriptor = Linkage::GetStubCallDescriptor(
2549 : jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0,
2550 12164 : CallDescriptor::kNoFlags, Operator::kNoThrow);
2551 6082 : allocate_heap_number_operator_.set(common->Call(descriptor));
2552 : }
2553 : Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
2554 : target, context, effect, control);
2555 : Node* store =
2556 : graph()->NewNode(machine->Store(StoreRepresentation(
2557 : MachineRepresentation::kFloat64, kNoWriteBarrier)),
2558 : heap_number, BuildHeapNumberValueIndexConstant(), value,
2559 25888 : heap_number, control);
2560 25888 : return graph()->NewNode(common->FinishRegion(), heap_number, store);
2561 : }
2562 :
2563 145222 : Node* WasmGraphBuilder::BuildLoadHeapNumberValue(Node* value, Node* control) {
2564 : return graph()->NewNode(jsgraph()->machine()->Load(MachineType::Float64()),
2565 : value, BuildHeapNumberValueIndexConstant(),
2566 217833 : graph()->start(), control);
2567 : }
2568 :
2569 12944 : Node* WasmGraphBuilder::BuildHeapNumberValueIndexConstant() {
2570 85555 : return jsgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
2571 : }
2572 :
2573 28389 : void WasmGraphBuilder::BuildJSToWasmWrapper(Handle<Code> wasm_code,
2574 223016 : wasm::FunctionSig* sig) {
2575 28389 : int wasm_count = static_cast<int>(sig->parameter_count());
2576 28389 : int count = wasm_count + 3;
2577 28389 : Node** args = Buffer(count);
2578 :
2579 : // Build the start and the JS parameter nodes.
2580 28389 : Node* start = Start(wasm_count + 5);
2581 28389 : *control_ = start;
2582 28389 : *effect_ = start;
2583 :
2584 : // Create the context parameter
2585 : Node* context = graph()->NewNode(
2586 : jsgraph()->common()->Parameter(
2587 : Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
2588 28389 : graph()->start());
2589 :
2590 : // Set the ThreadInWasm flag before we do the actual call.
2591 56778 : BuildModifyThreadInWasmFlag(true, jsgraph(), effect_, *control_);
2592 :
2593 28389 : if (!wasm::IsJSCompatibleSignature(sig_)) {
2594 : // Throw a TypeError. Use the context of the calling javascript function
2595 : // (passed as a parameter), such that the generated code is context
2596 : // independent.
2597 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, jsgraph(),
2598 186 : context, nullptr, 0, effect_, control_);
2599 :
2600 : // Add a dummy call to the wasm function so that the generated wrapper
2601 : // contains a reference to the wrapped wasm function. Without this reference
2602 : // the wasm function could not be re-imported into another wasm module.
2603 : int pos = 0;
2604 93 : args[pos++] = HeapConstant(wasm_code);
2605 93 : args[pos++] = *effect_;
2606 93 : args[pos++] = *control_;
2607 :
2608 : // We only need a dummy call descriptor.
2609 : wasm::FunctionSig::Builder dummy_sig_builder(jsgraph()->zone(), 0, 0);
2610 : CallDescriptor* desc = wasm::ModuleEnv::GetWasmCallDescriptor(
2611 186 : jsgraph()->zone(), dummy_sig_builder.Build());
2612 279 : *effect_ = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args);
2613 93 : Return(jsgraph()->UndefinedConstant());
2614 28389 : return;
2615 : }
2616 :
2617 : int pos = 0;
2618 28296 : args[pos++] = HeapConstant(wasm_code);
2619 :
2620 : // Convert JS parameters to WASM numbers.
2621 92917 : for (int i = 0; i < wasm_count; ++i) {
2622 36325 : Node* param = Param(i + 1);
2623 72650 : Node* wasm_param = FromJS(param, context, sig->GetParam(i));
2624 36325 : args[pos++] = wasm_param;
2625 : }
2626 :
2627 28296 : args[pos++] = *effect_;
2628 28296 : args[pos++] = *control_;
2629 :
2630 : // Call the WASM code.
2631 : CallDescriptor* desc =
2632 28296 : wasm::ModuleEnv::GetWasmCallDescriptor(jsgraph()->zone(), sig);
2633 :
2634 56592 : Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), count, args);
2635 28296 : *effect_ = call;
2636 :
2637 : // Clear the ThreadInWasmFlag
2638 56592 : BuildModifyThreadInWasmFlag(false, jsgraph(), effect_, *control_);
2639 :
2640 : Node* retval = call;
2641 : Node* jsval = ToJS(
2642 56592 : retval, sig->return_count() == 0 ? wasm::kWasmStmt : sig->GetReturn());
2643 : Return(jsval);
2644 : }
2645 :
2646 37808 : int WasmGraphBuilder::AddParameterNodes(Node** args, int pos, int param_count,
2647 44723 : wasm::FunctionSig* sig) {
2648 : // Convert WASM numbers to JS values.
2649 : int param_index = 0;
2650 82531 : for (int i = 0; i < param_count; ++i) {
2651 44723 : Node* param = Param(param_index++);
2652 89446 : args[pos++] = ToJS(param, sig->GetParam(i));
2653 : }
2654 37808 : return pos;
2655 : }
2656 :
2657 37883 : void WasmGraphBuilder::BuildWasmToJSWrapper(Handle<JSReceiver> target,
2658 379130 : wasm::FunctionSig* sig) {
2659 : DCHECK(target->IsCallable());
2660 :
2661 37883 : int wasm_count = static_cast<int>(sig->parameter_count());
2662 :
2663 : // Build the start and the parameter nodes.
2664 37883 : Isolate* isolate = jsgraph()->isolate();
2665 : CallDescriptor* desc;
2666 37883 : Node* start = Start(wasm_count + 3);
2667 37883 : *effect_ = start;
2668 37883 : *control_ = start;
2669 :
2670 37883 : if (!wasm::IsJSCompatibleSignature(sig_)) {
2671 : // Throw a TypeError. Embedding the context is ok here, since this code is
2672 : // regenerated at instantiation time.
2673 : Node* context =
2674 150 : jsgraph()->HeapConstant(jsgraph()->isolate()->native_context());
2675 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, jsgraph(),
2676 150 : context, nullptr, 0, effect_, control_);
2677 : // We don't need to return a value here, as the runtime call will not return
2678 : // anyway (the c entry stub will trigger stack unwinding).
2679 : ReturnVoid();
2680 37883 : return;
2681 : }
2682 :
2683 37808 : Node** args = Buffer(wasm_count + 7);
2684 :
2685 : Node* call = nullptr;
2686 :
2687 75616 : BuildModifyThreadInWasmFlag(false, jsgraph(), effect_, *control_);
2688 :
2689 37808 : if (target->IsJSFunction()) {
2690 : Handle<JSFunction> function = Handle<JSFunction>::cast(target);
2691 37763 : if (function->shared()->internal_formal_parameter_count() == wasm_count) {
2692 : int pos = 0;
2693 37208 : args[pos++] = jsgraph()->Constant(target); // target callable.
2694 : // Receiver.
2695 74356 : if (is_sloppy(function->shared()->language_mode()) &&
2696 : !function->shared()->native()) {
2697 : args[pos++] =
2698 74266 : HeapConstant(handle(function->context()->global_proxy(), isolate));
2699 : } else {
2700 : args[pos++] = jsgraph()->Constant(
2701 150 : handle(isolate->heap()->undefined_value(), isolate));
2702 : }
2703 :
2704 : desc = Linkage::GetJSCallDescriptor(
2705 74416 : graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
2706 :
2707 : // Convert WASM numbers to JS values.
2708 37208 : pos = AddParameterNodes(args, pos, wasm_count, sig);
2709 :
2710 74416 : args[pos++] = jsgraph()->UndefinedConstant(); // new target
2711 74416 : args[pos++] = jsgraph()->Int32Constant(wasm_count); // argument count
2712 74416 : args[pos++] = HeapConstant(handle(function->context()));
2713 37208 : args[pos++] = *effect_;
2714 37208 : args[pos++] = *control_;
2715 :
2716 74416 : call = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args);
2717 : }
2718 : }
2719 :
2720 : // We cannot call the target directly, we have to use the Call builtin.
2721 37808 : if (!call) {
2722 : int pos = 0;
2723 600 : Callable callable = CodeFactory::Call(isolate);
2724 600 : args[pos++] = jsgraph()->HeapConstant(callable.code());
2725 600 : args[pos++] = jsgraph()->Constant(target); // target callable
2726 600 : args[pos++] = jsgraph()->Int32Constant(wasm_count); // argument count
2727 : args[pos++] = jsgraph()->Constant(
2728 1200 : handle(isolate->heap()->undefined_value(), isolate)); // receiver
2729 :
2730 : desc = Linkage::GetStubCallDescriptor(isolate, graph()->zone(),
2731 : callable.descriptor(), wasm_count + 1,
2732 1800 : CallDescriptor::kNoFlags);
2733 :
2734 : // Convert WASM numbers to JS values.
2735 600 : pos = AddParameterNodes(args, pos, wasm_count, sig);
2736 :
2737 : // The native_context is sufficient here, because all kind of callables
2738 : // which depend on the context provide their own context. The context here
2739 : // is only needed if the target is a constructor to throw a TypeError, if
2740 : // the target is a native function, or if the target is a callable JSObject,
2741 : // which can only be constructed by the runtime.
2742 1200 : args[pos++] = HeapConstant(isolate->native_context());
2743 600 : args[pos++] = *effect_;
2744 600 : args[pos++] = *control_;
2745 :
2746 1200 : call = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args);
2747 : }
2748 :
2749 37808 : *effect_ = call;
2750 37808 : SetSourcePosition(call, 0);
2751 :
2752 75616 : BuildModifyThreadInWasmFlag(true, jsgraph(), effect_, *control_);
2753 :
2754 : // Convert the return value back.
2755 : Node* val = sig->return_count() == 0
2756 : ? jsgraph()->Int32Constant(0)
2757 : : FromJS(call, HeapConstant(isolate->native_context()),
2758 111782 : sig->GetReturn());
2759 : Return(val);
2760 : }
2761 :
2762 1369 : void WasmGraphBuilder::BuildWasmInterpreterEntry(
2763 11247 : uint32_t function_index, wasm::FunctionSig* sig,
2764 11605 : Handle<WasmInstanceObject> instance) {
2765 1369 : int wasm_count = static_cast<int>(sig->parameter_count());
2766 1369 : int param_count = jsgraph()->machine()->Is64()
2767 : ? wasm_count
2768 1369 : : Int64Lowering::GetParameterCountAfterLowering(sig);
2769 :
2770 : // Build the start and the parameter nodes.
2771 1369 : Node* start = Start(param_count + 3);
2772 1369 : *effect_ = start;
2773 1369 : *control_ = start;
2774 :
2775 : // Compute size for the argument buffer.
2776 1369 : int args_size_bytes = 0;
2777 2598 : for (int i = 0; i < wasm_count; i++) {
2778 2458 : args_size_bytes += 1 << ElementSizeLog2Of(sig->GetParam(i));
2779 : }
2780 :
2781 : // The return value is also passed via this buffer:
2782 : DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig->return_count());
2783 : // TODO(wasm): Handle multi-value returns.
2784 : DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
2785 : int return_size_bytes =
2786 2520 : sig->return_count() == 0 ? 0 : 1 << ElementSizeLog2Of(sig->GetReturn(0));
2787 :
2788 : // Get a stack slot for the arguments.
2789 1369 : Node* arg_buffer = args_size_bytes == 0 && return_size_bytes == 0
2790 : ? jsgraph()->IntPtrConstant(0)
2791 : : graph()->NewNode(jsgraph()->machine()->StackSlot(
2792 3998 : std::max(args_size_bytes, return_size_bytes)));
2793 :
2794 : // Now store all our arguments to the buffer.
2795 : int param_index = 0;
2796 : int offset = 0;
2797 :
2798 2598 : for (int i = 0; i < wasm_count; i++) {
2799 1229 : Node* param = Param(param_index++);
2800 1229 : if (Int64Lowering::IsI64AsTwoParameters(jsgraph()->machine(),
2801 2458 : sig->GetParam(i))) {
2802 : StoreRepresentation store_rep(wasm::kWasmI32,
2803 : WriteBarrierKind::kNoWriteBarrier);
2804 : *effect_ =
2805 : graph()->NewNode(jsgraph()->machine()->Store(store_rep), arg_buffer,
2806 : Int32Constant(offset + kInt64LowerHalfMemoryOffset),
2807 0 : param, *effect_, *control_);
2808 :
2809 0 : param = Param(param_index++);
2810 : *effect_ =
2811 : graph()->NewNode(jsgraph()->machine()->Store(store_rep), arg_buffer,
2812 : Int32Constant(offset + kInt64UpperHalfMemoryOffset),
2813 0 : param, *effect_, *control_);
2814 0 : offset += 8;
2815 :
2816 : } else {
2817 : MachineRepresentation param_rep = sig->GetParam(i);
2818 : StoreRepresentation store_rep(param_rep,
2819 : WriteBarrierKind::kNoWriteBarrier);
2820 : *effect_ =
2821 : graph()->NewNode(jsgraph()->machine()->Store(store_rep), arg_buffer,
2822 3687 : Int32Constant(offset), param, *effect_, *control_);
2823 1229 : offset += 1 << ElementSizeLog2Of(param_rep);
2824 : }
2825 : }
2826 : DCHECK_EQ(param_count, param_index);
2827 : DCHECK_EQ(args_size_bytes, offset);
2828 :
2829 : // We are passing the raw arg_buffer here. To the GC and other parts, it looks
2830 : // like a Smi (lowest bit not set). In the runtime function however, don't
2831 : // call Smi::value on it, but just cast it to a byte pointer.
2832 : Node* parameters[] = {
2833 2738 : jsgraph()->HeapConstant(instance), // wasm instance
2834 1369 : jsgraph()->SmiConstant(function_index), // function index
2835 : arg_buffer, // argument buffer
2836 2738 : };
2837 : BuildCallToRuntime(Runtime::kWasmRunInterpreter, jsgraph(), parameters,
2838 2738 : arraysize(parameters), effect_, control_);
2839 :
2840 : // Read back the return value.
2841 1369 : if (sig->return_count() == 0) {
2842 : Return(Int32Constant(0));
2843 1151 : } else if (Int64Lowering::IsI64AsTwoParameters(jsgraph()->machine(),
2844 1151 : sig->GetReturn())) {
2845 0 : MachineType load_rep = wasm::WasmOpcodes::MachineTypeFor(wasm::kWasmI32);
2846 : Node* lower =
2847 : graph()->NewNode(jsgraph()->machine()->Load(load_rep), arg_buffer,
2848 : Int32Constant(kInt64LowerHalfMemoryOffset), *effect_,
2849 0 : *control_);
2850 : Node* upper =
2851 : graph()->NewNode(jsgraph()->machine()->Load(load_rep), arg_buffer,
2852 : Int32Constant(kInt64UpperHalfMemoryOffset), lower,
2853 0 : *control_);
2854 0 : *effect_ = upper;
2855 : Return(lower, upper);
2856 : } else {
2857 1151 : MachineType load_rep = wasm::WasmOpcodes::MachineTypeFor(sig->GetReturn());
2858 : Node* val =
2859 : graph()->NewNode(jsgraph()->machine()->Load(load_rep), arg_buffer,
2860 2302 : Int32Constant(0), *effect_, *control_);
2861 : Return(val);
2862 : }
2863 1369 : }
2864 :
2865 144086 : Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
2866 : DCHECK_NOT_NULL(module_);
2867 : uintptr_t mem_start = reinterpret_cast<uintptr_t>(
2868 135151 : module_->instance ? module_->instance->mem_start : nullptr);
2869 135151 : if (offset == 0) {
2870 132867 : if (!mem_buffer_) {
2871 : mem_buffer_ = jsgraph()->RelocatableIntPtrConstant(
2872 13302 : mem_start, RelocInfo::WASM_MEMORY_REFERENCE);
2873 : }
2874 132865 : return mem_buffer_;
2875 : } else {
2876 : return jsgraph()->RelocatableIntPtrConstant(
2877 4568 : mem_start + offset, RelocInfo::WASM_MEMORY_REFERENCE);
2878 : }
2879 : }
2880 :
2881 1108 : Node* WasmGraphBuilder::CurrentMemoryPages() {
2882 : // CurrentMemoryPages will not be called from asm.js, hence we cannot be in
2883 : // lazy-compilation mode, hence the instance will be set.
2884 : DCHECK_EQ(wasm::kWasmOrigin, module_->module->get_origin());
2885 : DCHECK_NOT_NULL(module_);
2886 : DCHECK_NOT_NULL(module_->instance);
2887 : Node* call = BuildCallToRuntime(Runtime::kWasmMemorySize, jsgraph(), nullptr,
2888 1108 : 0, effect_, control_);
2889 554 : Node* result = BuildChangeSmiToInt32(call);
2890 554 : return result;
2891 : }
2892 :
2893 120368 : Node* WasmGraphBuilder::MemSize() {
2894 : DCHECK_NOT_NULL(module_);
2895 117278 : if (mem_size_) return mem_size_;
2896 3090 : uint32_t size = module_->instance ? module_->instance->mem_size : 0;
2897 : mem_size_ = jsgraph()->RelocatableInt32Constant(
2898 6180 : size, RelocInfo::WASM_MEMORY_SIZE_REFERENCE);
2899 3090 : return mem_size_;
2900 : }
2901 :
2902 4178 : void WasmGraphBuilder::EnsureFunctionTableNodes() {
2903 7092 : if (function_tables_.size() > 0) return;
2904 3628 : size_t tables_size = module_->module->function_tables.size();
2905 : if (module_->instance) {
2906 : DCHECK_EQ(tables_size, module_->instance->function_tables.size());
2907 : DCHECK_EQ(tables_size, module_->instance->signature_tables.size());
2908 : }
2909 3628 : for (size_t i = 0; i < tables_size; ++i) {
2910 3628 : auto function_handle = (*module_->function_tables)[i];
2911 3628 : auto signature_handle = (*module_->signature_tables)[i];
2912 3628 : function_tables_.push_back(HeapConstant(function_handle));
2913 3628 : signature_tables_.push_back(HeapConstant(signature_handle));
2914 3628 : uint32_t table_size = module_->module->function_tables[i].min_size;
2915 : function_table_sizes_.push_back(jsgraph()->RelocatableInt32Constant(
2916 : static_cast<uint32_t>(table_size),
2917 5442 : RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE));
2918 : }
2919 : }
2920 :
2921 31315 : Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
2922 : MachineType mem_type =
2923 15658 : wasm::WasmOpcodes::MachineTypeFor(module_->GetGlobalType(index));
2924 : byte* globals_start =
2925 7828 : module_->instance ? module_->instance->globals_start : nullptr;
2926 : uintptr_t global_addr = reinterpret_cast<uintptr_t>(
2927 15656 : globals_start + module_->module->globals[index].offset);
2928 : Node* addr = jsgraph()->RelocatableIntPtrConstant(
2929 15656 : global_addr, RelocInfo::WASM_GLOBAL_REFERENCE);
2930 7829 : const Operator* op = jsgraph()->machine()->Load(mem_type);
2931 : Node* node = graph()->NewNode(op, addr, jsgraph()->Int32Constant(0), *effect_,
2932 15658 : *control_);
2933 7829 : *effect_ = node;
2934 7829 : return node;
2935 : }
2936 :
2937 43800 : Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
2938 : MachineType mem_type =
2939 21900 : wasm::WasmOpcodes::MachineTypeFor(module_->GetGlobalType(index));
2940 : byte* globals_start =
2941 10950 : module_->instance ? module_->instance->globals_start : 0;
2942 : Node* addr = jsgraph()->RelocatableIntPtrConstant(
2943 10950 : reinterpret_cast<uintptr_t>(globals_start +
2944 10950 : module_->module->globals[index].offset),
2945 21900 : RelocInfo::WASM_GLOBAL_REFERENCE);
2946 : const Operator* op = jsgraph()->machine()->Store(
2947 21900 : StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
2948 : Node* node = graph()->NewNode(op, addr, jsgraph()->Int32Constant(0), val,
2949 21900 : *effect_, *control_);
2950 10950 : *effect_ = node;
2951 10950 : return node;
2952 : }
2953 :
2954 32445 : void WasmGraphBuilder::BoundsCheckMem(MachineType memtype, Node* index,
2955 : uint32_t offset,
2956 22127 : wasm::WasmCodePosition position) {
2957 16223 : if (FLAG_wasm_no_bounds_checks) return;
2958 : uint32_t size =
2959 16222 : module_ && module_->instance ? module_->instance->mem_size : 0;
2960 : byte memsize = wasm::WasmOpcodes::MemSize(memtype);
2961 :
2962 : size_t effective_size;
2963 16223 : if (size <= offset || size < (static_cast<uint64_t>(offset) + memsize)) {
2964 : // Two checks are needed in the case where the offset is statically
2965 : // out of bounds; one check for the offset being in bounds, and the next for
2966 : // the offset + index being out of bounds for code to be patched correctly
2967 : // on relocation.
2968 :
2969 : // Check for overflows.
2970 1604 : if ((std::numeric_limits<uint32_t>::max() - memsize) + 1 < offset) {
2971 : // Always trap. Do not use TrapAlways because it does not create a valid
2972 : // graph here.
2973 : TrapIfEq32(wasm::kTrapMemOutOfBounds, jsgraph()->Int32Constant(0), 0,
2974 6 : position);
2975 6 : return;
2976 : }
2977 1598 : size_t effective_offset = (offset - 1) + memsize;
2978 :
2979 : Node* cond = graph()->NewNode(jsgraph()->machine()->Uint32LessThan(),
2980 : jsgraph()->IntPtrConstant(effective_offset),
2981 : jsgraph()->RelocatableInt32Constant(
2982 : static_cast<uint32_t>(size),
2983 6391 : RelocInfo::WASM_MEMORY_SIZE_REFERENCE));
2984 1597 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
2985 : // For offset > effective size, this relies on check above to fail and
2986 : // effective size can be negative, relies on wrap around.
2987 1598 : effective_size = size - offset - memsize + 1;
2988 : } else {
2989 14619 : effective_size = size - offset - memsize + 1;
2990 14619 : CHECK(effective_size <= kMaxUInt32);
2991 :
2992 : Uint32Matcher m(index);
2993 14619 : if (m.HasValue()) {
2994 : uint32_t value = m.Value();
2995 8397 : if (value < effective_size) {
2996 : // The bounds check will always succeed.
2997 : return;
2998 : }
2999 : }
3000 : }
3001 :
3002 : Node* cond = graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index,
3003 : jsgraph()->RelocatableInt32Constant(
3004 : static_cast<uint32_t>(effective_size),
3005 25993 : RelocInfo::WASM_MEMORY_SIZE_REFERENCE));
3006 8662 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3007 : }
3008 :
3009 15990 : Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3010 : Node* index, uint32_t offset,
3011 : uint32_t alignment,
3012 28247 : wasm::WasmCodePosition position) {
3013 : Node* load;
3014 :
3015 : // WASM semantics throw on OOB. Introduce explicit bounds check.
3016 15990 : if (!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED) {
3017 14520 : BoundsCheckMem(memtype, index, offset, position);
3018 : }
3019 :
3020 26735 : if (memtype.representation() == MachineRepresentation::kWord8 ||
3021 10745 : jsgraph()->machine()->UnalignedLoadSupported(memtype, alignment)) {
3022 15990 : if (FLAG_wasm_trap_handler && V8_TRAP_HANDLER_SUPPORTED) {
3023 : DCHECK(FLAG_wasm_guard_pages);
3024 1470 : Node* position_node = jsgraph()->Int32Constant(position);
3025 : load = graph()->NewNode(jsgraph()->machine()->ProtectedLoad(memtype),
3026 : MemBuffer(offset), index, position_node, *effect_,
3027 2940 : *control_);
3028 : } else {
3029 : load = graph()->NewNode(jsgraph()->machine()->Load(memtype),
3030 29040 : MemBuffer(offset), index, *effect_, *control_);
3031 : }
3032 : } else {
3033 : // TODO(eholk): Support unaligned loads with trap handlers.
3034 : DCHECK(!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED);
3035 : load = graph()->NewNode(jsgraph()->machine()->UnalignedLoad(memtype),
3036 0 : MemBuffer(offset), index, *effect_, *control_);
3037 : }
3038 :
3039 15990 : *effect_ = load;
3040 :
3041 : #if defined(V8_TARGET_BIG_ENDIAN)
3042 : load = BuildChangeEndianness(load, memtype, type);
3043 : #endif
3044 :
3045 17092 : if (type == wasm::kWasmI64 &&
3046 1102 : ElementSizeLog2Of(memtype.representation()) < 3) {
3047 : // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3048 42 : if (memtype.IsSigned()) {
3049 : // sign extend
3050 42 : load = graph()->NewNode(jsgraph()->machine()->ChangeInt32ToInt64(), load);
3051 : } else {
3052 : // zero extend
3053 : load =
3054 0 : graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), load);
3055 : }
3056 : }
3057 :
3058 15990 : return load;
3059 : }
3060 :
3061 :
3062 1883 : Node* WasmGraphBuilder::StoreMem(MachineType memtype, Node* index,
3063 : uint32_t offset, uint32_t alignment, Node* val,
3064 3631 : wasm::WasmCodePosition position) {
3065 : Node* store;
3066 :
3067 : // WASM semantics throw on OOB. Introduce explicit bounds check.
3068 1883 : if (!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED) {
3069 1702 : BoundsCheckMem(memtype, index, offset, position);
3070 : }
3071 :
3072 : #if defined(V8_TARGET_BIG_ENDIAN)
3073 : val = BuildChangeEndianness(val, memtype);
3074 : #endif
3075 :
3076 3451 : if (memtype.representation() == MachineRepresentation::kWord8 ||
3077 1568 : jsgraph()->machine()->UnalignedStoreSupported(memtype, alignment)) {
3078 1883 : if (FLAG_wasm_trap_handler && V8_TRAP_HANDLER_SUPPORTED) {
3079 180 : Node* position_node = jsgraph()->Int32Constant(position);
3080 : store = graph()->NewNode(
3081 : jsgraph()->machine()->ProtectedStore(memtype.representation()),
3082 360 : MemBuffer(offset), index, val, position_node, *effect_, *control_);
3083 : } else {
3084 : StoreRepresentation rep(memtype.representation(), kNoWriteBarrier);
3085 : store =
3086 : graph()->NewNode(jsgraph()->machine()->Store(rep), MemBuffer(offset),
3087 3406 : index, val, *effect_, *control_);
3088 : }
3089 : } else {
3090 : // TODO(eholk): Support unaligned stores with trap handlers.
3091 : DCHECK(!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED);
3092 : UnalignedStoreRepresentation rep(memtype.representation());
3093 : store =
3094 : graph()->NewNode(jsgraph()->machine()->UnalignedStore(rep),
3095 0 : MemBuffer(offset), index, val, *effect_, *control_);
3096 : }
3097 :
3098 1883 : *effect_ = store;
3099 :
3100 1883 : return store;
3101 : }
3102 :
3103 68223 : Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3104 : // TODO(turbofan): fold bounds checks for constant asm.js loads.
3105 : // asm.js semantics use CheckedLoad (i.e. OOB reads return 0ish).
3106 68223 : const Operator* op = jsgraph()->machine()->CheckedLoad(type);
3107 : Node* load =
3108 68223 : graph()->NewNode(op, MemBuffer(0), index, MemSize(), *effect_, *control_);
3109 68223 : *effect_ = load;
3110 68223 : return load;
3111 : }
3112 :
3113 49055 : Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3114 49055 : Node* val) {
3115 : // TODO(turbofan): fold bounds checks for constant asm.js stores.
3116 : // asm.js semantics use CheckedStore (i.e. ignore OOB writes).
3117 : const Operator* op =
3118 98110 : jsgraph()->machine()->CheckedStore(type.representation());
3119 : Node* store = graph()->NewNode(op, MemBuffer(0), index, MemSize(), val,
3120 49055 : *effect_, *control_);
3121 49055 : *effect_ = store;
3122 49055 : return val;
3123 : }
3124 :
3125 0 : void WasmGraphBuilder::PrintDebugName(Node* node) {
3126 0 : PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3127 0 : }
3128 :
3129 0 : Node* WasmGraphBuilder::String(const char* string) {
3130 : return jsgraph()->Constant(
3131 0 : jsgraph()->isolate()->factory()->NewStringFromAsciiChecked(string));
3132 : }
3133 :
3134 4227249 : Graph* WasmGraphBuilder::graph() { return jsgraph()->graph(); }
3135 :
3136 24292 : void WasmGraphBuilder::Int64LoweringForTesting() {
3137 48584 : if (jsgraph()->machine()->Is32()) {
3138 : Int64Lowering r(jsgraph()->graph(), jsgraph()->machine(),
3139 0 : jsgraph()->common(), jsgraph()->zone(), sig_);
3140 0 : r.LowerGraph();
3141 : }
3142 24292 : }
3143 :
3144 0 : void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3145 0 : SimdScalarLowering(jsgraph(), sig_).LowerGraph();
3146 0 : }
3147 :
3148 218525 : void WasmGraphBuilder::SetSourcePosition(Node* node,
3149 : wasm::WasmCodePosition position) {
3150 : DCHECK_NE(position, wasm::kNoCodePosition);
3151 218525 : if (source_position_table_)
3152 112154 : source_position_table_->SetSourcePosition(node, SourcePosition(position));
3153 218519 : }
3154 :
3155 686 : Node* WasmGraphBuilder::S128Zero() {
3156 343 : has_simd_ = true;
3157 686 : return graph()->NewNode(jsgraph()->machine()->S128Zero());
3158 : }
3159 :
3160 0 : Node* WasmGraphBuilder::S1x4Zero() {
3161 0 : has_simd_ = true;
3162 0 : return graph()->NewNode(jsgraph()->machine()->S1x4Zero());
3163 : }
3164 :
3165 0 : Node* WasmGraphBuilder::S1x8Zero() {
3166 0 : has_simd_ = true;
3167 0 : return graph()->NewNode(jsgraph()->machine()->S1x8Zero());
3168 : }
3169 :
3170 0 : Node* WasmGraphBuilder::S1x16Zero() {
3171 0 : has_simd_ = true;
3172 0 : return graph()->NewNode(jsgraph()->machine()->S1x16Zero());
3173 : }
3174 :
3175 1029 : Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode,
3176 1029 : const NodeVector& inputs) {
3177 1029 : has_simd_ = true;
3178 1029 : switch (opcode) {
3179 : case wasm::kExprF32x4Splat:
3180 1029 : return graph()->NewNode(jsgraph()->machine()->F32x4Splat(), inputs[0]);
3181 : case wasm::kExprF32x4SConvertI32x4:
3182 : return graph()->NewNode(jsgraph()->machine()->F32x4SConvertI32x4(),
3183 0 : inputs[0]);
3184 : case wasm::kExprF32x4UConvertI32x4:
3185 : return graph()->NewNode(jsgraph()->machine()->F32x4UConvertI32x4(),
3186 0 : inputs[0]);
3187 : case wasm::kExprF32x4Abs:
3188 0 : return graph()->NewNode(jsgraph()->machine()->F32x4Abs(), inputs[0]);
3189 : case wasm::kExprF32x4Neg:
3190 0 : return graph()->NewNode(jsgraph()->machine()->F32x4Neg(), inputs[0]);
3191 : case wasm::kExprF32x4RecipApprox:
3192 : return graph()->NewNode(jsgraph()->machine()->F32x4RecipApprox(),
3193 0 : inputs[0]);
3194 : case wasm::kExprF32x4RecipSqrtApprox:
3195 : return graph()->NewNode(jsgraph()->machine()->F32x4RecipSqrtApprox(),
3196 0 : inputs[0]);
3197 : case wasm::kExprF32x4Add:
3198 : return graph()->NewNode(jsgraph()->machine()->F32x4Add(), inputs[0],
3199 0 : inputs[1]);
3200 : case wasm::kExprF32x4AddHoriz:
3201 : return graph()->NewNode(jsgraph()->machine()->F32x4AddHoriz(), inputs[0],
3202 0 : inputs[1]);
3203 : case wasm::kExprF32x4Sub:
3204 : return graph()->NewNode(jsgraph()->machine()->F32x4Sub(), inputs[0],
3205 0 : inputs[1]);
3206 : case wasm::kExprF32x4Mul:
3207 : return graph()->NewNode(jsgraph()->machine()->F32x4Mul(), inputs[0],
3208 0 : inputs[1]);
3209 : case wasm::kExprF32x4Min:
3210 : return graph()->NewNode(jsgraph()->machine()->F32x4Min(), inputs[0],
3211 0 : inputs[1]);
3212 : case wasm::kExprF32x4Max:
3213 : return graph()->NewNode(jsgraph()->machine()->F32x4Max(), inputs[0],
3214 0 : inputs[1]);
3215 : case wasm::kExprF32x4Eq:
3216 : return graph()->NewNode(jsgraph()->machine()->F32x4Eq(), inputs[0],
3217 0 : inputs[1]);
3218 : case wasm::kExprF32x4Ne:
3219 : return graph()->NewNode(jsgraph()->machine()->F32x4Ne(), inputs[0],
3220 0 : inputs[1]);
3221 : case wasm::kExprF32x4Lt:
3222 : return graph()->NewNode(jsgraph()->machine()->F32x4Lt(), inputs[0],
3223 0 : inputs[1]);
3224 : case wasm::kExprF32x4Le:
3225 : return graph()->NewNode(jsgraph()->machine()->F32x4Le(), inputs[0],
3226 0 : inputs[1]);
3227 : case wasm::kExprF32x4Gt:
3228 : return graph()->NewNode(jsgraph()->machine()->F32x4Lt(), inputs[1],
3229 0 : inputs[0]);
3230 : case wasm::kExprF32x4Ge:
3231 : return graph()->NewNode(jsgraph()->machine()->F32x4Le(), inputs[1],
3232 0 : inputs[0]);
3233 : case wasm::kExprI32x4Splat:
3234 630 : return graph()->NewNode(jsgraph()->machine()->I32x4Splat(), inputs[0]);
3235 : case wasm::kExprI32x4SConvertF32x4:
3236 : return graph()->NewNode(jsgraph()->machine()->I32x4SConvertF32x4(),
3237 0 : inputs[0]);
3238 : case wasm::kExprI32x4UConvertF32x4:
3239 : return graph()->NewNode(jsgraph()->machine()->I32x4UConvertF32x4(),
3240 0 : inputs[0]);
3241 : case wasm::kExprI32x4SConvertI16x8Low:
3242 : return graph()->NewNode(jsgraph()->machine()->I32x4SConvertI16x8Low(),
3243 0 : inputs[0]);
3244 : case wasm::kExprI32x4SConvertI16x8High:
3245 : return graph()->NewNode(jsgraph()->machine()->I32x4SConvertI16x8High(),
3246 0 : inputs[0]);
3247 : case wasm::kExprI32x4Neg:
3248 0 : return graph()->NewNode(jsgraph()->machine()->I32x4Neg(), inputs[0]);
3249 : case wasm::kExprI32x4Add:
3250 : return graph()->NewNode(jsgraph()->machine()->I32x4Add(), inputs[0],
3251 21 : inputs[1]);
3252 : case wasm::kExprI32x4AddHoriz:
3253 : return graph()->NewNode(jsgraph()->machine()->I32x4AddHoriz(), inputs[0],
3254 0 : inputs[1]);
3255 : case wasm::kExprI32x4Sub:
3256 : return graph()->NewNode(jsgraph()->machine()->I32x4Sub(), inputs[0],
3257 21 : inputs[1]);
3258 : case wasm::kExprI32x4Mul:
3259 : return graph()->NewNode(jsgraph()->machine()->I32x4Mul(), inputs[0],
3260 21 : inputs[1]);
3261 : case wasm::kExprI32x4MinS:
3262 : return graph()->NewNode(jsgraph()->machine()->I32x4MinS(), inputs[0],
3263 21 : inputs[1]);
3264 : case wasm::kExprI32x4MaxS:
3265 : return graph()->NewNode(jsgraph()->machine()->I32x4MaxS(), inputs[0],
3266 21 : inputs[1]);
3267 : case wasm::kExprI32x4Eq:
3268 : return graph()->NewNode(jsgraph()->machine()->I32x4Eq(), inputs[0],
3269 21 : inputs[1]);
3270 : case wasm::kExprI32x4Ne:
3271 : return graph()->NewNode(jsgraph()->machine()->I32x4Ne(), inputs[0],
3272 42 : inputs[1]);
3273 : case wasm::kExprI32x4LtS:
3274 : return graph()->NewNode(jsgraph()->machine()->I32x4LtS(), inputs[0],
3275 0 : inputs[1]);
3276 : case wasm::kExprI32x4LeS:
3277 : return graph()->NewNode(jsgraph()->machine()->I32x4LeS(), inputs[0],
3278 0 : inputs[1]);
3279 : case wasm::kExprI32x4GtS:
3280 : return graph()->NewNode(jsgraph()->machine()->I32x4LtS(), inputs[1],
3281 0 : inputs[0]);
3282 : case wasm::kExprI32x4GeS:
3283 : return graph()->NewNode(jsgraph()->machine()->I32x4LeS(), inputs[1],
3284 0 : inputs[0]);
3285 : case wasm::kExprI32x4UConvertI16x8Low:
3286 : return graph()->NewNode(jsgraph()->machine()->I32x4UConvertI16x8Low(),
3287 0 : inputs[0]);
3288 : case wasm::kExprI32x4UConvertI16x8High:
3289 : return graph()->NewNode(jsgraph()->machine()->I32x4UConvertI16x8High(),
3290 0 : inputs[0]);
3291 : case wasm::kExprI32x4MinU:
3292 : return graph()->NewNode(jsgraph()->machine()->I32x4MinU(), inputs[0],
3293 21 : inputs[1]);
3294 : case wasm::kExprI32x4MaxU:
3295 : return graph()->NewNode(jsgraph()->machine()->I32x4MaxU(), inputs[0],
3296 21 : inputs[1]);
3297 : case wasm::kExprI32x4LtU:
3298 : return graph()->NewNode(jsgraph()->machine()->I32x4LtU(), inputs[0],
3299 0 : inputs[1]);
3300 : case wasm::kExprI32x4LeU:
3301 : return graph()->NewNode(jsgraph()->machine()->I32x4LeU(), inputs[0],
3302 0 : inputs[1]);
3303 : case wasm::kExprI32x4GtU:
3304 : return graph()->NewNode(jsgraph()->machine()->I32x4LtU(), inputs[1],
3305 0 : inputs[0]);
3306 : case wasm::kExprI32x4GeU:
3307 : return graph()->NewNode(jsgraph()->machine()->I32x4LeU(), inputs[1],
3308 0 : inputs[0]);
3309 : case wasm::kExprI16x8Splat:
3310 798 : return graph()->NewNode(jsgraph()->machine()->I16x8Splat(), inputs[0]);
3311 : case wasm::kExprI16x8SConvertI8x16Low:
3312 : return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI8x16Low(),
3313 0 : inputs[0]);
3314 : case wasm::kExprI16x8SConvertI8x16High:
3315 : return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI8x16High(),
3316 0 : inputs[0]);
3317 : case wasm::kExprI16x8Neg:
3318 0 : return graph()->NewNode(jsgraph()->machine()->I16x8Neg(), inputs[0]);
3319 : case wasm::kExprI16x8SConvertI32x4:
3320 : return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI32x4(),
3321 0 : inputs[0], inputs[1]);
3322 : case wasm::kExprI16x8Add:
3323 : return graph()->NewNode(jsgraph()->machine()->I16x8Add(), inputs[0],
3324 21 : inputs[1]);
3325 : case wasm::kExprI16x8AddSaturateS:
3326 : return graph()->NewNode(jsgraph()->machine()->I16x8AddSaturateS(),
3327 21 : inputs[0], inputs[1]);
3328 : case wasm::kExprI16x8AddHoriz:
3329 : return graph()->NewNode(jsgraph()->machine()->I16x8AddHoriz(), inputs[0],
3330 0 : inputs[1]);
3331 : case wasm::kExprI16x8Sub:
3332 : return graph()->NewNode(jsgraph()->machine()->I16x8Sub(), inputs[0],
3333 21 : inputs[1]);
3334 : case wasm::kExprI16x8SubSaturateS:
3335 : return graph()->NewNode(jsgraph()->machine()->I16x8SubSaturateS(),
3336 21 : inputs[0], inputs[1]);
3337 : case wasm::kExprI16x8Mul:
3338 : return graph()->NewNode(jsgraph()->machine()->I16x8Mul(), inputs[0],
3339 21 : inputs[1]);
3340 : case wasm::kExprI16x8MinS:
3341 : return graph()->NewNode(jsgraph()->machine()->I16x8MinS(), inputs[0],
3342 21 : inputs[1]);
3343 : case wasm::kExprI16x8MaxS:
3344 : return graph()->NewNode(jsgraph()->machine()->I16x8MaxS(), inputs[0],
3345 21 : inputs[1]);
3346 : case wasm::kExprI16x8Eq:
3347 : return graph()->NewNode(jsgraph()->machine()->I16x8Eq(), inputs[0],
3348 21 : inputs[1]);
3349 : case wasm::kExprI16x8Ne:
3350 : return graph()->NewNode(jsgraph()->machine()->I16x8Ne(), inputs[0],
3351 42 : inputs[1]);
3352 : case wasm::kExprI16x8LtS:
3353 : return graph()->NewNode(jsgraph()->machine()->I16x8LtS(), inputs[0],
3354 0 : inputs[1]);
3355 : case wasm::kExprI16x8LeS:
3356 : return graph()->NewNode(jsgraph()->machine()->I16x8LeS(), inputs[0],
3357 0 : inputs[1]);
3358 : case wasm::kExprI16x8GtS:
3359 : return graph()->NewNode(jsgraph()->machine()->I16x8LtS(), inputs[1],
3360 0 : inputs[0]);
3361 : case wasm::kExprI16x8GeS:
3362 : return graph()->NewNode(jsgraph()->machine()->I16x8LeS(), inputs[1],
3363 0 : inputs[0]);
3364 : case wasm::kExprI16x8UConvertI8x16Low:
3365 : return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI8x16Low(),
3366 0 : inputs[0]);
3367 : case wasm::kExprI16x8UConvertI8x16High:
3368 : return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI8x16High(),
3369 0 : inputs[0]);
3370 : case wasm::kExprI16x8UConvertI32x4:
3371 : return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI32x4(),
3372 0 : inputs[0], inputs[1]);
3373 : case wasm::kExprI16x8AddSaturateU:
3374 : return graph()->NewNode(jsgraph()->machine()->I16x8AddSaturateU(),
3375 21 : inputs[0], inputs[1]);
3376 : case wasm::kExprI16x8SubSaturateU:
3377 : return graph()->NewNode(jsgraph()->machine()->I16x8SubSaturateU(),
3378 21 : inputs[0], inputs[1]);
3379 : case wasm::kExprI16x8MinU:
3380 : return graph()->NewNode(jsgraph()->machine()->I16x8MinU(), inputs[0],
3381 21 : inputs[1]);
3382 : case wasm::kExprI16x8MaxU:
3383 : return graph()->NewNode(jsgraph()->machine()->I16x8MaxU(), inputs[0],
3384 21 : inputs[1]);
3385 : case wasm::kExprI16x8LtU:
3386 : return graph()->NewNode(jsgraph()->machine()->I16x8LtU(), inputs[0],
3387 0 : inputs[1]);
3388 : case wasm::kExprI16x8LeU:
3389 : return graph()->NewNode(jsgraph()->machine()->I16x8LeU(), inputs[0],
3390 0 : inputs[1]);
3391 : case wasm::kExprI16x8GtU:
3392 : return graph()->NewNode(jsgraph()->machine()->I16x8LtU(), inputs[1],
3393 0 : inputs[0]);
3394 : case wasm::kExprI16x8GeU:
3395 : return graph()->NewNode(jsgraph()->machine()->I16x8LeU(), inputs[1],
3396 0 : inputs[0]);
3397 : case wasm::kExprI8x16Splat:
3398 693 : return graph()->NewNode(jsgraph()->machine()->I8x16Splat(), inputs[0]);
3399 : case wasm::kExprI8x16Neg:
3400 0 : return graph()->NewNode(jsgraph()->machine()->I8x16Neg(), inputs[0]);
3401 : case wasm::kExprI8x16SConvertI16x8:
3402 : return graph()->NewNode(jsgraph()->machine()->I8x16SConvertI16x8(),
3403 0 : inputs[0], inputs[1]);
3404 : case wasm::kExprI8x16Add:
3405 : return graph()->NewNode(jsgraph()->machine()->I8x16Add(), inputs[0],
3406 21 : inputs[1]);
3407 : case wasm::kExprI8x16AddSaturateS:
3408 : return graph()->NewNode(jsgraph()->machine()->I8x16AddSaturateS(),
3409 21 : inputs[0], inputs[1]);
3410 : case wasm::kExprI8x16Sub:
3411 : return graph()->NewNode(jsgraph()->machine()->I8x16Sub(), inputs[0],
3412 21 : inputs[1]);
3413 : case wasm::kExprI8x16SubSaturateS:
3414 : return graph()->NewNode(jsgraph()->machine()->I8x16SubSaturateS(),
3415 21 : inputs[0], inputs[1]);
3416 : case wasm::kExprI8x16Mul:
3417 : return graph()->NewNode(jsgraph()->machine()->I8x16Mul(), inputs[0],
3418 0 : inputs[1]);
3419 : case wasm::kExprI8x16MinS:
3420 : return graph()->NewNode(jsgraph()->machine()->I8x16MinS(), inputs[0],
3421 21 : inputs[1]);
3422 : case wasm::kExprI8x16MaxS:
3423 : return graph()->NewNode(jsgraph()->machine()->I8x16MaxS(), inputs[0],
3424 21 : inputs[1]);
3425 : case wasm::kExprI8x16Eq:
3426 : return graph()->NewNode(jsgraph()->machine()->I8x16Eq(), inputs[0],
3427 21 : inputs[1]);
3428 : case wasm::kExprI8x16Ne:
3429 : return graph()->NewNode(jsgraph()->machine()->I8x16Ne(), inputs[0],
3430 42 : inputs[1]);
3431 : case wasm::kExprI8x16LtS:
3432 : return graph()->NewNode(jsgraph()->machine()->I8x16LtS(), inputs[0],
3433 0 : inputs[1]);
3434 : case wasm::kExprI8x16LeS:
3435 : return graph()->NewNode(jsgraph()->machine()->I8x16LeS(), inputs[0],
3436 0 : inputs[1]);
3437 : case wasm::kExprI8x16GtS:
3438 : return graph()->NewNode(jsgraph()->machine()->I8x16LtS(), inputs[1],
3439 0 : inputs[0]);
3440 : case wasm::kExprI8x16GeS:
3441 : return graph()->NewNode(jsgraph()->machine()->I8x16LeS(), inputs[1],
3442 0 : inputs[0]);
3443 : case wasm::kExprI8x16UConvertI16x8:
3444 : return graph()->NewNode(jsgraph()->machine()->I8x16UConvertI16x8(),
3445 0 : inputs[0], inputs[1]);
3446 : case wasm::kExprI8x16AddSaturateU:
3447 : return graph()->NewNode(jsgraph()->machine()->I8x16AddSaturateU(),
3448 21 : inputs[0], inputs[1]);
3449 : case wasm::kExprI8x16SubSaturateU:
3450 : return graph()->NewNode(jsgraph()->machine()->I8x16SubSaturateU(),
3451 21 : inputs[0], inputs[1]);
3452 : case wasm::kExprI8x16MinU:
3453 : return graph()->NewNode(jsgraph()->machine()->I8x16MinU(), inputs[0],
3454 21 : inputs[1]);
3455 : case wasm::kExprI8x16MaxU:
3456 : return graph()->NewNode(jsgraph()->machine()->I8x16MaxU(), inputs[0],
3457 21 : inputs[1]);
3458 : case wasm::kExprI8x16LtU:
3459 : return graph()->NewNode(jsgraph()->machine()->I8x16LtU(), inputs[0],
3460 0 : inputs[1]);
3461 : case wasm::kExprI8x16LeU:
3462 : return graph()->NewNode(jsgraph()->machine()->I8x16LeU(), inputs[0],
3463 0 : inputs[1]);
3464 : case wasm::kExprI8x16GtU:
3465 : return graph()->NewNode(jsgraph()->machine()->I8x16LtU(), inputs[1],
3466 0 : inputs[0]);
3467 : case wasm::kExprI8x16GeU:
3468 : return graph()->NewNode(jsgraph()->machine()->I8x16LeU(), inputs[1],
3469 0 : inputs[0]);
3470 : case wasm::kExprS128And:
3471 : return graph()->NewNode(jsgraph()->machine()->S128And(), inputs[0],
3472 0 : inputs[1]);
3473 : case wasm::kExprS128Or:
3474 : return graph()->NewNode(jsgraph()->machine()->S128Or(), inputs[0],
3475 0 : inputs[1]);
3476 : case wasm::kExprS128Xor:
3477 : return graph()->NewNode(jsgraph()->machine()->S128Xor(), inputs[0],
3478 0 : inputs[1]);
3479 : case wasm::kExprS128Not:
3480 0 : return graph()->NewNode(jsgraph()->machine()->S128Not(), inputs[0]);
3481 : case wasm::kExprS32x4ZipLeft:
3482 : return graph()->NewNode(jsgraph()->machine()->S32x4ZipLeft(), inputs[0],
3483 0 : inputs[1]);
3484 : case wasm::kExprS32x4ZipRight:
3485 : return graph()->NewNode(jsgraph()->machine()->S32x4ZipRight(), inputs[0],
3486 0 : inputs[1]);
3487 : case wasm::kExprS32x4UnzipLeft:
3488 : return graph()->NewNode(jsgraph()->machine()->S32x4UnzipLeft(), inputs[0],
3489 0 : inputs[1]);
3490 : case wasm::kExprS32x4UnzipRight:
3491 : return graph()->NewNode(jsgraph()->machine()->S32x4UnzipRight(),
3492 0 : inputs[0], inputs[1]);
3493 : case wasm::kExprS32x4TransposeLeft:
3494 : return graph()->NewNode(jsgraph()->machine()->S32x4TransposeLeft(),
3495 0 : inputs[0], inputs[1]);
3496 : case wasm::kExprS32x4TransposeRight:
3497 : return graph()->NewNode(jsgraph()->machine()->S32x4TransposeRight(),
3498 0 : inputs[0], inputs[1]);
3499 : case wasm::kExprS32x4Select:
3500 : return graph()->NewNode(jsgraph()->machine()->S32x4Select(), inputs[0],
3501 63 : inputs[1], inputs[2]);
3502 : case wasm::kExprS16x8ZipLeft:
3503 : return graph()->NewNode(jsgraph()->machine()->S16x8ZipLeft(), inputs[0],
3504 0 : inputs[1]);
3505 : case wasm::kExprS16x8ZipRight:
3506 : return graph()->NewNode(jsgraph()->machine()->S16x8ZipRight(), inputs[0],
3507 0 : inputs[1]);
3508 : case wasm::kExprS16x8UnzipLeft:
3509 : return graph()->NewNode(jsgraph()->machine()->S16x8UnzipLeft(), inputs[0],
3510 0 : inputs[1]);
3511 : case wasm::kExprS16x8UnzipRight:
3512 : return graph()->NewNode(jsgraph()->machine()->S16x8UnzipRight(),
3513 0 : inputs[0], inputs[1]);
3514 : case wasm::kExprS16x8TransposeLeft:
3515 : return graph()->NewNode(jsgraph()->machine()->S16x8TransposeLeft(),
3516 0 : inputs[0], inputs[1]);
3517 : case wasm::kExprS16x8TransposeRight:
3518 : return graph()->NewNode(jsgraph()->machine()->S16x8TransposeRight(),
3519 0 : inputs[0], inputs[1]);
3520 : case wasm::kExprS16x8Select:
3521 : return graph()->NewNode(jsgraph()->machine()->S16x8Select(), inputs[0],
3522 63 : inputs[1], inputs[2]);
3523 : case wasm::kExprS8x16ZipLeft:
3524 : return graph()->NewNode(jsgraph()->machine()->S8x16ZipLeft(), inputs[0],
3525 0 : inputs[1]);
3526 : case wasm::kExprS8x16ZipRight:
3527 : return graph()->NewNode(jsgraph()->machine()->S8x16ZipRight(), inputs[0],
3528 0 : inputs[1]);
3529 : case wasm::kExprS8x16UnzipLeft:
3530 : return graph()->NewNode(jsgraph()->machine()->S8x16UnzipLeft(), inputs[0],
3531 0 : inputs[1]);
3532 : case wasm::kExprS8x16UnzipRight:
3533 : return graph()->NewNode(jsgraph()->machine()->S8x16UnzipRight(),
3534 0 : inputs[0], inputs[1]);
3535 : case wasm::kExprS8x16TransposeLeft:
3536 : return graph()->NewNode(jsgraph()->machine()->S8x16TransposeLeft(),
3537 0 : inputs[0], inputs[1]);
3538 : case wasm::kExprS8x16TransposeRight:
3539 : return graph()->NewNode(jsgraph()->machine()->S8x16TransposeRight(),
3540 0 : inputs[0], inputs[1]);
3541 : case wasm::kExprS8x16Select:
3542 : return graph()->NewNode(jsgraph()->machine()->S8x16Select(), inputs[0],
3543 63 : inputs[1], inputs[2]);
3544 : case wasm::kExprS32x2Reverse:
3545 0 : return graph()->NewNode(jsgraph()->machine()->S32x2Reverse(), inputs[0]);
3546 : case wasm::kExprS16x4Reverse:
3547 0 : return graph()->NewNode(jsgraph()->machine()->S16x4Reverse(), inputs[0]);
3548 : case wasm::kExprS16x2Reverse:
3549 0 : return graph()->NewNode(jsgraph()->machine()->S16x2Reverse(), inputs[0]);
3550 : case wasm::kExprS8x8Reverse:
3551 0 : return graph()->NewNode(jsgraph()->machine()->S8x8Reverse(), inputs[0]);
3552 : case wasm::kExprS8x4Reverse:
3553 0 : return graph()->NewNode(jsgraph()->machine()->S8x4Reverse(), inputs[0]);
3554 : case wasm::kExprS8x2Reverse:
3555 0 : return graph()->NewNode(jsgraph()->machine()->S8x2Reverse(), inputs[0]);
3556 : case wasm::kExprS1x4And:
3557 : return graph()->NewNode(jsgraph()->machine()->S1x4And(), inputs[0],
3558 0 : inputs[1]);
3559 : case wasm::kExprS1x4Or:
3560 : return graph()->NewNode(jsgraph()->machine()->S1x4Or(), inputs[0],
3561 0 : inputs[1]);
3562 : case wasm::kExprS1x4Xor:
3563 : return graph()->NewNode(jsgraph()->machine()->S1x4Xor(), inputs[0],
3564 0 : inputs[1]);
3565 : case wasm::kExprS1x4Not:
3566 0 : return graph()->NewNode(jsgraph()->machine()->S1x4Not(), inputs[0]);
3567 : case wasm::kExprS1x4AnyTrue:
3568 0 : return graph()->NewNode(jsgraph()->machine()->S1x4AnyTrue(), inputs[0]);
3569 : case wasm::kExprS1x4AllTrue:
3570 0 : return graph()->NewNode(jsgraph()->machine()->S1x4AllTrue(), inputs[0]);
3571 : case wasm::kExprS1x8And:
3572 : return graph()->NewNode(jsgraph()->machine()->S1x8And(), inputs[0],
3573 0 : inputs[1]);
3574 : case wasm::kExprS1x8Or:
3575 : return graph()->NewNode(jsgraph()->machine()->S1x8Or(), inputs[0],
3576 0 : inputs[1]);
3577 : case wasm::kExprS1x8Xor:
3578 : return graph()->NewNode(jsgraph()->machine()->S1x8Xor(), inputs[0],
3579 0 : inputs[1]);
3580 : case wasm::kExprS1x8Not:
3581 0 : return graph()->NewNode(jsgraph()->machine()->S1x8Not(), inputs[0]);
3582 : case wasm::kExprS1x8AnyTrue:
3583 0 : return graph()->NewNode(jsgraph()->machine()->S1x8AnyTrue(), inputs[0]);
3584 : case wasm::kExprS1x8AllTrue:
3585 0 : return graph()->NewNode(jsgraph()->machine()->S1x8AllTrue(), inputs[0]);
3586 : case wasm::kExprS1x16And:
3587 : return graph()->NewNode(jsgraph()->machine()->S1x16And(), inputs[0],
3588 0 : inputs[1]);
3589 : case wasm::kExprS1x16Or:
3590 : return graph()->NewNode(jsgraph()->machine()->S1x16Or(), inputs[0],
3591 0 : inputs[1]);
3592 : case wasm::kExprS1x16Xor:
3593 : return graph()->NewNode(jsgraph()->machine()->S1x16Xor(), inputs[0],
3594 0 : inputs[1]);
3595 : case wasm::kExprS1x16Not:
3596 0 : return graph()->NewNode(jsgraph()->machine()->S1x16Not(), inputs[0]);
3597 : case wasm::kExprS1x16AnyTrue:
3598 0 : return graph()->NewNode(jsgraph()->machine()->S1x16AnyTrue(), inputs[0]);
3599 : case wasm::kExprS1x16AllTrue:
3600 0 : return graph()->NewNode(jsgraph()->machine()->S1x16AllTrue(), inputs[0]);
3601 : default:
3602 0 : return graph()->NewNode(UnsupportedOpcode(opcode), nullptr);
3603 : }
3604 : }
3605 :
3606 5390 : Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
3607 5390 : const NodeVector& inputs) {
3608 5390 : has_simd_ = true;
3609 5390 : switch (opcode) {
3610 : case wasm::kExprF32x4ExtractLane:
3611 : return graph()->NewNode(jsgraph()->machine()->F32x4ExtractLane(lane),
3612 5390 : inputs[0]);
3613 : case wasm::kExprF32x4ReplaceLane:
3614 : return graph()->NewNode(jsgraph()->machine()->F32x4ReplaceLane(lane),
3615 0 : inputs[0], inputs[1]);
3616 : case wasm::kExprI32x4ExtractLane:
3617 : return graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane),
3618 1512 : inputs[0]);
3619 : case wasm::kExprI32x4ReplaceLane:
3620 : return graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(lane),
3621 126 : inputs[0], inputs[1]);
3622 : case wasm::kExprI16x8ExtractLane:
3623 : return graph()->NewNode(jsgraph()->machine()->I16x8ExtractLane(lane),
3624 4284 : inputs[0]);
3625 : case wasm::kExprI16x8ReplaceLane:
3626 : return graph()->NewNode(jsgraph()->machine()->I16x8ReplaceLane(lane),
3627 210 : inputs[0], inputs[1]);
3628 : case wasm::kExprI8x16ExtractLane:
3629 : return graph()->NewNode(jsgraph()->machine()->I8x16ExtractLane(lane),
3630 9660 : inputs[0]);
3631 : case wasm::kExprI8x16ReplaceLane:
3632 : return graph()->NewNode(jsgraph()->machine()->I8x16ReplaceLane(lane),
3633 378 : inputs[0], inputs[1]);
3634 : default:
3635 0 : return graph()->NewNode(UnsupportedOpcode(opcode), nullptr);
3636 : }
3637 : }
3638 :
3639 42 : Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
3640 42 : const NodeVector& inputs) {
3641 42 : has_simd_ = true;
3642 42 : switch (opcode) {
3643 : case wasm::kExprI32x4Shl:
3644 56 : return graph()->NewNode(jsgraph()->machine()->I32x4Shl(shift), inputs[0]);
3645 : case wasm::kExprI32x4ShrS:
3646 : return graph()->NewNode(jsgraph()->machine()->I32x4ShrS(shift),
3647 21 : inputs[0]);
3648 : case wasm::kExprI32x4ShrU:
3649 : return graph()->NewNode(jsgraph()->machine()->I32x4ShrU(shift),
3650 21 : inputs[0]);
3651 : case wasm::kExprI16x8Shl:
3652 21 : return graph()->NewNode(jsgraph()->machine()->I16x8Shl(shift), inputs[0]);
3653 : case wasm::kExprI16x8ShrS:
3654 : return graph()->NewNode(jsgraph()->machine()->I16x8ShrS(shift),
3655 21 : inputs[0]);
3656 : case wasm::kExprI16x8ShrU:
3657 : return graph()->NewNode(jsgraph()->machine()->I16x8ShrU(shift),
3658 21 : inputs[0]);
3659 : case wasm::kExprI8x16Shl:
3660 0 : return graph()->NewNode(jsgraph()->machine()->I8x16Shl(shift), inputs[0]);
3661 : case wasm::kExprI8x16ShrS:
3662 : return graph()->NewNode(jsgraph()->machine()->I8x16ShrS(shift),
3663 0 : inputs[0]);
3664 : case wasm::kExprI8x16ShrU:
3665 : return graph()->NewNode(jsgraph()->machine()->I8x16ShrU(shift),
3666 0 : inputs[0]);
3667 : default:
3668 0 : return graph()->NewNode(UnsupportedOpcode(opcode), nullptr);
3669 : }
3670 : }
3671 :
3672 0 : Node* WasmGraphBuilder::SimdConcatOp(uint8_t bytes, const NodeVector& inputs) {
3673 0 : has_simd_ = true;
3674 : return graph()->NewNode(jsgraph()->machine()->S8x16Concat(bytes), inputs[0],
3675 0 : inputs[1]);
3676 : }
3677 :
3678 0 : static void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
3679 : Isolate* isolate, Handle<Code> code,
3680 : const char* message, uint32_t index,
3681 0 : const wasm::WasmName& module_name,
3682 0 : const wasm::WasmName& func_name) {
3683 : DCHECK(isolate->logger()->is_logging_code_events() ||
3684 : isolate->is_profiling());
3685 :
3686 : ScopedVector<char> buffer(128);
3687 : SNPrintF(buffer, "%s#%d:%.*s:%.*s", message, index, module_name.length(),
3688 0 : module_name.start(), func_name.length(), func_name.start());
3689 : Handle<String> name_str =
3690 0 : isolate->factory()->NewStringFromAsciiChecked(buffer.start());
3691 : Handle<String> script_str =
3692 0 : isolate->factory()->NewStringFromAsciiChecked("(WASM)");
3693 : Handle<SharedFunctionInfo> shared =
3694 0 : isolate->factory()->NewSharedFunctionInfo(name_str, code, false);
3695 0 : PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *shared,
3696 : *script_str, 0, 0));
3697 0 : }
3698 :
3699 113556 : Handle<Code> CompileJSToWasmWrapper(Isolate* isolate,
3700 : const wasm::WasmModule* module,
3701 : Handle<Code> wasm_code, uint32_t index) {
3702 28389 : const wasm::WasmFunction* func = &module->functions[index];
3703 :
3704 : //----------------------------------------------------------------------------
3705 : // Create the Graph
3706 : //----------------------------------------------------------------------------
3707 28389 : Zone zone(isolate->allocator(), ZONE_NAME);
3708 28389 : Graph graph(&zone);
3709 28389 : CommonOperatorBuilder common(&zone);
3710 28389 : MachineOperatorBuilder machine(&zone);
3711 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
3712 :
3713 28389 : Node* control = nullptr;
3714 28389 : Node* effect = nullptr;
3715 :
3716 : wasm::ModuleEnv module_env(module, nullptr);
3717 28389 : WasmGraphBuilder builder(&module_env, &zone, &jsgraph, func->sig);
3718 : builder.set_control_ptr(&control);
3719 : builder.set_effect_ptr(&effect);
3720 28389 : builder.BuildJSToWasmWrapper(wasm_code, func->sig);
3721 :
3722 : //----------------------------------------------------------------------------
3723 : // Run the compilation pipeline.
3724 : //----------------------------------------------------------------------------
3725 28389 : if (FLAG_trace_turbo_graph) { // Simple textual RPO.
3726 0 : OFStream os(stdout);
3727 0 : os << "-- Graph after change lowering -- " << std::endl;
3728 0 : os << AsRPO(graph);
3729 : }
3730 :
3731 : // Schedule and compile to machine code.
3732 : int params = static_cast<int>(
3733 56778 : module_env.GetFunctionSignature(index)->parameter_count());
3734 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
3735 28389 : &zone, false, params + 1, CallDescriptor::kNoFlags);
3736 : Code::Flags flags = Code::ComputeFlags(Code::JS_TO_WASM_FUNCTION);
3737 : bool debugging =
3738 : #if DEBUG
3739 : true;
3740 : #else
3741 28389 : FLAG_print_opt_code || FLAG_trace_turbo || FLAG_trace_turbo_graph;
3742 : #endif
3743 28389 : Vector<const char> func_name = ArrayVector("js-to-wasm");
3744 :
3745 : static unsigned id = 0;
3746 : Vector<char> buffer;
3747 28389 : if (debugging) {
3748 0 : buffer = Vector<char>::New(128);
3749 0 : int chars = SNPrintF(buffer, "js-to-wasm#%d", id);
3750 0 : func_name = Vector<const char>::cast(buffer.SubVector(0, chars));
3751 : }
3752 :
3753 56778 : CompilationInfo info(func_name, isolate, &zone, flags);
3754 28389 : Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph);
3755 : #ifdef ENABLE_DISASSEMBLER
3756 : if (FLAG_print_opt_code && !code.is_null()) {
3757 : OFStream os(stdout);
3758 : code->Disassemble(buffer.start(), os);
3759 : }
3760 : #endif
3761 28389 : if (debugging) {
3762 : buffer.Dispose();
3763 : }
3764 :
3765 56778 : if (isolate->logger()->is_logging_code_events() || isolate->is_profiling()) {
3766 : char func_name[32];
3767 0 : SNPrintF(ArrayVector(func_name), "js-to-wasm#%d", func->func_index);
3768 : RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code,
3769 : "js-to-wasm", index, wasm::WasmName("export"),
3770 0 : CStrVector(func_name));
3771 : }
3772 56778 : return code;
3773 : }
3774 :
3775 113649 : Handle<Code> CompileWasmToJSWrapper(Isolate* isolate, Handle<JSReceiver> target,
3776 : wasm::FunctionSig* sig, uint32_t index,
3777 : Handle<String> module_name,
3778 : MaybeHandle<String> import_name,
3779 : wasm::ModuleOrigin origin) {
3780 : //----------------------------------------------------------------------------
3781 : // Create the Graph
3782 : //----------------------------------------------------------------------------
3783 37883 : Zone zone(isolate->allocator(), ZONE_NAME);
3784 37883 : Graph graph(&zone);
3785 37883 : CommonOperatorBuilder common(&zone);
3786 37883 : MachineOperatorBuilder machine(&zone);
3787 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
3788 :
3789 37883 : Node* control = nullptr;
3790 37883 : Node* effect = nullptr;
3791 :
3792 : SourcePositionTable* source_position_table =
3793 2269 : origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph)
3794 37883 : : nullptr;
3795 :
3796 : WasmGraphBuilder builder(nullptr, &zone, &jsgraph, sig,
3797 37883 : source_position_table);
3798 : builder.set_control_ptr(&control);
3799 : builder.set_effect_ptr(&effect);
3800 37883 : builder.BuildWasmToJSWrapper(target, sig);
3801 :
3802 : Handle<Code> code = Handle<Code>::null();
3803 : {
3804 37883 : if (FLAG_trace_turbo_graph) { // Simple textual RPO.
3805 0 : OFStream os(stdout);
3806 0 : os << "-- Graph after change lowering -- " << std::endl;
3807 0 : os << AsRPO(graph);
3808 : }
3809 :
3810 : // Schedule and compile to machine code.
3811 : CallDescriptor* incoming =
3812 37883 : wasm::ModuleEnv::GetWasmCallDescriptor(&zone, sig);
3813 37883 : if (machine.Is32()) {
3814 0 : incoming = wasm::ModuleEnv::GetI32WasmCallDescriptor(&zone, incoming);
3815 : }
3816 : Code::Flags flags = Code::ComputeFlags(Code::WASM_TO_JS_FUNCTION);
3817 : bool debugging =
3818 : #if DEBUG
3819 : true;
3820 : #else
3821 37883 : FLAG_print_opt_code || FLAG_trace_turbo || FLAG_trace_turbo_graph;
3822 : #endif
3823 37883 : Vector<const char> func_name = ArrayVector("wasm-to-js");
3824 : static unsigned id = 0;
3825 : Vector<char> buffer;
3826 37883 : if (debugging) {
3827 0 : buffer = Vector<char>::New(128);
3828 0 : int chars = SNPrintF(buffer, "wasm-to-js#%d", id);
3829 0 : func_name = Vector<const char>::cast(buffer.SubVector(0, chars));
3830 : }
3831 :
3832 37883 : CompilationInfo info(func_name, isolate, &zone, flags);
3833 : code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph, nullptr,
3834 37883 : source_position_table);
3835 : #ifdef ENABLE_DISASSEMBLER
3836 : if (FLAG_print_opt_code && !code.is_null()) {
3837 : OFStream os(stdout);
3838 : code->Disassemble(buffer.start(), os);
3839 : }
3840 : #endif
3841 37883 : if (debugging) {
3842 : buffer.Dispose();
3843 37883 : }
3844 : }
3845 75766 : if (isolate->logger()->is_logging_code_events() || isolate->is_profiling()) {
3846 : const char* function_name = nullptr;
3847 : int function_name_size = 0;
3848 0 : if (!import_name.is_null()) {
3849 : Handle<String> handle = import_name.ToHandleChecked();
3850 0 : function_name = handle->ToCString().get();
3851 : function_name_size = handle->length();
3852 : }
3853 : RecordFunctionCompilation(
3854 : CodeEventListener::FUNCTION_TAG, isolate, code, "wasm-to-js", index,
3855 0 : {module_name->ToCString().get(), module_name->length()},
3856 0 : {function_name, function_name_size});
3857 : }
3858 :
3859 37883 : return code;
3860 : }
3861 :
3862 4107 : Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index,
3863 : wasm::FunctionSig* sig,
3864 : Handle<WasmInstanceObject> instance) {
3865 : //----------------------------------------------------------------------------
3866 : // Create the Graph
3867 : //----------------------------------------------------------------------------
3868 1369 : Zone zone(isolate->allocator(), ZONE_NAME);
3869 1369 : Graph graph(&zone);
3870 1369 : CommonOperatorBuilder common(&zone);
3871 1369 : MachineOperatorBuilder machine(&zone);
3872 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
3873 :
3874 1369 : Node* control = nullptr;
3875 1369 : Node* effect = nullptr;
3876 :
3877 1369 : WasmGraphBuilder builder(nullptr, &zone, &jsgraph, sig);
3878 : builder.set_control_ptr(&control);
3879 : builder.set_effect_ptr(&effect);
3880 1369 : builder.BuildWasmInterpreterEntry(func_index, sig, instance);
3881 :
3882 : Handle<Code> code = Handle<Code>::null();
3883 : {
3884 1369 : if (FLAG_trace_turbo_graph) { // Simple textual RPO.
3885 0 : OFStream os(stdout);
3886 0 : os << "-- Wasm to interpreter graph -- " << std::endl;
3887 0 : os << AsRPO(graph);
3888 : }
3889 :
3890 : // Schedule and compile to machine code.
3891 : CallDescriptor* incoming =
3892 1369 : wasm::ModuleEnv::GetWasmCallDescriptor(&zone, sig);
3893 1369 : if (machine.Is32()) {
3894 0 : incoming = wasm::ModuleEnv::GetI32WasmCallDescriptor(&zone, incoming);
3895 : }
3896 : Code::Flags flags = Code::ComputeFlags(Code::WASM_INTERPRETER_ENTRY);
3897 : EmbeddedVector<char, 32> debug_name;
3898 1369 : int name_len = SNPrintF(debug_name, "wasm-to-interpreter#%d", func_index);
3899 : DCHECK(name_len > 0 && name_len < debug_name.length());
3900 : debug_name.Truncate(name_len);
3901 : DCHECK_EQ('\0', debug_name.start()[debug_name.length()]);
3902 :
3903 1369 : CompilationInfo info(debug_name, isolate, &zone, flags);
3904 1369 : code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph, nullptr);
3905 : #ifdef ENABLE_DISASSEMBLER
3906 : if (FLAG_print_opt_code && !code.is_null()) {
3907 : OFStream os(stdout);
3908 : code->Disassemble(debug_name.start(), os);
3909 : }
3910 : #endif
3911 :
3912 2738 : if (isolate->logger()->is_logging_code_events() ||
3913 : isolate->is_profiling()) {
3914 : RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code,
3915 : "wasm-to-interpreter", func_index,
3916 0 : wasm::WasmName("module"), debug_name);
3917 1369 : }
3918 : }
3919 :
3920 1369 : Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(1, TENURED);
3921 1369 : Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance);
3922 1369 : deopt_data->set(0, *weak_instance);
3923 1369 : code->set_deoptimization_data(*deopt_data);
3924 :
3925 1369 : return code;
3926 : }
3927 :
3928 47468 : SourcePositionTable* WasmCompilationUnit::BuildGraphForWasmFunction(
3929 : double* decode_ms) {
3930 : base::ElapsedTimer decode_timer;
3931 47468 : if (FLAG_trace_wasm_decode_time) {
3932 : decode_timer.Start();
3933 : }
3934 : // Create a TF graph during decoding.
3935 :
3936 47468 : Graph* graph = jsgraph_->graph();
3937 : CommonOperatorBuilder* common = jsgraph_->common();
3938 : MachineOperatorBuilder* machine = jsgraph_->machine();
3939 : SourcePositionTable* source_position_table =
3940 47482 : new (jsgraph_->zone()) SourcePositionTable(graph);
3941 : WasmGraphBuilder builder(module_env_, jsgraph_->zone(), jsgraph_,
3942 94920 : func_body_.sig, source_position_table);
3943 94942 : graph_construction_result_ =
3944 : wasm::BuildTFGraph(isolate_->allocator(), &builder, func_body_);
3945 :
3946 47500 : if (graph_construction_result_.failed()) {
3947 342 : if (FLAG_trace_wasm_compiler) {
3948 0 : OFStream os(stdout);
3949 0 : os << "Compilation failed: " << graph_construction_result_.error_msg
3950 0 : << std::endl;
3951 : }
3952 : return nullptr;
3953 : }
3954 :
3955 47158 : if (machine->Is32()) {
3956 : Int64Lowering(graph, machine, common, jsgraph_->zone(), func_body_.sig)
3957 0 : .LowerGraph();
3958 : }
3959 :
3960 47158 : if (builder.has_simd() && !CpuFeatures::SupportsWasmSimd128()) {
3961 0 : SimdScalarLowering(jsgraph_, func_body_.sig).LowerGraph();
3962 : }
3963 :
3964 94316 : if (func_index_ >= FLAG_trace_wasm_ast_start &&
3965 47158 : func_index_ < FLAG_trace_wasm_ast_end) {
3966 0 : PrintRawWasmCode(isolate_->allocator(), func_body_, module_env_->module);
3967 : }
3968 47158 : if (FLAG_trace_wasm_decode_time) {
3969 0 : *decode_ms = decode_timer.Elapsed().InMillisecondsF();
3970 : }
3971 47158 : return source_position_table;
3972 : }
3973 :
3974 : namespace {
3975 47502 : Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) {
3976 47502 : if (!name.is_empty()) {
3977 39800 : return name;
3978 : }
3979 : constexpr int kBufferLength = 15;
3980 :
3981 : EmbeddedVector<char, kBufferLength> name_vector;
3982 7702 : int name_len = SNPrintF(name_vector, "wasm#%d", index);
3983 : DCHECK(name_len > 0 && name_len < name_vector.length());
3984 :
3985 7702 : char* index_name = zone->NewArray<char>(name_len);
3986 7702 : memcpy(index_name, name_vector.start(), name_len);
3987 7702 : return Vector<const char>(index_name, name_len);
3988 : }
3989 : } // namespace
3990 :
3991 35006 : WasmCompilationUnit::WasmCompilationUnit(Isolate* isolate,
3992 : wasm::ModuleBytesEnv* module_env,
3993 35006 : const wasm::WasmFunction* function)
3994 : : WasmCompilationUnit(
3995 : isolate, &module_env->module_env,
3996 : wasm::FunctionBody{
3997 : function->sig, module_env->wire_bytes.start(),
3998 35006 : module_env->wire_bytes.start() + function->code_start_offset,
3999 35006 : module_env->wire_bytes.start() + function->code_end_offset},
4000 : module_env->wire_bytes.GetNameOrNull(function),
4001 175030 : function->func_index) {}
4002 :
4003 142506 : WasmCompilationUnit::WasmCompilationUnit(Isolate* isolate,
4004 : wasm::ModuleEnv* module_env,
4005 : wasm::FunctionBody body,
4006 : wasm::WasmName name, int index)
4007 : : isolate_(isolate),
4008 : module_env_(module_env),
4009 : func_body_(body),
4010 : func_name_(name),
4011 47502 : graph_zone_(new Zone(isolate->allocator(), ZONE_NAME)),
4012 : jsgraph_(new (graph_zone()) JSGraph(
4013 47502 : isolate, new (graph_zone()) Graph(graph_zone()),
4014 47502 : new (graph_zone()) CommonOperatorBuilder(graph_zone()), nullptr,
4015 : nullptr,
4016 : new (graph_zone()) MachineOperatorBuilder(
4017 : graph_zone(), MachineType::PointerRepresentation(),
4018 : InstructionSelector::SupportedMachineOperatorFlags(),
4019 47502 : InstructionSelector::AlignmentRequirements()))),
4020 : compilation_zone_(isolate->allocator(), ZONE_NAME),
4021 : info_(GetDebugName(&compilation_zone_, name, index), isolate,
4022 : &compilation_zone_, Code::ComputeFlags(Code::WASM_FUNCTION)),
4023 : func_index_(index),
4024 332514 : protected_instructions_(&compilation_zone_) {}
4025 :
4026 38968 : void WasmCompilationUnit::InitializeHandles() {
4027 : // Create and cache this node in the main thread, which contains a handle to
4028 : // the code object of the c-entry stub.
4029 47502 : jsgraph_->CEntryStubConstant(1);
4030 : DCHECK(!handles_initialized_);
4031 : #if DEBUG
4032 : handles_initialized_ = true;
4033 : #endif // DEBUG
4034 38968 : }
4035 :
4036 47483 : void WasmCompilationUnit::ExecuteCompilation() {
4037 : DCHECK(handles_initialized_);
4038 : // TODO(ahaas): The counters are not thread-safe at the moment.
4039 : // HistogramTimerScope wasm_compile_function_time_scope(
4040 : // isolate_->counters()->wasm_compile_function_time());
4041 47483 : if (FLAG_trace_wasm_compiler) {
4042 0 : if (func_name_.start() != nullptr) {
4043 : PrintF("Compiling WASM function %d:'%.*s'\n\n", func_index(),
4044 0 : func_name_.length(), func_name_.start());
4045 : } else {
4046 0 : PrintF("Compiling WASM function %d:<unnamed>\n\n", func_index());
4047 : }
4048 : }
4049 :
4050 47483 : double decode_ms = 0;
4051 : size_t node_count = 0;
4052 :
4053 : std::unique_ptr<Zone> graph_zone(graph_zone_.release());
4054 47483 : SourcePositionTable* source_positions = BuildGraphForWasmFunction(&decode_ms);
4055 :
4056 47499 : if (graph_construction_result_.failed()) {
4057 342 : ok_ = false;
4058 47502 : return;
4059 : }
4060 :
4061 : base::ElapsedTimer pipeline_timer;
4062 47157 : if (FLAG_trace_wasm_decode_time) {
4063 47154 : node_count = jsgraph_->graph()->NodeCount();
4064 : pipeline_timer.Start();
4065 : }
4066 :
4067 : // Run the compiler pipeline to generate machine code.
4068 : CallDescriptor* descriptor = wasm::ModuleEnv::GetWasmCallDescriptor(
4069 47157 : &compilation_zone_, func_body_.sig);
4070 94308 : if (jsgraph_->machine()->Is32()) {
4071 : descriptor =
4072 0 : module_env_->GetI32WasmCallDescriptor(&compilation_zone_, descriptor);
4073 : }
4074 : job_.reset(Pipeline::NewWasmCompilationJob(
4075 : &info_, jsgraph_, descriptor, source_positions, &protected_instructions_,
4076 47154 : !module_env_->module->is_wasm()));
4077 47121 : ok_ = job_->ExecuteJob() == CompilationJob::SUCCEEDED;
4078 : // TODO(bradnelson): Improve histogram handling of size_t.
4079 : // TODO(ahaas): The counters are not thread-safe at the moment.
4080 : // isolate_->counters()->wasm_compile_function_peak_memory_bytes()
4081 : // ->AddSample(
4082 : // static_cast<int>(jsgraph->graph()->zone()->allocation_size()));
4083 :
4084 47103 : if (FLAG_trace_wasm_decode_time) {
4085 0 : double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
4086 : PrintF(
4087 : "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
4088 : "%0.3f ms pipeline\n",
4089 0 : static_cast<unsigned>(func_body_.end - func_body_.start), decode_ms,
4090 0 : node_count, pipeline_ms);
4091 : }
4092 : }
4093 :
4094 47502 : Handle<Code> WasmCompilationUnit::FinishCompilation(
4095 : wasm::ErrorThrower* thrower) {
4096 47502 : if (!ok_) {
4097 342 : if (graph_construction_result_.failed()) {
4098 : // Add the function as another context for the exception
4099 : ScopedVector<char> buffer(128);
4100 342 : if (func_name_.start() == nullptr) {
4101 : SNPrintF(buffer,
4102 : "Compiling WASM function #%d:%.*s failed:", func_index_,
4103 27 : func_name_.length(), func_name_.start());
4104 : } else {
4105 315 : SNPrintF(buffer, "Compiling WASM function #%d failed:", func_index_);
4106 : }
4107 : thrower->CompileFailed(buffer.start(), graph_construction_result_);
4108 : }
4109 :
4110 : return Handle<Code>::null();
4111 : }
4112 : base::ElapsedTimer codegen_timer;
4113 47160 : if (FLAG_trace_wasm_decode_time) {
4114 : codegen_timer.Start();
4115 : }
4116 47160 : if (job_->FinalizeJob() != CompilationJob::SUCCEEDED) {
4117 : return Handle<Code>::null();
4118 : }
4119 : Handle<Code> code = info_.code();
4120 : DCHECK(!code.is_null());
4121 :
4122 94320 : if (isolate_->logger()->is_logging_code_events() ||
4123 : isolate_->is_profiling()) {
4124 : RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate_, code,
4125 : "WASM_function", func_index_,
4126 0 : wasm::WasmName("module"), func_name_);
4127 : }
4128 :
4129 47160 : if (FLAG_trace_wasm_decode_time) {
4130 0 : double codegen_ms = codegen_timer.Elapsed().InMillisecondsF();
4131 : PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n",
4132 0 : static_cast<unsigned>(func_body_.end - func_body_.start),
4133 0 : codegen_ms);
4134 : }
4135 :
4136 47160 : return code;
4137 : }
4138 :
4139 : // static
4140 8534 : Handle<Code> WasmCompilationUnit::CompileWasmFunction(
4141 : wasm::ErrorThrower* thrower, Isolate* isolate,
4142 : wasm::ModuleBytesEnv* module_env, const wasm::WasmFunction* function) {
4143 8534 : WasmCompilationUnit unit(isolate, module_env, function);
4144 : unit.InitializeHandles();
4145 8534 : unit.ExecuteCompilation();
4146 8534 : return unit.FinishCompilation(thrower);
4147 : }
4148 :
4149 : } // namespace compiler
4150 : } // namespace internal
4151 : } // namespace v8
|
