Coverage Report

Created: 2026-11-09 06:49

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/WasmEdge/include/executor/executor.h
Line
Count
Source
1
// SPDX-License-Identifier: Apache-2.0
2
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4
//===-- wasmedge/executor/executor.h - Executor class definition ----------===//
5
//
6
// Part of the WasmEdge Project.
7
//
8
//===----------------------------------------------------------------------===//
9
///
10
/// \file
11
/// This file contains the declaration of the Executor class, which instantiates
12
/// and runs Wasm modules.
13
///
14
//===----------------------------------------------------------------------===//
15
#pragma once
16
17
#include "ast/component/component.h"
18
#include "ast/module.h"
19
#include "common/async.h"
20
#include "common/configure.h"
21
#include "common/defines.h"
22
#include "common/errcode.h"
23
#include "common/statistics.h"
24
#include "common/types.h"
25
#include "runtime/callingframe.h"
26
#include "runtime/instance/component/component.h"
27
#include "runtime/instance/module.h"
28
#include "runtime/stackmgr.h"
29
#include "runtime/storemgr.h"
30
#include "system/stacktrace.h"
31
32
#include <atomic>
33
#include <csignal>
34
#include <cstddef>
35
#include <cstdint>
36
#include <functional>
37
#include <memory>
38
#include <mutex>
39
#include <optional>
40
#include <shared_mutex>
41
#include <string_view>
42
#include <type_traits>
43
#include <utility>
44
#include <vector>
45
46
namespace WasmEdge {
47
namespace Executor {
48
49
namespace {
50
51
// Template return type aliasing
52
/// Accept unsigned integer types. (uint32_t, uint64_t)
53
template <typename T>
54
using TypeU = typename std::enable_if_t<IsWasmUnsignV<T>, Expect<void>>;
55
/// Accept integer types. (uint32_t, int32_t, uint64_t, int64_t)
56
template <typename T>
57
using TypeI = typename std::enable_if_t<IsWasmIntV<T>, Expect<void>>;
58
/// Accept floating types. (float, double)
59
template <typename T>
60
using TypeF = typename std::enable_if_t<IsWasmFloatV<T>, Expect<void>>;
61
/// Accept all num types. (uint32_t, int32_t, uint64_t, int64_t, float, double)
62
template <typename T>
63
using TypeT = typename std::enable_if_t<IsWasmNumV<T>, Expect<void>>;
64
/// Accept Wasm built-in num types. (uint32_t, uint64_t, float, double)
65
template <typename T>
66
using TypeN = typename std::enable_if_t<IsWasmNativeNumV<T>, Expect<void>>;
67
68
/// Accept (unsigned integer types, unsigned integer types).
69
template <typename T1, typename T2>
70
using TypeUU = typename std::enable_if_t<IsWasmUnsignV<T1> && IsWasmUnsignV<T2>,
71
                                         Expect<void>>;
72
/// Accept (integer types, unsigned integer types).
73
template <typename T1, typename T2>
74
using TypeIU = typename std::enable_if_t<IsWasmIntV<T1> && IsWasmUnsignV<T2>,
75
                                         Expect<void>>;
76
/// Accept (floating types, floating types).
77
template <typename T1, typename T2>
78
using TypeFF = typename std::enable_if_t<IsWasmFloatV<T1> && IsWasmFloatV<T2>,
79
                                         Expect<void>>;
80
/// Accept (integer types, floating types).
81
template <typename T1, typename T2>
82
using TypeIF =
83
    typename std::enable_if_t<IsWasmIntV<T1> && IsWasmFloatV<T2>, Expect<void>>;
84
/// Accept (floating types, integer types).
85
template <typename T1, typename T2>
86
using TypeFI =
87
    typename std::enable_if_t<IsWasmFloatV<T1> && IsWasmIntV<T2>, Expect<void>>;
88
/// Accept (Wasm built-in num types, Wasm built-in num types).
89
template <typename T1, typename T2>
90
using TypeNN =
91
    typename std::enable_if_t<IsWasmNativeNumV<T1> && IsWasmNativeNumV<T2> &&
92
                                  sizeof(T1) == sizeof(T2),
93
                              Expect<void>>;
94
95
} // namespace
96
97
/// Helper class for handling the pre- and post-host functions.
98
class HostFuncHandler {
99
public:
100
0
  void setPreHost(void *HostData, std::function<void(void *)> HostFunc) {
101
0
    std::unique_lock Lock(Mutex);
102
0
    PreHostData = HostData;
103
0
    PreHostFunc = HostFunc;
104
0
  }
105
0
  void setPostHost(void *HostData, std::function<void(void *)> HostFunc) {
106
0
    std::unique_lock Lock(Mutex);
107
0
    PostHostData = HostData;
108
0
    PostHostFunc = HostFunc;
109
0
  }
110
0
  void invokePreHostFunc() {
111
0
    std::function<void(void *)> FuncSnapshot;
112
0
    void *DataSnapshot = nullptr;
113
0
    {
114
0
      std::shared_lock Lock(Mutex);
115
0
      FuncSnapshot = PreHostFunc;
116
0
      DataSnapshot = PreHostData;
117
0
    }
118
0
    if (FuncSnapshot.operator bool()) {
119
0
      FuncSnapshot(DataSnapshot);
120
0
    }
121
0
  }
122
0
  void invokePostHostFunc() {
123
0
    std::function<void(void *)> FuncSnapshot;
124
0
    void *DataSnapshot = nullptr;
125
0
    {
126
0
      std::shared_lock Lock(Mutex);
127
0
      FuncSnapshot = PostHostFunc;
128
0
      DataSnapshot = PostHostData;
129
0
    }
130
0
    if (FuncSnapshot.operator bool()) {
131
0
      FuncSnapshot(DataSnapshot);
132
0
    }
133
0
  }
134
135
private:
136
  void *PreHostData = nullptr;
137
  void *PostHostData = nullptr;
138
  std::function<void(void *)> PreHostFunc = {};
139
  std::function<void(void *)> PostHostFunc = {};
140
  mutable std::shared_mutex Mutex;
141
};
142
143
/// Executor flow control class.
144
class Executor {
145
public:
146
  Executor(const Configure &Conf, Statistics::Statistics *S = nullptr) noexcept
147
0
      : Conf(Conf) {
148
0
    if (Conf.getStatisticsConfigure().isInstructionCounting() ||
149
0
        Conf.getStatisticsConfigure().isCostMeasuring() ||
150
0
        Conf.getStatisticsConfigure().isTimeMeasuring()) {
151
0
      Stat = S;
152
0
    } else {
153
0
      Stat = nullptr;
154
0
    }
155
0
    if (Stat) {
156
0
      Stat->setCostLimit(Conf.getStatisticsConfigure().getCostLimit());
157
0
    }
158
0
  }
159
160
  /// Getter for configuration.
161
0
  const Configure &getConfigure() const { return Conf; }
162
163
  /// Instantiate a WASM Module as an anonymous module instance.
164
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
165
  instantiateModule(Runtime::StoreManager &StoreMgr, const AST::Module &Mod);
166
167
  /// Instantiate and register a WASM module as a named module instance.
168
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
169
  registerModule(Runtime::StoreManager &StoreMgr, const AST::Module &Mod,
170
                 std::string_view Name);
171
172
  /// Register an instantiated module as a named module instance.
173
  Expect<void> registerModule(Runtime::StoreManager &StoreMgr,
174
                              const Runtime::Instance::ModuleInstance &ModInst);
175
176
  /// Register an instantiated module under the given alias name.
177
  Expect<void> registerModule(Runtime::StoreManager &StoreMgr,
178
                              const Runtime::Instance::ModuleInstance &ModInst,
179
                              std::string_view Name);
180
181
  /// Instantiate a Component as an anonymous component instance.
182
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
183
  instantiateComponent(Runtime::StoreManager &StoreMgr,
184
                       const AST::Component::Component &Comp);
185
186
  /// Instantiate and register a Component as a named component instance.
187
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
188
  registerComponent(Runtime::StoreManager &StoreMgr,
189
                    const AST::Component::Component &Comp,
190
                    std::string_view Name);
191
192
  /// Register an instantiated component into a named component instance.
193
  Expect<void>
194
  registerComponent(Runtime::StoreManager &StoreMgr,
195
                    const Runtime::Instance::ComponentInstance &CompInst);
196
197
  /// Register a host function which will be invoked before calling a
198
  /// host function.
199
  Expect<void> registerPreHostFunction(void *HostData,
200
                                       std::function<void(void *)> HostFunc);
201
202
  /// Register a host function which will be invoked after calling a
203
  /// host function.
204
  Expect<void> registerPostHostFunction(void *HostData,
205
                                        std::function<void(void *)> HostFunc);
206
207
  /// Register a callback for lazy function compilation
208
  void registerLazyCompilationCallback(
209
      std::function<Expect<void>(const Runtime::Instance::FunctionInstance *)>
210
0
          Callback) {
211
0
    LazyCompilationHandler = std::move(Callback);
212
0
  }
213
214
  /// Invoke a WASM function by function instance.
215
  Expect<std::vector<std::pair<ValVariant, ValType>>>
216
  invoke(const Runtime::Instance::FunctionInstance *FuncInst,
217
         Span<const ValVariant> Params, Span<const ValType> ParamTypes);
218
219
  /// Invoke a Component function by function instance.
220
  Expect<std::vector<std::pair<ComponentValVariant, ComponentValType>>>
221
  invoke(const Runtime::Instance::Component::FunctionInstance *FuncInst,
222
         Span<const ComponentValVariant> Params,
223
         Span<const ComponentValType> ParamTypes);
224
225
  /// Module-qualified stack trace of the most recent failed invocation on this
226
  /// thread. Valid right after an invocation returns an error, else stale.
227
0
  static Span<const StackTraceEntry> getRecordedStackTrace() noexcept {
228
0
    return Span<const StackTraceEntry>{StackTrace}.first(StackTraceSize);
229
0
  }
230
231
  /// Asynchronous invoke a WASM function by function instance.
232
  Async<Expect<std::vector<std::pair<ValVariant, ValType>>>>
233
  asyncInvoke(const Runtime::Instance::FunctionInstance *FuncInst,
234
              Span<const ValVariant> Params, Span<const ValType> ParamTypes);
235
236
  /// Stop execution
237
0
  void stop() noexcept {
238
0
    StopToken.store(1, std::memory_order_relaxed);
239
0
    atomicNotifyAll();
240
0
  }
241
242
private:
243
  /// Run Wasm bytecode expression for initialization.
244
  Expect<void> runExpression(Runtime::StackManager &StackMgr,
245
                             AST::InstrView Instrs);
246
247
  /// Run Wasm function.
248
  Expect<void> runFunction(Runtime::StackManager &StackMgr,
249
                           const Runtime::Instance::FunctionInstance &Func,
250
                           Span<const ValVariant> Params);
251
252
  /// Execute instructions.
253
  Expect<void> execute(Runtime::StackManager &StackMgr,
254
                       const AST::InstrView::iterator Start,
255
                       const AST::InstrView::iterator End);
256
257
  /// \name Functions for instantiation.
258
  /// @{
259
  /// Instantiation of Module Instance.
260
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
261
  instantiate(Runtime::StoreManager &StoreMgr, const AST::Module &Mod,
262
              std::optional<std::string_view> Name = std::nullopt);
263
264
  /// Instantiation of Imports.
265
  Expect<void> instantiate(
266
      std::function<const Runtime::Instance::ModuleInstance *(std::string_view)>
267
          ModuleFinder,
268
      Runtime::Instance::ModuleInstance &ModInst,
269
      const AST::ImportSection &ImportSec);
270
271
  /// Instantiation of Function Instances.
272
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
273
                           const AST::FunctionSection &FuncSec,
274
                           const AST::CodeSection &CodeSec);
275
276
  /// Instantiation of Table Instances.
277
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
278
                           Runtime::Instance::ModuleInstance &ModInst,
279
                           const AST::TableSection &TabSec);
280
281
  /// Instantiation of Memory Instances.
282
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
283
                           const AST::MemorySection &MemSec);
284
285
  /// Instantiateion of Tag Instances.
286
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
287
                           const AST::TagSection &TagSec);
288
289
  /// Instantiation of Global Instances.
290
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
291
                           Runtime::Instance::ModuleInstance &ModInst,
292
                           const AST::GlobalSection &GlobSec);
293
294
  /// Instantiation of Element Instances.
295
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
296
                           Runtime::Instance::ModuleInstance &ModInst,
297
                           const AST::ElementSection &ElemSec);
298
299
  /// Initialize table with Element Instances.
300
  Expect<void> initTable(Runtime::StackManager &StackMgr,
301
                         const AST::ElementSection &ElemSec);
302
303
  /// Instantiation of Data Instances.
304
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
305
                           Runtime::Instance::ModuleInstance &ModInst,
306
                           const AST::DataSection &DataSec);
307
308
  /// Initialize memory with Data Instances.
309
  Expect<void> initMemory(Runtime::StackManager &StackMgr,
310
                          const AST::DataSection &DataSec);
311
312
  /// Instantiation of Exports.
313
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
314
                           const AST::ExportSection &ExportSec);
315
  /// @}
316
317
  /// \name Functions for instantiation of component model.
318
  /// @{
319
  /// Instantiation of Component Instance.
320
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
321
  instantiate(Runtime::StoreManager &StoreMgr,
322
              const AST::Component::Component &Comp,
323
              std::optional<std::string_view> Name = std::nullopt);
324
325
  /// Instantiation of Child Component Instance.
326
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
327
  instantiate(Runtime::Instance::ComponentImportManager &ImportMgr,
328
              const AST::Component::Component &Comp);
329
330
  /// Instantiation of Child Core Module Instance.
331
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
332
  instantiate(Runtime::Instance::ComponentImportManager &ImportMgr,
333
              const AST::Module &Mod);
334
335
  /// Instantiation of Core Module Section.
336
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
337
                           const AST::Component::CoreModuleSection &CoreModSec);
338
339
  /// Instantiation of Core Instance Section.
340
  Expect<void>
341
  instantiate(Runtime::Instance::ComponentInstance &CompInst,
342
              const AST::Component::CoreInstanceSection &CoreInstSec);
343
344
  /// Instantiation of Core Type Section.
345
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
346
                           const AST::Component::CoreTypeSection &CoreTypeSec);
347
348
  /// Instantiation of Component Section.
349
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
350
                           const AST::Component::ComponentSection &CompSec);
351
352
  /// Instantiation of Instance Section.
353
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
354
                           const AST::Component::InstanceSection &InstSec);
355
356
  /// Instantiation of Alias Section.
357
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
358
                           const AST::Component::AliasSection &AliasSec);
359
360
  /// Instantiation of Type Section.
361
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
362
                           const AST::Component::TypeSection &TypeSec);
363
364
  /// Instantiation of Canonical Section.
365
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
366
                           const AST::Component::CanonSection &CanonSec);
367
368
  /// Instantiation of Start Section.
369
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
370
                           const AST::Component::StartSection &StartSec);
371
372
  /// Instantiation of Import Section.
373
  Expect<void> instantiate(Runtime::StoreManager &StoreMgr,
374
                           Runtime::Instance::ComponentInstance &CompInst,
375
                           const AST::Component::ImportSection &ImportSec);
376
377
  /// Instantiation of Import Section in the child component instance.
378
  Expect<void> instantiate(Runtime::Instance::ComponentImportManager &ImportMgr,
379
                           Runtime::Instance::ComponentInstance &CompInst,
380
                           const AST::Component::ImportSection &ImportSec);
381
382
  /// Instantiation of Export Section.
383
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
384
                           const AST::Component::ExportSection &ExportSec);
385
  /// @}
386
387
  /// \name Helper Functions for canonical ABI
388
  /// @{
389
  Expect<std::vector<ValVariant>>
390
  convValsToCoreWASM(Span<const ComponentValVariant> Vals,
391
                     Span<const ComponentValType> ValTypes,
392
                     Runtime::Instance::FunctionInstance *RFuncInst,
393
                     Runtime::Instance::MemoryInstance *MemInst,
394
                     const Runtime::Instance::ComponentInstance *CompInst,
395
                     StringEncoding Enc = StringEncoding::UTF8);
396
397
  Expect<std::vector<std::pair<ComponentValVariant, ComponentValType>>>
398
  convValsToComponent(Span<const std::pair<ValVariant, ValType>> CoreVals,
399
                      Span<const ComponentValType> ValTypes,
400
                      Runtime::Instance::MemoryInstance *MemInst,
401
                      const Runtime::Instance::ComponentInstance *CompInst,
402
                      StringEncoding Enc = StringEncoding::UTF8);
403
  /// @}
404
405
  /// \name Helper Functions for block controls.
406
  /// @{
407
  /// Helper function for calling functions. Return the continuation iterator.
408
  /// Set `IsNativeEntry` when entering from the native code.
409
  Expect<AST::InstrView::iterator> enterFunction(
410
      Runtime::StackManager &StackMgr,
411
      const Runtime::Instance::FunctionInstance &Func,
412
      const AST::InstrView::iterator RetIt, bool IsTailCall = false,
413
      bool IsNativeEntry = false,
414
      const Runtime::Instance::ModuleInstance *CallerModInst = nullptr);
415
416
  /// Live module instances whose compiled code may be on the native stack at
417
  /// fault time: the current frame-stack modules plus every instance in a store
418
  /// reachable through their linked-module back-links. Rebuilt on each fault.
419
  std::vector<const Runtime::Instance::ModuleInstance *>
420
  collectLiveModules(const Runtime::StackManager &StackMgr) const noexcept;
421
422
  /// Helper function for branching to label.
423
  Expect<void> branchToLabel(Runtime::StackManager &StackMgr,
424
                             const AST::Instruction::JumpDescriptor &JumpDesc,
425
                             AST::InstrView::iterator &PC) noexcept;
426
427
  /// Helper function for throwing an exception. Pass `ExnInst` on `throw_ref`
428
  /// to reuse the exception instance and preserve exnref identity.
429
  Expect<void> throwException(
430
      Runtime::StackManager &StackMgr, Runtime::Instance::TagInstance &TagInst,
431
      AST::InstrView::iterator &PC,
432
      const Runtime::Instance::ExceptionInstance *ExnInst = nullptr) noexcept;
433
  /// @}
434
435
  /// \name Helper Function for checking memory offset boundary.
436
  /// @{
437
  Expect<void>
438
  checkOffsetOverflow(const Runtime::Instance::MemoryInstance &MemInst,
439
                      const AST::Instruction &Instr, const uint64_t Val,
440
                      const uint64_t Size) const noexcept;
441
  /// @}
442
443
  /// \name Helper Functions for GC instructions.
444
  /// @{
445
  Expect<RefVariant> structNew(const Runtime::Instance::ModuleInstance *ModInst,
446
                               const uint32_t TypeIdx,
447
                               Span<const ValVariant> Args = {}) const noexcept;
448
  Expect<ValVariant> structGet(const Runtime::Instance::ModuleInstance *ModInst,
449
                               const RefVariant Ref, const uint32_t TypeIdx,
450
                               const uint32_t Off,
451
                               const bool IsSigned = false) const noexcept;
452
  Expect<void> structSet(const Runtime::Instance::ModuleInstance *ModInst,
453
                         const RefVariant Ref, const ValVariant Val,
454
                         const uint32_t TypeIdx,
455
                         const uint32_t Off) const noexcept;
456
  Expect<RefVariant> arrayNew(const Runtime::Instance::ModuleInstance *ModInst,
457
                              const uint32_t TypeIdx, const uint32_t Length,
458
                              Span<const ValVariant> Args = {}) const noexcept;
459
  Expect<RefVariant>
460
  arrayNewData(const Runtime::Instance::ModuleInstance *ModInst,
461
               const uint32_t TypeIdx, const uint32_t DataIdx,
462
               const uint32_t Start, const uint32_t Length) const noexcept;
463
  Expect<RefVariant>
464
  arrayNewElem(const Runtime::Instance::ModuleInstance *ModInst,
465
               const uint32_t TypeIdx, const uint32_t ElemIdx,
466
               const uint32_t Start, const uint32_t Length) const noexcept;
467
  Expect<ValVariant> arrayGet(const Runtime::Instance::ModuleInstance *ModInst,
468
                              const RefVariant &Ref, const uint32_t TypeIdx,
469
                              const uint32_t Idx,
470
                              const bool IsSigned = false) const noexcept;
471
  Expect<void> arraySet(const Runtime::Instance::ModuleInstance *ModInst,
472
                        const RefVariant &Ref, const ValVariant &Val,
473
                        const uint32_t TypeIdx,
474
                        const uint32_t Idx) const noexcept;
475
  Expect<void> arrayFill(const Runtime::Instance::ModuleInstance *ModInst,
476
                         const RefVariant &Ref, const ValVariant &Val,
477
                         const uint32_t TypeIdx, const uint32_t Idx,
478
                         const uint32_t Cnt) const noexcept;
479
  Expect<void> arrayInitData(const Runtime::Instance::ModuleInstance *ModInst,
480
                             const RefVariant &Ref, const uint32_t TypeIdx,
481
                             const uint32_t DataIdx, const uint32_t DstIdx,
482
                             const uint32_t SrcIdx,
483
                             const uint32_t Cnt) const noexcept;
484
  Expect<void> arrayInitElem(const Runtime::Instance::ModuleInstance *ModInst,
485
                             const RefVariant &Ref, const uint32_t TypeIdx,
486
                             const uint32_t ElemIdx, const uint32_t DstIdx,
487
                             const uint32_t SrcIdx,
488
                             const uint32_t Cnt) const noexcept;
489
  Expect<void> arrayCopy(const Runtime::Instance::ModuleInstance *ModInst,
490
                         const RefVariant &DstRef, const uint32_t DstTypeIdx,
491
                         const uint32_t DstIdx, const RefVariant &SrcRef,
492
                         const uint32_t SrcTypeIdx, const uint32_t SrcIdx,
493
                         const uint32_t Cnt) const noexcept;
494
  /// @}
495
496
  /// \name Helper Functions for atomic operations.
497
  /// @{
498
  template <typename T>
499
  Expect<uint64_t> atomicWait(Runtime::Instance::MemoryInstance &MemInst,
500
                              uint64_t Address, EndianValue<T> Expected,
501
                              int64_t Timeout) noexcept;
502
  Expect<uint64_t> atomicNotify(Runtime::Instance::MemoryInstance &MemInst,
503
                                uint64_t Address, uint64_t Count) noexcept;
504
  void atomicNotifyAll() noexcept;
505
  /// @}
506
507
  /// \name Helper Functions for getting instances or types.
508
  /// @{
509
  /// Helper function for getting defined type by index.
510
  const AST::SubType *
511
  getDefTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
512
                  const uint32_t Idx) const;
513
514
  /// Helper function for getting composite type by index. Assuming validated.
515
  const WasmEdge::AST::CompositeType &
516
  getCompositeTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
517
                        const uint32_t Idx) const noexcept;
518
519
  /// Helper function for getting struct storage type by index.
520
  const ValType &
521
  getStructStorageTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
522
                            const uint32_t Idx,
523
                            const uint32_t Off) const noexcept;
524
525
  /// Helper function for getting array storage type by index.
526
  const ValType &
527
  getArrayStorageTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
528
                           const uint32_t Idx) const noexcept;
529
530
  /// Helper function for getting function instance by index.
531
  Runtime::Instance::FunctionInstance *
532
  getFuncInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
533
                   const uint32_t Idx) const;
534
535
  /// Helper function for getting table instance by index.
536
  Runtime::Instance::TableInstance *
537
  getTabInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
538
                  const uint32_t Idx) const;
539
540
  /// Helper function for getting memory instance by index.
541
  Runtime::Instance::MemoryInstance *
542
  getMemInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
543
                  const uint32_t Idx) const;
544
545
  /// Helper function for getting tag instance by index.
546
  Runtime::Instance::TagInstance *
547
  getTagInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
548
                  const uint32_t Idx) const;
549
550
  /// Helper function for getting global instance by index.
551
  Runtime::Instance::GlobalInstance *
552
  getGlobInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
553
                   const uint32_t Idx) const;
554
555
  /// Helper function for getting element instance by index.
556
  Runtime::Instance::ElementInstance *
557
  getElemInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
558
                   const uint32_t Idx) const;
559
560
  /// Helper function for getting data instance by index.
561
  Runtime::Instance::DataInstance *
562
  getDataInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
563
                   const uint32_t Idx) const;
564
565
  /// Helper function for converting into bottom abstract heap type.
566
  TypeCode toBottomType(const Runtime::Instance::ModuleInstance *ModInst,
567
                        const ValType &Type) const;
568
569
  /// Helper function for matching a reference against a reference type.
570
  bool matchRef(const Runtime::Instance::ModuleInstance *ModInst,
571
                const ValType &Exp, const RefVariant &Ref) const noexcept;
572
573
  /// Helper function for cleaning unused bits of numeric values in ValVariant.
574
  void cleanNumericVal(ValVariant &Val, const ValType &Type) const noexcept;
575
576
  /// Helper function for packing ValVariant for packed type.
577
  ValVariant packVal(const ValType &Type, const ValVariant &Val) const noexcept;
578
579
  /// Helper function for packing ValVariant vector for packed type.
580
  std::vector<ValVariant>
581
  packVals(const ValType &Type, std::vector<ValVariant> &&Vals) const noexcept;
582
583
  /// Helper function for unpacking ValVariant for packed type.
584
  ValVariant unpackVal(const ValType &Type, const ValVariant &Val,
585
                       bool IsSigned = false) const noexcept;
586
  /// @}
587
588
  /// \name Interpreter - Run instructions functions
589
  /// @{
590
  /// ======= Control instructions =======
591
  Expect<void> runIfElseOp(Runtime::StackManager &StackMgr,
592
                           const AST::Instruction &Instr,
593
                           AST::InstrView::iterator &PC) noexcept;
594
  Expect<void> runThrowOp(Runtime::StackManager &StackMgr,
595
                          const AST::Instruction &Instr,
596
                          AST::InstrView::iterator &PC) noexcept;
597
  Expect<void> runThrowRefOp(Runtime::StackManager &StackMgr,
598
                             const AST::Instruction &Instr,
599
                             AST::InstrView::iterator &PC) noexcept;
600
  Expect<void> runBrOp(Runtime::StackManager &StackMgr,
601
                       const AST::Instruction &Instr,
602
                       AST::InstrView::iterator &PC) noexcept;
603
  Expect<void> runBrIfOp(Runtime::StackManager &StackMgr,
604
                         const AST::Instruction &Instr,
605
                         AST::InstrView::iterator &PC) noexcept;
606
  Expect<void> runBrOnNullOp(Runtime::StackManager &StackMgr,
607
                             const AST::Instruction &Instr,
608
                             AST::InstrView::iterator &PC) noexcept;
609
  Expect<void> runBrOnNonNullOp(Runtime::StackManager &StackMgr,
610
                                const AST::Instruction &Instr,
611
                                AST::InstrView::iterator &PC) noexcept;
612
  Expect<void> runBrTableOp(Runtime::StackManager &StackMgr,
613
                            const AST::Instruction &Instr,
614
                            AST::InstrView::iterator &PC) noexcept;
615
  Expect<void> runBrOnCastOp(Runtime::StackManager &StackMgr,
616
                             const AST::Instruction &Instr,
617
                             AST::InstrView::iterator &PC,
618
                             bool IsReverse = false) noexcept;
619
  Expect<void> runReturnOp(Runtime::StackManager &StackMgr,
620
                           AST::InstrView::iterator &PC) noexcept;
621
  Expect<void> runCallOp(Runtime::StackManager &StackMgr,
622
                         const AST::Instruction &Instr,
623
                         AST::InstrView::iterator &PC,
624
                         bool IsTailCall = false) noexcept;
625
  Expect<void> runCallRefOp(Runtime::StackManager &StackMgr,
626
                            const AST::Instruction &Instr,
627
                            AST::InstrView::iterator &PC,
628
                            bool IsTailCall = false) noexcept;
629
  Expect<void> runCallIndirectOp(Runtime::StackManager &StackMgr,
630
                                 const AST::Instruction &Instr,
631
                                 AST::InstrView::iterator &PC,
632
                                 bool IsTailCall = false) noexcept;
633
  Expect<void> runTryTableOp(Runtime::StackManager &StackMgr,
634
                             const AST::Instruction &Instr,
635
                             AST::InstrView::iterator &PC) noexcept;
636
  /// ======= Variable instructions =======
637
  Expect<void> runLocalGetOp(Runtime::StackManager &StackMgr,
638
                             uint32_t StackOffset) const noexcept;
639
  Expect<void> runLocalSetOp(Runtime::StackManager &StackMgr,
640
                             uint32_t StackOffset) const noexcept;
641
  Expect<void> runLocalTeeOp(Runtime::StackManager &StackMgr,
642
                             uint32_t StackOffset) const noexcept;
643
  Expect<void> runGlobalGetOp(Runtime::StackManager &StackMgr,
644
                              uint32_t Idx) const noexcept;
645
  Expect<void> runGlobalSetOp(Runtime::StackManager &StackMgr,
646
                              uint32_t Idx) const noexcept;
647
  /// ======= Reference instructions =======
648
  Expect<void> runRefNullOp(Runtime::StackManager &StackMgr,
649
                            const ValType &Type) const noexcept;
650
  Expect<void> runRefIsNullOp(ValVariant &Val) const noexcept;
651
  Expect<void> runRefFuncOp(Runtime::StackManager &StackMgr,
652
                            uint32_t Idx) const noexcept;
653
  Expect<void> runRefEqOp(ValVariant &Val1,
654
                          const ValVariant &Val2) const noexcept;
655
  Expect<void> runRefAsNonNullOp(RefVariant &Val,
656
                                 const AST::Instruction &Instr) const noexcept;
657
  Expect<void> runStructNewOp(Runtime::StackManager &StackMgr,
658
                              const uint32_t TypeIdx,
659
                              const bool IsDefault = false) const noexcept;
660
  Expect<void> runStructGetOp(Runtime::StackManager &StackMgr,
661
                              const uint32_t TypeIdx, const uint32_t Off,
662
                              const AST::Instruction &Instr,
663
                              const bool IsSigned = false) const noexcept;
664
  Expect<void> runStructSetOp(Runtime::StackManager &StackMgr,
665
                              const ValVariant &Val, const uint32_t TypeIdx,
666
                              const uint32_t Off,
667
                              const AST::Instruction &Instr) const noexcept;
668
  Expect<void> runArrayNewOp(Runtime::StackManager &StackMgr,
669
                             const uint32_t TypeIdx, const uint32_t InitCnt,
670
                             uint32_t Length) const noexcept;
671
  Expect<void> runArrayNewDataOp(Runtime::StackManager &StackMgr,
672
                                 const uint32_t TypeIdx, const uint32_t DataIdx,
673
                                 const AST::Instruction &Instr) const noexcept;
674
  Expect<void> runArrayNewElemOp(Runtime::StackManager &StackMgr,
675
                                 const uint32_t TypeIdx, const uint32_t ElemIdx,
676
                                 const AST::Instruction &Instr) const noexcept;
677
  Expect<void> runArrayGetOp(Runtime::StackManager &StackMgr,
678
                             const uint32_t TypeIdx,
679
                             const AST::Instruction &Instr,
680
                             const bool IsSigned = false) const noexcept;
681
  Expect<void> runArraySetOp(Runtime::StackManager &StackMgr,
682
                             const ValVariant &Val, const uint32_t TypeIdx,
683
                             const AST::Instruction &Instr) const noexcept;
684
  Expect<void> runArrayLenOp(ValVariant &Val,
685
                             const AST::Instruction &Instr) const noexcept;
686
  Expect<void> runArrayFillOp(Runtime::StackManager &StackMgr,
687
                              const uint32_t Cnt, const ValVariant &Val,
688
                              const uint32_t TypeIdx,
689
                              const AST::Instruction &Instr) const noexcept;
690
  Expect<void> runArrayCopyOp(Runtime::StackManager &StackMgr,
691
                              const uint32_t Cnt, const uint32_t DstTypeIdx,
692
                              const uint32_t SrcTypeIdx,
693
                              const AST::Instruction &Instr) const noexcept;
694
  Expect<void> runArrayInitDataOp(Runtime::StackManager &StackMgr,
695
                                  const uint32_t Cnt, const uint32_t TypeIdx,
696
                                  const uint32_t DataIdx,
697
                                  const AST::Instruction &Instr) const noexcept;
698
  Expect<void> runArrayInitElemOp(Runtime::StackManager &StackMgr,
699
                                  const uint32_t Cnt, const uint32_t TypeIdx,
700
                                  const uint32_t ElemIdx,
701
                                  const AST::Instruction &Instr) const noexcept;
702
  Expect<void> runRefTestOp(const Runtime::Instance::ModuleInstance *ModInst,
703
                            ValVariant &Val, const AST::Instruction &Instr,
704
                            const bool IsCast = false) const noexcept;
705
  Expect<void> runRefConvOp(RefVariant &Val, TypeCode TCode) const noexcept;
706
  Expect<void> runRefI31Op(ValVariant &Val) const noexcept;
707
  Expect<void> runI31GetOp(ValVariant &Val, const AST::Instruction &Instr,
708
                           const bool IsSigned = false) const noexcept;
709
  /// ======= Table instructions =======
710
  Expect<void> runTableGetOp(Runtime::StackManager &StackMgr,
711
                             Runtime::Instance::TableInstance &TabInst,
712
                             const AST::Instruction &Instr);
713
  Expect<void> runTableSetOp(Runtime::StackManager &StackMgr,
714
                             Runtime::Instance::TableInstance &TabInst,
715
                             const AST::Instruction &Instr);
716
  Expect<void> runTableInitOp(Runtime::StackManager &StackMgr,
717
                              Runtime::Instance::TableInstance &TabInst,
718
                              Runtime::Instance::ElementInstance &ElemInst,
719
                              const AST::Instruction &Instr);
720
  Expect<void> runElemDropOp(Runtime::Instance::ElementInstance &ElemInst);
721
  Expect<void> runTableCopyOp(Runtime::StackManager &StackMgr,
722
                              Runtime::Instance::TableInstance &TabInstDst,
723
                              Runtime::Instance::TableInstance &TabInstSrc,
724
                              const AST::Instruction &Instr);
725
  Expect<void> runTableGrowOp(Runtime::StackManager &StackMgr,
726
                              Runtime::Instance::TableInstance &TabInst);
727
  Expect<void> runTableSizeOp(Runtime::StackManager &StackMgr,
728
                              Runtime::Instance::TableInstance &TabInst);
729
  Expect<void> runTableFillOp(Runtime::StackManager &StackMgr,
730
                              Runtime::Instance::TableInstance &TabInst,
731
                              const AST::Instruction &Instr);
732
  /// ======= Memory instructions =======
733
  template <typename T, uint32_t BitWidth = sizeof(T) * 8>
734
  TypeT<T> runLoadOp(Runtime::StackManager &StackMgr,
735
                     Runtime::Instance::MemoryInstance &MemInst,
736
                     const AST::Instruction &Instr);
737
  template <typename T, uint32_t BitWidth = sizeof(T) * 8>
738
  TypeN<T> runStoreOp(Runtime::StackManager &StackMgr,
739
                      Runtime::Instance::MemoryInstance &MemInst,
740
                      const AST::Instruction &Instr);
741
  Expect<void> runMemorySizeOp(Runtime::StackManager &StackMgr,
742
                               Runtime::Instance::MemoryInstance &MemInst);
743
  Expect<void> runMemoryGrowOp(Runtime::StackManager &StackMgr,
744
                               Runtime::Instance::MemoryInstance &MemInst);
745
  Expect<void> runMemoryInitOp(Runtime::StackManager &StackMgr,
746
                               Runtime::Instance::MemoryInstance &MemInst,
747
                               Runtime::Instance::DataInstance &DataInst,
748
                               const AST::Instruction &Instr);
749
  Expect<void> runDataDropOp(Runtime::Instance::DataInstance &DataInst);
750
  Expect<void> runMemoryCopyOp(Runtime::StackManager &StackMgr,
751
                               Runtime::Instance::MemoryInstance &MemInstDst,
752
                               Runtime::Instance::MemoryInstance &MemInstSrc,
753
                               const AST::Instruction &Instr);
754
  Expect<void> runMemoryFillOp(Runtime::StackManager &StackMgr,
755
                               Runtime::Instance::MemoryInstance &MemInst,
756
                               const AST::Instruction &Instr);
757
  /// ======= Test and Relation Numeric instructions =======
758
  template <typename T> TypeU<T> runEqzOp(ValVariant &Val) const;
759
  template <typename T>
760
  TypeT<T> runEqOp(ValVariant &Val1, const ValVariant &Val2) const;
761
  template <typename T>
762
  TypeT<T> runNeOp(ValVariant &Val1, const ValVariant &Val2) const;
763
  template <typename T>
764
  TypeT<T> runLtOp(ValVariant &Val1, const ValVariant &Val2) const;
765
  template <typename T>
766
  TypeT<T> runGtOp(ValVariant &Val1, const ValVariant &Val2) const;
767
  template <typename T>
768
  TypeT<T> runLeOp(ValVariant &Val1, const ValVariant &Val2) const;
769
  template <typename T>
770
  TypeT<T> runGeOp(ValVariant &Val1, const ValVariant &Val2) const;
771
  /// ======= Unary Numeric instructions =======
772
  template <typename T> TypeU<T> runClzOp(ValVariant &Val) const;
773
  template <typename T> TypeU<T> runCtzOp(ValVariant &Val) const;
774
  template <typename T> TypeU<T> runPopcntOp(ValVariant &Val) const;
775
  template <typename T> TypeF<T> runAbsOp(ValVariant &Val) const;
776
  template <typename T> TypeF<T> runNegOp(ValVariant &Val) const;
777
  template <typename T> TypeF<T> runCeilOp(ValVariant &Val) const;
778
  template <typename T> TypeF<T> runFloorOp(ValVariant &Val) const;
779
  template <typename T> TypeF<T> runTruncOp(ValVariant &Val) const;
780
  template <typename T> TypeF<T> runNearestOp(ValVariant &Val) const;
781
  template <typename T> TypeF<T> runSqrtOp(ValVariant &Val) const;
782
  /// ======= Binary Numeric instructions =======
783
  template <typename T>
784
  TypeN<T> runAddOp(ValVariant &Val1, const ValVariant &Val2) const;
785
  template <typename T>
786
  TypeN<T> runSubOp(ValVariant &Val1, const ValVariant &Val2) const;
787
  template <typename T>
788
  TypeN<T> runMulOp(ValVariant &Val1, const ValVariant &Val2) const;
789
  template <typename T>
790
  TypeT<T> runDivOp(const AST::Instruction &Instr, ValVariant &Val1,
791
                    const ValVariant &Val2) const;
792
  template <typename T>
793
  TypeI<T> runRemOp(const AST::Instruction &Instr, ValVariant &Val1,
794
                    const ValVariant &Val2) const;
795
  template <typename T>
796
  TypeU<T> runAndOp(ValVariant &Val1, const ValVariant &Val2) const;
797
  template <typename T>
798
  TypeU<T> runOrOp(ValVariant &Val1, const ValVariant &Val2) const;
799
  template <typename T>
800
  TypeU<T> runXorOp(ValVariant &Val1, const ValVariant &Val2) const;
801
  template <typename T>
802
  TypeU<T> runShlOp(ValVariant &Val1, const ValVariant &Val2) const;
803
  template <typename T>
804
  TypeI<T> runShrOp(ValVariant &Val1, const ValVariant &Val2) const;
805
  template <typename T>
806
  TypeU<T> runRotlOp(ValVariant &Val1, const ValVariant &Val2) const;
807
  template <typename T>
808
  TypeU<T> runRotrOp(ValVariant &Val1, const ValVariant &Val2) const;
809
  template <typename T>
810
  TypeF<T> runMinOp(ValVariant &Val1, const ValVariant &Val2) const;
811
  template <typename T>
812
  TypeF<T> runMaxOp(ValVariant &Val1, const ValVariant &Val2) const;
813
  template <typename T>
814
  TypeF<T> runCopysignOp(ValVariant &Val1, const ValVariant &Val2) const;
815
  /// ======= Cast Numeric instructions =======
816
  template <typename TIn, typename TOut>
817
  TypeUU<TIn, TOut> runWrapOp(ValVariant &Val) const;
818
  template <typename TIn, typename TOut>
819
  TypeFI<TIn, TOut> runTruncateOp(const AST::Instruction &Instr,
820
                                  ValVariant &Val) const;
821
  template <typename TIn, typename TOut>
822
  TypeFI<TIn, TOut> runTruncateSatOp(ValVariant &Val) const;
823
  template <typename TIn, typename TOut, size_t B = sizeof(TIn) * 8>
824
  TypeIU<TIn, TOut> runExtendOp(ValVariant &Val) const;
825
  template <typename TIn, typename TOut>
826
  TypeIF<TIn, TOut> runConvertOp(ValVariant &Val) const;
827
  template <typename TIn, typename TOut>
828
  TypeFF<TIn, TOut> runDemoteOp(ValVariant &Val) const;
829
  template <typename TIn, typename TOut>
830
  TypeFF<TIn, TOut> runPromoteOp(ValVariant &Val) const;
831
  template <typename TIn, typename TOut>
832
  TypeNN<TIn, TOut> runReinterpretOp(ValVariant &Val) const;
833
  /// ======= SIMD Memory instructions =======
834
  template <typename TIn, typename TOut>
835
  Expect<void> runLoadExpandOp(Runtime::StackManager &StackMgr,
836
                               Runtime::Instance::MemoryInstance &MemInst,
837
                               const AST::Instruction &Instr);
838
  template <typename T>
839
  Expect<void> runLoadSplatOp(Runtime::StackManager &StackMgr,
840
                              Runtime::Instance::MemoryInstance &MemInst,
841
                              const AST::Instruction &Instr);
842
  template <typename T>
843
  Expect<void> runLoadLaneOp(Runtime::StackManager &StackMgr,
844
                             Runtime::Instance::MemoryInstance &MemInst,
845
                             const AST::Instruction &Instr);
846
  template <typename T>
847
  Expect<void> runStoreLaneOp(Runtime::StackManager &StackMgr,
848
                              Runtime::Instance::MemoryInstance &MemInst,
849
                              const AST::Instruction &Instr);
850
  /// ======= SIMD Lane instructions =======
851
  template <typename TIn, typename TOut = TIn>
852
  Expect<void> runExtractLaneOp(ValVariant &Val, const uint8_t Index) const;
853
  template <typename TIn, typename TOut = TIn>
854
  Expect<void> runReplaceLaneOp(ValVariant &Val1, const ValVariant &Val2,
855
                                const uint8_t Index) const;
856
  /// ======= SIMD Numeric instructions =======
857
  template <typename TIn, typename TOut = TIn>
858
  Expect<void> runSplatOp(ValVariant &Val) const;
859
  template <typename T>
860
  Expect<void> runVectorEqOp(ValVariant &Val1, const ValVariant &Val2) const;
861
  template <typename T>
862
  Expect<void> runVectorNeOp(ValVariant &Val1, const ValVariant &Val2) const;
863
  template <typename T>
864
  Expect<void> runVectorLtOp(ValVariant &Val1, const ValVariant &Val2) const;
865
  template <typename T>
866
  Expect<void> runVectorGtOp(ValVariant &Val1, const ValVariant &Val2) const;
867
  template <typename T>
868
  Expect<void> runVectorLeOp(ValVariant &Val1, const ValVariant &Val2) const;
869
  template <typename T>
870
  Expect<void> runVectorGeOp(ValVariant &Val1, const ValVariant &Val2) const;
871
  template <typename T> Expect<void> runVectorAbsOp(ValVariant &Val) const;
872
  template <typename T> Expect<void> runVectorNegOp(ValVariant &Val) const;
873
  inline Expect<void> runVectorPopcntOp(ValVariant &Val) const;
874
  template <typename T> Expect<void> runVectorSqrtOp(ValVariant &Val) const;
875
  template <typename TIn, typename TOut>
876
  Expect<void> runVectorTruncSatOp(ValVariant &Val) const;
877
  template <typename TIn, typename TOut>
878
  Expect<void> runVectorConvertOp(ValVariant &Val) const;
879
  inline Expect<void> runVectorDemoteOp(ValVariant &Val) const;
880
  inline Expect<void> runVectorPromoteOp(ValVariant &Val) const;
881
  inline Expect<void> runVectorAnyTrueOp(ValVariant &Val) const;
882
  template <typename T> Expect<void> runVectorAllTrueOp(ValVariant &Val) const;
883
  template <typename T> Expect<void> runVectorBitMaskOp(ValVariant &Val) const;
884
  template <typename TIn, typename TOut>
885
  Expect<void> runVectorNarrowOp(ValVariant &Val1,
886
                                 const ValVariant &Val2) const;
887
  template <typename TIn, typename TOut>
888
  Expect<void> runVectorExtendLowOp(ValVariant &Val) const;
889
  template <typename TIn, typename TOut>
890
  Expect<void> runVectorExtendHighOp(ValVariant &Val) const;
891
  template <typename TIn, typename TOut>
892
  Expect<void> runVectorExtAddPairwiseOp(ValVariant &Val) const;
893
  template <typename TIn, typename TOut>
894
  Expect<void> runVectorExtMulLowOp(ValVariant &Val1,
895
                                    const ValVariant &Val2) const;
896
  template <typename TIn, typename TOut>
897
  Expect<void> runVectorExtMulHighOp(ValVariant &Val1,
898
                                     const ValVariant &Val2) const;
899
  inline Expect<void> runVectorQ15MulSatOp(ValVariant &Val1,
900
                                           const ValVariant &Val2) const;
901
  template <typename T>
902
  Expect<void> runVectorShlOp(ValVariant &Val1, const ValVariant &Val2) const;
903
  template <typename T>
904
  Expect<void> runVectorShrOp(ValVariant &Val1, const ValVariant &Val2) const;
905
  template <typename T>
906
  Expect<void> runVectorAddOp(ValVariant &Val1, const ValVariant &Val2) const;
907
  template <typename T>
908
  Expect<void> runVectorAddSatOp(ValVariant &Val1,
909
                                 const ValVariant &Val2) const;
910
  template <typename T>
911
  Expect<void> runVectorSubOp(ValVariant &Val1, const ValVariant &Val2) const;
912
  template <typename T>
913
  Expect<void> runVectorSubSatOp(ValVariant &Val1,
914
                                 const ValVariant &Val2) const;
915
  template <typename T>
916
  Expect<void> runVectorMulOp(ValVariant &Val1, const ValVariant &Val2) const;
917
  template <typename T>
918
  Expect<void> runVectorDivOp(ValVariant &Val1, const ValVariant &Val2) const;
919
  template <typename T>
920
  Expect<void> runVectorMinOp(ValVariant &Val1, const ValVariant &Val2) const;
921
  template <typename T>
922
  Expect<void> runVectorMaxOp(ValVariant &Val1, const ValVariant &Val2) const;
923
  template <typename T>
924
  Expect<void> runVectorFMinOp(ValVariant &Val1, const ValVariant &Val2) const;
925
  template <typename T>
926
  Expect<void> runVectorFMaxOp(ValVariant &Val1, const ValVariant &Val2) const;
927
  template <typename T, typename ET>
928
  Expect<void> runVectorAvgrOp(ValVariant &Val1, const ValVariant &Val2) const;
929
  template <typename T> Expect<void> runVectorCeilOp(ValVariant &Val) const;
930
  template <typename T> Expect<void> runVectorFloorOp(ValVariant &Val) const;
931
  template <typename T> Expect<void> runVectorTruncOp(ValVariant &Val) const;
932
  template <typename T> Expect<void> runVectorNearestOp(ValVariant &Val) const;
933
  /// ======= Relaxed SIMD instructions =======
934
  template <typename T>
935
  Expect<void> runVectorRelaxedLaneselectOp(ValVariant &Val1,
936
                                            const ValVariant &Val2,
937
                                            const ValVariant &Mask) const;
938
  inline Expect<void>
939
  runVectorRelaxedIntegerDotProductOp(ValVariant &Val1,
940
                                      const ValVariant &Val2) const;
941
  inline Expect<void> runVectorRelaxedIntegerDotProductOpAdd(
942
      ValVariant &Val1, const ValVariant &Val2, const ValVariant &C) const;
943
  /// ======= Atomic instructions =======
944
  Expect<void> runAtomicNotifyOp(Runtime::StackManager &StackMgr,
945
                                 Runtime::Instance::MemoryInstance &MemInst,
946
                                 const AST::Instruction &Instr);
947
  Expect<void> runMemoryFenceOp();
948
  template <typename T>
949
  TypeT<T> runAtomicWaitOp(Runtime::StackManager &StackMgr,
950
                           Runtime::Instance::MemoryInstance &MemInst,
951
                           const AST::Instruction &Instr);
952
  template <typename T, typename I>
953
  TypeT<T> runAtomicLoadOp(Runtime::StackManager &StackMgr,
954
                           Runtime::Instance::MemoryInstance &MemInst,
955
                           const AST::Instruction &Instr);
956
  template <typename T, typename I>
957
  TypeT<T> runAtomicStoreOp(Runtime::StackManager &StackMgr,
958
                            Runtime::Instance::MemoryInstance &MemInst,
959
                            const AST::Instruction &Instr);
960
  template <typename T, typename I>
961
  TypeT<T> runAtomicAddOp(Runtime::StackManager &StackMgr,
962
                          Runtime::Instance::MemoryInstance &MemInst,
963
                          const AST::Instruction &Instr);
964
  template <typename T, typename I>
965
  TypeT<T> runAtomicSubOp(Runtime::StackManager &StackMgr,
966
                          Runtime::Instance::MemoryInstance &MemInst,
967
                          const AST::Instruction &Instr);
968
  template <typename T, typename I>
969
  TypeT<T> runAtomicOrOp(Runtime::StackManager &StackMgr,
970
                         Runtime::Instance::MemoryInstance &MemInst,
971
                         const AST::Instruction &Instr);
972
  template <typename T, typename I>
973
  TypeT<T> runAtomicAndOp(Runtime::StackManager &StackMgr,
974
                          Runtime::Instance::MemoryInstance &MemInst,
975
                          const AST::Instruction &Instr);
976
  template <typename T, typename I>
977
  TypeT<T> runAtomicXorOp(Runtime::StackManager &StackMgr,
978
                          Runtime::Instance::MemoryInstance &MemInst,
979
                          const AST::Instruction &Instr);
980
  template <typename T, typename I>
981
  TypeT<T> runAtomicExchangeOp(Runtime::StackManager &StackMgr,
982
                               Runtime::Instance::MemoryInstance &MemInst,
983
                               const AST::Instruction &Instr);
984
  template <typename T, typename I>
985
  TypeT<T>
986
  runAtomicCompareExchangeOp(Runtime::StackManager &StackMgr,
987
                             Runtime::Instance::MemoryInstance &MemInst,
988
                             const AST::Instruction &Instr);
989
  /// @}
990
991
public:
992
  /// \name AOT/JIT - Run compiled functions
993
  /// @{
994
  Expect<void> proxyTrap(Runtime::StackManager &StackMgr,
995
                         const uint32_t Code) noexcept;
996
  Expect<void> proxyCall(Runtime::StackManager &StackMgr,
997
                         const Runtime::Instance::ModuleInstance *ModInst,
998
                         const uint32_t FuncIdx, const ValVariant *Args,
999
                         ValVariant *Rets) noexcept;
1000
  Expect<void>
1001
  proxyCallIndirect(Runtime::StackManager &StackMgr,
1002
                    const Runtime::Instance::ModuleInstance *ModInst,
1003
                    const uint32_t TableIdx, const uint32_t FuncTypeIdx,
1004
                    const uint64_t FuncIdx, const ValVariant *Args,
1005
                    ValVariant *Rets) noexcept;
1006
  Expect<void> proxyCallRef(Runtime::StackManager &StackMgr,
1007
                            const Runtime::Instance::ModuleInstance *ModInst,
1008
                            const RefVariant Ref, const ValVariant *Args,
1009
                            ValVariant *Rets) noexcept;
1010
  Expect<RefVariant>
1011
  proxyRefFunc(Runtime::StackManager &StackMgr,
1012
               const Runtime::Instance::ModuleInstance *ModInst,
1013
               const uint32_t FuncIdx) noexcept;
1014
  Expect<RefVariant>
1015
  proxyStructNew(Runtime::StackManager &StackMgr,
1016
                 const Runtime::Instance::ModuleInstance *ModInst,
1017
                 const uint32_t TypeIdx, const ValVariant *Args,
1018
                 const uint32_t ArgSize) noexcept;
1019
  Expect<void> proxyStructGet(Runtime::StackManager &StackMgr,
1020
                              const Runtime::Instance::ModuleInstance *ModInst,
1021
                              const RefVariant Ref, const uint32_t TypeIdx,
1022
                              const uint32_t Off, const bool IsSigned,
1023
                              ValVariant *Ret) noexcept;
1024
  Expect<void> proxyStructSet(Runtime::StackManager &StackMgr,
1025
                              const Runtime::Instance::ModuleInstance *ModInst,
1026
                              const RefVariant Ref, const uint32_t TypeIdx,
1027
                              const uint32_t Off,
1028
                              const ValVariant *Val) noexcept;
1029
  Expect<RefVariant>
1030
  proxyArrayNew(Runtime::StackManager &StackMgr,
1031
                const Runtime::Instance::ModuleInstance *ModInst,
1032
                const uint32_t TypeIdx, const uint32_t Length,
1033
                const ValVariant *Args, const uint32_t ArgSize) noexcept;
1034
  Expect<RefVariant>
1035
  proxyArrayNewData(Runtime::StackManager &StackMgr,
1036
                    const Runtime::Instance::ModuleInstance *ModInst,
1037
                    const uint32_t TypeIdx, const uint32_t DataIdx,
1038
                    const uint32_t Start, const uint32_t Length) noexcept;
1039
  Expect<RefVariant>
1040
  proxyArrayNewElem(Runtime::StackManager &StackMgr,
1041
                    const Runtime::Instance::ModuleInstance *ModInst,
1042
                    const uint32_t TypeIdx, const uint32_t ElemIdx,
1043
                    const uint32_t Start, const uint32_t Length) noexcept;
1044
  Expect<void> proxyArrayGet(Runtime::StackManager &StackMgr,
1045
                             const Runtime::Instance::ModuleInstance *ModInst,
1046
                             const RefVariant Ref, const uint32_t TypeIdx,
1047
                             const uint32_t Idx, const bool IsSigned,
1048
                             ValVariant *Ret) noexcept;
1049
  Expect<void> proxyArraySet(Runtime::StackManager &StackMgr,
1050
                             const Runtime::Instance::ModuleInstance *ModInst,
1051
                             const RefVariant Ref, const uint32_t TypeIdx,
1052
                             const uint32_t Idx,
1053
                             const ValVariant *Val) noexcept;
1054
  Expect<uint32_t> proxyArrayLen(Runtime::StackManager &StackMgr,
1055
                                 const RefVariant Ref) noexcept;
1056
  Expect<void> proxyArrayFill(Runtime::StackManager &StackMgr,
1057
                              const Runtime::Instance::ModuleInstance *ModInst,
1058
                              const RefVariant Ref, const uint32_t TypeIdx,
1059
                              const uint32_t Idx, const uint32_t Cnt,
1060
                              const ValVariant *Val) noexcept;
1061
  Expect<void> proxyArrayCopy(Runtime::StackManager &StackMgr,
1062
                              const Runtime::Instance::ModuleInstance *ModInst,
1063
                              const RefVariant DstRef,
1064
                              const uint32_t DstTypeIdx, const uint32_t DstIdx,
1065
                              const RefVariant SrcRef,
1066
                              const uint32_t SrcTypeIdx, const uint32_t SrcIdx,
1067
                              const uint32_t Cnt) noexcept;
1068
  Expect<void>
1069
  proxyArrayInitData(Runtime::StackManager &StackMgr,
1070
                     const Runtime::Instance::ModuleInstance *ModInst,
1071
                     const RefVariant Ref, const uint32_t TypeIdx,
1072
                     const uint32_t DataIdx, const uint32_t DstIdx,
1073
                     const uint32_t SrcIdx, const uint32_t Cnt) noexcept;
1074
  Expect<void>
1075
  proxyArrayInitElem(Runtime::StackManager &StackMgr,
1076
                     const Runtime::Instance::ModuleInstance *ModInst,
1077
                     const RefVariant Ref, const uint32_t TypeIdx,
1078
                     const uint32_t ElemIdx, const uint32_t DstIdx,
1079
                     const uint32_t SrcIdx, const uint32_t Cnt) noexcept;
1080
  Expect<uint32_t>
1081
  proxyRefTest(Runtime::StackManager &StackMgr,
1082
               const Runtime::Instance::ModuleInstance *ModInst,
1083
               const RefVariant Ref, ValType VTTest) noexcept;
1084
  Expect<RefVariant>
1085
  proxyRefCast(Runtime::StackManager &StackMgr,
1086
               const Runtime::Instance::ModuleInstance *ModInst,
1087
               const RefVariant Ref, ValType VTCast) noexcept;
1088
  Expect<void> proxyTableInit(Runtime::StackManager &StackMgr,
1089
                              const Runtime::Instance::ModuleInstance *ModInst,
1090
                              const uint32_t TableIdx, const uint32_t ElemIdx,
1091
                              const uint64_t DstOff, const uint32_t SrcOff,
1092
                              const uint32_t Len) noexcept;
1093
  Expect<void> proxyElemDrop(Runtime::StackManager &StackMgr,
1094
                             const Runtime::Instance::ModuleInstance *ModInst,
1095
                             const uint32_t ElemIdx) noexcept;
1096
  Expect<void> proxyTableCopy(Runtime::StackManager &StackMgr,
1097
                              const Runtime::Instance::ModuleInstance *ModInst,
1098
                              const uint32_t TableIdxDst,
1099
                              const uint32_t TableIdxSrc, const uint64_t DstOff,
1100
                              const uint64_t SrcOff,
1101
                              const uint64_t Len) noexcept;
1102
  Expect<uint64_t>
1103
  proxyTableGrow(Runtime::StackManager &StackMgr,
1104
                 const Runtime::Instance::ModuleInstance *ModInst,
1105
                 const uint32_t TableIdx, const RefVariant Val,
1106
                 const uint64_t NewSize) noexcept;
1107
  Expect<void> proxyTableFill(Runtime::StackManager &StackMgr,
1108
                              const Runtime::Instance::ModuleInstance *ModInst,
1109
                              const uint32_t TableIdx, const uint64_t Off,
1110
                              const RefVariant Ref,
1111
                              const uint64_t Len) noexcept;
1112
  Expect<uint64_t>
1113
  proxyMemGrow(Runtime::StackManager &StackMgr,
1114
               const Runtime::Instance::ModuleInstance *ModInst,
1115
               const uint32_t MemIdx, const uint64_t NewSize) noexcept;
1116
  Expect<void> proxyMemInit(Runtime::StackManager &StackMgr,
1117
                            const Runtime::Instance::ModuleInstance *ModInst,
1118
                            const uint32_t MemIdx, const uint32_t DataIdx,
1119
                            const uint64_t DstOff, const uint32_t SrcOff,
1120
                            const uint32_t Len) noexcept;
1121
  Expect<void> proxyDataDrop(Runtime::StackManager &StackMgr,
1122
                             const Runtime::Instance::ModuleInstance *ModInst,
1123
                             const uint32_t DataIdx) noexcept;
1124
  Expect<void> proxyMemCopy(Runtime::StackManager &StackMgr,
1125
                            const Runtime::Instance::ModuleInstance *ModInst,
1126
                            const uint32_t DstMemIdx, const uint32_t SrcMemIdx,
1127
                            const uint64_t DstOff, const uint64_t SrcOff,
1128
                            const uint64_t Len) noexcept;
1129
  Expect<void> proxyMemFill(Runtime::StackManager &StackMgr,
1130
                            const Runtime::Instance::ModuleInstance *ModInst,
1131
                            const uint32_t MemIdx, const uint64_t Off,
1132
                            const uint8_t Val, const uint64_t Len) noexcept;
1133
  Expect<uint64_t>
1134
  proxyMemAtomicNotify(Runtime::StackManager &StackMgr,
1135
                       const Runtime::Instance::ModuleInstance *ModInst,
1136
                       const uint32_t MemIdx, const uint64_t Offset,
1137
                       const uint64_t Count) noexcept;
1138
  Expect<uint64_t>
1139
  proxyMemAtomicWait(Runtime::StackManager &StackMgr,
1140
                     const Runtime::Instance::ModuleInstance *ModInst,
1141
                     const uint32_t MemIdx, const uint64_t Offset,
1142
                     const uint64_t Expected, const int64_t Timeout,
1143
                     const uint32_t BitWidth) noexcept;
1144
  Expect<void *> proxyTableGetFuncSymbol(
1145
      Runtime::StackManager &StackMgr,
1146
      const Runtime::Instance::ModuleInstance *ModInst, const uint32_t TableIdx,
1147
      const uint32_t FuncTypeIdx, const uint64_t FuncIdx,
1148
      Runtime::Instance::ModuleInstance::ModuleContext **CalleeCtxOut) noexcept;
1149
  Expect<void *> proxyRefGetFuncSymbol(
1150
      Runtime::StackManager &StackMgr,
1151
      const Runtime::Instance::ModuleInstance *ModInst, const RefVariant Ref,
1152
      Runtime::Instance::ModuleInstance::ModuleContext **CalleeCtxOut) noexcept;
1153
  Expect<void *>
1154
  proxyFuncGetFuncSymbol(Runtime::StackManager &StackMgr,
1155
                         const Runtime::Instance::ModuleInstance *ModInst,
1156
                         const uint32_t FuncIdx) noexcept;
1157
  Expect<void> proxyThrow(Runtime::StackManager &StackMgr,
1158
                          const Runtime::Instance::ModuleInstance *ModInst,
1159
                          const uint32_t TagIdx, const ValVariant *Vals,
1160
                          const uint32_t Num) noexcept;
1161
  Expect<void> proxyThrowRef(Runtime::StackManager &StackMgr,
1162
                             const RefVariant Ref) noexcept;
1163
  Expect<void> proxyCatchPop(Runtime::StackManager &StackMgr,
1164
                             const Runtime::Instance::ModuleInstance *ModInst,
1165
                             ValVariant *Out, const uint32_t PopPayload,
1166
                             const uint32_t NeedRef) noexcept;
1167
  /// @}
1168
1169
  /// Callbacks for compiled modules
1170
  static const Executable::IntrinsicsTable Intrinsics;
1171
  /// Proxy helper template struct
1172
  template <typename FuncPtr> struct ProxyHelper;
1173
1174
private:
1175
  /// Execution context for compiled functions.
1176
  struct ExecutorContext {
1177
    std::atomic_uint64_t *InstrCount;
1178
    uint64_t *CostTable;
1179
    std::atomic_uint64_t *Gas;
1180
    uint64_t GasLimit;
1181
    std::atomic_uint32_t *StopToken;
1182
    void *const *PendingExnTagAddr;
1183
    uint8_t *StackLimit;
1184
  };
1185
1186
  /// Compiled code reads this struct by field index through the mirrored
1187
  /// ExecCtx type built in lib/llvm/compiler/context.cpp. Keep both in the same
1188
  /// order.
1189
  static_assert(offsetof(ExecutorContext, InstrCount) == 0);
1190
  static_assert(offsetof(ExecutorContext, InstrCount) <
1191
                offsetof(ExecutorContext, CostTable));
1192
  static_assert(offsetof(ExecutorContext, CostTable) <
1193
                offsetof(ExecutorContext, Gas));
1194
  static_assert(offsetof(ExecutorContext, Gas) <
1195
                offsetof(ExecutorContext, GasLimit));
1196
  static_assert(offsetof(ExecutorContext, GasLimit) <
1197
                offsetof(ExecutorContext, StopToken));
1198
  static_assert(offsetof(ExecutorContext, StopToken) <
1199
                offsetof(ExecutorContext, PendingExnTagAddr));
1200
  static_assert(offsetof(ExecutorContext, PendingExnTagAddr) <
1201
                offsetof(ExecutorContext, StackLimit));
1202
1203
  /// Restores thread local VM reference after overwriting it.
1204
  struct SavedThreadLocal {
1205
    SavedThreadLocal(Executor &Ex, Runtime::StackManager &StackMgr,
1206
                     [[maybe_unused]] const Runtime::Instance::FunctionInstance
1207
                         &Func) noexcept;
1208
1209
    SavedThreadLocal(const SavedThreadLocal &) = delete;
1210
    SavedThreadLocal(SavedThreadLocal &&) = delete;
1211
1212
    ~SavedThreadLocal() noexcept;
1213
1214
    Executor *SavedThis;
1215
    Runtime::StackManager *SavedCurrentStack;
1216
    ExecutorContext SavedExecutionContext;
1217
  };
1218
1219
  /// Pending exception passing across the compiled and the native-entered
1220
  /// frames. A non-null tag means pending; the exception instance keeps the
1221
  /// throw_ref identity.
1222
  struct PendingExnStruct {
1223
    /// Tag instance of the pending exception. Null when no exception is
1224
    /// pending.
1225
    Runtime::Instance::TagInstance *TagInst = nullptr;
1226
    /// Exception instance identity. Null for a fresh throw; set on rethrowing
1227
    /// to keep the exnref identity.
1228
    const Runtime::Instance::ExceptionInstance *Inst = nullptr;
1229
1230
    /// Getter and setter of the payload values.
1231
0
    const std::vector<ValVariant> &getPayload() const noexcept {
1232
0
      return Payload;
1233
0
    }
1234
0
    void setPayload(Span<const ValVariant> Vals) noexcept {
1235
0
      Payload.assign(Vals.begin(), Vals.end());
1236
0
    }
1237
1238
  private:
1239
    std::vector<ValVariant> Payload;
1240
  };
1241
1242
  /// Pointer to current object.
1243
  static thread_local Executor *This;
1244
  /// Stack passed into compiled functions
1245
  static thread_local Runtime::StackManager *CurrentStack;
1246
  /// Execution context for compiled functions
1247
  static thread_local ExecutorContext ExecutionContext;
1248
  /// Pending exception for compiled functions
1249
  static thread_local PendingExnStruct PendingExn;
1250
  /// Record stack trace on error
1251
  static thread_local std::array<StackTraceEntry, 256> StackTrace;
1252
  static thread_local size_t StackTraceSize;
1253
1254
  /// WasmEdge configuration
1255
  const Configure Conf;
1256
  /// Executor statistics
1257
  Statistics::Statistics *Stat;
1258
  /// Stop execution
1259
  std::atomic_uint32_t StopToken = 0;
1260
  /// Memory instance this Executor is currently waiting on (for stop()).
1261
  std::atomic<Runtime::Instance::MemoryInstance *> WaitingMemory = nullptr;
1262
  /// Executor Host Function Handler
1263
  HostFuncHandler HostFuncHelper = {};
1264
  /// Callback for lazy function compilation
1265
  std::function<Expect<void>(const Runtime::Instance::FunctionInstance *)>
1266
      LazyCompilationHandler;
1267
1268
  /// Helper function for triggering lazy compilation.
1269
  /// XXX: Calling checkLazyCompilation in one thread while another thread calls
1270
  /// unsafeUpgradeToCompiled on the same FuncInst could result in a race
1271
  /// condition if checking FuncInst->isCompiledFunction() directly here. As a
1272
  /// temporary workaround, checks for compilation state are deferred to the
1273
  /// LazyCompilationHandler, which must serialize them against compiled-state
1274
  /// upgrades (the lazy JIT engine does so under its internal lock).
1275
  Expect<void> checkLazyCompilation(
1276
0
      const Runtime::Instance::FunctionInstance *FuncInst) const noexcept {
1277
0
    if (unlikely(LazyCompilationHandler != nullptr)) {
1278
0
      return LazyCompilationHandler(FuncInst);
1279
0
    }
1280
0
    return {};
1281
0
  }
1282
};
1283
1284
} // namespace Executor
1285
} // namespace WasmEdge
1286
1287
#include "engine/atomic.ipp"
1288
#include "engine/binary_numeric.ipp"
1289
#include "engine/cast_numeric.ipp"
1290
#include "engine/memory.ipp"
1291
#include "engine/relation_numeric.ipp"
1292
#include "engine/unary_numeric.ipp"