Coverage Report

Created: 2026-11-12 06:35

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/WasmEdge/lib/llvm/compiler/memoryInstr.cpp
Line
Count
Source
1
// SPDX-License-Identifier: Apache-2.0
2
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4
#include "compiler/function_compiler.h"
5
6
#include <limits>
7
8
namespace WasmEdge {
9
10
Expect<void>
11
621k
FunctionCompiler::compileMemoryOp(const AST::Instruction &Instr) noexcept {
12
621k
  switch (Instr.getOpCode()) {
13
1.08k
  case OpCode::I32__load:
14
1.08k
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
15
1.08k
                  Instr.getMemoryAlign(), Context.Int32Ty);
16
1.08k
    break;
17
2.44k
  case OpCode::I64__load:
18
2.44k
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
19
2.44k
                  Instr.getMemoryAlign(), Context.Int64Ty);
20
2.44k
    break;
21
118
  case OpCode::F32__load:
22
118
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
23
118
                  Instr.getMemoryAlign(), Context.FloatTy);
24
118
    break;
25
234
  case OpCode::F64__load:
26
234
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
27
234
                  Instr.getMemoryAlign(), Context.DoubleTy);
28
234
    break;
29
531
  case OpCode::I32__load8_s:
30
531
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
31
531
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
32
531
                  true);
33
531
    break;
34
412
  case OpCode::I32__load8_u:
35
412
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
36
412
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
37
412
                  false);
38
412
    break;
39
308
  case OpCode::I32__load16_s:
40
308
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
41
308
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
42
308
                  true);
43
308
    break;
44
1.53k
  case OpCode::I32__load16_u:
45
1.53k
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
46
1.53k
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
47
1.53k
                  false);
48
1.53k
    break;
49
687
  case OpCode::I64__load8_s:
50
687
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
51
687
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
52
687
                  true);
53
687
    break;
54
446
  case OpCode::I64__load8_u:
55
446
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
56
446
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
57
446
                  false);
58
446
    break;
59
319
  case OpCode::I64__load16_s:
60
319
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
61
319
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
62
319
                  true);
63
319
    break;
64
576
  case OpCode::I64__load16_u:
65
576
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
66
576
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
67
576
                  false);
68
576
    break;
69
366
  case OpCode::I64__load32_s:
70
366
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
71
366
                  Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
72
366
                  true);
73
366
    break;
74
476
  case OpCode::I64__load32_u:
75
476
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
76
476
                  Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
77
476
                  false);
78
476
    break;
79
417
  case OpCode::I32__store:
80
417
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
81
417
                   Instr.getMemoryAlign(), Context.Int32Ty);
82
417
    break;
83
1.35k
  case OpCode::I64__store:
84
1.35k
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
85
1.35k
                   Instr.getMemoryAlign(), Context.Int64Ty);
86
1.35k
    break;
87
65
  case OpCode::F32__store:
88
65
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
89
65
                   Instr.getMemoryAlign(), Context.FloatTy);
90
65
    break;
91
53
  case OpCode::F64__store:
92
53
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
93
53
                   Instr.getMemoryAlign(), Context.DoubleTy);
94
53
    break;
95
276
  case OpCode::I32__store8:
96
296
  case OpCode::I64__store8:
97
296
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
98
296
                   Instr.getMemoryAlign(), Context.Int8Ty, true);
99
296
    break;
100
241
  case OpCode::I32__store16:
101
366
  case OpCode::I64__store16:
102
366
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
103
366
                   Instr.getMemoryAlign(), Context.Int16Ty, true);
104
366
    break;
105
39
  case OpCode::I64__store32:
106
39
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
107
39
                   Instr.getMemoryAlign(), Context.Int32Ty, true);
108
39
    break;
109
762
  case OpCode::Memory__size:
110
762
    stackPush(Builder.createTrunc(
111
762
        Context.getMemorySize(Builder, ModCtx, Instr.getTargetIndex()),
112
762
        Context.MemoryAddrTypes[Instr.getTargetIndex()]));
113
762
    break;
114
809
  case OpCode::Memory__grow: {
115
809
    auto NewPageSize = Builder.createZExt(stackPop(), Context.Int64Ty);
116
809
    stackPush(Builder.createTrunc(
117
809
        Builder.createCall(
118
809
            Context.getIntrinsic(
119
809
                Builder, Executable::Intrinsics::kMemGrow,
120
809
                LLVM::Type::getFunctionType(
121
809
                    Context.Int64Ty,
122
809
                    {Context.Int8PtrTy, Context.Int32Ty, Context.Int64Ty},
123
809
                    false)),
124
809
            {Context.getModuleInst(Builder, ModCtx),
125
809
             LLContext.getInt32(Instr.getTargetIndex()), NewPageSize}),
126
809
        Context.MemoryAddrTypes[Instr.getTargetIndex()]));
127
809
    break;
128
241
  }
129
23
  case OpCode::Memory__init: {
130
23
    auto Len = stackPop();
131
23
    auto Src = stackPop();
132
23
    auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
133
23
    Builder.createCall(
134
23
        Context.getIntrinsic(
135
23
            Builder, Executable::Intrinsics::kMemInit,
136
23
            LLVM::Type::getFunctionType(Context.VoidTy,
137
23
                                        {Context.Int8PtrTy, Context.Int32Ty,
138
23
                                         Context.Int32Ty, Context.Int64Ty,
139
23
                                         Context.Int32Ty, Context.Int32Ty},
140
23
                                        false)),
141
23
        {Context.getModuleInst(Builder, ModCtx),
142
23
         LLContext.getInt32(Instr.getTargetIndex()),
143
23
         LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
144
23
    break;
145
241
  }
146
22
  case OpCode::Data__drop: {
147
22
    Builder.createCall(
148
22
        Context.getIntrinsic(
149
22
            Builder, Executable::Intrinsics::kDataDrop,
150
22
            LLVM::Type::getFunctionType(
151
22
                Context.VoidTy, {Context.Int8PtrTy, Context.Int32Ty}, false)),
152
22
        {Context.getModuleInst(Builder, ModCtx),
153
22
         LLContext.getInt32(Instr.getTargetIndex())});
154
22
    break;
155
241
  }
156
249
  case OpCode::Memory__copy: {
157
249
    auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
158
249
    auto Src = Builder.createZExt(stackPop(), Context.Int64Ty);
159
249
    auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
160
249
    Builder.createCall(
161
249
        Context.getIntrinsic(
162
249
            Builder, Executable::Intrinsics::kMemCopy,
163
249
            LLVM::Type::getFunctionType(Context.VoidTy,
164
249
                                        {Context.Int8PtrTy, Context.Int32Ty,
165
249
                                         Context.Int32Ty, Context.Int64Ty,
166
249
                                         Context.Int64Ty, Context.Int64Ty},
167
249
                                        false)),
168
249
        {Context.getModuleInst(Builder, ModCtx),
169
249
         LLContext.getInt32(Instr.getTargetIndex()),
170
249
         LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
171
249
    break;
172
241
  }
173
635
  case OpCode::Memory__fill: {
174
635
    auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
175
635
    auto Val = Builder.createTrunc(stackPop(), Context.Int8Ty);
176
635
    auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
177
635
    Builder.createCall(
178
635
        Context.getIntrinsic(
179
635
            Builder, Executable::Intrinsics::kMemFill,
180
635
            LLVM::Type::getFunctionType(Context.VoidTy,
181
635
                                        {Context.Int8PtrTy, Context.Int32Ty,
182
635
                                         Context.Int64Ty, Context.Int8Ty,
183
635
                                         Context.Int64Ty},
184
635
                                        false)),
185
635
        {Context.getModuleInst(Builder, ModCtx),
186
635
         LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});
187
635
    break;
188
241
  }
189
190
  // Const Numeric Instructions
191
516k
  case OpCode::I32__const:
192
516k
    stackPush(LLContext.getInt32(Instr.getNum().get<uint32_t>()));
193
516k
    break;
194
70.7k
  case OpCode::I64__const:
195
70.7k
    stackPush(LLContext.getInt64(Instr.getNum().get<uint64_t>()));
196
70.7k
    break;
197
13.7k
  case OpCode::F32__const:
198
13.7k
    stackPush(LLContext.getFloat(Instr.getNum().get<float>()));
199
13.7k
    break;
200
6.11k
  case OpCode::F64__const:
201
6.11k
    stackPush(LLContext.getDouble(Instr.getNum().get<double>()));
202
6.11k
    break;
203
204
0
  default:
205
0
    assumingUnreachable();
206
621k
  }
207
621k
  return {};
208
621k
}
209
210
// Memory32 cannot escape the reservation's guard region, so it keeps the
211
// guard-page fast path. Memory64 addresses are guest-controlled 64-bit values.
212
void FunctionCompiler::boundsCheckMemory64(unsigned MemoryIndex,
213
                                           LLVM::Value Addr, uint64_t Offset,
214
19.4k
                                           uint64_t AccessSize) noexcept {
215
19.4k
  if (Context.MemoryAddrTypes[MemoryIndex].getIntegerBitWidth() != 64) {
216
19.4k
    return;
217
19.4k
  }
218
1
  if (Offset > std::numeric_limits<uint64_t>::max() - AccessSize) {
219
    // No address can satisfy the access.
220
0
    Builder.createBr(getTrapBB(ErrCode::Value::MemoryOutOfBounds));
221
0
    Builder.positionAtEnd(
222
0
        LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok"));
223
0
    return;
224
0
  }
225
  // Addr < usub_sat(SizeBytes, Offset + AccessSize - 1) is exactly
226
  // Addr + Offset + AccessSize <= SizeBytes with every term kept in 64 bits.
227
1
  const uint64_t OffsetAndSize = Offset + AccessSize;
228
1
  auto SizeBytes =
229
1
      Builder.createShl(Context.getMemorySize(Builder, ModCtx, MemoryIndex),
230
1
                        LLContext.getInt64(16));
231
1
  auto Limit = Builder.createIntrinsic(
232
1
      LLVM::Core::USubSat, {Context.Int64Ty},
233
1
      {SizeBytes, LLContext.getInt64(OffsetAndSize - 1)});
234
1
  auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok");
235
1
  Builder.createCondBr(Builder.createLikely(Builder.createICmpULT(Addr, Limit)),
236
1
                       OkBB, getTrapBB(ErrCode::Value::MemoryOutOfBounds));
237
1
  Builder.positionAtEnd(OkBB);
238
1
}
239
240
void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
241
                                     unsigned Alignment,
242
16.2k
                                     LLVM::Type LoadTy) noexcept {
243
16.2k
  if constexpr (LLVM::kForceUnalignment) {
244
16.2k
    Alignment = 0;
245
16.2k
  }
246
16.2k
  auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
247
16.2k
  boundsCheckMemory64(MemoryIndex, Off, Offset,
248
16.2k
                      LoadTy.getPrimitiveSizeInBits() / 8);
249
16.2k
  if (Offset != 0) {
250
10.2k
    Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
251
10.2k
  }
252
253
16.2k
  auto VPtr = Builder.createInBoundsGEP1(
254
16.2k
      Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
255
16.2k
  auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
256
16.2k
  auto LoadInst = Builder.createLoad(LoadTy, Ptr, true);
257
16.2k
  LoadInst.setAlignment(1 << Alignment);
258
16.2k
  stackPush(switchEndian(LoadInst));
259
16.2k
}
260
261
void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
262
                                     unsigned Alignment, LLVM::Type LoadTy,
263
                                     LLVM::Type ExtendTy,
264
7.16k
                                     bool Signed) noexcept {
265
7.16k
  compileLoadOp(MemoryIndex, Offset, Alignment, LoadTy);
266
7.16k
  if (Signed) {
267
2.82k
    Stack.back() = Builder.createSExt(Stack.back(), ExtendTy);
268
4.33k
  } else {
269
4.33k
    Stack.back() = Builder.createZExt(Stack.back(), ExtendTy);
270
4.33k
  }
271
7.16k
}
272
273
void FunctionCompiler::compileStoreOp(uint32_t MemoryIndex, uint64_t Offset,
274
                                      uint32_t Alignment, LLVM::Type LoadTy,
275
3.14k
                                      bool Trunc, bool BitCast) noexcept {
276
3.14k
  if constexpr (LLVM::kForceUnalignment) {
277
3.14k
    Alignment = 0;
278
3.14k
  }
279
3.14k
  auto V = stackPop();
280
3.14k
  auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
281
3.14k
  boundsCheckMemory64(MemoryIndex, Off, Offset,
282
3.14k
                      LoadTy.getPrimitiveSizeInBits() / 8);
283
3.14k
  if (Offset != 0) {
284
2.35k
    Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
285
2.35k
  }
286
287
3.14k
  if (Trunc) {
288
701
    V = Builder.createTrunc(V, LoadTy);
289
701
  }
290
3.14k
  if (BitCast) {
291
226
    V = Builder.createBitCast(V, LoadTy);
292
226
  }
293
3.14k
  V = switchEndian(V);
294
3.14k
  auto VPtr = Builder.createInBoundsGEP1(
295
3.14k
      Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
296
3.14k
  auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
297
3.14k
  auto StoreInst = Builder.createStore(V, Ptr, true);
298
3.14k
  StoreInst.setAlignment(1 << Alignment);
299
3.14k
}
300
301
} // namespace WasmEdge