/src/WasmEdge/lib/llvm/compiler/memoryInstr.cpp

Source
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors

#include "compiler/function_compiler.h"

#include <limits>

namespace WasmEdge {

Expect<void>
FunctionCompiler::compileMemoryOp(const AST::Instruction &Instr) noexcept {
  switch (Instr.getOpCode()) {
  case OpCode::I32__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int32Ty);
    break;
  case OpCode::I64__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int64Ty);
    break;
  case OpCode::F32__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.FloatTy);
    break;
  case OpCode::F64__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.DoubleTy);
    break;
  case OpCode::I32__load8_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
                  true);
    break;
  case OpCode::I32__load8_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
                  false);
    break;
  case OpCode::I32__load16_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
                  true);
    break;
  case OpCode::I32__load16_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
                  false);
    break;
  case OpCode::I64__load8_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
                  true);
    break;
  case OpCode::I64__load8_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
                  false);
    break;
  case OpCode::I64__load16_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
                  true);
    break;
  case OpCode::I64__load16_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
                  false);
    break;
  case OpCode::I64__load32_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
                  true);
    break;
  case OpCode::I64__load32_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
                  false);
    break;
  case OpCode::I32__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int32Ty);
    break;
  case OpCode::I64__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int64Ty);
    break;
  case OpCode::F32__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.FloatTy);
    break;
  case OpCode::F64__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.DoubleTy);
    break;
  case OpCode::I32__store8:
  case OpCode::I64__store8:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int8Ty, true);
    break;
  case OpCode::I32__store16:
  case OpCode::I64__store16:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int16Ty, true);
    break;
  case OpCode::I64__store32:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int32Ty, true);
    break;
  case OpCode::Memory__size:
    stackPush(Builder.createTrunc(
        Context.getMemorySize(Builder, ModCtx, Instr.getTargetIndex()),
        Context.MemoryAddrTypes[Instr.getTargetIndex()]));
    break;
  case OpCode::Memory__grow: {
    auto NewPageSize = Builder.createZExt(stackPop(), Context.Int64Ty);
    stackPush(Builder.createTrunc(
        Builder.createCall(
            Context.getIntrinsic(
                Builder, Executable::Intrinsics::kMemGrow,
                LLVM::Type::getFunctionType(
                    Context.Int64Ty,
                    {Context.Int8PtrTy, Context.Int32Ty, Context.Int64Ty},
                    false)),
            {Context.getModuleInst(Builder, ModCtx),
             LLContext.getInt32(Instr.getTargetIndex()), NewPageSize}),
        Context.MemoryAddrTypes[Instr.getTargetIndex()]));
    break;
  }
  case OpCode::Memory__init: {
    auto Len = stackPop();
    auto Src = stackPop();
    auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kMemInit,
            LLVM::Type::getFunctionType(Context.VoidTy,
                                        {Context.Int8PtrTy, Context.Int32Ty,
                                         Context.Int32Ty, Context.Int64Ty,
                                         Context.Int32Ty, Context.Int32Ty},
                                        false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex()),
         LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
    break;
  }
  case OpCode::Data__drop: {
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kDataDrop,
            LLVM::Type::getFunctionType(
                Context.VoidTy, {Context.Int8PtrTy, Context.Int32Ty}, false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex())});
    break;
  }
  case OpCode::Memory__copy: {
    auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
    auto Src = Builder.createZExt(stackPop(), Context.Int64Ty);
    auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kMemCopy,
            LLVM::Type::getFunctionType(Context.VoidTy,
                                        {Context.Int8PtrTy, Context.Int32Ty,
                                         Context.Int32Ty, Context.Int64Ty,
                                         Context.Int64Ty, Context.Int64Ty},
                                        false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex()),
         LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
    break;
  }
  case OpCode::Memory__fill: {
    auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
    auto Val = Builder.createTrunc(stackPop(), Context.Int8Ty);
    auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kMemFill,
            LLVM::Type::getFunctionType(Context.VoidTy,
                                        {Context.Int8PtrTy, Context.Int32Ty,
                                         Context.Int64Ty, Context.Int8Ty,
                                         Context.Int64Ty},
                                        false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});
    break;
  }

  // Const Numeric Instructions
  case OpCode::I32__const:
    stackPush(LLContext.getInt32(Instr.getNum().get<uint32_t>()));
    break;
  case OpCode::I64__const:
    stackPush(LLContext.getInt64(Instr.getNum().get<uint64_t>()));
    break;
  case OpCode::F32__const:
    stackPush(LLContext.getFloat(Instr.getNum().get<float>()));
    break;
  case OpCode::F64__const:
    stackPush(LLContext.getDouble(Instr.getNum().get<double>()));
    break;

  default:
    assumingUnreachable();
  }
  return {};
}

// Memory32 cannot escape the reservation's guard region, so it keeps the
// guard-page fast path. Memory64 addresses are guest-controlled 64-bit values.
void FunctionCompiler::boundsCheckMemory64(unsigned MemoryIndex,
                                           LLVM::Value Addr, uint64_t Offset,
                                           uint64_t AccessSize) noexcept {
  if (Context.MemoryAddrTypes[MemoryIndex].getIntegerBitWidth() != 64) {
    return;
  }
  if (Offset > std::numeric_limits<uint64_t>::max() - AccessSize) {
    // No address can satisfy the access.
    Builder.createBr(getTrapBB(ErrCode::Value::MemoryOutOfBounds));
    Builder.positionAtEnd(
        LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok"));
    return;
  }
  // Addr < usub_sat(SizeBytes, Offset + AccessSize - 1) is exactly
  // Addr + Offset + AccessSize <= SizeBytes with every term kept in 64 bits.
  const uint64_t OffsetAndSize = Offset + AccessSize;
  auto SizeBytes =
      Builder.createShl(Context.getMemorySize(Builder, ModCtx, MemoryIndex),
                        LLContext.getInt64(16));
  auto Limit = Builder.createIntrinsic(
      LLVM::Core::USubSat, {Context.Int64Ty},
      {SizeBytes, LLContext.getInt64(OffsetAndSize - 1)});
  auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok");
  Builder.createCondBr(Builder.createLikely(Builder.createICmpULT(Addr, Limit)),
                       OkBB, getTrapBB(ErrCode::Value::MemoryOutOfBounds));
  Builder.positionAtEnd(OkBB);
}

void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
                                     unsigned Alignment,
                                     LLVM::Type LoadTy) noexcept {
  if constexpr (LLVM::kForceUnalignment) {
    Alignment = 0;
  }
  auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
  boundsCheckMemory64(MemoryIndex, Off, Offset,
                      LoadTy.getPrimitiveSizeInBits() / 8);
  if (Offset != 0) {
    Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
  }

  auto VPtr = Builder.createInBoundsGEP1(
      Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
  auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
  auto LoadInst = Builder.createLoad(LoadTy, Ptr, true);
  LoadInst.setAlignment(1 << Alignment);
  stackPush(switchEndian(LoadInst));
}

void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
                                     unsigned Alignment, LLVM::Type LoadTy,
                                     LLVM::Type ExtendTy,
                                     bool Signed) noexcept {
  compileLoadOp(MemoryIndex, Offset, Alignment, LoadTy);
  if (Signed) {
    Stack.back() = Builder.createSExt(Stack.back(), ExtendTy);
  } else {
    Stack.back() = Builder.createZExt(Stack.back(), ExtendTy);
  }
}

void FunctionCompiler::compileStoreOp(uint32_t MemoryIndex, uint64_t Offset,
                                      uint32_t Alignment, LLVM::Type LoadTy,
                                      bool Trunc, bool BitCast) noexcept {
  if constexpr (LLVM::kForceUnalignment) {
    Alignment = 0;
  }
  auto V = stackPop();
  auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
  boundsCheckMemory64(MemoryIndex, Off, Offset,
                      LoadTy.getPrimitiveSizeInBits() / 8);
  if (Offset != 0) {
    Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
  }

  if (Trunc) {
    V = Builder.createTrunc(V, LoadTy);
  }
  if (BitCast) {
    V = Builder.createBitCast(V, LoadTy);
  }
  V = switchEndian(V);
  auto VPtr = Builder.createInBoundsGEP1(
      Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
  auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
  auto StoreInst = Builder.createStore(V, Ptr, true);
  StoreInst.setAlignment(1 << Alignment);
}

} // namespace WasmEdge

Coverage Report

Created: 2026-09-28 06:22

Line	Count	Source
1		// SPDX-License-Identifier: Apache-2.0
2		// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4		#include "compiler/function_compiler.h"
5
6		#include <limits>
7
8		namespace WasmEdge {
9
10		Expect<void>
11	642k	FunctionCompiler::compileMemoryOp(const AST::Instruction &Instr) noexcept {
12	642k	switch (Instr.getOpCode()) {
13	1.20k	case OpCode::I32__load:
14	1.20k	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
15	1.20k	Instr.getMemoryAlign(), Context.Int32Ty);
16	1.20k	break;
17	2.70k	case OpCode::I64__load:
18	2.70k	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
19	2.70k	Instr.getMemoryAlign(), Context.Int64Ty);
20	2.70k	break;
21	107	case OpCode::F32__load:
22	107	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
23	107	Instr.getMemoryAlign(), Context.FloatTy);
24	107	break;
25	225	case OpCode::F64__load:
26	225	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
27	225	Instr.getMemoryAlign(), Context.DoubleTy);
28	225	break;
29	674	case OpCode::I32__load8_s:
30	674	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
31	674	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
32	674	true);
33	674	break;
34	438	case OpCode::I32__load8_u:
35	438	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
36	438	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
37	438	false);
38	438	break;
39	325	case OpCode::I32__load16_s:
40	325	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
41	325	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
42	325	true);
43	325	break;
44	1.56k	case OpCode::I32__load16_u:
45	1.56k	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
46	1.56k	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
47	1.56k	false);
48	1.56k	break;
49	750	case OpCode::I64__load8_s:
50	750	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
51	750	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
52	750	true);
53	750	break;
54	451	case OpCode::I64__load8_u:
55	451	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
56	451	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
57	451	false);
58	451	break;
59	375	case OpCode::I64__load16_s:
60	375	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
61	375	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
62	375	true);
63	375	break;
64	591	case OpCode::I64__load16_u:
65	591	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
66	591	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
67	591	false);
68	591	break;
69	404	case OpCode::I64__load32_s:
70	404	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
71	404	Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
72	404	true);
73	404	break;
74	500	case OpCode::I64__load32_u:
75	500	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
76	500	Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
77	500	false);
78	500	break;
79	396	case OpCode::I32__store:
80	396	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
81	396	Instr.getMemoryAlign(), Context.Int32Ty);
82	396	break;
83	1.34k	case OpCode::I64__store:
84	1.34k	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
85	1.34k	Instr.getMemoryAlign(), Context.Int64Ty);
86	1.34k	break;
87	64	case OpCode::F32__store:
88	64	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
89	64	Instr.getMemoryAlign(), Context.FloatTy);
90	64	break;
91	43	case OpCode::F64__store:
92	43	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
93	43	Instr.getMemoryAlign(), Context.DoubleTy);
94	43	break;
95	269	case OpCode::I32__store8:
96	285	case OpCode::I64__store8:
97	285	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
98	285	Instr.getMemoryAlign(), Context.Int8Ty, true);
99	285	break;
100	231	case OpCode::I32__store16:
101	351	case OpCode::I64__store16:
102	351	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
103	351	Instr.getMemoryAlign(), Context.Int16Ty, true);
104	351	break;
105	27	case OpCode::I64__store32:
106	27	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
107	27	Instr.getMemoryAlign(), Context.Int32Ty, true);
108	27	break;
109	697	case OpCode::Memory__size:
110	697	stackPush(Builder.createTrunc(
111	697	Context.getMemorySize(Builder, ModCtx, Instr.getTargetIndex()),
112	697	Context.MemoryAddrTypes[Instr.getTargetIndex()]));
113	697	break;
114	868	case OpCode::Memory__grow: {
115	868	auto NewPageSize = Builder.createZExt(stackPop(), Context.Int64Ty);
116	868	stackPush(Builder.createTrunc(
117	868	Builder.createCall(
118	868	Context.getIntrinsic(
119	868	Builder, Executable::Intrinsics::kMemGrow,
120	868	LLVM::Type::getFunctionType(
121	868	Context.Int64Ty,
122	868	{Context.Int8PtrTy, Context.Int32Ty, Context.Int64Ty},
123	868	false)),
124	868	{Context.getModuleInst(Builder, ModCtx),
125	868	LLContext.getInt32(Instr.getTargetIndex()), NewPageSize}),
126	868	Context.MemoryAddrTypes[Instr.getTargetIndex()]));
127	868	break;
128	231	}
129	23	case OpCode::Memory__init: {
130	23	auto Len = stackPop();
131	23	auto Src = stackPop();
132	23	auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
133	23	Builder.createCall(
134	23	Context.getIntrinsic(
135	23	Builder, Executable::Intrinsics::kMemInit,
136	23	LLVM::Type::getFunctionType(Context.VoidTy,
137	23	{Context.Int8PtrTy, Context.Int32Ty,
138	23	Context.Int32Ty, Context.Int64Ty,
139	23	Context.Int32Ty, Context.Int32Ty},
140	23	false)),
141	23	{Context.getModuleInst(Builder, ModCtx),
142	23	LLContext.getInt32(Instr.getTargetIndex()),
143	23	LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
144	23	break;
145	231	}
146	22	case OpCode::Data__drop: {
147	22	Builder.createCall(
148	22	Context.getIntrinsic(
149	22	Builder, Executable::Intrinsics::kDataDrop,
150	22	LLVM::Type::getFunctionType(
151	22	Context.VoidTy, {Context.Int8PtrTy, Context.Int32Ty}, false)),
152	22	{Context.getModuleInst(Builder, ModCtx),
153	22	LLContext.getInt32(Instr.getTargetIndex())});
154	22	break;
155	231	}
156	249	case OpCode::Memory__copy: {
157	249	auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
158	249	auto Src = Builder.createZExt(stackPop(), Context.Int64Ty);
159	249	auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
160	249	Builder.createCall(
161	249	Context.getIntrinsic(
162	249	Builder, Executable::Intrinsics::kMemCopy,
163	249	LLVM::Type::getFunctionType(Context.VoidTy,
164	249	{Context.Int8PtrTy, Context.Int32Ty,
165	249	Context.Int32Ty, Context.Int64Ty,
166	249	Context.Int64Ty, Context.Int64Ty},
167	249	false)),
168	249	{Context.getModuleInst(Builder, ModCtx),
169	249	LLContext.getInt32(Instr.getTargetIndex()),
170	249	LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
171	249	break;
172	231	}
173	634	case OpCode::Memory__fill: {
174	634	auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
175	634	auto Val = Builder.createTrunc(stackPop(), Context.Int8Ty);
176	634	auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
177	634	Builder.createCall(
178	634	Context.getIntrinsic(
179	634	Builder, Executable::Intrinsics::kMemFill,
180	634	LLVM::Type::getFunctionType(Context.VoidTy,
181	634	{Context.Int8PtrTy, Context.Int32Ty,
182	634	Context.Int64Ty, Context.Int8Ty,
183	634	Context.Int64Ty},
184	634	false)),
185	634	{Context.getModuleInst(Builder, ModCtx),
186	634	LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});
187	634	break;
188	231	}
189
190		// Const Numeric Instructions
191	535k	case OpCode::I32__const:
192	535k	stackPush(LLContext.getInt32(Instr.getNum().get<uint32_t>()));
193	535k	break;
194	70.4k	case OpCode::I64__const:
195	70.4k	stackPush(LLContext.getInt64(Instr.getNum().get<uint64_t>()));
196	70.4k	break;
197	14.3k	case OpCode::F32__const:
198	14.3k	stackPush(LLContext.getFloat(Instr.getNum().get<float>()));
199	14.3k	break;
200	6.69k	case OpCode::F64__const:
201	6.69k	stackPush(LLContext.getDouble(Instr.getNum().get<double>()));
202	6.69k	break;
203
204	0	default:
205	0	assumingUnreachable();
206	642k	}
207	642k	return {};
208	642k	}
209
210		// Memory32 cannot escape the reservation's guard region, so it keeps the
211		// guard-page fast path. Memory64 addresses are guest-controlled 64-bit values.
212		void FunctionCompiler::boundsCheckMemory64(unsigned MemoryIndex,
213		LLVM::Value Addr, uint64_t Offset,
214	20.8k	uint64_t AccessSize) noexcept {
215	20.8k	if (Context.MemoryAddrTypes[MemoryIndex].getIntegerBitWidth() != 64) {
216	20.8k	return;
217	20.8k	}
218	1	if (Offset > std::numeric_limits<uint64_t>::max() - AccessSize) {
219		// No address can satisfy the access.
220	0	Builder.createBr(getTrapBB(ErrCode::Value::MemoryOutOfBounds));
221	0	Builder.positionAtEnd(
222	0	LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok"));
223	0	return;
224	0	}
225		// Addr < usub_sat(SizeBytes, Offset + AccessSize - 1) is exactly
226		// Addr + Offset + AccessSize <= SizeBytes with every term kept in 64 bits.
227	1	const uint64_t OffsetAndSize = Offset + AccessSize;
228	1	auto SizeBytes =
229	1	Builder.createShl(Context.getMemorySize(Builder, ModCtx, MemoryIndex),
230	1	LLContext.getInt64(16));
231	1	auto Limit = Builder.createIntrinsic(
232	1	LLVM::Core::USubSat, {Context.Int64Ty},
233	1	{SizeBytes, LLContext.getInt64(OffsetAndSize - 1)});
234	1	auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok");
235	1	Builder.createCondBr(Builder.createLikely(Builder.createICmpULT(Addr, Limit)),
236	1	OkBB, getTrapBB(ErrCode::Value::MemoryOutOfBounds));
237	1	Builder.positionAtEnd(OkBB);
238	1	}
239
240		void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
241		unsigned Alignment,
242	17.6k	LLVM::Type LoadTy) noexcept {
243	17.6k	if constexpr (LLVM::kForceUnalignment) {
244	17.6k	Alignment = 0;
245	17.6k	}
246	17.6k	auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
247	17.6k	boundsCheckMemory64(MemoryIndex, Off, Offset,
248	17.6k	LoadTy.getPrimitiveSizeInBits() / 8);
249	17.6k	if (Offset != 0) {
250	11.2k	Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
251	11.2k	}
252
253	17.6k	auto VPtr = Builder.createInBoundsGEP1(
254	17.6k	Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
255	17.6k	auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
256	17.6k	auto LoadInst = Builder.createLoad(LoadTy, Ptr, true);
257	17.6k	LoadInst.setAlignment(1 << Alignment);
258	17.6k	stackPush(switchEndian(LoadInst));
259	17.6k	}
260
261		void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
262		unsigned Alignment, LLVM::Type LoadTy,
263		LLVM::Type ExtendTy,
264	7.71k	bool Signed) noexcept {
265	7.71k	compileLoadOp(MemoryIndex, Offset, Alignment, LoadTy);
266	7.71k	if (Signed) {
267	3.21k	Stack.back() = Builder.createSExt(Stack.back(), ExtendTy);
268	4.50k	} else {
269	4.50k	Stack.back() = Builder.createZExt(Stack.back(), ExtendTy);
270	4.50k	}
271	7.71k	}
272
273		void FunctionCompiler::compileStoreOp(uint32_t MemoryIndex, uint64_t Offset,
274		uint32_t Alignment, LLVM::Type LoadTy,
275	3.09k	bool Trunc, bool BitCast) noexcept {
276	3.09k	if constexpr (LLVM::kForceUnalignment) {
277	3.09k	Alignment = 0;
278	3.09k	}
279	3.09k	auto V = stackPop();
280	3.09k	auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
281	3.09k	boundsCheckMemory64(MemoryIndex, Off, Offset,
282	3.09k	LoadTy.getPrimitiveSizeInBits() / 8);
283	3.09k	if (Offset != 0) {
284	2.33k	Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
285	2.33k	}
286
287	3.09k	if (Trunc) {
288	663	V = Builder.createTrunc(V, LoadTy);
289	663	}
290	3.09k	if (BitCast) {
291	215	V = Builder.createBitCast(V, LoadTy);
292	215	}
293	3.09k	V = switchEndian(V);
294	3.09k	auto VPtr = Builder.createInBoundsGEP1(
295	3.09k	Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
296	3.09k	auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
297	3.09k	auto StoreInst = Builder.createStore(V, Ptr, true);
298	3.09k	StoreInst.setAlignment(1 << Alignment);
299	3.09k	}
300
301		} // namespace WasmEdge