/src/WasmEdge/lib/llvm/llvm.cpp

Source
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors

#include "llvm.h"

#include <llvm/IR/GlobalValue.h>
#include <llvm/Object/ObjectFile.h>
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#if LLVM_VERSION_MAJOR < 12 || WASMEDGE_OS_WINDOWS
#include <llvm/ExecutionEngine/Orc/Core.h>
#endif
#if LLVM_VERSION_MAJOR < 13
#include <llvm/ExecutionEngine/Orc/LLJIT.h>
#include <llvm/Support/CBindingWrapping.h>
#include <llvm/Support/Error.h>
#endif
#if LLVM_VERSION_MAJOR < 17
#include <llvm/IR/Instructions.h>
#include <llvm/Support/CBindingWrapping.h>
#endif

#if WASMEDGE_OS_WINDOWS
#include "common/spdlog.h"
#include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h>
#include <llvm/ExecutionEngine/SectionMemoryManager.h>
#include <llvm/Support/Process.h>
#include <system/winapi.h>
#endif

namespace llvm {
#if WASMEDGE_OS_WINDOWS
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ExecutionSession,
                                   LLVMOrcExecutionSessionRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ObjectLayer, LLVMOrcObjectLayerRef)
#endif
#if LLVM_VERSION_MAJOR < 12
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::LLJITBuilder, LLVMOrcLLJITBuilderRef)
#endif
#if LLVM_VERSION_MAJOR < 13
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule,
                                   LLVMOrcThreadSafeModuleRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::IRTransformLayer,
                                   LLVMOrcIRTransformLayerRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::MaterializationResponsibility,
                                   LLVMOrcMaterializationResponsibilityRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::LLJIT, LLVMOrcLLJITRef)
#endif
} // namespace llvm

#if LLVM_VERSION_MAJOR < 17
LLVMTailCallKind LLVMGetTailCallKind(LLVMValueRef Call) {
  return static_cast<LLVMTailCallKind>(
      llvm::unwrap<llvm::CallInst>(Call)->getTailCallKind());
}

void LLVMSetTailCallKind(LLVMValueRef Call, LLVMTailCallKind kind) {
  llvm::unwrap<llvm::CallInst>(Call)->setTailCallKind(
      static_cast<llvm::CallInst::TailCallKind>(kind));
}
#endif

namespace WasmEdge::LLVM {

void Value::setDSOLocal(bool Local) noexcept {
  llvm::cast<llvm::GlobalValue>(reinterpret_cast<llvm::Value *>(Ref))
      ->setDSOLocal(Local);
}

void Value::eliminateUnreachableBlocks() noexcept {
  llvm::EliminateUnreachableBlocks(
      *llvm::cast<llvm::Function>(reinterpret_cast<llvm::Value *>(Ref)));
}

bool SectionIterator::isText() const noexcept {
  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);
  return (*S)->isText();
}

bool SectionIterator::isData() const noexcept {
  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);
  return (*S)->isData();
}

bool SectionIterator::isBSS() const noexcept {
  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);
  return (*S)->isBSS();
}

bool SectionIterator::isPData() const noexcept {
#if WASMEDGE_OS_WINDOWS
  using namespace std::literals;
  return ".pdata"sv == getName();
#else
  return false;
#endif
}

bool SectionIterator::isEHFrame() const noexcept {
#if WASMEDGE_OS_LINUX
  using namespace std::literals;
  return ".eh_frame"sv == getName();
#elif WASMEDGE_OS_MACOS
  using namespace std::literals;
  return "__eh_frame"sv == getName();
#else
  return false;
#endif
}

bool SectionIterator::isVirtual() const noexcept {
  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);
  return (*S)->isVirtual();
}

#if WASMEDGE_OS_WINDOWS
namespace {
class DefaultMMapper final : public llvm::SectionMemoryManager::MemoryMapper {
public:
  llvm::sys::MemoryBlock allocateMappedMemory(
      llvm::SectionMemoryManager::AllocationPurpose /*Purpose*/,
      size_t NumBytes, const llvm::sys::MemoryBlock *const NearBlock,
      unsigned Flags, std::error_code &EC) override {
    return llvm::sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags,
                                                   EC);
  }
  std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &Block,
                                      unsigned Flags) override {
    return llvm::sys::Memory::protectMappedMemory(Block, Flags);
  }

  std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &M) override {
    return llvm::sys::Memory::releaseMappedMemory(M);
  }
};

class ContiguousSectionMemoryManager : public llvm::RTDyldMemoryManager {
public:
  explicit ContiguousSectionMemoryManager(
      llvm::SectionMemoryManager::MemoryMapper *UnownedMM = nullptr)
      : MMapper(UnownedMM), OwnedMMapper(nullptr) {
    if (!MMapper) {
      OwnedMMapper = std::make_unique<DefaultMMapper>();
      MMapper = OwnedMMapper.get();
    }
  }

  ~ContiguousSectionMemoryManager() noexcept override {
    using namespace std::literals;
    if (Preallocated.allocatedSize() != 0) {
      auto EC = MMapper->releaseMappedMemory(Preallocated);
      if (EC) {
        spdlog::error("releaseMappedMemory failed with error: {}"sv,
                      EC.message());
      }
    }
  }

  bool needsToReserveAllocationSpace() override { return true; }

  void reserveAllocationSpace(uintptr_t CodeSize, llvm::Align CodeAlign,
                              uintptr_t RODataSize, llvm::Align RODataAlign,
                              uintptr_t RWDataSize,
                              llvm::Align RWDataAlign) override {
    using namespace std::literals;
    assuming(Preallocated.allocatedSize() == 0);

    static const size_t PageSize = llvm::sys::Process::getPageSizeEstimate();
    assuming(CodeAlign.value() <= PageSize);
    assuming(RODataAlign.value() <= PageSize);
    assuming(RWDataAlign.value() <= PageSize);
    CodeSize = roundUpTo(CodeSize + CodeAlign.value(), PageSize);
    RODataSize = roundUpTo(RODataSize + RODataAlign.value(), PageSize);
    RWDataSize = roundUpTo(RWDataSize + RWDataAlign.value(), PageSize);
    const uintptr_t TotalSize =
        CodeSize + RODataSize + RWDataSize + PageSize * 3;

    std::error_code EC;
    Preallocated = MMapper->allocateMappedMemory(
        llvm::SectionMemoryManager::AllocationPurpose::Code, TotalSize, nullptr,
        llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE, EC);
    if (EC) {
      spdlog::error("allocateMappedMemory failed with error: {}"sv,
                    EC.message());
      return;
    }

    auto base = reinterpret_cast<std::uintptr_t>(Preallocated.base());
    CodeMem = CodeFree =
        llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), CodeSize);
    base += CodeSize;
    RODataMem = RODataFree =
        llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), RODataSize);
    base += RODataSize;
    RWDataMem = RWDataFree =
        llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), RWDataSize);
  }

  uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
                               unsigned /*SectionID*/,
                               llvm::StringRef /*SectionName*/,
                               bool IsReadOnly) override {
    if (IsReadOnly) {
      return Allocate(RODataFree, Size, Alignment);
    } else {
      return Allocate(RWDataFree, Size, Alignment);
    }
  }

  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
                               unsigned /*SectionID*/,
                               llvm::StringRef /*SectionName*/) override {
    return Allocate(CodeFree, Size, Alignment);
  }

  bool finalizeMemory(std::string *ErrMsg) override {
    std::error_code EC;

    EC = MMapper->protectMappedMemory(CodeMem, llvm::sys::Memory::MF_READ |
                                                   llvm::sys::Memory::MF_EXEC);
    if (EC) {
      if (ErrMsg) {
        *ErrMsg = EC.message();
      }
      return true;
    }
    EC = MMapper->protectMappedMemory(RODataMem, llvm::sys::Memory::MF_READ);
    if (EC) {
      if (ErrMsg) {
        *ErrMsg = EC.message();
      }
      return true;
    }

    llvm::sys::Memory::InvalidateInstructionCache(CodeMem.base(),
                                                  CodeMem.allocatedSize());
    return false;
  }

private:
  llvm::sys::MemoryBlock Preallocated;

  // Sections must be in the order code < rodata < rwdata.
  llvm::sys::MemoryBlock CodeMem;
  llvm::sys::MemoryBlock RODataMem;
  llvm::sys::MemoryBlock RWDataMem;

  llvm::sys::MemoryBlock CodeFree;
  llvm::sys::MemoryBlock RODataFree;
  llvm::sys::MemoryBlock RWDataFree;

  llvm::SectionMemoryManager::MemoryMapper *MMapper;
  std::unique_ptr<llvm::SectionMemoryManager::MemoryMapper> OwnedMMapper;

  uint8_t *Allocate(llvm::sys::MemoryBlock &FreeBlock, std::uintptr_t Size,
                    unsigned alignment) {
    using namespace std::literals;
    const auto Base = reinterpret_cast<uintptr_t>(FreeBlock.base());
    const auto Start = roundUpTo(Base, alignment);
    const uintptr_t PaddedSize = (Start - Base) + Size;
    if (PaddedSize > FreeBlock.allocatedSize()) {
      spdlog::error("Failed to satisfy suballocation request for {}"sv, Size);
      return nullptr;
    }
    FreeBlock =
        llvm::sys::MemoryBlock(reinterpret_cast<void *>(Base + PaddedSize),
                               FreeBlock.allocatedSize() - PaddedSize);
    return reinterpret_cast<uint8_t *>(Start);
  }

  static uintptr_t roundUpTo(uintptr_t Value, uintptr_t Divisor) noexcept {
    return ((Value + (Divisor - 1)) / Divisor) * Divisor;
  }
};

// Register stack unwind info for JIT functions
class Win64EHManager : public ContiguousSectionMemoryManager {
  using Base = ContiguousSectionMemoryManager;
  uint64_t CodeAddress = 0;

public:
  ~Win64EHManager() noexcept override {}

  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
                               unsigned SectionID,
                               llvm::StringRef SectionName) override {
    using namespace std::literals;
    const auto Allocated =
        Base::allocateCodeSection(Size, Alignment, SectionID, SectionName);
    if (SectionName == llvm::StringRef(".text"sv)) {
      CodeAddress = reinterpret_cast<uint64_t>(Allocated);
    }
    return Allocated;
  }

  void registerEHFrames(uint8_t *Addr, uint64_t /*LoadAddr*/,
                        size_t Size) noexcept override {
    using namespace std::literals;
    winapi::RUNTIME_FUNCTION_ *const FunctionTable =
        reinterpret_cast<winapi::RUNTIME_FUNCTION_ *>(Addr);
    const uint32_t EntryCount =
        static_cast<uint32_t>(Size / sizeof(winapi::RUNTIME_FUNCTION_));
    if (EntryCount == 0)
      return;
    // Calculate the object image base address by assuming that the address of
    // the first function is equal to the address of the code section.
    const auto ImageBase = CodeAddress - FunctionTable[0].BeginAddress;
    winapi::RtlAddFunctionTable(FunctionTable, EntryCount, ImageBase);
    EHFrames.push_back({Addr, Size});
  }
  void deregisterEHFrames() noexcept override {
    using namespace std::literals;
    for (auto &Frame : EHFrames) {
      winapi::RtlDeleteFunctionTable(
          reinterpret_cast<winapi::RUNTIME_FUNCTION_ *>(Frame.Addr));
    }
    EHFrames.clear();
  }
};
} // namespace

LLVMOrcLLJITBuilderRef OrcLLJIT::getBuilder() noexcept {
  using llvm::unwrap;
  using llvm::wrap;
  const LLVMOrcLLJITBuilderRef Builder = LLVMOrcCreateLLJITBuilder();
  LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
      Builder,
      [](void *, LLVMOrcExecutionSessionRef ES, const char *) noexcept {
        auto Layer = std::make_unique<llvm::orc::RTDyldObjectLinkingLayer>(
            *unwrap(ES), [](
#if LLVM_VERSION_MAJOR >= 21
                             const llvm::MemoryBuffer &
#endif
                         ) { return std::make_unique<Win64EHManager>(); });
        Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
        Layer->setAutoClaimResponsibilityForObjectSymbols(true);
        return wrap(static_cast<llvm::orc::ObjectLayer *>(Layer.release()));
      },
      nullptr);
  return Builder;
}
#else
LLVMOrcLLJITBuilderRef OrcLLJIT::getBuilder() noexcept { return nullptr; }
#endif

} // namespace WasmEdge::LLVM

#if LLVM_VERSION_MAJOR < 12 && WASMEDGE_OS_WINDOWS
void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
    LLVMOrcLLJITBuilderRef Builder,
    LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F,
    void *Ctx) noexcept {
  using llvm::unwrap;
  using llvm::wrap;
  unwrap(Builder)->setObjectLinkingLayerCreator(
      [=](llvm::orc::ExecutionSession &ES, const llvm::Triple &TT) {
        auto TTStr = TT.str();
        return std::unique_ptr<llvm::orc::ObjectLayer>(
            unwrap(F(Ctx, wrap(&ES), TTStr.c_str())));
      });
}
#endif
#if LLVM_VERSION_MAJOR < 13
LLVMOrcIRTransformLayerRef
LLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J) noexcept {
  using llvm::unwrap;
  using llvm::wrap;
  return wrap(&(unwrap(J)->getIRTransformLayer()));
}
void LLVMOrcIRTransformLayerSetTransform(
    LLVMOrcIRTransformLayerRef IRTransformLayer,
    LLVMOrcIRTransformLayerTransformFunction TransformFunction,
    void *Ctx) noexcept {
  using llvm::unwrap;
  using llvm::wrap;
  unwrap(IRTransformLayer)
      ->setTransform([=](llvm::orc::ThreadSafeModule TSM,
                         llvm::orc::MaterializationResponsibility &R)
                         -> llvm::Expected<llvm::orc::ThreadSafeModule> {
        LLVMOrcThreadSafeModuleRef TSMRef =
            wrap(new llvm::orc::ThreadSafeModule(std::move(TSM)));
        if (LLVMErrorRef Err = TransformFunction(Ctx, &TSMRef, wrap(&R))) {
          return unwrap(Err);
        }
        return std::move(*unwrap(TSMRef));
      });
}

LLVMErrorRef
LLVMOrcThreadSafeModuleWithModuleDo(LLVMOrcThreadSafeModuleRef TSM,
                                    LLVMOrcGenericIRModuleOperationFunction F,
                                    void *Ctx) noexcept {
  using llvm::unwrap;
  using llvm::wrap;
  return wrap(unwrap(TSM)->withModuleDo(
      [&](llvm::Module &M) { return unwrap(F(Ctx, wrap(&M))); }));
}
#endif

Coverage Report

Created: 2026-07-16 07:09

Line	Count	Source
1		// SPDX-License-Identifier: Apache-2.0
2		// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4		#include "llvm.h"
5
6		#include <llvm/IR/GlobalValue.h>
7		#include <llvm/Object/ObjectFile.h>
8		#include <llvm/Transforms/Utils/BasicBlockUtils.h>
9		#if LLVM_VERSION_MAJOR < 12 \|\| WASMEDGE_OS_WINDOWS
10		#include <llvm/ExecutionEngine/Orc/Core.h>
11		#endif
12		#if LLVM_VERSION_MAJOR < 13
13		#include <llvm/ExecutionEngine/Orc/LLJIT.h>
14		#include <llvm/Support/CBindingWrapping.h>
15		#include <llvm/Support/Error.h>
16		#endif
17		#if LLVM_VERSION_MAJOR < 17
18		#include <llvm/IR/Instructions.h>
19		#include <llvm/Support/CBindingWrapping.h>
20		#endif
21
22		#if WASMEDGE_OS_WINDOWS
23		#include "common/spdlog.h"
24		#include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h>
25		#include <llvm/ExecutionEngine/SectionMemoryManager.h>
26		#include <llvm/Support/Process.h>
27		#include <system/winapi.h>
28		#endif
29
30		namespace llvm {
31		#if WASMEDGE_OS_WINDOWS
32		DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ExecutionSession,
33		LLVMOrcExecutionSessionRef)
34		DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ObjectLayer, LLVMOrcObjectLayerRef)
35		#endif
36		#if LLVM_VERSION_MAJOR < 12
37		DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::LLJITBuilder, LLVMOrcLLJITBuilderRef)
38		#endif
39		#if LLVM_VERSION_MAJOR < 13
40		DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule,
41		LLVMOrcThreadSafeModuleRef)
42		DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::IRTransformLayer,
43		LLVMOrcIRTransformLayerRef)
44	0	DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::MaterializationResponsibility,
45	0	LLVMOrcMaterializationResponsibilityRef)
46	0	DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::LLJIT, LLVMOrcLLJITRef)
47	0	#endif
48	0	} // namespace llvm
49	0
50	0	#if LLVM_VERSION_MAJOR < 17
51	0	LLVMTailCallKind LLVMGetTailCallKind(LLVMValueRef Call) {
52	0	return static_cast<LLVMTailCallKind>(
53	0	llvm::unwrap<llvm::CallInst>(Call)->getTailCallKind());
54	0	}
55
56	233	void LLVMSetTailCallKind(LLVMValueRef Call, LLVMTailCallKind kind) {
57	233	llvm::unwrap<llvm::CallInst>(Call)->setTailCallKind(
58	233	static_cast<llvm::CallInst::TailCallKind>(kind));
59	233	}
60		#endif
61
62		namespace WasmEdge::LLVM {
63
64	22.1k	void Value::setDSOLocal(bool Local) noexcept {
65	22.1k	llvm::cast<llvm::GlobalValue>(reinterpret_cast<llvm::Value *>(Ref))
66	22.1k	->setDSOLocal(Local);
67	22.1k	}
68
69	10.5k	void Value::eliminateUnreachableBlocks() noexcept {
70	10.5k	llvm::EliminateUnreachableBlocks(
71	10.5k	llvm::cast<llvm::Function>(reinterpret_cast<llvm::Value >(Ref)));
72	10.5k	}
73
74	57.1k	bool SectionIterator::isText() const noexcept {
75	57.1k	auto S = reinterpret_cast<const llvm::object::section_iterator >(Ref);
76	57.1k	return (*S)->isText();
77	57.1k	}
78
79	53.0k	bool SectionIterator::isData() const noexcept {
80	53.0k	auto S = reinterpret_cast<const llvm::object::section_iterator >(Ref);
81	53.0k	return (*S)->isData();
82	53.0k	}
83
84	44.4k	bool SectionIterator::isBSS() const noexcept {
85	44.4k	auto S = reinterpret_cast<const llvm::object::section_iterator >(Ref);
86	44.4k	return (*S)->isBSS();
87	44.4k	}
88
89	57.1k	bool SectionIterator::isPData() const noexcept {
90		#if WASMEDGE_OS_WINDOWS
91		using namespace std::literals;
92		return ".pdata"sv == getName();
93		#else
94	57.1k	return false;
95	57.1k	#endif
96	57.1k	}
97
98	61.3k	bool SectionIterator::isEHFrame() const noexcept {
99	61.3k	#if WASMEDGE_OS_LINUX
100	61.3k	using namespace std::literals;
101	61.3k	return ".eh_frame"sv == getName();
102		#elif WASMEDGE_OS_MACOS
103		using namespace std::literals;
104		return "__eh_frame"sv == getName();
105		#else
106		return false;
107		#endif
108	61.3k	}
109
110	30.6k	bool SectionIterator::isVirtual() const noexcept {
111	30.6k	auto S = reinterpret_cast<const llvm::object::section_iterator >(Ref);
112	30.6k	return (*S)->isVirtual();
113	30.6k	}
114
115		#if WASMEDGE_OS_WINDOWS
116		namespace {
117		class DefaultMMapper final : public llvm::SectionMemoryManager::MemoryMapper {
118		public:
119		llvm::sys::MemoryBlock allocateMappedMemory(
120		llvm::SectionMemoryManager::AllocationPurpose /Purpose/,
121		size_t NumBytes, const llvm::sys::MemoryBlock *const NearBlock,
122		unsigned Flags, std::error_code &EC) override {
123		return llvm::sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags,
124		EC);
125		}
126		std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &Block,
127		unsigned Flags) override {
128		return llvm::sys::Memory::protectMappedMemory(Block, Flags);
129		}
130
131		std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &M) override {
132		return llvm::sys::Memory::releaseMappedMemory(M);
133		}
134		};
135
136		class ContiguousSectionMemoryManager : public llvm::RTDyldMemoryManager {
137		public:
138		explicit ContiguousSectionMemoryManager(
139		llvm::SectionMemoryManager::MemoryMapper *UnownedMM = nullptr)
140		: MMapper(UnownedMM), OwnedMMapper(nullptr) {
141		if (!MMapper) {
142		OwnedMMapper = std::make_unique<DefaultMMapper>();
143		MMapper = OwnedMMapper.get();
144		}
145		}
146
147		~ContiguousSectionMemoryManager() noexcept override {
148		using namespace std::literals;
149		if (Preallocated.allocatedSize() != 0) {
150		auto EC = MMapper->releaseMappedMemory(Preallocated);
151		if (EC) {
152		spdlog::error("releaseMappedMemory failed with error: {}"sv,
153		EC.message());
154		}
155		}
156		}
157
158		bool needsToReserveAllocationSpace() override { return true; }
159
160		void reserveAllocationSpace(uintptr_t CodeSize, llvm::Align CodeAlign,
161		uintptr_t RODataSize, llvm::Align RODataAlign,
162		uintptr_t RWDataSize,
163		llvm::Align RWDataAlign) override {
164		using namespace std::literals;
165		assuming(Preallocated.allocatedSize() == 0);
166
167		static const size_t PageSize = llvm::sys::Process::getPageSizeEstimate();
168		assuming(CodeAlign.value() <= PageSize);
169		assuming(RODataAlign.value() <= PageSize);
170		assuming(RWDataAlign.value() <= PageSize);
171		CodeSize = roundUpTo(CodeSize + CodeAlign.value(), PageSize);
172		RODataSize = roundUpTo(RODataSize + RODataAlign.value(), PageSize);
173		RWDataSize = roundUpTo(RWDataSize + RWDataAlign.value(), PageSize);
174		const uintptr_t TotalSize =
175		CodeSize + RODataSize + RWDataSize + PageSize * 3;
176
177		std::error_code EC;
178		Preallocated = MMapper->allocateMappedMemory(
179		llvm::SectionMemoryManager::AllocationPurpose::Code, TotalSize, nullptr,
180		llvm::sys::Memory::MF_READ \| llvm::sys::Memory::MF_WRITE, EC);
181		if (EC) {
182		spdlog::error("allocateMappedMemory failed with error: {}"sv,
183		EC.message());
184		return;
185		}
186
187		auto base = reinterpret_cast<std::uintptr_t>(Preallocated.base());
188		CodeMem = CodeFree =
189		llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), CodeSize);
190		base += CodeSize;
191		RODataMem = RODataFree =
192		llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), RODataSize);
193		base += RODataSize;
194		RWDataMem = RWDataFree =
195		llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), RWDataSize);
196		}
197
198		uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
199		unsigned /SectionID/,
200		llvm::StringRef /SectionName/,
201		bool IsReadOnly) override {
202		if (IsReadOnly) {
203		return Allocate(RODataFree, Size, Alignment);
204		} else {
205		return Allocate(RWDataFree, Size, Alignment);
206		}
207		}
208
209		uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
210		unsigned /SectionID/,
211		llvm::StringRef /SectionName/) override {
212		return Allocate(CodeFree, Size, Alignment);
213		}
214
215		bool finalizeMemory(std::string *ErrMsg) override {
216		std::error_code EC;
217
218		EC = MMapper->protectMappedMemory(CodeMem, llvm::sys::Memory::MF_READ \|
219		llvm::sys::Memory::MF_EXEC);
220		if (EC) {
221		if (ErrMsg) {
222		*ErrMsg = EC.message();
223		}
224		return true;
225		}
226		EC = MMapper->protectMappedMemory(RODataMem, llvm::sys::Memory::MF_READ);
227		if (EC) {
228		if (ErrMsg) {
229		*ErrMsg = EC.message();
230		}
231		return true;
232		}
233
234		llvm::sys::Memory::InvalidateInstructionCache(CodeMem.base(),
235		CodeMem.allocatedSize());
236		return false;
237		}
238
239		private:
240		llvm::sys::MemoryBlock Preallocated;
241
242		// Sections must be in the order code < rodata < rwdata.
243		llvm::sys::MemoryBlock CodeMem;
244		llvm::sys::MemoryBlock RODataMem;
245		llvm::sys::MemoryBlock RWDataMem;
246
247		llvm::sys::MemoryBlock CodeFree;
248		llvm::sys::MemoryBlock RODataFree;
249		llvm::sys::MemoryBlock RWDataFree;
250
251		llvm::SectionMemoryManager::MemoryMapper *MMapper;
252		std::unique_ptr<llvm::SectionMemoryManager::MemoryMapper> OwnedMMapper;
253
254		uint8_t *Allocate(llvm::sys::MemoryBlock &FreeBlock, std::uintptr_t Size,
255		unsigned alignment) {
256		using namespace std::literals;
257		const auto Base = reinterpret_cast<uintptr_t>(FreeBlock.base());
258		const auto Start = roundUpTo(Base, alignment);
259		const uintptr_t PaddedSize = (Start - Base) + Size;
260		if (PaddedSize > FreeBlock.allocatedSize()) {
261		spdlog::error("Failed to satisfy suballocation request for {}"sv, Size);
262		return nullptr;
263		}
264		FreeBlock =
265		llvm::sys::MemoryBlock(reinterpret_cast<void *>(Base + PaddedSize),
266		FreeBlock.allocatedSize() - PaddedSize);
267		return reinterpret_cast<uint8_t *>(Start);
268		}
269
270		static uintptr_t roundUpTo(uintptr_t Value, uintptr_t Divisor) noexcept {
271		return ((Value + (Divisor - 1)) / Divisor) * Divisor;
272		}
273		};
274
275		// Register stack unwind info for JIT functions
276		class Win64EHManager : public ContiguousSectionMemoryManager {
277		using Base = ContiguousSectionMemoryManager;
278		uint64_t CodeAddress = 0;
279
280		public:
281		~Win64EHManager() noexcept override {}
282
283		uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
284		unsigned SectionID,
285		llvm::StringRef SectionName) override {
286		using namespace std::literals;
287		const auto Allocated =
288		Base::allocateCodeSection(Size, Alignment, SectionID, SectionName);
289		if (SectionName == llvm::StringRef(".text"sv)) {
290		CodeAddress = reinterpret_cast<uint64_t>(Allocated);
291		}
292		return Allocated;
293		}
294
295		void registerEHFrames(uint8_t Addr, uint64_t /LoadAddr*/,
296		size_t Size) noexcept override {
297		using namespace std::literals;
298		winapi::RUNTIME_FUNCTION_ *const FunctionTable =
299		reinterpret_cast<winapi::RUNTIME_FUNCTION_ *>(Addr);
300		const uint32_t EntryCount =
301		static_cast<uint32_t>(Size / sizeof(winapi::RUNTIME_FUNCTION_));
302		if (EntryCount == 0)
303		return;
304		// Calculate the object image base address by assuming that the address of
305		// the first function is equal to the address of the code section.
306		const auto ImageBase = CodeAddress - FunctionTable[0].BeginAddress;
307		winapi::RtlAddFunctionTable(FunctionTable, EntryCount, ImageBase);
308		EHFrames.push_back({Addr, Size});
309		}
310		void deregisterEHFrames() noexcept override {
311		using namespace std::literals;
312		for (auto &Frame : EHFrames) {
313		winapi::RtlDeleteFunctionTable(
314		reinterpret_cast<winapi::RUNTIME_FUNCTION_ *>(Frame.Addr));
315		}
316		EHFrames.clear();
317		}
318		};
319		} // namespace
320
321		LLVMOrcLLJITBuilderRef OrcLLJIT::getBuilder() noexcept {
322		using llvm::unwrap;
323		using llvm::wrap;
324		const LLVMOrcLLJITBuilderRef Builder = LLVMOrcCreateLLJITBuilder();
325		LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
326		Builder,
327		[](void , LLVMOrcExecutionSessionRef ES, const char ) noexcept {
328		auto Layer = std::make_unique<llvm::orc::RTDyldObjectLinkingLayer>(
329		*unwrap(ES), [](
330		#if LLVM_VERSION_MAJOR >= 21
331		const llvm::MemoryBuffer &
332		#endif
333		) { return std::make_unique<Win64EHManager>(); });
334		Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
335		Layer->setAutoClaimResponsibilityForObjectSymbols(true);
336		return wrap(static_cast<llvm::orc::ObjectLayer *>(Layer.release()));
337		},
338		nullptr);
339		return Builder;
340		}
341		#else
342	0	LLVMOrcLLJITBuilderRef OrcLLJIT::getBuilder() noexcept { return nullptr; }
343		#endif
344
345		} // namespace WasmEdge::LLVM
346
347		#if LLVM_VERSION_MAJOR < 12 && WASMEDGE_OS_WINDOWS
348		void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(
349		LLVMOrcLLJITBuilderRef Builder,
350		LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F,
351		void *Ctx) noexcept {
352		using llvm::unwrap;
353		using llvm::wrap;
354		unwrap(Builder)->setObjectLinkingLayerCreator(
355		[=](llvm::orc::ExecutionSession &ES, const llvm::Triple &TT) {
356		auto TTStr = TT.str();
357		return std::unique_ptr<llvm::orc::ObjectLayer>(
358		unwrap(F(Ctx, wrap(&ES), TTStr.c_str())));
359		});
360		}
361		#endif
362		#if LLVM_VERSION_MAJOR < 13
363		LLVMOrcIRTransformLayerRef
364	0	LLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J) noexcept {
365	0	using llvm::unwrap;
366	0	using llvm::wrap;
367	0	return wrap(&(unwrap(J)->getIRTransformLayer()));
368	0	}
369		void LLVMOrcIRTransformLayerSetTransform(
370		LLVMOrcIRTransformLayerRef IRTransformLayer,
371		LLVMOrcIRTransformLayerTransformFunction TransformFunction,
372	0	void *Ctx) noexcept {
373	0	using llvm::unwrap;
374	0	using llvm::wrap;
375	0	unwrap(IRTransformLayer)
376	0	->setTransform([=](llvm::orc::ThreadSafeModule TSM,
377	0	llvm::orc::MaterializationResponsibility &R)
378	0	-> llvm::Expected<llvm::orc::ThreadSafeModule> {
379	0	LLVMOrcThreadSafeModuleRef TSMRef =
380	0	wrap(new llvm::orc::ThreadSafeModule(std::move(TSM)));
381	0	if (LLVMErrorRef Err = TransformFunction(Ctx, &TSMRef, wrap(&R))) {
382	0	return unwrap(Err);
383	0	}
384	0	return std::move(*unwrap(TSMRef));
385	0	});
386	0	}
387
388		LLVMErrorRef
389		LLVMOrcThreadSafeModuleWithModuleDo(LLVMOrcThreadSafeModuleRef TSM,
390		LLVMOrcGenericIRModuleOperationFunction F,
391	0	void *Ctx) noexcept {
392	0	using llvm::unwrap;
393	0	using llvm::wrap;
394	0	return wrap(unwrap(TSM)->withModuleDo(
395	0	[&](llvm::Module &M) { return unwrap(F(Ctx, wrap(&M))); }));
396	0	}
397		#endif