/src/llama.cpp/src/llama-mmap.cpp

Source
#include "llama-mmap.h"

#include "llama-impl.h"

#include "ggml.h"

#include <cstring>
#include <climits>
#include <stdexcept>
#include <cerrno>
#include <algorithm>

#ifdef __has_include
    #if __has_include(<unistd.h>)
        #include <unistd.h>
        #include <fcntl.h>
        #include <sys/stat.h>
        #if defined(_POSIX_MAPPED_FILES)
            #include <sys/mman.h>
        #endif
        #if defined(_POSIX_MEMLOCK_RANGE)
            #include <sys/resource.h>
        #endif
    #endif
#endif

#if defined(_WIN32)
    #define WIN32_LEAN_AND_MEAN
    #ifndef NOMINMAX
        #define NOMINMAX
    #endif
    #include <windows.h>
    #ifndef PATH_MAX
        #define PATH_MAX MAX_PATH
    #endif
    #include <io.h>
#endif

#if defined(__APPLE__)
#include <TargetConditionals.h>
#endif

// TODO: consider moving to llama-impl.h if needed in more places
#if defined(_WIN32)
static std::string llama_format_win_err(DWORD err) {
    LPSTR buf;
    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
    if (!size) {
        return "FormatMessageA failed";
    }
    std::string ret(buf, size);
    LocalFree(buf);
    return ret;
}
#endif

// llama_file

struct llama_file::impl {
#if defined(_WIN32)
    HANDLE fp_win32;
    std::string GetErrorMessageWin32(DWORD error_code) const {
        std::string ret;
        LPSTR lpMsgBuf = NULL;
        DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
                                    NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);
        if (!bufLen) {
            ret = format("Win32 error code: %lx", error_code);
        } else {
            ret = lpMsgBuf;
            LocalFree(lpMsgBuf);
        }

        return ret;
    }

    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
        fp = ggml_fopen(fname, mode);
        if (fp == NULL) {
            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
        }
        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
        seek(0, SEEK_END);
        size = tell();
        seek(0, SEEK_SET);
    }

    size_t tell() const {
        LARGE_INTEGER li;
        li.QuadPart = 0;
        BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
        if (!ret) {
            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
        }

        return li.QuadPart;
    }

    void seek(size_t offset, int whence) const {
        static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");
        static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");
        static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");

        LARGE_INTEGER li;
        li.QuadPart = offset;
        BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
        if (!ret) {
            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
        }
    }

    void read_raw(void * ptr, size_t len) const {
        size_t bytes_read = 0;
        while (bytes_read < len) {
            size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);
            DWORD chunk_read = 0;
            BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
            if (!result) {
                throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
            }
            if (chunk_read < chunk_size || chunk_read == 0) {
                throw std::runtime_error("unexpectedly reached end of file");
            }

            bytes_read += chunk_read;
        }
    }

    uint32_t read_u32() const {
        uint32_t val;
        read_raw(&val, sizeof(val));
        return val;
    }

    void write_raw(const void * ptr, size_t len) const {
        size_t bytes_written = 0;
        while (bytes_written < len) {
            size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);
            DWORD chunk_written = 0;
            BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
            if (!result) {
                throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
            }
            if (chunk_written < chunk_size || chunk_written == 0) {
                throw std::runtime_error("unexpectedly failed to write bytes");
            }

            bytes_written += chunk_written;
        }
    }

    void write_u32(uint32_t val) const {
        write_raw(&val, sizeof(val));
    }

    void read_aligned_chunk(size_t offset, void * dest, size_t size) const {
        throw std::runtime_error("DirectIO is not implemented on Windows.");
    }

    ~impl() {
        if (fp) {
            std::fclose(fp);
        }
    }
#else
    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
#ifdef __linux__
        // Try unbuffered I/O for read only
        if (use_direct_io && std::strcmp(mode, "rb") == 0) {
            fd = open(fname, O_RDONLY | O_DIRECT);

            if (fd != -1) {
                struct stat file_stats{};
                fstat(fd, &file_stats);

                size = file_stats.st_size;
                alignment = file_stats.st_blksize;

                off_t ret = lseek(fd, 0, SEEK_SET);
                if (ret == -1) {
                    throw std::runtime_error(format("seek error: %s", strerror(errno)));
                }
                return;
            }

            LLAMA_LOG_WARN("Failed to open model %s with error: %s. Falling back to buffered I/O",
                fname, strerror(errno));
        }
#endif
        fp = ggml_fopen(fname, mode);
        if (fp == NULL) {
            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
        }
        seek(0, SEEK_END);
        size = tell();
        seek(0, SEEK_SET);
    }

    size_t tell() const {
        if (fd == -1) {
            long ret = std::ftell(fp);
            if (ret == -1) {
                throw std::runtime_error(format("ftell error: %s", strerror(errno)));
            }

            return (size_t) ret;
        }

        off_t pos = lseek(fd, 0, SEEK_CUR);
        if (pos == -1) {
            throw std::runtime_error(format("lseek error: %s", strerror(errno)));
        }
        return (size_t) pos;
    }

    void seek(size_t offset, int whence) const {
        off_t ret = 0;
        if (fd == -1) {
            ret = std::fseek(fp, (long) offset, whence);
        } else {
            ret = lseek(fd, offset, whence);
        }
        if (ret == -1) {
            throw std::runtime_error(format("seek error: %s", strerror(errno)));
        }
    }

    void read_raw(void * ptr, size_t len) const {
        if (len == 0) {
            return;
        }
        errno = 0;
        if (fd == -1) {
            std::size_t ret = std::fread(ptr, len, 1, fp);
            if (ferror(fp)) {
                throw std::runtime_error(format("read error: %s", strerror(errno)));
            }
            if (ret != 1) {
                throw std::runtime_error("unexpectedly reached end of file");
            }
        } else {
            bool successful = false;
            while (!successful) {
                off_t ret = read(fd, ptr, len);

                if (ret == -1) {
                    if (errno == EINTR) {
                        continue;  // Interrupted by signal, retry
                    }
                    throw std::runtime_error(format("read error: %s", strerror(errno)));
                }
                if (ret == 0) {
                    throw std::runtime_error("unexpectedly reached end of file");
                }

                successful = true;
            }
        }
    }

    void read_aligned_chunk(size_t offset, void * dest, size_t size) const {
        off_t aligned_offset = offset & ~(alignment - 1);
        off_t offset_from_alignment = offset - aligned_offset;
        size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);

        void * raw_buffer = nullptr;
        int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);
        if (ret != 0) {
            throw std::runtime_error(format("posix_memalign failed with error %d", ret));
        }

        struct aligned_buffer_deleter {
            void operator()(void * p) const { free(p); }
        };
        std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);

        seek(aligned_offset, SEEK_SET);
        read_raw(buffer.get(), bytes_to_read);

        uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;
        memcpy(dest, reinterpret_cast<void *>(actual_data), size);
    }

    uint32_t read_u32() const {
        uint32_t ret;
        read_raw(&ret, sizeof(ret));
        return ret;
    }

    void write_raw(const void * ptr, size_t len) const {
        if (len == 0) {
            return;
        }
        errno = 0;
        size_t ret = std::fwrite(ptr, len, 1, fp);
        if (ret != 1) {
            throw std::runtime_error(format("write error: %s", strerror(errno)));
        }
    }

    void write_u32(uint32_t val) const {
        write_raw(&val, sizeof(val));
    }

    ~impl() {
        if (fd != -1) {
            close(fd);
        } else {
            std::fclose(fp);
        }
    }
    int fd = -1;
#endif

    void read_raw_at(void * ptr, size_t len, size_t offset) const {
        if (alignment != 1) {
            read_aligned_chunk(offset, ptr, len);
        } else {
            seek(offset, SEEK_SET);
            read_raw(ptr, len);
        }
    }

    size_t read_alignment() const {
        return alignment;
    }

    size_t alignment = 1;

    FILE * fp{};
    size_t size{};
};

llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :
    pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}
llama_file::~llama_file() = default;

size_t llama_file::tell() const { return pimpl->tell(); }
size_t llama_file::size() const { return pimpl->size; }

size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }

int llama_file::file_id() const {
#ifdef _WIN32
    return _fileno(pimpl->fp);
#else
#if defined(fileno)
    return fileno(pimpl->fp);
#else
    return ::fileno(pimpl->fp);
#endif
#endif
}

void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }
void llama_file::read_raw(void * ptr, size_t len) const { pimpl->read_raw(ptr, len); }
void llama_file::read_raw_at(void * ptr, size_t len, size_t offset) const { pimpl->read_raw_at(ptr, len, offset); }

uint32_t llama_file::read_u32() const { return pimpl->read_u32(); }

void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }

// llama_mmap

struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
    std::vector<std::pair<size_t, size_t>> mapped_fragments;

    impl(struct llama_file * file, size_t prefetch, bool numa) {
        size = file->size();
        int fd = file->file_id();
        int flags = MAP_SHARED;
        if (numa) { prefetch = 0; }
#ifdef __linux__
        if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
            LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
                    strerror(errno));
        }
        if (prefetch) { flags |= MAP_POPULATE; }
#endif
        addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);
        if (addr == MAP_FAILED) {
            throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
        }

        if (prefetch > 0) {
            if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {
                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",
                        strerror(errno));
            }
        }
        if (numa) {
            if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {
                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",
                        strerror(errno));
            }
        }

        mapped_fragments.emplace_back(0, file->size());
    }

    static void align_range(size_t * first, size_t * last, size_t page_size) {
        size_t offset_in_page = *first & (page_size - 1);
        size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
        *first += offset_to_page;

        *last = *last & ~(page_size - 1);

        if (*last <= *first) {
            *last = *first;
        }
    }

    void unmap_fragment(size_t first, size_t last) {
        int page_size = sysconf(_SC_PAGESIZE);
        align_range(&first, &last, page_size);
        size_t len = last - first;

        if (len == 0) {
            return;
        }

        GGML_ASSERT(first % page_size == 0);
        GGML_ASSERT(last % page_size == 0);
        GGML_ASSERT(last > first);

        void * next_page_start = (uint8_t *) addr + first;

        if (munmap(next_page_start, len)) {
            LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
        }

        std::vector<std::pair<size_t, size_t>> new_mapped_fragments;
        for (const auto & frag : mapped_fragments) {
            if (frag.first < first && frag.second > last) {
                new_mapped_fragments.emplace_back(frag.first, first);
                new_mapped_fragments.emplace_back(last, frag.second);
            } else if (frag.first < first && frag.second > first) {
                new_mapped_fragments.emplace_back(frag.first, first);
            } else if (frag.first < last && frag.second > last) {
                new_mapped_fragments.emplace_back(last, frag.second);
            } else if (frag.first >= first && frag.second <= last) {
            } else {
                new_mapped_fragments.push_back(frag);
            }
        }
        mapped_fragments = std::move(new_mapped_fragments);
    }

    ~impl() {
        for (const auto & frag : mapped_fragments) {
            if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
                LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
            }
        }
    }
#elif defined(_WIN32)
    impl(struct llama_file * file, size_t prefetch, bool numa) {
        GGML_UNUSED(numa);

        size = file->size();

        HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());

        HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);

        if (hMapping == NULL) {
            DWORD error = GetLastError();
            throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
        }

        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
        DWORD error = GetLastError();
        CloseHandle(hMapping);

        if (addr == NULL) {
            throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
        }

        if (prefetch > 0) {
#if _WIN32_WINNT >= 0x602
            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
            HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");

            pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");

            if (pPrefetchVirtualMemory) {
                WIN32_MEMORY_RANGE_ENTRY range;
                range.VirtualAddress = addr;
                range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);
                if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
                    LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
                            llama_format_win_err(GetLastError()).c_str());
                }
            }
#else
            LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");
#endif
        }
    }

    void unmap_fragment(size_t first, size_t last) {
        GGML_UNUSED(first);
        GGML_UNUSED(last);
    }

    ~impl() {
        if (!UnmapViewOfFile(addr)) {
            LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",
                    llama_format_win_err(GetLastError()).c_str());
        }
    }
#else
    impl(struct llama_file * file, size_t prefetch, bool numa) {
        GGML_UNUSED(file);
        GGML_UNUSED(prefetch);
        GGML_UNUSED(numa);

        throw std::runtime_error("mmap not supported");
    }

    void unmap_fragment(size_t first, size_t last) {
        GGML_UNUSED(first);
        GGML_UNUSED(last);

        throw std::runtime_error("mmap not supported");
    }
#endif

    void * addr;
    size_t size;
};

llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
llama_mmap::~llama_mmap() = default;

size_t llama_mmap::size() const { return pimpl->size; }
void * llama_mmap::addr() const { return pimpl->addr; }

void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }

#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
const bool llama_mmap::SUPPORTED  = true;
#else
const bool llama_mmap::SUPPORTED  = false;
#endif

// llama_mlock

struct llama_mlock::impl {
#ifdef _POSIX_MEMLOCK_RANGE
    static size_t lock_granularity() {
        return (size_t) sysconf(_SC_PAGESIZE);
    }

    bool raw_lock(const void * addr, size_t size) const {
        if (!mlock(addr, size)) {
            return true;
        }

#ifdef __APPLE__
#define MLOCK_SUGGESTION \
        "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
        "decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"
#else
#define MLOCK_SUGGESTION \
        "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"
#endif

        char* errmsg = std::strerror(errno);
        bool suggest = (errno == ENOMEM);
#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX)
        // visionOS/tvOS dont't support RLIMIT_MEMLOCK
        // Skip resource limit checks on visionOS/tvOS
        suggest = false;
#else
        struct rlimit lock_limit;
        if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {
            suggest = false;
        }
        if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {
            suggest = false;
        }
#endif

        LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
                size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
        return false;
    }

    static void raw_unlock(void * addr, size_t size) {
        if (munlock(addr, size)) {
            LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));
        }
    }
#elif defined(_WIN32)
    static size_t lock_granularity() {
        SYSTEM_INFO si;
        GetSystemInfo(&si);
        return (size_t) si.dwPageSize;
    }

    bool raw_lock(void * ptr, size_t len) const {
        for (int tries = 1; ; tries++) {
            if (VirtualLock(ptr, len)) {
                return true;
            }
            if (tries == 2) {
                LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
                    len, size, llama_format_win_err(GetLastError()).c_str());
                return false;
            }

            SIZE_T min_ws_size, max_ws_size;
            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
                LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",
                        llama_format_win_err(GetLastError()).c_str());
                return false;
            }
            size_t increment = len + 1048576;
            min_ws_size += increment;
            max_ws_size += increment;
            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
                LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",
                        llama_format_win_err(GetLastError()).c_str());
                return false;
            }
        }
    }

    static void raw_unlock(void * ptr, size_t len) {
        if (!VirtualUnlock(ptr, len)) {
            LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",
                    llama_format_win_err(GetLastError()).c_str());
        }
    }
#else
    static size_t lock_granularity() {
        return (size_t) 65536;
    }

    bool raw_lock(const void * addr, size_t len) const {
        LLAMA_LOG_WARN("warning: mlock not supported on this system\n");
        return false;
    }

    static void raw_unlock(const void * addr, size_t len) {}
#endif

    impl() : addr(NULL), size(0), failed_already(false) {}

    void init(void * ptr) {
        GGML_ASSERT(addr == NULL && size == 0);
        addr = ptr;
    }

    void grow_to(size_t target_size) {
        GGML_ASSERT(addr);
        if (failed_already) {
            return;
        }
        size_t granularity = lock_granularity();
        target_size = (target_size + granularity - 1) & ~(granularity - 1);
        if (target_size > size) {
            if (raw_lock((uint8_t *) addr + size, target_size - size)) {
                size = target_size;
            } else {
                failed_already = true;
            }
        }
    }

    void * addr;
    size_t size;

    bool failed_already;
};

llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}
llama_mlock::~llama_mlock() = default;

void llama_mlock::init(void * ptr) { pimpl->init(ptr); }
void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }

#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
const bool llama_mlock::SUPPORTED = true;
#else
const bool llama_mlock::SUPPORTED = false;
#endif

size_t llama_path_max() {
    return PATH_MAX;
}

Coverage Report

Created: 2025-12-28 06:25

Line	Count	Source
1		#include "llama-mmap.h"
2
3		#include "llama-impl.h"
4
5		#include "ggml.h"
6
7		#include <cstring>
8		#include <climits>
9		#include <stdexcept>
10		#include <cerrno>
11		#include <algorithm>
12
13		#ifdef __has_include
14		#if __has_include(<unistd.h>)
15		#include <unistd.h>
16		#include <fcntl.h>
17		#include <sys/stat.h>
18		#if defined(_POSIX_MAPPED_FILES)
19		#include <sys/mman.h>
20		#endif
21		#if defined(_POSIX_MEMLOCK_RANGE)
22		#include <sys/resource.h>
23		#endif
24		#endif
25		#endif
26
27		#if defined(_WIN32)
28		#define WIN32_LEAN_AND_MEAN
29		#ifndef NOMINMAX
30		#define NOMINMAX
31		#endif
32		#include <windows.h>
33		#ifndef PATH_MAX
34		#define PATH_MAX MAX_PATH
35		#endif
36		#include <io.h>
37		#endif
38
39		#if defined(__APPLE__)
40		#include <TargetConditionals.h>
41		#endif
42
43		// TODO: consider moving to llama-impl.h if needed in more places
44		#if defined(_WIN32)
45		static std::string llama_format_win_err(DWORD err) {
46		LPSTR buf;
47		size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER \| FORMAT_MESSAGE_FROM_SYSTEM \| FORMAT_MESSAGE_IGNORE_INSERTS,
48		NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
49		if (!size) {
50		return "FormatMessageA failed";
51		}
52		std::string ret(buf, size);
53		LocalFree(buf);
54		return ret;
55		}
56		#endif
57
58		// llama_file
59
60		struct llama_file::impl {
61		#if defined(_WIN32)
62		HANDLE fp_win32;
63		std::string GetErrorMessageWin32(DWORD error_code) const {
64		std::string ret;
65		LPSTR lpMsgBuf = NULL;
66		DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER \| FORMAT_MESSAGE_FROM_SYSTEM \| FORMAT_MESSAGE_IGNORE_INSERTS,
67		NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);
68		if (!bufLen) {
69		ret = format("Win32 error code: %lx", error_code);
70		} else {
71		ret = lpMsgBuf;
72		LocalFree(lpMsgBuf);
73		}
74
75		return ret;
76		}
77
78		impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
79		fp = ggml_fopen(fname, mode);
80		if (fp == NULL) {
81		throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
82		}
83		fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
84		seek(0, SEEK_END);
85		size = tell();
86		seek(0, SEEK_SET);
87		}
88
89		size_t tell() const {
90		LARGE_INTEGER li;
91		li.QuadPart = 0;
92		BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
93		if (!ret) {
94		throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
95		}
96
97		return li.QuadPart;
98		}
99
100		void seek(size_t offset, int whence) const {
101		static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");
102		static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");
103		static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");
104
105		LARGE_INTEGER li;
106		li.QuadPart = offset;
107		BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
108		if (!ret) {
109		throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
110		}
111		}
112
113		void read_raw(void * ptr, size_t len) const {
114		size_t bytes_read = 0;
115		while (bytes_read < len) {
116		size_t chunk_size = std::min<size_t>(len - bytes_read, 6410241024);
117		DWORD chunk_read = 0;
118		BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
119		if (!result) {
120		throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
121		}
122		if (chunk_read < chunk_size \|\| chunk_read == 0) {
123		throw std::runtime_error("unexpectedly reached end of file");
124		}
125
126		bytes_read += chunk_read;
127		}
128		}
129
130		uint32_t read_u32() const {
131		uint32_t val;
132		read_raw(&val, sizeof(val));
133		return val;
134		}
135
136		void write_raw(const void * ptr, size_t len) const {
137		size_t bytes_written = 0;
138		while (bytes_written < len) {
139		size_t chunk_size = std::min<size_t>(len - bytes_written, 6410241024);
140		DWORD chunk_written = 0;
141		BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
142		if (!result) {
143		throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
144		}
145		if (chunk_written < chunk_size \|\| chunk_written == 0) {
146		throw std::runtime_error("unexpectedly failed to write bytes");
147		}
148
149		bytes_written += chunk_written;
150		}
151		}
152
153		void write_u32(uint32_t val) const {
154		write_raw(&val, sizeof(val));
155		}
156
157		void read_aligned_chunk(size_t offset, void * dest, size_t size) const {
158		throw std::runtime_error("DirectIO is not implemented on Windows.");
159		}
160
161		~impl() {
162		if (fp) {
163		std::fclose(fp);
164		}
165		}
166		#else
167	0	impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
168	0	#ifdef __linux__
169		// Try unbuffered I/O for read only
170	0	if (use_direct_io && std::strcmp(mode, "rb") == 0) {
171	0	fd = open(fname, O_RDONLY \| O_DIRECT);
172
173	0	if (fd != -1) {
174	0	struct stat file_stats{};
175	0	fstat(fd, &file_stats);
176
177	0	size = file_stats.st_size;
178	0	alignment = file_stats.st_blksize;
179
180	0	off_t ret = lseek(fd, 0, SEEK_SET);
181	0	if (ret == -1) {
182	0	throw std::runtime_error(format("seek error: %s", strerror(errno)));
183	0	}
184	0	return;
185	0	}
186
187	0	LLAMA_LOG_WARN("Failed to open model %s with error: %s. Falling back to buffered I/O",
188	0	fname, strerror(errno));
189	0	}
190	0	#endif
191	0	fp = ggml_fopen(fname, mode);
192	0	if (fp == NULL) {
193	0	throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
194	0	}
195	0	seek(0, SEEK_END);
196	0	size = tell();
197	0	seek(0, SEEK_SET);
198	0	}
199
200	0	size_t tell() const {
201	0	if (fd == -1) {
202	0	long ret = std::ftell(fp);
203	0	if (ret == -1) {
204	0	throw std::runtime_error(format("ftell error: %s", strerror(errno)));
205	0	}
206
207	0	return (size_t) ret;
208	0	}
209
210	0	off_t pos = lseek(fd, 0, SEEK_CUR);
211	0	if (pos == -1) {
212	0	throw std::runtime_error(format("lseek error: %s", strerror(errno)));
213	0	}
214	0	return (size_t) pos;
215	0	}
216
217	0	void seek(size_t offset, int whence) const {
218	0	off_t ret = 0;
219	0	if (fd == -1) {
220	0	ret = std::fseek(fp, (long) offset, whence);
221	0	} else {
222	0	ret = lseek(fd, offset, whence);
223	0	}
224	0	if (ret == -1) {
225	0	throw std::runtime_error(format("seek error: %s", strerror(errno)));
226	0	}
227	0	}
228
229	0	void read_raw(void * ptr, size_t len) const {
230	0	if (len == 0) {
231	0	return;
232	0	}
233	0	errno = 0;
234	0	if (fd == -1) {
235	0	std::size_t ret = std::fread(ptr, len, 1, fp);
236	0	if (ferror(fp)) {
237	0	throw std::runtime_error(format("read error: %s", strerror(errno)));
238	0	}
239	0	if (ret != 1) {
240	0	throw std::runtime_error("unexpectedly reached end of file");
241	0	}
242	0	} else {
243	0	bool successful = false;
244	0	while (!successful) {
245	0	off_t ret = read(fd, ptr, len);
246
247	0	if (ret == -1) {
248	0	if (errno == EINTR) {
249	0	continue; // Interrupted by signal, retry
250	0	}
251	0	throw std::runtime_error(format("read error: %s", strerror(errno)));
252	0	}
253	0	if (ret == 0) {
254	0	throw std::runtime_error("unexpectedly reached end of file");
255	0	}
256
257	0	successful = true;
258	0	}
259	0	}
260	0	}
261
262	0	void read_aligned_chunk(size_t offset, void * dest, size_t size) const {
263	0	off_t aligned_offset = offset & ~(alignment - 1);
264	0	off_t offset_from_alignment = offset - aligned_offset;
265	0	size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);
266
267	0	void * raw_buffer = nullptr;
268	0	int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);
269	0	if (ret != 0) {
270	0	throw std::runtime_error(format("posix_memalign failed with error %d", ret));
271	0	}
272
273	0	struct aligned_buffer_deleter {
274	0	void operator()(void * p) const { free(p); }
275	0	};
276	0	std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);
277
278	0	seek(aligned_offset, SEEK_SET);
279	0	read_raw(buffer.get(), bytes_to_read);
280
281	0	uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;
282	0	memcpy(dest, reinterpret_cast<void *>(actual_data), size);
283	0	}
284
285	0	uint32_t read_u32() const {
286	0	uint32_t ret;
287	0	read_raw(&ret, sizeof(ret));
288	0	return ret;
289	0	}
290
291	0	void write_raw(const void * ptr, size_t len) const {
292	0	if (len == 0) {
293	0	return;
294	0	}
295	0	errno = 0;
296	0	size_t ret = std::fwrite(ptr, len, 1, fp);
297	0	if (ret != 1) {
298	0	throw std::runtime_error(format("write error: %s", strerror(errno)));
299	0	}
300	0	}
301
302	0	void write_u32(uint32_t val) const {
303	0	write_raw(&val, sizeof(val));
304	0	}
305
306	0	~impl() {
307	0	if (fd != -1) {
308	0	close(fd);
309	0	} else {
310	0	std::fclose(fp);
311	0	}
312	0	}
313		int fd = -1;
314		#endif
315
316	0	void read_raw_at(void * ptr, size_t len, size_t offset) const {
317	0	if (alignment != 1) {
318	0	read_aligned_chunk(offset, ptr, len);
319	0	} else {
320	0	seek(offset, SEEK_SET);
321	0	read_raw(ptr, len);
322	0	}
323	0	}
324
325	0	size_t read_alignment() const {
326	0	return alignment;
327	0	}
328
329		size_t alignment = 1;
330
331		FILE * fp{};
332		size_t size{};
333		};
334
335		llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :
336	0	pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}
337	0	llama_file::~llama_file() = default;
338
339	0	size_t llama_file::tell() const { return pimpl->tell(); }
340	0	size_t llama_file::size() const { return pimpl->size; }
341
342	0	size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
343
344	0	int llama_file::file_id() const {
345		#ifdef _WIN32
346		return _fileno(pimpl->fp);
347		#else
348		#if defined(fileno)
349		return fileno(pimpl->fp);
350		#else
351	0	return ::fileno(pimpl->fp);
352	0	#endif
353	0	#endif
354	0	}
355
356	0	void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }
357	0	void llama_file::read_raw(void * ptr, size_t len) const { pimpl->read_raw(ptr, len); }
358	0	void llama_file::read_raw_at(void * ptr, size_t len, size_t offset) const { pimpl->read_raw_at(ptr, len, offset); }
359
360	0	uint32_t llama_file::read_u32() const { return pimpl->read_u32(); }
361
362	0	void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
363	0	void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
364
365		// llama_mmap
366
367		struct llama_mmap::impl {
368		#ifdef _POSIX_MAPPED_FILES
369		std::vector<std::pair<size_t, size_t>> mapped_fragments;
370
371	0	impl(struct llama_file * file, size_t prefetch, bool numa) {
372	0	size = file->size();
373	0	int fd = file->file_id();
374	0	int flags = MAP_SHARED;
375	0	if (numa) { prefetch = 0; }
376	0	#ifdef __linux__
377	0	if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
378	0	LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
379	0	strerror(errno));
380	0	}
381	0	if (prefetch) { flags \|= MAP_POPULATE; }
382	0	#endif
383	0	addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);
384	0	if (addr == MAP_FAILED) {
385	0	throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
386	0	}
387
388	0	if (prefetch > 0) {
389	0	if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {
390	0	LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",
391	0	strerror(errno));
392	0	}
393	0	}
394	0	if (numa) {
395	0	if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {
396	0	LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",
397	0	strerror(errno));
398	0	}
399	0	}
400
401	0	mapped_fragments.emplace_back(0, file->size());
402	0	}
403
404	0	static void align_range(size_t * first, size_t * last, size_t page_size) {
405	0	size_t offset_in_page = *first & (page_size - 1);
406	0	size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
407	0	*first += offset_to_page;
408
409	0	last = last & ~(page_size - 1);
410
411	0	if (last <= first) {
412	0	last = first;
413	0	}
414	0	}
415
416	0	void unmap_fragment(size_t first, size_t last) {
417	0	int page_size = sysconf(_SC_PAGESIZE);
418	0	align_range(&first, &last, page_size);
419	0	size_t len = last - first;
420
421	0	if (len == 0) {
422	0	return;
423	0	}
424
425	0	GGML_ASSERT(first % page_size == 0);
426	0	GGML_ASSERT(last % page_size == 0);
427	0	GGML_ASSERT(last > first);
428
429	0	void * next_page_start = (uint8_t *) addr + first;
430
431	0	if (munmap(next_page_start, len)) {
432	0	LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
433	0	}
434
435	0	std::vector<std::pair<size_t, size_t>> new_mapped_fragments;
436	0	for (const auto & frag : mapped_fragments) {
437	0	if (frag.first < first && frag.second > last) {
438	0	new_mapped_fragments.emplace_back(frag.first, first);
439	0	new_mapped_fragments.emplace_back(last, frag.second);
440	0	} else if (frag.first < first && frag.second > first) {
441	0	new_mapped_fragments.emplace_back(frag.first, first);
442	0	} else if (frag.first < last && frag.second > last) {
443	0	new_mapped_fragments.emplace_back(last, frag.second);
444	0	} else if (frag.first >= first && frag.second <= last) {
445	0	} else {
446	0	new_mapped_fragments.push_back(frag);
447	0	}
448	0	}
449	0	mapped_fragments = std::move(new_mapped_fragments);
450	0	}
451
452	0	~impl() {
453	0	for (const auto & frag : mapped_fragments) {
454	0	if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
455	0	LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
456	0	}
457	0	}
458	0	}
459		#elif defined(_WIN32)
460		impl(struct llama_file * file, size_t prefetch, bool numa) {
461		GGML_UNUSED(numa);
462
463		size = file->size();
464
465		HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());
466
467		HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
468
469		if (hMapping == NULL) {
470		DWORD error = GetLastError();
471		throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
472		}
473
474		addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
475		DWORD error = GetLastError();
476		CloseHandle(hMapping);
477
478		if (addr == NULL) {
479		throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
480		}
481
482		if (prefetch > 0) {
483		#if _WIN32_WINNT >= 0x602
484		BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
485		HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
486
487		pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
488
489		if (pPrefetchVirtualMemory) {
490		WIN32_MEMORY_RANGE_ENTRY range;
491		range.VirtualAddress = addr;
492		range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);
493		if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
494		LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
495		llama_format_win_err(GetLastError()).c_str());
496		}
497		}
498		#else
499		LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");
500		#endif
501		}
502		}
503
504		void unmap_fragment(size_t first, size_t last) {
505		GGML_UNUSED(first);
506		GGML_UNUSED(last);
507		}
508
509		~impl() {
510		if (!UnmapViewOfFile(addr)) {
511		LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",
512		llama_format_win_err(GetLastError()).c_str());
513		}
514		}
515		#else
516		impl(struct llama_file * file, size_t prefetch, bool numa) {
517		GGML_UNUSED(file);
518		GGML_UNUSED(prefetch);
519		GGML_UNUSED(numa);
520
521		throw std::runtime_error("mmap not supported");
522		}
523
524		void unmap_fragment(size_t first, size_t last) {
525		GGML_UNUSED(first);
526		GGML_UNUSED(last);
527
528		throw std::runtime_error("mmap not supported");
529		}
530		#endif
531
532		void * addr;
533		size_t size;
534		};
535
536	0	llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
537	0	llama_mmap::~llama_mmap() = default;
538
539	0	size_t llama_mmap::size() const { return pimpl->size; }
540	0	void * llama_mmap::addr() const { return pimpl->addr; }
541
542	0	void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }
543
544		#if defined(_POSIX_MEMLOCK_RANGE) \|\| defined(_WIN32)
545		const bool llama_mmap::SUPPORTED = true;
546		#else
547		const bool llama_mmap::SUPPORTED = false;
548		#endif
549
550		// llama_mlock
551
552		struct llama_mlock::impl {
553		#ifdef _POSIX_MEMLOCK_RANGE
554	0	static size_t lock_granularity() {
555	0	return (size_t) sysconf(_SC_PAGESIZE);
556	0	}
557
558	0	bool raw_lock(const void * addr, size_t size) const {
559	0	if (!mlock(addr, size)) {
560	0	return true;
561	0	}
562
563		#ifdef __APPLE__
564		#define MLOCK_SUGGESTION \
565		"Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
566		"decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"
567		#else
568	0	#define MLOCK_SUGGESTION \
569	0	"Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"
570	0	#endif
571
572	0	char* errmsg = std::strerror(errno);
573	0	bool suggest = (errno == ENOMEM);
574		#if defined(TARGET_OS_VISION) \|\| defined(TARGET_OS_TV) \|\| defined(_AIX)
575		// visionOS/tvOS dont't support RLIMIT_MEMLOCK
576		// Skip resource limit checks on visionOS/tvOS
577		suggest = false;
578		#else
579	0	struct rlimit lock_limit;
580	0	if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {
581	0	suggest = false;
582	0	}
583	0	if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {
584	0	suggest = false;
585	0	}
586	0	#endif
587
588	0	LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
589	0	size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
590	0	return false;
591	0	}
592
593	0	static void raw_unlock(void * addr, size_t size) {
594	0	if (munlock(addr, size)) {
595	0	LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));
596	0	}
597	0	}
598		#elif defined(_WIN32)
599		static size_t lock_granularity() {
600		SYSTEM_INFO si;
601		GetSystemInfo(&si);
602		return (size_t) si.dwPageSize;
603		}
604
605		bool raw_lock(void * ptr, size_t len) const {
606		for (int tries = 1; ; tries++) {
607		if (VirtualLock(ptr, len)) {
608		return true;
609		}
610		if (tries == 2) {
611		LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
612		len, size, llama_format_win_err(GetLastError()).c_str());
613		return false;
614		}
615
616		SIZE_T min_ws_size, max_ws_size;
617		if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
618		LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",
619		llama_format_win_err(GetLastError()).c_str());
620		return false;
621		}
622		size_t increment = len + 1048576;
623		min_ws_size += increment;
624		max_ws_size += increment;
625		if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
626		LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",
627		llama_format_win_err(GetLastError()).c_str());
628		return false;
629		}
630		}
631		}
632
633		static void raw_unlock(void * ptr, size_t len) {
634		if (!VirtualUnlock(ptr, len)) {
635		LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",
636		llama_format_win_err(GetLastError()).c_str());
637		}
638		}
639		#else
640		static size_t lock_granularity() {
641		return (size_t) 65536;
642		}
643
644		bool raw_lock(const void * addr, size_t len) const {
645		LLAMA_LOG_WARN("warning: mlock not supported on this system\n");
646		return false;
647		}
648
649		static void raw_unlock(const void * addr, size_t len) {}
650		#endif
651
652	0	impl() : addr(NULL), size(0), failed_already(false) {}
653
654	0	void init(void * ptr) {
655	0	GGML_ASSERT(addr == NULL && size == 0);
656	0	addr = ptr;
657	0	}
658
659	0	void grow_to(size_t target_size) {
660	0	GGML_ASSERT(addr);
661	0	if (failed_already) {
662	0	return;
663	0	}
664	0	size_t granularity = lock_granularity();
665	0	target_size = (target_size + granularity - 1) & ~(granularity - 1);
666	0	if (target_size > size) {
667	0	if (raw_lock((uint8_t *) addr + size, target_size - size)) {
668	0	size = target_size;
669	0	} else {
670	0	failed_already = true;
671	0	}
672	0	}
673	0	}
674
675		void * addr;
676		size_t size;
677
678		bool failed_already;
679		};
680
681	0	llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}
682	0	llama_mlock::~llama_mlock() = default;
683
684	0	void llama_mlock::init(void * ptr) { pimpl->init(ptr); }
685	0	void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }
686
687		#if defined(_POSIX_MEMLOCK_RANGE) \|\| defined(_WIN32)
688		const bool llama_mlock::SUPPORTED = true;
689		#else
690		const bool llama_mlock::SUPPORTED = false;
691		#endif
692
693	0	size_t llama_path_max() {
694		return PATH_MAX;
695	0	}