/**********************************************************************

 *

 * Name:     cpl_virtualmem.cpp

 * Project:  CPL - Common Portability Library

 * Purpose:  Virtual memory

 * Author:   Even Rouault, <even dot rouault at spatialys.com>

 *

 **********************************************************************

 * Copyright (c) 2014, Even Rouault <even dot rouault at spatialys.com>

 *

 * SPDX-License-Identifier: MIT

 ****************************************************************************/

#ifndef _GNU_SOURCE

#define _GNU_SOURCE

#endif

// to have off_t on 64bit possibly

#ifndef _FILE_OFFSET_BITS

#define _FILE_OFFSET_BITS 64

#endif

#include "cpl_virtualmem.h"

#include <algorithm>

#include <cassert>

#include "cpl_atomic_ops.h"

#include "cpl_config.h"

#include "cpl_conv.h"

#include "cpl_error.h"

#include "cpl_multiproc.h"

#ifdef NDEBUG

// Non NDEBUG: Ignore the result.

#define IGNORE_OR_ASSERT_IN_DEBUG(expr) CPL_IGNORE_RET_VAL((expr))

#else

// Debug: Assert.

#define IGNORE_OR_ASSERT_IN_DEBUG(expr) assert((expr))

#endif

#if defined(__linux) && defined(CPL_MULTIPROC_PTHREAD)

#ifndef HAVE_5ARGS_MREMAP

// FIXME? gcore/virtualmem.py tests fail/crash when HAVE_5ARGS_MREMAP

// is not defined.

#warning "HAVE_5ARGS_MREMAP not found. Disabling HAVE_VIRTUAL_MEM_VMA"

#else

#define HAVE_VIRTUAL_MEM_VMA

#endif

#endif

#if defined(HAVE_MMAP) || defined(HAVE_VIRTUAL_MEM_VMA)

#include <unistd.h>    // read, write, close, pipe, sysconf

#include <sys/mman.h>  // mmap, munmap, mremap

#endif

typedef enum

{

    VIRTUAL_MEM_TYPE_FILE_MEMORY_MAPPED,

    VIRTUAL_MEM_TYPE_VMA

} CPLVirtualMemType;

struct CPLVirtualMem

{

    CPLVirtualMemType eType;

    struct CPLVirtualMem *pVMemBase;

    int nRefCount;

    CPLVirtualMemAccessMode eAccessMode;

    size_t nPageSize;

    // Aligned on nPageSize.

    void *pData;

    // Returned by mmap(), potentially lower than pData.

    void *pDataToFree;

    // Requested size (unrounded).

    size_t nSize;

    bool bSingleThreadUsage;

    void *pCbkUserData;

    CPLVirtualMemFreeUserData pfnFreeUserData;

};

#ifdef HAVE_VIRTUAL_MEM_VMA

#include <sys/select.h>  // select

#include <sys/stat.h>    // open()

#include <sys/types.h>   // open()

#include <errno.h>

#include <fcntl.h>   // open()

#include <signal.h>  // sigaction

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <pthread.h>

#ifndef HAVE_5ARGS_MREMAP

#include "cpl_atomic_ops.h"

#endif

/* Linux specific (i.e. non POSIX compliant) features used:

   - returning from a SIGSEGV handler is clearly a POSIX violation, but in

     practice most POSIX systems should be happy.

   - mremap() with 5 args is Linux specific. It is used when the user

     callback is invited to fill a page, we currently mmap() a

     writable page, let it filled it, and afterwards mremap() that

     temporary page onto the location where the fault occurred.

     If we have no mremap(), the workaround is to pause other threads that

     consume the current view while we are updating the faulted page, otherwise

     a non-paused thread could access a page that is in the middle of being

     filled... The way we pause those threads is quite original : we send them

     a SIGUSR1 and wait that they are stuck in the temporary SIGUSR1 handler...

   - MAP_ANONYMOUS isn't documented in POSIX, but very commonly found

     (sometimes called MAP_ANON)

   - dealing with the limitation of number of memory mapping regions,

     and the 65536 limit.

   - other things I've not identified

*/

#define ALIGN_DOWN(p, pagesize)                                                \

    reinterpret_cast<void *>((reinterpret_cast<GUIntptr_t>(p)) / (pagesize) *  \

                             (pagesize))

#define ALIGN_UP(p, pagesize)                                                  \

    reinterpret_cast<void *>(                                                  \

        cpl::div_round_up(reinterpret_cast<GUIntptr_t>(p), (pagesize)) *       \

        (pagesize))

#define DEFAULT_PAGE_SIZE (256 * 256)

#define MAXIMUM_PAGE_SIZE (32 * 1024 * 1024)

// Linux Kernel limit.

#define MAXIMUM_COUNT_OF_MAPPINGS 65536

#define BYEBYE_ADDR (reinterpret_cast<void *>(~static_cast<size_t>(0)))

#define MAPPING_FOUND "yeah"

#define MAPPING_NOT_FOUND "doh!"

#define SET_BIT(ar, bitnumber) ar[(bitnumber) / 8] |= 1 << ((bitnumber) % 8)

#define UNSET_BIT(ar, bitnumber)                                               \

    ar[(bitnumber) / 8] &= ~(1 << ((bitnumber) % 8))

#define TEST_BIT(ar, bitnumber) (ar[(bitnumber) / 8] & (1 << ((bitnumber) % 8)))

typedef enum

{

    OP_LOAD,

    OP_STORE,

    OP_MOVS_RSI_RDI,

    OP_UNKNOWN

} OpType;

typedef struct

{

    CPLVirtualMem sBase;

    GByte *pabitMappedPages;

    GByte *pabitRWMappedPages;

    int nCacheMaxSizeInPages;  // Maximum size of page array.

    int *panLRUPageIndices;    // Array with indices of cached pages.

    int iLRUStart;             // Index in array where to

                               // write next page index.

    int nLRUSize;              // Current size of the array.

    int iLastPage;  // Last page accessed.

    int nRetry;     // Number of consecutive

                    // retries to that last page.

    CPLVirtualMemCachePageCbk pfnCachePage;      // Called when a page is

                                                 // mapped.

    CPLVirtualMemUnCachePageCbk pfnUnCachePage;  // Called when a (writable)

                                                 // page is unmapped.

#ifndef HAVE_5ARGS_MREMAP

    CPLMutex *hMutexThreadArray;

    int nThreads;

    pthread_t *pahThreads;

#endif

} CPLVirtualMemVMA;

typedef struct

{

    // hVirtualMemManagerMutex protects the 2 following variables.

    CPLVirtualMemVMA **pasVirtualMem;

    int nVirtualMemCount;

    int pipefd_to_thread[2];

    int pipefd_from_thread[2];

    int pipefd_wait_thread[2];

    CPLJoinableThread *hHelperThread;

    // Using sigaction without testing HAVE_SIGACTION since we are in a Linux

    // specific code path

    struct sigaction oldact;

} CPLVirtualMemManager;

typedef struct

{

    void *pFaultAddr;

    OpType opType;

    pthread_t hRequesterThread;

} CPLVirtualMemMsgToWorkerThread;

// TODO: Singletons.

static CPLVirtualMemManager *pVirtualMemManager = nullptr;

static CPLMutex *hVirtualMemManagerMutex = nullptr;

static bool CPLVirtualMemManagerInit();

#ifdef DEBUG_VIRTUALMEM

/************************************************************************/

/*                           fprintfstderr()                            */

/************************************************************************/

// This function may be called from signal handlers where most functions

// from the C library are unsafe to be called. fprintf() is clearly one

// of those functions (see

// http://stackoverflow.com/questions/4554129/linux-glibc-can-i-use-fprintf-in-signal-handler)

// vsnprintf() is *probably* safer with respect to that (but there is no

// guarantee though).

// write() is async-signal-safe.

static void fprintfstderr(const char *fmt, ...)

{

    char buffer[80] = {};

    va_list ap;

    va_start(ap, fmt);

    vsnprintf(buffer, sizeof(buffer), fmt, ap);

    va_end(ap);

    int offset = 0;

    while (true)

    {

        const size_t nSizeToWrite = strlen(buffer + offset);

        int ret = static_cast<int>(write(2, buffer + offset, nSizeToWrite));

        if (ret < 0 && errno == EINTR)

        {

        }

        else

        {

            if (ret == static_cast<int>(nSizeToWrite))

                break;

            offset += ret;

        }

    }

}

#endif

/************************************************************************/

/*              CPLVirtualMemManagerRegisterVirtualMem()                */

/************************************************************************/

static bool CPLVirtualMemManagerRegisterVirtualMem(CPLVirtualMemVMA *ctxt)

{

    if (!CPLVirtualMemManagerInit())

        return false;

    bool bSuccess = true;

    IGNORE_OR_ASSERT_IN_DEBUG(ctxt);

    CPLAcquireMutex(hVirtualMemManagerMutex, 1000.0);

    CPLVirtualMemVMA **pasVirtualMemNew = static_cast<CPLVirtualMemVMA **>(

        VSI_REALLOC_VERBOSE(pVirtualMemManager->pasVirtualMem,

                            sizeof(CPLVirtualMemVMA *) *

                                (pVirtualMemManager->nVirtualMemCount + 1)));

    if (pasVirtualMemNew == nullptr)

    {

        bSuccess = false;

    }

    else

    {

        pVirtualMemManager->pasVirtualMem = pasVirtualMemNew;

        pVirtualMemManager

            ->pasVirtualMem[pVirtualMemManager->nVirtualMemCount] = ctxt;

        pVirtualMemManager->nVirtualMemCount++;

    }

    CPLReleaseMutex(hVirtualMemManagerMutex);

    return bSuccess;

}

/************************************************************************/

/*               CPLVirtualMemManagerUnregisterVirtualMem()             */

/************************************************************************/

static void CPLVirtualMemManagerUnregisterVirtualMem(CPLVirtualMemVMA *ctxt)

{

    CPLAcquireMutex(hVirtualMemManagerMutex, 1000.0);

    for (int i = 0; i < pVirtualMemManager->nVirtualMemCount; i++)

    {

        if (pVirtualMemManager->pasVirtualMem[i] == ctxt)

        {

            if (i < pVirtualMemManager->nVirtualMemCount - 1)

            {

                memmove(pVirtualMemManager->pasVirtualMem + i,

                        pVirtualMemManager->pasVirtualMem + i + 1,

                        sizeof(CPLVirtualMem *) *

                            (pVirtualMemManager->nVirtualMemCount - i - 1));

            }

            pVirtualMemManager->nVirtualMemCount--;

            break;

        }

    }

    CPLReleaseMutex(hVirtualMemManagerMutex);

}

/************************************************************************/

/*                           CPLVirtualMemNew()                         */

/************************************************************************/

static void CPLVirtualMemFreeFileMemoryMapped(CPLVirtualMemVMA *ctxt);

CPLVirtualMem *CPLVirtualMemNew(size_t nSize, size_t nCacheSize,

                                size_t nPageSizeHint, int bSingleThreadUsage,

                                CPLVirtualMemAccessMode eAccessMode,

                                CPLVirtualMemCachePageCbk pfnCachePage,

                                CPLVirtualMemUnCachePageCbk pfnUnCachePage,

                                CPLVirtualMemFreeUserData pfnFreeUserData,

                                void *pCbkUserData)

{

    size_t nMinPageSize = CPLGetPageSize();

    size_t nPageSize = DEFAULT_PAGE_SIZE;

    IGNORE_OR_ASSERT_IN_DEBUG(nSize > 0);

    IGNORE_OR_ASSERT_IN_DEBUG(pfnCachePage != nullptr);

    if (nPageSizeHint >= nMinPageSize && nPageSizeHint <= MAXIMUM_PAGE_SIZE)

    {

        if ((nPageSizeHint % nMinPageSize) == 0)

            nPageSize = nPageSizeHint;

        else

        {

            int nbits = 0;

            nPageSize = static_cast<size_t>(nPageSizeHint);

            do

            {

                nPageSize >>= 1;

                nbits++;

            } while (nPageSize > 0);

            nPageSize = static_cast<size_t>(1) << (nbits - 1);

            if (nPageSize < static_cast<size_t>(nPageSizeHint))

                nPageSize <<= 1;

        }

    }

    if ((nPageSize % nMinPageSize) != 0)

        nPageSize = nMinPageSize;

    if (nCacheSize > nSize)

        nCacheSize = nSize;

    else if (nCacheSize == 0)

        nCacheSize = 1;

    int nMappings = 0;

    // Linux specific:

    // Count the number of existing memory mappings.

    FILE *f = fopen("/proc/self/maps", "rb");

    if (f != nullptr)

    {

        char buffer[80] = {};

        while (fgets(buffer, sizeof(buffer), f) != nullptr)

            nMappings++;

        fclose(f);

    }

    size_t nCacheMaxSizeInPages = 0;

    while (true)

    {

        // /proc/self/maps must not have more than 65K lines.

        nCacheMaxSizeInPages = (nCacheSize + 2 * nPageSize - 1) / nPageSize;

        if (nCacheMaxSizeInPages >

            static_cast<size_t>((MAXIMUM_COUNT_OF_MAPPINGS * 9 / 10) -

                                nMappings))

            nPageSize <<= 1;

        else

            break;

    }

    size_t nRoundedMappingSize =

        ((nSize + 2 * nPageSize - 1) / nPageSize) * nPageSize;

    void *pData = mmap(nullptr, nRoundedMappingSize, PROT_NONE,

                       MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

    if (pData == MAP_FAILED)

    {

        perror("mmap");

        return nullptr;

    }

    CPLVirtualMemVMA *ctxt = static_cast<CPLVirtualMemVMA *>(

        VSI_CALLOC_VERBOSE(1, sizeof(CPLVirtualMemVMA)));

    if (ctxt == nullptr)

    {

        munmap(pData, nRoundedMappingSize);

        return nullptr;

    }

    ctxt->sBase.nRefCount = 1;

    ctxt->sBase.eType = VIRTUAL_MEM_TYPE_VMA;

    ctxt->sBase.eAccessMode = eAccessMode;

    ctxt->sBase.pDataToFree = pData;

    ctxt->sBase.pData = ALIGN_UP(pData, nPageSize);

    ctxt->sBase.nPageSize = nPageSize;

    ctxt->sBase.nSize = nSize;

    ctxt->sBase.bSingleThreadUsage = CPL_TO_BOOL(bSingleThreadUsage);

    ctxt->sBase.pfnFreeUserData = pfnFreeUserData;

    ctxt->sBase.pCbkUserData = pCbkUserData;

    ctxt->pabitMappedPages = static_cast<GByte *>(

        VSI_CALLOC_VERBOSE(1, (nRoundedMappingSize / nPageSize + 7) / 8));

    if (ctxt->pabitMappedPages == nullptr)

    {

        CPLVirtualMemFreeFileMemoryMapped(ctxt);

        CPLFree(ctxt);

        return nullptr;

    }

    ctxt->pabitRWMappedPages = static_cast<GByte *>(

        VSI_CALLOC_VERBOSE(1, (nRoundedMappingSize / nPageSize + 7) / 8));

    if (ctxt->pabitRWMappedPages == nullptr)

    {

        CPLVirtualMemFreeFileMemoryMapped(ctxt);

        CPLFree(ctxt);

        return nullptr;

    }

    // Need at least 2 pages in case for a rep movs instruction

    // that operate in the view.

    ctxt->nCacheMaxSizeInPages = static_cast<int>(nCacheMaxSizeInPages);

    ctxt->panLRUPageIndices = static_cast<int *>(

        VSI_MALLOC_VERBOSE(ctxt->nCacheMaxSizeInPages * sizeof(int)));

    if (ctxt->panLRUPageIndices == nullptr)

    {

        CPLVirtualMemFreeFileMemoryMapped(ctxt);

        CPLFree(ctxt);

        return nullptr;

    }

    ctxt->iLRUStart = 0;

    ctxt->nLRUSize = 0;

    ctxt->iLastPage = -1;

    ctxt->nRetry = 0;

    ctxt->pfnCachePage = pfnCachePage;

    ctxt->pfnUnCachePage = pfnUnCachePage;

#ifndef HAVE_5ARGS_MREMAP

    if (!ctxt->sBase.bSingleThreadUsage)

    {

        ctxt->hMutexThreadArray = CPLCreateMutex();

        IGNORE_OR_ASSERT_IN_DEBUG(ctxt->hMutexThreadArray != nullptr);

        CPLReleaseMutex(ctxt->hMutexThreadArray);

        ctxt->nThreads = 0;

        ctxt->pahThreads = nullptr;

    }

#endif

    if (!CPLVirtualMemManagerRegisterVirtualMem(ctxt))

    {

        CPLVirtualMemFreeFileMemoryMapped(ctxt);

        CPLFree(ctxt);

        return nullptr;

    }

    return reinterpret_cast<CPLVirtualMem *>(ctxt);

}

/************************************************************************/

/*                  CPLVirtualMemFreeFileMemoryMapped()                 */

/************************************************************************/

static void CPLVirtualMemFreeFileMemoryMapped(CPLVirtualMemVMA *ctxt)

{

    CPLVirtualMemManagerUnregisterVirtualMem(ctxt);

    size_t nRoundedMappingSize =

        ((ctxt->sBase.nSize + 2 * ctxt->sBase.nPageSize - 1) /

         ctxt->sBase.nPageSize) *

        ctxt->sBase.nPageSize;

    if (ctxt->sBase.eAccessMode == VIRTUALMEM_READWRITE &&

        ctxt->pabitRWMappedPages != nullptr && ctxt->pfnUnCachePage != nullptr)

    {

        for (size_t i = 0; i < nRoundedMappingSize / ctxt->sBase.nPageSize; i++)

        {

            if (TEST_BIT(ctxt->pabitRWMappedPages, i))

            {

                void *addr = static_cast<char *>(ctxt->sBase.pData) +

                             i * ctxt->sBase.nPageSize;

                ctxt->pfnUnCachePage(reinterpret_cast<CPLVirtualMem *>(ctxt),

                                     i * ctxt->sBase.nPageSize, addr,

                                     ctxt->sBase.nPageSize,

                                     ctxt->sBase.pCbkUserData);

            }

        }

    }

    int nRet = munmap(ctxt->sBase.pDataToFree, nRoundedMappingSize);

    IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

    CPLFree(ctxt->pabitMappedPages);

    CPLFree(ctxt->pabitRWMappedPages);

    CPLFree(ctxt->panLRUPageIndices);

#ifndef HAVE_5ARGS_MREMAP

    if (!ctxt->sBase.bSingleThreadUsage)

    {

        CPLFree(ctxt->pahThreads);

        CPLDestroyMutex(ctxt->hMutexThreadArray);

    }

#endif

}

#ifndef HAVE_5ARGS_MREMAP

static volatile int nCountThreadsInSigUSR1 = 0;

static volatile int nWaitHelperThread = 0;

/************************************************************************/

/*                   CPLVirtualMemSIGUSR1Handler()                      */

/************************************************************************/

static void CPLVirtualMemSIGUSR1Handler(int /* signum_unused */,

                                        siginfo_t * /* the_info_unused */,

                                        void * /* the_ctxt_unused */)

{

#if defined DEBUG_VIRTUALMEM && defined DEBUG_VERBOSE

    fprintfstderr("entering CPLVirtualMemSIGUSR1Handler %X\n", pthread_self());

#endif

    // Rouault guesses this is only POSIX correct if it is implemented by an

    // intrinsic.

    CPLAtomicInc(&nCountThreadsInSigUSR1);

    while (nWaitHelperThread)

        // Not explicitly indicated as signal-async-safe, but hopefully ok.

        usleep(1);

    CPLAtomicDec(&nCountThreadsInSigUSR1);

#if defined DEBUG_VIRTUALMEM && defined DEBUG_VERBOSE

    fprintfstderr("leaving CPLVirtualMemSIGUSR1Handler %X\n", pthread_self());

#endif

}

#endif

/************************************************************************/

/*                      CPLVirtualMemDeclareThread()                    */

/************************************************************************/

void CPLVirtualMemDeclareThread(CPLVirtualMem *ctxt)

{

    if (ctxt->eType == VIRTUAL_MEM_TYPE_FILE_MEMORY_MAPPED)

        return;

#ifndef HAVE_5ARGS_MREMAP

    CPLVirtualMemVMA *ctxtVMA = reinterpret_cast<CPLVirtualMemVMA *>(ctxt);

    IGNORE_OR_ASSERT_IN_DEBUG(!ctxt->bSingleThreadUsage);

    CPLAcquireMutex(ctxtVMA->hMutexThreadArray, 1000.0);

    ctxtVMA->pahThreads = static_cast<pthread_t *>(CPLRealloc(

        ctxtVMA->pahThreads, (ctxtVMA->nThreads + 1) * sizeof(pthread_t)));

    ctxtVMA->pahThreads[ctxtVMA->nThreads] = pthread_self();

    ctxtVMA->nThreads++;

    CPLReleaseMutex(ctxtVMA->hMutexThreadArray);

#endif

}

/************************************************************************/

/*                     CPLVirtualMemUnDeclareThread()                   */

/************************************************************************/

void CPLVirtualMemUnDeclareThread(CPLVirtualMem *ctxt)

{

    if (ctxt->eType == VIRTUAL_MEM_TYPE_FILE_MEMORY_MAPPED)

        return;

#ifndef HAVE_5ARGS_MREMAP

    CPLVirtualMemVMA *ctxtVMA = reinterpret_cast<CPLVirtualMemVMA *>(ctxt);

    pthread_t self = pthread_self();

    IGNORE_OR_ASSERT_IN_DEBUG(!ctxt->bSingleThreadUsage);

    CPLAcquireMutex(ctxtVMA->hMutexThreadArray, 1000.0);

    for (int i = 0; i < ctxtVMA->nThreads; i++)

    {

        if (ctxtVMA->pahThreads[i] == self)

        {

            if (i < ctxtVMA->nThreads - 1)

                memmove(ctxtVMA->pahThreads + i + 1, ctxtVMA->pahThreads + i,

                        (ctxtVMA->nThreads - 1 - i) * sizeof(pthread_t));

            ctxtVMA->nThreads--;

            break;

        }

    }

    CPLReleaseMutex(ctxtVMA->hMutexThreadArray);

#endif

}

/************************************************************************/

/*                     CPLVirtualMemGetPageToFill()                     */

/************************************************************************/

// Must be paired with CPLVirtualMemAddPage.

static void *CPLVirtualMemGetPageToFill(CPLVirtualMemVMA *ctxt,

                                        void *start_page_addr)

{

    void *pPageToFill = nullptr;

    if (ctxt->sBase.bSingleThreadUsage)

    {

        pPageToFill = start_page_addr;

        const int nRet = mprotect(pPageToFill, ctxt->sBase.nPageSize,

                                  PROT_READ | PROT_WRITE);

        IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

    }

    else

    {

#ifndef HAVE_5ARGS_MREMAP

        CPLAcquireMutex(ctxt->hMutexThreadArray, 1000.0);

        if (ctxt->nThreads == 1)

        {

            pPageToFill = start_page_addr;

            const int nRet = mprotect(pPageToFill, ctxt->sBase.nPageSize,

                                      PROT_READ | PROT_WRITE);

            IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

        }

        else

#endif

        {

            // Allocate a temporary writable page that the user

            // callback can fill.

            pPageToFill =

                mmap(nullptr, ctxt->sBase.nPageSize, PROT_READ | PROT_WRITE,

                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

            IGNORE_OR_ASSERT_IN_DEBUG(pPageToFill != MAP_FAILED);

        }

    }

    return pPageToFill;

}

/************************************************************************/

/*                        CPLVirtualMemAddPage()                        */

/************************************************************************/

static void CPLVirtualMemAddPage(CPLVirtualMemVMA *ctxt, void *target_addr,

                                 void *pPageToFill, OpType opType,

                                 pthread_t hRequesterThread)

{

    const int iPage =

        static_cast<int>((static_cast<char *>(target_addr) -

                          static_cast<char *>(ctxt->sBase.pData)) /

                         ctxt->sBase.nPageSize);

    if (ctxt->nLRUSize == ctxt->nCacheMaxSizeInPages)

    {

#if defined DEBUG_VIRTUALMEM && defined DEBUG_VERBOSE

        fprintfstderr("uncaching page %d\n", iPage);

#endif

        int nOldPage = ctxt->panLRUPageIndices[ctxt->iLRUStart];

        void *addr = static_cast<char *>(ctxt->sBase.pData) +

                     nOldPage * ctxt->sBase.nPageSize;

        if (ctxt->sBase.eAccessMode == VIRTUALMEM_READWRITE &&

            ctxt->pfnUnCachePage != nullptr &&

            TEST_BIT(ctxt->pabitRWMappedPages, nOldPage))

        {

            size_t nToBeEvicted = ctxt->sBase.nPageSize;

            if (static_cast<char *>(addr) + nToBeEvicted >=

                static_cast<char *>(ctxt->sBase.pData) + ctxt->sBase.nSize)

                nToBeEvicted = static_cast<char *>(ctxt->sBase.pData) +

                               ctxt->sBase.nSize - static_cast<char *>(addr);

            ctxt->pfnUnCachePage(reinterpret_cast<CPLVirtualMem *>(ctxt),

                                 nOldPage * ctxt->sBase.nPageSize, addr,

                                 nToBeEvicted, ctxt->sBase.pCbkUserData);

        }

        // "Free" the least recently used page.

        UNSET_BIT(ctxt->pabitMappedPages, nOldPage);

        UNSET_BIT(ctxt->pabitRWMappedPages, nOldPage);

        // Free the old page.

        // Not sure how portable it is to do that that way.

        const void *const pRet =

            mmap(addr, ctxt->sBase.nPageSize, PROT_NONE,

                 MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

        IGNORE_OR_ASSERT_IN_DEBUG(pRet == addr);

        // cppcheck-suppress memleak

    }

    ctxt->panLRUPageIndices[ctxt->iLRUStart] = iPage;

    ctxt->iLRUStart = (ctxt->iLRUStart + 1) % ctxt->nCacheMaxSizeInPages;

    if (ctxt->nLRUSize < ctxt->nCacheMaxSizeInPages)

    {

        ctxt->nLRUSize++;

    }

    SET_BIT(ctxt->pabitMappedPages, iPage);

    if (ctxt->sBase.bSingleThreadUsage)

    {

        if (opType == OP_STORE &&

            ctxt->sBase.eAccessMode == VIRTUALMEM_READWRITE)

        {

            // Let (and mark) the page writable since the instruction that

            // triggered the fault is a store.

            SET_BIT(ctxt->pabitRWMappedPages, iPage);

        }

        else if (ctxt->sBase.eAccessMode != VIRTUALMEM_READONLY)

        {

            const int nRet =

                mprotect(target_addr, ctxt->sBase.nPageSize, PROT_READ);

            IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

        }

    }

    else

    {

#ifdef HAVE_5ARGS_MREMAP

        (void)hRequesterThread;

        if (opType == OP_STORE &&

            ctxt->sBase.eAccessMode == VIRTUALMEM_READWRITE)

        {

            // Let (and mark) the page writable since the instruction that

            // triggered the fault is a store.

            SET_BIT(ctxt->pabitRWMappedPages, iPage);

        }

        else if (ctxt->sBase.eAccessMode != VIRTUALMEM_READONLY)

        {

            // Turn the temporary page read-only before remapping it.

            // Only turn it writtable when a new fault occurs (and the

            // mapping is writable).

            const int nRet =

                mprotect(pPageToFill, ctxt->sBase.nPageSize, PROT_READ);

            IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

        }

        /* Can now remap the pPageToFill onto the target page */

        const void *const pRet =

            mremap(pPageToFill, ctxt->sBase.nPageSize, ctxt->sBase.nPageSize,

                   MREMAP_MAYMOVE | MREMAP_FIXED, target_addr);

        IGNORE_OR_ASSERT_IN_DEBUG(pRet == target_addr);

#else

        if (ctxt->nThreads > 1)

        {

            /* Pause threads that share this mem view */

            CPLAtomicInc(&nWaitHelperThread);

            /* Install temporary SIGUSR1 signal handler */

            struct sigaction act, oldact;

            act.sa_sigaction = CPLVirtualMemSIGUSR1Handler;

            sigemptyset(&act.sa_mask);

            /* We don't want the sigsegv handler to be called when we are */

            /* running the sigusr1 handler */

            IGNORE_OR_ASSERT_IN_DEBUG(sigaddset(&act.sa_mask, SIGSEGV) == 0);

            act.sa_flags = 0;

            IGNORE_OR_ASSERT_IN_DEBUG(sigaction(SIGUSR1, &act, &oldact) == 0);

            for (int i = 0; i < ctxt->nThreads; i++)

            {

                if (ctxt->pahThreads[i] != hRequesterThread)

                {

#if defined DEBUG_VIRTUALMEM && defined DEBUG_VERBOSE

                    fprintfstderr("stopping thread %X\n", ctxt->pahThreads[i]);

#endif

                    IGNORE_OR_ASSERT_IN_DEBUG(

                        pthread_kill(ctxt->pahThreads[i], SIGUSR1) == 0);

                }

            }

            /* Wait that they are all paused */

            while (nCountThreadsInSigUSR1 != ctxt->nThreads - 1)

                usleep(1);

            /* Restore old SIGUSR1 signal handler */

            IGNORE_OR_ASSERT_IN_DEBUG(sigaction(SIGUSR1, &oldact, nullptr) ==

                                      0);

            int nRet = mprotect(target_addr, ctxt->sBase.nPageSize,

                                PROT_READ | PROT_WRITE);

            IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

#if defined DEBUG_VIRTUALMEM && defined DEBUG_VERBOSE

            fprintfstderr("memcpying page %d\n", iPage);

#endif

            memcpy(target_addr, pPageToFill, ctxt->sBase.nPageSize);

            if (opType == OP_STORE &&

                ctxt->sBase.eAccessMode == VIRTUALMEM_READWRITE)

            {

                // Let (and mark) the page writable since the instruction that

                // triggered the fault is a store.

                SET_BIT(ctxt->pabitRWMappedPages, iPage);

            }

            else

            {

                nRet = mprotect(target_addr, ctxt->sBase.nPageSize, PROT_READ);

                IGNORE_OR_ASSERT_IN_DEBUG(nRet == 0);

            }

            /* Wake up sleeping threads */

            CPLAtomicDec(&nWaitHelperThread);

            while (nCountThreadsInSigUSR1 != 0)

                usleep(1);

            IGNORE_OR_ASSERT_IN_DEBUG(

                munmap(pPageToFill, ctxt->sBase.nPageSize) == 0);

        }

        else

        {

            if (opType == OP_STORE &&

                ctxt->sBase.eAccessMode == VIRTUALMEM_READWRITE)

            {

                // Let (and mark) the page writable since the instruction that

                // triggered the fault is a store.

                SET_BIT(ctxt->pabitRWMappedPages, iPage);

            }

            else if (ctxt->sBase.eAccessMode != VIRTUALMEM_READONLY)

            {

                const int nRet2 =

                    mprotect(target_addr, ctxt->sBase.nPageSize, PROT_READ);

                IGNORE_OR_ASSERT_IN_DEBUG(nRet2 == 0);

            }

        }

        CPLReleaseMutex(ctxt->hMutexThreadArray);

#endif

    }

    // cppcheck-suppress memleak

}

/************************************************************************/

/*                    CPLVirtualMemGetOpTypeImm()                       */

/************************************************************************/

#if defined(__x86_64__) || defined(__i386__)

static OpType CPLVirtualMemGetOpTypeImm(GByte val_rip)

{

    OpType opType = OP_UNKNOWN;

    if ((/*val_rip >= 0x00 &&*/ val_rip <= 0x07) ||

        (val_rip >= 0x40 && val_rip <= 0x47))  // add $, (X)

        opType = OP_STORE;

    if ((val_rip >= 0x08 && val_rip <= 0x0f) ||

        (val_rip >= 0x48 && val_rip <= 0x4f))  // or $, (X)

        opType = OP_STORE;

    if ((val_rip >= 0x20 && val_rip <= 0x27) ||

        (val_rip >= 0x60 && val_rip <= 0x67))  // and $, (X)

        opType = OP_STORE;

    if ((val_rip >= 0x28 && val_rip <= 0x2f) ||

        (val_rip >= 0x68 && val_rip <= 0x6f))  // sub $, (X)

        opType = OP_STORE;

    if ((val_rip >= 0x30 && val_rip <= 0x37) ||

        (val_rip >= 0x70 && val_rip <= 0x77))  // xor $, (X)

        opType = OP_STORE;

    if ((val_rip >= 0x38 && val_rip <= 0x3f) ||

        (val_rip >= 0x78 && val_rip <= 0x7f))  // cmp $, (X)

        opType = OP_LOAD;

    return opType;

}

#endif

/************************************************************************/

/*                      CPLVirtualMemGetOpType()                        */

/************************************************************************/

// Don't need exhaustivity. It is just a hint for an optimization:

// If the fault occurs on a store operation, then we can directly put

// the page in writable mode if the mapping allows it.

#if defined(__x86_64__) || defined(__i386__)

static OpType CPLVirtualMemGetOpType(const GByte *rip)

{

    OpType opType = OP_UNKNOWN;

#if defined(__x86_64__) || defined(__i386__)

    switch (rip[0])

    {

        case 0x00: /* add %al,(%rax) */

        case 0x01: /* add %eax,(%rax) */

            opType = OP_STORE;

            break;

        case 0x02: /* add (%rax),%al */

        case 0x03: /* add (%rax),%eax */

            opType = OP_LOAD;

            break;

        case 0x08: /* or %al,(%rax) */

        case 0x09: /* or %eax,(%rax) */

            opType = OP_STORE;

            break;

        case 0x0a: /* or (%rax),%al */

        case 0x0b: /* or (%rax),%eax */

            opType = OP_LOAD;

            break;

        case 0x0f:

        {

            switch (rip[1])

            {

                case 0xb6: /* movzbl (%rax),%eax */

                case 0xb7: /* movzwl (%rax),%eax */

                case 0xbe: /* movsbl (%rax),%eax */

                case 0xbf: /* movswl (%rax),%eax */

                    opType = OP_LOAD;

                    break;

                default:

                    break;

            }

            break;

        }

        case 0xc6: /* movb $,(%rax) */

        case 0xc7: /* movl $,(%rax) */

            opType = OP_STORE;

            break;

        case 0x20: /* and %al,(%rax) */

        case 0x21: /* and %eax,(%rax) */

            opType = OP_STORE;

            break;

        case 0x22: /* and (%rax),%al */

        case 0x23: /* and (%rax),%eax */

            opType = OP_LOAD;

            break;

        case 0x28: /* sub %al,(%rax) */

        case 0x29: /* sub %eax,(%rax) */

            opType = OP_STORE;

            break;

        case 0x2a: /* sub (%rax),%al */

        case 0x2b: /* sub (%rax),%eax */

            opType = OP_LOAD;

            break;

        case 0x30: /* xor %al,(%rax) */

        case 0x31: /* xor %eax,(%rax) */

            opType = OP_STORE;

            break;

        case 0x32: /* xor (%rax),%al */

        case 0x33: /* xor (%rax),%eax */

            opType = OP_LOAD;

            break;

        case 0x38: /* cmp %al,(%rax) */

        case 0x39: /* cmp %eax,(%rax) */

            opType = OP_LOAD;

            break;

        case 0x40:

        {

            switch (rip[1])

            {

                case 0x00: /* add %spl,(%rax) */

                    opType = OP_STORE;

                    break;

                case 0x02: /* add (%rax),%spl */

                    opType = OP_LOAD;

                    break;

                case 0x28: /* sub %spl,(%rax) */

                    opType = OP_STORE;

                    break;

                case 0x2a: /* sub (%rax),%spl */

                    opType = OP_LOAD;

                    break;

                case 0x3a: /* cmp (%rax),%spl */

                    opType = OP_LOAD;

                    break;

                case 0x8a: /* mov (%rax),%spl */

                    opType = OP_LOAD;

                    break;

                default:

                    break;

            }

            break;

        }

#if defined(__x86_64__)

        case 0x41: /* reg=%al/%eax, X=%r8 */

        case 0x42: /* reg=%al/%eax, X=%rax,%r8,1 */

        case 0x43: /* reg=%al/%eax, X=%r8,%r8,1 */

        case 0x44: /* reg=%r8b/%r8w, X = %rax */

        case 0x45: /* reg=%r8b/%r8w, X = %r8 */

        case 0x46: /* reg=%r8b/%r8w, X = %rax,%r8,1 */

        case 0x47: /* reg=%r8b/%r8w, X = %r8,%r8,1 */

        {

            switch (rip[1])

            {

                case 0x00: /* add regb,(X) */

                case 0x01: /* add regl,(X) */