/src/icu/source/common/umapfile.cpp

Source
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
*
*   Copyright (C) 1999-2013, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
******************************************************************************/


/*----------------------------------------------------------------------------
 *
 *       Memory mapped file wrappers for use by the ICU Data Implementation
 *       All of the platform-specific implementation for mapping data files
 *         is here.  The rest of the ICU Data implementation uses only the
 *         wrapper functions.
 *
 *----------------------------------------------------------------------------*/
/* Defines _XOPEN_SOURCE for access to POSIX functions.
 * Must be before any other #includes. */
#include "uposixdefs.h"

#include "unicode/putil.h"
#include "unicode/ustring.h"
#include "udatamem.h"
#include "umapfile.h"

/* memory-mapping base definitions ------------------------------------------ */

#if MAP_IMPLEMENTATION==MAP_WIN32
#ifndef WIN32_LEAN_AND_MEAN
#   define WIN32_LEAN_AND_MEAN
#endif
#   define VC_EXTRALEAN
#   define NOUSER
#   define NOSERVICE
#   define NOIME
#   define NOMCX

#   if U_PLATFORM_HAS_WINUWP_API == 1
        // Some previous versions of the Windows 10 SDK don't expose various APIs for UWP applications
        // to use, even though UWP apps are allowed to call and use them.  Temporarily change the
        // WINAPI family partition below to Desktop, so that function declarations are visible for UWP.
#       include <winapifamily.h>
#       if !(WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM))
#           pragma push_macro("WINAPI_PARTITION_DESKTOP")
#           undef WINAPI_PARTITION_DESKTOP
#           define WINAPI_PARTITION_DESKTOP 1
#           define CHANGED_WINAPI_PARTITION_DESKTOP_VALUE
#       endif
#   endif

#   include <windows.h>

#   if U_PLATFORM_HAS_WINUWP_API == 1 && defined(CHANGED_WINAPI_PARTITION_DESKTOP_VALUE)
#       pragma pop_macro("WINAPI_PARTITION_DESKTOP")
#   endif

#   include "cmemory.h"

typedef HANDLE MemoryMap;

#   define IS_MAP(map) ((map)!=nullptr)

#elif MAP_IMPLEMENTATION==MAP_POSIX
    typedef size_t MemoryMap;

#   define IS_MAP(map) ((map)!=0)

#   include <unistd.h>
#   include <sys/mman.h>
#   include <sys/stat.h>
#   include <fcntl.h>

#   ifndef MAP_FAILED
#       define MAP_FAILED ((void*)-1)
#   endif
#elif MAP_IMPLEMENTATION==MAP_STDIO
#   include <stdio.h>
#   include "cmemory.h"

    typedef void *MemoryMap;

#   define IS_MAP(map) ((map)!=nullptr)
#endif

/*----------------------------------------------------------------------------*
 *                                                                            *
 *   Memory Mapped File support.  Platform dependent implementation of        *
 *                           functions used by the rest of the implementation.*
 *                                                                            *
 *----------------------------------------------------------------------------*/
#if MAP_IMPLEMENTATION==MAP_NONE
    U_CFUNC UBool
    uprv_mapFile(UDataMemory *pData, const char *path, UErrorCode *status) {
        if (U_FAILURE(*status)) {
            return false;
        }
        UDataMemory_init(pData); /* Clear the output struct. */
        return false;            /* no file access */
    }

    U_CFUNC void uprv_unmapFile(UDataMemory *pData) {
        /* nothing to do */
    }
#elif MAP_IMPLEMENTATION==MAP_WIN32
    U_CFUNC UBool
    uprv_mapFile(
         UDataMemory *pData,    /* Fill in with info on the result doing the mapping. */
                                /*   Output only; any original contents are cleared.  */
         const char *path,      /* File path to be opened/mapped.                     */
         UErrorCode *status     /* Error status, used to report out-of-memory errors. */
         )
    {
        if (U_FAILURE(*status)) {
            return false;
        }

        HANDLE map = nullptr;
        HANDLE file = INVALID_HANDLE_VALUE;
        DWORD fileLength = 0;

        UDataMemory_init(pData); /* Clear the output struct.        */

        /* open the input file */
#if U_PLATFORM_HAS_WINUWP_API == 0
        // Note: In the non-UWP code-path (ie: Win32), the value of the path variable might have come from 
        // the CRT 'getenv' function, and would be therefore be encoded in the default ANSI code page.
        // This means that we can't call the *W version of API below, whereas in the UWP code-path
        // there is no 'getenv' call, and thus the string will be only UTF-8/Invariant characters.
        file=CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr,
            OPEN_EXISTING,
            FILE_ATTRIBUTE_NORMAL|FILE_FLAG_RANDOM_ACCESS, nullptr);
#else
        // Convert from UTF-8 string to UTF-16 string.
        wchar_t utf16Path[MAX_PATH];
        int32_t pathUtf16Len = 0;
        u_strFromUTF8(reinterpret_cast<char16_t*>(utf16Path), static_cast<int32_t>(UPRV_LENGTHOF(utf16Path)), &pathUtf16Len, path, -1, status);

        if (U_FAILURE(*status)) {
            return false;
        }
        if (*status == U_STRING_NOT_TERMINATED_WARNING) {
            // Report back an error instead of a warning.
            *status = U_BUFFER_OVERFLOW_ERROR;
            return false;
        }

        file = CreateFileW(utf16Path, GENERIC_READ, FILE_SHARE_READ, nullptr,
            OPEN_EXISTING,
            FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, nullptr);
#endif
        if (file == INVALID_HANDLE_VALUE) {
            // If we failed to open the file due to an out-of-memory error, then we want
            // to report that error back to the caller.
            if (HRESULT_FROM_WIN32(GetLastError()) == E_OUTOFMEMORY) {
                *status = U_MEMORY_ALLOCATION_ERROR;
            }
            return false;
        }

        fileLength = GetFileSize(file, nullptr);

        // Note: We use nullptr/nullptr for lpAttributes parameter below.
        // This means our handle cannot be inherited and we will get the default security descriptor.
        /* create an unnamed Windows file-mapping object for the specified file */
        map = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr);

        CloseHandle(file);
        if (map == nullptr) {
            // If we failed to create the mapping due to an out-of-memory error, then 
            // we want to report that error back to the caller.
            if (HRESULT_FROM_WIN32(GetLastError()) == E_OUTOFMEMORY) {
                *status = U_MEMORY_ALLOCATION_ERROR;
            }
            return false;
        }

        /* map a view of the file into our address space */
        pData->pHeader = reinterpret_cast<const DataHeader *>(MapViewOfFile(map, FILE_MAP_READ, 0, 0, 0));
        if (pData->pHeader == nullptr) {
            CloseHandle(map);
            return false;
        }
        pData->map = map;
        pData->length = fileLength;

        return true;
    }

    U_CFUNC void
    uprv_unmapFile(UDataMemory *pData) {
        if (pData != nullptr && pData->map != nullptr) {
            UnmapViewOfFile(pData->pHeader);
            CloseHandle(pData->map);
            pData->pHeader = nullptr;
            pData->map = nullptr;
        }
    }



#elif MAP_IMPLEMENTATION==MAP_POSIX
    U_CFUNC UBool
    uprv_mapFile(UDataMemory *pData, const char *path, UErrorCode *status) {
        int fd;
        int length;
        struct stat mystat;
        void *data;

        if (U_FAILURE(*status)) {
            return false;
        }

        UDataMemory_init(pData); /* Clear the output struct.        */

        /* determine the length of the file */
        if(stat(path, &mystat)!=0 || mystat.st_size<=0) {
            return false;
        }
        length=mystat.st_size;

        /* open the file */
        fd=open(path, O_RDONLY);
        if(fd==-1) {
            return false;
        }

        /* get a view of the mapping */
#if U_PLATFORM != U_PF_HPUX
        data=mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0);
#else
        data=mmap(nullptr, length, PROT_READ, MAP_PRIVATE, fd, 0);
#endif
        close(fd); /* no longer needed */
        if(data==MAP_FAILED) {
            // Possibly check the errno value for ENOMEM, and report U_MEMORY_ALLOCATION_ERROR?
            return false;
        }

        pData->map = (char *)data + length;
        pData->pHeader=(const DataHeader *)data;
        pData->mapAddr = data;
        pData->length = length;
#if U_PLATFORM == U_PF_IPHONE || U_PLATFORM == U_PF_ANDROID
    // Apparently supported from Android 23 and higher:
    //   https://github.com/ggml-org/llama.cpp/pull/3631
    // Checking for the flag itself is safer than checking for __ANDROID_API__.
#   ifdef POSIX_MADV_RANDOM
        posix_madvise(data, length, POSIX_MADV_RANDOM);
#   endif
#endif
        return true;
    }

    U_CFUNC void
    uprv_unmapFile(UDataMemory *pData) {
        if(pData!=nullptr && pData->map!=nullptr) {
            size_t dataLen = (char *)pData->map - (char *)pData->mapAddr;
            if(munmap(pData->mapAddr, dataLen)==-1) {
            }
            pData->pHeader=nullptr;
            pData->map=nullptr;
            pData->mapAddr=nullptr;
        }
    }



#elif MAP_IMPLEMENTATION==MAP_STDIO
    /* copy of the filestrm.c/T_FileStream_size() implementation */
    static int32_t
    umap_fsize(FILE *f) {
        int32_t savedPos = ftell(f);
        int32_t size = 0;

        /*Changes by Bertrand A. D. doesn't affect the current position
        goes to the end of the file before ftell*/
        fseek(f, 0, SEEK_END);
        size = (int32_t)ftell(f);
        fseek(f, savedPos, SEEK_SET);
        return size;
    }

    U_CFUNC UBool
    uprv_mapFile(UDataMemory *pData, const char *path, UErrorCode *status) {
        FILE *file;
        int32_t fileLength;
        void *p;

        if (U_FAILURE(*status)) {
            return false;
        }

        UDataMemory_init(pData); /* Clear the output struct.        */
        /* open the input file */
        file=fopen(path, "rb");
        if(file==nullptr) {
            return false;
        }

        /* get the file length */
        fileLength=umap_fsize(file);
        if(ferror(file) || fileLength<=20) {
            fclose(file);
            return false;
        }

        /* allocate the memory to hold the file data */
        p=uprv_malloc(fileLength);
        if(p==nullptr) {
            fclose(file);
            *status = U_MEMORY_ALLOCATION_ERROR;
            return false;
        }

        /* read the file */
        if(fileLength!=fread(p, 1, fileLength, file)) {
            uprv_free(p);
            fclose(file);
            return false;
        }

        fclose(file);
        pData->map=p;
        pData->pHeader=(const DataHeader *)p;
        pData->mapAddr=p;
        pData->length = fileLength;
        return true;
    }

    U_CFUNC void
    uprv_unmapFile(UDataMemory *pData) {
        if(pData!=nullptr && pData->map!=nullptr) {
            uprv_free(pData->map);
            pData->map     = nullptr;
            pData->mapAddr = nullptr;
            pData->pHeader = nullptr;
        }
    }
#else
#   error MAP_IMPLEMENTATION is set incorrectly
#endif

Coverage Report

Created: 2026-06-13 06:44

Line	Count	Source
1		// © 2016 and later: Unicode, Inc. and others.
2		// License & terms of use: http://www.unicode.org/copyright.html
3		/*
4		******************************************************************************
5		*
6		* Copyright (C) 1999-2013, International Business Machines
7		* Corporation and others. All Rights Reserved.
8		*
9		******************************************************************************/
10
11
12		/*----------------------------------------------------------------------------
13		*
14		* Memory mapped file wrappers for use by the ICU Data Implementation
15		* All of the platform-specific implementation for mapping data files
16		* is here. The rest of the ICU Data implementation uses only the
17		* wrapper functions.
18		*
19		----------------------------------------------------------------------------/
20		/* Defines _XOPEN_SOURCE for access to POSIX functions.
21		* Must be before any other #includes. */
22		#include "uposixdefs.h"
23
24		#include "unicode/putil.h"
25		#include "unicode/ustring.h"
26		#include "udatamem.h"
27		#include "umapfile.h"
28
29		/* memory-mapping base definitions ------------------------------------------ */
30
31		#if MAP_IMPLEMENTATION==MAP_WIN32
32		#ifndef WIN32_LEAN_AND_MEAN
33		# define WIN32_LEAN_AND_MEAN
34		#endif
35		# define VC_EXTRALEAN
36		# define NOUSER
37		# define NOSERVICE
38		# define NOIME
39		# define NOMCX
40
41		# if U_PLATFORM_HAS_WINUWP_API == 1
42		// Some previous versions of the Windows 10 SDK don't expose various APIs for UWP applications
43		// to use, even though UWP apps are allowed to call and use them. Temporarily change the
44		// WINAPI family partition below to Desktop, so that function declarations are visible for UWP.
45		# include <winapifamily.h>
46		# if !(WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP \| WINAPI_PARTITION_SYSTEM))
47		# pragma push_macro("WINAPI_PARTITION_DESKTOP")
48		# undef WINAPI_PARTITION_DESKTOP
49		# define WINAPI_PARTITION_DESKTOP 1
50		# define CHANGED_WINAPI_PARTITION_DESKTOP_VALUE
51		# endif
52		# endif
53
54		# include <windows.h>
55
56		# if U_PLATFORM_HAS_WINUWP_API == 1 && defined(CHANGED_WINAPI_PARTITION_DESKTOP_VALUE)
57		# pragma pop_macro("WINAPI_PARTITION_DESKTOP")
58		# endif
59
60		# include "cmemory.h"
61
62		typedef HANDLE MemoryMap;
63
64		# define IS_MAP(map) ((map)!=nullptr)
65
66		#elif MAP_IMPLEMENTATION==MAP_POSIX
67		typedef size_t MemoryMap;
68
69		# define IS_MAP(map) ((map)!=0)
70
71		# include <unistd.h>
72		# include <sys/mman.h>
73		# include <sys/stat.h>
74		# include <fcntl.h>
75
76		# ifndef MAP_FAILED
77		# define MAP_FAILED ((void*)-1)
78		# endif
79		#elif MAP_IMPLEMENTATION==MAP_STDIO
80		# include <stdio.h>
81		# include "cmemory.h"
82
83		typedef void *MemoryMap;
84
85		# define IS_MAP(map) ((map)!=nullptr)
86		#endif
87
88		/----------------------------------------------------------------------------
89		* *
90		* Memory Mapped File support. Platform dependent implementation of *
91		* functions used by the rest of the implementation.*
92		* *
93		----------------------------------------------------------------------------/
94		#if MAP_IMPLEMENTATION==MAP_NONE
95		U_CFUNC UBool
96		uprv_mapFile(UDataMemory pData, const char path, UErrorCode *status) {
97		if (U_FAILURE(*status)) {
98		return false;
99		}
100		UDataMemory_init(pData); /* Clear the output struct. */
101		return false; /* no file access */
102		}
103
104		U_CFUNC void uprv_unmapFile(UDataMemory *pData) {
105		/* nothing to do */
106		}
107		#elif MAP_IMPLEMENTATION==MAP_WIN32
108		U_CFUNC UBool
109		uprv_mapFile(
110		UDataMemory pData, / Fill in with info on the result doing the mapping. */
111		/* Output only; any original contents are cleared. */
112		const char path, / File path to be opened/mapped. */
113		UErrorCode status / Error status, used to report out-of-memory errors. */
114		)
115		{
116		if (U_FAILURE(*status)) {
117		return false;
118		}
119
120		HANDLE map = nullptr;
121		HANDLE file = INVALID_HANDLE_VALUE;
122		DWORD fileLength = 0;
123
124		UDataMemory_init(pData); /* Clear the output struct. */
125
126		/* open the input file */
127		#if U_PLATFORM_HAS_WINUWP_API == 0
128		// Note: In the non-UWP code-path (ie: Win32), the value of the path variable might have come from
129		// the CRT 'getenv' function, and would be therefore be encoded in the default ANSI code page.
130		// This means that we can't call the *W version of API below, whereas in the UWP code-path
131		// there is no 'getenv' call, and thus the string will be only UTF-8/Invariant characters.
132		file=CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr,
133		OPEN_EXISTING,
134		FILE_ATTRIBUTE_NORMAL\|FILE_FLAG_RANDOM_ACCESS, nullptr);
135		#else
136		// Convert from UTF-8 string to UTF-16 string.
137		wchar_t utf16Path[MAX_PATH];
138		int32_t pathUtf16Len = 0;
139		u_strFromUTF8(reinterpret_cast<char16_t*>(utf16Path), static_cast<int32_t>(UPRV_LENGTHOF(utf16Path)), &pathUtf16Len, path, -1, status);
140
141		if (U_FAILURE(*status)) {
142		return false;
143		}
144		if (*status == U_STRING_NOT_TERMINATED_WARNING) {
145		// Report back an error instead of a warning.
146		*status = U_BUFFER_OVERFLOW_ERROR;
147		return false;
148		}
149
150		file = CreateFileW(utf16Path, GENERIC_READ, FILE_SHARE_READ, nullptr,
151		OPEN_EXISTING,
152		FILE_ATTRIBUTE_NORMAL \| FILE_FLAG_RANDOM_ACCESS, nullptr);
153		#endif
154		if (file == INVALID_HANDLE_VALUE) {
155		// If we failed to open the file due to an out-of-memory error, then we want
156		// to report that error back to the caller.
157		if (HRESULT_FROM_WIN32(GetLastError()) == E_OUTOFMEMORY) {
158		*status = U_MEMORY_ALLOCATION_ERROR;
159		}
160		return false;
161		}
162
163		fileLength = GetFileSize(file, nullptr);
164
165		// Note: We use nullptr/nullptr for lpAttributes parameter below.
166		// This means our handle cannot be inherited and we will get the default security descriptor.
167		/* create an unnamed Windows file-mapping object for the specified file */
168		map = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr);
169
170		CloseHandle(file);
171		if (map == nullptr) {
172		// If we failed to create the mapping due to an out-of-memory error, then
173		// we want to report that error back to the caller.
174		if (HRESULT_FROM_WIN32(GetLastError()) == E_OUTOFMEMORY) {
175		*status = U_MEMORY_ALLOCATION_ERROR;
176		}
177		return false;
178		}
179
180		/* map a view of the file into our address space */
181		pData->pHeader = reinterpret_cast<const DataHeader *>(MapViewOfFile(map, FILE_MAP_READ, 0, 0, 0));
182		if (pData->pHeader == nullptr) {
183		CloseHandle(map);
184		return false;
185		}
186		pData->map = map;
187		pData->length = fileLength;
188
189		return true;
190		}
191
192		U_CFUNC void
193		uprv_unmapFile(UDataMemory *pData) {
194		if (pData != nullptr && pData->map != nullptr) {
195		UnmapViewOfFile(pData->pHeader);
196		CloseHandle(pData->map);
197		pData->pHeader = nullptr;
198		pData->map = nullptr;
199		}
200		}
201
202
203
204		#elif MAP_IMPLEMENTATION==MAP_POSIX
205		U_CFUNC UBool
206	0	uprv_mapFile(UDataMemory pData, const char path, UErrorCode *status) {
207	0	int fd;
208	0	int length;
209	0	struct stat mystat;
210	0	void *data;
211
212	0	if (U_FAILURE(*status)) {
213	0	return false;
214	0	}
215
216	0	UDataMemory_init(pData); /* Clear the output struct. */
217
218		/* determine the length of the file */
219	0	if(stat(path, &mystat)!=0 \|\| mystat.st_size<=0) {
220	0	return false;
221	0	}
222	0	length=mystat.st_size;
223
224		/* open the file */
225	0	fd=open(path, O_RDONLY);
226	0	if(fd==-1) {
227	0	return false;
228	0	}
229
230		/* get a view of the mapping */
231	0	#if U_PLATFORM != U_PF_HPUX
232	0	data=mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, 0);
233		#else
234		data=mmap(nullptr, length, PROT_READ, MAP_PRIVATE, fd, 0);
235		#endif
236	0	close(fd); /* no longer needed */
237	0	if(data==MAP_FAILED) {
238		// Possibly check the errno value for ENOMEM, and report U_MEMORY_ALLOCATION_ERROR?
239	0	return false;
240	0	}
241
242	0	pData->map = (char *)data + length;
243	0	pData->pHeader=(const DataHeader *)data;
244	0	pData->mapAddr = data;
245	0	pData->length = length;
246		#if U_PLATFORM == U_PF_IPHONE \|\| U_PLATFORM == U_PF_ANDROID
247		// Apparently supported from Android 23 and higher:
248		// https://github.com/ggml-org/llama.cpp/pull/3631
249		// Checking for the flag itself is safer than checking for __ANDROID_API__.
250		# ifdef POSIX_MADV_RANDOM
251		posix_madvise(data, length, POSIX_MADV_RANDOM);
252		# endif
253		#endif
254	0	return true;
255	0	}
256
257		U_CFUNC void
258	0	uprv_unmapFile(UDataMemory *pData) {
259	0	if(pData!=nullptr && pData->map!=nullptr) {
260	0	size_t dataLen = (char )pData->map - (char )pData->mapAddr;
261	0	if(munmap(pData->mapAddr, dataLen)==-1) {
262	0	}
263	0	pData->pHeader=nullptr;
264	0	pData->map=nullptr;
265	0	pData->mapAddr=nullptr;
266	0	}
267	0	}
268
269
270
271		#elif MAP_IMPLEMENTATION==MAP_STDIO
272		/* copy of the filestrm.c/T_FileStream_size() implementation */
273		static int32_t
274		umap_fsize(FILE *f) {
275		int32_t savedPos = ftell(f);
276		int32_t size = 0;
277
278		/*Changes by Bertrand A. D. doesn't affect the current position
279		goes to the end of the file before ftell*/
280		fseek(f, 0, SEEK_END);
281		size = (int32_t)ftell(f);
282		fseek(f, savedPos, SEEK_SET);
283		return size;
284		}
285
286		U_CFUNC UBool
287		uprv_mapFile(UDataMemory pData, const char path, UErrorCode *status) {
288		FILE *file;
289		int32_t fileLength;
290		void *p;
291
292		if (U_FAILURE(*status)) {
293		return false;
294		}
295
296		UDataMemory_init(pData); /* Clear the output struct. */
297		/* open the input file */
298		file=fopen(path, "rb");
299		if(file==nullptr) {
300		return false;
301		}
302
303		/* get the file length */
304		fileLength=umap_fsize(file);
305		if(ferror(file) \|\| fileLength<=20) {
306		fclose(file);
307		return false;
308		}
309
310		/* allocate the memory to hold the file data */
311		p=uprv_malloc(fileLength);
312		if(p==nullptr) {
313		fclose(file);
314		*status = U_MEMORY_ALLOCATION_ERROR;
315		return false;
316		}
317
318		/* read the file */
319		if(fileLength!=fread(p, 1, fileLength, file)) {
320		uprv_free(p);
321		fclose(file);
322		return false;
323		}
324
325		fclose(file);
326		pData->map=p;
327		pData->pHeader=(const DataHeader *)p;
328		pData->mapAddr=p;
329		pData->length = fileLength;
330		return true;
331		}
332
333		U_CFUNC void
334		uprv_unmapFile(UDataMemory *pData) {
335		if(pData!=nullptr && pData->map!=nullptr) {
336		uprv_free(pData->map);
337		pData->map = nullptr;
338		pData->mapAddr = nullptr;
339		pData->pHeader = nullptr;
340		}
341		}
342		#else
343		# error MAP_IMPLEMENTATION is set incorrectly
344		#endif