Coverage Report

Created: 2025-11-16 06:25

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/gdal/port/cpl_vsisimple.cpp
Line
Count
Source
1
/******************************************************************************
2
 *
3
 * Project:  Common Portability Library
4
 * Purpose:  Simple implementation of POSIX VSI functions.
5
 * Author:   Frank Warmerdam, warmerdam@pobox.com
6
 *
7
 ******************************************************************************
8
 * Copyright (c) 1998, Frank Warmerdam
9
 * Copyright (c) 2008-2012, Even Rouault <even dot rouault at spatialys.com>
10
 *
11
 * SPDX-License-Identifier: MIT
12
 ****************************************************************************
13
 *
14
 * NB: Note that in wrappers we are always saving the error state (errno
15
 * variable) to avoid side effects during debug prints or other possible
16
 * standard function calls (error states will be overwritten after such
17
 * a call).
18
 *
19
 ****************************************************************************/
20
21
// Must be done before including cpl_port.h
22
// We need __MSVCRT_VERSION__ >= 0x0700 to have "_aligned_malloc"
23
// Latest versions of mingw32 define it, but with older ones,
24
// we need to define it manually.
25
#if defined(__MINGW32__)
26
#include <windows.h>
27
#ifndef __MSVCRT_VERSION__
28
#define __MSVCRT_VERSION__ 0x0700
29
#endif
30
#endif
31
32
#include "cpl_port.h"
33
#include "cpl_vsi.h"
34
35
#include <algorithm>
36
#include <cerrno>
37
#include <cstdarg>
38
#include <cstddef>
39
#include <cstdio>
40
#include <cstdlib>
41
#include <cstring>
42
#include <ctime>
43
#include <limits>
44
#include <sys/stat.h>
45
#if HAVE_GETRLIMIT
46
#include <sys/time.h>
47
#include <sys/resource.h>
48
#endif
49
50
#include "cpl_config.h"
51
#include "cpl_error.h"
52
#include "cpl_string.h"
53
54
#ifdef _WIN32
55
#include <malloc.h>  // For _aligned_malloc
56
#endif
57
58
// Uncomment to check consistent usage of VSIMalloc(), VSIRealloc(),
59
// VSICalloc(), VSIFree(), VSIStrdup().
60
// #define DEBUG_VSIMALLOC
61
62
// Uncomment to compute memory usage statistics.
63
// DEBUG_VSIMALLOC must also be defined.
64
// #define DEBUG_VSIMALLOC_STATS
65
66
// Highly experimental, and likely buggy. Do not use, except for fixing it!
67
// DEBUG_VSIMALLOC must also be defined.
68
// #define DEBUG_VSIMALLOC_MPROTECT
69
70
#ifdef DEBUG_VSIMALLOC_MPROTECT
71
#include <sys/mman.h>
72
#endif
73
74
// Uncomment to print every memory allocation or deallocation.
75
// DEBUG_VSIMALLOC must also be defined.
76
// #define DEBUG_VSIMALLOC_VERBOSE
77
78
// Number of bytes of the malloc/calloc/free that triggers a debug trace.
79
// Can be 0 for all allocs.
80
#define THRESHOLD_PRINT 10000
81
82
// Uncomment to print GDAL block cache use.
83
// Only used if DEBUG_VSIMALLOC_VERBOSE is enabled.
84
// #define DEBUG_BLOCK_CACHE_USE
85
86
#ifdef DEBUG_BLOCK_CACHE_USE
87
extern "C" GIntBig CPL_DLL CPL_STDCALL GDALGetCacheUsed64(void);
88
#endif
89
90
/* Unix or Windows NT/2000/XP */
91
#if !defined(_WIN32)
92
#include <unistd.h>
93
#else
94
#include <io.h>
95
#include <fcntl.h>
96
#include <direct.h>
97
#endif
98
99
/************************************************************************/
100
/*                              VSIFOpen()                              */
101
/************************************************************************/
102
103
FILE *VSIFOpen(const char *pszFilename, const char *pszAccess)
104
105
0
{
106
#if defined(_WIN32)
107
    FILE *fp = nullptr;
108
    if (CPLTestBool(CPLGetConfigOption("GDAL_FILENAME_IS_UTF8", "YES")))
109
    {
110
        wchar_t *pwszFilename =
111
            CPLRecodeToWChar(pszFilename, CPL_ENC_UTF8, CPL_ENC_UCS2);
112
        wchar_t *pwszAccess =
113
            CPLRecodeToWChar(pszAccess, CPL_ENC_UTF8, CPL_ENC_UCS2);
114
115
        fp = _wfopen(pwszFilename, pwszAccess);
116
117
        CPLFree(pwszFilename);
118
        CPLFree(pwszAccess);
119
    }
120
    else
121
    {
122
        // Are the casts really necessary?
123
        fp = fopen(const_cast<char *>(pszFilename),
124
                   const_cast<char *>(pszAccess));
125
    }
126
#else
127
0
    FILE *fp = fopen(pszFilename, pszAccess);
128
0
#endif
129
130
#ifdef VSI_DEBUG
131
    // Capture the error from fopen to avoid being overwritten by errors
132
    // from VSIDebug3.
133
    const int nError = errno;
134
    VSIDebug3("VSIFOpen(%s,%s) = %p", pszFilename, pszAccess, fp);
135
    errno = nError;
136
#endif
137
138
0
    return fp;
139
0
}
140
141
/************************************************************************/
142
/*                             VSIFClose()                              */
143
/************************************************************************/
144
145
int VSIFClose(FILE *fp)
146
147
0
{
148
0
    VSIDebug1("VSIClose(%p)", fp);
149
150
0
    return fclose(fp);
151
0
}
152
153
/************************************************************************/
154
/*                              VSIFSeek()                              */
155
/************************************************************************/
156
157
int VSIFSeek(FILE *fp, long nOffset, int nWhence)
158
159
0
{
160
0
#ifdef DEBUG
161
    // To workaround Coverity strange warning about potential negative seek
162
    // CID 1340084 when called from dgnwrite.cpp.
163
0
    if (nWhence == SEEK_SET && nOffset < 0)
164
0
        return -1;
165
0
#endif
166
0
    int nResult = fseek(fp, nOffset, nWhence);
167
168
#ifdef VSI_DEBUG
169
    // Capture the error from fseek to avoid being overwritten by errors
170
    // from VSIDebug.
171
    const int nError = errno;
172
173
    if (nWhence == SEEK_SET)
174
    {
175
        VSIDebug3("VSIFSeek(%p,%ld,SEEK_SET) = %d", fp, nOffset, nResult);
176
    }
177
    else if (nWhence == SEEK_END)
178
    {
179
        VSIDebug3("VSIFSeek(%p,%ld,SEEK_END) = %d", fp, nOffset, nResult);
180
    }
181
    else if (nWhence == SEEK_CUR)
182
    {
183
        VSIDebug3("VSIFSeek(%p,%ld,SEEK_CUR) = %d", fp, nOffset, nResult);
184
    }
185
    else
186
    {
187
        VSIDebug4("VSIFSeek(%p,%ld,%d-Unknown) = %d", fp, nOffset, nWhence,
188
                  nResult);
189
    }
190
191
    errno = nError;
192
#endif
193
194
0
    return nResult;
195
0
}
196
197
/************************************************************************/
198
/*                              VSIFTell()                              */
199
/************************************************************************/
200
201
long VSIFTell(FILE *fp)
202
203
0
{
204
0
    const long nOffset = ftell(fp);
205
206
#ifdef VSI_DEBUG
207
    // Capture the error from ftell to avoid being overwritten by errors
208
    // from VSIDebug.
209
    const int nError = errno;
210
    VSIDebug2("VSIFTell(%p) = %ld", fp, nOffset);
211
    errno = nError;
212
#endif
213
214
0
    return nOffset;
215
0
}
216
217
/************************************************************************/
218
/*                             VSIRewind()                              */
219
/************************************************************************/
220
221
void VSIRewind(FILE *fp)
222
223
0
{
224
0
    VSIDebug1("VSIRewind(%p)", fp);
225
0
    rewind(fp);
226
#ifdef VSI_DEBUG
227
    // Capture the error rewind ftell to avoid being overwritten by errors
228
    // from VSIDebug.
229
    const int nError = errno;
230
    VSIDebug2("VSIRewind(%p) errno = %d", fp, nError);
231
    errno = nError;
232
#endif
233
0
}
234
235
/************************************************************************/
236
/*                              VSIFRead()                              */
237
/************************************************************************/
238
239
size_t VSIFRead(void *pBuffer, size_t nSize, size_t nCount, FILE *fp)
240
241
0
{
242
0
    const size_t nResult = fread(pBuffer, nSize, nCount, fp);
243
244
#ifdef VSI_DEBUG
245
    // Capture the error from fread to avoid being overwritten by errors
246
    // from VSIDebug.
247
    const int nError = errno;
248
    VSIDebug4("VSIFRead(%p,%ld,%ld) = %ld", fp, static_cast<long>(nSize),
249
              static_cast<long>(nCount), static_cast<long>(nResult));
250
    errno = nError;
251
#endif
252
253
0
    return nResult;
254
0
}
255
256
/************************************************************************/
257
/*                             VSIFWrite()                              */
258
/************************************************************************/
259
260
size_t VSIFWrite(const void *pBuffer, size_t nSize, size_t nCount, FILE *fp)
261
262
0
{
263
0
    const size_t nResult = fwrite(pBuffer, nSize, nCount, fp);
264
265
#ifdef VSI_DEBUG
266
    // Capture the error from fwrite to avoid being overwritten by errors
267
    // from VSIDebug.
268
    const int nError = errno;
269
    VSIDebug4("VSIFWrite(%p,%ld,%ld) = %ld", fp, static_cast<long>(nSize),
270
              static_cast<long>(nCount), static_cast<long>(nResult));
271
    errno = nError;
272
#endif
273
274
0
    return nResult;
275
0
}
276
277
/************************************************************************/
278
/*                             VSIFFlush()                              */
279
/************************************************************************/
280
281
void VSIFFlush(FILE *fp)
282
283
0
{
284
#ifdef VSI_DEBUG
285
    VSIDebug1("VSIFFlush(%p)", fp);
286
    const int result =
287
#endif
288
0
        fflush(fp);
289
290
#ifdef VSI_DEBUG
291
    // Capture the error rewind ftell to avoid being overwritten by errors
292
    // from VSIDebug.
293
    const int nError = errno;
294
    VSIDebug2("VSIRewind(%p) errno = %d", fp, nError);
295
    if (result != 0)
296
    {
297
        CPLError(CE_Failure, CPLE_FileIO, "Flush failed.  errno = %d", nError);
298
    }
299
    errno = nError;
300
#endif
301
0
}
302
303
/************************************************************************/
304
/*                              VSIFGets()                              */
305
/************************************************************************/
306
307
char *VSIFGets(char *pszBuffer, int nBufferSize, FILE *fp)
308
309
0
{
310
0
    return fgets(pszBuffer, nBufferSize, fp);
311
0
}
312
313
/************************************************************************/
314
/*                              VSIFGetc()                              */
315
/************************************************************************/
316
317
int VSIFGetc(FILE *fp)
318
319
0
{
320
0
    return fgetc(fp);
321
0
}
322
323
/************************************************************************/
324
/*                             VSIUngetc()                              */
325
/************************************************************************/
326
327
int VSIUngetc(int c, FILE *fp)
328
329
0
{
330
0
    return ungetc(c, fp);
331
0
}
332
333
/************************************************************************/
334
/*                             VSIFPrintf()                             */
335
/*                                                                      */
336
/*      This is a little more complicated than just calling             */
337
/*      fprintf() because of the variable arguments.  Instead we        */
338
/*      have to use vfprintf().                                         */
339
/************************************************************************/
340
341
int VSIFPrintf(FILE *fp, CPL_FORMAT_STRING(const char *pszFormat), ...)
342
343
0
{
344
0
    va_list args;
345
346
0
    va_start(args, pszFormat);
347
0
    const int nReturn = vfprintf(fp, pszFormat, args);
348
0
    va_end(args);
349
350
0
    return nReturn;
351
0
}
352
353
/************************************************************************/
354
/*                              VSIFEof()                               */
355
/************************************************************************/
356
357
int VSIFEof(FILE *fp)
358
359
0
{
360
0
    return feof(fp);
361
0
}
362
363
/************************************************************************/
364
/*                              VSIFPuts()                              */
365
/************************************************************************/
366
367
int VSIFPuts(const char *pszString, FILE *fp)
368
369
0
{
370
0
    return fputs(pszString, fp);
371
0
}
372
373
/************************************************************************/
374
/*                              VSIFPutc()                              */
375
/************************************************************************/
376
377
int VSIFPutc(int nChar, FILE *fp)
378
379
0
{
380
0
    return fputc(nChar, fp);
381
0
}
382
383
#ifdef DEBUG_VSIMALLOC_STATS
384
#include "cpl_multiproc.h"
385
386
static CPLMutex *hMemStatMutex = nullptr;
387
static size_t nCurrentTotalAllocs = 0;
388
static size_t nMaxTotalAllocs = 0;
389
static GUIntBig nVSIMallocs = 0;
390
static GUIntBig nVSICallocs = 0;
391
static GUIntBig nVSIReallocs = 0;
392
static GUIntBig nVSIFrees = 0;
393
394
/************************************************************************/
395
/*                         VSIShowMemStats()                            */
396
/************************************************************************/
397
398
void VSIShowMemStats();
399
400
void VSIShowMemStats()
401
{
402
    char *pszShowMemStats = getenv("CPL_SHOW_MEM_STATS");
403
    if (pszShowMemStats == nullptr || pszShowMemStats[0] == '\0')
404
        return;
405
    printf("Current VSI memory usage        : " CPL_FRMT_GUIB " bytes\n", /*ok*/
406
           static_cast<GUIntBig>(nCurrentTotalAllocs));
407
    printf("Maximum VSI memory usage        : " CPL_FRMT_GUIB " bytes\n", /*ok*/
408
           static_cast<GUIntBig>(nMaxTotalAllocs));
409
    printf("Number of calls to VSIMalloc()  : " CPL_FRMT_GUIB "\n", /*ok*/
410
           nVSIMallocs);
411
    printf("Number of calls to VSICalloc()  : " CPL_FRMT_GUIB "\n", /*ok*/
412
           nVSICallocs);
413
    printf("Number of calls to VSIRealloc() : " CPL_FRMT_GUIB "\n", /*ok*/
414
           nVSIReallocs);
415
    printf("Number of calls to VSIFree()    : " CPL_FRMT_GUIB "\n", /*ok*/
416
           nVSIFrees);
417
    printf("VSIMalloc + VSICalloc - VSIFree : " CPL_FRMT_GUIB "\n", /*ok*/
418
           nVSIMallocs + nVSICallocs - nVSIFrees);
419
}
420
#endif
421
422
#ifdef DEBUG_VSIMALLOC
423
static GIntBig nMaxPeakAllocSize = -1;
424
static GIntBig nMaxCumulAllocSize = -1;
425
#endif
426
427
/************************************************************************/
428
/*                             VSICalloc()                              */
429
/************************************************************************/
430
431
#ifndef DEBUG_VSIMALLOC
432
433
/** Analog of calloc(). Use VSIFree() to free */
434
void *VSICalloc(size_t nCount, size_t nSize)
435
15.5k
{
436
    // cppcheck-suppress invalidFunctionArg
437
15.5k
    return calloc(nCount, nSize);
438
15.5k
}
439
440
#else  // DEBUG_VSIMALLOC
441
442
void *VSICalloc(size_t nCount, size_t nSize)
443
{
444
    size_t nMul = nCount * nSize;
445
    if (nCount != 0 && nMul / nCount != nSize)
446
    {
447
        fprintf(stderr, "Overflow in VSICalloc(%d, %d)\n", /*ok*/
448
                static_cast<int>(nCount), static_cast<int>(nSize));
449
        return nullptr;
450
    }
451
    if (nMaxPeakAllocSize < 0)
452
    {
453
        char *pszMaxPeakAllocSize = getenv("CPL_MAX_PEAK_ALLOC_SIZE");
454
        nMaxPeakAllocSize = pszMaxPeakAllocSize ? atoi(pszMaxPeakAllocSize) : 0;
455
        char *pszMaxCumulAllocSize = getenv("CPL_MAX_CUMUL_ALLOC_SIZE");
456
        nMaxCumulAllocSize =
457
            pszMaxCumulAllocSize ? atoi(pszMaxCumulAllocSize) : 0;
458
    }
459
    if (nMaxPeakAllocSize > 0 && static_cast<GIntBig>(nMul) > nMaxPeakAllocSize)
460
        return nullptr;
461
#ifdef DEBUG_VSIMALLOC_STATS
462
    if (nMaxCumulAllocSize > 0 &&
463
        static_cast<GIntBig>(nCurrentTotalAllocs) + static_cast<GIntBig>(nMul) >
464
            nMaxCumulAllocSize)
465
        return nullptr;
466
#endif
467
468
#ifdef DEBUG_VSIMALLOC_MPROTECT
469
    char *ptr = nullptr;
470
    const size_t nPageSize = getpagesize();
471
    const size_t nRequestedSize =
472
        (3 * sizeof(void *) + nMul + nPageSize - 1) & ~(nPageSize - 1);
473
    if (nRequestedSize < nMul)
474
        return nullptr;
475
    posix_memalign((void **)&ptr, nPageSize, nRequestedSize);
476
    if (ptr == nullptr)
477
        return nullptr;
478
    memset(ptr + 2 * sizeof(void *), 0, nMul);
479
#else
480
    const size_t nRequestedSize = 3 * sizeof(void *) + nMul;
481
    if (nRequestedSize < nMul)
482
        return nullptr;
483
    char *ptr = static_cast<char *>(calloc(1, nRequestedSize));
484
    if (ptr == nullptr)
485
        return nullptr;
486
#endif
487
488
    ptr[0] = 'V';
489
    ptr[1] = 'S';
490
    ptr[2] = 'I';
491
    ptr[3] = 'M';
492
    // cppcheck-suppress pointerSize
493
    memcpy(ptr + sizeof(void *), &nMul, sizeof(void *));
494
    ptr[2 * sizeof(void *) + nMul + 0] = 'E';
495
    ptr[2 * sizeof(void *) + nMul + 1] = 'V';
496
    ptr[2 * sizeof(void *) + nMul + 2] = 'S';
497
    ptr[2 * sizeof(void *) + nMul + 3] = 'I';
498
#if defined(DEBUG_VSIMALLOC_STATS) || defined(DEBUG_VSIMALLOC_VERBOSE)
499
    {
500
        CPLMutexHolderD(&hMemStatMutex);
501
#ifdef DEBUG_VSIMALLOC_VERBOSE
502
        if (nMul > THRESHOLD_PRINT)
503
        {
504
            fprintf(stderr, /*ok*/
505
                    "Thread[%p] VSICalloc(%d,%d) = %p"
506
#ifdef DEBUG_VSIMALLOC_STATS
507
                    ", current_cumul = " CPL_FRMT_GUIB
508
#ifdef DEBUG_BLOCK_CACHE_USE
509
                    ", block_cache_used = " CPL_FRMT_GIB
510
#endif
511
                    ", mal+cal-free = %d"
512
#endif
513
                    "\n",
514
                    (void *)CPLGetPID(), static_cast<int>(nCount),
515
                    static_cast<int>(nSize), ptr + 2 * sizeof(void *)
516
#ifdef DEBUG_VSIMALLOC_STATS
517
                                                 ,
518
                    static_cast<GUIntBig>(nCurrentTotalAllocs + nMul)
519
#ifdef DEBUG_BLOCK_CACHE_USE
520
                        ,
521
                    GDALGetCacheUsed64()
522
#endif
523
                        ,
524
                    static_cast<int>(nVSIMallocs + nVSICallocs - nVSIFrees)
525
#endif
526
            );
527
        }
528
#endif
529
#ifdef DEBUG_VSIMALLOC_STATS
530
        nVSICallocs++;
531
        if (nMaxTotalAllocs == 0)
532
            atexit(VSIShowMemStats);
533
        nCurrentTotalAllocs += nMul;
534
        if (nCurrentTotalAllocs > nMaxTotalAllocs)
535
            nMaxTotalAllocs = nCurrentTotalAllocs;
536
#endif
537
    }
538
#endif
539
    // cppcheck-suppress memleak
540
    return ptr + 2 * sizeof(void *);
541
}
542
#endif  // DEBUG_VSIMALLOC
543
544
/************************************************************************/
545
/*                             VSIMalloc()                              */
546
/************************************************************************/
547
548
#ifndef DEBUG_VSIMALLOC
549
/** Analog of malloc(). Use VSIFree() to free */
550
void *VSIMalloc(size_t nSize)
551
552
203k
{
553
203k
    return malloc(nSize);
554
203k
}
555
556
#else  // DEBUG_VSIMALLOC
557
558
void *VSIMalloc(size_t nSize)
559
560
{
561
    if (nMaxPeakAllocSize < 0)
562
    {
563
        char *pszMaxPeakAllocSize = getenv("CPL_MAX_PEAK_ALLOC_SIZE");
564
        nMaxPeakAllocSize = pszMaxPeakAllocSize ? atoi(pszMaxPeakAllocSize) : 0;
565
        char *pszMaxCumulAllocSize = getenv("CPL_MAX_CUMUL_ALLOC_SIZE");
566
        nMaxCumulAllocSize =
567
            pszMaxCumulAllocSize ? atoi(pszMaxCumulAllocSize) : 0;
568
    }
569
    if (nMaxPeakAllocSize > 0 &&
570
        static_cast<GIntBig>(nSize) > nMaxPeakAllocSize)
571
        return nullptr;
572
#ifdef DEBUG_VSIMALLOC_STATS
573
    if (nMaxCumulAllocSize > 0 && static_cast<GIntBig>(nCurrentTotalAllocs) +
574
                                          static_cast<GIntBig>(nSize) >
575
                                      nMaxCumulAllocSize)
576
        return nullptr;
577
#endif  // DEBUG_VSIMALLOC_STATS
578
579
#ifdef DEBUG_VSIMALLOC_MPROTECT
580
    char *ptr = nullptr;
581
    const size_t nPageSize = getpagesize();
582
    const size_t nRequestedSize =
583
        (3 * sizeof(void *) + nSize + nPageSize - 1) & ~(nPageSize - 1);
584
    if (nRequestedSize < nSize)
585
        return nullptr;
586
    posix_memalign((void **)&ptr, nPageSize, nRequestedSize);
587
#else
588
    const size_t nRequestedSize = 3 * sizeof(void *) + nSize;
589
    if (nRequestedSize < nSize)
590
        return nullptr;
591
    char *ptr = static_cast<char *>(malloc(nRequestedSize));
592
#endif  // DEBUG_VSIMALLOC_MPROTECT
593
    if (ptr == nullptr)
594
        return nullptr;
595
    ptr[0] = 'V';
596
    ptr[1] = 'S';
597
    ptr[2] = 'I';
598
    ptr[3] = 'M';
599
    // cppcheck-suppress pointerSize
600
    memcpy(ptr + sizeof(void *), &nSize, sizeof(void *));
601
    ptr[2 * sizeof(void *) + nSize + 0] = 'E';
602
    ptr[2 * sizeof(void *) + nSize + 1] = 'V';
603
    ptr[2 * sizeof(void *) + nSize + 2] = 'S';
604
    ptr[2 * sizeof(void *) + nSize + 3] = 'I';
605
#if defined(DEBUG_VSIMALLOC_STATS) || defined(DEBUG_VSIMALLOC_VERBOSE)
606
    {
607
        CPLMutexHolderD(&hMemStatMutex);
608
#ifdef DEBUG_VSIMALLOC_VERBOSE
609
        if (nSize > THRESHOLD_PRINT)
610
        {
611
            fprintf(stderr, /*ok*/
612
                    "Thread[%p] VSIMalloc(%d) = %p"
613
#ifdef DEBUG_VSIMALLOC_STATS
614
                    ", current_cumul = " CPL_FRMT_GUIB
615
#ifdef DEBUG_BLOCK_CACHE_USE
616
                    ", block_cache_used = " CPL_FRMT_GIB
617
#endif
618
                    ", mal+cal-free = %d"
619
#endif
620
                    "\n",
621
                    (void *)CPLGetPID(), static_cast<int>(nSize),
622
                    ptr + 2 * sizeof(void *)
623
#ifdef DEBUG_VSIMALLOC_STATS
624
                        ,
625
                    static_cast<GUIntBig>(nCurrentTotalAllocs + nSize)
626
#ifdef DEBUG_BLOCK_CACHE_USE
627
                        ,
628
                    GDALGetCacheUsed64()
629
#endif
630
                        ,
631
                    static_cast<int>(nVSIMallocs + nVSICallocs - nVSIFrees)
632
#endif
633
            );
634
        }
635
#endif  // DEBUG_VSIMALLOC_VERBOSE
636
#ifdef DEBUG_VSIMALLOC_STATS
637
        nVSIMallocs++;
638
        if (nMaxTotalAllocs == 0)
639
            atexit(VSIShowMemStats);
640
        nCurrentTotalAllocs += nSize;
641
        if (nCurrentTotalAllocs > nMaxTotalAllocs)
642
            nMaxTotalAllocs = nCurrentTotalAllocs;
643
#endif  // DEBUG_VSIMALLOC_STATS
644
    }
645
#endif  // DEBUG_VSIMALLOC_STATS || DEBUG_VSIMALLOC_VERBOSE
646
    // cppcheck-suppress memleak
647
    return ptr + 2 * sizeof(void *);
648
}
649
650
static void VSICheckMarkerBegin(char *ptr)
651
{
652
    if (memcmp(ptr, "VSIM", 4) != 0)
653
    {
654
        CPLError(CE_Fatal, CPLE_AppDefined,
655
                 "Inconsistent use of VSI memory allocation primitives "
656
                 "for %p : %c%c%c%c",
657
                 ptr, ptr[0], ptr[1], ptr[2], ptr[3]);
658
    }
659
}
660
661
static void VSICheckMarkerEnd(char *ptr, size_t nEnd)
662
{
663
    if (memcmp(ptr + nEnd, "EVSI", 4) != 0)
664
    {
665
        CPLError(CE_Fatal, CPLE_AppDefined,
666
                 "Memory has been written after the end of %p", ptr);
667
    }
668
}
669
670
#endif  // DEBUG_VSIMALLOC
671
672
/************************************************************************/
673
/*                             VSIRealloc()                             */
674
/************************************************************************/
675
676
/** Analog of realloc(). Use VSIFree() to free.
677
 *
678
 * If the pointer is NULL, VSIRealloc is equivalent to VSIMalloc.
679
 */
680
void *VSIRealloc(void *pData, size_t nNewSize)
681
682
43.3k
{
683
#ifdef DEBUG_VSIMALLOC
684
    if (pData == nullptr)
685
        return VSIMalloc(nNewSize);
686
687
    char *ptr = ((char *)pData) - 2 * sizeof(void *);
688
    VSICheckMarkerBegin(ptr);
689
690
    size_t nOldSize = 0;
691
    // cppcheck-suppress pointerSize
692
    memcpy(&nOldSize, ptr + sizeof(void *), sizeof(void *));
693
    VSICheckMarkerEnd(ptr, 2 * sizeof(void *) + nOldSize);
694
695
    if (nMaxPeakAllocSize < 0)
696
    {
697
        char *pszMaxPeakAllocSize = getenv("CPL_MAX_PEAK_ALLOC_SIZE");
698
        nMaxPeakAllocSize = pszMaxPeakAllocSize ? atoi(pszMaxPeakAllocSize) : 0;
699
    }
700
    if (nMaxPeakAllocSize > 0 &&
701
        static_cast<GIntBig>(nNewSize) > nMaxPeakAllocSize)
702
        return nullptr;
703
#ifdef DEBUG_VSIMALLOC_STATS
704
    if (nMaxCumulAllocSize > 0 && static_cast<GIntBig>(nCurrentTotalAllocs) +
705
                                          static_cast<GIntBig>(nNewSize) -
706
                                          static_cast<GIntBig>(nOldSize) >
707
                                      nMaxCumulAllocSize)
708
        return nullptr;
709
#endif
710
711
    ptr[2 * sizeof(void *) + nOldSize + 0] = 'I';
712
    ptr[2 * sizeof(void *) + nOldSize + 1] = 'S';
713
    ptr[2 * sizeof(void *) + nOldSize + 2] = 'V';
714
    ptr[2 * sizeof(void *) + nOldSize + 3] = 'E';
715
716
#ifdef DEBUG_VSIMALLOC_MPROTECT
717
    char *newptr = nullptr;
718
    const size_t nPageSize = getpagesize();
719
    const size_t nRequestedSize =
720
        (nNewSize + 3 * sizeof(void *) + nPageSize - 1) & ~(nPageSize - 1);
721
    if (nRequestedSize < nNewSize)
722
    {
723
        ptr[2 * sizeof(void *) + nOldSize + 0] = 'E';
724
        ptr[2 * sizeof(void *) + nOldSize + 1] = 'V';
725
        ptr[2 * sizeof(void *) + nOldSize + 2] = 'S';
726
        ptr[2 * sizeof(void *) + nOldSize + 3] = 'I';
727
        return nullptr;
728
    }
729
    posix_memalign((void **)&newptr, nPageSize, nRequestedSize);
730
    if (newptr == nullptr)
731
    {
732
        ptr[2 * sizeof(void *) + nOldSize + 0] = 'E';
733
        ptr[2 * sizeof(void *) + nOldSize + 1] = 'V';
734
        ptr[2 * sizeof(void *) + nOldSize + 2] = 'S';
735
        ptr[2 * sizeof(void *) + nOldSize + 3] = 'I';
736
        return nullptr;
737
    }
738
    memcpy(newptr + 2 * sizeof(void *), pData, nOldSize);
739
    ptr[0] = 'M';
740
    ptr[1] = 'I';
741
    ptr[2] = 'S';
742
    ptr[3] = 'V';
743
    free(ptr);
744
    newptr[0] = 'V';
745
    newptr[1] = 'S';
746
    newptr[2] = 'I';
747
    newptr[3] = 'M';
748
#else
749
    const size_t nRequestedSize = 3 * sizeof(void *) + nNewSize;
750
    if (nRequestedSize < nNewSize)
751
    {
752
        ptr[2 * sizeof(void *) + nOldSize + 0] = 'E';
753
        ptr[2 * sizeof(void *) + nOldSize + 1] = 'V';
754
        ptr[2 * sizeof(void *) + nOldSize + 2] = 'S';
755
        ptr[2 * sizeof(void *) + nOldSize + 3] = 'I';
756
        return nullptr;
757
    }
758
    void *newptr = realloc(ptr, nRequestedSize);
759
    if (newptr == nullptr)
760
    {
761
        ptr[2 * sizeof(void *) + nOldSize + 0] = 'E';
762
        ptr[2 * sizeof(void *) + nOldSize + 1] = 'V';
763
        ptr[2 * sizeof(void *) + nOldSize + 2] = 'S';
764
        ptr[2 * sizeof(void *) + nOldSize + 3] = 'I';
765
        return nullptr;
766
    }
767
#endif
768
    ptr = static_cast<char *>(newptr);
769
    // cppcheck-suppress pointerSize
770
    memcpy(ptr + sizeof(void *), &nNewSize, sizeof(void *));
771
    ptr[2 * sizeof(void *) + nNewSize + 0] = 'E';
772
    ptr[2 * sizeof(void *) + nNewSize + 1] = 'V';
773
    ptr[2 * sizeof(void *) + nNewSize + 2] = 'S';
774
    ptr[2 * sizeof(void *) + nNewSize + 3] = 'I';
775
776
#if defined(DEBUG_VSIMALLOC_STATS) || defined(DEBUG_VSIMALLOC_VERBOSE)
777
    {
778
        CPLMutexHolderD(&hMemStatMutex);
779
#ifdef DEBUG_VSIMALLOC_VERBOSE
780
        if (nNewSize > THRESHOLD_PRINT)
781
        {
782
            fprintf(
783
                stderr,
784
                "Thread[%p] VSIRealloc(%p, %d) = %p" /*ok*/
785
#ifdef DEBUG_VSIMALLOC_STATS
786
                ", current_cumul = " CPL_FRMT_GUIB
787
#ifdef DEBUG_BLOCK_CACHE_USE
788
                ", block_cache_used = " CPL_FRMT_GIB
789
#endif
790
                ", mal+cal-free = %d"
791
#endif
792
                "\n",
793
                (void *)CPLGetPID(), pData, static_cast<int>(nNewSize),
794
                ptr + 2 * sizeof(void *)
795
#ifdef DEBUG_VSIMALLOC_STATS
796
                    ,
797
                static_cast<GUIntBig>(nCurrentTotalAllocs - nOldSize + nNewSize)
798
#ifdef DEBUG_BLOCK_CACHE_USE
799
                    ,
800
                GDALGetCacheUsed64()
801
#endif
802
                    ,
803
                static_cast<int>(nVSIMallocs + nVSICallocs - nVSIFrees)
804
#endif
805
            );
806
        }
807
#endif
808
#ifdef DEBUG_VSIMALLOC_STATS
809
        nVSIReallocs++;
810
        nCurrentTotalAllocs -= nOldSize;
811
        nCurrentTotalAllocs += nNewSize;
812
        if (nCurrentTotalAllocs > nMaxTotalAllocs)
813
            nMaxTotalAllocs = nCurrentTotalAllocs;
814
#endif
815
    }
816
#endif
817
    return ptr + 2 * sizeof(void *);
818
#else
819
43.3k
    return realloc(pData, nNewSize);
820
43.3k
#endif
821
43.3k
}
822
823
/************************************************************************/
824
/*                              VSIFree()                               */
825
/************************************************************************/
826
827
/** Analog of free() for data allocated with VSIMalloc(), VSICalloc(),
828
 * VSIRealloc().
829
 *
830
 * It is not an error to call VSIFree with a NULL pointer, and it will
831
 * have no effect.
832
 */
833
void VSIFree(void *pData)
834
835
255k
{
836
#ifdef DEBUG_VSIMALLOC
837
    if (pData == nullptr)
838
        return;
839
840
    char *ptr = ((char *)pData) - 2 * sizeof(void *);
841
    VSICheckMarkerBegin(ptr);
842
    size_t nOldSize = 0;
843
    // cppcheck-suppress pointerSize
844
    memcpy(&nOldSize, ptr + sizeof(void *), sizeof(void *));
845
    VSICheckMarkerEnd(ptr, 2 * sizeof(void *) + nOldSize);
846
    ptr[0] = 'M';
847
    ptr[1] = 'I';
848
    ptr[2] = 'S';
849
    ptr[3] = 'V';
850
    ptr[2 * sizeof(void *) + nOldSize + 0] = 'I';
851
    ptr[2 * sizeof(void *) + nOldSize + 1] = 'S';
852
    ptr[2 * sizeof(void *) + nOldSize + 2] = 'V';
853
    ptr[2 * sizeof(void *) + nOldSize + 3] = 'E';
854
#if defined(DEBUG_VSIMALLOC_STATS) || defined(DEBUG_VSIMALLOC_VERBOSE)
855
    {
856
        CPLMutexHolderD(&hMemStatMutex);
857
#ifdef DEBUG_VSIMALLOC_VERBOSE
858
        if (nOldSize > THRESHOLD_PRINT)
859
        {
860
            fprintf(stderr, "Thread[%p] VSIFree(%p, (%d bytes))\n", /*ok*/
861
                    (void *)CPLGetPID(), pData, static_cast<int>(nOldSize));
862
        }
863
#endif
864
#ifdef DEBUG_VSIMALLOC_STATS
865
        nVSIFrees++;
866
        nCurrentTotalAllocs -= nOldSize;
867
#endif
868
    }
869
#endif
870
871
#ifdef DEBUG_VSIMALLOC_MPROTECT
872
    mprotect(ptr, nOldSize + 2 * sizeof(void *), PROT_NONE);
873
#else
874
    free(ptr);
875
#endif
876
877
#else
878
255k
    if (pData != nullptr)
879
222k
        free(pData);
880
255k
#endif
881
255k
}
882
883
/************************************************************************/
884
/*                      VSIMallocAligned()                              */
885
/************************************************************************/
886
887
/** Allocates a buffer with an alignment constraint.
888
 *
889
 * The return value must be freed with VSIFreeAligned().
890
 *
891
 * @param nAlignment Must be a power of 2, multiple of sizeof(void*), and
892
 *                   lesser than 256.
893
 * @param nSize Size of the buffer to allocate.
894
 * @return a buffer aligned on nAlignment and of size nSize, or NULL
895
 */
896
897
void *VSIMallocAligned(size_t nAlignment, size_t nSize)
898
0
{
899
    // In particular for posix_memalign() where behavior when passing
900
    // nSize == 0 is technically implementation defined (Valgrind complains),
901
    // so let's always return NULL.
902
0
    if (nSize == 0)
903
0
        return nullptr;
904
0
#if defined(HAVE_POSIX_MEMALIGN) && !defined(DEBUG_VSIMALLOC)
905
0
    void *pRet = nullptr;
906
0
    if (posix_memalign(&pRet, nAlignment, nSize) != 0)
907
0
    {
908
0
        pRet = nullptr;
909
0
    }
910
0
    return pRet;
911
#elif defined(_WIN32) && !defined(DEBUG_VSIMALLOC)
912
    return _aligned_malloc(nSize, nAlignment);
913
#else
914
    // Check constraints on alignment.
915
    if (nAlignment < sizeof(void *) || nAlignment >= 256 ||
916
        (nAlignment & (nAlignment - 1)) != 0)
917
        return nullptr;
918
    // Detect overflow.
919
    if (nSize + nAlignment < nSize)
920
        return nullptr;
921
    GByte *pabyData = static_cast<GByte *>(VSIMalloc(nSize + nAlignment));
922
    if (pabyData == nullptr)
923
        return nullptr;
924
    size_t nShift =
925
        nAlignment - (reinterpret_cast<size_t>(pabyData) % nAlignment);
926
    GByte *pabyAligned = pabyData + nShift;
927
    // Guaranteed to fit on a byte since nAlignment < 256.
928
    pabyAligned[-1] = static_cast<GByte>(nShift);
929
    return pabyAligned;
930
#endif
931
0
}
932
933
/************************************************************************/
934
/*                     VSIMallocAlignedAuto()                           */
935
/************************************************************************/
936
937
/** Allocates a buffer with an alignment constraint such that it can be
938
 * used by the most demanding vector instruction set on that platform.
939
 *
940
 * The return value must be freed with VSIFreeAligned().
941
 *
942
 * @param nSize Size of the buffer to allocate.
943
 * @return an aligned buffer of size nSize, or NULL
944
 */
945
946
void *VSIMallocAlignedAuto(size_t nSize)
947
0
{
948
    // We could potentially dynamically detect the capability of the CPU
949
    // but to simplify use 64 for AVX512 requirements (we use only AVX256
950
    // currently).
951
0
    return VSIMallocAligned(64, nSize);
952
0
}
953
954
/************************************************************************/
955
/*                        VSIMallocAlignedAutoVerbose()                 */
956
/************************************************************************/
957
958
/** See VSIMallocAlignedAuto() */
959
void *VSIMallocAlignedAutoVerbose(size_t nSize, const char *pszFile, int nLine)
960
0
{
961
0
    void *pRet = VSIMallocAlignedAuto(nSize);
962
0
    if (pRet == nullptr && nSize != 0)
963
0
    {
964
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
965
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB " bytes",
966
0
                 pszFile ? pszFile : "(unknown file)", nLine,
967
0
                 static_cast<GUIntBig>(nSize));
968
0
    }
969
0
    return pRet;
970
0
}
971
972
/************************************************************************/
973
/*                        VSIFreeAligned()                              */
974
/************************************************************************/
975
976
/** Free a buffer allocated with VSIMallocAligned().
977
 *
978
 * @param ptr Buffer to free.
979
 */
980
981
void VSIFreeAligned(void *ptr)
982
0
{
983
0
#if defined(HAVE_POSIX_MEMALIGN) && !defined(DEBUG_VSIMALLOC)
984
0
    free(ptr);
985
#elif defined(_WIN32) && !defined(DEBUG_VSIMALLOC)
986
    _aligned_free(ptr);
987
#else
988
    if (ptr == nullptr)
989
        return;
990
    GByte *pabyAligned = static_cast<GByte *>(ptr);
991
    size_t nShift = pabyAligned[-1];
992
    VSIFree(pabyAligned - nShift);
993
#endif
994
0
}
995
996
/************************************************************************/
997
/*                             VSIStrdup()                              */
998
/************************************************************************/
999
1000
/** Analog of strdup(). Use VSIFree() to free */
1001
char *VSIStrdup(const char *pszString)
1002
1003
79.5k
{
1004
79.5k
    const size_t nSize = strlen(pszString) + 1;
1005
79.5k
    char *ptr = static_cast<char *>(VSIMalloc(nSize));
1006
79.5k
    if (ptr == nullptr)
1007
0
        return nullptr;
1008
79.5k
    memcpy(ptr, pszString, nSize);
1009
79.5k
    return ptr;
1010
79.5k
}
1011
1012
/************************************************************************/
1013
/*                          VSICheckMul2()                              */
1014
/************************************************************************/
1015
1016
CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW
1017
static size_t VSICheckMul2(size_t mul1, size_t mul2, bool *pbOverflowFlag,
1018
                           const char *pszFile, int nLine)
1019
1.17k
{
1020
1.17k
    const size_t res = mul1 * mul2;
1021
1.17k
    if (mul1 != 0)
1022
1.17k
    {
1023
1.17k
        if (res / mul1 == mul2)
1024
1.17k
        {
1025
1.17k
            if (pbOverflowFlag)
1026
1.17k
                *pbOverflowFlag = FALSE;
1027
1.17k
            return res;
1028
1.17k
        }
1029
0
        else
1030
0
        {
1031
0
            if (pbOverflowFlag)
1032
0
                *pbOverflowFlag = TRUE;
1033
0
            CPLError(CE_Failure, CPLE_OutOfMemory,
1034
0
                     "%s: %d: Multiplication overflow : " CPL_FRMT_GUIB
1035
0
                     " * " CPL_FRMT_GUIB,
1036
0
                     pszFile ? pszFile : "(unknown file)", nLine,
1037
0
                     static_cast<GUIntBig>(mul1), static_cast<GUIntBig>(mul2));
1038
0
        }
1039
1.17k
    }
1040
0
    else
1041
0
    {
1042
0
        if (pbOverflowFlag)
1043
0
            *pbOverflowFlag = FALSE;
1044
0
    }
1045
0
    return 0;
1046
1.17k
}
1047
1048
/************************************************************************/
1049
/*                          VSICheckMul3()                              */
1050
/************************************************************************/
1051
1052
CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW
1053
static size_t VSICheckMul3(size_t mul1, size_t mul2, size_t mul3,
1054
                           bool *pbOverflowFlag, const char *pszFile, int nLine)
1055
0
{
1056
0
    if (mul1 != 0)
1057
0
    {
1058
0
        const size_t res = mul1 * mul2;
1059
0
        if (res / mul1 == mul2)
1060
0
        {
1061
0
            const size_t res2 = res * mul3;
1062
0
            if (mul3 != 0)
1063
0
            {
1064
0
                if (res2 / mul3 == res)
1065
0
                {
1066
0
                    if (pbOverflowFlag)
1067
0
                        *pbOverflowFlag = false;
1068
0
                    return res2;
1069
0
                }
1070
0
                else
1071
0
                {
1072
0
                    if (pbOverflowFlag)
1073
0
                        *pbOverflowFlag = true;
1074
0
                    CPLError(CE_Failure, CPLE_OutOfMemory,
1075
0
                             "%s: %d: Multiplication overflow : " CPL_FRMT_GUIB
1076
0
                             " * " CPL_FRMT_GUIB " * " CPL_FRMT_GUIB,
1077
0
                             pszFile ? pszFile : "(unknown file)", nLine,
1078
0
                             static_cast<GUIntBig>(mul1),
1079
0
                             static_cast<GUIntBig>(mul2),
1080
0
                             static_cast<GUIntBig>(mul3));
1081
0
                }
1082
0
            }
1083
0
            else
1084
0
            {
1085
0
                if (pbOverflowFlag)
1086
0
                    *pbOverflowFlag = false;
1087
0
            }
1088
0
        }
1089
0
        else
1090
0
        {
1091
0
            if (pbOverflowFlag)
1092
0
                *pbOverflowFlag = true;
1093
0
            CPLError(CE_Failure, CPLE_OutOfMemory,
1094
0
                     "%s: %d: Multiplication overflow : " CPL_FRMT_GUIB
1095
0
                     " * " CPL_FRMT_GUIB " * " CPL_FRMT_GUIB,
1096
0
                     pszFile ? pszFile : "(unknown file)", nLine,
1097
0
                     static_cast<GUIntBig>(mul1), static_cast<GUIntBig>(mul2),
1098
0
                     static_cast<GUIntBig>(mul3));
1099
0
        }
1100
0
    }
1101
0
    else
1102
0
    {
1103
0
        if (pbOverflowFlag)
1104
0
            *pbOverflowFlag = false;
1105
0
    }
1106
0
    return 0;
1107
0
}
1108
1109
/**
1110
 VSIMalloc2 allocates (nSize1 * nSize2) bytes.
1111
 In case of overflow of the multiplication, or if memory allocation fails, a
1112
 NULL pointer is returned and a CE_Failure error is raised with CPLError().
1113
 If nSize1 == 0 || nSize2 == 0, a NULL pointer will also be returned.
1114
 CPLFree() or VSIFree() can be used to free memory allocated by this function.
1115
*/
1116
void CPL_DLL *VSIMalloc2(size_t nSize1, size_t nSize2)
1117
0
{
1118
0
    return VSIMalloc2Verbose(nSize1, nSize2, nullptr, 0);
1119
0
}
1120
1121
/**
1122
 VSIMalloc3 allocates (nSize1 * nSize2 * nSize3) bytes.
1123
 In case of overflow of the multiplication, or if memory allocation fails, a
1124
 NULL pointer is returned and a CE_Failure error is raised with CPLError().
1125
 If nSize1 == 0 || nSize2 == 0 || nSize3 == 0, a NULL pointer will also be
1126
 returned.  CPLFree() or VSIFree() can be used to free memory allocated by this
1127
 function.
1128
*/
1129
void CPL_DLL *VSIMalloc3(size_t nSize1, size_t nSize2, size_t nSize3)
1130
0
{
1131
0
    return VSIMalloc3Verbose(nSize1, nSize2, nSize3, nullptr, 0);
1132
0
}
1133
1134
/************************************************************************/
1135
/*                          VSIMallocVerbose()                          */
1136
/************************************************************************/
1137
1138
void *VSIMallocVerbose(size_t nSize, const char *pszFile, int nLine)
1139
32
{
1140
32
    void *pRet = VSIMalloc(nSize);
1141
32
    if (pRet == nullptr && nSize != 0)
1142
0
    {
1143
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
1144
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB " bytes",
1145
0
                 pszFile ? pszFile : "(unknown file)", nLine,
1146
0
                 static_cast<GUIntBig>(nSize));
1147
0
    }
1148
32
    return pRet;
1149
32
}
1150
1151
/************************************************************************/
1152
/*                          VSIMalloc2Verbose()                         */
1153
/************************************************************************/
1154
1155
void *VSIMalloc2Verbose(size_t nSize1, size_t nSize2, const char *pszFile,
1156
                        int nLine)
1157
1.17k
{
1158
1.17k
    bool bOverflowFlag = false;
1159
1.17k
    const size_t nSizeToAllocate =
1160
1.17k
        VSICheckMul2(nSize1, nSize2, &bOverflowFlag, pszFile, nLine);
1161
1.17k
    if (bOverflowFlag || nSizeToAllocate == 0)
1162
0
        return nullptr;
1163
1164
1.17k
    void *pRet = VSIMalloc(nSizeToAllocate);
1165
1.17k
    if (pRet == nullptr)
1166
0
    {
1167
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
1168
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB " bytes",
1169
0
                 pszFile ? pszFile : "(unknown file)", nLine,
1170
0
                 static_cast<GUIntBig>(nSize1) * static_cast<GUIntBig>(nSize2));
1171
0
    }
1172
1.17k
    return pRet;
1173
1.17k
}
1174
1175
/************************************************************************/
1176
/*                          VSIMalloc3Verbose()                         */
1177
/************************************************************************/
1178
1179
void *VSIMalloc3Verbose(size_t nSize1, size_t nSize2, size_t nSize3,
1180
                        const char *pszFile, int nLine)
1181
0
{
1182
0
    bool bOverflowFlag = false;
1183
0
    size_t nSizeToAllocate =
1184
0
        VSICheckMul3(nSize1, nSize2, nSize3, &bOverflowFlag, pszFile, nLine);
1185
0
    if (bOverflowFlag || nSizeToAllocate == 0)
1186
0
        return nullptr;
1187
1188
0
    void *pRet = VSIMalloc(nSizeToAllocate);
1189
0
    if (pRet == nullptr)
1190
0
    {
1191
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
1192
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB " bytes",
1193
0
                 pszFile ? pszFile : "(unknown file)", nLine,
1194
0
                 static_cast<GUIntBig>(nSize1) * static_cast<GUIntBig>(nSize2) *
1195
0
                     static_cast<GUIntBig>(nSize3));
1196
0
    }
1197
0
    return pRet;
1198
0
}
1199
1200
/************************************************************************/
1201
/*                          VSICallocVerbose()                          */
1202
/************************************************************************/
1203
1204
void *VSICallocVerbose(size_t nCount, size_t nSize, const char *pszFile,
1205
                       int nLine)
1206
15.5k
{
1207
15.5k
    void *pRet = VSICalloc(nCount, nSize);
1208
15.5k
    if (pRet == nullptr && nCount != 0 && nSize != 0)
1209
0
    {
1210
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
1211
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB "x" CPL_FRMT_GUIB
1212
0
                 " bytes",
1213
0
                 pszFile ? pszFile : "(unknown file)", nLine,
1214
0
                 static_cast<GUIntBig>(nCount), static_cast<GUIntBig>(nSize));
1215
0
    }
1216
15.5k
    return pRet;
1217
15.5k
}
1218
1219
/************************************************************************/
1220
/*                          VSIReallocVerbose()                         */
1221
/************************************************************************/
1222
1223
void *VSIReallocVerbose(void *pOldPtr, size_t nNewSize, const char *pszFile,
1224
                        int nLine)
1225
42.2k
{
1226
42.2k
    void *pRet = VSIRealloc(pOldPtr, nNewSize);
1227
42.2k
    if (pRet == nullptr && nNewSize != 0)
1228
0
    {
1229
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
1230
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB " bytes",
1231
0
                 pszFile ? pszFile : "(unknown file)", nLine,
1232
0
                 static_cast<GUIntBig>(nNewSize));
1233
0
    }
1234
42.2k
    return pRet;
1235
42.2k
}
1236
1237
/************************************************************************/
1238
/*                          VSIStrdupVerbose()                          */
1239
/************************************************************************/
1240
1241
char *VSIStrdupVerbose(const char *pszStr, const char *pszFile, int nLine)
1242
79.5k
{
1243
79.5k
    char *pRet = VSIStrdup(pszStr);
1244
79.5k
    if (pRet == nullptr)
1245
0
    {
1246
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
1247
0
                 "%s, %d: cannot allocate " CPL_FRMT_GUIB " bytes",
1248
0
                 pszFile ? pszFile : "(unknown file)", nLine,
1249
0
                 static_cast<GUIntBig>(strlen(pszStr) + 1));
1250
0
    }
1251
79.5k
    return pRet;
1252
79.5k
}
1253
1254
/************************************************************************/
1255
/*                              VSIStat()                               */
1256
/************************************************************************/
1257
1258
int VSIStat(const char *pszFilename, VSIStatBuf *pStatBuf)
1259
1260
0
{
1261
#if defined(_WIN32)
1262
    if (CPLTestBool(CPLGetConfigOption("GDAL_FILENAME_IS_UTF8", "YES")))
1263
    {
1264
        wchar_t *pwszFilename =
1265
            CPLRecodeToWChar(pszFilename, CPL_ENC_UTF8, CPL_ENC_UCS2);
1266
1267
        int nResult =
1268
            _wstat(pwszFilename, reinterpret_cast<struct _stat *>(pStatBuf));
1269
1270
        CPLFree(pwszFilename);
1271
1272
        return nResult;
1273
    }
1274
1275
#endif
1276
0
    return stat(pszFilename, pStatBuf);
1277
0
}
1278
1279
/************************************************************************/
1280
/*                              VSITime()                               */
1281
/************************************************************************/
1282
1283
unsigned long VSITime(unsigned long *pnTimeToSet)
1284
1285
0
{
1286
0
    time_t tTime;
1287
1288
0
    tTime = time(nullptr);
1289
1290
0
    if (pnTimeToSet != nullptr)
1291
0
        *pnTimeToSet = static_cast<unsigned long>(tTime);
1292
1293
0
    return static_cast<unsigned long>(tTime);
1294
0
}
1295
1296
/************************************************************************/
1297
/*                              VSICTime()                              */
1298
/************************************************************************/
1299
1300
const char *VSICTime(unsigned long nTime)
1301
1302
1.54k
{
1303
1.54k
    time_t tTime = static_cast<time_t>(nTime);
1304
1.54k
#if HAVE_CTIME_R
1305
    // Cf https://linux.die.net/man/3/ctime_r:
1306
    // "The reentrant version ctime_r() does the same, but stores the string in
1307
    // a user-supplied buffer which should have room for at least 26 bytes"
1308
1.54k
    char buffer[26];
1309
1.54k
    char *ret = ctime_r(&tTime, buffer);
1310
1.54k
    return ret ? CPLSPrintf("%s", ret) : nullptr;
1311
#elif defined(_WIN32)
1312
    char buffer[26];
1313
    return ctime_s(buffer, sizeof(buffer), &tTime) == 0
1314
               ? CPLSPrintf("%s", buffer)
1315
               : nullptr;
1316
#else
1317
    return reinterpret_cast<const char *>(ctime(&tTime));
1318
#endif
1319
1.54k
}
1320
1321
/************************************************************************/
1322
/*                             VSIGMTime()                              */
1323
/************************************************************************/
1324
1325
struct tm *VSIGMTime(const time_t *pnTime, struct tm *poBrokenTime)
1326
0
{
1327
1328
0
#if HAVE_GMTIME_R
1329
0
    gmtime_r(pnTime, poBrokenTime);
1330
0
    return poBrokenTime;
1331
#elif defined(_WIN32)
1332
    return gmtime_s(poBrokenTime, pnTime) == 0 ? poBrokenTime : nullptr;
1333
#else
1334
    struct tm *poTime = gmtime(pnTime);
1335
    memcpy(poBrokenTime, poTime, sizeof(tm));
1336
    return poBrokenTime;
1337
#endif
1338
0
}
1339
1340
/************************************************************************/
1341
/*                             VSILocalTime()                           */
1342
/************************************************************************/
1343
1344
struct tm *VSILocalTime(const time_t *pnTime, struct tm *poBrokenTime)
1345
0
{
1346
1347
0
#if HAVE_LOCALTIME_R
1348
0
    localtime_r(pnTime, poBrokenTime);
1349
0
    return poBrokenTime;
1350
#elif defined(_WIN32)
1351
    return localtime_s(poBrokenTime, pnTime) == 0 ? poBrokenTime : nullptr;
1352
#else
1353
    struct tm *poTime = localtime(pnTime);
1354
    memcpy(poBrokenTime, poTime, sizeof(tm));
1355
    return poBrokenTime;
1356
#endif
1357
0
}
1358
1359
/************************************************************************/
1360
/*                            VSIStrerror()                             */
1361
/************************************************************************/
1362
1363
/** Return the error string corresponding to the error number. Do not free it */
1364
char *VSIStrerror(int nErrno)
1365
1366
0
{
1367
0
    return strerror(nErrno);
1368
0
}
1369
1370
/************************************************************************/
1371
/*                        CPLGetPhysicalRAM()                           */
1372
/************************************************************************/
1373
1374
#if HAVE_SC_PHYS_PAGES
1375
1376
/** Return the total physical RAM in bytes.
1377
 *
1378
 * In the context of a container using cgroups (typically Docker), this
1379
 * will take into account that limitation (with extra fixes in GDAL 3.6.3)
1380
 *
1381
 * You should generally use CPLGetUsablePhysicalRAM() instead.
1382
 *
1383
 * @return the total physical RAM in bytes (or 0 in case of failure).
1384
 */
1385
GIntBig CPLGetPhysicalRAM(void)
1386
0
{
1387
0
    static const GIntBig nPhysicalRAM = []() -> GIntBig
1388
0
    {
1389
0
        const long nPhysPages = sysconf(_SC_PHYS_PAGES);
1390
0
        const long nPageSize = sysconf(_SC_PAGESIZE);
1391
0
        if (nPhysPages <= 0 || nPageSize <= 0 ||
1392
0
            nPhysPages > std::numeric_limits<GIntBig>::max() / nPageSize)
1393
0
        {
1394
0
            return 0;
1395
0
        }
1396
1397
0
        GIntBig nVal = static_cast<GIntBig>(nPhysPages) * nPageSize;
1398
1399
0
#ifdef __linux
1400
0
        {
1401
            // Take into account MemTotal in /proc/meminfo
1402
            // which seems to be necessary for some container solutions
1403
            // Cf https://lists.osgeo.org/pipermail/gdal-dev/2023-January/056784.html
1404
0
            FILE *f = fopen("/proc/meminfo", "rb");
1405
0
            if (f)
1406
0
            {
1407
0
                char szLine[256];
1408
0
                while (fgets(szLine, sizeof(szLine), f))
1409
0
                {
1410
                    // Find line like "MemTotal:       32525176 kB"
1411
0
                    if (strncmp(szLine, "MemTotal:", strlen("MemTotal:")) == 0)
1412
0
                    {
1413
0
                        char *pszVal = szLine + strlen("MemTotal:");
1414
0
                        pszVal += strspn(pszVal, " ");
1415
0
                        char *pszEnd = strstr(pszVal, " kB");
1416
0
                        if (pszEnd)
1417
0
                        {
1418
0
                            *pszEnd = 0;
1419
0
                            if (CPLGetValueType(pszVal) == CPL_VALUE_INTEGER)
1420
0
                            {
1421
0
                                const GUIntBig nLimit =
1422
0
                                    CPLScanUIntBig(
1423
0
                                        pszVal,
1424
0
                                        static_cast<int>(strlen(pszVal))) *
1425
0
                                    1024;
1426
0
                                nVal = static_cast<GIntBig>(std::min(
1427
0
                                    static_cast<GUIntBig>(nVal), nLimit));
1428
0
                            }
1429
0
                        }
1430
0
                        break;
1431
0
                    }
1432
0
                }
1433
0
                fclose(f);
1434
0
            }
1435
0
        }
1436
1437
0
        char szGroupName[256];
1438
0
        bool bFromMemory = false;
1439
0
        szGroupName[0] = 0;
1440
0
        {
1441
0
            FILE *f = fopen("/proc/self/cgroup", "rb");
1442
0
            if (f)
1443
0
            {
1444
0
                char szLine[256];
1445
                // Find line like "6:memory:/user.slice/user-1000.slice/user@1000.service"
1446
                // and store "/user.slice/user-1000.slice/user@1000.service" in
1447
                // szMemoryPath for cgroup V1 or single line "0::/...." for cgroup V2.
1448
0
                while (fgets(szLine, sizeof(szLine), f))
1449
0
                {
1450
0
                    const char *pszMemory = strstr(szLine, ":memory:");
1451
0
                    if (pszMemory)
1452
0
                    {
1453
0
                        bFromMemory = true;
1454
0
                        snprintf(szGroupName, sizeof(szGroupName), "%s",
1455
0
                                 pszMemory + strlen(":memory:"));
1456
0
                        char *pszEOL = strchr(szGroupName, '\n');
1457
0
                        if (pszEOL)
1458
0
                            *pszEOL = '\0';
1459
0
                        break;
1460
0
                    }
1461
0
                    if (strncmp(szLine, "0::", strlen("0::")) == 0)
1462
0
                    {
1463
0
                        snprintf(szGroupName, sizeof(szGroupName), "%s",
1464
0
                                 szLine + strlen("0::"));
1465
0
                        char *pszEOL = strchr(szGroupName, '\n');
1466
0
                        if (pszEOL)
1467
0
                            *pszEOL = '\0';
1468
0
                        break;
1469
0
                    }
1470
0
                }
1471
0
                fclose(f);
1472
0
            }
1473
0
        }
1474
0
        if (szGroupName[0])
1475
0
        {
1476
0
            char szFilename[256 + 64];
1477
0
            if (bFromMemory)
1478
0
            {
1479
                // cgroup V1
1480
                // Read memory.limit_in_byte in the whole szGroupName hierarchy
1481
                // Make sure to end up by reading
1482
                // /sys/fs/cgroup/memory/memory.limit_in_bytes itself, for
1483
                // scenarios like https://github.com/OSGeo/gdal/issues/8968
1484
0
                while (true)
1485
0
                {
1486
0
                    snprintf(szFilename, sizeof(szFilename),
1487
0
                             "/sys/fs/cgroup/memory/%s/memory.limit_in_bytes",
1488
0
                             szGroupName);
1489
0
                    FILE *f = fopen(szFilename, "rb");
1490
0
                    if (f)
1491
0
                    {
1492
                        // If no limitation, on 64 bit, 9223372036854771712 is returned.
1493
0
                        char szBuffer[32];
1494
0
                        const int nRead = static_cast<int>(
1495
0
                            fread(szBuffer, 1, sizeof(szBuffer) - 1, f));
1496
0
                        szBuffer[nRead] = 0;
1497
0
                        fclose(f);
1498
0
                        const GUIntBig nLimit = CPLScanUIntBig(szBuffer, nRead);
1499
0
                        nVal = static_cast<GIntBig>(
1500
0
                            std::min(static_cast<GUIntBig>(nVal), nLimit));
1501
0
                    }
1502
0
                    char *pszLastSlash = strrchr(szGroupName, '/');
1503
0
                    if (!pszLastSlash)
1504
0
                        break;
1505
0
                    *pszLastSlash = '\0';
1506
0
                }
1507
0
            }
1508
0
            else
1509
0
            {
1510
                // cgroup V2
1511
                // Read memory.max in the whole szGroupName hierarchy
1512
0
                while (true)
1513
0
                {
1514
0
                    snprintf(szFilename, sizeof(szFilename),
1515
0
                             "/sys/fs/cgroup/%s/memory.max", szGroupName);
1516
0
                    FILE *f = fopen(szFilename, "rb");
1517
0
                    if (f)
1518
0
                    {
1519
                        // If no limitation, "max" is returned.
1520
0
                        char szBuffer[32];
1521
0
                        int nRead = static_cast<int>(
1522
0
                            fread(szBuffer, 1, sizeof(szBuffer) - 1, f));
1523
0
                        szBuffer[nRead] = 0;
1524
0
                        if (nRead > 0 && szBuffer[nRead - 1] == '\n')
1525
0
                        {
1526
0
                            nRead--;
1527
0
                            szBuffer[nRead] = 0;
1528
0
                        }
1529
0
                        fclose(f);
1530
0
                        if (CPLGetValueType(szBuffer) == CPL_VALUE_INTEGER)
1531
0
                        {
1532
0
                            const GUIntBig nLimit =
1533
0
                                CPLScanUIntBig(szBuffer, nRead);
1534
0
                            nVal = static_cast<GIntBig>(
1535
0
                                std::min(static_cast<GUIntBig>(nVal), nLimit));
1536
0
                        }
1537
0
                    }
1538
0
                    char *pszLastSlash = strrchr(szGroupName, '/');
1539
0
                    if (!pszLastSlash || pszLastSlash == szGroupName)
1540
0
                        break;
1541
0
                    *pszLastSlash = '\0';
1542
0
                }
1543
0
            }
1544
0
        }
1545
0
#endif
1546
1547
0
        return nVal;
1548
0
    }();
1549
1550
0
    return nPhysicalRAM;
1551
0
}
1552
1553
#elif defined(__MACH__) && defined(__APPLE__)
1554
1555
#include <sys/types.h>
1556
#include <sys/sysctl.h>
1557
1558
GIntBig CPLGetPhysicalRAM(void)
1559
{
1560
    GIntBig nPhysMem = 0;
1561
1562
    int mib[2] = {CTL_HW, HW_MEMSIZE};
1563
    size_t nLengthRes = sizeof(nPhysMem);
1564
    sysctl(mib, CPL_ARRAYSIZE(mib), &nPhysMem, &nLengthRes, nullptr, 0);
1565
1566
    return nPhysMem;
1567
}
1568
1569
#elif defined(_WIN32)
1570
1571
// GlobalMemoryStatusEx requires _WIN32_WINNT >= 0x0500.
1572
#ifndef _WIN32_WINNT
1573
#define _WIN32_WINNT 0x0500
1574
#endif
1575
#include <windows.h>
1576
1577
GIntBig CPLGetPhysicalRAM(void)
1578
{
1579
    MEMORYSTATUSEX statex;
1580
    statex.ullTotalPhys = 0;
1581
    statex.dwLength = sizeof(statex);
1582
    GlobalMemoryStatusEx(&statex);
1583
    return static_cast<GIntBig>(statex.ullTotalPhys);
1584
}
1585
1586
#else
1587
1588
GIntBig CPLGetPhysicalRAM(void)
1589
{
1590
    static bool bOnce = false;
1591
    if (!bOnce)
1592
    {
1593
        bOnce = true;
1594
        CPLDebug("PORT", "No implementation for CPLGetPhysicalRAM()");
1595
    }
1596
    return 0;
1597
}
1598
#endif
1599
1600
/************************************************************************/
1601
/*                       CPLGetUsablePhysicalRAM()                      */
1602
/************************************************************************/
1603
1604
/** Return the total physical RAM, usable by a process, in bytes.
1605
 *
1606
 * This is the same as CPLGetPhysicalRAM() except it will limit to 2 GB
1607
 * for 32 bit processes.
1608
 *
1609
 * Starting with GDAL 2.4.0, it will also take account resource limits (virtual
1610
 * memory) on Posix systems. Starting with GDAL 3.6.1, it will also take into
1611
 * account RLIMIT_RSS on Linux.
1612
 *
1613
 * Note: This memory may already be partly used by other processes.
1614
 *
1615
 * @return the total physical RAM, usable by a process, in bytes (or 0
1616
 * in case of failure).
1617
 */
1618
GIntBig CPLGetUsablePhysicalRAM(void)
1619
0
{
1620
0
    GIntBig nRAM = CPLGetPhysicalRAM();
1621
#if SIZEOF_VOIDP == 4
1622
    if (nRAM > INT_MAX)
1623
        nRAM = INT_MAX;
1624
#endif
1625
0
#if HAVE_GETRLIMIT
1626
0
    struct rlimit sLimit;
1627
0
#if HAVE_RLIMIT_AS
1628
0
    const int res = RLIMIT_AS;
1629
#else
1630
    // OpenBSD currently doesn't support RLIMIT_AS (mandated by Posix though)
1631
    const int res = RLIMIT_DATA;
1632
#endif
1633
0
    if (getrlimit(res, &sLimit) == 0 && sLimit.rlim_cur != RLIM_INFINITY &&
1634
0
        static_cast<GIntBig>(sLimit.rlim_cur) < nRAM)
1635
0
    {
1636
0
        nRAM = static_cast<GIntBig>(sLimit.rlim_cur);
1637
0
    }
1638
0
#ifdef RLIMIT_RSS
1639
    // Helps with RSS limit set by the srun utility. Cf
1640
    // https://github.com/OSGeo/gdal/issues/6669
1641
0
    if (getrlimit(RLIMIT_RSS, &sLimit) == 0 &&
1642
0
        sLimit.rlim_cur != RLIM_INFINITY &&
1643
0
        static_cast<GIntBig>(sLimit.rlim_cur) < nRAM)
1644
0
    {
1645
0
        nRAM = static_cast<GIntBig>(sLimit.rlim_cur);
1646
0
    }
1647
0
#endif
1648
0
#endif
1649
0
    return nRAM;
1650
0
}