Coverage Report

Created: 2025-06-13 06:29

/src/gdal/port/cpl_vsil_sparsefile.cpp
Line
Count
Source (jump to first uncovered line)
1
/******************************************************************************
2
 *
3
 * Project:  VSI Virtual File System
4
 * Purpose:  Implementation of sparse file virtual io driver.
5
 * Author:   Frank Warmerdam, warmerdam@pobox.com
6
 *
7
 ******************************************************************************
8
 * Copyright (c) 2010, Frank Warmerdam <warmerdam@pobox.com>
9
 * Copyright (c) 2010-2013, Even Rouault <even dot rouault at spatialys.com>
10
 *
11
 * SPDX-License-Identifier: MIT
12
 ****************************************************************************/
13
14
#include "cpl_port.h"
15
#include "cpl_vsi.h"
16
17
#include <cerrno>
18
#include <cstddef>
19
#include <cstdlib>
20
#include <cstring>
21
22
#if HAVE_FCNTL_H
23
#include <fcntl.h>
24
#endif
25
26
#include <algorithm>
27
#include <map>
28
#include <memory>
29
#include <vector>
30
31
#include "cpl_conv.h"
32
#include "cpl_error.h"
33
#include "cpl_minixml.h"
34
#include "cpl_multiproc.h"
35
#include "cpl_string.h"
36
#include "cpl_vsi_virtual.h"
37
38
class SFRegion
39
{
40
  public:
41
    CPLString osFilename{};
42
    VSILFILE *fp = nullptr;
43
    GUIntBig nDstOffset = 0;
44
    GUIntBig nSrcOffset = 0;
45
    GUIntBig nLength = 0;
46
    GByte byValue = 0;
47
    bool bTriedOpen = false;
48
};
49
50
/************************************************************************/
51
/* ==================================================================== */
52
/*                         VSISparseFileHandle                          */
53
/* ==================================================================== */
54
/************************************************************************/
55
56
class VSISparseFileFilesystemHandler;
57
58
class VSISparseFileHandle : public VSIVirtualHandle
59
{
60
    CPL_DISALLOW_COPY_ASSIGN(VSISparseFileHandle)
61
62
    VSISparseFileFilesystemHandler *m_poFS = nullptr;
63
    bool bEOF = false;
64
    bool bError = false;
65
66
  public:
67
    explicit VSISparseFileHandle(VSISparseFileFilesystemHandler *poFS)
68
0
        : m_poFS(poFS)
69
0
    {
70
0
    }
71
72
    GUIntBig nOverallLength = 0;
73
    GUIntBig nCurOffset = 0;
74
75
    std::vector<SFRegion> aoRegions{};
76
77
    int Seek(vsi_l_offset nOffset, int nWhence) override;
78
    vsi_l_offset Tell() override;
79
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
80
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
81
    void ClearErr() override;
82
    int Eof() override;
83
    int Error() override;
84
    int Close() override;
85
};
86
87
/************************************************************************/
88
/* ==================================================================== */
89
/*                   VSISparseFileFilesystemHandler                     */
90
/* ==================================================================== */
91
/************************************************************************/
92
93
class VSISparseFileFilesystemHandler : public VSIFilesystemHandler
94
{
95
    std::map<GIntBig, int> oRecOpenCount{};
96
    CPL_DISALLOW_COPY_ASSIGN(VSISparseFileFilesystemHandler)
97
98
  public:
99
3
    VSISparseFileFilesystemHandler() = default;
100
0
    ~VSISparseFileFilesystemHandler() override = default;
101
102
    int DecomposePath(const char *pszPath, CPLString &osFilename,
103
                      vsi_l_offset &nSparseFileOffset,
104
                      vsi_l_offset &nSparseFileSize);
105
106
    // TODO(schwehr): Fix VSISparseFileFilesystemHandler::Stat to not need
107
    // using.
108
    using VSIFilesystemHandler::Open;
109
110
    VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
111
                           bool bSetError,
112
                           CSLConstList /* papszOptions */) override;
113
    int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
114
             int nFlags) override;
115
    int Unlink(const char *pszFilename) override;
116
    int Mkdir(const char *pszDirname, long nMode) override;
117
    int Rmdir(const char *pszDirname) override;
118
    char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
119
120
    int GetRecCounter()
121
5.10k
    {
122
5.10k
        return oRecOpenCount[CPLGetPID()];
123
5.10k
    }
124
125
    void IncRecCounter()
126
0
    {
127
0
        oRecOpenCount[CPLGetPID()]++;
128
0
    }
129
130
    void DecRecCounter()
131
0
    {
132
0
        oRecOpenCount[CPLGetPID()]--;
133
0
    }
134
};
135
136
/************************************************************************/
137
/* ==================================================================== */
138
/*                             VSISparseFileHandle                      */
139
/* ==================================================================== */
140
/************************************************************************/
141
142
/************************************************************************/
143
/*                               Close()                                */
144
/************************************************************************/
145
146
int VSISparseFileHandle::Close()
147
148
0
{
149
0
    for (unsigned int i = 0; i < aoRegions.size(); i++)
150
0
    {
151
0
        if (aoRegions[i].fp != nullptr)
152
0
            CPL_IGNORE_RET_VAL(VSIFCloseL(aoRegions[i].fp));
153
0
    }
154
155
0
    return 0;
156
0
}
157
158
/************************************************************************/
159
/*                                Seek()                                */
160
/************************************************************************/
161
162
int VSISparseFileHandle::Seek(vsi_l_offset nOffset, int nWhence)
163
164
0
{
165
0
    bEOF = false;
166
0
    if (nWhence == SEEK_SET)
167
0
        nCurOffset = nOffset;
168
0
    else if (nWhence == SEEK_CUR)
169
0
    {
170
0
        nCurOffset += nOffset;
171
0
    }
172
0
    else if (nWhence == SEEK_END)
173
0
    {
174
0
        nCurOffset = nOverallLength + nOffset;
175
0
    }
176
0
    else
177
0
    {
178
0
        errno = EINVAL;
179
0
        return -1;
180
0
    }
181
182
0
    return 0;
183
0
}
184
185
/************************************************************************/
186
/*                                Tell()                                */
187
/************************************************************************/
188
189
vsi_l_offset VSISparseFileHandle::Tell()
190
191
0
{
192
0
    return nCurOffset;
193
0
}
194
195
/************************************************************************/
196
/*                                Read()                                */
197
/************************************************************************/
198
199
size_t VSISparseFileHandle::Read(void *pBuffer, size_t nSize, size_t nCount)
200
201
0
{
202
0
    if (nCurOffset >= nOverallLength)
203
0
    {
204
0
        bEOF = true;
205
0
        return 0;
206
0
    }
207
208
    /* -------------------------------------------------------------------- */
209
    /*      Find what region we are in, searching linearly from the         */
210
    /*      start.                                                          */
211
    /* -------------------------------------------------------------------- */
212
0
    unsigned int iRegion = 0;  // Used after for.
213
214
0
    for (; iRegion < aoRegions.size(); iRegion++)
215
0
    {
216
0
        if (nCurOffset >= aoRegions[iRegion].nDstOffset &&
217
0
            nCurOffset <
218
0
                aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength)
219
0
            break;
220
0
    }
221
222
0
    size_t nBytesRequested = nSize * nCount;
223
0
    if (nBytesRequested == 0)
224
0
    {
225
0
        return 0;
226
0
    }
227
0
    if (nCurOffset + nBytesRequested > nOverallLength)
228
0
    {
229
0
        nBytesRequested = static_cast<size_t>(nOverallLength - nCurOffset);
230
0
        bEOF = true;
231
0
    }
232
233
    /* -------------------------------------------------------------------- */
234
    /*      Default to zeroing the buffer if no corresponding region was    */
235
    /*      found.                                                          */
236
    /* -------------------------------------------------------------------- */
237
0
    if (iRegion == aoRegions.size())
238
0
    {
239
0
        memset(pBuffer, 0, nBytesRequested);
240
0
        nCurOffset += nBytesRequested;
241
0
        return nBytesRequested / nSize;
242
0
    }
243
244
    /* -------------------------------------------------------------------- */
245
    /*      If this request crosses region boundaries, split it into two    */
246
    /*      requests.                                                       */
247
    /* -------------------------------------------------------------------- */
248
0
    size_t nBytesReturnCount = 0;
249
0
    const GUIntBig nEndOffsetOfRegion =
250
0
        aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength;
251
252
0
    if (nCurOffset + nBytesRequested > nEndOffsetOfRegion)
253
0
    {
254
0
        const size_t nExtraBytes = static_cast<size_t>(
255
0
            nCurOffset + nBytesRequested - nEndOffsetOfRegion);
256
        // Recurse to get the rest of the request.
257
258
0
        const GUIntBig nCurOffsetSave = nCurOffset;
259
0
        nCurOffset += nBytesRequested - nExtraBytes;
260
0
        bool bEOFSave = bEOF;
261
0
        bEOF = false;
262
0
        const size_t nBytesRead = this->Read(static_cast<char *>(pBuffer) +
263
0
                                                 nBytesRequested - nExtraBytes,
264
0
                                             1, nExtraBytes);
265
0
        nCurOffset = nCurOffsetSave;
266
0
        bEOF = bEOFSave;
267
0
        if (nBytesRead < nExtraBytes)
268
0
        {
269
            // A short read in a region of a sparse file is always an error
270
0
            bError = true;
271
0
        }
272
273
0
        nBytesReturnCount += nBytesRead;
274
0
        nBytesRequested -= nExtraBytes;
275
0
    }
276
277
    /* -------------------------------------------------------------------- */
278
    /*      Handle a constant region.                                       */
279
    /* -------------------------------------------------------------------- */
280
0
    if (aoRegions[iRegion].osFilename.empty())
281
0
    {
282
0
        memset(pBuffer, aoRegions[iRegion].byValue,
283
0
               static_cast<size_t>(nBytesRequested));
284
285
0
        nBytesReturnCount += nBytesRequested;
286
0
    }
287
288
    /* -------------------------------------------------------------------- */
289
    /*      Otherwise handle as a file.                                     */
290
    /* -------------------------------------------------------------------- */
291
0
    else
292
0
    {
293
0
        if (aoRegions[iRegion].fp == nullptr)
294
0
        {
295
0
            if (!aoRegions[iRegion].bTriedOpen)
296
0
            {
297
0
                aoRegions[iRegion].fp =
298
0
                    VSIFOpenL(aoRegions[iRegion].osFilename, "r");
299
0
                if (aoRegions[iRegion].fp == nullptr)
300
0
                {
301
0
                    CPLDebug("/vsisparse/", "Failed to open '%s'.",
302
0
                             aoRegions[iRegion].osFilename.c_str());
303
0
                }
304
0
                aoRegions[iRegion].bTriedOpen = true;
305
0
            }
306
0
            if (aoRegions[iRegion].fp == nullptr)
307
0
            {
308
0
                bError = true;
309
0
                return 0;
310
0
            }
311
0
        }
312
313
0
        if (VSIFSeekL(aoRegions[iRegion].fp,
314
0
                      nCurOffset - aoRegions[iRegion].nDstOffset +
315
0
                          aoRegions[iRegion].nSrcOffset,
316
0
                      SEEK_SET) != 0)
317
0
        {
318
0
            bError = true;
319
0
            return 0;
320
0
        }
321
322
0
        m_poFS->IncRecCounter();
323
0
        const size_t nBytesRead =
324
0
            VSIFReadL(pBuffer, 1, static_cast<size_t>(nBytesRequested),
325
0
                      aoRegions[iRegion].fp);
326
0
        m_poFS->DecRecCounter();
327
0
        if (nBytesRead < static_cast<size_t>(nBytesRequested))
328
0
        {
329
            // A short read in a region of a sparse file is always an error
330
0
            bError = true;
331
0
        }
332
333
0
        nBytesReturnCount += nBytesRead;
334
0
    }
335
336
0
    nCurOffset += nBytesReturnCount;
337
338
0
    return nBytesReturnCount / nSize;
339
0
}
340
341
/************************************************************************/
342
/*                               Write()                                */
343
/************************************************************************/
344
345
size_t VSISparseFileHandle::Write(const void * /* pBuffer */,
346
                                  size_t /* nSize */, size_t /* nCount */)
347
0
{
348
0
    errno = EBADF;
349
0
    return 0;
350
0
}
351
352
/************************************************************************/
353
/*                                Eof()                                 */
354
/************************************************************************/
355
356
int VSISparseFileHandle::Eof()
357
358
0
{
359
0
    return bEOF ? 1 : 0;
360
0
}
361
362
/************************************************************************/
363
/*                               Error()                                */
364
/************************************************************************/
365
366
int VSISparseFileHandle::Error()
367
368
0
{
369
0
    return bError ? 1 : 0;
370
0
}
371
372
/************************************************************************/
373
/*                             ClearErr()                               */
374
/************************************************************************/
375
376
void VSISparseFileHandle::ClearErr()
377
378
0
{
379
0
    for (const auto &region : aoRegions)
380
0
    {
381
0
        if (region.fp)
382
0
            region.fp->ClearErr();
383
0
    }
384
0
    bEOF = false;
385
0
    bError = false;
386
0
}
387
388
/************************************************************************/
389
/* ==================================================================== */
390
/*                       VSISparseFileFilesystemHandler                 */
391
/* ==================================================================== */
392
/************************************************************************/
393
394
/************************************************************************/
395
/*                                Open()                                */
396
/************************************************************************/
397
398
VSIVirtualHandle *VSISparseFileFilesystemHandler::Open(
399
    const char *pszFilename, const char *pszAccess, bool /* bSetError */,
400
    CSLConstList /* papszOptions */)
401
402
5.18k
{
403
5.18k
    if (!STARTS_WITH_CI(pszFilename, "/vsisparse/"))
404
82
        return nullptr;
405
406
5.10k
    if (!EQUAL(pszAccess, "r") && !EQUAL(pszAccess, "rb"))
407
0
    {
408
0
        errno = EACCES;
409
0
        return nullptr;
410
0
    }
411
412
    // Arbitrary number.
413
5.10k
    if (GetRecCounter() == 32)
414
0
        return nullptr;
415
416
5.10k
    const CPLString osSparseFilePath = pszFilename + 11;
417
418
    /* -------------------------------------------------------------------- */
419
    /*      Does this file even exist?                                      */
420
    /* -------------------------------------------------------------------- */
421
5.10k
    VSILFILE *fp = VSIFOpenL(osSparseFilePath, "r");
422
5.10k
    if (fp == nullptr)
423
4.17k
        return nullptr;
424
924
    CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
425
426
    /* -------------------------------------------------------------------- */
427
    /*      Read the XML file.                                              */
428
    /* -------------------------------------------------------------------- */
429
924
    CPLXMLNode *psXMLRoot = CPLParseXMLFile(osSparseFilePath);
430
431
924
    if (psXMLRoot == nullptr)
432
924
        return nullptr;
433
434
    /* -------------------------------------------------------------------- */
435
    /*      Setup the file handle on this file.                             */
436
    /* -------------------------------------------------------------------- */
437
0
    VSISparseFileHandle *poHandle = new VSISparseFileHandle(this);
438
439
    /* -------------------------------------------------------------------- */
440
    /*      Translate the desired fields out of the XML tree.               */
441
    /* -------------------------------------------------------------------- */
442
0
    for (CPLXMLNode *psRegion = psXMLRoot->psChild; psRegion != nullptr;
443
0
         psRegion = psRegion->psNext)
444
0
    {
445
0
        if (psRegion->eType != CXT_Element)
446
0
            continue;
447
448
0
        if (!EQUAL(psRegion->pszValue, "SubfileRegion") &&
449
0
            !EQUAL(psRegion->pszValue, "ConstantRegion"))
450
0
            continue;
451
452
0
        SFRegion oRegion;
453
454
0
        oRegion.osFilename = CPLGetXMLValue(psRegion, "Filename", "");
455
0
        if (atoi(CPLGetXMLValue(psRegion, "Filename.relative", "0")) != 0)
456
0
        {
457
0
            const std::string osSFPath = CPLGetPathSafe(osSparseFilePath);
458
0
            oRegion.osFilename = CPLFormFilenameSafe(
459
0
                osSFPath.c_str(), oRegion.osFilename, nullptr);
460
0
        }
461
462
        // TODO(schwehr): Symbolic constant and an explanation for 32.
463
0
        oRegion.nDstOffset = CPLScanUIntBig(
464
0
            CPLGetXMLValue(psRegion, "DestinationOffset", "0"), 32);
465
466
0
        oRegion.nSrcOffset =
467
0
            CPLScanUIntBig(CPLGetXMLValue(psRegion, "SourceOffset", "0"), 32);
468
469
0
        oRegion.nLength =
470
0
            CPLScanUIntBig(CPLGetXMLValue(psRegion, "RegionLength", "0"), 32);
471
472
0
        oRegion.byValue =
473
0
            static_cast<GByte>(atoi(CPLGetXMLValue(psRegion, "Value", "0")));
474
475
0
        poHandle->aoRegions.push_back(std::move(oRegion));
476
0
    }
477
478
    /* -------------------------------------------------------------------- */
479
    /*      Get sparse file length, use maximum bound of regions if not     */
480
    /*      explicit in file.                                               */
481
    /* -------------------------------------------------------------------- */
482
0
    poHandle->nOverallLength =
483
0
        CPLScanUIntBig(CPLGetXMLValue(psXMLRoot, "Length", "0"), 32);
484
0
    if (poHandle->nOverallLength == 0)
485
0
    {
486
0
        for (unsigned int i = 0; i < poHandle->aoRegions.size(); i++)
487
0
        {
488
0
            poHandle->nOverallLength = std::max(
489
0
                poHandle->nOverallLength, poHandle->aoRegions[i].nDstOffset +
490
0
                                              poHandle->aoRegions[i].nLength);
491
0
        }
492
0
    }
493
494
0
    CPLDestroyXMLNode(psXMLRoot);
495
496
0
    return poHandle;
497
924
}
498
499
/************************************************************************/
500
/*                                Stat()                                */
501
/************************************************************************/
502
503
int VSISparseFileFilesystemHandler::Stat(const char *pszFilename,
504
                                         VSIStatBufL *psStatBuf, int nFlags)
505
506
5.04k
{
507
    // TODO(schwehr): Fix this so that the using statement is not needed.
508
    // Will just adding the bool for bSetError be okay?
509
5.04k
    VSIVirtualHandle *poFile = Open(pszFilename, "r");
510
511
5.04k
    memset(psStatBuf, 0, sizeof(VSIStatBufL));
512
513
5.04k
    if (poFile == nullptr)
514
5.04k
        return -1;
515
516
0
    poFile->Seek(0, SEEK_END);
517
0
    const vsi_l_offset nLength = poFile->Tell();
518
0
    delete poFile;
519
520
0
    const int nResult =
521
0
        VSIStatExL(pszFilename + strlen("/vsisparse/"), psStatBuf, nFlags);
522
523
0
    psStatBuf->st_size = nLength;
524
525
0
    return nResult;
526
5.04k
}
527
528
/************************************************************************/
529
/*                               Unlink()                               */
530
/************************************************************************/
531
532
int VSISparseFileFilesystemHandler::Unlink(const char * /* pszFilename */)
533
0
{
534
0
    errno = EACCES;
535
0
    return -1;
536
0
}
537
538
/************************************************************************/
539
/*                               Mkdir()                                */
540
/************************************************************************/
541
542
int VSISparseFileFilesystemHandler::Mkdir(const char * /* pszPathname */,
543
                                          long /* nMode */)
544
0
{
545
0
    errno = EACCES;
546
0
    return -1;
547
0
}
548
549
/************************************************************************/
550
/*                               Rmdir()                                */
551
/************************************************************************/
552
553
int VSISparseFileFilesystemHandler::Rmdir(const char * /* pszPathname */)
554
0
{
555
0
    errno = EACCES;
556
0
    return -1;
557
0
}
558
559
/************************************************************************/
560
/*                              ReadDirEx()                             */
561
/************************************************************************/
562
563
char **VSISparseFileFilesystemHandler::ReadDirEx(const char * /* pszPath */,
564
                                                 int /* nMaxFiles */)
565
0
{
566
0
    errno = EACCES;
567
0
    return nullptr;
568
0
}
569
570
/************************************************************************/
571
/*                 VSIInstallSparseFileFilesystemHandler()              */
572
/************************************************************************/
573
574
/*!
575
 \brief Install /vsisparse/ virtual file handler.
576
577
 \verbatim embed:rst
578
 See :ref:`/vsisparse/ documentation <vsisparse>`
579
 \endverbatim
580
 */
581
582
void VSIInstallSparseFileHandler()
583
3
{
584
3
    VSIFileManager::InstallHandler("/vsisparse/",
585
3
                                   new VSISparseFileFilesystemHandler);
586
3
}