Coverage Report

Created: 2025-12-31 06:48

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/gdal/port/cpl_vsil_gzip.cpp
Line
Count
Source
1
/******************************************************************************
2
 *
3
 * Project:  CPL - Common Portability Library
4
 * Purpose:  Implement VSI large file api for gz/zip files (.gz and .zip).
5
 * Author:   Even Rouault, even.rouault at spatialys.com
6
 *
7
 ******************************************************************************
8
 * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
9
 *
10
 * SPDX-License-Identifier: MIT
11
 ****************************************************************************/
12
13
//! @cond Doxygen_Suppress
14
15
/* gzio.c -- IO on .gz files
16
  Copyright (C) 1995-2005 Jean-loup Gailly.
17
18
  This software is provided 'as-is', without any express or implied
19
  warranty.  In no event will the authors be held liable for any damages
20
  arising from the use of this software.
21
22
  Permission is granted to anyone to use this software for any purpose,
23
  including commercial applications, and to alter it and redistribute it
24
  freely, subject to the following restrictions:
25
26
  1. The origin of this software must not be misrepresented; you must not
27
     claim that you wrote the original software. If you use this software
28
     in a product, an acknowledgment in the product documentation would be
29
     appreciated but is not required.
30
  2. Altered source versions must be plainly marked as such, and must not be
31
     misrepresented as being the original software.
32
  3. This notice may not be removed or altered from any source distribution.
33
34
  Jean-loup Gailly        Mark Adler
35
  jloup@gzip.org          madler@alumni.caltech.edu
36
37
  The data format used by the zlib library is described by RFCs (Request for
38
  Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
39
  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
40
*/
41
42
/* This file contains a refactoring of gzio.c from zlib project.
43
44
   It replaces classical calls operating on FILE* by calls to the VSI large file
45
   API. It also adds the capability to seek at the end of the file, which is not
46
   implemented in original gzSeek. It also implements a concept of in-memory
47
   "snapshots", that are a way of improving efficiency while seeking GZip
48
   files. Snapshots are created regularly when decompressing the data a snapshot
49
   of the gzip state.  Later we can seek directly in the compressed data to the
50
   closest snapshot in order to reduce the amount of data to uncompress again.
51
52
   For .gz files, an effort is done to cache the size of the uncompressed data
53
   in a .gz.properties file, so that we don't need to seek at the end of the
54
   file each time a Stat() is done.
55
56
   For .zip and .gz, both reading and writing are supported, but just one mode
57
   at a time (read-only or write-only).
58
*/
59
60
#include "cpl_port.h"
61
#include "cpl_conv.h"
62
#include "cpl_vsi.h"
63
64
#include <cerrno>
65
#include <cinttypes>
66
#include <climits>
67
#include <cstddef>
68
#include <cstdio>
69
#include <cstdlib>
70
#include <cstring>
71
#include <ctime>
72
73
#include <fcntl.h>
74
75
#include "cpl_zlib_header.h"  // to avoid warnings when including zlib.h
76
77
#ifdef HAVE_LIBDEFLATE
78
#include "libdeflate.h"
79
#endif
80
81
#include <algorithm>
82
#include <iterator>
83
#include <limits>
84
#include <list>
85
#include <map>
86
#include <memory>
87
#include <mutex>
88
#include <string>
89
#include <utility>
90
#include <vector>
91
92
#include "cpl_error.h"
93
#include "cpl_minizip_ioapi.h"
94
#include "cpl_minizip_unzip.h"
95
#include "cpl_multiproc.h"
96
#include "cpl_string.h"
97
#include "cpl_time.h"
98
#include "cpl_vsi_virtual.h"
99
#include "cpl_worker_thread_pool.h"
100
101
constexpr int Z_BUFSIZE = 65536;           // Original size is 16384
102
constexpr int gz_magic[2] = {0x1f, 0x8b};  // gzip magic header
103
104
// gzip flag byte.
105
#define ASCII_FLAG 0x01   // bit 0 set: file probably ascii text
106
0
#define HEAD_CRC 0x02     // bit 1 set: header CRC present
107
0
#define EXTRA_FIELD 0x04  // bit 2 set: extra field present
108
0
#define ORIG_NAME 0x08    // bit 3 set: original file name present
109
0
#define COMMENT 0x10      // bit 4 set: file comment present
110
0
#define RESERVED 0xE0     // bits 5..7: reserved
111
112
0
#define ALLOC(size) malloc(size)
113
#define TRYFREE(p)                                                             \
114
0
    {                                                                          \
115
0
        if (p)                                                                 \
116
0
            free(p);                                                           \
117
0
    }
118
119
#define CPL_VSIL_GZ_RETURN(ret)                                                \
120
0
    CPLError(CE_Failure, CPLE_AppDefined, "In file %s, at line %d, return %d", \
121
0
             __FILE__, __LINE__, ret)
122
123
// To avoid aliasing to CopyFile to CopyFileA on Windows
124
#ifdef CopyFile
125
#undef CopyFile
126
#endif
127
128
// #define ENABLE_DEBUG 1
129
130
/************************************************************************/
131
/* ==================================================================== */
132
/*                       VSIGZipHandle                                  */
133
/* ==================================================================== */
134
/************************************************************************/
135
136
typedef struct
137
{
138
    vsi_l_offset posInBaseHandle;
139
    z_stream stream;
140
    uLong crc;
141
    int transparent;
142
    vsi_l_offset in;
143
    vsi_l_offset out;
144
} GZipSnapshot;
145
146
class VSIGZipHandle final : public VSIVirtualHandle
147
{
148
    VSIVirtualHandleUniquePtr m_poBaseHandle{};
149
#ifdef DEBUG
150
    vsi_l_offset m_offset = 0;
151
#endif
152
    vsi_l_offset m_compressed_size = 0;
153
    vsi_l_offset m_uncompressed_size = 0;
154
    vsi_l_offset offsetEndCompressedData = 0;
155
    uLong m_expected_crc = 0;
156
    char *m_pszBaseFileName = nullptr; /* optional */
157
    bool m_bWriteProperties = false;
158
    bool m_bCanSaveInfo = false;
159
160
    /* Fields from gz_stream structure */
161
    z_stream stream;
162
    int z_err = Z_OK;    /* error code for last stream operation */
163
    int z_eof = 0;       /* set if end of input file (but not necessarily of the
164
                         uncompressed stream !) */
165
    bool m_bEOF = false; /* EOF flag for uncompressed stream */
166
    Byte *inbuf = nullptr;  /* input buffer */
167
    Byte *outbuf = nullptr; /* output buffer */
168
    uLong crc = 0;          /* crc32 of uncompressed data */
169
    int m_transparent = 0;  /* 1 if input file is not a .gz file */
170
    vsi_l_offset startOff =
171
        0; /* startOff of compressed data in file (header skipped) */
172
    vsi_l_offset in = 0;  /* bytes into deflate or inflate */
173
    vsi_l_offset out = 0; /* bytes out of deflate or inflate */
174
    vsi_l_offset m_nLastReadOffset = 0;
175
176
    GZipSnapshot *snapshots = nullptr;
177
    vsi_l_offset snapshot_byte_interval =
178
        0; /* number of compressed bytes at which we create a "snapshot" */
179
180
    void check_header();
181
    int get_byte();
182
    bool gzseek(vsi_l_offset nOffset, int nWhence);
183
    int gzrewind();
184
    uLong getLong();
185
186
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipHandle)
187
188
  public:
189
    VSIGZipHandle(VSIVirtualHandleUniquePtr poBaseHandleIn,
190
                  const char *pszBaseFileName, vsi_l_offset offset = 0,
191
                  vsi_l_offset compressed_size = 0,
192
                  vsi_l_offset uncompressed_size = 0, uLong expected_crc = 0,
193
                  int transparent = 0);
194
    ~VSIGZipHandle() override;
195
196
    bool IsInitOK() const
197
0
    {
198
0
        return inbuf != nullptr;
199
0
    }
200
201
    int Seek(vsi_l_offset nOffset, int nWhence) override;
202
    vsi_l_offset Tell() override;
203
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
204
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
205
    void ClearErr() override;
206
    int Eof() override;
207
    int Error() override;
208
    int Flush() override;
209
    int Close() override;
210
211
    VSIGZipHandle *Duplicate();
212
    bool CloseBaseHandle();
213
214
    vsi_l_offset GetLastReadOffset()
215
0
    {
216
0
        return m_nLastReadOffset;
217
0
    }
218
219
    const char *GetBaseFileName()
220
0
    {
221
0
        return m_pszBaseFileName;
222
0
    }
223
224
    void SetUncompressedSize(vsi_l_offset nUncompressedSize)
225
0
    {
226
0
        m_uncompressed_size = nUncompressedSize;
227
0
    }
228
229
    vsi_l_offset GetUncompressedSize()
230
0
    {
231
0
        return m_uncompressed_size;
232
0
    }
233
234
    void SaveInfo_unlocked();
235
236
    void UnsetCanSaveInfo()
237
0
    {
238
0
        m_bCanSaveInfo = false;
239
0
    }
240
};
241
242
#ifdef ENABLE_DEFLATE64
243
244
/************************************************************************/
245
/* ==================================================================== */
246
/*                           VSIDeflate64Handle                         */
247
/* ==================================================================== */
248
/************************************************************************/
249
250
struct VSIDeflate64Snapshot
251
{
252
    vsi_l_offset posInBaseHandle = 0;
253
    z_stream stream{};
254
    uLong crc = 0;
255
    vsi_l_offset in = 0;
256
    vsi_l_offset out = 0;
257
    std::vector<GByte> extraOutput{};
258
    bool m_bStreamEndReached = false;
259
};
260
261
class VSIDeflate64Handle final : public VSIVirtualHandle
262
{
263
    VSIVirtualHandleUniquePtr m_poBaseHandle{};
264
#ifdef DEBUG
265
    vsi_l_offset m_offset = 0;
266
#endif
267
    vsi_l_offset m_compressed_size = 0;
268
    vsi_l_offset m_uncompressed_size = 0;
269
    vsi_l_offset offsetEndCompressedData = 0;
270
    uLong m_expected_crc = 0;
271
    char *m_pszBaseFileName = nullptr; /* optional */
272
273
    /* Fields from gz_stream structure */
274
    z_stream stream;
275
    int z_err = Z_OK;    /* error code for last stream operation */
276
    int z_eof = 0;       /* set if end of input file (but not necessarily of the
277
                         uncompressed stream ! ) */
278
    bool m_bEOF = false; /* EOF flag for uncompressed stream */
279
    Byte *inbuf = nullptr;  /* input buffer */
280
    Byte *outbuf = nullptr; /* output buffer */
281
    std::vector<GByte> extraOutput{};
282
    bool m_bStreamEndReached = false;
283
    uLong crc = 0; /* crc32 of uncompressed data */
284
    vsi_l_offset startOff =
285
        0; /* startOff of compressed data in file (header skipped) */
286
    vsi_l_offset in = 0;  /* bytes into deflate or inflate */
287
    vsi_l_offset out = 0; /* bytes out of deflate or inflate */
288
289
    std::vector<VSIDeflate64Snapshot> snapshots{};
290
    vsi_l_offset snapshot_byte_interval =
291
        0; /* number of compressed bytes at which we create a "snapshot" */
292
293
    bool gzseek(vsi_l_offset nOffset, int nWhence);
294
    int gzrewind();
295
296
    CPL_DISALLOW_COPY_ASSIGN(VSIDeflate64Handle)
297
298
  public:
299
    VSIDeflate64Handle(VSIVirtualHandleUniquePtr poBaseHandleIn,
300
                       const char *pszBaseFileName, vsi_l_offset offset = 0,
301
                       vsi_l_offset compressed_size = 0,
302
                       vsi_l_offset uncompressed_size = 0,
303
                       uLong expected_crc = 0);
304
    ~VSIDeflate64Handle() override;
305
306
    bool IsInitOK() const
307
0
    {
308
0
        return inbuf != nullptr;
309
0
    }
310
311
    int Seek(vsi_l_offset nOffset, int nWhence) override;
312
    vsi_l_offset Tell() override;
313
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
314
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
315
    void ClearErr() override;
316
    int Eof() override;
317
    int Error() override;
318
    int Flush() override;
319
    int Close() override;
320
321
    VSIDeflate64Handle *Duplicate();
322
    bool CloseBaseHandle();
323
324
    const char *GetBaseFileName()
325
0
    {
326
0
        return m_pszBaseFileName;
327
0
    }
328
329
    void SetUncompressedSize(vsi_l_offset nUncompressedSize)
330
0
    {
331
0
        m_uncompressed_size = nUncompressedSize;
332
0
    }
333
334
    vsi_l_offset GetUncompressedSize()
335
0
    {
336
0
        return m_uncompressed_size;
337
0
    }
338
};
339
#endif
340
341
class VSIGZipFilesystemHandler final : public VSIFilesystemHandler
342
{
343
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipFilesystemHandler)
344
345
    std::recursive_mutex oMutex{};
346
    std::unique_ptr<VSIGZipHandle> poHandleLastGZipFile{};
347
    bool m_bInSaveInfo = false;
348
349
  public:
350
3
    VSIGZipFilesystemHandler() = default;
351
    ~VSIGZipFilesystemHandler() override;
352
353
    VSIVirtualHandleUniquePtr Open(const char *pszFilename,
354
                                   const char *pszAccess, bool bSetError,
355
                                   CSLConstList /* papszOptions */) override;
356
    VSIGZipHandle *OpenGZipReadOnly(const char *pszFilename,
357
                                    const char *pszAccess);
358
    int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
359
             int nFlags) override;
360
    char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
361
362
    const char *GetOptions() override;
363
364
    virtual bool SupportsSequentialWrite(const char *pszPath,
365
                                         bool bAllowLocalTempFile) override;
366
367
    virtual bool SupportsRandomWrite(const char * /* pszPath */,
368
                                     bool /* bAllowLocalTempFile */) override
369
0
    {
370
0
        return false;
371
0
    }
372
373
    void SaveInfo(VSIGZipHandle *poHandle);
374
    void SaveInfo_unlocked(VSIGZipHandle *poHandle);
375
};
376
377
/************************************************************************/
378
/*                            Duplicate()                               */
379
/************************************************************************/
380
381
VSIGZipHandle *VSIGZipHandle::Duplicate()
382
0
{
383
0
    CPLAssert(m_offset == 0);
384
0
    CPLAssert(m_compressed_size != 0);
385
0
    CPLAssert(m_pszBaseFileName != nullptr);
386
387
0
    VSIFilesystemHandler *poFSHandler =
388
0
        VSIFileManager::GetHandler(m_pszBaseFileName);
389
390
0
    auto poNewBaseHandle = poFSHandler->Open(m_pszBaseFileName, "rb");
391
392
0
    if (poNewBaseHandle == nullptr)
393
0
        return nullptr;
394
395
0
    auto poHandle = std::make_unique<VSIGZipHandle>(
396
0
        std::move(poNewBaseHandle), m_pszBaseFileName, 0, m_compressed_size,
397
0
        m_uncompressed_size);
398
0
    if (!(poHandle->IsInitOK()))
399
0
    {
400
0
        return nullptr;
401
0
    }
402
403
0
    poHandle->m_nLastReadOffset = m_nLastReadOffset;
404
405
    // Most important: duplicate the snapshots!
406
407
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
408
0
         i++)
409
0
    {
410
0
        if (snapshots[i].posInBaseHandle == 0)
411
0
            break;
412
413
0
        poHandle->snapshots[i].posInBaseHandle = snapshots[i].posInBaseHandle;
414
0
        inflateCopy(&poHandle->snapshots[i].stream, &snapshots[i].stream);
415
0
        poHandle->snapshots[i].crc = snapshots[i].crc;
416
0
        poHandle->snapshots[i].transparent = snapshots[i].transparent;
417
0
        poHandle->snapshots[i].in = snapshots[i].in;
418
0
        poHandle->snapshots[i].out = snapshots[i].out;
419
0
    }
420
421
0
    return poHandle.release();
422
0
}
423
424
/************************************************************************/
425
/*                     CloseBaseHandle()                                */
426
/************************************************************************/
427
428
bool VSIGZipHandle::CloseBaseHandle()
429
0
{
430
0
    bool bRet = true;
431
0
    if (m_poBaseHandle)
432
0
    {
433
0
        bRet = m_poBaseHandle->Close() == 0;
434
0
        m_poBaseHandle.reset();
435
0
    }
436
0
    return bRet;
437
0
}
438
439
/************************************************************************/
440
/*                       VSIGZipHandle()                                */
441
/************************************************************************/
442
443
VSIGZipHandle::VSIGZipHandle(VSIVirtualHandleUniquePtr poBaseHandleIn,
444
                             const char *pszBaseFileName, vsi_l_offset offset,
445
                             vsi_l_offset compressed_size,
446
                             vsi_l_offset uncompressed_size, uLong expected_crc,
447
                             int transparent)
448
0
    : m_poBaseHandle(std::move(poBaseHandleIn)),
449
#ifdef DEBUG
450
0
      m_offset(offset),
451
#endif
452
0
      m_uncompressed_size(uncompressed_size), m_expected_crc(expected_crc),
453
0
      m_pszBaseFileName(pszBaseFileName ? CPLStrdup(pszBaseFileName) : nullptr),
454
0
      m_bWriteProperties(CPLTestBool(
455
0
          CPLGetConfigOption("CPL_VSIL_GZIP_WRITE_PROPERTIES", "YES"))),
456
      m_bCanSaveInfo(
457
0
          CPLTestBool(CPLGetConfigOption("CPL_VSIL_GZIP_SAVE_INFO", "YES"))),
458
0
      stream(), crc(0), m_transparent(transparent)
459
0
{
460
0
    if (compressed_size || transparent)
461
0
    {
462
0
        m_compressed_size = compressed_size;
463
0
    }
464
0
    else
465
0
    {
466
0
        if (m_poBaseHandle->Seek(0, SEEK_END) != 0)
467
0
        {
468
0
            CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
469
0
            return;
470
0
        }
471
0
        const auto nFileSize = m_poBaseHandle->Tell();
472
0
        if (nFileSize < offset)
473
0
        {
474
0
            CPLError(CE_Failure, CPLE_FileIO, "/vsizip/: invalid file offset");
475
0
            return;
476
0
        }
477
0
        m_compressed_size = nFileSize - offset;
478
0
        compressed_size = m_compressed_size;
479
0
    }
480
0
    offsetEndCompressedData = offset + compressed_size;
481
482
0
    if (m_poBaseHandle->Seek(offset, SEEK_SET) != 0)
483
0
        CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
484
485
0
    stream.zalloc = nullptr;
486
0
    stream.zfree = nullptr;
487
0
    stream.opaque = nullptr;
488
0
    stream.next_in = inbuf = nullptr;
489
0
    stream.next_out = outbuf = nullptr;
490
0
    stream.avail_in = stream.avail_out = 0;
491
492
0
    inbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
493
0
    stream.next_in = inbuf;
494
495
0
    int err = inflateInit2(&(stream), -MAX_WBITS);
496
    // windowBits is passed < 0 to tell that there is no zlib header.
497
    // Note that in this case inflate *requires* an extra "dummy" byte
498
    // after the compressed stream in order to complete decompression and
499
    // return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
500
    // present after the compressed stream.
501
0
    if (err != Z_OK || inbuf == nullptr)
502
0
    {
503
0
        CPLError(CE_Failure, CPLE_NotSupported, "inflateInit2 init failed");
504
0
        TRYFREE(inbuf);
505
0
        inbuf = nullptr;
506
0
        return;
507
0
    }
508
0
    stream.avail_out = static_cast<uInt>(Z_BUFSIZE);
509
510
0
    if (offset == 0)
511
0
        check_header();  // Skip the .gz header.
512
0
    startOff = m_poBaseHandle->Tell() - stream.avail_in;
513
514
0
    if (transparent == 0)
515
0
    {
516
0
        snapshot_byte_interval = std::max(static_cast<vsi_l_offset>(Z_BUFSIZE),
517
0
                                          compressed_size / 100);
518
0
        snapshots = static_cast<GZipSnapshot *>(CPLCalloc(
519
0
            sizeof(GZipSnapshot),
520
0
            static_cast<size_t>(compressed_size / snapshot_byte_interval + 1)));
521
0
    }
522
0
}
523
524
/************************************************************************/
525
/*                      SaveInfo_unlocked()                             */
526
/************************************************************************/
527
528
void VSIGZipHandle::SaveInfo_unlocked()
529
0
{
530
0
    if (m_pszBaseFileName && m_bCanSaveInfo)
531
0
    {
532
0
        VSIFilesystemHandler *poFSHandler =
533
0
            VSIFileManager::GetHandler("/vsigzip/");
534
0
        cpl::down_cast<VSIGZipFilesystemHandler *>(poFSHandler)
535
0
            ->SaveInfo_unlocked(this);
536
0
        m_bCanSaveInfo = false;
537
0
    }
538
0
}
539
540
/************************************************************************/
541
/*                      ~VSIGZipHandle()                                */
542
/************************************************************************/
543
544
VSIGZipHandle::~VSIGZipHandle()
545
0
{
546
0
    if (m_pszBaseFileName && m_bCanSaveInfo)
547
0
    {
548
0
        VSIFilesystemHandler *poFSHandler =
549
0
            VSIFileManager::GetHandler("/vsigzip/");
550
0
        cpl::down_cast<VSIGZipFilesystemHandler *>(poFSHandler)->SaveInfo(this);
551
0
    }
552
553
0
    if (stream.state != nullptr)
554
0
    {
555
0
        inflateEnd(&(stream));
556
0
    }
557
558
0
    TRYFREE(inbuf);
559
0
    TRYFREE(outbuf);
560
561
0
    if (snapshots != nullptr)
562
0
    {
563
0
        for (size_t i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
564
0
             i++)
565
0
        {
566
0
            if (snapshots[i].posInBaseHandle)
567
0
            {
568
0
                inflateEnd(&(snapshots[i].stream));
569
0
            }
570
0
        }
571
0
        CPLFree(snapshots);
572
0
    }
573
0
    CPLFree(m_pszBaseFileName);
574
575
0
    CloseBaseHandle();
576
0
}
577
578
/************************************************************************/
579
/*                      check_header()                                  */
580
/************************************************************************/
581
582
void VSIGZipHandle::check_header()
583
0
{
584
    // Assure two bytes in the buffer so we can peek ahead -- handle case
585
    // where first byte of header is at the end of the buffer after the last
586
    // gzip segment.
587
0
    uInt len = stream.avail_in;
588
0
    if (len < 2)
589
0
    {
590
0
        if (len)
591
0
            inbuf[0] = stream.next_in[0];
592
0
        errno = 0;
593
0
        size_t nToRead = static_cast<size_t>(Z_BUFSIZE - len);
594
0
        CPLAssert(m_poBaseHandle->Tell() <= offsetEndCompressedData);
595
0
        if (m_poBaseHandle->Tell() + nToRead > offsetEndCompressedData)
596
0
            nToRead = static_cast<size_t>(offsetEndCompressedData -
597
0
                                          m_poBaseHandle->Tell());
598
599
0
        len = static_cast<uInt>(m_poBaseHandle->Read(inbuf + len, 1, nToRead));
600
#ifdef ENABLE_DEBUG
601
        CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
602
                 m_poBaseHandle->Tell(), offsetEndCompressedData);
603
#endif
604
0
        if (len == 0)  // && ferror(file)
605
0
        {
606
0
            if (m_poBaseHandle->Tell() != offsetEndCompressedData)
607
0
                z_err = Z_ERRNO;
608
0
        }
609
0
        stream.avail_in += len;
610
0
        stream.next_in = inbuf;
611
0
        if (stream.avail_in < 2)
612
0
        {
613
0
            m_transparent = stream.avail_in;
614
0
            return;
615
0
        }
616
0
    }
617
618
    // Peek ahead to check the gzip magic header.
619
0
    if (stream.next_in[0] != gz_magic[0] || stream.next_in[1] != gz_magic[1])
620
0
    {
621
0
        m_transparent = 1;
622
0
        return;
623
0
    }
624
0
    stream.avail_in -= 2;
625
0
    stream.next_in += 2;
626
627
    // Check the rest of the gzip header.
628
0
    const int method = get_byte();
629
0
    const int flags = get_byte();
630
0
    if (method != Z_DEFLATED || (flags & RESERVED) != 0)
631
0
    {
632
0
        z_err = Z_DATA_ERROR;
633
0
        return;
634
0
    }
635
636
    // Discard time, xflags and OS code:
637
0
    for (len = 0; len < 6; len++)
638
0
        CPL_IGNORE_RET_VAL(get_byte());
639
640
0
    if ((flags & EXTRA_FIELD) != 0)
641
0
    {
642
        // Skip the extra field.
643
0
        len = static_cast<uInt>(get_byte()) & 0xFF;
644
0
        len += (static_cast<uInt>(get_byte()) & 0xFF) << 8;
645
        // len is garbage if EOF but the loop below will quit anyway.
646
0
        while (len != 0 && get_byte() != EOF)
647
0
        {
648
0
            --len;
649
0
        }
650
0
    }
651
652
0
    if ((flags & ORIG_NAME) != 0)
653
0
    {
654
        // Skip the original file name.
655
0
        int c;
656
0
        while ((c = get_byte()) != 0 && c != EOF)
657
0
        {
658
0
        }
659
0
    }
660
0
    if ((flags & COMMENT) != 0)
661
0
    {
662
        // skip the .gz file comment.
663
0
        int c;
664
0
        while ((c = get_byte()) != 0 && c != EOF)
665
0
        {
666
0
        }
667
0
    }
668
0
    if ((flags & HEAD_CRC) != 0)
669
0
    {
670
        // Skip the header crc.
671
0
        for (len = 0; len < 2; len++)
672
0
            CPL_IGNORE_RET_VAL(get_byte());
673
0
    }
674
0
    z_err = z_eof ? Z_DATA_ERROR : Z_OK;
675
0
}
676
677
/************************************************************************/
678
/*                            get_byte()                                */
679
/************************************************************************/
680
681
int VSIGZipHandle::get_byte()
682
0
{
683
0
    if (z_eof)
684
0
        return EOF;
685
0
    if (stream.avail_in == 0)
686
0
    {
687
0
        errno = 0;
688
0
        size_t nToRead = static_cast<size_t>(Z_BUFSIZE);
689
0
        CPLAssert(m_poBaseHandle->Tell() <= offsetEndCompressedData);
690
0
        if (m_poBaseHandle->Tell() + nToRead > offsetEndCompressedData)
691
0
            nToRead = static_cast<size_t>(offsetEndCompressedData -
692
0
                                          m_poBaseHandle->Tell());
693
0
        stream.avail_in =
694
0
            static_cast<uInt>(m_poBaseHandle->Read(inbuf, 1, nToRead));
695
#ifdef ENABLE_DEBUG
696
        CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
697
                 m_poBaseHandle->Tell(), offsetEndCompressedData);
698
#endif
699
0
        if (stream.avail_in == 0)
700
0
        {
701
0
            z_eof = 1;
702
0
            if (m_poBaseHandle->Tell() != offsetEndCompressedData)
703
0
                z_err = Z_ERRNO;
704
            // if( ferror(file) ) z_err = Z_ERRNO;
705
0
            return EOF;
706
0
        }
707
0
        stream.next_in = inbuf;
708
0
    }
709
0
    stream.avail_in--;
710
0
    return *(stream.next_in)++;
711
0
}
712
713
/************************************************************************/
714
/*                            gzrewind()                                */
715
/************************************************************************/
716
717
int VSIGZipHandle::gzrewind()
718
0
{
719
0
    z_err = Z_OK;
720
0
    z_eof = 0;
721
0
    m_bEOF = false;
722
0
    stream.avail_in = 0;
723
0
    stream.next_in = inbuf;
724
0
    crc = 0;
725
0
    if (!m_transparent)
726
0
        CPL_IGNORE_RET_VAL(inflateReset(&stream));
727
0
    in = 0;
728
0
    out = 0;
729
0
    return m_poBaseHandle->Seek(startOff, SEEK_SET);
730
0
}
731
732
/************************************************************************/
733
/*                              Seek()                                  */
734
/************************************************************************/
735
736
int VSIGZipHandle::Seek(vsi_l_offset nOffset, int nWhence)
737
0
{
738
0
    m_bEOF = false;
739
740
0
    return gzseek(nOffset, nWhence) ? 0 : -1;
741
0
}
742
743
/************************************************************************/
744
/*                            gzseek()                                  */
745
/************************************************************************/
746
747
bool VSIGZipHandle::gzseek(vsi_l_offset offset, int whence)
748
0
{
749
0
    const vsi_l_offset original_offset = offset;
750
0
    const int original_nWhence = whence;
751
752
0
    z_eof = 0;
753
#ifdef ENABLE_DEBUG
754
    CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
755
#endif
756
757
0
    if (m_transparent)
758
0
    {
759
0
        stream.avail_in = 0;
760
0
        stream.next_in = inbuf;
761
0
        if (whence == SEEK_CUR)
762
0
        {
763
0
            if (out + offset > m_compressed_size)
764
0
            {
765
0
                CPL_VSIL_GZ_RETURN(FALSE);
766
0
                return false;
767
0
            }
768
769
0
            offset = startOff + out + offset;
770
0
        }
771
0
        else if (whence == SEEK_SET)
772
0
        {
773
0
            if (offset > m_compressed_size)
774
0
            {
775
0
                CPL_VSIL_GZ_RETURN(FALSE);
776
0
                return false;
777
0
            }
778
779
0
            offset = startOff + offset;
780
0
        }
781
0
        else if (whence == SEEK_END)
782
0
        {
783
            // Commented test: because vsi_l_offset is unsigned (for the moment)
784
            // so no way to seek backward. See #1590 */
785
0
            if (offset > 0)  // || -offset > compressed_size
786
0
            {
787
0
                CPL_VSIL_GZ_RETURN(FALSE);
788
0
                return false;
789
0
            }
790
791
0
            offset = startOff + m_compressed_size - offset;
792
0
        }
793
0
        else
794
0
        {
795
0
            CPL_VSIL_GZ_RETURN(FALSE);
796
0
            return false;
797
0
        }
798
799
0
        if (m_poBaseHandle->Seek(offset, SEEK_SET) < 0)
800
0
        {
801
0
            CPL_VSIL_GZ_RETURN(FALSE);
802
0
            return false;
803
0
        }
804
805
0
        out = offset - startOff;
806
0
        in = out;
807
0
        return true;
808
0
    }
809
810
    // whence == SEEK_END is unsuppored in original gzseek.
811
0
    if (whence == SEEK_END)
812
0
    {
813
        // If we known the uncompressed size, we can fake a jump to
814
        // the end of the stream.
815
0
        if (offset == 0 && m_uncompressed_size != 0)
816
0
        {
817
0
            out = m_uncompressed_size;
818
0
            return true;
819
0
        }
820
821
        // We don't know the uncompressed size. This is unfortunate.
822
        // Do the slow version.
823
0
        static int firstWarning = 1;
824
0
        if (m_compressed_size > 10 * 1024 * 1024 && firstWarning)
825
0
        {
826
0
            CPLError(CE_Warning, CPLE_AppDefined,
827
0
                     "VSIFSeekL(xxx, SEEK_END) may be really slow "
828
0
                     "on GZip streams.");
829
0
            firstWarning = 0;
830
0
        }
831
832
0
        whence = SEEK_CUR;
833
0
        offset = 1024 * 1024 * 1024;
834
0
        offset *= 1024 * 1024;
835
0
    }
836
837
    // Rest of function is for reading only.
838
839
    // Compute absolute position.
840
0
    if (whence == SEEK_CUR)
841
0
    {
842
0
        offset += out;
843
0
    }
844
845
    // For a negative seek, rewind and use positive seek.
846
0
    if (offset >= out)
847
0
    {
848
0
        offset -= out;
849
0
    }
850
0
    else if (gzrewind() < 0)
851
0
    {
852
0
        CPL_VSIL_GZ_RETURN(FALSE);
853
0
        return false;
854
0
    }
855
856
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
857
0
    {
858
0
        CPL_VSIL_GZ_RETURN(FALSE);
859
0
        return false;
860
0
    }
861
862
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
863
0
         i++)
864
0
    {
865
0
        if (snapshots[i].posInBaseHandle == 0)
866
0
            break;
867
0
        if (snapshots[i].out <= out + offset &&
868
0
            (i == m_compressed_size / snapshot_byte_interval ||
869
0
             snapshots[i + 1].out == 0 || snapshots[i + 1].out > out + offset))
870
0
        {
871
0
            if (out >= snapshots[i].out)
872
0
                break;
873
874
#ifdef ENABLE_DEBUG
875
            CPLDebug("SNAPSHOT",
876
                     "using snapshot %d : "
877
                     "posInBaseHandle(snapshot)=" CPL_FRMT_GUIB
878
                     " in(snapshot)=" CPL_FRMT_GUIB
879
                     " out(snapshot)=" CPL_FRMT_GUIB " out=" CPL_FRMT_GUIB
880
                     " offset=" CPL_FRMT_GUIB,
881
                     i, snapshots[i].posInBaseHandle, snapshots[i].in,
882
                     snapshots[i].out, out, offset);
883
#endif
884
0
            offset = out + offset - snapshots[i].out;
885
0
            if (m_poBaseHandle->Seek(snapshots[i].posInBaseHandle, SEEK_SET) !=
886
0
                0)
887
0
                CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
888
889
0
            inflateEnd(&stream);
890
0
            inflateCopy(&stream, &snapshots[i].stream);
891
0
            crc = snapshots[i].crc;
892
0
            m_transparent = snapshots[i].transparent;
893
0
            in = snapshots[i].in;
894
0
            out = snapshots[i].out;
895
0
            break;
896
0
        }
897
0
    }
898
899
    // Offset is now the number of bytes to skip.
900
901
0
    if (offset != 0 && outbuf == nullptr)
902
0
    {
903
0
        outbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
904
0
        if (outbuf == nullptr)
905
0
        {
906
0
            CPL_VSIL_GZ_RETURN(FALSE);
907
0
            return false;
908
0
        }
909
0
    }
910
911
0
    if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
912
0
    {
913
0
        return true;
914
0
    }
915
916
0
    while (offset > 0)
917
0
    {
918
0
        int size = Z_BUFSIZE;
919
0
        if (offset < static_cast<vsi_l_offset>(Z_BUFSIZE))
920
0
            size = static_cast<int>(offset);
921
922
0
        int read_size =
923
0
            static_cast<int>(Read(outbuf, 1, static_cast<uInt>(size)));
924
0
        if (original_nWhence == SEEK_END)
925
0
        {
926
0
            if (size != read_size)
927
0
            {
928
0
                z_err = Z_STREAM_END;
929
0
                break;
930
0
            }
931
0
        }
932
0
        else if (read_size == 0)
933
0
        {
934
            // CPL_VSIL_GZ_RETURN(FALSE);
935
0
            return false;
936
0
        }
937
0
        offset -= read_size;
938
0
    }
939
#ifdef ENABLE_DEBUG
940
    CPLDebug("GZIP", "gzseek at offset " CPL_FRMT_GUIB, out);
941
#endif
942
943
0
    if (original_offset == 0 && original_nWhence == SEEK_END)
944
0
    {
945
0
        m_uncompressed_size = out;
946
947
0
        if (m_pszBaseFileName && !STARTS_WITH(m_pszBaseFileName, "/vsicurl/") &&
948
0
            !STARTS_WITH(m_pszBaseFileName, "/vsitar/") &&
949
0
            !STARTS_WITH(m_pszBaseFileName, "/vsizip/") && m_bWriteProperties)
950
0
        {
951
0
            CPLErrorStateBackuper oErrorStateBackuper(CPLQuietErrorHandler);
952
953
0
            CPLString osCacheFilename(m_pszBaseFileName);
954
0
            osCacheFilename += ".properties";
955
956
            // Write a .properties file to avoid seeking next time.
957
0
            VSILFILE *fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "wb");
958
0
            if (fpCacheLength)
959
0
            {
960
0
                char szBuffer[32] = {};
961
962
0
                CPLPrintUIntBig(szBuffer, m_compressed_size, 31);
963
0
                char *pszFirstNonSpace = szBuffer;
964
0
                while (*pszFirstNonSpace == ' ')
965
0
                    pszFirstNonSpace++;
966
0
                CPL_IGNORE_RET_VAL(VSIFPrintfL(
967
0
                    fpCacheLength, "compressed_size=%s\n", pszFirstNonSpace));
968
969
0
                CPLPrintUIntBig(szBuffer, m_uncompressed_size, 31);
970
0
                pszFirstNonSpace = szBuffer;
971
0
                while (*pszFirstNonSpace == ' ')
972
0
                    pszFirstNonSpace++;
973
0
                CPL_IGNORE_RET_VAL(VSIFPrintfL(
974
0
                    fpCacheLength, "uncompressed_size=%s\n", pszFirstNonSpace));
975
976
0
                CPL_IGNORE_RET_VAL(VSIFCloseL(fpCacheLength));
977
0
            }
978
0
        }
979
0
    }
980
981
0
    return true;
982
0
}
983
984
/************************************************************************/
985
/*                              Tell()                                  */
986
/************************************************************************/
987
988
vsi_l_offset VSIGZipHandle::Tell()
989
0
{
990
#ifdef ENABLE_DEBUG
991
    CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
992
#endif
993
0
    return out;
994
0
}
995
996
/************************************************************************/
997
/*                              Read()                                  */
998
/************************************************************************/
999
1000
size_t VSIGZipHandle::Read(void *const buf, size_t const nSize,
1001
                           size_t const nMemb)
1002
0
{
1003
#ifdef ENABLE_DEBUG
1004
    CPLDebug("GZIP", "Read(%p, %d, %d)", buf, static_cast<int>(nSize),
1005
             static_cast<int>(nMemb));
1006
#endif
1007
1008
0
    if (m_bEOF || z_err != Z_OK)
1009
0
    {
1010
0
        if (z_err == Z_STREAM_END && nSize > 0 && nMemb > 0)
1011
0
            m_bEOF = true;
1012
0
        return 0;
1013
0
    }
1014
1015
0
    if (nSize > 0 && nMemb > UINT32_MAX / nSize)
1016
0
    {
1017
0
        CPLError(CE_Failure, CPLE_FileIO, "Too many bytes to read at once");
1018
0
        return 0;
1019
0
    }
1020
1021
0
    const unsigned len =
1022
0
        static_cast<unsigned int>(nSize) * static_cast<unsigned int>(nMemb);
1023
0
    Bytef *pStart =
1024
0
        static_cast<Bytef *>(buf);  // Start off point for crc computation.
1025
    // == stream.next_out but not forced far (for MSDOS).
1026
0
    Byte *next_out = static_cast<Byte *>(buf);
1027
0
    stream.next_out = static_cast<Bytef *>(buf);
1028
0
    stream.avail_out = len;
1029
1030
0
    while (stream.avail_out != 0)
1031
0
    {
1032
0
        if (m_transparent)
1033
0
        {
1034
            // Copy first the lookahead bytes:
1035
0
            uInt nRead = 0;
1036
0
            uInt n = stream.avail_in;
1037
0
            if (n > stream.avail_out)
1038
0
                n = stream.avail_out;
1039
0
            if (n > 0)
1040
0
            {
1041
0
                memcpy(stream.next_out, stream.next_in, n);
1042
0
                next_out += n;
1043
0
                stream.next_out = next_out;
1044
0
                stream.next_in += n;
1045
0
                stream.avail_out -= n;
1046
0
                stream.avail_in -= n;
1047
0
                nRead += n;
1048
0
            }
1049
0
            if (stream.avail_out > 0)
1050
0
            {
1051
0
                const uInt nToRead = static_cast<uInt>(
1052
0
                    std::min(m_compressed_size - (in + nRead),
1053
0
                             static_cast<vsi_l_offset>(stream.avail_out)));
1054
0
                const uInt nReadFromFile = static_cast<uInt>(
1055
0
                    m_poBaseHandle->Read(next_out, 1, nToRead));
1056
0
                if (nReadFromFile < nToRead && m_poBaseHandle->Error())
1057
0
                    z_err = Z_ERRNO;
1058
0
                stream.avail_out -= nReadFromFile;
1059
0
                nRead += nReadFromFile;
1060
0
            }
1061
0
            in += nRead;
1062
0
            out += nRead;
1063
0
            if (nRead < len)
1064
0
            {
1065
0
                m_bEOF = true;
1066
0
                z_eof = 1;
1067
0
            }
1068
#ifdef ENABLE_DEBUG
1069
            CPLDebug("GZIP", "Read return %d", static_cast<int>(nRead / nSize));
1070
#endif
1071
0
            return static_cast<int>(nRead) / nSize;
1072
0
        }
1073
0
        if (stream.avail_in == 0 && !z_eof)
1074
0
        {
1075
0
            vsi_l_offset posInBaseHandle = m_poBaseHandle->Tell();
1076
0
            if (posInBaseHandle - startOff > m_compressed_size)
1077
0
            {
1078
                // If we reach here, file size has changed (because at
1079
                // construction time startOff + m_compressed_size marked the
1080
                // end of file).
1081
                // We should probably have a better fix than that, by detecting
1082
                // at open time that the saved snapshot is not valid and
1083
                // discarding it.
1084
0
                CPLError(CE_Failure, CPLE_AppDefined,
1085
0
                         "File size of underlying /vsigzip/ file has changed");
1086
0
                z_err = Z_ERRNO;
1087
0
                CPL_VSIL_GZ_RETURN(0);
1088
0
                return 0;
1089
0
            }
1090
0
            GZipSnapshot *snapshot = &snapshots[(posInBaseHandle - startOff) /
1091
0
                                                snapshot_byte_interval];
1092
0
            if (snapshot->posInBaseHandle == 0)
1093
0
            {
1094
0
                snapshot->crc = crc32(
1095
0
                    crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1096
#ifdef ENABLE_DEBUG
1097
                CPLDebug("SNAPSHOT",
1098
                         "creating snapshot %d : "
1099
                         "posInBaseHandle=" CPL_FRMT_GUIB " in=" CPL_FRMT_GUIB
1100
                         " out=" CPL_FRMT_GUIB " crc=%X",
1101
                         static_cast<int>((posInBaseHandle - startOff) /
1102
                                          snapshot_byte_interval),
1103
                         posInBaseHandle, in, out,
1104
                         static_cast<unsigned int>(snapshot->crc));
1105
#endif
1106
0
                snapshot->posInBaseHandle = posInBaseHandle;
1107
0
                inflateCopy(&snapshot->stream, &stream);
1108
0
                snapshot->transparent = m_transparent;
1109
0
                snapshot->in = in;
1110
0
                snapshot->out = out;
1111
1112
0
                if (out > m_nLastReadOffset)
1113
0
                    m_nLastReadOffset = out;
1114
0
            }
1115
1116
0
            errno = 0;
1117
0
            stream.avail_in =
1118
0
                static_cast<uInt>(m_poBaseHandle->Read(inbuf, 1, Z_BUFSIZE));
1119
#ifdef ENABLE_DEBUG
1120
            CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
1121
                     m_poBaseHandle->Tell(), offsetEndCompressedData);
1122
#endif
1123
0
            if (m_poBaseHandle->Tell() > offsetEndCompressedData)
1124
0
            {
1125
#ifdef ENABLE_DEBUG
1126
                CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
1127
#endif
1128
0
                stream.avail_in = stream.avail_in -
1129
0
                                  static_cast<uInt>(m_poBaseHandle->Tell() -
1130
0
                                                    offsetEndCompressedData);
1131
0
                if (m_poBaseHandle->Seek(offsetEndCompressedData, SEEK_SET) !=
1132
0
                    0)
1133
0
                    CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1134
#ifdef ENABLE_DEBUG
1135
                CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
1136
#endif
1137
0
            }
1138
0
            if (stream.avail_in == 0)
1139
0
            {
1140
0
                z_eof = 1;
1141
0
                if (m_poBaseHandle->Error() ||
1142
0
                    m_poBaseHandle->Tell() != offsetEndCompressedData)
1143
0
                {
1144
0
                    z_err = Z_ERRNO;
1145
0
                    break;
1146
0
                }
1147
0
            }
1148
0
            stream.next_in = inbuf;
1149
0
        }
1150
0
        in += stream.avail_in;
1151
0
        out += stream.avail_out;
1152
0
        z_err = inflate(&(stream), Z_NO_FLUSH);
1153
0
        in -= stream.avail_in;
1154
0
        out -= stream.avail_out;
1155
1156
0
        if (z_err == Z_STREAM_END && m_compressed_size != 2)
1157
0
        {
1158
            // Check CRC and original size.
1159
0
            crc =
1160
0
                crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1161
0
            pStart = stream.next_out;
1162
0
            if (m_expected_crc)
1163
0
            {
1164
#ifdef ENABLE_DEBUG
1165
                CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X",
1166
                         static_cast<unsigned int>(crc),
1167
                         static_cast<unsigned int>(m_expected_crc));
1168
#endif
1169
0
            }
1170
0
            if (m_expected_crc != 0 && m_expected_crc != crc)
1171
0
            {
1172
0
                CPLError(CE_Failure, CPLE_FileIO,
1173
0
                         "CRC error. Got %X instead of %X",
1174
0
                         static_cast<unsigned int>(crc),
1175
0
                         static_cast<unsigned int>(m_expected_crc));
1176
0
                z_err = Z_DATA_ERROR;
1177
0
            }
1178
0
            else if (m_expected_crc == 0)
1179
0
            {
1180
0
                const uLong read_crc = static_cast<unsigned long>(getLong());
1181
0
                if (read_crc != crc)
1182
0
                {
1183
0
                    CPLError(CE_Failure, CPLE_FileIO,
1184
0
                             "CRC error. Got %X instead of %X",
1185
0
                             static_cast<unsigned int>(crc),
1186
0
                             static_cast<unsigned int>(read_crc));
1187
0
                    z_err = Z_DATA_ERROR;
1188
0
                }
1189
0
                else
1190
0
                {
1191
0
                    CPL_IGNORE_RET_VAL(getLong());
1192
                    // The uncompressed length returned by above getlong() may
1193
                    // be different from out in case of concatenated .gz files.
1194
                    // Check for such files:
1195
0
                    check_header();
1196
0
                    if (z_err == Z_OK)
1197
0
                    {
1198
0
                        inflateReset(&(stream));
1199
0
                        crc = 0;
1200
0
                    }
1201
0
                }
1202
0
            }
1203
0
        }
1204
0
        if (z_err != Z_OK || z_eof)
1205
0
            break;
1206
0
    }
1207
0
    crc = crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1208
1209
0
    size_t ret = (len - stream.avail_out) / nSize;
1210
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
1211
0
    {
1212
0
        CPLError(CE_Failure, CPLE_AppDefined,
1213
0
                 "In file %s, at line %d, decompression failed with "
1214
0
                 "z_err = %d, return = %d",
1215
0
                 __FILE__, __LINE__, z_err, static_cast<int>(ret));
1216
0
    }
1217
0
    else if (ret < nMemb)
1218
0
    {
1219
0
        m_bEOF = true;
1220
0
    }
1221
1222
#ifdef ENABLE_DEBUG
1223
    CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
1224
             static_cast<int>(ret), z_err, z_eof);
1225
#endif
1226
0
    return ret;
1227
0
}
1228
1229
/************************************************************************/
1230
/*                              getLong()                               */
1231
/************************************************************************/
1232
1233
uLong VSIGZipHandle::getLong()
1234
0
{
1235
0
    uLong x = static_cast<uLong>(get_byte()) & 0xFF;
1236
1237
0
    x += (static_cast<uLong>(get_byte()) & 0xFF) << 8;
1238
0
    x += (static_cast<uLong>(get_byte()) & 0xFF) << 16;
1239
0
    const int c = get_byte();
1240
0
    if (c == EOF)
1241
0
    {
1242
0
        z_err = Z_DATA_ERROR;
1243
0
        return 0;
1244
0
    }
1245
0
    x += static_cast<uLong>(c) << 24;
1246
    // coverity[overflow_sink]
1247
0
    return x;
1248
0
}
1249
1250
/************************************************************************/
1251
/*                              Write()                                 */
1252
/************************************************************************/
1253
1254
size_t VSIGZipHandle::Write(const void * /* pBuffer */, size_t /* nSize */,
1255
                            size_t /* nMemb */)
1256
0
{
1257
0
    CPLError(CE_Failure, CPLE_NotSupported,
1258
0
             "VSIFWriteL is not supported on GZip streams");
1259
0
    return 0;
1260
0
}
1261
1262
/************************************************************************/
1263
/*                               Eof()                                  */
1264
/************************************************************************/
1265
1266
int VSIGZipHandle::Eof()
1267
0
{
1268
#ifdef ENABLE_DEBUG
1269
    CPLDebug("GZIP", "Eof()");
1270
#endif
1271
0
    return m_bEOF;
1272
0
}
1273
1274
/************************************************************************/
1275
/*                             Error()                                  */
1276
/************************************************************************/
1277
1278
int VSIGZipHandle::Error()
1279
0
{
1280
#ifdef ENABLE_DEBUG
1281
    CPLDebug("GZIP", "Error()");
1282
#endif
1283
0
    return z_err != Z_OK && z_err != Z_STREAM_END;
1284
0
}
1285
1286
/************************************************************************/
1287
/*                             ClearErr()                               */
1288
/************************************************************************/
1289
1290
void VSIGZipHandle::ClearErr()
1291
0
{
1292
0
    m_poBaseHandle->ClearErr();
1293
0
    z_eof = 0;
1294
0
    m_bEOF = false;
1295
0
    z_err = Z_OK;
1296
0
}
1297
1298
/************************************************************************/
1299
/*                              Flush()                                 */
1300
/************************************************************************/
1301
1302
int VSIGZipHandle::Flush()
1303
0
{
1304
0
    return 0;
1305
0
}
1306
1307
/************************************************************************/
1308
/*                              Close()                                 */
1309
/************************************************************************/
1310
1311
int VSIGZipHandle::Close()
1312
0
{
1313
0
    return 0;
1314
0
}
1315
1316
#ifdef ENABLE_DEFLATE64
1317
1318
/************************************************************************/
1319
/*                            Duplicate()                               */
1320
/************************************************************************/
1321
1322
VSIDeflate64Handle *VSIDeflate64Handle::Duplicate()
1323
0
{
1324
0
    CPLAssert(m_offset == 0);
1325
0
    CPLAssert(m_compressed_size != 0);
1326
0
    CPLAssert(m_pszBaseFileName != nullptr);
1327
1328
0
    VSIFilesystemHandler *poFSHandler =
1329
0
        VSIFileManager::GetHandler(m_pszBaseFileName);
1330
1331
0
    VSIVirtualHandleUniquePtr poNewBaseHandle(
1332
0
        poFSHandler->Open(m_pszBaseFileName, "rb"));
1333
1334
0
    if (poNewBaseHandle == nullptr)
1335
0
        return nullptr;
1336
1337
0
    auto poHandle = std::make_unique<VSIDeflate64Handle>(
1338
0
        std::move(poNewBaseHandle), m_pszBaseFileName, 0, m_compressed_size,
1339
0
        m_uncompressed_size);
1340
0
    if (!(poHandle->IsInitOK()))
1341
0
    {
1342
0
        return nullptr;
1343
0
    }
1344
1345
    // Most important: duplicate the snapshots!
1346
1347
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
1348
0
         i++)
1349
0
    {
1350
0
        if (snapshots[i].posInBaseHandle == 0)
1351
0
            break;
1352
1353
0
        poHandle->snapshots[i].posInBaseHandle = snapshots[i].posInBaseHandle;
1354
0
        if (inflateBack9Copy(&poHandle->snapshots[i].stream,
1355
0
                             &snapshots[i].stream) != Z_OK)
1356
0
            CPLError(CE_Failure, CPLE_AppDefined, "inflateBack9Copy() failed");
1357
0
        poHandle->snapshots[i].crc = snapshots[i].crc;
1358
0
        poHandle->snapshots[i].in = snapshots[i].in;
1359
0
        poHandle->snapshots[i].out = snapshots[i].out;
1360
0
        poHandle->snapshots[i].extraOutput = snapshots[i].extraOutput;
1361
0
        poHandle->snapshots[i].m_bStreamEndReached =
1362
0
            snapshots[i].m_bStreamEndReached;
1363
0
    }
1364
1365
0
    return poHandle.release();
1366
0
}
1367
1368
/************************************************************************/
1369
/*                     CloseBaseHandle()                                */
1370
/************************************************************************/
1371
1372
bool VSIDeflate64Handle::CloseBaseHandle()
1373
0
{
1374
0
    bool bRet = true;
1375
0
    if (m_poBaseHandle)
1376
0
    {
1377
0
        bRet = m_poBaseHandle->Close() == 0;
1378
0
        m_poBaseHandle.reset();
1379
0
    }
1380
0
    return bRet;
1381
0
}
1382
1383
/************************************************************************/
1384
/*                       VSIDeflate64Handle()                           */
1385
/************************************************************************/
1386
1387
VSIDeflate64Handle::VSIDeflate64Handle(VSIVirtualHandleUniquePtr poBaseHandleIn,
1388
                                       const char *pszBaseFileName,
1389
                                       vsi_l_offset offset,
1390
                                       vsi_l_offset compressed_size,
1391
                                       vsi_l_offset uncompressed_size,
1392
                                       uLong expected_crc)
1393
0
    : m_poBaseHandle(std::move(poBaseHandleIn)),
1394
#ifdef DEBUG
1395
0
      m_offset(offset),
1396
#endif
1397
0
      m_uncompressed_size(uncompressed_size), m_expected_crc(expected_crc),
1398
0
      m_pszBaseFileName(pszBaseFileName ? CPLStrdup(pszBaseFileName) : nullptr),
1399
0
      stream(), crc(0)
1400
0
{
1401
0
    if (compressed_size)
1402
0
    {
1403
0
        m_compressed_size = compressed_size;
1404
0
    }
1405
0
    else
1406
0
    {
1407
0
        if (m_poBaseHandle->Seek(0, SEEK_END) != 0)
1408
0
            CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1409
0
        m_compressed_size = m_poBaseHandle->Tell() - offset;
1410
0
        compressed_size = m_compressed_size;
1411
0
    }
1412
0
    offsetEndCompressedData = offset + compressed_size;
1413
1414
0
    if (m_poBaseHandle->Seek(offset, SEEK_SET) != 0)
1415
0
        CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1416
1417
0
    stream.zalloc = nullptr;
1418
0
    stream.zfree = nullptr;
1419
0
    stream.opaque = nullptr;
1420
0
    stream.next_in = inbuf = nullptr;
1421
0
    stream.next_out = outbuf = nullptr;
1422
0
    stream.avail_in = stream.avail_out = 0;
1423
1424
0
    inbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
1425
0
    stream.next_in = inbuf;
1426
1427
0
    int err = inflateBack9Init(&(stream), nullptr);
1428
    // Note that in this case inflate *requires* an extra "dummy" byte
1429
    // after the compressed stream in order to complete decompression and
1430
    // return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
1431
    // present after the compressed stream.
1432
0
    if (err != Z_OK || inbuf == nullptr)
1433
0
    {
1434
0
        CPLError(CE_Failure, CPLE_NotSupported, "inflateBack9Init init failed");
1435
0
        TRYFREE(inbuf);
1436
0
        inbuf = nullptr;
1437
0
        return;
1438
0
    }
1439
0
    startOff = m_poBaseHandle->Tell() - stream.avail_in;
1440
1441
0
    snapshot_byte_interval =
1442
0
        std::max(static_cast<vsi_l_offset>(Z_BUFSIZE), compressed_size / 100);
1443
0
    snapshots.resize(
1444
0
        static_cast<size_t>(compressed_size / snapshot_byte_interval + 1));
1445
0
}
1446
1447
/************************************************************************/
1448
/*                      ~VSIDeflate64Handle()                       */
1449
/************************************************************************/
1450
1451
VSIDeflate64Handle::~VSIDeflate64Handle()
1452
0
{
1453
0
    if (stream.state != nullptr)
1454
0
    {
1455
0
        inflateBack9End(&(stream));
1456
0
    }
1457
1458
0
    TRYFREE(inbuf);
1459
0
    TRYFREE(outbuf);
1460
1461
0
    for (auto &snapshot : snapshots)
1462
0
    {
1463
0
        if (snapshot.posInBaseHandle)
1464
0
        {
1465
0
            inflateBack9End(&(snapshot.stream));
1466
0
        }
1467
0
    }
1468
0
    CPLFree(m_pszBaseFileName);
1469
1470
0
    CloseBaseHandle();
1471
0
}
1472
1473
/************************************************************************/
1474
/*                            gzrewind()                                */
1475
/************************************************************************/
1476
1477
int VSIDeflate64Handle::gzrewind()
1478
0
{
1479
0
    m_bStreamEndReached = false;
1480
0
    extraOutput.clear();
1481
0
    z_err = Z_OK;
1482
0
    z_eof = 0;
1483
0
    stream.avail_in = 0;
1484
0
    stream.next_in = inbuf;
1485
0
    crc = 0;
1486
0
    CPL_IGNORE_RET_VAL(inflateBack9End(&stream));
1487
0
    CPL_IGNORE_RET_VAL(inflateBack9Init(&stream, nullptr));
1488
0
    in = 0;
1489
0
    out = 0;
1490
0
    return m_poBaseHandle->Seek(startOff, SEEK_SET);
1491
0
}
1492
1493
/************************************************************************/
1494
/*                              Seek()                                  */
1495
/************************************************************************/
1496
1497
int VSIDeflate64Handle::Seek(vsi_l_offset nOffset, int nWhence)
1498
0
{
1499
0
    m_bEOF = false;
1500
0
    return gzseek(nOffset, nWhence) ? 0 : -1;
1501
0
}
1502
1503
/************************************************************************/
1504
/*                            gzseek()                                  */
1505
/************************************************************************/
1506
1507
bool VSIDeflate64Handle::gzseek(vsi_l_offset offset, int whence)
1508
0
{
1509
0
    const vsi_l_offset original_offset = offset;
1510
0
    const int original_nWhence = whence;
1511
1512
0
    z_eof = 0;
1513
#ifdef ENABLE_DEBUG
1514
    CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
1515
#endif
1516
1517
    // whence == SEEK_END is unsuppored in original gzseek.
1518
0
    if (whence == SEEK_END)
1519
0
    {
1520
        // If we known the uncompressed size, we can fake a jump to
1521
        // the end of the stream.
1522
0
        if (offset == 0 && m_uncompressed_size != 0)
1523
0
        {
1524
0
            out = m_uncompressed_size;
1525
0
            return true;
1526
0
        }
1527
1528
        // We don't know the uncompressed size. This is unfortunate.
1529
        // Do the slow version.
1530
0
        static int firstWarning = 1;
1531
0
        if (m_compressed_size > 10 * 1024 * 1024 && firstWarning)
1532
0
        {
1533
0
            CPLError(CE_Warning, CPLE_AppDefined,
1534
0
                     "VSIFSeekL(xxx, SEEK_END) may be really slow "
1535
0
                     "on GZip streams.");
1536
0
            firstWarning = 0;
1537
0
        }
1538
1539
0
        whence = SEEK_CUR;
1540
0
        offset = 1024 * 1024 * 1024;
1541
0
        offset *= 1024 * 1024;
1542
0
    }
1543
1544
    // Rest of function is for reading only.
1545
1546
    // Compute absolute position.
1547
0
    if (whence == SEEK_CUR)
1548
0
    {
1549
0
        offset += out;
1550
0
    }
1551
1552
    // For a negative seek, rewind and use positive seek.
1553
0
    if (offset >= out)
1554
0
    {
1555
0
        offset -= out;
1556
0
    }
1557
0
    else if (gzrewind() < 0)
1558
0
    {
1559
0
        CPL_VSIL_GZ_RETURN(FALSE);
1560
0
        return false;
1561
0
    }
1562
1563
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
1564
0
    {
1565
0
        CPL_VSIL_GZ_RETURN(FALSE);
1566
0
        return false;
1567
0
    }
1568
1569
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
1570
0
         i++)
1571
0
    {
1572
0
        if (snapshots[i].posInBaseHandle == 0)
1573
0
            break;
1574
0
        if (snapshots[i].out <= out + offset &&
1575
0
            (i == m_compressed_size / snapshot_byte_interval ||
1576
0
             snapshots[i + 1].out == 0 || snapshots[i + 1].out > out + offset))
1577
0
        {
1578
0
            if (out >= snapshots[i].out)
1579
0
                break;
1580
1581
#ifdef ENABLE_DEBUG
1582
            CPLDebug("SNAPSHOT",
1583
                     "using snapshot %d : "
1584
                     "posInBaseHandle(snapshot)=" CPL_FRMT_GUIB
1585
                     " in(snapshot)=" CPL_FRMT_GUIB
1586
                     " out(snapshot)=" CPL_FRMT_GUIB " out=" CPL_FRMT_GUIB
1587
                     " offset=" CPL_FRMT_GUIB,
1588
                     i, snapshots[i].posInBaseHandle, snapshots[i].in,
1589
                     snapshots[i].out, out, offset);
1590
#endif
1591
0
            offset = out + offset - snapshots[i].out;
1592
0
            if (m_poBaseHandle->Seek(snapshots[i].posInBaseHandle, SEEK_SET) !=
1593
0
                0)
1594
0
                CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1595
1596
0
            inflateBack9End(&stream);
1597
0
            if (inflateBack9Copy(&stream, &snapshots[i].stream) != Z_OK)
1598
0
                CPLError(CE_Failure, CPLE_AppDefined,
1599
0
                         "inflateBack9Copy() failed");
1600
0
            crc = snapshots[i].crc;
1601
0
            in = snapshots[i].in;
1602
0
            out = snapshots[i].out;
1603
0
            extraOutput = snapshots[i].extraOutput;
1604
0
            m_bStreamEndReached = snapshots[i].m_bStreamEndReached;
1605
0
            break;
1606
0
        }
1607
0
    }
1608
1609
    // Offset is now the number of bytes to skip.
1610
1611
0
    if (offset != 0 && outbuf == nullptr)
1612
0
    {
1613
0
        outbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
1614
0
        if (outbuf == nullptr)
1615
0
        {
1616
0
            CPL_VSIL_GZ_RETURN(FALSE);
1617
0
            return false;
1618
0
        }
1619
0
    }
1620
1621
0
    if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
1622
0
    {
1623
0
        return true;
1624
0
    }
1625
1626
0
    while (offset > 0)
1627
0
    {
1628
0
        int size = Z_BUFSIZE;
1629
0
        if (offset < static_cast<vsi_l_offset>(Z_BUFSIZE))
1630
0
            size = static_cast<int>(offset);
1631
1632
0
        int read_size =
1633
0
            static_cast<int>(Read(outbuf, 1, static_cast<uInt>(size)));
1634
0
        if (original_nWhence == SEEK_END)
1635
0
        {
1636
0
            if (size != read_size)
1637
0
            {
1638
0
                z_err = Z_STREAM_END;
1639
0
                break;
1640
0
            }
1641
0
        }
1642
0
        else if (read_size == 0)
1643
0
        {
1644
            // CPL_VSIL_GZ_RETURN(FALSE);
1645
0
            return false;
1646
0
        }
1647
0
        offset -= read_size;
1648
0
    }
1649
#ifdef ENABLE_DEBUG
1650
    CPLDebug("GZIP", "gzseek at offset " CPL_FRMT_GUIB, out);
1651
#endif
1652
1653
0
    if (original_offset == 0 && original_nWhence == SEEK_END)
1654
0
    {
1655
0
        m_uncompressed_size = out;
1656
0
    }
1657
1658
0
    return true;
1659
0
}
1660
1661
/************************************************************************/
1662
/*                              Tell()                                  */
1663
/************************************************************************/
1664
1665
vsi_l_offset VSIDeflate64Handle::Tell()
1666
0
{
1667
#ifdef ENABLE_DEBUG
1668
    CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
1669
#endif
1670
0
    return out;
1671
0
}
1672
1673
/************************************************************************/
1674
/*                              Read()                                  */
1675
/************************************************************************/
1676
1677
size_t VSIDeflate64Handle::Read(void *const buf, size_t const nSize,
1678
                                size_t const nMemb)
1679
0
{
1680
#ifdef ENABLE_DEBUG
1681
    CPLDebug("GZIP", "Read(%p, %d, %d)", buf, static_cast<int>(nSize),
1682
             static_cast<int>(nMemb));
1683
#endif
1684
1685
0
    if (m_bEOF || z_err != Z_OK)
1686
0
    {
1687
0
        if (z_err == Z_STREAM_END && nSize > 0 && nMemb > 0)
1688
0
            m_bEOF = true;
1689
0
        return 0;
1690
0
    }
1691
1692
0
    if (nSize > 0 && nMemb > UINT32_MAX / nSize)
1693
0
    {
1694
0
        CPLError(CE_Failure, CPLE_FileIO, "Too many bytes to read at once");
1695
0
        return 0;
1696
0
    }
1697
1698
0
    const unsigned len =
1699
0
        static_cast<unsigned int>(nSize) * static_cast<unsigned int>(nMemb);
1700
0
    Bytef *pStart =
1701
0
        static_cast<Bytef *>(buf);  // Start off point for crc computation.
1702
    // == stream.next_out but not forced far (for MSDOS).
1703
0
    stream.next_out = static_cast<Bytef *>(buf);
1704
0
    stream.avail_out = len;
1705
1706
0
    while (stream.avail_out != 0)
1707
0
    {
1708
0
        if (!extraOutput.empty())
1709
0
        {
1710
0
            if (extraOutput.size() >= stream.avail_out)
1711
0
            {
1712
0
                memcpy(stream.next_out, extraOutput.data(), stream.avail_out);
1713
0
                extraOutput.erase(extraOutput.begin(),
1714
0
                                  extraOutput.begin() + stream.avail_out);
1715
0
                out += stream.avail_out;
1716
0
                stream.next_out += stream.avail_out;
1717
0
                stream.avail_out = 0;
1718
0
            }
1719
0
            else
1720
0
            {
1721
0
                memcpy(stream.next_out, extraOutput.data(), extraOutput.size());
1722
0
                stream.next_out += extraOutput.size();
1723
0
                out += static_cast<uInt>(extraOutput.size());
1724
0
                stream.avail_out -= static_cast<uInt>(extraOutput.size());
1725
0
                CPLAssert(stream.avail_out > 0);
1726
0
                extraOutput.clear();
1727
0
            }
1728
0
            z_err = Z_OK;
1729
0
        }
1730
1731
0
        if (stream.avail_in == 0 && !z_eof)
1732
0
        {
1733
0
            vsi_l_offset posInBaseHandle = m_poBaseHandle->Tell();
1734
0
            if (posInBaseHandle - startOff > m_compressed_size)
1735
0
            {
1736
                // If we reach here, file size has changed (because at
1737
                // construction time startOff + m_compressed_size marked the
1738
                // end of file).
1739
                // We should probably have a better fix than that, by detecting
1740
                // at open time that the saved snapshot is not valid and
1741
                // discarding it.
1742
0
                CPLError(CE_Failure, CPLE_AppDefined,
1743
0
                         "File size of underlying /vsigzip/ file has changed");
1744
0
                z_err = Z_ERRNO;
1745
0
                CPL_VSIL_GZ_RETURN(0);
1746
0
                return 0;
1747
0
            }
1748
0
            auto snapshot = &snapshots[static_cast<size_t>(
1749
0
                (posInBaseHandle - startOff) / snapshot_byte_interval)];
1750
0
            if (snapshot->posInBaseHandle == 0)
1751
0
            {
1752
0
                snapshot->crc = crc32(
1753
0
                    crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1754
#ifdef ENABLE_DEBUG
1755
                CPLDebug("SNAPSHOT",
1756
                         "creating snapshot %d : "
1757
                         "posInBaseHandle=" CPL_FRMT_GUIB " in=" CPL_FRMT_GUIB
1758
                         " out=" CPL_FRMT_GUIB " crc=%X",
1759
                         static_cast<int>((posInBaseHandle - startOff) /
1760
                                          snapshot_byte_interval),
1761
                         posInBaseHandle, in, out,
1762
                         static_cast<unsigned int>(snapshot->crc));
1763
#endif
1764
0
                snapshot->posInBaseHandle = posInBaseHandle;
1765
0
                if (inflateBack9Copy(&snapshot->stream, &stream) != Z_OK)
1766
0
                    CPLError(CE_Failure, CPLE_AppDefined,
1767
0
                             "inflateBack9Copy() failed");
1768
0
                snapshot->in = in;
1769
0
                snapshot->out = out;
1770
0
                snapshot->extraOutput = extraOutput;
1771
0
                snapshot->m_bStreamEndReached = m_bStreamEndReached;
1772
0
            }
1773
1774
0
            errno = 0;
1775
0
            stream.avail_in =
1776
0
                static_cast<uInt>(m_poBaseHandle->Read(inbuf, 1, Z_BUFSIZE));
1777
#ifdef ENABLE_DEBUG
1778
            CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
1779
                     m_poBaseHandle->Tell(), offsetEndCompressedData);
1780
#endif
1781
0
            if (m_poBaseHandle->Tell() > offsetEndCompressedData)
1782
0
            {
1783
#ifdef ENABLE_DEBUG
1784
                CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
1785
#endif
1786
0
                stream.avail_in = stream.avail_in -
1787
0
                                  static_cast<uInt>(m_poBaseHandle->Tell() -
1788
0
                                                    offsetEndCompressedData);
1789
0
                if (m_poBaseHandle->Seek(offsetEndCompressedData, SEEK_SET) !=
1790
0
                    0)
1791
0
                    CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1792
#ifdef ENABLE_DEBUG
1793
                CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
1794
#endif
1795
0
            }
1796
0
            if (stream.avail_in == 0)
1797
0
            {
1798
0
                z_eof = 1;
1799
0
                if (m_poBaseHandle->Error() ||
1800
0
                    m_poBaseHandle->Tell() != offsetEndCompressedData)
1801
0
                {
1802
0
                    z_err = Z_ERRNO;
1803
0
                    break;
1804
0
                }
1805
0
            }
1806
0
            stream.next_in = inbuf;
1807
0
        }
1808
1809
0
        struct InOutCallback
1810
0
        {
1811
0
            vsi_l_offset *pOut = nullptr;
1812
0
            std::vector<GByte> *pExtraOutput = nullptr;
1813
0
            z_stream *pStream = nullptr;
1814
1815
0
            static unsigned inCbk(void FAR *, z_const unsigned char FAR *FAR *)
1816
0
            {
1817
0
                return 0;
1818
0
            }
1819
1820
0
            static int outCbk(void FAR *user_data, unsigned char FAR *data,
1821
0
                              unsigned len)
1822
0
            {
1823
0
                auto self = static_cast<InOutCallback *>(user_data);
1824
0
                if (self->pStream->avail_out >= len)
1825
0
                {
1826
0
                    memcpy(self->pStream->next_out, data, len);
1827
0
                    *(self->pOut) += len;
1828
0
                    self->pStream->next_out += len;
1829
0
                    self->pStream->avail_out -= len;
1830
0
                }
1831
0
                else
1832
0
                {
1833
0
                    if (self->pStream->avail_out != 0)
1834
0
                    {
1835
0
                        memcpy(self->pStream->next_out, data,
1836
0
                               self->pStream->avail_out);
1837
0
                        *(self->pOut) += self->pStream->avail_out;
1838
0
                        data += self->pStream->avail_out;
1839
0
                        len -= self->pStream->avail_out;
1840
0
                        self->pStream->next_out += self->pStream->avail_out;
1841
0
                        self->pStream->avail_out = 0;
1842
0
                    }
1843
0
                    if (len > 0)
1844
0
                    {
1845
0
                        self->pExtraOutput->insert(self->pExtraOutput->end(),
1846
0
                                                   data, data + len);
1847
0
                    }
1848
0
                }
1849
0
                return 0;
1850
0
            }
1851
0
        };
1852
1853
0
        InOutCallback cbkData;
1854
0
        cbkData.pOut = &out;
1855
0
        cbkData.pExtraOutput = &extraOutput;
1856
0
        cbkData.pStream = &stream;
1857
1858
0
        if (stream.avail_out)
1859
0
        {
1860
0
            if (m_bStreamEndReached)
1861
0
                z_err = Z_STREAM_END;
1862
0
            else
1863
0
            {
1864
0
                in += stream.avail_in;
1865
0
                z_err = inflateBack9(&(stream), InOutCallback::inCbk, &cbkData,
1866
0
                                     InOutCallback::outCbk, &cbkData);
1867
0
                in -= stream.avail_in;
1868
0
            }
1869
0
        }
1870
0
        if (z_err == Z_BUF_ERROR && stream.next_in == Z_NULL)
1871
0
            z_err = Z_OK;
1872
0
        else if (!extraOutput.empty() && z_err == Z_STREAM_END)
1873
0
        {
1874
0
            m_bStreamEndReached = true;
1875
0
            z_err = Z_OK;
1876
0
        }
1877
1878
0
        if (z_err == Z_STREAM_END /*&& m_compressed_size != 2*/)
1879
0
        {
1880
            // Check CRC and original size.
1881
0
            crc =
1882
0
                crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1883
0
            pStart = stream.next_out;
1884
0
            if (m_expected_crc)
1885
0
            {
1886
#ifdef ENABLE_DEBUG
1887
                CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X",
1888
                         static_cast<unsigned int>(crc),
1889
                         static_cast<unsigned int>(m_expected_crc));
1890
#endif
1891
0
            }
1892
0
            if (m_expected_crc != 0 && m_expected_crc != crc)
1893
0
            {
1894
0
                CPLError(CE_Failure, CPLE_FileIO,
1895
0
                         "CRC error. Got %X instead of %X",
1896
0
                         static_cast<unsigned int>(crc),
1897
0
                         static_cast<unsigned int>(m_expected_crc));
1898
0
                z_err = Z_DATA_ERROR;
1899
0
            }
1900
0
        }
1901
0
        if (z_err != Z_OK || z_eof)
1902
0
            break;
1903
0
    }
1904
0
    crc = crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1905
1906
0
    size_t ret = (len - stream.avail_out) / nSize;
1907
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
1908
0
    {
1909
0
        CPLError(CE_Failure, CPLE_AppDefined,
1910
0
                 "In file %s, at line %d, decompression failed with "
1911
0
                 "z_err = %d, return = %d",
1912
0
                 __FILE__, __LINE__, z_err, static_cast<int>(ret));
1913
0
    }
1914
0
    else if (ret < nMemb)
1915
0
    {
1916
0
        m_bEOF = true;
1917
0
    }
1918
1919
#ifdef ENABLE_DEBUG
1920
    CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
1921
             static_cast<int>(ret), z_err, z_eof);
1922
#endif
1923
0
    return ret;
1924
0
}
1925
1926
/************************************************************************/
1927
/*                              Write()                                 */
1928
/************************************************************************/
1929
1930
size_t VSIDeflate64Handle::Write(const void * /* pBuffer */, size_t /* nSize */,
1931
                                 size_t /* nMemb */)
1932
0
{
1933
0
    CPLError(CE_Failure, CPLE_NotSupported,
1934
0
             "VSIFWriteL is not supported on GZip streams");
1935
0
    return 0;
1936
0
}
1937
1938
/************************************************************************/
1939
/*                               Eof()                                  */
1940
/************************************************************************/
1941
1942
int VSIDeflate64Handle::Eof()
1943
0
{
1944
#ifdef ENABLE_DEBUG
1945
    CPLDebug("GZIP", "Eof()");
1946
#endif
1947
0
    return m_bEOF;
1948
0
}
1949
1950
/************************************************************************/
1951
/*                             Error()                                  */
1952
/************************************************************************/
1953
1954
int VSIDeflate64Handle::Error()
1955
0
{
1956
#ifdef ENABLE_DEBUG
1957
    CPLDebug("GZIP", "Error()");
1958
#endif
1959
0
    return z_err != Z_OK && z_err != Z_STREAM_END;
1960
0
}
1961
1962
/************************************************************************/
1963
/*                             ClearErr()                               */
1964
/************************************************************************/
1965
1966
void VSIDeflate64Handle::ClearErr()
1967
0
{
1968
0
    m_poBaseHandle->ClearErr();
1969
0
    z_eof = 0;
1970
0
    m_bEOF = false;
1971
0
    z_err = Z_OK;
1972
0
}
1973
1974
/************************************************************************/
1975
/*                              Flush()                                 */
1976
/************************************************************************/
1977
1978
int VSIDeflate64Handle::Flush()
1979
0
{
1980
0
    return 0;
1981
0
}
1982
1983
/************************************************************************/
1984
/*                              Close()                                 */
1985
/************************************************************************/
1986
1987
int VSIDeflate64Handle::Close()
1988
0
{
1989
0
    return 0;
1990
0
}
1991
#endif
1992
1993
/************************************************************************/
1994
/* ==================================================================== */
1995
/*                       VSIGZipWriteHandleMT                           */
1996
/* ==================================================================== */
1997
/************************************************************************/
1998
1999
class VSIGZipWriteHandleMT final : public VSIVirtualHandle
2000
{
2001
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipWriteHandleMT)
2002
2003
    VSIVirtualHandle *poBaseHandle_ = nullptr;
2004
    vsi_l_offset nCurOffset_ = 0;
2005
    uLong nCRC_ = 0;
2006
    int nDeflateType_ = CPL_DEFLATE_TYPE_GZIP;
2007
    bool bAutoCloseBaseHandle_ = false;
2008
    int nThreads_ = 0;
2009
    std::unique_ptr<CPLWorkerThreadPool> poPool_{};
2010
    std::list<std::string *> aposBuffers_{};
2011
    std::string *pCurBuffer_ = nullptr;
2012
    std::mutex sMutex_{};
2013
    int nSeqNumberGenerated_ = 0;
2014
    int nSeqNumberExpected_ = 0;
2015
    int nSeqNumberExpectedCRC_ = 0;
2016
    size_t nChunkSize_ = 0;
2017
    bool bHasErrored_ = false;
2018
2019
    struct Job
2020
    {
2021
        VSIGZipWriteHandleMT *pParent_ = nullptr;
2022
        std::string *pBuffer_ = nullptr;
2023
        int nSeqNumber_ = 0;
2024
        bool bFinish_ = false;
2025
        bool bInCRCComputation_ = false;
2026
2027
        std::string sCompressedData_{};
2028
        uLong nCRC_ = 0;
2029
    };
2030
2031
    std::list<Job *> apoFinishedJobs_{};
2032
    std::list<Job *> apoCRCFinishedJobs_{};
2033
    std::list<Job *> apoFreeJobs_{};
2034
    vsi_l_offset nStartOffset_ = 0;
2035
    size_t nSOZIPIndexEltSize_ = 0;
2036
    std::vector<uint8_t> *panSOZIPIndex_ = nullptr;
2037
2038
    static void DeflateCompress(void *inData);
2039
    static void CRCCompute(void *inData);
2040
    bool ProcessCompletedJobs();
2041
    Job *GetJobObject();
2042
#ifdef DEBUG_VERBOSE
2043
    void DumpState();
2044
#endif
2045
2046
  public:
2047
    VSIGZipWriteHandleMT(VSIVirtualHandle *poBaseHandle, int nDeflateType,
2048
                         bool bAutoCloseBaseHandleIn, int nThreads,
2049
                         size_t nChunkSize, size_t nSOZIPIndexEltSize,
2050
                         std::vector<uint8_t> *panSOZIPIndex);
2051
2052
    ~VSIGZipWriteHandleMT() override;
2053
2054
    int Seek(vsi_l_offset nOffset, int nWhence) override;
2055
    vsi_l_offset Tell() override;
2056
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
2057
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
2058
2059
    int Eof() override
2060
0
    {
2061
0
        return 0;
2062
0
    }
2063
2064
    int Error() override
2065
0
    {
2066
0
        return 0;
2067
0
    }
2068
2069
    void ClearErr() override
2070
0
    {
2071
0
    }
2072
2073
    int Flush() override;
2074
    int Close() override;
2075
};
2076
2077
/************************************************************************/
2078
/*                        VSIGZipWriteHandleMT()                        */
2079
/************************************************************************/
2080
2081
VSIGZipWriteHandleMT::VSIGZipWriteHandleMT(VSIVirtualHandle *poBaseHandle,
2082
                                           int nDeflateType,
2083
                                           bool bAutoCloseBaseHandleIn,
2084
                                           int nThreads, size_t nChunkSize,
2085
                                           size_t nSOZIPIndexEltSize,
2086
                                           std::vector<uint8_t> *panSOZIPIndex)
2087
0
    : poBaseHandle_(poBaseHandle), nDeflateType_(nDeflateType),
2088
0
      bAutoCloseBaseHandle_(bAutoCloseBaseHandleIn), nThreads_(nThreads),
2089
0
      nChunkSize_(nChunkSize), nSOZIPIndexEltSize_(nSOZIPIndexEltSize),
2090
0
      panSOZIPIndex_(panSOZIPIndex)
2091
0
{
2092
0
    if (nChunkSize_ == 0)
2093
0
    {
2094
0
        const char *pszChunkSize =
2095
0
            CPLGetConfigOption("CPL_VSIL_DEFLATE_CHUNK_SIZE", "1024K");
2096
0
        nChunkSize_ = static_cast<size_t>(atoi(pszChunkSize));
2097
0
        if (strchr(pszChunkSize, 'K'))
2098
0
            nChunkSize_ *= 1024;
2099
0
        else if (strchr(pszChunkSize, 'M'))
2100
0
            nChunkSize_ *= 1024 * 1024;
2101
0
        nChunkSize_ =
2102
0
            std::max(static_cast<size_t>(4 * 1024),
2103
0
                     std::min(static_cast<size_t>(UINT_MAX), nChunkSize_));
2104
0
    }
2105
2106
0
    for (int i = 0; i < 1 + nThreads_; i++)
2107
0
        aposBuffers_.emplace_back(new std::string());
2108
2109
0
    nStartOffset_ = poBaseHandle_->Tell();
2110
0
    if (nDeflateType == CPL_DEFLATE_TYPE_GZIP)
2111
0
    {
2112
0
        char header[11] = {};
2113
2114
        // Write a very simple .gz header:
2115
0
        snprintf(header, sizeof(header), "%c%c%c%c%c%c%c%c%c%c", gz_magic[0],
2116
0
                 gz_magic[1], Z_DEFLATED, 0 /*flags*/, 0, 0, 0, 0 /*time*/,
2117
0
                 0 /*xflags*/, 0x03);
2118
0
        poBaseHandle_->Write(header, 1, 10);
2119
0
    }
2120
0
}
2121
2122
/************************************************************************/
2123
/*                       ~VSIGZipWriteHandleMT()                        */
2124
/************************************************************************/
2125
2126
VSIGZipWriteHandleMT::~VSIGZipWriteHandleMT()
2127
2128
0
{
2129
0
    VSIGZipWriteHandleMT::Close();
2130
0
    for (auto &psJob : apoFinishedJobs_)
2131
0
    {
2132
0
        delete psJob->pBuffer_;
2133
0
        delete psJob;
2134
0
    }
2135
0
    for (auto &psJob : apoCRCFinishedJobs_)
2136
0
    {
2137
0
        delete psJob->pBuffer_;
2138
0
        delete psJob;
2139
0
    }
2140
0
    for (auto &psJob : apoFreeJobs_)
2141
0
    {
2142
0
        delete psJob->pBuffer_;
2143
0
        delete psJob;
2144
0
    }
2145
0
    for (auto &pstr : aposBuffers_)
2146
0
    {
2147
0
        delete pstr;
2148
0
    }
2149
0
    delete pCurBuffer_;
2150
0
}
2151
2152
/************************************************************************/
2153
/*                               Close()                                */
2154
/************************************************************************/
2155
2156
int VSIGZipWriteHandleMT::Close()
2157
2158
0
{
2159
0
    if (!poBaseHandle_)
2160
0
        return 0;
2161
2162
0
    int nRet = 0;
2163
2164
0
    if (!pCurBuffer_)
2165
0
        pCurBuffer_ = new std::string();
2166
2167
0
    {
2168
0
        auto psJob = GetJobObject();
2169
0
        psJob->bFinish_ = true;
2170
0
        psJob->pParent_ = this;
2171
0
        psJob->pBuffer_ = pCurBuffer_;
2172
0
        pCurBuffer_ = nullptr;
2173
0
        psJob->nSeqNumber_ = nSeqNumberGenerated_;
2174
0
        VSIGZipWriteHandleMT::DeflateCompress(psJob);
2175
0
    }
2176
2177
0
    if (poPool_)
2178
0
    {
2179
0
        poPool_->WaitCompletion(0);
2180
0
    }
2181
0
    if (!ProcessCompletedJobs())
2182
0
    {
2183
0
        nRet = -1;
2184
0
    }
2185
0
    else
2186
0
    {
2187
0
        CPLAssert(apoFinishedJobs_.empty());
2188
0
        if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2189
0
        {
2190
0
            if (poPool_)
2191
0
            {
2192
0
                poPool_->WaitCompletion(0);
2193
0
            }
2194
0
            ProcessCompletedJobs();
2195
0
        }
2196
0
        CPLAssert(apoCRCFinishedJobs_.empty());
2197
0
    }
2198
2199
0
    if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2200
0
    {
2201
0
        const GUInt32 anTrailer[2] = {
2202
0
            CPL_LSBWORD32(static_cast<GUInt32>(nCRC_)),
2203
0
            CPL_LSBWORD32(static_cast<GUInt32>(nCurOffset_))};
2204
2205
0
        if (poBaseHandle_->Write(anTrailer, 1, 8) < 8)
2206
0
        {
2207
0
            nRet = -1;
2208
0
        }
2209
0
    }
2210
2211
0
    if (bAutoCloseBaseHandle_)
2212
0
    {
2213
0
        int nRetClose = poBaseHandle_->Close();
2214
0
        if (nRet == 0)
2215
0
            nRet = nRetClose;
2216
2217
0
        delete poBaseHandle_;
2218
0
    }
2219
0
    poBaseHandle_ = nullptr;
2220
2221
0
    return nRet;
2222
0
}
2223
2224
/************************************************************************/
2225
/*                                Read()                                */
2226
/************************************************************************/
2227
2228
size_t VSIGZipWriteHandleMT::Read(void * /* pBuffer */, size_t /* nSize */,
2229
                                  size_t /* nMemb */)
2230
0
{
2231
0
    CPLError(CE_Failure, CPLE_NotSupported,
2232
0
             "VSIFReadL is not supported on GZip write streams");
2233
0
    return 0;
2234
0
}
2235
2236
/************************************************************************/
2237
/*                        DeflateCompress()                             */
2238
/************************************************************************/
2239
2240
void VSIGZipWriteHandleMT::DeflateCompress(void *inData)
2241
0
{
2242
0
    Job *psJob = static_cast<Job *>(inData);
2243
2244
0
    CPLAssert(psJob->pBuffer_);
2245
2246
0
    z_stream sStream;
2247
0
    memset(&sStream, 0, sizeof(sStream));
2248
0
    sStream.zalloc = nullptr;
2249
0
    sStream.zfree = nullptr;
2250
0
    sStream.opaque = nullptr;
2251
2252
0
    sStream.avail_in = static_cast<uInt>(psJob->pBuffer_->size());
2253
0
    sStream.next_in = reinterpret_cast<Bytef *>(&(*psJob->pBuffer_)[0]);
2254
2255
0
    int ret = deflateInit2(
2256
0
        &sStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
2257
0
        (psJob->pParent_->nDeflateType_ == CPL_DEFLATE_TYPE_ZLIB) ? MAX_WBITS
2258
0
                                                                  : -MAX_WBITS,
2259
0
        8, Z_DEFAULT_STRATEGY);
2260
0
    CPLAssertAlwaysEval(ret == Z_OK);
2261
2262
0
    size_t nRealSize = 0;
2263
2264
0
    while (sStream.avail_in > 0)
2265
0
    {
2266
0
        psJob->sCompressedData_.resize(nRealSize + Z_BUFSIZE);
2267
0
        sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2268
0
        sStream.next_out =
2269
0
            reinterpret_cast<Bytef *>(&psJob->sCompressedData_[0]) + nRealSize;
2270
2271
0
        const int zlibRet = deflate(&sStream, Z_NO_FLUSH);
2272
0
        CPLAssertAlwaysEval(zlibRet == Z_OK);
2273
2274
0
        nRealSize += static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2275
0
    }
2276
2277
0
    psJob->sCompressedData_.resize(nRealSize + Z_BUFSIZE);
2278
0
    sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2279
0
    sStream.next_out =
2280
0
        reinterpret_cast<Bytef *>(&psJob->sCompressedData_[0]) + nRealSize;
2281
2282
0
    if (psJob->bFinish_)
2283
0
    {
2284
0
        const int zlibRet = deflate(&sStream, Z_FINISH);
2285
0
        CPLAssertAlwaysEval(zlibRet == Z_STREAM_END);
2286
0
    }
2287
0
    else
2288
0
    {
2289
        // Do a Z_SYNC_FLUSH and Z_FULL_FLUSH, so as to have two markers when
2290
        // independent as pigz 2.3.4 or later. The following 9 byte sequence
2291
        // will be found: 0x00 0x00 0xff 0xff 0x00 0x00 0x00 0xff 0xff
2292
        // Z_FULL_FLUSH only is sufficient, but it is not obvious if a
2293
        // 0x00 0x00 0xff 0xff marker in the codestream is just a SYNC_FLUSH (
2294
        // without dictionary reset) or a FULL_FLUSH (with dictionary reset)
2295
0
        {
2296
0
            const int zlibRet = deflate(&sStream, Z_SYNC_FLUSH);
2297
0
            CPLAssertAlwaysEval(zlibRet == Z_OK);
2298
0
        }
2299
2300
0
        {
2301
0
            const int zlibRet = deflate(&sStream, Z_FULL_FLUSH);
2302
0
            CPLAssertAlwaysEval(zlibRet == Z_OK);
2303
0
        }
2304
0
    }
2305
2306
0
    nRealSize += static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2307
0
    psJob->sCompressedData_.resize(nRealSize);
2308
2309
0
    deflateEnd(&sStream);
2310
2311
0
    {
2312
0
        std::lock_guard<std::mutex> oLock(psJob->pParent_->sMutex_);
2313
0
        psJob->pParent_->apoFinishedJobs_.push_back(psJob);
2314
0
    }
2315
0
}
2316
2317
/************************************************************************/
2318
/*                          CRCCompute()                                */
2319
/************************************************************************/
2320
2321
void VSIGZipWriteHandleMT::CRCCompute(void *inData)
2322
0
{
2323
0
    Job *psJob = static_cast<Job *>(inData);
2324
0
    psJob->bInCRCComputation_ = true;
2325
0
    psJob->nCRC_ =
2326
0
        crc32(0U, reinterpret_cast<const Bytef *>(psJob->pBuffer_->data()),
2327
0
              static_cast<uInt>(psJob->pBuffer_->size()));
2328
2329
0
    {
2330
0
        std::lock_guard<std::mutex> oLock(psJob->pParent_->sMutex_);
2331
0
        psJob->pParent_->apoCRCFinishedJobs_.push_back(psJob);
2332
0
    }
2333
0
}
2334
2335
/************************************************************************/
2336
/*                                DumpState()                           */
2337
/************************************************************************/
2338
2339
#ifdef DEBUG_VERBOSE
2340
void VSIGZipWriteHandleMT::DumpState()
2341
{
2342
    fprintf(stderr, "Finished jobs (expected = %d):\n",  // ok
2343
            nSeqNumberExpected_);
2344
    for (const auto *psJob : apoFinishedJobs_)
2345
    {
2346
        fprintf(stderr, "seq number=%d, bInCRCComputation = %d\n",  // ok
2347
                psJob->nSeqNumber_, psJob->bInCRCComputation_ ? 1 : 0);
2348
    }
2349
    fprintf(stderr, "Finished CRC jobs (expected = %d):\n",  // ok
2350
            nSeqNumberExpectedCRC_);
2351
    for (const auto *psJob : apoFinishedJobs_)
2352
    {
2353
        fprintf(stderr, "seq number=%d\n",  // ok
2354
                psJob->nSeqNumber_);
2355
    }
2356
    fprintf(stderr, "apoFreeJobs_.size() = %d\n",  // ok
2357
            static_cast<int>(apoFreeJobs_.size()));
2358
    fprintf(stderr, "aposBuffers_.size() = %d\n",  // ok
2359
            static_cast<int>(aposBuffers_.size()));
2360
}
2361
#endif
2362
2363
/************************************************************************/
2364
/*                         ProcessCompletedJobs()                       */
2365
/************************************************************************/
2366
2367
bool VSIGZipWriteHandleMT::ProcessCompletedJobs()
2368
0
{
2369
0
    std::lock_guard<std::mutex> oLock(sMutex_);
2370
0
    bool do_it_again = true;
2371
0
    while (do_it_again)
2372
0
    {
2373
0
        do_it_again = false;
2374
0
        if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2375
0
        {
2376
0
            for (auto iter = apoFinishedJobs_.begin();
2377
0
                 iter != apoFinishedJobs_.end(); ++iter)
2378
0
            {
2379
0
                auto psJob = *iter;
2380
2381
0
                if (!psJob->bInCRCComputation_)
2382
0
                {
2383
0
                    psJob->bInCRCComputation_ = true;
2384
0
                    sMutex_.unlock();
2385
0
                    if (poPool_)
2386
0
                    {
2387
0
                        poPool_->SubmitJob(VSIGZipWriteHandleMT::CRCCompute,
2388
0
                                           psJob);
2389
0
                    }
2390
0
                    else
2391
0
                    {
2392
0
                        CRCCompute(psJob);
2393
0
                    }
2394
0
                    sMutex_.lock();
2395
0
                }
2396
0
            }
2397
0
        }
2398
2399
0
        for (auto iter = apoFinishedJobs_.begin();
2400
0
             iter != apoFinishedJobs_.end(); ++iter)
2401
0
        {
2402
0
            auto psJob = *iter;
2403
0
            if (psJob->nSeqNumber_ == nSeqNumberExpected_)
2404
0
            {
2405
0
                apoFinishedJobs_.erase(iter);
2406
2407
0
                const bool bIsSeqNumberExpectedZero =
2408
0
                    (nSeqNumberExpected_ == 0);
2409
0
                sMutex_.unlock();
2410
2411
0
                const size_t nToWrite = psJob->sCompressedData_.size();
2412
0
                if (panSOZIPIndex_ && !bIsSeqNumberExpectedZero &&
2413
0
                    !psJob->pBuffer_->empty())
2414
0
                {
2415
0
                    uint64_t nOffset = poBaseHandle_->Tell() - nStartOffset_;
2416
0
                    if (nSOZIPIndexEltSize_ == 8)
2417
0
                    {
2418
0
                        CPL_LSBPTR64(&nOffset);
2419
0
                        std::copy(reinterpret_cast<const uint8_t *>(&nOffset),
2420
0
                                  reinterpret_cast<const uint8_t *>(&nOffset) +
2421
0
                                      sizeof(nOffset),
2422
0
                                  std::back_inserter(*panSOZIPIndex_));
2423
0
                    }
2424
0
                    else
2425
0
                    {
2426
0
                        if (nOffset > std::numeric_limits<uint32_t>::max())
2427
0
                        {
2428
                            // shouldn't happen normally...
2429
0
                            CPLError(
2430
0
                                CE_Failure, CPLE_AppDefined,
2431
0
                                "Too big offset for SOZIP_OFFSET_SIZE = 4");
2432
0
                            panSOZIPIndex_->clear();
2433
0
                            panSOZIPIndex_ = nullptr;
2434
0
                        }
2435
0
                        else
2436
0
                        {
2437
0
                            uint32_t nOffset32 = static_cast<uint32_t>(nOffset);
2438
0
                            CPL_LSBPTR32(&nOffset32);
2439
0
                            std::copy(
2440
0
                                reinterpret_cast<const uint8_t *>(&nOffset32),
2441
0
                                reinterpret_cast<const uint8_t *>(&nOffset32) +
2442
0
                                    sizeof(nOffset32),
2443
0
                                std::back_inserter(*panSOZIPIndex_));
2444
0
                        }
2445
0
                    }
2446
0
                }
2447
0
                bool bError =
2448
0
                    poBaseHandle_->Write(psJob->sCompressedData_.data(), 1,
2449
0
                                         nToWrite) < nToWrite;
2450
0
                sMutex_.lock();
2451
0
                nSeqNumberExpected_++;
2452
2453
0
                if (nDeflateType_ != CPL_DEFLATE_TYPE_GZIP)
2454
0
                {
2455
0
                    aposBuffers_.push_back(psJob->pBuffer_);
2456
0
                    psJob->pBuffer_ = nullptr;
2457
2458
0
                    apoFreeJobs_.push_back(psJob);
2459
0
                }
2460
2461
0
                if (bError)
2462
0
                {
2463
0
                    return false;
2464
0
                }
2465
2466
0
                do_it_again = true;
2467
0
                break;
2468
0
            }
2469
0
        }
2470
2471
0
        if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2472
0
        {
2473
0
            for (auto iter = apoCRCFinishedJobs_.begin();
2474
0
                 iter != apoCRCFinishedJobs_.end(); ++iter)
2475
0
            {
2476
0
                auto psJob = *iter;
2477
0
                if (psJob->nSeqNumber_ == nSeqNumberExpectedCRC_)
2478
0
                {
2479
0
                    apoCRCFinishedJobs_.erase(iter);
2480
2481
0
                    nCRC_ = crc32_combine(
2482
0
                        nCRC_, psJob->nCRC_,
2483
0
                        static_cast<uLong>(psJob->pBuffer_->size()));
2484
2485
0
                    nSeqNumberExpectedCRC_++;
2486
2487
0
                    aposBuffers_.push_back(psJob->pBuffer_);
2488
0
                    psJob->pBuffer_ = nullptr;
2489
2490
0
                    apoFreeJobs_.push_back(psJob);
2491
0
                    do_it_again = true;
2492
0
                    break;
2493
0
                }
2494
0
            }
2495
0
        }
2496
0
    }
2497
0
    return true;
2498
0
}
2499
2500
/************************************************************************/
2501
/*                           GetJobObject()                             */
2502
/************************************************************************/
2503
2504
VSIGZipWriteHandleMT::Job *VSIGZipWriteHandleMT::GetJobObject()
2505
0
{
2506
0
    {
2507
0
        std::lock_guard<std::mutex> oLock(sMutex_);
2508
0
        if (!apoFreeJobs_.empty())
2509
0
        {
2510
0
            auto job = apoFreeJobs_.back();
2511
0
            apoFreeJobs_.pop_back();
2512
0
            job->sCompressedData_.clear();
2513
0
            job->bInCRCComputation_ = false;
2514
0
            return job;
2515
0
        }
2516
0
    }
2517
0
    return new Job();
2518
0
}
2519
2520
/************************************************************************/
2521
/*                               Write()                                */
2522
/************************************************************************/
2523
2524
size_t VSIGZipWriteHandleMT::Write(const void *const pBuffer,
2525
                                   size_t const nSize, size_t const nMemb)
2526
2527
0
{
2528
0
    if (bHasErrored_)
2529
0
        return 0;
2530
2531
0
    const char *pszBuffer = static_cast<const char *>(pBuffer);
2532
0
    size_t nBytesToWrite = nSize * nMemb;
2533
0
    while (nBytesToWrite > 0)
2534
0
    {
2535
0
        if (pCurBuffer_ == nullptr)
2536
0
        {
2537
0
            while (true)
2538
0
            {
2539
                // We store in a local variable instead of pCurBuffer_ directly
2540
                // to avoid Coverity Scan to be confused by the fact that we
2541
                // have used above pCurBuffer_ outside of the mutex. But what
2542
                // is protected by the mutex is aposBuffers_, not pCurBuffer_.
2543
0
                std::string *l_pCurBuffer = nullptr;
2544
0
                {
2545
0
                    std::lock_guard<std::mutex> oLock(sMutex_);
2546
0
                    if (!aposBuffers_.empty())
2547
0
                    {
2548
0
                        l_pCurBuffer = aposBuffers_.back();
2549
0
                        aposBuffers_.pop_back();
2550
0
                    }
2551
0
                }
2552
0
                pCurBuffer_ = l_pCurBuffer;
2553
0
                if (pCurBuffer_)
2554
0
                    break;
2555
2556
0
                if (poPool_)
2557
0
                {
2558
0
                    poPool_->WaitEvent();
2559
0
                }
2560
0
                if (!ProcessCompletedJobs())
2561
0
                {
2562
0
                    bHasErrored_ = true;
2563
0
                    return 0;
2564
0
                }
2565
0
            }
2566
0
            pCurBuffer_->clear();
2567
0
        }
2568
0
        size_t nConsumed =
2569
0
            std::min(nBytesToWrite, nChunkSize_ - pCurBuffer_->size());
2570
0
        pCurBuffer_->append(pszBuffer, nConsumed);
2571
0
        nCurOffset_ += nConsumed;
2572
0
        pszBuffer += nConsumed;
2573
0
        nBytesToWrite -= nConsumed;
2574
0
        if (pCurBuffer_->size() == nChunkSize_)
2575
0
        {
2576
0
            if (poPool_ == nullptr)
2577
0
            {
2578
0
                poPool_.reset(new CPLWorkerThreadPool());
2579
0
                if (!poPool_->Setup(nThreads_, nullptr, nullptr, false))
2580
0
                {
2581
0
                    bHasErrored_ = true;
2582
0
                    poPool_.reset();
2583
0
                    return 0;
2584
0
                }
2585
0
            }
2586
2587
0
            auto psJob = GetJobObject();
2588
0
            psJob->pParent_ = this;
2589
0
            psJob->pBuffer_ = pCurBuffer_;
2590
0
            psJob->nSeqNumber_ = nSeqNumberGenerated_;
2591
0
            nSeqNumberGenerated_++;
2592
0
            pCurBuffer_ = nullptr;
2593
0
            poPool_->SubmitJob(VSIGZipWriteHandleMT::DeflateCompress, psJob);
2594
0
        }
2595
0
    }
2596
2597
0
    return nMemb;
2598
0
}
2599
2600
/************************************************************************/
2601
/*                               Flush()                                */
2602
/************************************************************************/
2603
2604
int VSIGZipWriteHandleMT::Flush()
2605
2606
0
{
2607
    // we *could* do something for this but for now we choose not to.
2608
2609
0
    return 0;
2610
0
}
2611
2612
/************************************************************************/
2613
/*                                Seek()                                */
2614
/************************************************************************/
2615
2616
int VSIGZipWriteHandleMT::Seek(vsi_l_offset nOffset, int nWhence)
2617
2618
0
{
2619
0
    if (nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR))
2620
0
        return 0;
2621
0
    else if (nWhence == SEEK_SET && nOffset == nCurOffset_)
2622
0
        return 0;
2623
0
    else
2624
0
    {
2625
0
        CPLError(CE_Failure, CPLE_NotSupported,
2626
0
                 "Seeking on writable compressed data streams not supported.");
2627
2628
0
        return -1;
2629
0
    }
2630
0
}
2631
2632
/************************************************************************/
2633
/*                                Tell()                                */
2634
/************************************************************************/
2635
2636
vsi_l_offset VSIGZipWriteHandleMT::Tell()
2637
2638
0
{
2639
0
    return nCurOffset_;
2640
0
}
2641
2642
/************************************************************************/
2643
/* ==================================================================== */
2644
/*                       VSIGZipWriteHandle                             */
2645
/* ==================================================================== */
2646
/************************************************************************/
2647
2648
class VSIGZipWriteHandle final : public VSIVirtualHandle
2649
{
2650
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipWriteHandle)
2651
2652
    VSIVirtualHandle *m_poBaseHandle = nullptr;
2653
    z_stream sStream;
2654
    Byte *pabyInBuf = nullptr;
2655
    Byte *pabyOutBuf = nullptr;
2656
    bool bCompressActive = false;
2657
    vsi_l_offset nCurOffset = 0;
2658
    uLong nCRC = 0;
2659
    int nDeflateType = CPL_DEFLATE_TYPE_GZIP;
2660
    bool bAutoCloseBaseHandle = false;
2661
2662
  public:
2663
    VSIGZipWriteHandle(VSIVirtualHandle *poBaseHandle, int nDeflateType,
2664
                       bool bAutoCloseBaseHandleIn);
2665
2666
    ~VSIGZipWriteHandle() override;
2667
2668
    int Seek(vsi_l_offset nOffset, int nWhence) override;
2669
    vsi_l_offset Tell() override;
2670
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
2671
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
2672
2673
    int Eof() override
2674
0
    {
2675
0
        return 0;
2676
0
    }
2677
2678
    int Error() override
2679
0
    {
2680
0
        return 0;
2681
0
    }
2682
2683
    void ClearErr() override
2684
0
    {
2685
0
    }
2686
2687
    int Flush() override;
2688
    int Close() override;
2689
};
2690
2691
/************************************************************************/
2692
/*                         VSIGZipWriteHandle()                         */
2693
/************************************************************************/
2694
2695
VSIGZipWriteHandle::VSIGZipWriteHandle(VSIVirtualHandle *poBaseHandle,
2696
                                       int nDeflateTypeIn,
2697
                                       bool bAutoCloseBaseHandleIn)
2698
0
    : m_poBaseHandle(poBaseHandle), sStream(),
2699
0
      pabyInBuf(static_cast<Byte *>(CPLMalloc(Z_BUFSIZE))),
2700
0
      pabyOutBuf(static_cast<Byte *>(CPLMalloc(Z_BUFSIZE))),
2701
0
      nCRC(crc32(0L, nullptr, 0)), nDeflateType(nDeflateTypeIn),
2702
0
      bAutoCloseBaseHandle(bAutoCloseBaseHandleIn)
2703
0
{
2704
0
    sStream.zalloc = nullptr;
2705
0
    sStream.zfree = nullptr;
2706
0
    sStream.opaque = nullptr;
2707
0
    sStream.next_in = nullptr;
2708
0
    sStream.next_out = nullptr;
2709
0
    sStream.avail_in = sStream.avail_out = 0;
2710
2711
0
    sStream.next_in = pabyInBuf;
2712
2713
0
    if (deflateInit2(&sStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
2714
0
                     (nDeflateType == CPL_DEFLATE_TYPE_ZLIB) ? MAX_WBITS
2715
0
                                                             : -MAX_WBITS,
2716
0
                     8, Z_DEFAULT_STRATEGY) != Z_OK)
2717
0
    {
2718
0
        bCompressActive = false;
2719
0
    }
2720
0
    else
2721
0
    {
2722
0
        if (nDeflateType == CPL_DEFLATE_TYPE_GZIP)
2723
0
        {
2724
0
            char header[11] = {};
2725
2726
            // Write a very simple .gz header:
2727
0
            snprintf(header, sizeof(header), "%c%c%c%c%c%c%c%c%c%c",
2728
0
                     gz_magic[0], gz_magic[1], Z_DEFLATED, 0 /*flags*/, 0, 0, 0,
2729
0
                     0 /*time*/, 0 /*xflags*/, 0x03);
2730
0
            m_poBaseHandle->Write(header, 1, 10);
2731
0
        }
2732
2733
0
        bCompressActive = true;
2734
0
    }
2735
0
}
2736
2737
/************************************************************************/
2738
/*                       VSICreateGZipWritable()                        */
2739
/************************************************************************/
2740
2741
VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
2742
                                        int nDeflateTypeIn,
2743
                                        int bAutoCloseBaseHandle)
2744
0
{
2745
0
    return VSICreateGZipWritable(poBaseHandle, nDeflateTypeIn,
2746
0
                                 CPL_TO_BOOL(bAutoCloseBaseHandle), 0, 0, 0,
2747
0
                                 nullptr);
2748
0
}
2749
2750
VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
2751
                                        int nDeflateTypeIn,
2752
                                        bool bAutoCloseBaseHandle, int nThreads,
2753
                                        size_t nChunkSize,
2754
                                        size_t nSOZIPIndexEltSize,
2755
                                        std::vector<uint8_t> *panSOZIPIndex)
2756
0
{
2757
0
    const char *pszThreads = CPLGetConfigOption("GDAL_NUM_THREADS", nullptr);
2758
0
    if (pszThreads || nThreads > 0 || nChunkSize > 0)
2759
0
    {
2760
0
        if (nThreads == 0)
2761
0
        {
2762
0
            if (!pszThreads || EQUAL(pszThreads, "ALL_CPUS"))
2763
0
                nThreads = CPLGetNumCPUs();
2764
0
            else
2765
0
                nThreads = atoi(pszThreads);
2766
0
            nThreads = std::max(1, std::min(128, nThreads));
2767
0
        }
2768
0
        if (nThreads > 1 || nChunkSize > 0)
2769
0
        {
2770
            // coverity[tainted_data]
2771
0
            return new VSIGZipWriteHandleMT(
2772
0
                poBaseHandle, nDeflateTypeIn, bAutoCloseBaseHandle, nThreads,
2773
0
                nChunkSize, nSOZIPIndexEltSize, panSOZIPIndex);
2774
0
        }
2775
0
    }
2776
0
    return new VSIGZipWriteHandle(poBaseHandle, nDeflateTypeIn,
2777
0
                                  bAutoCloseBaseHandle);
2778
0
}
2779
2780
/************************************************************************/
2781
/*                        ~VSIGZipWriteHandle()                         */
2782
/************************************************************************/
2783
2784
VSIGZipWriteHandle::~VSIGZipWriteHandle()
2785
2786
0
{
2787
0
    if (bCompressActive)
2788
0
        VSIGZipWriteHandle::Close();
2789
2790
0
    CPLFree(pabyInBuf);
2791
0
    CPLFree(pabyOutBuf);
2792
0
}
2793
2794
/************************************************************************/
2795
/*                               Close()                                */
2796
/************************************************************************/
2797
2798
int VSIGZipWriteHandle::Close()
2799
2800
0
{
2801
0
    int nRet = 0;
2802
0
    if (bCompressActive)
2803
0
    {
2804
0
        sStream.next_out = pabyOutBuf;
2805
0
        sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2806
2807
0
        const int zlibRet = deflate(&sStream, Z_FINISH);
2808
0
        CPLAssertAlwaysEval(zlibRet == Z_STREAM_END);
2809
2810
0
        const size_t nOutBytes =
2811
0
            static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2812
2813
0
        deflateEnd(&sStream);
2814
2815
0
        if (m_poBaseHandle->Write(pabyOutBuf, 1, nOutBytes) < nOutBytes)
2816
0
        {
2817
0
            nRet = -1;
2818
0
        }
2819
2820
0
        if (nRet == 0 && nDeflateType == CPL_DEFLATE_TYPE_GZIP)
2821
0
        {
2822
0
            const GUInt32 anTrailer[2] = {
2823
0
                CPL_LSBWORD32(static_cast<GUInt32>(nCRC)),
2824
0
                CPL_LSBWORD32(static_cast<GUInt32>(nCurOffset))};
2825
2826
0
            if (m_poBaseHandle->Write(anTrailer, 1, 8) < 8)
2827
0
            {
2828
0
                nRet = -1;
2829
0
            }
2830
0
        }
2831
2832
0
        if (bAutoCloseBaseHandle)
2833
0
        {
2834
0
            if (nRet == 0)
2835
0
                nRet = m_poBaseHandle->Close();
2836
2837
0
            delete m_poBaseHandle;
2838
0
        }
2839
2840
0
        bCompressActive = false;
2841
0
    }
2842
2843
0
    return nRet;
2844
0
}
2845
2846
/************************************************************************/
2847
/*                                Read()                                */
2848
/************************************************************************/
2849
2850
size_t VSIGZipWriteHandle::Read(void * /* pBuffer */, size_t /* nSize */,
2851
                                size_t /* nMemb */)
2852
0
{
2853
0
    CPLError(CE_Failure, CPLE_NotSupported,
2854
0
             "VSIFReadL is not supported on GZip write streams");
2855
0
    return 0;
2856
0
}
2857
2858
/************************************************************************/
2859
/*                               Write()                                */
2860
/************************************************************************/
2861
2862
size_t VSIGZipWriteHandle::Write(const void *const pBuffer, size_t const nSize,
2863
                                 size_t const nMemb)
2864
2865
0
{
2866
0
    size_t nBytesToWrite = nSize * nMemb;
2867
2868
0
    {
2869
0
        size_t nOffset = 0;
2870
0
        while (nOffset < nBytesToWrite)
2871
0
        {
2872
0
            uInt nChunk = static_cast<uInt>(std::min(
2873
0
                static_cast<size_t>(UINT_MAX), nBytesToWrite - nOffset));
2874
0
            nCRC =
2875
0
                crc32(nCRC, reinterpret_cast<const Bytef *>(pBuffer) + nOffset,
2876
0
                      nChunk);
2877
0
            nOffset += nChunk;
2878
0
        }
2879
0
    }
2880
2881
0
    if (!bCompressActive)
2882
0
        return 0;
2883
2884
0
    size_t nNextByte = 0;
2885
0
    while (nNextByte < nBytesToWrite)
2886
0
    {
2887
0
        sStream.next_out = pabyOutBuf;
2888
0
        sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2889
2890
0
        if (sStream.avail_in > 0)
2891
0
            memmove(pabyInBuf, sStream.next_in, sStream.avail_in);
2892
2893
0
        const uInt nNewBytesToWrite = static_cast<uInt>(
2894
0
            std::min(static_cast<size_t>(Z_BUFSIZE - sStream.avail_in),
2895
0
                     nBytesToWrite - nNextByte));
2896
0
        memcpy(pabyInBuf + sStream.avail_in,
2897
0
               reinterpret_cast<const Byte *>(pBuffer) + nNextByte,
2898
0
               nNewBytesToWrite);
2899
2900
0
        sStream.next_in = pabyInBuf;
2901
0
        sStream.avail_in += nNewBytesToWrite;
2902
2903
0
        const int zlibRet = deflate(&sStream, Z_NO_FLUSH);
2904
0
        CPLAssertAlwaysEval(zlibRet == Z_OK);
2905
2906
0
        const size_t nOutBytes =
2907
0
            static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2908
2909
0
        if (nOutBytes > 0)
2910
0
        {
2911
0
            if (m_poBaseHandle->Write(pabyOutBuf, 1, nOutBytes) < nOutBytes)
2912
0
                return 0;
2913
0
        }
2914
2915
0
        nNextByte += nNewBytesToWrite;
2916
0
        nCurOffset += nNewBytesToWrite;
2917
0
    }
2918
2919
0
    return nMemb;
2920
0
}
2921
2922
/************************************************************************/
2923
/*                               Flush()                                */
2924
/************************************************************************/
2925
2926
int VSIGZipWriteHandle::Flush()
2927
2928
0
{
2929
    // we *could* do something for this but for now we choose not to.
2930
2931
0
    return 0;
2932
0
}
2933
2934
/************************************************************************/
2935
/*                                Seek()                                */
2936
/************************************************************************/
2937
2938
int VSIGZipWriteHandle::Seek(vsi_l_offset nOffset, int nWhence)
2939
2940
0
{
2941
0
    if (nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR))
2942
0
        return 0;
2943
0
    else if (nWhence == SEEK_SET && nOffset == nCurOffset)
2944
0
        return 0;
2945
0
    else
2946
0
    {
2947
0
        CPLError(CE_Failure, CPLE_NotSupported,
2948
0
                 "Seeking on writable compressed data streams not supported.");
2949
2950
0
        return -1;
2951
0
    }
2952
0
}
2953
2954
/************************************************************************/
2955
/*                                Tell()                                */
2956
/************************************************************************/
2957
2958
vsi_l_offset VSIGZipWriteHandle::Tell()
2959
2960
0
{
2961
0
    return nCurOffset;
2962
0
}
2963
2964
/************************************************************************/
2965
/* ==================================================================== */
2966
/*                       VSIGZipFilesystemHandler                       */
2967
/* ==================================================================== */
2968
/************************************************************************/
2969
2970
/************************************************************************/
2971
/*                  ~VSIGZipFilesystemHandler()                         */
2972
/************************************************************************/
2973
2974
VSIGZipFilesystemHandler::~VSIGZipFilesystemHandler()
2975
0
{
2976
0
    if (poHandleLastGZipFile)
2977
0
    {
2978
0
        poHandleLastGZipFile->UnsetCanSaveInfo();
2979
0
        poHandleLastGZipFile.reset();
2980
0
    }
2981
0
}
2982
2983
/************************************************************************/
2984
/*                            SaveInfo()                                */
2985
/************************************************************************/
2986
2987
void VSIGZipFilesystemHandler::SaveInfo(VSIGZipHandle *poHandle)
2988
0
{
2989
0
    std::unique_lock oLock(oMutex);
2990
0
    SaveInfo_unlocked(poHandle);
2991
0
}
2992
2993
void VSIGZipFilesystemHandler::SaveInfo_unlocked(VSIGZipHandle *poHandle)
2994
0
{
2995
0
    if (m_bInSaveInfo)
2996
0
        return;
2997
0
    m_bInSaveInfo = true;
2998
2999
0
    CPLAssert(poHandle != poHandleLastGZipFile.get());
3000
0
    CPLAssert(poHandle->GetBaseFileName() != nullptr);
3001
3002
0
    if (poHandleLastGZipFile == nullptr ||
3003
0
        strcmp(poHandleLastGZipFile->GetBaseFileName(),
3004
0
               poHandle->GetBaseFileName()) != 0 ||
3005
0
        poHandle->GetLastReadOffset() >
3006
0
            poHandleLastGZipFile->GetLastReadOffset())
3007
0
    {
3008
0
        std::unique_ptr<VSIGZipHandle> poTmp;
3009
0
        std::swap(poTmp, poHandleLastGZipFile);
3010
0
        if (poTmp)
3011
0
        {
3012
0
            poTmp->UnsetCanSaveInfo();
3013
0
            poTmp.reset();
3014
0
        }
3015
0
        poHandleLastGZipFile.reset(poHandle->Duplicate());
3016
0
        if (poHandleLastGZipFile)
3017
0
            poHandleLastGZipFile->CloseBaseHandle();
3018
0
    }
3019
0
    m_bInSaveInfo = false;
3020
0
}
3021
3022
/************************************************************************/
3023
/*                                Open()                                */
3024
/************************************************************************/
3025
3026
VSIVirtualHandleUniquePtr
3027
VSIGZipFilesystemHandler::Open(const char *pszFilename, const char *pszAccess,
3028
                               bool /* bSetError */,
3029
                               CSLConstList /* papszOptions */)
3030
2.41k
{
3031
2.41k
    if (!STARTS_WITH_CI(pszFilename, "/vsigzip/"))
3032
0
        return nullptr;
3033
3034
2.41k
    VSIFilesystemHandler *poFSHandler =
3035
2.41k
        VSIFileManager::GetHandler(pszFilename + strlen("/vsigzip/"));
3036
3037
    /* -------------------------------------------------------------------- */
3038
    /*      Is this an attempt to write a new file without update (w+)      */
3039
    /*      access?  If so, create a writable handle for the underlying     */
3040
    /*      filename.                                                       */
3041
    /* -------------------------------------------------------------------- */
3042
2.41k
    if (strchr(pszAccess, 'w') != nullptr)
3043
0
    {
3044
0
        if (strchr(pszAccess, '+') != nullptr)
3045
0
        {
3046
0
            CPLError(CE_Failure, CPLE_AppDefined,
3047
0
                     "Write+update (w+) not supported for /vsigzip, "
3048
0
                     "only read-only or write-only.");
3049
0
            return nullptr;
3050
0
        }
3051
3052
0
        auto poVirtualHandle =
3053
0
            poFSHandler->Open(pszFilename + strlen("/vsigzip/"), "wb");
3054
3055
0
        if (poVirtualHandle == nullptr)
3056
0
            return nullptr;
3057
3058
0
        return VSIVirtualHandleUniquePtr(
3059
0
            VSICreateGZipWritable(poVirtualHandle.release(),
3060
0
                                  strchr(pszAccess, 'z') != nullptr, TRUE));
3061
0
    }
3062
3063
    /* -------------------------------------------------------------------- */
3064
    /*      Otherwise we are in the read access case.                       */
3065
    /* -------------------------------------------------------------------- */
3066
3067
2.41k
    VSIGZipHandle *poGZIPHandle = OpenGZipReadOnly(pszFilename, pszAccess);
3068
2.41k
    if (poGZIPHandle)
3069
        // Wrap the VSIGZipHandle inside a buffered reader that will
3070
        // improve dramatically performance when doing small backward
3071
        // seeks.
3072
0
        return VSIVirtualHandleUniquePtr(
3073
0
            VSICreateBufferedReaderHandle(poGZIPHandle));
3074
3075
2.41k
    return nullptr;
3076
2.41k
}
3077
3078
/************************************************************************/
3079
/*                      SupportsSequentialWrite()                       */
3080
/************************************************************************/
3081
3082
bool VSIGZipFilesystemHandler::SupportsSequentialWrite(const char *pszPath,
3083
                                                       bool bAllowLocalTempFile)
3084
0
{
3085
0
    if (!STARTS_WITH_CI(pszPath, "/vsigzip/"))
3086
0
        return false;
3087
0
    const char *pszBaseFileName = pszPath + strlen("/vsigzip/");
3088
0
    VSIFilesystemHandler *poFSHandler =
3089
0
        VSIFileManager::GetHandler(pszBaseFileName);
3090
0
    return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
3091
0
}
3092
3093
/************************************************************************/
3094
/*                          OpenGZipReadOnly()                          */
3095
/************************************************************************/
3096
3097
VSIGZipHandle *
3098
VSIGZipFilesystemHandler::OpenGZipReadOnly(const char *pszFilename,
3099
                                           const char *pszAccess)
3100
2.72k
{
3101
2.72k
    VSIFilesystemHandler *poFSHandler =
3102
2.72k
        VSIFileManager::GetHandler(pszFilename + strlen("/vsigzip/"));
3103
3104
2.72k
    std::unique_lock oLock(oMutex);
3105
3106
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
3107
    // Disable caching in fuzzing mode as the /vsigzip/ file is likely to
3108
    // change very often
3109
    // TODO: filename-based logic isn't enough. We should probably check
3110
    // timestamp and/or file size.
3111
    if (poHandleLastGZipFile != nullptr &&
3112
        strcmp(pszFilename + strlen("/vsigzip/"),
3113
               poHandleLastGZipFile->GetBaseFileName()) == 0 &&
3114
        EQUAL(pszAccess, "rb"))
3115
    {
3116
        VSIGZipHandle *poHandle = poHandleLastGZipFile->Duplicate();
3117
        if (poHandle)
3118
            return poHandle;
3119
    }
3120
#else
3121
2.72k
    CPL_IGNORE_RET_VAL(pszAccess);
3122
2.72k
#endif
3123
3124
2.72k
    VSIVirtualHandleUniquePtr poVirtualHandle(
3125
2.72k
        poFSHandler->Open(pszFilename + strlen("/vsigzip/"), "rb"));
3126
3127
2.72k
    if (poVirtualHandle == nullptr)
3128
1.61k
        return nullptr;
3129
3130
1.11k
    unsigned char signature[2] = {'\0', '\0'};
3131
1.11k
    if (poVirtualHandle->Read(signature, 1, 2) != 2 ||
3132
0
        signature[0] != gz_magic[0] || signature[1] != gz_magic[1])
3133
1.11k
    {
3134
1.11k
        return nullptr;
3135
1.11k
    }
3136
3137
0
    if (poHandleLastGZipFile)
3138
0
    {
3139
0
        poHandleLastGZipFile->UnsetCanSaveInfo();
3140
0
        poHandleLastGZipFile.reset();
3141
0
    }
3142
3143
0
    auto poHandle = std::make_unique<VSIGZipHandle>(
3144
0
        std::move(poVirtualHandle), pszFilename + strlen("/vsigzip/"));
3145
0
    if (!(poHandle->IsInitOK()))
3146
0
    {
3147
0
        return nullptr;
3148
0
    }
3149
0
    return poHandle.release();
3150
0
}
3151
3152
/************************************************************************/
3153
/*                                Stat()                                */
3154
/************************************************************************/
3155
3156
int VSIGZipFilesystemHandler::Stat(const char *pszFilename,
3157
                                   VSIStatBufL *pStatBuf, int nFlags)
3158
5.57k
{
3159
5.57k
    if (!STARTS_WITH_CI(pszFilename, "/vsigzip/"))
3160
48
        return -1;
3161
3162
5.52k
    std::unique_lock oLock(oMutex);
3163
3164
5.52k
    memset(pStatBuf, 0, sizeof(VSIStatBufL));
3165
3166
5.52k
    if (poHandleLastGZipFile != nullptr &&
3167
0
        strcmp(pszFilename + strlen("/vsigzip/"),
3168
0
               poHandleLastGZipFile->GetBaseFileName()) == 0)
3169
0
    {
3170
0
        if (poHandleLastGZipFile->GetUncompressedSize() != 0)
3171
0
        {
3172
0
            pStatBuf->st_mode = S_IFREG;
3173
0
            pStatBuf->st_size = poHandleLastGZipFile->GetUncompressedSize();
3174
0
            return 0;
3175
0
        }
3176
0
    }
3177
3178
    // Begin by doing a stat on the real file.
3179
5.52k
    int ret = VSIStatExL(pszFilename + strlen("/vsigzip/"), pStatBuf, nFlags);
3180
3181
5.52k
    if (ret == 0 && (nFlags & VSI_STAT_SIZE_FLAG))
3182
317
    {
3183
317
        CPLString osCacheFilename(pszFilename + strlen("/vsigzip/"));
3184
317
        osCacheFilename += ".properties";
3185
3186
        // Can we save a bit of seeking by using a .properties file?
3187
317
        VSILFILE *fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "rb");
3188
317
        if (fpCacheLength)
3189
141
        {
3190
141
            const char *pszLine;
3191
141
            GUIntBig nCompressedSize = 0;
3192
141
            GUIntBig nUncompressedSize = 0;
3193
141
            while ((pszLine = CPLReadLineL(fpCacheLength)) != nullptr)
3194
0
            {
3195
0
                if (STARTS_WITH_CI(pszLine, "compressed_size="))
3196
0
                {
3197
0
                    const char *pszBuffer =
3198
0
                        pszLine + strlen("compressed_size=");
3199
0
                    nCompressedSize = CPLScanUIntBig(
3200
0
                        pszBuffer, static_cast<int>(strlen(pszBuffer)));
3201
0
                }
3202
0
                else if (STARTS_WITH_CI(pszLine, "uncompressed_size="))
3203
0
                {
3204
0
                    const char *pszBuffer =
3205
0
                        pszLine + strlen("uncompressed_size=");
3206
0
                    nUncompressedSize = CPLScanUIntBig(
3207
0
                        pszBuffer, static_cast<int>(strlen(pszBuffer)));
3208
0
                }
3209
0
            }
3210
3211
141
            CPL_IGNORE_RET_VAL(VSIFCloseL(fpCacheLength));
3212
3213
141
            if (nCompressedSize == static_cast<GUIntBig>(pStatBuf->st_size))
3214
141
            {
3215
                // Patch with the uncompressed size.
3216
141
                pStatBuf->st_size = nUncompressedSize;
3217
3218
141
                VSIGZipHandle *poHandle =
3219
141
                    VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename,
3220
141
                                                               "rb");
3221
141
                if (poHandle)
3222
0
                {
3223
0
                    poHandle->SetUncompressedSize(nUncompressedSize);
3224
0
                    SaveInfo_unlocked(poHandle);
3225
0
                    delete poHandle;
3226
0
                }
3227
3228
141
                return ret;
3229
141
            }
3230
141
        }
3231
3232
        // No, then seek at the end of the data (slow).
3233
176
        VSIGZipHandle *poHandle =
3234
176
            VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
3235
176
        if (poHandle)
3236
0
        {
3237
0
            poHandle->Seek(0, SEEK_END);
3238
0
            const GUIntBig uncompressed_size =
3239
0
                static_cast<GUIntBig>(poHandle->Tell());
3240
0
            poHandle->Seek(0, SEEK_SET);
3241
3242
            // Patch with the uncompressed size.
3243
0
            pStatBuf->st_size = uncompressed_size;
3244
3245
0
            delete poHandle;
3246
0
        }
3247
176
        else
3248
176
        {
3249
176
            ret = -1;
3250
176
        }
3251
176
    }
3252
3253
5.38k
    return ret;
3254
5.52k
}
3255
3256
/************************************************************************/
3257
/*                             ReadDirEx()                                */
3258
/************************************************************************/
3259
3260
char **VSIGZipFilesystemHandler::ReadDirEx(const char * /*pszDirname*/,
3261
                                           int /* nMaxFiles */)
3262
0
{
3263
0
    return nullptr;
3264
0
}
3265
3266
/************************************************************************/
3267
/*                           GetOptions()                               */
3268
/************************************************************************/
3269
3270
const char *VSIGZipFilesystemHandler::GetOptions()
3271
0
{
3272
0
    return "<Options>"
3273
0
           "  <Option name='GDAL_NUM_THREADS' type='string' "
3274
0
           "description='Number of threads for compression. Either a integer "
3275
0
           "or ALL_CPUS'/>"
3276
0
           "  <Option name='CPL_VSIL_DEFLATE_CHUNK_SIZE' type='string' "
3277
0
           "description='Chunk of uncompressed data for parallelization. "
3278
0
           "Use K(ilobytes) or M(egabytes) suffix' default='1M'/>"
3279
0
           "</Options>";
3280
0
}
3281
3282
//! @endcond
3283
/************************************************************************/
3284
/*                   VSIInstallGZipFileHandler()                        */
3285
/************************************************************************/
3286
3287
/*!
3288
 \brief Install GZip file system handler.
3289
3290
 A special file handler is installed that allows reading on-the-fly and
3291
 writing in GZip (.gz) files.
3292
3293
 All portions of the file system underneath the base
3294
 path "/vsigzip/" will be handled by this driver.
3295
3296
 \verbatim embed:rst
3297
 See :ref:`/vsigzip/ documentation <vsigzip>`
3298
 \endverbatim
3299
3300
 */
3301
3302
void VSIInstallGZipFileHandler()
3303
3
{
3304
3
    VSIFileManager::InstallHandler(
3305
3
        "/vsigzip/", std::make_shared<VSIGZipFilesystemHandler>());
3306
3
}
3307
3308
//! @cond Doxygen_Suppress
3309
3310
/************************************************************************/
3311
/* ==================================================================== */
3312
/*                         VSIZipEntryFileOffset                        */
3313
/* ==================================================================== */
3314
/************************************************************************/
3315
3316
class VSIZipEntryFileOffset final : public VSIArchiveEntryFileOffset
3317
{
3318
  public:
3319
    unz_file_pos m_file_pos;
3320
3321
0
    explicit VSIZipEntryFileOffset(unz_file_pos file_pos) : m_file_pos()
3322
0
    {
3323
0
        m_file_pos.pos_in_zip_directory = file_pos.pos_in_zip_directory;
3324
0
        m_file_pos.num_of_file = file_pos.num_of_file;
3325
0
    }
3326
3327
    ~VSIZipEntryFileOffset() override;
3328
};
3329
3330
0
VSIZipEntryFileOffset::~VSIZipEntryFileOffset() = default;
3331
3332
/************************************************************************/
3333
/* ==================================================================== */
3334
/*                             VSIZipReader                             */
3335
/* ==================================================================== */
3336
/************************************************************************/
3337
3338
class VSIZipReader final : public VSIArchiveReader
3339
{
3340
    CPL_DISALLOW_COPY_ASSIGN(VSIZipReader)
3341
3342
  private:
3343
    unzFile unzF = nullptr;
3344
    unz_file_pos file_pos;
3345
    GUIntBig nNextFileSize = 0;
3346
    CPLString osNextFileName{};
3347
    GIntBig nModifiedTime = 0;
3348
3349
    bool SetInfo();
3350
3351
  public:
3352
    explicit VSIZipReader(const char *pszZipFileName);
3353
    ~VSIZipReader() override;
3354
3355
    int IsValid()
3356
1.73k
    {
3357
1.73k
        return unzF != nullptr;
3358
1.73k
    }
3359
3360
    unzFile GetUnzFileHandle()
3361
0
    {
3362
0
        return unzF;
3363
0
    }
3364
3365
    int GotoFirstFile() override;
3366
    int GotoNextFile() override;
3367
3368
    VSIArchiveEntryFileOffset *GetFileOffset() override
3369
0
    {
3370
0
        return new VSIZipEntryFileOffset(file_pos);
3371
0
    }
3372
3373
    GUIntBig GetFileSize() override
3374
0
    {
3375
0
        return nNextFileSize;
3376
0
    }
3377
3378
    CPLString GetFileName() override
3379
0
    {
3380
0
        return osNextFileName;
3381
0
    }
3382
3383
    GIntBig GetModifiedTime() override
3384
0
    {
3385
0
        return nModifiedTime;
3386
0
    }
3387
3388
    int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) override;
3389
};
3390
3391
/************************************************************************/
3392
/*                           VSIZipReader()                             */
3393
/************************************************************************/
3394
3395
VSIZipReader::VSIZipReader(const char *pszZipFileName)
3396
1.73k
    : unzF(cpl_unzOpen(pszZipFileName)), file_pos()
3397
1.73k
{
3398
1.73k
    file_pos.pos_in_zip_directory = 0;
3399
1.73k
    file_pos.num_of_file = 0;
3400
1.73k
}
3401
3402
/************************************************************************/
3403
/*                          ~VSIZipReader()                             */
3404
/************************************************************************/
3405
3406
VSIZipReader::~VSIZipReader()
3407
1.73k
{
3408
1.73k
    if (unzF)
3409
0
        cpl_unzClose(unzF);
3410
1.73k
}
3411
3412
/************************************************************************/
3413
/*                              SetInfo()                               */
3414
/************************************************************************/
3415
3416
bool VSIZipReader::SetInfo()
3417
0
{
3418
0
    char fileName[8193] = {};
3419
0
    unz_file_info file_info;
3420
0
    if (UNZ_OK != cpl_unzGetCurrentFileInfo(unzF, &file_info, fileName,
3421
0
                                            sizeof(fileName) - 1, nullptr, 0,
3422
0
                                            nullptr, 0))
3423
0
    {
3424
0
        CPLError(CE_Failure, CPLE_FileIO, "cpl_unzGetCurrentFileInfo failed");
3425
0
        cpl_unzGetFilePos(unzF, &file_pos);
3426
0
        return false;
3427
0
    }
3428
0
    fileName[sizeof(fileName) - 1] = '\0';
3429
0
    osNextFileName = fileName;
3430
0
    nNextFileSize = file_info.uncompressed_size;
3431
0
    struct tm brokendowntime;
3432
0
    brokendowntime.tm_sec = file_info.tmu_date.tm_sec;
3433
0
    brokendowntime.tm_min = file_info.tmu_date.tm_min;
3434
0
    brokendowntime.tm_hour = file_info.tmu_date.tm_hour;
3435
0
    brokendowntime.tm_mday = file_info.tmu_date.tm_mday;
3436
0
    brokendowntime.tm_mon = file_info.tmu_date.tm_mon;
3437
    // The minizip conventions differs from the Unix one.
3438
0
    brokendowntime.tm_year = file_info.tmu_date.tm_year - 1900;
3439
0
    nModifiedTime = CPLYMDHMSToUnixTime(&brokendowntime);
3440
3441
0
    cpl_unzGetFilePos(unzF, &file_pos);
3442
0
    return true;
3443
0
}
3444
3445
/************************************************************************/
3446
/*                           GotoNextFile()                             */
3447
/************************************************************************/
3448
3449
int VSIZipReader::GotoNextFile()
3450
0
{
3451
0
    if (cpl_unzGoToNextFile(unzF) != UNZ_OK)
3452
0
        return FALSE;
3453
3454
0
    if (!SetInfo())
3455
0
        return FALSE;
3456
3457
0
    return TRUE;
3458
0
}
3459
3460
/************************************************************************/
3461
/*                          GotoFirstFile()                             */
3462
/************************************************************************/
3463
3464
int VSIZipReader::GotoFirstFile()
3465
0
{
3466
0
    if (cpl_unzGoToFirstFile(unzF) != UNZ_OK)
3467
0
        return FALSE;
3468
3469
0
    if (!SetInfo())
3470
0
        return FALSE;
3471
3472
0
    return TRUE;
3473
0
}
3474
3475
/************************************************************************/
3476
/*                         GotoFileOffset()                             */
3477
/************************************************************************/
3478
3479
int VSIZipReader::GotoFileOffset(VSIArchiveEntryFileOffset *pOffset)
3480
0
{
3481
0
    VSIZipEntryFileOffset *pZipEntryOffset =
3482
0
        reinterpret_cast<VSIZipEntryFileOffset *>(pOffset);
3483
0
    if (cpl_unzGoToFilePos(unzF, &(pZipEntryOffset->m_file_pos)) != UNZ_OK)
3484
0
    {
3485
0
        CPLError(CE_Failure, CPLE_AppDefined, "GotoFileOffset failed");
3486
0
        return FALSE;
3487
0
    }
3488
3489
0
    if (!SetInfo())
3490
0
        return FALSE;
3491
3492
0
    return TRUE;
3493
0
}
3494
3495
/************************************************************************/
3496
/* ==================================================================== */
3497
/*                       VSIZipFilesystemHandler                        */
3498
/* ==================================================================== */
3499
/************************************************************************/
3500
3501
class VSIZipWriteHandle;
3502
3503
class VSIZipFilesystemHandler final : public VSIArchiveFilesystemHandler
3504
{
3505
    CPL_DISALLOW_COPY_ASSIGN(VSIZipFilesystemHandler)
3506
3507
    std::map<CPLString, VSIZipWriteHandle *> oMapZipWriteHandles{};
3508
    VSIVirtualHandleUniquePtr OpenForWrite_unlocked(const char *pszFilename,
3509
                                                    const char *pszAccess);
3510
3511
    struct VSIFileInZipInfo
3512
    {
3513
        VSIVirtualHandleUniquePtr poVirtualHandle{};
3514
        std::map<std::string, std::string> oMapProperties{};
3515
        int nCompressionMethod = 0;
3516
        uint64_t nUncompressedSize = 0;
3517
        uint64_t nCompressedSize = 0;
3518
        uint64_t nStartDataStream = 0;
3519
        uLong nCRC = 0;
3520
        bool bSOZipIndexFound = false;
3521
        bool bSOZipIndexValid = false;
3522
        uint32_t nSOZIPVersion = 0;
3523
        uint32_t nSOZIPToSkip = 0;
3524
        uint32_t nSOZIPChunkSize = 0;
3525
        uint32_t nSOZIPOffsetSize = 0;
3526
        uint64_t nSOZIPStartData = 0;
3527
    };
3528
3529
    bool GetFileInfo(const char *pszFilename, VSIFileInZipInfo &info,
3530
                     bool bSetError);
3531
3532
  public:
3533
3
    VSIZipFilesystemHandler() = default;
3534
    ~VSIZipFilesystemHandler() override;
3535
3536
    const char *GetPrefix() const override
3537
20.3k
    {
3538
20.3k
        return "/vsizip";
3539
20.3k
    }
3540
3541
    std::vector<CPLString> GetExtensions() const override;
3542
    std::unique_ptr<VSIArchiveReader>
3543
    CreateReader(const char *pszZipFileName) override;
3544
3545
    VSIVirtualHandleUniquePtr Open(const char *pszFilename,
3546
                                   const char *pszAccess, bool bSetError,
3547
                                   CSLConstList /* papszOptions */) override;
3548
3549
    char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
3550
                           CSLConstList papszOptions) override;
3551
3552
    VSIVirtualHandleUniquePtr OpenForWrite(const char *pszFilename,
3553
                                           const char *pszAccess);
3554
3555
    int CopyFile(const char *pszSource, const char *pszTarget,
3556
                 VSILFILE *fpSource, vsi_l_offset nSourceSize,
3557
                 const char *const *papszOptions,
3558
                 GDALProgressFunc pProgressFunc, void *pProgressData) override;
3559
3560
    int Mkdir(const char *pszDirname, long nMode) override;
3561
    char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
3562
    int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
3563
             int nFlags) override;
3564
3565
    const char *GetOptions() override;
3566
3567
    void RemoveFromMap(VSIZipWriteHandle *poHandle);
3568
};
3569
3570
/************************************************************************/
3571
/* ==================================================================== */
3572
/*                       VSIZipWriteHandle                              */
3573
/* ==================================================================== */
3574
/************************************************************************/
3575
3576
class VSIZipWriteHandle final : public VSIVirtualHandle
3577
{
3578
    CPL_DISALLOW_COPY_ASSIGN(VSIZipWriteHandle)
3579
3580
    VSIZipFilesystemHandler *m_poFS = nullptr;
3581
    void *m_hZIP = nullptr;
3582
    VSIZipWriteHandle *poChildInWriting = nullptr;
3583
    VSIZipWriteHandle *m_poParent = nullptr;
3584
    bool bAutoDeleteParent = false;
3585
    vsi_l_offset nCurOffset = 0;
3586
3587
  public:
3588
    VSIZipWriteHandle(VSIZipFilesystemHandler *poFS, void *hZIP,
3589
                      VSIZipWriteHandle *poParent);
3590
3591
    ~VSIZipWriteHandle() override;
3592
3593
    int Seek(vsi_l_offset nOffset, int nWhence) override;
3594
    vsi_l_offset Tell() override;
3595
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
3596
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
3597
3598
    int Eof() override
3599
0
    {
3600
0
        return 0;
3601
0
    }
3602
3603
    int Error() override
3604
0
    {
3605
0
        return 0;
3606
0
    }
3607
3608
    void ClearErr() override
3609
0
    {
3610
0
    }
3611
3612
    int Flush() override;
3613
    int Close() override;
3614
3615
    void StartNewFile(VSIZipWriteHandle *poSubFile);
3616
    void StopCurrentFile();
3617
3618
    void *GetHandle()
3619
0
    {
3620
0
        return m_hZIP;
3621
0
    }
3622
3623
    VSIZipWriteHandle *GetChildInWriting()
3624
0
    {
3625
0
        return poChildInWriting;
3626
0
    }
3627
3628
    void SetAutoDeleteParent()
3629
0
    {
3630
0
        bAutoDeleteParent = true;
3631
0
    }
3632
};
3633
3634
/************************************************************************/
3635
/*                      ~VSIZipFilesystemHandler()                      */
3636
/************************************************************************/
3637
3638
VSIZipFilesystemHandler::~VSIZipFilesystemHandler()
3639
0
{
3640
0
    for (std::map<CPLString, VSIZipWriteHandle *>::const_iterator iter =
3641
0
             oMapZipWriteHandles.begin();
3642
0
         iter != oMapZipWriteHandles.end(); ++iter)
3643
0
    {
3644
0
        CPLError(CE_Failure, CPLE_AppDefined, "%s has not been closed",
3645
0
                 iter->first.c_str());
3646
0
    }
3647
0
}
3648
3649
/************************************************************************/
3650
/*                          GetExtensions()                             */
3651
/************************************************************************/
3652
3653
std::vector<CPLString> VSIZipFilesystemHandler::GetExtensions() const
3654
1.56k
{
3655
1.56k
    std::vector<CPLString> oList;
3656
1.56k
    oList.push_back(".zip");
3657
1.56k
    oList.push_back(".kmz");
3658
1.56k
    oList.push_back(".dwf");
3659
1.56k
    oList.push_back(".ods");
3660
1.56k
    oList.push_back(".xlsx");
3661
1.56k
    oList.push_back(".xlsm");
3662
3663
    // Add to zip FS handler extensions array additional extensions
3664
    // listed in CPL_VSIL_ZIP_ALLOWED_EXTENSIONS config option.
3665
    // The extensions are divided by commas.
3666
1.56k
    const char *pszAllowedExtensions =
3667
1.56k
        CPLGetConfigOption("CPL_VSIL_ZIP_ALLOWED_EXTENSIONS", nullptr);
3668
1.56k
    if (pszAllowedExtensions)
3669
0
    {
3670
0
        char **papszExtensions =
3671
0
            CSLTokenizeString2(pszAllowedExtensions, ", ", 0);
3672
0
        for (int i = 0; papszExtensions[i] != nullptr; i++)
3673
0
        {
3674
0
            oList.push_back(papszExtensions[i]);
3675
0
        }
3676
0
        CSLDestroy(papszExtensions);
3677
0
    }
3678
3679
1.56k
    return oList;
3680
1.56k
}
3681
3682
/************************************************************************/
3683
/*                           CreateReader()                             */
3684
/************************************************************************/
3685
3686
std::unique_ptr<VSIArchiveReader>
3687
VSIZipFilesystemHandler::CreateReader(const char *pszZipFileName)
3688
1.73k
{
3689
1.73k
    auto poReader = std::make_unique<VSIZipReader>(pszZipFileName);
3690
3691
1.73k
    if (!poReader->IsValid() || !poReader->GotoFirstFile())
3692
1.73k
    {
3693
1.73k
        return nullptr;
3694
1.73k
    }
3695
3696
0
    return poReader;
3697
1.73k
}
3698
3699
/************************************************************************/
3700
/*                         VSISOZipHandle                               */
3701
/************************************************************************/
3702
3703
class VSISOZipHandle final : public VSIVirtualHandle
3704
{
3705
    VSIVirtualHandleUniquePtr poBaseHandle_{};
3706
    vsi_l_offset nPosCompressedStream_;
3707
    uint64_t compressed_size_;
3708
    uint64_t uncompressed_size_;
3709
    vsi_l_offset indexPos_;
3710
    uint32_t nToSkip_;
3711
    uint32_t nChunkSize_;
3712
    bool bEOF_ = false;
3713
    bool bError_ = false;
3714
    vsi_l_offset nCurPos_ = 0;
3715
    bool bOK_ = true;
3716
#ifdef HAVE_LIBDEFLATE
3717
    struct libdeflate_decompressor *pDecompressor_ = nullptr;
3718
#else
3719
    z_stream sStream_{};
3720
#endif
3721
3722
    VSISOZipHandle(const VSISOZipHandle &) = delete;
3723
    VSISOZipHandle &operator=(const VSISOZipHandle &) = delete;
3724
3725
  public:
3726
    VSISOZipHandle(VSIVirtualHandleUniquePtr poVirtualHandleIn,
3727
                   vsi_l_offset nPosCompressedStream, uint64_t compressed_size,
3728
                   uint64_t uncompressed_size, vsi_l_offset indexPos,
3729
                   uint32_t nToSkip, uint32_t nChunkSize);
3730
    ~VSISOZipHandle() override;
3731
3732
    int Seek(vsi_l_offset nOffset, int nWhence) override;
3733
3734
    vsi_l_offset Tell() override
3735
0
    {
3736
0
        return nCurPos_;
3737
0
    }
3738
3739
    size_t Read(void *pBuffer, size_t nSize, size_t nCount) override;
3740
3741
    size_t Write(const void *, size_t, size_t) override
3742
0
    {
3743
0
        return 0;
3744
0
    }
3745
3746
    int Eof() override
3747
0
    {
3748
0
        return bEOF_;
3749
0
    }
3750
3751
    int Error() override
3752
0
    {
3753
0
        return bError_;
3754
0
    }
3755
3756
    void ClearErr() override
3757
0
    {
3758
0
        bEOF_ = false;
3759
0
        bError_ = false;
3760
0
    }
3761
3762
    int Close() override;
3763
3764
    bool IsOK() const
3765
0
    {
3766
0
        return bOK_;
3767
0
    }
3768
};
3769
3770
/************************************************************************/
3771
/*                         VSISOZipHandle()                             */
3772
/************************************************************************/
3773
3774
VSISOZipHandle::VSISOZipHandle(VSIVirtualHandleUniquePtr poVirtualHandleIn,
3775
                               vsi_l_offset nPosCompressedStream,
3776
                               uint64_t compressed_size,
3777
                               uint64_t uncompressed_size,
3778
                               vsi_l_offset indexPos, uint32_t nToSkip,
3779
                               uint32_t nChunkSize)
3780
0
    : poBaseHandle_(std::move(poVirtualHandleIn)),
3781
0
      nPosCompressedStream_(nPosCompressedStream),
3782
0
      compressed_size_(compressed_size), uncompressed_size_(uncompressed_size),
3783
0
      indexPos_(indexPos), nToSkip_(nToSkip), nChunkSize_(nChunkSize)
3784
0
{
3785
#ifdef HAVE_LIBDEFLATE
3786
    pDecompressor_ = libdeflate_alloc_decompressor();
3787
    if (!pDecompressor_)
3788
        bOK_ = false;
3789
#else
3790
0
    memset(&sStream_, 0, sizeof(sStream_));
3791
0
    int err = inflateInit2(&sStream_, -MAX_WBITS);
3792
0
    if (err != Z_OK)
3793
0
        bOK_ = false;
3794
0
#endif
3795
0
}
3796
3797
/************************************************************************/
3798
/*                        ~VSISOZipHandle()                             */
3799
/************************************************************************/
3800
3801
VSISOZipHandle::~VSISOZipHandle()
3802
0
{
3803
0
    VSISOZipHandle::Close();
3804
0
    if (bOK_)
3805
0
    {
3806
#ifdef HAVE_LIBDEFLATE
3807
        libdeflate_free_decompressor(pDecompressor_);
3808
#else
3809
0
        inflateEnd(&sStream_);
3810
0
#endif
3811
0
    }
3812
0
}
3813
3814
/************************************************************************/
3815
/*                              Close()                                 */
3816
/************************************************************************/
3817
3818
int VSISOZipHandle::Close()
3819
0
{
3820
0
    int ret = 0;
3821
0
    if (poBaseHandle_)
3822
0
    {
3823
0
        ret = poBaseHandle_->Close();
3824
0
        poBaseHandle_.reset();
3825
0
    }
3826
0
    return ret;
3827
0
}
3828
3829
/************************************************************************/
3830
/*                              Seek()                                  */
3831
/************************************************************************/
3832
3833
int VSISOZipHandle::Seek(vsi_l_offset nOffset, int nWhence)
3834
0
{
3835
0
    bEOF_ = false;
3836
0
    if (nWhence == SEEK_SET)
3837
0
        nCurPos_ = nOffset;
3838
0
    else if (nWhence == SEEK_END)
3839
0
        nCurPos_ = uncompressed_size_;
3840
0
    else
3841
0
        nCurPos_ += nOffset;
3842
0
    return 0;
3843
0
}
3844
3845
/************************************************************************/
3846
/*                              Read()                                  */
3847
/************************************************************************/
3848
3849
size_t VSISOZipHandle::Read(void *pBuffer, size_t nSize, size_t nCount)
3850
0
{
3851
0
    size_t nToRead = nSize * nCount;
3852
0
    if (nCurPos_ >= uncompressed_size_ && nToRead > 0)
3853
0
    {
3854
0
        bEOF_ = true;
3855
0
        return 0;
3856
0
    }
3857
3858
0
    if (nSize != 1)
3859
0
    {
3860
0
        bError_ = true;
3861
0
        CPLError(CE_Failure, CPLE_NotSupported, "Unsupported nSize");
3862
0
        return 0;
3863
0
    }
3864
0
    if ((nCurPos_ % nChunkSize_) != 0)
3865
0
    {
3866
0
        bError_ = true;
3867
0
        CPLError(CE_Failure, CPLE_NotSupported,
3868
0
                 "nCurPos is not a multiple of nChunkSize");
3869
0
        return 0;
3870
0
    }
3871
0
    if (nCurPos_ + nToRead > uncompressed_size_)
3872
0
    {
3873
0
        nToRead = static_cast<size_t>(uncompressed_size_ - nCurPos_);
3874
0
        nCount = nToRead;
3875
0
    }
3876
0
    else if ((nToRead % nChunkSize_) != 0)
3877
0
    {
3878
0
        bError_ = true;
3879
0
        CPLError(CE_Failure, CPLE_NotSupported,
3880
0
                 "nToRead is not a multiple of nChunkSize");
3881
0
        return 0;
3882
0
    }
3883
3884
0
    const auto ReadOffsetInCompressedStream =
3885
0
        [this](uint64_t nChunkIdx) -> uint64_t
3886
0
    {
3887
0
        if (nChunkIdx == 0)
3888
0
            return 0;
3889
0
        if (nChunkIdx == 1 + (uncompressed_size_ - 1) / nChunkSize_)
3890
0
            return compressed_size_;
3891
0
        constexpr size_t nOffsetSize = 8;
3892
0
        if (poBaseHandle_->Seek(indexPos_ + 32 + nToSkip_ +
3893
0
                                    (nChunkIdx - 1) * nOffsetSize,
3894
0
                                SEEK_SET) != 0)
3895
0
            return static_cast<uint64_t>(-1);
3896
3897
0
        uint64_t nOffset;
3898
0
        if (poBaseHandle_->Read(&nOffset, sizeof(nOffset), 1) != 1)
3899
0
            return static_cast<uint64_t>(-1);
3900
0
        CPL_LSBPTR64(&nOffset);
3901
0
        return nOffset;
3902
0
    };
3903
3904
0
    size_t nOffsetInOutputBuffer = 0;
3905
0
    while (true)
3906
0
    {
3907
0
        uint64_t nOffsetInCompressedStream =
3908
0
            ReadOffsetInCompressedStream(nCurPos_ / nChunkSize_);
3909
0
        if (nOffsetInCompressedStream == static_cast<uint64_t>(-1))
3910
0
        {
3911
0
            bError_ = true;
3912
0
            CPLError(CE_Failure, CPLE_AppDefined,
3913
0
                     "Cannot read nOffsetInCompressedStream");
3914
0
            return 0;
3915
0
        }
3916
0
        uint64_t nNextOffsetInCompressedStream =
3917
0
            ReadOffsetInCompressedStream(1 + nCurPos_ / nChunkSize_);
3918
0
        if (nNextOffsetInCompressedStream == static_cast<uint64_t>(-1))
3919
0
        {
3920
0
            bError_ = true;
3921
0
            CPLError(CE_Failure, CPLE_AppDefined,
3922
0
                     "Cannot read nNextOffsetInCompressedStream");
3923
0
            return 0;
3924
0
        }
3925
3926
0
        if (nNextOffsetInCompressedStream <= nOffsetInCompressedStream ||
3927
0
            nNextOffsetInCompressedStream - nOffsetInCompressedStream >
3928
0
                13 + 2 * nChunkSize_ ||
3929
0
            nNextOffsetInCompressedStream > compressed_size_)
3930
0
        {
3931
0
            bError_ = true;
3932
0
            CPLError(
3933
0
                CE_Failure, CPLE_AppDefined,
3934
0
                "Invalid values for nOffsetInCompressedStream (" CPL_FRMT_GUIB
3935
0
                ") / "
3936
0
                "nNextOffsetInCompressedStream(" CPL_FRMT_GUIB ")",
3937
0
                static_cast<GUIntBig>(nOffsetInCompressedStream),
3938
0
                static_cast<GUIntBig>(nNextOffsetInCompressedStream));
3939
0
            return 0;
3940
0
        }
3941
3942
        // CPLDebug("VSIZIP", "Seek to compressed data at offset "
3943
        // CPL_FRMT_GUIB, static_cast<GUIntBig>(nPosCompressedStream_ +
3944
        // nOffsetInCompressedStream));
3945
0
        if (poBaseHandle_->Seek(
3946
0
                nPosCompressedStream_ + nOffsetInCompressedStream, SEEK_SET) !=
3947
0
            0)
3948
0
        {
3949
0
            bError_ = true;
3950
0
            return 0;
3951
0
        }
3952
3953
0
        const int nCompressedToRead = static_cast<int>(
3954
0
            nNextOffsetInCompressedStream - nOffsetInCompressedStream);
3955
        // CPLDebug("VSIZIP", "nCompressedToRead = %d", nCompressedToRead);
3956
0
        std::vector<GByte> abyCompressedData(nCompressedToRead);
3957
0
        if (poBaseHandle_->Read(&abyCompressedData[0], nCompressedToRead, 1) !=
3958
0
            1)
3959
0
        {
3960
0
            bError_ = true;
3961
0
            return 0;
3962
0
        }
3963
3964
0
        size_t nToReadThisIter =
3965
0
            std::min(nToRead, static_cast<size_t>(nChunkSize_));
3966
3967
0
        if (nCompressedToRead >= 5 &&
3968
0
            abyCompressedData[nCompressedToRead - 5] == 0x00 &&
3969
0
            memcmp(&abyCompressedData[nCompressedToRead - 4],
3970
0
                   "\x00\x00\xFF\xFF", 4) == 0)
3971
0
        {
3972
            // Tag this flush block as the last one.
3973
0
            abyCompressedData[nCompressedToRead - 5] = 0x01;
3974
0
        }
3975
3976
#ifdef HAVE_LIBDEFLATE
3977
        size_t nOut = 0;
3978
        if (libdeflate_deflate_decompress(
3979
                pDecompressor_, &abyCompressedData[0], nCompressedToRead,
3980
                static_cast<Bytef *>(pBuffer) + nOffsetInOutputBuffer,
3981
                nToReadThisIter, &nOut) != LIBDEFLATE_SUCCESS)
3982
        {
3983
            bError_ = true;
3984
            CPLError(
3985
                CE_Failure, CPLE_AppDefined,
3986
                "libdeflate_deflate_decompress() failed at pos " CPL_FRMT_GUIB,
3987
                static_cast<GUIntBig>(nCurPos_));
3988
            return 0;
3989
        }
3990
        if (nOut != nToReadThisIter)
3991
        {
3992
            bError_ = true;
3993
            CPLError(CE_Failure, CPLE_AppDefined,
3994
                     "Only %u bytes decompressed at pos " CPL_FRMT_GUIB
3995
                     " whereas %u where expected",
3996
                     static_cast<unsigned>(nOut),
3997
                     static_cast<GUIntBig>(nCurPos_),
3998
                     static_cast<unsigned>(nToReadThisIter));
3999
            return 0;
4000
        }
4001
#else
4002
0
        sStream_.avail_in = nCompressedToRead;
4003
0
        sStream_.next_in = &abyCompressedData[0];
4004
0
        sStream_.avail_out = static_cast<int>(nToReadThisIter);
4005
0
        sStream_.next_out =
4006
0
            static_cast<Bytef *>(pBuffer) + nOffsetInOutputBuffer;
4007
4008
0
        int err = inflate(&sStream_, Z_FINISH);
4009
0
        if ((err != Z_OK && err != Z_STREAM_END))
4010
0
        {
4011
0
            bError_ = true;
4012
0
            CPLError(CE_Failure, CPLE_AppDefined,
4013
0
                     "inflate() failed at pos " CPL_FRMT_GUIB,
4014
0
                     static_cast<GUIntBig>(nCurPos_));
4015
0
            inflateReset(&sStream_);
4016
0
            return 0;
4017
0
        }
4018
0
        if (sStream_.avail_in != 0)
4019
0
            CPLDebug("VSIZIP", "avail_in = %d", sStream_.avail_in);
4020
0
        if (sStream_.avail_out != 0)
4021
0
        {
4022
0
            bError_ = true;
4023
0
            CPLError(
4024
0
                CE_Failure, CPLE_AppDefined,
4025
0
                "Only %u bytes decompressed at pos " CPL_FRMT_GUIB
4026
0
                " whereas %u where expected",
4027
0
                static_cast<unsigned>(nToReadThisIter - sStream_.avail_out),
4028
0
                static_cast<GUIntBig>(nCurPos_),
4029
0
                static_cast<unsigned>(nToReadThisIter));
4030
0
            inflateReset(&sStream_);
4031
0
            return 0;
4032
0
        }
4033
0
        inflateReset(&sStream_);
4034
0
#endif
4035
0
        nOffsetInOutputBuffer += nToReadThisIter;
4036
0
        nCurPos_ += nToReadThisIter;
4037
0
        nToRead -= nToReadThisIter;
4038
0
        if (nToRead == 0)
4039
0
            break;
4040
0
    }
4041
4042
0
    return nCount;
4043
0
}
4044
4045
/************************************************************************/
4046
/*                          GetFileInfo()                               */
4047
/************************************************************************/
4048
4049
bool VSIZipFilesystemHandler::GetFileInfo(const char *pszFilename,
4050
                                          VSIFileInZipInfo &info,
4051
                                          bool bSetError)
4052
2.08k
{
4053
4054
2.08k
    CPLString osZipInFileName;
4055
2.08k
    std::unique_ptr<char, VSIFreeReleaser> zipFilename(
4056
2.08k
        SplitFilename(pszFilename, osZipInFileName, true, bSetError));
4057
2.08k
    if (zipFilename == nullptr)
4058
1.45k
        return false;
4059
4060
624
    {
4061
624
        std::unique_lock oLock(oMutex);
4062
624
        if (oMapZipWriteHandles.find(zipFilename.get()) !=
4063
624
            oMapZipWriteHandles.end())
4064
0
        {
4065
0
            CPLError(CE_Failure, CPLE_AppDefined,
4066
0
                     "Cannot read a zip file being written");
4067
0
            return false;
4068
0
        }
4069
624
    }
4070
4071
624
    auto poReader = OpenArchiveFile(zipFilename.get(), osZipInFileName);
4072
624
    if (poReader == nullptr)
4073
624
    {
4074
624
        return false;
4075
624
    }
4076
4077
0
    VSIFilesystemHandler *poFSHandler =
4078
0
        VSIFileManager::GetHandler(zipFilename.get());
4079
4080
0
    VSIVirtualHandleUniquePtr poVirtualHandle(
4081
0
        poFSHandler->Open(zipFilename.get(), "rb"));
4082
4083
0
    if (poVirtualHandle == nullptr)
4084
0
    {
4085
0
        return false;
4086
0
    }
4087
4088
0
    unzFile unzF =
4089
0
        cpl::down_cast<VSIZipReader *>(poReader.get())->GetUnzFileHandle();
4090
4091
0
    if (cpl_unzOpenCurrentFile(unzF) != UNZ_OK)
4092
0
    {
4093
0
        CPLError(CE_Failure, CPLE_AppDefined,
4094
0
                 "cpl_unzOpenCurrentFile() failed");
4095
0
        return false;
4096
0
    }
4097
4098
0
    info.nStartDataStream = cpl_unzGetCurrentFileZStreamPos(unzF);
4099
4100
0
    unz_file_info file_info;
4101
0
    if (cpl_unzGetCurrentFileInfo(unzF, &file_info, nullptr, 0, nullptr, 0,
4102
0
                                  nullptr, 0) != UNZ_OK)
4103
0
    {
4104
0
        CPLError(CE_Failure, CPLE_AppDefined,
4105
0
                 "cpl_unzGetCurrentFileInfo() failed");
4106
0
        cpl_unzCloseCurrentFile(unzF);
4107
0
        return false;
4108
0
    }
4109
4110
0
    if (file_info.size_file_extra)
4111
0
    {
4112
0
        std::vector<GByte> abyExtra(file_info.size_file_extra);
4113
0
        poVirtualHandle->Seek(file_info.file_extra_abs_offset, SEEK_SET);
4114
0
        if (poVirtualHandle->Read(&abyExtra[0], abyExtra.size(), 1) == 1)
4115
0
        {
4116
0
            size_t nPos = 0;
4117
0
            while (nPos + 2 * sizeof(uint16_t) <= abyExtra.size())
4118
0
            {
4119
0
                uint16_t nId;
4120
0
                memcpy(&nId, &abyExtra[nPos], sizeof(uint16_t));
4121
0
                nPos += sizeof(uint16_t);
4122
0
                CPL_LSBPTR16(&nId);
4123
0
                uint16_t nSize;
4124
0
                memcpy(&nSize, &abyExtra[nPos], sizeof(uint16_t));
4125
0
                nPos += sizeof(uint16_t);
4126
0
                CPL_LSBPTR16(&nSize);
4127
0
                if (nId == 0x564b && nPos + nSize <= abyExtra.size())  // "KV"
4128
0
                {
4129
0
                    if (nSize >= strlen("KeyValuePairs") + 1 &&
4130
0
                        memcmp(&abyExtra[nPos], "KeyValuePairs",
4131
0
                               strlen("KeyValuePairs")) == 0)
4132
0
                    {
4133
0
                        int nPos2 = static_cast<int>(strlen("KeyValuePairs"));
4134
0
                        int nKVPairs = abyExtra[nPos + nPos2];
4135
0
                        nPos2++;
4136
0
                        for (int iKV = 0; iKV < nKVPairs; ++iKV)
4137
0
                        {
4138
0
                            if (nPos2 + sizeof(uint16_t) > nSize)
4139
0
                                break;
4140
0
                            uint16_t nKeyLen;
4141
0
                            memcpy(&nKeyLen, &abyExtra[nPos + nPos2],
4142
0
                                   sizeof(uint16_t));
4143
0
                            nPos2 += sizeof(uint16_t);
4144
0
                            CPL_LSBPTR16(&nKeyLen);
4145
0
                            if (nPos2 + nKeyLen > nSize)
4146
0
                                break;
4147
0
                            std::string osKey;
4148
0
                            osKey.resize(nKeyLen);
4149
0
                            memcpy(&osKey[0], &abyExtra[nPos + nPos2], nKeyLen);
4150
0
                            nPos2 += nKeyLen;
4151
4152
0
                            if (nPos2 + sizeof(uint16_t) > nSize)
4153
0
                                break;
4154
0
                            uint16_t nValLen;
4155
0
                            memcpy(&nValLen, &abyExtra[nPos + nPos2],
4156
0
                                   sizeof(uint16_t));
4157
0
                            nPos2 += sizeof(uint16_t);
4158
0
                            CPL_LSBPTR16(&nValLen);
4159
0
                            if (nPos2 + nValLen > nSize)
4160
0
                                break;
4161
0
                            std::string osVal;
4162
0
                            osVal.resize(nValLen);
4163
0
                            memcpy(&osVal[0], &abyExtra[nPos + nPos2], nValLen);
4164
0
                            nPos2 += nValLen;
4165
4166
0
                            info.oMapProperties[osKey] = std::move(osVal);
4167
0
                        }
4168
0
                    }
4169
0
                }
4170
0
                nPos += nSize;
4171
0
            }
4172
0
        }
4173
0
    }
4174
4175
0
    info.nCRC = file_info.crc;
4176
0
    info.nCompressionMethod = static_cast<int>(file_info.compression_method);
4177
0
    info.nUncompressedSize = static_cast<uint64_t>(file_info.uncompressed_size);
4178
0
    info.nCompressedSize = static_cast<uint64_t>(file_info.compressed_size);
4179
4180
    // Sanity checks
4181
0
    if (info.nCompressedSize >
4182
0
        std::numeric_limits<uint64_t>::max() - info.nStartDataStream)
4183
0
    {
4184
0
        CPLError(CE_Failure, CPLE_AppDefined,
4185
0
                 "Invalid compressed size for file %s", pszFilename);
4186
0
        return false;
4187
0
    }
4188
0
    const uLong64 afterFileOffset =
4189
0
        info.nStartDataStream + info.nCompressedSize;
4190
4191
    // Cf https://stackoverflow.com/questions/16792189/gzip-compression-ratio-for-zeros/16794960
4192
0
    constexpr unsigned MAX_DEFLATE_COMPRESSION_RATIO = 1032;
4193
0
    if (info.nCompressedSize == 0 && info.nUncompressedSize != 0)
4194
0
    {
4195
0
        CPLError(CE_Failure, CPLE_AppDefined,
4196
0
                 "Invalid compressed size (=0) vs uncompressed size (!=0) for "
4197
0
                 "file %s",
4198
0
                 pszFilename);
4199
0
        return false;
4200
0
    }
4201
0
    else if (info.nCompressedSize != 0 &&
4202
0
             info.nUncompressedSize / info.nCompressedSize >
4203
0
                 MAX_DEFLATE_COMPRESSION_RATIO)
4204
0
    {
4205
0
        CPLError(CE_Failure, CPLE_AppDefined,
4206
0
                 "Invalid compression ratio for file %s: %" PRIu64, pszFilename,
4207
0
                 info.nUncompressedSize / info.nCompressedSize);
4208
0
        return false;
4209
0
    }
4210
4211
    // A bit arbitrary
4212
0
    constexpr unsigned THRESHOLD_FOR_BIG_ALLOCS = 1024 * 1024 * 1024;
4213
0
    if (info.nUncompressedSize > THRESHOLD_FOR_BIG_ALLOCS)
4214
0
    {
4215
        // Check that the compressed file size is consistent with the ZIP file size
4216
0
        poVirtualHandle->Seek(0, SEEK_END);
4217
0
        if (afterFileOffset > poVirtualHandle->Tell())
4218
0
        {
4219
0
            CPLError(CE_Failure, CPLE_AppDefined,
4220
0
                     "Invalid compressed size for file %s: %" PRIu64,
4221
0
                     pszFilename, info.nCompressedSize);
4222
0
            return false;
4223
0
        }
4224
0
    }
4225
4226
    // Try to locate .sozip.idx file
4227
0
    unz_file_info file_info2;
4228
0
    std::string osAuxName;
4229
0
    osAuxName.resize(1024);
4230
0
    uLong64 indexPos;
4231
0
    if (file_info.compression_method == 8 &&
4232
0
        cpl_unzCurrentFileInfoFromLocalHeader(
4233
0
            unzF, afterFileOffset, &file_info2, &osAuxName[0], osAuxName.size(),
4234
0
            &indexPos) == UNZ_OK)
4235
0
    {
4236
0
        osAuxName.resize(strlen(osAuxName.c_str()));
4237
0
        if (osAuxName.find(".sozip.idx") != std::string::npos)
4238
0
        {
4239
0
            info.bSOZipIndexFound = true;
4240
0
            info.nSOZIPStartData = indexPos;
4241
0
            poVirtualHandle->Seek(indexPos, SEEK_SET);
4242
0
            uint32_t nVersion = 0;
4243
0
            poVirtualHandle->Read(&nVersion, sizeof(nVersion), 1);
4244
0
            CPL_LSBPTR32(&nVersion);
4245
0
            uint32_t nToSkip = 0;
4246
0
            poVirtualHandle->Read(&nToSkip, sizeof(nToSkip), 1);
4247
0
            CPL_LSBPTR32(&nToSkip);
4248
0
            uint32_t nChunkSize = 0;
4249
0
            poVirtualHandle->Read(&nChunkSize, sizeof(nChunkSize), 1);
4250
0
            CPL_LSBPTR32(&nChunkSize);
4251
0
            uint32_t nOffsetSize = 0;
4252
0
            poVirtualHandle->Read(&nOffsetSize, sizeof(nOffsetSize), 1);
4253
0
            CPL_LSBPTR32(&nOffsetSize);
4254
0
            uint64_t nUncompressedSize = 0;
4255
0
            poVirtualHandle->Read(&nUncompressedSize, sizeof(nUncompressedSize),
4256
0
                                  1);
4257
0
            CPL_LSBPTR64(&nUncompressedSize);
4258
0
            uint64_t nCompressedSize = 0;
4259
0
            poVirtualHandle->Read(&nCompressedSize, sizeof(nCompressedSize), 1);
4260
0
            CPL_LSBPTR64(&nCompressedSize);
4261
4262
0
            info.nSOZIPVersion = nVersion;
4263
0
            info.nSOZIPToSkip = nToSkip;
4264
0
            info.nSOZIPChunkSize = nChunkSize;
4265
0
            info.nSOZIPOffsetSize = nOffsetSize;
4266
4267
0
            bool bValid = true;
4268
0
            if (nVersion != 1)
4269
0
            {
4270
0
                CPLDebug("SOZIP", "version = %u, expected 1", nVersion);
4271
0
                bValid = false;
4272
0
            }
4273
0
            if (nCompressedSize != file_info.compressed_size)
4274
0
            {
4275
0
                CPLDebug("SOZIP",
4276
0
                         "compressedSize field inconsistent with file");
4277
0
                bValid = false;
4278
0
            }
4279
0
            if (nUncompressedSize != file_info.uncompressed_size)
4280
0
            {
4281
0
                CPLDebug("SOZIP",
4282
0
                         "uncompressedSize field inconsistent with file");
4283
0
                bValid = false;
4284
0
            }
4285
0
            if (!(nChunkSize > 0 && nChunkSize < 100 * 1024 * 1024))
4286
0
            {
4287
0
                CPLDebug("SOZIP", "invalid chunkSize = %u", nChunkSize);
4288
0
                bValid = false;
4289
0
            }
4290
0
            if (nOffsetSize != 8)
4291
0
            {
4292
0
                CPLDebug("SOZIP", "invalid offsetSize = %u", nOffsetSize);
4293
0
                bValid = false;
4294
0
            }
4295
0
            if (file_info2.compression_method != 0)
4296
0
            {
4297
0
                CPLDebug("SOZIP", "unexpected compression_method = %u",
4298
0
                         static_cast<unsigned>(file_info2.compression_method));
4299
0
                bValid = false;
4300
0
            }
4301
0
            if (bValid)
4302
0
            {
4303
0
                const auto nExpectedIndexSize =
4304
0
                    32 + static_cast<uint64_t>(nToSkip) +
4305
0
                    ((nUncompressedSize - 1) / nChunkSize) * nOffsetSize;
4306
0
                if (nExpectedIndexSize != file_info2.uncompressed_size)
4307
0
                {
4308
0
                    CPLDebug("SOZIP", "invalid file size for index");
4309
0
                    bValid = false;
4310
0
                }
4311
0
            }
4312
0
            if (bValid)
4313
0
            {
4314
0
                info.bSOZipIndexValid = true;
4315
0
                CPLDebug("SOZIP", "Found valid SOZIP index: %s",
4316
0
                         osAuxName.c_str());
4317
0
            }
4318
0
            else
4319
0
            {
4320
0
                CPLDebug("SOZIP", "Found *invalid* SOZIP index: %s",
4321
0
                         osAuxName.c_str());
4322
0
            }
4323
0
        }
4324
0
    }
4325
4326
0
    cpl_unzCloseCurrentFile(unzF);
4327
4328
0
    info.poVirtualHandle = std::move(poVirtualHandle);
4329
4330
0
    return true;
4331
0
}
4332
4333
/************************************************************************/
4334
/*                                 Open()                               */
4335
/************************************************************************/
4336
4337
VSIVirtualHandleUniquePtr
4338
VSIZipFilesystemHandler::Open(const char *pszFilename, const char *pszAccess,
4339
                              bool bSetError, CSLConstList /* papszOptions */)
4340
2.08k
{
4341
4342
2.08k
    if (strchr(pszAccess, 'w') != nullptr)
4343
0
    {
4344
0
        return OpenForWrite(pszFilename, pszAccess);
4345
0
    }
4346
4347
2.08k
    if (strchr(pszAccess, '+') != nullptr)
4348
0
    {
4349
0
        CPLError(CE_Failure, CPLE_AppDefined,
4350
0
                 "Read-write random access not supported for /vsizip");
4351
0
        return nullptr;
4352
0
    }
4353
4354
2.08k
    VSIFileInZipInfo info;
4355
2.08k
    if (!GetFileInfo(pszFilename, info, bSetError))
4356
2.08k
        return nullptr;
4357
4358
0
#ifdef ENABLE_DEFLATE64
4359
0
    if (info.nCompressionMethod == 9)
4360
0
    {
4361
0
        auto poGZIPHandle = std::make_unique<VSIDeflate64Handle>(
4362
0
            std::move(info.poVirtualHandle), nullptr, info.nStartDataStream,
4363
0
            info.nCompressedSize, info.nUncompressedSize, info.nCRC);
4364
0
        if (!(poGZIPHandle->IsInitOK()))
4365
0
        {
4366
0
            return nullptr;
4367
0
        }
4368
4369
        // Wrap the VSIGZipHandle inside a buffered reader that will
4370
        // improve dramatically performance when doing small backward
4371
        // seeks.
4372
0
        return VSIVirtualHandleUniquePtr(
4373
0
            VSICreateBufferedReaderHandle(poGZIPHandle.release()));
4374
0
    }
4375
0
    else
4376
0
#endif
4377
0
    {
4378
0
        if (info.bSOZipIndexValid)
4379
0
        {
4380
0
            auto poSOZIPHandle = std::make_unique<VSISOZipHandle>(
4381
0
                std::move(info.poVirtualHandle), info.nStartDataStream,
4382
0
                info.nCompressedSize, info.nUncompressedSize,
4383
0
                info.nSOZIPStartData, info.nSOZIPToSkip, info.nSOZIPChunkSize);
4384
0
            if (!poSOZIPHandle->IsOK())
4385
0
            {
4386
0
                return nullptr;
4387
0
            }
4388
0
            return VSIVirtualHandleUniquePtr(VSICreateCachedFile(
4389
0
                poSOZIPHandle.release(), info.nSOZIPChunkSize, 0));
4390
0
        }
4391
4392
0
        auto poGZIPHandle = std::make_unique<VSIGZipHandle>(
4393
0
            std::move(info.poVirtualHandle), nullptr, info.nStartDataStream,
4394
0
            info.nCompressedSize, info.nUncompressedSize, info.nCRC,
4395
0
            info.nCompressionMethod == 0);
4396
0
        if (!(poGZIPHandle->IsInitOK()))
4397
0
        {
4398
0
            return nullptr;
4399
0
        }
4400
4401
        // Wrap the VSIGZipHandle inside a buffered reader that will
4402
        // improve dramatically performance when doing small backward
4403
        // seeks.
4404
0
        return VSIVirtualHandleUniquePtr(
4405
0
            VSICreateBufferedReaderHandle(poGZIPHandle.release()));
4406
0
    }
4407
0
}
4408
4409
/************************************************************************/
4410
/*                          GetFileMetadata()                           */
4411
/************************************************************************/
4412
4413
char **VSIZipFilesystemHandler::GetFileMetadata(const char *pszFilename,
4414
                                                const char *pszDomain,
4415
                                                CSLConstList /*papszOptions*/)
4416
0
{
4417
0
    VSIFileInZipInfo info;
4418
0
    if (!GetFileInfo(pszFilename, info, true))
4419
0
        return nullptr;
4420
4421
0
    if (!pszDomain)
4422
0
    {
4423
0
        CPLStringList aosMetadata;
4424
0
        for (const auto &kv : info.oMapProperties)
4425
0
        {
4426
0
            aosMetadata.AddNameValue(kv.first.c_str(), kv.second.c_str());
4427
0
        }
4428
0
        return aosMetadata.StealList();
4429
0
    }
4430
0
    else if (EQUAL(pszDomain, "ZIP"))
4431
0
    {
4432
0
        CPLStringList aosMetadata;
4433
0
        aosMetadata.SetNameValue(
4434
0
            "START_DATA_OFFSET",
4435
0
            CPLSPrintf(CPL_FRMT_GUIB,
4436
0
                       static_cast<GUIntBig>(info.nStartDataStream)));
4437
4438
0
        if (info.nCompressionMethod == 0)
4439
0
            aosMetadata.SetNameValue("COMPRESSION_METHOD", "0 (STORED)");
4440
0
        else if (info.nCompressionMethod == 8)
4441
0
            aosMetadata.SetNameValue("COMPRESSION_METHOD", "8 (DEFLATE)");
4442
0
        else
4443
0
        {
4444
0
            aosMetadata.SetNameValue("COMPRESSION_METHOD",
4445
0
                                     CPLSPrintf("%d", info.nCompressionMethod));
4446
0
        }
4447
0
        aosMetadata.SetNameValue(
4448
0
            "COMPRESSED_SIZE",
4449
0
            CPLSPrintf(CPL_FRMT_GUIB,
4450
0
                       static_cast<GUIntBig>(info.nCompressedSize)));
4451
0
        aosMetadata.SetNameValue(
4452
0
            "UNCOMPRESSED_SIZE",
4453
0
            CPLSPrintf(CPL_FRMT_GUIB,
4454
0
                       static_cast<GUIntBig>(info.nUncompressedSize)));
4455
4456
0
        if (info.bSOZipIndexFound)
4457
0
        {
4458
0
            aosMetadata.SetNameValue("SOZIP_FOUND", "YES");
4459
4460
0
            aosMetadata.SetNameValue("SOZIP_VERSION",
4461
0
                                     CPLSPrintf("%u", info.nSOZIPVersion));
4462
4463
0
            aosMetadata.SetNameValue("SOZIP_OFFSET_SIZE",
4464
0
                                     CPLSPrintf("%u", info.nSOZIPOffsetSize));
4465
4466
0
            aosMetadata.SetNameValue("SOZIP_CHUNK_SIZE",
4467
0
                                     CPLSPrintf("%u", info.nSOZIPChunkSize));
4468
4469
0
            aosMetadata.SetNameValue(
4470
0
                "SOZIP_START_DATA_OFFSET",
4471
0
                CPLSPrintf(CPL_FRMT_GUIB,
4472
0
                           static_cast<GUIntBig>(info.nSOZIPStartData)));
4473
4474
0
            if (info.bSOZipIndexValid)
4475
0
            {
4476
0
                aosMetadata.SetNameValue("SOZIP_VALID", "YES");
4477
0
            }
4478
0
        }
4479
4480
0
        return aosMetadata.StealList();
4481
0
    }
4482
0
    return nullptr;
4483
0
}
4484
4485
/************************************************************************/
4486
/*                                Mkdir()                               */
4487
/************************************************************************/
4488
4489
int VSIZipFilesystemHandler::Mkdir(const char *pszDirname, long /* nMode */)
4490
0
{
4491
0
    CPLString osDirname = pszDirname;
4492
0
    if (!osDirname.empty() && osDirname.back() != '/')
4493
0
        osDirname += "/";
4494
0
    return OpenForWrite(osDirname, "wb") != nullptr ? 0 : -1;
4495
0
}
4496
4497
/************************************************************************/
4498
/*                               ReadDirEx()                            */
4499
/************************************************************************/
4500
4501
char **VSIZipFilesystemHandler::ReadDirEx(const char *pszDirname, int nMaxFiles)
4502
0
{
4503
0
    CPLString osInArchiveSubDir;
4504
0
    char *zipFilename =
4505
0
        SplitFilename(pszDirname, osInArchiveSubDir, true, true);
4506
0
    if (zipFilename == nullptr)
4507
0
        return nullptr;
4508
4509
0
    {
4510
0
        std::unique_lock oLock(oMutex);
4511
4512
0
        if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end())
4513
0
        {
4514
0
            CPLError(CE_Failure, CPLE_AppDefined,
4515
0
                     "Cannot read a zip file being written");
4516
0
            CPLFree(zipFilename);
4517
0
            return nullptr;
4518
0
        }
4519
0
    }
4520
0
    CPLFree(zipFilename);
4521
4522
0
    return VSIArchiveFilesystemHandler::ReadDirEx(pszDirname, nMaxFiles);
4523
0
}
4524
4525
/************************************************************************/
4526
/*                                 Stat()                               */
4527
/************************************************************************/
4528
4529
int VSIZipFilesystemHandler::Stat(const char *pszFilename,
4530
                                  VSIStatBufL *pStatBuf, int nFlags)
4531
2.90k
{
4532
2.90k
    CPLString osInArchiveSubDir;
4533
4534
2.90k
    memset(pStatBuf, 0, sizeof(VSIStatBufL));
4535
4536
2.90k
    char *zipFilename = SplitFilename(pszFilename, osInArchiveSubDir, true,
4537
2.90k
                                      (nFlags & VSI_STAT_SET_ERROR_FLAG) != 0);
4538
2.90k
    if (zipFilename == nullptr)
4539
1.69k
        return -1;
4540
4541
1.20k
    {
4542
1.20k
        std::unique_lock oLock(oMutex);
4543
4544
1.20k
        if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end())
4545
0
        {
4546
0
            CPLError(CE_Failure, CPLE_AppDefined,
4547
0
                     "Cannot read a zip file being written");
4548
0
            CPLFree(zipFilename);
4549
0
            return -1;
4550
0
        }
4551
1.20k
    }
4552
1.20k
    CPLFree(zipFilename);
4553
4554
1.20k
    return VSIArchiveFilesystemHandler::Stat(pszFilename, pStatBuf, nFlags);
4555
1.20k
}
4556
4557
/************************************************************************/
4558
/*                             RemoveFromMap()                           */
4559
/************************************************************************/
4560
4561
void VSIZipFilesystemHandler::RemoveFromMap(VSIZipWriteHandle *poHandle)
4562
0
{
4563
0
    std::unique_lock oLock(oMutex);
4564
4565
0
    for (std::map<CPLString, VSIZipWriteHandle *>::iterator iter =
4566
0
             oMapZipWriteHandles.begin();
4567
0
         iter != oMapZipWriteHandles.end(); ++iter)
4568
0
    {
4569
0
        if (iter->second == poHandle)
4570
0
        {
4571
0
            oMapZipWriteHandles.erase(iter);
4572
0
            break;
4573
0
        }
4574
0
    }
4575
0
}
4576
4577
/************************************************************************/
4578
/*                             OpenForWrite()                           */
4579
/************************************************************************/
4580
4581
VSIVirtualHandleUniquePtr
4582
VSIZipFilesystemHandler::OpenForWrite(const char *pszFilename,
4583
                                      const char *pszAccess)
4584
0
{
4585
0
    std::unique_lock oLock(oMutex);
4586
0
    return OpenForWrite_unlocked(pszFilename, pszAccess);
4587
0
}
4588
4589
VSIVirtualHandleUniquePtr
4590
VSIZipFilesystemHandler::OpenForWrite_unlocked(const char *pszFilename,
4591
                                               const char *pszAccess)
4592
0
{
4593
0
    CPLString osZipInFileName;
4594
4595
0
    char *zipFilename =
4596
0
        SplitFilename(pszFilename, osZipInFileName, false, false);
4597
0
    if (zipFilename == nullptr)
4598
0
        return nullptr;
4599
0
    CPLString osZipFilename = zipFilename;
4600
0
    CPLFree(zipFilename);
4601
0
    zipFilename = nullptr;
4602
4603
    // Invalidate cached file list.
4604
0
    auto iter = oFileList.find(osZipFilename);
4605
0
    if (iter != oFileList.end())
4606
0
    {
4607
0
        oFileList.erase(iter);
4608
0
    }
4609
4610
0
    auto oIter = oMapZipWriteHandles.find(osZipFilename);
4611
0
    if (oIter != oMapZipWriteHandles.end())
4612
0
    {
4613
0
        if (strchr(pszAccess, '+') != nullptr)
4614
0
        {
4615
0
            CPLError(
4616
0
                CE_Failure, CPLE_AppDefined,
4617
0
                "Random access not supported for writable file in /vsizip");
4618
0
            return nullptr;
4619
0
        }
4620
4621
0
        VSIZipWriteHandle *poZIPHandle = oIter->second;
4622
4623
0
        if (poZIPHandle->GetChildInWriting() != nullptr)
4624
0
        {
4625
0
            CPLError(CE_Failure, CPLE_AppDefined,
4626
0
                     "Cannot create %s while another file is being "
4627
0
                     "written in the .zip",
4628
0
                     osZipInFileName.c_str());
4629
0
            return nullptr;
4630
0
        }
4631
4632
0
        poZIPHandle->StopCurrentFile();
4633
4634
        // Re-add path separator when creating directories.
4635
0
        char chLastChar = pszFilename[strlen(pszFilename) - 1];
4636
0
        if (chLastChar == '/' || chLastChar == '\\')
4637
0
            osZipInFileName += chLastChar;
4638
4639
0
        if (CPLCreateFileInZip(poZIPHandle->GetHandle(), osZipInFileName,
4640
0
                               nullptr) != CE_None)
4641
0
            return nullptr;
4642
4643
0
        auto poChildHandle =
4644
0
            std::make_unique<VSIZipWriteHandle>(this, nullptr, poZIPHandle);
4645
4646
0
        poZIPHandle->StartNewFile(poChildHandle.get());
4647
4648
0
        return VSIVirtualHandleUniquePtr(poChildHandle.release());
4649
0
    }
4650
0
    else
4651
0
    {
4652
0
        char **papszOptions = nullptr;
4653
0
        if ((strchr(pszAccess, '+') && osZipInFileName.empty()) ||
4654
0
            !osZipInFileName.empty())
4655
0
        {
4656
0
            VSIStatBufL sBuf;
4657
0
            if (VSIStatExL(osZipFilename, &sBuf, VSI_STAT_EXISTS_FLAG) == 0)
4658
0
                papszOptions = CSLAddNameValue(papszOptions, "APPEND", "TRUE");
4659
0
        }
4660
4661
0
        void *hZIP = CPLCreateZip(osZipFilename, papszOptions);
4662
0
        CSLDestroy(papszOptions);
4663
4664
0
        if (hZIP == nullptr)
4665
0
            return nullptr;
4666
4667
0
        auto poHandle = new VSIZipWriteHandle(this, hZIP, nullptr);
4668
0
        oMapZipWriteHandles[osZipFilename] = poHandle;
4669
4670
0
        if (!osZipInFileName.empty())
4671
0
        {
4672
0
            auto poRes = std::unique_ptr<VSIZipWriteHandle>(
4673
0
                cpl::down_cast<VSIZipWriteHandle *>(
4674
0
                    OpenForWrite_unlocked(pszFilename, pszAccess).release()));
4675
0
            if (poRes == nullptr)
4676
0
            {
4677
0
                delete poHandle;
4678
0
                oMapZipWriteHandles.erase(osZipFilename);
4679
0
                return nullptr;
4680
0
            }
4681
4682
0
            poRes->SetAutoDeleteParent();
4683
4684
0
            return VSIVirtualHandleUniquePtr(poRes.release());
4685
0
        }
4686
4687
0
        return VSIVirtualHandleUniquePtr(poHandle);
4688
0
    }
4689
0
}
4690
4691
/************************************************************************/
4692
/*                           GetOptions()                               */
4693
/************************************************************************/
4694
4695
const char *VSIZipFilesystemHandler::GetOptions()
4696
0
{
4697
0
    return "<Options>"
4698
0
           "  <Option name='GDAL_NUM_THREADS' type='string' "
4699
0
           "description='Number of threads for compression. Either a integer "
4700
0
           "or ALL_CPUS'/>"
4701
0
           "  <Option name='CPL_VSIL_DEFLATE_CHUNK_SIZE' type='string' "
4702
0
           "description='Chunk of uncompressed data for parallelization. "
4703
0
           "Use K(ilobytes) or M(egabytes) suffix' default='1M'/>"
4704
0
           "</Options>";
4705
0
}
4706
4707
/************************************************************************/
4708
/*                           CopyFile()                                 */
4709
/************************************************************************/
4710
4711
int VSIZipFilesystemHandler::CopyFile(const char *pszSource,
4712
                                      const char *pszTarget, VSILFILE *fpSource,
4713
                                      vsi_l_offset /* nSourceSize */,
4714
                                      CSLConstList papszOptions,
4715
                                      GDALProgressFunc pProgressFunc,
4716
                                      void *pProgressData)
4717
0
{
4718
0
    CPLString osZipInFileName;
4719
4720
0
    char *zipFilename = SplitFilename(pszTarget, osZipInFileName, false, false);
4721
0
    if (zipFilename == nullptr)
4722
0
        return -1;
4723
0
    CPLString osZipFilename = zipFilename;
4724
0
    CPLFree(zipFilename);
4725
0
    zipFilename = nullptr;
4726
0
    if (osZipInFileName.empty())
4727
0
    {
4728
0
        CPLError(CE_Failure, CPLE_AppDefined,
4729
0
                 "Target filename should be of the form "
4730
0
                 "/vsizip/path_to.zip/filename_within_zip");
4731
0
        return -1;
4732
0
    }
4733
4734
    // Invalidate cached file list.
4735
0
    auto oIterFileList = oFileList.find(osZipFilename);
4736
0
    if (oIterFileList != oFileList.end())
4737
0
    {
4738
0
        oFileList.erase(oIterFileList);
4739
0
    }
4740
4741
0
    const auto oIter = oMapZipWriteHandles.find(osZipFilename);
4742
0
    if (oIter != oMapZipWriteHandles.end())
4743
0
    {
4744
0
        VSIZipWriteHandle *poZIPHandle = oIter->second;
4745
4746
0
        if (poZIPHandle->GetChildInWriting() != nullptr)
4747
0
        {
4748
0
            CPLError(CE_Failure, CPLE_AppDefined,
4749
0
                     "Cannot create %s while another file is being "
4750
0
                     "written in the .zip",
4751
0
                     osZipInFileName.c_str());
4752
0
            return -1;
4753
0
        }
4754
4755
0
        if (CPLAddFileInZip(poZIPHandle->GetHandle(), osZipInFileName.c_str(),
4756
0
                            pszSource, fpSource, papszOptions, pProgressFunc,
4757
0
                            pProgressData) != CE_None)
4758
0
        {
4759
0
            return -1;
4760
0
        }
4761
0
        return 0;
4762
0
    }
4763
0
    else
4764
0
    {
4765
0
        CPLStringList aosOptionsCreateZip;
4766
0
        VSIStatBufL sBuf;
4767
0
        if (VSIStatExL(osZipFilename, &sBuf, VSI_STAT_EXISTS_FLAG) == 0)
4768
0
            aosOptionsCreateZip.SetNameValue("APPEND", "TRUE");
4769
4770
0
        void *hZIP = CPLCreateZip(osZipFilename, aosOptionsCreateZip.List());
4771
4772
0
        if (hZIP == nullptr)
4773
0
            return -1;
4774
4775
0
        if (CPLAddFileInZip(hZIP, osZipInFileName.c_str(), pszSource, fpSource,
4776
0
                            papszOptions, pProgressFunc,
4777
0
                            pProgressData) != CE_None)
4778
0
        {
4779
0
            CPLCloseZip(hZIP);
4780
0
            return -1;
4781
0
        }
4782
0
        CPLCloseZip(hZIP);
4783
0
        return 0;
4784
0
    }
4785
0
}
4786
4787
/************************************************************************/
4788
/*                          VSIZipWriteHandle()                         */
4789
/************************************************************************/
4790
4791
VSIZipWriteHandle::VSIZipWriteHandle(VSIZipFilesystemHandler *poFS, void *hZIP,
4792
                                     VSIZipWriteHandle *poParent)
4793
0
    : m_poFS(poFS), m_hZIP(hZIP), m_poParent(poParent)
4794
0
{
4795
0
}
4796
4797
/************************************************************************/
4798
/*                         ~VSIZipWriteHandle()                         */
4799
/************************************************************************/
4800
4801
VSIZipWriteHandle::~VSIZipWriteHandle()
4802
0
{
4803
0
    VSIZipWriteHandle::Close();
4804
0
}
4805
4806
/************************************************************************/
4807
/*                               Seek()                                 */
4808
/************************************************************************/
4809
4810
int VSIZipWriteHandle::Seek(vsi_l_offset nOffset, int nWhence)
4811
0
{
4812
0
    if (nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR))
4813
0
        return 0;
4814
0
    if (nOffset == nCurOffset && nWhence == SEEK_SET)
4815
0
        return 0;
4816
4817
0
    CPLError(CE_Failure, CPLE_NotSupported,
4818
0
             "VSIFSeekL() is not supported on writable Zip files");
4819
0
    return -1;
4820
0
}
4821
4822
/************************************************************************/
4823
/*                               Tell()                                 */
4824
/************************************************************************/
4825
4826
vsi_l_offset VSIZipWriteHandle::Tell()
4827
0
{
4828
0
    return nCurOffset;
4829
0
}
4830
4831
/************************************************************************/
4832
/*                               Read()                                 */
4833
/************************************************************************/
4834
4835
size_t VSIZipWriteHandle::Read(void * /* pBuffer */, size_t /* nSize */,
4836
                               size_t /* nMemb */)
4837
0
{
4838
0
    CPLError(CE_Failure, CPLE_NotSupported,
4839
0
             "VSIFReadL() is not supported on writable Zip files");
4840
0
    return 0;
4841
0
}
4842
4843
/************************************************************************/
4844
/*                               Write()                                 */
4845
/************************************************************************/
4846
4847
size_t VSIZipWriteHandle::Write(const void *pBuffer, size_t nSize, size_t nMemb)
4848
0
{
4849
0
    if (m_poParent == nullptr)
4850
0
    {
4851
0
        CPLError(CE_Failure, CPLE_NotSupported,
4852
0
                 "VSIFWriteL() is not supported on "
4853
0
                 "main Zip file or closed subfiles");
4854
0
        return 0;
4855
0
    }
4856
4857
0
    const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer);
4858
0
    size_t nBytesToWrite = nSize * nMemb;
4859
0
    size_t nWritten = 0;
4860
0
    while (nWritten < nBytesToWrite)
4861
0
    {
4862
0
        int nToWrite = static_cast<int>(
4863
0
            std::min(static_cast<size_t>(INT_MAX), nBytesToWrite));
4864
0
        if (CPLWriteFileInZip(m_poParent->m_hZIP, pabyBuffer, nToWrite) !=
4865
0
            CE_None)
4866
0
            return 0;
4867
0
        nWritten += nToWrite;
4868
0
        pabyBuffer += nToWrite;
4869
0
    }
4870
4871
0
    nCurOffset += nSize * nMemb;
4872
4873
0
    return nMemb;
4874
0
}
4875
4876
/************************************************************************/
4877
/*                               Flush()                                */
4878
/************************************************************************/
4879
4880
int VSIZipWriteHandle::Flush()
4881
0
{
4882
    /*CPLError(CE_Failure, CPLE_NotSupported,
4883
             "VSIFFlushL() is not supported on writable Zip files");*/
4884
0
    return 0;
4885
0
}
4886
4887
/************************************************************************/
4888
/*                               Close()                                */
4889
/************************************************************************/
4890
4891
int VSIZipWriteHandle::Close()
4892
0
{
4893
0
    int nRet = 0;
4894
0
    if (m_poParent)
4895
0
    {
4896
0
        CPLCloseFileInZip(m_poParent->m_hZIP);
4897
0
        m_poParent->poChildInWriting = nullptr;
4898
0
        if (bAutoDeleteParent)
4899
0
        {
4900
0
            if (m_poParent->Close() != 0)
4901
0
                nRet = -1;
4902
0
            delete m_poParent;
4903
0
        }
4904
0
        m_poParent = nullptr;
4905
0
    }
4906
0
    if (poChildInWriting)
4907
0
    {
4908
0
        if (poChildInWriting->Close() != 0)
4909
0
            nRet = -1;
4910
0
        poChildInWriting = nullptr;
4911
0
    }
4912
0
    if (m_hZIP)
4913
0
    {
4914
0
        if (CPLCloseZip(m_hZIP) != CE_None)
4915
0
            nRet = -1;
4916
0
        m_hZIP = nullptr;
4917
4918
0
        m_poFS->RemoveFromMap(this);
4919
0
    }
4920
4921
0
    return nRet;
4922
0
}
4923
4924
/************************************************************************/
4925
/*                           StopCurrentFile()                          */
4926
/************************************************************************/
4927
4928
void VSIZipWriteHandle::StopCurrentFile()
4929
0
{
4930
0
    if (poChildInWriting)
4931
0
        poChildInWriting->Close();
4932
0
    poChildInWriting = nullptr;
4933
0
}
4934
4935
/************************************************************************/
4936
/*                           StartNewFile()                             */
4937
/************************************************************************/
4938
4939
void VSIZipWriteHandle::StartNewFile(VSIZipWriteHandle *poSubFile)
4940
0
{
4941
0
    poChildInWriting = poSubFile;
4942
0
}
4943
4944
//! @endcond
4945
4946
/************************************************************************/
4947
/*                    VSIInstallZipFileHandler()                        */
4948
/************************************************************************/
4949
4950
/*!
4951
 \brief Install ZIP file system handler.
4952
4953
 A special file handler is installed that allows reading on-the-fly in ZIP
4954
 (.zip) archives.
4955
4956
 All portions of the file system underneath the base path "/vsizip/" will be
4957
 handled by this driver.
4958
4959
 \verbatim embed:rst
4960
 See :ref:`/vsizip/ documentation <vsizip>`
4961
 \endverbatim
4962
4963
 */
4964
4965
void VSIInstallZipFileHandler()
4966
3
{
4967
3
    VSIFileManager::InstallHandler("/vsizip/",
4968
3
                                   std::make_shared<VSIZipFilesystemHandler>());
4969
3
}
4970
4971
/************************************************************************/
4972
/*                         CPLZLibDeflate()                             */
4973
/************************************************************************/
4974
4975
/**
4976
 * \brief Compress a buffer with ZLib compression.
4977
 *
4978
 * @param ptr input buffer.
4979
 * @param nBytes size of input buffer in bytes.
4980
 * @param nLevel ZLib compression level (-1 for default).
4981
 * @param outptr output buffer, or NULL to let the function allocate it.
4982
 * @param nOutAvailableBytes size of output buffer if provided, or ignored.
4983
 * @param pnOutBytes pointer to a size_t, where to store the size of the
4984
 *                   output buffer.
4985
 *
4986
 * @return the output buffer (to be freed with VSIFree() if not provided)
4987
 *         or NULL in case of error.
4988
 *
4989
 */
4990
4991
void *CPLZLibDeflate(const void *ptr, size_t nBytes, int nLevel, void *outptr,
4992
                     size_t nOutAvailableBytes, size_t *pnOutBytes)
4993
0
{
4994
0
    if (pnOutBytes != nullptr)
4995
0
        *pnOutBytes = 0;
4996
4997
0
    size_t nTmpSize = 0;
4998
0
    void *pTmp;
4999
#ifdef HAVE_LIBDEFLATE
5000
    struct libdeflate_compressor *enc =
5001
        libdeflate_alloc_compressor(nLevel < 0 ? 7 : nLevel);
5002
    if (enc == nullptr)
5003
    {
5004
        return nullptr;
5005
    }
5006
#endif
5007
0
    if (outptr == nullptr)
5008
0
    {
5009
#ifdef HAVE_LIBDEFLATE
5010
        nTmpSize = libdeflate_zlib_compress_bound(enc, nBytes);
5011
#else
5012
0
        nTmpSize = 32 + nBytes * 2;
5013
0
#endif
5014
0
        pTmp = VSIMalloc(nTmpSize);
5015
0
        if (pTmp == nullptr)
5016
0
        {
5017
#ifdef HAVE_LIBDEFLATE
5018
            libdeflate_free_compressor(enc);
5019
#endif
5020
0
            return nullptr;
5021
0
        }
5022
0
    }
5023
0
    else
5024
0
    {
5025
0
        pTmp = outptr;
5026
0
        nTmpSize = nOutAvailableBytes;
5027
0
    }
5028
5029
#ifdef HAVE_LIBDEFLATE
5030
    size_t nCompressedBytes =
5031
        libdeflate_zlib_compress(enc, ptr, nBytes, pTmp, nTmpSize);
5032
    libdeflate_free_compressor(enc);
5033
    if (nCompressedBytes == 0)
5034
    {
5035
        if (pTmp != outptr)
5036
            VSIFree(pTmp);
5037
        return nullptr;
5038
    }
5039
    if (pnOutBytes != nullptr)
5040
        *pnOutBytes = nCompressedBytes;
5041
#else
5042
0
    z_stream strm;
5043
0
    strm.zalloc = nullptr;
5044
0
    strm.zfree = nullptr;
5045
0
    strm.opaque = nullptr;
5046
0
    int ret = deflateInit(&strm, nLevel < 0 ? Z_DEFAULT_COMPRESSION : nLevel);
5047
0
    if (ret != Z_OK)
5048
0
    {
5049
0
        if (pTmp != outptr)
5050
0
            VSIFree(pTmp);
5051
0
        return nullptr;
5052
0
    }
5053
5054
0
    strm.avail_in = static_cast<uInt>(nBytes);
5055
0
    strm.next_in = reinterpret_cast<Bytef *>(const_cast<void *>(ptr));
5056
0
    strm.avail_out = static_cast<uInt>(nTmpSize);
5057
0
    strm.next_out = reinterpret_cast<Bytef *>(pTmp);
5058
0
    ret = deflate(&strm, Z_FINISH);
5059
0
    if (ret != Z_STREAM_END)
5060
0
    {
5061
0
        if (pTmp != outptr)
5062
0
            VSIFree(pTmp);
5063
0
        return nullptr;
5064
0
    }
5065
0
    if (pnOutBytes != nullptr)
5066
0
        *pnOutBytes = nTmpSize - strm.avail_out;
5067
0
    deflateEnd(&strm);
5068
0
#endif
5069
5070
0
    return pTmp;
5071
0
}
5072
5073
/************************************************************************/
5074
/*                         CPLZLibInflate()                             */
5075
/************************************************************************/
5076
5077
/**
5078
 * \brief Uncompress a buffer compressed with ZLib compression.
5079
 *
5080
 * @param ptr input buffer.
5081
 * @param nBytes size of input buffer in bytes.
5082
 * @param outptr output buffer, or NULL to let the function allocate it.
5083
 * @param nOutAvailableBytes size of output buffer if provided, or ignored.
5084
 * @param pnOutBytes pointer to a size_t, where to store the size of the
5085
 *                   output buffer.
5086
 *
5087
 * @return the output buffer (to be freed with VSIFree() if not provided)
5088
 *         or NULL in case of error.
5089
 *
5090
 */
5091
5092
void *CPLZLibInflate(const void *ptr, size_t nBytes, void *outptr,
5093
                     size_t nOutAvailableBytes, size_t *pnOutBytes)
5094
0
{
5095
0
    return CPLZLibInflateEx(ptr, nBytes, outptr, nOutAvailableBytes, false,
5096
0
                            pnOutBytes);
5097
0
}
5098
5099
/************************************************************************/
5100
/*                         CPLZLibInflateEx()                           */
5101
/************************************************************************/
5102
5103
/**
5104
 * \brief Uncompress a buffer compressed with ZLib compression.
5105
 *
5106
 * @param ptr input buffer.
5107
 * @param nBytes size of input buffer in bytes.
5108
 * @param outptr output buffer, or NULL to let the function allocate it.
5109
 * @param nOutAvailableBytes size of output buffer if provided, or ignored.
5110
 * @param bAllowResizeOutptr whether the function is allowed to grow outptr
5111
 *                           (using VSIRealloc) if its initial capacity
5112
 *                           provided by nOutAvailableBytes is not
5113
 *                           large enough. Ignored if outptr is NULL.
5114
 * @param pnOutBytes pointer to a size_t, where to store the size of the
5115
 *                   output buffer.
5116
 *
5117
 * @return the output buffer (to be freed with VSIFree() if not provided)
5118
 *         or NULL in case of error. If bAllowResizeOutptr is set to true,
5119
 *         only the returned pointer should be freed by the caller, as outptr
5120
 *         might have been reallocated or freed.
5121
 *
5122
 * @since GDAL 3.9.0
5123
 */
5124
5125
void *CPLZLibInflateEx(const void *ptr, size_t nBytes, void *outptr,
5126
                       size_t nOutAvailableBytes, bool bAllowResizeOutptr,
5127
                       size_t *pnOutBytes)
5128
0
{
5129
0
    if (pnOutBytes != nullptr)
5130
0
        *pnOutBytes = 0;
5131
0
    char *pszReallocatableBuf = nullptr;
5132
5133
#ifdef HAVE_LIBDEFLATE
5134
    if (outptr)
5135
    {
5136
        struct libdeflate_decompressor *dec = libdeflate_alloc_decompressor();
5137
        if (dec == nullptr)
5138
        {
5139
            if (bAllowResizeOutptr)
5140
                VSIFree(outptr);
5141
            return nullptr;
5142
        }
5143
        enum libdeflate_result res;
5144
        size_t nOutBytes = 0;
5145
        if (nBytes > 2 && static_cast<const GByte *>(ptr)[0] == 0x1F &&
5146
            static_cast<const GByte *>(ptr)[1] == 0x8B)
5147
        {
5148
            res = libdeflate_gzip_decompress(dec, ptr, nBytes, outptr,
5149
                                             nOutAvailableBytes, &nOutBytes);
5150
        }
5151
        else
5152
        {
5153
            res = libdeflate_zlib_decompress(dec, ptr, nBytes, outptr,
5154
                                             nOutAvailableBytes, &nOutBytes);
5155
        }
5156
        if (pnOutBytes)
5157
            *pnOutBytes = nOutBytes;
5158
        libdeflate_free_decompressor(dec);
5159
        if (res == LIBDEFLATE_INSUFFICIENT_SPACE && bAllowResizeOutptr)
5160
        {
5161
            if (nOutAvailableBytes >
5162
                (std::numeric_limits<size_t>::max() - 1) / 2)
5163
            {
5164
                VSIFree(outptr);
5165
                return nullptr;
5166
            }
5167
            size_t nOutBufSize = nOutAvailableBytes * 2;
5168
            pszReallocatableBuf = static_cast<char *>(
5169
                VSI_REALLOC_VERBOSE(outptr, nOutBufSize + 1));
5170
            if (!pszReallocatableBuf)
5171
            {
5172
                VSIFree(outptr);
5173
                return nullptr;
5174
            }
5175
            outptr = nullptr;
5176
            nOutAvailableBytes = nOutBufSize;
5177
        }
5178
        else if (res != LIBDEFLATE_SUCCESS)
5179
        {
5180
            if (bAllowResizeOutptr)
5181
                VSIFree(outptr);
5182
            return nullptr;
5183
        }
5184
        else
5185
        {
5186
            // Nul-terminate if possible.
5187
            if (nOutBytes < nOutAvailableBytes)
5188
            {
5189
                static_cast<char *>(outptr)[nOutBytes] = '\0';
5190
            }
5191
            return outptr;
5192
        }
5193
    }
5194
#endif
5195
5196
0
    z_stream strm;
5197
0
    memset(&strm, 0, sizeof(strm));
5198
0
    strm.zalloc = nullptr;
5199
0
    strm.zfree = nullptr;
5200
0
    strm.opaque = nullptr;
5201
0
    int ret;
5202
    // MAX_WBITS + 32 mode which detects automatically gzip vs zlib
5203
    // encapsulation seems to be broken with
5204
    // /opt/intel/oneapi/intelpython/latest/lib/libz.so.1 from
5205
    // intel/oneapi-basekit Docker image
5206
0
    if (nBytes > 2 && static_cast<const GByte *>(ptr)[0] == 0x1F &&
5207
0
        static_cast<const GByte *>(ptr)[1] == 0x8B)
5208
0
    {
5209
0
        ret = inflateInit2(&strm, MAX_WBITS + 16);  // gzip
5210
0
    }
5211
0
    else
5212
0
    {
5213
0
        ret = inflateInit2(&strm, MAX_WBITS);  // zlib
5214
0
    }
5215
0
    if (ret != Z_OK)
5216
0
    {
5217
0
        if (bAllowResizeOutptr)
5218
0
            VSIFree(outptr);
5219
0
        VSIFree(pszReallocatableBuf);
5220
0
        return nullptr;
5221
0
    }
5222
5223
0
    size_t nOutBufSize = 0;
5224
0
    char *pszOutBuf = nullptr;
5225
5226
#ifdef HAVE_LIBDEFLATE
5227
    if (pszReallocatableBuf)
5228
    {
5229
        pszOutBuf = pszReallocatableBuf;
5230
        nOutBufSize = nOutAvailableBytes;
5231
    }
5232
    else
5233
#endif
5234
0
        if (!outptr)
5235
0
    {
5236
0
        if (nBytes > (std::numeric_limits<size_t>::max() - 1) / 2)
5237
0
        {
5238
0
            inflateEnd(&strm);
5239
0
            return nullptr;
5240
0
        }
5241
0
        nOutBufSize = 2 * nBytes + 1;
5242
0
        pszOutBuf = static_cast<char *>(VSI_MALLOC_VERBOSE(nOutBufSize));
5243
0
        if (pszOutBuf == nullptr)
5244
0
        {
5245
0
            inflateEnd(&strm);
5246
0
            return nullptr;
5247
0
        }
5248
0
        pszReallocatableBuf = pszOutBuf;
5249
0
        bAllowResizeOutptr = true;
5250
0
    }
5251
0
#ifndef HAVE_LIBDEFLATE
5252
0
    else
5253
0
    {
5254
0
        pszOutBuf = static_cast<char *>(outptr);
5255
0
        nOutBufSize = nOutAvailableBytes;
5256
0
        if (bAllowResizeOutptr)
5257
0
            pszReallocatableBuf = pszOutBuf;
5258
0
    }
5259
0
#endif
5260
5261
0
    strm.next_in = static_cast<Bytef *>(const_cast<void *>(ptr));
5262
0
    strm.next_out = reinterpret_cast<Bytef *>(pszOutBuf);
5263
0
    size_t nInBytesRemaining = nBytes;
5264
0
    size_t nOutBytesRemaining = nOutBufSize;
5265
5266
0
    while (true)
5267
0
    {
5268
0
        strm.avail_in = static_cast<uInt>(std::min<size_t>(
5269
0
            nInBytesRemaining, std::numeric_limits<uInt>::max()));
5270
0
        const auto avail_in_before = strm.avail_in;
5271
0
        strm.avail_out = static_cast<uInt>(std::min<size_t>(
5272
0
            nOutBytesRemaining, std::numeric_limits<uInt>::max()));
5273
0
        const auto avail_out_before = strm.avail_out;
5274
0
        ret = inflate(&strm, Z_FINISH);
5275
0
        nInBytesRemaining -= (avail_in_before - strm.avail_in);
5276
0
        nOutBytesRemaining -= (avail_out_before - strm.avail_out);
5277
5278
0
        if (ret == Z_BUF_ERROR && strm.avail_out == 0)
5279
0
        {
5280
#ifdef HAVE_LIBDEFLATE
5281
            CPLAssert(bAllowResizeOutptr);
5282
#else
5283
0
            if (!bAllowResizeOutptr)
5284
0
            {
5285
0
                VSIFree(pszReallocatableBuf);
5286
0
                inflateEnd(&strm);
5287
0
                return nullptr;
5288
0
            }
5289
0
#endif
5290
5291
0
            const size_t nAlreadyWritten = nOutBufSize - nOutBytesRemaining;
5292
0
            if (nOutBufSize > (std::numeric_limits<size_t>::max() - 1) / 2)
5293
0
            {
5294
0
                VSIFree(pszReallocatableBuf);
5295
0
                inflateEnd(&strm);
5296
0
                return nullptr;
5297
0
            }
5298
0
            nOutBufSize = nOutBufSize * 2 + 1;
5299
0
            char *pszNew = static_cast<char *>(
5300
0
                VSI_REALLOC_VERBOSE(pszReallocatableBuf, nOutBufSize));
5301
0
            if (!pszNew)
5302
0
            {
5303
0
                VSIFree(pszReallocatableBuf);
5304
0
                inflateEnd(&strm);
5305
0
                return nullptr;
5306
0
            }
5307
0
            pszOutBuf = pszNew;
5308
0
            pszReallocatableBuf = pszOutBuf;
5309
0
            nOutBytesRemaining = nOutBufSize - nAlreadyWritten;
5310
0
            strm.next_out =
5311
0
                reinterpret_cast<Bytef *>(pszOutBuf + nAlreadyWritten);
5312
0
        }
5313
0
        else if (ret != Z_OK || nInBytesRemaining == 0)
5314
0
            break;
5315
0
    }
5316
5317
0
    if (ret == Z_OK || ret == Z_STREAM_END)
5318
0
    {
5319
0
        size_t nOutBytes = nOutBufSize - nOutBytesRemaining;
5320
        // Nul-terminate if possible.
5321
0
        if (nOutBytes < nOutBufSize)
5322
0
        {
5323
0
            pszOutBuf[nOutBytes] = '\0';
5324
0
        }
5325
0
        inflateEnd(&strm);
5326
0
        if (pnOutBytes != nullptr)
5327
0
            *pnOutBytes = nOutBytes;
5328
0
        return pszOutBuf;
5329
0
    }
5330
0
    else
5331
0
    {
5332
0
        VSIFree(pszReallocatableBuf);
5333
0
        inflateEnd(&strm);
5334
0
        return nullptr;
5335
0
    }
5336
0
}