Coverage Report

Created: 2025-06-13 06:18

/src/gdal/port/cpl_vsil_gzip.cpp
Line
Count
Source (jump to first uncovered line)
1
/******************************************************************************
2
 *
3
 * Project:  CPL - Common Portability Library
4
 * Purpose:  Implement VSI large file api for gz/zip files (.gz and .zip).
5
 * Author:   Even Rouault, even.rouault at spatialys.com
6
 *
7
 ******************************************************************************
8
 * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
9
 *
10
 * SPDX-License-Identifier: MIT
11
 ****************************************************************************/
12
13
//! @cond Doxygen_Suppress
14
15
/* gzio.c -- IO on .gz files
16
  Copyright (C) 1995-2005 Jean-loup Gailly.
17
18
  This software is provided 'as-is', without any express or implied
19
  warranty.  In no event will the authors be held liable for any damages
20
  arising from the use of this software.
21
22
  Permission is granted to anyone to use this software for any purpose,
23
  including commercial applications, and to alter it and redistribute it
24
  freely, subject to the following restrictions:
25
26
  1. The origin of this software must not be misrepresented; you must not
27
     claim that you wrote the original software. If you use this software
28
     in a product, an acknowledgment in the product documentation would be
29
     appreciated but is not required.
30
  2. Altered source versions must be plainly marked as such, and must not be
31
     misrepresented as being the original software.
32
  3. This notice may not be removed or altered from any source distribution.
33
34
  Jean-loup Gailly        Mark Adler
35
  jloup@gzip.org          madler@alumni.caltech.edu
36
37
  The data format used by the zlib library is described by RFCs (Request for
38
  Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
39
  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
40
*/
41
42
/* This file contains a refactoring of gzio.c from zlib project.
43
44
   It replaces classical calls operating on FILE* by calls to the VSI large file
45
   API. It also adds the capability to seek at the end of the file, which is not
46
   implemented in original gzSeek. It also implements a concept of in-memory
47
   "snapshots", that are a way of improving efficiency while seeking GZip
48
   files. Snapshots are created regularly when decompressing the data a snapshot
49
   of the gzip state.  Later we can seek directly in the compressed data to the
50
   closest snapshot in order to reduce the amount of data to uncompress again.
51
52
   For .gz files, an effort is done to cache the size of the uncompressed data
53
   in a .gz.properties file, so that we don't need to seek at the end of the
54
   file each time a Stat() is done.
55
56
   For .zip and .gz, both reading and writing are supported, but just one mode
57
   at a time (read-only or write-only).
58
*/
59
60
#include "cpl_port.h"
61
#include "cpl_conv.h"
62
#include "cpl_vsi.h"
63
64
#include <cerrno>
65
#include <climits>
66
#include <cstddef>
67
#include <cstdio>
68
#include <cstdlib>
69
#include <cstring>
70
#include <ctime>
71
72
#if HAVE_FCNTL_H
73
#include <fcntl.h>
74
#endif
75
#if HAVE_SYS_STAT_H
76
#include <sys/stat.h>
77
#endif
78
#include "cpl_zlib_header.h"  // to avoid warnings when including zlib.h
79
80
#ifdef HAVE_LIBDEFLATE
81
#include "libdeflate.h"
82
#endif
83
84
#include <algorithm>
85
#include <iterator>
86
#include <limits>
87
#include <list>
88
#include <map>
89
#include <memory>
90
#include <mutex>
91
#include <string>
92
#include <utility>
93
#include <vector>
94
95
#include "cpl_error.h"
96
#include "cpl_minizip_ioapi.h"
97
#include "cpl_minizip_unzip.h"
98
#include "cpl_multiproc.h"
99
#include "cpl_string.h"
100
#include "cpl_time.h"
101
#include "cpl_vsi_virtual.h"
102
#include "cpl_worker_thread_pool.h"
103
104
constexpr int Z_BUFSIZE = 65536;           // Original size is 16384
105
constexpr int gz_magic[2] = {0x1f, 0x8b};  // gzip magic header
106
107
// gzip flag byte.
108
#define ASCII_FLAG 0x01   // bit 0 set: file probably ascii text
109
0
#define HEAD_CRC 0x02     // bit 1 set: header CRC present
110
0
#define EXTRA_FIELD 0x04  // bit 2 set: extra field present
111
0
#define ORIG_NAME 0x08    // bit 3 set: original file name present
112
0
#define COMMENT 0x10      // bit 4 set: file comment present
113
0
#define RESERVED 0xE0     // bits 5..7: reserved
114
115
0
#define ALLOC(size) malloc(size)
116
#define TRYFREE(p)                                                             \
117
0
    {                                                                          \
118
0
        if (p)                                                                 \
119
0
            free(p);                                                           \
120
0
    }
121
122
#define CPL_VSIL_GZ_RETURN(ret)                                                \
123
0
    CPLError(CE_Failure, CPLE_AppDefined, "In file %s, at line %d, return %d", \
124
0
             __FILE__, __LINE__, ret)
125
126
// To avoid aliasing to CopyFile to CopyFileA on Windows
127
#ifdef CopyFile
128
#undef CopyFile
129
#endif
130
131
// #define ENABLE_DEBUG 1
132
133
/************************************************************************/
134
/* ==================================================================== */
135
/*                       VSIGZipHandle                                  */
136
/* ==================================================================== */
137
/************************************************************************/
138
139
typedef struct
140
{
141
    vsi_l_offset posInBaseHandle;
142
    z_stream stream;
143
    uLong crc;
144
    int transparent;
145
    vsi_l_offset in;
146
    vsi_l_offset out;
147
} GZipSnapshot;
148
149
class VSIGZipHandle final : public VSIVirtualHandle
150
{
151
    VSIVirtualHandle *m_poBaseHandle = nullptr;
152
#ifdef DEBUG
153
    vsi_l_offset m_offset = 0;
154
#endif
155
    vsi_l_offset m_compressed_size = 0;
156
    vsi_l_offset m_uncompressed_size = 0;
157
    vsi_l_offset offsetEndCompressedData = 0;
158
    uLong m_expected_crc = 0;
159
    char *m_pszBaseFileName = nullptr; /* optional */
160
    bool m_bWriteProperties = false;
161
    bool m_bCanSaveInfo = false;
162
163
    /* Fields from gz_stream structure */
164
    z_stream stream;
165
    int z_err = Z_OK;    /* error code for last stream operation */
166
    int z_eof = 0;       /* set if end of input file (but not necessarily of the
167
                         uncompressed stream !) */
168
    bool m_bEOF = false; /* EOF flag for uncompressed stream */
169
    Byte *inbuf = nullptr;  /* input buffer */
170
    Byte *outbuf = nullptr; /* output buffer */
171
    uLong crc = 0;          /* crc32 of uncompressed data */
172
    int m_transparent = 0;  /* 1 if input file is not a .gz file */
173
    vsi_l_offset startOff =
174
        0; /* startOff of compressed data in file (header skipped) */
175
    vsi_l_offset in = 0;  /* bytes into deflate or inflate */
176
    vsi_l_offset out = 0; /* bytes out of deflate or inflate */
177
    vsi_l_offset m_nLastReadOffset = 0;
178
179
    GZipSnapshot *snapshots = nullptr;
180
    vsi_l_offset snapshot_byte_interval =
181
        0; /* number of compressed bytes at which we create a "snapshot" */
182
183
    void check_header();
184
    int get_byte();
185
    bool gzseek(vsi_l_offset nOffset, int nWhence);
186
    int gzrewind();
187
    uLong getLong();
188
189
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipHandle)
190
191
  public:
192
    VSIGZipHandle(VSIVirtualHandle *poBaseHandle, const char *pszBaseFileName,
193
                  vsi_l_offset offset = 0, vsi_l_offset compressed_size = 0,
194
                  vsi_l_offset uncompressed_size = 0, uLong expected_crc = 0,
195
                  int transparent = 0);
196
    ~VSIGZipHandle() override;
197
198
    bool IsInitOK() const
199
0
    {
200
0
        return inbuf != nullptr;
201
0
    }
202
203
    int Seek(vsi_l_offset nOffset, int nWhence) override;
204
    vsi_l_offset Tell() override;
205
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
206
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
207
    void ClearErr() override;
208
    int Eof() override;
209
    int Error() override;
210
    int Flush() override;
211
    int Close() override;
212
213
    VSIGZipHandle *Duplicate();
214
    bool CloseBaseHandle();
215
216
    vsi_l_offset GetLastReadOffset()
217
0
    {
218
0
        return m_nLastReadOffset;
219
0
    }
220
221
    const char *GetBaseFileName()
222
0
    {
223
0
        return m_pszBaseFileName;
224
0
    }
225
226
    void SetUncompressedSize(vsi_l_offset nUncompressedSize)
227
0
    {
228
0
        m_uncompressed_size = nUncompressedSize;
229
0
    }
230
231
    vsi_l_offset GetUncompressedSize()
232
0
    {
233
0
        return m_uncompressed_size;
234
0
    }
235
236
    void SaveInfo_unlocked();
237
238
    void UnsetCanSaveInfo()
239
0
    {
240
0
        m_bCanSaveInfo = false;
241
0
    }
242
};
243
244
#ifdef ENABLE_DEFLATE64
245
246
/************************************************************************/
247
/* ==================================================================== */
248
/*                           VSIDeflate64Handle                         */
249
/* ==================================================================== */
250
/************************************************************************/
251
252
struct VSIDeflate64Snapshot
253
{
254
    vsi_l_offset posInBaseHandle = 0;
255
    z_stream stream{};
256
    uLong crc = 0;
257
    vsi_l_offset in = 0;
258
    vsi_l_offset out = 0;
259
    std::vector<GByte> extraOutput{};
260
    bool m_bStreamEndReached = false;
261
};
262
263
class VSIDeflate64Handle final : public VSIVirtualHandle
264
{
265
    VSIVirtualHandle *m_poBaseHandle = nullptr;
266
#ifdef DEBUG
267
    vsi_l_offset m_offset = 0;
268
#endif
269
    vsi_l_offset m_compressed_size = 0;
270
    vsi_l_offset m_uncompressed_size = 0;
271
    vsi_l_offset offsetEndCompressedData = 0;
272
    uLong m_expected_crc = 0;
273
    char *m_pszBaseFileName = nullptr; /* optional */
274
275
    /* Fields from gz_stream structure */
276
    z_stream stream;
277
    int z_err = Z_OK;    /* error code for last stream operation */
278
    int z_eof = 0;       /* set if end of input file (but not necessarily of the
279
                         uncompressed stream ! ) */
280
    bool m_bEOF = false; /* EOF flag for uncompressed stream */
281
    Byte *inbuf = nullptr;  /* input buffer */
282
    Byte *outbuf = nullptr; /* output buffer */
283
    std::vector<GByte> extraOutput{};
284
    bool m_bStreamEndReached = false;
285
    uLong crc = 0; /* crc32 of uncompressed data */
286
    vsi_l_offset startOff =
287
        0; /* startOff of compressed data in file (header skipped) */
288
    vsi_l_offset in = 0;  /* bytes into deflate or inflate */
289
    vsi_l_offset out = 0; /* bytes out of deflate or inflate */
290
291
    std::vector<VSIDeflate64Snapshot> snapshots{};
292
    vsi_l_offset snapshot_byte_interval =
293
        0; /* number of compressed bytes at which we create a "snapshot" */
294
295
    bool gzseek(vsi_l_offset nOffset, int nWhence);
296
    int gzrewind();
297
298
    CPL_DISALLOW_COPY_ASSIGN(VSIDeflate64Handle)
299
300
  public:
301
    VSIDeflate64Handle(VSIVirtualHandle *poBaseHandle,
302
                       const char *pszBaseFileName, vsi_l_offset offset = 0,
303
                       vsi_l_offset compressed_size = 0,
304
                       vsi_l_offset uncompressed_size = 0,
305
                       uLong expected_crc = 0);
306
    ~VSIDeflate64Handle() override;
307
308
    bool IsInitOK() const
309
0
    {
310
0
        return inbuf != nullptr;
311
0
    }
312
313
    int Seek(vsi_l_offset nOffset, int nWhence) override;
314
    vsi_l_offset Tell() override;
315
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
316
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
317
    void ClearErr() override;
318
    int Eof() override;
319
    int Error() override;
320
    int Flush() override;
321
    int Close() override;
322
323
    VSIDeflate64Handle *Duplicate();
324
    bool CloseBaseHandle();
325
326
    const char *GetBaseFileName()
327
0
    {
328
0
        return m_pszBaseFileName;
329
0
    }
330
331
    void SetUncompressedSize(vsi_l_offset nUncompressedSize)
332
0
    {
333
0
        m_uncompressed_size = nUncompressedSize;
334
0
    }
335
336
    vsi_l_offset GetUncompressedSize()
337
0
    {
338
0
        return m_uncompressed_size;
339
0
    }
340
};
341
#endif
342
343
class VSIGZipFilesystemHandler final : public VSIFilesystemHandler
344
{
345
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipFilesystemHandler)
346
347
    CPLMutex *hMutex = nullptr;
348
    VSIGZipHandle *poHandleLastGZipFile = nullptr;
349
    bool m_bInSaveInfo = false;
350
351
  public:
352
1
    VSIGZipFilesystemHandler() = default;
353
    ~VSIGZipFilesystemHandler() override;
354
355
    VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
356
                           bool bSetError,
357
                           CSLConstList /* papszOptions */) override;
358
    VSIGZipHandle *OpenGZipReadOnly(const char *pszFilename,
359
                                    const char *pszAccess);
360
    int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
361
             int nFlags) override;
362
    char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
363
364
    const char *GetOptions() override;
365
366
    virtual bool SupportsSequentialWrite(const char *pszPath,
367
                                         bool bAllowLocalTempFile) override;
368
369
    virtual bool SupportsRandomWrite(const char * /* pszPath */,
370
                                     bool /* bAllowLocalTempFile */) override
371
0
    {
372
0
        return false;
373
0
    }
374
375
    void SaveInfo(VSIGZipHandle *poHandle);
376
    void SaveInfo_unlocked(VSIGZipHandle *poHandle);
377
};
378
379
/************************************************************************/
380
/*                            Duplicate()                               */
381
/************************************************************************/
382
383
VSIGZipHandle *VSIGZipHandle::Duplicate()
384
0
{
385
0
    CPLAssert(m_offset == 0);
386
0
    CPLAssert(m_compressed_size != 0);
387
0
    CPLAssert(m_pszBaseFileName != nullptr);
388
389
0
    VSIFilesystemHandler *poFSHandler =
390
0
        VSIFileManager::GetHandler(m_pszBaseFileName);
391
392
0
    VSIVirtualHandle *poNewBaseHandle =
393
0
        poFSHandler->Open(m_pszBaseFileName, "rb");
394
395
0
    if (poNewBaseHandle == nullptr)
396
0
        return nullptr;
397
398
0
    VSIGZipHandle *poHandle =
399
0
        new VSIGZipHandle(poNewBaseHandle, m_pszBaseFileName, 0,
400
0
                          m_compressed_size, m_uncompressed_size);
401
0
    if (!(poHandle->IsInitOK()))
402
0
    {
403
0
        delete poHandle;
404
0
        return nullptr;
405
0
    }
406
407
0
    poHandle->m_nLastReadOffset = m_nLastReadOffset;
408
409
    // Most important: duplicate the snapshots!
410
411
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
412
0
         i++)
413
0
    {
414
0
        if (snapshots[i].posInBaseHandle == 0)
415
0
            break;
416
417
0
        poHandle->snapshots[i].posInBaseHandle = snapshots[i].posInBaseHandle;
418
0
        inflateCopy(&poHandle->snapshots[i].stream, &snapshots[i].stream);
419
0
        poHandle->snapshots[i].crc = snapshots[i].crc;
420
0
        poHandle->snapshots[i].transparent = snapshots[i].transparent;
421
0
        poHandle->snapshots[i].in = snapshots[i].in;
422
0
        poHandle->snapshots[i].out = snapshots[i].out;
423
0
    }
424
425
0
    return poHandle;
426
0
}
427
428
/************************************************************************/
429
/*                     CloseBaseHandle()                                */
430
/************************************************************************/
431
432
bool VSIGZipHandle::CloseBaseHandle()
433
0
{
434
0
    bool bRet = true;
435
0
    if (m_poBaseHandle)
436
0
    {
437
0
        bRet = m_poBaseHandle->Close() == 0;
438
0
        delete m_poBaseHandle;
439
0
    }
440
0
    m_poBaseHandle = nullptr;
441
0
    return bRet;
442
0
}
443
444
/************************************************************************/
445
/*                       VSIGZipHandle()                                */
446
/************************************************************************/
447
448
VSIGZipHandle::VSIGZipHandle(VSIVirtualHandle *poBaseHandle,
449
                             const char *pszBaseFileName, vsi_l_offset offset,
450
                             vsi_l_offset compressed_size,
451
                             vsi_l_offset uncompressed_size, uLong expected_crc,
452
                             int transparent)
453
0
    : m_poBaseHandle(poBaseHandle),
454
#ifdef DEBUG
455
0
      m_offset(offset),
456
#endif
457
0
      m_uncompressed_size(uncompressed_size), m_expected_crc(expected_crc),
458
0
      m_pszBaseFileName(pszBaseFileName ? CPLStrdup(pszBaseFileName) : nullptr),
459
0
      m_bWriteProperties(CPLTestBool(
460
0
          CPLGetConfigOption("CPL_VSIL_GZIP_WRITE_PROPERTIES", "YES"))),
461
      m_bCanSaveInfo(
462
0
          CPLTestBool(CPLGetConfigOption("CPL_VSIL_GZIP_SAVE_INFO", "YES"))),
463
0
      stream(), crc(0), m_transparent(transparent)
464
0
{
465
0
    if (compressed_size || transparent)
466
0
    {
467
0
        m_compressed_size = compressed_size;
468
0
    }
469
0
    else
470
0
    {
471
0
        if (poBaseHandle->Seek(0, SEEK_END) != 0)
472
0
            CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
473
0
        m_compressed_size = poBaseHandle->Tell() - offset;
474
0
        compressed_size = m_compressed_size;
475
0
    }
476
0
    offsetEndCompressedData = offset + compressed_size;
477
478
0
    if (poBaseHandle->Seek(offset, SEEK_SET) != 0)
479
0
        CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
480
481
0
    stream.zalloc = nullptr;
482
0
    stream.zfree = nullptr;
483
0
    stream.opaque = nullptr;
484
0
    stream.next_in = inbuf = nullptr;
485
0
    stream.next_out = outbuf = nullptr;
486
0
    stream.avail_in = stream.avail_out = 0;
487
488
0
    inbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
489
0
    stream.next_in = inbuf;
490
491
0
    int err = inflateInit2(&(stream), -MAX_WBITS);
492
    // windowBits is passed < 0 to tell that there is no zlib header.
493
    // Note that in this case inflate *requires* an extra "dummy" byte
494
    // after the compressed stream in order to complete decompression and
495
    // return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
496
    // present after the compressed stream.
497
0
    if (err != Z_OK || inbuf == nullptr)
498
0
    {
499
0
        CPLError(CE_Failure, CPLE_NotSupported, "inflateInit2 init failed");
500
0
        TRYFREE(inbuf);
501
0
        inbuf = nullptr;
502
0
        return;
503
0
    }
504
0
    stream.avail_out = static_cast<uInt>(Z_BUFSIZE);
505
506
0
    if (offset == 0)
507
0
        check_header();  // Skip the .gz header.
508
0
    startOff = poBaseHandle->Tell() - stream.avail_in;
509
510
0
    if (transparent == 0)
511
0
    {
512
0
        snapshot_byte_interval = std::max(static_cast<vsi_l_offset>(Z_BUFSIZE),
513
0
                                          compressed_size / 100);
514
0
        snapshots = static_cast<GZipSnapshot *>(CPLCalloc(
515
0
            sizeof(GZipSnapshot),
516
0
            static_cast<size_t>(compressed_size / snapshot_byte_interval + 1)));
517
0
    }
518
0
}
519
520
/************************************************************************/
521
/*                      SaveInfo_unlocked()                             */
522
/************************************************************************/
523
524
void VSIGZipHandle::SaveInfo_unlocked()
525
0
{
526
0
    if (m_pszBaseFileName && m_bCanSaveInfo)
527
0
    {
528
0
        VSIFilesystemHandler *poFSHandler =
529
0
            VSIFileManager::GetHandler("/vsigzip/");
530
0
        cpl::down_cast<VSIGZipFilesystemHandler *>(poFSHandler)
531
0
            ->SaveInfo_unlocked(this);
532
0
        m_bCanSaveInfo = false;
533
0
    }
534
0
}
535
536
/************************************************************************/
537
/*                      ~VSIGZipHandle()                                */
538
/************************************************************************/
539
540
VSIGZipHandle::~VSIGZipHandle()
541
0
{
542
0
    if (m_pszBaseFileName && m_bCanSaveInfo)
543
0
    {
544
0
        VSIFilesystemHandler *poFSHandler =
545
0
            VSIFileManager::GetHandler("/vsigzip/");
546
0
        cpl::down_cast<VSIGZipFilesystemHandler *>(poFSHandler)->SaveInfo(this);
547
0
    }
548
549
0
    if (stream.state != nullptr)
550
0
    {
551
0
        inflateEnd(&(stream));
552
0
    }
553
554
0
    TRYFREE(inbuf);
555
0
    TRYFREE(outbuf);
556
557
0
    if (snapshots != nullptr)
558
0
    {
559
0
        for (size_t i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
560
0
             i++)
561
0
        {
562
0
            if (snapshots[i].posInBaseHandle)
563
0
            {
564
0
                inflateEnd(&(snapshots[i].stream));
565
0
            }
566
0
        }
567
0
        CPLFree(snapshots);
568
0
    }
569
0
    CPLFree(m_pszBaseFileName);
570
571
0
    CloseBaseHandle();
572
0
}
573
574
/************************************************************************/
575
/*                      check_header()                                  */
576
/************************************************************************/
577
578
void VSIGZipHandle::check_header()
579
0
{
580
    // Assure two bytes in the buffer so we can peek ahead -- handle case
581
    // where first byte of header is at the end of the buffer after the last
582
    // gzip segment.
583
0
    uInt len = stream.avail_in;
584
0
    if (len < 2)
585
0
    {
586
0
        if (len)
587
0
            inbuf[0] = stream.next_in[0];
588
0
        errno = 0;
589
0
        size_t nToRead = static_cast<size_t>(Z_BUFSIZE - len);
590
0
        CPLAssert(m_poBaseHandle->Tell() <= offsetEndCompressedData);
591
0
        if (m_poBaseHandle->Tell() + nToRead > offsetEndCompressedData)
592
0
            nToRead = static_cast<size_t>(offsetEndCompressedData -
593
0
                                          m_poBaseHandle->Tell());
594
595
0
        len = static_cast<uInt>(m_poBaseHandle->Read(inbuf + len, 1, nToRead));
596
#ifdef ENABLE_DEBUG
597
        CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
598
                 m_poBaseHandle->Tell(), offsetEndCompressedData);
599
#endif
600
0
        if (len == 0)  // && ferror(file)
601
0
        {
602
0
            if (m_poBaseHandle->Tell() != offsetEndCompressedData)
603
0
                z_err = Z_ERRNO;
604
0
        }
605
0
        stream.avail_in += len;
606
0
        stream.next_in = inbuf;
607
0
        if (stream.avail_in < 2)
608
0
        {
609
0
            m_transparent = stream.avail_in;
610
0
            return;
611
0
        }
612
0
    }
613
614
    // Peek ahead to check the gzip magic header.
615
0
    if (stream.next_in[0] != gz_magic[0] || stream.next_in[1] != gz_magic[1])
616
0
    {
617
0
        m_transparent = 1;
618
0
        return;
619
0
    }
620
0
    stream.avail_in -= 2;
621
0
    stream.next_in += 2;
622
623
    // Check the rest of the gzip header.
624
0
    const int method = get_byte();
625
0
    const int flags = get_byte();
626
0
    if (method != Z_DEFLATED || (flags & RESERVED) != 0)
627
0
    {
628
0
        z_err = Z_DATA_ERROR;
629
0
        return;
630
0
    }
631
632
    // Discard time, xflags and OS code:
633
0
    for (len = 0; len < 6; len++)
634
0
        CPL_IGNORE_RET_VAL(get_byte());
635
636
0
    if ((flags & EXTRA_FIELD) != 0)
637
0
    {
638
        // Skip the extra field.
639
0
        len = static_cast<uInt>(get_byte()) & 0xFF;
640
0
        len += (static_cast<uInt>(get_byte()) & 0xFF) << 8;
641
        // len is garbage if EOF but the loop below will quit anyway.
642
0
        while (len != 0 && get_byte() != EOF)
643
0
        {
644
0
            --len;
645
0
        }
646
0
    }
647
648
0
    if ((flags & ORIG_NAME) != 0)
649
0
    {
650
        // Skip the original file name.
651
0
        int c;
652
0
        while ((c = get_byte()) != 0 && c != EOF)
653
0
        {
654
0
        }
655
0
    }
656
0
    if ((flags & COMMENT) != 0)
657
0
    {
658
        // skip the .gz file comment.
659
0
        int c;
660
0
        while ((c = get_byte()) != 0 && c != EOF)
661
0
        {
662
0
        }
663
0
    }
664
0
    if ((flags & HEAD_CRC) != 0)
665
0
    {
666
        // Skip the header crc.
667
0
        for (len = 0; len < 2; len++)
668
0
            CPL_IGNORE_RET_VAL(get_byte());
669
0
    }
670
0
    z_err = z_eof ? Z_DATA_ERROR : Z_OK;
671
0
}
672
673
/************************************************************************/
674
/*                            get_byte()                                */
675
/************************************************************************/
676
677
int VSIGZipHandle::get_byte()
678
0
{
679
0
    if (z_eof)
680
0
        return EOF;
681
0
    if (stream.avail_in == 0)
682
0
    {
683
0
        errno = 0;
684
0
        size_t nToRead = static_cast<size_t>(Z_BUFSIZE);
685
0
        CPLAssert(m_poBaseHandle->Tell() <= offsetEndCompressedData);
686
0
        if (m_poBaseHandle->Tell() + nToRead > offsetEndCompressedData)
687
0
            nToRead = static_cast<size_t>(offsetEndCompressedData -
688
0
                                          m_poBaseHandle->Tell());
689
0
        stream.avail_in =
690
0
            static_cast<uInt>(m_poBaseHandle->Read(inbuf, 1, nToRead));
691
#ifdef ENABLE_DEBUG
692
        CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
693
                 m_poBaseHandle->Tell(), offsetEndCompressedData);
694
#endif
695
0
        if (stream.avail_in == 0)
696
0
        {
697
0
            z_eof = 1;
698
0
            if (m_poBaseHandle->Tell() != offsetEndCompressedData)
699
0
                z_err = Z_ERRNO;
700
            // if( ferror(file) ) z_err = Z_ERRNO;
701
0
            return EOF;
702
0
        }
703
0
        stream.next_in = inbuf;
704
0
    }
705
0
    stream.avail_in--;
706
0
    return *(stream.next_in)++;
707
0
}
708
709
/************************************************************************/
710
/*                            gzrewind()                                */
711
/************************************************************************/
712
713
int VSIGZipHandle::gzrewind()
714
0
{
715
0
    z_err = Z_OK;
716
0
    z_eof = 0;
717
0
    m_bEOF = false;
718
0
    stream.avail_in = 0;
719
0
    stream.next_in = inbuf;
720
0
    crc = 0;
721
0
    if (!m_transparent)
722
0
        CPL_IGNORE_RET_VAL(inflateReset(&stream));
723
0
    in = 0;
724
0
    out = 0;
725
0
    return m_poBaseHandle->Seek(startOff, SEEK_SET);
726
0
}
727
728
/************************************************************************/
729
/*                              Seek()                                  */
730
/************************************************************************/
731
732
int VSIGZipHandle::Seek(vsi_l_offset nOffset, int nWhence)
733
0
{
734
0
    m_bEOF = false;
735
736
0
    return gzseek(nOffset, nWhence) ? 0 : -1;
737
0
}
738
739
/************************************************************************/
740
/*                            gzseek()                                  */
741
/************************************************************************/
742
743
bool VSIGZipHandle::gzseek(vsi_l_offset offset, int whence)
744
0
{
745
0
    const vsi_l_offset original_offset = offset;
746
0
    const int original_nWhence = whence;
747
748
0
    z_eof = 0;
749
#ifdef ENABLE_DEBUG
750
    CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
751
#endif
752
753
0
    if (m_transparent)
754
0
    {
755
0
        stream.avail_in = 0;
756
0
        stream.next_in = inbuf;
757
0
        if (whence == SEEK_CUR)
758
0
        {
759
0
            if (out + offset > m_compressed_size)
760
0
            {
761
0
                CPL_VSIL_GZ_RETURN(FALSE);
762
0
                return false;
763
0
            }
764
765
0
            offset = startOff + out + offset;
766
0
        }
767
0
        else if (whence == SEEK_SET)
768
0
        {
769
0
            if (offset > m_compressed_size)
770
0
            {
771
0
                CPL_VSIL_GZ_RETURN(FALSE);
772
0
                return false;
773
0
            }
774
775
0
            offset = startOff + offset;
776
0
        }
777
0
        else if (whence == SEEK_END)
778
0
        {
779
            // Commented test: because vsi_l_offset is unsigned (for the moment)
780
            // so no way to seek backward. See #1590 */
781
0
            if (offset > 0)  // || -offset > compressed_size
782
0
            {
783
0
                CPL_VSIL_GZ_RETURN(FALSE);
784
0
                return false;
785
0
            }
786
787
0
            offset = startOff + m_compressed_size - offset;
788
0
        }
789
0
        else
790
0
        {
791
0
            CPL_VSIL_GZ_RETURN(FALSE);
792
0
            return false;
793
0
        }
794
795
0
        if (m_poBaseHandle->Seek(offset, SEEK_SET) < 0)
796
0
        {
797
0
            CPL_VSIL_GZ_RETURN(FALSE);
798
0
            return false;
799
0
        }
800
801
0
        out = offset - startOff;
802
0
        in = out;
803
0
        return true;
804
0
    }
805
806
    // whence == SEEK_END is unsuppored in original gzseek.
807
0
    if (whence == SEEK_END)
808
0
    {
809
        // If we known the uncompressed size, we can fake a jump to
810
        // the end of the stream.
811
0
        if (offset == 0 && m_uncompressed_size != 0)
812
0
        {
813
0
            out = m_uncompressed_size;
814
0
            return true;
815
0
        }
816
817
        // We don't know the uncompressed size. This is unfortunate.
818
        // Do the slow version.
819
0
        static int firstWarning = 1;
820
0
        if (m_compressed_size > 10 * 1024 * 1024 && firstWarning)
821
0
        {
822
0
            CPLError(CE_Warning, CPLE_AppDefined,
823
0
                     "VSIFSeekL(xxx, SEEK_END) may be really slow "
824
0
                     "on GZip streams.");
825
0
            firstWarning = 0;
826
0
        }
827
828
0
        whence = SEEK_CUR;
829
0
        offset = 1024 * 1024 * 1024;
830
0
        offset *= 1024 * 1024;
831
0
    }
832
833
    // Rest of function is for reading only.
834
835
    // Compute absolute position.
836
0
    if (whence == SEEK_CUR)
837
0
    {
838
0
        offset += out;
839
0
    }
840
841
    // For a negative seek, rewind and use positive seek.
842
0
    if (offset >= out)
843
0
    {
844
0
        offset -= out;
845
0
    }
846
0
    else if (gzrewind() < 0)
847
0
    {
848
0
        CPL_VSIL_GZ_RETURN(FALSE);
849
0
        return false;
850
0
    }
851
852
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
853
0
    {
854
0
        CPL_VSIL_GZ_RETURN(FALSE);
855
0
        return false;
856
0
    }
857
858
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
859
0
         i++)
860
0
    {
861
0
        if (snapshots[i].posInBaseHandle == 0)
862
0
            break;
863
0
        if (snapshots[i].out <= out + offset &&
864
0
            (i == m_compressed_size / snapshot_byte_interval ||
865
0
             snapshots[i + 1].out == 0 || snapshots[i + 1].out > out + offset))
866
0
        {
867
0
            if (out >= snapshots[i].out)
868
0
                break;
869
870
#ifdef ENABLE_DEBUG
871
            CPLDebug("SNAPSHOT",
872
                     "using snapshot %d : "
873
                     "posInBaseHandle(snapshot)=" CPL_FRMT_GUIB
874
                     " in(snapshot)=" CPL_FRMT_GUIB
875
                     " out(snapshot)=" CPL_FRMT_GUIB " out=" CPL_FRMT_GUIB
876
                     " offset=" CPL_FRMT_GUIB,
877
                     i, snapshots[i].posInBaseHandle, snapshots[i].in,
878
                     snapshots[i].out, out, offset);
879
#endif
880
0
            offset = out + offset - snapshots[i].out;
881
0
            if (m_poBaseHandle->Seek(snapshots[i].posInBaseHandle, SEEK_SET) !=
882
0
                0)
883
0
                CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
884
885
0
            inflateEnd(&stream);
886
0
            inflateCopy(&stream, &snapshots[i].stream);
887
0
            crc = snapshots[i].crc;
888
0
            m_transparent = snapshots[i].transparent;
889
0
            in = snapshots[i].in;
890
0
            out = snapshots[i].out;
891
0
            break;
892
0
        }
893
0
    }
894
895
    // Offset is now the number of bytes to skip.
896
897
0
    if (offset != 0 && outbuf == nullptr)
898
0
    {
899
0
        outbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
900
0
        if (outbuf == nullptr)
901
0
        {
902
0
            CPL_VSIL_GZ_RETURN(FALSE);
903
0
            return false;
904
0
        }
905
0
    }
906
907
0
    if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
908
0
    {
909
0
        return true;
910
0
    }
911
912
0
    while (offset > 0)
913
0
    {
914
0
        int size = Z_BUFSIZE;
915
0
        if (offset < static_cast<vsi_l_offset>(Z_BUFSIZE))
916
0
            size = static_cast<int>(offset);
917
918
0
        int read_size =
919
0
            static_cast<int>(Read(outbuf, 1, static_cast<uInt>(size)));
920
0
        if (original_nWhence == SEEK_END)
921
0
        {
922
0
            if (size != read_size)
923
0
            {
924
0
                z_err = Z_STREAM_END;
925
0
                break;
926
0
            }
927
0
        }
928
0
        else if (read_size == 0)
929
0
        {
930
            // CPL_VSIL_GZ_RETURN(FALSE);
931
0
            return false;
932
0
        }
933
0
        offset -= read_size;
934
0
    }
935
#ifdef ENABLE_DEBUG
936
    CPLDebug("GZIP", "gzseek at offset " CPL_FRMT_GUIB, out);
937
#endif
938
939
0
    if (original_offset == 0 && original_nWhence == SEEK_END)
940
0
    {
941
0
        m_uncompressed_size = out;
942
943
0
        if (m_pszBaseFileName && !STARTS_WITH(m_pszBaseFileName, "/vsicurl/") &&
944
0
            !STARTS_WITH(m_pszBaseFileName, "/vsitar/") &&
945
0
            !STARTS_WITH(m_pszBaseFileName, "/vsizip/") && m_bWriteProperties)
946
0
        {
947
0
            CPLErrorStateBackuper oErrorStateBackuper(CPLQuietErrorHandler);
948
949
0
            CPLString osCacheFilename(m_pszBaseFileName);
950
0
            osCacheFilename += ".properties";
951
952
            // Write a .properties file to avoid seeking next time.
953
0
            VSILFILE *fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "wb");
954
0
            if (fpCacheLength)
955
0
            {
956
0
                char szBuffer[32] = {};
957
958
0
                CPLPrintUIntBig(szBuffer, m_compressed_size, 31);
959
0
                char *pszFirstNonSpace = szBuffer;
960
0
                while (*pszFirstNonSpace == ' ')
961
0
                    pszFirstNonSpace++;
962
0
                CPL_IGNORE_RET_VAL(VSIFPrintfL(
963
0
                    fpCacheLength, "compressed_size=%s\n", pszFirstNonSpace));
964
965
0
                CPLPrintUIntBig(szBuffer, m_uncompressed_size, 31);
966
0
                pszFirstNonSpace = szBuffer;
967
0
                while (*pszFirstNonSpace == ' ')
968
0
                    pszFirstNonSpace++;
969
0
                CPL_IGNORE_RET_VAL(VSIFPrintfL(
970
0
                    fpCacheLength, "uncompressed_size=%s\n", pszFirstNonSpace));
971
972
0
                CPL_IGNORE_RET_VAL(VSIFCloseL(fpCacheLength));
973
0
            }
974
0
        }
975
0
    }
976
977
0
    return true;
978
0
}
979
980
/************************************************************************/
981
/*                              Tell()                                  */
982
/************************************************************************/
983
984
vsi_l_offset VSIGZipHandle::Tell()
985
0
{
986
#ifdef ENABLE_DEBUG
987
    CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
988
#endif
989
0
    return out;
990
0
}
991
992
/************************************************************************/
993
/*                              Read()                                  */
994
/************************************************************************/
995
996
size_t VSIGZipHandle::Read(void *const buf, size_t const nSize,
997
                           size_t const nMemb)
998
0
{
999
#ifdef ENABLE_DEBUG
1000
    CPLDebug("GZIP", "Read(%p, %d, %d)", buf, static_cast<int>(nSize),
1001
             static_cast<int>(nMemb));
1002
#endif
1003
1004
0
    if (m_bEOF || z_err != Z_OK)
1005
0
    {
1006
0
        if (z_err == Z_STREAM_END && nSize > 0 && nMemb > 0)
1007
0
            m_bEOF = true;
1008
0
        return 0;
1009
0
    }
1010
1011
0
    if (nSize > 0 && nMemb > UINT32_MAX / nSize)
1012
0
    {
1013
0
        CPLError(CE_Failure, CPLE_FileIO, "Too many bytes to read at once");
1014
0
        return 0;
1015
0
    }
1016
1017
0
    const unsigned len =
1018
0
        static_cast<unsigned int>(nSize) * static_cast<unsigned int>(nMemb);
1019
0
    Bytef *pStart =
1020
0
        static_cast<Bytef *>(buf);  // Start off point for crc computation.
1021
    // == stream.next_out but not forced far (for MSDOS).
1022
0
    Byte *next_out = static_cast<Byte *>(buf);
1023
0
    stream.next_out = static_cast<Bytef *>(buf);
1024
0
    stream.avail_out = len;
1025
1026
0
    while (stream.avail_out != 0)
1027
0
    {
1028
0
        if (m_transparent)
1029
0
        {
1030
            // Copy first the lookahead bytes:
1031
0
            uInt nRead = 0;
1032
0
            uInt n = stream.avail_in;
1033
0
            if (n > stream.avail_out)
1034
0
                n = stream.avail_out;
1035
0
            if (n > 0)
1036
0
            {
1037
0
                memcpy(stream.next_out, stream.next_in, n);
1038
0
                next_out += n;
1039
0
                stream.next_out = next_out;
1040
0
                stream.next_in += n;
1041
0
                stream.avail_out -= n;
1042
0
                stream.avail_in -= n;
1043
0
                nRead += n;
1044
0
            }
1045
0
            if (stream.avail_out > 0)
1046
0
            {
1047
0
                const uInt nToRead = static_cast<uInt>(
1048
0
                    std::min(m_compressed_size - (in + nRead),
1049
0
                             static_cast<vsi_l_offset>(stream.avail_out)));
1050
0
                const uInt nReadFromFile = static_cast<uInt>(
1051
0
                    m_poBaseHandle->Read(next_out, 1, nToRead));
1052
0
                if (nReadFromFile < nToRead && m_poBaseHandle->Error())
1053
0
                    z_err = Z_ERRNO;
1054
0
                stream.avail_out -= nReadFromFile;
1055
0
                nRead += nReadFromFile;
1056
0
            }
1057
0
            in += nRead;
1058
0
            out += nRead;
1059
0
            if (nRead < len)
1060
0
            {
1061
0
                m_bEOF = true;
1062
0
                z_eof = 1;
1063
0
            }
1064
#ifdef ENABLE_DEBUG
1065
            CPLDebug("GZIP", "Read return %d", static_cast<int>(nRead / nSize));
1066
#endif
1067
0
            return static_cast<int>(nRead) / nSize;
1068
0
        }
1069
0
        if (stream.avail_in == 0 && !z_eof)
1070
0
        {
1071
0
            vsi_l_offset posInBaseHandle = m_poBaseHandle->Tell();
1072
0
            if (posInBaseHandle - startOff > m_compressed_size)
1073
0
            {
1074
                // If we reach here, file size has changed (because at
1075
                // construction time startOff + m_compressed_size marked the
1076
                // end of file).
1077
                // We should probably have a better fix than that, by detecting
1078
                // at open time that the saved snapshot is not valid and
1079
                // discarding it.
1080
0
                CPLError(CE_Failure, CPLE_AppDefined,
1081
0
                         "File size of underlying /vsigzip/ file has changed");
1082
0
                z_err = Z_ERRNO;
1083
0
                CPL_VSIL_GZ_RETURN(0);
1084
0
                return 0;
1085
0
            }
1086
0
            GZipSnapshot *snapshot = &snapshots[(posInBaseHandle - startOff) /
1087
0
                                                snapshot_byte_interval];
1088
0
            if (snapshot->posInBaseHandle == 0)
1089
0
            {
1090
0
                snapshot->crc = crc32(
1091
0
                    crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1092
#ifdef ENABLE_DEBUG
1093
                CPLDebug("SNAPSHOT",
1094
                         "creating snapshot %d : "
1095
                         "posInBaseHandle=" CPL_FRMT_GUIB " in=" CPL_FRMT_GUIB
1096
                         " out=" CPL_FRMT_GUIB " crc=%X",
1097
                         static_cast<int>((posInBaseHandle - startOff) /
1098
                                          snapshot_byte_interval),
1099
                         posInBaseHandle, in, out,
1100
                         static_cast<unsigned int>(snapshot->crc));
1101
#endif
1102
0
                snapshot->posInBaseHandle = posInBaseHandle;
1103
0
                inflateCopy(&snapshot->stream, &stream);
1104
0
                snapshot->transparent = m_transparent;
1105
0
                snapshot->in = in;
1106
0
                snapshot->out = out;
1107
1108
0
                if (out > m_nLastReadOffset)
1109
0
                    m_nLastReadOffset = out;
1110
0
            }
1111
1112
0
            errno = 0;
1113
0
            stream.avail_in =
1114
0
                static_cast<uInt>(m_poBaseHandle->Read(inbuf, 1, Z_BUFSIZE));
1115
#ifdef ENABLE_DEBUG
1116
            CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
1117
                     m_poBaseHandle->Tell(), offsetEndCompressedData);
1118
#endif
1119
0
            if (m_poBaseHandle->Tell() > offsetEndCompressedData)
1120
0
            {
1121
#ifdef ENABLE_DEBUG
1122
                CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
1123
#endif
1124
0
                stream.avail_in = stream.avail_in -
1125
0
                                  static_cast<uInt>(m_poBaseHandle->Tell() -
1126
0
                                                    offsetEndCompressedData);
1127
0
                if (m_poBaseHandle->Seek(offsetEndCompressedData, SEEK_SET) !=
1128
0
                    0)
1129
0
                    CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1130
#ifdef ENABLE_DEBUG
1131
                CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
1132
#endif
1133
0
            }
1134
0
            if (stream.avail_in == 0)
1135
0
            {
1136
0
                z_eof = 1;
1137
0
                if (m_poBaseHandle->Error() ||
1138
0
                    m_poBaseHandle->Tell() != offsetEndCompressedData)
1139
0
                {
1140
0
                    z_err = Z_ERRNO;
1141
0
                    break;
1142
0
                }
1143
0
            }
1144
0
            stream.next_in = inbuf;
1145
0
        }
1146
0
        in += stream.avail_in;
1147
0
        out += stream.avail_out;
1148
0
        z_err = inflate(&(stream), Z_NO_FLUSH);
1149
0
        in -= stream.avail_in;
1150
0
        out -= stream.avail_out;
1151
1152
0
        if (z_err == Z_STREAM_END && m_compressed_size != 2)
1153
0
        {
1154
            // Check CRC and original size.
1155
0
            crc =
1156
0
                crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1157
0
            pStart = stream.next_out;
1158
0
            if (m_expected_crc)
1159
0
            {
1160
#ifdef ENABLE_DEBUG
1161
                CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X",
1162
                         static_cast<unsigned int>(crc),
1163
                         static_cast<unsigned int>(m_expected_crc));
1164
#endif
1165
0
            }
1166
0
            if (m_expected_crc != 0 && m_expected_crc != crc)
1167
0
            {
1168
0
                CPLError(CE_Failure, CPLE_FileIO,
1169
0
                         "CRC error. Got %X instead of %X",
1170
0
                         static_cast<unsigned int>(crc),
1171
0
                         static_cast<unsigned int>(m_expected_crc));
1172
0
                z_err = Z_DATA_ERROR;
1173
0
            }
1174
0
            else if (m_expected_crc == 0)
1175
0
            {
1176
0
                const uLong read_crc = static_cast<unsigned long>(getLong());
1177
0
                if (read_crc != crc)
1178
0
                {
1179
0
                    CPLError(CE_Failure, CPLE_FileIO,
1180
0
                             "CRC error. Got %X instead of %X",
1181
0
                             static_cast<unsigned int>(crc),
1182
0
                             static_cast<unsigned int>(read_crc));
1183
0
                    z_err = Z_DATA_ERROR;
1184
0
                }
1185
0
                else
1186
0
                {
1187
0
                    CPL_IGNORE_RET_VAL(getLong());
1188
                    // The uncompressed length returned by above getlong() may
1189
                    // be different from out in case of concatenated .gz files.
1190
                    // Check for such files:
1191
0
                    check_header();
1192
0
                    if (z_err == Z_OK)
1193
0
                    {
1194
0
                        inflateReset(&(stream));
1195
0
                        crc = 0;
1196
0
                    }
1197
0
                }
1198
0
            }
1199
0
        }
1200
0
        if (z_err != Z_OK || z_eof)
1201
0
            break;
1202
0
    }
1203
0
    crc = crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1204
1205
0
    size_t ret = (len - stream.avail_out) / nSize;
1206
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
1207
0
    {
1208
0
        CPLError(CE_Failure, CPLE_AppDefined,
1209
0
                 "In file %s, at line %d, decompression failed with "
1210
0
                 "z_err = %d, return = %d",
1211
0
                 __FILE__, __LINE__, z_err, static_cast<int>(ret));
1212
0
    }
1213
0
    else if (ret < nMemb)
1214
0
    {
1215
0
        m_bEOF = true;
1216
0
    }
1217
1218
#ifdef ENABLE_DEBUG
1219
    CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
1220
             static_cast<int>(ret), z_err, z_eof);
1221
#endif
1222
0
    return ret;
1223
0
}
1224
1225
/************************************************************************/
1226
/*                              getLong()                               */
1227
/************************************************************************/
1228
1229
uLong VSIGZipHandle::getLong()
1230
0
{
1231
0
    uLong x = static_cast<uLong>(get_byte()) & 0xFF;
1232
1233
0
    x += (static_cast<uLong>(get_byte()) & 0xFF) << 8;
1234
0
    x += (static_cast<uLong>(get_byte()) & 0xFF) << 16;
1235
0
    const int c = get_byte();
1236
0
    if (c == EOF)
1237
0
    {
1238
0
        z_err = Z_DATA_ERROR;
1239
0
        return 0;
1240
0
    }
1241
0
    x += static_cast<uLong>(c) << 24;
1242
    // coverity[overflow_sink]
1243
0
    return x;
1244
0
}
1245
1246
/************************************************************************/
1247
/*                              Write()                                 */
1248
/************************************************************************/
1249
1250
size_t VSIGZipHandle::Write(const void * /* pBuffer */, size_t /* nSize */,
1251
                            size_t /* nMemb */)
1252
0
{
1253
0
    CPLError(CE_Failure, CPLE_NotSupported,
1254
0
             "VSIFWriteL is not supported on GZip streams");
1255
0
    return 0;
1256
0
}
1257
1258
/************************************************************************/
1259
/*                               Eof()                                  */
1260
/************************************************************************/
1261
1262
int VSIGZipHandle::Eof()
1263
0
{
1264
#ifdef ENABLE_DEBUG
1265
    CPLDebug("GZIP", "Eof()");
1266
#endif
1267
0
    return m_bEOF;
1268
0
}
1269
1270
/************************************************************************/
1271
/*                             Error()                                  */
1272
/************************************************************************/
1273
1274
int VSIGZipHandle::Error()
1275
0
{
1276
#ifdef ENABLE_DEBUG
1277
    CPLDebug("GZIP", "Error()");
1278
#endif
1279
0
    return z_err != Z_OK && z_err != Z_STREAM_END;
1280
0
}
1281
1282
/************************************************************************/
1283
/*                             ClearErr()                               */
1284
/************************************************************************/
1285
1286
void VSIGZipHandle::ClearErr()
1287
0
{
1288
0
    m_poBaseHandle->ClearErr();
1289
0
    z_eof = 0;
1290
0
    m_bEOF = false;
1291
0
    z_err = Z_OK;
1292
0
}
1293
1294
/************************************************************************/
1295
/*                              Flush()                                 */
1296
/************************************************************************/
1297
1298
int VSIGZipHandle::Flush()
1299
0
{
1300
0
    return 0;
1301
0
}
1302
1303
/************************************************************************/
1304
/*                              Close()                                 */
1305
/************************************************************************/
1306
1307
int VSIGZipHandle::Close()
1308
0
{
1309
0
    return 0;
1310
0
}
1311
1312
#ifdef ENABLE_DEFLATE64
1313
1314
/************************************************************************/
1315
/*                            Duplicate()                               */
1316
/************************************************************************/
1317
1318
VSIDeflate64Handle *VSIDeflate64Handle::Duplicate()
1319
0
{
1320
0
    CPLAssert(m_offset == 0);
1321
0
    CPLAssert(m_compressed_size != 0);
1322
0
    CPLAssert(m_pszBaseFileName != nullptr);
1323
1324
0
    VSIFilesystemHandler *poFSHandler =
1325
0
        VSIFileManager::GetHandler(m_pszBaseFileName);
1326
1327
0
    VSIVirtualHandle *poNewBaseHandle =
1328
0
        poFSHandler->Open(m_pszBaseFileName, "rb");
1329
1330
0
    if (poNewBaseHandle == nullptr)
1331
0
        return nullptr;
1332
1333
0
    VSIDeflate64Handle *poHandle =
1334
0
        new VSIDeflate64Handle(poNewBaseHandle, m_pszBaseFileName, 0,
1335
0
                               m_compressed_size, m_uncompressed_size);
1336
0
    if (!(poHandle->IsInitOK()))
1337
0
    {
1338
0
        delete poHandle;
1339
0
        return nullptr;
1340
0
    }
1341
1342
    // Most important: duplicate the snapshots!
1343
1344
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
1345
0
         i++)
1346
0
    {
1347
0
        if (snapshots[i].posInBaseHandle == 0)
1348
0
            break;
1349
1350
0
        poHandle->snapshots[i].posInBaseHandle = snapshots[i].posInBaseHandle;
1351
0
        if (inflateBack9Copy(&poHandle->snapshots[i].stream,
1352
0
                             &snapshots[i].stream) != Z_OK)
1353
0
            CPLError(CE_Failure, CPLE_AppDefined, "inflateBack9Copy() failed");
1354
0
        poHandle->snapshots[i].crc = snapshots[i].crc;
1355
0
        poHandle->snapshots[i].in = snapshots[i].in;
1356
0
        poHandle->snapshots[i].out = snapshots[i].out;
1357
0
        poHandle->snapshots[i].extraOutput = snapshots[i].extraOutput;
1358
0
        poHandle->snapshots[i].m_bStreamEndReached =
1359
0
            snapshots[i].m_bStreamEndReached;
1360
0
    }
1361
1362
0
    return poHandle;
1363
0
}
1364
1365
/************************************************************************/
1366
/*                     CloseBaseHandle()                                */
1367
/************************************************************************/
1368
1369
bool VSIDeflate64Handle::CloseBaseHandle()
1370
0
{
1371
0
    bool bRet = true;
1372
0
    if (m_poBaseHandle)
1373
0
    {
1374
0
        bRet = m_poBaseHandle->Close() == 0;
1375
0
        delete m_poBaseHandle;
1376
0
    }
1377
0
    m_poBaseHandle = nullptr;
1378
0
    return bRet;
1379
0
}
1380
1381
/************************************************************************/
1382
/*                       VSIDeflate64Handle()                                */
1383
/************************************************************************/
1384
1385
VSIDeflate64Handle::VSIDeflate64Handle(VSIVirtualHandle *poBaseHandle,
1386
                                       const char *pszBaseFileName,
1387
                                       vsi_l_offset offset,
1388
                                       vsi_l_offset compressed_size,
1389
                                       vsi_l_offset uncompressed_size,
1390
                                       uLong expected_crc)
1391
0
    : m_poBaseHandle(poBaseHandle),
1392
#ifdef DEBUG
1393
0
      m_offset(offset),
1394
#endif
1395
0
      m_uncompressed_size(uncompressed_size), m_expected_crc(expected_crc),
1396
0
      m_pszBaseFileName(pszBaseFileName ? CPLStrdup(pszBaseFileName) : nullptr),
1397
0
      stream(), crc(0)
1398
0
{
1399
0
    if (compressed_size)
1400
0
    {
1401
0
        m_compressed_size = compressed_size;
1402
0
    }
1403
0
    else
1404
0
    {
1405
0
        if (poBaseHandle->Seek(0, SEEK_END) != 0)
1406
0
            CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1407
0
        m_compressed_size = poBaseHandle->Tell() - offset;
1408
0
        compressed_size = m_compressed_size;
1409
0
    }
1410
0
    offsetEndCompressedData = offset + compressed_size;
1411
1412
0
    if (poBaseHandle->Seek(offset, SEEK_SET) != 0)
1413
0
        CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1414
1415
0
    stream.zalloc = nullptr;
1416
0
    stream.zfree = nullptr;
1417
0
    stream.opaque = nullptr;
1418
0
    stream.next_in = inbuf = nullptr;
1419
0
    stream.next_out = outbuf = nullptr;
1420
0
    stream.avail_in = stream.avail_out = 0;
1421
1422
0
    inbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
1423
0
    stream.next_in = inbuf;
1424
1425
0
    int err = inflateBack9Init(&(stream), nullptr);
1426
    // Note that in this case inflate *requires* an extra "dummy" byte
1427
    // after the compressed stream in order to complete decompression and
1428
    // return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
1429
    // present after the compressed stream.
1430
0
    if (err != Z_OK || inbuf == nullptr)
1431
0
    {
1432
0
        CPLError(CE_Failure, CPLE_NotSupported, "inflateBack9Init init failed");
1433
0
        TRYFREE(inbuf);
1434
0
        inbuf = nullptr;
1435
0
        return;
1436
0
    }
1437
0
    startOff = poBaseHandle->Tell() - stream.avail_in;
1438
1439
0
    snapshot_byte_interval =
1440
0
        std::max(static_cast<vsi_l_offset>(Z_BUFSIZE), compressed_size / 100);
1441
0
    snapshots.resize(
1442
0
        static_cast<size_t>(compressed_size / snapshot_byte_interval + 1));
1443
0
}
1444
1445
/************************************************************************/
1446
/*                      ~VSIDeflate64Handle()                       */
1447
/************************************************************************/
1448
1449
VSIDeflate64Handle::~VSIDeflate64Handle()
1450
0
{
1451
0
    if (stream.state != nullptr)
1452
0
    {
1453
0
        inflateBack9End(&(stream));
1454
0
    }
1455
1456
0
    TRYFREE(inbuf);
1457
0
    TRYFREE(outbuf);
1458
1459
0
    for (auto &snapshot : snapshots)
1460
0
    {
1461
0
        if (snapshot.posInBaseHandle)
1462
0
        {
1463
0
            inflateBack9End(&(snapshot.stream));
1464
0
        }
1465
0
    }
1466
0
    CPLFree(m_pszBaseFileName);
1467
1468
0
    CloseBaseHandle();
1469
0
}
1470
1471
/************************************************************************/
1472
/*                            gzrewind()                                */
1473
/************************************************************************/
1474
1475
int VSIDeflate64Handle::gzrewind()
1476
0
{
1477
0
    m_bStreamEndReached = false;
1478
0
    extraOutput.clear();
1479
0
    z_err = Z_OK;
1480
0
    z_eof = 0;
1481
0
    stream.avail_in = 0;
1482
0
    stream.next_in = inbuf;
1483
0
    crc = 0;
1484
0
    CPL_IGNORE_RET_VAL(inflateBack9End(&stream));
1485
0
    CPL_IGNORE_RET_VAL(inflateBack9Init(&stream, nullptr));
1486
0
    in = 0;
1487
0
    out = 0;
1488
0
    return m_poBaseHandle->Seek(startOff, SEEK_SET);
1489
0
}
1490
1491
/************************************************************************/
1492
/*                              Seek()                                  */
1493
/************************************************************************/
1494
1495
int VSIDeflate64Handle::Seek(vsi_l_offset nOffset, int nWhence)
1496
0
{
1497
0
    m_bEOF = false;
1498
0
    return gzseek(nOffset, nWhence) ? 0 : -1;
1499
0
}
1500
1501
/************************************************************************/
1502
/*                            gzseek()                                  */
1503
/************************************************************************/
1504
1505
bool VSIDeflate64Handle::gzseek(vsi_l_offset offset, int whence)
1506
0
{
1507
0
    const vsi_l_offset original_offset = offset;
1508
0
    const int original_nWhence = whence;
1509
1510
0
    z_eof = 0;
1511
#ifdef ENABLE_DEBUG
1512
    CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
1513
#endif
1514
1515
    // whence == SEEK_END is unsuppored in original gzseek.
1516
0
    if (whence == SEEK_END)
1517
0
    {
1518
        // If we known the uncompressed size, we can fake a jump to
1519
        // the end of the stream.
1520
0
        if (offset == 0 && m_uncompressed_size != 0)
1521
0
        {
1522
0
            out = m_uncompressed_size;
1523
0
            return true;
1524
0
        }
1525
1526
        // We don't know the uncompressed size. This is unfortunate.
1527
        // Do the slow version.
1528
0
        static int firstWarning = 1;
1529
0
        if (m_compressed_size > 10 * 1024 * 1024 && firstWarning)
1530
0
        {
1531
0
            CPLError(CE_Warning, CPLE_AppDefined,
1532
0
                     "VSIFSeekL(xxx, SEEK_END) may be really slow "
1533
0
                     "on GZip streams.");
1534
0
            firstWarning = 0;
1535
0
        }
1536
1537
0
        whence = SEEK_CUR;
1538
0
        offset = 1024 * 1024 * 1024;
1539
0
        offset *= 1024 * 1024;
1540
0
    }
1541
1542
    // Rest of function is for reading only.
1543
1544
    // Compute absolute position.
1545
0
    if (whence == SEEK_CUR)
1546
0
    {
1547
0
        offset += out;
1548
0
    }
1549
1550
    // For a negative seek, rewind and use positive seek.
1551
0
    if (offset >= out)
1552
0
    {
1553
0
        offset -= out;
1554
0
    }
1555
0
    else if (gzrewind() < 0)
1556
0
    {
1557
0
        CPL_VSIL_GZ_RETURN(FALSE);
1558
0
        return false;
1559
0
    }
1560
1561
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
1562
0
    {
1563
0
        CPL_VSIL_GZ_RETURN(FALSE);
1564
0
        return false;
1565
0
    }
1566
1567
0
    for (unsigned int i = 0; i < m_compressed_size / snapshot_byte_interval + 1;
1568
0
         i++)
1569
0
    {
1570
0
        if (snapshots[i].posInBaseHandle == 0)
1571
0
            break;
1572
0
        if (snapshots[i].out <= out + offset &&
1573
0
            (i == m_compressed_size / snapshot_byte_interval ||
1574
0
             snapshots[i + 1].out == 0 || snapshots[i + 1].out > out + offset))
1575
0
        {
1576
0
            if (out >= snapshots[i].out)
1577
0
                break;
1578
1579
#ifdef ENABLE_DEBUG
1580
            CPLDebug("SNAPSHOT",
1581
                     "using snapshot %d : "
1582
                     "posInBaseHandle(snapshot)=" CPL_FRMT_GUIB
1583
                     " in(snapshot)=" CPL_FRMT_GUIB
1584
                     " out(snapshot)=" CPL_FRMT_GUIB " out=" CPL_FRMT_GUIB
1585
                     " offset=" CPL_FRMT_GUIB,
1586
                     i, snapshots[i].posInBaseHandle, snapshots[i].in,
1587
                     snapshots[i].out, out, offset);
1588
#endif
1589
0
            offset = out + offset - snapshots[i].out;
1590
0
            if (m_poBaseHandle->Seek(snapshots[i].posInBaseHandle, SEEK_SET) !=
1591
0
                0)
1592
0
                CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1593
1594
0
            inflateBack9End(&stream);
1595
0
            if (inflateBack9Copy(&stream, &snapshots[i].stream) != Z_OK)
1596
0
                CPLError(CE_Failure, CPLE_AppDefined,
1597
0
                         "inflateBack9Copy() failed");
1598
0
            crc = snapshots[i].crc;
1599
0
            in = snapshots[i].in;
1600
0
            out = snapshots[i].out;
1601
0
            extraOutput = snapshots[i].extraOutput;
1602
0
            m_bStreamEndReached = snapshots[i].m_bStreamEndReached;
1603
0
            break;
1604
0
        }
1605
0
    }
1606
1607
    // Offset is now the number of bytes to skip.
1608
1609
0
    if (offset != 0 && outbuf == nullptr)
1610
0
    {
1611
0
        outbuf = static_cast<Byte *>(ALLOC(Z_BUFSIZE));
1612
0
        if (outbuf == nullptr)
1613
0
        {
1614
0
            CPL_VSIL_GZ_RETURN(FALSE);
1615
0
            return false;
1616
0
        }
1617
0
    }
1618
1619
0
    if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
1620
0
    {
1621
0
        return true;
1622
0
    }
1623
1624
0
    while (offset > 0)
1625
0
    {
1626
0
        int size = Z_BUFSIZE;
1627
0
        if (offset < static_cast<vsi_l_offset>(Z_BUFSIZE))
1628
0
            size = static_cast<int>(offset);
1629
1630
0
        int read_size =
1631
0
            static_cast<int>(Read(outbuf, 1, static_cast<uInt>(size)));
1632
0
        if (original_nWhence == SEEK_END)
1633
0
        {
1634
0
            if (size != read_size)
1635
0
            {
1636
0
                z_err = Z_STREAM_END;
1637
0
                break;
1638
0
            }
1639
0
        }
1640
0
        else if (read_size == 0)
1641
0
        {
1642
            // CPL_VSIL_GZ_RETURN(FALSE);
1643
0
            return false;
1644
0
        }
1645
0
        offset -= read_size;
1646
0
    }
1647
#ifdef ENABLE_DEBUG
1648
    CPLDebug("GZIP", "gzseek at offset " CPL_FRMT_GUIB, out);
1649
#endif
1650
1651
0
    if (original_offset == 0 && original_nWhence == SEEK_END)
1652
0
    {
1653
0
        m_uncompressed_size = out;
1654
0
    }
1655
1656
0
    return true;
1657
0
}
1658
1659
/************************************************************************/
1660
/*                              Tell()                                  */
1661
/************************************************************************/
1662
1663
vsi_l_offset VSIDeflate64Handle::Tell()
1664
0
{
1665
#ifdef ENABLE_DEBUG
1666
    CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
1667
#endif
1668
0
    return out;
1669
0
}
1670
1671
/************************************************************************/
1672
/*                              Read()                                  */
1673
/************************************************************************/
1674
1675
size_t VSIDeflate64Handle::Read(void *const buf, size_t const nSize,
1676
                                size_t const nMemb)
1677
0
{
1678
#ifdef ENABLE_DEBUG
1679
    CPLDebug("GZIP", "Read(%p, %d, %d)", buf, static_cast<int>(nSize),
1680
             static_cast<int>(nMemb));
1681
#endif
1682
1683
0
    if (m_bEOF || z_err != Z_OK)
1684
0
    {
1685
0
        if (z_err == Z_STREAM_END && nSize > 0 && nMemb > 0)
1686
0
            m_bEOF = true;
1687
0
        return 0;
1688
0
    }
1689
1690
0
    if (nSize > 0 && nMemb > UINT32_MAX / nSize)
1691
0
    {
1692
0
        CPLError(CE_Failure, CPLE_FileIO, "Too many bytes to read at once");
1693
0
        return 0;
1694
0
    }
1695
1696
0
    const unsigned len =
1697
0
        static_cast<unsigned int>(nSize) * static_cast<unsigned int>(nMemb);
1698
0
    Bytef *pStart =
1699
0
        static_cast<Bytef *>(buf);  // Start off point for crc computation.
1700
    // == stream.next_out but not forced far (for MSDOS).
1701
0
    stream.next_out = static_cast<Bytef *>(buf);
1702
0
    stream.avail_out = len;
1703
1704
0
    while (stream.avail_out != 0)
1705
0
    {
1706
0
        if (!extraOutput.empty())
1707
0
        {
1708
0
            if (extraOutput.size() >= stream.avail_out)
1709
0
            {
1710
0
                memcpy(stream.next_out, extraOutput.data(), stream.avail_out);
1711
0
                extraOutput.erase(extraOutput.begin(),
1712
0
                                  extraOutput.begin() + stream.avail_out);
1713
0
                out += stream.avail_out;
1714
0
                stream.next_out += stream.avail_out;
1715
0
                stream.avail_out = 0;
1716
0
            }
1717
0
            else
1718
0
            {
1719
0
                memcpy(stream.next_out, extraOutput.data(), extraOutput.size());
1720
0
                stream.next_out += extraOutput.size();
1721
0
                out += static_cast<uInt>(extraOutput.size());
1722
0
                stream.avail_out -= static_cast<uInt>(extraOutput.size());
1723
0
                CPLAssert(stream.avail_out > 0);
1724
0
                extraOutput.clear();
1725
0
            }
1726
0
            z_err = Z_OK;
1727
0
        }
1728
1729
0
        if (stream.avail_in == 0 && !z_eof)
1730
0
        {
1731
0
            vsi_l_offset posInBaseHandle = m_poBaseHandle->Tell();
1732
0
            if (posInBaseHandle - startOff > m_compressed_size)
1733
0
            {
1734
                // If we reach here, file size has changed (because at
1735
                // construction time startOff + m_compressed_size marked the
1736
                // end of file).
1737
                // We should probably have a better fix than that, by detecting
1738
                // at open time that the saved snapshot is not valid and
1739
                // discarding it.
1740
0
                CPLError(CE_Failure, CPLE_AppDefined,
1741
0
                         "File size of underlying /vsigzip/ file has changed");
1742
0
                z_err = Z_ERRNO;
1743
0
                CPL_VSIL_GZ_RETURN(0);
1744
0
                return 0;
1745
0
            }
1746
0
            auto snapshot = &snapshots[static_cast<size_t>(
1747
0
                (posInBaseHandle - startOff) / snapshot_byte_interval)];
1748
0
            if (snapshot->posInBaseHandle == 0)
1749
0
            {
1750
0
                snapshot->crc = crc32(
1751
0
                    crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1752
#ifdef ENABLE_DEBUG
1753
                CPLDebug("SNAPSHOT",
1754
                         "creating snapshot %d : "
1755
                         "posInBaseHandle=" CPL_FRMT_GUIB " in=" CPL_FRMT_GUIB
1756
                         " out=" CPL_FRMT_GUIB " crc=%X",
1757
                         static_cast<int>((posInBaseHandle - startOff) /
1758
                                          snapshot_byte_interval),
1759
                         posInBaseHandle, in, out,
1760
                         static_cast<unsigned int>(snapshot->crc));
1761
#endif
1762
0
                snapshot->posInBaseHandle = posInBaseHandle;
1763
0
                if (inflateBack9Copy(&snapshot->stream, &stream) != Z_OK)
1764
0
                    CPLError(CE_Failure, CPLE_AppDefined,
1765
0
                             "inflateBack9Copy() failed");
1766
0
                snapshot->in = in;
1767
0
                snapshot->out = out;
1768
0
                snapshot->extraOutput = extraOutput;
1769
0
                snapshot->m_bStreamEndReached = m_bStreamEndReached;
1770
0
            }
1771
1772
0
            errno = 0;
1773
0
            stream.avail_in =
1774
0
                static_cast<uInt>(m_poBaseHandle->Read(inbuf, 1, Z_BUFSIZE));
1775
#ifdef ENABLE_DEBUG
1776
            CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
1777
                     m_poBaseHandle->Tell(), offsetEndCompressedData);
1778
#endif
1779
0
            if (m_poBaseHandle->Tell() > offsetEndCompressedData)
1780
0
            {
1781
#ifdef ENABLE_DEBUG
1782
                CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
1783
#endif
1784
0
                stream.avail_in = stream.avail_in -
1785
0
                                  static_cast<uInt>(m_poBaseHandle->Tell() -
1786
0
                                                    offsetEndCompressedData);
1787
0
                if (m_poBaseHandle->Seek(offsetEndCompressedData, SEEK_SET) !=
1788
0
                    0)
1789
0
                    CPLError(CE_Failure, CPLE_FileIO, "Seek() failed");
1790
#ifdef ENABLE_DEBUG
1791
                CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
1792
#endif
1793
0
            }
1794
0
            if (stream.avail_in == 0)
1795
0
            {
1796
0
                z_eof = 1;
1797
0
                if (m_poBaseHandle->Error() ||
1798
0
                    m_poBaseHandle->Tell() != offsetEndCompressedData)
1799
0
                {
1800
0
                    z_err = Z_ERRNO;
1801
0
                    break;
1802
0
                }
1803
0
            }
1804
0
            stream.next_in = inbuf;
1805
0
        }
1806
1807
0
        struct InOutCallback
1808
0
        {
1809
0
            vsi_l_offset *pOut = nullptr;
1810
0
            std::vector<GByte> *pExtraOutput = nullptr;
1811
0
            z_stream *pStream = nullptr;
1812
1813
0
            static unsigned inCbk(void FAR *, z_const unsigned char FAR *FAR *)
1814
0
            {
1815
0
                return 0;
1816
0
            }
1817
1818
0
            static int outCbk(void FAR *user_data, unsigned char FAR *data,
1819
0
                              unsigned len)
1820
0
            {
1821
0
                auto self = static_cast<InOutCallback *>(user_data);
1822
0
                if (self->pStream->avail_out >= len)
1823
0
                {
1824
0
                    memcpy(self->pStream->next_out, data, len);
1825
0
                    *(self->pOut) += len;
1826
0
                    self->pStream->next_out += len;
1827
0
                    self->pStream->avail_out -= len;
1828
0
                }
1829
0
                else
1830
0
                {
1831
0
                    if (self->pStream->avail_out != 0)
1832
0
                    {
1833
0
                        memcpy(self->pStream->next_out, data,
1834
0
                               self->pStream->avail_out);
1835
0
                        *(self->pOut) += self->pStream->avail_out;
1836
0
                        data += self->pStream->avail_out;
1837
0
                        len -= self->pStream->avail_out;
1838
0
                        self->pStream->next_out += self->pStream->avail_out;
1839
0
                        self->pStream->avail_out = 0;
1840
0
                    }
1841
0
                    if (len > 0)
1842
0
                    {
1843
0
                        self->pExtraOutput->insert(self->pExtraOutput->end(),
1844
0
                                                   data, data + len);
1845
0
                    }
1846
0
                }
1847
0
                return 0;
1848
0
            }
1849
0
        };
1850
1851
0
        InOutCallback cbkData;
1852
0
        cbkData.pOut = &out;
1853
0
        cbkData.pExtraOutput = &extraOutput;
1854
0
        cbkData.pStream = &stream;
1855
1856
0
        if (stream.avail_out)
1857
0
        {
1858
0
            if (m_bStreamEndReached)
1859
0
                z_err = Z_STREAM_END;
1860
0
            else
1861
0
            {
1862
0
                in += stream.avail_in;
1863
0
                z_err = inflateBack9(&(stream), InOutCallback::inCbk, &cbkData,
1864
0
                                     InOutCallback::outCbk, &cbkData);
1865
0
                in -= stream.avail_in;
1866
0
            }
1867
0
        }
1868
0
        if (z_err == Z_BUF_ERROR && stream.next_in == Z_NULL)
1869
0
            z_err = Z_OK;
1870
0
        else if (!extraOutput.empty() && z_err == Z_STREAM_END)
1871
0
        {
1872
0
            m_bStreamEndReached = true;
1873
0
            z_err = Z_OK;
1874
0
        }
1875
1876
0
        if (z_err == Z_STREAM_END /*&& m_compressed_size != 2*/)
1877
0
        {
1878
            // Check CRC and original size.
1879
0
            crc =
1880
0
                crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1881
0
            pStart = stream.next_out;
1882
0
            if (m_expected_crc)
1883
0
            {
1884
#ifdef ENABLE_DEBUG
1885
                CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X",
1886
                         static_cast<unsigned int>(crc),
1887
                         static_cast<unsigned int>(m_expected_crc));
1888
#endif
1889
0
            }
1890
0
            if (m_expected_crc != 0 && m_expected_crc != crc)
1891
0
            {
1892
0
                CPLError(CE_Failure, CPLE_FileIO,
1893
0
                         "CRC error. Got %X instead of %X",
1894
0
                         static_cast<unsigned int>(crc),
1895
0
                         static_cast<unsigned int>(m_expected_crc));
1896
0
                z_err = Z_DATA_ERROR;
1897
0
            }
1898
0
        }
1899
0
        if (z_err != Z_OK || z_eof)
1900
0
            break;
1901
0
    }
1902
0
    crc = crc32(crc, pStart, static_cast<uInt>(stream.next_out - pStart));
1903
1904
0
    size_t ret = (len - stream.avail_out) / nSize;
1905
0
    if (z_err != Z_OK && z_err != Z_STREAM_END)
1906
0
    {
1907
0
        CPLError(CE_Failure, CPLE_AppDefined,
1908
0
                 "In file %s, at line %d, decompression failed with "
1909
0
                 "z_err = %d, return = %d",
1910
0
                 __FILE__, __LINE__, z_err, static_cast<int>(ret));
1911
0
    }
1912
0
    else if (ret < nMemb)
1913
0
    {
1914
0
        m_bEOF = true;
1915
0
    }
1916
1917
#ifdef ENABLE_DEBUG
1918
    CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
1919
             static_cast<int>(ret), z_err, z_eof);
1920
#endif
1921
0
    return ret;
1922
0
}
1923
1924
/************************************************************************/
1925
/*                              Write()                                 */
1926
/************************************************************************/
1927
1928
size_t VSIDeflate64Handle::Write(const void * /* pBuffer */, size_t /* nSize */,
1929
                                 size_t /* nMemb */)
1930
0
{
1931
0
    CPLError(CE_Failure, CPLE_NotSupported,
1932
0
             "VSIFWriteL is not supported on GZip streams");
1933
0
    return 0;
1934
0
}
1935
1936
/************************************************************************/
1937
/*                               Eof()                                  */
1938
/************************************************************************/
1939
1940
int VSIDeflate64Handle::Eof()
1941
0
{
1942
#ifdef ENABLE_DEBUG
1943
    CPLDebug("GZIP", "Eof()");
1944
#endif
1945
0
    return m_bEOF;
1946
0
}
1947
1948
/************************************************************************/
1949
/*                             Error()                                  */
1950
/************************************************************************/
1951
1952
int VSIDeflate64Handle::Error()
1953
0
{
1954
#ifdef ENABLE_DEBUG
1955
    CPLDebug("GZIP", "Error()");
1956
#endif
1957
0
    return z_err != Z_OK && z_err != Z_STREAM_END;
1958
0
}
1959
1960
/************************************************************************/
1961
/*                             ClearErr()                               */
1962
/************************************************************************/
1963
1964
void VSIDeflate64Handle::ClearErr()
1965
0
{
1966
0
    m_poBaseHandle->ClearErr();
1967
0
    z_eof = 0;
1968
0
    m_bEOF = false;
1969
0
    z_err = Z_OK;
1970
0
}
1971
1972
/************************************************************************/
1973
/*                              Flush()                                 */
1974
/************************************************************************/
1975
1976
int VSIDeflate64Handle::Flush()
1977
0
{
1978
0
    return 0;
1979
0
}
1980
1981
/************************************************************************/
1982
/*                              Close()                                 */
1983
/************************************************************************/
1984
1985
int VSIDeflate64Handle::Close()
1986
0
{
1987
0
    return 0;
1988
0
}
1989
#endif
1990
1991
/************************************************************************/
1992
/* ==================================================================== */
1993
/*                       VSIGZipWriteHandleMT                           */
1994
/* ==================================================================== */
1995
/************************************************************************/
1996
1997
class VSIGZipWriteHandleMT final : public VSIVirtualHandle
1998
{
1999
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipWriteHandleMT)
2000
2001
    VSIVirtualHandle *poBaseHandle_ = nullptr;
2002
    vsi_l_offset nCurOffset_ = 0;
2003
    uLong nCRC_ = 0;
2004
    int nDeflateType_ = CPL_DEFLATE_TYPE_GZIP;
2005
    bool bAutoCloseBaseHandle_ = false;
2006
    int nThreads_ = 0;
2007
    std::unique_ptr<CPLWorkerThreadPool> poPool_{};
2008
    std::list<std::string *> aposBuffers_{};
2009
    std::string *pCurBuffer_ = nullptr;
2010
    std::mutex sMutex_{};
2011
    int nSeqNumberGenerated_ = 0;
2012
    int nSeqNumberExpected_ = 0;
2013
    int nSeqNumberExpectedCRC_ = 0;
2014
    size_t nChunkSize_ = 0;
2015
    bool bHasErrored_ = false;
2016
2017
    struct Job
2018
    {
2019
        VSIGZipWriteHandleMT *pParent_ = nullptr;
2020
        std::string *pBuffer_ = nullptr;
2021
        int nSeqNumber_ = 0;
2022
        bool bFinish_ = false;
2023
        bool bInCRCComputation_ = false;
2024
2025
        std::string sCompressedData_{};
2026
        uLong nCRC_ = 0;
2027
    };
2028
2029
    std::list<Job *> apoFinishedJobs_{};
2030
    std::list<Job *> apoCRCFinishedJobs_{};
2031
    std::list<Job *> apoFreeJobs_{};
2032
    vsi_l_offset nStartOffset_ = 0;
2033
    size_t nSOZIPIndexEltSize_ = 0;
2034
    std::vector<uint8_t> *panSOZIPIndex_ = nullptr;
2035
2036
    static void DeflateCompress(void *inData);
2037
    static void CRCCompute(void *inData);
2038
    bool ProcessCompletedJobs();
2039
    Job *GetJobObject();
2040
#ifdef DEBUG_VERBOSE
2041
    void DumpState();
2042
#endif
2043
2044
  public:
2045
    VSIGZipWriteHandleMT(VSIVirtualHandle *poBaseHandle, int nDeflateType,
2046
                         bool bAutoCloseBaseHandleIn, int nThreads,
2047
                         size_t nChunkSize, size_t nSOZIPIndexEltSize,
2048
                         std::vector<uint8_t> *panSOZIPIndex);
2049
2050
    ~VSIGZipWriteHandleMT() override;
2051
2052
    int Seek(vsi_l_offset nOffset, int nWhence) override;
2053
    vsi_l_offset Tell() override;
2054
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
2055
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
2056
2057
    int Eof() override
2058
0
    {
2059
0
        return 0;
2060
0
    }
2061
2062
    int Error() override
2063
0
    {
2064
0
        return 0;
2065
0
    }
2066
2067
    void ClearErr() override
2068
0
    {
2069
0
    }
2070
2071
    int Flush() override;
2072
    int Close() override;
2073
};
2074
2075
/************************************************************************/
2076
/*                        VSIGZipWriteHandleMT()                        */
2077
/************************************************************************/
2078
2079
VSIGZipWriteHandleMT::VSIGZipWriteHandleMT(VSIVirtualHandle *poBaseHandle,
2080
                                           int nDeflateType,
2081
                                           bool bAutoCloseBaseHandleIn,
2082
                                           int nThreads, size_t nChunkSize,
2083
                                           size_t nSOZIPIndexEltSize,
2084
                                           std::vector<uint8_t> *panSOZIPIndex)
2085
0
    : poBaseHandle_(poBaseHandle), nDeflateType_(nDeflateType),
2086
0
      bAutoCloseBaseHandle_(bAutoCloseBaseHandleIn), nThreads_(nThreads),
2087
0
      nChunkSize_(nChunkSize), nSOZIPIndexEltSize_(nSOZIPIndexEltSize),
2088
0
      panSOZIPIndex_(panSOZIPIndex)
2089
0
{
2090
0
    if (nChunkSize_ == 0)
2091
0
    {
2092
0
        const char *pszChunkSize =
2093
0
            CPLGetConfigOption("CPL_VSIL_DEFLATE_CHUNK_SIZE", "1024K");
2094
0
        nChunkSize_ = static_cast<size_t>(atoi(pszChunkSize));
2095
0
        if (strchr(pszChunkSize, 'K'))
2096
0
            nChunkSize_ *= 1024;
2097
0
        else if (strchr(pszChunkSize, 'M'))
2098
0
            nChunkSize_ *= 1024 * 1024;
2099
0
        nChunkSize_ =
2100
0
            std::max(static_cast<size_t>(4 * 1024),
2101
0
                     std::min(static_cast<size_t>(UINT_MAX), nChunkSize_));
2102
0
    }
2103
2104
0
    for (int i = 0; i < 1 + nThreads_; i++)
2105
0
        aposBuffers_.emplace_back(new std::string());
2106
2107
0
    nStartOffset_ = poBaseHandle_->Tell();
2108
0
    if (nDeflateType == CPL_DEFLATE_TYPE_GZIP)
2109
0
    {
2110
0
        char header[11] = {};
2111
2112
        // Write a very simple .gz header:
2113
0
        snprintf(header, sizeof(header), "%c%c%c%c%c%c%c%c%c%c", gz_magic[0],
2114
0
                 gz_magic[1], Z_DEFLATED, 0 /*flags*/, 0, 0, 0, 0 /*time*/,
2115
0
                 0 /*xflags*/, 0x03);
2116
0
        poBaseHandle_->Write(header, 1, 10);
2117
0
    }
2118
0
}
2119
2120
/************************************************************************/
2121
/*                       ~VSIGZipWriteHandleMT()                        */
2122
/************************************************************************/
2123
2124
VSIGZipWriteHandleMT::~VSIGZipWriteHandleMT()
2125
2126
0
{
2127
0
    VSIGZipWriteHandleMT::Close();
2128
0
    for (auto &psJob : apoFinishedJobs_)
2129
0
    {
2130
0
        delete psJob->pBuffer_;
2131
0
        delete psJob;
2132
0
    }
2133
0
    for (auto &psJob : apoCRCFinishedJobs_)
2134
0
    {
2135
0
        delete psJob->pBuffer_;
2136
0
        delete psJob;
2137
0
    }
2138
0
    for (auto &psJob : apoFreeJobs_)
2139
0
    {
2140
0
        delete psJob->pBuffer_;
2141
0
        delete psJob;
2142
0
    }
2143
0
    for (auto &pstr : aposBuffers_)
2144
0
    {
2145
0
        delete pstr;
2146
0
    }
2147
0
    delete pCurBuffer_;
2148
0
}
2149
2150
/************************************************************************/
2151
/*                               Close()                                */
2152
/************************************************************************/
2153
2154
int VSIGZipWriteHandleMT::Close()
2155
2156
0
{
2157
0
    if (!poBaseHandle_)
2158
0
        return 0;
2159
2160
0
    int nRet = 0;
2161
2162
0
    if (!pCurBuffer_)
2163
0
        pCurBuffer_ = new std::string();
2164
2165
0
    {
2166
0
        auto psJob = GetJobObject();
2167
0
        psJob->bFinish_ = true;
2168
0
        psJob->pParent_ = this;
2169
0
        psJob->pBuffer_ = pCurBuffer_;
2170
0
        pCurBuffer_ = nullptr;
2171
0
        psJob->nSeqNumber_ = nSeqNumberGenerated_;
2172
0
        VSIGZipWriteHandleMT::DeflateCompress(psJob);
2173
0
    }
2174
2175
0
    if (poPool_)
2176
0
    {
2177
0
        poPool_->WaitCompletion(0);
2178
0
    }
2179
0
    if (!ProcessCompletedJobs())
2180
0
    {
2181
0
        nRet = -1;
2182
0
    }
2183
0
    else
2184
0
    {
2185
0
        CPLAssert(apoFinishedJobs_.empty());
2186
0
        if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2187
0
        {
2188
0
            if (poPool_)
2189
0
            {
2190
0
                poPool_->WaitCompletion(0);
2191
0
            }
2192
0
            ProcessCompletedJobs();
2193
0
        }
2194
0
        CPLAssert(apoCRCFinishedJobs_.empty());
2195
0
    }
2196
2197
0
    if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2198
0
    {
2199
0
        const GUInt32 anTrailer[2] = {
2200
0
            CPL_LSBWORD32(static_cast<GUInt32>(nCRC_)),
2201
0
            CPL_LSBWORD32(static_cast<GUInt32>(nCurOffset_))};
2202
2203
0
        if (poBaseHandle_->Write(anTrailer, 1, 8) < 8)
2204
0
        {
2205
0
            nRet = -1;
2206
0
        }
2207
0
    }
2208
2209
0
    if (bAutoCloseBaseHandle_)
2210
0
    {
2211
0
        int nRetClose = poBaseHandle_->Close();
2212
0
        if (nRet == 0)
2213
0
            nRet = nRetClose;
2214
2215
0
        delete poBaseHandle_;
2216
0
    }
2217
0
    poBaseHandle_ = nullptr;
2218
2219
0
    return nRet;
2220
0
}
2221
2222
/************************************************************************/
2223
/*                                Read()                                */
2224
/************************************************************************/
2225
2226
size_t VSIGZipWriteHandleMT::Read(void * /* pBuffer */, size_t /* nSize */,
2227
                                  size_t /* nMemb */)
2228
0
{
2229
0
    CPLError(CE_Failure, CPLE_NotSupported,
2230
0
             "VSIFReadL is not supported on GZip write streams");
2231
0
    return 0;
2232
0
}
2233
2234
/************************************************************************/
2235
/*                        DeflateCompress()                             */
2236
/************************************************************************/
2237
2238
void VSIGZipWriteHandleMT::DeflateCompress(void *inData)
2239
0
{
2240
0
    Job *psJob = static_cast<Job *>(inData);
2241
2242
0
    CPLAssert(psJob->pBuffer_);
2243
2244
0
    z_stream sStream;
2245
0
    memset(&sStream, 0, sizeof(sStream));
2246
0
    sStream.zalloc = nullptr;
2247
0
    sStream.zfree = nullptr;
2248
0
    sStream.opaque = nullptr;
2249
2250
0
    sStream.avail_in = static_cast<uInt>(psJob->pBuffer_->size());
2251
0
    sStream.next_in = reinterpret_cast<Bytef *>(&(*psJob->pBuffer_)[0]);
2252
2253
0
    int ret = deflateInit2(
2254
0
        &sStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
2255
0
        (psJob->pParent_->nDeflateType_ == CPL_DEFLATE_TYPE_ZLIB) ? MAX_WBITS
2256
0
                                                                  : -MAX_WBITS,
2257
0
        8, Z_DEFAULT_STRATEGY);
2258
0
    CPLAssertAlwaysEval(ret == Z_OK);
2259
2260
0
    size_t nRealSize = 0;
2261
2262
0
    while (sStream.avail_in > 0)
2263
0
    {
2264
0
        psJob->sCompressedData_.resize(nRealSize + Z_BUFSIZE);
2265
0
        sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2266
0
        sStream.next_out =
2267
0
            reinterpret_cast<Bytef *>(&psJob->sCompressedData_[0]) + nRealSize;
2268
2269
0
        const int zlibRet = deflate(&sStream, Z_NO_FLUSH);
2270
0
        CPLAssertAlwaysEval(zlibRet == Z_OK);
2271
2272
0
        nRealSize += static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2273
0
    }
2274
2275
0
    psJob->sCompressedData_.resize(nRealSize + Z_BUFSIZE);
2276
0
    sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2277
0
    sStream.next_out =
2278
0
        reinterpret_cast<Bytef *>(&psJob->sCompressedData_[0]) + nRealSize;
2279
2280
0
    if (psJob->bFinish_)
2281
0
    {
2282
0
        const int zlibRet = deflate(&sStream, Z_FINISH);
2283
0
        CPLAssertAlwaysEval(zlibRet == Z_STREAM_END);
2284
0
    }
2285
0
    else
2286
0
    {
2287
        // Do a Z_SYNC_FLUSH and Z_FULL_FLUSH, so as to have two markers when
2288
        // independent as pigz 2.3.4 or later. The following 9 byte sequence
2289
        // will be found: 0x00 0x00 0xff 0xff 0x00 0x00 0x00 0xff 0xff
2290
        // Z_FULL_FLUSH only is sufficient, but it is not obvious if a
2291
        // 0x00 0x00 0xff 0xff marker in the codestream is just a SYNC_FLUSH (
2292
        // without dictionary reset) or a FULL_FLUSH (with dictionary reset)
2293
0
        {
2294
0
            const int zlibRet = deflate(&sStream, Z_SYNC_FLUSH);
2295
0
            CPLAssertAlwaysEval(zlibRet == Z_OK);
2296
0
        }
2297
2298
0
        {
2299
0
            const int zlibRet = deflate(&sStream, Z_FULL_FLUSH);
2300
0
            CPLAssertAlwaysEval(zlibRet == Z_OK);
2301
0
        }
2302
0
    }
2303
2304
0
    nRealSize += static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2305
0
    psJob->sCompressedData_.resize(nRealSize);
2306
2307
0
    deflateEnd(&sStream);
2308
2309
0
    {
2310
0
        std::lock_guard<std::mutex> oLock(psJob->pParent_->sMutex_);
2311
0
        psJob->pParent_->apoFinishedJobs_.push_back(psJob);
2312
0
    }
2313
0
}
2314
2315
/************************************************************************/
2316
/*                          CRCCompute()                                */
2317
/************************************************************************/
2318
2319
void VSIGZipWriteHandleMT::CRCCompute(void *inData)
2320
0
{
2321
0
    Job *psJob = static_cast<Job *>(inData);
2322
0
    psJob->bInCRCComputation_ = true;
2323
0
    psJob->nCRC_ =
2324
0
        crc32(0U, reinterpret_cast<const Bytef *>(psJob->pBuffer_->data()),
2325
0
              static_cast<uInt>(psJob->pBuffer_->size()));
2326
2327
0
    {
2328
0
        std::lock_guard<std::mutex> oLock(psJob->pParent_->sMutex_);
2329
0
        psJob->pParent_->apoCRCFinishedJobs_.push_back(psJob);
2330
0
    }
2331
0
}
2332
2333
/************************************************************************/
2334
/*                                DumpState()                           */
2335
/************************************************************************/
2336
2337
#ifdef DEBUG_VERBOSE
2338
void VSIGZipWriteHandleMT::DumpState()
2339
{
2340
    fprintf(stderr, "Finished jobs (expected = %d):\n",  // ok
2341
            nSeqNumberExpected_);
2342
    for (const auto *psJob : apoFinishedJobs_)
2343
    {
2344
        fprintf(stderr, "seq number=%d, bInCRCComputation = %d\n",  // ok
2345
                psJob->nSeqNumber_, psJob->bInCRCComputation_ ? 1 : 0);
2346
    }
2347
    fprintf(stderr, "Finished CRC jobs (expected = %d):\n",  // ok
2348
            nSeqNumberExpectedCRC_);
2349
    for (const auto *psJob : apoFinishedJobs_)
2350
    {
2351
        fprintf(stderr, "seq number=%d\n",  // ok
2352
                psJob->nSeqNumber_);
2353
    }
2354
    fprintf(stderr, "apoFreeJobs_.size() = %d\n",  // ok
2355
            static_cast<int>(apoFreeJobs_.size()));
2356
    fprintf(stderr, "aposBuffers_.size() = %d\n",  // ok
2357
            static_cast<int>(aposBuffers_.size()));
2358
}
2359
#endif
2360
2361
/************************************************************************/
2362
/*                         ProcessCompletedJobs()                       */
2363
/************************************************************************/
2364
2365
bool VSIGZipWriteHandleMT::ProcessCompletedJobs()
2366
0
{
2367
0
    std::lock_guard<std::mutex> oLock(sMutex_);
2368
0
    bool do_it_again = true;
2369
0
    while (do_it_again)
2370
0
    {
2371
0
        do_it_again = false;
2372
0
        if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2373
0
        {
2374
0
            for (auto iter = apoFinishedJobs_.begin();
2375
0
                 iter != apoFinishedJobs_.end(); ++iter)
2376
0
            {
2377
0
                auto psJob = *iter;
2378
2379
0
                if (!psJob->bInCRCComputation_)
2380
0
                {
2381
0
                    psJob->bInCRCComputation_ = true;
2382
0
                    sMutex_.unlock();
2383
0
                    if (poPool_)
2384
0
                    {
2385
0
                        poPool_->SubmitJob(VSIGZipWriteHandleMT::CRCCompute,
2386
0
                                           psJob);
2387
0
                    }
2388
0
                    else
2389
0
                    {
2390
0
                        CRCCompute(psJob);
2391
0
                    }
2392
0
                    sMutex_.lock();
2393
0
                }
2394
0
            }
2395
0
        }
2396
2397
0
        for (auto iter = apoFinishedJobs_.begin();
2398
0
             iter != apoFinishedJobs_.end(); ++iter)
2399
0
        {
2400
0
            auto psJob = *iter;
2401
0
            if (psJob->nSeqNumber_ == nSeqNumberExpected_)
2402
0
            {
2403
0
                apoFinishedJobs_.erase(iter);
2404
2405
0
                const bool bIsSeqNumberExpectedZero =
2406
0
                    (nSeqNumberExpected_ == 0);
2407
0
                sMutex_.unlock();
2408
2409
0
                const size_t nToWrite = psJob->sCompressedData_.size();
2410
0
                if (panSOZIPIndex_ && !bIsSeqNumberExpectedZero &&
2411
0
                    !psJob->pBuffer_->empty())
2412
0
                {
2413
0
                    uint64_t nOffset = poBaseHandle_->Tell() - nStartOffset_;
2414
0
                    if (nSOZIPIndexEltSize_ == 8)
2415
0
                    {
2416
0
                        CPL_LSBPTR64(&nOffset);
2417
0
                        std::copy(reinterpret_cast<const uint8_t *>(&nOffset),
2418
0
                                  reinterpret_cast<const uint8_t *>(&nOffset) +
2419
0
                                      sizeof(nOffset),
2420
0
                                  std::back_inserter(*panSOZIPIndex_));
2421
0
                    }
2422
0
                    else
2423
0
                    {
2424
0
                        if (nOffset > std::numeric_limits<uint32_t>::max())
2425
0
                        {
2426
                            // shouldn't happen normally...
2427
0
                            CPLError(
2428
0
                                CE_Failure, CPLE_AppDefined,
2429
0
                                "Too big offset for SOZIP_OFFSET_SIZE = 4");
2430
0
                            panSOZIPIndex_->clear();
2431
0
                            panSOZIPIndex_ = nullptr;
2432
0
                        }
2433
0
                        else
2434
0
                        {
2435
0
                            uint32_t nOffset32 = static_cast<uint32_t>(nOffset);
2436
0
                            CPL_LSBPTR32(&nOffset32);
2437
0
                            std::copy(
2438
0
                                reinterpret_cast<const uint8_t *>(&nOffset32),
2439
0
                                reinterpret_cast<const uint8_t *>(&nOffset32) +
2440
0
                                    sizeof(nOffset32),
2441
0
                                std::back_inserter(*panSOZIPIndex_));
2442
0
                        }
2443
0
                    }
2444
0
                }
2445
0
                bool bError =
2446
0
                    poBaseHandle_->Write(psJob->sCompressedData_.data(), 1,
2447
0
                                         nToWrite) < nToWrite;
2448
0
                sMutex_.lock();
2449
0
                nSeqNumberExpected_++;
2450
2451
0
                if (nDeflateType_ != CPL_DEFLATE_TYPE_GZIP)
2452
0
                {
2453
0
                    aposBuffers_.push_back(psJob->pBuffer_);
2454
0
                    psJob->pBuffer_ = nullptr;
2455
2456
0
                    apoFreeJobs_.push_back(psJob);
2457
0
                }
2458
2459
0
                if (bError)
2460
0
                {
2461
0
                    return false;
2462
0
                }
2463
2464
0
                do_it_again = true;
2465
0
                break;
2466
0
            }
2467
0
        }
2468
2469
0
        if (nDeflateType_ == CPL_DEFLATE_TYPE_GZIP)
2470
0
        {
2471
0
            for (auto iter = apoCRCFinishedJobs_.begin();
2472
0
                 iter != apoCRCFinishedJobs_.end(); ++iter)
2473
0
            {
2474
0
                auto psJob = *iter;
2475
0
                if (psJob->nSeqNumber_ == nSeqNumberExpectedCRC_)
2476
0
                {
2477
0
                    apoCRCFinishedJobs_.erase(iter);
2478
2479
0
                    nCRC_ = crc32_combine(
2480
0
                        nCRC_, psJob->nCRC_,
2481
0
                        static_cast<uLong>(psJob->pBuffer_->size()));
2482
2483
0
                    nSeqNumberExpectedCRC_++;
2484
2485
0
                    aposBuffers_.push_back(psJob->pBuffer_);
2486
0
                    psJob->pBuffer_ = nullptr;
2487
2488
0
                    apoFreeJobs_.push_back(psJob);
2489
0
                    do_it_again = true;
2490
0
                    break;
2491
0
                }
2492
0
            }
2493
0
        }
2494
0
    }
2495
0
    return true;
2496
0
}
2497
2498
/************************************************************************/
2499
/*                           GetJobObject()                             */
2500
/************************************************************************/
2501
2502
VSIGZipWriteHandleMT::Job *VSIGZipWriteHandleMT::GetJobObject()
2503
0
{
2504
0
    {
2505
0
        std::lock_guard<std::mutex> oLock(sMutex_);
2506
0
        if (!apoFreeJobs_.empty())
2507
0
        {
2508
0
            auto job = apoFreeJobs_.back();
2509
0
            apoFreeJobs_.pop_back();
2510
0
            job->sCompressedData_.clear();
2511
0
            job->bInCRCComputation_ = false;
2512
0
            return job;
2513
0
        }
2514
0
    }
2515
0
    return new Job();
2516
0
}
2517
2518
/************************************************************************/
2519
/*                               Write()                                */
2520
/************************************************************************/
2521
2522
size_t VSIGZipWriteHandleMT::Write(const void *const pBuffer,
2523
                                   size_t const nSize, size_t const nMemb)
2524
2525
0
{
2526
0
    if (bHasErrored_)
2527
0
        return 0;
2528
2529
0
    const char *pszBuffer = static_cast<const char *>(pBuffer);
2530
0
    size_t nBytesToWrite = nSize * nMemb;
2531
0
    while (nBytesToWrite > 0)
2532
0
    {
2533
0
        if (pCurBuffer_ == nullptr)
2534
0
        {
2535
0
            while (true)
2536
0
            {
2537
                // We store in a local variable instead of pCurBuffer_ directly
2538
                // to avoid Coverity Scan to be confused by the fact that we
2539
                // have used above pCurBuffer_ outside of the mutex. But what
2540
                // is protected by the mutex is aposBuffers_, not pCurBuffer_.
2541
0
                std::string *l_pCurBuffer = nullptr;
2542
0
                {
2543
0
                    std::lock_guard<std::mutex> oLock(sMutex_);
2544
0
                    if (!aposBuffers_.empty())
2545
0
                    {
2546
0
                        l_pCurBuffer = aposBuffers_.back();
2547
0
                        aposBuffers_.pop_back();
2548
0
                    }
2549
0
                }
2550
0
                pCurBuffer_ = l_pCurBuffer;
2551
0
                if (pCurBuffer_)
2552
0
                    break;
2553
2554
0
                if (poPool_)
2555
0
                {
2556
0
                    poPool_->WaitEvent();
2557
0
                }
2558
0
                if (!ProcessCompletedJobs())
2559
0
                {
2560
0
                    bHasErrored_ = true;
2561
0
                    return 0;
2562
0
                }
2563
0
            }
2564
0
            pCurBuffer_->clear();
2565
0
        }
2566
0
        size_t nConsumed =
2567
0
            std::min(nBytesToWrite, nChunkSize_ - pCurBuffer_->size());
2568
0
        pCurBuffer_->append(pszBuffer, nConsumed);
2569
0
        nCurOffset_ += nConsumed;
2570
0
        pszBuffer += nConsumed;
2571
0
        nBytesToWrite -= nConsumed;
2572
0
        if (pCurBuffer_->size() == nChunkSize_)
2573
0
        {
2574
0
            if (poPool_ == nullptr)
2575
0
            {
2576
0
                poPool_.reset(new CPLWorkerThreadPool());
2577
0
                if (!poPool_->Setup(nThreads_, nullptr, nullptr, false))
2578
0
                {
2579
0
                    bHasErrored_ = true;
2580
0
                    poPool_.reset();
2581
0
                    return 0;
2582
0
                }
2583
0
            }
2584
2585
0
            auto psJob = GetJobObject();
2586
0
            psJob->pParent_ = this;
2587
0
            psJob->pBuffer_ = pCurBuffer_;
2588
0
            psJob->nSeqNumber_ = nSeqNumberGenerated_;
2589
0
            nSeqNumberGenerated_++;
2590
0
            pCurBuffer_ = nullptr;
2591
0
            poPool_->SubmitJob(VSIGZipWriteHandleMT::DeflateCompress, psJob);
2592
0
        }
2593
0
    }
2594
2595
0
    return nMemb;
2596
0
}
2597
2598
/************************************************************************/
2599
/*                               Flush()                                */
2600
/************************************************************************/
2601
2602
int VSIGZipWriteHandleMT::Flush()
2603
2604
0
{
2605
    // we *could* do something for this but for now we choose not to.
2606
2607
0
    return 0;
2608
0
}
2609
2610
/************************************************************************/
2611
/*                                Seek()                                */
2612
/************************************************************************/
2613
2614
int VSIGZipWriteHandleMT::Seek(vsi_l_offset nOffset, int nWhence)
2615
2616
0
{
2617
0
    if (nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR))
2618
0
        return 0;
2619
0
    else if (nWhence == SEEK_SET && nOffset == nCurOffset_)
2620
0
        return 0;
2621
0
    else
2622
0
    {
2623
0
        CPLError(CE_Failure, CPLE_NotSupported,
2624
0
                 "Seeking on writable compressed data streams not supported.");
2625
2626
0
        return -1;
2627
0
    }
2628
0
}
2629
2630
/************************************************************************/
2631
/*                                Tell()                                */
2632
/************************************************************************/
2633
2634
vsi_l_offset VSIGZipWriteHandleMT::Tell()
2635
2636
0
{
2637
0
    return nCurOffset_;
2638
0
}
2639
2640
/************************************************************************/
2641
/* ==================================================================== */
2642
/*                       VSIGZipWriteHandle                             */
2643
/* ==================================================================== */
2644
/************************************************************************/
2645
2646
class VSIGZipWriteHandle final : public VSIVirtualHandle
2647
{
2648
    CPL_DISALLOW_COPY_ASSIGN(VSIGZipWriteHandle)
2649
2650
    VSIVirtualHandle *m_poBaseHandle = nullptr;
2651
    z_stream sStream;
2652
    Byte *pabyInBuf = nullptr;
2653
    Byte *pabyOutBuf = nullptr;
2654
    bool bCompressActive = false;
2655
    vsi_l_offset nCurOffset = 0;
2656
    uLong nCRC = 0;
2657
    int nDeflateType = CPL_DEFLATE_TYPE_GZIP;
2658
    bool bAutoCloseBaseHandle = false;
2659
2660
  public:
2661
    VSIGZipWriteHandle(VSIVirtualHandle *poBaseHandle, int nDeflateType,
2662
                       bool bAutoCloseBaseHandleIn);
2663
2664
    ~VSIGZipWriteHandle() override;
2665
2666
    int Seek(vsi_l_offset nOffset, int nWhence) override;
2667
    vsi_l_offset Tell() override;
2668
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
2669
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
2670
2671
    int Eof() override
2672
0
    {
2673
0
        return 0;
2674
0
    }
2675
2676
    int Error() override
2677
0
    {
2678
0
        return 0;
2679
0
    }
2680
2681
    void ClearErr() override
2682
0
    {
2683
0
    }
2684
2685
    int Flush() override;
2686
    int Close() override;
2687
};
2688
2689
/************************************************************************/
2690
/*                         VSIGZipWriteHandle()                         */
2691
/************************************************************************/
2692
2693
VSIGZipWriteHandle::VSIGZipWriteHandle(VSIVirtualHandle *poBaseHandle,
2694
                                       int nDeflateTypeIn,
2695
                                       bool bAutoCloseBaseHandleIn)
2696
0
    : m_poBaseHandle(poBaseHandle), sStream(),
2697
0
      pabyInBuf(static_cast<Byte *>(CPLMalloc(Z_BUFSIZE))),
2698
0
      pabyOutBuf(static_cast<Byte *>(CPLMalloc(Z_BUFSIZE))),
2699
0
      nCRC(crc32(0L, nullptr, 0)), nDeflateType(nDeflateTypeIn),
2700
0
      bAutoCloseBaseHandle(bAutoCloseBaseHandleIn)
2701
0
{
2702
0
    sStream.zalloc = nullptr;
2703
0
    sStream.zfree = nullptr;
2704
0
    sStream.opaque = nullptr;
2705
0
    sStream.next_in = nullptr;
2706
0
    sStream.next_out = nullptr;
2707
0
    sStream.avail_in = sStream.avail_out = 0;
2708
2709
0
    sStream.next_in = pabyInBuf;
2710
2711
0
    if (deflateInit2(&sStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
2712
0
                     (nDeflateType == CPL_DEFLATE_TYPE_ZLIB) ? MAX_WBITS
2713
0
                                                             : -MAX_WBITS,
2714
0
                     8, Z_DEFAULT_STRATEGY) != Z_OK)
2715
0
    {
2716
0
        bCompressActive = false;
2717
0
    }
2718
0
    else
2719
0
    {
2720
0
        if (nDeflateType == CPL_DEFLATE_TYPE_GZIP)
2721
0
        {
2722
0
            char header[11] = {};
2723
2724
            // Write a very simple .gz header:
2725
0
            snprintf(header, sizeof(header), "%c%c%c%c%c%c%c%c%c%c",
2726
0
                     gz_magic[0], gz_magic[1], Z_DEFLATED, 0 /*flags*/, 0, 0, 0,
2727
0
                     0 /*time*/, 0 /*xflags*/, 0x03);
2728
0
            m_poBaseHandle->Write(header, 1, 10);
2729
0
        }
2730
2731
0
        bCompressActive = true;
2732
0
    }
2733
0
}
2734
2735
/************************************************************************/
2736
/*                       VSICreateGZipWritable()                        */
2737
/************************************************************************/
2738
2739
VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
2740
                                        int nDeflateTypeIn,
2741
                                        int bAutoCloseBaseHandle)
2742
0
{
2743
0
    return VSICreateGZipWritable(poBaseHandle, nDeflateTypeIn,
2744
0
                                 CPL_TO_BOOL(bAutoCloseBaseHandle), 0, 0, 0,
2745
0
                                 nullptr);
2746
0
}
2747
2748
VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
2749
                                        int nDeflateTypeIn,
2750
                                        bool bAutoCloseBaseHandle, int nThreads,
2751
                                        size_t nChunkSize,
2752
                                        size_t nSOZIPIndexEltSize,
2753
                                        std::vector<uint8_t> *panSOZIPIndex)
2754
0
{
2755
0
    const char *pszThreads = CPLGetConfigOption("GDAL_NUM_THREADS", nullptr);
2756
0
    if (pszThreads || nThreads > 0 || nChunkSize > 0)
2757
0
    {
2758
0
        if (nThreads == 0)
2759
0
        {
2760
0
            if (!pszThreads || EQUAL(pszThreads, "ALL_CPUS"))
2761
0
                nThreads = CPLGetNumCPUs();
2762
0
            else
2763
0
                nThreads = atoi(pszThreads);
2764
0
            nThreads = std::max(1, std::min(128, nThreads));
2765
0
        }
2766
0
        if (nThreads > 1 || nChunkSize > 0)
2767
0
        {
2768
            // coverity[tainted_data]
2769
0
            return new VSIGZipWriteHandleMT(
2770
0
                poBaseHandle, nDeflateTypeIn, bAutoCloseBaseHandle, nThreads,
2771
0
                nChunkSize, nSOZIPIndexEltSize, panSOZIPIndex);
2772
0
        }
2773
0
    }
2774
0
    return new VSIGZipWriteHandle(poBaseHandle, nDeflateTypeIn,
2775
0
                                  bAutoCloseBaseHandle);
2776
0
}
2777
2778
/************************************************************************/
2779
/*                        ~VSIGZipWriteHandle()                         */
2780
/************************************************************************/
2781
2782
VSIGZipWriteHandle::~VSIGZipWriteHandle()
2783
2784
0
{
2785
0
    if (bCompressActive)
2786
0
        VSIGZipWriteHandle::Close();
2787
2788
0
    CPLFree(pabyInBuf);
2789
0
    CPLFree(pabyOutBuf);
2790
0
}
2791
2792
/************************************************************************/
2793
/*                               Close()                                */
2794
/************************************************************************/
2795
2796
int VSIGZipWriteHandle::Close()
2797
2798
0
{
2799
0
    int nRet = 0;
2800
0
    if (bCompressActive)
2801
0
    {
2802
0
        sStream.next_out = pabyOutBuf;
2803
0
        sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2804
2805
0
        const int zlibRet = deflate(&sStream, Z_FINISH);
2806
0
        CPLAssertAlwaysEval(zlibRet == Z_STREAM_END);
2807
2808
0
        const size_t nOutBytes =
2809
0
            static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2810
2811
0
        deflateEnd(&sStream);
2812
2813
0
        if (m_poBaseHandle->Write(pabyOutBuf, 1, nOutBytes) < nOutBytes)
2814
0
        {
2815
0
            nRet = -1;
2816
0
        }
2817
2818
0
        if (nRet == 0 && nDeflateType == CPL_DEFLATE_TYPE_GZIP)
2819
0
        {
2820
0
            const GUInt32 anTrailer[2] = {
2821
0
                CPL_LSBWORD32(static_cast<GUInt32>(nCRC)),
2822
0
                CPL_LSBWORD32(static_cast<GUInt32>(nCurOffset))};
2823
2824
0
            if (m_poBaseHandle->Write(anTrailer, 1, 8) < 8)
2825
0
            {
2826
0
                nRet = -1;
2827
0
            }
2828
0
        }
2829
2830
0
        if (bAutoCloseBaseHandle)
2831
0
        {
2832
0
            if (nRet == 0)
2833
0
                nRet = m_poBaseHandle->Close();
2834
2835
0
            delete m_poBaseHandle;
2836
0
        }
2837
2838
0
        bCompressActive = false;
2839
0
    }
2840
2841
0
    return nRet;
2842
0
}
2843
2844
/************************************************************************/
2845
/*                                Read()                                */
2846
/************************************************************************/
2847
2848
size_t VSIGZipWriteHandle::Read(void * /* pBuffer */, size_t /* nSize */,
2849
                                size_t /* nMemb */)
2850
0
{
2851
0
    CPLError(CE_Failure, CPLE_NotSupported,
2852
0
             "VSIFReadL is not supported on GZip write streams");
2853
0
    return 0;
2854
0
}
2855
2856
/************************************************************************/
2857
/*                               Write()                                */
2858
/************************************************************************/
2859
2860
size_t VSIGZipWriteHandle::Write(const void *const pBuffer, size_t const nSize,
2861
                                 size_t const nMemb)
2862
2863
0
{
2864
0
    size_t nBytesToWrite = nSize * nMemb;
2865
2866
0
    {
2867
0
        size_t nOffset = 0;
2868
0
        while (nOffset < nBytesToWrite)
2869
0
        {
2870
0
            uInt nChunk = static_cast<uInt>(std::min(
2871
0
                static_cast<size_t>(UINT_MAX), nBytesToWrite - nOffset));
2872
0
            nCRC =
2873
0
                crc32(nCRC, reinterpret_cast<const Bytef *>(pBuffer) + nOffset,
2874
0
                      nChunk);
2875
0
            nOffset += nChunk;
2876
0
        }
2877
0
    }
2878
2879
0
    if (!bCompressActive)
2880
0
        return 0;
2881
2882
0
    size_t nNextByte = 0;
2883
0
    while (nNextByte < nBytesToWrite)
2884
0
    {
2885
0
        sStream.next_out = pabyOutBuf;
2886
0
        sStream.avail_out = static_cast<uInt>(Z_BUFSIZE);
2887
2888
0
        if (sStream.avail_in > 0)
2889
0
            memmove(pabyInBuf, sStream.next_in, sStream.avail_in);
2890
2891
0
        const uInt nNewBytesToWrite = static_cast<uInt>(
2892
0
            std::min(static_cast<size_t>(Z_BUFSIZE - sStream.avail_in),
2893
0
                     nBytesToWrite - nNextByte));
2894
0
        memcpy(pabyInBuf + sStream.avail_in,
2895
0
               reinterpret_cast<const Byte *>(pBuffer) + nNextByte,
2896
0
               nNewBytesToWrite);
2897
2898
0
        sStream.next_in = pabyInBuf;
2899
0
        sStream.avail_in += nNewBytesToWrite;
2900
2901
0
        const int zlibRet = deflate(&sStream, Z_NO_FLUSH);
2902
0
        CPLAssertAlwaysEval(zlibRet == Z_OK);
2903
2904
0
        const size_t nOutBytes =
2905
0
            static_cast<uInt>(Z_BUFSIZE) - sStream.avail_out;
2906
2907
0
        if (nOutBytes > 0)
2908
0
        {
2909
0
            if (m_poBaseHandle->Write(pabyOutBuf, 1, nOutBytes) < nOutBytes)
2910
0
                return 0;
2911
0
        }
2912
2913
0
        nNextByte += nNewBytesToWrite;
2914
0
        nCurOffset += nNewBytesToWrite;
2915
0
    }
2916
2917
0
    return nMemb;
2918
0
}
2919
2920
/************************************************************************/
2921
/*                               Flush()                                */
2922
/************************************************************************/
2923
2924
int VSIGZipWriteHandle::Flush()
2925
2926
0
{
2927
    // we *could* do something for this but for now we choose not to.
2928
2929
0
    return 0;
2930
0
}
2931
2932
/************************************************************************/
2933
/*                                Seek()                                */
2934
/************************************************************************/
2935
2936
int VSIGZipWriteHandle::Seek(vsi_l_offset nOffset, int nWhence)
2937
2938
0
{
2939
0
    if (nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR))
2940
0
        return 0;
2941
0
    else if (nWhence == SEEK_SET && nOffset == nCurOffset)
2942
0
        return 0;
2943
0
    else
2944
0
    {
2945
0
        CPLError(CE_Failure, CPLE_NotSupported,
2946
0
                 "Seeking on writable compressed data streams not supported.");
2947
2948
0
        return -1;
2949
0
    }
2950
0
}
2951
2952
/************************************************************************/
2953
/*                                Tell()                                */
2954
/************************************************************************/
2955
2956
vsi_l_offset VSIGZipWriteHandle::Tell()
2957
2958
0
{
2959
0
    return nCurOffset;
2960
0
}
2961
2962
/************************************************************************/
2963
/* ==================================================================== */
2964
/*                       VSIGZipFilesystemHandler                       */
2965
/* ==================================================================== */
2966
/************************************************************************/
2967
2968
/************************************************************************/
2969
/*                  ~VSIGZipFilesystemHandler()                         */
2970
/************************************************************************/
2971
2972
VSIGZipFilesystemHandler::~VSIGZipFilesystemHandler()
2973
0
{
2974
0
    if (poHandleLastGZipFile)
2975
0
    {
2976
0
        poHandleLastGZipFile->UnsetCanSaveInfo();
2977
0
        delete poHandleLastGZipFile;
2978
0
    }
2979
2980
0
    if (hMutex != nullptr)
2981
0
        CPLDestroyMutex(hMutex);
2982
0
    hMutex = nullptr;
2983
0
}
2984
2985
/************************************************************************/
2986
/*                            SaveInfo()                                */
2987
/************************************************************************/
2988
2989
void VSIGZipFilesystemHandler::SaveInfo(VSIGZipHandle *poHandle)
2990
0
{
2991
0
    CPLMutexHolder oHolder(&hMutex);
2992
0
    SaveInfo_unlocked(poHandle);
2993
0
}
2994
2995
void VSIGZipFilesystemHandler::SaveInfo_unlocked(VSIGZipHandle *poHandle)
2996
0
{
2997
0
    if (m_bInSaveInfo)
2998
0
        return;
2999
0
    m_bInSaveInfo = true;
3000
3001
0
    CPLAssert(poHandle != poHandleLastGZipFile);
3002
0
    CPLAssert(poHandle->GetBaseFileName() != nullptr);
3003
3004
0
    if (poHandleLastGZipFile == nullptr ||
3005
0
        strcmp(poHandleLastGZipFile->GetBaseFileName(),
3006
0
               poHandle->GetBaseFileName()) != 0 ||
3007
0
        poHandle->GetLastReadOffset() >
3008
0
            poHandleLastGZipFile->GetLastReadOffset())
3009
0
    {
3010
0
        VSIGZipHandle *poTmp = poHandleLastGZipFile;
3011
0
        poHandleLastGZipFile = nullptr;
3012
0
        if (poTmp)
3013
0
        {
3014
0
            poTmp->UnsetCanSaveInfo();
3015
0
            delete poTmp;
3016
0
        }
3017
0
        CPLAssert(poHandleLastGZipFile == nullptr);
3018
0
        poHandleLastGZipFile = poHandle->Duplicate();
3019
0
        if (poHandleLastGZipFile)
3020
0
            poHandleLastGZipFile->CloseBaseHandle();
3021
0
    }
3022
0
    m_bInSaveInfo = false;
3023
0
}
3024
3025
/************************************************************************/
3026
/*                                Open()                                */
3027
/************************************************************************/
3028
3029
VSIVirtualHandle *
3030
VSIGZipFilesystemHandler::Open(const char *pszFilename, const char *pszAccess,
3031
                               bool /* bSetError */,
3032
                               CSLConstList /* papszOptions */)
3033
0
{
3034
0
    if (!STARTS_WITH_CI(pszFilename, "/vsigzip/"))
3035
0
        return nullptr;
3036
3037
0
    VSIFilesystemHandler *poFSHandler =
3038
0
        VSIFileManager::GetHandler(pszFilename + strlen("/vsigzip/"));
3039
3040
    /* -------------------------------------------------------------------- */
3041
    /*      Is this an attempt to write a new file without update (w+)      */
3042
    /*      access?  If so, create a writable handle for the underlying     */
3043
    /*      filename.                                                       */
3044
    /* -------------------------------------------------------------------- */
3045
0
    if (strchr(pszAccess, 'w') != nullptr)
3046
0
    {
3047
0
        if (strchr(pszAccess, '+') != nullptr)
3048
0
        {
3049
0
            CPLError(CE_Failure, CPLE_AppDefined,
3050
0
                     "Write+update (w+) not supported for /vsigzip, "
3051
0
                     "only read-only or write-only.");
3052
0
            return nullptr;
3053
0
        }
3054
3055
0
        VSIVirtualHandle *poVirtualHandle =
3056
0
            poFSHandler->Open(pszFilename + strlen("/vsigzip/"), "wb");
3057
3058
0
        if (poVirtualHandle == nullptr)
3059
0
            return nullptr;
3060
3061
0
        return VSICreateGZipWritable(poVirtualHandle,
3062
0
                                     strchr(pszAccess, 'z') != nullptr, TRUE);
3063
0
    }
3064
3065
    /* -------------------------------------------------------------------- */
3066
    /*      Otherwise we are in the read access case.                       */
3067
    /* -------------------------------------------------------------------- */
3068
3069
0
    VSIGZipHandle *poGZIPHandle = OpenGZipReadOnly(pszFilename, pszAccess);
3070
0
    if (poGZIPHandle)
3071
        // Wrap the VSIGZipHandle inside a buffered reader that will
3072
        // improve dramatically performance when doing small backward
3073
        // seeks.
3074
0
        return VSICreateBufferedReaderHandle(poGZIPHandle);
3075
3076
0
    return nullptr;
3077
0
}
3078
3079
/************************************************************************/
3080
/*                      SupportsSequentialWrite()                       */
3081
/************************************************************************/
3082
3083
bool VSIGZipFilesystemHandler::SupportsSequentialWrite(const char *pszPath,
3084
                                                       bool bAllowLocalTempFile)
3085
0
{
3086
0
    if (!STARTS_WITH_CI(pszPath, "/vsigzip/"))
3087
0
        return false;
3088
0
    const char *pszBaseFileName = pszPath + strlen("/vsigzip/");
3089
0
    VSIFilesystemHandler *poFSHandler =
3090
0
        VSIFileManager::GetHandler(pszBaseFileName);
3091
0
    return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
3092
0
}
3093
3094
/************************************************************************/
3095
/*                          OpenGZipReadOnly()                          */
3096
/************************************************************************/
3097
3098
VSIGZipHandle *
3099
VSIGZipFilesystemHandler::OpenGZipReadOnly(const char *pszFilename,
3100
                                           const char *pszAccess)
3101
0
{
3102
0
    VSIFilesystemHandler *poFSHandler =
3103
0
        VSIFileManager::GetHandler(pszFilename + strlen("/vsigzip/"));
3104
3105
0
    CPLMutexHolder oHolder(&hMutex);
3106
3107
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
3108
    // Disable caching in fuzzing mode as the /vsigzip/ file is likely to
3109
    // change very often
3110
    // TODO: filename-based logic isn't enough. We should probably check
3111
    // timestamp and/or file size.
3112
    if (poHandleLastGZipFile != nullptr &&
3113
        strcmp(pszFilename + strlen("/vsigzip/"),
3114
               poHandleLastGZipFile->GetBaseFileName()) == 0 &&
3115
        EQUAL(pszAccess, "rb"))
3116
    {
3117
        VSIGZipHandle *poHandle = poHandleLastGZipFile->Duplicate();
3118
        if (poHandle)
3119
            return poHandle;
3120
    }
3121
#else
3122
0
    CPL_IGNORE_RET_VAL(pszAccess);
3123
0
#endif
3124
3125
0
    VSIVirtualHandle *poVirtualHandle =
3126
0
        poFSHandler->Open(pszFilename + strlen("/vsigzip/"), "rb");
3127
3128
0
    if (poVirtualHandle == nullptr)
3129
0
        return nullptr;
3130
3131
0
    unsigned char signature[2] = {'\0', '\0'};
3132
0
    if (poVirtualHandle->Read(signature, 1, 2) != 2 ||
3133
0
        signature[0] != gz_magic[0] || signature[1] != gz_magic[1])
3134
0
    {
3135
0
        poVirtualHandle->Close();
3136
0
        delete poVirtualHandle;
3137
0
        return nullptr;
3138
0
    }
3139
3140
0
    if (poHandleLastGZipFile)
3141
0
    {
3142
0
        poHandleLastGZipFile->UnsetCanSaveInfo();
3143
0
        delete poHandleLastGZipFile;
3144
0
        poHandleLastGZipFile = nullptr;
3145
0
    }
3146
3147
0
    VSIGZipHandle *poHandle =
3148
0
        new VSIGZipHandle(poVirtualHandle, pszFilename + strlen("/vsigzip/"));
3149
0
    if (!(poHandle->IsInitOK()))
3150
0
    {
3151
0
        delete poHandle;
3152
0
        return nullptr;
3153
0
    }
3154
0
    return poHandle;
3155
0
}
3156
3157
/************************************************************************/
3158
/*                                Stat()                                */
3159
/************************************************************************/
3160
3161
int VSIGZipFilesystemHandler::Stat(const char *pszFilename,
3162
                                   VSIStatBufL *pStatBuf, int nFlags)
3163
0
{
3164
0
    if (!STARTS_WITH_CI(pszFilename, "/vsigzip/"))
3165
0
        return -1;
3166
3167
0
    CPLMutexHolder oHolder(&hMutex);
3168
3169
0
    memset(pStatBuf, 0, sizeof(VSIStatBufL));
3170
3171
0
    if (poHandleLastGZipFile != nullptr &&
3172
0
        strcmp(pszFilename + strlen("/vsigzip/"),
3173
0
               poHandleLastGZipFile->GetBaseFileName()) == 0)
3174
0
    {
3175
0
        if (poHandleLastGZipFile->GetUncompressedSize() != 0)
3176
0
        {
3177
0
            pStatBuf->st_mode = S_IFREG;
3178
0
            pStatBuf->st_size = poHandleLastGZipFile->GetUncompressedSize();
3179
0
            return 0;
3180
0
        }
3181
0
    }
3182
3183
    // Begin by doing a stat on the real file.
3184
0
    int ret = VSIStatExL(pszFilename + strlen("/vsigzip/"), pStatBuf, nFlags);
3185
3186
0
    if (ret == 0 && (nFlags & VSI_STAT_SIZE_FLAG))
3187
0
    {
3188
0
        CPLString osCacheFilename(pszFilename + strlen("/vsigzip/"));
3189
0
        osCacheFilename += ".properties";
3190
3191
        // Can we save a bit of seeking by using a .properties file?
3192
0
        VSILFILE *fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "rb");
3193
0
        if (fpCacheLength)
3194
0
        {
3195
0
            const char *pszLine;
3196
0
            GUIntBig nCompressedSize = 0;
3197
0
            GUIntBig nUncompressedSize = 0;
3198
0
            while ((pszLine = CPLReadLineL(fpCacheLength)) != nullptr)
3199
0
            {
3200
0
                if (STARTS_WITH_CI(pszLine, "compressed_size="))
3201
0
                {
3202
0
                    const char *pszBuffer =
3203
0
                        pszLine + strlen("compressed_size=");
3204
0
                    nCompressedSize = CPLScanUIntBig(
3205
0
                        pszBuffer, static_cast<int>(strlen(pszBuffer)));
3206
0
                }
3207
0
                else if (STARTS_WITH_CI(pszLine, "uncompressed_size="))
3208
0
                {
3209
0
                    const char *pszBuffer =
3210
0
                        pszLine + strlen("uncompressed_size=");
3211
0
                    nUncompressedSize = CPLScanUIntBig(
3212
0
                        pszBuffer, static_cast<int>(strlen(pszBuffer)));
3213
0
                }
3214
0
            }
3215
3216
0
            CPL_IGNORE_RET_VAL(VSIFCloseL(fpCacheLength));
3217
3218
0
            if (nCompressedSize == static_cast<GUIntBig>(pStatBuf->st_size))
3219
0
            {
3220
                // Patch with the uncompressed size.
3221
0
                pStatBuf->st_size = nUncompressedSize;
3222
3223
0
                VSIGZipHandle *poHandle =
3224
0
                    VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename,
3225
0
                                                               "rb");
3226
0
                if (poHandle)
3227
0
                {
3228
0
                    poHandle->SetUncompressedSize(nUncompressedSize);
3229
0
                    SaveInfo_unlocked(poHandle);
3230
0
                    delete poHandle;
3231
0
                }
3232
3233
0
                return ret;
3234
0
            }
3235
0
        }
3236
3237
        // No, then seek at the end of the data (slow).
3238
0
        VSIGZipHandle *poHandle =
3239
0
            VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
3240
0
        if (poHandle)
3241
0
        {
3242
0
            poHandle->Seek(0, SEEK_END);
3243
0
            const GUIntBig uncompressed_size =
3244
0
                static_cast<GUIntBig>(poHandle->Tell());
3245
0
            poHandle->Seek(0, SEEK_SET);
3246
3247
            // Patch with the uncompressed size.
3248
0
            pStatBuf->st_size = uncompressed_size;
3249
3250
0
            delete poHandle;
3251
0
        }
3252
0
        else
3253
0
        {
3254
0
            ret = -1;
3255
0
        }
3256
0
    }
3257
3258
0
    return ret;
3259
0
}
3260
3261
/************************************************************************/
3262
/*                             ReadDirEx()                                */
3263
/************************************************************************/
3264
3265
char **VSIGZipFilesystemHandler::ReadDirEx(const char * /*pszDirname*/,
3266
                                           int /* nMaxFiles */)
3267
0
{
3268
0
    return nullptr;
3269
0
}
3270
3271
/************************************************************************/
3272
/*                           GetOptions()                               */
3273
/************************************************************************/
3274
3275
const char *VSIGZipFilesystemHandler::GetOptions()
3276
0
{
3277
0
    return "<Options>"
3278
0
           "  <Option name='GDAL_NUM_THREADS' type='string' "
3279
0
           "description='Number of threads for compression. Either a integer "
3280
0
           "or ALL_CPUS'/>"
3281
0
           "  <Option name='CPL_VSIL_DEFLATE_CHUNK_SIZE' type='string' "
3282
0
           "description='Chunk of uncompressed data for parallelization. "
3283
0
           "Use K(ilobytes) or M(egabytes) suffix' default='1M'/>"
3284
0
           "</Options>";
3285
0
}
3286
3287
//! @endcond
3288
/************************************************************************/
3289
/*                   VSIInstallGZipFileHandler()                        */
3290
/************************************************************************/
3291
3292
/*!
3293
 \brief Install GZip file system handler.
3294
3295
 A special file handler is installed that allows reading on-the-fly and
3296
 writing in GZip (.gz) files.
3297
3298
 All portions of the file system underneath the base
3299
 path "/vsigzip/" will be handled by this driver.
3300
3301
 \verbatim embed:rst
3302
 See :ref:`/vsigzip/ documentation <vsigzip>`
3303
 \endverbatim
3304
3305
 @since GDAL 1.6.0
3306
 */
3307
3308
void VSIInstallGZipFileHandler()
3309
1
{
3310
1
    VSIFileManager::InstallHandler("/vsigzip/", new VSIGZipFilesystemHandler);
3311
1
}
3312
3313
//! @cond Doxygen_Suppress
3314
3315
/************************************************************************/
3316
/* ==================================================================== */
3317
/*                         VSIZipEntryFileOffset                        */
3318
/* ==================================================================== */
3319
/************************************************************************/
3320
3321
class VSIZipEntryFileOffset final : public VSIArchiveEntryFileOffset
3322
{
3323
  public:
3324
    unz_file_pos m_file_pos;
3325
3326
0
    explicit VSIZipEntryFileOffset(unz_file_pos file_pos) : m_file_pos()
3327
0
    {
3328
0
        m_file_pos.pos_in_zip_directory = file_pos.pos_in_zip_directory;
3329
0
        m_file_pos.num_of_file = file_pos.num_of_file;
3330
0
    }
3331
3332
    ~VSIZipEntryFileOffset() override;
3333
};
3334
3335
0
VSIZipEntryFileOffset::~VSIZipEntryFileOffset() = default;
3336
3337
/************************************************************************/
3338
/* ==================================================================== */
3339
/*                             VSIZipReader                             */
3340
/* ==================================================================== */
3341
/************************************************************************/
3342
3343
class VSIZipReader final : public VSIArchiveReader
3344
{
3345
    CPL_DISALLOW_COPY_ASSIGN(VSIZipReader)
3346
3347
  private:
3348
    unzFile unzF = nullptr;
3349
    unz_file_pos file_pos;
3350
    GUIntBig nNextFileSize = 0;
3351
    CPLString osNextFileName{};
3352
    GIntBig nModifiedTime = 0;
3353
3354
    bool SetInfo();
3355
3356
  public:
3357
    explicit VSIZipReader(const char *pszZipFileName);
3358
    ~VSIZipReader() override;
3359
3360
    int IsValid()
3361
0
    {
3362
0
        return unzF != nullptr;
3363
0
    }
3364
3365
    unzFile GetUnzFileHandle()
3366
0
    {
3367
0
        return unzF;
3368
0
    }
3369
3370
    int GotoFirstFile() override;
3371
    int GotoNextFile() override;
3372
3373
    VSIArchiveEntryFileOffset *GetFileOffset() override
3374
0
    {
3375
0
        return new VSIZipEntryFileOffset(file_pos);
3376
0
    }
3377
3378
    GUIntBig GetFileSize() override
3379
0
    {
3380
0
        return nNextFileSize;
3381
0
    }
3382
3383
    CPLString GetFileName() override
3384
0
    {
3385
0
        return osNextFileName;
3386
0
    }
3387
3388
    GIntBig GetModifiedTime() override
3389
0
    {
3390
0
        return nModifiedTime;
3391
0
    }
3392
3393
    int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) override;
3394
};
3395
3396
/************************************************************************/
3397
/*                           VSIZipReader()                             */
3398
/************************************************************************/
3399
3400
VSIZipReader::VSIZipReader(const char *pszZipFileName)
3401
0
    : unzF(cpl_unzOpen(pszZipFileName)), file_pos()
3402
0
{
3403
0
    file_pos.pos_in_zip_directory = 0;
3404
0
    file_pos.num_of_file = 0;
3405
0
}
3406
3407
/************************************************************************/
3408
/*                          ~VSIZipReader()                             */
3409
/************************************************************************/
3410
3411
VSIZipReader::~VSIZipReader()
3412
0
{
3413
0
    if (unzF)
3414
0
        cpl_unzClose(unzF);
3415
0
}
3416
3417
/************************************************************************/
3418
/*                              SetInfo()                               */
3419
/************************************************************************/
3420
3421
bool VSIZipReader::SetInfo()
3422
0
{
3423
0
    char fileName[8193] = {};
3424
0
    unz_file_info file_info;
3425
0
    if (UNZ_OK != cpl_unzGetCurrentFileInfo(unzF, &file_info, fileName,
3426
0
                                            sizeof(fileName) - 1, nullptr, 0,
3427
0
                                            nullptr, 0))
3428
0
    {
3429
0
        CPLError(CE_Failure, CPLE_FileIO, "cpl_unzGetCurrentFileInfo failed");
3430
0
        cpl_unzGetFilePos(unzF, &file_pos);
3431
0
        return false;
3432
0
    }
3433
0
    fileName[sizeof(fileName) - 1] = '\0';
3434
0
    osNextFileName = fileName;
3435
0
    nNextFileSize = file_info.uncompressed_size;
3436
0
    struct tm brokendowntime;
3437
0
    brokendowntime.tm_sec = file_info.tmu_date.tm_sec;
3438
0
    brokendowntime.tm_min = file_info.tmu_date.tm_min;
3439
0
    brokendowntime.tm_hour = file_info.tmu_date.tm_hour;
3440
0
    brokendowntime.tm_mday = file_info.tmu_date.tm_mday;
3441
0
    brokendowntime.tm_mon = file_info.tmu_date.tm_mon;
3442
    // The minizip conventions differs from the Unix one.
3443
0
    brokendowntime.tm_year = file_info.tmu_date.tm_year - 1900;
3444
0
    nModifiedTime = CPLYMDHMSToUnixTime(&brokendowntime);
3445
3446
0
    cpl_unzGetFilePos(unzF, &file_pos);
3447
0
    return true;
3448
0
}
3449
3450
/************************************************************************/
3451
/*                           GotoNextFile()                             */
3452
/************************************************************************/
3453
3454
int VSIZipReader::GotoNextFile()
3455
0
{
3456
0
    if (cpl_unzGoToNextFile(unzF) != UNZ_OK)
3457
0
        return FALSE;
3458
3459
0
    if (!SetInfo())
3460
0
        return FALSE;
3461
3462
0
    return TRUE;
3463
0
}
3464
3465
/************************************************************************/
3466
/*                          GotoFirstFile()                             */
3467
/************************************************************************/
3468
3469
int VSIZipReader::GotoFirstFile()
3470
0
{
3471
0
    if (cpl_unzGoToFirstFile(unzF) != UNZ_OK)
3472
0
        return FALSE;
3473
3474
0
    if (!SetInfo())
3475
0
        return FALSE;
3476
3477
0
    return TRUE;
3478
0
}
3479
3480
/************************************************************************/
3481
/*                         GotoFileOffset()                             */
3482
/************************************************************************/
3483
3484
int VSIZipReader::GotoFileOffset(VSIArchiveEntryFileOffset *pOffset)
3485
0
{
3486
0
    VSIZipEntryFileOffset *pZipEntryOffset =
3487
0
        reinterpret_cast<VSIZipEntryFileOffset *>(pOffset);
3488
0
    if (cpl_unzGoToFilePos(unzF, &(pZipEntryOffset->m_file_pos)) != UNZ_OK)
3489
0
    {
3490
0
        CPLError(CE_Failure, CPLE_AppDefined, "GotoFileOffset failed");
3491
0
        return FALSE;
3492
0
    }
3493
3494
0
    if (!SetInfo())
3495
0
        return FALSE;
3496
3497
0
    return TRUE;
3498
0
}
3499
3500
/************************************************************************/
3501
/* ==================================================================== */
3502
/*                       VSIZipFilesystemHandler                  */
3503
/* ==================================================================== */
3504
/************************************************************************/
3505
3506
class VSIZipWriteHandle;
3507
3508
class VSIZipFilesystemHandler final : public VSIArchiveFilesystemHandler
3509
{
3510
    CPL_DISALLOW_COPY_ASSIGN(VSIZipFilesystemHandler)
3511
3512
    std::map<CPLString, VSIZipWriteHandle *> oMapZipWriteHandles{};
3513
    VSIVirtualHandle *OpenForWrite_unlocked(const char *pszFilename,
3514
                                            const char *pszAccess);
3515
3516
    struct VSIFileInZipInfo
3517
    {
3518
        VSIVirtualHandleUniquePtr poVirtualHandle{};
3519
        std::map<std::string, std::string> oMapProperties{};
3520
        int nCompressionMethod = 0;
3521
        uint64_t nUncompressedSize = 0;
3522
        uint64_t nCompressedSize = 0;
3523
        uint64_t nStartDataStream = 0;
3524
        uLong nCRC = 0;
3525
        bool bSOZipIndexFound = false;
3526
        bool bSOZipIndexValid = false;
3527
        uint32_t nSOZIPVersion = 0;
3528
        uint32_t nSOZIPToSkip = 0;
3529
        uint32_t nSOZIPChunkSize = 0;
3530
        uint32_t nSOZIPOffsetSize = 0;
3531
        uint64_t nSOZIPStartData = 0;
3532
    };
3533
3534
    bool GetFileInfo(const char *pszFilename, VSIFileInZipInfo &info);
3535
3536
  public:
3537
1
    VSIZipFilesystemHandler() = default;
3538
    ~VSIZipFilesystemHandler() override;
3539
3540
    const char *GetPrefix() override
3541
0
    {
3542
0
        return "/vsizip";
3543
0
    }
3544
3545
    std::vector<CPLString> GetExtensions() override;
3546
    VSIArchiveReader *CreateReader(const char *pszZipFileName) override;
3547
3548
    VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
3549
                           bool bSetError,
3550
                           CSLConstList /* papszOptions */) override;
3551
3552
    char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
3553
                           CSLConstList papszOptions) override;
3554
3555
    VSIVirtualHandle *OpenForWrite(const char *pszFilename,
3556
                                   const char *pszAccess);
3557
3558
    int CopyFile(const char *pszSource, const char *pszTarget,
3559
                 VSILFILE *fpSource, vsi_l_offset nSourceSize,
3560
                 const char *const *papszOptions,
3561
                 GDALProgressFunc pProgressFunc, void *pProgressData) override;
3562
3563
    int Mkdir(const char *pszDirname, long nMode) override;
3564
    char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
3565
    int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
3566
             int nFlags) override;
3567
3568
    const char *GetOptions() override;
3569
3570
    void RemoveFromMap(VSIZipWriteHandle *poHandle);
3571
};
3572
3573
/************************************************************************/
3574
/* ==================================================================== */
3575
/*                       VSIZipWriteHandle                              */
3576
/* ==================================================================== */
3577
/************************************************************************/
3578
3579
class VSIZipWriteHandle final : public VSIVirtualHandle
3580
{
3581
    CPL_DISALLOW_COPY_ASSIGN(VSIZipWriteHandle)
3582
3583
    VSIZipFilesystemHandler *m_poFS = nullptr;
3584
    void *m_hZIP = nullptr;
3585
    VSIZipWriteHandle *poChildInWriting = nullptr;
3586
    VSIZipWriteHandle *m_poParent = nullptr;
3587
    bool bAutoDeleteParent = false;
3588
    vsi_l_offset nCurOffset = 0;
3589
3590
  public:
3591
    VSIZipWriteHandle(VSIZipFilesystemHandler *poFS, void *hZIP,
3592
                      VSIZipWriteHandle *poParent);
3593
3594
    ~VSIZipWriteHandle() override;
3595
3596
    int Seek(vsi_l_offset nOffset, int nWhence) override;
3597
    vsi_l_offset Tell() override;
3598
    size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
3599
    size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
3600
3601
    int Eof() override
3602
0
    {
3603
0
        return 0;
3604
0
    }
3605
3606
    int Error() override
3607
0
    {
3608
0
        return 0;
3609
0
    }
3610
3611
    void ClearErr() override
3612
0
    {
3613
0
    }
3614
3615
    int Flush() override;
3616
    int Close() override;
3617
3618
    void StartNewFile(VSIZipWriteHandle *poSubFile);
3619
    void StopCurrentFile();
3620
3621
    void *GetHandle()
3622
0
    {
3623
0
        return m_hZIP;
3624
0
    }
3625
3626
    VSIZipWriteHandle *GetChildInWriting()
3627
0
    {
3628
0
        return poChildInWriting;
3629
0
    }
3630
3631
    void SetAutoDeleteParent()
3632
0
    {
3633
0
        bAutoDeleteParent = true;
3634
0
    }
3635
};
3636
3637
/************************************************************************/
3638
/*                      ~VSIZipFilesystemHandler()                      */
3639
/************************************************************************/
3640
3641
VSIZipFilesystemHandler::~VSIZipFilesystemHandler()
3642
0
{
3643
0
    for (std::map<CPLString, VSIZipWriteHandle *>::const_iterator iter =
3644
0
             oMapZipWriteHandles.begin();
3645
0
         iter != oMapZipWriteHandles.end(); ++iter)
3646
0
    {
3647
0
        CPLError(CE_Failure, CPLE_AppDefined, "%s has not been closed",
3648
0
                 iter->first.c_str());
3649
0
    }
3650
0
}
3651
3652
/************************************************************************/
3653
/*                          GetExtensions()                             */
3654
/************************************************************************/
3655
3656
std::vector<CPLString> VSIZipFilesystemHandler::GetExtensions()
3657
0
{
3658
0
    std::vector<CPLString> oList;
3659
0
    oList.push_back(".zip");
3660
0
    oList.push_back(".kmz");
3661
0
    oList.push_back(".dwf");
3662
0
    oList.push_back(".ods");
3663
0
    oList.push_back(".xlsx");
3664
0
    oList.push_back(".xlsm");
3665
3666
    // Add to zip FS handler extensions array additional extensions
3667
    // listed in CPL_VSIL_ZIP_ALLOWED_EXTENSIONS config option.
3668
    // The extensions are divided by commas.
3669
0
    const char *pszAllowedExtensions =
3670
0
        CPLGetConfigOption("CPL_VSIL_ZIP_ALLOWED_EXTENSIONS", nullptr);
3671
0
    if (pszAllowedExtensions)
3672
0
    {
3673
0
        char **papszExtensions =
3674
0
            CSLTokenizeString2(pszAllowedExtensions, ", ", 0);
3675
0
        for (int i = 0; papszExtensions[i] != nullptr; i++)
3676
0
        {
3677
0
            oList.push_back(papszExtensions[i]);
3678
0
        }
3679
0
        CSLDestroy(papszExtensions);
3680
0
    }
3681
3682
0
    return oList;
3683
0
}
3684
3685
/************************************************************************/
3686
/*                           CreateReader()                             */
3687
/************************************************************************/
3688
3689
VSIArchiveReader *
3690
VSIZipFilesystemHandler::CreateReader(const char *pszZipFileName)
3691
0
{
3692
0
    VSIZipReader *poReader = new VSIZipReader(pszZipFileName);
3693
3694
0
    if (!poReader->IsValid())
3695
0
    {
3696
0
        delete poReader;
3697
0
        return nullptr;
3698
0
    }
3699
3700
0
    if (!poReader->GotoFirstFile())
3701
0
    {
3702
0
        delete poReader;
3703
0
        return nullptr;
3704
0
    }
3705
3706
0
    return poReader;
3707
0
}
3708
3709
/************************************************************************/
3710
/*                         VSISOZipHandle                               */
3711
/************************************************************************/
3712
3713
class VSISOZipHandle final : public VSIVirtualHandle
3714
{
3715
    VSIVirtualHandle *poBaseHandle_;
3716
    vsi_l_offset nPosCompressedStream_;
3717
    uint64_t compressed_size_;
3718
    uint64_t uncompressed_size_;
3719
    vsi_l_offset indexPos_;
3720
    uint32_t nToSkip_;
3721
    uint32_t nChunkSize_;
3722
    bool bEOF_ = false;
3723
    bool bError_ = false;
3724
    vsi_l_offset nCurPos_ = 0;
3725
    bool bOK_ = true;
3726
#ifdef HAVE_LIBDEFLATE
3727
    struct libdeflate_decompressor *pDecompressor_ = nullptr;
3728
#else
3729
    z_stream sStream_{};
3730
#endif
3731
3732
    VSISOZipHandle(const VSISOZipHandle &) = delete;
3733
    VSISOZipHandle &operator=(const VSISOZipHandle &) = delete;
3734
3735
  public:
3736
    VSISOZipHandle(VSIVirtualHandle *poVirtualHandle,
3737
                   vsi_l_offset nPosCompressedStream, uint64_t compressed_size,
3738
                   uint64_t uncompressed_size, vsi_l_offset indexPos,
3739
                   uint32_t nToSkip, uint32_t nChunkSize);
3740
    ~VSISOZipHandle() override;
3741
3742
    virtual int Seek(vsi_l_offset nOffset, int nWhence) override;
3743
3744
    virtual vsi_l_offset Tell() override
3745
0
    {
3746
0
        return nCurPos_;
3747
0
    }
3748
3749
    virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) override;
3750
3751
    virtual size_t Write(const void *, size_t, size_t) override
3752
0
    {
3753
0
        return 0;
3754
0
    }
3755
3756
    virtual int Eof() override
3757
0
    {
3758
0
        return bEOF_;
3759
0
    }
3760
3761
    virtual int Error() override
3762
0
    {
3763
0
        return bError_;
3764
0
    }
3765
3766
    virtual void ClearErr() override
3767
0
    {
3768
0
        bEOF_ = false;
3769
0
        bError_ = false;
3770
0
    }
3771
3772
    virtual int Close() override;
3773
3774
    bool IsOK() const
3775
0
    {
3776
0
        return bOK_;
3777
0
    }
3778
};
3779
3780
/************************************************************************/
3781
/*                         VSISOZipHandle()                             */
3782
/************************************************************************/
3783
3784
VSISOZipHandle::VSISOZipHandle(VSIVirtualHandle *poVirtualHandle,
3785
                               vsi_l_offset nPosCompressedStream,
3786
                               uint64_t compressed_size,
3787
                               uint64_t uncompressed_size,
3788
                               vsi_l_offset indexPos, uint32_t nToSkip,
3789
                               uint32_t nChunkSize)
3790
0
    : poBaseHandle_(poVirtualHandle),
3791
0
      nPosCompressedStream_(nPosCompressedStream),
3792
0
      compressed_size_(compressed_size), uncompressed_size_(uncompressed_size),
3793
0
      indexPos_(indexPos), nToSkip_(nToSkip), nChunkSize_(nChunkSize)
3794
0
{
3795
#ifdef HAVE_LIBDEFLATE
3796
    pDecompressor_ = libdeflate_alloc_decompressor();
3797
    if (!pDecompressor_)
3798
        bOK_ = false;
3799
#else
3800
0
    memset(&sStream_, 0, sizeof(sStream_));
3801
0
    int err = inflateInit2(&sStream_, -MAX_WBITS);
3802
0
    if (err != Z_OK)
3803
0
        bOK_ = false;
3804
0
#endif
3805
0
}
3806
3807
/************************************************************************/
3808
/*                        ~VSISOZipHandle()                             */
3809
/************************************************************************/
3810
3811
VSISOZipHandle::~VSISOZipHandle()
3812
0
{
3813
0
    VSISOZipHandle::Close();
3814
0
    if (bOK_)
3815
0
    {
3816
#ifdef HAVE_LIBDEFLATE
3817
        libdeflate_free_decompressor(pDecompressor_);
3818
#else
3819
0
        inflateEnd(&sStream_);
3820
0
#endif
3821
0
    }
3822
0
}
3823
3824
/************************************************************************/
3825
/*                              Close()                                 */
3826
/************************************************************************/
3827
3828
int VSISOZipHandle::Close()
3829
0
{
3830
0
    delete poBaseHandle_;
3831
0
    poBaseHandle_ = nullptr;
3832
0
    return 0;
3833
0
}
3834
3835
/************************************************************************/
3836
/*                              Seek()                                  */
3837
/************************************************************************/
3838
3839
int VSISOZipHandle::Seek(vsi_l_offset nOffset, int nWhence)
3840
0
{
3841
0
    bEOF_ = false;
3842
0
    if (nWhence == SEEK_SET)
3843
0
        nCurPos_ = nOffset;
3844
0
    else if (nWhence == SEEK_END)
3845
0
        nCurPos_ = uncompressed_size_;
3846
0
    else
3847
0
        nCurPos_ += nOffset;
3848
0
    return 0;
3849
0
}
3850
3851
/************************************************************************/
3852
/*                              Read()                                  */
3853
/************************************************************************/
3854
3855
size_t VSISOZipHandle::Read(void *pBuffer, size_t nSize, size_t nCount)
3856
0
{
3857
0
    size_t nToRead = nSize * nCount;
3858
0
    if (nCurPos_ >= uncompressed_size_ && nToRead > 0)
3859
0
    {
3860
0
        bEOF_ = true;
3861
0
        return 0;
3862
0
    }
3863
3864
0
    if (nSize != 1)
3865
0
    {
3866
0
        bError_ = true;
3867
0
        CPLError(CE_Failure, CPLE_NotSupported, "Unsupported nSize");
3868
0
        return 0;
3869
0
    }
3870
0
    if ((nCurPos_ % nChunkSize_) != 0)
3871
0
    {
3872
0
        bError_ = true;
3873
0
        CPLError(CE_Failure, CPLE_NotSupported,
3874
0
                 "nCurPos is not a multiple of nChunkSize");
3875
0
        return 0;
3876
0
    }
3877
0
    if (nCurPos_ + nToRead > uncompressed_size_)
3878
0
    {
3879
0
        nToRead = static_cast<size_t>(uncompressed_size_ - nCurPos_);
3880
0
        nCount = nToRead;
3881
0
    }
3882
0
    else if ((nToRead % nChunkSize_) != 0)
3883
0
    {
3884
0
        bError_ = true;
3885
0
        CPLError(CE_Failure, CPLE_NotSupported,
3886
0
                 "nToRead is not a multiple of nChunkSize");
3887
0
        return 0;
3888
0
    }
3889
3890
0
    const auto ReadOffsetInCompressedStream =
3891
0
        [this](uint64_t nChunkIdx) -> uint64_t
3892
0
    {
3893
0
        if (nChunkIdx == 0)
3894
0
            return 0;
3895
0
        if (nChunkIdx == 1 + (uncompressed_size_ - 1) / nChunkSize_)
3896
0
            return compressed_size_;
3897
0
        constexpr size_t nOffsetSize = 8;
3898
0
        if (poBaseHandle_->Seek(indexPos_ + 32 + nToSkip_ +
3899
0
                                    (nChunkIdx - 1) * nOffsetSize,
3900
0
                                SEEK_SET) != 0)
3901
0
            return static_cast<uint64_t>(-1);
3902
3903
0
        uint64_t nOffset;
3904
0
        if (poBaseHandle_->Read(&nOffset, sizeof(nOffset), 1) != 1)
3905
0
            return static_cast<uint64_t>(-1);
3906
0
        CPL_LSBPTR64(&nOffset);
3907
0
        return nOffset;
3908
0
    };
3909
3910
0
    size_t nOffsetInOutputBuffer = 0;
3911
0
    while (true)
3912
0
    {
3913
0
        uint64_t nOffsetInCompressedStream =
3914
0
            ReadOffsetInCompressedStream(nCurPos_ / nChunkSize_);
3915
0
        if (nOffsetInCompressedStream == static_cast<uint64_t>(-1))
3916
0
        {
3917
0
            bError_ = true;
3918
0
            CPLError(CE_Failure, CPLE_AppDefined,
3919
0
                     "Cannot read nOffsetInCompressedStream");
3920
0
            return 0;
3921
0
        }
3922
0
        uint64_t nNextOffsetInCompressedStream =
3923
0
            ReadOffsetInCompressedStream(1 + nCurPos_ / nChunkSize_);
3924
0
        if (nNextOffsetInCompressedStream == static_cast<uint64_t>(-1))
3925
0
        {
3926
0
            bError_ = true;
3927
0
            CPLError(CE_Failure, CPLE_AppDefined,
3928
0
                     "Cannot read nNextOffsetInCompressedStream");
3929
0
            return 0;
3930
0
        }
3931
3932
0
        if (nNextOffsetInCompressedStream <= nOffsetInCompressedStream ||
3933
0
            nNextOffsetInCompressedStream - nOffsetInCompressedStream >
3934
0
                13 + 2 * nChunkSize_ ||
3935
0
            nNextOffsetInCompressedStream > compressed_size_)
3936
0
        {
3937
0
            bError_ = true;
3938
0
            CPLError(
3939
0
                CE_Failure, CPLE_AppDefined,
3940
0
                "Invalid values for nOffsetInCompressedStream (" CPL_FRMT_GUIB
3941
0
                ") / "
3942
0
                "nNextOffsetInCompressedStream(" CPL_FRMT_GUIB ")",
3943
0
                static_cast<GUIntBig>(nOffsetInCompressedStream),
3944
0
                static_cast<GUIntBig>(nNextOffsetInCompressedStream));
3945
0
            return 0;
3946
0
        }
3947
3948
        // CPLDebug("VSIZIP", "Seek to compressed data at offset "
3949
        // CPL_FRMT_GUIB, static_cast<GUIntBig>(nPosCompressedStream_ +
3950
        // nOffsetInCompressedStream));
3951
0
        if (poBaseHandle_->Seek(
3952
0
                nPosCompressedStream_ + nOffsetInCompressedStream, SEEK_SET) !=
3953
0
            0)
3954
0
        {
3955
0
            bError_ = true;
3956
0
            return 0;
3957
0
        }
3958
3959
0
        const int nCompressedToRead = static_cast<int>(
3960
0
            nNextOffsetInCompressedStream - nOffsetInCompressedStream);
3961
        // CPLDebug("VSIZIP", "nCompressedToRead = %d", nCompressedToRead);
3962
0
        std::vector<GByte> abyCompressedData(nCompressedToRead);
3963
0
        if (poBaseHandle_->Read(&abyCompressedData[0], nCompressedToRead, 1) !=
3964
0
            1)
3965
0
        {
3966
0
            bError_ = true;
3967
0
            return 0;
3968
0
        }
3969
3970
0
        size_t nToReadThisIter =
3971
0
            std::min(nToRead, static_cast<size_t>(nChunkSize_));
3972
3973
0
        if (nCompressedToRead >= 5 &&
3974
0
            abyCompressedData[nCompressedToRead - 5] == 0x00 &&
3975
0
            memcmp(&abyCompressedData[nCompressedToRead - 4],
3976
0
                   "\x00\x00\xFF\xFF", 4) == 0)
3977
0
        {
3978
            // Tag this flush block as the last one.
3979
0
            abyCompressedData[nCompressedToRead - 5] = 0x01;
3980
0
        }
3981
3982
#ifdef HAVE_LIBDEFLATE
3983
        size_t nOut = 0;
3984
        if (libdeflate_deflate_decompress(
3985
                pDecompressor_, &abyCompressedData[0], nCompressedToRead,
3986
                static_cast<Bytef *>(pBuffer) + nOffsetInOutputBuffer,
3987
                nToReadThisIter, &nOut) != LIBDEFLATE_SUCCESS)
3988
        {
3989
            bError_ = true;
3990
            CPLError(
3991
                CE_Failure, CPLE_AppDefined,
3992
                "libdeflate_deflate_decompress() failed at pos " CPL_FRMT_GUIB,
3993
                static_cast<GUIntBig>(nCurPos_));
3994
            return 0;
3995
        }
3996
        if (nOut != nToReadThisIter)
3997
        {
3998
            bError_ = true;
3999
            CPLError(CE_Failure, CPLE_AppDefined,
4000
                     "Only %u bytes decompressed at pos " CPL_FRMT_GUIB
4001
                     " whereas %u where expected",
4002
                     static_cast<unsigned>(nOut),
4003
                     static_cast<GUIntBig>(nCurPos_),
4004
                     static_cast<unsigned>(nToReadThisIter));
4005
            return 0;
4006
        }
4007
#else
4008
0
        sStream_.avail_in = nCompressedToRead;
4009
0
        sStream_.next_in = &abyCompressedData[0];
4010
0
        sStream_.avail_out = static_cast<int>(nToReadThisIter);
4011
0
        sStream_.next_out =
4012
0
            static_cast<Bytef *>(pBuffer) + nOffsetInOutputBuffer;
4013
4014
0
        int err = inflate(&sStream_, Z_FINISH);
4015
0
        if ((err != Z_OK && err != Z_STREAM_END))
4016
0
        {
4017
0
            bError_ = true;
4018
0
            CPLError(CE_Failure, CPLE_AppDefined,
4019
0
                     "inflate() failed at pos " CPL_FRMT_GUIB,
4020
0
                     static_cast<GUIntBig>(nCurPos_));
4021
0
            inflateReset(&sStream_);
4022
0
            return 0;
4023
0
        }
4024
0
        if (sStream_.avail_in != 0)
4025
0
            CPLDebug("VSIZIP", "avail_in = %d", sStream_.avail_in);
4026
0
        if (sStream_.avail_out != 0)
4027
0
        {
4028
0
            bError_ = true;
4029
0
            CPLError(
4030
0
                CE_Failure, CPLE_AppDefined,
4031
0
                "Only %u bytes decompressed at pos " CPL_FRMT_GUIB
4032
0
                " whereas %u where expected",
4033
0
                static_cast<unsigned>(nToReadThisIter - sStream_.avail_out),
4034
0
                static_cast<GUIntBig>(nCurPos_),
4035
0
                static_cast<unsigned>(nToReadThisIter));
4036
0
            inflateReset(&sStream_);
4037
0
            return 0;
4038
0
        }
4039
0
        inflateReset(&sStream_);
4040
0
#endif
4041
0
        nOffsetInOutputBuffer += nToReadThisIter;
4042
0
        nCurPos_ += nToReadThisIter;
4043
0
        nToRead -= nToReadThisIter;
4044
0
        if (nToRead == 0)
4045
0
            break;
4046
0
    }
4047
4048
0
    return nCount;
4049
0
}
4050
4051
/************************************************************************/
4052
/*                          GetFileInfo()                               */
4053
/************************************************************************/
4054
4055
bool VSIZipFilesystemHandler::GetFileInfo(const char *pszFilename,
4056
                                          VSIFileInZipInfo &info)
4057
0
{
4058
4059
0
    CPLString osZipInFileName;
4060
0
    char *zipFilename = SplitFilename(pszFilename, osZipInFileName, TRUE);
4061
0
    if (zipFilename == nullptr)
4062
0
        return false;
4063
4064
0
    {
4065
0
        CPLMutexHolder oHolder(&hMutex);
4066
0
        if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end())
4067
0
        {
4068
0
            CPLError(CE_Failure, CPLE_AppDefined,
4069
0
                     "Cannot read a zip file being written");
4070
0
            CPLFree(zipFilename);
4071
0
            return false;
4072
0
        }
4073
0
    }
4074
4075
0
    VSIArchiveReader *poReader = OpenArchiveFile(zipFilename, osZipInFileName);
4076
0
    if (poReader == nullptr)
4077
0
    {
4078
0
        CPLFree(zipFilename);
4079
0
        return false;
4080
0
    }
4081
4082
0
    VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(zipFilename);
4083
4084
0
    VSIVirtualHandle *poVirtualHandle = poFSHandler->Open(zipFilename, "rb");
4085
4086
0
    CPLFree(zipFilename);
4087
0
    zipFilename = nullptr;
4088
4089
0
    if (poVirtualHandle == nullptr)
4090
0
    {
4091
0
        delete poReader;
4092
0
        return false;
4093
0
    }
4094
4095
0
    unzFile unzF =
4096
0
        reinterpret_cast<VSIZipReader *>(poReader)->GetUnzFileHandle();
4097
4098
0
    if (cpl_unzOpenCurrentFile(unzF) != UNZ_OK)
4099
0
    {
4100
0
        CPLError(CE_Failure, CPLE_AppDefined,
4101
0
                 "cpl_unzOpenCurrentFile() failed");
4102
0
        delete poReader;
4103
0
        delete poVirtualHandle;
4104
0
        return false;
4105
0
    }
4106
4107
0
    info.nStartDataStream = cpl_unzGetCurrentFileZStreamPos(unzF);
4108
4109
0
    unz_file_info file_info;
4110
0
    if (cpl_unzGetCurrentFileInfo(unzF, &file_info, nullptr, 0, nullptr, 0,
4111
0
                                  nullptr, 0) != UNZ_OK)
4112
0
    {
4113
0
        CPLError(CE_Failure, CPLE_AppDefined,
4114
0
                 "cpl_unzGetCurrentFileInfo() failed");
4115
0
        cpl_unzCloseCurrentFile(unzF);
4116
0
        delete poReader;
4117
0
        delete poVirtualHandle;
4118
0
        return false;
4119
0
    }
4120
4121
0
    if (file_info.size_file_extra)
4122
0
    {
4123
0
        std::vector<GByte> abyExtra(file_info.size_file_extra);
4124
0
        poVirtualHandle->Seek(file_info.file_extra_abs_offset, SEEK_SET);
4125
0
        if (poVirtualHandle->Read(&abyExtra[0], abyExtra.size(), 1) == 1)
4126
0
        {
4127
0
            size_t nPos = 0;
4128
0
            while (nPos + 2 * sizeof(uint16_t) <= abyExtra.size())
4129
0
            {
4130
0
                uint16_t nId;
4131
0
                memcpy(&nId, &abyExtra[nPos], sizeof(uint16_t));
4132
0
                nPos += sizeof(uint16_t);
4133
0
                CPL_LSBPTR16(&nId);
4134
0
                uint16_t nSize;
4135
0
                memcpy(&nSize, &abyExtra[nPos], sizeof(uint16_t));
4136
0
                nPos += sizeof(uint16_t);
4137
0
                CPL_LSBPTR16(&nSize);
4138
0
                if (nId == 0x564b && nPos + nSize <= abyExtra.size())  // "KV"
4139
0
                {
4140
0
                    if (nSize >= strlen("KeyValuePairs") + 1 &&
4141
0
                        memcmp(&abyExtra[nPos], "KeyValuePairs",
4142
0
                               strlen("KeyValuePairs")) == 0)
4143
0
                    {
4144
0
                        int nPos2 = static_cast<int>(strlen("KeyValuePairs"));
4145
0
                        int nKVPairs = abyExtra[nPos + nPos2];
4146
0
                        nPos2++;
4147
0
                        for (int iKV = 0; iKV < nKVPairs; ++iKV)
4148
0
                        {
4149
0
                            if (nPos2 + sizeof(uint16_t) > nSize)
4150
0
                                break;
4151
0
                            uint16_t nKeyLen;
4152
0
                            memcpy(&nKeyLen, &abyExtra[nPos + nPos2],
4153
0
                                   sizeof(uint16_t));
4154
0
                            nPos2 += sizeof(uint16_t);
4155
0
                            CPL_LSBPTR16(&nKeyLen);
4156
0
                            if (nPos2 + nKeyLen > nSize)
4157
0
                                break;
4158
0
                            std::string osKey;
4159
0
                            osKey.resize(nKeyLen);
4160
0
                            memcpy(&osKey[0], &abyExtra[nPos + nPos2], nKeyLen);
4161
0
                            nPos2 += nKeyLen;
4162
4163
0
                            if (nPos2 + sizeof(uint16_t) > nSize)
4164
0
                                break;
4165
0
                            uint16_t nValLen;
4166
0
                            memcpy(&nValLen, &abyExtra[nPos + nPos2],
4167
0
                                   sizeof(uint16_t));
4168
0
                            nPos2 += sizeof(uint16_t);
4169
0
                            CPL_LSBPTR16(&nValLen);
4170
0
                            if (nPos2 + nValLen > nSize)
4171
0
                                break;
4172
0
                            std::string osVal;
4173
0
                            osVal.resize(nValLen);
4174
0
                            memcpy(&osVal[0], &abyExtra[nPos + nPos2], nValLen);
4175
0
                            nPos2 += nValLen;
4176
4177
0
                            info.oMapProperties[osKey] = std::move(osVal);
4178
0
                        }
4179
0
                    }
4180
0
                }
4181
0
                nPos += nSize;
4182
0
            }
4183
0
        }
4184
0
    }
4185
4186
0
    info.nCRC = file_info.crc;
4187
0
    info.nCompressionMethod = static_cast<int>(file_info.compression_method);
4188
0
    info.nUncompressedSize = static_cast<uint64_t>(file_info.uncompressed_size);
4189
0
    info.nCompressedSize = static_cast<uint64_t>(file_info.compressed_size);
4190
4191
    // Try to locate .sozip.idx file
4192
0
    uLong64 local_header_pos;
4193
0
    cpl_unzGetLocalHeaderPos(unzF, &local_header_pos);
4194
0
    local_header_pos = info.nStartDataStream + file_info.compressed_size;
4195
0
    unz_file_info file_info2;
4196
0
    std::string osAuxName;
4197
0
    osAuxName.resize(1024);
4198
0
    uLong64 indexPos;
4199
0
    if (file_info.compression_method == 8 &&
4200
0
        cpl_unzCurrentFileInfoFromLocalHeader(
4201
0
            unzF, local_header_pos, &file_info2, &osAuxName[0],
4202
0
            osAuxName.size(), &indexPos) == UNZ_OK)
4203
0
    {
4204
0
        osAuxName.resize(strlen(osAuxName.c_str()));
4205
0
        if (osAuxName.find(".sozip.idx") != std::string::npos)
4206
0
        {
4207
0
            info.bSOZipIndexFound = true;
4208
0
            info.nSOZIPStartData = indexPos;
4209
0
            poVirtualHandle->Seek(indexPos, SEEK_SET);
4210
0
            uint32_t nVersion = 0;
4211
0
            poVirtualHandle->Read(&nVersion, sizeof(nVersion), 1);
4212
0
            CPL_LSBPTR32(&nVersion);
4213
0
            uint32_t nToSkip = 0;
4214
0
            poVirtualHandle->Read(&nToSkip, sizeof(nToSkip), 1);
4215
0
            CPL_LSBPTR32(&nToSkip);
4216
0
            uint32_t nChunkSize = 0;
4217
0
            poVirtualHandle->Read(&nChunkSize, sizeof(nChunkSize), 1);
4218
0
            CPL_LSBPTR32(&nChunkSize);
4219
0
            uint32_t nOffsetSize = 0;
4220
0
            poVirtualHandle->Read(&nOffsetSize, sizeof(nOffsetSize), 1);
4221
0
            CPL_LSBPTR32(&nOffsetSize);
4222
0
            uint64_t nUncompressedSize = 0;
4223
0
            poVirtualHandle->Read(&nUncompressedSize, sizeof(nUncompressedSize),
4224
0
                                  1);
4225
0
            CPL_LSBPTR64(&nUncompressedSize);
4226
0
            uint64_t nCompressedSize = 0;
4227
0
            poVirtualHandle->Read(&nCompressedSize, sizeof(nCompressedSize), 1);
4228
0
            CPL_LSBPTR64(&nCompressedSize);
4229
4230
0
            info.nSOZIPVersion = nVersion;
4231
0
            info.nSOZIPToSkip = nToSkip;
4232
0
            info.nSOZIPChunkSize = nChunkSize;
4233
0
            info.nSOZIPOffsetSize = nOffsetSize;
4234
4235
0
            bool bValid = true;
4236
0
            if (nVersion != 1)
4237
0
            {
4238
0
                CPLDebug("SOZIP", "version = %u, expected 1", nVersion);
4239
0
                bValid = false;
4240
0
            }
4241
0
            if (nCompressedSize != file_info.compressed_size)
4242
0
            {
4243
0
                CPLDebug("SOZIP",
4244
0
                         "compressedSize field inconsistent with file");
4245
0
                bValid = false;
4246
0
            }
4247
0
            if (nUncompressedSize != file_info.uncompressed_size)
4248
0
            {
4249
0
                CPLDebug("SOZIP",
4250
0
                         "uncompressedSize field inconsistent with file");
4251
0
                bValid = false;
4252
0
            }
4253
0
            if (!(nChunkSize > 0 && nChunkSize < 100 * 1024 * 1024))
4254
0
            {
4255
0
                CPLDebug("SOZIP", "invalid chunkSize = %u", nChunkSize);
4256
0
                bValid = false;
4257
0
            }
4258
0
            if (nOffsetSize != 8)
4259
0
            {
4260
0
                CPLDebug("SOZIP", "invalid offsetSize = %u", nOffsetSize);
4261
0
                bValid = false;
4262
0
            }
4263
0
            if (file_info2.compression_method != 0)
4264
0
            {
4265
0
                CPLDebug("SOZIP", "unexpected compression_method = %u",
4266
0
                         static_cast<unsigned>(file_info2.compression_method));
4267
0
                bValid = false;
4268
0
            }
4269
0
            if (bValid)
4270
0
            {
4271
0
                const auto nExpectedIndexSize =
4272
0
                    32 + static_cast<uint64_t>(nToSkip) +
4273
0
                    ((nUncompressedSize - 1) / nChunkSize) * nOffsetSize;
4274
0
                if (nExpectedIndexSize != file_info2.uncompressed_size)
4275
0
                {
4276
0
                    CPLDebug("SOZIP", "invalid file size for index");
4277
0
                    bValid = false;
4278
0
                }
4279
0
            }
4280
0
            if (bValid)
4281
0
            {
4282
0
                info.bSOZipIndexValid = true;
4283
0
                CPLDebug("SOZIP", "Found valid SOZIP index: %s",
4284
0
                         osAuxName.c_str());
4285
0
            }
4286
0
            else
4287
0
            {
4288
0
                CPLDebug("SOZIP", "Found *invalid* SOZIP index: %s",
4289
0
                         osAuxName.c_str());
4290
0
            }
4291
0
        }
4292
0
    }
4293
4294
0
    cpl_unzCloseCurrentFile(unzF);
4295
4296
0
    delete poReader;
4297
4298
0
    info.poVirtualHandle.reset(poVirtualHandle);
4299
4300
0
    return true;
4301
0
}
4302
4303
/************************************************************************/
4304
/*                                 Open()                               */
4305
/************************************************************************/
4306
4307
VSIVirtualHandle *VSIZipFilesystemHandler::Open(const char *pszFilename,
4308
                                                const char *pszAccess,
4309
                                                bool /* bSetError */,
4310
                                                CSLConstList /* papszOptions */)
4311
0
{
4312
4313
0
    if (strchr(pszAccess, 'w') != nullptr)
4314
0
    {
4315
0
        return OpenForWrite(pszFilename, pszAccess);
4316
0
    }
4317
4318
0
    if (strchr(pszAccess, '+') != nullptr)
4319
0
    {
4320
0
        CPLError(CE_Failure, CPLE_AppDefined,
4321
0
                 "Read-write random access not supported for /vsizip");
4322
0
        return nullptr;
4323
0
    }
4324
4325
0
    VSIFileInZipInfo info;
4326
0
    if (!GetFileInfo(pszFilename, info))
4327
0
        return nullptr;
4328
4329
0
#ifdef ENABLE_DEFLATE64
4330
0
    if (info.nCompressionMethod == 9)
4331
0
    {
4332
0
        auto poGZIPHandle = new VSIDeflate64Handle(
4333
0
            info.poVirtualHandle.release(), nullptr, info.nStartDataStream,
4334
0
            info.nCompressedSize, info.nUncompressedSize, info.nCRC);
4335
0
        if (!(poGZIPHandle->IsInitOK()))
4336
0
        {
4337
0
            delete poGZIPHandle;
4338
0
            return nullptr;
4339
0
        }
4340
4341
        // Wrap the VSIGZipHandle inside a buffered reader that will
4342
        // improve dramatically performance when doing small backward
4343
        // seeks.
4344
0
        return VSICreateBufferedReaderHandle(poGZIPHandle);
4345
0
    }
4346
0
    else
4347
0
#endif
4348
0
    {
4349
0
        if (info.bSOZipIndexValid)
4350
0
        {
4351
0
            auto poSOZIPHandle = new VSISOZipHandle(
4352
0
                info.poVirtualHandle.release(), info.nStartDataStream,
4353
0
                info.nCompressedSize, info.nUncompressedSize,
4354
0
                info.nSOZIPStartData, info.nSOZIPToSkip, info.nSOZIPChunkSize);
4355
0
            if (!poSOZIPHandle->IsOK())
4356
0
            {
4357
0
                delete poSOZIPHandle;
4358
0
                return nullptr;
4359
0
            }
4360
0
            return VSICreateCachedFile(poSOZIPHandle, info.nSOZIPChunkSize, 0);
4361
0
        }
4362
4363
0
        VSIGZipHandle *poGZIPHandle = new VSIGZipHandle(
4364
0
            info.poVirtualHandle.release(), nullptr, info.nStartDataStream,
4365
0
            info.nCompressedSize, info.nUncompressedSize, info.nCRC,
4366
0
            info.nCompressionMethod == 0);
4367
0
        if (!(poGZIPHandle->IsInitOK()))
4368
0
        {
4369
0
            delete poGZIPHandle;
4370
0
            return nullptr;
4371
0
        }
4372
4373
        // Wrap the VSIGZipHandle inside a buffered reader that will
4374
        // improve dramatically performance when doing small backward
4375
        // seeks.
4376
0
        return VSICreateBufferedReaderHandle(poGZIPHandle);
4377
0
    }
4378
0
}
4379
4380
/************************************************************************/
4381
/*                          GetFileMetadata()                           */
4382
/************************************************************************/
4383
4384
char **VSIZipFilesystemHandler::GetFileMetadata(const char *pszFilename,
4385
                                                const char *pszDomain,
4386
                                                CSLConstList /*papszOptions*/)
4387
0
{
4388
0
    VSIFileInZipInfo info;
4389
0
    if (!GetFileInfo(pszFilename, info))
4390
0
        return nullptr;
4391
4392
0
    if (!pszDomain)
4393
0
    {
4394
0
        CPLStringList aosMetadata;
4395
0
        for (const auto &kv : info.oMapProperties)
4396
0
        {
4397
0
            aosMetadata.AddNameValue(kv.first.c_str(), kv.second.c_str());
4398
0
        }
4399
0
        return aosMetadata.StealList();
4400
0
    }
4401
0
    else if (EQUAL(pszDomain, "ZIP"))
4402
0
    {
4403
0
        CPLStringList aosMetadata;
4404
0
        aosMetadata.SetNameValue(
4405
0
            "START_DATA_OFFSET",
4406
0
            CPLSPrintf(CPL_FRMT_GUIB,
4407
0
                       static_cast<GUIntBig>(info.nStartDataStream)));
4408
4409
0
        if (info.nCompressionMethod == 0)
4410
0
            aosMetadata.SetNameValue("COMPRESSION_METHOD", "0 (STORED)");
4411
0
        else if (info.nCompressionMethod == 8)
4412
0
            aosMetadata.SetNameValue("COMPRESSION_METHOD", "8 (DEFLATE)");
4413
0
        else
4414
0
        {
4415
0
            aosMetadata.SetNameValue("COMPRESSION_METHOD",
4416
0
                                     CPLSPrintf("%d", info.nCompressionMethod));
4417
0
        }
4418
0
        aosMetadata.SetNameValue(
4419
0
            "COMPRESSED_SIZE",
4420
0
            CPLSPrintf(CPL_FRMT_GUIB,
4421
0
                       static_cast<GUIntBig>(info.nCompressedSize)));
4422
0
        aosMetadata.SetNameValue(
4423
0
            "UNCOMPRESSED_SIZE",
4424
0
            CPLSPrintf(CPL_FRMT_GUIB,
4425
0
                       static_cast<GUIntBig>(info.nUncompressedSize)));
4426
4427
0
        if (info.bSOZipIndexFound)
4428
0
        {
4429
0
            aosMetadata.SetNameValue("SOZIP_FOUND", "YES");
4430
4431
0
            aosMetadata.SetNameValue("SOZIP_VERSION",
4432
0
                                     CPLSPrintf("%u", info.nSOZIPVersion));
4433
4434
0
            aosMetadata.SetNameValue("SOZIP_OFFSET_SIZE",
4435
0
                                     CPLSPrintf("%u", info.nSOZIPOffsetSize));
4436
4437
0
            aosMetadata.SetNameValue("SOZIP_CHUNK_SIZE",
4438
0
                                     CPLSPrintf("%u", info.nSOZIPChunkSize));
4439
4440
0
            aosMetadata.SetNameValue(
4441
0
                "SOZIP_START_DATA_OFFSET",
4442
0
                CPLSPrintf(CPL_FRMT_GUIB,
4443
0
                           static_cast<GUIntBig>(info.nSOZIPStartData)));
4444
4445
0
            if (info.bSOZipIndexValid)
4446
0
            {
4447
0
                aosMetadata.SetNameValue("SOZIP_VALID", "YES");
4448
0
            }
4449
0
        }
4450
4451
0
        return aosMetadata.StealList();
4452
0
    }
4453
0
    return nullptr;
4454
0
}
4455
4456
/************************************************************************/
4457
/*                                Mkdir()                               */
4458
/************************************************************************/
4459
4460
int VSIZipFilesystemHandler::Mkdir(const char *pszDirname, long /* nMode */)
4461
0
{
4462
0
    CPLString osDirname = pszDirname;
4463
0
    if (!osDirname.empty() && osDirname.back() != '/')
4464
0
        osDirname += "/";
4465
0
    VSIVirtualHandle *poZIPHandle = OpenForWrite(osDirname, "wb");
4466
0
    if (poZIPHandle == nullptr)
4467
0
        return -1;
4468
0
    delete poZIPHandle;
4469
0
    return 0;
4470
0
}
4471
4472
/************************************************************************/
4473
/*                               ReadDirEx()                            */
4474
/************************************************************************/
4475
4476
char **VSIZipFilesystemHandler::ReadDirEx(const char *pszDirname, int nMaxFiles)
4477
0
{
4478
0
    CPLString osInArchiveSubDir;
4479
0
    char *zipFilename = SplitFilename(pszDirname, osInArchiveSubDir, TRUE);
4480
0
    if (zipFilename == nullptr)
4481
0
        return nullptr;
4482
4483
0
    {
4484
0
        CPLMutexHolder oHolder(&hMutex);
4485
4486
0
        if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end())
4487
0
        {
4488
0
            CPLError(CE_Failure, CPLE_AppDefined,
4489
0
                     "Cannot read a zip file being written");
4490
0
            CPLFree(zipFilename);
4491
0
            return nullptr;
4492
0
        }
4493
0
    }
4494
0
    CPLFree(zipFilename);
4495
4496
0
    return VSIArchiveFilesystemHandler::ReadDirEx(pszDirname, nMaxFiles);
4497
0
}
4498
4499
/************************************************************************/
4500
/*                                 Stat()                               */
4501
/************************************************************************/
4502
4503
int VSIZipFilesystemHandler::Stat(const char *pszFilename,
4504
                                  VSIStatBufL *pStatBuf, int nFlags)
4505
0
{
4506
0
    CPLString osInArchiveSubDir;
4507
4508
0
    memset(pStatBuf, 0, sizeof(VSIStatBufL));
4509
4510
0
    char *zipFilename = SplitFilename(pszFilename, osInArchiveSubDir, TRUE);
4511
0
    if (zipFilename == nullptr)
4512
0
        return -1;
4513
4514
0
    {
4515
0
        CPLMutexHolder oHolder(&hMutex);
4516
4517
0
        if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end())
4518
0
        {
4519
0
            CPLError(CE_Failure, CPLE_AppDefined,
4520
0
                     "Cannot read a zip file being written");
4521
0
            CPLFree(zipFilename);
4522
0
            return -1;
4523
0
        }
4524
0
    }
4525
0
    CPLFree(zipFilename);
4526
4527
0
    return VSIArchiveFilesystemHandler::Stat(pszFilename, pStatBuf, nFlags);
4528
0
}
4529
4530
/************************************************************************/
4531
/*                             RemoveFromMap()                           */
4532
/************************************************************************/
4533
4534
void VSIZipFilesystemHandler::RemoveFromMap(VSIZipWriteHandle *poHandle)
4535
0
{
4536
0
    CPLMutexHolder oHolder(&hMutex);
4537
4538
0
    for (std::map<CPLString, VSIZipWriteHandle *>::iterator iter =
4539
0
             oMapZipWriteHandles.begin();
4540
0
         iter != oMapZipWriteHandles.end(); ++iter)
4541
0
    {
4542
0
        if (iter->second == poHandle)
4543
0
        {
4544
0
            oMapZipWriteHandles.erase(iter);
4545
0
            break;
4546
0
        }
4547
0
    }
4548
0
}
4549
4550
/************************************************************************/
4551
/*                             OpenForWrite()                           */
4552
/************************************************************************/
4553
4554
VSIVirtualHandle *VSIZipFilesystemHandler::OpenForWrite(const char *pszFilename,
4555
                                                        const char *pszAccess)
4556
0
{
4557
0
    CPLMutexHolder oHolder(&hMutex);
4558
0
    return OpenForWrite_unlocked(pszFilename, pszAccess);
4559
0
}
4560
4561
VSIVirtualHandle *
4562
VSIZipFilesystemHandler::OpenForWrite_unlocked(const char *pszFilename,
4563
                                               const char *pszAccess)
4564
0
{
4565
0
    CPLString osZipInFileName;
4566
4567
0
    char *zipFilename = SplitFilename(pszFilename, osZipInFileName, FALSE);
4568
0
    if (zipFilename == nullptr)
4569
0
        return nullptr;
4570
0
    CPLString osZipFilename = zipFilename;
4571
0
    CPLFree(zipFilename);
4572
0
    zipFilename = nullptr;
4573
4574
    // Invalidate cached file list.
4575
0
    std::map<CPLString, VSIArchiveContent *>::iterator iter =
4576
0
        oFileList.find(osZipFilename);
4577
0
    if (iter != oFileList.end())
4578
0
    {
4579
0
        delete iter->second;
4580
4581
0
        oFileList.erase(iter);
4582
0
    }
4583
4584
0
    if (oMapZipWriteHandles.find(osZipFilename) != oMapZipWriteHandles.end())
4585
0
    {
4586
0
        if (strchr(pszAccess, '+') != nullptr)
4587
0
        {
4588
0
            CPLError(
4589
0
                CE_Failure, CPLE_AppDefined,
4590
0
                "Random access not supported for writable file in /vsizip");
4591
0
            return nullptr;
4592
0
        }
4593
4594
0
        VSIZipWriteHandle *poZIPHandle = oMapZipWriteHandles[osZipFilename];
4595
4596
0
        if (poZIPHandle->GetChildInWriting() != nullptr)
4597
0
        {
4598
0
            CPLError(CE_Failure, CPLE_AppDefined,
4599
0
                     "Cannot create %s while another file is being "
4600
0
                     "written in the .zip",
4601
0
                     osZipInFileName.c_str());
4602
0
            return nullptr;
4603
0
        }
4604
4605
0
        poZIPHandle->StopCurrentFile();
4606
4607
        // Re-add path separator when creating directories.
4608
0
        char chLastChar = pszFilename[strlen(pszFilename) - 1];
4609
0
        if (chLastChar == '/' || chLastChar == '\\')
4610
0
            osZipInFileName += chLastChar;
4611
4612
0
        if (CPLCreateFileInZip(poZIPHandle->GetHandle(), osZipInFileName,
4613
0
                               nullptr) != CE_None)
4614
0
            return nullptr;
4615
4616
0
        VSIZipWriteHandle *poChildHandle =
4617
0
            new VSIZipWriteHandle(this, nullptr, poZIPHandle);
4618
4619
0
        poZIPHandle->StartNewFile(poChildHandle);
4620
4621
0
        return poChildHandle;
4622
0
    }
4623
0
    else
4624
0
    {
4625
0
        char **papszOptions = nullptr;
4626
0
        if ((strchr(pszAccess, '+') && osZipInFileName.empty()) ||
4627
0
            !osZipInFileName.empty())
4628
0
        {
4629
0
            VSIStatBufL sBuf;
4630
0
            if (VSIStatExL(osZipFilename, &sBuf, VSI_STAT_EXISTS_FLAG) == 0)
4631
0
                papszOptions = CSLAddNameValue(papszOptions, "APPEND", "TRUE");
4632
0
        }
4633
4634
0
        void *hZIP = CPLCreateZip(osZipFilename, papszOptions);
4635
0
        CSLDestroy(papszOptions);
4636
4637
0
        if (hZIP == nullptr)
4638
0
            return nullptr;
4639
4640
0
        auto poHandle = new VSIZipWriteHandle(this, hZIP, nullptr);
4641
0
        oMapZipWriteHandles[osZipFilename] = poHandle;
4642
4643
0
        if (!osZipInFileName.empty())
4644
0
        {
4645
0
            VSIZipWriteHandle *poRes = reinterpret_cast<VSIZipWriteHandle *>(
4646
0
                OpenForWrite_unlocked(pszFilename, pszAccess));
4647
0
            if (poRes == nullptr)
4648
0
            {
4649
0
                delete poHandle;
4650
0
                oMapZipWriteHandles.erase(osZipFilename);
4651
0
                return nullptr;
4652
0
            }
4653
4654
0
            poRes->SetAutoDeleteParent();
4655
4656
0
            return poRes;
4657
0
        }
4658
4659
0
        return poHandle;
4660
0
    }
4661
0
}
4662
4663
/************************************************************************/
4664
/*                           GetOptions()                               */
4665
/************************************************************************/
4666
4667
const char *VSIZipFilesystemHandler::GetOptions()
4668
0
{
4669
0
    return "<Options>"
4670
0
           "  <Option name='GDAL_NUM_THREADS' type='string' "
4671
0
           "description='Number of threads for compression. Either a integer "
4672
0
           "or ALL_CPUS'/>"
4673
0
           "  <Option name='CPL_VSIL_DEFLATE_CHUNK_SIZE' type='string' "
4674
0
           "description='Chunk of uncompressed data for parallelization. "
4675
0
           "Use K(ilobytes) or M(egabytes) suffix' default='1M'/>"
4676
0
           "</Options>";
4677
0
}
4678
4679
/************************************************************************/
4680
/*                           CopyFile()                                 */
4681
/************************************************************************/
4682
4683
int VSIZipFilesystemHandler::CopyFile(const char *pszSource,
4684
                                      const char *pszTarget, VSILFILE *fpSource,
4685
                                      vsi_l_offset /* nSourceSize */,
4686
                                      CSLConstList papszOptions,
4687
                                      GDALProgressFunc pProgressFunc,
4688
                                      void *pProgressData)
4689
0
{
4690
0
    CPLString osZipInFileName;
4691
4692
0
    char *zipFilename = SplitFilename(pszTarget, osZipInFileName, FALSE);
4693
0
    if (zipFilename == nullptr)
4694
0
        return -1;
4695
0
    CPLString osZipFilename = zipFilename;
4696
0
    CPLFree(zipFilename);
4697
0
    zipFilename = nullptr;
4698
0
    if (osZipInFileName.empty())
4699
0
    {
4700
0
        CPLError(CE_Failure, CPLE_AppDefined,
4701
0
                 "Target filename should be of the form "
4702
0
                 "/vsizip/path_to.zip/filename_within_zip");
4703
0
        return -1;
4704
0
    }
4705
4706
    // Invalidate cached file list.
4707
0
    auto oIterFileList = oFileList.find(osZipFilename);
4708
0
    if (oIterFileList != oFileList.end())
4709
0
    {
4710
0
        delete oIterFileList->second;
4711
4712
0
        oFileList.erase(oIterFileList);
4713
0
    }
4714
4715
0
    const auto oIter = oMapZipWriteHandles.find(osZipFilename);
4716
0
    if (oIter != oMapZipWriteHandles.end())
4717
0
    {
4718
0
        VSIZipWriteHandle *poZIPHandle = oIter->second;
4719
4720
0
        if (poZIPHandle->GetChildInWriting() != nullptr)
4721
0
        {
4722
0
            CPLError(CE_Failure, CPLE_AppDefined,
4723
0
                     "Cannot create %s while another file is being "
4724
0
                     "written in the .zip",
4725
0
                     osZipInFileName.c_str());
4726
0
            return -1;
4727
0
        }
4728
4729
0
        if (CPLAddFileInZip(poZIPHandle->GetHandle(), osZipInFileName.c_str(),
4730
0
                            pszSource, fpSource, papszOptions, pProgressFunc,
4731
0
                            pProgressData) != CE_None)
4732
0
        {
4733
0
            return -1;
4734
0
        }
4735
0
        return 0;
4736
0
    }
4737
0
    else
4738
0
    {
4739
0
        CPLStringList aosOptionsCreateZip;
4740
0
        VSIStatBufL sBuf;
4741
0
        if (VSIStatExL(osZipFilename, &sBuf, VSI_STAT_EXISTS_FLAG) == 0)
4742
0
            aosOptionsCreateZip.SetNameValue("APPEND", "TRUE");
4743
4744
0
        void *hZIP = CPLCreateZip(osZipFilename, aosOptionsCreateZip.List());
4745
4746
0
        if (hZIP == nullptr)
4747
0
            return -1;
4748
4749
0
        if (CPLAddFileInZip(hZIP, osZipInFileName.c_str(), pszSource, fpSource,
4750
0
                            papszOptions, pProgressFunc,
4751
0
                            pProgressData) != CE_None)
4752
0
        {
4753
0
            CPLCloseZip(hZIP);
4754
0
            return -1;
4755
0
        }
4756
0
        CPLCloseZip(hZIP);
4757
0
        return 0;
4758
0
    }
4759
0
}
4760
4761
/************************************************************************/
4762
/*                          VSIZipWriteHandle()                         */
4763
/************************************************************************/
4764
4765
VSIZipWriteHandle::VSIZipWriteHandle(VSIZipFilesystemHandler *poFS, void *hZIP,
4766
                                     VSIZipWriteHandle *poParent)
4767
0
    : m_poFS(poFS), m_hZIP(hZIP), m_poParent(poParent)
4768
0
{
4769
0
}
4770
4771
/************************************************************************/
4772
/*                         ~VSIZipWriteHandle()                         */
4773
/************************************************************************/
4774
4775
VSIZipWriteHandle::~VSIZipWriteHandle()
4776
0
{
4777
0
    VSIZipWriteHandle::Close();
4778
0
}
4779
4780
/************************************************************************/
4781
/*                               Seek()                                 */
4782
/************************************************************************/
4783
4784
int VSIZipWriteHandle::Seek(vsi_l_offset nOffset, int nWhence)
4785
0
{
4786
0
    if (nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR))
4787
0
        return 0;
4788
0
    if (nOffset == nCurOffset && nWhence == SEEK_SET)
4789
0
        return 0;
4790
4791
0
    CPLError(CE_Failure, CPLE_NotSupported,
4792
0
             "VSIFSeekL() is not supported on writable Zip files");
4793
0
    return -1;
4794
0
}
4795
4796
/************************************************************************/
4797
/*                               Tell()                                 */
4798
/************************************************************************/
4799
4800
vsi_l_offset VSIZipWriteHandle::Tell()
4801
0
{
4802
0
    return nCurOffset;
4803
0
}
4804
4805
/************************************************************************/
4806
/*                               Read()                                 */
4807
/************************************************************************/
4808
4809
size_t VSIZipWriteHandle::Read(void * /* pBuffer */, size_t /* nSize */,
4810
                               size_t /* nMemb */)
4811
0
{
4812
0
    CPLError(CE_Failure, CPLE_NotSupported,
4813
0
             "VSIFReadL() is not supported on writable Zip files");
4814
0
    return 0;
4815
0
}
4816
4817
/************************************************************************/
4818
/*                               Write()                                 */
4819
/************************************************************************/
4820
4821
size_t VSIZipWriteHandle::Write(const void *pBuffer, size_t nSize, size_t nMemb)
4822
0
{
4823
0
    if (m_poParent == nullptr)
4824
0
    {
4825
0
        CPLError(CE_Failure, CPLE_NotSupported,
4826
0
                 "VSIFWriteL() is not supported on "
4827
0
                 "main Zip file or closed subfiles");
4828
0
        return 0;
4829
0
    }
4830
4831
0
    const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer);
4832
0
    size_t nBytesToWrite = nSize * nMemb;
4833
0
    size_t nWritten = 0;
4834
0
    while (nWritten < nBytesToWrite)
4835
0
    {
4836
0
        int nToWrite = static_cast<int>(
4837
0
            std::min(static_cast<size_t>(INT_MAX), nBytesToWrite));
4838
0
        if (CPLWriteFileInZip(m_poParent->m_hZIP, pabyBuffer, nToWrite) !=
4839
0
            CE_None)
4840
0
            return 0;
4841
0
        nWritten += nToWrite;
4842
0
        pabyBuffer += nToWrite;
4843
0
    }
4844
4845
0
    nCurOffset += nSize * nMemb;
4846
4847
0
    return nMemb;
4848
0
}
4849
4850
/************************************************************************/
4851
/*                               Flush()                                */
4852
/************************************************************************/
4853
4854
int VSIZipWriteHandle::Flush()
4855
0
{
4856
    /*CPLError(CE_Failure, CPLE_NotSupported,
4857
             "VSIFFlushL() is not supported on writable Zip files");*/
4858
0
    return 0;
4859
0
}
4860
4861
/************************************************************************/
4862
/*                               Close()                                */
4863
/************************************************************************/
4864
4865
int VSIZipWriteHandle::Close()
4866
0
{
4867
0
    int nRet = 0;
4868
0
    if (m_poParent)
4869
0
    {
4870
0
        CPLCloseFileInZip(m_poParent->m_hZIP);
4871
0
        m_poParent->poChildInWriting = nullptr;
4872
0
        if (bAutoDeleteParent)
4873
0
        {
4874
0
            if (m_poParent->Close() != 0)
4875
0
                nRet = -1;
4876
0
            delete m_poParent;
4877
0
        }
4878
0
        m_poParent = nullptr;
4879
0
    }
4880
0
    if (poChildInWriting)
4881
0
    {
4882
0
        if (poChildInWriting->Close() != 0)
4883
0
            nRet = -1;
4884
0
        poChildInWriting = nullptr;
4885
0
    }
4886
0
    if (m_hZIP)
4887
0
    {
4888
0
        if (CPLCloseZip(m_hZIP) != CE_None)
4889
0
            nRet = -1;
4890
0
        m_hZIP = nullptr;
4891
4892
0
        m_poFS->RemoveFromMap(this);
4893
0
    }
4894
4895
0
    return nRet;
4896
0
}
4897
4898
/************************************************************************/
4899
/*                           StopCurrentFile()                          */
4900
/************************************************************************/
4901
4902
void VSIZipWriteHandle::StopCurrentFile()
4903
0
{
4904
0
    if (poChildInWriting)
4905
0
        poChildInWriting->Close();
4906
0
    poChildInWriting = nullptr;
4907
0
}
4908
4909
/************************************************************************/
4910
/*                           StartNewFile()                             */
4911
/************************************************************************/
4912
4913
void VSIZipWriteHandle::StartNewFile(VSIZipWriteHandle *poSubFile)
4914
0
{
4915
0
    poChildInWriting = poSubFile;
4916
0
}
4917
4918
//! @endcond
4919
4920
/************************************************************************/
4921
/*                    VSIInstallZipFileHandler()                        */
4922
/************************************************************************/
4923
4924
/*!
4925
 \brief Install ZIP file system handler.
4926
4927
 A special file handler is installed that allows reading on-the-fly in ZIP
4928
 (.zip) archives.
4929
4930
 All portions of the file system underneath the base path "/vsizip/" will be
4931
 handled by this driver.
4932
4933
 \verbatim embed:rst
4934
 See :ref:`/vsizip/ documentation <vsizip>`
4935
 \endverbatim
4936
4937
 @since GDAL 1.6.0
4938
 */
4939
4940
void VSIInstallZipFileHandler()
4941
1
{
4942
1
    VSIFileManager::InstallHandler("/vsizip/", new VSIZipFilesystemHandler());
4943
1
}
4944
4945
/************************************************************************/
4946
/*                         CPLZLibDeflate()                             */
4947
/************************************************************************/
4948
4949
/**
4950
 * \brief Compress a buffer with ZLib compression.
4951
 *
4952
 * @param ptr input buffer.
4953
 * @param nBytes size of input buffer in bytes.
4954
 * @param nLevel ZLib compression level (-1 for default).
4955
 * @param outptr output buffer, or NULL to let the function allocate it.
4956
 * @param nOutAvailableBytes size of output buffer if provided, or ignored.
4957
 * @param pnOutBytes pointer to a size_t, where to store the size of the
4958
 *                   output buffer.
4959
 *
4960
 * @return the output buffer (to be freed with VSIFree() if not provided)
4961
 *         or NULL in case of error.
4962
 *
4963
 * @since GDAL 1.10.0
4964
 */
4965
4966
void *CPLZLibDeflate(const void *ptr, size_t nBytes, int nLevel, void *outptr,
4967
                     size_t nOutAvailableBytes, size_t *pnOutBytes)
4968
0
{
4969
0
    if (pnOutBytes != nullptr)
4970
0
        *pnOutBytes = 0;
4971
4972
0
    size_t nTmpSize = 0;
4973
0
    void *pTmp;
4974
#ifdef HAVE_LIBDEFLATE
4975
    struct libdeflate_compressor *enc =
4976
        libdeflate_alloc_compressor(nLevel < 0 ? 7 : nLevel);
4977
    if (enc == nullptr)
4978
    {
4979
        return nullptr;
4980
    }
4981
#endif
4982
0
    if (outptr == nullptr)
4983
0
    {
4984
#ifdef HAVE_LIBDEFLATE
4985
        nTmpSize = libdeflate_zlib_compress_bound(enc, nBytes);
4986
#else
4987
0
        nTmpSize = 32 + nBytes * 2;
4988
0
#endif
4989
0
        pTmp = VSIMalloc(nTmpSize);
4990
0
        if (pTmp == nullptr)
4991
0
        {
4992
#ifdef HAVE_LIBDEFLATE
4993
            libdeflate_free_compressor(enc);
4994
#endif
4995
0
            return nullptr;
4996
0
        }
4997
0
    }
4998
0
    else
4999
0
    {
5000
0
        pTmp = outptr;
5001
0
        nTmpSize = nOutAvailableBytes;
5002
0
    }
5003
5004
#ifdef HAVE_LIBDEFLATE
5005
    size_t nCompressedBytes =
5006
        libdeflate_zlib_compress(enc, ptr, nBytes, pTmp, nTmpSize);
5007
    libdeflate_free_compressor(enc);
5008
    if (nCompressedBytes == 0)
5009
    {
5010
        if (pTmp != outptr)
5011
            VSIFree(pTmp);
5012
        return nullptr;
5013
    }
5014
    if (pnOutBytes != nullptr)
5015
        *pnOutBytes = nCompressedBytes;
5016
#else
5017
0
    z_stream strm;
5018
0
    strm.zalloc = nullptr;
5019
0
    strm.zfree = nullptr;
5020
0
    strm.opaque = nullptr;
5021
0
    int ret = deflateInit(&strm, nLevel < 0 ? Z_DEFAULT_COMPRESSION : nLevel);
5022
0
    if (ret != Z_OK)
5023
0
    {
5024
0
        if (pTmp != outptr)
5025
0
            VSIFree(pTmp);
5026
0
        return nullptr;
5027
0
    }
5028
5029
0
    strm.avail_in = static_cast<uInt>(nBytes);
5030
0
    strm.next_in = reinterpret_cast<Bytef *>(const_cast<void *>(ptr));
5031
0
    strm.avail_out = static_cast<uInt>(nTmpSize);
5032
0
    strm.next_out = reinterpret_cast<Bytef *>(pTmp);
5033
0
    ret = deflate(&strm, Z_FINISH);
5034
0
    if (ret != Z_STREAM_END)
5035
0
    {
5036
0
        if (pTmp != outptr)
5037
0
            VSIFree(pTmp);
5038
0
        return nullptr;
5039
0
    }
5040
0
    if (pnOutBytes != nullptr)
5041
0
        *pnOutBytes = nTmpSize - strm.avail_out;
5042
0
    deflateEnd(&strm);
5043
0
#endif
5044
5045
0
    return pTmp;
5046
0
}
5047
5048
/************************************************************************/
5049
/*                         CPLZLibInflate()                             */
5050
/************************************************************************/
5051
5052
/**
5053
 * \brief Uncompress a buffer compressed with ZLib compression.
5054
 *
5055
 * @param ptr input buffer.
5056
 * @param nBytes size of input buffer in bytes.
5057
 * @param outptr output buffer, or NULL to let the function allocate it.
5058
 * @param nOutAvailableBytes size of output buffer if provided, or ignored.
5059
 * @param pnOutBytes pointer to a size_t, where to store the size of the
5060
 *                   output buffer.
5061
 *
5062
 * @return the output buffer (to be freed with VSIFree() if not provided)
5063
 *         or NULL in case of error.
5064
 *
5065
 * @since GDAL 1.10.0
5066
 */
5067
5068
void *CPLZLibInflate(const void *ptr, size_t nBytes, void *outptr,
5069
                     size_t nOutAvailableBytes, size_t *pnOutBytes)
5070
0
{
5071
0
    return CPLZLibInflateEx(ptr, nBytes, outptr, nOutAvailableBytes, false,
5072
0
                            pnOutBytes);
5073
0
}
5074
5075
/************************************************************************/
5076
/*                         CPLZLibInflateEx()                           */
5077
/************************************************************************/
5078
5079
/**
5080
 * \brief Uncompress a buffer compressed with ZLib compression.
5081
 *
5082
 * @param ptr input buffer.
5083
 * @param nBytes size of input buffer in bytes.
5084
 * @param outptr output buffer, or NULL to let the function allocate it.
5085
 * @param nOutAvailableBytes size of output buffer if provided, or ignored.
5086
 * @param bAllowResizeOutptr whether the function is allowed to grow outptr
5087
 *                           (using VSIRealloc) if its initial capacity
5088
 *                           provided by nOutAvailableBytes is not
5089
 *                           large enough. Ignored if outptr is NULL.
5090
 * @param pnOutBytes pointer to a size_t, where to store the size of the
5091
 *                   output buffer.
5092
 *
5093
 * @return the output buffer (to be freed with VSIFree() if not provided)
5094
 *         or NULL in case of error. If bAllowResizeOutptr is set to true,
5095
 *         only the returned pointer should be freed by the caller, as outptr
5096
 *         might have been reallocated or freed.
5097
 *
5098
 * @since GDAL 3.9.0
5099
 */
5100
5101
void *CPLZLibInflateEx(const void *ptr, size_t nBytes, void *outptr,
5102
                       size_t nOutAvailableBytes, bool bAllowResizeOutptr,
5103
                       size_t *pnOutBytes)
5104
0
{
5105
0
    if (pnOutBytes != nullptr)
5106
0
        *pnOutBytes = 0;
5107
0
    char *pszReallocatableBuf = nullptr;
5108
5109
#ifdef HAVE_LIBDEFLATE
5110
    if (outptr)
5111
    {
5112
        struct libdeflate_decompressor *dec = libdeflate_alloc_decompressor();
5113
        if (dec == nullptr)
5114
        {
5115
            if (bAllowResizeOutptr)
5116
                VSIFree(outptr);
5117
            return nullptr;
5118
        }
5119
        enum libdeflate_result res;
5120
        size_t nOutBytes = 0;
5121
        if (nBytes > 2 && static_cast<const GByte *>(ptr)[0] == 0x1F &&
5122
            static_cast<const GByte *>(ptr)[1] == 0x8B)
5123
        {
5124
            res = libdeflate_gzip_decompress(dec, ptr, nBytes, outptr,
5125
                                             nOutAvailableBytes, &nOutBytes);
5126
        }
5127
        else
5128
        {
5129
            res = libdeflate_zlib_decompress(dec, ptr, nBytes, outptr,
5130
                                             nOutAvailableBytes, &nOutBytes);
5131
        }
5132
        if (pnOutBytes)
5133
            *pnOutBytes = nOutBytes;
5134
        libdeflate_free_decompressor(dec);
5135
        if (res == LIBDEFLATE_INSUFFICIENT_SPACE && bAllowResizeOutptr)
5136
        {
5137
            if (nOutAvailableBytes >
5138
                (std::numeric_limits<size_t>::max() - 1) / 2)
5139
            {
5140
                VSIFree(outptr);
5141
                return nullptr;
5142
            }
5143
            size_t nOutBufSize = nOutAvailableBytes * 2;
5144
            pszReallocatableBuf = static_cast<char *>(
5145
                VSI_REALLOC_VERBOSE(outptr, nOutBufSize + 1));
5146
            if (!pszReallocatableBuf)
5147
            {
5148
                VSIFree(outptr);
5149
                return nullptr;
5150
            }
5151
            outptr = nullptr;
5152
            nOutAvailableBytes = nOutBufSize;
5153
        }
5154
        else if (res != LIBDEFLATE_SUCCESS)
5155
        {
5156
            if (bAllowResizeOutptr)
5157
                VSIFree(outptr);
5158
            return nullptr;
5159
        }
5160
        else
5161
        {
5162
            // Nul-terminate if possible.
5163
            if (nOutBytes < nOutAvailableBytes)
5164
            {
5165
                static_cast<char *>(outptr)[nOutBytes] = '\0';
5166
            }
5167
            return outptr;
5168
        }
5169
    }
5170
#endif
5171
5172
0
    z_stream strm;
5173
0
    memset(&strm, 0, sizeof(strm));
5174
0
    strm.zalloc = nullptr;
5175
0
    strm.zfree = nullptr;
5176
0
    strm.opaque = nullptr;
5177
0
    int ret;
5178
    // MAX_WBITS + 32 mode which detects automatically gzip vs zlib
5179
    // encapsulation seems to be broken with
5180
    // /opt/intel/oneapi/intelpython/latest/lib/libz.so.1 from
5181
    // intel/oneapi-basekit Docker image
5182
0
    if (nBytes > 2 && static_cast<const GByte *>(ptr)[0] == 0x1F &&
5183
0
        static_cast<const GByte *>(ptr)[1] == 0x8B)
5184
0
    {
5185
0
        ret = inflateInit2(&strm, MAX_WBITS + 16);  // gzip
5186
0
    }
5187
0
    else
5188
0
    {
5189
0
        ret = inflateInit2(&strm, MAX_WBITS);  // zlib
5190
0
    }
5191
0
    if (ret != Z_OK)
5192
0
    {
5193
0
        if (bAllowResizeOutptr)
5194
0
            VSIFree(outptr);
5195
0
        VSIFree(pszReallocatableBuf);
5196
0
        return nullptr;
5197
0
    }
5198
5199
0
    size_t nOutBufSize = 0;
5200
0
    char *pszOutBuf = nullptr;
5201
5202
#ifdef HAVE_LIBDEFLATE
5203
    if (pszReallocatableBuf)
5204
    {
5205
        pszOutBuf = pszReallocatableBuf;
5206
        nOutBufSize = nOutAvailableBytes;
5207
    }
5208
    else
5209
#endif
5210
0
        if (!outptr)
5211
0
    {
5212
0
        if (nBytes > (std::numeric_limits<size_t>::max() - 1) / 2)
5213
0
        {
5214
0
            inflateEnd(&strm);
5215
0
            return nullptr;
5216
0
        }
5217
0
        nOutBufSize = 2 * nBytes + 1;
5218
0
        pszOutBuf = static_cast<char *>(VSI_MALLOC_VERBOSE(nOutBufSize));
5219
0
        if (pszOutBuf == nullptr)
5220
0
        {
5221
0
            inflateEnd(&strm);
5222
0
            return nullptr;
5223
0
        }
5224
0
        pszReallocatableBuf = pszOutBuf;
5225
0
        bAllowResizeOutptr = true;
5226
0
    }
5227
0
#ifndef HAVE_LIBDEFLATE
5228
0
    else
5229
0
    {
5230
0
        pszOutBuf = static_cast<char *>(outptr);
5231
0
        nOutBufSize = nOutAvailableBytes;
5232
0
        if (bAllowResizeOutptr)
5233
0
            pszReallocatableBuf = pszOutBuf;
5234
0
    }
5235
0
#endif
5236
5237
0
    strm.next_in = static_cast<Bytef *>(const_cast<void *>(ptr));
5238
0
    strm.next_out = reinterpret_cast<Bytef *>(pszOutBuf);
5239
0
    size_t nInBytesRemaining = nBytes;
5240
0
    size_t nOutBytesRemaining = nOutBufSize;
5241
5242
0
    while (true)
5243
0
    {
5244
0
        strm.avail_in = static_cast<uInt>(std::min<size_t>(
5245
0
            nInBytesRemaining, std::numeric_limits<uInt>::max()));
5246
0
        const auto avail_in_before = strm.avail_in;
5247
0
        strm.avail_out = static_cast<uInt>(std::min<size_t>(
5248
0
            nOutBytesRemaining, std::numeric_limits<uInt>::max()));
5249
0
        const auto avail_out_before = strm.avail_out;
5250
0
        ret = inflate(&strm, Z_FINISH);
5251
0
        nInBytesRemaining -= (avail_in_before - strm.avail_in);
5252
0
        nOutBytesRemaining -= (avail_out_before - strm.avail_out);
5253
5254
0
        if (ret == Z_BUF_ERROR && strm.avail_out == 0)
5255
0
        {
5256
#ifdef HAVE_LIBDEFLATE
5257
            CPLAssert(bAllowResizeOutptr);
5258
#else
5259
0
            if (!bAllowResizeOutptr)
5260
0
            {
5261
0
                VSIFree(pszReallocatableBuf);
5262
0
                inflateEnd(&strm);
5263
0
                return nullptr;
5264
0
            }
5265
0
#endif
5266
5267
0
            const size_t nAlreadyWritten = nOutBufSize - nOutBytesRemaining;
5268
0
            if (nOutBufSize > (std::numeric_limits<size_t>::max() - 1) / 2)
5269
0
            {
5270
0
                VSIFree(pszReallocatableBuf);
5271
0
                inflateEnd(&strm);
5272
0
                return nullptr;
5273
0
            }
5274
0
            nOutBufSize = nOutBufSize * 2 + 1;
5275
0
            char *pszNew = static_cast<char *>(
5276
0
                VSI_REALLOC_VERBOSE(pszReallocatableBuf, nOutBufSize));
5277
0
            if (!pszNew)
5278
0
            {
5279
0
                VSIFree(pszReallocatableBuf);
5280
0
                inflateEnd(&strm);
5281
0
                return nullptr;
5282
0
            }
5283
0
            pszOutBuf = pszNew;
5284
0
            pszReallocatableBuf = pszOutBuf;
5285
0
            nOutBytesRemaining = nOutBufSize - nAlreadyWritten;
5286
0
            strm.next_out =
5287
0
                reinterpret_cast<Bytef *>(pszOutBuf + nAlreadyWritten);
5288
0
        }
5289
0
        else if (ret != Z_OK || nInBytesRemaining == 0)
5290
0
            break;
5291
0
    }
5292
5293
0
    if (ret == Z_OK || ret == Z_STREAM_END)
5294
0
    {
5295
0
        size_t nOutBytes = nOutBufSize - nOutBytesRemaining;
5296
        // Nul-terminate if possible.
5297
0
        if (nOutBytes < nOutBufSize)
5298
0
        {
5299
0
            pszOutBuf[nOutBytes] = '\0';
5300
0
        }
5301
0
        inflateEnd(&strm);
5302
0
        if (pnOutBytes != nullptr)
5303
0
            *pnOutBytes = nOutBytes;
5304
0
        return pszOutBuf;
5305
0
    }
5306
0
    else
5307
0
    {
5308
0
        VSIFree(pszReallocatableBuf);
5309
0
        inflateEnd(&strm);
5310
0
        return nullptr;
5311
0
    }
5312
0
}