Coverage Report

Created: 2026-07-25 06:50

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/gdal/apps/gdalalg_vector_partition.cpp
Line
Count
Source
1
/******************************************************************************
2
 *
3
 * Project:  GDAL
4
 * Purpose:  "partition" step of "vector pipeline"
5
 * Author:   Even Rouault <even dot rouault at spatialys.com>
6
 *
7
 ******************************************************************************
8
 * Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com>
9
 *
10
 * SPDX-License-Identifier: MIT
11
 ****************************************************************************/
12
13
#include "gdalalg_vector_partition.h"
14
15
#include "cpl_vsi.h"
16
#include "cpl_mem_cache.h"
17
#include "ogr_p.h"
18
19
#include <algorithm>
20
#include <set>
21
#include <string_view>
22
23
#ifndef _
24
0
#define _(x) (x)
25
#endif
26
27
//! @cond Doxygen_Suppress
28
29
constexpr int DIRECTORY_CREATION_MODE = 0755;
30
31
constexpr const char *NULL_MARKER = "__HIVE_DEFAULT_PARTITION__";
32
33
constexpr const char *DEFAULT_PATTERN_HIVE = "part_%010d";
34
constexpr const char *DEFAULT_PATTERN_FLAT_NO_FIELD = "{LAYER_NAME}_%010d";
35
constexpr const char *DEFAULT_PATTERN_FLAT = "{LAYER_NAME}_{FIELD_VALUE}_%010d";
36
37
constexpr char DIGIT_ZERO = '0';
38
39
/************************************************************************/
40
/*                       GetConstructorOptions()                        */
41
/************************************************************************/
42
43
/* static */
44
GDALVectorPartitionAlgorithm::ConstructorOptions
45
GDALVectorPartitionAlgorithm::GetConstructorOptions(bool standaloneStep)
46
0
{
47
0
    GDALVectorPartitionAlgorithm::ConstructorOptions options;
48
0
    options.SetStandaloneStep(standaloneStep);
49
0
    options.SetAddInputLayerNameArgument(false);
50
0
    options.SetAddDefaultArguments(false);
51
0
    return options;
52
0
}
53
54
/************************************************************************/
55
/*     GDALVectorPartitionAlgorithm::GDALVectorPartitionAlgorithm()     */
56
/************************************************************************/
57
58
GDALVectorPartitionAlgorithm::GDALVectorPartitionAlgorithm(bool standaloneStep)
59
0
    : GDALVectorPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL,
60
0
                                      GetConstructorOptions(standaloneStep))
61
0
{
62
0
    if (standaloneStep)
63
0
    {
64
0
        AddVectorInputArgs(false);
65
0
        AddProgressArg();
66
0
    }
67
0
    else
68
0
    {
69
0
        AddVectorHiddenInputDatasetArg();
70
0
    }
71
72
0
    AddArg(GDAL_ARG_NAME_OUTPUT, 'o', _("Output directory"), &m_output)
73
0
        .SetRequired()
74
0
        .SetIsInput()
75
0
        .SetMinCharCount(1)
76
0
        .SetPositional();
77
78
0
    constexpr const char *OVERWRITE_APPEND_EXCLUSION_GROUP = "overwrite-append";
79
0
    AddOverwriteArg(&m_overwrite)
80
0
        .SetMutualExclusionGroup(OVERWRITE_APPEND_EXCLUSION_GROUP);
81
0
    AddAppendLayerArg(&m_appendLayer)
82
0
        .SetMutualExclusionGroup(OVERWRITE_APPEND_EXCLUSION_GROUP);
83
0
    AddUpdateArg(&m_update).SetHidden();
84
85
0
    AddOutputFormatArg(&m_format, /* bStreamAllowed = */ false,
86
0
                       /* bGDALGAllowed = */ false)
87
0
        .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES,
88
0
                         {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE});
89
0
    AddCreationOptionsArg(&m_creationOptions);
90
0
    AddLayerCreationOptionsArg(&m_layerCreationOptions);
91
92
0
    auto &fieldNameArg = AddArg(
93
0
        "field", 0, _("Attribute or geometry field(s) on which to partition"),
94
0
        &m_fields);
95
0
    SetAutoCompleteFunctionForFieldName(fieldNameArg, nullptr,
96
0
                                        /* attributeFields = */ true,
97
0
                                        /* geometryFields = */ true,
98
0
                                        m_inputDataset);
99
0
    AddArg("scheme", 0, _("Partitioning scheme"), &m_scheme)
100
0
        .SetChoices(SCHEME_HIVE, SCHEME_FLAT)
101
0
        .SetDefault(m_scheme);
102
0
    AddArg("pattern", 0,
103
0
           _("Filename pattern ('part_%010d' for scheme=hive, "
104
0
             "'{LAYER_NAME}_{FIELD_VALUE}_%010d' for scheme=flat)"),
105
0
           &m_pattern)
106
0
        .SetMinCharCount(1)
107
0
        .AddValidationAction(
108
0
            [this]()
109
0
            {
110
0
                if (!m_pattern.empty())
111
0
                {
112
0
                    const auto nPercentPos = m_pattern.find('%');
113
0
                    if (nPercentPos == std::string::npos)
114
0
                    {
115
0
                        ReportError(CE_Failure, CPLE_IllegalArg, "%s",
116
0
                                    "Missing '%' character in pattern");
117
0
                        return false;
118
0
                    }
119
0
                    if (nPercentPos + 1 < m_pattern.size() &&
120
0
                        m_pattern.find('%', nPercentPos + 1) !=
121
0
                            std::string::npos)
122
0
                    {
123
0
                        ReportError(
124
0
                            CE_Failure, CPLE_IllegalArg, "%s",
125
0
                            "A single '%' character is expected in pattern");
126
0
                        return false;
127
0
                    }
128
0
                    bool percentFound = false;
129
0
                    for (size_t i = nPercentPos + 1; i < m_pattern.size(); ++i)
130
0
                    {
131
0
                        if (m_pattern[i] >= DIGIT_ZERO && m_pattern[i] <= '9')
132
0
                        {
133
                            // ok
134
0
                        }
135
0
                        else if (m_pattern[i] == 'd')
136
0
                        {
137
0
                            percentFound = true;
138
0
                            break;
139
0
                        }
140
0
                        else
141
0
                        {
142
0
                            break;
143
0
                        }
144
0
                    }
145
0
                    if (!percentFound)
146
0
                    {
147
0
                        ReportError(
148
0
                            CE_Failure, CPLE_IllegalArg, "%s",
149
0
                            "pattern value must include a single "
150
0
                            "'%[0]?[1-9]?[0]?d' part number specification");
151
0
                        return false;
152
0
                    }
153
0
                    m_partDigitCount =
154
0
                        atoi(m_pattern.c_str() + nPercentPos + 1);
155
0
                    if (m_partDigitCount > 10)
156
0
                    {
157
0
                        ReportError(CE_Failure, CPLE_IllegalArg,
158
0
                                    "Number of digits in part number "
159
0
                                    "specification should be in [1,10] range");
160
0
                        return false;
161
0
                    }
162
0
                    m_partDigitLeadingZeroes =
163
0
                        m_pattern[nPercentPos + 1] == DIGIT_ZERO;
164
0
                }
165
0
                return true;
166
0
            });
167
0
    AddArg("feature-limit", 0, _("Maximum number of features per file"),
168
0
           &m_featureLimit)
169
0
        .SetMinValueExcluded(0);
170
0
    AddArg("max-file-size", 0,
171
0
           _("Maximum file size (MB or GB suffix can be used)"),
172
0
           &m_maxFileSizeStr)
173
0
        .AddValidationAction(
174
0
            [this]()
175
0
            {
176
0
                bool ok;
177
0
                {
178
0
                    CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler);
179
0
                    ok = CPLParseMemorySize(m_maxFileSizeStr.c_str(),
180
0
                                            &m_maxFileSize,
181
0
                                            nullptr) == CE_None &&
182
0
                         m_maxFileSize > 0;
183
0
                }
184
0
                if (!ok)
185
0
                {
186
0
                    ReportError(CE_Failure, CPLE_IllegalArg,
187
0
                                "Invalid value for max-file-size");
188
0
                    return false;
189
0
                }
190
0
                else if (m_maxFileSize < 1024 * 1024)
191
0
                {
192
0
                    ReportError(CE_Failure, CPLE_IllegalArg,
193
0
                                "max-file-size should be at least one MB");
194
0
                    return false;
195
0
                }
196
0
                return true;
197
0
            });
198
0
    AddArg("omit-partitioned-field", 0,
199
0
           _("Whether to omit partitioned fields from target layer definition"),
200
0
           &m_omitPartitionedFields);
201
0
    AddArg("skip-errors", 0, _("Skip errors when writing features"),
202
0
           &m_skipErrors);
203
204
    // Hidden for now
205
206
0
    AddArg("max-cache-size", 0,
207
0
           _("Maximum number of datasets simultaneously opened"),
208
0
           &m_maxCacheSize)
209
0
        .SetMinValueIncluded(0)  // 0 = unlimited
210
0
        .SetDefault(m_maxCacheSize)
211
0
        .SetHidden();
212
213
0
    AddArg("transaction-size", 0,
214
0
           _("Maximum number of features per transaction"), &m_transactionSize)
215
0
        .SetMinValueIncluded(1)
216
0
        .SetDefault(m_transactionSize)
217
0
        .SetHidden();
218
219
0
    AddValidationAction(
220
0
        [this]()
221
0
        {
222
0
            if (m_fields.empty() && m_featureLimit == 0 && m_maxFileSize == 0)
223
0
            {
224
0
                ReportError(
225
0
                    CE_Failure, CPLE_IllegalArg,
226
0
                    "When 'fields' argument is not specified, "
227
0
                    "'feature-limit' and/or 'max-file-size' must be specified");
228
0
                return false;
229
0
            }
230
0
            return true;
231
0
        });
232
0
}
233
234
/************************************************************************/
235
/*                           PercentEncode()                            */
236
/************************************************************************/
237
238
static void PercentEncode(std::string &out, const std::string_view &s)
239
0
{
240
0
    for (unsigned char c : s)
241
0
    {
242
0
        if (c > 32 && c <= 127 && c != ':' && c != '/' && c != '\\' &&
243
0
            c != '>' && c != '%' && c != '=')
244
0
        {
245
0
            out += c;
246
0
        }
247
0
        else
248
0
        {
249
0
            out += CPLSPrintf("%%%02X", c);
250
0
        }
251
0
    }
252
0
}
253
254
static std::string PercentEncode(const std::string_view &s)
255
0
{
256
0
    std::string out;
257
0
    PercentEncode(out, s);
258
0
    return out;
259
0
}
260
261
/************************************************************************/
262
/*                      GetEstimatedFeatureSize()                       */
263
/************************************************************************/
264
265
static size_t GetEstimatedFeatureSize(
266
    const OGRFeature *poFeature, const std::vector<bool> &abPartitionedFields,
267
    const bool omitPartitionedFields,
268
    const std::vector<OGRFieldType> &aeSrcFieldTypes, bool bIsBinary)
269
0
{
270
0
    size_t nSize = 16;
271
0
    const int nFieldCount = poFeature->GetFieldCount();
272
0
    nSize += 4 * nFieldCount;
273
0
    for (int i = 0; i < nFieldCount; ++i)
274
0
    {
275
0
        if (!(omitPartitionedFields && abPartitionedFields[i]))
276
0
        {
277
0
            switch (aeSrcFieldTypes[i])
278
0
            {
279
0
                case OFTInteger:
280
0
                    nSize += bIsBinary ? sizeof(int) : 11;
281
0
                    break;
282
0
                case OFTInteger64:
283
0
                    nSize += bIsBinary ? sizeof(int64_t) : 21;
284
0
                    break;
285
0
                case OFTReal:
286
                    // Decimal representation
287
0
                    nSize += bIsBinary ? sizeof(double) : 15;
288
0
                    break;
289
0
                case OFTString:
290
0
                    nSize += 4 + strlen(poFeature->GetFieldAsStringUnsafe(i));
291
0
                    break;
292
0
                case OFTBinary:
293
0
                {
294
0
                    int nCount = 0;
295
0
                    CPL_IGNORE_RET_VAL(poFeature->GetFieldAsBinary(i, &nCount));
296
0
                    nSize += 4 + nCount;
297
0
                    break;
298
0
                }
299
0
                case OFTIntegerList:
300
0
                {
301
0
                    int nCount = 0;
302
0
                    CPL_IGNORE_RET_VAL(
303
0
                        poFeature->GetFieldAsIntegerList(i, &nCount));
304
0
                    nSize += 4 + (bIsBinary ? sizeof(int) : 11) * nCount;
305
0
                    break;
306
0
                }
307
0
                case OFTInteger64List:
308
0
                {
309
0
                    int nCount = 0;
310
0
                    CPL_IGNORE_RET_VAL(
311
0
                        poFeature->GetFieldAsInteger64List(i, &nCount));
312
0
                    nSize += 4 + (bIsBinary ? sizeof(int64_t) : 21) * nCount;
313
0
                    break;
314
0
                }
315
0
                case OFTRealList:
316
0
                {
317
0
                    int nCount = 0;
318
0
                    CPL_IGNORE_RET_VAL(
319
0
                        poFeature->GetFieldAsDoubleList(i, &nCount));
320
0
                    nSize += 4 + (bIsBinary ? sizeof(double) : 15) * nCount;
321
0
                    break;
322
0
                }
323
0
                case OFTStringList:
324
0
                {
325
0
                    CSLConstList papszIter = poFeature->GetFieldAsStringList(i);
326
0
                    nSize += 4;
327
0
                    for (; papszIter && *papszIter; ++papszIter)
328
0
                        nSize += 4 + strlen(*papszIter);
329
0
                    break;
330
0
                }
331
0
                case OFTTime:
332
                    // Decimal representation
333
0
                    nSize += 4 + sizeof("HH:MM:SS.sss");
334
0
                    break;
335
0
                case OFTDate:
336
                    // Decimal representation
337
0
                    nSize += 4 + sizeof("YYYY-MM-DD");
338
0
                    break;
339
0
                case OFTDateTime:
340
                    // Decimal representation
341
0
                    nSize += 4 + sizeof("YYYY-MM-DDTHH:MM:SS.sss+HH:MM");
342
0
                    break;
343
0
                case OFTWideString:
344
0
                case OFTWideStringList:
345
0
                    break;
346
0
            }
347
0
        }
348
0
    }
349
350
0
    const int nGeomFieldCount = poFeature->GetGeomFieldCount();
351
0
    nSize += 4 * nGeomFieldCount;
352
0
    for (int i = 0; i < nGeomFieldCount; ++i)
353
0
    {
354
0
        const auto poGeom = poFeature->GetGeomFieldRef(i);
355
0
        if (poGeom)
356
0
            nSize += poGeom->WkbSize();
357
0
    }
358
359
0
    return nSize;
360
0
}
361
362
/************************************************************************/
363
/*                       GetCurrentOutputLayer()                        */
364
/************************************************************************/
365
366
constexpr int MIN_FILE_SIZE = 65536;
367
368
namespace
369
{
370
struct Layer
371
{
372
    bool bUseTransactions = false;
373
    std::unique_ptr<GDALDataset> poDS{};
374
    OGRLayer *poLayer = nullptr;
375
    GIntBig nFeatureCount = 0;
376
    int nFileCounter = 1;
377
    GIntBig nFileSize = MIN_FILE_SIZE;
378
379
    ~Layer()
380
0
    {
381
0
        if (poDS)
382
0
        {
383
0
            CPL_IGNORE_RET_VAL(poDS->CommitTransaction());
384
0
        }
385
0
    }
386
};
387
}  // namespace
388
389
static bool GetCurrentOutputLayer(
390
    GDALAlgorithm *const alg, const OGRFeatureDefn *const poSrcFeatureDefn,
391
    OGRLayer *const poSrcLayer, const std::string &osKey,
392
    const std::vector<OGRwkbGeometryType> &aeGeomTypes,
393
    const std::string &osLayerDir, const std::string &osScheme,
394
    const std::string &osPatternIn, bool partDigitLeadingZeroes,
395
    size_t partDigitCount, const int featureLimit, const GIntBig maxFileSize,
396
    const bool omitPartitionedFields,
397
    const std::vector<bool> &abPartitionedFields,
398
    const std::vector<bool> &abPartitionedGeomFields, const char *pszExtension,
399
    GDALDriver *const poOutDriver, const CPLStringList &datasetCreationOptions,
400
    const CPLStringList &layerCreationOptions,
401
    const OGRFeatureDefn *const poFeatureDefnWithoutPartitionedFields,
402
    const int nSpatialIndexPerFeatureConstant,
403
    const int nSpatialIndexPerLog2FeatureCountConstant, bool bUseTransactions,
404
    lru11::Cache<std::string, std::shared_ptr<Layer>> &oCacheOutputLayer,
405
    std::shared_ptr<Layer> &outputLayer)
406
0
{
407
0
    const std::string osPattern =
408
0
        !osPatternIn.empty() ? osPatternIn
409
0
        : osScheme == GDALVectorPartitionAlgorithm::SCHEME_HIVE
410
0
            ? DEFAULT_PATTERN_HIVE
411
0
        : osKey.empty() ? DEFAULT_PATTERN_FLAT_NO_FIELD
412
0
                        : DEFAULT_PATTERN_FLAT;
413
414
0
    bool bLimitReached = false;
415
0
    bool bOpenOrCreateNewFile = true;
416
0
    if (oCacheOutputLayer.tryGet(osKey, outputLayer))
417
0
    {
418
0
        if (featureLimit > 0 && outputLayer->nFeatureCount >= featureLimit)
419
0
        {
420
0
            bLimitReached = true;
421
0
        }
422
0
        else if (maxFileSize > 0 &&
423
0
                 outputLayer->nFileSize +
424
0
                         (nSpatialIndexPerFeatureConstant > 0
425
0
                              ? (outputLayer->nFeatureCount *
426
0
                                     nSpatialIndexPerFeatureConstant +
427
0
                                 static_cast<int>(std::ceil(
428
0
                                     log2(outputLayer->nFeatureCount)))) *
429
0
                                    nSpatialIndexPerLog2FeatureCountConstant
430
0
                              : 0) >=
431
0
                     maxFileSize)
432
0
        {
433
0
            bLimitReached = true;
434
0
        }
435
0
        else
436
0
        {
437
0
            bOpenOrCreateNewFile = false;
438
0
        }
439
0
    }
440
0
    else
441
0
    {
442
0
        outputLayer = std::make_unique<Layer>();
443
0
        outputLayer->bUseTransactions = bUseTransactions;
444
0
    }
445
446
0
    const auto SubstituteVariables = [&osKey, poSrcLayer](const std::string &s)
447
0
    {
448
0
        CPLString ret(s);
449
0
        ret.replaceAll("{LAYER_NAME}",
450
0
                       PercentEncode(poSrcLayer->GetDescription()));
451
452
0
        if (ret.find("{FIELD_VALUE}") != std::string::npos)
453
0
        {
454
0
            std::string fieldValue;
455
0
            const CPLStringList aosTokens(
456
0
                CSLTokenizeString2(osKey.c_str(), "/", 0));
457
0
            for (int i = 0; i < aosTokens.size(); ++i)
458
0
            {
459
0
                const CPLStringList aosFieldNameValue(
460
0
                    CSLTokenizeString2(aosTokens[i], "=", 0));
461
0
                if (!fieldValue.empty())
462
0
                    fieldValue += '_';
463
0
                fieldValue +=
464
0
                    aosFieldNameValue.size() == 2
465
0
                        ? (strcmp(aosFieldNameValue[1], NULL_MARKER) == 0
466
0
                               ? std::string("__NULL__")
467
0
                               : aosFieldNameValue[1])
468
0
                        : std::string("__EMPTY__");
469
0
            }
470
0
            ret.replaceAll("{FIELD_VALUE}", fieldValue);
471
0
        }
472
0
        return ret;
473
0
    };
474
475
0
    const auto nPercentPos = osPattern.find('%');
476
0
    CPLAssert(nPercentPos !=
477
0
              std::string::npos);  // checked by validation action
478
0
    const std::string osPatternPrefix =
479
0
        SubstituteVariables(osPattern.substr(0, nPercentPos));
480
0
    const auto nAfterDPos = osPattern.find('d', nPercentPos + 1) + 1;
481
0
    const std::string osPatternSuffix =
482
0
        nAfterDPos < osPattern.size()
483
0
            ? SubstituteVariables(osPattern.substr(nAfterDPos))
484
0
            : std::string();
485
486
0
    const auto GetBasenameFromCounter = [partDigitCount, partDigitLeadingZeroes,
487
0
                                         &osPatternPrefix,
488
0
                                         &osPatternSuffix](int nCounter)
489
0
    {
490
0
        const std::string sCounter(CPLSPrintf("%d", nCounter));
491
0
        std::string s(osPatternPrefix);
492
0
        if (sCounter.size() < partDigitCount)
493
0
        {
494
0
            s += std::string(partDigitCount - sCounter.size(),
495
0
                             partDigitLeadingZeroes ? DIGIT_ZERO : ' ');
496
0
        }
497
0
        s += sCounter;
498
0
        s += osPatternSuffix;
499
0
        return s;
500
0
    };
501
502
0
    if (bOpenOrCreateNewFile)
503
0
    {
504
0
        std::string osDatasetDir =
505
0
            osScheme == GDALVectorPartitionAlgorithm::SCHEME_HIVE
506
0
                ? CPLFormFilenameSafe(osLayerDir.c_str(), osKey.c_str(),
507
0
                                      nullptr)
508
0
                : osLayerDir;
509
0
        outputLayer->nFeatureCount = 0;
510
511
0
        bool bCreateNewFile = true;
512
0
        if (bLimitReached)
513
0
        {
514
0
            ++outputLayer->nFileCounter;
515
0
        }
516
0
        else
517
0
        {
518
0
            outputLayer->nFileCounter = 1;
519
520
0
            VSIStatBufL sStat;
521
0
            if (VSIStatL(osDatasetDir.c_str(), &sStat) != 0)
522
0
            {
523
0
                if (VSIMkdirRecursive(osDatasetDir.c_str(),
524
0
                                      DIRECTORY_CREATION_MODE) != 0)
525
0
                {
526
0
                    alg->ReportError(CE_Failure, CPLE_AppDefined,
527
0
                                     "Cannot create directory '%s'",
528
0
                                     osDatasetDir.c_str());
529
0
                    return false;
530
0
                }
531
0
            }
532
533
0
            int nMaxCounter = 0;
534
0
            std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> psDir(
535
0
                VSIOpenDir(osDatasetDir.c_str(), 0, nullptr), VSICloseDir);
536
0
            if (psDir)
537
0
            {
538
0
                while (const auto *psEntry = VSIGetNextDirEntry(psDir.get()))
539
0
                {
540
0
                    const std::string osName(
541
0
                        CPLGetBasenameSafe(psEntry->pszName));
542
0
                    if (cpl::starts_with(osName, osPatternPrefix) &&
543
0
                        cpl::ends_with(osName, osPatternSuffix))
544
0
                    {
545
0
                        nMaxCounter = std::max(
546
0
                            nMaxCounter,
547
0
                            atoi(osName
548
0
                                     .substr(osPatternPrefix.size(),
549
0
                                             osName.size() -
550
0
                                                 osPatternPrefix.size() -
551
0
                                                 osPatternSuffix.size())
552
0
                                     .c_str()));
553
0
                    }
554
0
                }
555
0
            }
556
557
0
            if (nMaxCounter > 0)
558
0
            {
559
0
                outputLayer->nFileCounter = nMaxCounter;
560
561
0
                const std::string osFilename = CPLFormFilenameSafe(
562
0
                    osDatasetDir.c_str(),
563
0
                    GetBasenameFromCounter(nMaxCounter).c_str(), pszExtension);
564
0
                auto poDS = std::unique_ptr<GDALDataset>(GDALDataset::Open(
565
0
                    osFilename.c_str(),
566
0
                    GDAL_OF_VECTOR | GDAL_OF_UPDATE | GDAL_OF_VERBOSE_ERROR));
567
0
                if (!poDS)
568
0
                    return false;
569
0
                auto poDstLayer = poDS->GetLayer(0);
570
0
                if (!poDstLayer)
571
0
                {
572
0
                    alg->ReportError(CE_Failure, CPLE_AppDefined,
573
0
                                     "No layer in %s", osFilename.c_str());
574
0
                    return false;
575
0
                }
576
577
                // Check if the existing output layer has the expected layer
578
                // definition
579
0
                const auto poRefFeatureDefn =
580
0
                    poFeatureDefnWithoutPartitionedFields
581
0
                        ? poFeatureDefnWithoutPartitionedFields
582
0
                        : poSrcFeatureDefn;
583
0
                const auto poDstFeatureDefn = poDstLayer->GetLayerDefn();
584
0
                bool bSameDefinition = (poDstFeatureDefn->GetFieldCount() ==
585
0
                                        poRefFeatureDefn->GetFieldCount());
586
0
                for (int i = 0;
587
0
                     bSameDefinition && i < poRefFeatureDefn->GetFieldCount();
588
0
                     ++i)
589
0
                {
590
0
                    const auto poRefFieldDefn =
591
0
                        poRefFeatureDefn->GetFieldDefn(i);
592
0
                    const auto poDstFieldDefn =
593
0
                        poDstFeatureDefn->GetFieldDefn(i);
594
0
                    bSameDefinition =
595
0
                        EQUAL(poRefFieldDefn->GetNameRef(),
596
0
                              poDstFieldDefn->GetNameRef()) &&
597
0
                        poRefFieldDefn->GetType() == poDstFieldDefn->GetType();
598
0
                }
599
0
                bSameDefinition =
600
0
                    bSameDefinition && (poDstFeatureDefn->GetGeomFieldCount() ==
601
0
                                        poRefFeatureDefn->GetGeomFieldCount());
602
0
                for (int i = 0; bSameDefinition &&
603
0
                                i < poRefFeatureDefn->GetGeomFieldCount();
604
0
                     ++i)
605
0
                {
606
0
                    const auto poRefFieldDefn =
607
0
                        poRefFeatureDefn->GetGeomFieldDefn(i);
608
0
                    const auto poDstFieldDefn =
609
0
                        poDstFeatureDefn->GetGeomFieldDefn(i);
610
0
                    bSameDefinition =
611
0
                        (poRefFeatureDefn->GetGeomFieldCount() == 1 ||
612
0
                         EQUAL(poRefFieldDefn->GetNameRef(),
613
0
                               poDstFieldDefn->GetNameRef()));
614
0
                }
615
616
0
                if (!bSameDefinition)
617
0
                {
618
0
                    alg->ReportError(CE_Failure, CPLE_AppDefined,
619
0
                                     "%s does not have the same feature "
620
0
                                     "definition as the source layer",
621
0
                                     osFilename.c_str());
622
0
                    return false;
623
0
                }
624
625
0
                if (VSIStatL(osFilename.c_str(), &sStat) == 0)
626
0
                {
627
0
                    outputLayer->nFileSize = sStat.st_size;
628
0
                }
629
630
0
                GIntBig nFeatureCount = 0;
631
0
                if (((featureLimit == 0 ||
632
0
                      (nFeatureCount = poDstLayer->GetFeatureCount(true)) <
633
0
                          featureLimit)) &&
634
0
                    (maxFileSize == 0 || outputLayer->nFileSize < maxFileSize))
635
0
                {
636
0
                    bCreateNewFile = false;
637
0
                    outputLayer->poDS = std::move(poDS);
638
0
                    outputLayer->poLayer = poDstLayer;
639
0
                    outputLayer->nFeatureCount = nFeatureCount;
640
641
0
                    if (bUseTransactions)
642
0
                    {
643
0
                        if (outputLayer->poDS->StartTransaction() !=
644
0
                            OGRERR_NONE)
645
0
                        {
646
0
                            return false;
647
0
                        }
648
0
                    }
649
0
                }
650
0
                else
651
0
                {
652
0
                    ++outputLayer->nFileCounter;
653
0
                }
654
0
            }
655
0
        }
656
657
0
        if (bCreateNewFile)
658
0
        {
659
0
            outputLayer->nFileSize = MIN_FILE_SIZE;
660
661
0
            if (bUseTransactions && outputLayer->poDS &&
662
0
                outputLayer->poDS->CommitTransaction() != OGRERR_NONE)
663
0
            {
664
0
                return false;
665
0
            }
666
667
0
            const std::string osFilename = CPLFormFilenameSafe(
668
0
                osDatasetDir.c_str(),
669
0
                GetBasenameFromCounter(outputLayer->nFileCounter).c_str(),
670
0
                pszExtension);
671
0
            outputLayer->poDS.reset(
672
0
                poOutDriver->Create(osFilename.c_str(), 0, 0, 0, GDT_Unknown,
673
0
                                    datasetCreationOptions.List()));
674
0
            if (!outputLayer->poDS)
675
0
            {
676
0
                alg->ReportError(CE_Failure, CPLE_AppDefined,
677
0
                                 "Cannot create dataset '%s'",
678
0
                                 osFilename.c_str());
679
0
                return false;
680
0
            }
681
682
0
            CPLStringList modLayerCreationOptions(layerCreationOptions);
683
0
            const char *pszSrcFIDColumn = poSrcLayer->GetFIDColumn();
684
0
            if (pszSrcFIDColumn[0])
685
0
            {
686
0
                const char *pszLCO = poOutDriver->GetMetadataItem(
687
0
                    GDAL_DS_LAYER_CREATIONOPTIONLIST);
688
0
                if (pszLCO && strstr(pszLCO, "'FID'") &&
689
0
                    layerCreationOptions.FetchNameValue("FID") == nullptr)
690
0
                    modLayerCreationOptions.SetNameValue("FID",
691
0
                                                         pszSrcFIDColumn);
692
0
            }
693
694
0
            std::unique_ptr<OGRGeomFieldDefn> poFirstGeomFieldDefn;
695
0
            if (poSrcFeatureDefn->GetGeomFieldCount())
696
0
            {
697
0
                poFirstGeomFieldDefn = std::make_unique<OGRGeomFieldDefn>(
698
0
                    *poSrcFeatureDefn->GetGeomFieldDefn(0));
699
0
                if (abPartitionedGeomFields[0])
700
0
                {
701
0
                    if (aeGeomTypes[0] == wkbNone)
702
0
                        poFirstGeomFieldDefn.reset();
703
0
                    else
704
0
                        whileUnsealing(poFirstGeomFieldDefn.get())
705
0
                            ->SetType(aeGeomTypes[0]);
706
0
                }
707
0
            }
708
0
            auto poLayer = outputLayer->poDS->CreateLayer(
709
0
                poSrcLayer->GetDescription(), poFirstGeomFieldDefn.get(),
710
0
                modLayerCreationOptions.List());
711
0
            if (!poLayer)
712
0
            {
713
0
                return false;
714
0
            }
715
0
            outputLayer->poLayer = poLayer;
716
0
            int iField = -1;
717
0
            for (const auto *poFieldDefn : poSrcFeatureDefn->GetFields())
718
0
            {
719
0
                ++iField;
720
0
                if (omitPartitionedFields && abPartitionedFields[iField])
721
0
                    continue;
722
0
                if (poLayer->CreateField(poFieldDefn) != OGRERR_NONE)
723
0
                {
724
0
                    alg->ReportError(CE_Failure, CPLE_AppDefined,
725
0
                                     "Cannot create field '%s'",
726
0
                                     poFieldDefn->GetNameRef());
727
0
                    return false;
728
0
                }
729
0
            }
730
0
            int iGeomField = -1;
731
0
            for (const auto *poGeomFieldDefn :
732
0
                 poSrcFeatureDefn->GetGeomFields())
733
0
            {
734
0
                ++iGeomField;
735
0
                if (iGeomField > 0)
736
0
                {
737
0
                    OGRGeomFieldDefn oClone(poGeomFieldDefn);
738
0
                    if (abPartitionedGeomFields[iGeomField])
739
0
                    {
740
0
                        if (aeGeomTypes[iGeomField] == wkbNone)
741
0
                            continue;
742
0
                        whileUnsealing(&oClone)->SetType(
743
0
                            aeGeomTypes[iGeomField]);
744
0
                    }
745
0
                    if (poLayer->CreateGeomField(&oClone) != OGRERR_NONE)
746
0
                    {
747
0
                        alg->ReportError(CE_Failure, CPLE_AppDefined,
748
0
                                         "Cannot create geometry field '%s'",
749
0
                                         poGeomFieldDefn->GetNameRef());
750
0
                        return false;
751
0
                    }
752
0
                }
753
0
            }
754
755
0
            if (bUseTransactions)
756
0
            {
757
0
                if (outputLayer->poDS->StartTransaction() != OGRERR_NONE)
758
0
                    return false;
759
0
            }
760
0
        }
761
762
0
        const auto nCounter = CPLGetErrorCounter();
763
0
        oCacheOutputLayer.insert(osKey, outputLayer);
764
        // In case insertion caused an eviction and old dataset
765
        // flushing caused an error
766
0
        if (CPLGetErrorCounter() != nCounter)
767
0
            return false;
768
0
    }
769
770
0
    return true;
771
0
}
772
773
/************************************************************************/
774
/*               GDALVectorPartitionAlgorithm::RunStep()                */
775
/************************************************************************/
776
777
bool GDALVectorPartitionAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt)
778
0
{
779
0
    auto poSrcDS = m_inputDataset[0].GetDatasetRef();
780
0
    CPLAssert(poSrcDS);
781
782
0
    auto poOutDriver = poSrcDS->GetDriver();
783
0
    const char *pszExtensions =
784
0
        poOutDriver ? poOutDriver->GetMetadataItem(GDAL_DMD_EXTENSIONS)
785
0
                    : nullptr;
786
0
    if (m_format.empty())
787
0
    {
788
0
        if (!pszExtensions)
789
0
        {
790
0
            ReportError(CE_Failure, CPLE_AppDefined,
791
0
                        "Cannot infer output format. Please specify "
792
0
                        "'output-format' argument");
793
0
            return false;
794
0
        }
795
0
    }
796
0
    else
797
0
    {
798
0
        poOutDriver = GetGDALDriverManager()->GetDriverByName(m_format.c_str());
799
0
        if (!(poOutDriver && (pszExtensions = poOutDriver->GetMetadataItem(
800
0
                                  GDAL_DMD_EXTENSIONS)) != nullptr))
801
0
        {
802
0
            ReportError(CE_Failure, CPLE_AppDefined,
803
0
                        "Output driver has no known file extension");
804
0
            return false;
805
0
        }
806
0
    }
807
0
    CPLAssert(poOutDriver);
808
809
0
    const bool bFormatSupportsAppend =
810
0
        poOutDriver->GetMetadataItem(GDAL_DCAP_UPDATE) ||
811
0
        poOutDriver->GetMetadataItem(GDAL_DCAP_APPEND);
812
0
    if (m_appendLayer && !bFormatSupportsAppend)
813
0
    {
814
0
        ReportError(CE_Failure, CPLE_AppDefined,
815
0
                    "Driver '%s' does not support update",
816
0
                    poOutDriver->GetDescription());
817
0
        return false;
818
0
    }
819
820
0
    const bool bParquetOutput = EQUAL(poOutDriver->GetDescription(), "PARQUET");
821
0
    if (bParquetOutput && m_scheme == SCHEME_HIVE)
822
0
    {
823
        // Required for Parquet Hive partitioning
824
0
        m_omitPartitionedFields = true;
825
0
    }
826
827
0
    const CPLStringList aosExtensions(CSLTokenizeString(pszExtensions));
828
0
    const char *pszExtension = aosExtensions[0];
829
830
0
    const CPLStringList datasetCreationOptions(m_creationOptions);
831
0
    const CPLStringList layerCreationOptions(m_layerCreationOptions);
832
833
    // We don't have driver metadata for that (and that would be a bit
834
    // tricky because some formats are half-text/half-binary), so...
835
0
    const bool bOutputFormatIsBinary =
836
0
        bParquetOutput || EQUAL(poOutDriver->GetDescription(), "GPKG") ||
837
0
        EQUAL(poOutDriver->GetDescription(), "SQLite") ||
838
0
        EQUAL(poOutDriver->GetDescription(), "FlatGeoBuf");
839
840
    // Below values have been experimentally determined and are not based
841
    // on rocket science...
842
0
    int nSpatialIndexPerFeatureConstant = 0;
843
0
    int nSpatialIndexPerLog2FeatureCountConstant = 0;
844
0
    if (CPLTestBool(
845
0
            layerCreationOptions.FetchNameValueDef("SPATIAL_INDEX", "YES")))
846
0
    {
847
0
        if (EQUAL(poOutDriver->GetDescription(), "GPKG"))
848
0
        {
849
0
            nSpatialIndexPerFeatureConstant =
850
0
                static_cast<int>(sizeof(double) * 4 + sizeof(uint32_t));
851
0
            nSpatialIndexPerLog2FeatureCountConstant = 1;
852
0
        }
853
0
        else if (EQUAL(poOutDriver->GetDescription(), "FlatGeoBuf"))
854
0
        {
855
0
            nSpatialIndexPerFeatureConstant = 1;
856
0
            nSpatialIndexPerLog2FeatureCountConstant =
857
0
                static_cast<int>(sizeof(double) * 4 + sizeof(uint64_t));
858
0
        }
859
0
    }
860
861
0
    const bool bUseTransactions =
862
0
        (EQUAL(poOutDriver->GetDescription(), "GPKG") ||
863
0
         EQUAL(poOutDriver->GetDescription(), "SQLite")) &&
864
0
        !m_skipErrors;
865
866
0
    VSIStatBufL sStat;
867
0
    if (VSIStatL(m_output.c_str(), &sStat) == 0)
868
0
    {
869
0
        if (m_overwrite)
870
0
        {
871
0
            bool emptyDir = true;
872
0
            bool hasDirLevel1WithEqual = false;
873
874
            // Do a sanity check to verify that this looks like a directory
875
            // generated by partition
876
877
0
            if (m_scheme == SCHEME_HIVE)
878
0
            {
879
0
                std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> psDir(
880
0
                    VSIOpenDir(m_output.c_str(), -1, nullptr), VSICloseDir);
881
0
                if (psDir)
882
0
                {
883
0
                    while (const auto *psEntry =
884
0
                               VSIGetNextDirEntry(psDir.get()))
885
0
                    {
886
0
                        emptyDir = false;
887
0
                        if (VSI_ISDIR(psEntry->nMode))
888
0
                        {
889
0
                            std::string_view v(psEntry->pszName);
890
0
                            if (std::count_if(
891
0
                                    v.begin(), v.end(), [](char c)
892
0
                                    { return c == '/' || c == '\\'; }) == 1)
893
0
                            {
894
0
                                const auto nPosDirSep = v.find_first_of("/\\");
895
0
                                const auto nPosEqual = v.find('=', nPosDirSep);
896
0
                                if (nPosEqual != std::string::npos)
897
0
                                {
898
0
                                    hasDirLevel1WithEqual = true;
899
0
                                    break;
900
0
                                }
901
0
                            }
902
0
                        }
903
0
                    }
904
0
                }
905
906
0
                if (!hasDirLevel1WithEqual && !emptyDir)
907
0
                {
908
0
                    ReportError(
909
0
                        CE_Failure, CPLE_AppDefined,
910
0
                        "Rejecting removing '%s' as it does not look like "
911
0
                        "a directory generated by this utility. If you are "
912
0
                        "sure, remove it manually and re-run",
913
0
                        m_output.c_str());
914
0
                    return false;
915
0
                }
916
0
            }
917
0
            else
918
0
            {
919
0
                bool hasSubDir = false;
920
0
                std::unique_ptr<VSIDIR, decltype(&VSICloseDir)> psDir(
921
0
                    VSIOpenDir(m_output.c_str(), 0, nullptr), VSICloseDir);
922
0
                if (psDir)
923
0
                {
924
0
                    while (const auto *psEntry =
925
0
                               VSIGetNextDirEntry(psDir.get()))
926
0
                    {
927
0
                        if (VSI_ISDIR(psEntry->nMode))
928
0
                        {
929
0
                            hasSubDir = true;
930
0
                            break;
931
0
                        }
932
0
                    }
933
0
                }
934
935
0
                if (hasSubDir)
936
0
                {
937
0
                    ReportError(
938
0
                        CE_Failure, CPLE_AppDefined,
939
0
                        "Rejecting removing '%s' as it does not look like "
940
0
                        "a directory generated by this utility. If you are "
941
0
                        "sure, remove it manually and re-run",
942
0
                        m_output.c_str());
943
0
                    return false;
944
0
                }
945
0
            }
946
947
0
            if (VSIRmdirRecursive(m_output.c_str()) != 0)
948
0
            {
949
0
                ReportError(CE_Failure, CPLE_AppDefined, "Cannot remove '%s'",
950
0
                            m_output.c_str());
951
0
                return false;
952
0
            }
953
0
        }
954
0
        else if (!m_appendLayer)
955
0
        {
956
0
            ReportError(CE_Failure, CPLE_AppDefined,
957
0
                        "'%s' already exists. Specify --overwrite or --append",
958
0
                        m_output.c_str());
959
0
            return false;
960
0
        }
961
0
    }
962
0
    if (VSIStatL(m_output.c_str(), &sStat) != 0)
963
0
    {
964
0
        if (VSIMkdir(m_output.c_str(), DIRECTORY_CREATION_MODE) != 0)
965
0
        {
966
0
            ReportError(CE_Failure, CPLE_AppDefined,
967
0
                        "Cannot create directory '%s'", m_output.c_str());
968
0
            return false;
969
0
        }
970
0
    }
971
972
0
    for (OGRLayer *poSrcLayer : poSrcDS->GetLayers())
973
0
    {
974
0
        const std::string osLayerDir =
975
0
            m_scheme == SCHEME_HIVE
976
0
                ? CPLFormFilenameSafe(
977
0
                      m_output.c_str(),
978
0
                      PercentEncode(poSrcLayer->GetDescription()).c_str(),
979
0
                      nullptr)
980
0
                : m_output;
981
0
        if (m_scheme == SCHEME_HIVE &&
982
0
            VSIStatL(osLayerDir.c_str(), &sStat) != 0)
983
0
        {
984
0
            if (VSIMkdir(osLayerDir.c_str(), DIRECTORY_CREATION_MODE) != 0)
985
0
            {
986
0
                ReportError(CE_Failure, CPLE_AppDefined,
987
0
                            "Cannot create directory '%s'", osLayerDir.c_str());
988
0
                return false;
989
0
            }
990
0
        }
991
992
0
        const auto poSrcFeatureDefn = poSrcLayer->GetLayerDefn();
993
994
0
        struct Field
995
0
        {
996
0
            int nIdx{};
997
0
            bool bIsGeom = false;
998
0
            std::string encodedFieldName{};
999
0
            OGRFieldType eType{};
1000
0
        };
1001
1002
0
        std::vector<Field> asFields;
1003
0
        std::vector<bool> abPartitionedFields(poSrcFeatureDefn->GetFieldCount(),
1004
0
                                              false);
1005
0
        std::vector<bool> abPartitionedGeomFields(
1006
0
            poSrcFeatureDefn->GetGeomFieldCount(), false);
1007
0
        for (const std::string &fieldName : m_fields)
1008
0
        {
1009
0
            int nIdx = poSrcFeatureDefn->GetFieldIndex(fieldName.c_str());
1010
0
            if (nIdx < 0)
1011
0
            {
1012
0
                if (EQUAL(fieldName.c_str(), "OGR_GEOMETRY") &&
1013
0
                    poSrcFeatureDefn->GetGeomFieldCount() > 0)
1014
0
                {
1015
0
                    CPLError(CE_Warning, CPLE_AppDefined,
1016
0
                             "'%s' is deprecated. Please use '%s' instead",
1017
0
                             "OGR_GEOMETRY",
1018
0
                             OGR_GEOMETRY_DEFAULT_NON_EMPTY_NAME);
1019
0
                    nIdx = 0;
1020
0
                }
1021
0
                else if (EQUAL(fieldName.c_str(),
1022
0
                               OGR_GEOMETRY_DEFAULT_NON_EMPTY_NAME) &&
1023
0
                         poSrcFeatureDefn->GetGeomFieldCount() > 0)
1024
0
                {
1025
0
                    nIdx = 0;
1026
0
                }
1027
0
                else
1028
0
                    nIdx =
1029
0
                        poSrcFeatureDefn->GetGeomFieldIndex(fieldName.c_str());
1030
0
                if (nIdx < 0)
1031
0
                {
1032
0
                    ReportError(CE_Failure, CPLE_AppDefined,
1033
0
                                "Cannot find field '%s' in layer '%s'",
1034
0
                                fieldName.c_str(),
1035
0
                                poSrcLayer->GetDescription());
1036
0
                    return false;
1037
0
                }
1038
0
                else
1039
0
                {
1040
0
                    abPartitionedGeomFields[nIdx] = true;
1041
0
                    Field f;
1042
0
                    f.nIdx = nIdx;
1043
0
                    f.bIsGeom = true;
1044
0
                    if (fieldName.empty())
1045
0
                        f.encodedFieldName =
1046
0
                            OGR_GEOMETRY_DEFAULT_NON_EMPTY_NAME;
1047
0
                    else
1048
0
                        f.encodedFieldName = PercentEncode(fieldName);
1049
0
                    asFields.push_back(std::move(f));
1050
0
                }
1051
0
            }
1052
0
            else
1053
0
            {
1054
0
                const auto eType =
1055
0
                    poSrcFeatureDefn->GetFieldDefn(nIdx)->GetType();
1056
0
                if (eType != OFTString && eType != OFTInteger &&
1057
0
                    eType != OFTInteger64)
1058
0
                {
1059
0
                    ReportError(
1060
0
                        CE_Failure, CPLE_NotSupported,
1061
0
                        "Field '%s' not valid for partitioning. Only fields of "
1062
0
                        "type String, Integer or Integer64, or geometry fields,"
1063
0
                        " are accepted",
1064
0
                        fieldName.c_str());
1065
0
                    return false;
1066
0
                }
1067
0
                abPartitionedFields[nIdx] = true;
1068
0
                Field f;
1069
0
                f.nIdx = nIdx;
1070
0
                f.bIsGeom = false;
1071
0
                f.encodedFieldName = PercentEncode(fieldName);
1072
0
                f.eType = eType;
1073
0
                asFields.push_back(std::move(f));
1074
0
            }
1075
0
        }
1076
1077
0
        std::vector<OGRFieldType> aeSrcFieldTypes;
1078
0
        for (const auto *poFieldDefn : poSrcFeatureDefn->GetFields())
1079
0
        {
1080
0
            aeSrcFieldTypes.push_back(poFieldDefn->GetType());
1081
0
        }
1082
1083
0
        std::unique_ptr<OGRFeatureDefn> poFeatureDefnWithoutPartitionedFields(
1084
0
            poSrcFeatureDefn->Clone());
1085
0
        std::vector<int> anMapForSetFrom;
1086
0
        if (m_omitPartitionedFields)
1087
0
        {
1088
            // Sort fields by descending index (so we can delete them easily)
1089
0
            std::vector<Field> sortedFields(asFields);
1090
0
            std::sort(sortedFields.begin(), sortedFields.end(),
1091
0
                      [](const Field &a, const Field &b)
1092
0
                      { return a.nIdx > b.nIdx; });
1093
0
            for (const auto &field : sortedFields)
1094
0
            {
1095
0
                if (!field.bIsGeom)
1096
0
                    poFeatureDefnWithoutPartitionedFields->DeleteFieldDefn(
1097
0
                        field.nIdx);
1098
0
            }
1099
0
            anMapForSetFrom =
1100
0
                poFeatureDefnWithoutPartitionedFields->ComputeMapForSetFrom(
1101
0
                    poSrcFeatureDefn);
1102
0
        }
1103
1104
0
        lru11::Cache<std::string, std::shared_ptr<Layer>> oCacheOutputLayer(
1105
0
            m_maxCacheSize, 0);
1106
0
        std::shared_ptr<Layer> outputLayer = std::make_unique<Layer>();
1107
0
        outputLayer->bUseTransactions = bUseTransactions;
1108
1109
0
        GIntBig nTotalFeatures = 1;
1110
0
        GIntBig nFeatureIter = 0;
1111
0
        if (ctxt.m_pfnProgress)
1112
0
            nTotalFeatures = poSrcLayer->GetFeatureCount(true);
1113
0
        const double dfInvTotalFeatures =
1114
0
            1.0 / static_cast<double>(std::max<GIntBig>(1, nTotalFeatures));
1115
1116
0
        std::string osAttrQueryString;
1117
0
        if (const char *pszAttrQueryString = poSrcLayer->GetAttrQueryString())
1118
0
            osAttrQueryString = pszAttrQueryString;
1119
1120
0
        std::string osKeyTmp;
1121
0
        std::vector<OGRwkbGeometryType> aeGeomTypesTmp;
1122
0
        const auto BuildKey =
1123
0
            [&osKeyTmp, &aeGeomTypesTmp](const std::vector<Field> &fields,
1124
0
                                         const OGRFeature *poFeature)
1125
0
            -> std::pair<const std::string &,
1126
0
                         const std::vector<OGRwkbGeometryType> &>
1127
0
        {
1128
0
            osKeyTmp.clear();
1129
0
            aeGeomTypesTmp.resize(poFeature->GetDefnRef()->GetGeomFieldCount());
1130
0
            for (const auto &field : fields)
1131
0
            {
1132
0
                if (!osKeyTmp.empty())
1133
0
                    osKeyTmp += '/';
1134
0
                osKeyTmp += field.encodedFieldName;
1135
0
                osKeyTmp += '=';
1136
0
                if (field.bIsGeom)
1137
0
                {
1138
0
                    const auto poGeom = poFeature->GetGeomFieldRef(field.nIdx);
1139
0
                    if (poGeom)
1140
0
                    {
1141
0
                        aeGeomTypesTmp[field.nIdx] = poGeom->getGeometryType();
1142
0
                        osKeyTmp += poGeom->getGeometryName();
1143
0
                        if (poGeom->Is3D())
1144
0
                            osKeyTmp += 'Z';
1145
0
                        if (poGeom->IsMeasured())
1146
0
                            osKeyTmp += 'M';
1147
0
                    }
1148
0
                    else
1149
0
                    {
1150
0
                        aeGeomTypesTmp[field.nIdx] = wkbNone;
1151
0
                        osKeyTmp += NULL_MARKER;
1152
0
                    }
1153
0
                }
1154
0
                else if (poFeature->IsFieldSetAndNotNull(field.nIdx))
1155
0
                {
1156
0
                    if (field.eType == OFTString)
1157
0
                    {
1158
0
                        PercentEncode(
1159
0
                            osKeyTmp,
1160
0
                            poFeature->GetFieldAsStringUnsafe(field.nIdx));
1161
0
                    }
1162
0
                    else if (field.eType == OFTInteger)
1163
0
                    {
1164
0
                        osKeyTmp += CPLSPrintf(
1165
0
                            "%d",
1166
0
                            poFeature->GetFieldAsIntegerUnsafe(field.nIdx));
1167
0
                    }
1168
0
                    else
1169
0
                    {
1170
0
                        osKeyTmp += CPLSPrintf(
1171
0
                            CPL_FRMT_GIB,
1172
0
                            poFeature->GetFieldAsInteger64Unsafe(field.nIdx));
1173
0
                    }
1174
0
                }
1175
0
                else
1176
0
                {
1177
0
                    osKeyTmp += NULL_MARKER;
1178
0
                }
1179
0
            }
1180
0
            return {osKeyTmp, aeGeomTypesTmp};
1181
0
        };
1182
1183
0
        std::set<std::string> oSetKeys;
1184
0
        if (!bFormatSupportsAppend)
1185
0
        {
1186
0
            CPLDebug(
1187
0
                "GDAL",
1188
0
                "First pass to determine all distinct partitioned values...");
1189
1190
0
            if (asFields.size() == 1 && !asFields[0].bIsGeom)
1191
0
            {
1192
0
                std::string osSQL = "SELECT DISTINCT \"";
1193
0
                osSQL += CPLString(m_fields[0]).replaceAll('"', "\"\"");
1194
0
                osSQL += "\" FROM \"";
1195
0
                osSQL += CPLString(poSrcLayer->GetDescription())
1196
0
                             .replaceAll('"', "\"\"");
1197
0
                osSQL += '"';
1198
0
                if (!osAttrQueryString.empty())
1199
0
                {
1200
0
                    osSQL += " WHERE ";
1201
0
                    osSQL += osAttrQueryString;
1202
0
                }
1203
0
                auto poSQLLayer =
1204
0
                    poSrcDS->ExecuteSQL(osSQL.c_str(), nullptr, nullptr);
1205
0
                if (!poSQLLayer)
1206
0
                    return false;
1207
0
                std::vector<Field> asSingleField{asFields[0]};
1208
0
                asSingleField[0].nIdx = 0;
1209
0
                for (auto &poFeature : *poSQLLayer)
1210
0
                {
1211
0
                    const auto sPair = BuildKey(asFields, poFeature.get());
1212
0
                    const std::string &osKey = sPair.first;
1213
0
                    oSetKeys.insert(osKey);
1214
#ifdef DEBUG_VERBOSE
1215
                    CPLDebug("GDAL", "Found %s", osKey.c_str());
1216
#endif
1217
0
                }
1218
0
                poSrcDS->ReleaseResultSet(poSQLLayer);
1219
1220
0
                if (!osAttrQueryString.empty())
1221
0
                {
1222
0
                    poSrcLayer->SetAttributeFilter(osAttrQueryString.c_str());
1223
0
                }
1224
0
            }
1225
0
            else
1226
0
            {
1227
0
                for (auto &poFeature : *poSrcLayer)
1228
0
                {
1229
0
                    const auto sPair = BuildKey(asFields, poFeature.get());
1230
0
                    const std::string &osKey = sPair.first;
1231
0
                    if (oSetKeys.insert(osKey).second)
1232
0
                    {
1233
#ifdef DEBUG_VERBOSE
1234
                        CPLDebug("GDAL", "Found %s", osKey.c_str());
1235
#endif
1236
0
                    }
1237
0
                }
1238
0
            }
1239
0
            CPLDebug("GDAL",
1240
0
                     "End of first pass: %d unique partitioning keys found -> "
1241
0
                     "%d pass(es) needed",
1242
0
                     static_cast<int>(oSetKeys.size()),
1243
0
                     static_cast<int>((oSetKeys.size() + m_maxCacheSize - 1) /
1244
0
                                      m_maxCacheSize));
1245
1246
            // If we have less distinct values as the maximum cache size, we
1247
            // can do a single iteration.
1248
0
            if (oSetKeys.size() <= static_cast<size_t>(m_maxCacheSize))
1249
0
                oSetKeys.clear();
1250
0
        }
1251
1252
0
        std::set<std::string> oSetOutputDatasets;
1253
0
        auto oSetKeysIter = oSetKeys.begin();
1254
0
        while (true)
1255
0
        {
1256
            // Determine which keys are allowed for the current pass
1257
0
            std::set<std::string> oSetKeysAllowedInThisPass;
1258
0
            if (!oSetKeys.empty())
1259
0
            {
1260
0
                while (oSetKeysAllowedInThisPass.size() <
1261
0
                           static_cast<size_t>(m_maxCacheSize) &&
1262
0
                       oSetKeysIter != oSetKeys.end())
1263
0
                {
1264
0
                    oSetKeysAllowedInThisPass.insert(*oSetKeysIter);
1265
0
                    ++oSetKeysIter;
1266
0
                }
1267
0
                if (oSetKeysAllowedInThisPass.empty())
1268
0
                    break;
1269
0
            }
1270
1271
0
            for (auto &poFeature : *poSrcLayer)
1272
0
            {
1273
0
                const auto sPair = BuildKey(asFields, poFeature.get());
1274
0
                const std::string &osKey = sPair.first;
1275
0
                const auto &aeGeomTypes = sPair.second;
1276
1277
0
                if (!oSetKeysAllowedInThisPass.empty() &&
1278
0
                    !cpl::contains(oSetKeysAllowedInThisPass, osKey))
1279
0
                {
1280
0
                    continue;
1281
0
                }
1282
1283
0
                if (!GetCurrentOutputLayer(
1284
0
                        this, poSrcFeatureDefn, poSrcLayer, osKey, aeGeomTypes,
1285
0
                        osLayerDir, m_scheme, m_pattern,
1286
0
                        m_partDigitLeadingZeroes, m_partDigitCount,
1287
0
                        m_featureLimit, m_maxFileSize, m_omitPartitionedFields,
1288
0
                        abPartitionedFields, abPartitionedGeomFields,
1289
0
                        pszExtension, poOutDriver, datasetCreationOptions,
1290
0
                        layerCreationOptions,
1291
0
                        poFeatureDefnWithoutPartitionedFields.get(),
1292
0
                        poFeature->GetGeometryRef()
1293
0
                            ? nSpatialIndexPerFeatureConstant
1294
0
                            : 0,
1295
0
                        nSpatialIndexPerLog2FeatureCountConstant,
1296
0
                        bUseTransactions, oCacheOutputLayer, outputLayer))
1297
0
                {
1298
0
                    return false;
1299
0
                }
1300
1301
0
                if (bParquetOutput)
1302
0
                {
1303
0
                    oSetOutputDatasets.insert(
1304
0
                        outputLayer->poDS->GetDescription());
1305
0
                }
1306
1307
0
                if (m_appendLayer)
1308
0
                    poFeature->SetFID(OGRNullFID);
1309
1310
0
                OGRErr eErr;
1311
0
                if (m_omitPartitionedFields ||
1312
0
                    std::find(aeGeomTypes.begin(), aeGeomTypes.end(),
1313
0
                              wkbNone) != aeGeomTypes.end())
1314
0
                {
1315
0
                    OGRFeature oFeat(outputLayer->poLayer->GetLayerDefn());
1316
0
                    oFeat.SetFrom(poFeature.get(), anMapForSetFrom.data());
1317
0
                    oFeat.SetFID(poFeature->GetFID());
1318
0
                    eErr = outputLayer->poLayer->CreateFeature(&oFeat);
1319
0
                }
1320
0
                else
1321
0
                {
1322
0
                    poFeature->SetFDefnUnsafe(
1323
0
                        outputLayer->poLayer->GetLayerDefn());
1324
0
                    eErr = outputLayer->poLayer->CreateFeature(poFeature.get());
1325
0
                }
1326
0
                if (eErr != OGRERR_NONE)
1327
0
                {
1328
0
                    ReportError(m_skipErrors ? CE_Warning : CE_Failure,
1329
0
                                CPLE_AppDefined,
1330
0
                                "Cannot insert feature " CPL_FRMT_GIB,
1331
0
                                poFeature->GetFID());
1332
0
                    if (m_skipErrors)
1333
0
                        continue;
1334
0
                    return false;
1335
0
                }
1336
0
                ++outputLayer->nFeatureCount;
1337
1338
0
                if (bUseTransactions &&
1339
0
                    (outputLayer->nFeatureCount % m_transactionSize) == 0)
1340
0
                {
1341
0
                    if (outputLayer->poDS->CommitTransaction() != OGRERR_NONE ||
1342
0
                        outputLayer->poDS->StartTransaction() != OGRERR_NONE)
1343
0
                    {
1344
0
                        return false;
1345
0
                    }
1346
0
                }
1347
1348
                // Compute a rough estimate of the space taken by the feature
1349
0
                if (m_maxFileSize > 0)
1350
0
                {
1351
0
                    outputLayer->nFileSize += GetEstimatedFeatureSize(
1352
0
                        poFeature.get(), abPartitionedFields,
1353
0
                        m_omitPartitionedFields, aeSrcFieldTypes,
1354
0
                        bOutputFormatIsBinary);
1355
0
                }
1356
1357
0
                ++nFeatureIter;
1358
0
                if (ctxt.m_pfnProgress &&
1359
0
                    !ctxt.m_pfnProgress(
1360
0
                        std::min(1.0, static_cast<double>(nFeatureIter) *
1361
0
                                          dfInvTotalFeatures),
1362
0
                        "", ctxt.m_pProgressData))
1363
0
                {
1364
0
                    ReportError(CE_Failure, CPLE_UserInterrupt,
1365
0
                                "Interrupted by user");
1366
0
                    return false;
1367
0
                }
1368
0
            }
1369
1370
0
            if (oSetKeysIter == oSetKeys.end())
1371
0
                break;
1372
0
        }
1373
1374
0
        const auto nCounter = CPLGetErrorCounter();
1375
0
        outputLayer.reset();
1376
0
        oCacheOutputLayer.clear();
1377
0
        if (CPLGetErrorCounter() != nCounter)
1378
0
            return false;
1379
1380
        // For Parquet output, create special "_metadata" file that contains
1381
        // the schema and references the individual files
1382
0
        if (bParquetOutput && !oSetOutputDatasets.empty())
1383
0
        {
1384
0
            auto poAlg =
1385
0
                GDALGlobalAlgorithmRegistry::GetSingleton().Instantiate(
1386
0
                    "driver", "parquet", "create-metadata-file");
1387
0
            if (poAlg)
1388
0
            {
1389
0
                auto inputArg = poAlg->GetArg(GDAL_ARG_NAME_INPUT);
1390
0
                auto outputArg = poAlg->GetArg(GDAL_ARG_NAME_OUTPUT);
1391
0
                if (inputArg && inputArg->GetType() == GAAT_DATASET_LIST &&
1392
0
                    outputArg && outputArg->GetType() == GAAT_DATASET)
1393
0
                {
1394
0
                    std::vector<std::string> asInputFilenames;
1395
0
                    asInputFilenames.insert(asInputFilenames.end(),
1396
0
                                            oSetOutputDatasets.begin(),
1397
0
                                            oSetOutputDatasets.end());
1398
0
                    inputArg->Set(asInputFilenames);
1399
0
                    outputArg->Set(CPLFormFilenameSafe(osLayerDir.c_str(),
1400
0
                                                       "_metadata", nullptr));
1401
0
                    if (!poAlg->Run())
1402
0
                        return false;
1403
0
                }
1404
0
            }
1405
0
        }
1406
0
    }
1407
1408
0
    return true;
1409
0
}
1410
1411
/************************************************************************/
1412
/*               GDALVectorPartitionAlgorithm::RunImpl()                */
1413
/************************************************************************/
1414
1415
bool GDALVectorPartitionAlgorithm::RunImpl(GDALProgressFunc pfnProgress,
1416
                                           void *pProgressData)
1417
0
{
1418
0
    GDALPipelineStepRunContext stepCtxt;
1419
0
    stepCtxt.m_pfnProgress = pfnProgress;
1420
0
    stepCtxt.m_pProgressData = pProgressData;
1421
0
    return RunStep(stepCtxt);
1422
0
}
1423
1424
GDALVectorPartitionAlgorithmStandalone::
1425
0
    ~GDALVectorPartitionAlgorithmStandalone() = default;
1426
//! @endcond