/src/gdal/apps/gdalalg_abstract_pipeline.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: GDAL |
4 | | * Purpose: gdal "raster/vector pipeline" subcommand |
5 | | * Author: Even Rouault <even dot rouault at spatialys.com> |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2024-2025, Even Rouault <even dot rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_conv.h" |
14 | | #include "cpl_enumerate.h" |
15 | | #include "cpl_error_internal.h" |
16 | | #include "cpl_json.h" |
17 | | |
18 | | #include "gdalalg_abstract_pipeline.h" |
19 | | #include "gdalalg_materialize.h" |
20 | | #include "gdalalg_raster_read.h" |
21 | | #include "gdalalg_raster_write.h" |
22 | | #include "gdalalg_vector_read.h" |
23 | | #include "gdalalg_tee.h" |
24 | | |
25 | | #include "vrtdataset.h" |
26 | | |
27 | | #include <algorithm> |
28 | | #include <cassert> |
29 | | |
30 | | //! @cond Doxygen_Suppress |
31 | | |
32 | | /* clang-format off */ |
33 | | constexpr const char *const apszReadParametersPrefixOmitted[] = { |
34 | | GDAL_ARG_NAME_INPUT, |
35 | | GDAL_ARG_NAME_INPUT_FORMAT, |
36 | | GDAL_ARG_NAME_OPEN_OPTION, |
37 | | GDAL_ARG_NAME_INPUT_LAYER}; |
38 | | |
39 | | constexpr const char *const apszWriteParametersPrefixOmitted[] = { |
40 | | GDAL_ARG_NAME_OUTPUT, |
41 | | GDAL_ARG_NAME_OUTPUT_FORMAT, |
42 | | GDAL_ARG_NAME_CREATION_OPTION, |
43 | | GDAL_ARG_NAME_OUTPUT_LAYER, |
44 | | GDAL_ARG_NAME_LAYER_CREATION_OPTION, |
45 | | GDAL_ARG_NAME_UPDATE, |
46 | | GDAL_ARG_NAME_OVERWRITE, |
47 | | GDAL_ARG_NAME_APPEND, |
48 | | GDAL_ARG_NAME_OVERWRITE_LAYER}; |
49 | | |
50 | | /* clang-format on */ |
51 | | |
52 | | /************************************************************************/ |
53 | | /* IsReadSpecificArgument() */ |
54 | | /************************************************************************/ |
55 | | |
56 | | /* static */ |
57 | | bool GDALAbstractPipelineAlgorithm::IsReadSpecificArgument( |
58 | | const char *pszArgName) |
59 | 0 | { |
60 | 0 | return std::find_if(std::begin(apszReadParametersPrefixOmitted), |
61 | 0 | std::end(apszReadParametersPrefixOmitted), |
62 | 0 | [pszArgName](const char *pszStr) |
63 | 0 | { return strcmp(pszStr, pszArgName) == 0; }) != |
64 | 0 | std::end(apszReadParametersPrefixOmitted); |
65 | 0 | } |
66 | | |
67 | | /************************************************************************/ |
68 | | /* IsWriteSpecificArgument() */ |
69 | | /************************************************************************/ |
70 | | |
71 | | /* static */ |
72 | | bool GDALAbstractPipelineAlgorithm::IsWriteSpecificArgument( |
73 | | const char *pszArgName) |
74 | 0 | { |
75 | 0 | return std::find_if(std::begin(apszWriteParametersPrefixOmitted), |
76 | 0 | std::end(apszWriteParametersPrefixOmitted), |
77 | 0 | [pszArgName](const char *pszStr) |
78 | 0 | { return strcmp(pszStr, pszArgName) == 0; }) != |
79 | 0 | std::end(apszWriteParametersPrefixOmitted); |
80 | 0 | } |
81 | | |
82 | | /************************************************************************/ |
83 | | /* GDALAbstractPipelineAlgorithm::CheckFirstAndLastStep() */ |
84 | | /************************************************************************/ |
85 | | |
86 | | bool GDALAbstractPipelineAlgorithm::CheckFirstAndLastStep( |
87 | | const std::vector<GDALPipelineStepAlgorithm *> &steps, |
88 | | bool forAutoComplete) const |
89 | 0 | { |
90 | 0 | if (m_bExpectReadStep && !steps.front()->CanBeFirstStep()) |
91 | 0 | { |
92 | 0 | std::set<CPLString> setFirstStepNames; |
93 | 0 | for (const auto &stepName : GetStepRegistry().GetNames()) |
94 | 0 | { |
95 | 0 | auto alg = GetStepAlg(stepName); |
96 | 0 | if (alg && alg->CanBeFirstStep() && |
97 | 0 | stepName != GDALRasterReadAlgorithm::NAME) |
98 | 0 | { |
99 | 0 | setFirstStepNames.insert(CPLString(stepName) |
100 | 0 | .replaceAll(RASTER_SUFFIX, "") |
101 | 0 | .replaceAll(VECTOR_SUFFIX, "")); |
102 | 0 | } |
103 | 0 | } |
104 | 0 | std::vector<std::string> firstStepNames{GDALRasterReadAlgorithm::NAME}; |
105 | 0 | for (const std::string &s : setFirstStepNames) |
106 | 0 | firstStepNames.push_back(s); |
107 | |
|
108 | 0 | std::string msg = "First step should be "; |
109 | 0 | for (size_t i = 0; i < firstStepNames.size(); ++i) |
110 | 0 | { |
111 | 0 | if (i == firstStepNames.size() - 1) |
112 | 0 | msg += " or "; |
113 | 0 | else if (i > 0) |
114 | 0 | msg += ", "; |
115 | 0 | msg += '\''; |
116 | 0 | msg += firstStepNames[i]; |
117 | 0 | msg += '\''; |
118 | 0 | } |
119 | |
|
120 | 0 | ReportError(CE_Failure, CPLE_AppDefined, "%s", msg.c_str()); |
121 | 0 | return false; |
122 | 0 | } |
123 | | |
124 | 0 | if (!m_bExpectReadStep) |
125 | 0 | { |
126 | 0 | if (steps.front()->CanBeFirstStep()) |
127 | 0 | { |
128 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
129 | 0 | "No read-like step like '%s' is allowed", |
130 | 0 | steps.front()->GetName().c_str()); |
131 | 0 | return false; |
132 | 0 | } |
133 | 0 | } |
134 | | |
135 | 0 | if (forAutoComplete) |
136 | 0 | return true; |
137 | | |
138 | 0 | if (m_eLastStepAsWrite == StepConstraint::CAN_NOT_BE) |
139 | 0 | { |
140 | 0 | if (steps.back()->CanBeLastStep() && !steps.back()->CanBeMiddleStep()) |
141 | 0 | { |
142 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
143 | 0 | "Last step in %s pipeline must not be a " |
144 | 0 | "write-like step.", |
145 | 0 | m_bInnerPipeline ? "an inner" : "a"); |
146 | 0 | return false; |
147 | 0 | } |
148 | 0 | } |
149 | | |
150 | 0 | for (size_t i = 1; i < steps.size() - 1; ++i) |
151 | 0 | { |
152 | 0 | if (!steps[i]->CanBeMiddleStep()) |
153 | 0 | { |
154 | 0 | if (steps[i]->CanBeFirstStep() && m_bExpectReadStep) |
155 | 0 | { |
156 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
157 | 0 | "Only first step can be '%s'", |
158 | 0 | steps[i]->GetName().c_str()); |
159 | 0 | } |
160 | 0 | else if (steps[i]->CanBeLastStep() && |
161 | 0 | m_eLastStepAsWrite != StepConstraint::CAN_NOT_BE) |
162 | 0 | { |
163 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
164 | 0 | "Only last step can be '%s'", |
165 | 0 | steps[i]->GetName().c_str()); |
166 | 0 | } |
167 | 0 | else |
168 | 0 | { |
169 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
170 | 0 | "'%s' is not allowed as an intermediate step", |
171 | 0 | steps[i]->GetName().c_str()); |
172 | 0 | return false; |
173 | 0 | } |
174 | 0 | } |
175 | 0 | } |
176 | | |
177 | 0 | if (steps.size() >= 2 && steps.back()->CanBeFirstStep() && |
178 | 0 | !steps.back()->CanBeLastStep()) |
179 | 0 | { |
180 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
181 | 0 | "'%s' is only allowed as a first step", |
182 | 0 | steps.back()->GetName().c_str()); |
183 | 0 | return false; |
184 | 0 | } |
185 | | |
186 | 0 | if (m_eLastStepAsWrite == StepConstraint::MUST_BE && |
187 | 0 | !steps.back()->CanBeLastStep()) |
188 | 0 | { |
189 | 0 | std::set<CPLString> setLastStepNames; |
190 | 0 | for (const auto &stepName : GetStepRegistry().GetNames()) |
191 | 0 | { |
192 | 0 | auto alg = GetStepAlg(stepName); |
193 | 0 | if (alg && alg->CanBeLastStep()) |
194 | 0 | { |
195 | 0 | const CPLString nameWithoutSuffix = |
196 | 0 | CPLString(stepName) |
197 | 0 | .replaceAll(RASTER_SUFFIX, "") |
198 | 0 | .replaceAll(VECTOR_SUFFIX, ""); |
199 | 0 | if (nameWithoutSuffix != GDALRasterWriteAlgorithm::NAME) |
200 | 0 | { |
201 | 0 | setLastStepNames.insert(nameWithoutSuffix); |
202 | 0 | } |
203 | 0 | } |
204 | 0 | } |
205 | 0 | std::vector<std::string> lastStepNames{GDALRasterWriteAlgorithm::NAME}; |
206 | 0 | for (const std::string &s : setLastStepNames) |
207 | 0 | lastStepNames.push_back(s); |
208 | |
|
209 | 0 | std::string msg = "Last step should be "; |
210 | 0 | for (size_t i = 0; i < lastStepNames.size(); ++i) |
211 | 0 | { |
212 | 0 | if (i == lastStepNames.size() - 1) |
213 | 0 | msg += " or "; |
214 | 0 | else if (i > 0) |
215 | 0 | msg += ", "; |
216 | 0 | msg += '\''; |
217 | 0 | msg += lastStepNames[i]; |
218 | 0 | msg += '\''; |
219 | 0 | } |
220 | |
|
221 | 0 | ReportError(CE_Failure, CPLE_AppDefined, "%s", msg.c_str()); |
222 | 0 | return false; |
223 | 0 | } |
224 | | |
225 | 0 | return true; |
226 | 0 | } |
227 | | |
228 | | /************************************************************************/ |
229 | | /* GDALAbstractPipelineAlgorithm::GetStepAlg() */ |
230 | | /************************************************************************/ |
231 | | |
232 | | std::unique_ptr<GDALPipelineStepAlgorithm> |
233 | | GDALAbstractPipelineAlgorithm::GetStepAlg(const std::string &name) const |
234 | 0 | { |
235 | 0 | auto alg = GetStepRegistry().Instantiate(name); |
236 | 0 | return std::unique_ptr<GDALPipelineStepAlgorithm>( |
237 | 0 | cpl::down_cast<GDALPipelineStepAlgorithm *>(alg.release())); |
238 | 0 | } |
239 | | |
240 | | /************************************************************************/ |
241 | | /* GetDatasetType() */ |
242 | | /************************************************************************/ |
243 | | |
244 | | /** Return GDAL_OF_RASTER, GDAL_OF_VECTOR or 0 */ |
245 | | static int GetDatasetType(GDALDataset *poDS) |
246 | 0 | { |
247 | 0 | if (poDS->GetLayerCount() > 0 && poDS->GetRasterCount() == 0) |
248 | 0 | return GDAL_OF_VECTOR; |
249 | | |
250 | 0 | if (poDS->GetLayerCount() == 0 && |
251 | 0 | (poDS->GetRasterCount() > 0 || |
252 | 0 | poDS->GetMetadata(GDAL_MDD_SUBDATASETS) != nullptr)) |
253 | 0 | { |
254 | 0 | return GDAL_OF_RASTER; |
255 | 0 | } |
256 | | |
257 | 0 | return 0; |
258 | 0 | } |
259 | | |
260 | | /************************************************************************/ |
261 | | /* GetInputDatasetType() */ |
262 | | /************************************************************************/ |
263 | | |
264 | | /** Return GDAL_OF_RASTER, GDAL_OF_VECTOR or 0 */ |
265 | | /* static */ |
266 | | int GDALAbstractPipelineAlgorithm::GetInputDatasetType( |
267 | | const GDALPipelineStepAlgorithm *alg) |
268 | 0 | { |
269 | 0 | int ret = 0; |
270 | 0 | const auto stepInputArg = alg->GetArg(GDAL_ARG_NAME_INPUT); |
271 | 0 | if (stepInputArg && stepInputArg->IsExplicitlySet() && |
272 | 0 | (stepInputArg->GetType() == GAAT_DATASET || |
273 | 0 | stepInputArg->GetType() == GAAT_DATASET_LIST)) |
274 | 0 | { |
275 | 0 | std::string inputDatasetName; |
276 | 0 | if (stepInputArg->GetType() == GAAT_DATASET) |
277 | 0 | { |
278 | 0 | inputDatasetName = |
279 | 0 | stepInputArg->Get<GDALArgDatasetValue>().GetName(); |
280 | 0 | } |
281 | 0 | else |
282 | 0 | { |
283 | 0 | auto &val = stepInputArg->Get<std::vector<GDALArgDatasetValue>>(); |
284 | 0 | if (!val.empty()) |
285 | 0 | { |
286 | 0 | inputDatasetName = val[0].GetName(); |
287 | 0 | } |
288 | 0 | } |
289 | |
|
290 | 0 | if (!inputDatasetName.empty()) |
291 | 0 | { |
292 | 0 | std::unique_ptr<GDALDataset> datasetHolder; |
293 | 0 | GDALDataset *poDS; |
294 | 0 | const auto oIter = |
295 | 0 | alg->m_oMapDatasetNameToDataset.find(inputDatasetName); |
296 | 0 | if (oIter != alg->m_oMapDatasetNameToDataset.end()) |
297 | 0 | poDS = oIter->second; |
298 | 0 | else |
299 | 0 | { |
300 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
301 | 0 | datasetHolder.reset( |
302 | 0 | GDALDataset::Open(inputDatasetName.c_str())); |
303 | 0 | poDS = datasetHolder.get(); |
304 | 0 | } |
305 | 0 | if (poDS) |
306 | 0 | { |
307 | 0 | ret = GetDatasetType(poDS); |
308 | 0 | } |
309 | 0 | } |
310 | 0 | } |
311 | 0 | return ret; |
312 | 0 | } |
313 | | |
314 | | /************************************************************************/ |
315 | | /* GDALAbstractPipelineAlgorithm::CopyStepAlgorithmFromAnother() */ |
316 | | /************************************************************************/ |
317 | | |
318 | | /** Copy arguments and other parameters from \a src to \a dst, typically |
319 | | * when turning a raster algorithm to the vector one of the same name, or |
320 | | * vice-versa. |
321 | | */ |
322 | | bool GDALAbstractPipelineAlgorithm::CopyStepAlgorithmFromAnother( |
323 | | GDALPipelineStepAlgorithm *dst, const GDALPipelineStepAlgorithm *src, |
324 | | bool maybeWriteStep) const |
325 | 0 | { |
326 | 0 | if (src->GetName() == GDALTeeStepAlgorithmAbstract::NAME) |
327 | 0 | { |
328 | 0 | const auto poSrcTeeAlg = |
329 | 0 | dynamic_cast<const GDALTeeStepAlgorithmAbstract *>(src); |
330 | 0 | auto poDstTeeAlg = dynamic_cast<GDALTeeStepAlgorithmAbstract *>(dst); |
331 | 0 | CPLAssert(poSrcTeeAlg); |
332 | 0 | CPLAssert(poDstTeeAlg); |
333 | 0 | poDstTeeAlg->CopyFilenameBindingsFrom(poSrcTeeAlg); |
334 | 0 | } |
335 | | |
336 | 0 | if (maybeWriteStep) |
337 | 0 | { |
338 | | // Propagate output parameters set at the pipeline level to the |
339 | | // "write" step |
340 | 0 | for (auto &arg : dst->GetArgs()) |
341 | 0 | { |
342 | 0 | if (!arg->IsHidden()) |
343 | 0 | { |
344 | 0 | const auto pipelineArg = GetArg(arg->GetName()); |
345 | 0 | if (pipelineArg && pipelineArg->IsExplicitlySet() && |
346 | 0 | pipelineArg->GetType() == arg->GetType()) |
347 | 0 | { |
348 | 0 | arg->SetSkipIfAlreadySet(true); |
349 | 0 | [[maybe_unused]] bool ret = arg->SetFrom(*pipelineArg); |
350 | 0 | CPLAssert(ret); |
351 | 0 | } |
352 | 0 | } |
353 | 0 | } |
354 | 0 | } |
355 | | |
356 | | // Propagate parameters set on the old algorithm to the new one |
357 | 0 | for (const auto &srcArg : src->GetArgs()) |
358 | 0 | { |
359 | 0 | if (srcArg->IsExplicitlySet()) |
360 | 0 | { |
361 | 0 | auto dstArg = dst->GetArg(srcArg->GetName()); |
362 | 0 | if (!dstArg) |
363 | 0 | { |
364 | 0 | dst->ReportError(CE_Failure, CPLE_IllegalArg, |
365 | 0 | "Option '--%s' is unknown", |
366 | 0 | srcArg->GetName().c_str()); |
367 | 0 | return false; |
368 | 0 | } |
369 | 0 | else |
370 | 0 | { |
371 | 0 | dstArg->SetSkipIfAlreadySet(true); |
372 | 0 | if (!dstArg->SetFrom(*srcArg)) |
373 | 0 | return false; |
374 | 0 | } |
375 | 0 | } |
376 | 0 | } |
377 | | |
378 | 0 | dst->m_oMapDatasetNameToDataset = |
379 | 0 | std::move(src->m_oMapDatasetNameToDataset); |
380 | 0 | dst->SetCallPath({dst->GetName()}); |
381 | 0 | dst->SetReferencePathForRelativePaths(GetReferencePathForRelativePaths()); |
382 | 0 | if (IsCalledFromCommandLine()) |
383 | 0 | dst->SetCalledFromCommandLine(); |
384 | |
|
385 | 0 | return true; |
386 | 0 | } |
387 | | |
388 | | /************************************************************************/ |
389 | | /* GDALAbstractPipelineAlgorithm::ParseCommandLineArguments() */ |
390 | | /************************************************************************/ |
391 | | |
392 | | bool GDALAbstractPipelineAlgorithm::ParseCommandLineArguments( |
393 | | const std::vector<std::string> &argsIn) |
394 | 0 | { |
395 | 0 | return ParseCommandLineArguments(argsIn, /*forAutoComplete=*/false, |
396 | 0 | /*pCurArgsForAutocomplete=*/nullptr); |
397 | 0 | } |
398 | | |
399 | | /** Parse arguments of a pipeline. |
400 | | * |
401 | | * @param argsIn Pipeline arguments |
402 | | * @param forAutoComplete true if this method is called from GetAutoComplete() |
403 | | * @param[out] pCurArgsForAutocomplete Pointer to a vector of string, or null. |
404 | | * If provided, it will contain the arguments |
405 | | * of the active pipeline. Useful for |
406 | | * completion in nested pipelines. |
407 | | */ |
408 | | bool GDALAbstractPipelineAlgorithm::ParseCommandLineArguments( |
409 | | const std::vector<std::string> &argsIn, bool forAutoComplete, |
410 | | std::vector<std::string> *pCurArgsForAutocomplete) |
411 | 0 | { |
412 | 0 | std::vector<std::string> args = argsIn; |
413 | 0 | if (pCurArgsForAutocomplete) |
414 | 0 | *pCurArgsForAutocomplete = args; |
415 | |
|
416 | 0 | if (!m_bInnerPipeline && IsCalledFromCommandLine()) |
417 | 0 | { |
418 | 0 | m_eLastStepAsWrite = StepConstraint::MUST_BE; |
419 | 0 | } |
420 | |
|
421 | 0 | if (args.size() == 1 && (args[0] == "-h" || args[0] == "--help" || |
422 | 0 | args[0] == "help" || args[0] == "--json-usage")) |
423 | 0 | { |
424 | 0 | return GDALAlgorithm::ParseCommandLineArguments(args); |
425 | 0 | } |
426 | 0 | else if (args.size() == 1 && STARTS_WITH(args[0].c_str(), "--help-doc=")) |
427 | 0 | { |
428 | 0 | m_helpDocCategory = args[0].substr(strlen("--help-doc=")); |
429 | 0 | return GDALAlgorithm::ParseCommandLineArguments({"--help-doc"}); |
430 | 0 | } |
431 | | |
432 | 0 | bool foundStepMarker = false; |
433 | |
|
434 | 0 | for (size_t i = 0; i < args.size(); ++i) |
435 | 0 | { |
436 | 0 | const auto &arg = args[i]; |
437 | 0 | if (arg == "--pipeline") |
438 | 0 | { |
439 | 0 | if (i + 1 < args.size() && |
440 | 0 | CPLString(args[i + 1]).ifind(".json") != std::string::npos) |
441 | 0 | break; |
442 | 0 | return GDALAlgorithm::ParseCommandLineArguments(args); |
443 | 0 | } |
444 | | |
445 | 0 | else if (cpl::starts_with(arg, "--pipeline=")) |
446 | 0 | { |
447 | 0 | if (CPLString(arg).ifind(".json") != std::string::npos) |
448 | 0 | break; |
449 | 0 | return GDALAlgorithm::ParseCommandLineArguments(args); |
450 | 0 | } |
451 | | |
452 | | // gdal pipeline [--quiet] "read poly.gpkg ..." |
453 | 0 | if (arg.find("read ") == 0) |
454 | 0 | return GDALAlgorithm::ParseCommandLineArguments(args); |
455 | | |
456 | 0 | if (arg == "!") |
457 | 0 | foundStepMarker = true; |
458 | 0 | } |
459 | | |
460 | 0 | bool runExistingPipeline = false; |
461 | 0 | if (!foundStepMarker && !m_executionForStreamOutput) |
462 | 0 | { |
463 | 0 | std::string osCommandLine; |
464 | 0 | for (const auto &arg : args) |
465 | 0 | { |
466 | 0 | if (((!arg.empty() && arg[0] != '-') || |
467 | 0 | cpl::starts_with(arg, "--pipeline=")) && |
468 | 0 | CPLString(arg).ifind(".json") != std::string::npos) |
469 | 0 | { |
470 | 0 | bool ret; |
471 | 0 | if (m_pipeline == arg) |
472 | 0 | ret = true; |
473 | 0 | else |
474 | 0 | { |
475 | 0 | const std::string filename = |
476 | 0 | cpl::starts_with(arg, "--pipeline=") |
477 | 0 | ? arg.substr(strlen("--pipeline=")) |
478 | 0 | : arg; |
479 | 0 | if (forAutoComplete) |
480 | 0 | { |
481 | 0 | SetParseForAutoCompletion(); |
482 | 0 | } |
483 | 0 | ret = GDALAlgorithm::ParseCommandLineArguments(args) || |
484 | 0 | forAutoComplete; |
485 | 0 | if (ret) |
486 | 0 | { |
487 | 0 | ret = m_pipeline == filename; |
488 | 0 | } |
489 | 0 | } |
490 | 0 | if (ret) |
491 | 0 | { |
492 | 0 | CPLJSONDocument oDoc; |
493 | 0 | ret = oDoc.Load(m_pipeline); |
494 | 0 | if (ret) |
495 | 0 | { |
496 | 0 | osCommandLine = |
497 | 0 | oDoc.GetRoot().GetString("command_line"); |
498 | 0 | if (osCommandLine.empty()) |
499 | 0 | { |
500 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
501 | 0 | "command_line missing in %s", |
502 | 0 | m_pipeline.c_str()); |
503 | 0 | return false; |
504 | 0 | } |
505 | | |
506 | 0 | for (const char *prefix : |
507 | 0 | {"gdal pipeline ", "gdal raster pipeline ", |
508 | 0 | "gdal vector pipeline "}) |
509 | 0 | { |
510 | 0 | if (cpl::starts_with(osCommandLine, prefix)) |
511 | 0 | osCommandLine = |
512 | 0 | osCommandLine.substr(strlen(prefix)); |
513 | 0 | } |
514 | |
|
515 | 0 | if (oDoc.GetRoot().GetBool( |
516 | 0 | "relative_paths_relative_to_this_file", true)) |
517 | 0 | { |
518 | 0 | SetReferencePathForRelativePaths( |
519 | 0 | CPLGetPathSafe(m_pipeline.c_str()).c_str()); |
520 | 0 | } |
521 | |
|
522 | 0 | runExistingPipeline = true; |
523 | 0 | } |
524 | 0 | } |
525 | 0 | if (ret) |
526 | 0 | break; |
527 | 0 | else |
528 | 0 | return false; |
529 | 0 | } |
530 | 0 | } |
531 | 0 | if (runExistingPipeline) |
532 | 0 | { |
533 | 0 | const CPLStringList aosArgs( |
534 | 0 | CSLTokenizeString(osCommandLine.c_str())); |
535 | |
|
536 | 0 | args = aosArgs; |
537 | 0 | } |
538 | 0 | } |
539 | | |
540 | 0 | if (!m_steps.empty()) |
541 | 0 | { |
542 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
543 | 0 | "ParseCommandLineArguments() can only be called once per " |
544 | 0 | "instance."); |
545 | 0 | return false; |
546 | 0 | } |
547 | | |
548 | 0 | const bool bIsGenericPipeline = |
549 | 0 | (GetInputType() == (GDAL_OF_RASTER | GDAL_OF_VECTOR)); |
550 | |
|
551 | 0 | struct Step |
552 | 0 | { |
553 | 0 | std::unique_ptr<GDALPipelineStepAlgorithm> alg{}; |
554 | 0 | std::vector<std::string> args{}; |
555 | 0 | bool alreadyChangedType = false; |
556 | 0 | bool isSubAlgorithm = false; |
557 | 0 | }; |
558 | |
|
559 | 0 | int nDatasetType = GetInputType(); |
560 | 0 | const auto SetCurStepAlg = |
561 | 0 | [this, bIsGenericPipeline, &nDatasetType]( |
562 | 0 | Step &curStep, const std::string &algName, bool firstStep) |
563 | 0 | { |
564 | 0 | if (bIsGenericPipeline) |
565 | 0 | { |
566 | 0 | if (algName == GDALRasterReadAlgorithm::NAME) |
567 | 0 | { |
568 | 0 | curStep.alg = std::make_unique<GDALRasterReadAlgorithm>(true); |
569 | 0 | } |
570 | 0 | else |
571 | 0 | { |
572 | 0 | if (nDatasetType == GDAL_OF_RASTER) |
573 | 0 | curStep.alg = GetStepAlg(algName + RASTER_SUFFIX); |
574 | 0 | else if (nDatasetType == GDAL_OF_VECTOR) |
575 | 0 | curStep.alg = GetStepAlg(algName + VECTOR_SUFFIX); |
576 | 0 | if (!curStep.alg) |
577 | 0 | curStep.alg = GetStepAlg(algName); |
578 | 0 | if (!curStep.alg) |
579 | 0 | curStep.alg = GetStepAlg(algName + RASTER_SUFFIX); |
580 | 0 | if (curStep.alg) |
581 | 0 | nDatasetType = curStep.alg->GetOutputType(); |
582 | 0 | } |
583 | 0 | } |
584 | 0 | else |
585 | 0 | { |
586 | 0 | curStep.alg = GetStepAlg(algName); |
587 | 0 | } |
588 | 0 | if (!curStep.alg) |
589 | 0 | { |
590 | 0 | ReportError(CE_Failure, CPLE_AppDefined, "unknown step name: %s", |
591 | 0 | algName.c_str()); |
592 | 0 | return false; |
593 | 0 | } |
594 | | // We don't want to accept '_PIPE_' dataset placeholder for the first |
595 | | // step of a pipeline. |
596 | 0 | curStep.alg->m_inputDatasetCanBeOmitted = |
597 | 0 | !firstStep || !m_bExpectReadStep; |
598 | 0 | curStep.alg->SetCallPath({algName}); |
599 | 0 | curStep.alg->SetReferencePathForRelativePaths( |
600 | 0 | GetReferencePathForRelativePaths()); |
601 | 0 | return true; |
602 | 0 | }; |
603 | |
|
604 | 0 | std::vector<Step> steps; |
605 | 0 | steps.resize(1); |
606 | |
|
607 | 0 | int nNestLevel = 0; |
608 | 0 | std::vector<std::string> nestedPipelineArgs; |
609 | |
|
610 | 0 | for (const auto &argIn : args) |
611 | 0 | { |
612 | 0 | std::string arg(argIn); |
613 | | |
614 | | // If outputting to stdout, automatically turn off progress bar |
615 | 0 | if (arg == "/vsistdout/") |
616 | 0 | { |
617 | 0 | auto quietArg = GetArg(GDAL_ARG_NAME_QUIET); |
618 | 0 | if (quietArg && quietArg->GetType() == GAAT_BOOLEAN) |
619 | 0 | quietArg->Set(true); |
620 | 0 | } |
621 | |
|
622 | 0 | auto &curStep = steps.back(); |
623 | |
|
624 | 0 | if (nNestLevel > 0) |
625 | 0 | { |
626 | 0 | if (arg == CLOSE_NESTED_PIPELINE) |
627 | 0 | { |
628 | 0 | if ((--nNestLevel) == 0) |
629 | 0 | { |
630 | 0 | arg = BuildNestedPipeline(curStep.alg.get(), |
631 | 0 | nestedPipelineArgs, |
632 | 0 | forAutoComplete, nullptr); |
633 | 0 | if (arg.empty()) |
634 | 0 | { |
635 | 0 | return false; |
636 | 0 | } |
637 | 0 | if (pCurArgsForAutocomplete) |
638 | 0 | *pCurArgsForAutocomplete = args; |
639 | 0 | } |
640 | 0 | else |
641 | 0 | { |
642 | 0 | nestedPipelineArgs.push_back(std::move(arg)); |
643 | 0 | continue; |
644 | 0 | } |
645 | 0 | } |
646 | 0 | else |
647 | 0 | { |
648 | 0 | if (arg == OPEN_NESTED_PIPELINE) |
649 | 0 | { |
650 | 0 | if (++nNestLevel == MAX_NESTING_LEVEL) |
651 | 0 | { |
652 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
653 | 0 | "Too many nested pipelines"); |
654 | 0 | return false; |
655 | 0 | } |
656 | 0 | } |
657 | 0 | nestedPipelineArgs.push_back(std::move(arg)); |
658 | 0 | continue; |
659 | 0 | } |
660 | 0 | } |
661 | | |
662 | 0 | if (arg == "--progress") |
663 | 0 | { |
664 | 0 | m_progressBarRequested = true; |
665 | 0 | continue; |
666 | 0 | } |
667 | 0 | if (arg == "-q" || arg == "--quiet") |
668 | 0 | { |
669 | 0 | m_quiet = true; |
670 | 0 | m_progressBarRequested = false; |
671 | 0 | continue; |
672 | 0 | } |
673 | | |
674 | 0 | if (IsCalledFromCommandLine() && (arg == "-h" || arg == "--help")) |
675 | 0 | { |
676 | 0 | if (!steps.back().alg) |
677 | 0 | steps.pop_back(); |
678 | 0 | if (steps.empty()) |
679 | 0 | { |
680 | 0 | return GDALAlgorithm::ParseCommandLineArguments(args); |
681 | 0 | } |
682 | 0 | else |
683 | 0 | { |
684 | 0 | m_stepOnWhichHelpIsRequested = std::move(steps.back().alg); |
685 | 0 | return true; |
686 | 0 | } |
687 | 0 | } |
688 | | |
689 | 0 | if (arg == "!" || arg == "|") |
690 | 0 | { |
691 | 0 | if (curStep.alg) |
692 | 0 | { |
693 | 0 | steps.resize(steps.size() + 1); |
694 | 0 | } |
695 | 0 | } |
696 | 0 | else if (arg == OPEN_NESTED_PIPELINE) |
697 | 0 | { |
698 | 0 | if (!curStep.alg) |
699 | 0 | { |
700 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
701 | 0 | "Open bracket must be placed where an input " |
702 | 0 | "dataset is expected"); |
703 | 0 | return false; |
704 | 0 | } |
705 | 0 | ++nNestLevel; |
706 | 0 | } |
707 | 0 | else if (arg == CLOSE_NESTED_PIPELINE) |
708 | 0 | { |
709 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
710 | 0 | "Closing bracket found without matching open bracket"); |
711 | 0 | return false; |
712 | 0 | } |
713 | 0 | #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA |
714 | 0 | else if (arg == "+step") |
715 | 0 | { |
716 | 0 | if (curStep.alg) |
717 | 0 | { |
718 | 0 | steps.resize(steps.size() + 1); |
719 | 0 | } |
720 | 0 | } |
721 | 0 | else if (arg.find("+gdal=") == 0) |
722 | 0 | { |
723 | 0 | const std::string algName = arg.substr(strlen("+gdal=")); |
724 | 0 | if (!SetCurStepAlg(curStep, algName, steps.size() == 1)) |
725 | 0 | return false; |
726 | 0 | } |
727 | 0 | #endif |
728 | 0 | else if (!curStep.alg) |
729 | 0 | { |
730 | 0 | std::string algName = std::move(arg); |
731 | 0 | #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA |
732 | 0 | if (!algName.empty() && algName[0] == '+') |
733 | 0 | algName = algName.substr(1); |
734 | 0 | #endif |
735 | 0 | if (!SetCurStepAlg(curStep, algName, steps.size() == 1)) |
736 | 0 | return false; |
737 | 0 | } |
738 | 0 | else |
739 | 0 | { |
740 | 0 | if (curStep.alg->HasSubAlgorithms()) |
741 | 0 | { |
742 | 0 | auto subAlg = std::unique_ptr<GDALPipelineStepAlgorithm>( |
743 | 0 | cpl::down_cast<GDALPipelineStepAlgorithm *>( |
744 | 0 | curStep.alg->InstantiateSubAlgorithm(arg).release())); |
745 | 0 | if (!subAlg) |
746 | 0 | { |
747 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
748 | 0 | "'%s' is a unknown sub-algorithm of '%s'", |
749 | 0 | arg.c_str(), curStep.alg->GetName().c_str()); |
750 | 0 | return false; |
751 | 0 | } |
752 | 0 | curStep.isSubAlgorithm = true; |
753 | 0 | subAlg->m_inputDatasetCanBeOmitted = |
754 | 0 | steps.size() > 1 || !m_bExpectReadStep; |
755 | 0 | curStep.alg = std::move(subAlg); |
756 | 0 | continue; |
757 | 0 | } |
758 | | |
759 | 0 | #ifdef GDAL_PIPELINE_PROJ_NOSTALGIA |
760 | 0 | if (!arg.empty() && arg[0] == '+' && |
761 | 0 | arg.find(' ') == std::string::npos) |
762 | 0 | { |
763 | 0 | curStep.args.push_back("--" + arg.substr(1)); |
764 | 0 | continue; |
765 | 0 | } |
766 | 0 | #endif |
767 | 0 | curStep.args.push_back(std::move(arg)); |
768 | 0 | } |
769 | 0 | } |
770 | | |
771 | 0 | if (nNestLevel > 0) |
772 | 0 | { |
773 | 0 | if (forAutoComplete) |
774 | 0 | { |
775 | 0 | BuildNestedPipeline(steps.back().alg.get(), nestedPipelineArgs, |
776 | 0 | forAutoComplete, pCurArgsForAutocomplete); |
777 | 0 | return true; |
778 | 0 | } |
779 | 0 | else |
780 | 0 | { |
781 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
782 | 0 | "Open bracket has no matching closing bracket"); |
783 | 0 | return false; |
784 | 0 | } |
785 | 0 | } |
786 | | |
787 | | // As we initially added a step without alg to bootstrap things, make |
788 | | // sure to remove it if it hasn't been filled, or the user has terminated |
789 | | // the pipeline with a '!' separator. |
790 | 0 | if (!steps.back().alg) |
791 | 0 | steps.pop_back(); |
792 | |
|
793 | 0 | if (runExistingPipeline) |
794 | 0 | { |
795 | | // Add a final "write" step if there is no explicit allowed last step |
796 | 0 | if (!steps.empty() && !steps.back().alg->CanBeLastStep()) |
797 | 0 | { |
798 | 0 | steps.resize(steps.size() + 1); |
799 | 0 | steps.back().alg = GetStepAlg( |
800 | 0 | std::string(GDALRasterWriteAlgorithm::NAME) |
801 | 0 | .append(bIsGenericPipeline ? RASTER_SUFFIX : "")); |
802 | 0 | steps.back().alg->m_inputDatasetCanBeOmitted = true; |
803 | 0 | } |
804 | | |
805 | | // Remove "--output-format=stream" and "streamed_dataset" if found |
806 | 0 | if (steps.back().alg->GetName() == GDALRasterWriteAlgorithm::NAME) |
807 | 0 | { |
808 | 0 | for (auto oIter = steps.back().args.begin(); |
809 | 0 | oIter != steps.back().args.end();) |
810 | 0 | { |
811 | 0 | if (*oIter == std::string("--") |
812 | 0 | .append(GDAL_ARG_NAME_OUTPUT_FORMAT) |
813 | 0 | .append("=stream") || |
814 | 0 | *oIter == std::string("--") |
815 | 0 | .append(GDAL_ARG_NAME_OUTPUT) |
816 | 0 | .append("=streamed_dataset") || |
817 | 0 | *oIter == "streamed_dataset") |
818 | 0 | { |
819 | 0 | oIter = steps.back().args.erase(oIter); |
820 | 0 | } |
821 | 0 | else |
822 | 0 | { |
823 | 0 | ++oIter; |
824 | 0 | } |
825 | 0 | } |
826 | 0 | } |
827 | 0 | } |
828 | |
|
829 | 0 | bool helpRequested = false; |
830 | 0 | if (IsCalledFromCommandLine()) |
831 | 0 | { |
832 | 0 | for (auto &step : steps) |
833 | 0 | step.alg->SetCalledFromCommandLine(); |
834 | |
|
835 | 0 | for (const std::string &v : args) |
836 | 0 | { |
837 | 0 | if (cpl::ends_with(v, "=?")) |
838 | 0 | helpRequested = true; |
839 | 0 | } |
840 | 0 | } |
841 | |
|
842 | 0 | if (m_eLastStepAsWrite == StepConstraint::MUST_BE) |
843 | 0 | { |
844 | 0 | if (steps.size() < 2) |
845 | 0 | { |
846 | 0 | if (!steps.empty() && helpRequested) |
847 | 0 | { |
848 | 0 | steps.back().alg->ParseCommandLineArguments(steps.back().args); |
849 | 0 | return false; |
850 | 0 | } |
851 | | |
852 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
853 | 0 | "At least 2 steps must be provided"); |
854 | 0 | return false; |
855 | 0 | } |
856 | | |
857 | 0 | if (!steps.back().alg->CanBeLastStep()) |
858 | 0 | { |
859 | 0 | if (helpRequested) |
860 | 0 | { |
861 | 0 | steps.back().alg->ParseCommandLineArguments(steps.back().args); |
862 | 0 | return false; |
863 | 0 | } |
864 | 0 | } |
865 | 0 | } |
866 | 0 | else |
867 | 0 | { |
868 | 0 | if (steps.empty()) |
869 | 0 | { |
870 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
871 | 0 | "At least one step must be provided in %s pipeline.", |
872 | 0 | m_bInnerPipeline ? "an inner" : "a"); |
873 | 0 | return false; |
874 | 0 | } |
875 | 0 | } |
876 | | |
877 | 0 | std::vector<GDALPipelineStepAlgorithm *> stepAlgs; |
878 | 0 | for (const auto &step : steps) |
879 | 0 | stepAlgs.push_back(step.alg.get()); |
880 | 0 | if (!CheckFirstAndLastStep(stepAlgs, forAutoComplete)) |
881 | 0 | return false; // CheckFirstAndLastStep emits an error |
882 | | |
883 | 0 | for (auto &step : steps) |
884 | 0 | { |
885 | 0 | step.alg->SetReferencePathForRelativePaths( |
886 | 0 | GetReferencePathForRelativePaths()); |
887 | 0 | } |
888 | |
|
889 | 0 | const auto PropagateArgsFromPipeline = |
890 | 0 | [this](GDALPipelineStepAlgorithm *alg) |
891 | 0 | { |
892 | 0 | const GDALAbstractPipelineAlgorithm *constThis = this; |
893 | 0 | for (auto &arg : alg->GetArgs()) |
894 | 0 | { |
895 | 0 | if (!arg->IsHidden()) |
896 | 0 | { |
897 | 0 | const auto pipelineArg = constThis->GetArg(arg->GetName()); |
898 | 0 | if (pipelineArg && pipelineArg->IsExplicitlySet() && |
899 | 0 | pipelineArg->GetType() == arg->GetType()) |
900 | 0 | { |
901 | 0 | arg->SetSkipIfAlreadySet(true); |
902 | 0 | arg->SetFrom(*pipelineArg); |
903 | 0 | } |
904 | 0 | } |
905 | 0 | } |
906 | 0 | }; |
907 | | |
908 | | // Propagate input parameters set at the pipeline level to the |
909 | | // "read" step |
910 | 0 | if (m_bExpectReadStep) |
911 | 0 | { |
912 | 0 | PropagateArgsFromPipeline(steps.front().alg.get()); |
913 | 0 | } |
914 | | |
915 | | // Same with "write" step |
916 | 0 | if (m_eLastStepAsWrite != StepConstraint::CAN_NOT_BE && |
917 | 0 | steps.back().alg->CanBeLastStep()) |
918 | 0 | { |
919 | 0 | PropagateArgsFromPipeline(steps.back().alg.get()); |
920 | 0 | } |
921 | |
|
922 | 0 | if (runExistingPipeline) |
923 | 0 | { |
924 | 0 | std::set<std::pair<Step *, std::string>> alreadyCleanedArgs; |
925 | |
|
926 | 0 | for (const auto &arg : GetArgs()) |
927 | 0 | { |
928 | 0 | if (arg->IsUserProvided() || |
929 | 0 | ((arg->GetName() == GDAL_ARG_NAME_INPUT || |
930 | 0 | arg->GetName() == GDAL_ARG_NAME_INPUT_LAYER || |
931 | 0 | arg->GetName() == GDAL_ARG_NAME_OUTPUT || |
932 | 0 | arg->GetName() == GDAL_ARG_NAME_OUTPUT_FORMAT) && |
933 | 0 | arg->IsExplicitlySet())) |
934 | 0 | { |
935 | 0 | CPLStringList tokens( |
936 | 0 | CSLTokenizeString2(arg->GetName().c_str(), ".", 0)); |
937 | 0 | std::string stepName; |
938 | 0 | std::string stepArgName; |
939 | 0 | if (tokens.size() == 1 && IsReadSpecificArgument(tokens[0])) |
940 | 0 | { |
941 | 0 | stepName = steps.front().alg->GetName(); |
942 | 0 | stepArgName = tokens[0]; |
943 | 0 | } |
944 | 0 | else if (tokens.size() == 1 && |
945 | 0 | IsWriteSpecificArgument(tokens[0])) |
946 | 0 | { |
947 | 0 | stepName = steps.back().alg->GetName(); |
948 | 0 | stepArgName = tokens[0]; |
949 | 0 | } |
950 | 0 | else if (tokens.size() == 2) |
951 | 0 | { |
952 | 0 | stepName = tokens[0]; |
953 | 0 | stepArgName = tokens[1]; |
954 | 0 | } |
955 | 0 | else |
956 | 0 | { |
957 | 0 | if (tokens.size() == 1) |
958 | 0 | { |
959 | 0 | const Step *matchingStep = nullptr; |
960 | 0 | for (auto &step : steps) |
961 | 0 | { |
962 | 0 | if (step.alg->GetArg(tokens[0])) |
963 | 0 | { |
964 | 0 | if (!matchingStep) |
965 | 0 | matchingStep = &step; |
966 | 0 | else |
967 | 0 | { |
968 | 0 | ReportError( |
969 | 0 | CE_Failure, CPLE_AppDefined, |
970 | 0 | "Ambiguous argument name '%s', because " |
971 | 0 | "it is valid for several steps in the " |
972 | 0 | "pipeline. It should be specified with " |
973 | 0 | "the form " |
974 | 0 | "<algorithm-name>.<argument-name>.", |
975 | 0 | tokens[0]); |
976 | 0 | return false; |
977 | 0 | } |
978 | 0 | } |
979 | 0 | } |
980 | 0 | if (!matchingStep) |
981 | 0 | { |
982 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
983 | 0 | "No step in the pipeline has an " |
984 | 0 | "argument named '%s'", |
985 | 0 | tokens[0]); |
986 | 0 | return false; |
987 | 0 | } |
988 | 0 | stepName = matchingStep->alg->GetName(); |
989 | 0 | stepArgName = tokens[0]; |
990 | 0 | } |
991 | 0 | else |
992 | 0 | { |
993 | 0 | ReportError( |
994 | 0 | CE_Failure, CPLE_AppDefined, |
995 | 0 | "Invalid argument name '%s'. It should of the " |
996 | 0 | "form <algorithm-name>.<argument-name>.", |
997 | 0 | arg->GetName().c_str()); |
998 | 0 | return false; |
999 | 0 | } |
1000 | 0 | } |
1001 | 0 | const auto nPosBracket = stepName.find('['); |
1002 | 0 | int iRequestedStepIdx = -1; |
1003 | 0 | if (nPosBracket != std::string::npos && stepName.back() == ']') |
1004 | 0 | { |
1005 | 0 | iRequestedStepIdx = |
1006 | 0 | atoi(stepName.c_str() + nPosBracket + 1); |
1007 | 0 | stepName.resize(nPosBracket); |
1008 | 0 | } |
1009 | 0 | int iMatchingStepIdx = 0; |
1010 | 0 | Step *matchingStep = nullptr; |
1011 | 0 | for (auto &step : steps) |
1012 | 0 | { |
1013 | 0 | if (step.alg->GetName() == stepName) |
1014 | 0 | { |
1015 | 0 | if (iRequestedStepIdx >= 0) |
1016 | 0 | { |
1017 | 0 | if (iRequestedStepIdx == iMatchingStepIdx) |
1018 | 0 | { |
1019 | 0 | matchingStep = &step; |
1020 | 0 | break; |
1021 | 0 | } |
1022 | 0 | ++iMatchingStepIdx; |
1023 | 0 | } |
1024 | 0 | else if (matchingStep == nullptr) |
1025 | 0 | { |
1026 | 0 | matchingStep = &step; |
1027 | 0 | } |
1028 | 0 | else |
1029 | 0 | { |
1030 | 0 | ReportError( |
1031 | 0 | CE_Failure, CPLE_AppDefined, |
1032 | 0 | "Argument '%s' is ambiguous as there are " |
1033 | 0 | "several '%s' steps in the pipeline. Qualify " |
1034 | 0 | "it as '%s[<zero-based-index>]' to remove " |
1035 | 0 | "ambiguity.", |
1036 | 0 | arg->GetName().c_str(), stepName.c_str(), |
1037 | 0 | stepName.c_str()); |
1038 | 0 | return false; |
1039 | 0 | } |
1040 | 0 | } |
1041 | 0 | } |
1042 | 0 | if (!matchingStep) |
1043 | 0 | { |
1044 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
1045 | 0 | "Argument '%s' refers to a non-existing '%s' " |
1046 | 0 | "step in the pipeline.", |
1047 | 0 | arg->GetName().c_str(), tokens[0]); |
1048 | 0 | return false; |
1049 | 0 | } |
1050 | | |
1051 | 0 | auto &step = *matchingStep; |
1052 | 0 | std::string stepArgNameDashDash = |
1053 | 0 | std::string("--").append(stepArgName); |
1054 | |
|
1055 | 0 | auto oKeyPair = std::make_pair(matchingStep, stepArgName); |
1056 | 0 | if (!cpl::contains(alreadyCleanedArgs, oKeyPair)) |
1057 | 0 | { |
1058 | 0 | alreadyCleanedArgs.insert(std::move(oKeyPair)); |
1059 | |
|
1060 | 0 | std::vector<GDALAlgorithmArg *> positionalArgs; |
1061 | 0 | for (auto &stepArg : step.alg->GetArgs()) |
1062 | 0 | { |
1063 | 0 | if (stepArg->IsPositional()) |
1064 | 0 | positionalArgs.push_back(stepArg.get()); |
1065 | 0 | } |
1066 | | |
1067 | | // Remove step arguments that match the user override |
1068 | 0 | const std::string stepArgNameDashDashEqual = |
1069 | 0 | stepArgNameDashDash + '='; |
1070 | 0 | size_t idxPositional = 0; |
1071 | 0 | for (auto oIter = step.args.begin(); |
1072 | 0 | oIter != step.args.end();) |
1073 | 0 | { |
1074 | 0 | const auto &iterArgName = *oIter; |
1075 | 0 | if (iterArgName == stepArgNameDashDash) |
1076 | 0 | { |
1077 | 0 | oIter = step.args.erase(oIter); |
1078 | 0 | auto stepArg = step.alg->GetArg(stepArgName); |
1079 | 0 | if (stepArg && stepArg->GetType() != GAAT_BOOLEAN) |
1080 | 0 | { |
1081 | 0 | if (oIter != step.args.end()) |
1082 | 0 | oIter = step.args.erase(oIter); |
1083 | 0 | } |
1084 | 0 | } |
1085 | 0 | else if (cpl::starts_with(iterArgName, |
1086 | 0 | stepArgNameDashDashEqual)) |
1087 | 0 | { |
1088 | 0 | oIter = step.args.erase(oIter); |
1089 | 0 | } |
1090 | 0 | else if (!iterArgName.empty() && iterArgName[0] == '-') |
1091 | 0 | { |
1092 | 0 | const auto equalPos = iterArgName.find('='); |
1093 | 0 | auto stepArg = step.alg->GetArg( |
1094 | 0 | equalPos == std::string::npos |
1095 | 0 | ? iterArgName |
1096 | 0 | : iterArgName.substr(0, equalPos)); |
1097 | 0 | if (stepArg && stepArg->GetName() == stepArgName) |
1098 | 0 | { |
1099 | 0 | oIter = step.args.erase(oIter); |
1100 | 0 | if (equalPos == std::string::npos && |
1101 | 0 | stepArg->GetType() != GAAT_BOOLEAN && |
1102 | 0 | oIter != step.args.end()) |
1103 | 0 | { |
1104 | 0 | oIter = step.args.erase(oIter); |
1105 | 0 | } |
1106 | 0 | } |
1107 | 0 | else |
1108 | 0 | { |
1109 | 0 | ++oIter; |
1110 | 0 | if (stepArg && equalPos == std::string::npos && |
1111 | 0 | stepArg->GetType() != GAAT_BOOLEAN && |
1112 | 0 | oIter != step.args.end()) |
1113 | 0 | { |
1114 | 0 | ++oIter; |
1115 | 0 | } |
1116 | 0 | } |
1117 | 0 | } |
1118 | 0 | else if (idxPositional < positionalArgs.size()) |
1119 | 0 | { |
1120 | 0 | if (positionalArgs[idxPositional]->GetName() == |
1121 | 0 | stepArgName) |
1122 | 0 | { |
1123 | 0 | oIter = step.args.erase(oIter); |
1124 | 0 | } |
1125 | 0 | else |
1126 | 0 | { |
1127 | 0 | ++oIter; |
1128 | 0 | } |
1129 | 0 | ++idxPositional; |
1130 | 0 | } |
1131 | 0 | else |
1132 | 0 | { |
1133 | 0 | ++oIter; |
1134 | 0 | } |
1135 | 0 | } |
1136 | 0 | } |
1137 | |
|
1138 | 0 | if (arg->IsUserProvided()) |
1139 | 0 | { |
1140 | | // Add user override |
1141 | 0 | step.args.push_back(std::move(stepArgNameDashDash)); |
1142 | 0 | auto stepArg = step.alg->GetArg(stepArgName); |
1143 | 0 | if (stepArg && stepArg->GetType() != GAAT_BOOLEAN) |
1144 | 0 | { |
1145 | 0 | step.args.push_back(arg->Get<std::string>()); |
1146 | 0 | } |
1147 | 0 | } |
1148 | 0 | } |
1149 | 0 | } |
1150 | 0 | } |
1151 | | |
1152 | 0 | int nInitialDatasetType = 0; |
1153 | 0 | if (bIsGenericPipeline) |
1154 | 0 | { |
1155 | 0 | if (!m_bExpectReadStep) |
1156 | 0 | { |
1157 | 0 | CPLAssert(m_inputDataset.size() == 1 && |
1158 | 0 | m_inputDataset[0].GetDatasetRef()); |
1159 | 0 | nInitialDatasetType = |
1160 | 0 | GetDatasetType(m_inputDataset[0].GetDatasetRef()); |
1161 | 0 | } |
1162 | | |
1163 | | // Parse each step, but without running the validation |
1164 | 0 | nDatasetType = nInitialDatasetType; |
1165 | 0 | bool firstStep = nDatasetType == 0; |
1166 | |
|
1167 | 0 | for (auto &step : steps) |
1168 | 0 | { |
1169 | 0 | bool ret = false; |
1170 | 0 | CPLErrorAccumulator oAccumulator; |
1171 | 0 | bool hasTriedRaster = false; |
1172 | 0 | if (nDatasetType == 0 || nDatasetType == GDAL_OF_RASTER) |
1173 | 0 | { |
1174 | 0 | hasTriedRaster = true; |
1175 | 0 | [[maybe_unused]] auto context = |
1176 | 0 | oAccumulator.InstallForCurrentScope(); |
1177 | 0 | step.alg->m_skipValidationInParseCommandLine = true; |
1178 | 0 | ret = step.alg->ParseCommandLineArguments(step.args); |
1179 | 0 | if (ret && nDatasetType == 0 && forAutoComplete) |
1180 | 0 | { |
1181 | 0 | ret = step.alg->ValidateArguments(); |
1182 | 0 | if (ret && firstStep && |
1183 | 0 | step.alg->m_inputDataset.size() == 1) |
1184 | 0 | { |
1185 | 0 | auto poDS = step.alg->m_inputDataset[0].GetDatasetRef(); |
1186 | 0 | if (poDS && poDS->GetLayerCount() > 0) |
1187 | 0 | ret = false; |
1188 | 0 | } |
1189 | 0 | else if (!ret && firstStep) |
1190 | 0 | ret = true; |
1191 | 0 | } |
1192 | 0 | } |
1193 | 0 | else if (!m_bExpectReadStep && |
1194 | 0 | nDatasetType == step.alg->GetInputType()) |
1195 | 0 | { |
1196 | 0 | step.alg->m_skipValidationInParseCommandLine = true; |
1197 | 0 | ret = step.alg->ParseCommandLineArguments(step.args); |
1198 | 0 | if (!ret) |
1199 | 0 | return false; |
1200 | 0 | } |
1201 | | |
1202 | 0 | if (!ret) |
1203 | 0 | { |
1204 | 0 | auto algVector = |
1205 | 0 | GetStepAlg(step.alg->GetName() + VECTOR_SUFFIX); |
1206 | 0 | if (algVector && |
1207 | 0 | (nDatasetType == 0 || nDatasetType == GDAL_OF_VECTOR)) |
1208 | 0 | { |
1209 | 0 | step.alg = std::move(algVector); |
1210 | 0 | step.alg->m_inputDatasetCanBeOmitted = |
1211 | 0 | !firstStep || !m_bExpectReadStep; |
1212 | 0 | step.alg->m_skipValidationInParseCommandLine = true; |
1213 | 0 | ret = step.alg->ParseCommandLineArguments(step.args); |
1214 | 0 | if (ret) |
1215 | 0 | { |
1216 | 0 | step.alg->SetCallPath({step.alg->GetName()}); |
1217 | 0 | step.alg->SetReferencePathForRelativePaths( |
1218 | 0 | GetReferencePathForRelativePaths()); |
1219 | 0 | step.alreadyChangedType = true; |
1220 | 0 | } |
1221 | 0 | else if (!forAutoComplete) |
1222 | 0 | return false; |
1223 | 0 | } |
1224 | 0 | if (!ret && hasTriedRaster && !forAutoComplete) |
1225 | 0 | { |
1226 | 0 | for (const auto &sError : oAccumulator.GetErrors()) |
1227 | 0 | { |
1228 | 0 | CPLError(sError.type, sError.no, "%s", |
1229 | 0 | sError.msg.c_str()); |
1230 | 0 | } |
1231 | 0 | return false; |
1232 | 0 | } |
1233 | 0 | } |
1234 | 0 | if (ret && forAutoComplete) |
1235 | 0 | nDatasetType = step.alg->GetOutputType(); |
1236 | 0 | firstStep = false; |
1237 | 0 | } |
1238 | 0 | } |
1239 | 0 | else |
1240 | 0 | { |
1241 | 0 | for (auto &step : steps) |
1242 | 0 | { |
1243 | 0 | step.alg->m_skipValidationInParseCommandLine = true; |
1244 | 0 | if (!step.alg->ParseCommandLineArguments(step.args) && |
1245 | 0 | !forAutoComplete) |
1246 | 0 | return false; |
1247 | 0 | } |
1248 | 0 | } |
1249 | | |
1250 | | // Evaluate "input" argument of "read" step, together with the "output" |
1251 | | // argument of the "write" step, in case they point to the same dataset. |
1252 | 0 | auto inputArg = steps.front().alg->GetArg(GDAL_ARG_NAME_INPUT); |
1253 | 0 | if (inputArg && inputArg->IsExplicitlySet() && |
1254 | 0 | inputArg->GetType() == GAAT_DATASET_LIST && |
1255 | 0 | inputArg->Get<std::vector<GDALArgDatasetValue>>().size() == 1) |
1256 | 0 | { |
1257 | 0 | int nCountChangeFieldTypeStepsToBeRemoved = 0; |
1258 | 0 | std::string osTmpJSONFilename; |
1259 | | |
1260 | | // Check if there are steps like change-field-type just after the read |
1261 | | // step. If so, we can convert them into a OGR_SCHEMA open option for |
1262 | | // drivers that support it. |
1263 | 0 | auto &inputVals = inputArg->Get<std::vector<GDALArgDatasetValue>>(); |
1264 | 0 | if (!inputVals[0].GetDatasetRef() && steps.size() >= 2 && |
1265 | 0 | steps[0].alg->GetName() == GDALVectorReadAlgorithm::NAME && |
1266 | 0 | !steps.back().alg->IsGDALGOutput()) |
1267 | 0 | { |
1268 | 0 | auto openOptionArgs = |
1269 | 0 | steps.front().alg->GetArg(GDAL_ARG_NAME_OPEN_OPTION); |
1270 | 0 | if (openOptionArgs && !openOptionArgs->IsExplicitlySet() && |
1271 | 0 | openOptionArgs->GetType() == GAAT_STRING_LIST) |
1272 | 0 | { |
1273 | 0 | const auto &openOptionVals = |
1274 | 0 | openOptionArgs->Get<std::vector<std::string>>(); |
1275 | 0 | if (CPLStringList(openOptionVals) |
1276 | 0 | .FetchNameValue("OGR_SCHEMA") == nullptr) |
1277 | 0 | { |
1278 | 0 | CPLJSONArray oLayers; |
1279 | 0 | for (size_t iStep = 1; iStep < steps.size(); ++iStep) |
1280 | 0 | { |
1281 | 0 | auto oObj = |
1282 | 0 | steps[iStep].alg->Get_OGR_SCHEMA_OpenOption_Layer(); |
1283 | 0 | if (!oObj.IsValid()) |
1284 | 0 | break; |
1285 | 0 | oLayers.Add(oObj); |
1286 | 0 | ++nCountChangeFieldTypeStepsToBeRemoved; |
1287 | 0 | } |
1288 | |
|
1289 | 0 | if (nCountChangeFieldTypeStepsToBeRemoved > 0) |
1290 | 0 | { |
1291 | 0 | CPLJSONDocument oDoc; |
1292 | 0 | oDoc.GetRoot().Set("layers", oLayers); |
1293 | 0 | osTmpJSONFilename = |
1294 | 0 | VSIMemGenerateHiddenFilename(nullptr); |
1295 | | // CPLDebug("GDAL", "OGR_SCHEMA: %s", oDoc.SaveAsString().c_str()); |
1296 | 0 | oDoc.Save(osTmpJSONFilename); |
1297 | |
|
1298 | 0 | openOptionArgs->Set(std::vector<std::string>{ |
1299 | 0 | std::string("@OGR_SCHEMA=") |
1300 | 0 | .append(osTmpJSONFilename)}); |
1301 | 0 | } |
1302 | 0 | } |
1303 | 0 | } |
1304 | 0 | } |
1305 | |
|
1306 | 0 | const bool bOK = steps.front().alg->ProcessDatasetArg( |
1307 | 0 | inputArg, steps.back().alg.get()) || |
1308 | 0 | forAutoComplete; |
1309 | |
|
1310 | 0 | if (!osTmpJSONFilename.empty()) |
1311 | 0 | VSIUnlink(osTmpJSONFilename.c_str()); |
1312 | |
|
1313 | 0 | if (!bOK) |
1314 | 0 | { |
1315 | 0 | return false; |
1316 | 0 | } |
1317 | | |
1318 | | // Now check if the driver of the input dataset actually supports |
1319 | | // the OGR_SCHEMA open option. If so, we can remove the steps from |
1320 | | // the pipeline |
1321 | 0 | if (nCountChangeFieldTypeStepsToBeRemoved) |
1322 | 0 | { |
1323 | 0 | if (auto poDS = inputVals[0].GetDatasetRef()) |
1324 | 0 | { |
1325 | 0 | if (auto poDriver = poDS->GetDriver()) |
1326 | 0 | { |
1327 | 0 | const char *pszOpenOptionList = |
1328 | 0 | poDriver->GetMetadataItem(GDAL_DMD_OPENOPTIONLIST); |
1329 | 0 | if (pszOpenOptionList && |
1330 | 0 | strstr(pszOpenOptionList, "OGR_SCHEMA")) |
1331 | 0 | { |
1332 | 0 | CPLDebug("GDAL", |
1333 | 0 | "Merging %d step(s) as OGR_SCHEMA open option", |
1334 | 0 | nCountChangeFieldTypeStepsToBeRemoved); |
1335 | 0 | steps.erase(steps.begin() + 1, |
1336 | 0 | steps.begin() + 1 + |
1337 | 0 | nCountChangeFieldTypeStepsToBeRemoved); |
1338 | 0 | } |
1339 | 0 | } |
1340 | 0 | } |
1341 | 0 | } |
1342 | 0 | } |
1343 | | |
1344 | 0 | if (bIsGenericPipeline) |
1345 | 0 | { |
1346 | 0 | int nLastStepOutputType = nInitialDatasetType; |
1347 | 0 | if (m_bExpectReadStep) |
1348 | 0 | { |
1349 | 0 | nLastStepOutputType = GDAL_OF_VECTOR; |
1350 | 0 | if (steps.front().alg->GetName() != |
1351 | 0 | std::string(GDALRasterReadAlgorithm::NAME) && |
1352 | 0 | steps.front().alg->GetOutputType() == GDAL_OF_RASTER) |
1353 | 0 | { |
1354 | 0 | nLastStepOutputType = GDAL_OF_RASTER; |
1355 | 0 | } |
1356 | 0 | else |
1357 | 0 | { |
1358 | 0 | auto &inputDatasets = steps.front().alg->GetInputDatasets(); |
1359 | 0 | if (!inputDatasets.empty()) |
1360 | 0 | { |
1361 | 0 | auto poSrcDS = inputDatasets[0].GetDatasetRef(); |
1362 | 0 | if (poSrcDS) |
1363 | 0 | { |
1364 | 0 | if (poSrcDS->GetRasterCount() != 0) |
1365 | 0 | nLastStepOutputType = GDAL_OF_RASTER; |
1366 | 0 | } |
1367 | 0 | } |
1368 | 0 | } |
1369 | 0 | } |
1370 | |
|
1371 | 0 | for (size_t i = |
1372 | 0 | ((m_bExpectReadStep && steps[0].alg->GetOutputType() != 0) |
1373 | 0 | ? 1 |
1374 | 0 | : 0); |
1375 | 0 | !forAutoComplete && i < steps.size(); ++i) |
1376 | 0 | { |
1377 | 0 | auto &step = steps[i]; |
1378 | |
|
1379 | 0 | if (!step.alreadyChangedType && !step.isSubAlgorithm && |
1380 | 0 | GetStepAlg(step.alg->GetName()) == nullptr) |
1381 | 0 | { |
1382 | | // GetInputDatasetType() is to deal with pipelines like |
1383 | | // gdal pipeline read input_vector ! clip --input raster_dataset --like _PIPE_ ! write output_raster |
1384 | 0 | const int nThisDatasetType = |
1385 | 0 | GetInputDatasetType(step.alg.get()); |
1386 | 0 | const int nThisStepType = |
1387 | 0 | nThisDatasetType ? nThisDatasetType : nLastStepOutputType; |
1388 | |
|
1389 | 0 | if (step.alg->GetInputType() != 0 && |
1390 | 0 | nThisStepType != step.alg->GetInputType()) |
1391 | 0 | { |
1392 | 0 | auto newAlg = GetStepAlg(step.alg->GetName() + |
1393 | 0 | (nThisStepType == GDAL_OF_RASTER |
1394 | 0 | ? RASTER_SUFFIX |
1395 | 0 | : VECTOR_SUFFIX)); |
1396 | 0 | CPLAssert(newAlg); |
1397 | | |
1398 | 0 | const bool maybeWriteStep = |
1399 | 0 | (i == steps.size() - 1 && |
1400 | 0 | m_eLastStepAsWrite != StepConstraint::CAN_NOT_BE); |
1401 | |
|
1402 | 0 | if (!CopyStepAlgorithmFromAnother( |
1403 | 0 | newAlg.get(), step.alg.get(), maybeWriteStep)) |
1404 | 0 | return false; |
1405 | | |
1406 | 0 | newAlg->m_inputDatasetCanBeOmitted = |
1407 | 0 | i > 0 || !m_bExpectReadStep; |
1408 | 0 | step.alg = std::move(newAlg); |
1409 | 0 | step.alreadyChangedType = true; |
1410 | 0 | } |
1411 | 0 | } |
1412 | | |
1413 | 0 | if (i > 0) |
1414 | 0 | { |
1415 | 0 | bool emitError = |
1416 | 0 | (step.alg->GetInputType() != 0 && |
1417 | 0 | step.alg->GetInputType() != nLastStepOutputType); |
1418 | | |
1419 | | // Check if a dataset argument, which has as value the |
1420 | | // placeholder value, has the same dataset type as the output |
1421 | | // of the last step |
1422 | 0 | for (const auto &arg : step.alg->GetArgs()) |
1423 | 0 | { |
1424 | 0 | if (!arg->IsOutput() && |
1425 | 0 | (arg->GetType() == GAAT_DATASET || |
1426 | 0 | arg->GetType() == GAAT_DATASET_LIST)) |
1427 | 0 | { |
1428 | 0 | if (arg->GetType() == GAAT_DATASET) |
1429 | 0 | { |
1430 | 0 | if (arg->Get<GDALArgDatasetValue>().GetName() == |
1431 | 0 | GDAL_DATASET_PIPELINE_PLACEHOLDER_VALUE) |
1432 | 0 | { |
1433 | 0 | if ((arg->GetDatasetType() & |
1434 | 0 | nLastStepOutputType) != 0) |
1435 | 0 | { |
1436 | 0 | emitError = false; |
1437 | 0 | break; |
1438 | 0 | } |
1439 | 0 | } |
1440 | 0 | } |
1441 | 0 | else |
1442 | 0 | { |
1443 | 0 | CPLAssert(arg->GetType() == GAAT_DATASET_LIST); |
1444 | 0 | auto &val = |
1445 | 0 | arg->Get<std::vector<GDALArgDatasetValue>>(); |
1446 | 0 | if (val.size() == 1 && |
1447 | 0 | val[0].GetName() == |
1448 | 0 | GDAL_DATASET_PIPELINE_PLACEHOLDER_VALUE) |
1449 | 0 | { |
1450 | 0 | if ((arg->GetDatasetType() & |
1451 | 0 | nLastStepOutputType) != 0) |
1452 | 0 | { |
1453 | 0 | emitError = false; |
1454 | 0 | break; |
1455 | 0 | } |
1456 | 0 | } |
1457 | 0 | } |
1458 | 0 | } |
1459 | 0 | } |
1460 | 0 | if (emitError) |
1461 | 0 | { |
1462 | 0 | ReportError( |
1463 | 0 | CE_Failure, CPLE_AppDefined, |
1464 | 0 | "Step '%s' expects a %s input dataset, but " |
1465 | 0 | "previous step '%s' " |
1466 | 0 | "generates a %s output dataset", |
1467 | 0 | step.alg->GetName().c_str(), |
1468 | 0 | step.alg->GetInputType() == GDAL_OF_RASTER ? "raster" |
1469 | 0 | : step.alg->GetInputType() == GDAL_OF_VECTOR |
1470 | 0 | ? "vector" |
1471 | 0 | : "unknown", |
1472 | 0 | steps[i - 1].alg->GetName().c_str(), |
1473 | 0 | nLastStepOutputType == GDAL_OF_RASTER ? "raster" |
1474 | 0 | : nLastStepOutputType == GDAL_OF_VECTOR ? "vector" |
1475 | 0 | : "unknown"); |
1476 | 0 | return false; |
1477 | 0 | } |
1478 | 0 | } |
1479 | | |
1480 | 0 | nLastStepOutputType = step.alg->GetOutputType(); |
1481 | 0 | if (!forAutoComplete && nLastStepOutputType == 0) |
1482 | 0 | { |
1483 | | // If this step has no precise output dataset (unique instance |
1484 | | // at time of writing is 'external'), we stop trying to fix |
1485 | | // the raster/vector nature of ambiguous steps for now, and |
1486 | | // defer doing that during pipeline execution itself. |
1487 | 0 | m_nFirstStepWithUnknownInputType = static_cast<int>(i + 1); |
1488 | 0 | break; |
1489 | 0 | } |
1490 | 0 | } |
1491 | 0 | } |
1492 | | |
1493 | 0 | int iStep = 0; |
1494 | 0 | for (const auto &step : steps) |
1495 | 0 | { |
1496 | 0 | if (iStep == m_nFirstStepWithUnknownInputType) |
1497 | 0 | break; |
1498 | 0 | if (!step.alg->ValidateArguments() && !forAutoComplete) |
1499 | 0 | return false; |
1500 | 0 | ++iStep; |
1501 | 0 | } |
1502 | | |
1503 | 0 | for (auto &step : steps) |
1504 | 0 | m_steps.push_back(std::move(step.alg)); |
1505 | |
|
1506 | 0 | return true; |
1507 | 0 | } |
1508 | | |
1509 | | /************************************************************************/ |
1510 | | /* GDALAbstractPipelineAlgorithm::BuildNestedPipeline() */ |
1511 | | /************************************************************************/ |
1512 | | |
1513 | | /** Build a nested pipeline |
1514 | | * |
1515 | | * @param curAlg Current algorithm for which the nested pipeline will be a child. |
1516 | | * e.g in "gdal pipeline read ... ! clip --like [ ... ]", |
1517 | | * curAlg is "clip". |
1518 | | * @param nestedPipelineArgs Arguments of the nested pipeline, i.e. values |
1519 | | * between square brackets. |
1520 | | * @param forAutoComplete true if this method is called from GetAutoComplete() |
1521 | | * @param[out] pCurArgsForAutocomplete Pointer to a vector of string, or null. |
1522 | | * If provided, it will contain the arguments |
1523 | | * of the active pipeline. Useful for |
1524 | | * completion in nested pipelines. |
1525 | | */ |
1526 | | std::string GDALAbstractPipelineAlgorithm::BuildNestedPipeline( |
1527 | | GDALPipelineStepAlgorithm *curAlg, |
1528 | | std::vector<std::string> &nestedPipelineArgs, bool forAutoComplete, |
1529 | | std::vector<std::string> *pCurArgsForAutocomplete) |
1530 | 0 | { |
1531 | 0 | std::string datasetNameOut; |
1532 | 0 | CPLAssert(curAlg); |
1533 | | |
1534 | 0 | auto nestedPipeline = CreateNestedPipeline(); |
1535 | 0 | if (curAlg->GetName() == GDALTeeStepAlgorithmAbstract::NAME) |
1536 | 0 | nestedPipeline->m_bExpectReadStep = false; |
1537 | 0 | else |
1538 | 0 | nestedPipeline->m_eLastStepAsWrite = StepConstraint::CAN_NOT_BE; |
1539 | 0 | nestedPipeline->m_executionForStreamOutput = m_executionForStreamOutput; |
1540 | 0 | if (IsCalledFromCommandLine()) |
1541 | 0 | nestedPipeline->SetCalledFromCommandLine(); |
1542 | 0 | nestedPipeline->SetReferencePathForRelativePaths( |
1543 | 0 | GetReferencePathForRelativePaths()); |
1544 | |
|
1545 | 0 | std::string argsStr = OPEN_NESTED_PIPELINE; |
1546 | 0 | for (const std::string &str : nestedPipelineArgs) |
1547 | 0 | { |
1548 | 0 | argsStr += ' '; |
1549 | 0 | argsStr += GDALAlgorithmArg::GetEscapedString(str); |
1550 | 0 | } |
1551 | 0 | argsStr += ' '; |
1552 | 0 | argsStr += CLOSE_NESTED_PIPELINE; |
1553 | |
|
1554 | 0 | if (curAlg->GetName() != GDALTeeStepAlgorithmAbstract::NAME) |
1555 | 0 | { |
1556 | 0 | if (!nestedPipeline->ParseCommandLineArguments( |
1557 | 0 | nestedPipelineArgs, forAutoComplete, pCurArgsForAutocomplete) || |
1558 | 0 | (!forAutoComplete && !nestedPipeline->Run())) |
1559 | 0 | { |
1560 | 0 | return datasetNameOut; |
1561 | 0 | } |
1562 | 0 | auto poDS = nestedPipeline->GetOutputDataset().GetDatasetRef(); |
1563 | 0 | if (!poDS) |
1564 | 0 | { |
1565 | | // That shouldn't happen normally for well-behaved algorithms, but |
1566 | | // it doesn't hurt checking. |
1567 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
1568 | 0 | "Nested pipeline does not generate an output dataset"); |
1569 | 0 | return datasetNameOut; |
1570 | 0 | } |
1571 | 0 | datasetNameOut = |
1572 | 0 | CPLSPrintf("$$nested_pipeline_%p$$", nestedPipeline.get()); |
1573 | 0 | curAlg->m_oMapDatasetNameToDataset[datasetNameOut] = poDS; |
1574 | |
|
1575 | 0 | poDS->SetDescription(argsStr.c_str()); |
1576 | 0 | auto poVRTDataset = dynamic_cast<VRTDataset *>(poDS); |
1577 | 0 | if (poVRTDataset) |
1578 | 0 | poVRTDataset->SetWritable(false); |
1579 | 0 | } |
1580 | | |
1581 | 0 | m_apoNestedPipelines.emplace_back(std::move(nestedPipeline)); |
1582 | |
|
1583 | 0 | if (curAlg->GetName() == GDALTeeStepAlgorithmAbstract::NAME) |
1584 | 0 | { |
1585 | 0 | auto teeAlg = dynamic_cast<GDALTeeStepAlgorithmAbstract *>(curAlg); |
1586 | 0 | if (teeAlg) |
1587 | 0 | { |
1588 | 0 | datasetNameOut = std::move(argsStr); |
1589 | 0 | if (!teeAlg->BindFilename(datasetNameOut, |
1590 | 0 | m_apoNestedPipelines.back().get(), |
1591 | 0 | nestedPipelineArgs)) |
1592 | 0 | { |
1593 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
1594 | 0 | "Another identical nested pipeline exists"); |
1595 | 0 | datasetNameOut.clear(); |
1596 | 0 | } |
1597 | 0 | } |
1598 | 0 | } |
1599 | |
|
1600 | 0 | nestedPipelineArgs.clear(); |
1601 | |
|
1602 | 0 | return datasetNameOut; |
1603 | 0 | } |
1604 | | |
1605 | | /************************************************************************/ |
1606 | | /* GDALAbstractPipelineAlgorithm::GetAutoComplete() */ |
1607 | | /************************************************************************/ |
1608 | | |
1609 | | std::vector<std::string> |
1610 | | GDALAbstractPipelineAlgorithm::GetAutoComplete(std::vector<std::string> &argsIn, |
1611 | | bool lastWordIsComplete, |
1612 | | bool showAllOptions) |
1613 | 0 | { |
1614 | 0 | std::vector<std::string> args; |
1615 | 0 | { |
1616 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
1617 | 0 | ParseCommandLineArguments(argsIn, /*forAutoComplete=*/true, &args); |
1618 | 0 | } |
1619 | 0 | VSIStatBufL sStat; |
1620 | 0 | if (!m_pipeline.empty() && VSIStatL(m_pipeline.c_str(), &sStat) == 0 && |
1621 | 0 | !m_steps.empty() && !args.empty()) |
1622 | 0 | { |
1623 | 0 | std::map<std::string, std::vector<GDALAlgorithm *>> mapSteps; |
1624 | 0 | for (const auto &step : m_steps) |
1625 | 0 | { |
1626 | 0 | mapSteps[step->GetName()].push_back(step.get()); |
1627 | 0 | } |
1628 | |
|
1629 | 0 | std::vector<std::string> ret; |
1630 | 0 | const auto &lastArg = args.back(); |
1631 | 0 | if (!lastArg.empty() && lastArg[0] == '-' && |
1632 | 0 | lastArg.find('=') == std::string::npos && !lastWordIsComplete) |
1633 | 0 | { |
1634 | 0 | for (const auto &step : m_steps) |
1635 | 0 | { |
1636 | 0 | const int iterCount = |
1637 | 0 | static_cast<int>(mapSteps[step->GetName()].size()); |
1638 | 0 | for (int i = 0; i < iterCount; ++i) |
1639 | 0 | { |
1640 | 0 | for (const auto &arg : step->GetArgs()) |
1641 | 0 | { |
1642 | 0 | if (!arg->IsHiddenForCLI() && |
1643 | 0 | arg->GetCategory() != GAAC_COMMON) |
1644 | 0 | { |
1645 | 0 | std::string s = std::string("--"); |
1646 | 0 | if (!((step->GetName() == |
1647 | 0 | GDALRasterReadAlgorithm::NAME && |
1648 | 0 | IsReadSpecificArgument( |
1649 | 0 | arg->GetName().c_str())) || |
1650 | 0 | (step->GetName() == |
1651 | 0 | GDALRasterWriteAlgorithm::NAME && |
1652 | 0 | IsWriteSpecificArgument( |
1653 | 0 | arg->GetName().c_str())))) |
1654 | 0 | { |
1655 | 0 | s += step->GetName(); |
1656 | 0 | if (iterCount > 1) |
1657 | 0 | { |
1658 | 0 | s += '['; |
1659 | 0 | s += std::to_string(i); |
1660 | 0 | s += ']'; |
1661 | 0 | } |
1662 | 0 | s += '.'; |
1663 | 0 | } |
1664 | 0 | s += arg->GetName(); |
1665 | 0 | if (arg->GetType() == GAAT_BOOLEAN) |
1666 | 0 | ret.push_back(std::move(s)); |
1667 | 0 | else |
1668 | 0 | ret.push_back(s + "="); |
1669 | 0 | } |
1670 | 0 | } |
1671 | 0 | } |
1672 | 0 | } |
1673 | 0 | } |
1674 | 0 | else if (cpl::starts_with(lastArg, "--") && |
1675 | 0 | lastArg.find('=') != std::string::npos && !lastWordIsComplete) |
1676 | 0 | { |
1677 | 0 | const auto nDotPos = lastArg.find('.'); |
1678 | 0 | std::string stepName; |
1679 | 0 | std::string argName; |
1680 | 0 | int idx = 0; |
1681 | 0 | if (nDotPos != std::string::npos) |
1682 | 0 | { |
1683 | 0 | stepName = lastArg.substr(strlen("--"), nDotPos - strlen("--")); |
1684 | 0 | const auto nBracketPos = stepName.find('['); |
1685 | 0 | if (nBracketPos != std::string::npos) |
1686 | 0 | { |
1687 | 0 | idx = atoi(stepName.c_str() + nBracketPos + 1); |
1688 | 0 | stepName.resize(nBracketPos); |
1689 | 0 | } |
1690 | 0 | argName = "--" + lastArg.substr(nDotPos + 1); |
1691 | 0 | } |
1692 | 0 | else |
1693 | 0 | { |
1694 | 0 | argName = lastArg; |
1695 | 0 | for (const char *prefix : apszReadParametersPrefixOmitted) |
1696 | 0 | { |
1697 | 0 | if (cpl::starts_with(lastArg.substr(strlen("--")), |
1698 | 0 | std::string(prefix) + "=")) |
1699 | 0 | { |
1700 | 0 | stepName = GDALRasterReadAlgorithm::NAME; |
1701 | 0 | break; |
1702 | 0 | } |
1703 | 0 | } |
1704 | |
|
1705 | 0 | for (const char *prefix : apszWriteParametersPrefixOmitted) |
1706 | 0 | { |
1707 | 0 | if (cpl::starts_with(lastArg.substr(strlen("--")), |
1708 | 0 | std::string(prefix) + "=")) |
1709 | 0 | { |
1710 | 0 | stepName = GDALRasterWriteAlgorithm::NAME; |
1711 | 0 | break; |
1712 | 0 | } |
1713 | 0 | } |
1714 | 0 | } |
1715 | |
|
1716 | 0 | auto iter = mapSteps.find(stepName); |
1717 | 0 | if (iter != mapSteps.end() && idx >= 0 && |
1718 | 0 | static_cast<size_t>(idx) < iter->second.size()) |
1719 | 0 | { |
1720 | 0 | auto &step = iter->second[idx]; |
1721 | 0 | std::vector<std::string> subArgs; |
1722 | 0 | for (const auto &arg : step->GetArgs()) |
1723 | 0 | { |
1724 | 0 | std::string strArg; |
1725 | 0 | if (arg->IsExplicitlySet() && |
1726 | 0 | arg->Serialize(strArg, /* absolutePath=*/false)) |
1727 | 0 | { |
1728 | 0 | subArgs.push_back(std::move(strArg)); |
1729 | 0 | } |
1730 | 0 | } |
1731 | 0 | subArgs.push_back(std::move(argName)); |
1732 | 0 | ret = step->GetAutoComplete(subArgs, lastWordIsComplete, |
1733 | 0 | showAllOptions); |
1734 | 0 | } |
1735 | 0 | } |
1736 | 0 | return ret; |
1737 | 0 | } |
1738 | 0 | else |
1739 | 0 | { |
1740 | 0 | std::vector<std::string> ret; |
1741 | 0 | std::set<std::string> setSuggestions; |
1742 | 0 | if (args.size() <= 1) |
1743 | 0 | { |
1744 | 0 | for (const std::string &name : GetStepRegistry().GetNames()) |
1745 | 0 | { |
1746 | 0 | auto alg = GetStepRegistry().Instantiate(name); |
1747 | 0 | auto stepAlg = |
1748 | 0 | dynamic_cast<GDALPipelineStepAlgorithm *>(alg.get()); |
1749 | 0 | if (stepAlg && stepAlg->CanBeFirstStep()) |
1750 | 0 | { |
1751 | 0 | std::string suggestionName = |
1752 | 0 | CPLString(name) |
1753 | 0 | .replaceAll(RASTER_SUFFIX, "") |
1754 | 0 | .replaceAll(VECTOR_SUFFIX, ""); |
1755 | 0 | if (!cpl::contains(setSuggestions, suggestionName)) |
1756 | 0 | { |
1757 | 0 | if (!args.empty() && suggestionName == args[0]) |
1758 | 0 | return {}; |
1759 | 0 | if (args.empty() || |
1760 | 0 | cpl::starts_with(suggestionName, args[0])) |
1761 | 0 | { |
1762 | 0 | setSuggestions.insert(suggestionName); |
1763 | 0 | ret.push_back(std::move(suggestionName)); |
1764 | 0 | } |
1765 | 0 | } |
1766 | 0 | } |
1767 | 0 | } |
1768 | 0 | } |
1769 | 0 | else |
1770 | 0 | { |
1771 | 0 | int nDatasetType = GetInputType(); |
1772 | 0 | constexpr int MIXED_TYPE = GDAL_OF_RASTER | GDAL_OF_VECTOR; |
1773 | 0 | const bool isMixedTypePipeline = nDatasetType == MIXED_TYPE; |
1774 | 0 | std::string lastStep = args[0]; |
1775 | 0 | std::vector<std::string> lastArgs; |
1776 | 0 | bool firstStep = true; |
1777 | 0 | bool foundSlowStep = false; |
1778 | 0 | for (size_t i = 1; i < args.size(); ++i) |
1779 | 0 | { |
1780 | 0 | if (firstStep && isMixedTypePipeline && |
1781 | 0 | nDatasetType == MIXED_TYPE && !args[i].empty() && |
1782 | 0 | args[i][0] != '-') |
1783 | 0 | { |
1784 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
1785 | 0 | auto poDS = std::unique_ptr<GDALDataset>( |
1786 | 0 | GDALDataset::Open(args[i].c_str())); |
1787 | 0 | if (poDS) |
1788 | 0 | { |
1789 | 0 | const int nThisDatasetType = GetDatasetType(poDS.get()); |
1790 | 0 | if (nThisDatasetType) |
1791 | 0 | nDatasetType = nThisDatasetType; |
1792 | 0 | } |
1793 | 0 | } |
1794 | 0 | lastArgs.push_back(args[i]); |
1795 | 0 | if (i + 1 < args.size() && args[i] == "!") |
1796 | 0 | { |
1797 | 0 | firstStep = false; |
1798 | 0 | ++i; |
1799 | 0 | lastArgs.clear(); |
1800 | 0 | lastStep = args[i]; |
1801 | 0 | auto curAlg = GetStepAlg(lastStep); |
1802 | 0 | if (isMixedTypePipeline && !curAlg) |
1803 | 0 | { |
1804 | 0 | if (nDatasetType == GDAL_OF_RASTER) |
1805 | 0 | curAlg = GetStepAlg(lastStep + RASTER_SUFFIX); |
1806 | 0 | else if (nDatasetType == GDAL_OF_VECTOR) |
1807 | 0 | curAlg = GetStepAlg(lastStep + VECTOR_SUFFIX); |
1808 | 0 | } |
1809 | 0 | if (curAlg) |
1810 | 0 | { |
1811 | 0 | foundSlowStep = |
1812 | 0 | foundSlowStep || |
1813 | 0 | !curAlg->IsNativelyStreamingCompatible(); |
1814 | 0 | nDatasetType = curAlg->GetOutputType(); |
1815 | 0 | } |
1816 | 0 | } |
1817 | 0 | } |
1818 | |
|
1819 | 0 | if (args.back() == "!" || |
1820 | 0 | (args[args.size() - 2] == "!" && !GetStepAlg(args.back()) && |
1821 | 0 | !GetStepAlg(args.back() + RASTER_SUFFIX) && |
1822 | 0 | !GetStepAlg(args.back() + VECTOR_SUFFIX))) |
1823 | 0 | { |
1824 | 0 | for (const std::string &name : GetStepRegistry().GetNames()) |
1825 | 0 | { |
1826 | 0 | auto alg = GetStepRegistry().Instantiate(name); |
1827 | 0 | auto stepAlg = |
1828 | 0 | dynamic_cast<GDALPipelineStepAlgorithm *>(alg.get()); |
1829 | 0 | if (stepAlg && isMixedTypePipeline && |
1830 | 0 | nDatasetType != MIXED_TYPE && |
1831 | 0 | stepAlg->GetInputType() != nDatasetType) |
1832 | 0 | { |
1833 | 0 | continue; |
1834 | 0 | } |
1835 | 0 | if (stepAlg && !stepAlg->CanBeFirstStep()) |
1836 | 0 | { |
1837 | 0 | std::string suggestionName = |
1838 | 0 | CPLString(name) |
1839 | 0 | .replaceAll(RASTER_SUFFIX, "") |
1840 | 0 | .replaceAll(VECTOR_SUFFIX, ""); |
1841 | 0 | if (!cpl::contains(setSuggestions, suggestionName)) |
1842 | 0 | { |
1843 | 0 | setSuggestions.insert(suggestionName); |
1844 | 0 | ret.push_back(std::move(suggestionName)); |
1845 | 0 | } |
1846 | 0 | } |
1847 | 0 | } |
1848 | 0 | } |
1849 | 0 | else |
1850 | 0 | { |
1851 | 0 | if (!foundSlowStep) |
1852 | 0 | { |
1853 | | // Try to run the pipeline so that the last step gets its |
1854 | | // input dataset. |
1855 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
1856 | 0 | GDALPipelineStepRunContext ctxt; |
1857 | 0 | RunStep(ctxt); |
1858 | 0 | if (!m_steps.empty() && |
1859 | 0 | m_steps.back()->GetName() == lastStep) |
1860 | 0 | { |
1861 | 0 | return m_steps.back()->GetAutoComplete( |
1862 | 0 | lastArgs, lastWordIsComplete, |
1863 | 0 | /* showAllOptions = */ false); |
1864 | 0 | } |
1865 | 0 | } |
1866 | | |
1867 | 0 | auto curAlg = GetStepAlg(lastStep); |
1868 | 0 | if (isMixedTypePipeline && !curAlg) |
1869 | 0 | { |
1870 | 0 | if (nDatasetType == GDAL_OF_RASTER) |
1871 | 0 | curAlg = GetStepAlg(lastStep + RASTER_SUFFIX); |
1872 | 0 | else if (nDatasetType == GDAL_OF_VECTOR) |
1873 | 0 | curAlg = GetStepAlg(lastStep + VECTOR_SUFFIX); |
1874 | 0 | else |
1875 | 0 | { |
1876 | 0 | for (const char *suffix : |
1877 | 0 | {RASTER_SUFFIX, VECTOR_SUFFIX}) |
1878 | 0 | { |
1879 | 0 | curAlg = GetStepAlg(lastStep + suffix); |
1880 | 0 | if (curAlg) |
1881 | 0 | { |
1882 | 0 | for (const auto &v : curAlg->GetAutoComplete( |
1883 | 0 | lastArgs, lastWordIsComplete, |
1884 | 0 | /* showAllOptions = */ false)) |
1885 | 0 | { |
1886 | 0 | if (!cpl::contains(setSuggestions, v)) |
1887 | 0 | { |
1888 | 0 | setSuggestions.insert(v); |
1889 | 0 | ret.push_back(std::move(v)); |
1890 | 0 | } |
1891 | 0 | } |
1892 | 0 | } |
1893 | 0 | } |
1894 | 0 | curAlg.reset(); |
1895 | 0 | } |
1896 | 0 | } |
1897 | 0 | if (curAlg) |
1898 | 0 | { |
1899 | 0 | ret = curAlg->GetAutoComplete(lastArgs, lastWordIsComplete, |
1900 | 0 | /* showAllOptions = */ false); |
1901 | 0 | } |
1902 | 0 | } |
1903 | 0 | } |
1904 | 0 | return ret; |
1905 | 0 | } |
1906 | 0 | } |
1907 | | |
1908 | | /************************************************************************/ |
1909 | | /* GDALAbstractPipelineAlgorithm::SaveGDALGIntoFileOrString() */ |
1910 | | /************************************************************************/ |
1911 | | |
1912 | | /** Save the pipeline either into the file of name outFilename, if |
1913 | | * outFilename is not empty, or into the output string outString if |
1914 | | * outFilename is empty. |
1915 | | */ |
1916 | | bool GDALAbstractPipelineAlgorithm::SaveGDALGIntoFileOrString( |
1917 | | const std::string &outFilename, std::string &outString) const |
1918 | 0 | { |
1919 | 0 | std::string osCommandLine; |
1920 | |
|
1921 | 0 | for (const auto &path : GDALAlgorithm::m_callPath) |
1922 | 0 | { |
1923 | 0 | if (!osCommandLine.empty()) |
1924 | 0 | osCommandLine += ' '; |
1925 | 0 | osCommandLine += path; |
1926 | 0 | } |
1927 | | |
1928 | | // Do not include the last step |
1929 | 0 | for (size_t i = 0; i + 1 < m_steps.size(); ++i) |
1930 | 0 | { |
1931 | 0 | const auto &step = m_steps[i]; |
1932 | 0 | if (!step->IsNativelyStreamingCompatible()) |
1933 | 0 | { |
1934 | 0 | GDALAlgorithm::ReportError( |
1935 | 0 | CE_Warning, CPLE_AppDefined, |
1936 | 0 | "Step %s is not natively streaming compatible, and " |
1937 | 0 | "may cause significant processing time at opening", |
1938 | 0 | step->GDALAlgorithm::GetName().c_str()); |
1939 | 0 | } |
1940 | |
|
1941 | 0 | if (i > 0) |
1942 | 0 | osCommandLine += " !"; |
1943 | 0 | for (const auto &path : step->GDALAlgorithm::m_callPath) |
1944 | 0 | { |
1945 | 0 | if (!osCommandLine.empty()) |
1946 | 0 | osCommandLine += ' '; |
1947 | 0 | osCommandLine += path; |
1948 | 0 | } |
1949 | |
|
1950 | 0 | for (const auto &arg : step->GetArgs()) |
1951 | 0 | { |
1952 | 0 | if (arg->IsExplicitlySet()) |
1953 | 0 | { |
1954 | 0 | osCommandLine += ' '; |
1955 | 0 | std::string strArg; |
1956 | 0 | if (!arg->Serialize(strArg, /* absolutePath=*/false)) |
1957 | 0 | { |
1958 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
1959 | 0 | "Cannot serialize argument %s", |
1960 | 0 | arg->GetName().c_str()); |
1961 | 0 | return false; |
1962 | 0 | } |
1963 | 0 | osCommandLine += strArg; |
1964 | 0 | } |
1965 | 0 | } |
1966 | 0 | } |
1967 | | |
1968 | 0 | return GDALAlgorithm::SaveGDALG(outFilename, outString, osCommandLine); |
1969 | 0 | } |
1970 | | |
1971 | | /************************************************************************/ |
1972 | | /* RunStepDealWithGDALGJson() */ |
1973 | | /************************************************************************/ |
1974 | | |
1975 | | GDALAbstractPipelineAlgorithm::RunStepState |
1976 | | GDALAbstractPipelineAlgorithm::RunStepDealWithGDALGJson() |
1977 | 0 | { |
1978 | | // Handle output to GDALG file |
1979 | 0 | if (!m_steps.empty() && |
1980 | 0 | m_steps.back()->GetName() == GDALRasterWriteAlgorithm::NAME) |
1981 | 0 | { |
1982 | 0 | const auto outputArg = m_steps.back()->GetArg(GDAL_ARG_NAME_OUTPUT); |
1983 | 0 | const auto outputFormatArg = |
1984 | 0 | m_steps.back()->GetArg(GDAL_ARG_NAME_OUTPUT_FORMAT); |
1985 | 0 | if (outputArg && outputArg->GetType() == GAAT_DATASET && |
1986 | 0 | outputArg->IsExplicitlySet()) |
1987 | 0 | { |
1988 | 0 | const std::string &outputFileName = |
1989 | 0 | outputArg->Get<GDALArgDatasetValue>().GetName(); |
1990 | 0 | if (m_steps.back()->IsGDALGOutput()) |
1991 | 0 | { |
1992 | 0 | std::string outStringUnused; |
1993 | 0 | return SaveGDALGIntoFileOrString(outputFileName, |
1994 | 0 | outStringUnused) |
1995 | 0 | ? RunStepState::PROCESSED |
1996 | 0 | : RunStepState::ERROR; |
1997 | 0 | } |
1998 | | |
1999 | 0 | bool isVRTOutput; |
2000 | 0 | if (outputFormatArg && outputFormatArg->GetType() == GAAT_STRING && |
2001 | 0 | outputFormatArg->IsExplicitlySet()) |
2002 | 0 | { |
2003 | 0 | const auto &val = outputFormatArg->Get<std::string>(); |
2004 | 0 | isVRTOutput = EQUAL(val.c_str(), "vrt"); |
2005 | 0 | } |
2006 | 0 | else |
2007 | 0 | { |
2008 | 0 | isVRTOutput = EQUAL( |
2009 | 0 | CPLGetExtensionSafe(outputFileName.c_str()).c_str(), "vrt"); |
2010 | 0 | } |
2011 | 0 | if (isVRTOutput && !outputFileName.empty() && m_steps.size() > 3) |
2012 | 0 | { |
2013 | 0 | ReportError( |
2014 | 0 | CE_Failure, CPLE_NotSupported, |
2015 | 0 | "VRT output is not supported when there are more than 3 " |
2016 | 0 | "steps. Consider using the GDALG driver (files with " |
2017 | 0 | ".gdalg.json extension)"); |
2018 | 0 | return RunStepState::ERROR; |
2019 | 0 | } |
2020 | 0 | if (isVRTOutput) |
2021 | 0 | { |
2022 | 0 | for (const auto &step : m_steps) |
2023 | 0 | { |
2024 | 0 | if (!step->m_outputVRTCompatible) |
2025 | 0 | { |
2026 | 0 | step->ReportError( |
2027 | 0 | CE_Failure, CPLE_NotSupported, |
2028 | 0 | "VRT output is not supported. Consider using the " |
2029 | 0 | "GDALG driver instead (files with .gdalg.json " |
2030 | 0 | "extension)"); |
2031 | 0 | return RunStepState::ERROR; |
2032 | 0 | } |
2033 | 0 | } |
2034 | 0 | } |
2035 | 0 | } |
2036 | 0 | } |
2037 | | |
2038 | 0 | if (m_executionForStreamOutput && |
2039 | 0 | !CPLTestBool( |
2040 | 0 | CPLGetConfigOption("GDAL_ALGORITHM_ALLOW_WRITES_IN_STREAM", "NO"))) |
2041 | 0 | { |
2042 | | // For security reasons, to avoid that reading a .gdalg.json file writes |
2043 | | // a file on the file system. |
2044 | 0 | for (const auto &step : m_steps) |
2045 | 0 | { |
2046 | 0 | if (step->GetName() == GDALRasterWriteAlgorithm::NAME) |
2047 | 0 | { |
2048 | 0 | if (!EQUAL(step->m_format.c_str(), "stream")) |
2049 | 0 | { |
2050 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2051 | 0 | "in streamed execution, --format " |
2052 | 0 | "stream should be used"); |
2053 | 0 | return RunStepState::ERROR; |
2054 | 0 | } |
2055 | 0 | } |
2056 | 0 | else if (step->GeneratesFilesFromUserInput()) |
2057 | 0 | { |
2058 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2059 | 0 | "Step '%s' not allowed in stream execution, unless " |
2060 | 0 | "the GDAL_ALGORITHM_ALLOW_WRITES_IN_STREAM " |
2061 | 0 | "configuration option is set.", |
2062 | 0 | step->GetName().c_str()); |
2063 | 0 | return RunStepState::ERROR; |
2064 | 0 | } |
2065 | 0 | } |
2066 | 0 | } |
2067 | | |
2068 | 0 | return RunStepState::GO_ON; |
2069 | 0 | } |
2070 | | |
2071 | | /************************************************************************/ |
2072 | | /* RunStepDealWithMultiProcessing() */ |
2073 | | /************************************************************************/ |
2074 | | |
2075 | | GDALAbstractPipelineAlgorithm::RunStepState |
2076 | | GDALAbstractPipelineAlgorithm::RunStepDealWithMultiProcessing( |
2077 | | GDALPipelineStepRunContext &ctxt) |
2078 | 0 | { |
2079 | | // Because of multiprocessing in gdal raster tile, make sure that all |
2080 | | // steps before it are either materialized or serialized in a .gdal.json file |
2081 | 0 | if (m_steps.size() >= 2 && m_steps.back()->SupportsInputMultiThreading() && |
2082 | 0 | m_steps.back() |
2083 | 0 | ->GetArg(GDAL_ARG_NAME_NUM_THREADS_INT_HIDDEN) |
2084 | 0 | ->Get<int>() > 1 && |
2085 | 0 | !(m_steps.size() == 2 && |
2086 | 0 | m_steps[0]->GetName() == GDALRasterReadAlgorithm::NAME)) |
2087 | 0 | { |
2088 | 0 | if (m_steps[m_steps.size() - 2]->GetName() == |
2089 | 0 | GDALMaterializeRasterAlgorithm::NAME) |
2090 | 0 | { |
2091 | | // If the step immediately before the last one is materialize, |
2092 | | // then make sure the file it generates can be re-opened |
2093 | 0 | auto poArg = m_steps[m_steps.size() - 2]->GetArg( |
2094 | 0 | GDALMaterializeRasterAlgorithm:: |
2095 | 0 | ARG_NAME_REOPEN_AND_DO_NOT_EARLY_DELETE); |
2096 | 0 | CPLAssert(poArg); |
2097 | 0 | poArg->Set(true); |
2098 | 0 | } |
2099 | 0 | else |
2100 | 0 | { |
2101 | 0 | for (size_t i = m_steps.size() - 2; i > 0;) |
2102 | 0 | { |
2103 | 0 | --i; |
2104 | 0 | if (m_steps[i]->GetName() == |
2105 | 0 | GDALMaterializeRasterAlgorithm::NAME) |
2106 | 0 | { |
2107 | 0 | auto poArg = m_steps[i]->GetArg(GDAL_ARG_NAME_OUTPUT); |
2108 | 0 | CPLAssert(poArg); |
2109 | 0 | if (poArg->IsExplicitlySet()) |
2110 | 0 | { |
2111 | | // We could potentially support that scenario but that |
2112 | | // would require executing the pipeline up to that |
2113 | | // step, and serializing the rest to GDALG |
2114 | 0 | ReportError( |
2115 | 0 | CE_Failure, CPLE_AppDefined, |
2116 | 0 | "Cannot execute this pipeline in parallel mode due " |
2117 | 0 | "to the presence of a materialize step that has a " |
2118 | 0 | "'output' argument and is not immediately before " |
2119 | 0 | "the last step. " |
2120 | 0 | "Move/create a materialize step immediately before " |
2121 | 0 | "the last step, or add '-j 1' to the last step " |
2122 | 0 | "'%s'", |
2123 | 0 | m_steps.back()->GetName().c_str()); |
2124 | 0 | return RunStepState::ERROR; |
2125 | 0 | } |
2126 | 0 | } |
2127 | 0 | } |
2128 | | |
2129 | 0 | bool ret = false; |
2130 | 0 | auto poSrcDS = m_inputDataset.size() == 1 |
2131 | 0 | ? m_inputDataset[0].GetDatasetRef() |
2132 | 0 | : nullptr; |
2133 | 0 | if (poSrcDS) |
2134 | 0 | { |
2135 | 0 | auto poSrcDriver = poSrcDS->GetDriver(); |
2136 | 0 | if (!poSrcDriver || EQUAL(poSrcDriver->GetDescription(), "MEM")) |
2137 | 0 | { |
2138 | 0 | ReportError( |
2139 | 0 | CE_Failure, CPLE_AppDefined, |
2140 | 0 | "Cannot execute this pipeline in parallel mode due to " |
2141 | 0 | "input dataset of last step being a non-materialized " |
2142 | 0 | "dataset. " |
2143 | 0 | "Materialize it first, or add '-j 1' to the last step " |
2144 | 0 | "'%s'", |
2145 | 0 | m_steps.back()->GetName().c_str()); |
2146 | 0 | return RunStepState::ERROR; |
2147 | 0 | } |
2148 | 0 | } |
2149 | 0 | std::string outString; |
2150 | 0 | if (SaveGDALGIntoFileOrString(std::string(), outString)) |
2151 | 0 | { |
2152 | 0 | const char *const apszAllowedDrivers[] = {"GDALG", nullptr}; |
2153 | 0 | auto poCurDS = GDALDataset::Open( |
2154 | 0 | outString.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, |
2155 | 0 | apszAllowedDrivers); |
2156 | 0 | if (poCurDS) |
2157 | 0 | { |
2158 | 0 | auto &lastAlg = m_steps.back(); |
2159 | 0 | lastAlg->m_inputDataset.clear(); |
2160 | 0 | lastAlg->m_inputDataset.resize(1); |
2161 | 0 | lastAlg->m_inputDataset[0].Set(poCurDS); |
2162 | 0 | lastAlg->m_inputDataset[0].SetDatasetOpenedByAlgorithm(); |
2163 | 0 | poCurDS->Release(); |
2164 | 0 | ret = lastAlg->RunStep(ctxt); |
2165 | 0 | lastAlg->m_inputDataset[0].Close(); |
2166 | 0 | } |
2167 | 0 | } |
2168 | 0 | else |
2169 | 0 | { |
2170 | 0 | ReportError( |
2171 | 0 | CE_Failure, CPLE_AppDefined, |
2172 | 0 | "Cannot execute this pipeline in parallel mode due to " |
2173 | 0 | "an unexpected error. " |
2174 | 0 | "Trying adding a materialize step before the last step " |
2175 | 0 | "'%s', or add '-j 1' to the last step.", |
2176 | 0 | m_steps.back()->GetName().c_str()); |
2177 | 0 | return RunStepState::ERROR; |
2178 | 0 | } |
2179 | 0 | return ret ? RunStepState::PROCESSED : RunStepState::ERROR; |
2180 | 0 | } |
2181 | 0 | } |
2182 | | |
2183 | 0 | return RunStepState::GO_ON; |
2184 | 0 | } |
2185 | | |
2186 | | /************************************************************************/ |
2187 | | /* RunStepDealWithStepUnknownInputType() */ |
2188 | | /************************************************************************/ |
2189 | | |
2190 | | bool GDALAbstractPipelineAlgorithm::RunStepDealWithStepUnknownInputType( |
2191 | | size_t i, int nCurDatasetType) |
2192 | 0 | { |
2193 | | // We go here if there was a step such as "external" where at |
2194 | | // ParseCommandLineArguments() time we could not determine its |
2195 | | // type of output dataset. Now we must check for steps afterwards |
2196 | | // such as "write" or "reproject" that exist both as separate raster |
2197 | | // and vector commands if the one we initially picked is appropriate. |
2198 | | // If not, then switch to the other type. |
2199 | | |
2200 | | // GetInputDatasetType() is to deal with pipelines like: |
2201 | | // gdal pipeline read input_vector ! |
2202 | | // external --command "cp <INPUT> <OUTPUT>" ! |
2203 | | // clip --input raster_dataset --like _PIPE_ ! |
2204 | | // write output_raster |
2205 | |
|
2206 | 0 | auto &step = m_steps[i]; |
2207 | 0 | const int nThisDatasetType = GetInputDatasetType(step.get()); |
2208 | 0 | const int nThisStepType = |
2209 | 0 | nThisDatasetType ? nThisDatasetType : nCurDatasetType; |
2210 | 0 | if (step->GetInputType() != 0 && step->GetInputType() != nThisStepType) |
2211 | 0 | { |
2212 | 0 | auto newAlg = GetStepAlg( |
2213 | 0 | step->GetName() + |
2214 | 0 | (nThisStepType == GDAL_OF_RASTER ? RASTER_SUFFIX : VECTOR_SUFFIX)); |
2215 | 0 | if (newAlg) |
2216 | 0 | { |
2217 | 0 | const bool maybeWriteStep = |
2218 | 0 | (i == m_steps.size() - 1 && |
2219 | 0 | m_eLastStepAsWrite != StepConstraint::CAN_NOT_BE); |
2220 | 0 | if (!CopyStepAlgorithmFromAnother(newAlg.get(), step.get(), |
2221 | 0 | maybeWriteStep)) |
2222 | 0 | return false; |
2223 | 0 | newAlg->m_inputDatasetCanBeOmitted = true; |
2224 | |
|
2225 | 0 | step = std::move(newAlg); |
2226 | 0 | } |
2227 | 0 | } |
2228 | | |
2229 | 0 | return step->ValidateArguments(); |
2230 | 0 | } |
2231 | | |
2232 | | /************************************************************************/ |
2233 | | /* CheckStepHasNoInputDatasetAlreadySet() */ |
2234 | | /************************************************************************/ |
2235 | | |
2236 | | bool GDALAbstractPipelineAlgorithm::CheckStepHasNoInputDatasetAlreadySet( |
2237 | | size_t i, GDALDataset *poCurDS) |
2238 | 0 | { |
2239 | 0 | auto &step = *(m_steps[i]); |
2240 | 0 | bool prevStepOutputSetToThisStep = false; |
2241 | 0 | for (auto &arg : step.GetArgs()) |
2242 | 0 | { |
2243 | 0 | if (!arg->IsOutput() && (arg->GetType() == GAAT_DATASET || |
2244 | 0 | arg->GetType() == GAAT_DATASET_LIST)) |
2245 | 0 | { |
2246 | 0 | if (arg->GetType() == GAAT_DATASET) |
2247 | 0 | { |
2248 | 0 | if ((arg->GetName() == GDAL_ARG_NAME_INPUT && |
2249 | 0 | !arg->IsExplicitlySet()) || |
2250 | 0 | arg->Get<GDALArgDatasetValue>().GetName() == |
2251 | 0 | GDAL_DATASET_PIPELINE_PLACEHOLDER_VALUE) |
2252 | 0 | { |
2253 | 0 | auto &val = arg->Get<GDALArgDatasetValue>(); |
2254 | 0 | if (val.GetDatasetRef()) |
2255 | 0 | { |
2256 | | // Shouldn't happen |
2257 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2258 | 0 | "Step nr %d (%s) has already an " |
2259 | 0 | "input dataset for argument %s", |
2260 | 0 | static_cast<int>(i), step.GetName().c_str(), |
2261 | 0 | arg->GetName().c_str()); |
2262 | 0 | return false; |
2263 | 0 | } |
2264 | 0 | prevStepOutputSetToThisStep = true; |
2265 | 0 | val.Set(poCurDS); |
2266 | 0 | arg->NotifyValueSet(); |
2267 | 0 | } |
2268 | 0 | } |
2269 | 0 | else |
2270 | 0 | { |
2271 | 0 | CPLAssert(arg->GetType() == GAAT_DATASET_LIST); |
2272 | 0 | auto &val = arg->Get<std::vector<GDALArgDatasetValue>>(); |
2273 | 0 | if ((arg->GetName() == GDAL_ARG_NAME_INPUT && |
2274 | 0 | !arg->IsExplicitlySet()) || |
2275 | 0 | (val.size() == 1 && |
2276 | 0 | val[0].GetName() == |
2277 | 0 | GDAL_DATASET_PIPELINE_PLACEHOLDER_VALUE)) |
2278 | 0 | { |
2279 | 0 | if (val.size() == 1 && val[0].GetDatasetRef()) |
2280 | 0 | { |
2281 | | // Shouldn't happen |
2282 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2283 | 0 | "Step nr %d (%s) has already an " |
2284 | 0 | "input dataset for argument %s", |
2285 | 0 | static_cast<int>(i), step.GetName().c_str(), |
2286 | 0 | arg->GetName().c_str()); |
2287 | 0 | return false; |
2288 | 0 | } |
2289 | 0 | prevStepOutputSetToThisStep = true; |
2290 | 0 | val.clear(); |
2291 | 0 | val.resize(1); |
2292 | 0 | val[0].Set(poCurDS); |
2293 | 0 | arg->NotifyValueSet(); |
2294 | 0 | } |
2295 | 0 | } |
2296 | 0 | } |
2297 | 0 | } |
2298 | 0 | if (!prevStepOutputSetToThisStep) |
2299 | 0 | { |
2300 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2301 | 0 | "Step nr %d (%s) does not use input dataset from " |
2302 | 0 | "previous step", |
2303 | 0 | static_cast<int>(i), step.GetName().c_str()); |
2304 | 0 | return false; |
2305 | 0 | } |
2306 | | |
2307 | 0 | return true; |
2308 | 0 | } |
2309 | | |
2310 | | /************************************************************************/ |
2311 | | /* GDALAbstractPipelineAlgorithm::RunStep() */ |
2312 | | /************************************************************************/ |
2313 | | |
2314 | | bool GDALAbstractPipelineAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt) |
2315 | 0 | { |
2316 | 0 | if (m_stepOnWhichHelpIsRequested) |
2317 | 0 | { |
2318 | 0 | printf( |
2319 | 0 | "%s", |
2320 | 0 | m_stepOnWhichHelpIsRequested->GetUsageForCLI(false).c_str()); /*ok*/ |
2321 | 0 | return true; |
2322 | 0 | } |
2323 | | |
2324 | 0 | if (m_steps.empty()) |
2325 | 0 | { |
2326 | | // If invoked programmatically, not from the command line. |
2327 | |
|
2328 | 0 | if (m_pipeline.empty()) |
2329 | 0 | { |
2330 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2331 | 0 | "'pipeline' argument not set"); |
2332 | 0 | return false; |
2333 | 0 | } |
2334 | | |
2335 | 0 | const CPLStringList aosTokens(CSLTokenizeString(m_pipeline.c_str())); |
2336 | 0 | if (!ParseCommandLineArguments(aosTokens)) |
2337 | 0 | return false; |
2338 | 0 | } |
2339 | | |
2340 | 0 | switch (RunStepDealWithGDALGJson()) |
2341 | 0 | { |
2342 | 0 | case RunStepState::PROCESSED: |
2343 | 0 | return true; |
2344 | 0 | case RunStepState::ERROR: |
2345 | 0 | return false; |
2346 | 0 | case RunStepState::GO_ON: |
2347 | 0 | break; |
2348 | 0 | } |
2349 | 0 | switch (RunStepDealWithMultiProcessing(ctxt)) |
2350 | 0 | { |
2351 | 0 | case RunStepState::PROCESSED: |
2352 | 0 | return true; |
2353 | 0 | case RunStepState::ERROR: |
2354 | 0 | return false; |
2355 | 0 | case RunStepState::GO_ON: |
2356 | 0 | break; |
2357 | 0 | } |
2358 | | |
2359 | 0 | int countPipelinesWithProgress = 0; |
2360 | 0 | for (size_t i = (m_bExpectReadStep ? 0 : 1); i < m_steps.size(); ++i) |
2361 | 0 | { |
2362 | 0 | const bool bCanHandleNextStep = |
2363 | 0 | i < m_steps.size() - 1 && |
2364 | 0 | !m_steps[i]->CanHandleNextStep(m_steps[i + 1].get()); |
2365 | 0 | if (bCanHandleNextStep && |
2366 | 0 | !m_steps[i + 1]->IsNativelyStreamingCompatible()) |
2367 | 0 | ++countPipelinesWithProgress; |
2368 | 0 | else if (!m_steps[i]->IsNativelyStreamingCompatible()) |
2369 | 0 | ++countPipelinesWithProgress; |
2370 | 0 | if (bCanHandleNextStep) |
2371 | 0 | ++i; |
2372 | 0 | } |
2373 | 0 | if (countPipelinesWithProgress == 0) |
2374 | 0 | countPipelinesWithProgress = 1; |
2375 | |
|
2376 | 0 | bool ret = true; |
2377 | 0 | GDALDataset *poCurDS = nullptr; |
2378 | 0 | int iCurStepWithProgress = 0; |
2379 | |
|
2380 | 0 | if (!m_bExpectReadStep) |
2381 | 0 | { |
2382 | 0 | CPLAssert(m_inputDataset.size() == 1); |
2383 | 0 | poCurDS = m_inputDataset[0].GetDatasetRef(); |
2384 | 0 | CPLAssert(poCurDS); |
2385 | 0 | } |
2386 | | |
2387 | 0 | GDALProgressFunc pfnProgress = ctxt.m_pfnProgress; |
2388 | 0 | void *pProgressData = ctxt.m_pProgressData; |
2389 | 0 | if (IsCalledFromCommandLine() && HasOutputString()) |
2390 | 0 | { |
2391 | 0 | pfnProgress = nullptr; |
2392 | 0 | pProgressData = nullptr; |
2393 | 0 | } |
2394 | |
|
2395 | 0 | int nCurDatasetType = poCurDS ? GetDatasetType(poCurDS) : 0; |
2396 | |
|
2397 | 0 | for (size_t i = 0; i < m_steps.size(); ++i) |
2398 | 0 | { |
2399 | 0 | auto &step = m_steps[i]; |
2400 | |
|
2401 | 0 | if (i > 0 && m_nFirstStepWithUnknownInputType >= 0 && |
2402 | 0 | i >= static_cast<size_t>(m_nFirstStepWithUnknownInputType) && |
2403 | 0 | nCurDatasetType != 0 && GetStepAlg(step->GetName()) == nullptr) |
2404 | 0 | { |
2405 | 0 | if (!RunStepDealWithStepUnknownInputType(i, nCurDatasetType)) |
2406 | 0 | return false; |
2407 | 0 | } |
2408 | | |
2409 | 0 | if (i > 0 || poCurDS) |
2410 | 0 | { |
2411 | 0 | if (!CheckStepHasNoInputDatasetAlreadySet(i, poCurDS)) |
2412 | 0 | return false; |
2413 | 0 | } |
2414 | | |
2415 | 0 | if (i + 1 < m_steps.size() && step->m_outputDataset.GetDatasetRef() && |
2416 | 0 | !step->OutputDatasetAllowedBeforeRunningStep()) |
2417 | 0 | { |
2418 | | // Shouldn't happen |
2419 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2420 | 0 | "Step nr %d (%s) has already an output dataset", |
2421 | 0 | static_cast<int>(i), step->GetName().c_str()); |
2422 | 0 | return false; |
2423 | 0 | } |
2424 | | |
2425 | 0 | const bool bCanHandleNextStep = |
2426 | 0 | i < m_steps.size() - 1 && |
2427 | 0 | step->CanHandleNextStep(m_steps[i + 1].get()); |
2428 | |
|
2429 | 0 | std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)> pScaledData( |
2430 | 0 | nullptr, GDALDestroyScaledProgress); |
2431 | 0 | GDALPipelineStepRunContext stepCtxt; |
2432 | 0 | if ((bCanHandleNextStep && |
2433 | 0 | m_steps[i + 1]->IsNativelyStreamingCompatible()) || |
2434 | 0 | !step->IsNativelyStreamingCompatible()) |
2435 | 0 | { |
2436 | 0 | pScaledData.reset(GDALCreateScaledProgress( |
2437 | 0 | iCurStepWithProgress / |
2438 | 0 | static_cast<double>(countPipelinesWithProgress), |
2439 | 0 | (iCurStepWithProgress + 1) / |
2440 | 0 | static_cast<double>(countPipelinesWithProgress), |
2441 | 0 | pfnProgress, pProgressData)); |
2442 | 0 | ++iCurStepWithProgress; |
2443 | 0 | stepCtxt.m_pfnProgress = pScaledData ? GDALScaledProgress : nullptr; |
2444 | 0 | stepCtxt.m_pProgressData = pScaledData.get(); |
2445 | 0 | } |
2446 | 0 | if (bCanHandleNextStep) |
2447 | 0 | { |
2448 | 0 | stepCtxt.m_poNextUsableStep = m_steps[i + 1].get(); |
2449 | 0 | } |
2450 | 0 | if (i + 1 == m_steps.size() && m_stdout && |
2451 | 0 | step->GetArg(GDAL_ARG_NAME_STDOUT) != nullptr) |
2452 | 0 | { |
2453 | 0 | step->m_stdout = true; |
2454 | 0 | } |
2455 | 0 | step->m_inputDatasetCanBeOmitted = false; |
2456 | 0 | step->m_quiet = m_quiet; |
2457 | 0 | if (!step->ValidateArguments() || !step->RunStep(stepCtxt)) |
2458 | 0 | { |
2459 | 0 | ret = false; |
2460 | 0 | break; |
2461 | 0 | } |
2462 | 0 | poCurDS = step->m_outputDataset.GetDatasetRef(); |
2463 | 0 | nCurDatasetType = 0; |
2464 | 0 | if (poCurDS) |
2465 | 0 | { |
2466 | 0 | nCurDatasetType = GetDatasetType(poCurDS); |
2467 | 0 | } |
2468 | 0 | else if (!(i + 1 == m_steps.size() && |
2469 | 0 | (!step->m_output.empty() || |
2470 | 0 | step->GetArg(GDAL_ARG_NAME_STDOUT) != nullptr || |
2471 | 0 | step->GetOutputType() == 0))) |
2472 | 0 | { |
2473 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
2474 | 0 | "Step nr %d (%s) failed to produce an output dataset", |
2475 | 0 | static_cast<int>(i), step->GetName().c_str()); |
2476 | 0 | return false; |
2477 | 0 | } |
2478 | | |
2479 | 0 | m_output += step->GetOutputString(); |
2480 | |
|
2481 | 0 | if (bCanHandleNextStep) |
2482 | 0 | { |
2483 | 0 | ++i; |
2484 | 0 | } |
2485 | 0 | } |
2486 | | |
2487 | 0 | if (pfnProgress && m_output.empty()) |
2488 | 0 | pfnProgress(1.0, "", pProgressData); |
2489 | |
|
2490 | 0 | if (!m_output.empty()) |
2491 | 0 | { |
2492 | 0 | auto outputStringArg = GetArg(GDAL_ARG_NAME_OUTPUT_STRING); |
2493 | 0 | if (outputStringArg && outputStringArg->GetType() == GAAT_STRING) |
2494 | 0 | outputStringArg->Set(m_output); |
2495 | 0 | } |
2496 | |
|
2497 | 0 | if (ret && poCurDS && !m_outputDataset.GetDatasetRef()) |
2498 | 0 | { |
2499 | 0 | m_outputDataset.Set(poCurDS); |
2500 | 0 | } |
2501 | |
|
2502 | 0 | return ret; |
2503 | 0 | } |
2504 | | |
2505 | | /************************************************************************/ |
2506 | | /* GDALAbstractPipelineAlgorithm::HasOutputString() */ |
2507 | | /************************************************************************/ |
2508 | | |
2509 | | bool GDALAbstractPipelineAlgorithm::HasOutputString() const |
2510 | 0 | { |
2511 | 0 | for (const auto &step : m_steps) |
2512 | 0 | { |
2513 | 0 | if (step->HasOutputString()) |
2514 | 0 | return true; |
2515 | 0 | } |
2516 | 0 | return false; |
2517 | 0 | } |
2518 | | |
2519 | | /************************************************************************/ |
2520 | | /* GDALAbstractPipelineAlgorithm::Finalize() */ |
2521 | | /************************************************************************/ |
2522 | | |
2523 | | bool GDALAbstractPipelineAlgorithm::Finalize() |
2524 | 0 | { |
2525 | 0 | bool ret = GDALPipelineStepAlgorithm::Finalize(); |
2526 | | // Finalize steps in reverse order, typically to make sure later steps |
2527 | | // have dropped their reference on datasets passed by previous ones. |
2528 | | // This helps for example for the "external" step that needs to delete |
2529 | | // temporary files. |
2530 | 0 | for (auto iter = m_steps.rbegin(); iter != m_steps.rend(); ++iter) |
2531 | 0 | { |
2532 | 0 | ret = (*iter)->Finalize() && ret; |
2533 | 0 | } |
2534 | 0 | return ret; |
2535 | 0 | } |
2536 | | |
2537 | | /************************************************************************/ |
2538 | | /* GDALAbstractPipelineAlgorithm::GetUsageAsJSON() */ |
2539 | | /************************************************************************/ |
2540 | | |
2541 | | std::string GDALAbstractPipelineAlgorithm::GetUsageAsJSON() const |
2542 | 0 | { |
2543 | 0 | CPLJSONDocument oDoc; |
2544 | 0 | CPL_IGNORE_RET_VAL(oDoc.LoadMemory(GDALAlgorithm::GetUsageAsJSON())); |
2545 | |
|
2546 | 0 | CPLJSONArray jPipelineSteps; |
2547 | 0 | for (const std::string &name : GetStepRegistry().GetNames()) |
2548 | 0 | { |
2549 | 0 | auto alg = GetStepAlg(name); |
2550 | 0 | if (!alg->IsHidden()) |
2551 | 0 | { |
2552 | 0 | CPLJSONDocument oStepDoc; |
2553 | 0 | CPL_IGNORE_RET_VAL(oStepDoc.LoadMemory(alg->GetUsageAsJSON())); |
2554 | 0 | jPipelineSteps.Add(oStepDoc.GetRoot()); |
2555 | 0 | } |
2556 | 0 | } |
2557 | 0 | oDoc.GetRoot().Add("pipeline_algorithms", jPipelineSteps); |
2558 | |
|
2559 | 0 | return oDoc.SaveAsString(); |
2560 | 0 | } |
2561 | | |
2562 | | //! @endcond |