/src/gdal/apps/gdalalg_mdim_mosaic.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: GDAL |
4 | | * Purpose: gdal "mdim mosaic" subcommand |
5 | | * Author: Even Rouault <even dot rouault at spatialys.com> |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "gdalalg_mdim_mosaic.h" |
14 | | #include "gdalalg_mdim_write.h" |
15 | | |
16 | | #include "cpl_conv.h" |
17 | | #include "cpl_vsi_virtual.h" |
18 | | #include "gdal_priv.h" |
19 | | #include "vrtdataset.h" |
20 | | |
21 | | #include <algorithm> |
22 | | #include <cmath> |
23 | | #include <optional> |
24 | | |
25 | | //! @cond Doxygen_Suppress |
26 | | |
27 | | #ifndef _ |
28 | 0 | #define _(x) (x) |
29 | | #endif |
30 | | |
31 | | /************************************************************************/ |
32 | | /* GDALMdimMosaicAlgorithm::GDALMdimMosaicAlgorithm() */ |
33 | | /************************************************************************/ |
34 | | |
35 | | GDALMdimMosaicAlgorithm::GDALMdimMosaicAlgorithm(bool bStandalone) |
36 | 0 | : GDALMdimPipelineStepAlgorithm(NAME, DESCRIPTION, HELP_URL, |
37 | 0 | ConstructorOptions() |
38 | 0 | .SetStandaloneStep(bStandalone) |
39 | 0 | .SetAddDefaultArguments(false) |
40 | 0 | .SetAutoOpenInputDatasets(false) |
41 | 0 | .SetInputDatasetInputFlags(GADV_NAME) |
42 | 0 | .SetInputDatasetMaxCount(INT_MAX)) |
43 | 0 | { |
44 | 0 | AddMdimInputArgs(/* openForMixedMdimVector = */ false, |
45 | 0 | /* hiddenForCLI = */ false, /* acceptRaster = */ true); |
46 | 0 | if (bStandalone) |
47 | 0 | { |
48 | 0 | AddProgressArg(); |
49 | 0 | AddMdimOutputArgs(false); |
50 | 0 | } |
51 | |
|
52 | 0 | AddArrayNameArg(&m_array, _("Name of array(s) to mosaic.")); |
53 | 0 | } |
54 | | |
55 | | /************************************************************************/ |
56 | | /* GetDimensionDesc() */ |
57 | | /************************************************************************/ |
58 | | |
59 | | std::optional<GDALMdimMosaicAlgorithm::DimensionDesc> |
60 | | GDALMdimMosaicAlgorithm::GetDimensionDesc( |
61 | | const std::string &osDSName, |
62 | | const std::shared_ptr<GDALDimension> &poDim) const |
63 | 0 | { |
64 | 0 | auto poVar = poDim->GetIndexingVariable(); |
65 | 0 | if (poVar) |
66 | 0 | { |
67 | 0 | if (poVar->GetDimensionCount() != 1) |
68 | 0 | { |
69 | 0 | ReportError( |
70 | 0 | CE_Failure, CPLE_AppDefined, |
71 | 0 | "Dataset %s: indexing variable %s of dimension %s is not 1D", |
72 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
73 | 0 | poDim->GetName().c_str()); |
74 | 0 | return std::nullopt; |
75 | 0 | } |
76 | 0 | if (poVar->GetDataType().GetClass() != GEDTC_NUMERIC) |
77 | 0 | { |
78 | 0 | ReportError( |
79 | 0 | CE_Failure, CPLE_AppDefined, |
80 | 0 | "Dataset %s: indexing variable %s of dimension %s has a " |
81 | 0 | "non-numeric data type", |
82 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
83 | 0 | poDim->GetName().c_str()); |
84 | 0 | return std::nullopt; |
85 | 0 | } |
86 | 0 | } |
87 | 0 | DimensionDesc desc; |
88 | 0 | desc.osName = poDim->GetName(); |
89 | 0 | desc.osType = poDim->GetType(); |
90 | 0 | desc.osDirection = poDim->GetDirection(); |
91 | 0 | const auto nSize = poDim->GetSize(); |
92 | 0 | if (poVar) |
93 | 0 | { |
94 | 0 | desc.attributes = poVar->GetAttributes(); |
95 | 0 | desc.osUnit = poVar->GetUnit(); |
96 | 0 | } |
97 | 0 | CPLAssert(nSize > 0); |
98 | 0 | desc.nSize = nSize; |
99 | 0 | desc.bHasIndexingVar = poVar != nullptr; |
100 | 0 | if (!poVar) |
101 | 0 | { |
102 | 0 | desc.dfStart = 0; |
103 | 0 | desc.dfIncrement = 1; |
104 | 0 | } |
105 | 0 | else if (nSize <= 2 || |
106 | 0 | !poVar->IsRegularlySpaced(desc.dfStart, desc.dfIncrement)) |
107 | 0 | { |
108 | 0 | constexpr uint64_t LIMIT = 100 * 1000 * 1000; |
109 | 0 | if (nSize > LIMIT) |
110 | 0 | { |
111 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
112 | 0 | "Dataset %s: indexing variable %s of dimension %s has " |
113 | 0 | "too large size", |
114 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
115 | 0 | desc.osName.c_str()); |
116 | 0 | return std::nullopt; |
117 | 0 | } |
118 | 0 | std::vector<double> adfValues(static_cast<size_t>(nSize)); |
119 | 0 | GUInt64 arrayStartIdx[] = {0}; |
120 | 0 | size_t anCount[] = {adfValues.size()}; |
121 | 0 | GInt64 arrayStep[] = {1}; |
122 | 0 | GPtrDiff_t bufferStride[] = {1}; |
123 | 0 | if (!poVar->Read(arrayStartIdx, anCount, arrayStep, bufferStride, |
124 | 0 | GDALExtendedDataType::Create(GDT_Float64), |
125 | 0 | adfValues.data())) |
126 | 0 | { |
127 | 0 | return std::nullopt; |
128 | 0 | } |
129 | 0 | if (std::isnan(adfValues[0])) |
130 | 0 | { |
131 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
132 | 0 | "Dataset %s: indexing variable %s of dimension %s has " |
133 | 0 | "NaN values", |
134 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
135 | 0 | desc.osName.c_str()); |
136 | 0 | return std::nullopt; |
137 | 0 | } |
138 | 0 | if (nSize > 1) |
139 | 0 | { |
140 | 0 | const int nSign = adfValues[1] > adfValues[0] ? 1 : -1; |
141 | 0 | if (std::isnan(adfValues[1])) |
142 | 0 | { |
143 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
144 | 0 | "Dataset %s: indexing variable %s of dimension %s " |
145 | 0 | "has NaN values", |
146 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
147 | 0 | desc.osName.c_str()); |
148 | 0 | return std::nullopt; |
149 | 0 | } |
150 | 0 | for (size_t i = 2; i < nSize; ++i) |
151 | 0 | { |
152 | 0 | if (std::isnan(adfValues[i])) |
153 | 0 | { |
154 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
155 | 0 | "Dataset %s: indexing variable %s of dimension " |
156 | 0 | "%s has NaN values", |
157 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
158 | 0 | desc.osName.c_str()); |
159 | 0 | return std::nullopt; |
160 | 0 | } |
161 | 0 | const int nSign2 = adfValues[i] > adfValues[i - 1] ? 1 : -1; |
162 | 0 | if (nSign != nSign2) |
163 | 0 | { |
164 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
165 | 0 | "Dataset %s: indexing variable %s of dimension " |
166 | 0 | "%s is not strictly increasing or decreasing", |
167 | 0 | osDSName.c_str(), poVar->GetName().c_str(), |
168 | 0 | desc.osName.c_str()); |
169 | 0 | return std::nullopt; |
170 | 0 | } |
171 | 0 | } |
172 | 0 | desc.nProgressionSign = nSign; |
173 | 0 | } |
174 | 0 | std::sort(adfValues.begin(), adfValues.end()); |
175 | 0 | desc.aaValues.push_back(std::move(adfValues)); |
176 | 0 | } |
177 | 0 | return desc; |
178 | 0 | } |
179 | | |
180 | | /************************************************************************/ |
181 | | /* GDALMdimMosaicAlgorithm::BuildArrayParameters() */ |
182 | | /************************************************************************/ |
183 | | |
184 | | bool GDALMdimMosaicAlgorithm::BuildArrayParameters( |
185 | | const CPLStringList &aosInputDatasetNames, |
186 | | std::vector<ArrayParameters> &aoArrayParameters) |
187 | 0 | { |
188 | 0 | for (const char *pszDatasetName : cpl::Iterate(aosInputDatasetNames)) |
189 | 0 | { |
190 | 0 | auto poDS = std::unique_ptr<GDALDataset>(GDALDataset::Open( |
191 | 0 | pszDatasetName, GDAL_OF_MULTIDIM_RASTER | GDAL_OF_VERBOSE_ERROR, |
192 | 0 | CPLStringList(m_inputFormats).List(), |
193 | 0 | CPLStringList(m_openOptions).List(), nullptr)); |
194 | 0 | if (!poDS) |
195 | 0 | return false; |
196 | 0 | auto poRG = poDS->GetRootGroup(); |
197 | 0 | if (!poRG) |
198 | 0 | { |
199 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
200 | 0 | "Cannot get root group for dataset %s", pszDatasetName); |
201 | 0 | return false; |
202 | 0 | } |
203 | 0 | std::vector<std::shared_ptr<GDALMDArray>> apoArrays; |
204 | 0 | if (!m_array.empty()) |
205 | 0 | { |
206 | 0 | for (const auto &array : m_array) |
207 | 0 | { |
208 | 0 | auto poArray = poRG->OpenMDArrayFromFullname(array); |
209 | 0 | if (!poArray) |
210 | 0 | { |
211 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
212 | 0 | "Cannot find array %s in dataset %s", |
213 | 0 | array.c_str(), pszDatasetName); |
214 | 0 | return false; |
215 | 0 | } |
216 | 0 | apoArrays.push_back(std::move(poArray)); |
217 | 0 | } |
218 | 0 | } |
219 | 0 | else |
220 | 0 | { |
221 | 0 | for (const std::string &arrayName : |
222 | 0 | poRG->GetMDArrayFullNamesRecursive()) |
223 | 0 | { |
224 | 0 | auto poArray = poRG->OpenMDArrayFromFullname(arrayName); |
225 | 0 | if (!poArray) |
226 | 0 | { |
227 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
228 | 0 | "Cannot open array %s of dataset %s", |
229 | 0 | arrayName.c_str(), pszDatasetName); |
230 | 0 | return false; |
231 | 0 | } |
232 | 0 | if (poArray->GetDimensionCount() < 2) |
233 | 0 | continue; |
234 | 0 | m_array.push_back(arrayName); |
235 | 0 | apoArrays.push_back(std::move(poArray)); |
236 | 0 | } |
237 | 0 | if (apoArrays.empty()) |
238 | 0 | { |
239 | 0 | ReportError( |
240 | 0 | CE_Failure, CPLE_AppDefined, |
241 | 0 | "No array of dimension count >= 2 found in dataset %s", |
242 | 0 | pszDatasetName); |
243 | 0 | return false; |
244 | 0 | } |
245 | 0 | } |
246 | | |
247 | 0 | if (aoArrayParameters.empty()) |
248 | 0 | aoArrayParameters.resize(apoArrays.size()); |
249 | |
|
250 | 0 | for (size_t iArray = 0; iArray < apoArrays.size(); ++iArray) |
251 | 0 | { |
252 | 0 | auto &arrayParameters = aoArrayParameters[iArray]; |
253 | 0 | auto &poArray = apoArrays[iArray]; |
254 | 0 | if (poArray->GetDimensionCount() == 0) |
255 | 0 | { |
256 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
257 | 0 | "Array %s of dataset %s has no dimensions", |
258 | 0 | poArray->GetName().c_str(), pszDatasetName); |
259 | 0 | return false; |
260 | 0 | } |
261 | | |
262 | 0 | std::vector<SourceShortDimDesc> aoSourceShortDimDesc; |
263 | 0 | const auto AddToSourceShortDimDesc = |
264 | 0 | [&aoSourceShortDimDesc](const DimensionDesc &dimDesc, |
265 | 0 | uint64_t nSize) |
266 | 0 | { |
267 | 0 | SourceShortDimDesc sourceDesc; |
268 | 0 | sourceDesc.nSize = nSize; |
269 | 0 | sourceDesc.bIsRegularlySpaced = dimDesc.aaValues.empty(); |
270 | 0 | if (sourceDesc.bIsRegularlySpaced) |
271 | 0 | sourceDesc.dfStart = dimDesc.dfStart; |
272 | 0 | else |
273 | 0 | sourceDesc.dfStart = dimDesc.aaValues[0][0]; |
274 | 0 | aoSourceShortDimDesc.push_back(std::move(sourceDesc)); |
275 | 0 | }; |
276 | |
|
277 | 0 | const auto anBlockSize = poArray->GetBlockSize(); |
278 | 0 | CPLAssert(anBlockSize.size() == poArray->GetDimensionCount()); |
279 | | |
280 | 0 | if (!arrayParameters.poFirstSourceArray) |
281 | 0 | { |
282 | 0 | arrayParameters.poFirstSourceArray = poArray; |
283 | 0 | CPLAssert(arrayParameters.mosaicDimensions.empty()); |
284 | 0 | size_t iDim = 0; |
285 | 0 | for (auto &poDim : poArray->GetDimensions()) |
286 | 0 | { |
287 | 0 | auto descOpt = GetDimensionDesc(pszDatasetName, poDim); |
288 | 0 | if (!descOpt.has_value()) |
289 | 0 | return false; |
290 | 0 | auto &desc = descOpt.value(); |
291 | 0 | AddToSourceShortDimDesc(desc, poDim->GetSize()); |
292 | 0 | desc.nBlockSize = anBlockSize[iDim]; |
293 | 0 | arrayParameters.mosaicDimensions.push_back(std::move(desc)); |
294 | 0 | ++iDim; |
295 | 0 | } |
296 | 0 | } |
297 | 0 | else |
298 | 0 | { |
299 | 0 | if (poArray->GetDimensionCount() != |
300 | 0 | arrayParameters.mosaicDimensions.size()) |
301 | 0 | { |
302 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
303 | 0 | "Array %s of dataset %s does not have the same " |
304 | 0 | "number of dimensions as in other datasets", |
305 | 0 | poArray->GetName().c_str(), pszDatasetName); |
306 | 0 | return false; |
307 | 0 | } |
308 | 0 | if (poArray->GetDataType() != |
309 | 0 | arrayParameters.poFirstSourceArray->GetDataType()) |
310 | 0 | { |
311 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
312 | 0 | "Array %s of dataset %s does not have the same " |
313 | 0 | "data type as in other datasets", |
314 | 0 | poArray->GetName().c_str(), pszDatasetName); |
315 | 0 | return false; |
316 | 0 | } |
317 | 0 | const void *poFirstRawNoData = |
318 | 0 | arrayParameters.poFirstSourceArray->GetRawNoDataValue(); |
319 | 0 | const void *poRawNoData = poArray->GetRawNoDataValue(); |
320 | 0 | if (!((!poFirstRawNoData && !poRawNoData) || |
321 | 0 | (poFirstRawNoData && poRawNoData && |
322 | 0 | memcmp(poFirstRawNoData, poRawNoData, |
323 | 0 | poArray->GetDataType().GetSize()) == 0))) |
324 | 0 | { |
325 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
326 | 0 | "Array %s of dataset %s does not have the same " |
327 | 0 | "nodata value as in other datasets", |
328 | 0 | poArray->GetName().c_str(), pszDatasetName); |
329 | 0 | return false; |
330 | 0 | } |
331 | 0 | const std::vector<std::shared_ptr<GDALDimension>> apoDims = |
332 | 0 | poArray->GetDimensions(); |
333 | 0 | for (size_t iDim = 0; |
334 | 0 | iDim < arrayParameters.mosaicDimensions.size(); ++iDim) |
335 | 0 | { |
336 | 0 | DimensionDesc &desc = |
337 | 0 | arrayParameters.mosaicDimensions[iDim]; |
338 | 0 | auto &poDim = apoDims[iDim]; |
339 | 0 | if (poDim->GetName() != desc.osName) |
340 | 0 | { |
341 | 0 | ReportError( |
342 | 0 | CE_Failure, CPLE_AppDefined, |
343 | 0 | "Dimension %d of array %s of dataset %s does " |
344 | 0 | "not have the same name as in other datasets", |
345 | 0 | static_cast<int>(iDim), poArray->GetName().c_str(), |
346 | 0 | pszDatasetName); |
347 | 0 | return false; |
348 | 0 | } |
349 | 0 | if (desc.nBlockSize != anBlockSize[iDim]) |
350 | 0 | desc.nBlockSize = 0; |
351 | |
|
352 | 0 | auto descThisDatasetOpt = |
353 | 0 | GetDimensionDesc(pszDatasetName, poDim); |
354 | 0 | if (!descThisDatasetOpt.has_value()) |
355 | 0 | return false; |
356 | 0 | auto &descThisDataset = descThisDatasetOpt.value(); |
357 | 0 | AddToSourceShortDimDesc(descThisDataset, poDim->GetSize()); |
358 | 0 | if (descThisDataset.bHasIndexingVar && |
359 | 0 | !desc.bHasIndexingVar) |
360 | 0 | { |
361 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
362 | 0 | "Dimension %s of array %s of dataset %s " |
363 | 0 | "has an indexing variable, contrary " |
364 | 0 | "to other datasets", |
365 | 0 | poDim->GetName().c_str(), |
366 | 0 | poArray->GetName().c_str(), pszDatasetName); |
367 | 0 | return false; |
368 | 0 | } |
369 | 0 | else if (!descThisDataset.bHasIndexingVar && |
370 | 0 | desc.bHasIndexingVar) |
371 | 0 | { |
372 | 0 | ReportError( |
373 | 0 | CE_Failure, CPLE_AppDefined, |
374 | 0 | "Dimension %s of array %s of dataset %s " |
375 | 0 | "does not have an indexing variable, contrary " |
376 | 0 | "to other datasets", |
377 | 0 | poDim->GetName().c_str(), |
378 | 0 | poArray->GetName().c_str(), pszDatasetName); |
379 | 0 | return false; |
380 | 0 | } |
381 | 0 | else if (!desc.bHasIndexingVar && |
382 | 0 | descThisDataset.nSize != desc.nSize) |
383 | 0 | { |
384 | 0 | ReportError( |
385 | 0 | CE_Failure, CPLE_AppDefined, |
386 | 0 | "Dimension %s of array %s of dataset %s " |
387 | 0 | "does not have the same size as in other datasets", |
388 | 0 | poDim->GetName().c_str(), |
389 | 0 | poArray->GetName().c_str(), pszDatasetName); |
390 | 0 | return false; |
391 | 0 | } |
392 | 0 | else if (desc.bHasIndexingVar && desc.aaValues.empty()) |
393 | 0 | { |
394 | 0 | if (!descThisDataset.aaValues.empty()) |
395 | 0 | { |
396 | 0 | ReportError( |
397 | 0 | CE_Failure, CPLE_AppDefined, |
398 | 0 | "Dimension %s of array %s of dataset %s " |
399 | 0 | "has irregularly-spaced values, contrary " |
400 | 0 | "to other datasets", |
401 | 0 | poDim->GetName().c_str(), |
402 | 0 | poArray->GetName().c_str(), pszDatasetName); |
403 | 0 | return false; |
404 | 0 | } |
405 | 0 | if (std::fabs(descThisDataset.dfIncrement - |
406 | 0 | desc.dfIncrement) > |
407 | 0 | 1e-10 * std::fabs(desc.dfIncrement)) |
408 | 0 | { |
409 | 0 | ReportError( |
410 | 0 | CE_Failure, CPLE_AppDefined, |
411 | 0 | "Dimension %s of array %s of dataset %s is " |
412 | 0 | "indexed by a variable with spacing %g, " |
413 | 0 | "whereas it is %g in other datasets", |
414 | 0 | poDim->GetName().c_str(), |
415 | 0 | poArray->GetName().c_str(), pszDatasetName, |
416 | 0 | descThisDataset.dfIncrement, desc.dfIncrement); |
417 | 0 | return false; |
418 | 0 | } |
419 | 0 | const double dfPos = |
420 | 0 | (descThisDataset.dfStart - desc.dfStart) / |
421 | 0 | desc.dfIncrement; |
422 | 0 | if (std::fabs(std::round(dfPos) - dfPos) > 1e-3) |
423 | 0 | { |
424 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
425 | 0 | "Dimension %s of array %s of dataset " |
426 | 0 | "%s is indexed " |
427 | 0 | "by a variable whose start value is " |
428 | 0 | "not aligned " |
429 | 0 | "with the one of other datasets", |
430 | 0 | poDim->GetName().c_str(), |
431 | 0 | poArray->GetName().c_str(), |
432 | 0 | pszDatasetName); |
433 | 0 | return false; |
434 | 0 | } |
435 | 0 | const double dfEnd = std::max( |
436 | 0 | desc.dfStart + static_cast<double>(desc.nSize) * |
437 | 0 | desc.dfIncrement, |
438 | 0 | descThisDataset.dfStart + |
439 | 0 | static_cast<double>(descThisDataset.nSize) * |
440 | 0 | descThisDataset.dfIncrement); |
441 | 0 | desc.dfStart = |
442 | 0 | std::min(desc.dfStart, descThisDataset.dfStart); |
443 | 0 | const double dfSize = |
444 | 0 | (dfEnd - desc.dfStart) / desc.dfIncrement; |
445 | 0 | constexpr double MAX_INTEGER_REPRESENTABLE = |
446 | 0 | static_cast<double>(1ULL << 53); |
447 | 0 | if (dfSize > MAX_INTEGER_REPRESENTABLE) |
448 | 0 | { |
449 | 0 | ReportError( |
450 | 0 | CE_Failure, CPLE_AppDefined, |
451 | 0 | "Dimension %s of array %s of dataset %s " |
452 | 0 | "would be too large if merged.", |
453 | 0 | poDim->GetName().c_str(), |
454 | 0 | poArray->GetName().c_str(), pszDatasetName); |
455 | 0 | return false; |
456 | 0 | } |
457 | 0 | desc.nSize = static_cast<uint64_t>(dfSize + 0.5); |
458 | 0 | } |
459 | 0 | else if (desc.bHasIndexingVar) |
460 | 0 | { |
461 | 0 | if (descThisDataset.aaValues.empty()) |
462 | 0 | { |
463 | 0 | ReportError( |
464 | 0 | CE_Failure, CPLE_AppDefined, |
465 | 0 | "Dimension %s of array %s of dataset %s " |
466 | 0 | "has regularly spaced labels, contrary to " |
467 | 0 | "other datasets", |
468 | 0 | poDim->GetName().c_str(), |
469 | 0 | poArray->GetName().c_str(), pszDatasetName); |
470 | 0 | return false; |
471 | 0 | } |
472 | 0 | if (descThisDataset.nProgressionSign != |
473 | 0 | desc.nProgressionSign) |
474 | 0 | { |
475 | 0 | ReportError( |
476 | 0 | CE_Failure, CPLE_AppDefined, |
477 | 0 | "Dataset %s: values in indexing variable %s of " |
478 | 0 | "dimension %s must be either increasing or " |
479 | 0 | "decreasing in all input datasets.", |
480 | 0 | pszDatasetName, |
481 | 0 | poDim->GetIndexingVariable()->GetName().c_str(), |
482 | 0 | desc.osName.c_str()); |
483 | 0 | return false; |
484 | 0 | } |
485 | 0 | CPLAssert(descThisDataset.aaValues.size() == 1); |
486 | 0 | if (descThisDataset.aaValues[0][0] < |
487 | 0 | desc.aaValues[0][0]) |
488 | 0 | { |
489 | 0 | if (descThisDataset.aaValues[0].back() >= |
490 | 0 | desc.aaValues[0][0]) |
491 | 0 | { |
492 | 0 | ReportError( |
493 | 0 | CE_Failure, CPLE_AppDefined, |
494 | 0 | "Dataset %s: values in indexing variable " |
495 | 0 | "%s of " |
496 | 0 | "dimension %s are not the same as in other " |
497 | 0 | "datasets", |
498 | 0 | pszDatasetName, |
499 | 0 | poDim->GetIndexingVariable() |
500 | 0 | ->GetName() |
501 | 0 | .c_str(), |
502 | 0 | desc.osName.c_str()); |
503 | 0 | return false; |
504 | 0 | } |
505 | 0 | desc.aaValues.insert( |
506 | 0 | desc.aaValues.begin(), |
507 | 0 | std::move(descThisDataset.aaValues[0])); |
508 | 0 | } |
509 | 0 | else |
510 | 0 | { |
511 | 0 | for (size_t i = 0; i < desc.aaValues.size(); ++i) |
512 | 0 | { |
513 | 0 | if (descThisDataset.aaValues[0][0] == |
514 | 0 | desc.aaValues[i][0]) |
515 | 0 | { |
516 | 0 | if (descThisDataset.aaValues[0] != |
517 | 0 | desc.aaValues[i]) |
518 | 0 | { |
519 | 0 | ReportError( |
520 | 0 | CE_Failure, CPLE_AppDefined, |
521 | 0 | "Dataset %s: values in indexing " |
522 | 0 | "variable %s of dimension %s are " |
523 | 0 | "not " |
524 | 0 | "the same as in other datasets", |
525 | 0 | pszDatasetName, |
526 | 0 | poDim->GetIndexingVariable() |
527 | 0 | ->GetName() |
528 | 0 | .c_str(), |
529 | 0 | desc.osName.c_str()); |
530 | 0 | return false; |
531 | 0 | } |
532 | 0 | } |
533 | 0 | else if (descThisDataset.aaValues[0][0] > |
534 | 0 | desc.aaValues[i][0] && |
535 | 0 | (i + 1 == desc.aaValues.size() || |
536 | 0 | descThisDataset.aaValues[0][0] < |
537 | 0 | desc.aaValues[i + 1][0])) |
538 | 0 | { |
539 | 0 | if (descThisDataset.aaValues[0][0] <= |
540 | 0 | desc.aaValues[i].back()) |
541 | 0 | { |
542 | 0 | ReportError( |
543 | 0 | CE_Failure, CPLE_AppDefined, |
544 | 0 | "Dataset %s: values in indexing " |
545 | 0 | "variable %s of dimension %s are " |
546 | 0 | "overlapping with the ones of " |
547 | 0 | "other " |
548 | 0 | "datasets", |
549 | 0 | pszDatasetName, |
550 | 0 | poDim->GetIndexingVariable() |
551 | 0 | ->GetName() |
552 | 0 | .c_str(), |
553 | 0 | desc.osName.c_str()); |
554 | 0 | return false; |
555 | 0 | } |
556 | 0 | if (i + 1 < desc.aaValues.size() && |
557 | 0 | descThisDataset.aaValues[0].back() >= |
558 | 0 | desc.aaValues[i + 1][0]) |
559 | 0 | { |
560 | 0 | ReportError( |
561 | 0 | CE_Failure, CPLE_AppDefined, |
562 | 0 | "Dataset %s: values in indexing " |
563 | 0 | "variable %s of dimension %s are " |
564 | 0 | "overlapping with the ones of " |
565 | 0 | "other " |
566 | 0 | "datasets", |
567 | 0 | pszDatasetName, |
568 | 0 | poDim->GetIndexingVariable() |
569 | 0 | ->GetName() |
570 | 0 | .c_str(), |
571 | 0 | desc.osName.c_str()); |
572 | 0 | return false; |
573 | 0 | } |
574 | 0 | desc.aaValues.insert( |
575 | 0 | desc.aaValues.begin() + i + 1, |
576 | 0 | std::move(descThisDataset.aaValues[0])); |
577 | 0 | break; |
578 | 0 | } |
579 | 0 | } |
580 | 0 | } |
581 | 0 | } |
582 | 0 | } |
583 | 0 | } |
584 | | |
585 | 0 | arrayParameters.aaoSourceShortDimDesc.push_back( |
586 | 0 | std::move(aoSourceShortDimDesc)); |
587 | 0 | } |
588 | 0 | } |
589 | | |
590 | 0 | return true; |
591 | 0 | } |
592 | | |
593 | | /************************************************************************/ |
594 | | /* GDALMdimMosaicAlgorithm::GetInputDatasetNames() */ |
595 | | /************************************************************************/ |
596 | | |
597 | | bool GDALMdimMosaicAlgorithm::GetInputDatasetNames( |
598 | | GDALProgressFunc pfnProgress, void *pProgressData, |
599 | | CPLStringList &aosInputDatasetNames) const |
600 | 0 | { |
601 | 0 | for (auto &ds : m_inputDataset) |
602 | 0 | { |
603 | 0 | if (ds.GetName()[0] == '@') |
604 | 0 | { |
605 | 0 | auto f = VSIVirtualHandleUniquePtr( |
606 | 0 | VSIFOpenL(ds.GetName().c_str() + 1, "r")); |
607 | 0 | if (!f) |
608 | 0 | { |
609 | 0 | ReportError(CE_Failure, CPLE_FileIO, "Cannot open %s", |
610 | 0 | ds.GetName().c_str() + 1); |
611 | 0 | return false; |
612 | 0 | } |
613 | 0 | while (const char *filename = CPLReadLineL(f.get())) |
614 | 0 | { |
615 | 0 | aosInputDatasetNames.push_back(filename); |
616 | 0 | } |
617 | 0 | } |
618 | 0 | else if (ds.GetName().find_first_of("*?[") != std::string::npos) |
619 | 0 | { |
620 | 0 | CPLStringList aosMatches(VSIGlob(ds.GetName().c_str(), nullptr, |
621 | 0 | pfnProgress, pProgressData)); |
622 | 0 | for (const char *pszStr : aosMatches) |
623 | 0 | { |
624 | 0 | aosInputDatasetNames.push_back(pszStr); |
625 | 0 | } |
626 | 0 | } |
627 | 0 | else |
628 | 0 | { |
629 | 0 | std::string osDatasetName = ds.GetName(); |
630 | 0 | if (!GetReferencePathForRelativePaths().empty()) |
631 | 0 | { |
632 | 0 | osDatasetName = GDALDataset::BuildFilename( |
633 | 0 | osDatasetName.c_str(), |
634 | 0 | GetReferencePathForRelativePaths().c_str(), true); |
635 | 0 | } |
636 | 0 | aosInputDatasetNames.push_back(osDatasetName.c_str()); |
637 | 0 | } |
638 | 0 | } |
639 | 0 | return true; |
640 | 0 | } |
641 | | |
642 | | /************************************************************************/ |
643 | | /* GDALMdimMosaicAlgorithm::RunStep() */ |
644 | | /************************************************************************/ |
645 | | |
646 | | bool GDALMdimMosaicAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt) |
647 | 0 | { |
648 | 0 | CPLAssert(!m_outputDataset.GetDatasetRef()); |
649 | | |
650 | 0 | GDALDriver *poOutDrv = nullptr; |
651 | 0 | if (m_standaloneStep) |
652 | 0 | { |
653 | 0 | if (m_format.empty()) |
654 | 0 | { |
655 | 0 | const auto aosFormats = |
656 | 0 | CPLStringList(GDALGetOutputDriversForDatasetName( |
657 | 0 | m_outputDataset.GetName().c_str(), GDAL_OF_MULTIDIM_RASTER, |
658 | 0 | /* bSingleMatch = */ true, |
659 | 0 | /* bWarn = */ true)); |
660 | 0 | if (aosFormats.size() != 1) |
661 | 0 | { |
662 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
663 | 0 | "Cannot guess driver for %s", |
664 | 0 | m_outputDataset.GetName().c_str()); |
665 | 0 | return false; |
666 | 0 | } |
667 | 0 | m_format = aosFormats[0]; |
668 | 0 | } |
669 | 0 | poOutDrv = GetGDALDriverManager()->GetDriverByName(m_format.c_str()); |
670 | 0 | if (!poOutDrv) |
671 | 0 | { |
672 | 0 | ReportError(CE_Failure, CPLE_AppDefined, "Driver %s does not exist", |
673 | 0 | m_format.c_str()); |
674 | 0 | return false; |
675 | 0 | } |
676 | 0 | } |
677 | | |
678 | 0 | const double dfIntermediatePercentage = !m_standaloneStep ? 1.0 : 0.1; |
679 | 0 | std::unique_ptr<void, decltype(&GDALDestroyScaledProgress)> pScaledData( |
680 | 0 | GDALCreateScaledProgress(0.0, dfIntermediatePercentage, |
681 | 0 | ctxt.m_pfnProgress, ctxt.m_pProgressData), |
682 | 0 | GDALDestroyScaledProgress); |
683 | |
|
684 | 0 | CPLStringList aosInputDatasetNames; |
685 | 0 | if (!GetInputDatasetNames(GDALScaledProgress, pScaledData.get(), |
686 | 0 | aosInputDatasetNames)) |
687 | 0 | return false; |
688 | | |
689 | 0 | for (std::string &name : m_array) |
690 | 0 | { |
691 | 0 | if (!name.empty() && name[0] != '/') |
692 | 0 | name = "/" + name; |
693 | 0 | } |
694 | |
|
695 | 0 | std::vector<ArrayParameters> aoArrayParameters; |
696 | 0 | if (!BuildArrayParameters(aosInputDatasetNames, aoArrayParameters)) |
697 | 0 | { |
698 | 0 | return false; |
699 | 0 | } |
700 | | |
701 | 0 | auto poVRTDS = VRTDataset::CreateVRTMultiDimensional("", nullptr, nullptr); |
702 | 0 | CPLAssert(poVRTDS); |
703 | | |
704 | 0 | auto poDstGroup = poVRTDS->GetRootVRTGroup(); |
705 | 0 | CPLAssert(poDstGroup); |
706 | | |
707 | 0 | std::map<std::string, std::shared_ptr<GDALDimension>> |
708 | 0 | oMapAlreadyCreatedDims; |
709 | | |
710 | | // Iterate over arrays |
711 | 0 | for (auto &arrayParameters : aoArrayParameters) |
712 | 0 | { |
713 | 0 | auto &poFirstSourceArray = arrayParameters.poFirstSourceArray; |
714 | 0 | CPLAssert(poFirstSourceArray); |
715 | 0 | auto &aaoSourceShortDimDesc = arrayParameters.aaoSourceShortDimDesc; |
716 | 0 | CPLAssert(aaoSourceShortDimDesc.size() == |
717 | 0 | static_cast<size_t>(aosInputDatasetNames.size())); |
718 | | |
719 | | // Create mosaic array dimensions |
720 | 0 | std::vector<std::shared_ptr<GDALDimension>> apoDstDims; |
721 | 0 | for (auto &desc : arrayParameters.mosaicDimensions) |
722 | 0 | { |
723 | 0 | auto oIterCreatedDims = oMapAlreadyCreatedDims.find(desc.osName); |
724 | 0 | if (oIterCreatedDims != oMapAlreadyCreatedDims.end()) |
725 | 0 | { |
726 | 0 | apoDstDims.push_back(oIterCreatedDims->second); |
727 | 0 | } |
728 | 0 | else |
729 | 0 | { |
730 | 0 | uint64_t nDimSize64 = desc.nSize; |
731 | 0 | if (!desc.aaValues.empty()) |
732 | 0 | { |
733 | 0 | nDimSize64 = 0; |
734 | 0 | for (const auto &aValues : desc.aaValues) |
735 | 0 | nDimSize64 += aValues.size(); |
736 | 0 | } |
737 | 0 | auto dstDim = poDstGroup->CreateDimension( |
738 | 0 | desc.osName, desc.osType, desc.osDirection, nDimSize64); |
739 | 0 | if (!dstDim) |
740 | 0 | return false; |
741 | | |
742 | 0 | if (desc.bHasIndexingVar) |
743 | 0 | { |
744 | 0 | auto var = poDstGroup->CreateVRTMDArray( |
745 | 0 | desc.osName, {dstDim}, |
746 | 0 | GDALExtendedDataType::Create(GDT_Float64)); |
747 | 0 | if (!var) |
748 | 0 | return false; |
749 | | |
750 | 0 | for (const auto &attr : desc.attributes) |
751 | 0 | { |
752 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
753 | 0 | auto dstAttr = var->CreateAttribute( |
754 | 0 | attr->GetName(), attr->GetDimensionsSize(), |
755 | 0 | attr->GetDataType()); |
756 | 0 | if (dstAttr) |
757 | 0 | { |
758 | 0 | auto raw(attr->ReadAsRaw()); |
759 | 0 | CPL_IGNORE_RET_VAL( |
760 | 0 | dstAttr->Write(raw.data(), raw.size())); |
761 | 0 | } |
762 | 0 | } |
763 | 0 | if (!desc.osUnit.empty()) |
764 | 0 | { |
765 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
766 | 0 | var->SetUnit(desc.osUnit); |
767 | 0 | } |
768 | |
|
769 | 0 | if (desc.aaValues.empty()) |
770 | 0 | { |
771 | 0 | auto poSource = |
772 | 0 | std::make_unique<VRTMDArraySourceRegularlySpaced>( |
773 | 0 | desc.dfStart, desc.dfIncrement); |
774 | 0 | var->AddSource(std::move(poSource)); |
775 | 0 | } |
776 | 0 | else |
777 | 0 | { |
778 | 0 | const size_t nDimSize = static_cast<size_t>(nDimSize64); |
779 | 0 | std::vector<GUInt64> anOffset = {0}; |
780 | 0 | std::vector<size_t> anCount = {nDimSize}; |
781 | 0 | std::vector<double> adfValues; |
782 | 0 | adfValues.reserve(nDimSize); |
783 | 0 | if (desc.nProgressionSign >= 0) |
784 | 0 | { |
785 | 0 | for (const auto &aValues : desc.aaValues) |
786 | 0 | adfValues.insert(adfValues.end(), |
787 | 0 | aValues.begin(), |
788 | 0 | aValues.end()); |
789 | 0 | } |
790 | 0 | else |
791 | 0 | { |
792 | 0 | for (auto oIter = desc.aaValues.rbegin(); |
793 | 0 | oIter != desc.aaValues.rend(); ++oIter) |
794 | 0 | { |
795 | 0 | adfValues.insert(adfValues.end(), |
796 | 0 | oIter->rbegin(), |
797 | 0 | oIter->rend()); |
798 | 0 | } |
799 | 0 | } |
800 | 0 | std::vector<GByte> abyValues(nDimSize * sizeof(double)); |
801 | 0 | memcpy(abyValues.data(), adfValues.data(), |
802 | 0 | nDimSize * sizeof(double)); |
803 | 0 | auto poSource = |
804 | 0 | std::make_unique<VRTMDArraySourceInlinedValues>( |
805 | 0 | var.get(), |
806 | 0 | /* bIsConstantValue = */ false, |
807 | 0 | std::move(anOffset), std::move(anCount), |
808 | 0 | std::move(abyValues)); |
809 | 0 | var->AddSource(std::move(poSource)); |
810 | 0 | } |
811 | 0 | dstDim->SetIndexingVariable(std::move(var)); |
812 | 0 | } |
813 | | |
814 | 0 | oMapAlreadyCreatedDims[dstDim->GetName()] = dstDim; |
815 | 0 | apoDstDims.push_back(std::move(dstDim)); |
816 | 0 | } |
817 | 0 | } |
818 | | |
819 | | // Create mosaic array |
820 | 0 | CPLStringList aosArrayCO; |
821 | 0 | std::string osBlockSize; |
822 | 0 | for (size_t i = 0; i < apoDstDims.size(); ++i) |
823 | 0 | { |
824 | 0 | if (i > 0) |
825 | 0 | osBlockSize += ','; |
826 | 0 | osBlockSize += |
827 | 0 | std::to_string(arrayParameters.mosaicDimensions[i].nBlockSize); |
828 | 0 | } |
829 | 0 | if (!osBlockSize.empty()) |
830 | 0 | aosArrayCO.SetNameValue("BLOCKSIZE", osBlockSize.c_str()); |
831 | |
|
832 | 0 | auto poDstArray = poDstGroup->CreateVRTMDArray( |
833 | 0 | CPLGetFilename(poFirstSourceArray->GetName().c_str()), apoDstDims, |
834 | 0 | poFirstSourceArray->GetDataType(), aosArrayCO); |
835 | 0 | if (!poDstArray) |
836 | 0 | return false; |
837 | | |
838 | 0 | GUInt64 nCurCost = 0; |
839 | 0 | poDstArray->CopyFromAllExceptValues(poFirstSourceArray.get(), false, |
840 | 0 | nCurCost, 0, nullptr, nullptr); |
841 | | |
842 | | // Add sources to mosaic array |
843 | 0 | for (int iDS = 0; iDS < aosInputDatasetNames.size(); ++iDS) |
844 | 0 | { |
845 | 0 | const auto &aoSourceShortDimDesc = aaoSourceShortDimDesc[iDS]; |
846 | |
|
847 | 0 | const auto dimCount = arrayParameters.mosaicDimensions.size(); |
848 | 0 | std::vector<GUInt64> anSrcOffset(dimCount); |
849 | 0 | std::vector<GUInt64> anCount(dimCount); |
850 | 0 | std::vector<GUInt64> anDstOffset; |
851 | 0 | CPLAssert(aoSourceShortDimDesc.size() == dimCount); |
852 | | |
853 | 0 | for (size_t iDim = 0; iDim < dimCount; ++iDim) |
854 | 0 | { |
855 | 0 | const DimensionDesc &desc = |
856 | 0 | arrayParameters.mosaicDimensions[iDim]; |
857 | 0 | const SourceShortDimDesc &sourceDesc = |
858 | 0 | aoSourceShortDimDesc[iDim]; |
859 | 0 | if (sourceDesc.bIsRegularlySpaced) |
860 | 0 | { |
861 | 0 | const double dfPos = |
862 | 0 | (sourceDesc.dfStart - desc.dfStart) / desc.dfIncrement; |
863 | 0 | anDstOffset.push_back(static_cast<uint64_t>(dfPos + 0.5)); |
864 | 0 | } |
865 | 0 | else |
866 | 0 | { |
867 | 0 | uint64_t nPos = 0; |
868 | 0 | bool bFound = false; |
869 | 0 | for (size_t i = 0; i < desc.aaValues.size(); ++i) |
870 | 0 | { |
871 | 0 | if (sourceDesc.dfStart == desc.aaValues[i][0]) |
872 | 0 | { |
873 | 0 | bFound = true; |
874 | 0 | break; |
875 | 0 | } |
876 | 0 | else |
877 | 0 | { |
878 | 0 | nPos += desc.aaValues[i].size(); |
879 | 0 | } |
880 | 0 | } |
881 | 0 | CPLAssert(bFound); |
882 | 0 | CPL_IGNORE_RET_VAL(bFound); |
883 | |
|
884 | 0 | anDstOffset.push_back(nPos); |
885 | 0 | } |
886 | | |
887 | 0 | anCount[iDim] = sourceDesc.nSize; |
888 | 0 | } |
889 | | |
890 | 0 | std::vector<GUInt64> anStep(dimCount, 1); |
891 | 0 | std::vector<int> anTransposedAxis; |
892 | 0 | auto poSource = std::make_unique<VRTMDArraySourceFromArray>( |
893 | 0 | poDstArray.get(), false, false, aosInputDatasetNames[iDS], |
894 | 0 | poFirstSourceArray->GetFullName(), std::string(), |
895 | 0 | std::move(anTransposedAxis), |
896 | 0 | std::string(), // viewExpr |
897 | 0 | std::move(anSrcOffset), std::move(anCount), std::move(anStep), |
898 | 0 | std::move(anDstOffset)); |
899 | 0 | poDstArray->AddSource(std::move(poSource)); |
900 | 0 | } |
901 | 0 | } |
902 | | |
903 | 0 | if (poOutDrv) |
904 | 0 | { |
905 | 0 | auto poOutDS = std::unique_ptr<GDALDataset>(poOutDrv->CreateCopy( |
906 | 0 | m_outputDataset.GetName().c_str(), poVRTDS.get(), false, |
907 | 0 | CPLStringList(m_creationOptions).List(), GDALScaledProgress, |
908 | 0 | pScaledData.get())); |
909 | 0 | if (poOutDS) |
910 | 0 | m_outputDataset.Set(std::move(poOutDS)); |
911 | 0 | } |
912 | 0 | else |
913 | 0 | { |
914 | 0 | m_outputDataset.Set(std::move(poVRTDS)); |
915 | 0 | } |
916 | |
|
917 | 0 | return m_outputDataset.GetDatasetRef() != nullptr; |
918 | 0 | } |
919 | | |
920 | | /************************************************************************/ |
921 | | /* GDALMdimMosaicAlgorithm::RunImpl() */ |
922 | | /************************************************************************/ |
923 | | |
924 | | bool GDALMdimMosaicAlgorithm::RunImpl(GDALProgressFunc pfnProgress, |
925 | | void *pProgressData) |
926 | 0 | { |
927 | 0 | GDALPipelineStepRunContext stepCtxt; |
928 | 0 | stepCtxt.m_pfnProgress = pfnProgress; |
929 | 0 | stepCtxt.m_pProgressData = pProgressData; |
930 | 0 | return RunStep(stepCtxt); |
931 | 0 | } |
932 | | |
933 | 0 | GDALMdimMosaicAlgorithmStandalone::~GDALMdimMosaicAlgorithmStandalone() = |
934 | | default; |
935 | | |
936 | | //! @endcond |