/src/gdal/apps/gdalalg_materialize.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: GDAL |
4 | | * Purpose: gdal "materialize" pipeline step |
5 | | * Author: Even Rouault <even dot rouault at spatialys.com> |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2025, Even Rouault <even dot rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "gdalalg_materialize.h" |
14 | | #include "gdal_utils.h" |
15 | | #include "gdal_priv.h" |
16 | | #include "ogrsf_frmts.h" |
17 | | |
18 | | //! @cond Doxygen_Suppress |
19 | | |
20 | | #ifndef _ |
21 | 0 | #define _(x) (x) |
22 | | #endif |
23 | | |
24 | | /************************************************************************/ |
25 | | /* GDALMaterializeRasterAlgorithm() */ |
26 | | /************************************************************************/ |
27 | | |
28 | | GDALMaterializeRasterAlgorithm::GDALMaterializeRasterAlgorithm() |
29 | 0 | : GDALMaterializeStepAlgorithm<GDALRasterPipelineStepAlgorithm, |
30 | 0 | GDAL_OF_RASTER>(HELP_URL) |
31 | 0 | { |
32 | 0 | AddRasterHiddenInputDatasetArg(); |
33 | |
|
34 | 0 | AddOutputDatasetArg(&m_outputDataset, GDAL_OF_RASTER, |
35 | 0 | /* positionalAndRequired = */ false, |
36 | 0 | _("Materialized dataset name")) |
37 | 0 | .SetDatasetInputFlags(GADV_NAME); |
38 | |
|
39 | 0 | AddOutputFormatArg(&m_format) |
40 | 0 | .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, |
41 | 0 | {GDAL_DCAP_RASTER, GDAL_DCAP_CREATECOPY, |
42 | 0 | GDAL_DCAP_OPEN, GDAL_DMD_EXTENSIONS}) |
43 | 0 | .AddMetadataItem(GAAMDI_ALLOWED_FORMATS, {"MEM", "COG"}) |
44 | 0 | .AddMetadataItem(GAAMDI_EXCLUDED_FORMATS, {"VRT"}); |
45 | |
|
46 | 0 | AddCreationOptionsArg(&m_creationOptions); |
47 | 0 | AddOverwriteArg(&m_overwrite); |
48 | |
|
49 | 0 | AddArg(ARG_NAME_REOPEN_AND_DO_NOT_EARLY_DELETE, 0, |
50 | 0 | _("Reopen after materialization and do not early deleted"), |
51 | 0 | &m_reopenAndDoNotEarlyDelete) |
52 | 0 | .SetHidden(); |
53 | 0 | } |
54 | | |
55 | | /************************************************************************/ |
56 | | /* GDALMaterializeRasterAlgorithm::RunStep() */ |
57 | | /************************************************************************/ |
58 | | |
59 | | bool GDALMaterializeRasterAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt) |
60 | 0 | { |
61 | 0 | auto pfnProgress = ctxt.m_pfnProgress; |
62 | 0 | auto pProgressData = ctxt.m_pProgressData; |
63 | |
|
64 | 0 | auto poSrcDS = m_inputDataset[0].GetDatasetRef(); |
65 | 0 | CPLAssert(poSrcDS); |
66 | 0 | CPLAssert(!m_outputDataset.GetDatasetRef()); |
67 | | |
68 | 0 | std::string filename = m_outputDataset.GetName(); |
69 | 0 | if (m_format.empty()) |
70 | 0 | { |
71 | 0 | if (filename.empty()) |
72 | 0 | { |
73 | 0 | m_format = "GTiff"; |
74 | 0 | } |
75 | 0 | else |
76 | 0 | { |
77 | 0 | const auto aosFormats = |
78 | 0 | CPLStringList(GDALGetOutputDriversForDatasetName( |
79 | 0 | filename.c_str(), GDAL_OF_RASTER, |
80 | 0 | /* bSingleMatch = */ true, |
81 | 0 | /* bWarn = */ true)); |
82 | 0 | if (aosFormats.size() != 1) |
83 | 0 | { |
84 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
85 | 0 | "Cannot guess driver for %s", filename.c_str()); |
86 | 0 | return false; |
87 | 0 | } |
88 | 0 | m_format = aosFormats[0]; |
89 | 0 | } |
90 | 0 | } |
91 | | |
92 | 0 | auto poDrv = GetGDALDriverManager()->GetDriverByName(m_format.c_str()); |
93 | 0 | if (!poDrv) |
94 | 0 | { |
95 | 0 | ReportError(CE_Failure, CPLE_AppDefined, "Driver %s does not exist", |
96 | 0 | m_format.c_str()); |
97 | 0 | return false; |
98 | 0 | } |
99 | | |
100 | 0 | const bool autoDeleteFile = !m_reopenAndDoNotEarlyDelete && |
101 | 0 | filename.empty() && |
102 | 0 | !EQUAL(m_format.c_str(), "MEM"); |
103 | 0 | if (filename.empty() && !EQUAL(m_format.c_str(), "MEM")) |
104 | 0 | { |
105 | 0 | filename = CPLGenerateTempFilenameSafe(nullptr); |
106 | |
|
107 | 0 | const char *pszExt = poDrv->GetMetadataItem(GDAL_DMD_EXTENSIONS); |
108 | 0 | if (pszExt) |
109 | 0 | { |
110 | 0 | filename += '.'; |
111 | 0 | filename += CPLStringList(CSLTokenizeString(pszExt))[0]; |
112 | 0 | } |
113 | 0 | } |
114 | |
|
115 | 0 | CPLStringList aosOptions(m_creationOptions); |
116 | 0 | if (EQUAL(m_format.c_str(), "GTiff")) |
117 | 0 | { |
118 | 0 | if (aosOptions.FetchNameValue("TILED") == nullptr) |
119 | 0 | { |
120 | 0 | aosOptions.SetNameValue("TILED", "YES"); |
121 | 0 | } |
122 | 0 | if (aosOptions.FetchNameValue("COPY_SRC_OVERVIEWS") == nullptr) |
123 | 0 | { |
124 | 0 | aosOptions.SetNameValue("COPY_SRC_OVERVIEWS", "YES"); |
125 | 0 | } |
126 | 0 | if (aosOptions.FetchNameValue("COMPRESS") == nullptr) |
127 | 0 | { |
128 | 0 | const char *pszCOList = |
129 | 0 | poDrv->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST); |
130 | 0 | aosOptions.SetNameValue( |
131 | 0 | "COMPRESS", |
132 | 0 | pszCOList && strstr(pszCOList, "ZSTD") ? "ZSTD" : "DEFLATE"); |
133 | 0 | } |
134 | 0 | } |
135 | |
|
136 | 0 | if (autoDeleteFile) |
137 | 0 | { |
138 | 0 | aosOptions.SetNameValue("@SUPPRESS_ASAP", "YES"); |
139 | 0 | } |
140 | |
|
141 | 0 | auto poOutDS = std::unique_ptr<GDALDataset>( |
142 | 0 | poDrv->CreateCopy(filename.c_str(), poSrcDS, false, aosOptions.List(), |
143 | 0 | pfnProgress, pProgressData)); |
144 | 0 | bool ok = poOutDS != nullptr && poOutDS->FlushCache() == CE_None; |
145 | 0 | if (poOutDS) |
146 | 0 | { |
147 | 0 | if (m_reopenAndDoNotEarlyDelete || |
148 | 0 | poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED)) |
149 | 0 | { |
150 | 0 | ok = poOutDS->Close() == CE_None; |
151 | 0 | poOutDS.reset(); |
152 | 0 | if (ok) |
153 | 0 | { |
154 | 0 | std::string reopenFormat = m_format; |
155 | 0 | if (m_format == "COG") |
156 | 0 | reopenFormat = "GTiff"; |
157 | 0 | const char *const apszAllowedDrivers[] = {reopenFormat.c_str(), |
158 | 0 | nullptr}; |
159 | 0 | poOutDS.reset(GDALDataset::Open( |
160 | 0 | filename.c_str(), GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, |
161 | 0 | apszAllowedDrivers)); |
162 | 0 | ok = poOutDS != nullptr; |
163 | 0 | } |
164 | 0 | } |
165 | 0 | if (ok) |
166 | 0 | { |
167 | 0 | if (autoDeleteFile) |
168 | 0 | { |
169 | 0 | #if !defined(_WIN32) |
170 | 0 | if (poDrv->GetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE)) |
171 | 0 | { |
172 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
173 | 0 | poDrv->Delete(poOutDS.get(), |
174 | 0 | CPLStringList(poOutDS->GetFileList()).List()); |
175 | 0 | } |
176 | 0 | #endif |
177 | 0 | poOutDS->MarkSuppressOnClose(); |
178 | 0 | } |
179 | |
|
180 | 0 | m_outputDataset.Set(std::move(poOutDS)); |
181 | 0 | } |
182 | 0 | } |
183 | 0 | return ok; |
184 | 0 | } |
185 | | |
186 | | /************************************************************************/ |
187 | | /* GDALMaterializeVectorAlgorithm() */ |
188 | | /************************************************************************/ |
189 | | |
190 | | GDALMaterializeVectorAlgorithm::GDALMaterializeVectorAlgorithm() |
191 | 0 | : GDALMaterializeStepAlgorithm<GDALVectorPipelineStepAlgorithm, |
192 | 0 | GDAL_OF_VECTOR>(HELP_URL) |
193 | 0 | { |
194 | 0 | AddVectorHiddenInputDatasetArg(); |
195 | |
|
196 | 0 | AddOutputDatasetArg(&m_outputDataset, GDAL_OF_VECTOR, |
197 | 0 | /* positionalAndRequired = */ false, |
198 | 0 | _("Materialized dataset name")) |
199 | 0 | .SetDatasetInputFlags(GADV_NAME); |
200 | |
|
201 | 0 | AddOutputFormatArg(&m_format) |
202 | 0 | .AddMetadataItem(GAAMDI_REQUIRED_CAPABILITIES, |
203 | 0 | {GDAL_DCAP_VECTOR, GDAL_DCAP_CREATE, GDAL_DCAP_OPEN, |
204 | 0 | GDAL_DMD_EXTENSIONS}) |
205 | 0 | .AddMetadataItem(GAAMDI_ALLOWED_FORMATS, {"MEM"}) |
206 | 0 | .AddMetadataItem(GAAMDI_EXCLUDED_FORMATS, |
207 | 0 | {"MBTiles", "MVT", "PMTiles", "JP2ECW"}); |
208 | |
|
209 | 0 | AddCreationOptionsArg(&m_creationOptions); |
210 | 0 | AddLayerCreationOptionsArg(&m_layerCreationOptions); |
211 | 0 | AddOverwriteArg(&m_overwrite); |
212 | 0 | } |
213 | | |
214 | | /************************************************************************/ |
215 | | /* GDALMaterializeVectorAlgorithm::RunStep() */ |
216 | | /************************************************************************/ |
217 | | |
218 | | bool GDALMaterializeVectorAlgorithm::RunStep(GDALPipelineStepRunContext &ctxt) |
219 | 0 | { |
220 | 0 | auto pfnProgress = ctxt.m_pfnProgress; |
221 | 0 | auto pProgressData = ctxt.m_pProgressData; |
222 | |
|
223 | 0 | auto poSrcDS = m_inputDataset[0].GetDatasetRef(); |
224 | 0 | CPLAssert(poSrcDS); |
225 | 0 | CPLAssert(!m_outputDataset.GetDatasetRef()); |
226 | | |
227 | 0 | std::string filename = m_outputDataset.GetName(); |
228 | 0 | if (m_format.empty()) |
229 | 0 | { |
230 | 0 | if (filename.empty()) |
231 | 0 | { |
232 | 0 | bool bSeveralGeomFields = false; |
233 | 0 | for (const auto *poLayer : poSrcDS->GetLayers()) |
234 | 0 | { |
235 | 0 | if (!bSeveralGeomFields) |
236 | 0 | bSeveralGeomFields = |
237 | 0 | poLayer->GetLayerDefn()->GetGeomFieldCount() > 1; |
238 | 0 | if (!bSeveralGeomFields && |
239 | 0 | poLayer->GetLayerDefn()->GetGeomFieldCount() > 0) |
240 | 0 | { |
241 | 0 | for (const auto *poFieldDefn : |
242 | 0 | poLayer->GetLayerDefn()->GetFields()) |
243 | 0 | { |
244 | 0 | const auto eType = poFieldDefn->GetType(); |
245 | 0 | if (eType == OFTStringList || eType == OFTIntegerList || |
246 | 0 | eType == OFTRealList || eType == OFTInteger64List) |
247 | 0 | { |
248 | 0 | bSeveralGeomFields = true; |
249 | 0 | } |
250 | 0 | } |
251 | 0 | } |
252 | 0 | } |
253 | 0 | m_format = bSeveralGeomFields ? "SQLite" : "GPKG"; |
254 | 0 | } |
255 | 0 | else |
256 | 0 | { |
257 | 0 | const auto aosFormats = |
258 | 0 | CPLStringList(GDALGetOutputDriversForDatasetName( |
259 | 0 | filename.c_str(), GDAL_OF_VECTOR, |
260 | 0 | /* bSingleMatch = */ true, |
261 | 0 | /* bWarn = */ true)); |
262 | 0 | if (aosFormats.size() != 1) |
263 | 0 | { |
264 | 0 | ReportError(CE_Failure, CPLE_AppDefined, |
265 | 0 | "Cannot guess driver for %s", filename.c_str()); |
266 | 0 | return false; |
267 | 0 | } |
268 | 0 | m_format = aosFormats[0]; |
269 | 0 | } |
270 | 0 | } |
271 | | |
272 | 0 | auto poDrv = GetGDALDriverManager()->GetDriverByName(m_format.c_str()); |
273 | 0 | if (!poDrv) |
274 | 0 | { |
275 | 0 | ReportError(CE_Failure, CPLE_AppDefined, "Driver %s does not exist", |
276 | 0 | m_format.c_str()); |
277 | 0 | return false; |
278 | 0 | } |
279 | | |
280 | 0 | const bool autoDeleteFile = !m_reopenAndDoNotEarlyDelete && |
281 | 0 | filename.empty() && |
282 | 0 | !EQUAL(m_format.c_str(), "MEM"); |
283 | 0 | if (filename.empty() && !EQUAL(m_format.c_str(), "MEM")) |
284 | 0 | { |
285 | 0 | filename = CPLGenerateTempFilenameSafe(nullptr); |
286 | |
|
287 | 0 | const char *pszExt = poDrv->GetMetadataItem(GDAL_DMD_EXTENSIONS); |
288 | 0 | if (pszExt) |
289 | 0 | { |
290 | 0 | filename += '.'; |
291 | 0 | filename += CPLStringList(CSLTokenizeString(pszExt))[0]; |
292 | 0 | } |
293 | 0 | } |
294 | |
|
295 | 0 | CPLStringList aosOptions; |
296 | 0 | aosOptions.AddString("--invoked-from-gdal-algorithm"); |
297 | 0 | if (!m_overwrite) |
298 | 0 | { |
299 | 0 | aosOptions.AddString("--no-overwrite"); |
300 | 0 | } |
301 | |
|
302 | 0 | aosOptions.AddString("-of"); |
303 | 0 | aosOptions.AddString(m_format.c_str()); |
304 | 0 | for (const auto &co : m_creationOptions) |
305 | 0 | { |
306 | 0 | aosOptions.AddString("-dsco"); |
307 | 0 | aosOptions.AddString(co.c_str()); |
308 | 0 | } |
309 | 0 | CPLStringList aosReopenOpenOptions; |
310 | 0 | if (EQUAL(m_format.c_str(), "SQLite")) |
311 | 0 | { |
312 | 0 | const char *pszCOList = |
313 | 0 | poDrv->GetMetadataItem(GDAL_DMD_CREATIONOPTIONLIST); |
314 | 0 | if (pszCOList && strstr(pszCOList, "SPATIALITE") && |
315 | 0 | CPLStringList(m_creationOptions).FetchNameValue("SPATIALITE") == |
316 | 0 | nullptr) |
317 | 0 | { |
318 | 0 | aosOptions.AddString("-dsco"); |
319 | 0 | aosOptions.AddString("SPATIALITE=YES"); |
320 | 0 | } |
321 | 0 | aosReopenOpenOptions.AddString("LIST_ALL_TABLES=YES"); |
322 | 0 | } |
323 | 0 | for (const auto &co : m_layerCreationOptions) |
324 | 0 | { |
325 | 0 | aosOptions.AddString("-lco"); |
326 | 0 | aosOptions.AddString(co.c_str()); |
327 | 0 | } |
328 | 0 | if (pfnProgress && pfnProgress != GDALDummyProgress) |
329 | 0 | { |
330 | 0 | aosOptions.AddString("-progress"); |
331 | 0 | } |
332 | |
|
333 | 0 | if (autoDeleteFile) |
334 | 0 | { |
335 | 0 | aosOptions.AddString("-dsco"); |
336 | 0 | aosOptions.AddString("@SUPPRESS_ASAP=YES"); |
337 | 0 | } |
338 | |
|
339 | 0 | GDALVectorTranslateOptions *psOptions = |
340 | 0 | GDALVectorTranslateOptionsNew(aosOptions.List(), nullptr); |
341 | 0 | GDALVectorTranslateOptionsSetProgress(psOptions, pfnProgress, |
342 | 0 | pProgressData); |
343 | |
|
344 | 0 | GDALDatasetH hSrcDS = GDALDataset::ToHandle(poSrcDS); |
345 | 0 | auto poOutDS = std::unique_ptr<GDALDataset>( |
346 | 0 | GDALDataset::FromHandle(GDALVectorTranslate( |
347 | 0 | filename.c_str(), nullptr, 1, &hSrcDS, psOptions, nullptr))); |
348 | 0 | GDALVectorTranslateOptionsFree(psOptions); |
349 | |
|
350 | 0 | bool ok = poOutDS != nullptr && poOutDS->FlushCache() == CE_None; |
351 | 0 | if (poOutDS) |
352 | 0 | { |
353 | 0 | if (m_reopenAndDoNotEarlyDelete || |
354 | 0 | poDrv->GetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED)) |
355 | 0 | { |
356 | 0 | ok = poOutDS->Close() == CE_None; |
357 | 0 | poOutDS.reset(); |
358 | 0 | if (ok) |
359 | 0 | { |
360 | 0 | const char *const apszAllowedDrivers[] = {m_format.c_str(), |
361 | 0 | nullptr}; |
362 | 0 | poOutDS.reset(GDALDataset::Open( |
363 | 0 | filename.c_str(), GDAL_OF_VECTOR | GDAL_OF_VERBOSE_ERROR, |
364 | 0 | apszAllowedDrivers, aosReopenOpenOptions.List())); |
365 | 0 | ok = poOutDS != nullptr; |
366 | 0 | } |
367 | 0 | } |
368 | 0 | if (ok) |
369 | 0 | { |
370 | 0 | if (autoDeleteFile) |
371 | 0 | { |
372 | 0 | #if !defined(_WIN32) |
373 | 0 | if (poDrv->GetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE)) |
374 | 0 | { |
375 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
376 | 0 | poDrv->Delete(poOutDS.get(), |
377 | 0 | CPLStringList(poOutDS->GetFileList()).List()); |
378 | 0 | } |
379 | 0 | #endif |
380 | 0 | poOutDS->MarkSuppressOnClose(); |
381 | 0 | } |
382 | |
|
383 | 0 | m_outputDataset.Set(std::move(poOutDS)); |
384 | 0 | } |
385 | 0 | } |
386 | 0 | return ok; |
387 | 0 | } |
388 | | |
389 | | //! @endcond |