/src/gdal/ogr/ogr_schema_override.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * Project: OGR_SCHEMA open options handling |
3 | | * Purpose: Class for representing a layer schema override. |
4 | | * Author: Alessandro Pasotti, elpaso@itopen.it |
5 | | * |
6 | | ****************************************************************************** |
7 | | * Copyright (c) 2024, Alessandro Pasotti <elpaso at itopen dot it> |
8 | | * |
9 | | * SPDX-License-Identifier: MIT |
10 | | ****************************************************************************/ |
11 | | |
12 | | //! @cond Doxygen_Suppress |
13 | | |
14 | | #include "ogr_schema_override.h" |
15 | | #include "ogrsf_frmts.h" |
16 | | |
17 | | bool OGRSchemaOverride::LoadFromJSON(const std::string &osJSON) |
18 | 0 | { |
19 | 0 | std::string osFieldsSchemaOverride; |
20 | 0 | bool bFieldsSchemaOverrideIsFilePath{false}; |
21 | | |
22 | | // Try to load the content of the file |
23 | 0 | GByte *pabyRet = nullptr; |
24 | 0 | if (VSIIngestFile(nullptr, osJSON.c_str(), &pabyRet, nullptr, -1) == TRUE) |
25 | 0 | { |
26 | 0 | bFieldsSchemaOverrideIsFilePath = true; |
27 | 0 | osFieldsSchemaOverride = std::string(reinterpret_cast<char *>(pabyRet)); |
28 | 0 | VSIFree(pabyRet); |
29 | 0 | } |
30 | |
|
31 | 0 | if (!bFieldsSchemaOverrideIsFilePath) |
32 | 0 | { |
33 | 0 | osFieldsSchemaOverride = osJSON; |
34 | 0 | } |
35 | |
|
36 | 0 | CPLJSONDocument oSchemaDoc; |
37 | 0 | if (oSchemaDoc.LoadMemory(osFieldsSchemaOverride)) |
38 | 0 | { |
39 | 0 | const CPLJSONObject oRoot = oSchemaDoc.GetRoot(); |
40 | 0 | if (oRoot.IsValid()) |
41 | 0 | { |
42 | 0 | const auto aoLayers = oRoot.GetArray("layers"); |
43 | | // Loop through layer names and get the field details for each field. |
44 | 0 | for (const auto &oLayer : aoLayers) |
45 | 0 | { |
46 | 0 | if (oLayer.IsValid()) |
47 | 0 | { |
48 | 0 | const auto oLayerFields = oLayer.GetArray("fields"); |
49 | | // Parse fields |
50 | 0 | const auto osLayerName = oLayer.GetString("name"); |
51 | 0 | const auto osSchemaType = oLayer.GetString("schemaType"); |
52 | | // Default schemaType is "Patch" |
53 | 0 | const auto bSchemaFullOverride = |
54 | 0 | CPLString(osSchemaType).tolower() == "full"; |
55 | 0 | OGRLayerSchemaOverride oLayerOverride; |
56 | 0 | oLayerOverride.SetLayerName(osLayerName); |
57 | 0 | oLayerOverride.SetFullOverride(bSchemaFullOverride); |
58 | |
|
59 | 0 | if (oLayerFields.Size() > 0 && !osLayerName.empty()) |
60 | 0 | { |
61 | 0 | for (const auto &oField : oLayerFields) |
62 | 0 | { |
63 | 0 | const auto osFieldName = oField.GetString("name"); |
64 | 0 | OGRFieldDefnOverride oFieldOverride; |
65 | |
|
66 | 0 | const CPLString oSrcType( |
67 | 0 | CPLString(oField.GetString("srcType")) |
68 | 0 | .tolower()); |
69 | 0 | const CPLString oSrcSubType( |
70 | 0 | CPLString(oField.GetString("srcSubType")) |
71 | 0 | .tolower()); |
72 | 0 | const CPLString oType( |
73 | 0 | CPLString(oField.GetString("type")).tolower()); |
74 | 0 | const CPLString oSubType( |
75 | 0 | CPLString(oField.GetString("subType")) |
76 | 0 | .tolower()); |
77 | 0 | const CPLString osNewName( |
78 | 0 | CPLString(oField.GetString("newName")) |
79 | 0 | .tolower()); |
80 | 0 | const auto nWidth = oField.GetInteger("width", 0); |
81 | 0 | const auto nPrecision = |
82 | 0 | oField.GetInteger("precision", 0); |
83 | |
|
84 | 0 | if (!osNewName.empty()) |
85 | 0 | { |
86 | 0 | oFieldOverride.SetFieldName(osNewName); |
87 | 0 | } |
88 | |
|
89 | 0 | if (!oSrcType.empty()) |
90 | 0 | { |
91 | 0 | if (bSchemaFullOverride) |
92 | 0 | { |
93 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
94 | 0 | "Non-patch OGR_SCHEMA definition " |
95 | 0 | "is not allowed with specifying " |
96 | 0 | "source field type"); |
97 | 0 | return false; |
98 | 0 | } |
99 | 0 | if (!osFieldName.empty() || !osNewName.empty()) |
100 | 0 | { |
101 | 0 | CPLError(CE_Warning, CPLE_AppDefined, |
102 | 0 | "Field name and source field type " |
103 | 0 | "are mutually exclusive"); |
104 | 0 | return false; |
105 | 0 | } |
106 | 0 | const OGRFieldType eType = |
107 | 0 | OGRFieldDefn::GetFieldTypeByName( |
108 | 0 | oSrcType.c_str()); |
109 | | // Check if the field type is valid |
110 | 0 | if (eType == OFTString && oSrcType != "string") |
111 | 0 | { |
112 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
113 | 0 | "Unsupported source field type: " |
114 | 0 | "%s", |
115 | 0 | oSrcType.c_str()); |
116 | 0 | return false; |
117 | 0 | } |
118 | 0 | oFieldOverride.SetSrcFieldType(eType); |
119 | 0 | } |
120 | | |
121 | 0 | if (!oSrcSubType.empty()) |
122 | 0 | { |
123 | 0 | if (bSchemaFullOverride) |
124 | 0 | { |
125 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
126 | 0 | "Non-patch OGR_SCHEMA definition " |
127 | 0 | "is not allowed with specifying " |
128 | 0 | "source field subtype"); |
129 | 0 | return false; |
130 | 0 | } |
131 | 0 | if (!osFieldName.empty() || !osNewName.empty()) |
132 | 0 | { |
133 | 0 | CPLError(CE_Warning, CPLE_AppDefined, |
134 | 0 | "Field name and source field " |
135 | 0 | "subtype are mutually exclusive"); |
136 | 0 | return false; |
137 | 0 | } |
138 | 0 | const OGRFieldSubType eSubType = |
139 | 0 | OGRFieldDefn::GetFieldSubTypeByName( |
140 | 0 | oSubType.c_str()); |
141 | | // Check if the field subType is valid |
142 | 0 | if (eSubType == OFSTNone && |
143 | 0 | oSrcSubType != "none") |
144 | 0 | { |
145 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
146 | 0 | "Unsupported source field subType:" |
147 | 0 | " %s", |
148 | 0 | oSubType.c_str()); |
149 | 0 | return false; |
150 | 0 | } |
151 | 0 | oFieldOverride.SetSrcFieldSubType(eSubType); |
152 | 0 | } |
153 | | |
154 | 0 | if (oSrcType.empty() && oSrcSubType.empty() && |
155 | 0 | osFieldName.empty()) |
156 | 0 | { |
157 | 0 | CPLError(CE_Warning, CPLE_AppDefined, |
158 | 0 | "Field name is missing"); |
159 | 0 | return false; |
160 | 0 | } |
161 | | |
162 | 0 | if (!oType.empty()) |
163 | 0 | { |
164 | 0 | const OGRFieldType eType = |
165 | 0 | OGRFieldDefn::GetFieldTypeByName( |
166 | 0 | oType.c_str()); |
167 | | // Check if the field type is valid |
168 | 0 | if (eType == OFTString && oType != "string") |
169 | 0 | { |
170 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
171 | 0 | "Unsupported field type: %s " |
172 | 0 | "for field %s", |
173 | 0 | oType.c_str(), |
174 | 0 | osFieldName.c_str()); |
175 | 0 | return false; |
176 | 0 | } |
177 | 0 | oFieldOverride.SetFieldType(eType); |
178 | 0 | } |
179 | | |
180 | 0 | if (!oSubType.empty()) |
181 | 0 | { |
182 | 0 | const OGRFieldSubType eSubType = |
183 | 0 | OGRFieldDefn::GetFieldSubTypeByName( |
184 | 0 | oSubType.c_str()); |
185 | | // Check if the field subType is valid |
186 | 0 | if (eSubType == OFSTNone && oSubType != "none") |
187 | 0 | { |
188 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
189 | 0 | "Unsupported field subType: " |
190 | 0 | "%s for field %s", |
191 | 0 | oSubType.c_str(), |
192 | 0 | osFieldName.c_str()); |
193 | 0 | return false; |
194 | 0 | } |
195 | 0 | oFieldOverride.SetFieldSubType(eSubType); |
196 | 0 | } |
197 | | |
198 | 0 | if (nWidth != 0) |
199 | 0 | { |
200 | 0 | oFieldOverride.SetFieldWidth(nWidth); |
201 | 0 | } |
202 | |
|
203 | 0 | if (nPrecision != 0) |
204 | 0 | { |
205 | 0 | oFieldOverride.SetFieldPrecision(nPrecision); |
206 | 0 | } |
207 | |
|
208 | 0 | if (bSchemaFullOverride || oFieldOverride.IsValid()) |
209 | 0 | { |
210 | 0 | if (osFieldName.empty()) |
211 | 0 | { |
212 | 0 | oLayerOverride.AddUnnamedFieldOverride( |
213 | 0 | oFieldOverride); |
214 | 0 | } |
215 | 0 | else |
216 | 0 | { |
217 | 0 | oLayerOverride.AddNamedFieldOverride( |
218 | 0 | osFieldName, oFieldOverride); |
219 | 0 | } |
220 | 0 | } |
221 | 0 | else |
222 | 0 | { |
223 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
224 | 0 | "Field %s has no valid overrides " |
225 | 0 | "and schemaType is not \"Full\"", |
226 | 0 | osFieldName.c_str()); |
227 | 0 | return false; |
228 | 0 | } |
229 | 0 | } |
230 | 0 | } |
231 | | |
232 | 0 | if (oLayerOverride.IsValid()) |
233 | 0 | { |
234 | 0 | AddLayerOverride(oLayerOverride); |
235 | 0 | } |
236 | 0 | else |
237 | 0 | { |
238 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
239 | 0 | "Layer %s has no valid overrides", |
240 | 0 | osLayerName.c_str()); |
241 | 0 | return false; |
242 | 0 | } |
243 | 0 | } |
244 | 0 | else |
245 | 0 | { |
246 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
247 | 0 | "SCHEMA info is invalid JSON"); |
248 | 0 | return false; |
249 | 0 | } |
250 | 0 | } |
251 | 0 | return true; |
252 | 0 | } |
253 | 0 | else |
254 | 0 | { |
255 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
256 | 0 | "SCHEMA info is invalid JSON"); |
257 | 0 | return false; |
258 | 0 | } |
259 | 0 | } |
260 | 0 | else |
261 | 0 | { |
262 | 0 | CPLError(CE_Failure, CPLE_AppDefined, "SCHEMA info is invalid JSON"); |
263 | 0 | return false; |
264 | 0 | } |
265 | 0 | } |
266 | | |
267 | | bool OGRSchemaOverride::IsValid() const |
268 | 0 | { |
269 | 0 | bool isValid = !m_aoLayerOverrides.empty(); |
270 | 0 | for (const auto &oLayerOverride : m_aoLayerOverrides) |
271 | 0 | { |
272 | 0 | isValid &= oLayerOverride.IsValid(); |
273 | 0 | } |
274 | 0 | return isValid; |
275 | 0 | } |
276 | | |
277 | | bool OGRSchemaOverride::DefaultApply( |
278 | | GDALDataset *poDS, const char *pszDebugKey, |
279 | | std::function<void(OGRLayer *, int)> callbackWhenRemovingField) const |
280 | 0 | { |
281 | 0 | const auto &oLayerOverrides = GetLayerOverrides(); |
282 | 0 | for (const auto &oLayerFieldOverride : oLayerOverrides) |
283 | 0 | { |
284 | 0 | const auto &osLayerName = oLayerFieldOverride.GetLayerName(); |
285 | 0 | const bool bIsFullOverride{oLayerFieldOverride.IsFullOverride()}; |
286 | 0 | auto oNamedFieldOverrides = |
287 | 0 | oLayerFieldOverride.GetNamedFieldOverrides(); |
288 | 0 | const auto &oUnnamedFieldOverrides = |
289 | 0 | oLayerFieldOverride.GetUnnamedFieldOverrides(); |
290 | |
|
291 | 0 | const auto ProcessLayer = |
292 | 0 | [&callbackWhenRemovingField, &osLayerName, &oNamedFieldOverrides, |
293 | 0 | &oUnnamedFieldOverrides, bIsFullOverride](OGRLayer *poLayer) |
294 | 0 | { |
295 | 0 | std::vector<OGRFieldDefn *> aoFields; |
296 | | // Patch field definitions |
297 | 0 | auto poLayerDefn = poLayer->GetLayerDefn(); |
298 | 0 | for (int i = 0; i < poLayerDefn->GetFieldCount(); i++) |
299 | 0 | { |
300 | 0 | auto poFieldDefn = poLayerDefn->GetFieldDefn(i); |
301 | |
|
302 | 0 | const auto PatchFieldDefn = |
303 | 0 | [poFieldDefn](const OGRFieldDefnOverride &oFieldOverride) |
304 | 0 | { |
305 | 0 | if (oFieldOverride.GetFieldType().has_value()) |
306 | 0 | whileUnsealing(poFieldDefn) |
307 | 0 | ->SetType(oFieldOverride.GetFieldType().value()); |
308 | 0 | if (oFieldOverride.GetFieldWidth().has_value()) |
309 | 0 | whileUnsealing(poFieldDefn) |
310 | 0 | ->SetWidth(oFieldOverride.GetFieldWidth().value()); |
311 | 0 | if (oFieldOverride.GetFieldPrecision().has_value()) |
312 | 0 | whileUnsealing(poFieldDefn) |
313 | 0 | ->SetPrecision( |
314 | 0 | oFieldOverride.GetFieldPrecision().value()); |
315 | 0 | if (oFieldOverride.GetFieldSubType().has_value()) |
316 | 0 | whileUnsealing(poFieldDefn) |
317 | 0 | ->SetSubType( |
318 | 0 | oFieldOverride.GetFieldSubType().value()); |
319 | 0 | if (oFieldOverride.GetFieldName().has_value()) |
320 | 0 | whileUnsealing(poFieldDefn) |
321 | 0 | ->SetName( |
322 | 0 | oFieldOverride.GetFieldName().value().c_str()); |
323 | 0 | }; |
324 | |
|
325 | 0 | auto oFieldOverrideIter = |
326 | 0 | oNamedFieldOverrides.find(poFieldDefn->GetNameRef()); |
327 | 0 | if (oFieldOverrideIter != oNamedFieldOverrides.cend()) |
328 | 0 | { |
329 | 0 | const auto &oFieldOverride = oFieldOverrideIter->second; |
330 | 0 | PatchFieldDefn(oFieldOverride); |
331 | |
|
332 | 0 | if (bIsFullOverride) |
333 | 0 | { |
334 | 0 | aoFields.push_back(poFieldDefn); |
335 | 0 | } |
336 | 0 | oNamedFieldOverrides.erase(oFieldOverrideIter); |
337 | 0 | } |
338 | 0 | else |
339 | 0 | { |
340 | 0 | for (const auto &oFieldOverride : oUnnamedFieldOverrides) |
341 | 0 | { |
342 | 0 | if ((!oFieldOverride.GetSrcFieldType().has_value() || |
343 | 0 | oFieldOverride.GetSrcFieldType().value() == |
344 | 0 | poFieldDefn->GetType()) && |
345 | 0 | (!oFieldOverride.GetSrcFieldSubType().has_value() || |
346 | 0 | oFieldOverride.GetSrcFieldSubType().value() == |
347 | 0 | poFieldDefn->GetSubType())) |
348 | 0 | { |
349 | 0 | PatchFieldDefn(oFieldOverride); |
350 | 0 | break; |
351 | 0 | } |
352 | 0 | } |
353 | 0 | } |
354 | 0 | } |
355 | | |
356 | | // Error if any field override is not found |
357 | 0 | if (!oNamedFieldOverrides.empty()) |
358 | 0 | { |
359 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
360 | 0 | "Field %s not found in layer %s", |
361 | 0 | oNamedFieldOverrides.cbegin()->first.c_str(), |
362 | 0 | osLayerName.c_str()); |
363 | 0 | return false; |
364 | 0 | } |
365 | | |
366 | | // Remove fields not in the override |
367 | 0 | if (bIsFullOverride) |
368 | 0 | { |
369 | 0 | for (int i = poLayerDefn->GetFieldCount() - 1; i >= 0; i--) |
370 | 0 | { |
371 | 0 | auto poFieldDefn = poLayerDefn->GetFieldDefn(i); |
372 | 0 | if (std::find(aoFields.begin(), aoFields.end(), |
373 | 0 | poFieldDefn) == aoFields.end()) |
374 | 0 | { |
375 | 0 | callbackWhenRemovingField(poLayer, i); |
376 | 0 | whileUnsealing(poLayerDefn)->DeleteFieldDefn(i); |
377 | 0 | } |
378 | 0 | } |
379 | 0 | } |
380 | |
|
381 | 0 | return true; |
382 | 0 | }; |
383 | |
|
384 | 0 | CPLDebug(pszDebugKey, "Applying schema override for layer %s", |
385 | 0 | osLayerName.c_str()); |
386 | |
|
387 | 0 | if (osLayerName == "*") |
388 | 0 | { |
389 | 0 | for (auto *poLayer : poDS->GetLayers()) |
390 | 0 | { |
391 | 0 | if (!ProcessLayer(poLayer)) |
392 | 0 | return false; |
393 | 0 | } |
394 | 0 | } |
395 | 0 | else |
396 | 0 | { |
397 | | // Fail if the layer name does not exist |
398 | 0 | auto poLayer = poDS->GetLayerByName(osLayerName.c_str()); |
399 | 0 | if (poLayer == nullptr) |
400 | 0 | { |
401 | 0 | CPLError(CE_Failure, CPLE_AppDefined, "Layer %s not found", |
402 | 0 | osLayerName.c_str()); |
403 | 0 | return false; |
404 | 0 | } |
405 | 0 | if (!ProcessLayer(poLayer)) |
406 | 0 | return false; |
407 | 0 | } |
408 | 0 | } |
409 | | |
410 | 0 | return true; |
411 | 0 | } |
412 | | |
413 | | bool OGRLayerSchemaOverride::IsValid() const |
414 | 0 | { |
415 | 0 | bool isValid = |
416 | 0 | !m_osLayerName.empty() && |
417 | 0 | (!m_oNamedFieldOverrides.empty() || !m_aoUnnamedFieldOverrides.empty()); |
418 | 0 | for (const auto &oFieldOverrideIter : m_oNamedFieldOverrides) |
419 | 0 | { |
420 | 0 | isValid &= !oFieldOverrideIter.first.empty(); |
421 | | // When schemaType is "full" override we don't need to check if the field |
422 | | // overrides are valid: a list of fields to keep is enough. |
423 | 0 | if (!m_bIsFullOverride) |
424 | 0 | { |
425 | 0 | isValid &= oFieldOverrideIter.second.IsValid(); |
426 | 0 | } |
427 | 0 | } |
428 | 0 | return isValid; |
429 | 0 | } |
430 | | |
431 | | bool OGRFieldDefnOverride::IsValid() const |
432 | 0 | { |
433 | 0 | return m_osName.has_value() || m_eType.has_value() || |
434 | 0 | m_eSubType.has_value() || m_eSrcType.has_value() || |
435 | 0 | m_eSrcSubType.has_value() || m_nWidth.has_value() || |
436 | 0 | m_nPrecision.has_value(); |
437 | 0 | } |
438 | | |
439 | | //! @endcond |