/src/gdal/ogr/ogrsf_frmts/pmtiles/ogr_pmtiles.h
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: OpenGIS Simple Features Reference Implementation |
4 | | * Purpose: Implementation of PMTiles |
5 | | * Author: Even Rouault <even.rouault at spatialys.com> |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2023, Planet Labs |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #ifndef OGR_PMTILES_H_INCLUDED |
14 | | #define OGR_PMTILES_H_INCLUDED |
15 | | |
16 | | #include "gdal_priv.h" |
17 | | #include "ogrsf_frmts.h" |
18 | | |
19 | | #include "cpl_compressor.h" |
20 | | #include "cpl_vsi_virtual.h" |
21 | | |
22 | | #include "include_pmtiles.h" |
23 | | |
24 | | #include <array> |
25 | | #include <limits> |
26 | | #include <set> |
27 | | #include <stack> |
28 | | |
29 | | // #define DEBUG_PMTILES |
30 | | |
31 | 0 | #define SPHERICAL_RADIUS 6378137.0 |
32 | 0 | #define MAX_GM (SPHERICAL_RADIUS * M_PI) // 20037508.342789244 |
33 | | |
34 | | #if defined(HAVE_SQLITE) && defined(HAVE_GEOS) |
35 | | // Needed by mvtutils.h |
36 | | #define HAVE_MVT_WRITE_SUPPORT |
37 | | #endif |
38 | | |
39 | | constexpr int PMTILES_HEADER_LENGTH = 127; |
40 | | |
41 | | /************************************************************************/ |
42 | | /* OGRPMTilesDataset */ |
43 | | /************************************************************************/ |
44 | | |
45 | | class OGRPMTilesDataset final : public GDALDataset |
46 | | { |
47 | | public: |
48 | 0 | OGRPMTilesDataset() = default; |
49 | | |
50 | | ~OGRPMTilesDataset() override; |
51 | | |
52 | | bool Open(GDALOpenInfo *poOpenInfo); |
53 | | |
54 | | int GetLayerCount() const override |
55 | 0 | { |
56 | 0 | return static_cast<int>(m_apoLayers.size()); |
57 | 0 | } |
58 | | |
59 | | const OGRLayer *GetLayer(int) const override; |
60 | | |
61 | | const OGRSpatialReference *GetSpatialRef() const override |
62 | 0 | { |
63 | 0 | return !m_oSRS.IsEmpty() ? &m_oSRS : nullptr; |
64 | 0 | } |
65 | | |
66 | | CPLErr GetGeoTransform(GDALGeoTransform >) const override |
67 | 0 | { |
68 | 0 | gt = m_gt; |
69 | 0 | return !m_oSRS.IsEmpty() ? CE_None : CE_Failure; |
70 | 0 | } |
71 | | |
72 | | inline int GetMinZoomLevel() const |
73 | 0 | { |
74 | 0 | return m_nMinZoomLevel; |
75 | 0 | } |
76 | | |
77 | | inline int GetMaxZoomLevel() const |
78 | 0 | { |
79 | 0 | return m_nMaxZoomLevel; |
80 | 0 | } |
81 | | |
82 | | inline const pmtiles::headerv3 &GetHeader() const |
83 | 0 | { |
84 | 0 | return m_sHeader; |
85 | 0 | } |
86 | | |
87 | | static const char *GetCompression(uint8_t nVal); |
88 | | |
89 | | static const char *GetTileType(const pmtiles::headerv3 &sHeader); |
90 | | |
91 | | inline const std::string &GetMetadataContent() const |
92 | 0 | { |
93 | 0 | return m_osMetadata; |
94 | 0 | } |
95 | | |
96 | | inline const std::string &GetMetadataFilename() const |
97 | 0 | { |
98 | 0 | return m_osMetadataFilename; |
99 | 0 | } |
100 | | |
101 | | inline const std::string &GetClipOpenOption() const |
102 | 0 | { |
103 | 0 | return m_osClipOpenOption; |
104 | 0 | } |
105 | | |
106 | | /** Return a short-lived decompressed buffer for metadata or directory |
107 | | * entries or nullptr in case of error. |
108 | | */ |
109 | | const std::string *ReadInternal(uint64_t nOffset, uint64_t nSize, |
110 | | const char *pszDataType); |
111 | | |
112 | | /** Return a short-lived decompressed buffer for tile data. |
113 | | * or nullptr in case of error. |
114 | | */ |
115 | | const std::string *ReadTileData(uint64_t nOffset, uint64_t nSize); |
116 | | |
117 | | protected: |
118 | | friend class GDALPMTilesRasterBand; |
119 | | |
120 | | CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, |
121 | | int nYSize, void *pData, int nBufXSize, int nBufYSize, |
122 | | GDALDataType eBufType, int nBandCount, |
123 | | BANDMAP_TYPE panBandMap, GSpacing nPixelSpace, |
124 | | GSpacing nLineSpace, GSpacing nBandSpace, |
125 | | GDALRasterIOExtraArg *psExtraArg) override; |
126 | | |
127 | | private: |
128 | | VSIVirtualHandleUniquePtr m_poFileUniquePtr{}; |
129 | | |
130 | | VSIVirtualHandle *m_poFile = nullptr; |
131 | | |
132 | | //! PMTiles header |
133 | | pmtiles::headerv3 m_sHeader{}; |
134 | | |
135 | | //! JSON serialized metadata |
136 | | std::string m_osMetadata{}; |
137 | | |
138 | | //! /vsimem/ filename with the m_osMetadata content |
139 | | std::string m_osMetadataFilename{}; |
140 | | |
141 | | //! Value of the CLIP open option |
142 | | std::string m_osClipOpenOption{}; |
143 | | |
144 | | //! Decompressor for metadata and directories |
145 | | const CPLCompressor *m_psInternalDecompressor = nullptr; |
146 | | |
147 | | //! Decompressor for tile |
148 | | const CPLCompressor *m_psTileDataDecompressor = nullptr; |
149 | | |
150 | | //! Last raw data read by Read() |
151 | | std::string m_osBuffer{}; |
152 | | |
153 | | //! Last uncompressed data read by Read(). Only used if compression |
154 | | std::string m_osDecompressedBuffer{}; |
155 | | |
156 | | std::vector<std::unique_ptr<OGRLayer>> m_apoLayers{}; |
157 | | |
158 | | //! Minimum zoom level got from header |
159 | | int m_nMinZoomLevel = 0; |
160 | | |
161 | | //! Maximum zoom level got from header |
162 | | int m_nMaxZoomLevel = 0; |
163 | | |
164 | | //! Zoom level got (for raster) |
165 | | int m_nZoomLevel = 0; |
166 | | |
167 | | //! CRS (for raster) |
168 | | OGRSpatialReference m_oSRS{}; |
169 | | |
170 | | //! Geotransform |
171 | | GDALGeoTransform m_gt{}; |
172 | | |
173 | | //! GeoPackage driver (for raster) |
174 | | GDALDriver *m_poGPKGDriver = nullptr; |
175 | | |
176 | | //! Temporary GeoPackage filename (for raster) |
177 | | std::string m_osTmpGPKGFilename{}; |
178 | | |
179 | | //! Overview datasets |
180 | | std::vector<std::unique_ptr<OGRPMTilesDataset>> m_apoOverviews{}; |
181 | | |
182 | | bool OpenVector(const GDALOpenInfo *poOpenInfo, |
183 | | const CPLJSONObject &oJsonRoot, int nZoomLevel); |
184 | | |
185 | | bool OpenRaster(int nZoomLevel); |
186 | | |
187 | | /** Return a short-lived decompressed buffer, or nullptr in case of error |
188 | | */ |
189 | | const std::string *Read(const CPLCompressor *psDecompressor, |
190 | | uint64_t nOffset, uint64_t nSize, |
191 | | const char *pszDataType); |
192 | | |
193 | | CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesDataset) |
194 | | }; |
195 | | |
196 | | /************************************************************************/ |
197 | | /* GDALPMTilesRasterBand */ |
198 | | /************************************************************************/ |
199 | | |
200 | | class GDALPMTilesRasterBand final : public GDALRasterBand |
201 | | { |
202 | | public: |
203 | | GDALPMTilesRasterBand(OGRPMTilesDataset *poDSIn, int nBandIn, |
204 | | int nBlockSize); |
205 | | |
206 | | GDALColorInterp GetColorInterpretation() override; |
207 | | |
208 | | int GetOverviewCount() override; |
209 | | |
210 | | GDALRasterBand *GetOverview(int) override; |
211 | | |
212 | | protected: |
213 | | CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) override; |
214 | | |
215 | | CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, |
216 | | int nYSize, void *pData, int nBufXSize, int nBufYSize, |
217 | | GDALDataType eBufType, GSpacing nPixelSpace, |
218 | | GSpacing nLineSpace, |
219 | | GDALRasterIOExtraArg *psExtraArg) override; |
220 | | }; |
221 | | |
222 | | /************************************************************************/ |
223 | | /* OGRPMTilesTileIterator */ |
224 | | /************************************************************************/ |
225 | | |
226 | | //! Iterator to browse through tiles |
227 | | class OGRPMTilesTileIterator |
228 | | { |
229 | | public: |
230 | | //! Constructor to iterate over all tiles (possibly limited to a zoom level |
231 | | explicit OGRPMTilesTileIterator(OGRPMTilesDataset *poDS, |
232 | | int nZoomLevel = -1) |
233 | 0 | : m_poDS(poDS), m_nZoomLevel(nZoomLevel) |
234 | 0 | { |
235 | 0 | } |
236 | | |
237 | | //! Constructor with a window of interest in tile coordinates |
238 | | OGRPMTilesTileIterator(OGRPMTilesDataset *poDS, int nZoomLevel, int nMinX, |
239 | | int nMinY, int nMaxX, int nMaxY) |
240 | 0 | : m_poDS(poDS), m_nZoomLevel(nZoomLevel), m_nMinX(nMinX), |
241 | 0 | m_nMinY(nMinY), m_nMaxX(nMaxX), m_nMaxY(nMaxY) |
242 | 0 | { |
243 | 0 | } |
244 | | |
245 | | /** Return the (z, x, y, offset, length) of the next tile. |
246 | | * |
247 | | * If entry_zxy.offset == 0, the iteration has stopped. |
248 | | */ |
249 | | pmtiles::entry_zxy GetNextTile(uint32_t *pnRunLength = nullptr); |
250 | | |
251 | | void SkipRunLength(); |
252 | | |
253 | | #ifdef DEBUG_PMTILES |
254 | | void DumpTiles(); |
255 | | #endif |
256 | | |
257 | | private: |
258 | | // Input parameters |
259 | | OGRPMTilesDataset *m_poDS = nullptr; |
260 | | int m_nZoomLevel = -1; |
261 | | int m_nMinX = -1; |
262 | | int m_nMinY = -1; |
263 | | int m_nMaxX = -1; |
264 | | int m_nMaxY = -1; |
265 | | |
266 | | // Used when iterating over tile id is inefficient |
267 | | int m_nCurX = -1; |
268 | | int m_nCurY = -1; |
269 | | |
270 | | // for sanity checks. Must be increasing when walking through entries |
271 | | static constexpr uint64_t INVALID_LAST_TILE_ID = |
272 | | std::numeric_limits<uint64_t>::max(); |
273 | | uint64_t m_nLastTileId = INVALID_LAST_TILE_ID; |
274 | | |
275 | | // Computed values from zoom leven and min/max x/y |
276 | | uint64_t m_nMinTileId = std::numeric_limits<uint64_t>::max(); |
277 | | uint64_t m_nMaxTileId = 0; |
278 | | |
279 | | bool m_bEOF = false; |
280 | | |
281 | | // State of exploration of a directory |
282 | | struct DirectoryContext |
283 | | { |
284 | | // Entries, either tiles (sEntry.run_length > 0) or subdiretories |
285 | | // (sEntry.run_length == 0) |
286 | | std::vector<pmtiles::entryv3> sEntries{}; |
287 | | |
288 | | // Next index of sEntries[] to explore |
289 | | uint32_t nIdxInEntries = 0; |
290 | | |
291 | | // For tiles, value between 0 and |
292 | | // sEntries[nIdxInEntries].run_length - 1 |
293 | | uint32_t nIdxInRunLength = 0; |
294 | | }; |
295 | | |
296 | | // Stack of directories: bottom is root directory, and then we |
297 | | // push subdiretories we browse throw |
298 | | std::stack<DirectoryContext> m_aoStack{}; |
299 | | |
300 | | bool LoadRootDirectory(); |
301 | | |
302 | | CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesTileIterator) |
303 | | }; |
304 | | |
305 | | /************************************************************************/ |
306 | | /* OGRPMTilesVectorLayer */ |
307 | | /************************************************************************/ |
308 | | |
309 | | class OGRPMTilesVectorLayer final |
310 | | : public OGRLayer, |
311 | | public OGRGetNextFeatureThroughRaw<OGRPMTilesVectorLayer> |
312 | | { |
313 | | public: |
314 | | OGRPMTilesVectorLayer(OGRPMTilesDataset *poDS, const char *pszLayerName, |
315 | | const CPLJSONObject &oFields, |
316 | | const CPLJSONArray &oAttributesFromTileStats, |
317 | | bool bJsonField, double dfMinX, double dfMinY, |
318 | | double dfMaxX, double dfMaxY, |
319 | | OGRwkbGeometryType eGeomType, int nZoomLevel, |
320 | | bool bZoomLevelFromSpatialFilter); |
321 | | ~OGRPMTilesVectorLayer() override; |
322 | | |
323 | | void ResetReading() override; |
324 | | |
325 | | OGRFeature *GetNextRawFeature(); |
326 | | DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(OGRPMTilesVectorLayer) |
327 | | |
328 | | const OGRFeatureDefn *GetLayerDefn() const override |
329 | 0 | { |
330 | 0 | return m_poFeatureDefn; |
331 | 0 | } |
332 | | |
333 | | int TestCapability(const char *) const override; |
334 | | |
335 | | OGRErr IGetExtent(int iGeomField, OGREnvelope *psExtent, |
336 | | bool bForce) override; |
337 | | |
338 | | OGRErr ISetSpatialFilter(int iGeomField, |
339 | | const OGRGeometry *poGeom) override; |
340 | | |
341 | | GIntBig GetFeatureCount(int bForce) override; |
342 | | |
343 | | OGRFeature *GetFeature(GIntBig nFID) override; |
344 | | |
345 | | static OGRwkbGeometryType GuessGeometryType(OGRPMTilesDataset *poDS, |
346 | | const char *pszLayerName, |
347 | | int nZoomLevel); |
348 | | |
349 | | private: |
350 | | OGRPMTilesDataset *m_poDS = nullptr; |
351 | | OGRFeatureDefn *m_poFeatureDefn = nullptr; |
352 | | |
353 | | //! Iterator over tiles |
354 | | std::unique_ptr<OGRPMTilesTileIterator> m_poTileIterator{}; |
355 | | |
356 | | //! Total feature count (may over-estimate due to not applying clipping) |
357 | | GIntBig m_nFeatureCount = -1; |
358 | | |
359 | | //! X tile value of currently opened tile |
360 | | uint32_t m_nX = 0; |
361 | | |
362 | | //! Y tile value of currently opened tile |
363 | | uint32_t m_nY = 0; |
364 | | |
365 | | //! Offset of the currently opened tile |
366 | | uint64_t m_nLastTileOffset = 0; |
367 | | |
368 | | //! Uncompressed MVT tile |
369 | | std::string m_osTileData{}; |
370 | | |
371 | | //! In-memory MVT dataset of the currently opened tile |
372 | | std::unique_ptr<GDALDataset> m_poTileDS{}; |
373 | | |
374 | | //! Layer of m_poTileDS |
375 | | OGRLayer *m_poTileLayer = nullptr; |
376 | | |
377 | | //! Layer extent |
378 | | OGREnvelope m_sExtent{}; |
379 | | |
380 | | //! Minimum X tile value corresponding to m_sFilterEnvelope |
381 | | int m_nFilterMinX = 0; |
382 | | |
383 | | //! Minimum Y tile value corresponding to m_sFilterEnvelope |
384 | | int m_nFilterMinY = 0; |
385 | | |
386 | | //! Maximum X tile value corresponding to m_sFilterEnvelope |
387 | | int m_nFilterMaxX = 0; |
388 | | |
389 | | //! Maximum Y tile value corresponding to m_sFilterEnvelope |
390 | | int m_nFilterMaxY = 0; |
391 | | |
392 | | //! Currently used zoom level |
393 | | int m_nZoomLevel = 0; |
394 | | |
395 | | //! Whether we should auto-adapt m_nZoomLevel from the spatial filter extent |
396 | | bool m_bZoomLevelAuto = false; |
397 | | |
398 | | //! Whether we should expose the tile fields in a "json" field |
399 | | bool m_bJsonField = false; |
400 | | |
401 | | std::unique_ptr<OGRFeature> GetNextSrcFeature(); |
402 | | std::unique_ptr<OGRFeature> CreateFeatureFrom(OGRFeature *poSrcFeature); |
403 | | GIntBig GetTotalFeatureCount() const; |
404 | | void ExtentToTileExtent(const OGREnvelope &sEnvelope, int &nTileMinX, |
405 | | int &nTileMinY, int &nTileMaxX, |
406 | | int &nTileMaxY) const; |
407 | | |
408 | | CPL_DISALLOW_COPY_ASSIGN(OGRPMTilesVectorLayer) |
409 | | }; |
410 | | |
411 | | const char *VSIPMTilesGetTileExtension(OGRPMTilesDataset *poDS); |
412 | | |
413 | | #ifdef HAVE_MVT_WRITE_SUPPORT |
414 | | |
415 | | /************************************************************************/ |
416 | | /* OGRPMTilesWriterDataset */ |
417 | | /************************************************************************/ |
418 | | |
419 | | class OGRPMTilesWriterDataset final : public GDALDataset |
420 | | { |
421 | | std::unique_ptr<GDALDataset> m_poMBTilesWriterDataset{}; |
422 | | |
423 | | public: |
424 | | OGRPMTilesWriterDataset() = default; |
425 | | |
426 | | ~OGRPMTilesWriterDataset() override; |
427 | | |
428 | | bool Create(const char *pszFilename, CSLConstList papszOptions); |
429 | | |
430 | | CPLErr Close(GDALProgressFunc = nullptr, void * = nullptr) override; |
431 | | |
432 | | OGRLayer *ICreateLayer(const char *pszName, |
433 | | const OGRGeomFieldDefn *poGeomFieldDefn, |
434 | | CSLConstList papszOptions) override; |
435 | | |
436 | | int TestCapability(const char *) const override; |
437 | | }; |
438 | | |
439 | | #endif // HAVE_MVT_WRITE_SUPPORT |
440 | | |
441 | | /************************************************************************/ |
442 | | /* HashArray() */ |
443 | | /************************************************************************/ |
444 | | |
445 | | // From https://codereview.stackexchange.com/questions/171999/specializing-stdhash-for-stdarray |
446 | | // We do not use std::hash<std::array<T, N>> as the name of the struct |
447 | | // because with gcc 5.4 we get the following error: |
448 | | // https://stackoverflow.com/questions/25594644/warning-specialization-of-template-in-different-namespace |
449 | | template <class T, size_t N> struct HashArray |
450 | | { |
451 | | CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW |
452 | | size_t operator()(const std::array<T, N> &key) const |
453 | 0 | { |
454 | 0 | std::hash<T> hasher; |
455 | 0 | size_t result = 0; |
456 | 0 | for (size_t i = 0; i < N; ++i) |
457 | 0 | { |
458 | 0 | result = result * 31 + hasher(key[i]); |
459 | 0 | } |
460 | 0 | return result; |
461 | 0 | } |
462 | | }; |
463 | | |
464 | | #endif // OGR_PMTILES_H_INCLUDED |