/src/gdal/ogr/ogrsf_frmts/pmtiles/vsipmtiles.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: OpenGIS Simple Features Reference Implementation |
4 | | * Purpose: Virtual file system /vsipmtiles/ PMTiles |
5 | | * Author: Even Rouault <even.rouault at spatialys.com> |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2023, Planet Labs |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_vsi_virtual.h" |
14 | | |
15 | | #include "vsipmtiles.h" |
16 | | #include "ogr_pmtiles.h" |
17 | | |
18 | | #include "cpl_json.h" |
19 | | |
20 | | #include <set> |
21 | | |
22 | | #define ENDS_WITH_CI(a, b) \ |
23 | 0 | (strlen(a) >= strlen(b) && EQUAL(a + strlen(a) - strlen(b), b)) |
24 | | |
25 | | constexpr const char *PMTILES_HEADER_JSON = "pmtiles_header.json"; |
26 | | constexpr const char *METADATA_JSON = "metadata.json"; |
27 | | |
28 | | /************************************************************************/ |
29 | | /* VSIPMTilesFilesystemHandler */ |
30 | | /************************************************************************/ |
31 | | |
32 | | class VSIPMTilesFilesystemHandler final : public VSIFilesystemHandler |
33 | | { |
34 | | public: |
35 | 22 | VSIPMTilesFilesystemHandler() = default; |
36 | | |
37 | | VSIVirtualHandleUniquePtr Open(const char *pszFilename, |
38 | | const char *pszAccess, bool bSetError, |
39 | | CSLConstList papszOptions) override; |
40 | | int Stat(const char *pszFilename, VSIStatBufL *pStatBuf, |
41 | | int nFlags) override; |
42 | | char **ReadDirEx(const char *pszDirname, int nMaxFiles) override; |
43 | | }; |
44 | | |
45 | | /************************************************************************/ |
46 | | /* VSIPMTilesGetTileExtension() */ |
47 | | /************************************************************************/ |
48 | | |
49 | | static const char *VSIPMTilesGetTileExtension(OGRPMTilesDataset *poDS) |
50 | 0 | { |
51 | 0 | const auto &sHeader = poDS->GetHeader(); |
52 | 0 | switch (sHeader.tile_type) |
53 | 0 | { |
54 | 0 | case pmtiles::TILETYPE_PNG: |
55 | 0 | return ".png"; |
56 | 0 | case pmtiles::TILETYPE_JPEG: |
57 | 0 | return ".jpg"; |
58 | 0 | case pmtiles::TILETYPE_WEBP: |
59 | 0 | return ".webp"; |
60 | 0 | case pmtiles::TILETYPE_MVT: |
61 | 0 | return ".mvt"; |
62 | 0 | } |
63 | 0 | if (sHeader.tile_compression == pmtiles::COMPRESSION_GZIP) |
64 | 0 | return ".bin.gz"; |
65 | 0 | if (sHeader.tile_compression == pmtiles::COMPRESSION_ZSTD) |
66 | 0 | return ".bin.zstd"; |
67 | 0 | return ".bin"; |
68 | 0 | } |
69 | | |
70 | | /************************************************************************/ |
71 | | /* VSIPMTilesGetPMTilesHeaderJson() */ |
72 | | /************************************************************************/ |
73 | | |
74 | | static std::string VSIPMTilesGetPMTilesHeaderJson(OGRPMTilesDataset *poDS) |
75 | 0 | { |
76 | 0 | const auto &sHeader = poDS->GetHeader(); |
77 | 0 | CPLJSONDocument oDoc; |
78 | 0 | CPLJSONObject oRoot; |
79 | 0 | oRoot.Set("root_dir_offset", sHeader.root_dir_offset); |
80 | 0 | oRoot.Set("json_metadata_offset", sHeader.json_metadata_offset); |
81 | 0 | oRoot.Set("json_metadata_bytes", sHeader.json_metadata_bytes); |
82 | 0 | oRoot.Set("leaf_dirs_offset", sHeader.leaf_dirs_offset); |
83 | 0 | oRoot.Set("leaf_dirs_bytes", sHeader.leaf_dirs_bytes); |
84 | 0 | oRoot.Set("tile_data_offset", sHeader.tile_data_offset); |
85 | 0 | oRoot.Set("tile_data_bytes", sHeader.tile_data_bytes); |
86 | 0 | oRoot.Set("addressed_tiles_count", sHeader.addressed_tiles_count); |
87 | 0 | oRoot.Set("tile_entries_count", sHeader.tile_entries_count); |
88 | 0 | oRoot.Set("tile_contents_count", sHeader.tile_contents_count); |
89 | 0 | oRoot.Set("clustered", sHeader.clustered); |
90 | 0 | oRoot.Set("internal_compression", sHeader.internal_compression); |
91 | 0 | oRoot.Set("internal_compression_str", |
92 | 0 | OGRPMTilesDataset::GetCompression(sHeader.internal_compression)); |
93 | 0 | oRoot.Set("tile_compression", sHeader.tile_compression); |
94 | 0 | oRoot.Set("tile_compression_str", |
95 | 0 | OGRPMTilesDataset::GetCompression(sHeader.tile_compression)); |
96 | 0 | oRoot.Set("tile_type", sHeader.tile_type); |
97 | 0 | oRoot.Set("tile_type_str", OGRPMTilesDataset::GetTileType(sHeader)); |
98 | 0 | oRoot.Set("min_zoom", sHeader.min_zoom); |
99 | 0 | oRoot.Set("max_zoom", sHeader.max_zoom); |
100 | 0 | oRoot.Set("min_lon_e7", sHeader.min_lon_e7); |
101 | 0 | oRoot.Set("min_lon_e7_float", sHeader.min_lon_e7 / 10e6); |
102 | 0 | oRoot.Set("min_lat_e7", sHeader.min_lat_e7); |
103 | 0 | oRoot.Set("min_lat_e7_float", sHeader.min_lat_e7 / 10e6); |
104 | 0 | oRoot.Set("max_lon_e7", sHeader.max_lon_e7); |
105 | 0 | oRoot.Set("max_lon_e7_float", sHeader.max_lon_e7 / 10e6); |
106 | 0 | oRoot.Set("max_lat_e7", sHeader.max_lat_e7); |
107 | 0 | oRoot.Set("max_lat_e7_float", sHeader.max_lat_e7 / 10e6); |
108 | 0 | oRoot.Set("center_zoom", sHeader.center_zoom); |
109 | 0 | oRoot.Set("center_lon_e7", sHeader.center_lon_e7); |
110 | 0 | oRoot.Set("center_lat_e7", sHeader.center_lat_e7); |
111 | 0 | oDoc.SetRoot(oRoot); |
112 | 0 | return oDoc.SaveAsString(); |
113 | 0 | } |
114 | | |
115 | | /************************************************************************/ |
116 | | /* VSIPMTilesOpen() */ |
117 | | /************************************************************************/ |
118 | | |
119 | | static std::unique_ptr<OGRPMTilesDataset> |
120 | | VSIPMTilesOpen(const char *pszFilename, std::string &osSubfilename, |
121 | | int &nComponents, int &nZ, int &nX, int &nY) |
122 | 516 | { |
123 | 516 | if (!STARTS_WITH(pszFilename, "/vsipmtiles/")) |
124 | 0 | return nullptr; |
125 | 516 | pszFilename += strlen("/vsipmtiles/"); |
126 | | |
127 | 516 | std::string osFilename(pszFilename); |
128 | 516 | if (!osFilename.empty() && osFilename.back() == '/') |
129 | 9 | osFilename.pop_back(); |
130 | 516 | pszFilename = osFilename.c_str(); |
131 | | |
132 | 516 | nZ = nX = nY = -1; |
133 | 516 | nComponents = 0; |
134 | 516 | std::string osPmtilesFilename; |
135 | | |
136 | 516 | const char *pszPmtilesExt = strstr(pszFilename, ".pmtiles"); |
137 | 516 | if (!pszPmtilesExt) |
138 | 516 | return nullptr; |
139 | | |
140 | 0 | CPLStringList aosTokens; |
141 | 0 | do |
142 | 0 | { |
143 | 0 | if (pszPmtilesExt[strlen(".pmtiles")] == '/') |
144 | 0 | { |
145 | 0 | const char *pszSubFile = pszPmtilesExt + strlen(".pmtiles/"); |
146 | 0 | osPmtilesFilename.assign(pszFilename, pszSubFile - pszFilename - 1); |
147 | 0 | osSubfilename = pszPmtilesExt + strlen(".pmtiles/"); |
148 | 0 | if (osSubfilename == METADATA_JSON || |
149 | 0 | osSubfilename == PMTILES_HEADER_JSON) |
150 | 0 | { |
151 | 0 | break; |
152 | 0 | } |
153 | 0 | } |
154 | 0 | else |
155 | 0 | { |
156 | 0 | osPmtilesFilename = pszFilename; |
157 | 0 | osSubfilename.clear(); |
158 | 0 | break; |
159 | 0 | } |
160 | | |
161 | 0 | aosTokens = CSLTokenizeString2(osSubfilename.c_str(), "/", 0); |
162 | 0 | nComponents = aosTokens.size(); |
163 | 0 | if (nComponents >= 4) |
164 | 0 | return nullptr; |
165 | | |
166 | 0 | if (CPLGetValueType(aosTokens[0]) != CPL_VALUE_INTEGER) |
167 | 0 | return nullptr; |
168 | 0 | nZ = atoi(aosTokens[0]); |
169 | 0 | if (nComponents == 1) |
170 | 0 | break; |
171 | | |
172 | 0 | if (CPLGetValueType(aosTokens[1]) != CPL_VALUE_INTEGER) |
173 | 0 | return nullptr; |
174 | 0 | nX = atoi(aosTokens[1]); |
175 | 0 | if (nComponents == 2) |
176 | 0 | break; |
177 | | |
178 | 0 | break; |
179 | 0 | } while (false); |
180 | | |
181 | 0 | GDALOpenInfo oOpenInfo(osPmtilesFilename.c_str(), GA_ReadOnly); |
182 | 0 | CPLStringList aosOptions; |
183 | 0 | aosOptions.SetNameValue("DECOMPRESS_TILES", "NO"); |
184 | 0 | aosOptions.SetNameValue("ACCEPT_ANY_TILE_TYPE", "YES"); |
185 | 0 | oOpenInfo.papszOpenOptions = aosOptions.List(); |
186 | 0 | auto poDS = std::make_unique<OGRPMTilesDataset>(); |
187 | 0 | { |
188 | 0 | CPLErrorHandlerPusher oErrorHandler(CPLQuietErrorHandler); |
189 | 0 | if (!poDS->Open(&oOpenInfo)) |
190 | 0 | return nullptr; |
191 | 0 | } |
192 | | |
193 | 0 | if (nComponents == 3) |
194 | 0 | { |
195 | 0 | const char *pszTileExt = VSIPMTilesGetTileExtension(poDS.get()); |
196 | 0 | if (!ENDS_WITH_CI(aosTokens[2], pszTileExt)) |
197 | 0 | return nullptr; |
198 | 0 | aosTokens[2][strlen(aosTokens[2]) - strlen(pszTileExt)] = 0; |
199 | 0 | if (CPLGetValueType(aosTokens[2]) != CPL_VALUE_INTEGER) |
200 | 0 | return nullptr; |
201 | 0 | nY = atoi(aosTokens[2]); |
202 | 0 | } |
203 | 0 | return poDS; |
204 | 0 | } |
205 | | |
206 | | /************************************************************************/ |
207 | | /* Open() */ |
208 | | /************************************************************************/ |
209 | | |
210 | | VSIVirtualHandleUniquePtr |
211 | | VSIPMTilesFilesystemHandler::Open(const char *pszFilename, |
212 | | const char *pszAccess, bool /*bSetError*/, |
213 | | CSLConstList /*papszOptions*/) |
214 | 195 | { |
215 | 195 | if (strchr(pszAccess, '+') || strchr(pszAccess, 'w') || |
216 | 195 | strchr(pszAccess, 'a')) |
217 | 0 | return nullptr; |
218 | 195 | std::string osSubfilename; |
219 | 195 | int nComponents; |
220 | 195 | int nZ; |
221 | 195 | int nX; |
222 | 195 | int nY; |
223 | 195 | auto poDS = |
224 | 195 | VSIPMTilesOpen(pszFilename, osSubfilename, nComponents, nZ, nX, nY); |
225 | 195 | if (!poDS) |
226 | 195 | return nullptr; |
227 | | |
228 | 0 | if (osSubfilename == METADATA_JSON) |
229 | 0 | { |
230 | 0 | return VSIVirtualHandleUniquePtr( |
231 | 0 | VSIFileFromMemBuffer(nullptr, |
232 | 0 | reinterpret_cast<GByte *>(CPLStrdup( |
233 | 0 | poDS->GetMetadataContent().c_str())), |
234 | 0 | poDS->GetMetadataContent().size(), true)); |
235 | 0 | } |
236 | | |
237 | 0 | if (osSubfilename == PMTILES_HEADER_JSON) |
238 | 0 | { |
239 | 0 | const auto osStr = VSIPMTilesGetPMTilesHeaderJson(poDS.get()); |
240 | 0 | return VSIVirtualHandleUniquePtr(VSIFileFromMemBuffer( |
241 | 0 | nullptr, reinterpret_cast<GByte *>(CPLStrdup(osStr.c_str())), |
242 | 0 | osStr.size(), true)); |
243 | 0 | } |
244 | | |
245 | 0 | if (nComponents != 3) |
246 | 0 | return nullptr; |
247 | | |
248 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
249 | |
|
250 | 0 | OGRPMTilesTileIterator oIter(poDS.get(), nZ, nX, nY, nX, nY); |
251 | 0 | auto sTile = oIter.GetNextTile(); |
252 | 0 | if (sTile.offset == 0) |
253 | 0 | return nullptr; |
254 | | |
255 | 0 | const auto *posStr = poDS->ReadTileData(sTile.offset, sTile.length); |
256 | 0 | if (!posStr) |
257 | 0 | { |
258 | 0 | return nullptr; |
259 | 0 | } |
260 | | |
261 | 0 | GByte *pabyData = static_cast<GByte *>(CPLMalloc(posStr->size())); |
262 | 0 | memcpy(pabyData, posStr->data(), posStr->size()); |
263 | 0 | return VSIVirtualHandleUniquePtr( |
264 | 0 | VSIFileFromMemBuffer(nullptr, pabyData, posStr->size(), true)); |
265 | 0 | } |
266 | | |
267 | | /************************************************************************/ |
268 | | /* Stat() */ |
269 | | /************************************************************************/ |
270 | | |
271 | | int VSIPMTilesFilesystemHandler::Stat(const char *pszFilename, |
272 | | VSIStatBufL *pStatBuf, int /*nFlags*/) |
273 | 305 | { |
274 | 305 | memset(pStatBuf, 0, sizeof(VSIStatBufL)); |
275 | | |
276 | 305 | std::string osSubfilename; |
277 | 305 | int nComponents; |
278 | 305 | int nZ; |
279 | 305 | int nX; |
280 | 305 | int nY; |
281 | 305 | auto poDS = |
282 | 305 | VSIPMTilesOpen(pszFilename, osSubfilename, nComponents, nZ, nX, nY); |
283 | 305 | if (!poDS) |
284 | 305 | return -1; |
285 | | |
286 | 0 | VSIStatBufL sStatPmtiles; |
287 | 0 | if (VSIStatL(poDS->GetDescription(), &sStatPmtiles) == 0) |
288 | 0 | { |
289 | 0 | pStatBuf->st_mtime = sStatPmtiles.st_mtime; |
290 | 0 | } |
291 | |
|
292 | 0 | if (osSubfilename.empty()) |
293 | 0 | { |
294 | 0 | pStatBuf->st_mode = S_IFDIR; |
295 | 0 | return 0; |
296 | 0 | } |
297 | 0 | else if (osSubfilename == METADATA_JSON) |
298 | 0 | { |
299 | 0 | pStatBuf->st_mode = S_IFREG; |
300 | 0 | pStatBuf->st_size = poDS->GetMetadataContent().size(); |
301 | 0 | return 0; |
302 | 0 | } |
303 | 0 | else if (osSubfilename == PMTILES_HEADER_JSON) |
304 | 0 | { |
305 | 0 | pStatBuf->st_mode = S_IFREG; |
306 | 0 | pStatBuf->st_size = VSIPMTilesGetPMTilesHeaderJson(poDS.get()).size(); |
307 | 0 | return 0; |
308 | 0 | } |
309 | | |
310 | 0 | CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); |
311 | |
|
312 | 0 | OGRPMTilesTileIterator oIter(poDS.get(), nZ, nX, nY, nX, nY); |
313 | 0 | auto sTile = oIter.GetNextTile(); |
314 | 0 | if (sTile.offset == 0) |
315 | 0 | return -1; |
316 | | |
317 | 0 | if (nComponents <= 2) |
318 | 0 | { |
319 | 0 | pStatBuf->st_mode = S_IFDIR; |
320 | 0 | return 0; |
321 | 0 | } |
322 | | |
323 | 0 | pStatBuf->st_mode = S_IFREG; |
324 | 0 | pStatBuf->st_size = sTile.length; |
325 | 0 | return 0; |
326 | 0 | } |
327 | | |
328 | | /************************************************************************/ |
329 | | /* ReadDirEx() */ |
330 | | /************************************************************************/ |
331 | | |
332 | | char **VSIPMTilesFilesystemHandler::ReadDirEx(const char *pszFilename, |
333 | | int nMaxFiles) |
334 | 16 | { |
335 | 16 | std::string osSubfilename; |
336 | 16 | int nComponents; |
337 | 16 | int nZ; |
338 | 16 | int nX; |
339 | 16 | int nY; |
340 | 16 | auto poDS = |
341 | 16 | VSIPMTilesOpen(pszFilename, osSubfilename, nComponents, nZ, nX, nY); |
342 | 16 | if (!poDS) |
343 | 16 | return nullptr; |
344 | | |
345 | 0 | if (osSubfilename.empty()) |
346 | 0 | { |
347 | 0 | CPLStringList aosFiles; |
348 | 0 | aosFiles.AddString(PMTILES_HEADER_JSON); |
349 | 0 | aosFiles.AddString(METADATA_JSON); |
350 | 0 | for (int i = poDS->GetMinZoomLevel(); i <= poDS->GetMaxZoomLevel(); ++i) |
351 | 0 | { |
352 | 0 | OGRPMTilesTileIterator oIter(poDS.get(), i); |
353 | 0 | auto sTile = oIter.GetNextTile(); |
354 | 0 | if (sTile.offset != 0) |
355 | 0 | { |
356 | 0 | if (nMaxFiles > 0 && aosFiles.size() >= nMaxFiles) |
357 | 0 | break; |
358 | 0 | aosFiles.AddString(CPLSPrintf("%d", i)); |
359 | 0 | } |
360 | 0 | } |
361 | 0 | return aosFiles.StealList(); |
362 | 0 | } |
363 | | |
364 | 0 | if (nComponents == 1) |
365 | 0 | { |
366 | 0 | std::set<int> oSetX; |
367 | 0 | OGRPMTilesTileIterator oIter(poDS.get(), nZ); |
368 | 0 | while (true) |
369 | 0 | { |
370 | 0 | auto sTile = oIter.GetNextTile(); |
371 | 0 | if (sTile.offset == 0) |
372 | 0 | break; |
373 | 0 | oSetX.insert(sTile.x); |
374 | 0 | if (nMaxFiles > 0 && static_cast<int>(oSetX.size()) >= nMaxFiles) |
375 | 0 | break; |
376 | 0 | if (oSetX.size() == 1024 * 1024) |
377 | 0 | { |
378 | 0 | CPLError(CE_Failure, CPLE_AppDefined, "Too many tiles"); |
379 | 0 | return nullptr; |
380 | 0 | } |
381 | 0 | } |
382 | 0 | CPLStringList aosFiles; |
383 | 0 | for (int x : oSetX) |
384 | 0 | { |
385 | 0 | aosFiles.AddString(CPLSPrintf("%d", x)); |
386 | 0 | } |
387 | 0 | return aosFiles.StealList(); |
388 | 0 | } |
389 | | |
390 | 0 | if (nComponents == 2) |
391 | 0 | { |
392 | 0 | std::set<int> oSetY; |
393 | 0 | OGRPMTilesTileIterator oIter(poDS.get(), nZ, nX, -1, nX, -1); |
394 | 0 | while (true) |
395 | 0 | { |
396 | 0 | auto sTile = oIter.GetNextTile(); |
397 | 0 | if (sTile.offset == 0) |
398 | 0 | break; |
399 | 0 | oSetY.insert(sTile.y); |
400 | 0 | if (nMaxFiles > 0 && static_cast<int>(oSetY.size()) >= nMaxFiles) |
401 | 0 | break; |
402 | 0 | if (oSetY.size() == 1024 * 1024) |
403 | 0 | { |
404 | 0 | CPLError(CE_Failure, CPLE_AppDefined, "Too many tiles"); |
405 | 0 | return nullptr; |
406 | 0 | } |
407 | 0 | } |
408 | 0 | CPLStringList aosFiles; |
409 | 0 | const char *pszTileExt = VSIPMTilesGetTileExtension(poDS.get()); |
410 | 0 | for (int y : oSetY) |
411 | 0 | { |
412 | 0 | aosFiles.AddString(CPLSPrintf("%d%s", y, pszTileExt)); |
413 | 0 | } |
414 | 0 | return aosFiles.StealList(); |
415 | 0 | } |
416 | | |
417 | 0 | return nullptr; |
418 | 0 | } |
419 | | |
420 | | /************************************************************************/ |
421 | | /* VSIPMTilesRegister() */ |
422 | | /************************************************************************/ |
423 | | |
424 | | void VSIPMTilesRegister() |
425 | 22 | { |
426 | 22 | if (VSIFileManager::GetHandler("/vsipmtiles/") == |
427 | 22 | VSIFileManager::GetHandler("/")) |
428 | 22 | { |
429 | 22 | VSIFileManager::InstallHandler( |
430 | 22 | "/vsipmtiles/", std::make_shared<VSIPMTilesFilesystemHandler>()); |
431 | 22 | } |
432 | 22 | } |