/src/gdal/frmts/wms/gdalwmscache.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: WMS Client Driver |
4 | | * Purpose: Implementation of Dataset and RasterBand classes for WMS |
5 | | * and other similar services. |
6 | | * Author: Adam Nowacki, nowak@xpam.de |
7 | | * |
8 | | ****************************************************************************** |
9 | | * Copyright (c) 2007, Adam Nowacki |
10 | | * Copyright (c) 2017, Dmitry Baryshnikov, <polimax@mail.ru> |
11 | | * Copyright (c) 2017, NextGIS, <info@nextgis.com> |
12 | | * |
13 | | * SPDX-License-Identifier: MIT |
14 | | ****************************************************************************/ |
15 | | |
16 | | #include "cpl_md5.h" |
17 | | #include "wmsdriver.h" |
18 | | |
19 | | static void CleanCacheThread(void *pData) |
20 | 2 | { |
21 | 2 | GDALWMSCache *pCache = static_cast<GDALWMSCache *>(pData); |
22 | 2 | pCache->Clean(); |
23 | 2 | } |
24 | | |
25 | | //------------------------------------------------------------------------------ |
26 | | // GDALWMSFileCache |
27 | | //------------------------------------------------------------------------------ |
28 | | class GDALWMSFileCache : public GDALWMSCacheImpl |
29 | | { |
30 | | public: |
31 | | GDALWMSFileCache(const CPLString &soPath, CPLXMLNode *pConfig) |
32 | 165 | : GDALWMSCacheImpl(soPath, pConfig), m_osPostfix(""), m_nDepth(2), |
33 | 165 | m_nExpires(604800), // 7 days |
34 | 165 | m_nMaxSize(67108864), // 64 Mb |
35 | 165 | m_nCleanThreadRunTimeout(120) // 3 min |
36 | 165 | { |
37 | 165 | const char *pszCacheDepth = CPLGetXMLValue(pConfig, "Depth", "2"); |
38 | 165 | if (pszCacheDepth != nullptr) |
39 | 165 | m_nDepth = atoi(pszCacheDepth); |
40 | | |
41 | 165 | const char *pszCacheExtension = |
42 | 165 | CPLGetXMLValue(pConfig, "Extension", nullptr); |
43 | 165 | if (pszCacheExtension != nullptr) |
44 | 0 | m_osPostfix = pszCacheExtension; |
45 | | |
46 | 165 | const char *pszCacheExpires = |
47 | 165 | CPLGetXMLValue(pConfig, "Expires", nullptr); |
48 | 165 | if (pszCacheExpires != nullptr) |
49 | 0 | { |
50 | 0 | m_nExpires = atoi(pszCacheExpires); |
51 | 0 | CPLDebug("WMS", "Cache expires in %d sec", m_nExpires); |
52 | 0 | } |
53 | | |
54 | 165 | const char *pszCacheMaxSize = |
55 | 165 | CPLGetXMLValue(pConfig, "MaxSize", nullptr); |
56 | 165 | if (pszCacheMaxSize != nullptr) |
57 | 0 | m_nMaxSize = atol(pszCacheMaxSize); |
58 | | |
59 | 165 | const char *pszCleanThreadRunTimeout = |
60 | 165 | CPLGetXMLValue(pConfig, "CleanTimeout", nullptr); |
61 | 165 | if (pszCleanThreadRunTimeout != nullptr) |
62 | 0 | { |
63 | 0 | m_nCleanThreadRunTimeout = atoi(pszCleanThreadRunTimeout); |
64 | 0 | CPLDebug("WMS", "Clean Thread Run Timeout is %d sec", |
65 | 0 | m_nCleanThreadRunTimeout); |
66 | 0 | } |
67 | 165 | } |
68 | | |
69 | | virtual int GetCleanThreadRunTimeout() override; |
70 | | |
71 | | virtual CPLErr Insert(const char *pszKey, |
72 | | const CPLString &osFileName) override |
73 | 36 | { |
74 | | // Warns if it fails to write, but returns success |
75 | 36 | CPLString soFilePath = GetFilePath(pszKey); |
76 | 36 | MakeDirs(CPLGetDirnameSafe(soFilePath).c_str()); |
77 | 36 | if (CPLCopyFile(soFilePath, osFileName) == CE_None) |
78 | 36 | return CE_None; |
79 | | // Warn if it fails after folder creation |
80 | 0 | CPLError(CE_Warning, CPLE_FileIO, "Error writing to WMS cache %s", |
81 | 0 | m_soPath.c_str()); |
82 | 0 | return CE_None; |
83 | 36 | } |
84 | | |
85 | | virtual enum GDALWMSCacheItemStatus |
86 | | GetItemStatus(const char *pszKey) const override |
87 | 192 | { |
88 | 192 | VSIStatBufL sStatBuf; |
89 | 192 | if (VSIStatL(GetFilePath(pszKey), &sStatBuf) == 0) |
90 | 136 | { |
91 | 136 | long seconds = static_cast<long>(time(nullptr) - sStatBuf.st_mtime); |
92 | 136 | return seconds < m_nExpires ? CACHE_ITEM_OK : CACHE_ITEM_EXPIRED; |
93 | 136 | } |
94 | 56 | return CACHE_ITEM_NOT_FOUND; |
95 | 192 | } |
96 | | |
97 | | virtual GDALDataset *GetDataset(const char *pszKey, |
98 | | char **papszOpenOptions) const override |
99 | 144 | { |
100 | 144 | return GDALDataset::FromHandle(GDALOpenEx( |
101 | 144 | GetFilePath(pszKey), |
102 | 144 | GDAL_OF_RASTER | GDAL_OF_READONLY | GDAL_OF_VERBOSE_ERROR, nullptr, |
103 | 144 | papszOpenOptions, nullptr)); |
104 | 144 | } |
105 | | |
106 | | virtual void Clean() override |
107 | 2 | { |
108 | 2 | char **papszList = VSIReadDirRecursive(m_soPath); |
109 | 2 | if (papszList == nullptr) |
110 | 0 | { |
111 | 0 | return; |
112 | 0 | } |
113 | | |
114 | 2 | int counter = 0; |
115 | 2 | std::vector<int> toDelete; |
116 | 2 | long nSize = 0; |
117 | 2 | time_t nTime = time(nullptr); |
118 | 8 | while (papszList[counter] != nullptr) |
119 | 6 | { |
120 | 6 | const std::string osPath = |
121 | 6 | CPLFormFilenameSafe(m_soPath, papszList[counter], nullptr); |
122 | 6 | VSIStatBufL sStatBuf; |
123 | 6 | if (VSIStatL(osPath.c_str(), &sStatBuf) == 0) |
124 | 6 | { |
125 | 6 | if (!VSI_ISDIR(sStatBuf.st_mode)) |
126 | 2 | { |
127 | 2 | long seconds = static_cast<long>(nTime - sStatBuf.st_mtime); |
128 | 2 | if (seconds > m_nExpires) |
129 | 0 | { |
130 | 0 | toDelete.push_back(counter); |
131 | 0 | } |
132 | | |
133 | 2 | nSize += static_cast<long>(sStatBuf.st_size); |
134 | 2 | } |
135 | 6 | } |
136 | 6 | counter++; |
137 | 6 | } |
138 | | |
139 | 2 | if (nSize > m_nMaxSize) |
140 | 0 | { |
141 | 0 | CPLDebug("WMS", "Delete %u items from cache", |
142 | 0 | static_cast<unsigned int>(toDelete.size())); |
143 | 0 | for (size_t i = 0; i < toDelete.size(); ++i) |
144 | 0 | { |
145 | 0 | const std::string osPath = CPLFormFilenameSafe( |
146 | 0 | m_soPath, papszList[toDelete[i]], nullptr); |
147 | 0 | VSIUnlink(osPath.c_str()); |
148 | 0 | } |
149 | 0 | } |
150 | | |
151 | 2 | CSLDestroy(papszList); |
152 | 2 | } |
153 | | |
154 | | private: |
155 | | CPLString GetFilePath(const char *pszKey) const |
156 | 372 | { |
157 | 372 | CPLString soHash(CPLMD5String(pszKey)); |
158 | 372 | CPLString soCacheFile(m_soPath); |
159 | | |
160 | 372 | if (!soCacheFile.empty() && soCacheFile.back() != '/') |
161 | 372 | { |
162 | 372 | soCacheFile.append(1, '/'); |
163 | 372 | } |
164 | | |
165 | 1.11k | for (int i = 0; i < m_nDepth; ++i) |
166 | 744 | { |
167 | 744 | soCacheFile.append(1, soHash[i]); |
168 | 744 | soCacheFile.append(1, '/'); |
169 | 744 | } |
170 | 372 | soCacheFile.append(soHash); |
171 | 372 | soCacheFile.append(m_osPostfix); |
172 | 372 | return soCacheFile; |
173 | 372 | } |
174 | | |
175 | | static void MakeDirs(const char *pszPath) |
176 | 93 | { |
177 | 93 | if (IsPathExists(pszPath)) |
178 | 36 | { |
179 | 36 | return; |
180 | 36 | } |
181 | | // Recursive makedirs, ignoring errors |
182 | 57 | MakeDirs(CPLGetDirnameSafe(pszPath).c_str()); |
183 | | |
184 | 57 | VSIMkdir(pszPath, 0744); |
185 | 57 | } |
186 | | |
187 | | static bool IsPathExists(const char *pszPath) |
188 | 93 | { |
189 | 93 | VSIStatBufL sbuf; |
190 | 93 | return VSIStatL(pszPath, &sbuf) == 0; |
191 | 93 | } |
192 | | |
193 | | private: |
194 | | CPLString m_osPostfix; |
195 | | int m_nDepth; |
196 | | int m_nExpires; |
197 | | long m_nMaxSize; |
198 | | int m_nCleanThreadRunTimeout; |
199 | | }; |
200 | | |
201 | | int GDALWMSFileCache::GetCleanThreadRunTimeout() |
202 | 36 | { |
203 | 36 | return m_nCleanThreadRunTimeout; |
204 | 36 | } |
205 | | |
206 | | //------------------------------------------------------------------------------ |
207 | | // GDALWMSCache |
208 | | //------------------------------------------------------------------------------ |
209 | | |
210 | 165 | GDALWMSCache::GDALWMSCache() = default; |
211 | | |
212 | | GDALWMSCache::~GDALWMSCache() |
213 | 165 | { |
214 | 165 | if (m_hThread) |
215 | 2 | CPLJoinThread(m_hThread); |
216 | 165 | delete m_poCache; |
217 | 165 | } |
218 | | |
219 | | CPLErr GDALWMSCache::Initialize(const char *pszUrl, CPLXMLNode *pConfig) |
220 | 165 | { |
221 | 165 | const auto NullifyIfEmpty = [](const char *pszStr) |
222 | 660 | { return pszStr && pszStr[0] != 0 ? pszStr : nullptr; }; |
223 | | |
224 | 165 | if (const char *pszXmlCachePath = |
225 | 165 | NullifyIfEmpty(CPLGetXMLValue(pConfig, "Path", nullptr))) |
226 | 0 | { |
227 | 0 | m_osCachePath = pszXmlCachePath; |
228 | 0 | } |
229 | 165 | else if (const char *pszUserCachePath = NullifyIfEmpty( |
230 | 165 | CPLGetConfigOption("GDAL_DEFAULT_WMS_CACHE_PATH", nullptr))) |
231 | 0 | { |
232 | 0 | m_osCachePath = pszUserCachePath; |
233 | 0 | } |
234 | 165 | else if (const char *pszXDG_CACHE_HOME = |
235 | 165 | NullifyIfEmpty(CPLGetConfigOption("XDG_CACHE_HOME", nullptr))) |
236 | 0 | { |
237 | 0 | m_osCachePath = |
238 | 0 | CPLFormFilenameSafe(pszXDG_CACHE_HOME, "gdalwmscache", nullptr); |
239 | 0 | } |
240 | 165 | else |
241 | 165 | { |
242 | | #ifdef _WIN32 |
243 | | const char *pszHome = |
244 | | NullifyIfEmpty(CPLGetConfigOption("USERPROFILE", nullptr)); |
245 | | #else |
246 | 165 | const char *pszHome = |
247 | 165 | NullifyIfEmpty(CPLGetConfigOption("HOME", nullptr)); |
248 | 165 | #endif |
249 | 165 | if (pszHome) |
250 | 165 | { |
251 | 165 | m_osCachePath = CPLFormFilenameSafe( |
252 | 165 | CPLFormFilenameSafe(pszHome, ".cache", nullptr).c_str(), |
253 | 165 | "gdalwmscache", nullptr); |
254 | 165 | } |
255 | 0 | else |
256 | 0 | { |
257 | 0 | const char *pszDir = |
258 | 0 | NullifyIfEmpty(CPLGetConfigOption("CPL_TMPDIR", nullptr)); |
259 | |
|
260 | 0 | if (!pszDir) |
261 | 0 | pszDir = NullifyIfEmpty(CPLGetConfigOption("TMPDIR", nullptr)); |
262 | |
|
263 | 0 | if (!pszDir) |
264 | 0 | pszDir = NullifyIfEmpty(CPLGetConfigOption("TEMP", nullptr)); |
265 | |
|
266 | 0 | if (!pszDir) |
267 | 0 | pszDir = "."; |
268 | |
|
269 | 0 | const char *pszUsername = |
270 | 0 | NullifyIfEmpty(CPLGetConfigOption("USERNAME", nullptr)); |
271 | 0 | if (!pszUsername) |
272 | 0 | pszUsername = |
273 | 0 | NullifyIfEmpty(CPLGetConfigOption("USER", nullptr)); |
274 | |
|
275 | 0 | if (pszUsername) |
276 | 0 | { |
277 | 0 | m_osCachePath = CPLFormFilenameSafe( |
278 | 0 | pszDir, CPLSPrintf("gdalwmscache_%s", pszUsername), |
279 | 0 | nullptr); |
280 | 0 | } |
281 | 0 | else |
282 | 0 | { |
283 | 0 | m_osCachePath = CPLFormFilenameSafe( |
284 | 0 | pszDir, |
285 | 0 | CPLSPrintf("gdalwmscache_%s", |
286 | 0 | CPLMD5String(pszUrl ? pszUrl : "")), |
287 | 0 | nullptr); |
288 | 0 | } |
289 | 0 | } |
290 | 165 | } |
291 | | |
292 | | // Separate folder for each unique dataset url |
293 | 165 | if (CPLTestBool(CPLGetXMLValue(pConfig, "Unique", "True"))) |
294 | 165 | { |
295 | 165 | m_osCachePath = CPLFormFilenameSafe( |
296 | 165 | m_osCachePath, CPLMD5String(pszUrl ? pszUrl : ""), nullptr); |
297 | 165 | } |
298 | 165 | CPLDebug("WMS", "Using %s for cache", m_osCachePath.c_str()); |
299 | | |
300 | | // TODO: Add sqlite db cache type |
301 | 165 | const char *pszType = CPLGetXMLValue(pConfig, "Type", "file"); |
302 | 165 | if (EQUAL(pszType, "file")) |
303 | 165 | { |
304 | 165 | m_poCache = new GDALWMSFileCache(m_osCachePath, pConfig); |
305 | 165 | } |
306 | | |
307 | 165 | return CE_None; |
308 | 165 | } |
309 | | |
310 | | CPLErr GDALWMSCache::Insert(const char *pszKey, const CPLString &soFileName) |
311 | 36 | { |
312 | 36 | if (m_poCache != nullptr && pszKey != nullptr) |
313 | 36 | { |
314 | | // Add file to cache |
315 | 36 | CPLErr result = m_poCache->Insert(pszKey, soFileName); |
316 | 36 | if (result == CE_None) |
317 | 36 | { |
318 | | // Start clean thread |
319 | 36 | int cleanThreadRunTimeout = m_poCache->GetCleanThreadRunTimeout(); |
320 | 36 | if (cleanThreadRunTimeout > 0 && !m_bIsCleanThreadRunning && |
321 | 36 | time(nullptr) - m_nCleanThreadLastRunTime > |
322 | 36 | cleanThreadRunTimeout) |
323 | 2 | { |
324 | 2 | if (m_hThread) |
325 | 0 | CPLJoinThread(m_hThread); |
326 | 2 | m_bIsCleanThreadRunning = true; |
327 | 2 | m_hThread = CPLCreateJoinableThread(CleanCacheThread, this); |
328 | 2 | } |
329 | 36 | } |
330 | 36 | return result; |
331 | 36 | } |
332 | | |
333 | 0 | return CE_Failure; |
334 | 36 | } |
335 | | |
336 | | enum GDALWMSCacheItemStatus |
337 | | GDALWMSCache::GetItemStatus(const char *pszKey) const |
338 | 192 | { |
339 | 192 | if (m_poCache != nullptr) |
340 | 192 | { |
341 | 192 | return m_poCache->GetItemStatus(pszKey); |
342 | 192 | } |
343 | 0 | return CACHE_ITEM_NOT_FOUND; |
344 | 192 | } |
345 | | |
346 | | GDALDataset *GDALWMSCache::GetDataset(const char *pszKey, |
347 | | char **papszOpenOptions) const |
348 | 144 | { |
349 | 144 | if (m_poCache != nullptr) |
350 | 144 | { |
351 | 144 | return m_poCache->GetDataset(pszKey, papszOpenOptions); |
352 | 144 | } |
353 | 0 | return nullptr; |
354 | 144 | } |
355 | | |
356 | | void GDALWMSCache::Clean() |
357 | 2 | { |
358 | 2 | if (m_poCache != nullptr) |
359 | 2 | { |
360 | 2 | CPLDebug("WMS", "Clean cache"); |
361 | 2 | m_poCache->Clean(); |
362 | 2 | } |
363 | | |
364 | 2 | m_nCleanThreadLastRunTime = time(nullptr); |
365 | 2 | m_bIsCleanThreadRunning = false; |
366 | 2 | } |