/src/gdal/port/cpl_vsil_curl.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: CPL - Common Portability Library |
4 | | * Purpose: Implement VSI large file api for HTTP/FTP files |
5 | | * Author: Even Rouault, even.rouault at spatialys.com |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_port.h" |
14 | | #include "cpl_vsil_curl_priv.h" |
15 | | #include "cpl_vsil_curl_class.h" |
16 | | |
17 | | #include <algorithm> |
18 | | #include <array> |
19 | | #include <limits> |
20 | | #include <map> |
21 | | #include <memory> |
22 | | #include <set> |
23 | | |
24 | | #include "cpl_aws.h" |
25 | | #include "cpl_json.h" |
26 | | #include "cpl_json_header.h" |
27 | | #include "cpl_minixml.h" |
28 | | #include "cpl_multiproc.h" |
29 | | #include "cpl_string.h" |
30 | | #include "cpl_time.h" |
31 | | #include "cpl_vsi.h" |
32 | | #include "cpl_vsi_virtual.h" |
33 | | #include "cpl_http.h" |
34 | | #include "cpl_mem_cache.h" |
35 | | |
36 | | #ifndef S_IRUSR |
37 | | #define S_IRUSR 00400 |
38 | | #define S_IWUSR 00200 |
39 | | #define S_IXUSR 00100 |
40 | | #define S_IRGRP 00040 |
41 | | #define S_IWGRP 00020 |
42 | | #define S_IXGRP 00010 |
43 | | #define S_IROTH 00004 |
44 | | #define S_IWOTH 00002 |
45 | | #define S_IXOTH 00001 |
46 | | #endif |
47 | | |
48 | | #ifndef HAVE_CURL |
49 | | |
50 | | void VSIInstallCurlFileHandler(void) |
51 | 0 | { |
52 | | // Not supported. |
53 | 0 | } |
54 | | |
55 | | void VSICurlClearCache(void) |
56 | 0 | { |
57 | | // Not supported. |
58 | 0 | } |
59 | | |
60 | | void VSICurlPartialClearCache(const char *) |
61 | 0 | { |
62 | | // Not supported. |
63 | 0 | } |
64 | | |
65 | | void VSICurlAuthParametersChanged() |
66 | 815 | { |
67 | | // Not supported. |
68 | 815 | } |
69 | | |
70 | | void VSINetworkStatsReset(void) |
71 | 0 | { |
72 | | // Not supported |
73 | 0 | } |
74 | | |
75 | | char *VSINetworkStatsGetAsSerializedJSON(char ** /* papszOptions */) |
76 | 0 | { |
77 | | // Not supported |
78 | 0 | return nullptr; |
79 | 0 | } |
80 | | |
81 | | /************************************************************************/ |
82 | | /* VSICurlInstallReadCbk() */ |
83 | | /************************************************************************/ |
84 | | |
85 | | int VSICurlInstallReadCbk(VSILFILE * /* fp */, |
86 | | VSICurlReadCbkFunc /* pfnReadCbk */, |
87 | | void * /* pfnUserData */, |
88 | | int /* bStopOnInterruptUntilUninstall */) |
89 | 0 | { |
90 | 0 | return FALSE; |
91 | 0 | } |
92 | | |
93 | | /************************************************************************/ |
94 | | /* VSICurlUninstallReadCbk() */ |
95 | | /************************************************************************/ |
96 | | |
97 | | int VSICurlUninstallReadCbk(VSILFILE * /* fp */) |
98 | 0 | { |
99 | 0 | return FALSE; |
100 | 0 | } |
101 | | |
102 | | #else |
103 | | |
104 | | //! @cond Doxygen_Suppress |
105 | | #ifndef DOXYGEN_SKIP |
106 | | |
107 | | #define ENABLE_DEBUG 1 |
108 | | #define ENABLE_DEBUG_VERBOSE 0 |
109 | | |
110 | | #define unchecked_curl_easy_setopt(handle, opt, param) \ |
111 | | CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param)) |
112 | | |
113 | | /***********************************************************รน************/ |
114 | | /* VSICurlAuthParametersChanged() */ |
115 | | /************************************************************************/ |
116 | | |
117 | | static unsigned int gnGenerationAuthParameters = 0; |
118 | | |
119 | | void VSICurlAuthParametersChanged() |
120 | | { |
121 | | gnGenerationAuthParameters++; |
122 | | } |
123 | | |
124 | | // Do not access those variables directly ! |
125 | | // Use VSICURLGetDownloadChunkSize() and GetMaxRegions() |
126 | | static int N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = 0; |
127 | | static int DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = 0; |
128 | | |
129 | | /************************************************************************/ |
130 | | /* VSICURLReadGlobalEnvVariables() */ |
131 | | /************************************************************************/ |
132 | | |
133 | | static void VSICURLReadGlobalEnvVariables() |
134 | | { |
135 | | struct Initializer |
136 | | { |
137 | | Initializer() |
138 | | { |
139 | | constexpr int DOWNLOAD_CHUNK_SIZE_DEFAULT = 16384; |
140 | | const char *pszChunkSize = |
141 | | CPLGetConfigOption("CPL_VSIL_CURL_CHUNK_SIZE", nullptr); |
142 | | GIntBig nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
143 | | |
144 | | if (pszChunkSize) |
145 | | { |
146 | | if (CPLParseMemorySize(pszChunkSize, &nChunkSize, nullptr) != |
147 | | CE_None) |
148 | | { |
149 | | CPLError( |
150 | | CE_Warning, CPLE_AppDefined, |
151 | | "Could not parse value for CPL_VSIL_CURL_CHUNK_SIZE. " |
152 | | "Using default value of %d instead.", |
153 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
154 | | } |
155 | | } |
156 | | |
157 | | constexpr int MIN_CHUNK_SIZE = 1024; |
158 | | constexpr int MAX_CHUNK_SIZE = 10 * 1024 * 1024; |
159 | | if (nChunkSize < MIN_CHUNK_SIZE || nChunkSize > MAX_CHUNK_SIZE) |
160 | | { |
161 | | nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
162 | | CPLError(CE_Warning, CPLE_AppDefined, |
163 | | "Invalid value for CPL_VSIL_CURL_CHUNK_SIZE. " |
164 | | "Allowed range is [%d, %d]. " |
165 | | "Using CPL_VSIL_CURL_CHUNK_SIZE=%d instead", |
166 | | MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, |
167 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
168 | | } |
169 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = |
170 | | static_cast<int>(nChunkSize); |
171 | | |
172 | | constexpr int N_MAX_REGIONS_DEFAULT = 1000; |
173 | | constexpr int CACHE_SIZE_DEFAULT = |
174 | | N_MAX_REGIONS_DEFAULT * DOWNLOAD_CHUNK_SIZE_DEFAULT; |
175 | | |
176 | | const char *pszCacheSize = |
177 | | CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE", nullptr); |
178 | | GIntBig nCacheSize = CACHE_SIZE_DEFAULT; |
179 | | |
180 | | if (pszCacheSize) |
181 | | { |
182 | | if (CPLParseMemorySize(pszCacheSize, &nCacheSize, nullptr) != |
183 | | CE_None) |
184 | | { |
185 | | CPLError( |
186 | | CE_Warning, CPLE_AppDefined, |
187 | | "Could not parse value for CPL_VSIL_CURL_CACHE_SIZE. " |
188 | | "Using default value of " CPL_FRMT_GIB " instead.", |
189 | | nCacheSize); |
190 | | } |
191 | | } |
192 | | |
193 | | const auto nMaxRAM = CPLGetUsablePhysicalRAM(); |
194 | | const auto nMinVal = DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
195 | | auto nMaxVal = static_cast<GIntBig>(INT_MAX) * |
196 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
197 | | if (nMaxRAM > 0 && nMaxVal > nMaxRAM) |
198 | | nMaxVal = nMaxRAM; |
199 | | if (nCacheSize < nMinVal || nCacheSize > nMaxVal) |
200 | | { |
201 | | nCacheSize = nCacheSize < nMinVal ? nMinVal : nMaxVal; |
202 | | CPLError(CE_Warning, CPLE_AppDefined, |
203 | | "Invalid value for CPL_VSIL_CURL_CACHE_SIZE. " |
204 | | "Allowed range is [%d, " CPL_FRMT_GIB "]. " |
205 | | "Using CPL_VSIL_CURL_CACHE_SIZE=" CPL_FRMT_GIB |
206 | | " instead", |
207 | | nMinVal, nMaxVal, nCacheSize); |
208 | | } |
209 | | N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = std::max( |
210 | | 1, static_cast<int>(nCacheSize / |
211 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY)); |
212 | | } |
213 | | }; |
214 | | |
215 | | static Initializer initializer; |
216 | | } |
217 | | |
218 | | /************************************************************************/ |
219 | | /* VSICURLGetDownloadChunkSize() */ |
220 | | /************************************************************************/ |
221 | | |
222 | | int VSICURLGetDownloadChunkSize() |
223 | | { |
224 | | VSICURLReadGlobalEnvVariables(); |
225 | | return DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
226 | | } |
227 | | |
228 | | /************************************************************************/ |
229 | | /* GetMaxRegions() */ |
230 | | /************************************************************************/ |
231 | | |
232 | | static int GetMaxRegions() |
233 | | { |
234 | | VSICURLReadGlobalEnvVariables(); |
235 | | return N_MAX_REGIONS_DO_NOT_USE_DIRECTLY; |
236 | | } |
237 | | |
238 | | /************************************************************************/ |
239 | | /* VSICurlFindStringSensitiveExceptEscapeSequences() */ |
240 | | /************************************************************************/ |
241 | | |
242 | | static int |
243 | | VSICurlFindStringSensitiveExceptEscapeSequences(CSLConstList papszList, |
244 | | const char *pszTarget) |
245 | | |
246 | | { |
247 | | if (papszList == nullptr) |
248 | | return -1; |
249 | | |
250 | | for (int i = 0; papszList[i] != nullptr; i++) |
251 | | { |
252 | | const char *pszIter1 = papszList[i]; |
253 | | const char *pszIter2 = pszTarget; |
254 | | char ch1 = '\0'; |
255 | | char ch2 = '\0'; |
256 | | /* The comparison is case-sensitive, except for escaped */ |
257 | | /* sequences where letters of the hexadecimal sequence */ |
258 | | /* can be uppercase or lowercase depending on the quoting algorithm */ |
259 | | while (true) |
260 | | { |
261 | | ch1 = *pszIter1; |
262 | | ch2 = *pszIter2; |
263 | | if (ch1 == '\0' || ch2 == '\0') |
264 | | break; |
265 | | if (ch1 == '%' && ch2 == '%' && pszIter1[1] != '\0' && |
266 | | pszIter1[2] != '\0' && pszIter2[1] != '\0' && |
267 | | pszIter2[2] != '\0') |
268 | | { |
269 | | if (!EQUALN(pszIter1 + 1, pszIter2 + 1, 2)) |
270 | | break; |
271 | | pszIter1 += 2; |
272 | | pszIter2 += 2; |
273 | | } |
274 | | if (ch1 != ch2) |
275 | | break; |
276 | | pszIter1++; |
277 | | pszIter2++; |
278 | | } |
279 | | if (ch1 == ch2 && ch1 == '\0') |
280 | | return i; |
281 | | } |
282 | | |
283 | | return -1; |
284 | | } |
285 | | |
286 | | /************************************************************************/ |
287 | | /* VSICurlIsFileInList() */ |
288 | | /************************************************************************/ |
289 | | |
290 | | static int VSICurlIsFileInList(CSLConstList papszList, const char *pszTarget) |
291 | | { |
292 | | int nRet = |
293 | | VSICurlFindStringSensitiveExceptEscapeSequences(papszList, pszTarget); |
294 | | if (nRet >= 0) |
295 | | return nRet; |
296 | | |
297 | | // If we didn't find anything, try to URL-escape the target filename. |
298 | | char *pszEscaped = CPLEscapeString(pszTarget, -1, CPLES_URL); |
299 | | if (strcmp(pszTarget, pszEscaped) != 0) |
300 | | { |
301 | | nRet = VSICurlFindStringSensitiveExceptEscapeSequences(papszList, |
302 | | pszEscaped); |
303 | | } |
304 | | CPLFree(pszEscaped); |
305 | | return nRet; |
306 | | } |
307 | | |
308 | | /************************************************************************/ |
309 | | /* VSICurlGetURLFromFilename() */ |
310 | | /************************************************************************/ |
311 | | |
312 | | static std::string VSICurlGetURLFromFilename( |
313 | | const char *pszFilename, CPLHTTPRetryParameters *poRetryParameters, |
314 | | bool *pbUseHead, bool *pbUseRedirectURLIfNoQueryStringParams, |
315 | | bool *pbListDir, bool *pbEmptyDir, CPLStringList *paosHTTPOptions, |
316 | | bool *pbPlanetaryComputerURLSigning, char **ppszPlanetaryComputerCollection) |
317 | | { |
318 | | if (ppszPlanetaryComputerCollection) |
319 | | *ppszPlanetaryComputerCollection = nullptr; |
320 | | |
321 | | if (!STARTS_WITH(pszFilename, "/vsicurl/") && |
322 | | !STARTS_WITH(pszFilename, "/vsicurl?")) |
323 | | return pszFilename; |
324 | | |
325 | | if (pbPlanetaryComputerURLSigning) |
326 | | { |
327 | | // It may be more convenient sometimes to store Planetary Computer URL |
328 | | // signing as a per-path specific option rather than capturing it in |
329 | | // the filename with the &pc_url_signing=yes option. |
330 | | if (CPLTestBool(VSIGetPathSpecificOption( |
331 | | pszFilename, "VSICURL_PC_URL_SIGNING", "FALSE"))) |
332 | | { |
333 | | *pbPlanetaryComputerURLSigning = true; |
334 | | } |
335 | | } |
336 | | |
337 | | pszFilename += strlen("/vsicurl/"); |
338 | | if (!STARTS_WITH(pszFilename, "http://") && |
339 | | !STARTS_WITH(pszFilename, "https://") && |
340 | | !STARTS_WITH(pszFilename, "ftp://") && |
341 | | !STARTS_WITH(pszFilename, "file://")) |
342 | | { |
343 | | if (*pszFilename == '?') |
344 | | pszFilename++; |
345 | | char **papszTokens = CSLTokenizeString2(pszFilename, "&", 0); |
346 | | for (int i = 0; papszTokens[i] != nullptr; i++) |
347 | | { |
348 | | char *pszUnescaped = |
349 | | CPLUnescapeString(papszTokens[i], nullptr, CPLES_URL); |
350 | | CPLFree(papszTokens[i]); |
351 | | papszTokens[i] = pszUnescaped; |
352 | | } |
353 | | |
354 | | std::string osURL; |
355 | | std::string osHeaders; |
356 | | for (int i = 0; papszTokens[i]; i++) |
357 | | { |
358 | | char *pszKey = nullptr; |
359 | | const char *pszValue = CPLParseNameValue(papszTokens[i], &pszKey); |
360 | | if (pszKey && pszValue) |
361 | | { |
362 | | if (EQUAL(pszKey, "max_retry")) |
363 | | { |
364 | | if (poRetryParameters) |
365 | | poRetryParameters->nMaxRetry = atoi(pszValue); |
366 | | } |
367 | | else if (EQUAL(pszKey, "retry_delay")) |
368 | | { |
369 | | if (poRetryParameters) |
370 | | poRetryParameters->dfInitialDelay = CPLAtof(pszValue); |
371 | | } |
372 | | else if (EQUAL(pszKey, "retry_codes")) |
373 | | { |
374 | | if (poRetryParameters) |
375 | | poRetryParameters->osRetryCodes = pszValue; |
376 | | } |
377 | | else if (EQUAL(pszKey, "use_head")) |
378 | | { |
379 | | if (pbUseHead) |
380 | | *pbUseHead = CPLTestBool(pszValue); |
381 | | } |
382 | | else if (EQUAL(pszKey, |
383 | | "use_redirect_url_if_no_query_string_params")) |
384 | | { |
385 | | /* Undocumented. Used by PLScenes driver */ |
386 | | if (pbUseRedirectURLIfNoQueryStringParams) |
387 | | *pbUseRedirectURLIfNoQueryStringParams = |
388 | | CPLTestBool(pszValue); |
389 | | } |
390 | | else if (EQUAL(pszKey, "list_dir")) |
391 | | { |
392 | | if (pbListDir) |
393 | | *pbListDir = CPLTestBool(pszValue); |
394 | | } |
395 | | else if (EQUAL(pszKey, "empty_dir")) |
396 | | { |
397 | | if (pbEmptyDir) |
398 | | *pbEmptyDir = CPLTestBool(pszValue); |
399 | | } |
400 | | else if (EQUAL(pszKey, "useragent") || |
401 | | EQUAL(pszKey, "referer") || EQUAL(pszKey, "cookie") || |
402 | | EQUAL(pszKey, "header_file") || |
403 | | EQUAL(pszKey, "unsafessl") || |
404 | | #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION |
405 | | EQUAL(pszKey, "timeout") || |
406 | | EQUAL(pszKey, "connecttimeout") || |
407 | | #endif |
408 | | EQUAL(pszKey, "low_speed_time") || |
409 | | EQUAL(pszKey, "low_speed_limit") || |
410 | | EQUAL(pszKey, "proxy") || EQUAL(pszKey, "proxyauth") || |
411 | | EQUAL(pszKey, "proxyuserpwd")) |
412 | | { |
413 | | // Above names are the ones supported by |
414 | | // CPLHTTPSetOptions() |
415 | | if (paosHTTPOptions) |
416 | | { |
417 | | paosHTTPOptions->SetNameValue(pszKey, pszValue); |
418 | | } |
419 | | } |
420 | | else if (EQUAL(pszKey, "url")) |
421 | | { |
422 | | osURL = pszValue; |
423 | | } |
424 | | else if (EQUAL(pszKey, "pc_url_signing")) |
425 | | { |
426 | | if (pbPlanetaryComputerURLSigning) |
427 | | *pbPlanetaryComputerURLSigning = CPLTestBool(pszValue); |
428 | | } |
429 | | else if (EQUAL(pszKey, "pc_collection")) |
430 | | { |
431 | | if (ppszPlanetaryComputerCollection) |
432 | | { |
433 | | CPLFree(*ppszPlanetaryComputerCollection); |
434 | | *ppszPlanetaryComputerCollection = CPLStrdup(pszValue); |
435 | | } |
436 | | } |
437 | | else if (STARTS_WITH(pszKey, "header.")) |
438 | | { |
439 | | osHeaders += (pszKey + strlen("header.")); |
440 | | osHeaders += ':'; |
441 | | osHeaders += pszValue; |
442 | | osHeaders += "\r\n"; |
443 | | } |
444 | | else |
445 | | { |
446 | | CPLError(CE_Warning, CPLE_NotSupported, |
447 | | "Unsupported option: %s", pszKey); |
448 | | } |
449 | | } |
450 | | CPLFree(pszKey); |
451 | | } |
452 | | |
453 | | if (paosHTTPOptions && !osHeaders.empty()) |
454 | | paosHTTPOptions->SetNameValue("HEADERS", osHeaders.c_str()); |
455 | | |
456 | | CSLDestroy(papszTokens); |
457 | | if (osURL.empty()) |
458 | | { |
459 | | CPLError(CE_Failure, CPLE_IllegalArg, "Missing url parameter"); |
460 | | return pszFilename; |
461 | | } |
462 | | |
463 | | return osURL; |
464 | | } |
465 | | |
466 | | return pszFilename; |
467 | | } |
468 | | |
469 | | namespace cpl |
470 | | { |
471 | | |
472 | | /************************************************************************/ |
473 | | /* VSICurlHandle() */ |
474 | | /************************************************************************/ |
475 | | |
476 | | VSICurlHandle::VSICurlHandle(VSICurlFilesystemHandlerBase *poFSIn, |
477 | | const char *pszFilename, const char *pszURLIn) |
478 | | : poFS(poFSIn), m_osFilename(pszFilename), |
479 | | m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv(pszFilename)), |
480 | | m_oRetryParameters(m_aosHTTPOptions), |
481 | | m_bUseHead( |
482 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_USE_HEAD", "YES"))) |
483 | | { |
484 | | if (pszURLIn) |
485 | | { |
486 | | m_pszURL = CPLStrdup(pszURLIn); |
487 | | } |
488 | | else |
489 | | { |
490 | | char *pszPCCollection = nullptr; |
491 | | m_pszURL = |
492 | | CPLStrdup(VSICurlGetURLFromFilename( |
493 | | pszFilename, &m_oRetryParameters, &m_bUseHead, |
494 | | &m_bUseRedirectURLIfNoQueryStringParams, nullptr, |
495 | | nullptr, &m_aosHTTPOptions, |
496 | | &m_bPlanetaryComputerURLSigning, &pszPCCollection) |
497 | | .c_str()); |
498 | | if (pszPCCollection) |
499 | | m_osPlanetaryComputerCollection = pszPCCollection; |
500 | | CPLFree(pszPCCollection); |
501 | | } |
502 | | |
503 | | m_bCached = poFSIn->AllowCachedDataFor(pszFilename); |
504 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
505 | | } |
506 | | |
507 | | /************************************************************************/ |
508 | | /* ~VSICurlHandle() */ |
509 | | /************************************************************************/ |
510 | | |
511 | | VSICurlHandle::~VSICurlHandle() |
512 | | { |
513 | | if (m_oThreadAdviseRead.joinable()) |
514 | | { |
515 | | m_oThreadAdviseRead.join(); |
516 | | } |
517 | | if (m_hCurlMultiHandleForAdviseRead) |
518 | | { |
519 | | curl_multi_cleanup(m_hCurlMultiHandleForAdviseRead); |
520 | | } |
521 | | |
522 | | if (!m_bCached) |
523 | | { |
524 | | poFS->InvalidateCachedData(m_pszURL); |
525 | | poFS->InvalidateDirContent(CPLGetDirnameSafe(m_osFilename.c_str())); |
526 | | } |
527 | | CPLFree(m_pszURL); |
528 | | } |
529 | | |
530 | | /************************************************************************/ |
531 | | /* SetURL() */ |
532 | | /************************************************************************/ |
533 | | |
534 | | void VSICurlHandle::SetURL(const char *pszURLIn) |
535 | | { |
536 | | CPLFree(m_pszURL); |
537 | | m_pszURL = CPLStrdup(pszURLIn); |
538 | | } |
539 | | |
540 | | /************************************************************************/ |
541 | | /* InstallReadCbk() */ |
542 | | /************************************************************************/ |
543 | | |
544 | | int VSICurlHandle::InstallReadCbk(VSICurlReadCbkFunc pfnReadCbkIn, |
545 | | void *pfnUserDataIn, |
546 | | int bStopOnInterruptUntilUninstallIn) |
547 | | { |
548 | | if (pfnReadCbk != nullptr) |
549 | | return FALSE; |
550 | | |
551 | | pfnReadCbk = pfnReadCbkIn; |
552 | | pReadCbkUserData = pfnUserDataIn; |
553 | | bStopOnInterruptUntilUninstall = |
554 | | CPL_TO_BOOL(bStopOnInterruptUntilUninstallIn); |
555 | | bInterrupted = false; |
556 | | return TRUE; |
557 | | } |
558 | | |
559 | | /************************************************************************/ |
560 | | /* UninstallReadCbk() */ |
561 | | /************************************************************************/ |
562 | | |
563 | | int VSICurlHandle::UninstallReadCbk() |
564 | | { |
565 | | if (pfnReadCbk == nullptr) |
566 | | return FALSE; |
567 | | |
568 | | pfnReadCbk = nullptr; |
569 | | pReadCbkUserData = nullptr; |
570 | | bStopOnInterruptUntilUninstall = false; |
571 | | bInterrupted = false; |
572 | | return TRUE; |
573 | | } |
574 | | |
575 | | /************************************************************************/ |
576 | | /* Seek() */ |
577 | | /************************************************************************/ |
578 | | |
579 | | int VSICurlHandle::Seek(vsi_l_offset nOffset, int nWhence) |
580 | | { |
581 | | if (nWhence == SEEK_SET) |
582 | | { |
583 | | curOffset = nOffset; |
584 | | } |
585 | | else if (nWhence == SEEK_CUR) |
586 | | { |
587 | | curOffset = curOffset + nOffset; |
588 | | } |
589 | | else |
590 | | { |
591 | | curOffset = GetFileSize(false) + nOffset; |
592 | | } |
593 | | bEOF = false; |
594 | | return 0; |
595 | | } |
596 | | |
597 | | } // namespace cpl |
598 | | |
599 | | /************************************************************************/ |
600 | | /* VSICurlGetTimeStampFromRFC822DateTime() */ |
601 | | /************************************************************************/ |
602 | | |
603 | | static GIntBig VSICurlGetTimeStampFromRFC822DateTime(const char *pszDT) |
604 | | { |
605 | | // Sun, 03 Apr 2016 12:07:27 GMT |
606 | | if (strlen(pszDT) >= 5 && pszDT[3] == ',' && pszDT[4] == ' ') |
607 | | pszDT += 5; |
608 | | int nDay = 0; |
609 | | int nYear = 0; |
610 | | int nHour = 0; |
611 | | int nMinute = 0; |
612 | | int nSecond = 0; |
613 | | char szMonth[4] = {}; |
614 | | szMonth[3] = 0; |
615 | | if (sscanf(pszDT, "%02d %03s %04d %02d:%02d:%02d GMT", &nDay, szMonth, |
616 | | &nYear, &nHour, &nMinute, &nSecond) == 6) |
617 | | { |
618 | | static const char *const aszMonthStr[] = {"Jan", "Feb", "Mar", "Apr", |
619 | | "May", "Jun", "Jul", "Aug", |
620 | | "Sep", "Oct", "Nov", "Dec"}; |
621 | | |
622 | | int nMonthIdx0 = -1; |
623 | | for (int i = 0; i < 12; i++) |
624 | | { |
625 | | if (EQUAL(szMonth, aszMonthStr[i])) |
626 | | { |
627 | | nMonthIdx0 = i; |
628 | | break; |
629 | | } |
630 | | } |
631 | | if (nMonthIdx0 >= 0) |
632 | | { |
633 | | struct tm brokendowntime; |
634 | | brokendowntime.tm_year = nYear - 1900; |
635 | | brokendowntime.tm_mon = nMonthIdx0; |
636 | | brokendowntime.tm_mday = nDay; |
637 | | brokendowntime.tm_hour = nHour; |
638 | | brokendowntime.tm_min = nMinute; |
639 | | brokendowntime.tm_sec = nSecond; |
640 | | return CPLYMDHMSToUnixTime(&brokendowntime); |
641 | | } |
642 | | } |
643 | | return 0; |
644 | | } |
645 | | |
646 | | /************************************************************************/ |
647 | | /* VSICURLInitWriteFuncStruct() */ |
648 | | /************************************************************************/ |
649 | | |
650 | | void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp, |
651 | | VSICurlReadCbkFunc pfnReadCbk, |
652 | | void *pReadCbkUserData) |
653 | | { |
654 | | psStruct->pBuffer = nullptr; |
655 | | psStruct->nSize = 0; |
656 | | psStruct->bIsHTTP = false; |
657 | | psStruct->bMultiRange = false; |
658 | | psStruct->nStartOffset = 0; |
659 | | psStruct->nEndOffset = 0; |
660 | | psStruct->nHTTPCode = 0; |
661 | | psStruct->nFirstHTTPCode = 0; |
662 | | psStruct->nContentLength = 0; |
663 | | psStruct->bFoundContentRange = false; |
664 | | psStruct->bError = false; |
665 | | psStruct->bDetectRangeDownloadingError = true; |
666 | | psStruct->nTimestampDate = 0; |
667 | | |
668 | | psStruct->fp = fp; |
669 | | psStruct->pfnReadCbk = pfnReadCbk; |
670 | | psStruct->pReadCbkUserData = pReadCbkUserData; |
671 | | psStruct->bInterrupted = false; |
672 | | } |
673 | | |
674 | | /************************************************************************/ |
675 | | /* VSICurlHandleWriteFunc() */ |
676 | | /************************************************************************/ |
677 | | |
678 | | size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, |
679 | | void *req) |
680 | | { |
681 | | cpl::WriteFuncStruct *psStruct = static_cast<cpl::WriteFuncStruct *>(req); |
682 | | const size_t nSize = count * nmemb; |
683 | | |
684 | | if (psStruct->bInterrupted) |
685 | | { |
686 | | return 0; |
687 | | } |
688 | | |
689 | | char *pNewBuffer = static_cast<char *>( |
690 | | VSIRealloc(psStruct->pBuffer, psStruct->nSize + nSize + 1)); |
691 | | if (pNewBuffer) |
692 | | { |
693 | | psStruct->pBuffer = pNewBuffer; |
694 | | memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize); |
695 | | psStruct->pBuffer[psStruct->nSize + nSize] = '\0'; |
696 | | if (psStruct->bIsHTTP) |
697 | | { |
698 | | char *pszLine = psStruct->pBuffer + psStruct->nSize; |
699 | | if (STARTS_WITH_CI(pszLine, "HTTP/")) |
700 | | { |
701 | | char *pszSpace = strchr(pszLine, ' '); |
702 | | if (pszSpace) |
703 | | { |
704 | | const int nHTTPCode = atoi(pszSpace + 1); |
705 | | if (psStruct->nFirstHTTPCode == 0) |
706 | | psStruct->nFirstHTTPCode = nHTTPCode; |
707 | | psStruct->nHTTPCode = nHTTPCode; |
708 | | } |
709 | | } |
710 | | else if (STARTS_WITH_CI(pszLine, "Content-Length: ")) |
711 | | { |
712 | | psStruct->nContentLength = CPLScanUIntBig( |
713 | | pszLine + 16, static_cast<int>(strlen(pszLine + 16))); |
714 | | } |
715 | | else if (STARTS_WITH_CI(pszLine, "Content-Range: ")) |
716 | | { |
717 | | psStruct->bFoundContentRange = true; |
718 | | } |
719 | | else if (STARTS_WITH_CI(pszLine, "Date: ")) |
720 | | { |
721 | | CPLString osDate = pszLine + strlen("Date: "); |
722 | | size_t nSizeLine = osDate.size(); |
723 | | while (nSizeLine && (osDate[nSizeLine - 1] == '\r' || |
724 | | osDate[nSizeLine - 1] == '\n')) |
725 | | { |
726 | | osDate.resize(nSizeLine - 1); |
727 | | nSizeLine--; |
728 | | } |
729 | | osDate.Trim(); |
730 | | |
731 | | GIntBig nTimestampDate = |
732 | | VSICurlGetTimeStampFromRFC822DateTime(osDate.c_str()); |
733 | | #if DEBUG_VERBOSE |
734 | | CPLDebug("VSICURL", "Timestamp = " CPL_FRMT_GIB, |
735 | | nTimestampDate); |
736 | | #endif |
737 | | psStruct->nTimestampDate = nTimestampDate; |
738 | | } |
739 | | /*if( nSize > 2 && pszLine[nSize - 2] == '\r' && |
740 | | pszLine[nSize - 1] == '\n' ) |
741 | | { |
742 | | pszLine[nSize - 2] = 0; |
743 | | CPLDebug("VSICURL", "%s", pszLine); |
744 | | pszLine[nSize - 2] = '\r'; |
745 | | }*/ |
746 | | |
747 | | if (pszLine[0] == '\r' && pszLine[1] == '\n') |
748 | | { |
749 | | // Detect servers that don't support range downloading. |
750 | | if (psStruct->nHTTPCode == 200 && |
751 | | psStruct->bDetectRangeDownloadingError && |
752 | | !psStruct->bMultiRange && !psStruct->bFoundContentRange && |
753 | | (psStruct->nStartOffset != 0 || |
754 | | psStruct->nContentLength > |
755 | | 10 * (psStruct->nEndOffset - psStruct->nStartOffset + |
756 | | 1))) |
757 | | { |
758 | | CPLError(CE_Failure, CPLE_AppDefined, |
759 | | "Range downloading not supported by this " |
760 | | "server!"); |
761 | | psStruct->bError = true; |
762 | | return 0; |
763 | | } |
764 | | } |
765 | | } |
766 | | else |
767 | | { |
768 | | if (psStruct->pfnReadCbk) |
769 | | { |
770 | | if (!psStruct->pfnReadCbk(psStruct->fp, buffer, nSize, |
771 | | psStruct->pReadCbkUserData)) |
772 | | { |
773 | | psStruct->bInterrupted = true; |
774 | | return 0; |
775 | | } |
776 | | } |
777 | | } |
778 | | psStruct->nSize += nSize; |
779 | | return nmemb; |
780 | | } |
781 | | else |
782 | | { |
783 | | return 0; |
784 | | } |
785 | | } |
786 | | |
787 | | /************************************************************************/ |
788 | | /* VSICurlIsS3LikeSignedURL() */ |
789 | | /************************************************************************/ |
790 | | |
791 | | static bool VSICurlIsS3LikeSignedURL(const char *pszURL) |
792 | | { |
793 | | return ((strstr(pszURL, ".s3.amazonaws.com/") != nullptr || |
794 | | strstr(pszURL, ".s3.amazonaws.com:") != nullptr || |
795 | | strstr(pszURL, ".storage.googleapis.com/") != nullptr || |
796 | | strstr(pszURL, ".storage.googleapis.com:") != nullptr || |
797 | | strstr(pszURL, ".cloudfront.net/") != nullptr || |
798 | | strstr(pszURL, ".cloudfront.net:") != nullptr) && |
799 | | (strstr(pszURL, "&Signature=") != nullptr || |
800 | | strstr(pszURL, "?Signature=") != nullptr)) || |
801 | | strstr(pszURL, "&X-Amz-Signature=") != nullptr || |
802 | | strstr(pszURL, "?X-Amz-Signature=") != nullptr; |
803 | | } |
804 | | |
805 | | /************************************************************************/ |
806 | | /* VSICurlGetExpiresFromS3LikeSignedURL() */ |
807 | | /************************************************************************/ |
808 | | |
809 | | static GIntBig VSICurlGetExpiresFromS3LikeSignedURL(const char *pszURL) |
810 | | { |
811 | | const auto GetParamValue = [pszURL](const char *pszKey) -> const char * |
812 | | { |
813 | | for (const char *pszPrefix : {"&", "?"}) |
814 | | { |
815 | | std::string osNeedle(pszPrefix); |
816 | | osNeedle += pszKey; |
817 | | osNeedle += '='; |
818 | | const char *pszStr = strstr(pszURL, osNeedle.c_str()); |
819 | | if (pszStr) |
820 | | return pszStr + osNeedle.size(); |
821 | | } |
822 | | return nullptr; |
823 | | }; |
824 | | |
825 | | { |
826 | | // Expires= is a Unix timestamp |
827 | | const char *pszExpires = GetParamValue("Expires"); |
828 | | if (pszExpires != nullptr) |
829 | | return CPLAtoGIntBig(pszExpires); |
830 | | } |
831 | | |
832 | | // X-Amz-Expires= is a delay, to be combined with X-Amz-Date= |
833 | | const char *pszAmzExpires = GetParamValue("X-Amz-Expires"); |
834 | | if (pszAmzExpires == nullptr) |
835 | | return 0; |
836 | | const int nDelay = atoi(pszAmzExpires); |
837 | | |
838 | | const char *pszAmzDate = GetParamValue("X-Amz-Date"); |
839 | | if (pszAmzDate == nullptr) |
840 | | return 0; |
841 | | // pszAmzDate should be YYYYMMDDTHHMMSSZ |
842 | | if (strlen(pszAmzDate) < strlen("YYYYMMDDTHHMMSSZ")) |
843 | | return 0; |
844 | | if (pszAmzDate[strlen("YYYYMMDDTHHMMSSZ") - 1] != 'Z') |
845 | | return 0; |
846 | | struct tm brokendowntime; |
847 | | brokendowntime.tm_year = |
848 | | atoi(std::string(pszAmzDate).substr(0, 4).c_str()) - 1900; |
849 | | brokendowntime.tm_mon = |
850 | | atoi(std::string(pszAmzDate).substr(4, 2).c_str()) - 1; |
851 | | brokendowntime.tm_mday = atoi(std::string(pszAmzDate).substr(6, 2).c_str()); |
852 | | brokendowntime.tm_hour = atoi(std::string(pszAmzDate).substr(9, 2).c_str()); |
853 | | brokendowntime.tm_min = atoi(std::string(pszAmzDate).substr(11, 2).c_str()); |
854 | | brokendowntime.tm_sec = atoi(std::string(pszAmzDate).substr(13, 2).c_str()); |
855 | | return CPLYMDHMSToUnixTime(&brokendowntime) + nDelay; |
856 | | } |
857 | | |
858 | | /************************************************************************/ |
859 | | /* VSICURLMultiPerform() */ |
860 | | /************************************************************************/ |
861 | | |
862 | | void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle, |
863 | | std::atomic<bool> *pbInterrupt) |
864 | | { |
865 | | int repeats = 0; |
866 | | |
867 | | if (hEasyHandle) |
868 | | curl_multi_add_handle(hCurlMultiHandle, hEasyHandle); |
869 | | |
870 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
871 | | while (true) |
872 | | { |
873 | | int still_running; |
874 | | while (curl_multi_perform(hCurlMultiHandle, &still_running) == |
875 | | CURLM_CALL_MULTI_PERFORM) |
876 | | { |
877 | | // loop |
878 | | } |
879 | | if (!still_running) |
880 | | { |
881 | | break; |
882 | | } |
883 | | |
884 | | #ifdef undef |
885 | | CURLMsg *msg; |
886 | | do |
887 | | { |
888 | | int msgq = 0; |
889 | | msg = curl_multi_info_read(hCurlMultiHandle, &msgq); |
890 | | if (msg && (msg->msg == CURLMSG_DONE)) |
891 | | { |
892 | | CURL *e = msg->easy_handle; |
893 | | } |
894 | | } while (msg); |
895 | | #endif |
896 | | |
897 | | CPLMultiPerformWait(hCurlMultiHandle, repeats); |
898 | | |
899 | | if (pbInterrupt && *pbInterrupt) |
900 | | break; |
901 | | } |
902 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
903 | | |
904 | | if (hEasyHandle) |
905 | | curl_multi_remove_handle(hCurlMultiHandle, hEasyHandle); |
906 | | } |
907 | | |
908 | | /************************************************************************/ |
909 | | /* VSICurlDummyWriteFunc() */ |
910 | | /************************************************************************/ |
911 | | |
912 | | static size_t VSICurlDummyWriteFunc(void *, size_t, size_t, void *) |
913 | | { |
914 | | return 0; |
915 | | } |
916 | | |
917 | | /************************************************************************/ |
918 | | /* VSICURLResetHeaderAndWriterFunctions() */ |
919 | | /************************************************************************/ |
920 | | |
921 | | void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle) |
922 | | { |
923 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
924 | | VSICurlDummyWriteFunc); |
925 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
926 | | VSICurlDummyWriteFunc); |
927 | | } |
928 | | |
929 | | /************************************************************************/ |
930 | | /* Iso8601ToUnixTime() */ |
931 | | /************************************************************************/ |
932 | | |
933 | | static bool Iso8601ToUnixTime(const char *pszDT, GIntBig *pnUnixTime) |
934 | | { |
935 | | int nYear; |
936 | | int nMonth; |
937 | | int nDay; |
938 | | int nHour; |
939 | | int nMinute; |
940 | | int nSecond; |
941 | | if (sscanf(pszDT, "%04d-%02d-%02dT%02d:%02d:%02d", &nYear, &nMonth, &nDay, |
942 | | &nHour, &nMinute, &nSecond) == 6) |
943 | | { |
944 | | struct tm brokendowntime; |
945 | | brokendowntime.tm_year = nYear - 1900; |
946 | | brokendowntime.tm_mon = nMonth - 1; |
947 | | brokendowntime.tm_mday = nDay; |
948 | | brokendowntime.tm_hour = nHour; |
949 | | brokendowntime.tm_min = nMinute; |
950 | | brokendowntime.tm_sec = nSecond; |
951 | | *pnUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
952 | | return true; |
953 | | } |
954 | | return false; |
955 | | } |
956 | | |
957 | | namespace cpl |
958 | | { |
959 | | |
960 | | /************************************************************************/ |
961 | | /* ManagePlanetaryComputerSigning() */ |
962 | | /************************************************************************/ |
963 | | |
964 | | void VSICurlHandle::ManagePlanetaryComputerSigning() const |
965 | | { |
966 | | // Take global lock |
967 | | static std::mutex goMutex; |
968 | | std::lock_guard<std::mutex> oLock(goMutex); |
969 | | |
970 | | struct PCSigningInfo |
971 | | { |
972 | | std::string osQueryString{}; |
973 | | GIntBig nExpireTimestamp = 0; |
974 | | }; |
975 | | |
976 | | PCSigningInfo sSigningInfo; |
977 | | constexpr int knExpirationDelayMargin = 60; |
978 | | |
979 | | if (!m_osPlanetaryComputerCollection.empty()) |
980 | | { |
981 | | // key is the name of a collection |
982 | | static lru11::Cache<std::string, PCSigningInfo> goCacheCollection{1024}; |
983 | | |
984 | | if (goCacheCollection.tryGet(m_osPlanetaryComputerCollection, |
985 | | sSigningInfo) && |
986 | | time(nullptr) + knExpirationDelayMargin <= |
987 | | sSigningInfo.nExpireTimestamp) |
988 | | { |
989 | | m_osQueryString = sSigningInfo.osQueryString; |
990 | | } |
991 | | else |
992 | | { |
993 | | const auto psResult = |
994 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
995 | | "VSICURL_PC_SAS_TOKEN_URL", |
996 | | "https://planetarycomputer.microsoft.com/api/" |
997 | | "sas/v1/token/")) + |
998 | | m_osPlanetaryComputerCollection) |
999 | | .c_str(), |
1000 | | nullptr); |
1001 | | if (psResult) |
1002 | | { |
1003 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1004 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1005 | | const char *pszToken = aosKeyVals.FetchNameValue("token"); |
1006 | | if (pszToken) |
1007 | | { |
1008 | | m_osQueryString = '?'; |
1009 | | m_osQueryString += pszToken; |
1010 | | |
1011 | | sSigningInfo.osQueryString = m_osQueryString; |
1012 | | sSigningInfo.nExpireTimestamp = 0; |
1013 | | const char *pszExpiry = |
1014 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1015 | | if (pszExpiry) |
1016 | | { |
1017 | | Iso8601ToUnixTime(pszExpiry, |
1018 | | &sSigningInfo.nExpireTimestamp); |
1019 | | } |
1020 | | goCacheCollection.insert(m_osPlanetaryComputerCollection, |
1021 | | sSigningInfo); |
1022 | | |
1023 | | CPLDebug("VSICURL", "Got token from Planetary Computer: %s", |
1024 | | m_osQueryString.c_str()); |
1025 | | } |
1026 | | CPLHTTPDestroyResult(psResult); |
1027 | | } |
1028 | | } |
1029 | | } |
1030 | | else |
1031 | | { |
1032 | | // key is a URL |
1033 | | static lru11::Cache<std::string, PCSigningInfo> goCacheURL{1024}; |
1034 | | |
1035 | | if (goCacheURL.tryGet(m_pszURL, sSigningInfo) && |
1036 | | time(nullptr) + knExpirationDelayMargin <= |
1037 | | sSigningInfo.nExpireTimestamp) |
1038 | | { |
1039 | | m_osQueryString = sSigningInfo.osQueryString; |
1040 | | } |
1041 | | else |
1042 | | { |
1043 | | const auto psResult = |
1044 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
1045 | | "VSICURL_PC_SAS_SIGN_HREF_URL", |
1046 | | "https://planetarycomputer.microsoft.com/api/" |
1047 | | "sas/v1/sign?href=")) + |
1048 | | m_pszURL) |
1049 | | .c_str(), |
1050 | | nullptr); |
1051 | | if (psResult) |
1052 | | { |
1053 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1054 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1055 | | const char *pszHref = aosKeyVals.FetchNameValue("href"); |
1056 | | if (pszHref && STARTS_WITH(pszHref, m_pszURL)) |
1057 | | { |
1058 | | m_osQueryString = pszHref + strlen(m_pszURL); |
1059 | | |
1060 | | sSigningInfo.osQueryString = m_osQueryString; |
1061 | | sSigningInfo.nExpireTimestamp = 0; |
1062 | | const char *pszExpiry = |
1063 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1064 | | if (pszExpiry) |
1065 | | { |
1066 | | Iso8601ToUnixTime(pszExpiry, |
1067 | | &sSigningInfo.nExpireTimestamp); |
1068 | | } |
1069 | | goCacheURL.insert(m_pszURL, sSigningInfo); |
1070 | | |
1071 | | CPLDebug("VSICURL", |
1072 | | "Got signature from Planetary Computer: %s", |
1073 | | m_osQueryString.c_str()); |
1074 | | } |
1075 | | CPLHTTPDestroyResult(psResult); |
1076 | | } |
1077 | | } |
1078 | | } |
1079 | | } |
1080 | | |
1081 | | /************************************************************************/ |
1082 | | /* UpdateQueryString() */ |
1083 | | /************************************************************************/ |
1084 | | |
1085 | | void VSICurlHandle::UpdateQueryString() const |
1086 | | { |
1087 | | if (m_bPlanetaryComputerURLSigning) |
1088 | | { |
1089 | | ManagePlanetaryComputerSigning(); |
1090 | | } |
1091 | | else |
1092 | | { |
1093 | | const char *pszQueryString = VSIGetPathSpecificOption( |
1094 | | m_osFilename.c_str(), "VSICURL_QUERY_STRING", nullptr); |
1095 | | if (pszQueryString) |
1096 | | { |
1097 | | if (m_osFilename.back() == '?') |
1098 | | { |
1099 | | if (pszQueryString[0] == '?') |
1100 | | m_osQueryString = pszQueryString + 1; |
1101 | | else |
1102 | | m_osQueryString = pszQueryString; |
1103 | | } |
1104 | | else |
1105 | | { |
1106 | | if (pszQueryString[0] == '?') |
1107 | | m_osQueryString = pszQueryString; |
1108 | | else |
1109 | | { |
1110 | | m_osQueryString = "?"; |
1111 | | m_osQueryString.append(pszQueryString); |
1112 | | } |
1113 | | } |
1114 | | } |
1115 | | } |
1116 | | } |
1117 | | |
1118 | | /************************************************************************/ |
1119 | | /* GetFileSizeOrHeaders() */ |
1120 | | /************************************************************************/ |
1121 | | |
1122 | | vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, |
1123 | | bool bGetHeaders) |
1124 | | { |
1125 | | if (oFileProp.bHasComputedFileSize && !bGetHeaders) |
1126 | | return oFileProp.fileSize; |
1127 | | |
1128 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
1129 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
1130 | | NetworkStatisticsAction oContextAction("GetFileSize"); |
1131 | | |
1132 | | oFileProp.bHasComputedFileSize = true; |
1133 | | |
1134 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
1135 | | |
1136 | | UpdateQueryString(); |
1137 | | |
1138 | | std::string osURL(m_pszURL + m_osQueryString); |
1139 | | bool bRetryWithGet = false; |
1140 | | bool bS3LikeRedirect = false; |
1141 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
1142 | | |
1143 | | retry: |
1144 | | CURL *hCurlHandle = curl_easy_init(); |
1145 | | |
1146 | | struct curl_slist *headers = |
1147 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), m_aosHTTPOptions.List()); |
1148 | | |
1149 | | WriteFuncStruct sWriteFuncHeaderData; |
1150 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
1151 | | nullptr); |
1152 | | sWriteFuncHeaderData.bDetectRangeDownloadingError = false; |
1153 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(osURL.c_str(), "http"); |
1154 | | |
1155 | | WriteFuncStruct sWriteFuncData; |
1156 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
1157 | | |
1158 | | std::string osVerb; |
1159 | | std::string osRange; // leave in this scope ! |
1160 | | int nRoundedBufSize = 0; |
1161 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
1162 | | if (UseLimitRangeGetInsteadOfHead()) |
1163 | | { |
1164 | | osVerb = "GET"; |
1165 | | const int nBufSize = std::max( |
1166 | | 1024, std::min(10 * 1024 * 1024, |
1167 | | atoi(CPLGetConfigOption( |
1168 | | "GDAL_INGESTED_BYTES_AT_OPEN", "1024")))); |
1169 | | nRoundedBufSize = cpl::div_round_up(nBufSize, knDOWNLOAD_CHUNK_SIZE) * |
1170 | | knDOWNLOAD_CHUNK_SIZE; |
1171 | | |
1172 | | // so it gets included in Azure signature |
1173 | | osRange = CPLSPrintf("Range: bytes=0-%d", nRoundedBufSize - 1); |
1174 | | headers = curl_slist_append(headers, osRange.c_str()); |
1175 | | } |
1176 | | // HACK for mbtiles driver: http://a.tiles.mapbox.com/v3/ doesn't accept |
1177 | | // HEAD, as it is a redirect to AWS S3 signed URL, but those are only valid |
1178 | | // for a given type of HTTP request, and thus GET. This is valid for any |
1179 | | // signed URL for AWS S3. |
1180 | | else if (bRetryWithGet || |
1181 | | strstr(osURL.c_str(), ".tiles.mapbox.com/") != nullptr || |
1182 | | VSICurlIsS3LikeSignedURL(osURL.c_str()) || !m_bUseHead) |
1183 | | { |
1184 | | sWriteFuncData.bInterrupted = true; |
1185 | | osVerb = "GET"; |
1186 | | } |
1187 | | else |
1188 | | { |
1189 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1); |
1190 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 0); |
1191 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 1); |
1192 | | osVerb = "HEAD"; |
1193 | | } |
1194 | | |
1195 | | if (!AllowAutomaticRedirection()) |
1196 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
1197 | | |
1198 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
1199 | | &sWriteFuncHeaderData); |
1200 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1201 | | VSICurlHandleWriteFunc); |
1202 | | |
1203 | | // Bug with older curl versions (<=7.16.4) and FTP. |
1204 | | // See http://curl.haxx.se/mail/lib-2007-08/0312.html |
1205 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
1206 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1207 | | VSICurlHandleWriteFunc); |
1208 | | |
1209 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
1210 | | szCurlErrBuf[0] = '\0'; |
1211 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
1212 | | |
1213 | | headers = GetCurlHeaders(osVerb, headers); |
1214 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
1215 | | |
1216 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
1217 | | |
1218 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); |
1219 | | |
1220 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
1221 | | |
1222 | | curl_slist_free_all(headers); |
1223 | | |
1224 | | oFileProp.eExists = EXIST_UNKNOWN; |
1225 | | |
1226 | | long mtime = 0; |
1227 | | curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME, &mtime); |
1228 | | |
1229 | | if (osVerb == "GET") |
1230 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
1231 | | else |
1232 | | NetworkStatisticsLogger::LogHEAD(); |
1233 | | |
1234 | | if (STARTS_WITH(osURL.c_str(), "ftp")) |
1235 | | { |
1236 | | if (sWriteFuncData.pBuffer != nullptr) |
1237 | | { |
1238 | | const char *pszContentLength = |
1239 | | strstr(const_cast<const char *>(sWriteFuncData.pBuffer), |
1240 | | "Content-Length: "); |
1241 | | if (pszContentLength) |
1242 | | { |
1243 | | pszContentLength += strlen("Content-Length: "); |
1244 | | oFileProp.eExists = EXIST_YES; |
1245 | | oFileProp.fileSize = |
1246 | | CPLScanUIntBig(pszContentLength, |
1247 | | static_cast<int>(strlen(pszContentLength))); |
1248 | | if (ENABLE_DEBUG) |
1249 | | CPLDebug(poFS->GetDebugKey(), |
1250 | | "GetFileSize(%s)=" CPL_FRMT_GUIB, osURL.c_str(), |
1251 | | oFileProp.fileSize); |
1252 | | } |
1253 | | } |
1254 | | } |
1255 | | |
1256 | | double dfSize = 0; |
1257 | | long response_code = -1; |
1258 | | if (oFileProp.eExists != EXIST_YES) |
1259 | | { |
1260 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
1261 | | |
1262 | | bool bAlreadyLogged = false; |
1263 | | if (response_code >= 400 && szCurlErrBuf[0] == '\0') |
1264 | | { |
1265 | | const bool bLogResponse = |
1266 | | CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")); |
1267 | | if (bLogResponse && sWriteFuncData.pBuffer) |
1268 | | { |
1269 | | const char *pszErrorMsg = |
1270 | | static_cast<const char *>(sWriteFuncData.pBuffer); |
1271 | | bAlreadyLogged = true; |
1272 | | CPLDebug( |
1273 | | poFS->GetDebugKey(), |
1274 | | "GetFileSize(%s): response_code=%d, server error msg=%s", |
1275 | | osURL.c_str(), static_cast<int>(response_code), |
1276 | | pszErrorMsg[0] ? pszErrorMsg : "(no message provided)"); |
1277 | | } |
1278 | | } |
1279 | | else if (szCurlErrBuf[0] != '\0') |
1280 | | { |
1281 | | bAlreadyLogged = true; |
1282 | | CPLDebug(poFS->GetDebugKey(), |
1283 | | "GetFileSize(%s): response_code=%d, curl error msg=%s", |
1284 | | osURL.c_str(), static_cast<int>(response_code), |
1285 | | szCurlErrBuf); |
1286 | | } |
1287 | | |
1288 | | std::string osEffectiveURL; |
1289 | | { |
1290 | | char *pszEffectiveURL = nullptr; |
1291 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
1292 | | &pszEffectiveURL); |
1293 | | if (pszEffectiveURL) |
1294 | | osEffectiveURL = pszEffectiveURL; |
1295 | | } |
1296 | | |
1297 | | if (!osEffectiveURL.empty() && |
1298 | | strstr(osEffectiveURL.c_str(), osURL.c_str()) == nullptr) |
1299 | | { |
1300 | | // Moved permanently ? |
1301 | | if (sWriteFuncHeaderData.nFirstHTTPCode == 301 || |
1302 | | (m_bUseRedirectURLIfNoQueryStringParams && |
1303 | | osEffectiveURL.find('?') == std::string::npos)) |
1304 | | { |
1305 | | CPLDebug(poFS->GetDebugKey(), |
1306 | | "Using effective URL %s permanently", |
1307 | | osEffectiveURL.c_str()); |
1308 | | oFileProp.osRedirectURL = osEffectiveURL; |
1309 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1310 | | } |
1311 | | else |
1312 | | { |
1313 | | CPLDebug(poFS->GetDebugKey(), |
1314 | | "Using effective URL %s temporarily", |
1315 | | osEffectiveURL.c_str()); |
1316 | | } |
1317 | | |
1318 | | // Is this is a redirect to a S3 URL? |
1319 | | if (VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
1320 | | !VSICurlIsS3LikeSignedURL(osURL.c_str())) |
1321 | | { |
1322 | | // Note that this is a redirect as we won't notice after the |
1323 | | // retry. |
1324 | | bS3LikeRedirect = true; |
1325 | | |
1326 | | if (!bRetryWithGet && osVerb == "HEAD" && response_code == 403) |
1327 | | { |
1328 | | CPLDebug(poFS->GetDebugKey(), |
1329 | | "Redirected to a AWS S3 signed URL. Retrying " |
1330 | | "with GET request instead of HEAD since the URL " |
1331 | | "might be valid only for GET"); |
1332 | | bRetryWithGet = true; |
1333 | | osURL = std::move(osEffectiveURL); |
1334 | | CPLFree(sWriteFuncData.pBuffer); |
1335 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1336 | | curl_easy_cleanup(hCurlHandle); |
1337 | | goto retry; |
1338 | | } |
1339 | | } |
1340 | | } |
1341 | | |
1342 | | if (bS3LikeRedirect && response_code >= 200 && response_code < 300 && |
1343 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
1344 | | !osEffectiveURL.empty() && |
1345 | | CPLTestBool( |
1346 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
1347 | | { |
1348 | | const GIntBig nExpireTimestamp = |
1349 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
1350 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
1351 | | { |
1352 | | const int nValidity = static_cast<int>( |
1353 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
1354 | | CPLDebug(poFS->GetDebugKey(), |
1355 | | "Will use redirect URL for the next %d seconds", |
1356 | | nValidity); |
1357 | | // As our local clock might not be in sync with server clock, |
1358 | | // figure out the expiration timestamp in local time |
1359 | | oFileProp.bS3LikeRedirect = true; |
1360 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
1361 | | oFileProp.osRedirectURL = osEffectiveURL; |
1362 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1363 | | } |
1364 | | } |
1365 | | |
1366 | | if (response_code < 400) |
1367 | | { |
1368 | | curl_off_t nSizeTmp = 0; |
1369 | | const CURLcode code = curl_easy_getinfo( |
1370 | | hCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &nSizeTmp); |
1371 | | CPL_IGNORE_RET_VAL(dfSize); |
1372 | | dfSize = static_cast<double>(nSizeTmp); |
1373 | | if (code == 0) |
1374 | | { |
1375 | | oFileProp.eExists = EXIST_YES; |
1376 | | if (dfSize < 0) |
1377 | | { |
1378 | | if (osVerb == "HEAD" && !bRetryWithGet && |
1379 | | response_code == 200) |
1380 | | { |
1381 | | CPLDebug(poFS->GetDebugKey(), |
1382 | | "HEAD did not provide file size. Retrying " |
1383 | | "with GET"); |
1384 | | bRetryWithGet = true; |
1385 | | CPLFree(sWriteFuncData.pBuffer); |
1386 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1387 | | curl_easy_cleanup(hCurlHandle); |
1388 | | goto retry; |
1389 | | } |
1390 | | oFileProp.fileSize = 0; |
1391 | | } |
1392 | | else |
1393 | | oFileProp.fileSize = static_cast<GUIntBig>(dfSize); |
1394 | | } |
1395 | | } |
1396 | | |
1397 | | if (sWriteFuncHeaderData.pBuffer != nullptr && |
1398 | | (response_code == 200 || response_code == 206)) |
1399 | | { |
1400 | | { |
1401 | | char **papszHeaders = |
1402 | | CSLTokenizeString2(sWriteFuncHeaderData.pBuffer, "\r\n", 0); |
1403 | | for (int i = 0; papszHeaders[i]; ++i) |
1404 | | { |
1405 | | char *pszKey = nullptr; |
1406 | | const char *pszValue = |
1407 | | CPLParseNameValue(papszHeaders[i], &pszKey); |
1408 | | if (pszKey && pszValue) |
1409 | | { |
1410 | | if (bGetHeaders) |
1411 | | { |
1412 | | m_aosHeaders.SetNameValue(pszKey, pszValue); |
1413 | | } |
1414 | | if (EQUAL(pszKey, "Cache-Control") && |
1415 | | EQUAL(pszValue, "no-cache") && |
1416 | | CPLTestBool(CPLGetConfigOption( |
1417 | | "CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES"))) |
1418 | | { |
1419 | | m_bCached = false; |
1420 | | } |
1421 | | |
1422 | | else if (EQUAL(pszKey, "ETag")) |
1423 | | { |
1424 | | std::string osValue(pszValue); |
1425 | | if (osValue.size() >= 2 && osValue.front() == '"' && |
1426 | | osValue.back() == '"') |
1427 | | osValue = osValue.substr(1, osValue.size() - 2); |
1428 | | oFileProp.ETag = std::move(osValue); |
1429 | | } |
1430 | | |
1431 | | // Azure Data Lake Storage |
1432 | | else if (EQUAL(pszKey, "x-ms-resource-type")) |
1433 | | { |
1434 | | if (EQUAL(pszValue, "file")) |
1435 | | { |
1436 | | oFileProp.nMode |= S_IFREG; |
1437 | | } |
1438 | | else if (EQUAL(pszValue, "directory")) |
1439 | | { |
1440 | | oFileProp.bIsDirectory = true; |
1441 | | oFileProp.nMode |= S_IFDIR; |
1442 | | } |
1443 | | } |
1444 | | else if (EQUAL(pszKey, "x-ms-permissions")) |
1445 | | { |
1446 | | oFileProp.nMode |= |
1447 | | VSICurlParseUnixPermissions(pszValue); |
1448 | | } |
1449 | | |
1450 | | // https://overturemapswestus2.blob.core.windows.net/release/2024-11-13.0/theme%3Ddivisions/type%3Ddivision_area |
1451 | | // returns a x-ms-meta-hdi_isfolder: true header |
1452 | | else if (EQUAL(pszKey, "x-ms-meta-hdi_isfolder") && |
1453 | | EQUAL(pszValue, "true")) |
1454 | | { |
1455 | | oFileProp.bIsAzureFolder = true; |
1456 | | oFileProp.bIsDirectory = true; |
1457 | | oFileProp.nMode |= S_IFDIR; |
1458 | | } |
1459 | | } |
1460 | | CPLFree(pszKey); |
1461 | | } |
1462 | | CSLDestroy(papszHeaders); |
1463 | | } |
1464 | | } |
1465 | | |
1466 | | if (UseLimitRangeGetInsteadOfHead() && response_code == 206) |
1467 | | { |
1468 | | oFileProp.eExists = EXIST_NO; |
1469 | | oFileProp.fileSize = 0; |
1470 | | if (sWriteFuncHeaderData.pBuffer != nullptr) |
1471 | | { |
1472 | | const char *pszContentRange = strstr( |
1473 | | sWriteFuncHeaderData.pBuffer, "Content-Range: bytes "); |
1474 | | if (pszContentRange == nullptr) |
1475 | | pszContentRange = strstr(sWriteFuncHeaderData.pBuffer, |
1476 | | "content-range: bytes "); |
1477 | | if (pszContentRange) |
1478 | | pszContentRange = strchr(pszContentRange, '/'); |
1479 | | if (pszContentRange) |
1480 | | { |
1481 | | oFileProp.eExists = EXIST_YES; |
1482 | | oFileProp.fileSize = static_cast<GUIntBig>( |
1483 | | CPLAtoGIntBig(pszContentRange + 1)); |
1484 | | } |
1485 | | |
1486 | | // Add first bytes to cache |
1487 | | if (sWriteFuncData.pBuffer != nullptr) |
1488 | | { |
1489 | | size_t nOffset = 0; |
1490 | | while (nOffset < sWriteFuncData.nSize) |
1491 | | { |
1492 | | const size_t nToCache = |
1493 | | std::min<size_t>(sWriteFuncData.nSize - nOffset, |
1494 | | knDOWNLOAD_CHUNK_SIZE); |
1495 | | poFS->AddRegion(m_pszURL, nOffset, nToCache, |
1496 | | sWriteFuncData.pBuffer + nOffset); |
1497 | | nOffset += nToCache; |
1498 | | } |
1499 | | } |
1500 | | } |
1501 | | } |
1502 | | else if (IsDirectoryFromExists(osVerb.c_str(), |
1503 | | static_cast<int>(response_code))) |
1504 | | { |
1505 | | oFileProp.eExists = EXIST_YES; |
1506 | | oFileProp.fileSize = 0; |
1507 | | oFileProp.bIsDirectory = true; |
1508 | | } |
1509 | | // 405 = Method not allowed |
1510 | | else if (response_code == 405 && !bRetryWithGet && osVerb == "HEAD") |
1511 | | { |
1512 | | CPLDebug(poFS->GetDebugKey(), |
1513 | | "HEAD not allowed. Retrying with GET"); |
1514 | | bRetryWithGet = true; |
1515 | | CPLFree(sWriteFuncData.pBuffer); |
1516 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1517 | | curl_easy_cleanup(hCurlHandle); |
1518 | | goto retry; |
1519 | | } |
1520 | | else if (response_code == 416) |
1521 | | { |
1522 | | oFileProp.eExists = EXIST_YES; |
1523 | | oFileProp.fileSize = 0; |
1524 | | } |
1525 | | else if (response_code != 200) |
1526 | | { |
1527 | | // Look if we should attempt a retry |
1528 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
1529 | | sWriteFuncHeaderData.pBuffer, |
1530 | | szCurlErrBuf)) |
1531 | | { |
1532 | | CPLError(CE_Warning, CPLE_AppDefined, |
1533 | | "HTTP error code: %d - %s. " |
1534 | | "Retrying again in %.1f secs", |
1535 | | static_cast<int>(response_code), m_pszURL, |
1536 | | oRetryContext.GetCurrentDelay()); |
1537 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
1538 | | CPLFree(sWriteFuncData.pBuffer); |
1539 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1540 | | curl_easy_cleanup(hCurlHandle); |
1541 | | goto retry; |
1542 | | } |
1543 | | |
1544 | | if (sWriteFuncData.pBuffer != nullptr) |
1545 | | { |
1546 | | if (UseLimitRangeGetInsteadOfHead() && |
1547 | | CanRestartOnError(sWriteFuncData.pBuffer, |
1548 | | sWriteFuncHeaderData.pBuffer, bSetError)) |
1549 | | { |
1550 | | oFileProp.bHasComputedFileSize = false; |
1551 | | CPLFree(sWriteFuncData.pBuffer); |
1552 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1553 | | curl_easy_cleanup(hCurlHandle); |
1554 | | return GetFileSizeOrHeaders(bSetError, bGetHeaders); |
1555 | | } |
1556 | | else |
1557 | | { |
1558 | | CPL_IGNORE_RET_VAL(CanRestartOnError( |
1559 | | sWriteFuncData.pBuffer, sWriteFuncHeaderData.pBuffer, |
1560 | | bSetError)); |
1561 | | } |
1562 | | } |
1563 | | |
1564 | | // If there was no VSI error thrown in the process, |
1565 | | // fail by reporting the HTTP response code. |
1566 | | if (bSetError && VSIGetLastErrorNo() == 0) |
1567 | | { |
1568 | | if (strlen(szCurlErrBuf) > 0) |
1569 | | { |
1570 | | if (response_code == 0) |
1571 | | { |
1572 | | VSIError(VSIE_HttpError, "CURL error: %s", |
1573 | | szCurlErrBuf); |
1574 | | } |
1575 | | else |
1576 | | { |
1577 | | VSIError(VSIE_HttpError, "HTTP response code: %d - %s", |
1578 | | static_cast<int>(response_code), szCurlErrBuf); |
1579 | | } |
1580 | | } |
1581 | | else |
1582 | | { |
1583 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
1584 | | static_cast<int>(response_code)); |
1585 | | } |
1586 | | } |
1587 | | else |
1588 | | { |
1589 | | if (response_code != 400 && response_code != 404) |
1590 | | { |
1591 | | CPLError(CE_Warning, CPLE_AppDefined, |
1592 | | "HTTP response code on %s: %d", osURL.c_str(), |
1593 | | static_cast<int>(response_code)); |
1594 | | } |
1595 | | // else a CPLDebug() is emitted below |
1596 | | } |
1597 | | |
1598 | | oFileProp.eExists = EXIST_NO; |
1599 | | oFileProp.nHTTPCode = static_cast<int>(response_code); |
1600 | | oFileProp.fileSize = 0; |
1601 | | } |
1602 | | else if (sWriteFuncData.pBuffer != nullptr) |
1603 | | { |
1604 | | ProcessGetFileSizeResult( |
1605 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer)); |
1606 | | } |
1607 | | |
1608 | | // Try to guess if this is a directory. Generally if this is a |
1609 | | // directory, curl will retry with an URL with slash added. |
1610 | | if (!osEffectiveURL.empty() && |
1611 | | strncmp(osURL.c_str(), osEffectiveURL.c_str(), osURL.size()) == 0 && |
1612 | | osEffectiveURL[osURL.size()] == '/' && |
1613 | | oFileProp.eExists != EXIST_NO) |
1614 | | { |
1615 | | oFileProp.eExists = EXIST_YES; |
1616 | | oFileProp.fileSize = 0; |
1617 | | oFileProp.bIsDirectory = true; |
1618 | | } |
1619 | | else if (osURL.back() == '/') |
1620 | | { |
1621 | | oFileProp.bIsDirectory = true; |
1622 | | } |
1623 | | |
1624 | | if (!bAlreadyLogged) |
1625 | | { |
1626 | | CPLDebug(poFS->GetDebugKey(), |
1627 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
1628 | | osURL.c_str(), oFileProp.fileSize, |
1629 | | static_cast<int>(response_code)); |
1630 | | } |
1631 | | } |
1632 | | |
1633 | | CPLFree(sWriteFuncData.pBuffer); |
1634 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1635 | | curl_easy_cleanup(hCurlHandle); |
1636 | | |
1637 | | oFileProp.bHasComputedFileSize = true; |
1638 | | if (mtime > 0) |
1639 | | oFileProp.mTime = mtime; |
1640 | | // Do not update cached file properties if cURL returned a non-HTTP error |
1641 | | if (response_code != 0) |
1642 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1643 | | |
1644 | | return oFileProp.fileSize; |
1645 | | } |
1646 | | |
1647 | | /************************************************************************/ |
1648 | | /* Exists() */ |
1649 | | /************************************************************************/ |
1650 | | |
1651 | | bool VSICurlHandle::Exists(bool bSetError) |
1652 | | { |
1653 | | if (oFileProp.eExists == EXIST_UNKNOWN) |
1654 | | { |
1655 | | GetFileSize(bSetError); |
1656 | | } |
1657 | | else if (oFileProp.eExists == EXIST_NO) |
1658 | | { |
1659 | | // If there was no VSI error thrown in the process, |
1660 | | // and we know the HTTP error code of the first request where the |
1661 | | // file could not be retrieved, fail by reporting the HTTP code. |
1662 | | if (bSetError && VSIGetLastErrorNo() == 0 && oFileProp.nHTTPCode) |
1663 | | { |
1664 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
1665 | | oFileProp.nHTTPCode); |
1666 | | } |
1667 | | } |
1668 | | |
1669 | | return oFileProp.eExists == EXIST_YES; |
1670 | | } |
1671 | | |
1672 | | /************************************************************************/ |
1673 | | /* Tell() */ |
1674 | | /************************************************************************/ |
1675 | | |
1676 | | vsi_l_offset VSICurlHandle::Tell() |
1677 | | { |
1678 | | return curOffset; |
1679 | | } |
1680 | | |
1681 | | /************************************************************************/ |
1682 | | /* GetRedirectURLIfValid() */ |
1683 | | /************************************************************************/ |
1684 | | |
1685 | | std::string |
1686 | | VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired, |
1687 | | CPLStringList &aosHTTPOptions) const |
1688 | | { |
1689 | | bHasExpired = false; |
1690 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
1691 | | |
1692 | | std::string osURL(m_pszURL + m_osQueryString); |
1693 | | if (oFileProp.bS3LikeRedirect) |
1694 | | { |
1695 | | if (time(nullptr) + 1 < oFileProp.nExpireTimestampLocal) |
1696 | | { |
1697 | | CPLDebug(poFS->GetDebugKey(), |
1698 | | "Using redirect URL as it looks to be still valid " |
1699 | | "(%d seconds left)", |
1700 | | static_cast<int>(oFileProp.nExpireTimestampLocal - |
1701 | | time(nullptr))); |
1702 | | osURL = oFileProp.osRedirectURL; |
1703 | | } |
1704 | | else |
1705 | | { |
1706 | | CPLDebug(poFS->GetDebugKey(), |
1707 | | "Redirect URL has expired. Using original URL"); |
1708 | | oFileProp.bS3LikeRedirect = false; |
1709 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1710 | | bHasExpired = true; |
1711 | | } |
1712 | | } |
1713 | | else if (!oFileProp.osRedirectURL.empty()) |
1714 | | { |
1715 | | osURL = oFileProp.osRedirectURL; |
1716 | | bHasExpired = false; |
1717 | | } |
1718 | | |
1719 | | if (m_pszURL != osURL) |
1720 | | { |
1721 | | const char *pszAuthorizationHeaderAllowed = CPLGetConfigOption( |
1722 | | "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT", |
1723 | | "IF_SAME_HOST"); |
1724 | | if (EQUAL(pszAuthorizationHeaderAllowed, "IF_SAME_HOST")) |
1725 | | { |
1726 | | const auto ExtractServer = [](const std::string &s) |
1727 | | { |
1728 | | size_t afterHTTPPos = 0; |
1729 | | if (STARTS_WITH(s.c_str(), "http://")) |
1730 | | afterHTTPPos = strlen("http://"); |
1731 | | else if (STARTS_WITH(s.c_str(), "https://")) |
1732 | | afterHTTPPos = strlen("https://"); |
1733 | | const auto posSlash = s.find('/', afterHTTPPos); |
1734 | | if (posSlash != std::string::npos) |
1735 | | return s.substr(afterHTTPPos, posSlash - afterHTTPPos); |
1736 | | else |
1737 | | return s.substr(afterHTTPPos); |
1738 | | }; |
1739 | | |
1740 | | if (ExtractServer(osURL) != ExtractServer(m_pszURL)) |
1741 | | { |
1742 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", |
1743 | | "NO"); |
1744 | | } |
1745 | | } |
1746 | | else if (!CPLTestBool(pszAuthorizationHeaderAllowed)) |
1747 | | { |
1748 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", "NO"); |
1749 | | } |
1750 | | } |
1751 | | |
1752 | | return osURL; |
1753 | | } |
1754 | | |
1755 | | /************************************************************************/ |
1756 | | /* CurrentDownload */ |
1757 | | /************************************************************************/ |
1758 | | |
1759 | | namespace |
1760 | | { |
1761 | | struct CurrentDownload |
1762 | | { |
1763 | | VSICurlFilesystemHandlerBase *m_poFS = nullptr; |
1764 | | std::string m_osURL{}; |
1765 | | vsi_l_offset m_nStartOffset = 0; |
1766 | | int m_nBlocks = 0; |
1767 | | std::string m_osAlreadyDownloadedData{}; |
1768 | | bool m_bHasAlreadyDownloadedData = false; |
1769 | | |
1770 | | CurrentDownload(VSICurlFilesystemHandlerBase *poFS, const char *pszURL, |
1771 | | vsi_l_offset startOffset, int nBlocks) |
1772 | | : m_poFS(poFS), m_osURL(pszURL), m_nStartOffset(startOffset), |
1773 | | m_nBlocks(nBlocks) |
1774 | | { |
1775 | | auto res = m_poFS->NotifyStartDownloadRegion(m_osURL, m_nStartOffset, |
1776 | | m_nBlocks); |
1777 | | m_bHasAlreadyDownloadedData = res.first; |
1778 | | m_osAlreadyDownloadedData = std::move(res.second); |
1779 | | } |
1780 | | |
1781 | | bool HasAlreadyDownloadedData() const |
1782 | | { |
1783 | | return m_bHasAlreadyDownloadedData; |
1784 | | } |
1785 | | |
1786 | | const std::string &GetAlreadyDownloadedData() const |
1787 | | { |
1788 | | return m_osAlreadyDownloadedData; |
1789 | | } |
1790 | | |
1791 | | void SetData(const std::string &osData) |
1792 | | { |
1793 | | CPLAssert(!m_bHasAlreadyDownloadedData); |
1794 | | m_bHasAlreadyDownloadedData = true; |
1795 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
1796 | | osData); |
1797 | | } |
1798 | | |
1799 | | ~CurrentDownload() |
1800 | | { |
1801 | | if (!m_bHasAlreadyDownloadedData) |
1802 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
1803 | | std::string()); |
1804 | | } |
1805 | | |
1806 | | CurrentDownload(const CurrentDownload &) = delete; |
1807 | | CurrentDownload &operator=(const CurrentDownload &) = delete; |
1808 | | }; |
1809 | | } // namespace |
1810 | | |
1811 | | /************************************************************************/ |
1812 | | /* NotifyStartDownloadRegion() */ |
1813 | | /************************************************************************/ |
1814 | | |
1815 | | /** Indicate intent at downloading a new region. |
1816 | | * |
1817 | | * If the region is already in download in another thread, then wait for its |
1818 | | * completion. |
1819 | | * |
1820 | | * Returns: |
1821 | | * - (false, empty string) if a new download is needed |
1822 | | * - (true, region_content) if we have been waiting for a download of the same |
1823 | | * region to be completed and got its result. Note that region_content will be |
1824 | | * empty if the download of that region failed. |
1825 | | */ |
1826 | | std::pair<bool, std::string> |
1827 | | VSICurlFilesystemHandlerBase::NotifyStartDownloadRegion( |
1828 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks) |
1829 | | { |
1830 | | std::string osId(osURL); |
1831 | | osId += '_'; |
1832 | | osId += std::to_string(startOffset); |
1833 | | osId += '_'; |
1834 | | osId += std::to_string(nBlocks); |
1835 | | |
1836 | | m_oMutex.lock(); |
1837 | | auto oIter = m_oMapRegionInDownload.find(osId); |
1838 | | if (oIter != m_oMapRegionInDownload.end()) |
1839 | | { |
1840 | | auto ®ion = *(oIter->second); |
1841 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
1842 | | m_oMutex.unlock(); |
1843 | | region.nWaiters++; |
1844 | | while (region.bDownloadInProgress) |
1845 | | { |
1846 | | region.oCond.wait(oRegionLock); |
1847 | | } |
1848 | | std::string osRet = region.osData; |
1849 | | region.nWaiters--; |
1850 | | region.oCond.notify_one(); |
1851 | | return std::pair<bool, std::string>(true, osRet); |
1852 | | } |
1853 | | else |
1854 | | { |
1855 | | auto poRegionInDownload = std::make_unique<RegionInDownload>(); |
1856 | | poRegionInDownload->bDownloadInProgress = true; |
1857 | | m_oMapRegionInDownload[osId] = std::move(poRegionInDownload); |
1858 | | m_oMutex.unlock(); |
1859 | | return std::pair<bool, std::string>(false, std::string()); |
1860 | | } |
1861 | | } |
1862 | | |
1863 | | /************************************************************************/ |
1864 | | /* NotifyStopDownloadRegion() */ |
1865 | | /************************************************************************/ |
1866 | | |
1867 | | void VSICurlFilesystemHandlerBase::NotifyStopDownloadRegion( |
1868 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks, |
1869 | | const std::string &osData) |
1870 | | { |
1871 | | std::string osId(osURL); |
1872 | | osId += '_'; |
1873 | | osId += std::to_string(startOffset); |
1874 | | osId += '_'; |
1875 | | osId += std::to_string(nBlocks); |
1876 | | |
1877 | | m_oMutex.lock(); |
1878 | | auto oIter = m_oMapRegionInDownload.find(osId); |
1879 | | CPLAssert(oIter != m_oMapRegionInDownload.end()); |
1880 | | auto ®ion = *(oIter->second); |
1881 | | { |
1882 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
1883 | | if (region.nWaiters) |
1884 | | { |
1885 | | region.osData = osData; |
1886 | | region.bDownloadInProgress = false; |
1887 | | region.oCond.notify_all(); |
1888 | | |
1889 | | while (region.nWaiters) |
1890 | | { |
1891 | | region.oCond.wait(oRegionLock); |
1892 | | } |
1893 | | } |
1894 | | } |
1895 | | m_oMapRegionInDownload.erase(oIter); |
1896 | | m_oMutex.unlock(); |
1897 | | } |
1898 | | |
1899 | | /************************************************************************/ |
1900 | | /* DownloadRegion() */ |
1901 | | /************************************************************************/ |
1902 | | |
1903 | | std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, |
1904 | | const int nBlocks) |
1905 | | { |
1906 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
1907 | | return std::string(); |
1908 | | |
1909 | | if (oFileProp.eExists == EXIST_NO) |
1910 | | return std::string(); |
1911 | | |
1912 | | // Check if there is not a download of the same region in progress in |
1913 | | // another thread, and if so wait for it to be completed |
1914 | | CurrentDownload currentDownload(poFS, m_pszURL, startOffset, nBlocks); |
1915 | | if (currentDownload.HasAlreadyDownloadedData()) |
1916 | | { |
1917 | | return currentDownload.GetAlreadyDownloadedData(); |
1918 | | } |
1919 | | |
1920 | | begin: |
1921 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
1922 | | |
1923 | | UpdateQueryString(); |
1924 | | |
1925 | | bool bHasExpired = false; |
1926 | | |
1927 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
1928 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
1929 | | bool bUsedRedirect = osURL != m_pszURL; |
1930 | | |
1931 | | WriteFuncStruct sWriteFuncData; |
1932 | | WriteFuncStruct sWriteFuncHeaderData; |
1933 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
1934 | | |
1935 | | retry: |
1936 | | CURL *hCurlHandle = curl_easy_init(); |
1937 | | struct curl_slist *headers = |
1938 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
1939 | | |
1940 | | if (!AllowAutomaticRedirection()) |
1941 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
1942 | | |
1943 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
1944 | | pReadCbkUserData); |
1945 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
1946 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1947 | | VSICurlHandleWriteFunc); |
1948 | | |
1949 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
1950 | | nullptr); |
1951 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
1952 | | &sWriteFuncHeaderData); |
1953 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1954 | | VSICurlHandleWriteFunc); |
1955 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
1956 | | sWriteFuncHeaderData.nStartOffset = startOffset; |
1957 | | sWriteFuncHeaderData.nEndOffset = |
1958 | | startOffset + |
1959 | | static_cast<vsi_l_offset>(nBlocks) * VSICURLGetDownloadChunkSize() - 1; |
1960 | | // Some servers don't like we try to read after end-of-file (#5786). |
1961 | | if (oFileProp.bHasComputedFileSize && |
1962 | | sWriteFuncHeaderData.nEndOffset >= oFileProp.fileSize) |
1963 | | { |
1964 | | sWriteFuncHeaderData.nEndOffset = oFileProp.fileSize - 1; |
1965 | | } |
1966 | | |
1967 | | char rangeStr[512] = {}; |
1968 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
1969 | | startOffset, sWriteFuncHeaderData.nEndOffset); |
1970 | | |
1971 | | if (ENABLE_DEBUG) |
1972 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
1973 | | osURL.c_str()); |
1974 | | |
1975 | | std::string osHeaderRange; // leave in this scope |
1976 | | if (sWriteFuncHeaderData.bIsHTTP) |
1977 | | { |
1978 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
1979 | | // So it gets included in Azure signature |
1980 | | headers = curl_slist_append(headers, osHeaderRange.c_str()); |
1981 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
1982 | | } |
1983 | | else |
1984 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
1985 | | |
1986 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
1987 | | szCurlErrBuf[0] = '\0'; |
1988 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
1989 | | |
1990 | | headers = GetCurlHeaders("GET", headers); |
1991 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
1992 | | |
1993 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
1994 | | |
1995 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); |
1996 | | |
1997 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
1998 | | |
1999 | | curl_slist_free_all(headers); |
2000 | | |
2001 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
2002 | | |
2003 | | if (sWriteFuncData.bInterrupted || m_bInterrupt) |
2004 | | { |
2005 | | bInterrupted = true; |
2006 | | |
2007 | | // Notify that the download of the current region is finished |
2008 | | currentDownload.SetData(std::string()); |
2009 | | |
2010 | | CPLFree(sWriteFuncData.pBuffer); |
2011 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2012 | | curl_easy_cleanup(hCurlHandle); |
2013 | | |
2014 | | return std::string(); |
2015 | | } |
2016 | | |
2017 | | long response_code = 0; |
2018 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2019 | | |
2020 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
2021 | | { |
2022 | | CPLDebug(poFS->GetDebugKey(), |
2023 | | "DownloadRegion(%s): response_code=%d, msg=%s", osURL.c_str(), |
2024 | | static_cast<int>(response_code), szCurlErrBuf); |
2025 | | } |
2026 | | |
2027 | | long mtime = 0; |
2028 | | curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME, &mtime); |
2029 | | if (mtime > 0) |
2030 | | { |
2031 | | oFileProp.mTime = mtime; |
2032 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2033 | | } |
2034 | | |
2035 | | if (ENABLE_DEBUG) |
2036 | | CPLDebug(poFS->GetDebugKey(), "Got response_code=%ld", response_code); |
2037 | | |
2038 | | if (bUsedRedirect && |
2039 | | (response_code == 403 || |
2040 | | // Below case is in particular for |
2041 | | // gdalinfo |
2042 | | // /vsicurl/https://lpdaac.earthdata.nasa.gov/lp-prod-protected/HLSS30.015/HLS.S30.T10TEK.2020273T190109.v1.5.B8A.tif |
2043 | | // --config GDAL_DISABLE_READDIR_ON_OPEN EMPTY_DIR --config |
2044 | | // GDAL_HTTP_COOKIEFILE /tmp/cookie.txt --config GDAL_HTTP_COOKIEJAR |
2045 | | // /tmp/cookie.txt We got the redirect URL from a HEAD request, but it |
2046 | | // is not valid for a GET. So retry with GET on original URL to get a |
2047 | | // redirect URL valid for it. |
2048 | | (response_code == 400 && |
2049 | | osURL.find(".cloudfront.net") != std::string::npos))) |
2050 | | { |
2051 | | CPLDebug(poFS->GetDebugKey(), |
2052 | | "Got an error with redirect URL. Retrying with original one"); |
2053 | | oFileProp.bS3LikeRedirect = false; |
2054 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2055 | | bUsedRedirect = false; |
2056 | | osURL = m_pszURL; |
2057 | | CPLFree(sWriteFuncData.pBuffer); |
2058 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2059 | | curl_easy_cleanup(hCurlHandle); |
2060 | | goto retry; |
2061 | | } |
2062 | | |
2063 | | if (response_code == 401 && oRetryContext.CanRetry()) |
2064 | | { |
2065 | | CPLDebug(poFS->GetDebugKey(), "Unauthorized, trying to authenticate"); |
2066 | | CPLFree(sWriteFuncData.pBuffer); |
2067 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2068 | | curl_easy_cleanup(hCurlHandle); |
2069 | | if (Authenticate(m_osFilename.c_str())) |
2070 | | goto retry; |
2071 | | return std::string(); |
2072 | | } |
2073 | | |
2074 | | UpdateRedirectInfo(hCurlHandle, sWriteFuncHeaderData); |
2075 | | |
2076 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
2077 | | response_code != 226 && response_code != 426) || |
2078 | | sWriteFuncHeaderData.bError) |
2079 | | { |
2080 | | if (sWriteFuncData.pBuffer != nullptr && |
2081 | | CanRestartOnError( |
2082 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer), |
2083 | | reinterpret_cast<const char *>(sWriteFuncHeaderData.pBuffer), |
2084 | | true)) |
2085 | | { |
2086 | | CPLFree(sWriteFuncData.pBuffer); |
2087 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2088 | | curl_easy_cleanup(hCurlHandle); |
2089 | | goto begin; |
2090 | | } |
2091 | | |
2092 | | // Look if we should attempt a retry |
2093 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
2094 | | sWriteFuncHeaderData.pBuffer, szCurlErrBuf)) |
2095 | | { |
2096 | | CPLError(CE_Warning, CPLE_AppDefined, |
2097 | | "HTTP error code: %d - %s. " |
2098 | | "Retrying again in %.1f secs", |
2099 | | static_cast<int>(response_code), m_pszURL, |
2100 | | oRetryContext.GetCurrentDelay()); |
2101 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
2102 | | CPLFree(sWriteFuncData.pBuffer); |
2103 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2104 | | curl_easy_cleanup(hCurlHandle); |
2105 | | goto retry; |
2106 | | } |
2107 | | |
2108 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
2109 | | { |
2110 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
2111 | | CPLError( |
2112 | | CE_Failure, CPLE_AppDefined, |
2113 | | "%d: %s, Range downloading not supported by this server!", |
2114 | | static_cast<int>(response_code), szCurlErrBuf); |
2115 | | else |
2116 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
2117 | | static_cast<int>(response_code), szCurlErrBuf); |
2118 | | } |
2119 | | else if (response_code == 416) /* Range Not Satisfiable */ |
2120 | | { |
2121 | | if (sWriteFuncData.pBuffer) |
2122 | | { |
2123 | | CPLError( |
2124 | | CE_Failure, CPLE_AppDefined, |
2125 | | "%d: Range downloading not supported by this server: %s", |
2126 | | static_cast<int>(response_code), sWriteFuncData.pBuffer); |
2127 | | } |
2128 | | else |
2129 | | { |
2130 | | CPLError(CE_Failure, CPLE_AppDefined, |
2131 | | "%d: Range downloading not supported by this server", |
2132 | | static_cast<int>(response_code)); |
2133 | | } |
2134 | | } |
2135 | | if (!oFileProp.bHasComputedFileSize && startOffset == 0) |
2136 | | { |
2137 | | oFileProp.bHasComputedFileSize = true; |
2138 | | oFileProp.fileSize = 0; |
2139 | | oFileProp.eExists = EXIST_NO; |
2140 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2141 | | } |
2142 | | CPLFree(sWriteFuncData.pBuffer); |
2143 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2144 | | curl_easy_cleanup(hCurlHandle); |
2145 | | return std::string(); |
2146 | | } |
2147 | | |
2148 | | if (!oFileProp.bHasComputedFileSize && sWriteFuncHeaderData.pBuffer) |
2149 | | { |
2150 | | // Try to retrieve the filesize from the HTTP headers |
2151 | | // if in the form: "Content-Range: bytes x-y/filesize". |
2152 | | char *pszContentRange = |
2153 | | strstr(sWriteFuncHeaderData.pBuffer, "Content-Range: bytes "); |
2154 | | if (pszContentRange == nullptr) |
2155 | | pszContentRange = |
2156 | | strstr(sWriteFuncHeaderData.pBuffer, "content-range: bytes "); |
2157 | | if (pszContentRange) |
2158 | | { |
2159 | | char *pszEOL = strchr(pszContentRange, '\n'); |
2160 | | if (pszEOL) |
2161 | | { |
2162 | | *pszEOL = 0; |
2163 | | pszEOL = strchr(pszContentRange, '\r'); |
2164 | | if (pszEOL) |
2165 | | *pszEOL = 0; |
2166 | | char *pszSlash = strchr(pszContentRange, '/'); |
2167 | | if (pszSlash) |
2168 | | { |
2169 | | pszSlash++; |
2170 | | oFileProp.fileSize = CPLScanUIntBig( |
2171 | | pszSlash, static_cast<int>(strlen(pszSlash))); |
2172 | | } |
2173 | | } |
2174 | | } |
2175 | | else if (STARTS_WITH(m_pszURL, "ftp")) |
2176 | | { |
2177 | | // Parse 213 answer for FTP protocol. |
2178 | | char *pszSize = strstr(sWriteFuncHeaderData.pBuffer, "213 "); |
2179 | | if (pszSize) |
2180 | | { |
2181 | | pszSize += 4; |
2182 | | char *pszEOL = strchr(pszSize, '\n'); |
2183 | | if (pszEOL) |
2184 | | { |
2185 | | *pszEOL = 0; |
2186 | | pszEOL = strchr(pszSize, '\r'); |
2187 | | if (pszEOL) |
2188 | | *pszEOL = 0; |
2189 | | |
2190 | | oFileProp.fileSize = CPLScanUIntBig( |
2191 | | pszSize, static_cast<int>(strlen(pszSize))); |
2192 | | } |
2193 | | } |
2194 | | } |
2195 | | |
2196 | | if (oFileProp.fileSize != 0) |
2197 | | { |
2198 | | oFileProp.eExists = EXIST_YES; |
2199 | | |
2200 | | if (ENABLE_DEBUG) |
2201 | | CPLDebug(poFS->GetDebugKey(), |
2202 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
2203 | | m_pszURL, oFileProp.fileSize, |
2204 | | static_cast<int>(response_code)); |
2205 | | |
2206 | | oFileProp.bHasComputedFileSize = true; |
2207 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2208 | | } |
2209 | | } |
2210 | | |
2211 | | DownloadRegionPostProcess(startOffset, nBlocks, sWriteFuncData.pBuffer, |
2212 | | sWriteFuncData.nSize); |
2213 | | |
2214 | | std::string osRet; |
2215 | | osRet.assign(sWriteFuncData.pBuffer, sWriteFuncData.nSize); |
2216 | | |
2217 | | // Notify that the download of the current region is finished |
2218 | | currentDownload.SetData(osRet); |
2219 | | |
2220 | | CPLFree(sWriteFuncData.pBuffer); |
2221 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2222 | | curl_easy_cleanup(hCurlHandle); |
2223 | | |
2224 | | return osRet; |
2225 | | } |
2226 | | |
2227 | | /************************************************************************/ |
2228 | | /* UpdateRedirectInfo() */ |
2229 | | /************************************************************************/ |
2230 | | |
2231 | | void VSICurlHandle::UpdateRedirectInfo( |
2232 | | CURL *hCurlHandle, const WriteFuncStruct &sWriteFuncHeaderData) |
2233 | | { |
2234 | | std::string osEffectiveURL; |
2235 | | { |
2236 | | char *pszEffectiveURL = nullptr; |
2237 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
2238 | | &pszEffectiveURL); |
2239 | | if (pszEffectiveURL) |
2240 | | osEffectiveURL = pszEffectiveURL; |
2241 | | } |
2242 | | |
2243 | | if (!oFileProp.bS3LikeRedirect && !osEffectiveURL.empty() && |
2244 | | strstr(osEffectiveURL.c_str(), m_pszURL) == nullptr) |
2245 | | { |
2246 | | CPLDebug(poFS->GetDebugKey(), "Effective URL: %s", |
2247 | | osEffectiveURL.c_str()); |
2248 | | |
2249 | | long response_code = 0; |
2250 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2251 | | if (response_code >= 200 && response_code < 300 && |
2252 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
2253 | | VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
2254 | | !VSICurlIsS3LikeSignedURL(m_pszURL) && |
2255 | | CPLTestBool( |
2256 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
2257 | | { |
2258 | | GIntBig nExpireTimestamp = |
2259 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
2260 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
2261 | | { |
2262 | | const int nValidity = static_cast<int>( |
2263 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
2264 | | CPLDebug(poFS->GetDebugKey(), |
2265 | | "Will use redirect URL for the next %d seconds", |
2266 | | nValidity); |
2267 | | // As our local clock might not be in sync with server clock, |
2268 | | // figure out the expiration timestamp in local time. |
2269 | | oFileProp.bS3LikeRedirect = true; |
2270 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
2271 | | oFileProp.osRedirectURL = std::move(osEffectiveURL); |
2272 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2273 | | } |
2274 | | } |
2275 | | } |
2276 | | } |
2277 | | |
2278 | | /************************************************************************/ |
2279 | | /* DownloadRegionPostProcess() */ |
2280 | | /************************************************************************/ |
2281 | | |
2282 | | void VSICurlHandle::DownloadRegionPostProcess(const vsi_l_offset startOffset, |
2283 | | const int nBlocks, |
2284 | | const char *pBuffer, size_t nSize) |
2285 | | { |
2286 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2287 | | lastDownloadedOffset = startOffset + static_cast<vsi_l_offset>(nBlocks) * |
2288 | | knDOWNLOAD_CHUNK_SIZE; |
2289 | | |
2290 | | if (nSize > static_cast<size_t>(nBlocks) * knDOWNLOAD_CHUNK_SIZE) |
2291 | | { |
2292 | | if (ENABLE_DEBUG) |
2293 | | CPLDebug( |
2294 | | poFS->GetDebugKey(), |
2295 | | "Got more data than expected : %u instead of %u", |
2296 | | static_cast<unsigned int>(nSize), |
2297 | | static_cast<unsigned int>(nBlocks * knDOWNLOAD_CHUNK_SIZE)); |
2298 | | } |
2299 | | |
2300 | | vsi_l_offset l_startOffset = startOffset; |
2301 | | while (nSize > 0) |
2302 | | { |
2303 | | #if DEBUG_VERBOSE |
2304 | | if (ENABLE_DEBUG) |
2305 | | CPLDebug(poFS->GetDebugKey(), "Add region %u - %u", |
2306 | | static_cast<unsigned int>(startOffset), |
2307 | | static_cast<unsigned int>(std::min( |
2308 | | static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize))); |
2309 | | #endif |
2310 | | const size_t nChunkSize = |
2311 | | std::min(static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize); |
2312 | | poFS->AddRegion(m_pszURL, l_startOffset, nChunkSize, pBuffer); |
2313 | | l_startOffset += nChunkSize; |
2314 | | pBuffer += nChunkSize; |
2315 | | nSize -= nChunkSize; |
2316 | | } |
2317 | | } |
2318 | | |
2319 | | /************************************************************************/ |
2320 | | /* Read() */ |
2321 | | /************************************************************************/ |
2322 | | |
2323 | | size_t VSICurlHandle::Read(void *const pBufferIn, size_t const nSize, |
2324 | | size_t const nMemb) |
2325 | | { |
2326 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2327 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2328 | | NetworkStatisticsAction oContextAction("Read"); |
2329 | | |
2330 | | size_t nBufferRequestSize = nSize * nMemb; |
2331 | | if (nBufferRequestSize == 0) |
2332 | | return 0; |
2333 | | |
2334 | | void *pBuffer = pBufferIn; |
2335 | | |
2336 | | #if DEBUG_VERBOSE |
2337 | | CPLDebug(poFS->GetDebugKey(), "offset=%d, size=%d", |
2338 | | static_cast<int>(curOffset), static_cast<int>(nBufferRequestSize)); |
2339 | | #endif |
2340 | | |
2341 | | vsi_l_offset iterOffset = curOffset; |
2342 | | const int knMAX_REGIONS = GetMaxRegions(); |
2343 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2344 | | while (nBufferRequestSize) |
2345 | | { |
2346 | | // Don't try to read after end of file. |
2347 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2348 | | if (oFileProp.bHasComputedFileSize && iterOffset >= oFileProp.fileSize) |
2349 | | { |
2350 | | if (iterOffset == curOffset) |
2351 | | { |
2352 | | CPLDebug(poFS->GetDebugKey(), |
2353 | | "Request at offset " CPL_FRMT_GUIB |
2354 | | ", after end of file", |
2355 | | iterOffset); |
2356 | | } |
2357 | | break; |
2358 | | } |
2359 | | |
2360 | | const vsi_l_offset nOffsetToDownload = |
2361 | | (iterOffset / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
2362 | | std::string osRegion; |
2363 | | std::shared_ptr<std::string> psRegion = |
2364 | | poFS->GetRegion(m_pszURL, nOffsetToDownload); |
2365 | | if (psRegion != nullptr) |
2366 | | { |
2367 | | osRegion = *psRegion; |
2368 | | } |
2369 | | else |
2370 | | { |
2371 | | if (nOffsetToDownload == lastDownloadedOffset) |
2372 | | { |
2373 | | // In case of consecutive reads (of small size), we use a |
2374 | | // heuristic that we will read the file sequentially, so |
2375 | | // we double the requested size to decrease the number of |
2376 | | // client/server roundtrips. |
2377 | | constexpr int MAX_CHUNK_SIZE_INCREASE_FACTOR = 128; |
2378 | | if (nBlocksToDownload < MAX_CHUNK_SIZE_INCREASE_FACTOR) |
2379 | | nBlocksToDownload *= 2; |
2380 | | } |
2381 | | else |
2382 | | { |
2383 | | // Random reads. Cancel the above heuristics. |
2384 | | nBlocksToDownload = 1; |
2385 | | } |
2386 | | |
2387 | | // Ensure that we will request at least the number of blocks |
2388 | | // to satisfy the remaining buffer size to read. |
2389 | | const vsi_l_offset nEndOffsetToDownload = |
2390 | | ((iterOffset + nBufferRequestSize + knDOWNLOAD_CHUNK_SIZE - 1) / |
2391 | | knDOWNLOAD_CHUNK_SIZE) * |
2392 | | knDOWNLOAD_CHUNK_SIZE; |
2393 | | const int nMinBlocksToDownload = |
2394 | | static_cast<int>((nEndOffsetToDownload - nOffsetToDownload) / |
2395 | | knDOWNLOAD_CHUNK_SIZE); |
2396 | | if (nBlocksToDownload < nMinBlocksToDownload) |
2397 | | nBlocksToDownload = nMinBlocksToDownload; |
2398 | | |
2399 | | // Avoid reading already cached data. |
2400 | | // Note: this might get evicted if concurrent reads are done, but |
2401 | | // this should not cause bugs. Just missed optimization. |
2402 | | for (int i = 1; i < nBlocksToDownload; i++) |
2403 | | { |
2404 | | if (poFS->GetRegion(m_pszURL, nOffsetToDownload + |
2405 | | static_cast<vsi_l_offset>(i) * |
2406 | | knDOWNLOAD_CHUNK_SIZE) != |
2407 | | nullptr) |
2408 | | { |
2409 | | nBlocksToDownload = i; |
2410 | | break; |
2411 | | } |
2412 | | } |
2413 | | |
2414 | | // We can't download more than knMAX_REGIONS chunks at a time, |
2415 | | // otherwise the cache will not be big enough to store them and |
2416 | | // copy their content to the target buffer. |
2417 | | if (nBlocksToDownload > knMAX_REGIONS) |
2418 | | nBlocksToDownload = knMAX_REGIONS; |
2419 | | |
2420 | | osRegion = DownloadRegion(nOffsetToDownload, nBlocksToDownload); |
2421 | | if (osRegion.empty()) |
2422 | | { |
2423 | | if (!bInterrupted) |
2424 | | bError = true; |
2425 | | return 0; |
2426 | | } |
2427 | | } |
2428 | | |
2429 | | const vsi_l_offset nRegionOffset = iterOffset - nOffsetToDownload; |
2430 | | if (osRegion.size() < nRegionOffset) |
2431 | | { |
2432 | | if (iterOffset == curOffset) |
2433 | | { |
2434 | | CPLDebug(poFS->GetDebugKey(), |
2435 | | "Request at offset " CPL_FRMT_GUIB |
2436 | | ", after end of file", |
2437 | | iterOffset); |
2438 | | } |
2439 | | break; |
2440 | | } |
2441 | | |
2442 | | const int nToCopy = static_cast<int>( |
2443 | | std::min(static_cast<vsi_l_offset>(nBufferRequestSize), |
2444 | | osRegion.size() - nRegionOffset)); |
2445 | | memcpy(pBuffer, osRegion.data() + nRegionOffset, nToCopy); |
2446 | | pBuffer = static_cast<char *>(pBuffer) + nToCopy; |
2447 | | iterOffset += nToCopy; |
2448 | | nBufferRequestSize -= nToCopy; |
2449 | | if (osRegion.size() < static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE) && |
2450 | | nBufferRequestSize != 0) |
2451 | | { |
2452 | | break; |
2453 | | } |
2454 | | } |
2455 | | |
2456 | | const size_t ret = static_cast<size_t>((iterOffset - curOffset) / nSize); |
2457 | | if (ret != nMemb) |
2458 | | bEOF = true; |
2459 | | |
2460 | | curOffset = iterOffset; |
2461 | | |
2462 | | return ret; |
2463 | | } |
2464 | | |
2465 | | /************************************************************************/ |
2466 | | /* ReadMultiRange() */ |
2467 | | /************************************************************************/ |
2468 | | |
2469 | | int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, |
2470 | | const vsi_l_offset *const panOffsets, |
2471 | | const size_t *const panSizes) |
2472 | | { |
2473 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
2474 | | return FALSE; |
2475 | | |
2476 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2477 | | if (oFileProp.eExists == EXIST_NO) |
2478 | | return -1; |
2479 | | |
2480 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2481 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2482 | | NetworkStatisticsAction oContextAction("ReadMultiRange"); |
2483 | | |
2484 | | const char *pszMultiRangeStrategy = |
2485 | | CPLGetConfigOption("GDAL_HTTP_MULTIRANGE", ""); |
2486 | | if (EQUAL(pszMultiRangeStrategy, "SINGLE_GET")) |
2487 | | { |
2488 | | // Just in case someone needs it, but the interest of this mode is |
2489 | | // rather dubious now. We could probably remove it |
2490 | | return ReadMultiRangeSingleGet(nRanges, ppData, panOffsets, panSizes); |
2491 | | } |
2492 | | else if (nRanges == 1 || EQUAL(pszMultiRangeStrategy, "SERIAL")) |
2493 | | { |
2494 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
2495 | | panSizes); |
2496 | | } |
2497 | | |
2498 | | UpdateQueryString(); |
2499 | | |
2500 | | bool bHasExpired = false; |
2501 | | |
2502 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
2503 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
2504 | | if (bHasExpired) |
2505 | | { |
2506 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
2507 | | panSizes); |
2508 | | } |
2509 | | |
2510 | | CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); |
2511 | | #ifdef CURLPIPE_MULTIPLEX |
2512 | | // Enable HTTP/2 multiplexing (ignored if an older version of HTTP is |
2513 | | // used) |
2514 | | // Not that this does not enable HTTP/1.1 pipeling, which is not |
2515 | | // recommended for example by Google Cloud Storage. |
2516 | | // For HTTP/1.1, parallel connections work better since you can get |
2517 | | // results out of order. |
2518 | | if (CPLTestBool(CPLGetConfigOption("GDAL_HTTP_MULTIPLEX", "YES"))) |
2519 | | { |
2520 | | curl_multi_setopt(hMultiHandle, CURLMOPT_PIPELINING, |
2521 | | CURLPIPE_MULTIPLEX); |
2522 | | } |
2523 | | #endif |
2524 | | |
2525 | | std::vector<CURL *> aHandles; |
2526 | | std::vector<WriteFuncStruct> asWriteFuncData(nRanges); |
2527 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData(nRanges); |
2528 | | std::vector<char *> apszRanges; |
2529 | | std::vector<struct curl_slist *> aHeaders; |
2530 | | |
2531 | | struct CurlErrBuffer |
2532 | | { |
2533 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
2534 | | }; |
2535 | | |
2536 | | std::vector<CurlErrBuffer> asCurlErrors(nRanges); |
2537 | | |
2538 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
2539 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
2540 | | |
2541 | | for (int i = 0, iRequest = 0; i < nRanges;) |
2542 | | { |
2543 | | size_t nSize = 0; |
2544 | | int iNext = i; |
2545 | | // Identify consecutive ranges |
2546 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
2547 | | panOffsets[iNext] + panSizes[iNext] == panOffsets[iNext + 1]) |
2548 | | { |
2549 | | nSize += panSizes[iNext]; |
2550 | | iNext++; |
2551 | | } |
2552 | | nSize += panSizes[iNext]; |
2553 | | |
2554 | | if (nSize == 0) |
2555 | | { |
2556 | | i = iNext + 1; |
2557 | | continue; |
2558 | | } |
2559 | | |
2560 | | CURL *hCurlHandle = curl_easy_init(); |
2561 | | aHandles.push_back(hCurlHandle); |
2562 | | |
2563 | | // As the multi-range request is likely not the first one, we don't |
2564 | | // need to wait as we already know if pipelining is possible |
2565 | | // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); |
2566 | | |
2567 | | struct curl_slist *headers = VSICurlSetOptions( |
2568 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
2569 | | |
2570 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[iRequest], this, pfnReadCbk, |
2571 | | pReadCbkUserData); |
2572 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
2573 | | &asWriteFuncData[iRequest]); |
2574 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2575 | | VSICurlHandleWriteFunc); |
2576 | | |
2577 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[iRequest], nullptr, |
2578 | | nullptr, nullptr); |
2579 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2580 | | &asWriteFuncHeaderData[iRequest]); |
2581 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2582 | | VSICurlHandleWriteFunc); |
2583 | | asWriteFuncHeaderData[iRequest].bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2584 | | asWriteFuncHeaderData[iRequest].nStartOffset = panOffsets[i]; |
2585 | | |
2586 | | asWriteFuncHeaderData[iRequest].nEndOffset = panOffsets[i] + nSize - 1; |
2587 | | |
2588 | | char rangeStr[512] = {}; |
2589 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2590 | | asWriteFuncHeaderData[iRequest].nStartOffset, |
2591 | | asWriteFuncHeaderData[iRequest].nEndOffset); |
2592 | | |
2593 | | if (ENABLE_DEBUG) |
2594 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
2595 | | osURL.c_str()); |
2596 | | |
2597 | | if (asWriteFuncHeaderData[iRequest].bIsHTTP) |
2598 | | { |
2599 | | // So it gets included in Azure signature |
2600 | | char *pszRange = CPLStrdup(CPLSPrintf("Range: bytes=%s", rangeStr)); |
2601 | | apszRanges.push_back(pszRange); |
2602 | | headers = curl_slist_append(headers, pszRange); |
2603 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
2604 | | } |
2605 | | else |
2606 | | { |
2607 | | apszRanges.push_back(nullptr); |
2608 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
2609 | | } |
2610 | | |
2611 | | asCurlErrors[iRequest].szCurlErrBuf[0] = '\0'; |
2612 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
2613 | | &asCurlErrors[iRequest].szCurlErrBuf[0]); |
2614 | | |
2615 | | headers = GetCurlHeaders("GET", headers); |
2616 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
2617 | | aHeaders.push_back(headers); |
2618 | | curl_multi_add_handle(hMultiHandle, hCurlHandle); |
2619 | | |
2620 | | i = iNext + 1; |
2621 | | iRequest++; |
2622 | | } |
2623 | | |
2624 | | if (!aHandles.empty()) |
2625 | | { |
2626 | | VSICURLMultiPerform(hMultiHandle); |
2627 | | } |
2628 | | |
2629 | | int nRet = 0; |
2630 | | size_t iReq = 0; |
2631 | | int iRange = 0; |
2632 | | size_t nTotalDownloaded = 0; |
2633 | | for (; iReq < aHandles.size(); iReq++, iRange++) |
2634 | | { |
2635 | | while (iRange < nRanges && panSizes[iRange] == 0) |
2636 | | { |
2637 | | iRange++; |
2638 | | } |
2639 | | if (iRange == nRanges) |
2640 | | break; |
2641 | | |
2642 | | long response_code = 0; |
2643 | | curl_easy_getinfo(aHandles[iReq], CURLINFO_HTTP_CODE, &response_code); |
2644 | | |
2645 | | if (ENABLE_DEBUG && asCurlErrors[iRange].szCurlErrBuf[0] != '\0') |
2646 | | { |
2647 | | char rangeStr[512] = {}; |
2648 | | snprintf(rangeStr, sizeof(rangeStr), |
2649 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2650 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2651 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2652 | | |
2653 | | const char *pszErrorMsg = &asCurlErrors[iRange].szCurlErrBuf[0]; |
2654 | | CPLDebug(poFS->GetDebugKey(), |
2655 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
2656 | | osURL.c_str(), rangeStr, static_cast<int>(response_code), |
2657 | | pszErrorMsg); |
2658 | | } |
2659 | | |
2660 | | if ((response_code != 206 && response_code != 225) || |
2661 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
2662 | | asWriteFuncHeaderData[iReq].nStartOffset + |
2663 | | asWriteFuncData[iReq].nSize) |
2664 | | { |
2665 | | char rangeStr[512] = {}; |
2666 | | snprintf(rangeStr, sizeof(rangeStr), |
2667 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2668 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2669 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2670 | | |
2671 | | CPLError(CE_Failure, CPLE_AppDefined, |
2672 | | "Request for %s failed with response_code=%ld", rangeStr, |
2673 | | response_code); |
2674 | | nRet = -1; |
2675 | | } |
2676 | | else if (nRet == 0) |
2677 | | { |
2678 | | size_t nOffset = 0; |
2679 | | size_t nRemainingSize = asWriteFuncData[iReq].nSize; |
2680 | | nTotalDownloaded += nRemainingSize; |
2681 | | CPLAssert(iRange < nRanges); |
2682 | | while (true) |
2683 | | { |
2684 | | if (nRemainingSize < panSizes[iRange]) |
2685 | | { |
2686 | | nRet = -1; |
2687 | | break; |
2688 | | } |
2689 | | |
2690 | | if (panSizes[iRange] > 0) |
2691 | | { |
2692 | | memcpy(ppData[iRange], |
2693 | | asWriteFuncData[iReq].pBuffer + nOffset, |
2694 | | panSizes[iRange]); |
2695 | | } |
2696 | | |
2697 | | if (bMergeConsecutiveRanges && iRange + 1 < nRanges && |
2698 | | panOffsets[iRange] + panSizes[iRange] == |
2699 | | panOffsets[iRange + 1]) |
2700 | | { |
2701 | | nOffset += panSizes[iRange]; |
2702 | | nRemainingSize -= panSizes[iRange]; |
2703 | | iRange++; |
2704 | | } |
2705 | | else |
2706 | | { |
2707 | | break; |
2708 | | } |
2709 | | } |
2710 | | } |
2711 | | |
2712 | | curl_multi_remove_handle(hMultiHandle, aHandles[iReq]); |
2713 | | VSICURLResetHeaderAndWriterFunctions(aHandles[iReq]); |
2714 | | curl_easy_cleanup(aHandles[iReq]); |
2715 | | CPLFree(apszRanges[iReq]); |
2716 | | CPLFree(asWriteFuncData[iReq].pBuffer); |
2717 | | CPLFree(asWriteFuncHeaderData[iReq].pBuffer); |
2718 | | curl_slist_free_all(aHeaders[iReq]); |
2719 | | } |
2720 | | |
2721 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
2722 | | |
2723 | | if (ENABLE_DEBUG) |
2724 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
2725 | | |
2726 | | return nRet; |
2727 | | } |
2728 | | |
2729 | | /************************************************************************/ |
2730 | | /* ReadMultiRangeSingleGet() */ |
2731 | | /************************************************************************/ |
2732 | | |
2733 | | // TODO: the interest of this mode is rather dubious now. We could probably |
2734 | | // remove it |
2735 | | int VSICurlHandle::ReadMultiRangeSingleGet(int const nRanges, |
2736 | | void **const ppData, |
2737 | | const vsi_l_offset *const panOffsets, |
2738 | | const size_t *const panSizes) |
2739 | | { |
2740 | | std::string osRanges; |
2741 | | std::string osFirstRange; |
2742 | | std::string osLastRange; |
2743 | | int nMergedRanges = 0; |
2744 | | vsi_l_offset nTotalReqSize = 0; |
2745 | | for (int i = 0; i < nRanges; i++) |
2746 | | { |
2747 | | std::string osCurRange; |
2748 | | if (i != 0) |
2749 | | osRanges.append(","); |
2750 | | osCurRange = CPLSPrintf(CPL_FRMT_GUIB "-", panOffsets[i]); |
2751 | | while (i + 1 < nRanges && |
2752 | | panOffsets[i] + panSizes[i] == panOffsets[i + 1]) |
2753 | | { |
2754 | | nTotalReqSize += panSizes[i]; |
2755 | | i++; |
2756 | | } |
2757 | | nTotalReqSize += panSizes[i]; |
2758 | | osCurRange.append( |
2759 | | CPLSPrintf(CPL_FRMT_GUIB, panOffsets[i] + panSizes[i] - 1)); |
2760 | | nMergedRanges++; |
2761 | | |
2762 | | osRanges += osCurRange; |
2763 | | |
2764 | | if (nMergedRanges == 1) |
2765 | | osFirstRange = osCurRange; |
2766 | | osLastRange = std::move(osCurRange); |
2767 | | } |
2768 | | |
2769 | | const char *pszMaxRanges = |
2770 | | CPLGetConfigOption("CPL_VSIL_CURL_MAX_RANGES", "250"); |
2771 | | int nMaxRanges = atoi(pszMaxRanges); |
2772 | | if (nMaxRanges <= 0) |
2773 | | nMaxRanges = 250; |
2774 | | if (nMergedRanges > nMaxRanges) |
2775 | | { |
2776 | | const int nHalf = nRanges / 2; |
2777 | | const int nRet = ReadMultiRange(nHalf, ppData, panOffsets, panSizes); |
2778 | | if (nRet != 0) |
2779 | | return nRet; |
2780 | | return ReadMultiRange(nRanges - nHalf, ppData + nHalf, |
2781 | | panOffsets + nHalf, panSizes + nHalf); |
2782 | | } |
2783 | | |
2784 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
2785 | | CURL *hCurlHandle = curl_easy_init(); |
2786 | | |
2787 | | struct curl_slist *headers = |
2788 | | VSICurlSetOptions(hCurlHandle, m_pszURL, m_aosHTTPOptions.List()); |
2789 | | |
2790 | | WriteFuncStruct sWriteFuncData; |
2791 | | WriteFuncStruct sWriteFuncHeaderData; |
2792 | | |
2793 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
2794 | | pReadCbkUserData); |
2795 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
2796 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2797 | | VSICurlHandleWriteFunc); |
2798 | | |
2799 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
2800 | | nullptr); |
2801 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2802 | | &sWriteFuncHeaderData); |
2803 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2804 | | VSICurlHandleWriteFunc); |
2805 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2806 | | sWriteFuncHeaderData.bMultiRange = nMergedRanges > 1; |
2807 | | if (nMergedRanges == 1) |
2808 | | { |
2809 | | sWriteFuncHeaderData.nStartOffset = panOffsets[0]; |
2810 | | sWriteFuncHeaderData.nEndOffset = panOffsets[0] + nTotalReqSize - 1; |
2811 | | } |
2812 | | |
2813 | | if (ENABLE_DEBUG) |
2814 | | { |
2815 | | if (nMergedRanges == 1) |
2816 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
2817 | | osRanges.c_str(), m_pszURL); |
2818 | | else |
2819 | | CPLDebug(poFS->GetDebugKey(), |
2820 | | "Downloading %s, ..., %s (" CPL_FRMT_GUIB " bytes, %s)...", |
2821 | | osFirstRange.c_str(), osLastRange.c_str(), |
2822 | | static_cast<GUIntBig>(nTotalReqSize), m_pszURL); |
2823 | | } |
2824 | | |
2825 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, osRanges.c_str()); |
2826 | | |
2827 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
2828 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
2829 | | |
2830 | | headers = GetCurlHeaders("GET", headers); |
2831 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
2832 | | |
2833 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
2834 | | |
2835 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
2836 | | |
2837 | | curl_slist_free_all(headers); |
2838 | | |
2839 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
2840 | | |
2841 | | if (sWriteFuncData.bInterrupted) |
2842 | | { |
2843 | | bInterrupted = true; |
2844 | | |
2845 | | CPLFree(sWriteFuncData.pBuffer); |
2846 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2847 | | curl_easy_cleanup(hCurlHandle); |
2848 | | |
2849 | | return -1; |
2850 | | } |
2851 | | |
2852 | | long response_code = 0; |
2853 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2854 | | |
2855 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
2856 | | response_code != 226 && response_code != 426) || |
2857 | | sWriteFuncHeaderData.bError) |
2858 | | { |
2859 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
2860 | | { |
2861 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
2862 | | CPLError( |
2863 | | CE_Failure, CPLE_AppDefined, |
2864 | | "%d: %s, Range downloading not supported by this server!", |
2865 | | static_cast<int>(response_code), szCurlErrBuf); |
2866 | | else |
2867 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
2868 | | static_cast<int>(response_code), szCurlErrBuf); |
2869 | | } |
2870 | | /* |
2871 | | if( !bHasComputedFileSize && startOffset == 0 ) |
2872 | | { |
2873 | | cachedFileProp->bHasComputedFileSize = bHasComputedFileSize = true; |
2874 | | cachedFileProp->fileSize = fileSize = 0; |
2875 | | cachedFileProp->eExists = eExists = EXIST_NO; |
2876 | | } |
2877 | | */ |
2878 | | CPLFree(sWriteFuncData.pBuffer); |
2879 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2880 | | curl_easy_cleanup(hCurlHandle); |
2881 | | return -1; |
2882 | | } |
2883 | | |
2884 | | char *pBuffer = sWriteFuncData.pBuffer; |
2885 | | size_t nSize = sWriteFuncData.nSize; |
2886 | | |
2887 | | // TODO(schwehr): Localize after removing gotos. |
2888 | | int nRet = -1; |
2889 | | char *pszBoundary; |
2890 | | std::string osBoundary; |
2891 | | char *pszNext = nullptr; |
2892 | | int iRange = 0; |
2893 | | int iPart = 0; |
2894 | | char *pszEOL = nullptr; |
2895 | | |
2896 | | /* -------------------------------------------------------------------- */ |
2897 | | /* No multipart if a single range has been requested */ |
2898 | | /* -------------------------------------------------------------------- */ |
2899 | | |
2900 | | if (nMergedRanges == 1) |
2901 | | { |
2902 | | size_t nAccSize = 0; |
2903 | | if (static_cast<vsi_l_offset>(nSize) < nTotalReqSize) |
2904 | | goto end; |
2905 | | |
2906 | | for (int i = 0; i < nRanges; i++) |
2907 | | { |
2908 | | memcpy(ppData[i], pBuffer + nAccSize, panSizes[i]); |
2909 | | nAccSize += panSizes[i]; |
2910 | | } |
2911 | | |
2912 | | nRet = 0; |
2913 | | goto end; |
2914 | | } |
2915 | | |
2916 | | /* -------------------------------------------------------------------- */ |
2917 | | /* Extract boundary name */ |
2918 | | /* -------------------------------------------------------------------- */ |
2919 | | |
2920 | | pszBoundary = strstr(sWriteFuncHeaderData.pBuffer, |
2921 | | "Content-Type: multipart/byteranges; boundary="); |
2922 | | if (pszBoundary == nullptr) |
2923 | | { |
2924 | | CPLError(CE_Failure, CPLE_AppDefined, "Could not find '%s'", |
2925 | | "Content-Type: multipart/byteranges; boundary="); |
2926 | | goto end; |
2927 | | } |
2928 | | |
2929 | | pszBoundary += strlen("Content-Type: multipart/byteranges; boundary="); |
2930 | | |
2931 | | pszEOL = strchr(pszBoundary, '\r'); |
2932 | | if (pszEOL) |
2933 | | *pszEOL = 0; |
2934 | | pszEOL = strchr(pszBoundary, '\n'); |
2935 | | if (pszEOL) |
2936 | | *pszEOL = 0; |
2937 | | |
2938 | | /* Remove optional double-quote character around boundary name */ |
2939 | | if (pszBoundary[0] == '"') |
2940 | | { |
2941 | | pszBoundary++; |
2942 | | char *pszLastDoubleQuote = strrchr(pszBoundary, '"'); |
2943 | | if (pszLastDoubleQuote) |
2944 | | *pszLastDoubleQuote = 0; |
2945 | | } |
2946 | | |
2947 | | osBoundary = "--"; |
2948 | | osBoundary += pszBoundary; |
2949 | | |
2950 | | /* -------------------------------------------------------------------- */ |
2951 | | /* Find the start of the first chunk. */ |
2952 | | /* -------------------------------------------------------------------- */ |
2953 | | pszNext = strstr(pBuffer, osBoundary.c_str()); |
2954 | | if (pszNext == nullptr) |
2955 | | { |
2956 | | CPLError(CE_Failure, CPLE_AppDefined, "No parts found."); |
2957 | | goto end; |
2958 | | } |
2959 | | |
2960 | | pszNext += osBoundary.size(); |
2961 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
2962 | | pszNext++; |
2963 | | if (*pszNext == '\r') |
2964 | | pszNext++; |
2965 | | if (*pszNext == '\n') |
2966 | | pszNext++; |
2967 | | |
2968 | | /* -------------------------------------------------------------------- */ |
2969 | | /* Loop over parts... */ |
2970 | | /* -------------------------------------------------------------------- */ |
2971 | | while (iPart < nRanges) |
2972 | | { |
2973 | | /* -------------------------------------------------------------------- |
2974 | | */ |
2975 | | /* Collect headers. */ |
2976 | | /* -------------------------------------------------------------------- |
2977 | | */ |
2978 | | bool bExpectedRange = false; |
2979 | | |
2980 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
2981 | | { |
2982 | | pszEOL = strstr(pszNext, "\n"); |
2983 | | |
2984 | | if (pszEOL == nullptr) |
2985 | | { |
2986 | | CPLError(CE_Failure, CPLE_AppDefined, |
2987 | | "Error while parsing multipart content (at line %d)", |
2988 | | __LINE__); |
2989 | | goto end; |
2990 | | } |
2991 | | |
2992 | | *pszEOL = '\0'; |
2993 | | bool bRestoreAntislashR = false; |
2994 | | if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r') |
2995 | | { |
2996 | | bRestoreAntislashR = true; |
2997 | | pszEOL[-1] = '\0'; |
2998 | | } |
2999 | | |
3000 | | if (STARTS_WITH_CI(pszNext, "Content-Range: bytes ")) |
3001 | | { |
3002 | | bExpectedRange = true; /* FIXME */ |
3003 | | } |
3004 | | |
3005 | | if (bRestoreAntislashR) |
3006 | | pszEOL[-1] = '\r'; |
3007 | | *pszEOL = '\n'; |
3008 | | |
3009 | | pszNext = pszEOL + 1; |
3010 | | } |
3011 | | |
3012 | | if (!bExpectedRange) |
3013 | | { |
3014 | | CPLError(CE_Failure, CPLE_AppDefined, |
3015 | | "Error while parsing multipart content (at line %d)", |
3016 | | __LINE__); |
3017 | | goto end; |
3018 | | } |
3019 | | |
3020 | | if (*pszNext == '\r') |
3021 | | pszNext++; |
3022 | | if (*pszNext == '\n') |
3023 | | pszNext++; |
3024 | | |
3025 | | /* -------------------------------------------------------------------- |
3026 | | */ |
3027 | | /* Work out the data block size. */ |
3028 | | /* -------------------------------------------------------------------- |
3029 | | */ |
3030 | | size_t nBytesAvail = nSize - (pszNext - pBuffer); |
3031 | | |
3032 | | while (true) |
3033 | | { |
3034 | | if (nBytesAvail < panSizes[iRange]) |
3035 | | { |
3036 | | CPLError(CE_Failure, CPLE_AppDefined, |
3037 | | "Error while parsing multipart content (at line %d)", |
3038 | | __LINE__); |
3039 | | goto end; |
3040 | | } |
3041 | | |
3042 | | memcpy(ppData[iRange], pszNext, panSizes[iRange]); |
3043 | | pszNext += panSizes[iRange]; |
3044 | | nBytesAvail -= panSizes[iRange]; |
3045 | | if (iRange + 1 < nRanges && |
3046 | | panOffsets[iRange] + panSizes[iRange] == panOffsets[iRange + 1]) |
3047 | | { |
3048 | | iRange++; |
3049 | | } |
3050 | | else |
3051 | | { |
3052 | | break; |
3053 | | } |
3054 | | } |
3055 | | |
3056 | | iPart++; |
3057 | | iRange++; |
3058 | | |
3059 | | while (nBytesAvail > 0 && |
3060 | | (*pszNext != '-' || |
3061 | | strncmp(pszNext, osBoundary.c_str(), osBoundary.size()) != 0)) |
3062 | | { |
3063 | | pszNext++; |
3064 | | nBytesAvail--; |
3065 | | } |
3066 | | |
3067 | | if (nBytesAvail == 0) |
3068 | | { |
3069 | | CPLError(CE_Failure, CPLE_AppDefined, |
3070 | | "Error while parsing multipart content (at line %d)", |
3071 | | __LINE__); |
3072 | | goto end; |
3073 | | } |
3074 | | |
3075 | | pszNext += osBoundary.size(); |
3076 | | if (STARTS_WITH(pszNext, "--")) |
3077 | | { |
3078 | | // End of multipart. |
3079 | | break; |
3080 | | } |
3081 | | |
3082 | | if (*pszNext == '\r') |
3083 | | pszNext++; |
3084 | | if (*pszNext == '\n') |
3085 | | pszNext++; |
3086 | | else |
3087 | | { |
3088 | | CPLError(CE_Failure, CPLE_AppDefined, |
3089 | | "Error while parsing multipart content (at line %d)", |
3090 | | __LINE__); |
3091 | | goto end; |
3092 | | } |
3093 | | } |
3094 | | |
3095 | | if (iPart == nMergedRanges) |
3096 | | nRet = 0; |
3097 | | else |
3098 | | CPLError(CE_Failure, CPLE_AppDefined, |
3099 | | "Got only %d parts, where %d were expected", iPart, |
3100 | | nMergedRanges); |
3101 | | |
3102 | | end: |
3103 | | CPLFree(sWriteFuncData.pBuffer); |
3104 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3105 | | curl_easy_cleanup(hCurlHandle); |
3106 | | |
3107 | | return nRet; |
3108 | | } |
3109 | | |
3110 | | /************************************************************************/ |
3111 | | /* PRead() */ |
3112 | | /************************************************************************/ |
3113 | | |
3114 | | size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, |
3115 | | vsi_l_offset nOffset) const |
3116 | | { |
3117 | | // Try to use AdviseRead ranges fetched asynchronously |
3118 | | if (!m_aoAdviseReadRanges.empty()) |
3119 | | { |
3120 | | for (auto &poRange : m_aoAdviseReadRanges) |
3121 | | { |
3122 | | if (nOffset >= poRange->nStartOffset && |
3123 | | nOffset + nSize <= poRange->nStartOffset + poRange->nSize) |
3124 | | { |
3125 | | { |
3126 | | std::unique_lock<std::mutex> oLock(poRange->oMutex); |
3127 | | // coverity[missing_lock:FALSE] |
3128 | | while (!poRange->bDone) |
3129 | | { |
3130 | | poRange->oCV.wait(oLock); |
3131 | | } |
3132 | | } |
3133 | | if (poRange->abyData.empty()) |
3134 | | return 0; |
3135 | | |
3136 | | auto nEndOffset = |
3137 | | poRange->nStartOffset + poRange->abyData.size(); |
3138 | | if (nOffset >= nEndOffset) |
3139 | | return 0; |
3140 | | const size_t nToCopy = static_cast<size_t>( |
3141 | | std::min<vsi_l_offset>(nSize, nEndOffset - nOffset)); |
3142 | | memcpy(pBuffer, |
3143 | | poRange->abyData.data() + |
3144 | | static_cast<size_t>(nOffset - poRange->nStartOffset), |
3145 | | nToCopy); |
3146 | | return nToCopy; |
3147 | | } |
3148 | | } |
3149 | | } |
3150 | | |
3151 | | // poFS has a global mutex |
3152 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
3153 | | if (oFileProp.eExists == EXIST_NO) |
3154 | | return static_cast<size_t>(-1); |
3155 | | |
3156 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3157 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3158 | | NetworkStatisticsAction oContextAction("PRead"); |
3159 | | |
3160 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3161 | | std::string osURL; |
3162 | | { |
3163 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3164 | | UpdateQueryString(); |
3165 | | bool bHasExpired; |
3166 | | osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions); |
3167 | | } |
3168 | | |
3169 | | CURL *hCurlHandle = curl_easy_init(); |
3170 | | |
3171 | | struct curl_slist *headers = |
3172 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3173 | | |
3174 | | WriteFuncStruct sWriteFuncData; |
3175 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
3176 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
3177 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3178 | | VSICurlHandleWriteFunc); |
3179 | | |
3180 | | WriteFuncStruct sWriteFuncHeaderData; |
3181 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
3182 | | nullptr); |
3183 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3184 | | &sWriteFuncHeaderData); |
3185 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3186 | | VSICurlHandleWriteFunc); |
3187 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3188 | | sWriteFuncHeaderData.nStartOffset = nOffset; |
3189 | | |
3190 | | sWriteFuncHeaderData.nEndOffset = nOffset + nSize - 1; |
3191 | | |
3192 | | char rangeStr[512] = {}; |
3193 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3194 | | sWriteFuncHeaderData.nStartOffset, |
3195 | | sWriteFuncHeaderData.nEndOffset); |
3196 | | |
3197 | | #if 0 |
3198 | | if( ENABLE_DEBUG ) |
3199 | | CPLDebug(poFS->GetDebugKey(), |
3200 | | "Downloading %s (%s)...", rangeStr, osURL.c_str()); |
3201 | | #endif |
3202 | | |
3203 | | std::string osHeaderRange; |
3204 | | if (sWriteFuncHeaderData.bIsHTTP) |
3205 | | { |
3206 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
3207 | | // So it gets included in Azure signature |
3208 | | headers = curl_slist_append(headers, osHeaderRange.data()); |
3209 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
3210 | | } |
3211 | | else |
3212 | | { |
3213 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
3214 | | } |
3215 | | |
3216 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3217 | | szCurlErrBuf[0] = '\0'; |
3218 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3219 | | &szCurlErrBuf[0]); |
3220 | | |
3221 | | { |
3222 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3223 | | headers = |
3224 | | const_cast<VSICurlHandle *>(this)->GetCurlHeaders("GET", headers); |
3225 | | } |
3226 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
3227 | | |
3228 | | CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); |
3229 | | VSICURLMultiPerform(hMultiHandle, hCurlHandle, &m_bInterrupt); |
3230 | | |
3231 | | { |
3232 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3233 | | const_cast<VSICurlHandle *>(this)->UpdateRedirectInfo( |
3234 | | hCurlHandle, sWriteFuncHeaderData); |
3235 | | } |
3236 | | |
3237 | | long response_code = 0; |
3238 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
3239 | | |
3240 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
3241 | | { |
3242 | | const char *pszErrorMsg = &szCurlErrBuf[0]; |
3243 | | CPLDebug(poFS->GetDebugKey(), "PRead(%s), %s: response_code=%d, msg=%s", |
3244 | | osURL.c_str(), rangeStr, static_cast<int>(response_code), |
3245 | | pszErrorMsg); |
3246 | | } |
3247 | | |
3248 | | size_t nRet; |
3249 | | if ((response_code != 206 && response_code != 225) || |
3250 | | sWriteFuncData.nSize == 0) |
3251 | | { |
3252 | | if (!m_bInterrupt) |
3253 | | { |
3254 | | CPLDebug(poFS->GetDebugKey(), |
3255 | | "Request for %s failed with response_code=%ld", rangeStr, |
3256 | | response_code); |
3257 | | } |
3258 | | nRet = static_cast<size_t>(-1); |
3259 | | } |
3260 | | else |
3261 | | { |
3262 | | nRet = std::min(sWriteFuncData.nSize, nSize); |
3263 | | if (nRet > 0) |
3264 | | memcpy(pBuffer, sWriteFuncData.pBuffer, nRet); |
3265 | | } |
3266 | | |
3267 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
3268 | | curl_easy_cleanup(hCurlHandle); |
3269 | | CPLFree(sWriteFuncData.pBuffer); |
3270 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3271 | | curl_slist_free_all(headers); |
3272 | | |
3273 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
3274 | | |
3275 | | #if 0 |
3276 | | if( ENABLE_DEBUG ) |
3277 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
3278 | | #endif |
3279 | | |
3280 | | return nRet; |
3281 | | } |
3282 | | |
3283 | | /************************************************************************/ |
3284 | | /* GetAdviseReadTotalBytesLimit() */ |
3285 | | /************************************************************************/ |
3286 | | |
3287 | | size_t VSICurlHandle::GetAdviseReadTotalBytesLimit() const |
3288 | | { |
3289 | | return static_cast<size_t>(std::min<unsigned long long>( |
3290 | | std::numeric_limits<size_t>::max(), |
3291 | | // 100 MB |
3292 | | std::strtoull( |
3293 | | CPLGetConfigOption("CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT", |
3294 | | "104857600"), |
3295 | | nullptr, 10))); |
3296 | | } |
3297 | | |
3298 | | /************************************************************************/ |
3299 | | /* VSICURLMultiInit() */ |
3300 | | /************************************************************************/ |
3301 | | |
3302 | | static CURLM *VSICURLMultiInit() |
3303 | | { |
3304 | | CURLM *hCurlMultiHandle = curl_multi_init(); |
3305 | | |
3306 | | if (const char *pszMAXCONNECTS = |
3307 | | CPLGetConfigOption("GDAL_HTTP_MAX_CACHED_CONNECTIONS", nullptr)) |
3308 | | { |
3309 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAXCONNECTS, |
3310 | | atoi(pszMAXCONNECTS)); |
3311 | | } |
3312 | | |
3313 | | if (const char *pszMAX_TOTAL_CONNECTIONS = |
3314 | | CPLGetConfigOption("GDAL_HTTP_MAX_TOTAL_CONNECTIONS", nullptr)) |
3315 | | { |
3316 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, |
3317 | | atoi(pszMAX_TOTAL_CONNECTIONS)); |
3318 | | } |
3319 | | |
3320 | | return hCurlMultiHandle; |
3321 | | } |
3322 | | |
3323 | | /************************************************************************/ |
3324 | | /* AdviseRead() */ |
3325 | | /************************************************************************/ |
3326 | | |
3327 | | void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, |
3328 | | const size_t *panSizes) |
3329 | | { |
3330 | | if (!CPLTestBool( |
3331 | | CPLGetConfigOption("GDAL_HTTP_ENABLE_ADVISE_READ", "TRUE"))) |
3332 | | return; |
3333 | | |
3334 | | if (m_oThreadAdviseRead.joinable()) |
3335 | | { |
3336 | | m_oThreadAdviseRead.join(); |
3337 | | } |
3338 | | |
3339 | | // Give up if we need to allocate too much memory |
3340 | | vsi_l_offset nMaxSize = 0; |
3341 | | const size_t nLimit = GetAdviseReadTotalBytesLimit(); |
3342 | | for (int i = 0; i < nRanges; ++i) |
3343 | | { |
3344 | | if (panSizes[i] > nLimit - nMaxSize) |
3345 | | { |
3346 | | CPLDebug(poFS->GetDebugKey(), |
3347 | | "Trying to request too many bytes in AdviseRead()"); |
3348 | | return; |
3349 | | } |
3350 | | nMaxSize += panSizes[i]; |
3351 | | } |
3352 | | |
3353 | | UpdateQueryString(); |
3354 | | |
3355 | | bool bHasExpired = false; |
3356 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3357 | | const std::string l_osURL( |
3358 | | GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
3359 | | if (bHasExpired) |
3360 | | { |
3361 | | return; |
3362 | | } |
3363 | | |
3364 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
3365 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
3366 | | |
3367 | | try |
3368 | | { |
3369 | | m_aoAdviseReadRanges.clear(); |
3370 | | m_aoAdviseReadRanges.reserve(nRanges); |
3371 | | for (int i = 0; i < nRanges;) |
3372 | | { |
3373 | | int iNext = i; |
3374 | | // Identify consecutive ranges |
3375 | | constexpr size_t SIZE_COG_MARKERS = 2 * sizeof(uint32_t); |
3376 | | auto nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3377 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
3378 | | panOffsets[iNext + 1] > panOffsets[iNext] && |
3379 | | panOffsets[iNext] + panSizes[iNext] + SIZE_COG_MARKERS >= |
3380 | | panOffsets[iNext + 1] && |
3381 | | panOffsets[iNext + 1] + panSizes[iNext + 1] > nEndOffset) |
3382 | | { |
3383 | | iNext++; |
3384 | | nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3385 | | } |
3386 | | CPLAssert(panOffsets[i] <= nEndOffset); |
3387 | | const size_t nSize = |
3388 | | static_cast<size_t>(nEndOffset - panOffsets[i]); |
3389 | | |
3390 | | if (nSize == 0) |
3391 | | { |
3392 | | i = iNext + 1; |
3393 | | continue; |
3394 | | } |
3395 | | |
3396 | | auto newAdviseReadRange = |
3397 | | std::make_unique<AdviseReadRange>(m_oRetryParameters); |
3398 | | newAdviseReadRange->nStartOffset = panOffsets[i]; |
3399 | | newAdviseReadRange->nSize = nSize; |
3400 | | newAdviseReadRange->abyData.resize(nSize); |
3401 | | m_aoAdviseReadRanges.push_back(std::move(newAdviseReadRange)); |
3402 | | |
3403 | | i = iNext + 1; |
3404 | | } |
3405 | | } |
3406 | | catch (const std::exception &) |
3407 | | { |
3408 | | CPLError(CE_Failure, CPLE_OutOfMemory, |
3409 | | "Out of memory in VSICurlHandle::AdviseRead()"); |
3410 | | m_aoAdviseReadRanges.clear(); |
3411 | | } |
3412 | | |
3413 | | if (m_aoAdviseReadRanges.empty()) |
3414 | | return; |
3415 | | |
3416 | | #ifdef DEBUG |
3417 | | CPLDebug(poFS->GetDebugKey(), "AdviseRead(): fetching %u ranges", |
3418 | | static_cast<unsigned>(m_aoAdviseReadRanges.size())); |
3419 | | #endif |
3420 | | |
3421 | | const auto task = [this, aosHTTPOptions = std::move(aosHTTPOptions)]( |
3422 | | const std::string &osURL) |
3423 | | { |
3424 | | if (!m_hCurlMultiHandleForAdviseRead) |
3425 | | m_hCurlMultiHandleForAdviseRead = VSICURLMultiInit(); |
3426 | | |
3427 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3428 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3429 | | NetworkStatisticsAction oContextAction("AdviseRead"); |
3430 | | |
3431 | | #ifdef CURLPIPE_MULTIPLEX |
3432 | | // Enable HTTP/2 multiplexing (ignored if an older version of HTTP is |
3433 | | // used) |
3434 | | // Not that this does not enable HTTP/1.1 pipeling, which is not |
3435 | | // recommended for example by Google Cloud Storage. |
3436 | | // For HTTP/1.1, parallel connections work better since you can get |
3437 | | // results out of order. |
3438 | | if (CPLTestBool(CPLGetConfigOption("GDAL_HTTP_MULTIPLEX", "YES"))) |
3439 | | { |
3440 | | curl_multi_setopt(m_hCurlMultiHandleForAdviseRead, |
3441 | | CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); |
3442 | | } |
3443 | | #endif |
3444 | | |
3445 | | size_t nTotalDownloaded = 0; |
3446 | | |
3447 | | while (true) |
3448 | | { |
3449 | | |
3450 | | std::vector<CURL *> aHandles; |
3451 | | std::vector<WriteFuncStruct> asWriteFuncData( |
3452 | | m_aoAdviseReadRanges.size()); |
3453 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData( |
3454 | | m_aoAdviseReadRanges.size()); |
3455 | | std::vector<char *> apszRanges; |
3456 | | std::vector<struct curl_slist *> aHeaders; |
3457 | | |
3458 | | struct CurlErrBuffer |
3459 | | { |
3460 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3461 | | }; |
3462 | | std::vector<CurlErrBuffer> asCurlErrors( |
3463 | | m_aoAdviseReadRanges.size()); |
3464 | | |
3465 | | std::map<CURL *, size_t> oMapHandleToIdx; |
3466 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
3467 | | { |
3468 | | if (!m_aoAdviseReadRanges[i]->bToRetry) |
3469 | | { |
3470 | | aHandles.push_back(nullptr); |
3471 | | apszRanges.push_back(nullptr); |
3472 | | aHeaders.push_back(nullptr); |
3473 | | continue; |
3474 | | } |
3475 | | m_aoAdviseReadRanges[i]->bToRetry = false; |
3476 | | |
3477 | | CURL *hCurlHandle = curl_easy_init(); |
3478 | | oMapHandleToIdx[hCurlHandle] = i; |
3479 | | aHandles.push_back(hCurlHandle); |
3480 | | |
3481 | | // As the multi-range request is likely not the first one, we don't |
3482 | | // need to wait as we already know if pipelining is possible |
3483 | | // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); |
3484 | | |
3485 | | struct curl_slist *headers = VSICurlSetOptions( |
3486 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3487 | | |
3488 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[i], this, |
3489 | | pfnReadCbk, pReadCbkUserData); |
3490 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
3491 | | &asWriteFuncData[i]); |
3492 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3493 | | VSICurlHandleWriteFunc); |
3494 | | |
3495 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[i], nullptr, |
3496 | | nullptr, nullptr); |
3497 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3498 | | &asWriteFuncHeaderData[i]); |
3499 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3500 | | VSICurlHandleWriteFunc); |
3501 | | asWriteFuncHeaderData[i].bIsHTTP = |
3502 | | STARTS_WITH(m_pszURL, "http"); |
3503 | | asWriteFuncHeaderData[i].nStartOffset = |
3504 | | m_aoAdviseReadRanges[i]->nStartOffset; |
3505 | | |
3506 | | asWriteFuncHeaderData[i].nEndOffset = |
3507 | | m_aoAdviseReadRanges[i]->nStartOffset + |
3508 | | m_aoAdviseReadRanges[i]->nSize - 1; |
3509 | | |
3510 | | char rangeStr[512] = {}; |
3511 | | snprintf(rangeStr, sizeof(rangeStr), |
3512 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3513 | | asWriteFuncHeaderData[i].nStartOffset, |
3514 | | asWriteFuncHeaderData[i].nEndOffset); |
3515 | | |
3516 | | if (ENABLE_DEBUG) |
3517 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
3518 | | rangeStr, osURL.c_str()); |
3519 | | |
3520 | | if (asWriteFuncHeaderData[i].bIsHTTP) |
3521 | | { |
3522 | | std::string osHeaderRange( |
3523 | | CPLSPrintf("Range: bytes=%s", rangeStr)); |
3524 | | // So it gets included in Azure signature |
3525 | | char *pszRange = CPLStrdup(osHeaderRange.c_str()); |
3526 | | apszRanges.push_back(pszRange); |
3527 | | headers = curl_slist_append(headers, pszRange); |
3528 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3529 | | nullptr); |
3530 | | } |
3531 | | else |
3532 | | { |
3533 | | apszRanges.push_back(nullptr); |
3534 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3535 | | rangeStr); |
3536 | | } |
3537 | | |
3538 | | asCurlErrors[i].szCurlErrBuf[0] = '\0'; |
3539 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3540 | | &asCurlErrors[i].szCurlErrBuf[0]); |
3541 | | |
3542 | | headers = GetCurlHeaders("GET", headers); |
3543 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
3544 | | headers); |
3545 | | aHeaders.push_back(headers); |
3546 | | curl_multi_add_handle(m_hCurlMultiHandleForAdviseRead, |
3547 | | hCurlHandle); |
3548 | | } |
3549 | | |
3550 | | const auto DealWithRequest = [this, &osURL, &nTotalDownloaded, |
3551 | | &oMapHandleToIdx, &asCurlErrors, |
3552 | | &asWriteFuncHeaderData, |
3553 | | &asWriteFuncData](CURL *hCurlHandle) |
3554 | | { |
3555 | | auto oIter = oMapHandleToIdx.find(hCurlHandle); |
3556 | | CPLAssert(oIter != oMapHandleToIdx.end()); |
3557 | | const auto iReq = oIter->second; |
3558 | | |
3559 | | long response_code = 0; |
3560 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, |
3561 | | &response_code); |
3562 | | |
3563 | | if (ENABLE_DEBUG && asCurlErrors[iReq].szCurlErrBuf[0] != '\0') |
3564 | | { |
3565 | | char rangeStr[512] = {}; |
3566 | | snprintf(rangeStr, sizeof(rangeStr), |
3567 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3568 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3569 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3570 | | |
3571 | | const char *pszErrorMsg = |
3572 | | &asCurlErrors[iReq].szCurlErrBuf[0]; |
3573 | | CPLDebug(poFS->GetDebugKey(), |
3574 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
3575 | | osURL.c_str(), rangeStr, |
3576 | | static_cast<int>(response_code), pszErrorMsg); |
3577 | | } |
3578 | | |
3579 | | bool bToRetry = false; |
3580 | | if ((response_code != 206 && response_code != 225) || |
3581 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
3582 | | asWriteFuncHeaderData[iReq].nStartOffset + |
3583 | | asWriteFuncData[iReq].nSize) |
3584 | | { |
3585 | | char rangeStr[512] = {}; |
3586 | | snprintf(rangeStr, sizeof(rangeStr), |
3587 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3588 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3589 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3590 | | |
3591 | | // Look if we should attempt a retry |
3592 | | if (m_aoAdviseReadRanges[iReq]->retryContext.CanRetry( |
3593 | | static_cast<int>(response_code), |
3594 | | asWriteFuncData[iReq].pBuffer, |
3595 | | &asCurlErrors[iReq].szCurlErrBuf[0])) |
3596 | | { |
3597 | | CPLError(CE_Warning, CPLE_AppDefined, |
3598 | | "HTTP error code for %s range %s: %d. " |
3599 | | "Retrying again in %.1f secs", |
3600 | | osURL.c_str(), rangeStr, |
3601 | | static_cast<int>(response_code), |
3602 | | m_aoAdviseReadRanges[iReq] |
3603 | | ->retryContext.GetCurrentDelay()); |
3604 | | m_aoAdviseReadRanges[iReq]->dfSleepDelay = |
3605 | | m_aoAdviseReadRanges[iReq] |
3606 | | ->retryContext.GetCurrentDelay(); |
3607 | | bToRetry = true; |
3608 | | } |
3609 | | else |
3610 | | { |
3611 | | CPLError(CE_Failure, CPLE_AppDefined, |
3612 | | "Request for %s range %s failed with " |
3613 | | "response_code=%ld", |
3614 | | osURL.c_str(), rangeStr, response_code); |
3615 | | } |
3616 | | } |
3617 | | else |
3618 | | { |
3619 | | const size_t nSize = asWriteFuncData[iReq].nSize; |
3620 | | memcpy(&m_aoAdviseReadRanges[iReq]->abyData[0], |
3621 | | asWriteFuncData[iReq].pBuffer, nSize); |
3622 | | m_aoAdviseReadRanges[iReq]->abyData.resize(nSize); |
3623 | | |
3624 | | nTotalDownloaded += nSize; |
3625 | | } |
3626 | | |
3627 | | m_aoAdviseReadRanges[iReq]->bToRetry = bToRetry; |
3628 | | |
3629 | | if (!bToRetry) |
3630 | | { |
3631 | | std::lock_guard<std::mutex> oLock( |
3632 | | m_aoAdviseReadRanges[iReq]->oMutex); |
3633 | | m_aoAdviseReadRanges[iReq]->bDone = true; |
3634 | | m_aoAdviseReadRanges[iReq]->oCV.notify_all(); |
3635 | | } |
3636 | | }; |
3637 | | |
3638 | | int repeats = 0; |
3639 | | |
3640 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
3641 | | while (true) |
3642 | | { |
3643 | | int still_running; |
3644 | | while (curl_multi_perform(m_hCurlMultiHandleForAdviseRead, |
3645 | | &still_running) == |
3646 | | CURLM_CALL_MULTI_PERFORM) |
3647 | | { |
3648 | | // loop |
3649 | | } |
3650 | | if (!still_running) |
3651 | | { |
3652 | | break; |
3653 | | } |
3654 | | |
3655 | | CURLMsg *msg; |
3656 | | do |
3657 | | { |
3658 | | int msgq = 0; |
3659 | | msg = curl_multi_info_read(m_hCurlMultiHandleForAdviseRead, |
3660 | | &msgq); |
3661 | | if (msg && (msg->msg == CURLMSG_DONE)) |
3662 | | { |
3663 | | DealWithRequest(msg->easy_handle); |
3664 | | } |
3665 | | } while (msg); |
3666 | | |
3667 | | CPLMultiPerformWait(m_hCurlMultiHandleForAdviseRead, repeats); |
3668 | | } |
3669 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
3670 | | |
3671 | | bool bRetry = false; |
3672 | | double dfDelay = 0.0; |
3673 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
3674 | | { |
3675 | | bool bReqDone; |
3676 | | { |
3677 | | // To please Coverity Scan |
3678 | | std::lock_guard<std::mutex> oLock( |
3679 | | m_aoAdviseReadRanges[i]->oMutex); |
3680 | | bReqDone = m_aoAdviseReadRanges[i]->bDone; |
3681 | | } |
3682 | | if (!bReqDone && !m_aoAdviseReadRanges[i]->bToRetry) |
3683 | | { |
3684 | | DealWithRequest(aHandles[i]); |
3685 | | } |
3686 | | if (m_aoAdviseReadRanges[i]->bToRetry) |
3687 | | dfDelay = std::max(dfDelay, |
3688 | | m_aoAdviseReadRanges[i]->dfSleepDelay); |
3689 | | bRetry = bRetry || m_aoAdviseReadRanges[i]->bToRetry; |
3690 | | if (aHandles[i]) |
3691 | | { |
3692 | | curl_multi_remove_handle(m_hCurlMultiHandleForAdviseRead, |
3693 | | aHandles[i]); |
3694 | | VSICURLResetHeaderAndWriterFunctions(aHandles[i]); |
3695 | | curl_easy_cleanup(aHandles[i]); |
3696 | | } |
3697 | | CPLFree(apszRanges[i]); |
3698 | | CPLFree(asWriteFuncData[i].pBuffer); |
3699 | | CPLFree(asWriteFuncHeaderData[i].pBuffer); |
3700 | | if (aHeaders[i]) |
3701 | | curl_slist_free_all(aHeaders[i]); |
3702 | | } |
3703 | | if (!bRetry) |
3704 | | break; |
3705 | | CPLSleep(dfDelay); |
3706 | | } |
3707 | | |
3708 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
3709 | | }; |
3710 | | |
3711 | | m_oThreadAdviseRead = std::thread(task, l_osURL); |
3712 | | } |
3713 | | |
3714 | | /************************************************************************/ |
3715 | | /* Write() */ |
3716 | | /************************************************************************/ |
3717 | | |
3718 | | size_t VSICurlHandle::Write(const void * /* pBuffer */, size_t /* nSize */, |
3719 | | size_t /* nMemb */) |
3720 | | { |
3721 | | return 0; |
3722 | | } |
3723 | | |
3724 | | /************************************************************************/ |
3725 | | /* ClearErr() */ |
3726 | | /************************************************************************/ |
3727 | | |
3728 | | void VSICurlHandle::ClearErr() |
3729 | | |
3730 | | { |
3731 | | bEOF = false; |
3732 | | bError = false; |
3733 | | } |
3734 | | |
3735 | | /************************************************************************/ |
3736 | | /* Error() */ |
3737 | | /************************************************************************/ |
3738 | | |
3739 | | int VSICurlHandle::Error() |
3740 | | |
3741 | | { |
3742 | | return bError ? TRUE : FALSE; |
3743 | | } |
3744 | | |
3745 | | /************************************************************************/ |
3746 | | /* Eof() */ |
3747 | | /************************************************************************/ |
3748 | | |
3749 | | int VSICurlHandle::Eof() |
3750 | | |
3751 | | { |
3752 | | return bEOF ? TRUE : FALSE; |
3753 | | } |
3754 | | |
3755 | | /************************************************************************/ |
3756 | | /* Flush() */ |
3757 | | /************************************************************************/ |
3758 | | |
3759 | | int VSICurlHandle::Flush() |
3760 | | { |
3761 | | return 0; |
3762 | | } |
3763 | | |
3764 | | /************************************************************************/ |
3765 | | /* Close() */ |
3766 | | /************************************************************************/ |
3767 | | |
3768 | | int VSICurlHandle::Close() |
3769 | | { |
3770 | | return 0; |
3771 | | } |
3772 | | |
3773 | | /************************************************************************/ |
3774 | | /* VSICurlFilesystemHandlerBase() */ |
3775 | | /************************************************************************/ |
3776 | | |
3777 | | VSICurlFilesystemHandlerBase::VSICurlFilesystemHandlerBase() |
3778 | | : oCacheFileProp{100 * 1024}, oCacheDirList{1024, 0} |
3779 | | { |
3780 | | } |
3781 | | |
3782 | | /************************************************************************/ |
3783 | | /* CachedConnection */ |
3784 | | /************************************************************************/ |
3785 | | |
3786 | | namespace |
3787 | | { |
3788 | | struct CachedConnection |
3789 | | { |
3790 | | CURLM *hCurlMultiHandle = nullptr; |
3791 | | void clear(); |
3792 | | |
3793 | | ~CachedConnection() |
3794 | | { |
3795 | | clear(); |
3796 | | } |
3797 | | }; |
3798 | | } // namespace |
3799 | | |
3800 | | #ifdef _WIN32 |
3801 | | // Currently thread_local and C++ objects don't work well with DLL on Windows |
3802 | | static void FreeCachedConnection(void *pData) |
3803 | | { |
3804 | | delete static_cast< |
3805 | | std::map<VSICurlFilesystemHandlerBase *, CachedConnection> *>(pData); |
3806 | | } |
3807 | | |
3808 | | // Per-thread and per-filesystem Curl connection cache. |
3809 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> & |
3810 | | GetConnectionCache() |
3811 | | { |
3812 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> |
3813 | | dummyCache; |
3814 | | int bMemoryErrorOccurred = false; |
3815 | | void *pData = |
3816 | | CPLGetTLSEx(CTLS_VSICURL_CACHEDCONNECTION, &bMemoryErrorOccurred); |
3817 | | if (bMemoryErrorOccurred) |
3818 | | { |
3819 | | return dummyCache; |
3820 | | } |
3821 | | if (pData == nullptr) |
3822 | | { |
3823 | | auto cachedConnection = |
3824 | | new std::map<VSICurlFilesystemHandlerBase *, CachedConnection>(); |
3825 | | CPLSetTLSWithFreeFuncEx(CTLS_VSICURL_CACHEDCONNECTION, cachedConnection, |
3826 | | FreeCachedConnection, &bMemoryErrorOccurred); |
3827 | | if (bMemoryErrorOccurred) |
3828 | | { |
3829 | | delete cachedConnection; |
3830 | | return dummyCache; |
3831 | | } |
3832 | | return *cachedConnection; |
3833 | | } |
3834 | | return *static_cast< |
3835 | | std::map<VSICurlFilesystemHandlerBase *, CachedConnection> *>(pData); |
3836 | | } |
3837 | | #else |
3838 | | static thread_local std::map<VSICurlFilesystemHandlerBase *, CachedConnection> |
3839 | | g_tls_connectionCache; |
3840 | | |
3841 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> & |
3842 | | GetConnectionCache() |
3843 | | { |
3844 | | return g_tls_connectionCache; |
3845 | | } |
3846 | | #endif |
3847 | | |
3848 | | /************************************************************************/ |
3849 | | /* clear() */ |
3850 | | /************************************************************************/ |
3851 | | |
3852 | | void CachedConnection::clear() |
3853 | | { |
3854 | | if (hCurlMultiHandle) |
3855 | | { |
3856 | | VSICURLMultiCleanup(hCurlMultiHandle); |
3857 | | hCurlMultiHandle = nullptr; |
3858 | | } |
3859 | | } |
3860 | | |
3861 | | /************************************************************************/ |
3862 | | /* ~VSICurlFilesystemHandlerBase() */ |
3863 | | /************************************************************************/ |
3864 | | |
3865 | | VSICurlFilesystemHandlerBase::~VSICurlFilesystemHandlerBase() |
3866 | | { |
3867 | | VSICurlFilesystemHandlerBase::ClearCache(); |
3868 | | GetConnectionCache().erase(this); |
3869 | | |
3870 | | if (hMutex != nullptr) |
3871 | | CPLDestroyMutex(hMutex); |
3872 | | hMutex = nullptr; |
3873 | | } |
3874 | | |
3875 | | /************************************************************************/ |
3876 | | /* AllowCachedDataFor() */ |
3877 | | /************************************************************************/ |
3878 | | |
3879 | | bool VSICurlFilesystemHandlerBase::AllowCachedDataFor(const char *pszFilename) |
3880 | | { |
3881 | | bool bCachedAllowed = true; |
3882 | | char **papszTokens = CSLTokenizeString2( |
3883 | | CPLGetConfigOption("CPL_VSIL_CURL_NON_CACHED", ""), ":", 0); |
3884 | | for (int i = 0; papszTokens && papszTokens[i]; i++) |
3885 | | { |
3886 | | if (STARTS_WITH(pszFilename, papszTokens[i])) |
3887 | | { |
3888 | | bCachedAllowed = false; |
3889 | | break; |
3890 | | } |
3891 | | } |
3892 | | CSLDestroy(papszTokens); |
3893 | | return bCachedAllowed; |
3894 | | } |
3895 | | |
3896 | | /************************************************************************/ |
3897 | | /* GetCurlMultiHandleFor() */ |
3898 | | /************************************************************************/ |
3899 | | |
3900 | | CURLM *VSICurlFilesystemHandlerBase::GetCurlMultiHandleFor( |
3901 | | const std::string & /*osURL*/) |
3902 | | { |
3903 | | auto &conn = GetConnectionCache()[this]; |
3904 | | if (conn.hCurlMultiHandle == nullptr) |
3905 | | { |
3906 | | conn.hCurlMultiHandle = VSICURLMultiInit(); |
3907 | | } |
3908 | | return conn.hCurlMultiHandle; |
3909 | | } |
3910 | | |
3911 | | /************************************************************************/ |
3912 | | /* GetRegionCache() */ |
3913 | | /************************************************************************/ |
3914 | | |
3915 | | VSICurlFilesystemHandlerBase::RegionCacheType * |
3916 | | VSICurlFilesystemHandlerBase::GetRegionCache() |
3917 | | { |
3918 | | // should be called under hMutex taken |
3919 | | if (m_poRegionCacheDoNotUseDirectly == nullptr) |
3920 | | { |
3921 | | m_poRegionCacheDoNotUseDirectly.reset( |
3922 | | new RegionCacheType(static_cast<size_t>(GetMaxRegions()))); |
3923 | | } |
3924 | | return m_poRegionCacheDoNotUseDirectly.get(); |
3925 | | } |
3926 | | |
3927 | | /************************************************************************/ |
3928 | | /* GetRegion() */ |
3929 | | /************************************************************************/ |
3930 | | |
3931 | | std::shared_ptr<std::string> |
3932 | | VSICurlFilesystemHandlerBase::GetRegion(const char *pszURL, |
3933 | | vsi_l_offset nFileOffsetStart) |
3934 | | { |
3935 | | CPLMutexHolder oHolder(&hMutex); |
3936 | | |
3937 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
3938 | | nFileOffsetStart = |
3939 | | (nFileOffsetStart / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
3940 | | |
3941 | | std::shared_ptr<std::string> out; |
3942 | | if (GetRegionCache()->tryGet( |
3943 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), out)) |
3944 | | { |
3945 | | return out; |
3946 | | } |
3947 | | |
3948 | | return nullptr; |
3949 | | } |
3950 | | |
3951 | | /************************************************************************/ |
3952 | | /* AddRegion() */ |
3953 | | /************************************************************************/ |
3954 | | |
3955 | | void VSICurlFilesystemHandlerBase::AddRegion(const char *pszURL, |
3956 | | vsi_l_offset nFileOffsetStart, |
3957 | | size_t nSize, const char *pData) |
3958 | | { |
3959 | | CPLMutexHolder oHolder(&hMutex); |
3960 | | |
3961 | | auto value = std::make_shared<std::string>(); |
3962 | | value->assign(pData, nSize); |
3963 | | GetRegionCache()->insert( |
3964 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), |
3965 | | std::move(value)); |
3966 | | } |
3967 | | |
3968 | | /************************************************************************/ |
3969 | | /* GetCachedFileProp() */ |
3970 | | /************************************************************************/ |
3971 | | |
3972 | | bool VSICurlFilesystemHandlerBase::GetCachedFileProp(const char *pszURL, |
3973 | | FileProp &oFileProp) |
3974 | | { |
3975 | | CPLMutexHolder oHolder(&hMutex); |
3976 | | bool inCache; |
3977 | | if (oCacheFileProp.tryGet(std::string(pszURL), inCache)) |
3978 | | { |
3979 | | if (VSICURLGetCachedFileProp(pszURL, oFileProp)) |
3980 | | { |
3981 | | return true; |
3982 | | } |
3983 | | oCacheFileProp.remove(std::string(pszURL)); |
3984 | | } |
3985 | | return false; |
3986 | | } |
3987 | | |
3988 | | /************************************************************************/ |
3989 | | /* SetCachedFileProp() */ |
3990 | | /************************************************************************/ |
3991 | | |
3992 | | void VSICurlFilesystemHandlerBase::SetCachedFileProp(const char *pszURL, |
3993 | | FileProp &oFileProp) |
3994 | | { |
3995 | | CPLMutexHolder oHolder(&hMutex); |
3996 | | oCacheFileProp.insert(std::string(pszURL), true); |
3997 | | VSICURLSetCachedFileProp(pszURL, oFileProp); |
3998 | | } |
3999 | | |
4000 | | /************************************************************************/ |
4001 | | /* GetCachedDirList() */ |
4002 | | /************************************************************************/ |
4003 | | |
4004 | | bool VSICurlFilesystemHandlerBase::GetCachedDirList( |
4005 | | const char *pszURL, CachedDirList &oCachedDirList) |
4006 | | { |
4007 | | CPLMutexHolder oHolder(&hMutex); |
4008 | | |
4009 | | return oCacheDirList.tryGet(std::string(pszURL), oCachedDirList) && |
4010 | | // Let a chance to use new auth parameters |
4011 | | gnGenerationAuthParameters == |
4012 | | oCachedDirList.nGenerationAuthParameters; |
4013 | | } |
4014 | | |
4015 | | /************************************************************************/ |
4016 | | /* SetCachedDirList() */ |
4017 | | /************************************************************************/ |
4018 | | |
4019 | | void VSICurlFilesystemHandlerBase::SetCachedDirList( |
4020 | | const char *pszURL, CachedDirList &oCachedDirList) |
4021 | | { |
4022 | | CPLMutexHolder oHolder(&hMutex); |
4023 | | |
4024 | | std::string key(pszURL); |
4025 | | CachedDirList oldValue; |
4026 | | if (oCacheDirList.tryGet(key, oldValue)) |
4027 | | { |
4028 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4029 | | oCacheDirList.remove(key); |
4030 | | } |
4031 | | |
4032 | | while ((!oCacheDirList.empty() && |
4033 | | nCachedFilesInDirList + oCachedDirList.oFileList.size() > |
4034 | | 1024 * 1024) || |
4035 | | oCacheDirList.size() == oCacheDirList.getMaxAllowedSize()) |
4036 | | { |
4037 | | std::string oldestKey; |
4038 | | oCacheDirList.getOldestEntry(oldestKey, oldValue); |
4039 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4040 | | oCacheDirList.remove(oldestKey); |
4041 | | } |
4042 | | oCachedDirList.nGenerationAuthParameters = gnGenerationAuthParameters; |
4043 | | |
4044 | | nCachedFilesInDirList += oCachedDirList.oFileList.size(); |
4045 | | oCacheDirList.insert(key, oCachedDirList); |
4046 | | } |
4047 | | |
4048 | | /************************************************************************/ |
4049 | | /* ExistsInCacheDirList() */ |
4050 | | /************************************************************************/ |
4051 | | |
4052 | | bool VSICurlFilesystemHandlerBase::ExistsInCacheDirList( |
4053 | | const std::string &osDirname, bool *pbIsDir) |
4054 | | { |
4055 | | CachedDirList cachedDirList; |
4056 | | if (GetCachedDirList(osDirname.c_str(), cachedDirList)) |
4057 | | { |
4058 | | if (pbIsDir) |
4059 | | *pbIsDir = !cachedDirList.oFileList.empty(); |
4060 | | return false; |
4061 | | } |
4062 | | else |
4063 | | { |
4064 | | if (pbIsDir) |
4065 | | *pbIsDir = false; |
4066 | | return false; |
4067 | | } |
4068 | | } |
4069 | | |
4070 | | /************************************************************************/ |
4071 | | /* InvalidateCachedData() */ |
4072 | | /************************************************************************/ |
4073 | | |
4074 | | void VSICurlFilesystemHandlerBase::InvalidateCachedData(const char *pszURL) |
4075 | | { |
4076 | | CPLMutexHolder oHolder(&hMutex); |
4077 | | |
4078 | | oCacheFileProp.remove(std::string(pszURL)); |
4079 | | |
4080 | | // Invalidate all cached regions for this URL |
4081 | | std::list<FilenameOffsetPair> keysToRemove; |
4082 | | std::string osURL(pszURL); |
4083 | | auto lambda = |
4084 | | [&keysToRemove, |
4085 | | &osURL](const lru11::KeyValuePair<FilenameOffsetPair, |
4086 | | std::shared_ptr<std::string>> &kv) |
4087 | | { |
4088 | | if (kv.key.filename_ == osURL) |
4089 | | keysToRemove.push_back(kv.key); |
4090 | | }; |
4091 | | auto *poRegionCache = GetRegionCache(); |
4092 | | poRegionCache->cwalk(lambda); |
4093 | | for (const auto &key : keysToRemove) |
4094 | | poRegionCache->remove(key); |
4095 | | } |
4096 | | |
4097 | | /************************************************************************/ |
4098 | | /* ClearCache() */ |
4099 | | /************************************************************************/ |
4100 | | |
4101 | | void VSICurlFilesystemHandlerBase::ClearCache() |
4102 | | { |
4103 | | CPLMutexHolder oHolder(&hMutex); |
4104 | | |
4105 | | GetRegionCache()->clear(); |
4106 | | |
4107 | | { |
4108 | | const auto lambda = [](const lru11::KeyValuePair<std::string, bool> &kv) |
4109 | | { VSICURLInvalidateCachedFileProp(kv.key.c_str()); }; |
4110 | | oCacheFileProp.cwalk(lambda); |
4111 | | oCacheFileProp.clear(); |
4112 | | } |
4113 | | |
4114 | | oCacheDirList.clear(); |
4115 | | nCachedFilesInDirList = 0; |
4116 | | |
4117 | | GetConnectionCache()[this].clear(); |
4118 | | } |
4119 | | |
4120 | | /************************************************************************/ |
4121 | | /* PartialClearCache() */ |
4122 | | /************************************************************************/ |
4123 | | |
4124 | | void VSICurlFilesystemHandlerBase::PartialClearCache( |
4125 | | const char *pszFilenamePrefix) |
4126 | | { |
4127 | | CPLMutexHolder oHolder(&hMutex); |
4128 | | |
4129 | | std::string osURL = GetURLFromFilename(pszFilenamePrefix); |
4130 | | { |
4131 | | std::list<FilenameOffsetPair> keysToRemove; |
4132 | | auto lambda = |
4133 | | [&keysToRemove, &osURL]( |
4134 | | const lru11::KeyValuePair<FilenameOffsetPair, |
4135 | | std::shared_ptr<std::string>> &kv) |
4136 | | { |
4137 | | if (strncmp(kv.key.filename_.c_str(), osURL.c_str(), |
4138 | | osURL.size()) == 0) |
4139 | | keysToRemove.push_back(kv.key); |
4140 | | }; |
4141 | | auto *poRegionCache = GetRegionCache(); |
4142 | | poRegionCache->cwalk(lambda); |
4143 | | for (const auto &key : keysToRemove) |
4144 | | poRegionCache->remove(key); |
4145 | | } |
4146 | | |
4147 | | { |
4148 | | std::list<std::string> keysToRemove; |
4149 | | auto lambda = [&keysToRemove, |
4150 | | &osURL](const lru11::KeyValuePair<std::string, bool> &kv) |
4151 | | { |
4152 | | if (strncmp(kv.key.c_str(), osURL.c_str(), osURL.size()) == 0) |
4153 | | keysToRemove.push_back(kv.key); |
4154 | | }; |
4155 | | oCacheFileProp.cwalk(lambda); |
4156 | | for (const auto &key : keysToRemove) |
4157 | | oCacheFileProp.remove(key); |
4158 | | } |
4159 | | VSICURLInvalidateCachedFilePropPrefix(osURL.c_str()); |
4160 | | |
4161 | | { |
4162 | | const size_t nLen = strlen(pszFilenamePrefix); |
4163 | | std::list<std::string> keysToRemove; |
4164 | | auto lambda = |
4165 | | [this, &keysToRemove, pszFilenamePrefix, |
4166 | | nLen](const lru11::KeyValuePair<std::string, CachedDirList> &kv) |
4167 | | { |
4168 | | if (strncmp(kv.key.c_str(), pszFilenamePrefix, nLen) == 0) |
4169 | | { |
4170 | | keysToRemove.push_back(kv.key); |
4171 | | nCachedFilesInDirList -= kv.value.oFileList.size(); |
4172 | | } |
4173 | | }; |
4174 | | oCacheDirList.cwalk(lambda); |
4175 | | for (const auto &key : keysToRemove) |
4176 | | oCacheDirList.remove(key); |
4177 | | } |
4178 | | } |
4179 | | |
4180 | | /************************************************************************/ |
4181 | | /* CreateFileHandle() */ |
4182 | | /************************************************************************/ |
4183 | | |
4184 | | VSICurlHandle * |
4185 | | VSICurlFilesystemHandlerBase::CreateFileHandle(const char *pszFilename) |
4186 | | { |
4187 | | return new VSICurlHandle(this, pszFilename); |
4188 | | } |
4189 | | |
4190 | | /************************************************************************/ |
4191 | | /* GetActualURL() */ |
4192 | | /************************************************************************/ |
4193 | | |
4194 | | const char *VSICurlFilesystemHandlerBase::GetActualURL(const char *pszFilename) |
4195 | | { |
4196 | | VSICurlHandle *poHandle = CreateFileHandle(pszFilename); |
4197 | | if (poHandle == nullptr) |
4198 | | return pszFilename; |
4199 | | std::string osURL(poHandle->GetURL()); |
4200 | | delete poHandle; |
4201 | | return CPLSPrintf("%s", osURL.c_str()); |
4202 | | } |
4203 | | |
4204 | | /************************************************************************/ |
4205 | | /* GetOptions() */ |
4206 | | /************************************************************************/ |
4207 | | |
4208 | | #define VSICURL_OPTIONS \ |
4209 | | " <Option name='GDAL_HTTP_MAX_RETRY' type='int' " \ |
4210 | | "description='Maximum number of retries' default='0'/>" \ |
4211 | | " <Option name='GDAL_HTTP_RETRY_DELAY' type='double' " \ |
4212 | | "description='Retry delay in seconds' default='30'/>" \ |
4213 | | " <Option name='GDAL_HTTP_HEADER_FILE' type='string' " \ |
4214 | | "description='Filename of a file that contains HTTP headers to " \ |
4215 | | "forward to the server'/>" \ |
4216 | | " <Option name='CPL_VSIL_CURL_USE_HEAD' type='boolean' " \ |
4217 | | "description='Whether to use HTTP HEAD verb to retrieve " \ |
4218 | | "file information' default='YES'/>" \ |
4219 | | " <Option name='GDAL_HTTP_MULTIRANGE' type='string-select' " \ |
4220 | | "description='Strategy to apply to run multi-range requests' " \ |
4221 | | "default='PARALLEL'>" \ |
4222 | | " <Value>PARALLEL</Value>" \ |
4223 | | " <Value>SERIAL</Value>" \ |
4224 | | " </Option>" \ |
4225 | | " <Option name='GDAL_HTTP_MULTIPLEX' type='boolean' " \ |
4226 | | "description='Whether to enable HTTP/2 multiplexing' default='YES'/>" \ |
4227 | | " <Option name='GDAL_HTTP_MERGE_CONSECUTIVE_RANGES' type='boolean' " \ |
4228 | | "description='Whether to merge consecutive ranges in multirange " \ |
4229 | | "requests' default='YES'/>" \ |
4230 | | " <Option name='CPL_VSIL_CURL_NON_CACHED' type='string' " \ |
4231 | | "description='Colon-separated list of filenames whose content" \ |
4232 | | "must not be cached across open attempts'/>" \ |
4233 | | " <Option name='CPL_VSIL_CURL_ALLOWED_FILENAME' type='string' " \ |
4234 | | "description='Single filename that is allowed to be opened'/>" \ |
4235 | | " <Option name='CPL_VSIL_CURL_ALLOWED_EXTENSIONS' type='string' " \ |
4236 | | "description='Comma or space separated list of allowed file " \ |
4237 | | "extensions'/>" \ |
4238 | | " <Option name='GDAL_DISABLE_READDIR_ON_OPEN' type='string-select' " \ |
4239 | | "description='Whether to disable establishing the list of files in " \ |
4240 | | "the directory of the current filename' default='NO'>" \ |
4241 | | " <Value>NO</Value>" \ |
4242 | | " <Value>YES</Value>" \ |
4243 | | " <Value>EMPTY_DIR</Value>" \ |
4244 | | " </Option>" \ |
4245 | | " <Option name='VSI_CACHE' type='boolean' " \ |
4246 | | "description='Whether to cache in memory the contents of the opened " \ |
4247 | | "file as soon as they are read' default='NO'/>" \ |
4248 | | " <Option name='CPL_VSIL_CURL_CHUNK_SIZE' type='integer' " \ |
4249 | | "description='Size in bytes of the minimum amount of data read in a " \ |
4250 | | "file' default='16384' min='1024' max='10485760'/>" \ |
4251 | | " <Option name='CPL_VSIL_CURL_CACHE_SIZE' type='integer' " \ |
4252 | | "description='Size in bytes of the global /vsicurl/ cache' " \ |
4253 | | "default='16384000'/>" \ |
4254 | | " <Option name='CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE' type='boolean' " \ |
4255 | | "description='Whether to skip files with Glacier storage class in " \ |
4256 | | "directory listing.' default='YES'/>" \ |
4257 | | " <Option name='CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT' " \ |
4258 | | "type='integer' description='Maximum number of bytes AdviseRead() is " \ |
4259 | | "allowed to fetch at once' default='104857600'/>" \ |
4260 | | " <Option name='GDAL_HTTP_MAX_CACHED_CONNECTIONS' type='integer' " \ |
4261 | | "description='Maximum amount of connections that libcurl may keep alive " \ |
4262 | | "in its connection cache after use'/>" \ |
4263 | | " <Option name='GDAL_HTTP_MAX_TOTAL_CONNECTIONS' type='integer' " \ |
4264 | | "description='Maximum number of simultaneously open connections in " \ |
4265 | | "total'/>" |
4266 | | |
4267 | | const char *VSICurlFilesystemHandlerBase::GetOptionsStatic() |
4268 | | { |
4269 | | return VSICURL_OPTIONS; |
4270 | | } |
4271 | | |
4272 | | const char *VSICurlFilesystemHandlerBase::GetOptions() |
4273 | | { |
4274 | | static std::string osOptions(std::string("<Options>") + GetOptionsStatic() + |
4275 | | "</Options>"); |
4276 | | return osOptions.c_str(); |
4277 | | } |
4278 | | |
4279 | | /************************************************************************/ |
4280 | | /* IsAllowedFilename() */ |
4281 | | /************************************************************************/ |
4282 | | |
4283 | | bool VSICurlFilesystemHandlerBase::IsAllowedFilename(const char *pszFilename) |
4284 | | { |
4285 | | const char *pszAllowedFilename = |
4286 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_FILENAME", nullptr); |
4287 | | if (pszAllowedFilename != nullptr) |
4288 | | { |
4289 | | return strcmp(pszFilename, pszAllowedFilename) == 0; |
4290 | | } |
4291 | | |
4292 | | // Consider that only the files whose extension ends up with one that is |
4293 | | // listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server. This can |
4294 | | // speeds up dramatically open experience, in case the server cannot return |
4295 | | // a file list. {noext} can be used as a special token to mean file with no |
4296 | | // extension. |
4297 | | // For example: |
4298 | | // gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" |
4299 | | // /vsicurl/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif |
4300 | | const char *pszAllowedExtensions = |
4301 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", nullptr); |
4302 | | if (pszAllowedExtensions) |
4303 | | { |
4304 | | char **papszExtensions = |
4305 | | CSLTokenizeString2(pszAllowedExtensions, ", ", 0); |
4306 | | const char *queryStart = strchr(pszFilename, '?'); |
4307 | | char *pszFilenameWithoutQuery = nullptr; |
4308 | | if (queryStart != nullptr) |
4309 | | { |
4310 | | pszFilenameWithoutQuery = CPLStrdup(pszFilename); |
4311 | | pszFilenameWithoutQuery[queryStart - pszFilename] = '\0'; |
4312 | | pszFilename = pszFilenameWithoutQuery; |
4313 | | } |
4314 | | const size_t nURLLen = strlen(pszFilename); |
4315 | | bool bFound = false; |
4316 | | for (int i = 0; papszExtensions[i] != nullptr; i++) |
4317 | | { |
4318 | | const size_t nExtensionLen = strlen(papszExtensions[i]); |
4319 | | if (EQUAL(papszExtensions[i], "{noext}")) |
4320 | | { |
4321 | | const char *pszLastSlash = strrchr(pszFilename, '/'); |
4322 | | if (pszLastSlash != nullptr && |
4323 | | strchr(pszLastSlash, '.') == nullptr) |
4324 | | { |
4325 | | bFound = true; |
4326 | | break; |
4327 | | } |
4328 | | } |
4329 | | else if (nURLLen > nExtensionLen && |
4330 | | EQUAL(pszFilename + nURLLen - nExtensionLen, |
4331 | | papszExtensions[i])) |
4332 | | { |
4333 | | bFound = true; |
4334 | | break; |
4335 | | } |
4336 | | } |
4337 | | |
4338 | | CSLDestroy(papszExtensions); |
4339 | | if (pszFilenameWithoutQuery) |
4340 | | { |
4341 | | CPLFree(pszFilenameWithoutQuery); |
4342 | | } |
4343 | | |
4344 | | return bFound; |
4345 | | } |
4346 | | return TRUE; |
4347 | | } |
4348 | | |
4349 | | /************************************************************************/ |
4350 | | /* Open() */ |
4351 | | /************************************************************************/ |
4352 | | |
4353 | | VSIVirtualHandleUniquePtr |
4354 | | VSICurlFilesystemHandlerBase::Open(const char *pszFilename, |
4355 | | const char *pszAccess, bool bSetError, |
4356 | | CSLConstList papszOptions) |
4357 | | { |
4358 | | if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str()) && |
4359 | | !STARTS_WITH_CI(pszFilename, "/vsicurl?")) |
4360 | | return nullptr; |
4361 | | |
4362 | | if (strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, '+') != nullptr) |
4363 | | { |
4364 | | if (bSetError) |
4365 | | { |
4366 | | VSIError(VSIE_FileError, |
4367 | | "Only read-only mode is supported for /vsicurl"); |
4368 | | } |
4369 | | return nullptr; |
4370 | | } |
4371 | | if (!papszOptions || |
4372 | | !CPLTestBool(CSLFetchNameValueDef( |
4373 | | papszOptions, "IGNORE_FILENAME_RESTRICTIONS", "NO"))) |
4374 | | { |
4375 | | if (!IsAllowedFilename(pszFilename)) |
4376 | | return nullptr; |
4377 | | } |
4378 | | |
4379 | | bool bListDir = true; |
4380 | | bool bEmptyDir = false; |
4381 | | CPL_IGNORE_RET_VAL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
4382 | | nullptr, &bListDir, &bEmptyDir, |
4383 | | nullptr, nullptr, nullptr)); |
4384 | | |
4385 | | const char *pszOptionVal = CSLFetchNameValueDef( |
4386 | | papszOptions, "DISABLE_READDIR_ON_OPEN", |
4387 | | VSIGetPathSpecificOption(pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", |
4388 | | "NO")); |
4389 | | const bool bCache = CPLTestBool(CSLFetchNameValueDef( |
4390 | | papszOptions, "CACHE", AllowCachedDataFor(pszFilename) ? "YES" : "NO")); |
4391 | | const bool bSkipReadDir = !bListDir || bEmptyDir || |
4392 | | EQUAL(pszOptionVal, "EMPTY_DIR") || |
4393 | | CPLTestBool(pszOptionVal) || !bCache; |
4394 | | |
4395 | | std::string osFilename(pszFilename); |
4396 | | bool bGotFileList = !bSkipReadDir; |
4397 | | bool bForceExistsCheck = false; |
4398 | | FileProp cachedFileProp; |
4399 | | if (!bSkipReadDir && |
4400 | | !(GetCachedFileProp(osFilename.c_str() + strlen(GetFSPrefix().c_str()), |
4401 | | cachedFileProp) && |
4402 | | cachedFileProp.eExists == EXIST_YES) && |
4403 | | strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
4404 | | !STARTS_WITH(CPLGetExtensionSafe(osFilename.c_str()).c_str(), "zip") && |
4405 | | // Likely a Kerchunk JSON reference file: no need to list siblings |
4406 | | !cpl::ends_with(osFilename, ".nc.zarr")) |
4407 | | { |
4408 | | // 1000 corresponds to the default page size of S3. |
4409 | | constexpr int FILE_COUNT_LIMIT = 1000; |
4410 | | const CPLStringList aosFileList(ReadDirInternal( |
4411 | | (CPLGetDirnameSafe(osFilename.c_str()) + '/').c_str(), |
4412 | | FILE_COUNT_LIMIT, &bGotFileList)); |
4413 | | const bool bFound = |
4414 | | VSICurlIsFileInList(aosFileList.List(), |
4415 | | CPLGetFilename(osFilename.c_str())) != -1; |
4416 | | if (bGotFileList && !bFound && aosFileList.size() < FILE_COUNT_LIMIT) |
4417 | | { |
4418 | | // Some file servers are case insensitive, so in case there is a |
4419 | | // match with case difference, do a full check just in case. |
4420 | | // e.g. |
4421 | | // http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/MEGA90N000CB.IMG |
4422 | | // that is queried by |
4423 | | // gdalinfo |
4424 | | // /vsicurl/http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/mega90n000cb.lbl |
4425 | | if (aosFileList.FindString(CPLGetFilename(osFilename.c_str())) != |
4426 | | -1) |
4427 | | { |
4428 | | bForceExistsCheck = true; |
4429 | | } |
4430 | | else |
4431 | | { |
4432 | | return nullptr; |
4433 | | } |
4434 | | } |
4435 | | } |
4436 | | |
4437 | | auto poHandle = |
4438 | | std::unique_ptr<VSICurlHandle>(CreateFileHandle(osFilename.c_str())); |
4439 | | if (poHandle == nullptr) |
4440 | | return nullptr; |
4441 | | poHandle->SetCache(bCache); |
4442 | | if (!bGotFileList || bForceExistsCheck) |
4443 | | { |
4444 | | // If we didn't get a filelist, check that the file really exists. |
4445 | | if (!poHandle->Exists(bSetError)) |
4446 | | { |
4447 | | return nullptr; |
4448 | | } |
4449 | | } |
4450 | | |
4451 | | if (CPLTestBool(CPLGetConfigOption("VSI_CACHE", "FALSE"))) |
4452 | | return VSIVirtualHandleUniquePtr( |
4453 | | VSICreateCachedFile(poHandle.release())); |
4454 | | else |
4455 | | return VSIVirtualHandleUniquePtr(poHandle.release()); |
4456 | | } |
4457 | | |
4458 | | /************************************************************************/ |
4459 | | /* VSICurlParserFindEOL() */ |
4460 | | /* */ |
4461 | | /* Small helper function for VSICurlPaseHTMLFileList() to find */ |
4462 | | /* the end of a line in the directory listing. Either a <br> */ |
4463 | | /* or newline. */ |
4464 | | /************************************************************************/ |
4465 | | |
4466 | | static char *VSICurlParserFindEOL(char *pszData) |
4467 | | |
4468 | | { |
4469 | | while (*pszData != '\0' && *pszData != '\n' && |
4470 | | !STARTS_WITH_CI(pszData, "<br>")) |
4471 | | pszData++; |
4472 | | |
4473 | | if (*pszData == '\0') |
4474 | | return nullptr; |
4475 | | |
4476 | | return pszData; |
4477 | | } |
4478 | | |
4479 | | /************************************************************************/ |
4480 | | /* VSICurlParseHTMLDateTimeFileSize() */ |
4481 | | /************************************************************************/ |
4482 | | |
4483 | | static const char *const apszMonths[] = { |
4484 | | "January", "February", "March", "April", "May", "June", |
4485 | | "July", "August", "September", "October", "November", "December"}; |
4486 | | |
4487 | | static bool VSICurlParseHTMLDateTimeFileSize(const char *pszStr, |
4488 | | struct tm &brokendowntime, |
4489 | | GUIntBig &nFileSize, |
4490 | | GIntBig &mTime) |
4491 | | { |
4492 | | for (int iMonth = 0; iMonth < 12; iMonth++) |
4493 | | { |
4494 | | char szMonth[32] = {}; |
4495 | | szMonth[0] = '-'; |
4496 | | memcpy(szMonth + 1, apszMonths[iMonth], 3); |
4497 | | szMonth[4] = '-'; |
4498 | | szMonth[5] = '\0'; |
4499 | | const char *pszMonthFound = strstr(pszStr, szMonth); |
4500 | | if (pszMonthFound) |
4501 | | { |
4502 | | // Format of Apache, like in |
4503 | | // http://download.osgeo.org/gdal/data/gtiff/ |
4504 | | // "17-May-2010 12:26" |
4505 | | const auto nMonthFoundLen = strlen(pszMonthFound); |
4506 | | if (pszMonthFound - pszStr > 2 && nMonthFoundLen > 15 && |
4507 | | pszMonthFound[-2 + 11] == ' ' && pszMonthFound[-2 + 14] == ':') |
4508 | | { |
4509 | | pszMonthFound -= 2; |
4510 | | int nDay = atoi(pszMonthFound); |
4511 | | int nYear = atoi(pszMonthFound + 7); |
4512 | | int nHour = atoi(pszMonthFound + 12); |
4513 | | int nMin = atoi(pszMonthFound + 15); |
4514 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4515 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4516 | | { |
4517 | | brokendowntime.tm_year = nYear - 1900; |
4518 | | brokendowntime.tm_mon = iMonth; |
4519 | | brokendowntime.tm_mday = nDay; |
4520 | | brokendowntime.tm_hour = nHour; |
4521 | | brokendowntime.tm_min = nMin; |
4522 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4523 | | |
4524 | | if (nMonthFoundLen > 15 + 2) |
4525 | | { |
4526 | | const char *pszFilesize = pszMonthFound + 15 + 2; |
4527 | | while (*pszFilesize == ' ') |
4528 | | pszFilesize++; |
4529 | | if (*pszFilesize >= '1' && *pszFilesize <= '9') |
4530 | | nFileSize = CPLScanUIntBig( |
4531 | | pszFilesize, |
4532 | | static_cast<int>(strlen(pszFilesize))); |
4533 | | } |
4534 | | |
4535 | | return true; |
4536 | | } |
4537 | | } |
4538 | | return false; |
4539 | | } |
4540 | | |
4541 | | /* Microsoft IIS */ |
4542 | | snprintf(szMonth, sizeof(szMonth), " %s ", apszMonths[iMonth]); |
4543 | | pszMonthFound = strstr(pszStr, szMonth); |
4544 | | if (pszMonthFound) |
4545 | | { |
4546 | | int nLenMonth = static_cast<int>(strlen(apszMonths[iMonth])); |
4547 | | if (pszMonthFound - pszStr > 2 && pszMonthFound[-1] != ',' && |
4548 | | pszMonthFound[-2] != ' ' && |
4549 | | static_cast<int>(strlen(pszMonthFound - 2)) > |
4550 | | 2 + 1 + nLenMonth + 1 + 4 + 1 + 5 + 1 + 4) |
4551 | | { |
4552 | | /* Format of http://ortho.linz.govt.nz/tifs/1994_95/ */ |
4553 | | /* " Friday, 21 April 2006 12:05 p.m. 48062343 |
4554 | | * m35a_fy_94_95.tif" */ |
4555 | | pszMonthFound -= 2; |
4556 | | int nDay = atoi(pszMonthFound); |
4557 | | int nCurOffset = 2 + 1 + nLenMonth + 1; |
4558 | | int nYear = atoi(pszMonthFound + nCurOffset); |
4559 | | nCurOffset += 4 + 1; |
4560 | | int nHour = atoi(pszMonthFound + nCurOffset); |
4561 | | if (nHour < 10) |
4562 | | nCurOffset += 1 + 1; |
4563 | | else |
4564 | | nCurOffset += 2 + 1; |
4565 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
4566 | | nCurOffset += 2 + 1; |
4567 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "p.m.")) |
4568 | | nHour += 12; |
4569 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "a.m.")) |
4570 | | nHour = -1; |
4571 | | nCurOffset += 4; |
4572 | | |
4573 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
4574 | | while (*pszFilesize == ' ') |
4575 | | pszFilesize++; |
4576 | | if (*pszFilesize >= '1' && *pszFilesize <= '9') |
4577 | | nFileSize = CPLScanUIntBig( |
4578 | | pszFilesize, static_cast<int>(strlen(pszFilesize))); |
4579 | | |
4580 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4581 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4582 | | { |
4583 | | brokendowntime.tm_year = nYear - 1900; |
4584 | | brokendowntime.tm_mon = iMonth; |
4585 | | brokendowntime.tm_mday = nDay; |
4586 | | brokendowntime.tm_hour = nHour; |
4587 | | brokendowntime.tm_min = nMin; |
4588 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4589 | | |
4590 | | return true; |
4591 | | } |
4592 | | nFileSize = 0; |
4593 | | } |
4594 | | else if (pszMonthFound - pszStr > 1 && pszMonthFound[-1] == ',' && |
4595 | | static_cast<int>(strlen(pszMonthFound)) > |
4596 | | 1 + nLenMonth + 1 + 2 + 1 + 1 + 4 + 1 + 5 + 1 + 2) |
4597 | | { |
4598 | | // Format of |
4599 | | // http://publicfiles.dep.state.fl.us/dear/BWR_GIS/2007NWFLULC/ |
4600 | | // " Sunday, June 20, 2010 6:46 PM 233170905 |
4601 | | // NWF2007LULCForSDE.zip" |
4602 | | pszMonthFound += 1; |
4603 | | int nCurOffset = nLenMonth + 1; |
4604 | | int nDay = atoi(pszMonthFound + nCurOffset); |
4605 | | nCurOffset += 2 + 1 + 1; |
4606 | | int nYear = atoi(pszMonthFound + nCurOffset); |
4607 | | nCurOffset += 4 + 1; |
4608 | | int nHour = atoi(pszMonthFound + nCurOffset); |
4609 | | nCurOffset += 2 + 1; |
4610 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
4611 | | nCurOffset += 2 + 1; |
4612 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "PM")) |
4613 | | nHour += 12; |
4614 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "AM")) |
4615 | | nHour = -1; |
4616 | | nCurOffset += 2; |
4617 | | |
4618 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
4619 | | while (*pszFilesize == ' ') |
4620 | | pszFilesize++; |
4621 | | if (*pszFilesize >= '1' && *pszFilesize <= '9') |
4622 | | nFileSize = CPLScanUIntBig( |
4623 | | pszFilesize, static_cast<int>(strlen(pszFilesize))); |
4624 | | |
4625 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4626 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4627 | | { |
4628 | | brokendowntime.tm_year = nYear - 1900; |
4629 | | brokendowntime.tm_mon = iMonth; |
4630 | | brokendowntime.tm_mday = nDay; |
4631 | | brokendowntime.tm_hour = nHour; |
4632 | | brokendowntime.tm_min = nMin; |
4633 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4634 | | |
4635 | | return true; |
4636 | | } |
4637 | | nFileSize = 0; |
4638 | | } |
4639 | | return false; |
4640 | | } |
4641 | | } |
4642 | | |
4643 | | return false; |
4644 | | } |
4645 | | |
4646 | | /************************************************************************/ |
4647 | | /* ParseHTMLFileList() */ |
4648 | | /* */ |
4649 | | /* Parse a file list document and return all the components. */ |
4650 | | /************************************************************************/ |
4651 | | |
4652 | | char **VSICurlFilesystemHandlerBase::ParseHTMLFileList(const char *pszFilename, |
4653 | | int nMaxFiles, |
4654 | | char *pszData, |
4655 | | bool *pbGotFileList) |
4656 | | { |
4657 | | *pbGotFileList = false; |
4658 | | |
4659 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
4660 | | nullptr, nullptr, nullptr, |
4661 | | nullptr, nullptr, nullptr)); |
4662 | | const char *pszDir = nullptr; |
4663 | | if (STARTS_WITH_CI(osURL.c_str(), "http://")) |
4664 | | pszDir = strchr(osURL.c_str() + strlen("http://"), '/'); |
4665 | | else if (STARTS_WITH_CI(osURL.c_str(), "https://")) |
4666 | | pszDir = strchr(osURL.c_str() + strlen("https://"), '/'); |
4667 | | else if (STARTS_WITH_CI(osURL.c_str(), "ftp://")) |
4668 | | pszDir = strchr(osURL.c_str() + strlen("ftp://"), '/'); |
4669 | | if (pszDir == nullptr) |
4670 | | pszDir = ""; |
4671 | | |
4672 | | /* Apache / Nginx */ |
4673 | | /* Most of the time the format is <title>Index of {pszDir[/]}</title>, but |
4674 | | * there are special cases like https://cdn.star.nesdis.noaa.gov/GOES18/ABI/MESO/M1/GEOCOLOR/ |
4675 | | * where a CDN stuff makes that the title is <title>Index of /ma-cdn02/GOES/data/GOES18/ABI/MESO/M1/GEOCOLOR/</title> |
4676 | | */ |
4677 | | const std::string osTitleIndexOfPrefix = "<title>Index of "; |
4678 | | const std::string osExpectedSuffix = std::string(pszDir).append("</title>"); |
4679 | | const std::string osExpectedSuffixWithSlash = |
4680 | | std::string(pszDir).append("/</title>"); |
4681 | | /* FTP */ |
4682 | | const std::string osExpectedStringFTP = |
4683 | | std::string("FTP Listing of ").append(pszDir).append("/"); |
4684 | | /* Apache 1.3.33 */ |
4685 | | const std::string osExpectedStringOldApache = |
4686 | | std::string("<TITLE>Index of ").append(pszDir).append("</TITLE>"); |
4687 | | |
4688 | | // The listing of |
4689 | | // http://dds.cr.usgs.gov/srtm/SRTM_image_sample/picture%20examples/ |
4690 | | // has |
4691 | | // "<title>Index of /srtm/SRTM_image_sample/picture examples</title>" |
4692 | | // so we must try unescaped %20 also. |
4693 | | // Similar with |
4694 | | // http://datalib.usask.ca/gis/Data/Central_America_goodbutdoweown%3f/ |
4695 | | std::string osExpectedString_unescaped; |
4696 | | if (strchr(pszDir, '%')) |
4697 | | { |
4698 | | char *pszUnescapedDir = CPLUnescapeString(pszDir, nullptr, CPLES_URL); |
4699 | | osExpectedString_unescaped = osTitleIndexOfPrefix; |
4700 | | osExpectedString_unescaped += pszUnescapedDir; |
4701 | | osExpectedString_unescaped += "</title>"; |
4702 | | CPLFree(pszUnescapedDir); |
4703 | | } |
4704 | | |
4705 | | char *c = nullptr; |
4706 | | int nCount = 0; |
4707 | | int nCountTable = 0; |
4708 | | CPLStringList oFileList; |
4709 | | char *pszLine = pszData; |
4710 | | bool bIsHTMLDirList = false; |
4711 | | |
4712 | | while ((c = VSICurlParserFindEOL(pszLine)) != nullptr) |
4713 | | { |
4714 | | *c = '\0'; |
4715 | | |
4716 | | // To avoid false positive on pages such as |
4717 | | // http://www.ngs.noaa.gov/PC_PROD/USGG2009BETA |
4718 | | // This is a heuristics, but normal HTML listing of files have not more |
4719 | | // than one table. |
4720 | | if (strstr(pszLine, "<table")) |
4721 | | { |
4722 | | nCountTable++; |
4723 | | if (nCountTable == 2) |
4724 | | { |
4725 | | *pbGotFileList = false; |
4726 | | return nullptr; |
4727 | | } |
4728 | | } |
4729 | | |
4730 | | if (!bIsHTMLDirList && |
4731 | | ((strstr(pszLine, osTitleIndexOfPrefix.c_str()) && |
4732 | | (strstr(pszLine, osExpectedSuffix.c_str()) || |
4733 | | strstr(pszLine, osExpectedSuffixWithSlash.c_str()))) || |
4734 | | strstr(pszLine, osExpectedStringFTP.c_str()) || |
4735 | | strstr(pszLine, osExpectedStringOldApache.c_str()) || |
4736 | | (!osExpectedString_unescaped.empty() && |
4737 | | strstr(pszLine, osExpectedString_unescaped.c_str())))) |
4738 | | { |
4739 | | bIsHTMLDirList = true; |
4740 | | *pbGotFileList = true; |
4741 | | } |
4742 | | // Subversion HTTP listing |
4743 | | // or Microsoft-IIS/6.0 listing |
4744 | | // (e.g. http://ortho.linz.govt.nz/tifs/2005_06/) */ |
4745 | | else if (!bIsHTMLDirList && strstr(pszLine, "<title>")) |
4746 | | { |
4747 | | // Detect something like: |
4748 | | // <html><head><title>gdal - Revision 20739: |
4749 | | // /trunk/autotest/gcore/data</title></head> */ The annoying thing |
4750 | | // is that what is after ': ' is a subpart of what is after |
4751 | | // http://server/ |
4752 | | char *pszSubDir = strstr(pszLine, ": "); |
4753 | | if (pszSubDir == nullptr) |
4754 | | // or <title>ortho.linz.govt.nz - /tifs/2005_06/</title> |
4755 | | pszSubDir = strstr(pszLine, "- "); |
4756 | | if (pszSubDir) |
4757 | | { |
4758 | | pszSubDir += 2; |
4759 | | char *pszTmp = strstr(pszSubDir, "</title>"); |
4760 | | if (pszTmp) |
4761 | | { |
4762 | | if (pszTmp[-1] == '/') |
4763 | | pszTmp[-1] = 0; |
4764 | | else |
4765 | | *pszTmp = 0; |
4766 | | if (strstr(pszDir, pszSubDir)) |
4767 | | { |
4768 | | bIsHTMLDirList = true; |
4769 | | *pbGotFileList = true; |
4770 | | } |
4771 | | } |
4772 | | } |
4773 | | } |
4774 | | else if (bIsHTMLDirList && |
4775 | | (strstr(pszLine, "<a href=\"") != nullptr || |
4776 | | strstr(pszLine, "<A HREF=\"") != nullptr) && |
4777 | | // Exclude absolute links, like to subversion home. |
4778 | | strstr(pszLine, "<a href=\"http://") == nullptr && |
4779 | | // exclude parent directory. |
4780 | | strstr(pszLine, "Parent Directory") == nullptr) |
4781 | | { |
4782 | | char *beginFilename = strstr(pszLine, "<a href=\""); |
4783 | | if (beginFilename == nullptr) |
4784 | | beginFilename = strstr(pszLine, "<A HREF=\""); |
4785 | | beginFilename += strlen("<a href=\""); |
4786 | | char *endQuote = strchr(beginFilename, '"'); |
4787 | | if (endQuote && !STARTS_WITH(beginFilename, "?C=") && |
4788 | | !STARTS_WITH(beginFilename, "?N=")) |
4789 | | { |
4790 | | struct tm brokendowntime; |
4791 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
4792 | | GUIntBig nFileSize = 0; |
4793 | | GIntBig mTime = 0; |
4794 | | |
4795 | | VSICurlParseHTMLDateTimeFileSize(pszLine, brokendowntime, |
4796 | | nFileSize, mTime); |
4797 | | |
4798 | | *endQuote = '\0'; |
4799 | | |
4800 | | // Remove trailing slash, that are returned for directories by |
4801 | | // Apache. |
4802 | | bool bIsDirectory = false; |
4803 | | if (endQuote[-1] == '/') |
4804 | | { |
4805 | | bIsDirectory = true; |
4806 | | endQuote[-1] = 0; |
4807 | | } |
4808 | | |
4809 | | // shttpd links include slashes from the root directory. |
4810 | | // Skip them. |
4811 | | while (strchr(beginFilename, '/')) |
4812 | | beginFilename = strchr(beginFilename, '/') + 1; |
4813 | | |
4814 | | if (strcmp(beginFilename, ".") != 0 && |
4815 | | strcmp(beginFilename, "..") != 0) |
4816 | | { |
4817 | | std::string osCachedFilename = |
4818 | | CPLSPrintf("%s/%s", osURL.c_str(), beginFilename); |
4819 | | |
4820 | | FileProp cachedFileProp; |
4821 | | GetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
4822 | | cachedFileProp.eExists = EXIST_YES; |
4823 | | cachedFileProp.bIsDirectory = bIsDirectory; |
4824 | | cachedFileProp.mTime = static_cast<time_t>(mTime); |
4825 | | cachedFileProp.bHasComputedFileSize = nFileSize > 0; |
4826 | | cachedFileProp.fileSize = nFileSize; |
4827 | | SetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
4828 | | |
4829 | | oFileList.AddString(beginFilename); |
4830 | | if (ENABLE_DEBUG_VERBOSE) |
4831 | | { |
4832 | | CPLDebug( |
4833 | | GetDebugKey(), |
4834 | | "File[%d] = %s, is_dir = %d, size = " CPL_FRMT_GUIB |
4835 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
4836 | | nCount, osCachedFilename.c_str(), |
4837 | | bIsDirectory ? 1 : 0, nFileSize, |
4838 | | brokendowntime.tm_year + 1900, |
4839 | | brokendowntime.tm_mon + 1, brokendowntime.tm_mday, |
4840 | | brokendowntime.tm_hour, brokendowntime.tm_min, |
4841 | | brokendowntime.tm_sec); |
4842 | | } |
4843 | | nCount++; |
4844 | | |
4845 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
4846 | | break; |
4847 | | } |
4848 | | } |
4849 | | } |
4850 | | pszLine = c + 1; |
4851 | | } |
4852 | | |
4853 | | return oFileList.StealList(); |
4854 | | } |
4855 | | |
4856 | | /************************************************************************/ |
4857 | | /* GetStreamingFilename() */ |
4858 | | /************************************************************************/ |
4859 | | |
4860 | | std::string VSICurlFilesystemHandler::GetStreamingFilename( |
4861 | | const std::string &osFilename) const |
4862 | | { |
4863 | | if (STARTS_WITH(osFilename.c_str(), GetFSPrefix().c_str())) |
4864 | | return "/vsicurl_streaming/" + osFilename.substr(GetFSPrefix().size()); |
4865 | | return osFilename; |
4866 | | } |
4867 | | |
4868 | | /************************************************************************/ |
4869 | | /* VSICurlGetToken() */ |
4870 | | /************************************************************************/ |
4871 | | |
4872 | | static char *VSICurlGetToken(char *pszCurPtr, char **ppszNextToken) |
4873 | | { |
4874 | | if (pszCurPtr == nullptr) |
4875 | | return nullptr; |
4876 | | |
4877 | | while ((*pszCurPtr) == ' ') |
4878 | | pszCurPtr++; |
4879 | | if (*pszCurPtr == '\0') |
4880 | | return nullptr; |
4881 | | |
4882 | | char *pszToken = pszCurPtr; |
4883 | | while ((*pszCurPtr) != ' ' && (*pszCurPtr) != '\0') |
4884 | | pszCurPtr++; |
4885 | | if (*pszCurPtr == '\0') |
4886 | | { |
4887 | | *ppszNextToken = nullptr; |
4888 | | } |
4889 | | else |
4890 | | { |
4891 | | *pszCurPtr = '\0'; |
4892 | | pszCurPtr++; |
4893 | | while ((*pszCurPtr) == ' ') |
4894 | | pszCurPtr++; |
4895 | | *ppszNextToken = pszCurPtr; |
4896 | | } |
4897 | | |
4898 | | return pszToken; |
4899 | | } |
4900 | | |
4901 | | /************************************************************************/ |
4902 | | /* VSICurlParseFullFTPLine() */ |
4903 | | /************************************************************************/ |
4904 | | |
4905 | | /* Parse lines like the following ones : |
4906 | | -rw-r--r-- 1 10003 100 430 Jul 04 2008 COPYING |
4907 | | lrwxrwxrwx 1 ftp ftp 28 Jun 14 14:13 MPlayer -> |
4908 | | mirrors/mplayerhq.hu/MPlayer -rw-r--r-- 1 ftp ftp 725614592 May 13 |
4909 | | 20:13 Fedora-15-x86_64-Live-KDE.iso drwxr-xr-x 280 1003 1003 6656 Aug 26 |
4910 | | 04:17 gnu |
4911 | | */ |
4912 | | |
4913 | | static bool VSICurlParseFullFTPLine(char *pszLine, char *&pszFilename, |
4914 | | bool &bSizeValid, GUIntBig &nSize, |
4915 | | bool &bIsDirectory, GIntBig &nUnixTime) |
4916 | | { |
4917 | | char *pszNextToken = pszLine; |
4918 | | char *pszPermissions = VSICurlGetToken(pszNextToken, &pszNextToken); |
4919 | | if (pszPermissions == nullptr || strlen(pszPermissions) != 10) |
4920 | | return false; |
4921 | | bIsDirectory = pszPermissions[0] == 'd'; |
4922 | | |
4923 | | for (int i = 0; i < 3; i++) |
4924 | | { |
4925 | | if (VSICurlGetToken(pszNextToken, &pszNextToken) == nullptr) |
4926 | | return false; |
4927 | | } |
4928 | | |
4929 | | char *pszSize = VSICurlGetToken(pszNextToken, &pszNextToken); |
4930 | | if (pszSize == nullptr) |
4931 | | return false; |
4932 | | |
4933 | | if (pszPermissions[0] == '-') |
4934 | | { |
4935 | | // Regular file. |
4936 | | bSizeValid = true; |
4937 | | nSize = CPLScanUIntBig(pszSize, static_cast<int>(strlen(pszSize))); |
4938 | | } |
4939 | | |
4940 | | struct tm brokendowntime; |
4941 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
4942 | | bool bBrokenDownTimeValid = true; |
4943 | | |
4944 | | char *pszMonth = VSICurlGetToken(pszNextToken, &pszNextToken); |
4945 | | if (pszMonth == nullptr || strlen(pszMonth) != 3) |
4946 | | return false; |
4947 | | |
4948 | | int i = 0; // Used after for. |
4949 | | for (; i < 12; i++) |
4950 | | { |
4951 | | if (EQUALN(pszMonth, apszMonths[i], 3)) |
4952 | | break; |
4953 | | } |
4954 | | if (i < 12) |
4955 | | brokendowntime.tm_mon = i; |
4956 | | else |
4957 | | bBrokenDownTimeValid = false; |
4958 | | |
4959 | | char *pszDay = VSICurlGetToken(pszNextToken, &pszNextToken); |
4960 | | if (pszDay == nullptr || (strlen(pszDay) != 1 && strlen(pszDay) != 2)) |
4961 | | return false; |
4962 | | int nDay = atoi(pszDay); |
4963 | | if (nDay >= 1 && nDay <= 31) |
4964 | | brokendowntime.tm_mday = nDay; |
4965 | | else |
4966 | | bBrokenDownTimeValid = false; |
4967 | | |
4968 | | char *pszHourOrYear = VSICurlGetToken(pszNextToken, &pszNextToken); |
4969 | | if (pszHourOrYear == nullptr || |
4970 | | (strlen(pszHourOrYear) != 4 && strlen(pszHourOrYear) != 5)) |
4971 | | return false; |
4972 | | if (strlen(pszHourOrYear) == 4) |
4973 | | { |
4974 | | brokendowntime.tm_year = atoi(pszHourOrYear) - 1900; |
4975 | | } |
4976 | | else |
4977 | | { |
4978 | | time_t sTime; |
4979 | | time(&sTime); |
4980 | | struct tm currentBrokendowntime; |
4981 | | CPLUnixTimeToYMDHMS(static_cast<GIntBig>(sTime), |
4982 | | ¤tBrokendowntime); |
4983 | | brokendowntime.tm_year = currentBrokendowntime.tm_year; |
4984 | | brokendowntime.tm_hour = atoi(pszHourOrYear); |
4985 | | brokendowntime.tm_min = atoi(pszHourOrYear + 3); |
4986 | | } |
4987 | | |
4988 | | if (bBrokenDownTimeValid) |
4989 | | nUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4990 | | else |
4991 | | nUnixTime = 0; |
4992 | | |
4993 | | if (pszNextToken == nullptr) |
4994 | | return false; |
4995 | | |
4996 | | pszFilename = pszNextToken; |
4997 | | |
4998 | | char *pszCurPtr = pszFilename; |
4999 | | while (*pszCurPtr != '\0') |
5000 | | { |
5001 | | // In case of a link, stop before the pointed part of the link. |
5002 | | if (pszPermissions[0] == 'l' && STARTS_WITH(pszCurPtr, " -> ")) |
5003 | | { |
5004 | | break; |
5005 | | } |
5006 | | pszCurPtr++; |
5007 | | } |
5008 | | *pszCurPtr = '\0'; |
5009 | | |
5010 | | return true; |
5011 | | } |
5012 | | |
5013 | | /************************************************************************/ |
5014 | | /* GetURLFromFilename() */ |
5015 | | /************************************************************************/ |
5016 | | |
5017 | | std::string VSICurlFilesystemHandlerBase::GetURLFromFilename( |
5018 | | const std::string &osFilename) const |
5019 | | { |
5020 | | return VSICurlGetURLFromFilename(osFilename.c_str(), nullptr, nullptr, |
5021 | | nullptr, nullptr, nullptr, nullptr, |
5022 | | nullptr, nullptr); |
5023 | | } |
5024 | | |
5025 | | /************************************************************************/ |
5026 | | /* RegisterEmptyDir() */ |
5027 | | /************************************************************************/ |
5028 | | |
5029 | | void VSICurlFilesystemHandlerBase::RegisterEmptyDir( |
5030 | | const std::string &osDirname) |
5031 | | { |
5032 | | CachedDirList cachedDirList; |
5033 | | cachedDirList.bGotFileList = true; |
5034 | | cachedDirList.oFileList.AddString("."); |
5035 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5036 | | } |
5037 | | |
5038 | | /************************************************************************/ |
5039 | | /* GetFileList() */ |
5040 | | /************************************************************************/ |
5041 | | |
5042 | | char **VSICurlFilesystemHandlerBase::GetFileList(const char *pszDirname, |
5043 | | int nMaxFiles, |
5044 | | bool *pbGotFileList) |
5045 | | { |
5046 | | if (ENABLE_DEBUG) |
5047 | | CPLDebug(GetDebugKey(), "GetFileList(%s)", pszDirname); |
5048 | | |
5049 | | *pbGotFileList = false; |
5050 | | |
5051 | | bool bListDir = true; |
5052 | | bool bEmptyDir = false; |
5053 | | std::string osURL(VSICurlGetURLFromFilename(pszDirname, nullptr, nullptr, |
5054 | | nullptr, &bListDir, &bEmptyDir, |
5055 | | nullptr, nullptr, nullptr)); |
5056 | | if (bEmptyDir) |
5057 | | { |
5058 | | *pbGotFileList = true; |
5059 | | return CSLAddString(nullptr, "."); |
5060 | | } |
5061 | | if (!bListDir) |
5062 | | return nullptr; |
5063 | | |
5064 | | // Deal with publicly visible Azure directories. |
5065 | | if (STARTS_WITH(osURL.c_str(), "https://")) |
5066 | | { |
5067 | | const char *pszBlobCore = |
5068 | | strstr(osURL.c_str(), ".blob.core.windows.net/"); |
5069 | | if (pszBlobCore) |
5070 | | { |
5071 | | FileProp cachedFileProp; |
5072 | | GetCachedFileProp(osURL.c_str(), cachedFileProp); |
5073 | | if (cachedFileProp.bIsAzureFolder) |
5074 | | { |
5075 | | const char *pszURLWithoutHTTPS = |
5076 | | osURL.c_str() + strlen("https://"); |
5077 | | const std::string osStorageAccount( |
5078 | | pszURLWithoutHTTPS, pszBlobCore - pszURLWithoutHTTPS); |
5079 | | CPLConfigOptionSetter oSetter1("AZURE_NO_SIGN_REQUEST", "YES", |
5080 | | false); |
5081 | | CPLConfigOptionSetter oSetter2("AZURE_STORAGE_ACCOUNT", |
5082 | | osStorageAccount.c_str(), false); |
5083 | | const std::string osVSIAZ(std::string("/vsiaz/").append( |
5084 | | pszBlobCore + strlen(".blob.core.windows.net/"))); |
5085 | | char **papszFileList = VSIReadDirEx(osVSIAZ.c_str(), nMaxFiles); |
5086 | | if (papszFileList) |
5087 | | { |
5088 | | *pbGotFileList = true; |
5089 | | return papszFileList; |
5090 | | } |
5091 | | } |
5092 | | } |
5093 | | } |
5094 | | |
5095 | | // HACK (optimization in fact) for MBTiles driver. |
5096 | | if (strstr(pszDirname, ".tiles.mapbox.com") != nullptr) |
5097 | | return nullptr; |
5098 | | |
5099 | | if (STARTS_WITH(osURL.c_str(), "ftp://")) |
5100 | | { |
5101 | | WriteFuncStruct sWriteFuncData; |
5102 | | sWriteFuncData.pBuffer = nullptr; |
5103 | | |
5104 | | std::string osDirname(osURL); |
5105 | | osDirname += '/'; |
5106 | | |
5107 | | char **papszFileList = nullptr; |
5108 | | |
5109 | | CURLM *hCurlMultiHandle = GetCurlMultiHandleFor(osDirname); |
5110 | | CURL *hCurlHandle = curl_easy_init(); |
5111 | | |
5112 | | for (int iTry = 0; iTry < 2; iTry++) |
5113 | | { |
5114 | | struct curl_slist *headers = |
5115 | | VSICurlSetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5116 | | |
5117 | | // On the first pass, we want to try fetching all the possible |
5118 | | // information (filename, file/directory, size). If that does not |
5119 | | // work, then try again with CURLOPT_DIRLISTONLY set. |
5120 | | if (iTry == 1) |
5121 | | { |
5122 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 1); |
5123 | | } |
5124 | | |
5125 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, |
5126 | | nullptr); |
5127 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5128 | | &sWriteFuncData); |
5129 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5130 | | VSICurlHandleWriteFunc); |
5131 | | |
5132 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5133 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5134 | | szCurlErrBuf); |
5135 | | |
5136 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
5137 | | headers); |
5138 | | |
5139 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
5140 | | |
5141 | | curl_slist_free_all(headers); |
5142 | | |
5143 | | if (sWriteFuncData.pBuffer == nullptr) |
5144 | | { |
5145 | | curl_easy_cleanup(hCurlHandle); |
5146 | | return nullptr; |
5147 | | } |
5148 | | |
5149 | | char *pszLine = sWriteFuncData.pBuffer; |
5150 | | char *c = nullptr; |
5151 | | int nCount = 0; |
5152 | | |
5153 | | if (STARTS_WITH_CI(pszLine, "<!DOCTYPE HTML") || |
5154 | | STARTS_WITH_CI(pszLine, "<HTML>")) |
5155 | | { |
5156 | | papszFileList = |
5157 | | ParseHTMLFileList(pszDirname, nMaxFiles, |
5158 | | sWriteFuncData.pBuffer, pbGotFileList); |
5159 | | break; |
5160 | | } |
5161 | | else if (iTry == 0) |
5162 | | { |
5163 | | CPLStringList oFileList; |
5164 | | *pbGotFileList = true; |
5165 | | |
5166 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5167 | | { |
5168 | | *c = 0; |
5169 | | if (c - pszLine > 0 && c[-1] == '\r') |
5170 | | c[-1] = 0; |
5171 | | |
5172 | | char *pszFilename = nullptr; |
5173 | | bool bSizeValid = false; |
5174 | | GUIntBig nFileSize = 0; |
5175 | | bool bIsDirectory = false; |
5176 | | GIntBig mUnixTime = 0; |
5177 | | if (!VSICurlParseFullFTPLine(pszLine, pszFilename, |
5178 | | bSizeValid, nFileSize, |
5179 | | bIsDirectory, mUnixTime)) |
5180 | | break; |
5181 | | |
5182 | | if (strcmp(pszFilename, ".") != 0 && |
5183 | | strcmp(pszFilename, "..") != 0) |
5184 | | { |
5185 | | if (CPLHasUnbalancedPathTraversal(pszFilename)) |
5186 | | { |
5187 | | CPLError(CE_Warning, CPLE_AppDefined, |
5188 | | "Ignoring '%s' that has a path traversal " |
5189 | | "pattern", |
5190 | | pszFilename); |
5191 | | } |
5192 | | else |
5193 | | { |
5194 | | std::string osCachedFilename = |
5195 | | CPLSPrintf("%s/%s", osURL.c_str(), pszFilename); |
5196 | | |
5197 | | FileProp cachedFileProp; |
5198 | | GetCachedFileProp(osCachedFilename.c_str(), |
5199 | | cachedFileProp); |
5200 | | cachedFileProp.eExists = EXIST_YES; |
5201 | | cachedFileProp.bIsDirectory = bIsDirectory; |
5202 | | cachedFileProp.mTime = |
5203 | | static_cast<time_t>(mUnixTime); |
5204 | | cachedFileProp.bHasComputedFileSize = bSizeValid; |
5205 | | cachedFileProp.fileSize = nFileSize; |
5206 | | SetCachedFileProp(osCachedFilename.c_str(), |
5207 | | cachedFileProp); |
5208 | | |
5209 | | oFileList.AddString(pszFilename); |
5210 | | if (ENABLE_DEBUG_VERBOSE) |
5211 | | { |
5212 | | struct tm brokendowntime; |
5213 | | CPLUnixTimeToYMDHMS(mUnixTime, &brokendowntime); |
5214 | | CPLDebug( |
5215 | | GetDebugKey(), |
5216 | | "File[%d] = %s, is_dir = %d, size " |
5217 | | "= " CPL_FRMT_GUIB |
5218 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
5219 | | nCount, pszFilename, bIsDirectory ? 1 : 0, |
5220 | | nFileSize, brokendowntime.tm_year + 1900, |
5221 | | brokendowntime.tm_mon + 1, |
5222 | | brokendowntime.tm_mday, |
5223 | | brokendowntime.tm_hour, |
5224 | | brokendowntime.tm_min, |
5225 | | brokendowntime.tm_sec); |
5226 | | } |
5227 | | |
5228 | | nCount++; |
5229 | | |
5230 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
5231 | | break; |
5232 | | } |
5233 | | } |
5234 | | |
5235 | | pszLine = c + 1; |
5236 | | } |
5237 | | |
5238 | | if (c == nullptr) |
5239 | | { |
5240 | | papszFileList = oFileList.StealList(); |
5241 | | break; |
5242 | | } |
5243 | | } |
5244 | | else |
5245 | | { |
5246 | | CPLStringList oFileList; |
5247 | | *pbGotFileList = true; |
5248 | | |
5249 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5250 | | { |
5251 | | *c = 0; |
5252 | | if (c - pszLine > 0 && c[-1] == '\r') |
5253 | | c[-1] = 0; |
5254 | | |
5255 | | if (strcmp(pszLine, ".") != 0 && strcmp(pszLine, "..") != 0) |
5256 | | { |
5257 | | oFileList.AddString(pszLine); |
5258 | | if (ENABLE_DEBUG_VERBOSE) |
5259 | | { |
5260 | | CPLDebug(GetDebugKey(), "File[%d] = %s", nCount, |
5261 | | pszLine); |
5262 | | } |
5263 | | nCount++; |
5264 | | } |
5265 | | |
5266 | | pszLine = c + 1; |
5267 | | } |
5268 | | |
5269 | | papszFileList = oFileList.StealList(); |
5270 | | } |
5271 | | |
5272 | | CPLFree(sWriteFuncData.pBuffer); |
5273 | | sWriteFuncData.pBuffer = nullptr; |
5274 | | } |
5275 | | |
5276 | | CPLFree(sWriteFuncData.pBuffer); |
5277 | | curl_easy_cleanup(hCurlHandle); |
5278 | | |
5279 | | return papszFileList; |
5280 | | } |
5281 | | |
5282 | | // Try to recognize HTML pages that list the content of a directory. |
5283 | | // Currently this supports what Apache and shttpd can return. |
5284 | | else if (STARTS_WITH(osURL.c_str(), "http://") || |
5285 | | STARTS_WITH(osURL.c_str(), "https://")) |
5286 | | { |
5287 | | std::string osDirname(std::move(osURL)); |
5288 | | osDirname += '/'; |
5289 | | |
5290 | | CURLM *hCurlMultiHandle = GetCurlMultiHandleFor(osDirname); |
5291 | | CURL *hCurlHandle = curl_easy_init(); |
5292 | | |
5293 | | struct curl_slist *headers = |
5294 | | VSICurlSetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5295 | | |
5296 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
5297 | | |
5298 | | WriteFuncStruct sWriteFuncData; |
5299 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
5300 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5301 | | &sWriteFuncData); |
5302 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5303 | | VSICurlHandleWriteFunc); |
5304 | | |
5305 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5306 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5307 | | szCurlErrBuf); |
5308 | | |
5309 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
5310 | | |
5311 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
5312 | | |
5313 | | curl_slist_free_all(headers); |
5314 | | |
5315 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
5316 | | |
5317 | | if (sWriteFuncData.pBuffer == nullptr) |
5318 | | { |
5319 | | curl_easy_cleanup(hCurlHandle); |
5320 | | return nullptr; |
5321 | | } |
5322 | | |
5323 | | char **papszFileList = nullptr; |
5324 | | if (STARTS_WITH_CI(sWriteFuncData.pBuffer, "<?xml") && |
5325 | | strstr(sWriteFuncData.pBuffer, "<ListBucketResult") != nullptr) |
5326 | | { |
5327 | | CPLStringList osFileList; |
5328 | | std::string osBaseURL(pszDirname); |
5329 | | osBaseURL += "/"; |
5330 | | bool bIsTruncated = true; |
5331 | | bool ret = AnalyseS3FileList( |
5332 | | osBaseURL, sWriteFuncData.pBuffer, osFileList, nMaxFiles, |
5333 | | GetS3IgnoredStorageClasses(), bIsTruncated); |
5334 | | // If the list is truncated, then don't report it. |
5335 | | if (ret && !bIsTruncated) |
5336 | | { |
5337 | | if (osFileList.empty()) |
5338 | | { |
5339 | | // To avoid an error to be reported |
5340 | | osFileList.AddString("."); |
5341 | | } |
5342 | | papszFileList = osFileList.StealList(); |
5343 | | *pbGotFileList = true; |
5344 | | } |
5345 | | } |
5346 | | else |
5347 | | { |
5348 | | papszFileList = ParseHTMLFileList( |
5349 | | pszDirname, nMaxFiles, sWriteFuncData.pBuffer, pbGotFileList); |
5350 | | } |
5351 | | |
5352 | | CPLFree(sWriteFuncData.pBuffer); |
5353 | | curl_easy_cleanup(hCurlHandle); |
5354 | | return papszFileList; |
5355 | | } |
5356 | | |
5357 | | return nullptr; |
5358 | | } |
5359 | | |
5360 | | /************************************************************************/ |
5361 | | /* GetS3IgnoredStorageClasses() */ |
5362 | | /************************************************************************/ |
5363 | | |
5364 | | std::set<std::string> VSICurlFilesystemHandlerBase::GetS3IgnoredStorageClasses() |
5365 | | { |
5366 | | std::set<std::string> oSetIgnoredStorageClasses; |
5367 | | const char *pszIgnoredStorageClasses = |
5368 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_STORAGE_CLASSES", nullptr); |
5369 | | const char *pszIgnoreGlacierStorage = |
5370 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE", nullptr); |
5371 | | CPLStringList aosIgnoredStorageClasses( |
5372 | | CSLTokenizeString2(pszIgnoredStorageClasses ? pszIgnoredStorageClasses |
5373 | | : "GLACIER,DEEP_ARCHIVE", |
5374 | | ",", 0)); |
5375 | | for (int i = 0; i < aosIgnoredStorageClasses.size(); ++i) |
5376 | | oSetIgnoredStorageClasses.insert(aosIgnoredStorageClasses[i]); |
5377 | | if (pszIgnoredStorageClasses == nullptr && |
5378 | | pszIgnoreGlacierStorage != nullptr && |
5379 | | !CPLTestBool(pszIgnoreGlacierStorage)) |
5380 | | { |
5381 | | oSetIgnoredStorageClasses.clear(); |
5382 | | } |
5383 | | return oSetIgnoredStorageClasses; |
5384 | | } |
5385 | | |
5386 | | /************************************************************************/ |
5387 | | /* Stat() */ |
5388 | | /************************************************************************/ |
5389 | | |
5390 | | int VSICurlFilesystemHandlerBase::Stat(const char *pszFilename, |
5391 | | VSIStatBufL *pStatBuf, int nFlags) |
5392 | | { |
5393 | | if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str()) && |
5394 | | !STARTS_WITH_CI(pszFilename, "/vsicurl?")) |
5395 | | return -1; |
5396 | | |
5397 | | memset(pStatBuf, 0, sizeof(VSIStatBufL)); |
5398 | | |
5399 | | if ((nFlags & VSI_STAT_CACHE_ONLY) != 0) |
5400 | | { |
5401 | | cpl::FileProp oFileProp; |
5402 | | if (!GetCachedFileProp(GetURLFromFilename(pszFilename).c_str(), |
5403 | | oFileProp) || |
5404 | | oFileProp.eExists != EXIST_YES) |
5405 | | { |
5406 | | return -1; |
5407 | | } |
5408 | | pStatBuf->st_mode = static_cast<unsigned short>(oFileProp.nMode); |
5409 | | pStatBuf->st_mtime = oFileProp.mTime; |
5410 | | pStatBuf->st_size = oFileProp.fileSize; |
5411 | | return 0; |
5412 | | } |
5413 | | |
5414 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5415 | | NetworkStatisticsAction oContextAction("Stat"); |
5416 | | |
5417 | | const std::string osFilename(pszFilename); |
5418 | | |
5419 | | if (!IsAllowedFilename(pszFilename)) |
5420 | | return -1; |
5421 | | |
5422 | | bool bListDir = true; |
5423 | | bool bEmptyDir = false; |
5424 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
5425 | | nullptr, &bListDir, &bEmptyDir, |
5426 | | nullptr, nullptr, nullptr)); |
5427 | | |
5428 | | const char *pszOptionVal = VSIGetPathSpecificOption( |
5429 | | pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", "NO"); |
5430 | | const bool bSkipReadDir = |
5431 | | !bListDir || bEmptyDir || EQUAL(pszOptionVal, "EMPTY_DIR") || |
5432 | | CPLTestBool(pszOptionVal) || !AllowCachedDataFor(pszFilename); |
5433 | | |
5434 | | // Does it look like a FTP directory? |
5435 | | if (STARTS_WITH(osURL.c_str(), "ftp://") && osFilename.back() == '/' && |
5436 | | !bSkipReadDir) |
5437 | | { |
5438 | | char **papszFileList = ReadDirEx(osFilename.c_str(), 0); |
5439 | | if (papszFileList) |
5440 | | { |
5441 | | pStatBuf->st_mode = S_IFDIR; |
5442 | | pStatBuf->st_size = 0; |
5443 | | |
5444 | | CSLDestroy(papszFileList); |
5445 | | |
5446 | | return 0; |
5447 | | } |
5448 | | return -1; |
5449 | | } |
5450 | | else if (strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
5451 | | !STARTS_WITH_CI(CPLGetExtensionSafe(osFilename.c_str()).c_str(), |
5452 | | "zip") && |
5453 | | strstr(osFilename.c_str(), ".zip.") != nullptr && |
5454 | | strstr(osFilename.c_str(), ".ZIP.") != nullptr && !bSkipReadDir) |
5455 | | { |
5456 | | bool bGotFileList = false; |
5457 | | char **papszFileList = ReadDirInternal( |
5458 | | CPLGetDirnameSafe(osFilename.c_str()).c_str(), 0, &bGotFileList); |
5459 | | const bool bFound = |
5460 | | VSICurlIsFileInList(papszFileList, |
5461 | | CPLGetFilename(osFilename.c_str())) != -1; |
5462 | | CSLDestroy(papszFileList); |
5463 | | if (bGotFileList && !bFound) |
5464 | | { |
5465 | | return -1; |
5466 | | } |
5467 | | } |
5468 | | |
5469 | | VSICurlHandle *poHandle = CreateFileHandle(osFilename.c_str()); |
5470 | | if (poHandle == nullptr) |
5471 | | return -1; |
5472 | | |
5473 | | if (poHandle->IsKnownFileSize() || |
5474 | | ((nFlags & VSI_STAT_SIZE_FLAG) && !poHandle->IsDirectory() && |
5475 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES")))) |
5476 | | { |
5477 | | pStatBuf->st_size = poHandle->GetFileSize(true); |
5478 | | } |
5479 | | |
5480 | | const int nRet = |
5481 | | poHandle->Exists((nFlags & VSI_STAT_SET_ERROR_FLAG) > 0) ? 0 : -1; |
5482 | | pStatBuf->st_mtime = poHandle->GetMTime(); |
5483 | | pStatBuf->st_mode = static_cast<unsigned short>(poHandle->GetMode()); |
5484 | | if (pStatBuf->st_mode == 0) |
5485 | | pStatBuf->st_mode = poHandle->IsDirectory() ? S_IFDIR : S_IFREG; |
5486 | | delete poHandle; |
5487 | | return nRet; |
5488 | | } |
5489 | | |
5490 | | /************************************************************************/ |
5491 | | /* ReadDirInternal() */ |
5492 | | /************************************************************************/ |
5493 | | |
5494 | | char **VSICurlFilesystemHandlerBase::ReadDirInternal(const char *pszDirname, |
5495 | | int nMaxFiles, |
5496 | | bool *pbGotFileList) |
5497 | | { |
5498 | | std::string osDirname(pszDirname); |
5499 | | |
5500 | | // Replace a/b/../c by a/c |
5501 | | const auto posSlashDotDot = osDirname.find("/.."); |
5502 | | if (posSlashDotDot != std::string::npos && posSlashDotDot >= 1) |
5503 | | { |
5504 | | const auto posPrecedingSlash = |
5505 | | osDirname.find_last_of('/', posSlashDotDot - 1); |
5506 | | if (posPrecedingSlash != std::string::npos && posPrecedingSlash >= 1) |
5507 | | { |
5508 | | osDirname.erase(osDirname.begin() + posPrecedingSlash, |
5509 | | osDirname.begin() + posSlashDotDot + strlen("/..")); |
5510 | | } |
5511 | | } |
5512 | | |
5513 | | std::string osDirnameOri(osDirname); |
5514 | | if (osDirname + "/" == GetFSPrefix()) |
5515 | | { |
5516 | | osDirname += "/"; |
5517 | | } |
5518 | | else if (osDirname != GetFSPrefix()) |
5519 | | { |
5520 | | while (!osDirname.empty() && osDirname.back() == '/') |
5521 | | osDirname.erase(osDirname.size() - 1); |
5522 | | } |
5523 | | |
5524 | | if (osDirname.size() < GetFSPrefix().size()) |
5525 | | { |
5526 | | if (pbGotFileList) |
5527 | | *pbGotFileList = true; |
5528 | | return nullptr; |
5529 | | } |
5530 | | |
5531 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5532 | | NetworkStatisticsAction oContextAction("ReadDir"); |
5533 | | |
5534 | | CPLMutexHolder oHolder(&hMutex); |
5535 | | |
5536 | | // If we know the file exists and is not a directory, |
5537 | | // then don't try to list its content. |
5538 | | FileProp cachedFileProp; |
5539 | | if (GetCachedFileProp(GetURLFromFilename(osDirname.c_str()).c_str(), |
5540 | | cachedFileProp) && |
5541 | | cachedFileProp.eExists == EXIST_YES && !cachedFileProp.bIsDirectory) |
5542 | | { |
5543 | | if (osDirnameOri != osDirname) |
5544 | | { |
5545 | | if (GetCachedFileProp((GetURLFromFilename(osDirname) + "/").c_str(), |
5546 | | cachedFileProp) && |
5547 | | cachedFileProp.eExists == EXIST_YES && |
5548 | | !cachedFileProp.bIsDirectory) |
5549 | | { |
5550 | | if (pbGotFileList) |
5551 | | *pbGotFileList = true; |
5552 | | return nullptr; |
5553 | | } |
5554 | | } |
5555 | | else |
5556 | | { |
5557 | | if (pbGotFileList) |
5558 | | *pbGotFileList = true; |
5559 | | return nullptr; |
5560 | | } |
5561 | | } |
5562 | | |
5563 | | CachedDirList cachedDirList; |
5564 | | if (!GetCachedDirList(osDirname.c_str(), cachedDirList)) |
5565 | | { |
5566 | | cachedDirList.oFileList.Assign(GetFileList(osDirname.c_str(), nMaxFiles, |
5567 | | &cachedDirList.bGotFileList), |
5568 | | true); |
5569 | | if (cachedDirList.bGotFileList && cachedDirList.oFileList.empty()) |
5570 | | { |
5571 | | // To avoid an error to be reported |
5572 | | cachedDirList.oFileList.AddString("."); |
5573 | | } |
5574 | | if (nMaxFiles <= 0 || cachedDirList.oFileList.size() < nMaxFiles) |
5575 | | { |
5576 | | // Only cache content if we didn't hit the limitation |
5577 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5578 | | } |
5579 | | } |
5580 | | |
5581 | | if (pbGotFileList) |
5582 | | *pbGotFileList = cachedDirList.bGotFileList; |
5583 | | |
5584 | | return CSLDuplicate(cachedDirList.oFileList.List()); |
5585 | | } |
5586 | | |
5587 | | /************************************************************************/ |
5588 | | /* InvalidateDirContent() */ |
5589 | | /************************************************************************/ |
5590 | | |
5591 | | void VSICurlFilesystemHandlerBase::InvalidateDirContent( |
5592 | | const std::string &osDirname) |
5593 | | { |
5594 | | CPLMutexHolder oHolder(&hMutex); |
5595 | | |
5596 | | CachedDirList oCachedDirList; |
5597 | | if (oCacheDirList.tryGet(osDirname, oCachedDirList)) |
5598 | | { |
5599 | | nCachedFilesInDirList -= oCachedDirList.oFileList.size(); |
5600 | | oCacheDirList.remove(osDirname); |
5601 | | } |
5602 | | } |
5603 | | |
5604 | | /************************************************************************/ |
5605 | | /* ReadDirEx() */ |
5606 | | /************************************************************************/ |
5607 | | |
5608 | | char **VSICurlFilesystemHandlerBase::ReadDirEx(const char *pszDirname, |
5609 | | int nMaxFiles) |
5610 | | { |
5611 | | return ReadDirInternal(pszDirname, nMaxFiles, nullptr); |
5612 | | } |
5613 | | |
5614 | | /************************************************************************/ |
5615 | | /* SiblingFiles() */ |
5616 | | /************************************************************************/ |
5617 | | |
5618 | | char **VSICurlFilesystemHandlerBase::SiblingFiles(const char *pszFilename) |
5619 | | { |
5620 | | /* Small optimization to avoid unnecessary stat'ing from PAux or ENVI */ |
5621 | | /* drivers. The MBTiles driver needs no companion file. */ |
5622 | | if (EQUAL(CPLGetExtensionSafe(pszFilename).c_str(), "mbtiles")) |
5623 | | { |
5624 | | return static_cast<char **>(CPLCalloc(1, sizeof(char *))); |
5625 | | } |
5626 | | return nullptr; |
5627 | | } |
5628 | | |
5629 | | /************************************************************************/ |
5630 | | /* GetFileMetadata() */ |
5631 | | /************************************************************************/ |
5632 | | |
5633 | | char **VSICurlFilesystemHandlerBase::GetFileMetadata(const char *pszFilename, |
5634 | | const char *pszDomain, |
5635 | | CSLConstList) |
5636 | | { |
5637 | | if (pszDomain == nullptr || !EQUAL(pszDomain, "HEADERS")) |
5638 | | return nullptr; |
5639 | | std::unique_ptr<VSICurlHandle> poHandle(CreateFileHandle(pszFilename)); |
5640 | | if (poHandle == nullptr) |
5641 | | return nullptr; |
5642 | | |
5643 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5644 | | NetworkStatisticsAction oContextAction("GetFileMetadata"); |
5645 | | |
5646 | | poHandle->GetFileSizeOrHeaders(true, true); |
5647 | | return CSLDuplicate(poHandle->GetHeaders().List()); |
5648 | | } |
5649 | | |
5650 | | /************************************************************************/ |
5651 | | /* VSIAppendWriteHandle() */ |
5652 | | /************************************************************************/ |
5653 | | |
5654 | | VSIAppendWriteHandle::VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS, |
5655 | | const char *pszFSPrefix, |
5656 | | const char *pszFilename, |
5657 | | int nChunkSize) |
5658 | | : m_poFS(poFS), m_osFSPrefix(pszFSPrefix), m_osFilename(pszFilename), |
5659 | | m_oRetryParameters(CPLStringList(CPLHTTPGetOptionsFromEnv(pszFilename))), |
5660 | | m_nBufferSize(nChunkSize) |
5661 | | { |
5662 | | m_pabyBuffer = static_cast<GByte *>(VSIMalloc(m_nBufferSize)); |
5663 | | if (m_pabyBuffer == nullptr) |
5664 | | { |
5665 | | CPLError(CE_Failure, CPLE_AppDefined, |
5666 | | "Cannot allocate working buffer for %s writing", |
5667 | | m_osFSPrefix.c_str()); |
5668 | | } |
5669 | | } |
5670 | | |
5671 | | /************************************************************************/ |
5672 | | /* ~VSIAppendWriteHandle() */ |
5673 | | /************************************************************************/ |
5674 | | |
5675 | | VSIAppendWriteHandle::~VSIAppendWriteHandle() |
5676 | | { |
5677 | | /* WARNING: implementation should call Close() themselves */ |
5678 | | /* cannot be done safely from here, since Send() can be called. */ |
5679 | | CPLFree(m_pabyBuffer); |
5680 | | } |
5681 | | |
5682 | | /************************************************************************/ |
5683 | | /* Seek() */ |
5684 | | /************************************************************************/ |
5685 | | |
5686 | | int VSIAppendWriteHandle::Seek(vsi_l_offset nOffset, int nWhence) |
5687 | | { |
5688 | | if (!((nWhence == SEEK_SET && nOffset == m_nCurOffset) || |
5689 | | (nWhence == SEEK_CUR && nOffset == 0) || |
5690 | | (nWhence == SEEK_END && nOffset == 0))) |
5691 | | { |
5692 | | CPLError(CE_Failure, CPLE_NotSupported, |
5693 | | "Seek not supported on writable %s files", |
5694 | | m_osFSPrefix.c_str()); |
5695 | | m_bError = true; |
5696 | | return -1; |
5697 | | } |
5698 | | return 0; |
5699 | | } |
5700 | | |
5701 | | /************************************************************************/ |
5702 | | /* Tell() */ |
5703 | | /************************************************************************/ |
5704 | | |
5705 | | vsi_l_offset VSIAppendWriteHandle::Tell() |
5706 | | { |
5707 | | return m_nCurOffset; |
5708 | | } |
5709 | | |
5710 | | /************************************************************************/ |
5711 | | /* Read() */ |
5712 | | /************************************************************************/ |
5713 | | |
5714 | | size_t VSIAppendWriteHandle::Read(void * /* pBuffer */, size_t /* nSize */, |
5715 | | size_t /* nMemb */) |
5716 | | { |
5717 | | CPLError(CE_Failure, CPLE_NotSupported, |
5718 | | "Read not supported on writable %s files", m_osFSPrefix.c_str()); |
5719 | | m_bError = true; |
5720 | | return 0; |
5721 | | } |
5722 | | |
5723 | | /************************************************************************/ |
5724 | | /* ReadCallBackBuffer() */ |
5725 | | /************************************************************************/ |
5726 | | |
5727 | | size_t VSIAppendWriteHandle::ReadCallBackBuffer(char *buffer, size_t size, |
5728 | | size_t nitems, void *instream) |
5729 | | { |
5730 | | VSIAppendWriteHandle *poThis = |
5731 | | static_cast<VSIAppendWriteHandle *>(instream); |
5732 | | const int nSizeMax = static_cast<int>(size * nitems); |
5733 | | const int nSizeToWrite = std::min( |
5734 | | nSizeMax, poThis->m_nBufferOff - poThis->m_nBufferOffReadCallback); |
5735 | | memcpy(buffer, poThis->m_pabyBuffer + poThis->m_nBufferOffReadCallback, |
5736 | | nSizeToWrite); |
5737 | | poThis->m_nBufferOffReadCallback += nSizeToWrite; |
5738 | | return nSizeToWrite; |
5739 | | } |
5740 | | |
5741 | | /************************************************************************/ |
5742 | | /* Write() */ |
5743 | | /************************************************************************/ |
5744 | | |
5745 | | size_t VSIAppendWriteHandle::Write(const void *pBuffer, size_t nSize, |
5746 | | size_t nMemb) |
5747 | | { |
5748 | | if (m_bError) |
5749 | | return 0; |
5750 | | |
5751 | | size_t nBytesToWrite = nSize * nMemb; |
5752 | | if (nBytesToWrite == 0) |
5753 | | return 0; |
5754 | | |
5755 | | const GByte *pabySrcBuffer = reinterpret_cast<const GByte *>(pBuffer); |
5756 | | while (nBytesToWrite > 0) |
5757 | | { |
5758 | | if (m_nBufferOff == m_nBufferSize) |
5759 | | { |
5760 | | if (!Send(false)) |
5761 | | { |
5762 | | m_bError = true; |
5763 | | return 0; |
5764 | | } |
5765 | | m_nBufferOff = 0; |
5766 | | } |
5767 | | |
5768 | | const int nToWriteInBuffer = static_cast<int>(std::min( |
5769 | | static_cast<size_t>(m_nBufferSize - m_nBufferOff), nBytesToWrite)); |
5770 | | memcpy(m_pabyBuffer + m_nBufferOff, pabySrcBuffer, nToWriteInBuffer); |
5771 | | pabySrcBuffer += nToWriteInBuffer; |
5772 | | m_nBufferOff += nToWriteInBuffer; |
5773 | | m_nCurOffset += nToWriteInBuffer; |
5774 | | nBytesToWrite -= nToWriteInBuffer; |
5775 | | } |
5776 | | return nMemb; |
5777 | | } |
5778 | | |
5779 | | /************************************************************************/ |
5780 | | /* Close() */ |
5781 | | /************************************************************************/ |
5782 | | |
5783 | | int VSIAppendWriteHandle::Close() |
5784 | | { |
5785 | | int nRet = 0; |
5786 | | if (!m_bClosed) |
5787 | | { |
5788 | | m_bClosed = true; |
5789 | | if (!m_bError && !Send(true)) |
5790 | | nRet = -1; |
5791 | | } |
5792 | | return nRet; |
5793 | | } |
5794 | | |
5795 | | /************************************************************************/ |
5796 | | /* CurlRequestHelper() */ |
5797 | | /************************************************************************/ |
5798 | | |
5799 | | CurlRequestHelper::CurlRequestHelper() |
5800 | | { |
5801 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
5802 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
5803 | | nullptr); |
5804 | | } |
5805 | | |
5806 | | /************************************************************************/ |
5807 | | /* ~CurlRequestHelper() */ |
5808 | | /************************************************************************/ |
5809 | | |
5810 | | CurlRequestHelper::~CurlRequestHelper() |
5811 | | { |
5812 | | CPLFree(sWriteFuncData.pBuffer); |
5813 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
5814 | | } |
5815 | | |
5816 | | /************************************************************************/ |
5817 | | /* perform() */ |
5818 | | /************************************************************************/ |
5819 | | |
5820 | | long CurlRequestHelper::perform(CURL *hCurlHandle, struct curl_slist *headers, |
5821 | | VSICurlFilesystemHandlerBase *poFS, |
5822 | | IVSIS3LikeHandleHelper *poS3HandleHelper) |
5823 | | { |
5824 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
5825 | | |
5826 | | poS3HandleHelper->ResetQueryParameters(); |
5827 | | |
5828 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
5829 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5830 | | VSICurlHandleWriteFunc); |
5831 | | |
5832 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
5833 | | &sWriteFuncHeaderData); |
5834 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
5835 | | VSICurlHandleWriteFunc); |
5836 | | |
5837 | | szCurlErrBuf[0] = '\0'; |
5838 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
5839 | | |
5840 | | VSICURLMultiPerform(poFS->GetCurlMultiHandleFor(poS3HandleHelper->GetURL()), |
5841 | | hCurlHandle); |
5842 | | |
5843 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
5844 | | |
5845 | | curl_slist_free_all(headers); |
5846 | | |
5847 | | long response_code = 0; |
5848 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
5849 | | return response_code; |
5850 | | } |
5851 | | |
5852 | | /************************************************************************/ |
5853 | | /* NetworkStatisticsLogger */ |
5854 | | /************************************************************************/ |
5855 | | |
5856 | | // Global variable |
5857 | | NetworkStatisticsLogger NetworkStatisticsLogger::gInstance{}; |
5858 | | int NetworkStatisticsLogger::gnEnabled = -1; // unknown state |
5859 | | |
5860 | | static void ShowNetworkStats() |
5861 | | { |
5862 | | printf("Network statistics:\n%s\n", // ok |
5863 | | NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
5864 | | } |
5865 | | |
5866 | | void NetworkStatisticsLogger::ReadEnabled() |
5867 | | { |
5868 | | const bool bShowNetworkStats = |
5869 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_SHOW_NETWORK_STATS", "NO")); |
5870 | | gnEnabled = |
5871 | | (bShowNetworkStats || CPLTestBool(CPLGetConfigOption( |
5872 | | "CPL_VSIL_NETWORK_STATS_ENABLED", "NO"))) |
5873 | | ? TRUE |
5874 | | : FALSE; |
5875 | | if (bShowNetworkStats) |
5876 | | { |
5877 | | static bool bRegistered = false; |
5878 | | if (!bRegistered) |
5879 | | { |
5880 | | bRegistered = true; |
5881 | | atexit(ShowNetworkStats); |
5882 | | } |
5883 | | } |
5884 | | } |
5885 | | |
5886 | | void NetworkStatisticsLogger::EnterFileSystem(const char *pszName) |
5887 | | { |
5888 | | if (!IsEnabled()) |
5889 | | return; |
5890 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5891 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
5892 | | ContextPathItem(ContextPathType::FILESYSTEM, pszName)); |
5893 | | } |
5894 | | |
5895 | | void NetworkStatisticsLogger::LeaveFileSystem() |
5896 | | { |
5897 | | if (!IsEnabled()) |
5898 | | return; |
5899 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5900 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
5901 | | } |
5902 | | |
5903 | | void NetworkStatisticsLogger::EnterFile(const char *pszName) |
5904 | | { |
5905 | | if (!IsEnabled()) |
5906 | | return; |
5907 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5908 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
5909 | | ContextPathItem(ContextPathType::FILE, pszName)); |
5910 | | } |
5911 | | |
5912 | | void NetworkStatisticsLogger::LeaveFile() |
5913 | | { |
5914 | | if (!IsEnabled()) |
5915 | | return; |
5916 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5917 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
5918 | | } |
5919 | | |
5920 | | void NetworkStatisticsLogger::EnterAction(const char *pszName) |
5921 | | { |
5922 | | if (!IsEnabled()) |
5923 | | return; |
5924 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5925 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
5926 | | ContextPathItem(ContextPathType::ACTION, pszName)); |
5927 | | } |
5928 | | |
5929 | | void NetworkStatisticsLogger::LeaveAction() |
5930 | | { |
5931 | | if (!IsEnabled()) |
5932 | | return; |
5933 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5934 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
5935 | | } |
5936 | | |
5937 | | std::vector<NetworkStatisticsLogger::Counters *> |
5938 | | NetworkStatisticsLogger::GetCountersForContext() |
5939 | | { |
5940 | | std::vector<Counters *> v; |
5941 | | const auto &contextPath = gInstance.m_mapThreadIdToContextPath[CPLGetPID()]; |
5942 | | |
5943 | | Stats *curStats = &m_stats; |
5944 | | v.push_back(&(curStats->counters)); |
5945 | | |
5946 | | bool inFileSystem = false; |
5947 | | bool inFile = false; |
5948 | | bool inAction = false; |
5949 | | for (const auto &item : contextPath) |
5950 | | { |
5951 | | if (item.eType == ContextPathType::FILESYSTEM) |
5952 | | { |
5953 | | if (inFileSystem) |
5954 | | continue; |
5955 | | inFileSystem = true; |
5956 | | } |
5957 | | else if (item.eType == ContextPathType::FILE) |
5958 | | { |
5959 | | if (inFile) |
5960 | | continue; |
5961 | | inFile = true; |
5962 | | } |
5963 | | else if (item.eType == ContextPathType::ACTION) |
5964 | | { |
5965 | | if (inAction) |
5966 | | continue; |
5967 | | inAction = true; |
5968 | | } |
5969 | | |
5970 | | curStats = &(curStats->children[item]); |
5971 | | v.push_back(&(curStats->counters)); |
5972 | | } |
5973 | | |
5974 | | return v; |
5975 | | } |
5976 | | |
5977 | | void NetworkStatisticsLogger::LogGET(size_t nDownloadedBytes) |
5978 | | { |
5979 | | if (!IsEnabled()) |
5980 | | return; |
5981 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5982 | | for (auto counters : gInstance.GetCountersForContext()) |
5983 | | { |
5984 | | counters->nGET++; |
5985 | | counters->nGETDownloadedBytes += nDownloadedBytes; |
5986 | | } |
5987 | | } |
5988 | | |
5989 | | void NetworkStatisticsLogger::LogPUT(size_t nUploadedBytes) |
5990 | | { |
5991 | | if (!IsEnabled()) |
5992 | | return; |
5993 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5994 | | for (auto counters : gInstance.GetCountersForContext()) |
5995 | | { |
5996 | | counters->nPUT++; |
5997 | | counters->nPUTUploadedBytes += nUploadedBytes; |
5998 | | } |
5999 | | } |
6000 | | |
6001 | | void NetworkStatisticsLogger::LogHEAD() |
6002 | | { |
6003 | | if (!IsEnabled()) |
6004 | | return; |
6005 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6006 | | for (auto counters : gInstance.GetCountersForContext()) |
6007 | | { |
6008 | | counters->nHEAD++; |
6009 | | } |
6010 | | } |
6011 | | |
6012 | | void NetworkStatisticsLogger::LogPOST(size_t nUploadedBytes, |
6013 | | size_t nDownloadedBytes) |
6014 | | { |
6015 | | if (!IsEnabled()) |
6016 | | return; |
6017 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6018 | | for (auto counters : gInstance.GetCountersForContext()) |
6019 | | { |
6020 | | counters->nPOST++; |
6021 | | counters->nPOSTUploadedBytes += nUploadedBytes; |
6022 | | counters->nPOSTDownloadedBytes += nDownloadedBytes; |
6023 | | } |
6024 | | } |
6025 | | |
6026 | | void NetworkStatisticsLogger::LogDELETE() |
6027 | | { |
6028 | | if (!IsEnabled()) |
6029 | | return; |
6030 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6031 | | for (auto counters : gInstance.GetCountersForContext()) |
6032 | | { |
6033 | | counters->nDELETE++; |
6034 | | } |
6035 | | } |
6036 | | |
6037 | | void NetworkStatisticsLogger::Reset() |
6038 | | { |
6039 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6040 | | gInstance.m_stats = Stats(); |
6041 | | gnEnabled = -1; |
6042 | | } |
6043 | | |
6044 | | void NetworkStatisticsLogger::Stats::AsJSON(CPLJSONObject &oJSON) const |
6045 | | { |
6046 | | CPLJSONObject oMethods; |
6047 | | if (counters.nHEAD) |
6048 | | oMethods.Add("HEAD/count", counters.nHEAD); |
6049 | | if (counters.nGET) |
6050 | | oMethods.Add("GET/count", counters.nGET); |
6051 | | if (counters.nGETDownloadedBytes) |
6052 | | oMethods.Add("GET/downloaded_bytes", counters.nGETDownloadedBytes); |
6053 | | if (counters.nPUT) |
6054 | | oMethods.Add("PUT/count", counters.nPUT); |
6055 | | if (counters.nPUTUploadedBytes) |
6056 | | oMethods.Add("PUT/uploaded_bytes", counters.nPUTUploadedBytes); |
6057 | | if (counters.nPOST) |
6058 | | oMethods.Add("POST/count", counters.nPOST); |
6059 | | if (counters.nPOSTUploadedBytes) |
6060 | | oMethods.Add("POST/uploaded_bytes", counters.nPOSTUploadedBytes); |
6061 | | if (counters.nPOSTDownloadedBytes) |
6062 | | oMethods.Add("POST/downloaded_bytes", counters.nPOSTDownloadedBytes); |
6063 | | if (counters.nDELETE) |
6064 | | oMethods.Add("DELETE/count", counters.nDELETE); |
6065 | | oJSON.Add("methods", oMethods); |
6066 | | CPLJSONObject oFiles; |
6067 | | bool bFilesAdded = false; |
6068 | | for (const auto &kv : children) |
6069 | | { |
6070 | | CPLJSONObject childJSON; |
6071 | | kv.second.AsJSON(childJSON); |
6072 | | if (kv.first.eType == ContextPathType::FILESYSTEM) |
6073 | | { |
6074 | | std::string osName(kv.first.osName); |
6075 | | if (!osName.empty() && osName[0] == '/') |
6076 | | osName = osName.substr(1); |
6077 | | if (!osName.empty() && osName.back() == '/') |
6078 | | osName.pop_back(); |
6079 | | oJSON.Add(("handlers/" + osName).c_str(), childJSON); |
6080 | | } |
6081 | | else if (kv.first.eType == ContextPathType::FILE) |
6082 | | { |
6083 | | if (!bFilesAdded) |
6084 | | { |
6085 | | bFilesAdded = true; |
6086 | | oJSON.Add("files", oFiles); |
6087 | | } |
6088 | | oFiles.AddNoSplitName(kv.first.osName.c_str(), childJSON); |
6089 | | } |
6090 | | else if (kv.first.eType == ContextPathType::ACTION) |
6091 | | { |
6092 | | oJSON.Add(("actions/" + kv.first.osName).c_str(), childJSON); |
6093 | | } |
6094 | | } |
6095 | | } |
6096 | | |
6097 | | std::string NetworkStatisticsLogger::GetReportAsSerializedJSON() |
6098 | | { |
6099 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6100 | | |
6101 | | CPLJSONObject oJSON; |
6102 | | gInstance.m_stats.AsJSON(oJSON); |
6103 | | return oJSON.Format(CPLJSONObject::PrettyFormat::Pretty); |
6104 | | } |
6105 | | |
6106 | | } /* end of namespace cpl */ |
6107 | | |
6108 | | /************************************************************************/ |
6109 | | /* VSICurlParseUnixPermissions() */ |
6110 | | /************************************************************************/ |
6111 | | |
6112 | | int VSICurlParseUnixPermissions(const char *pszPermissions) |
6113 | | { |
6114 | | if (strlen(pszPermissions) != 9) |
6115 | | return 0; |
6116 | | int nMode = 0; |
6117 | | if (pszPermissions[0] == 'r') |
6118 | | nMode |= S_IRUSR; |
6119 | | if (pszPermissions[1] == 'w') |
6120 | | nMode |= S_IWUSR; |
6121 | | if (pszPermissions[2] == 'x') |
6122 | | nMode |= S_IXUSR; |
6123 | | if (pszPermissions[3] == 'r') |
6124 | | nMode |= S_IRGRP; |
6125 | | if (pszPermissions[4] == 'w') |
6126 | | nMode |= S_IWGRP; |
6127 | | if (pszPermissions[5] == 'x') |
6128 | | nMode |= S_IXGRP; |
6129 | | if (pszPermissions[6] == 'r') |
6130 | | nMode |= S_IROTH; |
6131 | | if (pszPermissions[7] == 'w') |
6132 | | nMode |= S_IWOTH; |
6133 | | if (pszPermissions[8] == 'x') |
6134 | | nMode |= S_IXOTH; |
6135 | | return nMode; |
6136 | | } |
6137 | | |
6138 | | /************************************************************************/ |
6139 | | /* Cache of file properties. */ |
6140 | | /************************************************************************/ |
6141 | | |
6142 | | static std::mutex oCacheFilePropMutex; |
6143 | | static lru11::Cache<std::string, cpl::FileProp> *poCacheFileProp = nullptr; |
6144 | | |
6145 | | /************************************************************************/ |
6146 | | /* VSICURLGetCachedFileProp() */ |
6147 | | /************************************************************************/ |
6148 | | |
6149 | | bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6150 | | { |
6151 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6152 | | return poCacheFileProp != nullptr && |
6153 | | poCacheFileProp->tryGet(std::string(pszURL), oFileProp) && |
6154 | | // Let a chance to use new auth parameters |
6155 | | !(oFileProp.eExists == cpl::EXIST_NO && |
6156 | | gnGenerationAuthParameters != oFileProp.nGenerationAuthParameters); |
6157 | | } |
6158 | | |
6159 | | /************************************************************************/ |
6160 | | /* VSICURLSetCachedFileProp() */ |
6161 | | /************************************************************************/ |
6162 | | |
6163 | | void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6164 | | { |
6165 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6166 | | if (poCacheFileProp == nullptr) |
6167 | | poCacheFileProp = |
6168 | | new lru11::Cache<std::string, cpl::FileProp>(100 * 1024); |
6169 | | oFileProp.nGenerationAuthParameters = gnGenerationAuthParameters; |
6170 | | poCacheFileProp->insert(std::string(pszURL), oFileProp); |
6171 | | } |
6172 | | |
6173 | | /************************************************************************/ |
6174 | | /* VSICURLInvalidateCachedFileProp() */ |
6175 | | /************************************************************************/ |
6176 | | |
6177 | | void VSICURLInvalidateCachedFileProp(const char *pszURL) |
6178 | | { |
6179 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6180 | | if (poCacheFileProp != nullptr) |
6181 | | poCacheFileProp->remove(std::string(pszURL)); |
6182 | | } |
6183 | | |
6184 | | /************************************************************************/ |
6185 | | /* VSICURLInvalidateCachedFilePropPrefix() */ |
6186 | | /************************************************************************/ |
6187 | | |
6188 | | void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL) |
6189 | | { |
6190 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6191 | | if (poCacheFileProp != nullptr) |
6192 | | { |
6193 | | std::list<std::string> keysToRemove; |
6194 | | const size_t nURLSize = strlen(pszURL); |
6195 | | auto lambda = |
6196 | | [&keysToRemove, &pszURL, nURLSize]( |
6197 | | const lru11::KeyValuePair<std::string, cpl::FileProp> &kv) |
6198 | | { |
6199 | | if (strncmp(kv.key.c_str(), pszURL, nURLSize) == 0) |
6200 | | keysToRemove.push_back(kv.key); |
6201 | | }; |
6202 | | poCacheFileProp->cwalk(lambda); |
6203 | | for (const auto &key : keysToRemove) |
6204 | | poCacheFileProp->remove(key); |
6205 | | } |
6206 | | } |
6207 | | |
6208 | | /************************************************************************/ |
6209 | | /* VSICURLDestroyCacheFileProp() */ |
6210 | | /************************************************************************/ |
6211 | | |
6212 | | void VSICURLDestroyCacheFileProp() |
6213 | | { |
6214 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6215 | | delete poCacheFileProp; |
6216 | | poCacheFileProp = nullptr; |
6217 | | } |
6218 | | |
6219 | | /************************************************************************/ |
6220 | | /* VSICURLMultiCleanup() */ |
6221 | | /************************************************************************/ |
6222 | | |
6223 | | void VSICURLMultiCleanup(CURLM *hCurlMultiHandle) |
6224 | | { |
6225 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
6226 | | curl_multi_cleanup(hCurlMultiHandle); |
6227 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
6228 | | } |
6229 | | |
6230 | | /************************************************************************/ |
6231 | | /* VSICurlInstallReadCbk() */ |
6232 | | /************************************************************************/ |
6233 | | |
6234 | | int VSICurlInstallReadCbk(VSILFILE *fp, VSICurlReadCbkFunc pfnReadCbk, |
6235 | | void *pfnUserData, int bStopOnInterruptUntilUninstall) |
6236 | | { |
6237 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->InstallReadCbk( |
6238 | | pfnReadCbk, pfnUserData, bStopOnInterruptUntilUninstall); |
6239 | | } |
6240 | | |
6241 | | /************************************************************************/ |
6242 | | /* VSICurlUninstallReadCbk() */ |
6243 | | /************************************************************************/ |
6244 | | |
6245 | | int VSICurlUninstallReadCbk(VSILFILE *fp) |
6246 | | { |
6247 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->UninstallReadCbk(); |
6248 | | } |
6249 | | |
6250 | | /************************************************************************/ |
6251 | | /* VSICurlSetOptions() */ |
6252 | | /************************************************************************/ |
6253 | | |
6254 | | struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL, |
6255 | | const char *const *papszOptions) |
6256 | | { |
6257 | | struct curl_slist *headers = static_cast<struct curl_slist *>( |
6258 | | CPLHTTPSetOptions(hCurlHandle, pszURL, papszOptions)); |
6259 | | |
6260 | | long option = CURLFTPMETHOD_SINGLECWD; |
6261 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_FILEMETHOD, option); |
6262 | | |
6263 | | // ftp://ftp2.cits.rncan.gc.ca/pub/cantopo/250k_tif/ |
6264 | | // doesn't like EPSV command, |
6265 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_USE_EPSV, 0); |
6266 | | |
6267 | | return headers; |
6268 | | } |
6269 | | |
6270 | | /************************************************************************/ |
6271 | | /* VSICurlSetContentTypeFromExt() */ |
6272 | | /************************************************************************/ |
6273 | | |
6274 | | struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *poList, |
6275 | | const char *pszPath) |
6276 | | { |
6277 | | struct curl_slist *iter = poList; |
6278 | | while (iter != nullptr) |
6279 | | { |
6280 | | if (STARTS_WITH_CI(iter->data, "Content-Type")) |
6281 | | { |
6282 | | return poList; |
6283 | | } |
6284 | | iter = iter->next; |
6285 | | } |
6286 | | |
6287 | | static const struct |
6288 | | { |
6289 | | const char *ext; |
6290 | | const char *mime; |
6291 | | } aosExtMimePairs[] = { |
6292 | | {"txt", "text/plain"}, {"json", "application/json"}, |
6293 | | {"tif", "image/tiff"}, {"tiff", "image/tiff"}, |
6294 | | {"jpg", "image/jpeg"}, {"jpeg", "image/jpeg"}, |
6295 | | {"jp2", "image/jp2"}, {"jpx", "image/jp2"}, |
6296 | | {"j2k", "image/jp2"}, {"jpc", "image/jp2"}, |
6297 | | {"png", "image/png"}, |
6298 | | }; |
6299 | | |
6300 | | const std::string osExt = CPLGetExtensionSafe(pszPath); |
6301 | | if (!osExt.empty()) |
6302 | | { |
6303 | | for (const auto &pair : aosExtMimePairs) |
6304 | | { |
6305 | | if (EQUAL(osExt.c_str(), pair.ext)) |
6306 | | { |
6307 | | |
6308 | | const std::string osContentType( |
6309 | | CPLSPrintf("Content-Type: %s", pair.mime)); |
6310 | | poList = curl_slist_append(poList, osContentType.c_str()); |
6311 | | #ifdef DEBUG_VERBOSE |
6312 | | CPLDebug("HTTP", "Setting %s, based on lookup table.", |
6313 | | osContentType.c_str()); |
6314 | | #endif |
6315 | | break; |
6316 | | } |
6317 | | } |
6318 | | } |
6319 | | |
6320 | | return poList; |
6321 | | } |
6322 | | |
6323 | | /************************************************************************/ |
6324 | | /* VSICurlSetCreationHeadersFromOptions() */ |
6325 | | /************************************************************************/ |
6326 | | |
6327 | | struct curl_slist *VSICurlSetCreationHeadersFromOptions( |
6328 | | struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath) |
6329 | | { |
6330 | | bool bContentTypeFound = false; |
6331 | | for (CSLConstList papszIter = papszOptions; papszIter && *papszIter; |
6332 | | ++papszIter) |
6333 | | { |
6334 | | char *pszKey = nullptr; |
6335 | | const char *pszValue = CPLParseNameValue(*papszIter, &pszKey); |
6336 | | if (pszKey && pszValue) |
6337 | | { |
6338 | | if (EQUAL(pszKey, "Content-Type")) |
6339 | | { |
6340 | | bContentTypeFound = true; |
6341 | | } |
6342 | | headers = curl_slist_append(headers, |
6343 | | CPLSPrintf("%s: %s", pszKey, pszValue)); |
6344 | | } |
6345 | | CPLFree(pszKey); |
6346 | | } |
6347 | | |
6348 | | // If Content-type not found in papszOptions, try to set it from the |
6349 | | // filename exstension. |
6350 | | if (!bContentTypeFound) |
6351 | | { |
6352 | | headers = VSICurlSetContentTypeFromExt(headers, pszPath); |
6353 | | } |
6354 | | |
6355 | | return headers; |
6356 | | } |
6357 | | |
6358 | | #endif // DOXYGEN_SKIP |
6359 | | //! @endcond |
6360 | | |
6361 | | /************************************************************************/ |
6362 | | /* VSIInstallCurlFileHandler() */ |
6363 | | /************************************************************************/ |
6364 | | |
6365 | | /*! |
6366 | | \brief Install /vsicurl/ HTTP/FTP file system handler (requires libcurl) |
6367 | | |
6368 | | \verbatim embed:rst |
6369 | | See :ref:`/vsicurl/ documentation <vsicurl>` |
6370 | | \endverbatim |
6371 | | |
6372 | | */ |
6373 | | void VSIInstallCurlFileHandler(void) |
6374 | | { |
6375 | | auto poHandler = std::make_shared<cpl::VSICurlFilesystemHandler>(); |
6376 | | VSIFileManager::InstallHandler("/vsicurl/", poHandler); |
6377 | | VSIFileManager::InstallHandler("/vsicurl?", poHandler); |
6378 | | } |
6379 | | |
6380 | | /************************************************************************/ |
6381 | | /* VSICurlClearCache() */ |
6382 | | /************************************************************************/ |
6383 | | |
6384 | | /** |
6385 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6386 | | * |
6387 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6388 | | * /vsiswift/) cache a number of |
6389 | | * metadata and data for faster execution in read-only scenarios. But when the |
6390 | | * content on the server-side may change during the same process, those |
6391 | | * mechanisms can prevent opening new files, or give an outdated version of |
6392 | | * them. |
6393 | | * |
6394 | | */ |
6395 | | |
6396 | | void VSICurlClearCache(void) |
6397 | | { |
6398 | | // FIXME ? Currently we have different filesystem instances for |
6399 | | // vsicurl/, /vsis3/, /vsigs/ . So each one has its own cache of regions. |
6400 | | // File properties cache are now shared |
6401 | | char **papszPrefix = VSIFileManager::GetPrefixes(); |
6402 | | for (size_t i = 0; papszPrefix && papszPrefix[i]; ++i) |
6403 | | { |
6404 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6405 | | VSIFileManager::GetHandler(papszPrefix[i])); |
6406 | | |
6407 | | if (poFSHandler) |
6408 | | poFSHandler->ClearCache(); |
6409 | | } |
6410 | | CSLDestroy(papszPrefix); |
6411 | | |
6412 | | VSICurlStreamingClearCache(); |
6413 | | } |
6414 | | |
6415 | | /************************************************************************/ |
6416 | | /* VSICurlPartialClearCache() */ |
6417 | | /************************************************************************/ |
6418 | | |
6419 | | /** |
6420 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6421 | | * for a given filename (and its subfiles and subdirectories if it is a |
6422 | | * directory) |
6423 | | * |
6424 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6425 | | * /vsiswift/) cache a number of |
6426 | | * metadata and data for faster execution in read-only scenarios. But when the |
6427 | | * content on the server-side may change during the same process, those |
6428 | | * mechanisms can prevent opening new files, or give an outdated version of |
6429 | | * them. |
6430 | | * |
6431 | | * The filename prefix must start with the name of a known virtual file system |
6432 | | * (such as "/vsicurl/", "/vsis3/") |
6433 | | * |
6434 | | * VSICurlPartialClearCache("/vsis3/b") will clear all cached state for any file |
6435 | | * or directory starting with that prefix, so potentially "/vsis3/bucket", |
6436 | | * "/vsis3/basket/" or "/vsis3/basket/object". |
6437 | | * |
6438 | | * @param pszFilenamePrefix Filename prefix |
6439 | | */ |
6440 | | |
6441 | | void VSICurlPartialClearCache(const char *pszFilenamePrefix) |
6442 | | { |
6443 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6444 | | VSIFileManager::GetHandler(pszFilenamePrefix)); |
6445 | | |
6446 | | if (poFSHandler) |
6447 | | poFSHandler->PartialClearCache(pszFilenamePrefix); |
6448 | | } |
6449 | | |
6450 | | /************************************************************************/ |
6451 | | /* VSINetworkStatsReset() */ |
6452 | | /************************************************************************/ |
6453 | | |
6454 | | /** |
6455 | | * \brief Clear network related statistics. |
6456 | | * |
6457 | | * The effect of the CPL_VSIL_NETWORK_STATS_ENABLED configuration option |
6458 | | * will also be reset. That is, that the next network access will check its |
6459 | | * value again. |
6460 | | * |
6461 | | * @since GDAL 3.2.0 |
6462 | | */ |
6463 | | |
6464 | | void VSINetworkStatsReset(void) |
6465 | | { |
6466 | | cpl::NetworkStatisticsLogger::Reset(); |
6467 | | } |
6468 | | |
6469 | | /************************************************************************/ |
6470 | | /* VSINetworkStatsGetAsSerializedJSON() */ |
6471 | | /************************************************************************/ |
6472 | | |
6473 | | /** |
6474 | | * \brief Return network related statistics, as a JSON serialized object. |
6475 | | * |
6476 | | * Statistics collecting should be enabled with the |
6477 | | CPL_VSIL_NETWORK_STATS_ENABLED |
6478 | | * configuration option set to YES before any network activity starts |
6479 | | * (for efficiency, reading it is cached on first access, until |
6480 | | VSINetworkStatsReset() is called) |
6481 | | * |
6482 | | * Statistics can also be emitted on standard output at process termination if |
6483 | | * the CPL_VSIL_SHOW_NETWORK_STATS configuration option is set to YES. |
6484 | | * |
6485 | | * Example of output: |
6486 | | * \code{.js} |
6487 | | * { |
6488 | | * "methods":{ |
6489 | | * "GET":{ |
6490 | | * "count":6, |
6491 | | * "downloaded_bytes":40825 |
6492 | | * }, |
6493 | | * "PUT":{ |
6494 | | * "count":1, |
6495 | | * "uploaded_bytes":35472 |
6496 | | * } |
6497 | | * }, |
6498 | | * "handlers":{ |
6499 | | * "vsigs":{ |
6500 | | * "methods":{ |
6501 | | * "GET":{ |
6502 | | * "count":2, |
6503 | | * "downloaded_bytes":446 |
6504 | | * }, |
6505 | | * "PUT":{ |
6506 | | * "count":1, |
6507 | | * "uploaded_bytes":35472 |
6508 | | * } |
6509 | | * }, |
6510 | | * "files":{ |
6511 | | * "\/vsigs\/spatialys\/byte.tif":{ |
6512 | | * "methods":{ |
6513 | | * "PUT":{ |
6514 | | * "count":1, |
6515 | | * "uploaded_bytes":35472 |
6516 | | * } |
6517 | | * }, |
6518 | | * "actions":{ |
6519 | | * "Write":{ |
6520 | | * "methods":{ |
6521 | | * "PUT":{ |
6522 | | * "count":1, |
6523 | | * "uploaded_bytes":35472 |
6524 | | * } |
6525 | | * } |
6526 | | * } |
6527 | | * } |
6528 | | * } |
6529 | | * }, |
6530 | | * "actions":{ |
6531 | | * "Stat":{ |
6532 | | * "methods":{ |
6533 | | * "GET":{ |
6534 | | * "count":2, |
6535 | | * "downloaded_bytes":446 |
6536 | | * } |
6537 | | * }, |
6538 | | * "files":{ |
6539 | | * "\/vsigs\/spatialys\/byte.tif\/":{ |
6540 | | * "methods":{ |
6541 | | * "GET":{ |
6542 | | * "count":1, |
6543 | | * "downloaded_bytes":181 |
6544 | | * } |
6545 | | * } |
6546 | | * } |
6547 | | * } |
6548 | | * } |
6549 | | * } |
6550 | | * }, |
6551 | | * "vsis3":{ |
6552 | | * [...] |
6553 | | * } |
6554 | | * } |
6555 | | * } |
6556 | | * \endcode |
6557 | | * |
6558 | | * @param papszOptions Unused. |
6559 | | * @return a JSON serialized string to free with VSIFree(), or nullptr |
6560 | | * @since GDAL 3.2.0 |
6561 | | */ |
6562 | | |
6563 | | char *VSINetworkStatsGetAsSerializedJSON(CPL_UNUSED char **papszOptions) |
6564 | | { |
6565 | | return CPLStrdup( |
6566 | | cpl::NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
6567 | | } |
6568 | | |
6569 | | #endif /* HAVE_CURL */ |
6570 | | |
6571 | | #undef ENABLE_DEBUG |