/src/gdal/port/cpl_vsil_curl.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: CPL - Common Portability Library |
4 | | * Purpose: Implement VSI large file api for HTTP/FTP files |
5 | | * Author: Even Rouault, even.rouault at spatialys.com |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_port.h" |
14 | | #include "cpl_vsil_curl_priv.h" |
15 | | #include "cpl_vsil_curl_class.h" |
16 | | |
17 | | #include <algorithm> |
18 | | #include <array> |
19 | | #include <limits> |
20 | | #include <map> |
21 | | #include <memory> |
22 | | #include <numeric> |
23 | | #include <set> |
24 | | #include <string_view> |
25 | | |
26 | | #include "cpl_aws.h" |
27 | | #include "cpl_json.h" |
28 | | #include "cpl_json_header.h" |
29 | | #include "cpl_minixml.h" |
30 | | #include "cpl_multiproc.h" |
31 | | #include "cpl_string.h" |
32 | | #include "cpl_time.h" |
33 | | #include "cpl_vsi.h" |
34 | | #include "cpl_vsi_virtual.h" |
35 | | #include "cpl_http.h" |
36 | | #include "cpl_mem_cache.h" |
37 | | |
38 | | #ifndef S_IRUSR |
39 | | #define S_IRUSR 00400 |
40 | | #define S_IWUSR 00200 |
41 | | #define S_IXUSR 00100 |
42 | | #define S_IRGRP 00040 |
43 | | #define S_IWGRP 00020 |
44 | | #define S_IXGRP 00010 |
45 | | #define S_IROTH 00004 |
46 | | #define S_IWOTH 00002 |
47 | | #define S_IXOTH 00001 |
48 | | #endif |
49 | | |
50 | | #ifndef HAVE_CURL |
51 | | |
52 | | void VSIInstallCurlFileHandler(void) |
53 | 0 | { |
54 | | // Not supported. |
55 | 0 | } |
56 | | |
57 | | void VSICurlClearCache(void) |
58 | 0 | { |
59 | | // Not supported. |
60 | 0 | } |
61 | | |
62 | | void VSICurlPartialClearCache(const char *) |
63 | 0 | { |
64 | | // Not supported. |
65 | 0 | } |
66 | | |
67 | | void VSICurlAuthParametersChanged() |
68 | 370 | { |
69 | | // Not supported. |
70 | 370 | } |
71 | | |
72 | | void VSINetworkStatsReset(void) |
73 | 0 | { |
74 | | // Not supported |
75 | 0 | } |
76 | | |
77 | | char *VSINetworkStatsGetAsSerializedJSON(char ** /* papszOptions */) |
78 | 0 | { |
79 | | // Not supported |
80 | 0 | return nullptr; |
81 | 0 | } |
82 | | |
83 | | /************************************************************************/ |
84 | | /* VSICurlInstallReadCbk() */ |
85 | | /************************************************************************/ |
86 | | |
87 | | int VSICurlInstallReadCbk(VSILFILE * /* fp */, |
88 | | VSICurlReadCbkFunc /* pfnReadCbk */, |
89 | | void * /* pfnUserData */, |
90 | | int /* bStopOnInterruptUntilUninstall */) |
91 | 0 | { |
92 | 0 | return FALSE; |
93 | 0 | } |
94 | | |
95 | | /************************************************************************/ |
96 | | /* VSICurlUninstallReadCbk() */ |
97 | | /************************************************************************/ |
98 | | |
99 | | int VSICurlUninstallReadCbk(VSILFILE * /* fp */) |
100 | 0 | { |
101 | 0 | return FALSE; |
102 | 0 | } |
103 | | |
104 | | #else |
105 | | |
106 | | //! @cond Doxygen_Suppress |
107 | | #ifndef DOXYGEN_SKIP |
108 | | |
109 | | #define ENABLE_DEBUG 1 |
110 | | #define ENABLE_DEBUG_VERBOSE 0 |
111 | | |
112 | | #define unchecked_curl_easy_setopt(handle, opt, param) \ |
113 | | CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param)) |
114 | | |
115 | | constexpr const char *const VSICURL_PREFIXES[] = {"/vsicurl/", "/vsicurl?"}; |
116 | | |
117 | | extern "C" bool CPL_DLL GDALIsInGlobalDestructorFromDLLMain(); |
118 | | |
119 | | /***********************************************************รน************/ |
120 | | /* VSICurlAuthParametersChanged() */ |
121 | | /************************************************************************/ |
122 | | |
123 | | static unsigned int gnGenerationAuthParameters = 0; |
124 | | |
125 | | void VSICurlAuthParametersChanged() |
126 | | { |
127 | | gnGenerationAuthParameters++; |
128 | | } |
129 | | |
130 | | // Do not access those variables directly ! |
131 | | // Use VSICURLGetDownloadChunkSize() and GetMaxRegions() |
132 | | static int N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = 0; |
133 | | static int DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = 0; |
134 | | |
135 | | /************************************************************************/ |
136 | | /* VSICURLReadGlobalEnvVariables() */ |
137 | | /************************************************************************/ |
138 | | |
139 | | static void VSICURLReadGlobalEnvVariables() |
140 | | { |
141 | | struct Initializer |
142 | | { |
143 | | Initializer() |
144 | | { |
145 | | constexpr int DOWNLOAD_CHUNK_SIZE_DEFAULT = 16384; |
146 | | const char *pszChunkSize = |
147 | | CPLGetConfigOption("CPL_VSIL_CURL_CHUNK_SIZE", nullptr); |
148 | | GIntBig nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
149 | | |
150 | | if (pszChunkSize) |
151 | | { |
152 | | if (CPLParseMemorySize(pszChunkSize, &nChunkSize, nullptr) != |
153 | | CE_None) |
154 | | { |
155 | | CPLError( |
156 | | CE_Warning, CPLE_AppDefined, |
157 | | "Could not parse value for CPL_VSIL_CURL_CHUNK_SIZE. " |
158 | | "Using default value of %d instead.", |
159 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
160 | | } |
161 | | } |
162 | | |
163 | | constexpr int MIN_CHUNK_SIZE = 1024; |
164 | | constexpr int MAX_CHUNK_SIZE = 10 * 1024 * 1024; |
165 | | if (nChunkSize < MIN_CHUNK_SIZE || nChunkSize > MAX_CHUNK_SIZE) |
166 | | { |
167 | | nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
168 | | CPLError(CE_Warning, CPLE_AppDefined, |
169 | | "Invalid value for CPL_VSIL_CURL_CHUNK_SIZE. " |
170 | | "Allowed range is [%d, %d]. " |
171 | | "Using CPL_VSIL_CURL_CHUNK_SIZE=%d instead", |
172 | | MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, |
173 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
174 | | } |
175 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = |
176 | | static_cast<int>(nChunkSize); |
177 | | |
178 | | constexpr int N_MAX_REGIONS_DEFAULT = 1000; |
179 | | constexpr int CACHE_SIZE_DEFAULT = |
180 | | N_MAX_REGIONS_DEFAULT * DOWNLOAD_CHUNK_SIZE_DEFAULT; |
181 | | |
182 | | const char *pszCacheSize = |
183 | | CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE", nullptr); |
184 | | GIntBig nCacheSize = CACHE_SIZE_DEFAULT; |
185 | | |
186 | | if (pszCacheSize) |
187 | | { |
188 | | if (CPLParseMemorySize(pszCacheSize, &nCacheSize, nullptr) != |
189 | | CE_None) |
190 | | { |
191 | | CPLError( |
192 | | CE_Warning, CPLE_AppDefined, |
193 | | "Could not parse value for CPL_VSIL_CURL_CACHE_SIZE. " |
194 | | "Using default value of " CPL_FRMT_GIB " instead.", |
195 | | nCacheSize); |
196 | | } |
197 | | } |
198 | | |
199 | | const auto nMaxRAM = CPLGetUsablePhysicalRAM(); |
200 | | const auto nMinVal = DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
201 | | auto nMaxVal = static_cast<GIntBig>(INT_MAX) * |
202 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
203 | | if (nMaxRAM > 0 && nMaxVal > nMaxRAM) |
204 | | nMaxVal = nMaxRAM; |
205 | | if (nCacheSize < nMinVal || nCacheSize > nMaxVal) |
206 | | { |
207 | | nCacheSize = nCacheSize < nMinVal ? nMinVal : nMaxVal; |
208 | | CPLError(CE_Warning, CPLE_AppDefined, |
209 | | "Invalid value for CPL_VSIL_CURL_CACHE_SIZE. " |
210 | | "Allowed range is [%d, " CPL_FRMT_GIB "]. " |
211 | | "Using CPL_VSIL_CURL_CACHE_SIZE=" CPL_FRMT_GIB |
212 | | " instead", |
213 | | nMinVal, nMaxVal, nCacheSize); |
214 | | } |
215 | | N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = std::max( |
216 | | 1, static_cast<int>(nCacheSize / |
217 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY)); |
218 | | } |
219 | | }; |
220 | | |
221 | | static Initializer initializer; |
222 | | } |
223 | | |
224 | | /************************************************************************/ |
225 | | /* VSICURLGetDownloadChunkSize() */ |
226 | | /************************************************************************/ |
227 | | |
228 | | int VSICURLGetDownloadChunkSize() |
229 | | { |
230 | | VSICURLReadGlobalEnvVariables(); |
231 | | return DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
232 | | } |
233 | | |
234 | | /************************************************************************/ |
235 | | /* GetMaxRegions() */ |
236 | | /************************************************************************/ |
237 | | |
238 | | static int GetMaxRegions() |
239 | | { |
240 | | VSICURLReadGlobalEnvVariables(); |
241 | | return N_MAX_REGIONS_DO_NOT_USE_DIRECTLY; |
242 | | } |
243 | | |
244 | | /************************************************************************/ |
245 | | /* VSICurlFindStringSensitiveExceptEscapeSequences() */ |
246 | | /************************************************************************/ |
247 | | |
248 | | static int |
249 | | VSICurlFindStringSensitiveExceptEscapeSequences(CSLConstList papszList, |
250 | | const char *pszTarget) |
251 | | |
252 | | { |
253 | | if (papszList == nullptr) |
254 | | return -1; |
255 | | |
256 | | for (int i = 0; papszList[i] != nullptr; i++) |
257 | | { |
258 | | const char *pszIter1 = papszList[i]; |
259 | | const char *pszIter2 = pszTarget; |
260 | | char ch1 = '\0'; |
261 | | char ch2 = '\0'; |
262 | | /* The comparison is case-sensitive, except for escaped */ |
263 | | /* sequences where letters of the hexadecimal sequence */ |
264 | | /* can be uppercase or lowercase depending on the quoting algorithm */ |
265 | | while (true) |
266 | | { |
267 | | ch1 = *pszIter1; |
268 | | ch2 = *pszIter2; |
269 | | if (ch1 == '\0' || ch2 == '\0') |
270 | | break; |
271 | | if (ch1 == '%' && ch2 == '%' && pszIter1[1] != '\0' && |
272 | | pszIter1[2] != '\0' && pszIter2[1] != '\0' && |
273 | | pszIter2[2] != '\0') |
274 | | { |
275 | | if (!EQUALN(pszIter1 + 1, pszIter2 + 1, 2)) |
276 | | break; |
277 | | pszIter1 += 2; |
278 | | pszIter2 += 2; |
279 | | } |
280 | | if (ch1 != ch2) |
281 | | break; |
282 | | pszIter1++; |
283 | | pszIter2++; |
284 | | } |
285 | | if (ch1 == ch2 && ch1 == '\0') |
286 | | return i; |
287 | | } |
288 | | |
289 | | return -1; |
290 | | } |
291 | | |
292 | | /************************************************************************/ |
293 | | /* VSICurlIsFileInList() */ |
294 | | /************************************************************************/ |
295 | | |
296 | | static int VSICurlIsFileInList(CSLConstList papszList, const char *pszTarget) |
297 | | { |
298 | | int nRet = |
299 | | VSICurlFindStringSensitiveExceptEscapeSequences(papszList, pszTarget); |
300 | | if (nRet >= 0) |
301 | | return nRet; |
302 | | |
303 | | // If we didn't find anything, try to URL-escape the target filename. |
304 | | char *pszEscaped = CPLEscapeString(pszTarget, -1, CPLES_URL); |
305 | | if (strcmp(pszTarget, pszEscaped) != 0) |
306 | | { |
307 | | nRet = VSICurlFindStringSensitiveExceptEscapeSequences(papszList, |
308 | | pszEscaped); |
309 | | } |
310 | | CPLFree(pszEscaped); |
311 | | return nRet; |
312 | | } |
313 | | |
314 | | /************************************************************************/ |
315 | | /* StartsWithVSICurlPrefix() */ |
316 | | /************************************************************************/ |
317 | | |
318 | | static bool StartsWithVSICurlPrefix(const char *pszFilename) |
319 | | { |
320 | | for (const char *pszPrefix : VSICURL_PREFIXES) |
321 | | { |
322 | | if (STARTS_WITH(pszFilename, pszPrefix)) |
323 | | { |
324 | | return true; |
325 | | } |
326 | | } |
327 | | return false; |
328 | | } |
329 | | |
330 | | /************************************************************************/ |
331 | | /* VSICurlGetURLFromFilename() */ |
332 | | /************************************************************************/ |
333 | | |
334 | | static std::string VSICurlGetURLFromFilename( |
335 | | const char *pszFilename, CPLHTTPRetryParameters *poRetryParameters, |
336 | | bool *pbUseHead, bool *pbUseRedirectURLIfNoQueryStringParams, |
337 | | bool *pbListDir, bool *pbEmptyDir, CPLStringList *paosHTTPOptions, |
338 | | bool *pbPlanetaryComputerURLSigning, char **ppszPlanetaryComputerCollection) |
339 | | { |
340 | | if (ppszPlanetaryComputerCollection) |
341 | | *ppszPlanetaryComputerCollection = nullptr; |
342 | | |
343 | | if (!StartsWithVSICurlPrefix(pszFilename)) |
344 | | return pszFilename; |
345 | | |
346 | | if (pbPlanetaryComputerURLSigning) |
347 | | { |
348 | | // It may be more convenient sometimes to store Planetary Computer URL |
349 | | // signing as a per-path specific option rather than capturing it in |
350 | | // the filename with the &pc_url_signing=yes option. |
351 | | if (CPLTestBool(VSIGetPathSpecificOption( |
352 | | pszFilename, "VSICURL_PC_URL_SIGNING", "FALSE"))) |
353 | | { |
354 | | *pbPlanetaryComputerURLSigning = true; |
355 | | } |
356 | | } |
357 | | |
358 | | pszFilename += strlen("/vsicurl/"); |
359 | | if (!STARTS_WITH(pszFilename, "http://") && |
360 | | !STARTS_WITH(pszFilename, "https://") && |
361 | | !STARTS_WITH(pszFilename, "ftp://") && |
362 | | !STARTS_WITH(pszFilename, "file://")) |
363 | | { |
364 | | if (*pszFilename == '?') |
365 | | pszFilename++; |
366 | | char **papszTokens = CSLTokenizeString2(pszFilename, "&", 0); |
367 | | for (int i = 0; papszTokens[i] != nullptr; i++) |
368 | | { |
369 | | char *pszUnescaped = |
370 | | CPLUnescapeString(papszTokens[i], nullptr, CPLES_URL); |
371 | | CPLFree(papszTokens[i]); |
372 | | papszTokens[i] = pszUnescaped; |
373 | | } |
374 | | |
375 | | std::string osURL; |
376 | | std::string osHeaders; |
377 | | for (int i = 0; papszTokens[i]; i++) |
378 | | { |
379 | | char *pszKey = nullptr; |
380 | | const char *pszValue = CPLParseNameValue(papszTokens[i], &pszKey); |
381 | | if (pszKey && pszValue) |
382 | | { |
383 | | if (EQUAL(pszKey, "max_retry")) |
384 | | { |
385 | | if (poRetryParameters) |
386 | | poRetryParameters->nMaxRetry = atoi(pszValue); |
387 | | } |
388 | | else if (EQUAL(pszKey, "retry_delay")) |
389 | | { |
390 | | if (poRetryParameters) |
391 | | poRetryParameters->dfInitialDelay = CPLAtof(pszValue); |
392 | | } |
393 | | else if (EQUAL(pszKey, "retry_codes")) |
394 | | { |
395 | | if (poRetryParameters) |
396 | | poRetryParameters->osRetryCodes = pszValue; |
397 | | } |
398 | | else if (EQUAL(pszKey, "use_head")) |
399 | | { |
400 | | if (pbUseHead) |
401 | | *pbUseHead = CPLTestBool(pszValue); |
402 | | } |
403 | | else if (EQUAL(pszKey, |
404 | | "use_redirect_url_if_no_query_string_params")) |
405 | | { |
406 | | /* Undocumented. Used by PLScenes driver */ |
407 | | if (pbUseRedirectURLIfNoQueryStringParams) |
408 | | *pbUseRedirectURLIfNoQueryStringParams = |
409 | | CPLTestBool(pszValue); |
410 | | } |
411 | | else if (EQUAL(pszKey, "list_dir")) |
412 | | { |
413 | | if (pbListDir) |
414 | | *pbListDir = CPLTestBool(pszValue); |
415 | | } |
416 | | else if (EQUAL(pszKey, "empty_dir")) |
417 | | { |
418 | | if (pbEmptyDir) |
419 | | *pbEmptyDir = CPLTestBool(pszValue); |
420 | | } |
421 | | else if (EQUAL(pszKey, "header_file")) |
422 | | { |
423 | | #if defined(CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED) |
424 | | constexpr bool CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED = |
425 | | true; |
426 | | #else |
427 | | constexpr bool CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED = |
428 | | false; |
429 | | #endif |
430 | | if (CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED) |
431 | | { |
432 | | CPLError(CE_Failure, CPLE_AppDefined, |
433 | | "Use of 'header_file' key-value pair in " |
434 | | "/vsicurl? is disabled in this build"); |
435 | | } |
436 | | else |
437 | | { |
438 | | bool bSetValue = false; |
439 | | const char *pszAllowHeaderFileKVP = CPLGetConfigOption( |
440 | | "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED", nullptr); |
441 | | if (!pszAllowHeaderFileKVP || |
442 | | pszAllowHeaderFileKVP[0] == 0 || |
443 | | EQUAL(pszAllowHeaderFileKVP, "ONLY_IN_TEMP")) |
444 | | { |
445 | | if (STARTS_WITH(pszValue, "/vsimem/")) |
446 | | { |
447 | | bSetValue = !CPLHasUnbalancedPathTraversal( |
448 | | pszValue + strlen("/vsimem/")); |
449 | | } |
450 | | else if (STARTS_WITH(pszValue, "/tmp/")) |
451 | | { |
452 | | bSetValue = !CPLHasUnbalancedPathTraversal( |
453 | | pszValue + strlen("/tmp/")); |
454 | | } |
455 | | else |
456 | | { |
457 | | for (const char *pszEnvVar : {"TEMP", "TMP"}) |
458 | | { |
459 | | if (const char *pszTemp = |
460 | | CPLGetConfigOption(pszEnvVar, |
461 | | nullptr)) |
462 | | { |
463 | | std::string osTemp = pszTemp; |
464 | | if (!osTemp.empty() && |
465 | | (osTemp.back() == '/' || |
466 | | osTemp.back() == '\\')) |
467 | | osTemp.pop_back(); |
468 | | if (!osTemp.empty() && |
469 | | cpl::starts_with( |
470 | | std::string_view(pszValue), |
471 | | osTemp) && |
472 | | (pszValue[osTemp.size()] == '/' || |
473 | | pszValue[osTemp.size()] == '\\')) |
474 | | { |
475 | | bSetValue = |
476 | | !CPLHasUnbalancedPathTraversal( |
477 | | pszValue + osTemp.size()); |
478 | | break; |
479 | | } |
480 | | } |
481 | | } |
482 | | } |
483 | | if (!bSetValue) |
484 | | { |
485 | | CPLError(CE_Failure, CPLE_AppDefined, |
486 | | "Use of 'header_file=%s' " |
487 | | "key-value pair in /vsicurl? is " |
488 | | "disabled because it refers to a " |
489 | | "file stored in a non-temporary " |
490 | | "location. You may set the " |
491 | | "CPL_VSIL_CURL_HEADER_FILE_KVP_" |
492 | | "ENABLED configuration option to " |
493 | | "YES to remove that restriction.", |
494 | | pszValue); |
495 | | } |
496 | | } |
497 | | else if (CPLTestBool(pszAllowHeaderFileKVP)) |
498 | | { |
499 | | bSetValue = true; |
500 | | } |
501 | | else |
502 | | { |
503 | | CPLError(CE_Failure, CPLE_AppDefined, |
504 | | "Use of 'header_file' key-value pair in " |
505 | | "/vsicurl? is disabled by the " |
506 | | "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED " |
507 | | "configuration option"); |
508 | | } |
509 | | |
510 | | if (bSetValue && paosHTTPOptions) |
511 | | { |
512 | | paosHTTPOptions->SetNameValue(pszKey, pszValue); |
513 | | } |
514 | | } |
515 | | } |
516 | | else if (EQUAL(pszKey, "useragent") || |
517 | | EQUAL(pszKey, "referer") || EQUAL(pszKey, "cookie") || |
518 | | EQUAL(pszKey, "unsafessl") || |
519 | | #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION |
520 | | EQUAL(pszKey, "timeout") || |
521 | | EQUAL(pszKey, "connecttimeout") || |
522 | | #endif |
523 | | EQUAL(pszKey, "low_speed_time") || |
524 | | EQUAL(pszKey, "low_speed_limit") || |
525 | | EQUAL(pszKey, "proxy") || EQUAL(pszKey, "proxyauth") || |
526 | | EQUAL(pszKey, "proxyuserpwd")) |
527 | | { |
528 | | // Above names are the ones supported by |
529 | | // CPLHTTPSetOptions() |
530 | | if (paosHTTPOptions) |
531 | | { |
532 | | paosHTTPOptions->SetNameValue(pszKey, pszValue); |
533 | | } |
534 | | } |
535 | | else if (EQUAL(pszKey, "url")) |
536 | | { |
537 | | osURL = pszValue; |
538 | | } |
539 | | else if (EQUAL(pszKey, "pc_url_signing")) |
540 | | { |
541 | | if (pbPlanetaryComputerURLSigning) |
542 | | *pbPlanetaryComputerURLSigning = CPLTestBool(pszValue); |
543 | | } |
544 | | else if (EQUAL(pszKey, "pc_collection")) |
545 | | { |
546 | | if (ppszPlanetaryComputerCollection) |
547 | | { |
548 | | CPLFree(*ppszPlanetaryComputerCollection); |
549 | | *ppszPlanetaryComputerCollection = CPLStrdup(pszValue); |
550 | | } |
551 | | } |
552 | | else if (STARTS_WITH(pszKey, "header.")) |
553 | | { |
554 | | osHeaders += (pszKey + strlen("header.")); |
555 | | osHeaders += ':'; |
556 | | osHeaders += pszValue; |
557 | | osHeaders += "\r\n"; |
558 | | } |
559 | | else |
560 | | { |
561 | | CPLError(CE_Warning, CPLE_NotSupported, |
562 | | "Unsupported option: %s", pszKey); |
563 | | } |
564 | | } |
565 | | CPLFree(pszKey); |
566 | | } |
567 | | |
568 | | if (paosHTTPOptions && !osHeaders.empty()) |
569 | | paosHTTPOptions->SetNameValue("HEADERS", osHeaders.c_str()); |
570 | | |
571 | | CSLDestroy(papszTokens); |
572 | | if (osURL.empty()) |
573 | | { |
574 | | CPLError(CE_Failure, CPLE_IllegalArg, "Missing url parameter"); |
575 | | return pszFilename; |
576 | | } |
577 | | |
578 | | return osURL; |
579 | | } |
580 | | |
581 | | return pszFilename; |
582 | | } |
583 | | |
584 | | namespace cpl |
585 | | { |
586 | | |
587 | | /************************************************************************/ |
588 | | /* VSICurlHandle() */ |
589 | | /************************************************************************/ |
590 | | |
591 | | VSICurlHandle::VSICurlHandle(VSICurlFilesystemHandlerBase *poFSIn, |
592 | | const char *pszFilename, const char *pszURLIn) |
593 | | : poFS(poFSIn), m_osFilename(pszFilename), |
594 | | m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv(pszFilename)), |
595 | | m_oRetryParameters(m_aosHTTPOptions), |
596 | | m_bUseHead( |
597 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_USE_HEAD", "YES"))) |
598 | | { |
599 | | if (pszURLIn) |
600 | | { |
601 | | m_pszURL = CPLStrdup(pszURLIn); |
602 | | } |
603 | | else |
604 | | { |
605 | | char *pszPCCollection = nullptr; |
606 | | m_pszURL = |
607 | | CPLStrdup(VSICurlGetURLFromFilename( |
608 | | pszFilename, &m_oRetryParameters, &m_bUseHead, |
609 | | &m_bUseRedirectURLIfNoQueryStringParams, nullptr, |
610 | | nullptr, &m_aosHTTPOptions, |
611 | | &m_bPlanetaryComputerURLSigning, &pszPCCollection) |
612 | | .c_str()); |
613 | | if (pszPCCollection) |
614 | | m_osPlanetaryComputerCollection = pszPCCollection; |
615 | | CPLFree(pszPCCollection); |
616 | | } |
617 | | |
618 | | m_bCached = poFSIn->AllowCachedDataFor(pszFilename); |
619 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
620 | | } |
621 | | |
622 | | /************************************************************************/ |
623 | | /* ~VSICurlHandle() */ |
624 | | /************************************************************************/ |
625 | | |
626 | | VSICurlHandle::~VSICurlHandle() |
627 | | { |
628 | | if (m_oThreadAdviseRead.joinable()) |
629 | | { |
630 | | m_oThreadAdviseRead.join(); |
631 | | } |
632 | | if (m_hCurlMultiHandleForAdviseRead) |
633 | | { |
634 | | VSICURLMultiCleanup(m_hCurlMultiHandleForAdviseRead); |
635 | | } |
636 | | |
637 | | if (!m_bCached) |
638 | | { |
639 | | poFS->InvalidateCachedData(m_pszURL); |
640 | | poFS->InvalidateDirContent(CPLGetDirnameSafe(m_osFilename.c_str())); |
641 | | } |
642 | | CPLFree(m_pszURL); |
643 | | } |
644 | | |
645 | | /************************************************************************/ |
646 | | /* SetURL() */ |
647 | | /************************************************************************/ |
648 | | |
649 | | void VSICurlHandle::SetURL(const char *pszURLIn) |
650 | | { |
651 | | CPLFree(m_pszURL); |
652 | | m_pszURL = CPLStrdup(pszURLIn); |
653 | | } |
654 | | |
655 | | /************************************************************************/ |
656 | | /* InstallReadCbk() */ |
657 | | /************************************************************************/ |
658 | | |
659 | | int VSICurlHandle::InstallReadCbk(VSICurlReadCbkFunc pfnReadCbkIn, |
660 | | void *pfnUserDataIn, |
661 | | int bStopOnInterruptUntilUninstallIn) |
662 | | { |
663 | | if (pfnReadCbk != nullptr) |
664 | | return FALSE; |
665 | | |
666 | | pfnReadCbk = pfnReadCbkIn; |
667 | | pReadCbkUserData = pfnUserDataIn; |
668 | | bStopOnInterruptUntilUninstall = |
669 | | CPL_TO_BOOL(bStopOnInterruptUntilUninstallIn); |
670 | | bInterrupted = false; |
671 | | return TRUE; |
672 | | } |
673 | | |
674 | | /************************************************************************/ |
675 | | /* UninstallReadCbk() */ |
676 | | /************************************************************************/ |
677 | | |
678 | | int VSICurlHandle::UninstallReadCbk() |
679 | | { |
680 | | if (pfnReadCbk == nullptr) |
681 | | return FALSE; |
682 | | |
683 | | pfnReadCbk = nullptr; |
684 | | pReadCbkUserData = nullptr; |
685 | | bStopOnInterruptUntilUninstall = false; |
686 | | bInterrupted = false; |
687 | | return TRUE; |
688 | | } |
689 | | |
690 | | /************************************************************************/ |
691 | | /* Seek() */ |
692 | | /************************************************************************/ |
693 | | |
694 | | int VSICurlHandle::Seek(vsi_l_offset nOffset, int nWhence) |
695 | | { |
696 | | if (nWhence == SEEK_SET) |
697 | | { |
698 | | curOffset = nOffset; |
699 | | } |
700 | | else if (nWhence == SEEK_CUR) |
701 | | { |
702 | | curOffset = curOffset + nOffset; |
703 | | } |
704 | | else |
705 | | { |
706 | | curOffset = GetFileSize(false) + nOffset; |
707 | | } |
708 | | bEOF = false; |
709 | | return 0; |
710 | | } |
711 | | |
712 | | } // namespace cpl |
713 | | |
714 | | /************************************************************************/ |
715 | | /* VSICurlGetTimeStampFromRFC822DateTime() */ |
716 | | /************************************************************************/ |
717 | | |
718 | | static GIntBig VSICurlGetTimeStampFromRFC822DateTime(const char *pszDT) |
719 | | { |
720 | | // Sun, 03 Apr 2016 12:07:27 GMT |
721 | | if (strlen(pszDT) >= 5 && pszDT[3] == ',' && pszDT[4] == ' ') |
722 | | pszDT += 5; |
723 | | int nDay = 0; |
724 | | int nYear = 0; |
725 | | int nHour = 0; |
726 | | int nMinute = 0; |
727 | | int nSecond = 0; |
728 | | char szMonth[4] = {}; |
729 | | szMonth[3] = 0; |
730 | | if (sscanf(pszDT, "%02d %03s %04d %02d:%02d:%02d GMT", &nDay, szMonth, |
731 | | &nYear, &nHour, &nMinute, &nSecond) == 6) |
732 | | { |
733 | | static const char *const aszMonthStr[] = {"Jan", "Feb", "Mar", "Apr", |
734 | | "May", "Jun", "Jul", "Aug", |
735 | | "Sep", "Oct", "Nov", "Dec"}; |
736 | | |
737 | | int nMonthIdx0 = -1; |
738 | | for (int i = 0; i < 12; i++) |
739 | | { |
740 | | if (EQUAL(szMonth, aszMonthStr[i])) |
741 | | { |
742 | | nMonthIdx0 = i; |
743 | | break; |
744 | | } |
745 | | } |
746 | | if (nMonthIdx0 >= 0) |
747 | | { |
748 | | struct tm brokendowntime; |
749 | | brokendowntime.tm_year = nYear - 1900; |
750 | | brokendowntime.tm_mon = nMonthIdx0; |
751 | | brokendowntime.tm_mday = nDay; |
752 | | brokendowntime.tm_hour = nHour; |
753 | | brokendowntime.tm_min = nMinute; |
754 | | brokendowntime.tm_sec = nSecond; |
755 | | return CPLYMDHMSToUnixTime(&brokendowntime); |
756 | | } |
757 | | } |
758 | | return 0; |
759 | | } |
760 | | |
761 | | /************************************************************************/ |
762 | | /* VSICURLInitWriteFuncStruct() */ |
763 | | /************************************************************************/ |
764 | | |
765 | | void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp, |
766 | | VSICurlReadCbkFunc pfnReadCbk, |
767 | | void *pReadCbkUserData) |
768 | | { |
769 | | psStruct->pBuffer = nullptr; |
770 | | psStruct->nSize = 0; |
771 | | psStruct->bIsHTTP = false; |
772 | | psStruct->bMultiRange = false; |
773 | | psStruct->nStartOffset = 0; |
774 | | psStruct->nEndOffset = 0; |
775 | | psStruct->nHTTPCode = 0; |
776 | | psStruct->nFirstHTTPCode = 0; |
777 | | psStruct->nContentLength = 0; |
778 | | psStruct->bFoundContentRange = false; |
779 | | psStruct->bError = false; |
780 | | psStruct->bDetectRangeDownloadingError = true; |
781 | | psStruct->nTimestampDate = 0; |
782 | | |
783 | | psStruct->fp = fp; |
784 | | psStruct->pfnReadCbk = pfnReadCbk; |
785 | | psStruct->pReadCbkUserData = pReadCbkUserData; |
786 | | psStruct->bInterrupted = false; |
787 | | } |
788 | | |
789 | | /************************************************************************/ |
790 | | /* VSICurlHandleWriteFunc() */ |
791 | | /************************************************************************/ |
792 | | |
793 | | size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, |
794 | | void *req) |
795 | | { |
796 | | cpl::WriteFuncStruct *psStruct = static_cast<cpl::WriteFuncStruct *>(req); |
797 | | const size_t nSize = count * nmemb; |
798 | | |
799 | | if (psStruct->bInterrupted) |
800 | | { |
801 | | return 0; |
802 | | } |
803 | | |
804 | | char *pNewBuffer = static_cast<char *>( |
805 | | VSIRealloc(psStruct->pBuffer, psStruct->nSize + nSize + 1)); |
806 | | if (pNewBuffer) |
807 | | { |
808 | | psStruct->pBuffer = pNewBuffer; |
809 | | memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize); |
810 | | psStruct->pBuffer[psStruct->nSize + nSize] = '\0'; |
811 | | if (psStruct->bIsHTTP) |
812 | | { |
813 | | char *pszLine = psStruct->pBuffer + psStruct->nSize; |
814 | | if (STARTS_WITH_CI(pszLine, "HTTP/")) |
815 | | { |
816 | | char *pszSpace = strchr(pszLine, ' '); |
817 | | if (pszSpace) |
818 | | { |
819 | | const int nHTTPCode = atoi(pszSpace + 1); |
820 | | if (psStruct->nFirstHTTPCode == 0) |
821 | | psStruct->nFirstHTTPCode = nHTTPCode; |
822 | | psStruct->nHTTPCode = nHTTPCode; |
823 | | } |
824 | | } |
825 | | else if (STARTS_WITH_CI(pszLine, "Content-Length: ")) |
826 | | { |
827 | | psStruct->nContentLength = CPLScanUIntBig( |
828 | | pszLine + 16, static_cast<int>(strlen(pszLine + 16))); |
829 | | } |
830 | | else if (STARTS_WITH_CI(pszLine, "Content-Range: ")) |
831 | | { |
832 | | psStruct->bFoundContentRange = true; |
833 | | } |
834 | | else if (STARTS_WITH_CI(pszLine, "Date: ")) |
835 | | { |
836 | | CPLString osDate = pszLine + strlen("Date: "); |
837 | | size_t nSizeLine = osDate.size(); |
838 | | while (nSizeLine && (osDate[nSizeLine - 1] == '\r' || |
839 | | osDate[nSizeLine - 1] == '\n')) |
840 | | { |
841 | | osDate.resize(nSizeLine - 1); |
842 | | nSizeLine--; |
843 | | } |
844 | | osDate.Trim(); |
845 | | |
846 | | GIntBig nTimestampDate = |
847 | | VSICurlGetTimeStampFromRFC822DateTime(osDate.c_str()); |
848 | | #if DEBUG_VERBOSE |
849 | | CPLDebug("VSICURL", "Timestamp = " CPL_FRMT_GIB, |
850 | | nTimestampDate); |
851 | | #endif |
852 | | psStruct->nTimestampDate = nTimestampDate; |
853 | | } |
854 | | /*if( nSize > 2 && pszLine[nSize - 2] == '\r' && |
855 | | pszLine[nSize - 1] == '\n' ) |
856 | | { |
857 | | pszLine[nSize - 2] = 0; |
858 | | CPLDebug("VSICURL", "%s", pszLine); |
859 | | pszLine[nSize - 2] = '\r'; |
860 | | }*/ |
861 | | |
862 | | if (pszLine[0] == '\r' && pszLine[1] == '\n') |
863 | | { |
864 | | // Detect servers that don't support range downloading. |
865 | | if (psStruct->nHTTPCode == 200 && |
866 | | psStruct->bDetectRangeDownloadingError && |
867 | | !psStruct->bMultiRange && !psStruct->bFoundContentRange && |
868 | | (psStruct->nStartOffset != 0 || |
869 | | psStruct->nContentLength > |
870 | | 10 * (psStruct->nEndOffset - psStruct->nStartOffset + |
871 | | 1))) |
872 | | { |
873 | | CPLError(CE_Failure, CPLE_AppDefined, |
874 | | "Range downloading not supported by this " |
875 | | "server!"); |
876 | | psStruct->bError = true; |
877 | | return 0; |
878 | | } |
879 | | } |
880 | | } |
881 | | else |
882 | | { |
883 | | if (psStruct->pfnReadCbk) |
884 | | { |
885 | | if (!psStruct->pfnReadCbk(psStruct->fp, buffer, nSize, |
886 | | psStruct->pReadCbkUserData)) |
887 | | { |
888 | | psStruct->bInterrupted = true; |
889 | | return 0; |
890 | | } |
891 | | } |
892 | | } |
893 | | psStruct->nSize += nSize; |
894 | | return nmemb; |
895 | | } |
896 | | else |
897 | | { |
898 | | return 0; |
899 | | } |
900 | | } |
901 | | |
902 | | /************************************************************************/ |
903 | | /* VSICurlIsS3LikeSignedURL() */ |
904 | | /************************************************************************/ |
905 | | |
906 | | static bool VSICurlIsS3LikeSignedURL(const char *pszURL) |
907 | | { |
908 | | return ((strstr(pszURL, ".s3.amazonaws.com/") != nullptr || |
909 | | strstr(pszURL, ".s3.amazonaws.com:") != nullptr || |
910 | | strstr(pszURL, ".storage.googleapis.com/") != nullptr || |
911 | | strstr(pszURL, ".storage.googleapis.com:") != nullptr || |
912 | | strstr(pszURL, ".cloudfront.net/") != nullptr || |
913 | | strstr(pszURL, ".cloudfront.net:") != nullptr) && |
914 | | (strstr(pszURL, "&Signature=") != nullptr || |
915 | | strstr(pszURL, "?Signature=") != nullptr)) || |
916 | | strstr(pszURL, "&X-Amz-Signature=") != nullptr || |
917 | | strstr(pszURL, "?X-Amz-Signature=") != nullptr; |
918 | | } |
919 | | |
920 | | /************************************************************************/ |
921 | | /* VSICurlGetExpiresFromS3LikeSignedURL() */ |
922 | | /************************************************************************/ |
923 | | |
924 | | static GIntBig VSICurlGetExpiresFromS3LikeSignedURL(const char *pszURL) |
925 | | { |
926 | | const auto GetParamValue = [pszURL](const char *pszKey) -> const char * |
927 | | { |
928 | | for (const char *pszPrefix : {"&", "?"}) |
929 | | { |
930 | | std::string osNeedle(pszPrefix); |
931 | | osNeedle += pszKey; |
932 | | osNeedle += '='; |
933 | | const char *pszStr = strstr(pszURL, osNeedle.c_str()); |
934 | | if (pszStr) |
935 | | return pszStr + osNeedle.size(); |
936 | | } |
937 | | return nullptr; |
938 | | }; |
939 | | |
940 | | { |
941 | | // Expires= is a Unix timestamp |
942 | | const char *pszExpires = GetParamValue("Expires"); |
943 | | if (pszExpires != nullptr) |
944 | | return CPLAtoGIntBig(pszExpires); |
945 | | } |
946 | | |
947 | | // X-Amz-Expires= is a delay, to be combined with X-Amz-Date= |
948 | | const char *pszAmzExpires = GetParamValue("X-Amz-Expires"); |
949 | | if (pszAmzExpires == nullptr) |
950 | | return 0; |
951 | | const int nDelay = atoi(pszAmzExpires); |
952 | | |
953 | | const char *pszAmzDate = GetParamValue("X-Amz-Date"); |
954 | | if (pszAmzDate == nullptr) |
955 | | return 0; |
956 | | // pszAmzDate should be YYYYMMDDTHHMMSSZ |
957 | | if (strlen(pszAmzDate) < strlen("YYYYMMDDTHHMMSSZ")) |
958 | | return 0; |
959 | | if (pszAmzDate[strlen("YYYYMMDDTHHMMSSZ") - 1] != 'Z') |
960 | | return 0; |
961 | | struct tm brokendowntime; |
962 | | brokendowntime.tm_year = |
963 | | atoi(std::string(pszAmzDate).substr(0, 4).c_str()) - 1900; |
964 | | brokendowntime.tm_mon = |
965 | | atoi(std::string(pszAmzDate).substr(4, 2).c_str()) - 1; |
966 | | brokendowntime.tm_mday = atoi(std::string(pszAmzDate).substr(6, 2).c_str()); |
967 | | brokendowntime.tm_hour = atoi(std::string(pszAmzDate).substr(9, 2).c_str()); |
968 | | brokendowntime.tm_min = atoi(std::string(pszAmzDate).substr(11, 2).c_str()); |
969 | | brokendowntime.tm_sec = atoi(std::string(pszAmzDate).substr(13, 2).c_str()); |
970 | | return CPLYMDHMSToUnixTime(&brokendowntime) + nDelay; |
971 | | } |
972 | | |
973 | | /************************************************************************/ |
974 | | /* VSICURLMultiPerform() */ |
975 | | /************************************************************************/ |
976 | | |
977 | | void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle, |
978 | | std::atomic<bool> *pbInterrupt) |
979 | | { |
980 | | int repeats = 0; |
981 | | |
982 | | if (hEasyHandle) |
983 | | curl_multi_add_handle(hCurlMultiHandle, hEasyHandle); |
984 | | |
985 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
986 | | while (true) |
987 | | { |
988 | | int still_running; |
989 | | while (curl_multi_perform(hCurlMultiHandle, &still_running) == |
990 | | CURLM_CALL_MULTI_PERFORM) |
991 | | { |
992 | | // loop |
993 | | } |
994 | | if (!still_running) |
995 | | { |
996 | | break; |
997 | | } |
998 | | |
999 | | #ifdef undef |
1000 | | CURLMsg *msg; |
1001 | | do |
1002 | | { |
1003 | | int msgq = 0; |
1004 | | msg = curl_multi_info_read(hCurlMultiHandle, &msgq); |
1005 | | if (msg && (msg->msg == CURLMSG_DONE)) |
1006 | | { |
1007 | | CURL *e = msg->easy_handle; |
1008 | | } |
1009 | | } while (msg); |
1010 | | #endif |
1011 | | |
1012 | | CPLMultiPerformWait(hCurlMultiHandle, repeats); |
1013 | | |
1014 | | if (pbInterrupt && *pbInterrupt) |
1015 | | break; |
1016 | | } |
1017 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
1018 | | |
1019 | | if (hEasyHandle) |
1020 | | curl_multi_remove_handle(hCurlMultiHandle, hEasyHandle); |
1021 | | } |
1022 | | |
1023 | | /************************************************************************/ |
1024 | | /* VSICurlDummyWriteFunc() */ |
1025 | | /************************************************************************/ |
1026 | | |
1027 | | static size_t VSICurlDummyWriteFunc(void *, size_t, size_t, void *) |
1028 | | { |
1029 | | return 0; |
1030 | | } |
1031 | | |
1032 | | /************************************************************************/ |
1033 | | /* VSICURLResetHeaderAndWriterFunctions() */ |
1034 | | /************************************************************************/ |
1035 | | |
1036 | | void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle) |
1037 | | { |
1038 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1039 | | VSICurlDummyWriteFunc); |
1040 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1041 | | VSICurlDummyWriteFunc); |
1042 | | } |
1043 | | |
1044 | | /************************************************************************/ |
1045 | | /* Iso8601ToUnixTime() */ |
1046 | | /************************************************************************/ |
1047 | | |
1048 | | static bool Iso8601ToUnixTime(const char *pszDT, GIntBig *pnUnixTime) |
1049 | | { |
1050 | | int nYear; |
1051 | | int nMonth; |
1052 | | int nDay; |
1053 | | int nHour; |
1054 | | int nMinute; |
1055 | | int nSecond; |
1056 | | if (sscanf(pszDT, "%04d-%02d-%02dT%02d:%02d:%02d", &nYear, &nMonth, &nDay, |
1057 | | &nHour, &nMinute, &nSecond) == 6) |
1058 | | { |
1059 | | struct tm brokendowntime; |
1060 | | brokendowntime.tm_year = nYear - 1900; |
1061 | | brokendowntime.tm_mon = nMonth - 1; |
1062 | | brokendowntime.tm_mday = nDay; |
1063 | | brokendowntime.tm_hour = nHour; |
1064 | | brokendowntime.tm_min = nMinute; |
1065 | | brokendowntime.tm_sec = nSecond; |
1066 | | *pnUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
1067 | | return true; |
1068 | | } |
1069 | | return false; |
1070 | | } |
1071 | | |
1072 | | namespace cpl |
1073 | | { |
1074 | | |
1075 | | /************************************************************************/ |
1076 | | /* ManagePlanetaryComputerSigning() */ |
1077 | | /************************************************************************/ |
1078 | | |
1079 | | void VSICurlHandle::ManagePlanetaryComputerSigning() const |
1080 | | { |
1081 | | // Take global lock |
1082 | | static std::mutex goMutex; |
1083 | | std::lock_guard<std::mutex> oLock(goMutex); |
1084 | | |
1085 | | struct PCSigningInfo |
1086 | | { |
1087 | | std::string osQueryString{}; |
1088 | | GIntBig nExpireTimestamp = 0; |
1089 | | }; |
1090 | | |
1091 | | PCSigningInfo sSigningInfo; |
1092 | | constexpr int knExpirationDelayMargin = 60; |
1093 | | |
1094 | | if (!m_osPlanetaryComputerCollection.empty()) |
1095 | | { |
1096 | | // key is the name of a collection |
1097 | | static lru11::Cache<std::string, PCSigningInfo> goCacheCollection{1024}; |
1098 | | |
1099 | | if (goCacheCollection.tryGet(m_osPlanetaryComputerCollection, |
1100 | | sSigningInfo) && |
1101 | | time(nullptr) + knExpirationDelayMargin <= |
1102 | | sSigningInfo.nExpireTimestamp) |
1103 | | { |
1104 | | m_osQueryString = sSigningInfo.osQueryString; |
1105 | | } |
1106 | | else |
1107 | | { |
1108 | | const auto psResult = |
1109 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
1110 | | "VSICURL_PC_SAS_TOKEN_URL", |
1111 | | "https://planetarycomputer.microsoft.com/api/" |
1112 | | "sas/v1/token/")) + |
1113 | | m_osPlanetaryComputerCollection) |
1114 | | .c_str(), |
1115 | | nullptr); |
1116 | | if (psResult) |
1117 | | { |
1118 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1119 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1120 | | const char *pszToken = aosKeyVals.FetchNameValue("token"); |
1121 | | if (pszToken) |
1122 | | { |
1123 | | m_osQueryString = '?'; |
1124 | | m_osQueryString += pszToken; |
1125 | | |
1126 | | sSigningInfo.osQueryString = m_osQueryString; |
1127 | | sSigningInfo.nExpireTimestamp = 0; |
1128 | | const char *pszExpiry = |
1129 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1130 | | if (pszExpiry) |
1131 | | { |
1132 | | Iso8601ToUnixTime(pszExpiry, |
1133 | | &sSigningInfo.nExpireTimestamp); |
1134 | | } |
1135 | | goCacheCollection.insert(m_osPlanetaryComputerCollection, |
1136 | | sSigningInfo); |
1137 | | |
1138 | | CPLDebug("VSICURL", "Got token from Planetary Computer: %s", |
1139 | | m_osQueryString.c_str()); |
1140 | | } |
1141 | | CPLHTTPDestroyResult(psResult); |
1142 | | } |
1143 | | } |
1144 | | } |
1145 | | else |
1146 | | { |
1147 | | // key is a URL |
1148 | | static lru11::Cache<std::string, PCSigningInfo> goCacheURL{1024}; |
1149 | | |
1150 | | if (goCacheURL.tryGet(m_pszURL, sSigningInfo) && |
1151 | | time(nullptr) + knExpirationDelayMargin <= |
1152 | | sSigningInfo.nExpireTimestamp) |
1153 | | { |
1154 | | m_osQueryString = sSigningInfo.osQueryString; |
1155 | | } |
1156 | | else |
1157 | | { |
1158 | | const auto psResult = |
1159 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
1160 | | "VSICURL_PC_SAS_SIGN_HREF_URL", |
1161 | | "https://planetarycomputer.microsoft.com/api/" |
1162 | | "sas/v1/sign?href=")) + |
1163 | | m_pszURL) |
1164 | | .c_str(), |
1165 | | nullptr); |
1166 | | if (psResult) |
1167 | | { |
1168 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1169 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1170 | | const char *pszHref = aosKeyVals.FetchNameValue("href"); |
1171 | | if (pszHref && STARTS_WITH(pszHref, m_pszURL)) |
1172 | | { |
1173 | | m_osQueryString = pszHref + strlen(m_pszURL); |
1174 | | |
1175 | | sSigningInfo.osQueryString = m_osQueryString; |
1176 | | sSigningInfo.nExpireTimestamp = 0; |
1177 | | const char *pszExpiry = |
1178 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1179 | | if (pszExpiry) |
1180 | | { |
1181 | | Iso8601ToUnixTime(pszExpiry, |
1182 | | &sSigningInfo.nExpireTimestamp); |
1183 | | } |
1184 | | goCacheURL.insert(m_pszURL, sSigningInfo); |
1185 | | |
1186 | | CPLDebug("VSICURL", |
1187 | | "Got signature from Planetary Computer: %s", |
1188 | | m_osQueryString.c_str()); |
1189 | | } |
1190 | | CPLHTTPDestroyResult(psResult); |
1191 | | } |
1192 | | } |
1193 | | } |
1194 | | } |
1195 | | |
1196 | | /************************************************************************/ |
1197 | | /* UpdateQueryString() */ |
1198 | | /************************************************************************/ |
1199 | | |
1200 | | void VSICurlHandle::UpdateQueryString() const |
1201 | | { |
1202 | | if (m_bPlanetaryComputerURLSigning) |
1203 | | { |
1204 | | ManagePlanetaryComputerSigning(); |
1205 | | } |
1206 | | else |
1207 | | { |
1208 | | const char *pszQueryString = VSIGetPathSpecificOption( |
1209 | | m_osFilename.c_str(), "VSICURL_QUERY_STRING", nullptr); |
1210 | | if (pszQueryString) |
1211 | | { |
1212 | | if (m_osFilename.back() == '?') |
1213 | | { |
1214 | | if (pszQueryString[0] == '?') |
1215 | | m_osQueryString = pszQueryString + 1; |
1216 | | else |
1217 | | m_osQueryString = pszQueryString; |
1218 | | } |
1219 | | else |
1220 | | { |
1221 | | if (pszQueryString[0] == '?') |
1222 | | m_osQueryString = pszQueryString; |
1223 | | else |
1224 | | { |
1225 | | m_osQueryString = "?"; |
1226 | | m_osQueryString.append(pszQueryString); |
1227 | | } |
1228 | | } |
1229 | | } |
1230 | | } |
1231 | | } |
1232 | | |
1233 | | /************************************************************************/ |
1234 | | /* GetFileSizeOrHeaders() */ |
1235 | | /************************************************************************/ |
1236 | | |
1237 | | vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, |
1238 | | bool bGetHeaders) |
1239 | | { |
1240 | | if (oFileProp.bHasComputedFileSize && !bGetHeaders) |
1241 | | return oFileProp.fileSize; |
1242 | | |
1243 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
1244 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
1245 | | NetworkStatisticsAction oContextAction("GetFileSize"); |
1246 | | |
1247 | | oFileProp.bHasComputedFileSize = true; |
1248 | | |
1249 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
1250 | | |
1251 | | UpdateQueryString(); |
1252 | | |
1253 | | std::string osURL(m_pszURL + m_osQueryString); |
1254 | | int nTryCount = 0; |
1255 | | bool bRetryWithGet = false; |
1256 | | bool bRetryWithLimitedRangeGet = false; |
1257 | | bool bS3LikeRedirect = false; |
1258 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
1259 | | |
1260 | | retry: |
1261 | | ++nTryCount; |
1262 | | CURL *hCurlHandle = curl_easy_init(); |
1263 | | |
1264 | | struct curl_slist *headers = nullptr; |
1265 | | if (bS3LikeRedirect) |
1266 | | { |
1267 | | // Do not propagate authentication sent to the original URL to a S3-like |
1268 | | // redirect. |
1269 | | CPLStringList aosHTTPOptions{}; |
1270 | | for (const auto &pszOption : m_aosHTTPOptions) |
1271 | | { |
1272 | | if (STARTS_WITH_CI(pszOption, "HTTPAUTH") || |
1273 | | STARTS_WITH_CI(pszOption, "HTTP_BEARER")) |
1274 | | continue; |
1275 | | aosHTTPOptions.AddString(pszOption); |
1276 | | } |
1277 | | headers = VSICurlSetOptions(hCurlHandle, osURL.c_str(), |
1278 | | aosHTTPOptions.List()); |
1279 | | } |
1280 | | else |
1281 | | { |
1282 | | headers = VSICurlSetOptions(hCurlHandle, osURL.c_str(), |
1283 | | m_aosHTTPOptions.List()); |
1284 | | } |
1285 | | |
1286 | | WriteFuncStruct sWriteFuncHeaderData; |
1287 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
1288 | | nullptr); |
1289 | | sWriteFuncHeaderData.bDetectRangeDownloadingError = false; |
1290 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(osURL.c_str(), "http"); |
1291 | | |
1292 | | WriteFuncStruct sWriteFuncData; |
1293 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
1294 | | |
1295 | | std::string osVerb; |
1296 | | std::string osRange; // leave in this scope ! |
1297 | | int nRoundedBufSize = 0; |
1298 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
1299 | | bool bHasUsedLimitedRangeGet = false; |
1300 | | if (bRetryWithLimitedRangeGet || UseLimitRangeGetInsteadOfHead()) |
1301 | | { |
1302 | | bHasUsedLimitedRangeGet = true; |
1303 | | osVerb = "GET"; |
1304 | | const int nBufSize = std::clamp( |
1305 | | atoi(CPLGetConfigOption("GDAL_INGESTED_BYTES_AT_OPEN", "1024")), |
1306 | | 1024, 10 * 1024 * 1024); |
1307 | | nRoundedBufSize = cpl::div_round_up(nBufSize, knDOWNLOAD_CHUNK_SIZE) * |
1308 | | knDOWNLOAD_CHUNK_SIZE; |
1309 | | |
1310 | | // so it gets included in Azure signature |
1311 | | osRange = CPLSPrintf("Range: bytes=0-%d", nRoundedBufSize - 1); |
1312 | | headers = curl_slist_append(headers, osRange.c_str()); |
1313 | | } |
1314 | | // HACK for mbtiles driver: http://a.tiles.mapbox.com/v3/ doesn't accept |
1315 | | // HEAD, as it is a redirect to AWS S3 signed URL, but those are only valid |
1316 | | // for a given type of HTTP request, and thus GET. This is valid for any |
1317 | | // signed URL for AWS S3. |
1318 | | else if (bRetryWithGet || |
1319 | | strstr(osURL.c_str(), ".tiles.mapbox.com/") != nullptr || |
1320 | | VSICurlIsS3LikeSignedURL(osURL.c_str()) || !m_bUseHead) |
1321 | | { |
1322 | | sWriteFuncData.bInterrupted = true; |
1323 | | osVerb = "GET"; |
1324 | | } |
1325 | | else |
1326 | | { |
1327 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1); |
1328 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 0); |
1329 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 1); |
1330 | | osVerb = "HEAD"; |
1331 | | } |
1332 | | |
1333 | | bRetryWithLimitedRangeGet = false; |
1334 | | |
1335 | | if (!AllowAutomaticRedirection()) |
1336 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
1337 | | |
1338 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
1339 | | &sWriteFuncHeaderData); |
1340 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1341 | | VSICurlHandleWriteFunc); |
1342 | | |
1343 | | // Bug with older curl versions (<=7.16.4) and FTP. |
1344 | | // See http://curl.haxx.se/mail/lib-2007-08/0312.html |
1345 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
1346 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1347 | | VSICurlHandleWriteFunc); |
1348 | | |
1349 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
1350 | | szCurlErrBuf[0] = '\0'; |
1351 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
1352 | | |
1353 | | headers = GetCurlHeaders(osVerb, headers); |
1354 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
1355 | | |
1356 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
1357 | | |
1358 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); |
1359 | | |
1360 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
1361 | | |
1362 | | curl_slist_free_all(headers); |
1363 | | |
1364 | | oFileProp.eExists = EXIST_UNKNOWN; |
1365 | | |
1366 | | curl_off_t filetime = -1; |
1367 | | GIntBig mtime = 0; |
1368 | | if (curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME_T, &filetime) == |
1369 | | CURLE_OK && |
1370 | | filetime != -1) |
1371 | | { |
1372 | | mtime = static_cast<GIntBig>(filetime); |
1373 | | } |
1374 | | |
1375 | | if (osVerb == "GET") |
1376 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
1377 | | else |
1378 | | NetworkStatisticsLogger::LogHEAD(); |
1379 | | |
1380 | | if (STARTS_WITH(osURL.c_str(), "ftp")) |
1381 | | { |
1382 | | if (sWriteFuncData.pBuffer != nullptr) |
1383 | | { |
1384 | | const char *pszContentLength = |
1385 | | strstr(const_cast<const char *>(sWriteFuncData.pBuffer), |
1386 | | "Content-Length: "); |
1387 | | if (pszContentLength) |
1388 | | { |
1389 | | pszContentLength += strlen("Content-Length: "); |
1390 | | oFileProp.eExists = EXIST_YES; |
1391 | | oFileProp.fileSize = |
1392 | | CPLScanUIntBig(pszContentLength, |
1393 | | static_cast<int>(strlen(pszContentLength))); |
1394 | | if constexpr (ENABLE_DEBUG) |
1395 | | { |
1396 | | CPLDebug(poFS->GetDebugKey(), |
1397 | | "GetFileSize(%s)=" CPL_FRMT_GUIB, osURL.c_str(), |
1398 | | oFileProp.fileSize); |
1399 | | } |
1400 | | } |
1401 | | } |
1402 | | } |
1403 | | |
1404 | | double dfSize = 0; |
1405 | | long response_code = -1; |
1406 | | if (oFileProp.eExists != EXIST_YES) |
1407 | | { |
1408 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
1409 | | |
1410 | | bool bAlreadyLogged = false; |
1411 | | if (response_code >= 400 && szCurlErrBuf[0] == '\0') |
1412 | | { |
1413 | | const bool bLogResponse = |
1414 | | CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")); |
1415 | | if (bLogResponse && sWriteFuncData.pBuffer) |
1416 | | { |
1417 | | const char *pszErrorMsg = |
1418 | | static_cast<const char *>(sWriteFuncData.pBuffer); |
1419 | | bAlreadyLogged = true; |
1420 | | CPLDebug( |
1421 | | poFS->GetDebugKey(), |
1422 | | "GetFileSize(%s): response_code=%d, server error msg=%s", |
1423 | | osURL.c_str(), static_cast<int>(response_code), |
1424 | | pszErrorMsg[0] ? pszErrorMsg : "(no message provided)"); |
1425 | | } |
1426 | | } |
1427 | | else if (szCurlErrBuf[0] != '\0') |
1428 | | { |
1429 | | bAlreadyLogged = true; |
1430 | | CPLDebug(poFS->GetDebugKey(), |
1431 | | "GetFileSize(%s): response_code=%d, curl error msg=%s", |
1432 | | osURL.c_str(), static_cast<int>(response_code), |
1433 | | szCurlErrBuf); |
1434 | | } |
1435 | | |
1436 | | std::string osEffectiveURL; |
1437 | | { |
1438 | | char *pszEffectiveURL = nullptr; |
1439 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
1440 | | &pszEffectiveURL); |
1441 | | if (pszEffectiveURL) |
1442 | | osEffectiveURL = pszEffectiveURL; |
1443 | | } |
1444 | | |
1445 | | if (!osEffectiveURL.empty() && |
1446 | | strstr(osEffectiveURL.c_str(), osURL.c_str()) == nullptr) |
1447 | | { |
1448 | | // Moved permanently ? |
1449 | | if (sWriteFuncHeaderData.nFirstHTTPCode == 301 || |
1450 | | (m_bUseRedirectURLIfNoQueryStringParams && |
1451 | | osEffectiveURL.find('?') == std::string::npos)) |
1452 | | { |
1453 | | CPLDebug(poFS->GetDebugKey(), |
1454 | | "Using effective URL %s permanently", |
1455 | | osEffectiveURL.c_str()); |
1456 | | oFileProp.osRedirectURL = osEffectiveURL; |
1457 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1458 | | } |
1459 | | else |
1460 | | { |
1461 | | CPLDebug(poFS->GetDebugKey(), |
1462 | | "Using effective URL %s temporarily", |
1463 | | osEffectiveURL.c_str()); |
1464 | | } |
1465 | | |
1466 | | // Is this is a redirect to a S3 URL? |
1467 | | if (VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
1468 | | !VSICurlIsS3LikeSignedURL(osURL.c_str())) |
1469 | | { |
1470 | | // Note that this is a redirect as we won't notice after the |
1471 | | // retry. |
1472 | | bS3LikeRedirect = true; |
1473 | | |
1474 | | if (!bRetryWithGet && osVerb == "HEAD" && response_code == 403) |
1475 | | { |
1476 | | CPLDebug(poFS->GetDebugKey(), |
1477 | | "Redirected to a AWS S3 signed URL. Retrying " |
1478 | | "with GET request instead of HEAD since the URL " |
1479 | | "might be valid only for GET"); |
1480 | | bRetryWithGet = true; |
1481 | | osURL = std::move(osEffectiveURL); |
1482 | | CPLFree(sWriteFuncData.pBuffer); |
1483 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1484 | | curl_easy_cleanup(hCurlHandle); |
1485 | | goto retry; |
1486 | | } |
1487 | | } |
1488 | | else if (oFileProp.osRedirectURL.empty() && nTryCount == 1 && |
1489 | | ((response_code >= 300 && response_code < 400) || |
1490 | | (osVerb == "HEAD" && response_code == 403))) |
1491 | | { |
1492 | | if (response_code == 403) |
1493 | | { |
1494 | | CPLDebug( |
1495 | | poFS->GetDebugKey(), |
1496 | | "Retrying redirected URL with GET instead of HEAD"); |
1497 | | bRetryWithGet = true; |
1498 | | } |
1499 | | osURL = std::move(osEffectiveURL); |
1500 | | CPLFree(sWriteFuncData.pBuffer); |
1501 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1502 | | curl_easy_cleanup(hCurlHandle); |
1503 | | goto retry; |
1504 | | } |
1505 | | } |
1506 | | |
1507 | | if (bS3LikeRedirect && response_code >= 200 && response_code < 300 && |
1508 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
1509 | | !osEffectiveURL.empty() && |
1510 | | CPLTestBool( |
1511 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
1512 | | { |
1513 | | const GIntBig nExpireTimestamp = |
1514 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
1515 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
1516 | | { |
1517 | | const int nValidity = static_cast<int>( |
1518 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
1519 | | CPLDebug(poFS->GetDebugKey(), |
1520 | | "Will use redirect URL for the next %d seconds", |
1521 | | nValidity); |
1522 | | // As our local clock might not be in sync with server clock, |
1523 | | // figure out the expiration timestamp in local time |
1524 | | oFileProp.bS3LikeRedirect = true; |
1525 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
1526 | | oFileProp.osRedirectURL = osEffectiveURL; |
1527 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1528 | | } |
1529 | | } |
1530 | | |
1531 | | // Split a string with the raw HTTP response headers as a key/value |
1532 | | // CPLStringList |
1533 | | const auto TokenizeHeaders = [](const char *pszHeaders) -> CPLStringList |
1534 | | { |
1535 | | CPLStringList aosHeaders; |
1536 | | while (pszHeaders) |
1537 | | { |
1538 | | const char *pszDelim = strchr(pszHeaders, ':'); |
1539 | | if (!pszDelim) |
1540 | | break; |
1541 | | const char *pszValue = pszDelim + 1; |
1542 | | |
1543 | | // Skip whitespace after colon |
1544 | | while (*pszValue == ' ' || *pszValue == '\t') |
1545 | | ++pszValue; |
1546 | | |
1547 | | // Find end of value |
1548 | | const char *pszEndOfValue = pszValue; |
1549 | | while (*pszEndOfValue && |
1550 | | !(*pszEndOfValue == '\r' && pszEndOfValue[1] == '\n')) |
1551 | | ++pszEndOfValue; |
1552 | | |
1553 | | aosHeaders.SetNameValue( |
1554 | | std::string(pszHeaders, pszDelim - pszHeaders).c_str(), |
1555 | | std::string(pszValue, pszEndOfValue - pszValue).c_str()); |
1556 | | |
1557 | | if (*pszEndOfValue == '\r' && pszEndOfValue[1] == '\n') |
1558 | | pszHeaders = pszEndOfValue + 2; |
1559 | | else |
1560 | | break; |
1561 | | } |
1562 | | return aosHeaders; |
1563 | | }; |
1564 | | |
1565 | | if (response_code < 300) |
1566 | | { |
1567 | | curl_off_t nSizeTmp = 0; |
1568 | | const CURLcode code = curl_easy_getinfo( |
1569 | | hCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &nSizeTmp); |
1570 | | CPL_IGNORE_RET_VAL(dfSize); |
1571 | | dfSize = static_cast<double>(nSizeTmp); |
1572 | | if (code == 0) |
1573 | | { |
1574 | | if (dfSize < 0) |
1575 | | { |
1576 | | if (osVerb == "HEAD" && !bRetryWithGet && |
1577 | | response_code == 200) |
1578 | | { |
1579 | | if (sWriteFuncHeaderData.pBuffer) |
1580 | | { |
1581 | | const CPLStringList aosHeaders( |
1582 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1583 | | if (strcmp(aosHeaders.FetchNameValueDef( |
1584 | | "accept-ranges", ""), |
1585 | | "bytes") == 0) |
1586 | | { |
1587 | | CPLDebug( |
1588 | | poFS->GetDebugKey(), |
1589 | | "HEAD did not provide file size. Retrying " |
1590 | | "with limited range GET"); |
1591 | | bRetryWithLimitedRangeGet = true; |
1592 | | CPLFree(sWriteFuncData.pBuffer); |
1593 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1594 | | curl_easy_cleanup(hCurlHandle); |
1595 | | goto retry; |
1596 | | } |
1597 | | } |
1598 | | |
1599 | | CPLDebug(poFS->GetDebugKey(), |
1600 | | "HEAD did not provide file size. Retrying " |
1601 | | "with GET"); |
1602 | | bRetryWithGet = true; |
1603 | | CPLFree(sWriteFuncData.pBuffer); |
1604 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1605 | | curl_easy_cleanup(hCurlHandle); |
1606 | | goto retry; |
1607 | | } |
1608 | | |
1609 | | if (poFS->GetFSPrefix() == "/vsicurl/" || |
1610 | | poFS->GetFSPrefix() == "/vsicurl?") |
1611 | | { |
1612 | | const CPLStringList aosHeaders( |
1613 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1614 | | if (strcmp(aosHeaders.FetchNameValueDef( |
1615 | | "transfer-encoding", ""), |
1616 | | "chunked") == 0) |
1617 | | { |
1618 | | CPLError( |
1619 | | CE_Failure, CPLE_AppDefined, |
1620 | | "Server does not seem to support range " |
1621 | | "requests. " |
1622 | | "Maybe retry with /vsicurl_streaming/ if the " |
1623 | | "read " |
1624 | | "access pattern is compatible with sequential " |
1625 | | "reading, or download the file entirely"); |
1626 | | } |
1627 | | } |
1628 | | } |
1629 | | else |
1630 | | { |
1631 | | oFileProp.eExists = EXIST_YES; |
1632 | | oFileProp.fileSize = static_cast<GUIntBig>(dfSize); |
1633 | | } |
1634 | | } |
1635 | | } |
1636 | | |
1637 | | if (sWriteFuncHeaderData.pBuffer != nullptr && |
1638 | | (response_code == 200 || response_code == 206)) |
1639 | | { |
1640 | | { |
1641 | | const CPLStringList aosHeaders( |
1642 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1643 | | for (const auto &[pszKey, pszValue] : |
1644 | | cpl::IterateNameValue(aosHeaders)) |
1645 | | { |
1646 | | if (bGetHeaders) |
1647 | | { |
1648 | | m_aosHeaders.SetNameValue(pszKey, pszValue); |
1649 | | } |
1650 | | if (EQUAL(pszKey, "Cache-Control") && |
1651 | | EQUAL(pszValue, "no-cache") && |
1652 | | CPLTestBool(CPLGetConfigOption( |
1653 | | "CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES"))) |
1654 | | { |
1655 | | m_bCached = false; |
1656 | | } |
1657 | | |
1658 | | else if (EQUAL(pszKey, "ETag")) |
1659 | | { |
1660 | | std::string osValue(pszValue); |
1661 | | if (osValue.size() >= 2 && osValue.front() == '"' && |
1662 | | osValue.back() == '"') |
1663 | | osValue = osValue.substr(1, osValue.size() - 2); |
1664 | | oFileProp.ETag = std::move(osValue); |
1665 | | } |
1666 | | |
1667 | | // Azure Data Lake Storage |
1668 | | else if (EQUAL(pszKey, "x-ms-resource-type")) |
1669 | | { |
1670 | | if (EQUAL(pszValue, "file")) |
1671 | | { |
1672 | | oFileProp.nMode |= S_IFREG; |
1673 | | } |
1674 | | else if (EQUAL(pszValue, "directory")) |
1675 | | { |
1676 | | oFileProp.bIsDirectory = true; |
1677 | | oFileProp.nMode |= S_IFDIR; |
1678 | | } |
1679 | | } |
1680 | | else if (EQUAL(pszKey, "x-ms-permissions")) |
1681 | | { |
1682 | | oFileProp.nMode |= |
1683 | | VSICurlParseUnixPermissions(pszValue); |
1684 | | } |
1685 | | |
1686 | | // https://overturemapswestus2.blob.core.windows.net/release/2024-11-13.0/theme%3Ddivisions/type%3Ddivision_area |
1687 | | // returns a x-ms-meta-hdi_isfolder: true header |
1688 | | else if (EQUAL(pszKey, "x-ms-meta-hdi_isfolder") && |
1689 | | EQUAL(pszValue, "true")) |
1690 | | { |
1691 | | oFileProp.bIsAzureFolder = true; |
1692 | | oFileProp.bIsDirectory = true; |
1693 | | oFileProp.nMode |= S_IFDIR; |
1694 | | } |
1695 | | } |
1696 | | } |
1697 | | } |
1698 | | |
1699 | | if (bHasUsedLimitedRangeGet && response_code == 206) |
1700 | | { |
1701 | | oFileProp.eExists = EXIST_NO; |
1702 | | oFileProp.fileSize = 0; |
1703 | | if (sWriteFuncHeaderData.pBuffer != nullptr) |
1704 | | { |
1705 | | const CPLStringList aosHeaders( |
1706 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1707 | | const char *pszContentRange = |
1708 | | aosHeaders.FetchNameValue("content-range"); |
1709 | | // Trailing space in string intended |
1710 | | if (pszContentRange && |
1711 | | STARTS_WITH_CI(pszContentRange, "bytes ")) |
1712 | | { |
1713 | | pszContentRange += strlen("bytes "); |
1714 | | pszContentRange = strchr(pszContentRange, '/'); |
1715 | | if (pszContentRange) |
1716 | | { |
1717 | | oFileProp.eExists = EXIST_YES; |
1718 | | oFileProp.fileSize = static_cast<GUIntBig>( |
1719 | | CPLAtoGIntBig(pszContentRange + 1)); |
1720 | | } |
1721 | | } |
1722 | | |
1723 | | // Add first bytes to cache |
1724 | | if (sWriteFuncData.pBuffer != nullptr) |
1725 | | { |
1726 | | size_t nOffset = 0; |
1727 | | while (nOffset < sWriteFuncData.nSize) |
1728 | | { |
1729 | | const size_t nToCache = |
1730 | | std::min<size_t>(sWriteFuncData.nSize - nOffset, |
1731 | | knDOWNLOAD_CHUNK_SIZE); |
1732 | | poFS->AddRegion(m_pszURL, nOffset, nToCache, |
1733 | | sWriteFuncData.pBuffer + nOffset); |
1734 | | nOffset += nToCache; |
1735 | | } |
1736 | | } |
1737 | | } |
1738 | | } |
1739 | | else if (IsDirectoryFromExists(osVerb.c_str(), |
1740 | | static_cast<int>(response_code))) |
1741 | | { |
1742 | | oFileProp.eExists = EXIST_YES; |
1743 | | oFileProp.fileSize = 0; |
1744 | | oFileProp.bIsDirectory = true; |
1745 | | } |
1746 | | // 405 = Method not allowed |
1747 | | else if (response_code == 405 && !bRetryWithGet && osVerb == "HEAD") |
1748 | | { |
1749 | | CPLDebug(poFS->GetDebugKey(), |
1750 | | "HEAD not allowed. Retrying with GET"); |
1751 | | bRetryWithGet = true; |
1752 | | CPLFree(sWriteFuncData.pBuffer); |
1753 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1754 | | curl_easy_cleanup(hCurlHandle); |
1755 | | goto retry; |
1756 | | } |
1757 | | else if (response_code == 416) |
1758 | | { |
1759 | | oFileProp.eExists = EXIST_YES; |
1760 | | oFileProp.fileSize = 0; |
1761 | | } |
1762 | | else if (response_code != 200) |
1763 | | { |
1764 | | // Look if we should attempt a retry |
1765 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
1766 | | sWriteFuncHeaderData.pBuffer, |
1767 | | szCurlErrBuf)) |
1768 | | { |
1769 | | CPLError(CE_Warning, CPLE_AppDefined, |
1770 | | "HTTP error code: %d - %s. " |
1771 | | "Retrying again in %.1f secs", |
1772 | | static_cast<int>(response_code), m_pszURL, |
1773 | | oRetryContext.GetCurrentDelay()); |
1774 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
1775 | | CPLFree(sWriteFuncData.pBuffer); |
1776 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1777 | | curl_easy_cleanup(hCurlHandle); |
1778 | | goto retry; |
1779 | | } |
1780 | | |
1781 | | if (sWriteFuncData.pBuffer != nullptr) |
1782 | | { |
1783 | | if (UseLimitRangeGetInsteadOfHead() && |
1784 | | CanRestartOnError(sWriteFuncData.pBuffer, |
1785 | | sWriteFuncHeaderData.pBuffer, bSetError)) |
1786 | | { |
1787 | | oFileProp.bHasComputedFileSize = false; |
1788 | | CPLFree(sWriteFuncData.pBuffer); |
1789 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1790 | | curl_easy_cleanup(hCurlHandle); |
1791 | | return GetFileSizeOrHeaders(bSetError, bGetHeaders); |
1792 | | } |
1793 | | else |
1794 | | { |
1795 | | CPL_IGNORE_RET_VAL(CanRestartOnError( |
1796 | | sWriteFuncData.pBuffer, sWriteFuncHeaderData.pBuffer, |
1797 | | bSetError)); |
1798 | | } |
1799 | | } |
1800 | | |
1801 | | // If there was no VSI error thrown in the process, |
1802 | | // fail by reporting the HTTP response code. |
1803 | | if (bSetError && VSIGetLastErrorNo() == 0) |
1804 | | { |
1805 | | if (strlen(szCurlErrBuf) > 0) |
1806 | | { |
1807 | | if (response_code == 0) |
1808 | | { |
1809 | | VSIError(VSIE_HttpError, "CURL error: %s", |
1810 | | szCurlErrBuf); |
1811 | | } |
1812 | | else |
1813 | | { |
1814 | | VSIError(VSIE_HttpError, "HTTP response code: %d - %s", |
1815 | | static_cast<int>(response_code), szCurlErrBuf); |
1816 | | } |
1817 | | } |
1818 | | else |
1819 | | { |
1820 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
1821 | | static_cast<int>(response_code)); |
1822 | | } |
1823 | | } |
1824 | | else |
1825 | | { |
1826 | | if (response_code != 400 && response_code != 404) |
1827 | | { |
1828 | | CPLError(CE_Warning, CPLE_AppDefined, |
1829 | | "HTTP response code on %s: %d", osURL.c_str(), |
1830 | | static_cast<int>(response_code)); |
1831 | | } |
1832 | | // else a CPLDebug() is emitted below |
1833 | | } |
1834 | | |
1835 | | oFileProp.eExists = EXIST_NO; |
1836 | | oFileProp.nHTTPCode = static_cast<int>(response_code); |
1837 | | oFileProp.fileSize = 0; |
1838 | | } |
1839 | | else if (sWriteFuncData.pBuffer != nullptr) |
1840 | | { |
1841 | | ProcessGetFileSizeResult( |
1842 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer)); |
1843 | | } |
1844 | | |
1845 | | // Try to guess if this is a directory. Generally if this is a |
1846 | | // directory, curl will retry with an URL with slash added. |
1847 | | if (!osEffectiveURL.empty() && |
1848 | | strncmp(osURL.c_str(), osEffectiveURL.c_str(), osURL.size()) == 0 && |
1849 | | osEffectiveURL[osURL.size()] == '/' && |
1850 | | oFileProp.eExists != EXIST_NO) |
1851 | | { |
1852 | | oFileProp.eExists = EXIST_YES; |
1853 | | oFileProp.fileSize = 0; |
1854 | | oFileProp.bIsDirectory = true; |
1855 | | } |
1856 | | else if (osURL.back() == '/') |
1857 | | { |
1858 | | oFileProp.bIsDirectory = true; |
1859 | | } |
1860 | | |
1861 | | if (!bAlreadyLogged) |
1862 | | { |
1863 | | CPLDebug(poFS->GetDebugKey(), |
1864 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
1865 | | osURL.c_str(), oFileProp.fileSize, |
1866 | | static_cast<int>(response_code)); |
1867 | | } |
1868 | | } |
1869 | | |
1870 | | CPLFree(sWriteFuncData.pBuffer); |
1871 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1872 | | curl_easy_cleanup(hCurlHandle); |
1873 | | |
1874 | | oFileProp.bHasComputedFileSize = true; |
1875 | | if (mtime > 0) |
1876 | | oFileProp.mTime = mtime; |
1877 | | // Do not update cached file properties if cURL returned a non-HTTP error |
1878 | | if (response_code != 0) |
1879 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1880 | | |
1881 | | return oFileProp.fileSize; |
1882 | | } |
1883 | | |
1884 | | /************************************************************************/ |
1885 | | /* Exists() */ |
1886 | | /************************************************************************/ |
1887 | | |
1888 | | bool VSICurlHandle::Exists(bool bSetError) |
1889 | | { |
1890 | | if (oFileProp.eExists == EXIST_UNKNOWN) |
1891 | | { |
1892 | | GetFileSize(bSetError); |
1893 | | } |
1894 | | else if (oFileProp.eExists == EXIST_NO) |
1895 | | { |
1896 | | // If there was no VSI error thrown in the process, |
1897 | | // and we know the HTTP error code of the first request where the |
1898 | | // file could not be retrieved, fail by reporting the HTTP code. |
1899 | | if (bSetError && VSIGetLastErrorNo() == 0 && oFileProp.nHTTPCode) |
1900 | | { |
1901 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
1902 | | oFileProp.nHTTPCode); |
1903 | | } |
1904 | | } |
1905 | | |
1906 | | return oFileProp.eExists == EXIST_YES; |
1907 | | } |
1908 | | |
1909 | | /************************************************************************/ |
1910 | | /* Tell() */ |
1911 | | /************************************************************************/ |
1912 | | |
1913 | | vsi_l_offset VSICurlHandle::Tell() |
1914 | | { |
1915 | | return curOffset; |
1916 | | } |
1917 | | |
1918 | | /************************************************************************/ |
1919 | | /* GetRedirectURLIfValid() */ |
1920 | | /************************************************************************/ |
1921 | | |
1922 | | std::string |
1923 | | VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired, |
1924 | | CPLStringList &aosHTTPOptions) const |
1925 | | { |
1926 | | bHasExpired = false; |
1927 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
1928 | | |
1929 | | std::string osURL(m_pszURL + m_osQueryString); |
1930 | | if (oFileProp.bS3LikeRedirect) |
1931 | | { |
1932 | | if (time(nullptr) + 1 < oFileProp.nExpireTimestampLocal) |
1933 | | { |
1934 | | CPLDebug(poFS->GetDebugKey(), |
1935 | | "Using redirect URL as it looks to be still valid " |
1936 | | "(%d seconds left)", |
1937 | | static_cast<int>(oFileProp.nExpireTimestampLocal - |
1938 | | time(nullptr))); |
1939 | | osURL = oFileProp.osRedirectURL; |
1940 | | } |
1941 | | else |
1942 | | { |
1943 | | CPLDebug(poFS->GetDebugKey(), |
1944 | | "Redirect URL has expired. Using original URL"); |
1945 | | oFileProp.bS3LikeRedirect = false; |
1946 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1947 | | bHasExpired = true; |
1948 | | } |
1949 | | } |
1950 | | else if (!oFileProp.osRedirectURL.empty()) |
1951 | | { |
1952 | | osURL = oFileProp.osRedirectURL; |
1953 | | bHasExpired = false; |
1954 | | } |
1955 | | |
1956 | | if (m_pszURL != osURL) |
1957 | | { |
1958 | | const char *pszAuthorizationHeaderAllowed = VSIGetPathSpecificOption( |
1959 | | m_osFilename.c_str(), |
1960 | | "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT", |
1961 | | "IF_SAME_HOST"); |
1962 | | if (EQUAL(pszAuthorizationHeaderAllowed, "IF_SAME_HOST")) |
1963 | | { |
1964 | | const auto ExtractServer = [](const std::string &s) |
1965 | | { |
1966 | | size_t afterHTTPPos = 0; |
1967 | | if (STARTS_WITH(s.c_str(), "http://")) |
1968 | | afterHTTPPos = strlen("http://"); |
1969 | | else if (STARTS_WITH(s.c_str(), "https://")) |
1970 | | afterHTTPPos = strlen("https://"); |
1971 | | const auto posSlash = s.find('/', afterHTTPPos); |
1972 | | if (posSlash != std::string::npos) |
1973 | | return s.substr(afterHTTPPos, posSlash - afterHTTPPos); |
1974 | | else |
1975 | | return s.substr(afterHTTPPos); |
1976 | | }; |
1977 | | |
1978 | | if (ExtractServer(osURL) != ExtractServer(m_pszURL)) |
1979 | | { |
1980 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", |
1981 | | "NO"); |
1982 | | } |
1983 | | } |
1984 | | else if (!CPLTestBool(pszAuthorizationHeaderAllowed)) |
1985 | | { |
1986 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", "NO"); |
1987 | | } |
1988 | | } |
1989 | | |
1990 | | return osURL; |
1991 | | } |
1992 | | |
1993 | | /************************************************************************/ |
1994 | | /* CurrentDownload */ |
1995 | | /************************************************************************/ |
1996 | | |
1997 | | namespace |
1998 | | { |
1999 | | struct CurrentDownload |
2000 | | { |
2001 | | VSICurlFilesystemHandlerBase *m_poFS = nullptr; |
2002 | | std::string m_osURL{}; |
2003 | | vsi_l_offset m_nStartOffset = 0; |
2004 | | int m_nBlocks = 0; |
2005 | | std::string m_osAlreadyDownloadedData{}; |
2006 | | bool m_bHasAlreadyDownloadedData = false; |
2007 | | |
2008 | | CurrentDownload(VSICurlFilesystemHandlerBase *poFS, const char *pszURL, |
2009 | | vsi_l_offset startOffset, int nBlocks) |
2010 | | : m_poFS(poFS), m_osURL(pszURL), m_nStartOffset(startOffset), |
2011 | | m_nBlocks(nBlocks) |
2012 | | { |
2013 | | auto res = m_poFS->NotifyStartDownloadRegion(m_osURL, m_nStartOffset, |
2014 | | m_nBlocks); |
2015 | | m_bHasAlreadyDownloadedData = res.first; |
2016 | | m_osAlreadyDownloadedData = std::move(res.second); |
2017 | | } |
2018 | | |
2019 | | bool HasAlreadyDownloadedData() const |
2020 | | { |
2021 | | return m_bHasAlreadyDownloadedData; |
2022 | | } |
2023 | | |
2024 | | const std::string &GetAlreadyDownloadedData() const |
2025 | | { |
2026 | | return m_osAlreadyDownloadedData; |
2027 | | } |
2028 | | |
2029 | | void SetData(const std::string &osData) |
2030 | | { |
2031 | | CPLAssert(!m_bHasAlreadyDownloadedData); |
2032 | | m_bHasAlreadyDownloadedData = true; |
2033 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
2034 | | osData); |
2035 | | } |
2036 | | |
2037 | | ~CurrentDownload() |
2038 | | { |
2039 | | if (!m_bHasAlreadyDownloadedData) |
2040 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
2041 | | std::string()); |
2042 | | } |
2043 | | |
2044 | | CurrentDownload(const CurrentDownload &) = delete; |
2045 | | CurrentDownload &operator=(const CurrentDownload &) = delete; |
2046 | | }; |
2047 | | } // namespace |
2048 | | |
2049 | | /************************************************************************/ |
2050 | | /* NotifyStartDownloadRegion() */ |
2051 | | /************************************************************************/ |
2052 | | |
2053 | | /** Indicate intent at downloading a new region. |
2054 | | * |
2055 | | * If the region is already in download in another thread, then wait for its |
2056 | | * completion. |
2057 | | * |
2058 | | * Returns: |
2059 | | * - (false, empty string) if a new download is needed |
2060 | | * - (true, region_content) if we have been waiting for a download of the same |
2061 | | * region to be completed and got its result. Note that region_content will be |
2062 | | * empty if the download of that region failed. |
2063 | | */ |
2064 | | std::pair<bool, std::string> |
2065 | | VSICurlFilesystemHandlerBase::NotifyStartDownloadRegion( |
2066 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks) |
2067 | | { |
2068 | | std::string osId(osURL); |
2069 | | osId += '_'; |
2070 | | osId += std::to_string(startOffset); |
2071 | | osId += '_'; |
2072 | | osId += std::to_string(nBlocks); |
2073 | | |
2074 | | m_oMutex.lock(); |
2075 | | auto oIter = m_oMapRegionInDownload.find(osId); |
2076 | | if (oIter != m_oMapRegionInDownload.end()) |
2077 | | { |
2078 | | auto ®ion = *(oIter->second); |
2079 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
2080 | | m_oMutex.unlock(); |
2081 | | region.nWaiters++; |
2082 | | while (region.bDownloadInProgress) |
2083 | | { |
2084 | | region.oCond.wait(oRegionLock); |
2085 | | } |
2086 | | std::string osRet = region.osData; |
2087 | | region.nWaiters--; |
2088 | | region.oCond.notify_one(); |
2089 | | return std::pair<bool, std::string>(true, osRet); |
2090 | | } |
2091 | | else |
2092 | | { |
2093 | | auto poRegionInDownload = std::make_unique<RegionInDownload>(); |
2094 | | poRegionInDownload->bDownloadInProgress = true; |
2095 | | m_oMapRegionInDownload[osId] = std::move(poRegionInDownload); |
2096 | | m_oMutex.unlock(); |
2097 | | return std::pair<bool, std::string>(false, std::string()); |
2098 | | } |
2099 | | } |
2100 | | |
2101 | | /************************************************************************/ |
2102 | | /* NotifyStopDownloadRegion() */ |
2103 | | /************************************************************************/ |
2104 | | |
2105 | | void VSICurlFilesystemHandlerBase::NotifyStopDownloadRegion( |
2106 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks, |
2107 | | const std::string &osData) |
2108 | | { |
2109 | | std::string osId(osURL); |
2110 | | osId += '_'; |
2111 | | osId += std::to_string(startOffset); |
2112 | | osId += '_'; |
2113 | | osId += std::to_string(nBlocks); |
2114 | | |
2115 | | m_oMutex.lock(); |
2116 | | auto oIter = m_oMapRegionInDownload.find(osId); |
2117 | | CPLAssert(oIter != m_oMapRegionInDownload.end()); |
2118 | | auto ®ion = *(oIter->second); |
2119 | | { |
2120 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
2121 | | if (region.nWaiters) |
2122 | | { |
2123 | | region.osData = osData; |
2124 | | region.bDownloadInProgress = false; |
2125 | | region.oCond.notify_all(); |
2126 | | |
2127 | | while (region.nWaiters) |
2128 | | { |
2129 | | region.oCond.wait(oRegionLock); |
2130 | | } |
2131 | | } |
2132 | | } |
2133 | | m_oMapRegionInDownload.erase(oIter); |
2134 | | m_oMutex.unlock(); |
2135 | | } |
2136 | | |
2137 | | /************************************************************************/ |
2138 | | /* DownloadRegion() */ |
2139 | | /************************************************************************/ |
2140 | | |
2141 | | std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, |
2142 | | const int nBlocks) |
2143 | | { |
2144 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
2145 | | return std::string(); |
2146 | | |
2147 | | if (oFileProp.eExists == EXIST_NO) |
2148 | | return std::string(); |
2149 | | |
2150 | | // Check if there is not a download of the same region in progress in |
2151 | | // another thread, and if so wait for it to be completed |
2152 | | CurrentDownload currentDownload(poFS, m_pszURL, startOffset, nBlocks); |
2153 | | if (currentDownload.HasAlreadyDownloadedData()) |
2154 | | { |
2155 | | return currentDownload.GetAlreadyDownloadedData(); |
2156 | | } |
2157 | | |
2158 | | begin: |
2159 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
2160 | | |
2161 | | UpdateQueryString(); |
2162 | | |
2163 | | bool bHasExpired = false; |
2164 | | |
2165 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
2166 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
2167 | | bool bUsedRedirect = osURL != m_pszURL; |
2168 | | |
2169 | | WriteFuncStruct sWriteFuncData; |
2170 | | WriteFuncStruct sWriteFuncHeaderData; |
2171 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
2172 | | |
2173 | | retry: |
2174 | | CURL *hCurlHandle = curl_easy_init(); |
2175 | | struct curl_slist *headers = |
2176 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
2177 | | |
2178 | | if (!AllowAutomaticRedirection()) |
2179 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
2180 | | |
2181 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
2182 | | pReadCbkUserData); |
2183 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
2184 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2185 | | VSICurlHandleWriteFunc); |
2186 | | |
2187 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
2188 | | nullptr); |
2189 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2190 | | &sWriteFuncHeaderData); |
2191 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2192 | | VSICurlHandleWriteFunc); |
2193 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2194 | | sWriteFuncHeaderData.nStartOffset = startOffset; |
2195 | | sWriteFuncHeaderData.nEndOffset = |
2196 | | startOffset + |
2197 | | static_cast<vsi_l_offset>(nBlocks) * VSICURLGetDownloadChunkSize() - 1; |
2198 | | // Some servers don't like we try to read after end-of-file (#5786). |
2199 | | if (oFileProp.bHasComputedFileSize && |
2200 | | sWriteFuncHeaderData.nEndOffset >= oFileProp.fileSize) |
2201 | | { |
2202 | | sWriteFuncHeaderData.nEndOffset = oFileProp.fileSize - 1; |
2203 | | } |
2204 | | |
2205 | | char rangeStr[512] = {}; |
2206 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2207 | | startOffset, sWriteFuncHeaderData.nEndOffset); |
2208 | | |
2209 | | if constexpr (ENABLE_DEBUG) |
2210 | | { |
2211 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
2212 | | osURL.c_str()); |
2213 | | } |
2214 | | |
2215 | | std::string osHeaderRange; // leave in this scope |
2216 | | if (sWriteFuncHeaderData.bIsHTTP) |
2217 | | { |
2218 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
2219 | | // So it gets included in Azure signature |
2220 | | headers = curl_slist_append(headers, osHeaderRange.c_str()); |
2221 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
2222 | | } |
2223 | | else |
2224 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
2225 | | |
2226 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
2227 | | szCurlErrBuf[0] = '\0'; |
2228 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
2229 | | |
2230 | | headers = GetCurlHeaders("GET", headers); |
2231 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
2232 | | |
2233 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
2234 | | |
2235 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); |
2236 | | |
2237 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
2238 | | |
2239 | | curl_slist_free_all(headers); |
2240 | | |
2241 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
2242 | | |
2243 | | if (sWriteFuncData.bInterrupted || m_bInterrupt) |
2244 | | { |
2245 | | bInterrupted = true; |
2246 | | |
2247 | | // Notify that the download of the current region is finished |
2248 | | currentDownload.SetData(std::string()); |
2249 | | |
2250 | | CPLFree(sWriteFuncData.pBuffer); |
2251 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2252 | | curl_easy_cleanup(hCurlHandle); |
2253 | | |
2254 | | return std::string(); |
2255 | | } |
2256 | | |
2257 | | long response_code = 0; |
2258 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2259 | | |
2260 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
2261 | | { |
2262 | | CPLDebug(poFS->GetDebugKey(), |
2263 | | "DownloadRegion(%s): response_code=%d, msg=%s", osURL.c_str(), |
2264 | | static_cast<int>(response_code), szCurlErrBuf); |
2265 | | } |
2266 | | |
2267 | | long mtime = 0; |
2268 | | curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME, &mtime); |
2269 | | if (mtime > 0) |
2270 | | { |
2271 | | oFileProp.mTime = mtime; |
2272 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2273 | | } |
2274 | | |
2275 | | if constexpr (ENABLE_DEBUG) |
2276 | | { |
2277 | | CPLDebug(poFS->GetDebugKey(), "Got response_code=%ld", response_code); |
2278 | | } |
2279 | | |
2280 | | if (bUsedRedirect && |
2281 | | (response_code == 403 || |
2282 | | // Below case is in particular for |
2283 | | // gdalinfo |
2284 | | // /vsicurl/https://lpdaac.earthdata.nasa.gov/lp-prod-protected/HLSS30.015/HLS.S30.T10TEK.2020273T190109.v1.5.B8A.tif |
2285 | | // --config GDAL_DISABLE_READDIR_ON_OPEN EMPTY_DIR --config |
2286 | | // GDAL_HTTP_COOKIEFILE /tmp/cookie.txt --config GDAL_HTTP_COOKIEJAR |
2287 | | // /tmp/cookie.txt We got the redirect URL from a HEAD request, but it |
2288 | | // is not valid for a GET. So retry with GET on original URL to get a |
2289 | | // redirect URL valid for it. |
2290 | | (response_code == 400 && |
2291 | | osURL.find(".cloudfront.net") != std::string::npos))) |
2292 | | { |
2293 | | CPLDebug(poFS->GetDebugKey(), |
2294 | | "Got an error with redirect URL. Retrying with original one"); |
2295 | | oFileProp.bS3LikeRedirect = false; |
2296 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2297 | | bUsedRedirect = false; |
2298 | | osURL = m_pszURL; |
2299 | | CPLFree(sWriteFuncData.pBuffer); |
2300 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2301 | | curl_easy_cleanup(hCurlHandle); |
2302 | | goto retry; |
2303 | | } |
2304 | | |
2305 | | if (response_code == 401 && oRetryContext.CanRetry()) |
2306 | | { |
2307 | | CPLDebug(poFS->GetDebugKey(), "Unauthorized, trying to authenticate"); |
2308 | | CPLFree(sWriteFuncData.pBuffer); |
2309 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2310 | | curl_easy_cleanup(hCurlHandle); |
2311 | | if (Authenticate(m_osFilename.c_str())) |
2312 | | goto retry; |
2313 | | return std::string(); |
2314 | | } |
2315 | | |
2316 | | UpdateRedirectInfo(hCurlHandle, sWriteFuncHeaderData); |
2317 | | |
2318 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
2319 | | response_code != 226 && response_code != 426) || |
2320 | | sWriteFuncHeaderData.bError) |
2321 | | { |
2322 | | if (sWriteFuncData.pBuffer != nullptr && |
2323 | | CanRestartOnError( |
2324 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer), |
2325 | | reinterpret_cast<const char *>(sWriteFuncHeaderData.pBuffer), |
2326 | | true)) |
2327 | | { |
2328 | | CPLFree(sWriteFuncData.pBuffer); |
2329 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2330 | | curl_easy_cleanup(hCurlHandle); |
2331 | | goto begin; |
2332 | | } |
2333 | | |
2334 | | // Look if we should attempt a retry |
2335 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
2336 | | sWriteFuncHeaderData.pBuffer, szCurlErrBuf)) |
2337 | | { |
2338 | | CPLError(CE_Warning, CPLE_AppDefined, |
2339 | | "HTTP error code: %d - %s. " |
2340 | | "Retrying again in %.1f secs", |
2341 | | static_cast<int>(response_code), m_pszURL, |
2342 | | oRetryContext.GetCurrentDelay()); |
2343 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
2344 | | CPLFree(sWriteFuncData.pBuffer); |
2345 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2346 | | curl_easy_cleanup(hCurlHandle); |
2347 | | goto retry; |
2348 | | } |
2349 | | |
2350 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
2351 | | { |
2352 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
2353 | | CPLError( |
2354 | | CE_Failure, CPLE_AppDefined, |
2355 | | "%d: %s, Range downloading not supported by this server!", |
2356 | | static_cast<int>(response_code), szCurlErrBuf); |
2357 | | else |
2358 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
2359 | | static_cast<int>(response_code), szCurlErrBuf); |
2360 | | } |
2361 | | else if (response_code == 416) /* Range Not Satisfiable */ |
2362 | | { |
2363 | | if (sWriteFuncData.pBuffer) |
2364 | | { |
2365 | | CPLError( |
2366 | | CE_Failure, CPLE_AppDefined, |
2367 | | "%d: Range downloading not supported by this server: %s", |
2368 | | static_cast<int>(response_code), sWriteFuncData.pBuffer); |
2369 | | } |
2370 | | else |
2371 | | { |
2372 | | CPLError(CE_Failure, CPLE_AppDefined, |
2373 | | "%d: Range downloading not supported by this server", |
2374 | | static_cast<int>(response_code)); |
2375 | | } |
2376 | | } |
2377 | | if (!oFileProp.bHasComputedFileSize && startOffset == 0) |
2378 | | { |
2379 | | oFileProp.bHasComputedFileSize = true; |
2380 | | oFileProp.fileSize = 0; |
2381 | | oFileProp.eExists = EXIST_NO; |
2382 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2383 | | } |
2384 | | CPLFree(sWriteFuncData.pBuffer); |
2385 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2386 | | curl_easy_cleanup(hCurlHandle); |
2387 | | return std::string(); |
2388 | | } |
2389 | | |
2390 | | if (!oFileProp.bHasComputedFileSize && sWriteFuncHeaderData.pBuffer) |
2391 | | { |
2392 | | // Try to retrieve the filesize from the HTTP headers |
2393 | | // if in the form: "Content-Range: bytes x-y/filesize". |
2394 | | char *pszContentRange = |
2395 | | strstr(sWriteFuncHeaderData.pBuffer, "Content-Range: bytes "); |
2396 | | if (pszContentRange == nullptr) |
2397 | | pszContentRange = |
2398 | | strstr(sWriteFuncHeaderData.pBuffer, "content-range: bytes "); |
2399 | | if (pszContentRange) |
2400 | | { |
2401 | | char *pszEOL = strchr(pszContentRange, '\n'); |
2402 | | if (pszEOL) |
2403 | | { |
2404 | | *pszEOL = 0; |
2405 | | pszEOL = strchr(pszContentRange, '\r'); |
2406 | | if (pszEOL) |
2407 | | *pszEOL = 0; |
2408 | | char *pszSlash = strchr(pszContentRange, '/'); |
2409 | | if (pszSlash) |
2410 | | { |
2411 | | pszSlash++; |
2412 | | oFileProp.fileSize = CPLScanUIntBig( |
2413 | | pszSlash, static_cast<int>(strlen(pszSlash))); |
2414 | | } |
2415 | | } |
2416 | | } |
2417 | | else if (STARTS_WITH(m_pszURL, "ftp")) |
2418 | | { |
2419 | | // Parse 213 answer for FTP protocol. |
2420 | | char *pszSize = strstr(sWriteFuncHeaderData.pBuffer, "213 "); |
2421 | | if (pszSize) |
2422 | | { |
2423 | | pszSize += 4; |
2424 | | char *pszEOL = strchr(pszSize, '\n'); |
2425 | | if (pszEOL) |
2426 | | { |
2427 | | *pszEOL = 0; |
2428 | | pszEOL = strchr(pszSize, '\r'); |
2429 | | if (pszEOL) |
2430 | | *pszEOL = 0; |
2431 | | |
2432 | | oFileProp.fileSize = CPLScanUIntBig( |
2433 | | pszSize, static_cast<int>(strlen(pszSize))); |
2434 | | } |
2435 | | } |
2436 | | } |
2437 | | |
2438 | | if (oFileProp.fileSize != 0) |
2439 | | { |
2440 | | oFileProp.eExists = EXIST_YES; |
2441 | | |
2442 | | if constexpr (ENABLE_DEBUG) |
2443 | | { |
2444 | | CPLDebug(poFS->GetDebugKey(), |
2445 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
2446 | | m_pszURL, oFileProp.fileSize, |
2447 | | static_cast<int>(response_code)); |
2448 | | } |
2449 | | |
2450 | | oFileProp.bHasComputedFileSize = true; |
2451 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2452 | | } |
2453 | | } |
2454 | | |
2455 | | DownloadRegionPostProcess(startOffset, nBlocks, sWriteFuncData.pBuffer, |
2456 | | sWriteFuncData.nSize); |
2457 | | |
2458 | | std::string osRet; |
2459 | | osRet.assign(sWriteFuncData.pBuffer, sWriteFuncData.nSize); |
2460 | | |
2461 | | // Notify that the download of the current region is finished |
2462 | | currentDownload.SetData(osRet); |
2463 | | |
2464 | | CPLFree(sWriteFuncData.pBuffer); |
2465 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2466 | | curl_easy_cleanup(hCurlHandle); |
2467 | | |
2468 | | return osRet; |
2469 | | } |
2470 | | |
2471 | | /************************************************************************/ |
2472 | | /* UpdateRedirectInfo() */ |
2473 | | /************************************************************************/ |
2474 | | |
2475 | | void VSICurlHandle::UpdateRedirectInfo( |
2476 | | CURL *hCurlHandle, const WriteFuncStruct &sWriteFuncHeaderData) |
2477 | | { |
2478 | | std::string osEffectiveURL; |
2479 | | { |
2480 | | char *pszEffectiveURL = nullptr; |
2481 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
2482 | | &pszEffectiveURL); |
2483 | | if (pszEffectiveURL) |
2484 | | osEffectiveURL = pszEffectiveURL; |
2485 | | } |
2486 | | |
2487 | | if (!oFileProp.bS3LikeRedirect && !osEffectiveURL.empty() && |
2488 | | strstr(osEffectiveURL.c_str(), m_pszURL) == nullptr) |
2489 | | { |
2490 | | CPLDebug(poFS->GetDebugKey(), "Effective URL: %s", |
2491 | | osEffectiveURL.c_str()); |
2492 | | |
2493 | | long response_code = 0; |
2494 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2495 | | if (response_code >= 200 && response_code < 300 && |
2496 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
2497 | | VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
2498 | | !VSICurlIsS3LikeSignedURL(m_pszURL) && |
2499 | | CPLTestBool( |
2500 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
2501 | | { |
2502 | | GIntBig nExpireTimestamp = |
2503 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
2504 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
2505 | | { |
2506 | | const int nValidity = static_cast<int>( |
2507 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
2508 | | CPLDebug(poFS->GetDebugKey(), |
2509 | | "Will use redirect URL for the next %d seconds", |
2510 | | nValidity); |
2511 | | // As our local clock might not be in sync with server clock, |
2512 | | // figure out the expiration timestamp in local time. |
2513 | | oFileProp.bS3LikeRedirect = true; |
2514 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
2515 | | oFileProp.osRedirectURL = std::move(osEffectiveURL); |
2516 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2517 | | } |
2518 | | } |
2519 | | } |
2520 | | } |
2521 | | |
2522 | | /************************************************************************/ |
2523 | | /* DownloadRegionPostProcess() */ |
2524 | | /************************************************************************/ |
2525 | | |
2526 | | void VSICurlHandle::DownloadRegionPostProcess(const vsi_l_offset startOffset, |
2527 | | const int nBlocks, |
2528 | | const char *pBuffer, size_t nSize) |
2529 | | { |
2530 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2531 | | lastDownloadedOffset = startOffset + static_cast<vsi_l_offset>(nBlocks) * |
2532 | | knDOWNLOAD_CHUNK_SIZE; |
2533 | | |
2534 | | if (nSize > static_cast<size_t>(nBlocks) * knDOWNLOAD_CHUNK_SIZE) |
2535 | | { |
2536 | | if constexpr (ENABLE_DEBUG) |
2537 | | { |
2538 | | CPLDebug( |
2539 | | poFS->GetDebugKey(), |
2540 | | "Got more data than expected : %u instead of %u", |
2541 | | static_cast<unsigned int>(nSize), |
2542 | | static_cast<unsigned int>(nBlocks * knDOWNLOAD_CHUNK_SIZE)); |
2543 | | } |
2544 | | } |
2545 | | |
2546 | | vsi_l_offset l_startOffset = startOffset; |
2547 | | while (nSize > 0) |
2548 | | { |
2549 | | #if DEBUG_VERBOSE |
2550 | | if constexpr (ENABLE_DEBUG) |
2551 | | { |
2552 | | CPLDebug(poFS->GetDebugKey(), "Add region %u - %u", |
2553 | | static_cast<unsigned int>(startOffset), |
2554 | | static_cast<unsigned int>(std::min( |
2555 | | static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize))); |
2556 | | } |
2557 | | #endif |
2558 | | const size_t nChunkSize = |
2559 | | std::min(static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize); |
2560 | | poFS->AddRegion(m_pszURL, l_startOffset, nChunkSize, pBuffer); |
2561 | | l_startOffset += nChunkSize; |
2562 | | pBuffer += nChunkSize; |
2563 | | nSize -= nChunkSize; |
2564 | | } |
2565 | | } |
2566 | | |
2567 | | /************************************************************************/ |
2568 | | /* Read() */ |
2569 | | /************************************************************************/ |
2570 | | |
2571 | | size_t VSICurlHandle::Read(void *const pBufferIn, size_t const nBytes) |
2572 | | { |
2573 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2574 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2575 | | NetworkStatisticsAction oContextAction("Read"); |
2576 | | |
2577 | | size_t nBufferRequestSize = nBytes; |
2578 | | if (nBufferRequestSize == 0) |
2579 | | return 0; |
2580 | | |
2581 | | void *pBuffer = pBufferIn; |
2582 | | |
2583 | | #if DEBUG_VERBOSE |
2584 | | CPLDebug(poFS->GetDebugKey(), "offset=%d, size=%d", |
2585 | | static_cast<int>(curOffset), static_cast<int>(nBufferRequestSize)); |
2586 | | #endif |
2587 | | |
2588 | | vsi_l_offset iterOffset = curOffset; |
2589 | | const int knMAX_REGIONS = GetMaxRegions(); |
2590 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2591 | | while (nBufferRequestSize) |
2592 | | { |
2593 | | // Don't try to read after end of file. |
2594 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2595 | | if (oFileProp.bHasComputedFileSize && iterOffset >= oFileProp.fileSize) |
2596 | | { |
2597 | | if (iterOffset == curOffset) |
2598 | | { |
2599 | | CPLDebug(poFS->GetDebugKey(), |
2600 | | "Request at offset " CPL_FRMT_GUIB |
2601 | | ", after end of file", |
2602 | | iterOffset); |
2603 | | } |
2604 | | break; |
2605 | | } |
2606 | | |
2607 | | const vsi_l_offset nOffsetToDownload = |
2608 | | (iterOffset / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
2609 | | std::string osRegion; |
2610 | | std::shared_ptr<std::string> psRegion = |
2611 | | poFS->GetRegion(m_pszURL, nOffsetToDownload); |
2612 | | if (psRegion != nullptr) |
2613 | | { |
2614 | | osRegion = *psRegion; |
2615 | | } |
2616 | | else |
2617 | | { |
2618 | | if (nOffsetToDownload == lastDownloadedOffset) |
2619 | | { |
2620 | | // In case of consecutive reads (of small size), we use a |
2621 | | // heuristic that we will read the file sequentially, so |
2622 | | // we double the requested size to decrease the number of |
2623 | | // client/server roundtrips. |
2624 | | constexpr int MAX_CHUNK_SIZE_INCREASE_FACTOR = 128; |
2625 | | if (nBlocksToDownload < MAX_CHUNK_SIZE_INCREASE_FACTOR) |
2626 | | nBlocksToDownload *= 2; |
2627 | | } |
2628 | | else |
2629 | | { |
2630 | | // Random reads. Cancel the above heuristics. |
2631 | | nBlocksToDownload = 1; |
2632 | | } |
2633 | | |
2634 | | // Ensure that we will request at least the number of blocks |
2635 | | // to satisfy the remaining buffer size to read. |
2636 | | const vsi_l_offset nEndOffsetToDownload = |
2637 | | ((iterOffset + nBufferRequestSize + knDOWNLOAD_CHUNK_SIZE - 1) / |
2638 | | knDOWNLOAD_CHUNK_SIZE) * |
2639 | | knDOWNLOAD_CHUNK_SIZE; |
2640 | | const int nMinBlocksToDownload = |
2641 | | static_cast<int>((nEndOffsetToDownload - nOffsetToDownload) / |
2642 | | knDOWNLOAD_CHUNK_SIZE); |
2643 | | if (nBlocksToDownload < nMinBlocksToDownload) |
2644 | | nBlocksToDownload = nMinBlocksToDownload; |
2645 | | |
2646 | | // Avoid reading already cached data. |
2647 | | // Note: this might get evicted if concurrent reads are done, but |
2648 | | // this should not cause bugs. Just missed optimization. |
2649 | | for (int i = 1; i < nBlocksToDownload; i++) |
2650 | | { |
2651 | | if (poFS->GetRegion(m_pszURL, nOffsetToDownload + |
2652 | | static_cast<vsi_l_offset>(i) * |
2653 | | knDOWNLOAD_CHUNK_SIZE) != |
2654 | | nullptr) |
2655 | | { |
2656 | | nBlocksToDownload = i; |
2657 | | break; |
2658 | | } |
2659 | | } |
2660 | | |
2661 | | // We can't download more than knMAX_REGIONS chunks at a time, |
2662 | | // otherwise the cache will not be big enough to store them and |
2663 | | // copy their content to the target buffer. |
2664 | | if (nBlocksToDownload > knMAX_REGIONS) |
2665 | | nBlocksToDownload = knMAX_REGIONS; |
2666 | | |
2667 | | osRegion = DownloadRegion(nOffsetToDownload, nBlocksToDownload); |
2668 | | if (osRegion.empty()) |
2669 | | { |
2670 | | if (!bInterrupted) |
2671 | | bError = true; |
2672 | | return 0; |
2673 | | } |
2674 | | } |
2675 | | |
2676 | | const vsi_l_offset nRegionOffset = iterOffset - nOffsetToDownload; |
2677 | | if (osRegion.size() < nRegionOffset) |
2678 | | { |
2679 | | if (iterOffset == curOffset) |
2680 | | { |
2681 | | CPLDebug(poFS->GetDebugKey(), |
2682 | | "Request at offset " CPL_FRMT_GUIB |
2683 | | ", after end of file", |
2684 | | iterOffset); |
2685 | | } |
2686 | | break; |
2687 | | } |
2688 | | |
2689 | | const int nToCopy = static_cast<int>( |
2690 | | std::min(static_cast<vsi_l_offset>(nBufferRequestSize), |
2691 | | osRegion.size() - nRegionOffset)); |
2692 | | memcpy(pBuffer, osRegion.data() + nRegionOffset, nToCopy); |
2693 | | pBuffer = static_cast<char *>(pBuffer) + nToCopy; |
2694 | | iterOffset += nToCopy; |
2695 | | nBufferRequestSize -= nToCopy; |
2696 | | if (osRegion.size() < static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE) && |
2697 | | nBufferRequestSize != 0) |
2698 | | { |
2699 | | break; |
2700 | | } |
2701 | | } |
2702 | | |
2703 | | const size_t ret = static_cast<size_t>(iterOffset - curOffset); |
2704 | | if (ret != nBytes) |
2705 | | bEOF = true; |
2706 | | |
2707 | | curOffset = iterOffset; |
2708 | | |
2709 | | return ret; |
2710 | | } |
2711 | | |
2712 | | /************************************************************************/ |
2713 | | /* ReadMultiRange() */ |
2714 | | /************************************************************************/ |
2715 | | |
2716 | | int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, |
2717 | | const vsi_l_offset *const panOffsets, |
2718 | | const size_t *const panSizes) |
2719 | | { |
2720 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
2721 | | return FALSE; |
2722 | | |
2723 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2724 | | if (oFileProp.eExists == EXIST_NO) |
2725 | | return -1; |
2726 | | |
2727 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2728 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2729 | | NetworkStatisticsAction oContextAction("ReadMultiRange"); |
2730 | | |
2731 | | const char *pszMultiRangeStrategy = |
2732 | | CPLGetConfigOption("GDAL_HTTP_MULTIRANGE", ""); |
2733 | | if (EQUAL(pszMultiRangeStrategy, "SINGLE_GET")) |
2734 | | { |
2735 | | // Just in case someone needs it, but the interest of this mode is |
2736 | | // rather dubious now. We could probably remove it |
2737 | | return ReadMultiRangeSingleGet(nRanges, ppData, panOffsets, panSizes); |
2738 | | } |
2739 | | else if (nRanges == 1 || EQUAL(pszMultiRangeStrategy, "SERIAL")) |
2740 | | { |
2741 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
2742 | | panSizes); |
2743 | | } |
2744 | | |
2745 | | UpdateQueryString(); |
2746 | | |
2747 | | bool bHasExpired = false; |
2748 | | |
2749 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
2750 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
2751 | | if (bHasExpired) |
2752 | | { |
2753 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
2754 | | panSizes); |
2755 | | } |
2756 | | |
2757 | | CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); |
2758 | | #ifdef CURLPIPE_MULTIPLEX |
2759 | | // Enable HTTP/2 multiplexing (ignored if an older version of HTTP is |
2760 | | // used) |
2761 | | // Not that this does not enable HTTP/1.1 pipeling, which is not |
2762 | | // recommended for example by Google Cloud Storage. |
2763 | | // For HTTP/1.1, parallel connections work better since you can get |
2764 | | // results out of order. |
2765 | | if (CPLTestBool(CPLGetConfigOption("GDAL_HTTP_MULTIPLEX", "YES"))) |
2766 | | { |
2767 | | curl_multi_setopt(hMultiHandle, CURLMOPT_PIPELINING, |
2768 | | CURLPIPE_MULTIPLEX); |
2769 | | } |
2770 | | #endif |
2771 | | |
2772 | | struct CurlErrBuffer |
2773 | | { |
2774 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
2775 | | }; |
2776 | | |
2777 | | // Sort ranges by file offset so the merge loop below can coalesce |
2778 | | // adjacent ranges regardless of the order the caller passed them. |
2779 | | // The ppData buffer pointers travel with their offsets, so the |
2780 | | // distribute logic fills the correct caller buffers after reading. |
2781 | | std::vector<int> anSortOrder(nRanges); |
2782 | | std::iota(anSortOrder.begin(), anSortOrder.end(), 0); |
2783 | | std::sort(anSortOrder.begin(), anSortOrder.end(), [panOffsets](int a, int b) |
2784 | | { return panOffsets[a] < panOffsets[b]; }); |
2785 | | |
2786 | | std::vector<void *> apSortedData(nRanges); |
2787 | | std::vector<vsi_l_offset> anSortedOffsets(nRanges); |
2788 | | std::vector<size_t> anSortedSizes(nRanges); |
2789 | | for (int i = 0; i < nRanges; ++i) |
2790 | | { |
2791 | | apSortedData[i] = ppData[anSortOrder[i]]; |
2792 | | anSortedOffsets[i] = panOffsets[anSortOrder[i]]; |
2793 | | anSortedSizes[i] = panSizes[anSortOrder[i]]; |
2794 | | } |
2795 | | |
2796 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
2797 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
2798 | | |
2799 | | // Build list of merged requests upfront, each with its own retry context |
2800 | | struct MergedRequest |
2801 | | { |
2802 | | int iFirstRange; |
2803 | | int iLastRange; |
2804 | | vsi_l_offset nStartOffset; |
2805 | | size_t nSize; |
2806 | | CPLHTTPRetryContext retryContext; |
2807 | | bool bToRetry = true; // true initially to trigger first attempt |
2808 | | |
2809 | | MergedRequest(int first, int last, vsi_l_offset start, size_t size, |
2810 | | const CPLHTTPRetryParameters ¶ms) |
2811 | | : iFirstRange(first), iLastRange(last), nStartOffset(start), |
2812 | | nSize(size), retryContext(params) |
2813 | | { |
2814 | | } |
2815 | | }; |
2816 | | |
2817 | | std::vector<MergedRequest> asMergedRequests; |
2818 | | for (int i = 0; i < nRanges;) |
2819 | | { |
2820 | | size_t nSize = 0; |
2821 | | int iNext = i; |
2822 | | // Identify consecutive ranges |
2823 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
2824 | | anSortedOffsets[iNext] + anSortedSizes[iNext] == |
2825 | | anSortedOffsets[iNext + 1]) |
2826 | | { |
2827 | | nSize += anSortedSizes[iNext]; |
2828 | | iNext++; |
2829 | | } |
2830 | | nSize += anSortedSizes[iNext]; |
2831 | | |
2832 | | if (nSize == 0) |
2833 | | { |
2834 | | i = iNext + 1; |
2835 | | continue; |
2836 | | } |
2837 | | |
2838 | | asMergedRequests.emplace_back(i, iNext, anSortedOffsets[i], nSize, |
2839 | | m_oRetryParameters); |
2840 | | i = iNext + 1; |
2841 | | } |
2842 | | |
2843 | | if (asMergedRequests.empty()) |
2844 | | return 0; |
2845 | | |
2846 | | int nRet = 0; |
2847 | | size_t nTotalDownloaded = 0; |
2848 | | |
2849 | | // Retry loop: re-issue only failed requests that are retryable |
2850 | | while (true) |
2851 | | { |
2852 | | const size_t nRequests = asMergedRequests.size(); |
2853 | | std::vector<CURL *> aHandles(nRequests, nullptr); |
2854 | | std::vector<WriteFuncStruct> asWriteFuncData(nRequests); |
2855 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData(nRequests); |
2856 | | std::vector<char *> apszRanges(nRequests, nullptr); |
2857 | | std::vector<struct curl_slist *> aHeaders(nRequests, nullptr); |
2858 | | std::vector<CurlErrBuffer> asCurlErrors(nRequests); |
2859 | | |
2860 | | bool bAnyHandle = false; |
2861 | | for (size_t iReq = 0; iReq < nRequests; iReq++) |
2862 | | { |
2863 | | if (!asMergedRequests[iReq].bToRetry) |
2864 | | continue; |
2865 | | asMergedRequests[iReq].bToRetry = false; |
2866 | | |
2867 | | CURL *hCurlHandle = curl_easy_init(); |
2868 | | aHandles[iReq] = hCurlHandle; |
2869 | | bAnyHandle = true; |
2870 | | |
2871 | | struct curl_slist *headers = VSICurlSetOptions( |
2872 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
2873 | | |
2874 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[iReq], this, pfnReadCbk, |
2875 | | pReadCbkUserData); |
2876 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
2877 | | &asWriteFuncData[iReq]); |
2878 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2879 | | VSICurlHandleWriteFunc); |
2880 | | |
2881 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[iReq], nullptr, |
2882 | | nullptr, nullptr); |
2883 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2884 | | &asWriteFuncHeaderData[iReq]); |
2885 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2886 | | VSICurlHandleWriteFunc); |
2887 | | asWriteFuncHeaderData[iReq].bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2888 | | asWriteFuncHeaderData[iReq].nStartOffset = |
2889 | | asMergedRequests[iReq].nStartOffset; |
2890 | | asWriteFuncHeaderData[iReq].nEndOffset = |
2891 | | asMergedRequests[iReq].nStartOffset + |
2892 | | asMergedRequests[iReq].nSize - 1; |
2893 | | |
2894 | | char rangeStr[512] = {}; |
2895 | | snprintf(rangeStr, sizeof(rangeStr), |
2896 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2897 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2898 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2899 | | |
2900 | | if constexpr (ENABLE_DEBUG) |
2901 | | { |
2902 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
2903 | | rangeStr, osURL.c_str()); |
2904 | | } |
2905 | | |
2906 | | if (asWriteFuncHeaderData[iReq].bIsHTTP) |
2907 | | { |
2908 | | // So it gets included in Azure signature |
2909 | | char *pszRange = |
2910 | | CPLStrdup(CPLSPrintf("Range: bytes=%s", rangeStr)); |
2911 | | apszRanges[iReq] = pszRange; |
2912 | | headers = curl_slist_append(headers, pszRange); |
2913 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
2914 | | } |
2915 | | else |
2916 | | { |
2917 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
2918 | | rangeStr); |
2919 | | } |
2920 | | |
2921 | | asCurlErrors[iReq].szCurlErrBuf[0] = '\0'; |
2922 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
2923 | | &asCurlErrors[iReq].szCurlErrBuf[0]); |
2924 | | |
2925 | | headers = GetCurlHeaders("GET", headers); |
2926 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
2927 | | headers); |
2928 | | aHeaders[iReq] = headers; |
2929 | | curl_multi_add_handle(hMultiHandle, hCurlHandle); |
2930 | | } |
2931 | | |
2932 | | if (bAnyHandle) |
2933 | | { |
2934 | | VSICURLMultiPerform(hMultiHandle); |
2935 | | } |
2936 | | |
2937 | | // Process results |
2938 | | bool bRetry = false; |
2939 | | double dfMaxDelay = 0.0; |
2940 | | for (size_t iReq = 0; iReq < nRequests; iReq++) |
2941 | | { |
2942 | | if (!aHandles[iReq]) |
2943 | | continue; |
2944 | | |
2945 | | long response_code = 0; |
2946 | | curl_easy_getinfo(aHandles[iReq], CURLINFO_HTTP_CODE, |
2947 | | &response_code); |
2948 | | |
2949 | | if (ENABLE_DEBUG && asCurlErrors[iReq].szCurlErrBuf[0] != '\0') |
2950 | | { |
2951 | | char rangeStr[512] = {}; |
2952 | | snprintf(rangeStr, sizeof(rangeStr), |
2953 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2954 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2955 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2956 | | |
2957 | | const char *pszErrorMsg = &asCurlErrors[iReq].szCurlErrBuf[0]; |
2958 | | CPLDebug(poFS->GetDebugKey(), |
2959 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
2960 | | osURL.c_str(), rangeStr, |
2961 | | static_cast<int>(response_code), pszErrorMsg); |
2962 | | } |
2963 | | |
2964 | | if ((response_code != 206 && response_code != 225) || |
2965 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
2966 | | asWriteFuncHeaderData[iReq].nStartOffset + |
2967 | | asWriteFuncData[iReq].nSize) |
2968 | | { |
2969 | | char rangeStr[512] = {}; |
2970 | | snprintf(rangeStr, sizeof(rangeStr), |
2971 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2972 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2973 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2974 | | |
2975 | | // Look if we should attempt a retry |
2976 | | if (asMergedRequests[iReq].retryContext.CanRetry( |
2977 | | static_cast<int>(response_code), |
2978 | | asWriteFuncData[iReq].pBuffer, |
2979 | | &asCurlErrors[iReq].szCurlErrBuf[0])) |
2980 | | { |
2981 | | CPLError( |
2982 | | CE_Warning, CPLE_AppDefined, |
2983 | | "HTTP error code for %s range %s: %d. " |
2984 | | "Retrying again in %.1f secs", |
2985 | | osURL.c_str(), rangeStr, |
2986 | | static_cast<int>(response_code), |
2987 | | asMergedRequests[iReq].retryContext.GetCurrentDelay()); |
2988 | | dfMaxDelay = std::max( |
2989 | | dfMaxDelay, |
2990 | | asMergedRequests[iReq].retryContext.GetCurrentDelay()); |
2991 | | asMergedRequests[iReq].bToRetry = true; |
2992 | | bRetry = true; |
2993 | | } |
2994 | | else |
2995 | | { |
2996 | | CPLError(CE_Failure, CPLE_AppDefined, |
2997 | | "Request for %s failed with response_code=%ld", |
2998 | | rangeStr, response_code); |
2999 | | nRet = -1; |
3000 | | } |
3001 | | } |
3002 | | else if (nRet == 0) |
3003 | | { |
3004 | | size_t nOffset = 0; |
3005 | | size_t nRemainingSize = asWriteFuncData[iReq].nSize; |
3006 | | nTotalDownloaded += nRemainingSize; |
3007 | | for (int iRange = asMergedRequests[iReq].iFirstRange; |
3008 | | iRange <= asMergedRequests[iReq].iLastRange; iRange++) |
3009 | | { |
3010 | | if (nRemainingSize < anSortedSizes[iRange]) |
3011 | | { |
3012 | | nRet = -1; |
3013 | | break; |
3014 | | } |
3015 | | |
3016 | | if (anSortedSizes[iRange] > 0) |
3017 | | { |
3018 | | memcpy(apSortedData[iRange], |
3019 | | asWriteFuncData[iReq].pBuffer + nOffset, |
3020 | | anSortedSizes[iRange]); |
3021 | | } |
3022 | | nOffset += anSortedSizes[iRange]; |
3023 | | nRemainingSize -= anSortedSizes[iRange]; |
3024 | | } |
3025 | | } |
3026 | | |
3027 | | curl_multi_remove_handle(hMultiHandle, aHandles[iReq]); |
3028 | | VSICURLResetHeaderAndWriterFunctions(aHandles[iReq]); |
3029 | | curl_easy_cleanup(aHandles[iReq]); |
3030 | | CPLFree(apszRanges[iReq]); |
3031 | | CPLFree(asWriteFuncData[iReq].pBuffer); |
3032 | | CPLFree(asWriteFuncHeaderData[iReq].pBuffer); |
3033 | | if (aHeaders[iReq]) |
3034 | | curl_slist_free_all(aHeaders[iReq]); |
3035 | | } |
3036 | | |
3037 | | if (!bRetry || nRet != 0) |
3038 | | break; |
3039 | | CPLSleep(dfMaxDelay); |
3040 | | } |
3041 | | |
3042 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
3043 | | |
3044 | | if constexpr (ENABLE_DEBUG) |
3045 | | { |
3046 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
3047 | | } |
3048 | | |
3049 | | return nRet; |
3050 | | } |
3051 | | |
3052 | | /************************************************************************/ |
3053 | | /* ReadMultiRangeSingleGet() */ |
3054 | | /************************************************************************/ |
3055 | | |
3056 | | // TODO: the interest of this mode is rather dubious now. We could probably |
3057 | | // remove it |
3058 | | int VSICurlHandle::ReadMultiRangeSingleGet(int const nRanges, |
3059 | | void **const ppData, |
3060 | | const vsi_l_offset *const panOffsets, |
3061 | | const size_t *const panSizes) |
3062 | | { |
3063 | | std::string osRanges; |
3064 | | std::string osFirstRange; |
3065 | | std::string osLastRange; |
3066 | | int nMergedRanges = 0; |
3067 | | vsi_l_offset nTotalReqSize = 0; |
3068 | | for (int i = 0; i < nRanges; i++) |
3069 | | { |
3070 | | std::string osCurRange; |
3071 | | if (i != 0) |
3072 | | osRanges.append(","); |
3073 | | osCurRange = CPLSPrintf(CPL_FRMT_GUIB "-", panOffsets[i]); |
3074 | | while (i + 1 < nRanges && |
3075 | | panOffsets[i] + panSizes[i] == panOffsets[i + 1]) |
3076 | | { |
3077 | | nTotalReqSize += panSizes[i]; |
3078 | | i++; |
3079 | | } |
3080 | | nTotalReqSize += panSizes[i]; |
3081 | | osCurRange.append( |
3082 | | CPLSPrintf(CPL_FRMT_GUIB, panOffsets[i] + panSizes[i] - 1)); |
3083 | | nMergedRanges++; |
3084 | | |
3085 | | osRanges += osCurRange; |
3086 | | |
3087 | | if (nMergedRanges == 1) |
3088 | | osFirstRange = osCurRange; |
3089 | | osLastRange = std::move(osCurRange); |
3090 | | } |
3091 | | |
3092 | | const char *pszMaxRanges = |
3093 | | CPLGetConfigOption("CPL_VSIL_CURL_MAX_RANGES", "250"); |
3094 | | int nMaxRanges = atoi(pszMaxRanges); |
3095 | | if (nMaxRanges <= 0) |
3096 | | nMaxRanges = 250; |
3097 | | if (nMergedRanges > nMaxRanges) |
3098 | | { |
3099 | | const int nHalf = nRanges / 2; |
3100 | | const int nRet = ReadMultiRange(nHalf, ppData, panOffsets, panSizes); |
3101 | | if (nRet != 0) |
3102 | | return nRet; |
3103 | | return ReadMultiRange(nRanges - nHalf, ppData + nHalf, |
3104 | | panOffsets + nHalf, panSizes + nHalf); |
3105 | | } |
3106 | | |
3107 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
3108 | | CURL *hCurlHandle = curl_easy_init(); |
3109 | | |
3110 | | struct curl_slist *headers = |
3111 | | VSICurlSetOptions(hCurlHandle, m_pszURL, m_aosHTTPOptions.List()); |
3112 | | |
3113 | | WriteFuncStruct sWriteFuncData; |
3114 | | WriteFuncStruct sWriteFuncHeaderData; |
3115 | | |
3116 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
3117 | | pReadCbkUserData); |
3118 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
3119 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3120 | | VSICurlHandleWriteFunc); |
3121 | | |
3122 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
3123 | | nullptr); |
3124 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3125 | | &sWriteFuncHeaderData); |
3126 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3127 | | VSICurlHandleWriteFunc); |
3128 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3129 | | sWriteFuncHeaderData.bMultiRange = nMergedRanges > 1; |
3130 | | if (nMergedRanges == 1) |
3131 | | { |
3132 | | sWriteFuncHeaderData.nStartOffset = panOffsets[0]; |
3133 | | sWriteFuncHeaderData.nEndOffset = panOffsets[0] + nTotalReqSize - 1; |
3134 | | } |
3135 | | |
3136 | | if constexpr (ENABLE_DEBUG) |
3137 | | { |
3138 | | if (nMergedRanges == 1) |
3139 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
3140 | | osRanges.c_str(), m_pszURL); |
3141 | | else |
3142 | | CPLDebug(poFS->GetDebugKey(), |
3143 | | "Downloading %s, ..., %s (" CPL_FRMT_GUIB " bytes, %s)...", |
3144 | | osFirstRange.c_str(), osLastRange.c_str(), |
3145 | | static_cast<GUIntBig>(nTotalReqSize), m_pszURL); |
3146 | | } |
3147 | | |
3148 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, osRanges.c_str()); |
3149 | | |
3150 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
3151 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
3152 | | |
3153 | | headers = GetCurlHeaders("GET", headers); |
3154 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
3155 | | |
3156 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
3157 | | |
3158 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
3159 | | |
3160 | | curl_slist_free_all(headers); |
3161 | | |
3162 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
3163 | | |
3164 | | if (sWriteFuncData.bInterrupted) |
3165 | | { |
3166 | | bInterrupted = true; |
3167 | | |
3168 | | CPLFree(sWriteFuncData.pBuffer); |
3169 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3170 | | curl_easy_cleanup(hCurlHandle); |
3171 | | |
3172 | | return -1; |
3173 | | } |
3174 | | |
3175 | | long response_code = 0; |
3176 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
3177 | | |
3178 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
3179 | | response_code != 226 && response_code != 426) || |
3180 | | sWriteFuncHeaderData.bError) |
3181 | | { |
3182 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
3183 | | { |
3184 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
3185 | | CPLError( |
3186 | | CE_Failure, CPLE_AppDefined, |
3187 | | "%d: %s, Range downloading not supported by this server!", |
3188 | | static_cast<int>(response_code), szCurlErrBuf); |
3189 | | else |
3190 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
3191 | | static_cast<int>(response_code), szCurlErrBuf); |
3192 | | } |
3193 | | /* |
3194 | | if( !bHasComputedFileSize && startOffset == 0 ) |
3195 | | { |
3196 | | cachedFileProp->bHasComputedFileSize = bHasComputedFileSize = true; |
3197 | | cachedFileProp->fileSize = fileSize = 0; |
3198 | | cachedFileProp->eExists = eExists = EXIST_NO; |
3199 | | } |
3200 | | */ |
3201 | | CPLFree(sWriteFuncData.pBuffer); |
3202 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3203 | | curl_easy_cleanup(hCurlHandle); |
3204 | | return -1; |
3205 | | } |
3206 | | |
3207 | | char *pBuffer = sWriteFuncData.pBuffer; |
3208 | | size_t nSize = sWriteFuncData.nSize; |
3209 | | |
3210 | | // TODO(schwehr): Localize after removing gotos. |
3211 | | int nRet = -1; |
3212 | | char *pszBoundary; |
3213 | | std::string osBoundary; |
3214 | | char *pszNext = nullptr; |
3215 | | int iRange = 0; |
3216 | | int iPart = 0; |
3217 | | char *pszEOL = nullptr; |
3218 | | |
3219 | | /* -------------------------------------------------------------------- */ |
3220 | | /* No multipart if a single range has been requested */ |
3221 | | /* -------------------------------------------------------------------- */ |
3222 | | |
3223 | | if (nMergedRanges == 1) |
3224 | | { |
3225 | | size_t nAccSize = 0; |
3226 | | if (static_cast<vsi_l_offset>(nSize) < nTotalReqSize) |
3227 | | goto end; |
3228 | | |
3229 | | for (int i = 0; i < nRanges; i++) |
3230 | | { |
3231 | | memcpy(ppData[i], pBuffer + nAccSize, panSizes[i]); |
3232 | | nAccSize += panSizes[i]; |
3233 | | } |
3234 | | |
3235 | | nRet = 0; |
3236 | | goto end; |
3237 | | } |
3238 | | |
3239 | | /* -------------------------------------------------------------------- */ |
3240 | | /* Extract boundary name */ |
3241 | | /* -------------------------------------------------------------------- */ |
3242 | | |
3243 | | pszBoundary = strstr(sWriteFuncHeaderData.pBuffer, |
3244 | | "Content-Type: multipart/byteranges; boundary="); |
3245 | | if (pszBoundary == nullptr) |
3246 | | { |
3247 | | CPLError(CE_Failure, CPLE_AppDefined, "Could not find '%s'", |
3248 | | "Content-Type: multipart/byteranges; boundary="); |
3249 | | goto end; |
3250 | | } |
3251 | | |
3252 | | pszBoundary += strlen("Content-Type: multipart/byteranges; boundary="); |
3253 | | |
3254 | | pszEOL = strchr(pszBoundary, '\r'); |
3255 | | if (pszEOL) |
3256 | | *pszEOL = 0; |
3257 | | pszEOL = strchr(pszBoundary, '\n'); |
3258 | | if (pszEOL) |
3259 | | *pszEOL = 0; |
3260 | | |
3261 | | /* Remove optional double-quote character around boundary name */ |
3262 | | if (pszBoundary[0] == '"') |
3263 | | { |
3264 | | pszBoundary++; |
3265 | | char *pszLastDoubleQuote = strrchr(pszBoundary, '"'); |
3266 | | if (pszLastDoubleQuote) |
3267 | | *pszLastDoubleQuote = 0; |
3268 | | } |
3269 | | |
3270 | | osBoundary = "--"; |
3271 | | osBoundary += pszBoundary; |
3272 | | |
3273 | | /* -------------------------------------------------------------------- */ |
3274 | | /* Find the start of the first chunk. */ |
3275 | | /* -------------------------------------------------------------------- */ |
3276 | | pszNext = strstr(pBuffer, osBoundary.c_str()); |
3277 | | if (pszNext == nullptr) |
3278 | | { |
3279 | | CPLError(CE_Failure, CPLE_AppDefined, "No parts found."); |
3280 | | goto end; |
3281 | | } |
3282 | | |
3283 | | pszNext += osBoundary.size(); |
3284 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
3285 | | pszNext++; |
3286 | | if (*pszNext == '\r') |
3287 | | pszNext++; |
3288 | | if (*pszNext == '\n') |
3289 | | pszNext++; |
3290 | | |
3291 | | /* -------------------------------------------------------------------- */ |
3292 | | /* Loop over parts... */ |
3293 | | /* -------------------------------------------------------------------- */ |
3294 | | while (iPart < nRanges) |
3295 | | { |
3296 | | /* -------------------------------------------------------------------- |
3297 | | */ |
3298 | | /* Collect headers. */ |
3299 | | /* -------------------------------------------------------------------- |
3300 | | */ |
3301 | | bool bExpectedRange = false; |
3302 | | |
3303 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
3304 | | { |
3305 | | pszEOL = strstr(pszNext, "\n"); |
3306 | | |
3307 | | if (pszEOL == nullptr) |
3308 | | { |
3309 | | CPLError(CE_Failure, CPLE_AppDefined, |
3310 | | "Error while parsing multipart content (at line %d)", |
3311 | | __LINE__); |
3312 | | goto end; |
3313 | | } |
3314 | | |
3315 | | *pszEOL = '\0'; |
3316 | | bool bRestoreAntislashR = false; |
3317 | | if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r') |
3318 | | { |
3319 | | bRestoreAntislashR = true; |
3320 | | pszEOL[-1] = '\0'; |
3321 | | } |
3322 | | |
3323 | | if (STARTS_WITH_CI(pszNext, "Content-Range: bytes ")) |
3324 | | { |
3325 | | bExpectedRange = true; /* FIXME */ |
3326 | | } |
3327 | | |
3328 | | if (bRestoreAntislashR) |
3329 | | pszEOL[-1] = '\r'; |
3330 | | *pszEOL = '\n'; |
3331 | | |
3332 | | pszNext = pszEOL + 1; |
3333 | | } |
3334 | | |
3335 | | if (!bExpectedRange) |
3336 | | { |
3337 | | CPLError(CE_Failure, CPLE_AppDefined, |
3338 | | "Error while parsing multipart content (at line %d)", |
3339 | | __LINE__); |
3340 | | goto end; |
3341 | | } |
3342 | | |
3343 | | if (*pszNext == '\r') |
3344 | | pszNext++; |
3345 | | if (*pszNext == '\n') |
3346 | | pszNext++; |
3347 | | |
3348 | | /* -------------------------------------------------------------------- |
3349 | | */ |
3350 | | /* Work out the data block size. */ |
3351 | | /* -------------------------------------------------------------------- |
3352 | | */ |
3353 | | size_t nBytesAvail = nSize - (pszNext - pBuffer); |
3354 | | |
3355 | | while (true) |
3356 | | { |
3357 | | if (nBytesAvail < panSizes[iRange]) |
3358 | | { |
3359 | | CPLError(CE_Failure, CPLE_AppDefined, |
3360 | | "Error while parsing multipart content (at line %d)", |
3361 | | __LINE__); |
3362 | | goto end; |
3363 | | } |
3364 | | |
3365 | | memcpy(ppData[iRange], pszNext, panSizes[iRange]); |
3366 | | pszNext += panSizes[iRange]; |
3367 | | nBytesAvail -= panSizes[iRange]; |
3368 | | if (iRange + 1 < nRanges && |
3369 | | panOffsets[iRange] + panSizes[iRange] == panOffsets[iRange + 1]) |
3370 | | { |
3371 | | iRange++; |
3372 | | } |
3373 | | else |
3374 | | { |
3375 | | break; |
3376 | | } |
3377 | | } |
3378 | | |
3379 | | iPart++; |
3380 | | iRange++; |
3381 | | |
3382 | | while (nBytesAvail > 0 && |
3383 | | (*pszNext != '-' || |
3384 | | strncmp(pszNext, osBoundary.c_str(), osBoundary.size()) != 0)) |
3385 | | { |
3386 | | pszNext++; |
3387 | | nBytesAvail--; |
3388 | | } |
3389 | | |
3390 | | if (nBytesAvail == 0) |
3391 | | { |
3392 | | CPLError(CE_Failure, CPLE_AppDefined, |
3393 | | "Error while parsing multipart content (at line %d)", |
3394 | | __LINE__); |
3395 | | goto end; |
3396 | | } |
3397 | | |
3398 | | pszNext += osBoundary.size(); |
3399 | | if (STARTS_WITH(pszNext, "--")) |
3400 | | { |
3401 | | // End of multipart. |
3402 | | break; |
3403 | | } |
3404 | | |
3405 | | if (*pszNext == '\r') |
3406 | | pszNext++; |
3407 | | if (*pszNext == '\n') |
3408 | | pszNext++; |
3409 | | else |
3410 | | { |
3411 | | CPLError(CE_Failure, CPLE_AppDefined, |
3412 | | "Error while parsing multipart content (at line %d)", |
3413 | | __LINE__); |
3414 | | goto end; |
3415 | | } |
3416 | | } |
3417 | | |
3418 | | if (iPart == nMergedRanges) |
3419 | | nRet = 0; |
3420 | | else |
3421 | | CPLError(CE_Failure, CPLE_AppDefined, |
3422 | | "Got only %d parts, where %d were expected", iPart, |
3423 | | nMergedRanges); |
3424 | | |
3425 | | end: |
3426 | | CPLFree(sWriteFuncData.pBuffer); |
3427 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3428 | | curl_easy_cleanup(hCurlHandle); |
3429 | | |
3430 | | return nRet; |
3431 | | } |
3432 | | |
3433 | | /************************************************************************/ |
3434 | | /* PRead() */ |
3435 | | /************************************************************************/ |
3436 | | |
3437 | | size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, |
3438 | | vsi_l_offset nOffset) const |
3439 | | { |
3440 | | // Try to use AdviseRead ranges fetched asynchronously |
3441 | | if (!m_aoAdviseReadRanges.empty()) |
3442 | | { |
3443 | | for (auto &poRange : m_aoAdviseReadRanges) |
3444 | | { |
3445 | | if (nOffset >= poRange->nStartOffset && |
3446 | | nOffset + nSize <= poRange->nStartOffset + poRange->nSize) |
3447 | | { |
3448 | | { |
3449 | | std::unique_lock<std::mutex> oLock(poRange->oMutex); |
3450 | | // coverity[missing_lock:FALSE] |
3451 | | while (!poRange->bDone) |
3452 | | { |
3453 | | poRange->oCV.wait(oLock); |
3454 | | } |
3455 | | } |
3456 | | if (poRange->abyData.empty()) |
3457 | | return 0; |
3458 | | |
3459 | | auto nEndOffset = |
3460 | | poRange->nStartOffset + poRange->abyData.size(); |
3461 | | if (nOffset >= nEndOffset) |
3462 | | return 0; |
3463 | | const size_t nToCopy = static_cast<size_t>( |
3464 | | std::min<vsi_l_offset>(nSize, nEndOffset - nOffset)); |
3465 | | memcpy(pBuffer, |
3466 | | poRange->abyData.data() + |
3467 | | static_cast<size_t>(nOffset - poRange->nStartOffset), |
3468 | | nToCopy); |
3469 | | return nToCopy; |
3470 | | } |
3471 | | } |
3472 | | } |
3473 | | |
3474 | | // poFS has a global mutex |
3475 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
3476 | | if (oFileProp.eExists == EXIST_NO) |
3477 | | return static_cast<size_t>(-1); |
3478 | | |
3479 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3480 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3481 | | NetworkStatisticsAction oContextAction("PRead"); |
3482 | | |
3483 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3484 | | std::string osURL; |
3485 | | { |
3486 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3487 | | UpdateQueryString(); |
3488 | | bool bHasExpired; |
3489 | | osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions); |
3490 | | } |
3491 | | |
3492 | | CURL *hCurlHandle = curl_easy_init(); |
3493 | | |
3494 | | struct curl_slist *headers = |
3495 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3496 | | |
3497 | | WriteFuncStruct sWriteFuncData; |
3498 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
3499 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
3500 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3501 | | VSICurlHandleWriteFunc); |
3502 | | |
3503 | | WriteFuncStruct sWriteFuncHeaderData; |
3504 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
3505 | | nullptr); |
3506 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3507 | | &sWriteFuncHeaderData); |
3508 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3509 | | VSICurlHandleWriteFunc); |
3510 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3511 | | sWriteFuncHeaderData.nStartOffset = nOffset; |
3512 | | |
3513 | | sWriteFuncHeaderData.nEndOffset = nOffset + nSize - 1; |
3514 | | |
3515 | | char rangeStr[512] = {}; |
3516 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3517 | | sWriteFuncHeaderData.nStartOffset, |
3518 | | sWriteFuncHeaderData.nEndOffset); |
3519 | | |
3520 | | if constexpr (ENABLE_DEBUG) |
3521 | | { |
3522 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
3523 | | osURL.c_str()); |
3524 | | } |
3525 | | |
3526 | | std::string osHeaderRange; |
3527 | | if (sWriteFuncHeaderData.bIsHTTP) |
3528 | | { |
3529 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
3530 | | // So it gets included in Azure signature |
3531 | | headers = curl_slist_append(headers, osHeaderRange.data()); |
3532 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
3533 | | } |
3534 | | else |
3535 | | { |
3536 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
3537 | | } |
3538 | | |
3539 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3540 | | szCurlErrBuf[0] = '\0'; |
3541 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3542 | | &szCurlErrBuf[0]); |
3543 | | |
3544 | | { |
3545 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3546 | | headers = |
3547 | | const_cast<VSICurlHandle *>(this)->GetCurlHeaders("GET", headers); |
3548 | | } |
3549 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
3550 | | |
3551 | | CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); |
3552 | | VSICURLMultiPerform(hMultiHandle, hCurlHandle, &m_bInterrupt); |
3553 | | |
3554 | | { |
3555 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3556 | | const_cast<VSICurlHandle *>(this)->UpdateRedirectInfo( |
3557 | | hCurlHandle, sWriteFuncHeaderData); |
3558 | | } |
3559 | | |
3560 | | long response_code = 0; |
3561 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
3562 | | |
3563 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
3564 | | { |
3565 | | const char *pszErrorMsg = &szCurlErrBuf[0]; |
3566 | | CPLDebug(poFS->GetDebugKey(), "PRead(%s), %s: response_code=%d, msg=%s", |
3567 | | osURL.c_str(), rangeStr, static_cast<int>(response_code), |
3568 | | pszErrorMsg); |
3569 | | } |
3570 | | |
3571 | | size_t nRet; |
3572 | | if ((response_code != 206 && response_code != 225) || |
3573 | | sWriteFuncData.nSize == 0) |
3574 | | { |
3575 | | if (!m_bInterrupt) |
3576 | | { |
3577 | | CPLDebug(poFS->GetDebugKey(), |
3578 | | "Request for %s failed with response_code=%ld", rangeStr, |
3579 | | response_code); |
3580 | | } |
3581 | | nRet = static_cast<size_t>(-1); |
3582 | | } |
3583 | | else |
3584 | | { |
3585 | | nRet = std::min(sWriteFuncData.nSize, nSize); |
3586 | | if (nRet > 0) |
3587 | | memcpy(pBuffer, sWriteFuncData.pBuffer, nRet); |
3588 | | } |
3589 | | |
3590 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
3591 | | curl_easy_cleanup(hCurlHandle); |
3592 | | CPLFree(sWriteFuncData.pBuffer); |
3593 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3594 | | curl_slist_free_all(headers); |
3595 | | |
3596 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
3597 | | |
3598 | | #if 0 |
3599 | | if( ENABLE_DEBUG ) |
3600 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
3601 | | #endif |
3602 | | |
3603 | | return nRet; |
3604 | | } |
3605 | | |
3606 | | /************************************************************************/ |
3607 | | /* GetAdviseReadTotalBytesLimit() */ |
3608 | | /************************************************************************/ |
3609 | | |
3610 | | size_t VSICurlHandle::GetAdviseReadTotalBytesLimit() const |
3611 | | { |
3612 | | return static_cast<size_t>(std::min<unsigned long long>( |
3613 | | std::numeric_limits<size_t>::max(), |
3614 | | // 100 MB |
3615 | | std::strtoull( |
3616 | | CPLGetConfigOption("CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT", |
3617 | | "104857600"), |
3618 | | nullptr, 10))); |
3619 | | } |
3620 | | |
3621 | | /************************************************************************/ |
3622 | | /* VSICURLMultiInit() */ |
3623 | | /************************************************************************/ |
3624 | | |
3625 | | static CURLM *VSICURLMultiInit() |
3626 | | { |
3627 | | CURLM *hCurlMultiHandle = curl_multi_init(); |
3628 | | |
3629 | | if (const char *pszMAXCONNECTS = |
3630 | | CPLGetConfigOption("GDAL_HTTP_MAX_CACHED_CONNECTIONS", nullptr)) |
3631 | | { |
3632 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAXCONNECTS, |
3633 | | atoi(pszMAXCONNECTS)); |
3634 | | } |
3635 | | |
3636 | | if (const char *pszMAX_TOTAL_CONNECTIONS = |
3637 | | CPLGetConfigOption("GDAL_HTTP_MAX_TOTAL_CONNECTIONS", nullptr)) |
3638 | | { |
3639 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, |
3640 | | atoi(pszMAX_TOTAL_CONNECTIONS)); |
3641 | | } |
3642 | | |
3643 | | return hCurlMultiHandle; |
3644 | | } |
3645 | | |
3646 | | /************************************************************************/ |
3647 | | /* AdviseRead() */ |
3648 | | /************************************************************************/ |
3649 | | |
3650 | | void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, |
3651 | | const size_t *panSizes) |
3652 | | { |
3653 | | if (!CPLTestBool( |
3654 | | CPLGetConfigOption("GDAL_HTTP_ENABLE_ADVISE_READ", "TRUE"))) |
3655 | | return; |
3656 | | |
3657 | | if (m_oThreadAdviseRead.joinable()) |
3658 | | { |
3659 | | m_oThreadAdviseRead.join(); |
3660 | | } |
3661 | | |
3662 | | // Give up if we need to allocate too much memory |
3663 | | vsi_l_offset nMaxSize = 0; |
3664 | | const size_t nLimit = GetAdviseReadTotalBytesLimit(); |
3665 | | for (int i = 0; i < nRanges; ++i) |
3666 | | { |
3667 | | if (panSizes[i] > nLimit - nMaxSize) |
3668 | | { |
3669 | | CPLDebug(poFS->GetDebugKey(), |
3670 | | "Trying to request too many bytes in AdviseRead()"); |
3671 | | return; |
3672 | | } |
3673 | | nMaxSize += panSizes[i]; |
3674 | | } |
3675 | | |
3676 | | UpdateQueryString(); |
3677 | | |
3678 | | bool bHasExpired = false; |
3679 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3680 | | const std::string l_osURL( |
3681 | | GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
3682 | | if (bHasExpired) |
3683 | | { |
3684 | | return; |
3685 | | } |
3686 | | |
3687 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
3688 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
3689 | | |
3690 | | try |
3691 | | { |
3692 | | m_aoAdviseReadRanges.clear(); |
3693 | | m_aoAdviseReadRanges.reserve(nRanges); |
3694 | | for (int i = 0; i < nRanges;) |
3695 | | { |
3696 | | int iNext = i; |
3697 | | // Identify consecutive ranges |
3698 | | constexpr size_t SIZE_COG_MARKERS = 2 * sizeof(uint32_t); |
3699 | | auto nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3700 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
3701 | | panOffsets[iNext + 1] > panOffsets[iNext] && |
3702 | | panOffsets[iNext] + panSizes[iNext] + SIZE_COG_MARKERS >= |
3703 | | panOffsets[iNext + 1] && |
3704 | | panOffsets[iNext + 1] + panSizes[iNext + 1] > nEndOffset) |
3705 | | { |
3706 | | iNext++; |
3707 | | nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3708 | | } |
3709 | | CPLAssert(panOffsets[i] <= nEndOffset); |
3710 | | const size_t nSize = |
3711 | | static_cast<size_t>(nEndOffset - panOffsets[i]); |
3712 | | |
3713 | | if (nSize == 0) |
3714 | | { |
3715 | | i = iNext + 1; |
3716 | | continue; |
3717 | | } |
3718 | | |
3719 | | auto newAdviseReadRange = |
3720 | | std::make_unique<AdviseReadRange>(m_oRetryParameters); |
3721 | | newAdviseReadRange->nStartOffset = panOffsets[i]; |
3722 | | newAdviseReadRange->nSize = nSize; |
3723 | | newAdviseReadRange->abyData.resize(nSize); |
3724 | | m_aoAdviseReadRanges.push_back(std::move(newAdviseReadRange)); |
3725 | | |
3726 | | i = iNext + 1; |
3727 | | } |
3728 | | } |
3729 | | catch (const std::exception &) |
3730 | | { |
3731 | | CPLError(CE_Failure, CPLE_OutOfMemory, |
3732 | | "Out of memory in VSICurlHandle::AdviseRead()"); |
3733 | | m_aoAdviseReadRanges.clear(); |
3734 | | } |
3735 | | |
3736 | | if (m_aoAdviseReadRanges.empty()) |
3737 | | return; |
3738 | | |
3739 | | #ifdef DEBUG |
3740 | | CPLDebug(poFS->GetDebugKey(), "AdviseRead(): fetching %u ranges", |
3741 | | static_cast<unsigned>(m_aoAdviseReadRanges.size())); |
3742 | | #endif |
3743 | | |
3744 | | const auto task = [this, aosHTTPOptions = std::move(aosHTTPOptions)]( |
3745 | | const std::string &osURL) |
3746 | | { |
3747 | | if (!m_hCurlMultiHandleForAdviseRead) |
3748 | | m_hCurlMultiHandleForAdviseRead = VSICURLMultiInit(); |
3749 | | |
3750 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3751 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3752 | | NetworkStatisticsAction oContextAction("AdviseRead"); |
3753 | | |
3754 | | #ifdef CURLPIPE_MULTIPLEX |
3755 | | // Enable HTTP/2 multiplexing (ignored if an older version of HTTP is |
3756 | | // used) |
3757 | | // Not that this does not enable HTTP/1.1 pipeling, which is not |
3758 | | // recommended for example by Google Cloud Storage. |
3759 | | // For HTTP/1.1, parallel connections work better since you can get |
3760 | | // results out of order. |
3761 | | if (CPLTestBool(CPLGetConfigOption("GDAL_HTTP_MULTIPLEX", "YES"))) |
3762 | | { |
3763 | | curl_multi_setopt(m_hCurlMultiHandleForAdviseRead, |
3764 | | CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); |
3765 | | } |
3766 | | #endif |
3767 | | |
3768 | | size_t nTotalDownloaded = 0; |
3769 | | |
3770 | | while (true) |
3771 | | { |
3772 | | |
3773 | | std::vector<CURL *> aHandles; |
3774 | | std::vector<WriteFuncStruct> asWriteFuncData( |
3775 | | m_aoAdviseReadRanges.size()); |
3776 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData( |
3777 | | m_aoAdviseReadRanges.size()); |
3778 | | std::vector<char *> apszRanges; |
3779 | | std::vector<struct curl_slist *> aHeaders; |
3780 | | |
3781 | | struct CurlErrBuffer |
3782 | | { |
3783 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3784 | | }; |
3785 | | std::vector<CurlErrBuffer> asCurlErrors( |
3786 | | m_aoAdviseReadRanges.size()); |
3787 | | |
3788 | | std::map<CURL *, size_t> oMapHandleToIdx; |
3789 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
3790 | | { |
3791 | | if (!m_aoAdviseReadRanges[i]->bToRetry) |
3792 | | { |
3793 | | aHandles.push_back(nullptr); |
3794 | | apszRanges.push_back(nullptr); |
3795 | | aHeaders.push_back(nullptr); |
3796 | | continue; |
3797 | | } |
3798 | | m_aoAdviseReadRanges[i]->bToRetry = false; |
3799 | | |
3800 | | CURL *hCurlHandle = curl_easy_init(); |
3801 | | oMapHandleToIdx[hCurlHandle] = i; |
3802 | | aHandles.push_back(hCurlHandle); |
3803 | | |
3804 | | // As the multi-range request is likely not the first one, we don't |
3805 | | // need to wait as we already know if pipelining is possible |
3806 | | // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); |
3807 | | |
3808 | | struct curl_slist *headers = VSICurlSetOptions( |
3809 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3810 | | |
3811 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[i], this, |
3812 | | pfnReadCbk, pReadCbkUserData); |
3813 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
3814 | | &asWriteFuncData[i]); |
3815 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3816 | | VSICurlHandleWriteFunc); |
3817 | | |
3818 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[i], nullptr, |
3819 | | nullptr, nullptr); |
3820 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3821 | | &asWriteFuncHeaderData[i]); |
3822 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3823 | | VSICurlHandleWriteFunc); |
3824 | | asWriteFuncHeaderData[i].bIsHTTP = |
3825 | | STARTS_WITH(m_pszURL, "http"); |
3826 | | asWriteFuncHeaderData[i].nStartOffset = |
3827 | | m_aoAdviseReadRanges[i]->nStartOffset; |
3828 | | |
3829 | | asWriteFuncHeaderData[i].nEndOffset = |
3830 | | m_aoAdviseReadRanges[i]->nStartOffset + |
3831 | | m_aoAdviseReadRanges[i]->nSize - 1; |
3832 | | |
3833 | | char rangeStr[512] = {}; |
3834 | | snprintf(rangeStr, sizeof(rangeStr), |
3835 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3836 | | asWriteFuncHeaderData[i].nStartOffset, |
3837 | | asWriteFuncHeaderData[i].nEndOffset); |
3838 | | |
3839 | | if constexpr (ENABLE_DEBUG) |
3840 | | { |
3841 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
3842 | | rangeStr, osURL.c_str()); |
3843 | | } |
3844 | | |
3845 | | if (asWriteFuncHeaderData[i].bIsHTTP) |
3846 | | { |
3847 | | std::string osHeaderRange( |
3848 | | CPLSPrintf("Range: bytes=%s", rangeStr)); |
3849 | | // So it gets included in Azure signature |
3850 | | char *pszRange = CPLStrdup(osHeaderRange.c_str()); |
3851 | | apszRanges.push_back(pszRange); |
3852 | | headers = curl_slist_append(headers, pszRange); |
3853 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3854 | | nullptr); |
3855 | | } |
3856 | | else |
3857 | | { |
3858 | | apszRanges.push_back(nullptr); |
3859 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3860 | | rangeStr); |
3861 | | } |
3862 | | |
3863 | | asCurlErrors[i].szCurlErrBuf[0] = '\0'; |
3864 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3865 | | &asCurlErrors[i].szCurlErrBuf[0]); |
3866 | | |
3867 | | headers = GetCurlHeaders("GET", headers); |
3868 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
3869 | | headers); |
3870 | | aHeaders.push_back(headers); |
3871 | | curl_multi_add_handle(m_hCurlMultiHandleForAdviseRead, |
3872 | | hCurlHandle); |
3873 | | } |
3874 | | |
3875 | | const auto DealWithRequest = [this, &osURL, &nTotalDownloaded, |
3876 | | &oMapHandleToIdx, &asCurlErrors, |
3877 | | &asWriteFuncHeaderData, |
3878 | | &asWriteFuncData](CURL *hCurlHandle) |
3879 | | { |
3880 | | auto oIter = oMapHandleToIdx.find(hCurlHandle); |
3881 | | CPLAssert(oIter != oMapHandleToIdx.end()); |
3882 | | const auto iReq = oIter->second; |
3883 | | |
3884 | | long response_code = 0; |
3885 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, |
3886 | | &response_code); |
3887 | | |
3888 | | if (ENABLE_DEBUG && asCurlErrors[iReq].szCurlErrBuf[0] != '\0') |
3889 | | { |
3890 | | char rangeStr[512] = {}; |
3891 | | snprintf(rangeStr, sizeof(rangeStr), |
3892 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3893 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3894 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3895 | | |
3896 | | const char *pszErrorMsg = |
3897 | | &asCurlErrors[iReq].szCurlErrBuf[0]; |
3898 | | CPLDebug(poFS->GetDebugKey(), |
3899 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
3900 | | osURL.c_str(), rangeStr, |
3901 | | static_cast<int>(response_code), pszErrorMsg); |
3902 | | } |
3903 | | |
3904 | | bool bToRetry = false; |
3905 | | if ((response_code != 206 && response_code != 225) || |
3906 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
3907 | | asWriteFuncHeaderData[iReq].nStartOffset + |
3908 | | asWriteFuncData[iReq].nSize) |
3909 | | { |
3910 | | char rangeStr[512] = {}; |
3911 | | snprintf(rangeStr, sizeof(rangeStr), |
3912 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3913 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3914 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3915 | | |
3916 | | // Look if we should attempt a retry |
3917 | | if (m_aoAdviseReadRanges[iReq]->retryContext.CanRetry( |
3918 | | static_cast<int>(response_code), |
3919 | | asWriteFuncData[iReq].pBuffer, |
3920 | | &asCurlErrors[iReq].szCurlErrBuf[0])) |
3921 | | { |
3922 | | CPLError(CE_Warning, CPLE_AppDefined, |
3923 | | "HTTP error code for %s range %s: %d. " |
3924 | | "Retrying again in %.1f secs", |
3925 | | osURL.c_str(), rangeStr, |
3926 | | static_cast<int>(response_code), |
3927 | | m_aoAdviseReadRanges[iReq] |
3928 | | ->retryContext.GetCurrentDelay()); |
3929 | | m_aoAdviseReadRanges[iReq]->dfSleepDelay = |
3930 | | m_aoAdviseReadRanges[iReq] |
3931 | | ->retryContext.GetCurrentDelay(); |
3932 | | bToRetry = true; |
3933 | | } |
3934 | | else |
3935 | | { |
3936 | | CPLError(CE_Failure, CPLE_AppDefined, |
3937 | | "Request for %s range %s failed with " |
3938 | | "response_code=%ld", |
3939 | | osURL.c_str(), rangeStr, response_code); |
3940 | | } |
3941 | | } |
3942 | | else |
3943 | | { |
3944 | | const size_t nSize = asWriteFuncData[iReq].nSize; |
3945 | | memcpy(&m_aoAdviseReadRanges[iReq]->abyData[0], |
3946 | | asWriteFuncData[iReq].pBuffer, nSize); |
3947 | | m_aoAdviseReadRanges[iReq]->abyData.resize(nSize); |
3948 | | |
3949 | | nTotalDownloaded += nSize; |
3950 | | } |
3951 | | |
3952 | | m_aoAdviseReadRanges[iReq]->bToRetry = bToRetry; |
3953 | | |
3954 | | if (!bToRetry) |
3955 | | { |
3956 | | std::lock_guard<std::mutex> oLock( |
3957 | | m_aoAdviseReadRanges[iReq]->oMutex); |
3958 | | m_aoAdviseReadRanges[iReq]->bDone = true; |
3959 | | m_aoAdviseReadRanges[iReq]->oCV.notify_all(); |
3960 | | } |
3961 | | }; |
3962 | | |
3963 | | int repeats = 0; |
3964 | | |
3965 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
3966 | | while (true) |
3967 | | { |
3968 | | int still_running; |
3969 | | while (curl_multi_perform(m_hCurlMultiHandleForAdviseRead, |
3970 | | &still_running) == |
3971 | | CURLM_CALL_MULTI_PERFORM) |
3972 | | { |
3973 | | // loop |
3974 | | } |
3975 | | if (!still_running) |
3976 | | { |
3977 | | break; |
3978 | | } |
3979 | | |
3980 | | CURLMsg *msg; |
3981 | | do |
3982 | | { |
3983 | | int msgq = 0; |
3984 | | msg = curl_multi_info_read(m_hCurlMultiHandleForAdviseRead, |
3985 | | &msgq); |
3986 | | if (msg && (msg->msg == CURLMSG_DONE)) |
3987 | | { |
3988 | | DealWithRequest(msg->easy_handle); |
3989 | | } |
3990 | | } while (msg); |
3991 | | |
3992 | | CPLMultiPerformWait(m_hCurlMultiHandleForAdviseRead, repeats); |
3993 | | } |
3994 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
3995 | | |
3996 | | bool bRetry = false; |
3997 | | double dfDelay = 0.0; |
3998 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
3999 | | { |
4000 | | bool bReqDone; |
4001 | | { |
4002 | | // To please Coverity Scan |
4003 | | std::lock_guard<std::mutex> oLock( |
4004 | | m_aoAdviseReadRanges[i]->oMutex); |
4005 | | bReqDone = m_aoAdviseReadRanges[i]->bDone; |
4006 | | } |
4007 | | if (!bReqDone && !m_aoAdviseReadRanges[i]->bToRetry) |
4008 | | { |
4009 | | DealWithRequest(aHandles[i]); |
4010 | | } |
4011 | | if (m_aoAdviseReadRanges[i]->bToRetry) |
4012 | | dfDelay = std::max(dfDelay, |
4013 | | m_aoAdviseReadRanges[i]->dfSleepDelay); |
4014 | | bRetry = bRetry || m_aoAdviseReadRanges[i]->bToRetry; |
4015 | | if (aHandles[i]) |
4016 | | { |
4017 | | curl_multi_remove_handle(m_hCurlMultiHandleForAdviseRead, |
4018 | | aHandles[i]); |
4019 | | VSICURLResetHeaderAndWriterFunctions(aHandles[i]); |
4020 | | curl_easy_cleanup(aHandles[i]); |
4021 | | } |
4022 | | CPLFree(apszRanges[i]); |
4023 | | CPLFree(asWriteFuncData[i].pBuffer); |
4024 | | CPLFree(asWriteFuncHeaderData[i].pBuffer); |
4025 | | if (aHeaders[i]) |
4026 | | curl_slist_free_all(aHeaders[i]); |
4027 | | } |
4028 | | if (!bRetry) |
4029 | | break; |
4030 | | CPLSleep(dfDelay); |
4031 | | } |
4032 | | |
4033 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
4034 | | }; |
4035 | | |
4036 | | m_oThreadAdviseRead = std::thread(task, l_osURL); |
4037 | | } |
4038 | | |
4039 | | /************************************************************************/ |
4040 | | /* Write() */ |
4041 | | /************************************************************************/ |
4042 | | |
4043 | | size_t VSICurlHandle::Write(const void * /* pBuffer */, size_t /* nBytes */) |
4044 | | { |
4045 | | return 0; |
4046 | | } |
4047 | | |
4048 | | /************************************************************************/ |
4049 | | /* ClearErr() */ |
4050 | | /************************************************************************/ |
4051 | | |
4052 | | void VSICurlHandle::ClearErr() |
4053 | | |
4054 | | { |
4055 | | bEOF = false; |
4056 | | bError = false; |
4057 | | } |
4058 | | |
4059 | | /************************************************************************/ |
4060 | | /* Error() */ |
4061 | | /************************************************************************/ |
4062 | | |
4063 | | int VSICurlHandle::Error() |
4064 | | |
4065 | | { |
4066 | | return bError ? TRUE : FALSE; |
4067 | | } |
4068 | | |
4069 | | /************************************************************************/ |
4070 | | /* Eof() */ |
4071 | | /************************************************************************/ |
4072 | | |
4073 | | int VSICurlHandle::Eof() |
4074 | | |
4075 | | { |
4076 | | return bEOF ? TRUE : FALSE; |
4077 | | } |
4078 | | |
4079 | | /************************************************************************/ |
4080 | | /* Flush() */ |
4081 | | /************************************************************************/ |
4082 | | |
4083 | | int VSICurlHandle::Flush() |
4084 | | { |
4085 | | return 0; |
4086 | | } |
4087 | | |
4088 | | /************************************************************************/ |
4089 | | /* Close() */ |
4090 | | /************************************************************************/ |
4091 | | |
4092 | | int VSICurlHandle::Close() |
4093 | | { |
4094 | | return 0; |
4095 | | } |
4096 | | |
4097 | | /************************************************************************/ |
4098 | | /* VSICurlFilesystemHandlerBase() */ |
4099 | | /************************************************************************/ |
4100 | | |
4101 | | VSICurlFilesystemHandlerBase::VSICurlFilesystemHandlerBase() |
4102 | | : oCacheDirList{1024, 0} |
4103 | | { |
4104 | | } |
4105 | | |
4106 | | /************************************************************************/ |
4107 | | /* CachedConnection */ |
4108 | | /************************************************************************/ |
4109 | | |
4110 | | namespace |
4111 | | { |
4112 | | struct CachedConnection |
4113 | | { |
4114 | | CURLM *hCurlMultiHandle = nullptr; |
4115 | | void clear(); |
4116 | | |
4117 | | ~CachedConnection() |
4118 | | { |
4119 | | clear(); |
4120 | | } |
4121 | | }; |
4122 | | } // namespace |
4123 | | |
4124 | | #ifdef _WIN32 |
4125 | | // Currently thread_local and C++ objects don't work well with DLL on Windows |
4126 | | static void FreeCachedConnection(void *pData) |
4127 | | { |
4128 | | delete static_cast< |
4129 | | std::map<VSICurlFilesystemHandlerBase *, CachedConnection> *>(pData); |
4130 | | } |
4131 | | |
4132 | | // Per-thread and per-filesystem Curl connection cache. |
4133 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> & |
4134 | | GetConnectionCache() |
4135 | | { |
4136 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> |
4137 | | dummyCache; |
4138 | | int bMemoryErrorOccurred = false; |
4139 | | void *pData = |
4140 | | CPLGetTLSEx(CTLS_VSICURL_CACHEDCONNECTION, &bMemoryErrorOccurred); |
4141 | | if (bMemoryErrorOccurred) |
4142 | | { |
4143 | | return dummyCache; |
4144 | | } |
4145 | | if (pData == nullptr) |
4146 | | { |
4147 | | auto cachedConnection = |
4148 | | new std::map<VSICurlFilesystemHandlerBase *, CachedConnection>(); |
4149 | | CPLSetTLSWithFreeFuncEx(CTLS_VSICURL_CACHEDCONNECTION, cachedConnection, |
4150 | | FreeCachedConnection, &bMemoryErrorOccurred); |
4151 | | if (bMemoryErrorOccurred) |
4152 | | { |
4153 | | delete cachedConnection; |
4154 | | return dummyCache; |
4155 | | } |
4156 | | return *cachedConnection; |
4157 | | } |
4158 | | return *static_cast< |
4159 | | std::map<VSICurlFilesystemHandlerBase *, CachedConnection> *>(pData); |
4160 | | } |
4161 | | #else |
4162 | | static thread_local std::map<VSICurlFilesystemHandlerBase *, CachedConnection> |
4163 | | g_tls_connectionCache; |
4164 | | |
4165 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> & |
4166 | | GetConnectionCache() |
4167 | | { |
4168 | | return g_tls_connectionCache; |
4169 | | } |
4170 | | #endif |
4171 | | |
4172 | | /************************************************************************/ |
4173 | | /* clear() */ |
4174 | | /************************************************************************/ |
4175 | | |
4176 | | void CachedConnection::clear() |
4177 | | { |
4178 | | if (hCurlMultiHandle) |
4179 | | { |
4180 | | VSICURLMultiCleanup(hCurlMultiHandle); |
4181 | | hCurlMultiHandle = nullptr; |
4182 | | } |
4183 | | } |
4184 | | |
4185 | | /************************************************************************/ |
4186 | | /* ~VSICurlFilesystemHandlerBase() */ |
4187 | | /************************************************************************/ |
4188 | | |
4189 | | VSICurlFilesystemHandlerBase::~VSICurlFilesystemHandlerBase() |
4190 | | { |
4191 | | VSICurlFilesystemHandlerBase::ClearCache(); |
4192 | | GetConnectionCache().erase(this); |
4193 | | |
4194 | | if (hMutex != nullptr) |
4195 | | CPLDestroyMutex(hMutex); |
4196 | | hMutex = nullptr; |
4197 | | } |
4198 | | |
4199 | | /************************************************************************/ |
4200 | | /* AllowCachedDataFor() */ |
4201 | | /************************************************************************/ |
4202 | | |
4203 | | bool VSICurlFilesystemHandlerBase::AllowCachedDataFor(const char *pszFilename) |
4204 | | { |
4205 | | bool bCachedAllowed = true; |
4206 | | char **papszTokens = CSLTokenizeString2( |
4207 | | CPLGetConfigOption("CPL_VSIL_CURL_NON_CACHED", ""), ":", 0); |
4208 | | for (int i = 0; papszTokens && papszTokens[i]; i++) |
4209 | | { |
4210 | | if (STARTS_WITH(pszFilename, papszTokens[i])) |
4211 | | { |
4212 | | bCachedAllowed = false; |
4213 | | break; |
4214 | | } |
4215 | | } |
4216 | | CSLDestroy(papszTokens); |
4217 | | return bCachedAllowed; |
4218 | | } |
4219 | | |
4220 | | /************************************************************************/ |
4221 | | /* GetCurlMultiHandleFor() */ |
4222 | | /************************************************************************/ |
4223 | | |
4224 | | CURLM *VSICurlFilesystemHandlerBase::GetCurlMultiHandleFor( |
4225 | | const std::string & /*osURL*/) |
4226 | | { |
4227 | | auto &conn = GetConnectionCache()[this]; |
4228 | | if (conn.hCurlMultiHandle == nullptr) |
4229 | | { |
4230 | | conn.hCurlMultiHandle = VSICURLMultiInit(); |
4231 | | } |
4232 | | return conn.hCurlMultiHandle; |
4233 | | } |
4234 | | |
4235 | | /************************************************************************/ |
4236 | | /* GetRegionCache() */ |
4237 | | /************************************************************************/ |
4238 | | |
4239 | | VSICurlFilesystemHandlerBase::RegionCacheType * |
4240 | | VSICurlFilesystemHandlerBase::GetRegionCache() |
4241 | | { |
4242 | | // should be called under hMutex taken |
4243 | | if (m_poRegionCacheDoNotUseDirectly == nullptr) |
4244 | | { |
4245 | | m_poRegionCacheDoNotUseDirectly.reset( |
4246 | | new RegionCacheType(static_cast<size_t>(GetMaxRegions()))); |
4247 | | } |
4248 | | return m_poRegionCacheDoNotUseDirectly.get(); |
4249 | | } |
4250 | | |
4251 | | /************************************************************************/ |
4252 | | /* GetRegion() */ |
4253 | | /************************************************************************/ |
4254 | | |
4255 | | std::shared_ptr<std::string> |
4256 | | VSICurlFilesystemHandlerBase::GetRegion(const char *pszURL, |
4257 | | vsi_l_offset nFileOffsetStart) |
4258 | | { |
4259 | | CPLMutexHolder oHolder(&hMutex); |
4260 | | |
4261 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
4262 | | nFileOffsetStart = |
4263 | | (nFileOffsetStart / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
4264 | | |
4265 | | std::shared_ptr<std::string> out; |
4266 | | if (GetRegionCache()->tryGet( |
4267 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), out)) |
4268 | | { |
4269 | | return out; |
4270 | | } |
4271 | | |
4272 | | return nullptr; |
4273 | | } |
4274 | | |
4275 | | /************************************************************************/ |
4276 | | /* AddRegion() */ |
4277 | | /************************************************************************/ |
4278 | | |
4279 | | void VSICurlFilesystemHandlerBase::AddRegion(const char *pszURL, |
4280 | | vsi_l_offset nFileOffsetStart, |
4281 | | size_t nSize, const char *pData) |
4282 | | { |
4283 | | CPLMutexHolder oHolder(&hMutex); |
4284 | | |
4285 | | auto value = std::make_shared<std::string>(); |
4286 | | value->assign(pData, nSize); |
4287 | | GetRegionCache()->insert( |
4288 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), |
4289 | | std::move(value)); |
4290 | | } |
4291 | | |
4292 | | /************************************************************************/ |
4293 | | /* GetCachedFileProp() */ |
4294 | | /************************************************************************/ |
4295 | | |
4296 | | bool VSICurlFilesystemHandlerBase::GetCachedFileProp(const char *pszURL, |
4297 | | FileProp &oFileProp) |
4298 | | { |
4299 | | return VSICURLGetCachedFileProp(pszURL, oFileProp); |
4300 | | } |
4301 | | |
4302 | | /************************************************************************/ |
4303 | | /* SetCachedFileProp() */ |
4304 | | /************************************************************************/ |
4305 | | |
4306 | | void VSICurlFilesystemHandlerBase::SetCachedFileProp(const char *pszURL, |
4307 | | FileProp &oFileProp) |
4308 | | { |
4309 | | VSICURLSetCachedFileProp(pszURL, oFileProp); |
4310 | | } |
4311 | | |
4312 | | /************************************************************************/ |
4313 | | /* GetCachedDirList() */ |
4314 | | /************************************************************************/ |
4315 | | |
4316 | | bool VSICurlFilesystemHandlerBase::GetCachedDirList( |
4317 | | const char *pszURL, CachedDirList &oCachedDirList) |
4318 | | { |
4319 | | CPLMutexHolder oHolder(&hMutex); |
4320 | | |
4321 | | return oCacheDirList.tryGet(std::string(pszURL), oCachedDirList) && |
4322 | | // Let a chance to use new auth parameters |
4323 | | gnGenerationAuthParameters == |
4324 | | oCachedDirList.nGenerationAuthParameters; |
4325 | | } |
4326 | | |
4327 | | /************************************************************************/ |
4328 | | /* SetCachedDirList() */ |
4329 | | /************************************************************************/ |
4330 | | |
4331 | | void VSICurlFilesystemHandlerBase::SetCachedDirList( |
4332 | | const char *pszURL, CachedDirList &oCachedDirList) |
4333 | | { |
4334 | | CPLMutexHolder oHolder(&hMutex); |
4335 | | |
4336 | | std::string key(pszURL); |
4337 | | CachedDirList oldValue; |
4338 | | if (oCacheDirList.tryGet(key, oldValue)) |
4339 | | { |
4340 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4341 | | oCacheDirList.remove(key); |
4342 | | } |
4343 | | |
4344 | | while ((!oCacheDirList.empty() && |
4345 | | nCachedFilesInDirList + oCachedDirList.oFileList.size() > |
4346 | | 1024 * 1024) || |
4347 | | oCacheDirList.size() == oCacheDirList.getMaxAllowedSize()) |
4348 | | { |
4349 | | std::string oldestKey; |
4350 | | oCacheDirList.getOldestEntry(oldestKey, oldValue); |
4351 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4352 | | oCacheDirList.remove(oldestKey); |
4353 | | } |
4354 | | oCachedDirList.nGenerationAuthParameters = gnGenerationAuthParameters; |
4355 | | |
4356 | | nCachedFilesInDirList += oCachedDirList.oFileList.size(); |
4357 | | oCacheDirList.insert(key, oCachedDirList); |
4358 | | } |
4359 | | |
4360 | | /************************************************************************/ |
4361 | | /* ExistsInCacheDirList() */ |
4362 | | /************************************************************************/ |
4363 | | |
4364 | | bool VSICurlFilesystemHandlerBase::ExistsInCacheDirList( |
4365 | | const std::string &osDirname, bool *pbIsDir) |
4366 | | { |
4367 | | CachedDirList cachedDirList; |
4368 | | if (GetCachedDirList(osDirname.c_str(), cachedDirList)) |
4369 | | { |
4370 | | if (pbIsDir) |
4371 | | *pbIsDir = !cachedDirList.oFileList.empty(); |
4372 | | return false; |
4373 | | } |
4374 | | else |
4375 | | { |
4376 | | if (pbIsDir) |
4377 | | *pbIsDir = false; |
4378 | | return false; |
4379 | | } |
4380 | | } |
4381 | | |
4382 | | /************************************************************************/ |
4383 | | /* InvalidateCachedData() */ |
4384 | | /************************************************************************/ |
4385 | | |
4386 | | void VSICurlFilesystemHandlerBase::InvalidateCachedData(const char *pszURL) |
4387 | | { |
4388 | | CPLMutexHolder oHolder(&hMutex); |
4389 | | |
4390 | | VSICURLInvalidateCachedFileProp(pszURL); |
4391 | | |
4392 | | // Invalidate all cached regions for this URL |
4393 | | std::list<FilenameOffsetPair> keysToRemove; |
4394 | | std::string osURL(pszURL); |
4395 | | auto lambda = |
4396 | | [&keysToRemove, |
4397 | | &osURL](const lru11::KeyValuePair<FilenameOffsetPair, |
4398 | | std::shared_ptr<std::string>> &kv) |
4399 | | { |
4400 | | if (kv.key.filename_ == osURL) |
4401 | | keysToRemove.push_back(kv.key); |
4402 | | }; |
4403 | | auto *poRegionCache = GetRegionCache(); |
4404 | | poRegionCache->cwalk(lambda); |
4405 | | for (const auto &key : keysToRemove) |
4406 | | poRegionCache->remove(key); |
4407 | | } |
4408 | | |
4409 | | /************************************************************************/ |
4410 | | /* ClearCache() */ |
4411 | | /************************************************************************/ |
4412 | | |
4413 | | void VSICurlFilesystemHandlerBase::ClearCache() |
4414 | | { |
4415 | | CPLMutexHolder oHolder(&hMutex); |
4416 | | |
4417 | | GetRegionCache()->clear(); |
4418 | | |
4419 | | VSICURLDestroyCacheFileProp(); |
4420 | | |
4421 | | oCacheDirList.clear(); |
4422 | | nCachedFilesInDirList = 0; |
4423 | | |
4424 | | GetConnectionCache()[this].clear(); |
4425 | | } |
4426 | | |
4427 | | /************************************************************************/ |
4428 | | /* PartialClearCache() */ |
4429 | | /************************************************************************/ |
4430 | | |
4431 | | void VSICurlFilesystemHandlerBase::PartialClearCache( |
4432 | | const char *pszFilenamePrefix) |
4433 | | { |
4434 | | CPLMutexHolder oHolder(&hMutex); |
4435 | | |
4436 | | std::string osURL = GetURLFromFilename(pszFilenamePrefix); |
4437 | | { |
4438 | | std::list<FilenameOffsetPair> keysToRemove; |
4439 | | auto lambda = |
4440 | | [&keysToRemove, &osURL]( |
4441 | | const lru11::KeyValuePair<FilenameOffsetPair, |
4442 | | std::shared_ptr<std::string>> &kv) |
4443 | | { |
4444 | | if (strncmp(kv.key.filename_.c_str(), osURL.c_str(), |
4445 | | osURL.size()) == 0) |
4446 | | keysToRemove.push_back(kv.key); |
4447 | | }; |
4448 | | auto *poRegionCache = GetRegionCache(); |
4449 | | poRegionCache->cwalk(lambda); |
4450 | | for (const auto &key : keysToRemove) |
4451 | | poRegionCache->remove(key); |
4452 | | } |
4453 | | |
4454 | | VSICURLInvalidateCachedFilePropPrefix(osURL.c_str()); |
4455 | | |
4456 | | { |
4457 | | const size_t nLen = strlen(pszFilenamePrefix); |
4458 | | std::list<std::string> keysToRemove; |
4459 | | auto lambda = |
4460 | | [this, &keysToRemove, pszFilenamePrefix, |
4461 | | nLen](const lru11::KeyValuePair<std::string, CachedDirList> &kv) |
4462 | | { |
4463 | | if (strncmp(kv.key.c_str(), pszFilenamePrefix, nLen) == 0) |
4464 | | { |
4465 | | keysToRemove.push_back(kv.key); |
4466 | | nCachedFilesInDirList -= kv.value.oFileList.size(); |
4467 | | } |
4468 | | }; |
4469 | | oCacheDirList.cwalk(lambda); |
4470 | | for (const auto &key : keysToRemove) |
4471 | | oCacheDirList.remove(key); |
4472 | | } |
4473 | | } |
4474 | | |
4475 | | /************************************************************************/ |
4476 | | /* CreateFileHandle() */ |
4477 | | /************************************************************************/ |
4478 | | |
4479 | | VSICurlHandle * |
4480 | | VSICurlFilesystemHandlerBase::CreateFileHandle(const char *pszFilename) |
4481 | | { |
4482 | | return new VSICurlHandle(this, pszFilename); |
4483 | | } |
4484 | | |
4485 | | /************************************************************************/ |
4486 | | /* GetActualURL() */ |
4487 | | /************************************************************************/ |
4488 | | |
4489 | | const char *VSICurlFilesystemHandlerBase::GetActualURL(const char *pszFilename) |
4490 | | { |
4491 | | VSICurlHandle *poHandle = CreateFileHandle(pszFilename); |
4492 | | if (poHandle == nullptr) |
4493 | | return pszFilename; |
4494 | | std::string osURL(poHandle->GetURL()); |
4495 | | delete poHandle; |
4496 | | return CPLSPrintf("%s", osURL.c_str()); |
4497 | | } |
4498 | | |
4499 | | /************************************************************************/ |
4500 | | /* GetOptions() */ |
4501 | | /************************************************************************/ |
4502 | | |
4503 | | #define VSICURL_OPTIONS \ |
4504 | | " <Option name='GDAL_HTTP_MAX_RETRY' type='int' " \ |
4505 | | "description='Maximum number of retries' default='0'/>" \ |
4506 | | " <Option name='GDAL_HTTP_RETRY_DELAY' type='double' " \ |
4507 | | "description='Retry delay in seconds' default='30'/>" \ |
4508 | | " <Option name='GDAL_HTTP_HEADER_FILE' type='string' " \ |
4509 | | "description='Filename of a file that contains HTTP headers to " \ |
4510 | | "forward to the server'/>" \ |
4511 | | " <Option name='CPL_VSIL_CURL_USE_HEAD' type='boolean' " \ |
4512 | | "description='Whether to use HTTP HEAD verb to retrieve " \ |
4513 | | "file information' default='YES'/>" \ |
4514 | | " <Option name='GDAL_HTTP_MULTIRANGE' type='string-select' " \ |
4515 | | "description='Strategy to apply to run multi-range requests' " \ |
4516 | | "default='PARALLEL'>" \ |
4517 | | " <Value>PARALLEL</Value>" \ |
4518 | | " <Value>SERIAL</Value>" \ |
4519 | | " </Option>" \ |
4520 | | " <Option name='GDAL_HTTP_MULTIPLEX' type='boolean' " \ |
4521 | | "description='Whether to enable HTTP/2 multiplexing' default='YES'/>" \ |
4522 | | " <Option name='GDAL_HTTP_MERGE_CONSECUTIVE_RANGES' type='boolean' " \ |
4523 | | "description='Whether to merge consecutive ranges in multirange " \ |
4524 | | "requests' default='YES'/>" \ |
4525 | | " <Option name='CPL_VSIL_CURL_NON_CACHED' type='string' " \ |
4526 | | "description='Colon-separated list of filenames whose content" \ |
4527 | | "must not be cached across open attempts'/>" \ |
4528 | | " <Option name='CPL_VSIL_CURL_ALLOWED_FILENAME' type='string' " \ |
4529 | | "description='Single filename that is allowed to be opened'/>" \ |
4530 | | " <Option name='CPL_VSIL_CURL_ALLOWED_EXTENSIONS' type='string' " \ |
4531 | | "description='Comma or space separated list of allowed file " \ |
4532 | | "extensions'/>" \ |
4533 | | " <Option name='GDAL_DISABLE_READDIR_ON_OPEN' type='string-select' " \ |
4534 | | "description='Whether to disable establishing the list of files in " \ |
4535 | | "the directory of the current filename' default='NO'>" \ |
4536 | | " <Value>NO</Value>" \ |
4537 | | " <Value>YES</Value>" \ |
4538 | | " <Value>EMPTY_DIR</Value>" \ |
4539 | | " </Option>" \ |
4540 | | " <Option name='VSI_CACHE' type='boolean' " \ |
4541 | | "description='Whether to cache in memory the contents of the opened " \ |
4542 | | "file as soon as they are read' default='NO'/>" \ |
4543 | | " <Option name='CPL_VSIL_CURL_CHUNK_SIZE' type='integer' " \ |
4544 | | "description='Size in bytes of the minimum amount of data read in a " \ |
4545 | | "file' default='16384' min='1024' max='10485760'/>" \ |
4546 | | " <Option name='CPL_VSIL_CURL_CACHE_SIZE' type='integer' " \ |
4547 | | "description='Size in bytes of the global /vsicurl/ cache' " \ |
4548 | | "default='16384000'/>" \ |
4549 | | " <Option name='CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE' type='boolean' " \ |
4550 | | "description='Whether to skip files with Glacier storage class in " \ |
4551 | | "directory listing.' default='YES'/>" \ |
4552 | | " <Option name='CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT' " \ |
4553 | | "type='integer' description='Maximum number of bytes AdviseRead() is " \ |
4554 | | "allowed to fetch at once' default='104857600'/>" \ |
4555 | | " <Option name='CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED' " \ |
4556 | | "type='string-select' description='Whether the header-file key-value " \ |
4557 | | "pair can be used in /vsicurl? filenames' default='ONLY_IN_TEMP'>" \ |
4558 | | " <Value>ONLY_IN_TEMP</Value>" \ |
4559 | | " <Value>NO</Value>" \ |
4560 | | " <Value>YES</Value>" \ |
4561 | | " </Option>" \ |
4562 | | " <Option name='GDAL_HTTP_MAX_CACHED_CONNECTIONS' type='integer' " \ |
4563 | | "description='Maximum amount of connections that libcurl may keep alive " \ |
4564 | | "in its connection cache after use'/>" \ |
4565 | | " <Option name='GDAL_HTTP_MAX_TOTAL_CONNECTIONS' type='integer' " \ |
4566 | | "description='Maximum number of simultaneously open connections in " \ |
4567 | | "total'/>" |
4568 | | |
4569 | | const char *VSICurlFilesystemHandlerBase::GetOptionsStatic() |
4570 | | { |
4571 | | return VSICURL_OPTIONS; |
4572 | | } |
4573 | | |
4574 | | const char *VSICurlFilesystemHandlerBase::GetOptions() |
4575 | | { |
4576 | | static std::string osOptions(std::string("<Options>") + GetOptionsStatic() + |
4577 | | "</Options>"); |
4578 | | return osOptions.c_str(); |
4579 | | } |
4580 | | |
4581 | | /************************************************************************/ |
4582 | | /* IsAllowedFilename() */ |
4583 | | /************************************************************************/ |
4584 | | |
4585 | | bool VSICurlFilesystemHandlerBase::IsAllowedFilename(const char *pszFilename) |
4586 | | { |
4587 | | const char *pszAllowedFilename = |
4588 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_FILENAME", nullptr); |
4589 | | if (pszAllowedFilename != nullptr) |
4590 | | { |
4591 | | return strcmp(pszFilename, pszAllowedFilename) == 0; |
4592 | | } |
4593 | | |
4594 | | // Consider that only the files whose extension ends up with one that is |
4595 | | // listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server. This can |
4596 | | // speeds up dramatically open experience, in case the server cannot return |
4597 | | // a file list. {noext} can be used as a special token to mean file with no |
4598 | | // extension. |
4599 | | // For example: |
4600 | | // gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" |
4601 | | // /vsicurl/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif |
4602 | | const char *pszAllowedExtensions = |
4603 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", nullptr); |
4604 | | if (pszAllowedExtensions) |
4605 | | { |
4606 | | char **papszExtensions = |
4607 | | CSLTokenizeString2(pszAllowedExtensions, ", ", 0); |
4608 | | const char *queryStart = strchr(pszFilename, '?'); |
4609 | | char *pszFilenameWithoutQuery = nullptr; |
4610 | | if (queryStart != nullptr) |
4611 | | { |
4612 | | pszFilenameWithoutQuery = CPLStrdup(pszFilename); |
4613 | | pszFilenameWithoutQuery[queryStart - pszFilename] = '\0'; |
4614 | | pszFilename = pszFilenameWithoutQuery; |
4615 | | } |
4616 | | const size_t nURLLen = strlen(pszFilename); |
4617 | | bool bFound = false; |
4618 | | for (int i = 0; papszExtensions[i] != nullptr; i++) |
4619 | | { |
4620 | | const size_t nExtensionLen = strlen(papszExtensions[i]); |
4621 | | if (EQUAL(papszExtensions[i], "{noext}")) |
4622 | | { |
4623 | | const char *pszLastSlash = strrchr(pszFilename, '/'); |
4624 | | if (pszLastSlash != nullptr && |
4625 | | strchr(pszLastSlash, '.') == nullptr) |
4626 | | { |
4627 | | bFound = true; |
4628 | | break; |
4629 | | } |
4630 | | } |
4631 | | else if (nURLLen > nExtensionLen && |
4632 | | EQUAL(pszFilename + nURLLen - nExtensionLen, |
4633 | | papszExtensions[i])) |
4634 | | { |
4635 | | bFound = true; |
4636 | | break; |
4637 | | } |
4638 | | } |
4639 | | |
4640 | | CSLDestroy(papszExtensions); |
4641 | | if (pszFilenameWithoutQuery) |
4642 | | { |
4643 | | CPLFree(pszFilenameWithoutQuery); |
4644 | | } |
4645 | | |
4646 | | return bFound; |
4647 | | } |
4648 | | return TRUE; |
4649 | | } |
4650 | | |
4651 | | /************************************************************************/ |
4652 | | /* Open() */ |
4653 | | /************************************************************************/ |
4654 | | |
4655 | | VSIVirtualHandleUniquePtr |
4656 | | VSICurlFilesystemHandlerBase::Open(const char *pszFilename, |
4657 | | const char *pszAccess, bool bSetError, |
4658 | | CSLConstList papszOptions) |
4659 | | { |
4660 | | const bool bStartsWithVSICurlPrefix = StartsWithVSICurlPrefix(pszFilename); |
4661 | | if (!bStartsWithVSICurlPrefix && |
4662 | | !cpl::starts_with(std::string_view(pszFilename), GetFSPrefix())) |
4663 | | { |
4664 | | return nullptr; |
4665 | | } |
4666 | | |
4667 | | if (strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, '+') != nullptr) |
4668 | | { |
4669 | | if (bSetError) |
4670 | | { |
4671 | | VSIError(VSIE_FileError, |
4672 | | "Only read-only mode is supported for /vsicurl"); |
4673 | | } |
4674 | | return nullptr; |
4675 | | } |
4676 | | if (!papszOptions || |
4677 | | !CPLTestBool(CSLFetchNameValueDef( |
4678 | | papszOptions, "IGNORE_FILENAME_RESTRICTIONS", "NO"))) |
4679 | | { |
4680 | | if (!IsAllowedFilename(pszFilename)) |
4681 | | return nullptr; |
4682 | | } |
4683 | | |
4684 | | bool bListDir = true; |
4685 | | bool bEmptyDir = false; |
4686 | | std::string osURL = |
4687 | | bStartsWithVSICurlPrefix |
4688 | | ? VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, nullptr, |
4689 | | &bListDir, &bEmptyDir, nullptr, nullptr, |
4690 | | nullptr) |
4691 | | : GetURLFromFilename(pszFilename); |
4692 | | |
4693 | | const char *pszOptionVal = CSLFetchNameValueDef( |
4694 | | papszOptions, "DISABLE_READDIR_ON_OPEN", |
4695 | | VSIGetPathSpecificOption(pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", |
4696 | | "NO")); |
4697 | | const bool bCache = CPLTestBool(CSLFetchNameValueDef( |
4698 | | papszOptions, "CACHE", AllowCachedDataFor(pszFilename) ? "YES" : "NO")); |
4699 | | const bool bSkipReadDir = !bListDir || bEmptyDir || |
4700 | | EQUAL(pszOptionVal, "EMPTY_DIR") || |
4701 | | CPLTestBool(pszOptionVal) || !bCache; |
4702 | | |
4703 | | std::string osFilename(pszFilename); |
4704 | | bool bGotFileList = !bSkipReadDir; |
4705 | | bool bForceExistsCheck = false; |
4706 | | FileProp cachedFileProp; |
4707 | | if (!bSkipReadDir && |
4708 | | !(GetCachedFileProp(osURL.c_str(), cachedFileProp) && |
4709 | | cachedFileProp.eExists == EXIST_YES) && |
4710 | | strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
4711 | | !STARTS_WITH(CPLGetExtensionSafe(osFilename.c_str()).c_str(), "zip") && |
4712 | | // Likely a Kerchunk JSON reference file: no need to list siblings |
4713 | | !cpl::ends_with(osFilename, ".nc.zarr")) |
4714 | | { |
4715 | | // 1000 corresponds to the default page size of S3. |
4716 | | constexpr int FILE_COUNT_LIMIT = 1000; |
4717 | | const CPLStringList aosFileList(ReadDirInternal( |
4718 | | (CPLGetDirnameSafe(osFilename.c_str()) + '/').c_str(), |
4719 | | FILE_COUNT_LIMIT, &bGotFileList)); |
4720 | | const bool bFound = |
4721 | | VSICurlIsFileInList(aosFileList.List(), |
4722 | | CPLGetFilename(osFilename.c_str())) != -1; |
4723 | | if (bGotFileList && !bFound && aosFileList.size() < FILE_COUNT_LIMIT) |
4724 | | { |
4725 | | // Some file servers are case insensitive, so in case there is a |
4726 | | // match with case difference, do a full check just in case. |
4727 | | // e.g. |
4728 | | // http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/MEGA90N000CB.IMG |
4729 | | // that is queried by |
4730 | | // gdalinfo |
4731 | | // /vsicurl/http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/mega90n000cb.lbl |
4732 | | if (aosFileList.FindString(CPLGetFilename(osFilename.c_str())) != |
4733 | | -1) |
4734 | | { |
4735 | | bForceExistsCheck = true; |
4736 | | } |
4737 | | else |
4738 | | { |
4739 | | return nullptr; |
4740 | | } |
4741 | | } |
4742 | | } |
4743 | | if (!bStartsWithVSICurlPrefix) |
4744 | | osURL = GetURLFromFilename(pszFilename); |
4745 | | if (GetCachedFileProp(osURL.c_str(), cachedFileProp) && |
4746 | | cachedFileProp.eExists == EXIST_YES && cachedFileProp.bIsDirectory) |
4747 | | { |
4748 | | return nullptr; |
4749 | | } |
4750 | | |
4751 | | auto poHandle = |
4752 | | std::unique_ptr<VSICurlHandle>(CreateFileHandle(osFilename.c_str())); |
4753 | | if (poHandle == nullptr) |
4754 | | return nullptr; |
4755 | | poHandle->SetCache(bCache); |
4756 | | if (!bGotFileList || bForceExistsCheck) |
4757 | | { |
4758 | | // If we didn't get a filelist, check that the file really exists. |
4759 | | if (!poHandle->Exists(bSetError)) |
4760 | | { |
4761 | | return nullptr; |
4762 | | } |
4763 | | } |
4764 | | |
4765 | | if (CPLTestBool(CPLGetConfigOption("VSI_CACHE", "FALSE"))) |
4766 | | return VSIVirtualHandleUniquePtr( |
4767 | | VSICreateCachedFile(poHandle.release())); |
4768 | | else |
4769 | | return VSIVirtualHandleUniquePtr(poHandle.release()); |
4770 | | } |
4771 | | |
4772 | | /************************************************************************/ |
4773 | | /* VSICurlParserFindEOL() */ |
4774 | | /* */ |
4775 | | /* Small helper function for VSICurlPaseHTMLFileList() to find */ |
4776 | | /* the end of a line in the directory listing. Either a <br> */ |
4777 | | /* or newline. */ |
4778 | | /************************************************************************/ |
4779 | | |
4780 | | static char *VSICurlParserFindEOL(char *pszData) |
4781 | | |
4782 | | { |
4783 | | while (*pszData != '\0' && *pszData != '\n' && |
4784 | | !STARTS_WITH_CI(pszData, "<br>")) |
4785 | | pszData++; |
4786 | | |
4787 | | if (*pszData == '\0') |
4788 | | return nullptr; |
4789 | | |
4790 | | return pszData; |
4791 | | } |
4792 | | |
4793 | | /************************************************************************/ |
4794 | | /* ParseFileSize() */ |
4795 | | /************************************************************************/ |
4796 | | |
4797 | | static GUIntBig ParseFileSize(const char *pszStr) |
4798 | | { |
4799 | | GUIntBig nFileSize = 0; |
4800 | | while (*pszStr == ' ') |
4801 | | pszStr++; |
4802 | | if (*pszStr >= '1' && *pszStr <= '9') |
4803 | | { |
4804 | | const char *pszIter = pszStr + 1; |
4805 | | while (*pszIter >= '0' && *pszIter <= '9') |
4806 | | ++pszIter; |
4807 | | if (*pszIter == 0 || *pszIter == ' ' || *pszIter == '\t' || |
4808 | | *pszIter == '\r' || *pszIter == '\n') |
4809 | | { |
4810 | | nFileSize = |
4811 | | CPLScanUIntBig(pszStr, static_cast<int>(pszIter - pszStr)); |
4812 | | } |
4813 | | } |
4814 | | return nFileSize; |
4815 | | } |
4816 | | |
4817 | | /************************************************************************/ |
4818 | | /* VSICurlParseHTMLDateTimeFileSize() */ |
4819 | | /************************************************************************/ |
4820 | | |
4821 | | static const char *const apszMonths[] = { |
4822 | | "January", "February", "March", "April", "May", "June", |
4823 | | "July", "August", "September", "October", "November", "December"}; |
4824 | | |
4825 | | static bool VSICurlParseHTMLDateTimeFileSize(const char *pszStr, |
4826 | | struct tm &brokendowntime, |
4827 | | GUIntBig &nFileSize, |
4828 | | GIntBig &mTime) |
4829 | | { |
4830 | | for (int iMonth = 0; iMonth < 12; iMonth++) |
4831 | | { |
4832 | | nFileSize = 0; |
4833 | | |
4834 | | char szMonth[32] = {}; |
4835 | | szMonth[0] = '-'; |
4836 | | memcpy(szMonth + 1, apszMonths[iMonth], 3); |
4837 | | szMonth[4] = '-'; |
4838 | | szMonth[5] = '\0'; |
4839 | | const char *pszMonthFound = strstr(pszStr, szMonth); |
4840 | | if (pszMonthFound) |
4841 | | { |
4842 | | // Format of Apache, like in |
4843 | | // http://download.osgeo.org/gdal/data/gtiff/ |
4844 | | // "17-May-2010 12:26" |
4845 | | const auto nMonthFoundLen = strlen(pszMonthFound); |
4846 | | if (pszMonthFound - pszStr > 2 && nMonthFoundLen > 15 && |
4847 | | pszMonthFound[-2 + 11] == ' ' && pszMonthFound[-2 + 14] == ':') |
4848 | | { |
4849 | | pszMonthFound -= 2; |
4850 | | int nDay = atoi(pszMonthFound); |
4851 | | int nYear = atoi(pszMonthFound + 7); |
4852 | | int nHour = atoi(pszMonthFound + 12); |
4853 | | int nMin = atoi(pszMonthFound + 15); |
4854 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4855 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4856 | | { |
4857 | | brokendowntime.tm_year = nYear - 1900; |
4858 | | brokendowntime.tm_mon = iMonth; |
4859 | | brokendowntime.tm_mday = nDay; |
4860 | | brokendowntime.tm_hour = nHour; |
4861 | | brokendowntime.tm_min = nMin; |
4862 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4863 | | |
4864 | | if (nMonthFoundLen > 15 + 2) |
4865 | | { |
4866 | | const char *pszFilesize = pszMonthFound + 15 + 2; |
4867 | | nFileSize = ParseFileSize(pszFilesize); |
4868 | | } |
4869 | | } |
4870 | | } |
4871 | | return nFileSize > 0; |
4872 | | } |
4873 | | |
4874 | | /* Microsoft IIS */ |
4875 | | snprintf(szMonth, sizeof(szMonth), " %s ", apszMonths[iMonth]); |
4876 | | pszMonthFound = strstr(pszStr, szMonth); |
4877 | | if (pszMonthFound) |
4878 | | { |
4879 | | int nLenMonth = static_cast<int>(strlen(apszMonths[iMonth])); |
4880 | | if (pszMonthFound - pszStr > 2 && pszMonthFound[-1] != ',' && |
4881 | | pszMonthFound[-2] != ' ' && |
4882 | | static_cast<int>(strlen(pszMonthFound - 2)) > |
4883 | | 2 + 1 + nLenMonth + 1 + 4 + 1 + 5 + 1 + 4) |
4884 | | { |
4885 | | /* Format of http://ortho.linz.govt.nz/tifs/1994_95/ */ |
4886 | | /* " Friday, 21 April 2006 12:05 p.m. 48062343 |
4887 | | * m35a_fy_94_95.tif" */ |
4888 | | pszMonthFound -= 2; |
4889 | | int nDay = atoi(pszMonthFound); |
4890 | | int nCurOffset = 2 + 1 + nLenMonth + 1; |
4891 | | int nYear = atoi(pszMonthFound + nCurOffset); |
4892 | | nCurOffset += 4 + 1; |
4893 | | int nHour = atoi(pszMonthFound + nCurOffset); |
4894 | | if (nHour < 10) |
4895 | | nCurOffset += 1 + 1; |
4896 | | else |
4897 | | nCurOffset += 2 + 1; |
4898 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
4899 | | nCurOffset += 2 + 1; |
4900 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "p.m.")) |
4901 | | nHour += 12; |
4902 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "a.m.")) |
4903 | | nHour = -1; |
4904 | | nCurOffset += 4; |
4905 | | |
4906 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4907 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4908 | | { |
4909 | | brokendowntime.tm_year = nYear - 1900; |
4910 | | brokendowntime.tm_mon = iMonth; |
4911 | | brokendowntime.tm_mday = nDay; |
4912 | | brokendowntime.tm_hour = nHour; |
4913 | | brokendowntime.tm_min = nMin; |
4914 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4915 | | |
4916 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
4917 | | nFileSize = ParseFileSize(pszFilesize); |
4918 | | } |
4919 | | } |
4920 | | else if (pszMonthFound - pszStr > 1 && pszMonthFound[-1] == ',' && |
4921 | | static_cast<int>(strlen(pszMonthFound)) > |
4922 | | 1 + nLenMonth + 1 + 2 + 1 + 1 + 4 + 1 + 5 + 1 + 2) |
4923 | | { |
4924 | | // Format of |
4925 | | // http://publicfiles.dep.state.fl.us/dear/BWR_GIS/2007NWFLULC/ |
4926 | | // " Sunday, June 20, 2010 6:46 PM 233170905 |
4927 | | // NWF2007LULCForSDE.zip" |
4928 | | pszMonthFound += 1; |
4929 | | int nCurOffset = nLenMonth + 1; |
4930 | | int nDay = atoi(pszMonthFound + nCurOffset); |
4931 | | nCurOffset += 2 + 1 + 1; |
4932 | | int nYear = atoi(pszMonthFound + nCurOffset); |
4933 | | nCurOffset += 4 + 1; |
4934 | | int nHour = atoi(pszMonthFound + nCurOffset); |
4935 | | nCurOffset += 2 + 1; |
4936 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
4937 | | nCurOffset += 2 + 1; |
4938 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "PM")) |
4939 | | nHour += 12; |
4940 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "AM")) |
4941 | | nHour = -1; |
4942 | | nCurOffset += 2; |
4943 | | |
4944 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4945 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4946 | | { |
4947 | | brokendowntime.tm_year = nYear - 1900; |
4948 | | brokendowntime.tm_mon = iMonth; |
4949 | | brokendowntime.tm_mday = nDay; |
4950 | | brokendowntime.tm_hour = nHour; |
4951 | | brokendowntime.tm_min = nMin; |
4952 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4953 | | |
4954 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
4955 | | nFileSize = ParseFileSize(pszFilesize); |
4956 | | } |
4957 | | } |
4958 | | |
4959 | | return nFileSize > 0; |
4960 | | } |
4961 | | } |
4962 | | |
4963 | | return false; |
4964 | | } |
4965 | | |
4966 | | /************************************************************************/ |
4967 | | /* ParseHTMLFileList() */ |
4968 | | /* */ |
4969 | | /* Parse a file list document and return all the components. */ |
4970 | | /************************************************************************/ |
4971 | | |
4972 | | char **VSICurlFilesystemHandlerBase::ParseHTMLFileList(const char *pszFilename, |
4973 | | int nMaxFiles, |
4974 | | char *pszData, |
4975 | | bool *pbGotFileList) |
4976 | | { |
4977 | | *pbGotFileList = false; |
4978 | | |
4979 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
4980 | | nullptr, nullptr, nullptr, |
4981 | | nullptr, nullptr, nullptr)); |
4982 | | const char *pszDir = nullptr; |
4983 | | if (STARTS_WITH_CI(osURL.c_str(), "http://")) |
4984 | | pszDir = strchr(osURL.c_str() + strlen("http://"), '/'); |
4985 | | else if (STARTS_WITH_CI(osURL.c_str(), "https://")) |
4986 | | pszDir = strchr(osURL.c_str() + strlen("https://"), '/'); |
4987 | | else if (STARTS_WITH_CI(osURL.c_str(), "ftp://")) |
4988 | | pszDir = strchr(osURL.c_str() + strlen("ftp://"), '/'); |
4989 | | if (pszDir == nullptr) |
4990 | | pszDir = ""; |
4991 | | |
4992 | | /* Apache / Nginx */ |
4993 | | /* Most of the time the format is <title>Index of {pszDir[/]}</title>, but |
4994 | | * there are special cases like https://cdn.star.nesdis.noaa.gov/GOES18/ABI/MESO/M1/GEOCOLOR/ |
4995 | | * where a CDN stuff makes that the title is <title>Index of /ma-cdn02/GOES/data/GOES18/ABI/MESO/M1/GEOCOLOR/</title> |
4996 | | */ |
4997 | | const std::string osTitleIndexOfPrefix = "<title>Index of "; |
4998 | | const std::string osExpectedSuffix = std::string(pszDir).append("</title>"); |
4999 | | const std::string osExpectedSuffixWithSlash = |
5000 | | std::string(pszDir).append("/</title>"); |
5001 | | /* FTP */ |
5002 | | const std::string osExpectedStringFTP = |
5003 | | std::string("FTP Listing of ").append(pszDir).append("/"); |
5004 | | /* Apache 1.3.33 */ |
5005 | | const std::string osExpectedStringOldApache = |
5006 | | std::string("<TITLE>Index of ").append(pszDir).append("</TITLE>"); |
5007 | | |
5008 | | // The listing of |
5009 | | // http://dds.cr.usgs.gov/srtm/SRTM_image_sample/picture%20examples/ |
5010 | | // has |
5011 | | // "<title>Index of /srtm/SRTM_image_sample/picture examples</title>" |
5012 | | // so we must try unescaped %20 also. |
5013 | | // Similar with |
5014 | | // http://datalib.usask.ca/gis/Data/Central_America_goodbutdoweown%3f/ |
5015 | | std::string osExpectedString_unescaped; |
5016 | | if (strchr(pszDir, '%')) |
5017 | | { |
5018 | | char *pszUnescapedDir = CPLUnescapeString(pszDir, nullptr, CPLES_URL); |
5019 | | osExpectedString_unescaped = osTitleIndexOfPrefix; |
5020 | | osExpectedString_unescaped += pszUnescapedDir; |
5021 | | osExpectedString_unescaped += "</title>"; |
5022 | | CPLFree(pszUnescapedDir); |
5023 | | } |
5024 | | |
5025 | | char *c = nullptr; |
5026 | | int nCount = 0; |
5027 | | int nCountTable = 0; |
5028 | | CPLStringList oFileList; |
5029 | | char *pszLine = pszData; |
5030 | | bool bIsHTMLDirList = false; |
5031 | | |
5032 | | while ((c = VSICurlParserFindEOL(pszLine)) != nullptr) |
5033 | | { |
5034 | | *c = '\0'; |
5035 | | |
5036 | | // To avoid false positive on pages such as |
5037 | | // http://www.ngs.noaa.gov/PC_PROD/USGG2009BETA |
5038 | | // This is a heuristics, but normal HTML listing of files have not more |
5039 | | // than one table. |
5040 | | if (strstr(pszLine, "<table")) |
5041 | | { |
5042 | | nCountTable++; |
5043 | | if (nCountTable == 2) |
5044 | | { |
5045 | | *pbGotFileList = false; |
5046 | | return nullptr; |
5047 | | } |
5048 | | } |
5049 | | |
5050 | | if (!bIsHTMLDirList && |
5051 | | ((strstr(pszLine, osTitleIndexOfPrefix.c_str()) && |
5052 | | (strstr(pszLine, osExpectedSuffix.c_str()) || |
5053 | | strstr(pszLine, osExpectedSuffixWithSlash.c_str()))) || |
5054 | | strstr(pszLine, osExpectedStringFTP.c_str()) || |
5055 | | strstr(pszLine, osExpectedStringOldApache.c_str()) || |
5056 | | (!osExpectedString_unescaped.empty() && |
5057 | | strstr(pszLine, osExpectedString_unescaped.c_str())))) |
5058 | | { |
5059 | | bIsHTMLDirList = true; |
5060 | | *pbGotFileList = true; |
5061 | | } |
5062 | | // Subversion HTTP listing |
5063 | | // or Microsoft-IIS/6.0 listing |
5064 | | // (e.g. http://ortho.linz.govt.nz/tifs/2005_06/) */ |
5065 | | else if (!bIsHTMLDirList && strstr(pszLine, "<title>")) |
5066 | | { |
5067 | | // Detect something like: |
5068 | | // <html><head><title>gdal - Revision 20739: |
5069 | | // /trunk/autotest/gcore/data</title></head> */ The annoying thing |
5070 | | // is that what is after ': ' is a subpart of what is after |
5071 | | // http://server/ |
5072 | | char *pszSubDir = strstr(pszLine, ": "); |
5073 | | if (pszSubDir == nullptr) |
5074 | | // or <title>ortho.linz.govt.nz - /tifs/2005_06/</title> |
5075 | | pszSubDir = strstr(pszLine, "- "); |
5076 | | if (pszSubDir) |
5077 | | { |
5078 | | pszSubDir += 2; |
5079 | | char *pszTmp = strstr(pszSubDir, "</title>"); |
5080 | | if (pszTmp) |
5081 | | { |
5082 | | if (pszTmp[-1] == '/') |
5083 | | pszTmp[-1] = 0; |
5084 | | else |
5085 | | *pszTmp = 0; |
5086 | | if (strstr(pszDir, pszSubDir)) |
5087 | | { |
5088 | | bIsHTMLDirList = true; |
5089 | | *pbGotFileList = true; |
5090 | | } |
5091 | | } |
5092 | | } |
5093 | | } |
5094 | | else if (bIsHTMLDirList && |
5095 | | (strstr(pszLine, "<a href=\"") != nullptr || |
5096 | | strstr(pszLine, "<A HREF=\"") != nullptr) && |
5097 | | // Exclude absolute links, like to subversion home. |
5098 | | strstr(pszLine, "<a href=\"http://") == nullptr && |
5099 | | // exclude parent directory. |
5100 | | strstr(pszLine, "Parent Directory") == nullptr) |
5101 | | { |
5102 | | char *beginFilename = strstr(pszLine, "<a href=\""); |
5103 | | if (beginFilename == nullptr) |
5104 | | beginFilename = strstr(pszLine, "<A HREF=\""); |
5105 | | beginFilename += strlen("<a href=\""); |
5106 | | char *endQuote = strchr(beginFilename, '"'); |
5107 | | if (endQuote && !STARTS_WITH(beginFilename, "?C=") && |
5108 | | !STARTS_WITH(beginFilename, "?N=")) |
5109 | | { |
5110 | | struct tm brokendowntime; |
5111 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
5112 | | GUIntBig nFileSize = 0; |
5113 | | GIntBig mTime = 0; |
5114 | | |
5115 | | VSICurlParseHTMLDateTimeFileSize(pszLine, brokendowntime, |
5116 | | nFileSize, mTime); |
5117 | | |
5118 | | *endQuote = '\0'; |
5119 | | |
5120 | | // Remove trailing slash, that are returned for directories by |
5121 | | // Apache. |
5122 | | bool bIsDirectory = false; |
5123 | | if (endQuote[-1] == '/') |
5124 | | { |
5125 | | bIsDirectory = true; |
5126 | | endQuote[-1] = 0; |
5127 | | } |
5128 | | |
5129 | | // shttpd links include slashes from the root directory. |
5130 | | // Skip them. |
5131 | | while (strchr(beginFilename, '/')) |
5132 | | beginFilename = strchr(beginFilename, '/') + 1; |
5133 | | |
5134 | | if (strcmp(beginFilename, ".") != 0 && |
5135 | | strcmp(beginFilename, "..") != 0) |
5136 | | { |
5137 | | std::string osCachedFilename = |
5138 | | CPLSPrintf("%s/%s", osURL.c_str(), beginFilename); |
5139 | | |
5140 | | FileProp cachedFileProp; |
5141 | | GetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
5142 | | cachedFileProp.eExists = EXIST_YES; |
5143 | | cachedFileProp.bIsDirectory = bIsDirectory; |
5144 | | if (mTime > 0) |
5145 | | { |
5146 | | cachedFileProp.mTime = static_cast<time_t>(mTime); |
5147 | | } |
5148 | | if (!cachedFileProp.bHasComputedFileSize) |
5149 | | { |
5150 | | cachedFileProp.bHasComputedFileSize = nFileSize > 0; |
5151 | | cachedFileProp.fileSize = nFileSize; |
5152 | | } |
5153 | | SetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
5154 | | |
5155 | | oFileList.AddString(beginFilename); |
5156 | | if constexpr (ENABLE_DEBUG_VERBOSE) |
5157 | | { |
5158 | | CPLDebug( |
5159 | | GetDebugKey(), |
5160 | | "File[%d] = %s, is_dir = %d, size = " CPL_FRMT_GUIB |
5161 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
5162 | | nCount, osCachedFilename.c_str(), |
5163 | | bIsDirectory ? 1 : 0, nFileSize, |
5164 | | brokendowntime.tm_year + 1900, |
5165 | | brokendowntime.tm_mon + 1, brokendowntime.tm_mday, |
5166 | | brokendowntime.tm_hour, brokendowntime.tm_min, |
5167 | | brokendowntime.tm_sec); |
5168 | | } |
5169 | | nCount++; |
5170 | | |
5171 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
5172 | | break; |
5173 | | } |
5174 | | } |
5175 | | } |
5176 | | pszLine = c + 1; |
5177 | | } |
5178 | | |
5179 | | return oFileList.StealList(); |
5180 | | } |
5181 | | |
5182 | | /************************************************************************/ |
5183 | | /* GetStreamingFilename() */ |
5184 | | /************************************************************************/ |
5185 | | |
5186 | | std::string VSICurlFilesystemHandler::GetStreamingFilename( |
5187 | | const std::string &osFilename) const |
5188 | | { |
5189 | | if (STARTS_WITH(osFilename.c_str(), GetFSPrefix().c_str())) |
5190 | | return "/vsicurl_streaming/" + osFilename.substr(GetFSPrefix().size()); |
5191 | | return osFilename; |
5192 | | } |
5193 | | |
5194 | | /************************************************************************/ |
5195 | | /* GetHintForPotentiallyRecognizedPath() */ |
5196 | | /************************************************************************/ |
5197 | | |
5198 | | std::string VSICurlFilesystemHandler::GetHintForPotentiallyRecognizedPath( |
5199 | | const std::string &osPath) |
5200 | | { |
5201 | | if (!StartsWithVSICurlPrefix(osPath.c_str()) && |
5202 | | !cpl::starts_with(osPath, GetStreamingFilename(GetFSPrefix()))) |
5203 | | { |
5204 | | for (const char *pszPrefix : {"http://", "https://"}) |
5205 | | { |
5206 | | if (cpl::starts_with(osPath, pszPrefix)) |
5207 | | { |
5208 | | return GetFSPrefix() + osPath; |
5209 | | } |
5210 | | } |
5211 | | } |
5212 | | return std::string(); |
5213 | | } |
5214 | | |
5215 | | /************************************************************************/ |
5216 | | /* VSICurlGetToken() */ |
5217 | | /************************************************************************/ |
5218 | | |
5219 | | static char *VSICurlGetToken(char *pszCurPtr, char **ppszNextToken) |
5220 | | { |
5221 | | if (pszCurPtr == nullptr) |
5222 | | return nullptr; |
5223 | | |
5224 | | while ((*pszCurPtr) == ' ') |
5225 | | pszCurPtr++; |
5226 | | if (*pszCurPtr == '\0') |
5227 | | return nullptr; |
5228 | | |
5229 | | char *pszToken = pszCurPtr; |
5230 | | while ((*pszCurPtr) != ' ' && (*pszCurPtr) != '\0') |
5231 | | pszCurPtr++; |
5232 | | if (*pszCurPtr == '\0') |
5233 | | { |
5234 | | *ppszNextToken = nullptr; |
5235 | | } |
5236 | | else |
5237 | | { |
5238 | | *pszCurPtr = '\0'; |
5239 | | pszCurPtr++; |
5240 | | while ((*pszCurPtr) == ' ') |
5241 | | pszCurPtr++; |
5242 | | *ppszNextToken = pszCurPtr; |
5243 | | } |
5244 | | |
5245 | | return pszToken; |
5246 | | } |
5247 | | |
5248 | | /************************************************************************/ |
5249 | | /* VSICurlParseFullFTPLine() */ |
5250 | | /************************************************************************/ |
5251 | | |
5252 | | /* Parse lines like the following ones : |
5253 | | -rw-r--r-- 1 10003 100 430 Jul 04 2008 COPYING |
5254 | | lrwxrwxrwx 1 ftp ftp 28 Jun 14 14:13 MPlayer -> |
5255 | | mirrors/mplayerhq.hu/MPlayer -rw-r--r-- 1 ftp ftp 725614592 May 13 |
5256 | | 20:13 Fedora-15-x86_64-Live-KDE.iso drwxr-xr-x 280 1003 1003 6656 Aug 26 |
5257 | | 04:17 gnu |
5258 | | */ |
5259 | | |
5260 | | static bool VSICurlParseFullFTPLine(char *pszLine, char *&pszFilename, |
5261 | | bool &bSizeValid, GUIntBig &nSize, |
5262 | | bool &bIsDirectory, GIntBig &nUnixTime) |
5263 | | { |
5264 | | char *pszNextToken = pszLine; |
5265 | | char *pszPermissions = VSICurlGetToken(pszNextToken, &pszNextToken); |
5266 | | if (pszPermissions == nullptr || strlen(pszPermissions) != 10) |
5267 | | return false; |
5268 | | bIsDirectory = pszPermissions[0] == 'd'; |
5269 | | |
5270 | | for (int i = 0; i < 3; i++) |
5271 | | { |
5272 | | if (VSICurlGetToken(pszNextToken, &pszNextToken) == nullptr) |
5273 | | return false; |
5274 | | } |
5275 | | |
5276 | | char *pszSize = VSICurlGetToken(pszNextToken, &pszNextToken); |
5277 | | if (pszSize == nullptr) |
5278 | | return false; |
5279 | | |
5280 | | if (pszPermissions[0] == '-') |
5281 | | { |
5282 | | // Regular file. |
5283 | | bSizeValid = true; |
5284 | | nSize = CPLScanUIntBig(pszSize, static_cast<int>(strlen(pszSize))); |
5285 | | } |
5286 | | |
5287 | | struct tm brokendowntime; |
5288 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
5289 | | bool bBrokenDownTimeValid = true; |
5290 | | |
5291 | | char *pszMonth = VSICurlGetToken(pszNextToken, &pszNextToken); |
5292 | | if (pszMonth == nullptr || strlen(pszMonth) != 3) |
5293 | | return false; |
5294 | | |
5295 | | int i = 0; // Used after for. |
5296 | | for (; i < 12; i++) |
5297 | | { |
5298 | | if (EQUALN(pszMonth, apszMonths[i], 3)) |
5299 | | break; |
5300 | | } |
5301 | | if (i < 12) |
5302 | | brokendowntime.tm_mon = i; |
5303 | | else |
5304 | | bBrokenDownTimeValid = false; |
5305 | | |
5306 | | char *pszDay = VSICurlGetToken(pszNextToken, &pszNextToken); |
5307 | | if (pszDay == nullptr || (strlen(pszDay) != 1 && strlen(pszDay) != 2)) |
5308 | | return false; |
5309 | | int nDay = atoi(pszDay); |
5310 | | if (nDay >= 1 && nDay <= 31) |
5311 | | brokendowntime.tm_mday = nDay; |
5312 | | else |
5313 | | bBrokenDownTimeValid = false; |
5314 | | |
5315 | | char *pszHourOrYear = VSICurlGetToken(pszNextToken, &pszNextToken); |
5316 | | if (pszHourOrYear == nullptr || |
5317 | | (strlen(pszHourOrYear) != 4 && strlen(pszHourOrYear) != 5)) |
5318 | | return false; |
5319 | | if (strlen(pszHourOrYear) == 4) |
5320 | | { |
5321 | | brokendowntime.tm_year = atoi(pszHourOrYear) - 1900; |
5322 | | } |
5323 | | else |
5324 | | { |
5325 | | time_t sTime; |
5326 | | time(&sTime); |
5327 | | struct tm currentBrokendowntime; |
5328 | | CPLUnixTimeToYMDHMS(static_cast<GIntBig>(sTime), |
5329 | | ¤tBrokendowntime); |
5330 | | brokendowntime.tm_year = currentBrokendowntime.tm_year; |
5331 | | brokendowntime.tm_hour = atoi(pszHourOrYear); |
5332 | | brokendowntime.tm_min = atoi(pszHourOrYear + 3); |
5333 | | } |
5334 | | |
5335 | | if (bBrokenDownTimeValid) |
5336 | | nUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
5337 | | else |
5338 | | nUnixTime = 0; |
5339 | | |
5340 | | if (pszNextToken == nullptr) |
5341 | | return false; |
5342 | | |
5343 | | pszFilename = pszNextToken; |
5344 | | |
5345 | | char *pszCurPtr = pszFilename; |
5346 | | while (*pszCurPtr != '\0') |
5347 | | { |
5348 | | // In case of a link, stop before the pointed part of the link. |
5349 | | if (pszPermissions[0] == 'l' && STARTS_WITH(pszCurPtr, " -> ")) |
5350 | | { |
5351 | | break; |
5352 | | } |
5353 | | pszCurPtr++; |
5354 | | } |
5355 | | *pszCurPtr = '\0'; |
5356 | | |
5357 | | return true; |
5358 | | } |
5359 | | |
5360 | | /************************************************************************/ |
5361 | | /* GetURLFromFilename() */ |
5362 | | /************************************************************************/ |
5363 | | |
5364 | | std::string VSICurlFilesystemHandlerBase::GetURLFromFilename( |
5365 | | const std::string &osFilename) const |
5366 | | { |
5367 | | return VSICurlGetURLFromFilename(osFilename.c_str(), nullptr, nullptr, |
5368 | | nullptr, nullptr, nullptr, nullptr, |
5369 | | nullptr, nullptr); |
5370 | | } |
5371 | | |
5372 | | /************************************************************************/ |
5373 | | /* RegisterEmptyDir() */ |
5374 | | /************************************************************************/ |
5375 | | |
5376 | | void VSICurlFilesystemHandlerBase::RegisterEmptyDir( |
5377 | | const std::string &osDirname) |
5378 | | { |
5379 | | CachedDirList cachedDirList; |
5380 | | cachedDirList.bGotFileList = true; |
5381 | | cachedDirList.oFileList.AddString("."); |
5382 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5383 | | } |
5384 | | |
5385 | | /************************************************************************/ |
5386 | | /* GetFileList() */ |
5387 | | /************************************************************************/ |
5388 | | |
5389 | | char **VSICurlFilesystemHandlerBase::GetFileList(const char *pszDirname, |
5390 | | int nMaxFiles, |
5391 | | bool *pbGotFileList) |
5392 | | { |
5393 | | if constexpr (ENABLE_DEBUG) |
5394 | | { |
5395 | | CPLDebug(GetDebugKey(), "GetFileList(%s)", pszDirname); |
5396 | | } |
5397 | | |
5398 | | *pbGotFileList = false; |
5399 | | |
5400 | | bool bListDir = true; |
5401 | | bool bEmptyDir = false; |
5402 | | std::string osURL(VSICurlGetURLFromFilename(pszDirname, nullptr, nullptr, |
5403 | | nullptr, &bListDir, &bEmptyDir, |
5404 | | nullptr, nullptr, nullptr)); |
5405 | | if (bEmptyDir) |
5406 | | { |
5407 | | *pbGotFileList = true; |
5408 | | return CSLAddString(nullptr, "."); |
5409 | | } |
5410 | | if (!bListDir) |
5411 | | return nullptr; |
5412 | | |
5413 | | // Deal with publicly visible Azure directories. |
5414 | | if (STARTS_WITH(osURL.c_str(), "https://")) |
5415 | | { |
5416 | | const char *pszBlobCore = |
5417 | | strstr(osURL.c_str(), ".blob.core.windows.net/"); |
5418 | | if (pszBlobCore) |
5419 | | { |
5420 | | FileProp cachedFileProp; |
5421 | | GetCachedFileProp(osURL.c_str(), cachedFileProp); |
5422 | | if (cachedFileProp.bIsAzureFolder) |
5423 | | { |
5424 | | const char *pszURLWithoutHTTPS = |
5425 | | osURL.c_str() + strlen("https://"); |
5426 | | const std::string osStorageAccount( |
5427 | | pszURLWithoutHTTPS, pszBlobCore - pszURLWithoutHTTPS); |
5428 | | CPLConfigOptionSetter oSetter1("AZURE_NO_SIGN_REQUEST", "YES", |
5429 | | false); |
5430 | | CPLConfigOptionSetter oSetter2("AZURE_STORAGE_ACCOUNT", |
5431 | | osStorageAccount.c_str(), false); |
5432 | | const std::string osVSIAZ(std::string("/vsiaz/").append( |
5433 | | pszBlobCore + strlen(".blob.core.windows.net/"))); |
5434 | | char **papszFileList = VSIReadDirEx(osVSIAZ.c_str(), nMaxFiles); |
5435 | | if (papszFileList) |
5436 | | { |
5437 | | *pbGotFileList = true; |
5438 | | return papszFileList; |
5439 | | } |
5440 | | } |
5441 | | } |
5442 | | } |
5443 | | |
5444 | | // HACK (optimization in fact) for MBTiles driver. |
5445 | | if (strstr(pszDirname, ".tiles.mapbox.com") != nullptr) |
5446 | | return nullptr; |
5447 | | |
5448 | | if (STARTS_WITH(osURL.c_str(), "ftp://")) |
5449 | | { |
5450 | | WriteFuncStruct sWriteFuncData; |
5451 | | sWriteFuncData.pBuffer = nullptr; |
5452 | | |
5453 | | std::string osDirname(osURL); |
5454 | | osDirname += '/'; |
5455 | | |
5456 | | char **papszFileList = nullptr; |
5457 | | |
5458 | | CURLM *hCurlMultiHandle = GetCurlMultiHandleFor(osDirname); |
5459 | | CURL *hCurlHandle = curl_easy_init(); |
5460 | | |
5461 | | for (int iTry = 0; iTry < 2; iTry++) |
5462 | | { |
5463 | | struct curl_slist *headers = |
5464 | | VSICurlSetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5465 | | |
5466 | | // On the first pass, we want to try fetching all the possible |
5467 | | // information (filename, file/directory, size). If that does not |
5468 | | // work, then try again with CURLOPT_DIRLISTONLY set. |
5469 | | if (iTry == 1) |
5470 | | { |
5471 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 1); |
5472 | | } |
5473 | | |
5474 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, |
5475 | | nullptr); |
5476 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5477 | | &sWriteFuncData); |
5478 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5479 | | VSICurlHandleWriteFunc); |
5480 | | |
5481 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5482 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5483 | | szCurlErrBuf); |
5484 | | |
5485 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
5486 | | headers); |
5487 | | |
5488 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
5489 | | |
5490 | | curl_slist_free_all(headers); |
5491 | | |
5492 | | if (sWriteFuncData.pBuffer == nullptr) |
5493 | | { |
5494 | | curl_easy_cleanup(hCurlHandle); |
5495 | | return nullptr; |
5496 | | } |
5497 | | |
5498 | | char *pszLine = sWriteFuncData.pBuffer; |
5499 | | char *c = nullptr; |
5500 | | int nCount = 0; |
5501 | | |
5502 | | if (STARTS_WITH_CI(pszLine, "<!DOCTYPE HTML") || |
5503 | | STARTS_WITH_CI(pszLine, "<HTML>")) |
5504 | | { |
5505 | | papszFileList = |
5506 | | ParseHTMLFileList(pszDirname, nMaxFiles, |
5507 | | sWriteFuncData.pBuffer, pbGotFileList); |
5508 | | break; |
5509 | | } |
5510 | | else if (iTry == 0) |
5511 | | { |
5512 | | CPLStringList oFileList; |
5513 | | *pbGotFileList = true; |
5514 | | |
5515 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5516 | | { |
5517 | | *c = 0; |
5518 | | if (c - pszLine > 0 && c[-1] == '\r') |
5519 | | c[-1] = 0; |
5520 | | |
5521 | | char *pszFilename = nullptr; |
5522 | | bool bSizeValid = false; |
5523 | | GUIntBig nFileSize = 0; |
5524 | | bool bIsDirectory = false; |
5525 | | GIntBig mUnixTime = 0; |
5526 | | if (!VSICurlParseFullFTPLine(pszLine, pszFilename, |
5527 | | bSizeValid, nFileSize, |
5528 | | bIsDirectory, mUnixTime)) |
5529 | | break; |
5530 | | |
5531 | | if (strcmp(pszFilename, ".") != 0 && |
5532 | | strcmp(pszFilename, "..") != 0) |
5533 | | { |
5534 | | if (CPLHasUnbalancedPathTraversal(pszFilename)) |
5535 | | { |
5536 | | CPLError(CE_Warning, CPLE_AppDefined, |
5537 | | "Ignoring '%s' that has a path traversal " |
5538 | | "pattern", |
5539 | | pszFilename); |
5540 | | } |
5541 | | else |
5542 | | { |
5543 | | std::string osCachedFilename = |
5544 | | CPLSPrintf("%s/%s", osURL.c_str(), pszFilename); |
5545 | | |
5546 | | FileProp cachedFileProp; |
5547 | | GetCachedFileProp(osCachedFilename.c_str(), |
5548 | | cachedFileProp); |
5549 | | cachedFileProp.eExists = EXIST_YES; |
5550 | | cachedFileProp.bIsDirectory = bIsDirectory; |
5551 | | cachedFileProp.mTime = |
5552 | | static_cast<time_t>(mUnixTime); |
5553 | | cachedFileProp.bHasComputedFileSize = bSizeValid; |
5554 | | cachedFileProp.fileSize = nFileSize; |
5555 | | SetCachedFileProp(osCachedFilename.c_str(), |
5556 | | cachedFileProp); |
5557 | | |
5558 | | oFileList.AddString(pszFilename); |
5559 | | if constexpr (ENABLE_DEBUG_VERBOSE) |
5560 | | { |
5561 | | struct tm brokendowntime; |
5562 | | CPLUnixTimeToYMDHMS(mUnixTime, &brokendowntime); |
5563 | | CPLDebug( |
5564 | | GetDebugKey(), |
5565 | | "File[%d] = %s, is_dir = %d, size " |
5566 | | "= " CPL_FRMT_GUIB |
5567 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
5568 | | nCount, pszFilename, bIsDirectory ? 1 : 0, |
5569 | | nFileSize, brokendowntime.tm_year + 1900, |
5570 | | brokendowntime.tm_mon + 1, |
5571 | | brokendowntime.tm_mday, |
5572 | | brokendowntime.tm_hour, |
5573 | | brokendowntime.tm_min, |
5574 | | brokendowntime.tm_sec); |
5575 | | } |
5576 | | |
5577 | | nCount++; |
5578 | | |
5579 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
5580 | | break; |
5581 | | } |
5582 | | } |
5583 | | |
5584 | | pszLine = c + 1; |
5585 | | } |
5586 | | |
5587 | | if (c == nullptr) |
5588 | | { |
5589 | | papszFileList = oFileList.StealList(); |
5590 | | break; |
5591 | | } |
5592 | | } |
5593 | | else |
5594 | | { |
5595 | | CPLStringList oFileList; |
5596 | | *pbGotFileList = true; |
5597 | | |
5598 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5599 | | { |
5600 | | *c = 0; |
5601 | | if (c - pszLine > 0 && c[-1] == '\r') |
5602 | | c[-1] = 0; |
5603 | | |
5604 | | if (strcmp(pszLine, ".") != 0 && strcmp(pszLine, "..") != 0) |
5605 | | { |
5606 | | oFileList.AddString(pszLine); |
5607 | | if constexpr (ENABLE_DEBUG_VERBOSE) |
5608 | | { |
5609 | | CPLDebug(GetDebugKey(), "File[%d] = %s", nCount, |
5610 | | pszLine); |
5611 | | } |
5612 | | nCount++; |
5613 | | } |
5614 | | |
5615 | | pszLine = c + 1; |
5616 | | } |
5617 | | |
5618 | | papszFileList = oFileList.StealList(); |
5619 | | } |
5620 | | |
5621 | | CPLFree(sWriteFuncData.pBuffer); |
5622 | | sWriteFuncData.pBuffer = nullptr; |
5623 | | } |
5624 | | |
5625 | | CPLFree(sWriteFuncData.pBuffer); |
5626 | | curl_easy_cleanup(hCurlHandle); |
5627 | | |
5628 | | return papszFileList; |
5629 | | } |
5630 | | |
5631 | | // Try to recognize HTML pages that list the content of a directory. |
5632 | | // Currently this supports what Apache and shttpd can return. |
5633 | | else if (STARTS_WITH(osURL.c_str(), "http://") || |
5634 | | STARTS_WITH(osURL.c_str(), "https://")) |
5635 | | { |
5636 | | std::string osDirname(std::move(osURL)); |
5637 | | osDirname += '/'; |
5638 | | |
5639 | | CURLM *hCurlMultiHandle = GetCurlMultiHandleFor(osDirname); |
5640 | | CURL *hCurlHandle = curl_easy_init(); |
5641 | | |
5642 | | struct curl_slist *headers = |
5643 | | VSICurlSetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5644 | | |
5645 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
5646 | | |
5647 | | WriteFuncStruct sWriteFuncData; |
5648 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
5649 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5650 | | &sWriteFuncData); |
5651 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5652 | | VSICurlHandleWriteFunc); |
5653 | | |
5654 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5655 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5656 | | szCurlErrBuf); |
5657 | | |
5658 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
5659 | | |
5660 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
5661 | | |
5662 | | curl_slist_free_all(headers); |
5663 | | |
5664 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
5665 | | |
5666 | | if (sWriteFuncData.pBuffer == nullptr) |
5667 | | { |
5668 | | curl_easy_cleanup(hCurlHandle); |
5669 | | return nullptr; |
5670 | | } |
5671 | | |
5672 | | char **papszFileList = nullptr; |
5673 | | if (STARTS_WITH_CI(sWriteFuncData.pBuffer, "<?xml") && |
5674 | | strstr(sWriteFuncData.pBuffer, "<ListBucketResult") != nullptr) |
5675 | | { |
5676 | | CPLStringList osFileList; |
5677 | | std::string osBaseURL(pszDirname); |
5678 | | osBaseURL += "/"; |
5679 | | bool bIsTruncated = true; |
5680 | | bool ret = AnalyseS3FileList( |
5681 | | osBaseURL, sWriteFuncData.pBuffer, osFileList, nMaxFiles, |
5682 | | GetS3IgnoredStorageClasses(), bIsTruncated); |
5683 | | // If the list is truncated, then don't report it. |
5684 | | if (ret && !bIsTruncated) |
5685 | | { |
5686 | | if (osFileList.empty()) |
5687 | | { |
5688 | | // To avoid an error to be reported |
5689 | | osFileList.AddString("."); |
5690 | | } |
5691 | | papszFileList = osFileList.StealList(); |
5692 | | *pbGotFileList = true; |
5693 | | } |
5694 | | } |
5695 | | else |
5696 | | { |
5697 | | papszFileList = ParseHTMLFileList( |
5698 | | pszDirname, nMaxFiles, sWriteFuncData.pBuffer, pbGotFileList); |
5699 | | } |
5700 | | |
5701 | | CPLFree(sWriteFuncData.pBuffer); |
5702 | | curl_easy_cleanup(hCurlHandle); |
5703 | | return papszFileList; |
5704 | | } |
5705 | | |
5706 | | return nullptr; |
5707 | | } |
5708 | | |
5709 | | /************************************************************************/ |
5710 | | /* GetS3IgnoredStorageClasses() */ |
5711 | | /************************************************************************/ |
5712 | | |
5713 | | std::set<std::string> VSICurlFilesystemHandlerBase::GetS3IgnoredStorageClasses() |
5714 | | { |
5715 | | std::set<std::string> oSetIgnoredStorageClasses; |
5716 | | const char *pszIgnoredStorageClasses = |
5717 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_STORAGE_CLASSES", nullptr); |
5718 | | const char *pszIgnoreGlacierStorage = |
5719 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE", nullptr); |
5720 | | CPLStringList aosIgnoredStorageClasses( |
5721 | | CSLTokenizeString2(pszIgnoredStorageClasses ? pszIgnoredStorageClasses |
5722 | | : "GLACIER,DEEP_ARCHIVE", |
5723 | | ",", 0)); |
5724 | | for (int i = 0; i < aosIgnoredStorageClasses.size(); ++i) |
5725 | | oSetIgnoredStorageClasses.insert(aosIgnoredStorageClasses[i]); |
5726 | | if (pszIgnoredStorageClasses == nullptr && |
5727 | | pszIgnoreGlacierStorage != nullptr && |
5728 | | !CPLTestBool(pszIgnoreGlacierStorage)) |
5729 | | { |
5730 | | oSetIgnoredStorageClasses.clear(); |
5731 | | } |
5732 | | return oSetIgnoredStorageClasses; |
5733 | | } |
5734 | | |
5735 | | /************************************************************************/ |
5736 | | /* Stat() */ |
5737 | | /************************************************************************/ |
5738 | | |
5739 | | int VSICurlFilesystemHandlerBase::Stat(const char *pszFilename, |
5740 | | VSIStatBufL *pStatBuf, int nFlags) |
5741 | | { |
5742 | | if (!cpl::starts_with(std::string_view(pszFilename), GetFSPrefix()) && |
5743 | | !StartsWithVSICurlPrefix(pszFilename)) |
5744 | | { |
5745 | | return -1; |
5746 | | } |
5747 | | |
5748 | | memset(pStatBuf, 0, sizeof(VSIStatBufL)); |
5749 | | |
5750 | | if ((nFlags & VSI_STAT_CACHE_ONLY) != 0) |
5751 | | { |
5752 | | cpl::FileProp oFileProp; |
5753 | | if (!GetCachedFileProp(GetURLFromFilename(pszFilename).c_str(), |
5754 | | oFileProp) || |
5755 | | oFileProp.eExists != EXIST_YES) |
5756 | | { |
5757 | | return -1; |
5758 | | } |
5759 | | pStatBuf->st_mode = static_cast<unsigned short>(oFileProp.nMode); |
5760 | | pStatBuf->st_mtime = oFileProp.mTime; |
5761 | | pStatBuf->st_size = oFileProp.fileSize; |
5762 | | return 0; |
5763 | | } |
5764 | | |
5765 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5766 | | NetworkStatisticsAction oContextAction("Stat"); |
5767 | | |
5768 | | const std::string osFilename(pszFilename); |
5769 | | |
5770 | | if (!IsAllowedFilename(pszFilename)) |
5771 | | return -1; |
5772 | | |
5773 | | bool bListDir = true; |
5774 | | bool bEmptyDir = false; |
5775 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
5776 | | nullptr, &bListDir, &bEmptyDir, |
5777 | | nullptr, nullptr, nullptr)); |
5778 | | |
5779 | | const char *pszOptionVal = VSIGetPathSpecificOption( |
5780 | | pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", "NO"); |
5781 | | const bool bSkipReadDir = |
5782 | | !bListDir || bEmptyDir || EQUAL(pszOptionVal, "EMPTY_DIR") || |
5783 | | CPLTestBool(pszOptionVal) || !AllowCachedDataFor(pszFilename); |
5784 | | |
5785 | | // Does it look like a FTP directory? |
5786 | | if (STARTS_WITH(osURL.c_str(), "ftp://") && osFilename.back() == '/' && |
5787 | | !bSkipReadDir) |
5788 | | { |
5789 | | char **papszFileList = ReadDirEx(osFilename.c_str(), 0); |
5790 | | if (papszFileList) |
5791 | | { |
5792 | | pStatBuf->st_mode = S_IFDIR; |
5793 | | pStatBuf->st_size = 0; |
5794 | | |
5795 | | CSLDestroy(papszFileList); |
5796 | | |
5797 | | return 0; |
5798 | | } |
5799 | | return -1; |
5800 | | } |
5801 | | else if (strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
5802 | | !STARTS_WITH_CI(CPLGetExtensionSafe(osFilename.c_str()).c_str(), |
5803 | | "zip") && |
5804 | | strstr(osFilename.c_str(), ".zip.") != nullptr && |
5805 | | strstr(osFilename.c_str(), ".ZIP.") != nullptr && !bSkipReadDir) |
5806 | | { |
5807 | | bool bGotFileList = false; |
5808 | | char **papszFileList = ReadDirInternal( |
5809 | | CPLGetDirnameSafe(osFilename.c_str()).c_str(), 0, &bGotFileList); |
5810 | | const bool bFound = |
5811 | | VSICurlIsFileInList(papszFileList, |
5812 | | CPLGetFilename(osFilename.c_str())) != -1; |
5813 | | CSLDestroy(papszFileList); |
5814 | | if (bGotFileList && !bFound) |
5815 | | { |
5816 | | return -1; |
5817 | | } |
5818 | | } |
5819 | | |
5820 | | VSICurlHandle *poHandle = CreateFileHandle(osFilename.c_str()); |
5821 | | if (poHandle == nullptr) |
5822 | | return -1; |
5823 | | |
5824 | | if (poHandle->IsKnownFileSize() || |
5825 | | ((nFlags & VSI_STAT_SIZE_FLAG) && !poHandle->IsDirectory() && |
5826 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES")))) |
5827 | | { |
5828 | | pStatBuf->st_size = poHandle->GetFileSize(true); |
5829 | | } |
5830 | | |
5831 | | const int nRet = |
5832 | | poHandle->Exists((nFlags & VSI_STAT_SET_ERROR_FLAG) > 0) ? 0 : -1; |
5833 | | pStatBuf->st_mtime = poHandle->GetMTime(); |
5834 | | pStatBuf->st_mode = static_cast<unsigned short>(poHandle->GetMode()); |
5835 | | if (pStatBuf->st_mode == 0) |
5836 | | pStatBuf->st_mode = poHandle->IsDirectory() ? S_IFDIR : S_IFREG; |
5837 | | delete poHandle; |
5838 | | return nRet; |
5839 | | } |
5840 | | |
5841 | | /************************************************************************/ |
5842 | | /* ReadDirInternal() */ |
5843 | | /************************************************************************/ |
5844 | | |
5845 | | char **VSICurlFilesystemHandlerBase::ReadDirInternal(const char *pszDirname, |
5846 | | int nMaxFiles, |
5847 | | bool *pbGotFileList) |
5848 | | { |
5849 | | std::string osDirname(pszDirname); |
5850 | | |
5851 | | // Replace a/b/../c by a/c |
5852 | | const auto posSlashDotDot = osDirname.find("/.."); |
5853 | | if (posSlashDotDot != std::string::npos && posSlashDotDot >= 1) |
5854 | | { |
5855 | | const auto posPrecedingSlash = |
5856 | | osDirname.find_last_of('/', posSlashDotDot - 1); |
5857 | | if (posPrecedingSlash != std::string::npos && posPrecedingSlash >= 1) |
5858 | | { |
5859 | | osDirname.erase(osDirname.begin() + posPrecedingSlash, |
5860 | | osDirname.begin() + posSlashDotDot + strlen("/..")); |
5861 | | } |
5862 | | } |
5863 | | |
5864 | | std::string osDirnameOri(osDirname); |
5865 | | if (osDirname + "/" == GetFSPrefix()) |
5866 | | { |
5867 | | osDirname += "/"; |
5868 | | } |
5869 | | else if (osDirname != GetFSPrefix()) |
5870 | | { |
5871 | | while (!osDirname.empty() && osDirname.back() == '/') |
5872 | | osDirname.erase(osDirname.size() - 1); |
5873 | | } |
5874 | | |
5875 | | if (osDirname.size() < GetFSPrefix().size()) |
5876 | | { |
5877 | | if (pbGotFileList) |
5878 | | *pbGotFileList = true; |
5879 | | return nullptr; |
5880 | | } |
5881 | | |
5882 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5883 | | NetworkStatisticsAction oContextAction("ReadDir"); |
5884 | | |
5885 | | CPLMutexHolder oHolder(&hMutex); |
5886 | | |
5887 | | // If we know the file exists and is not a directory, |
5888 | | // then don't try to list its content. |
5889 | | FileProp cachedFileProp; |
5890 | | if (GetCachedFileProp(GetURLFromFilename(osDirname.c_str()).c_str(), |
5891 | | cachedFileProp) && |
5892 | | cachedFileProp.eExists == EXIST_YES && !cachedFileProp.bIsDirectory) |
5893 | | { |
5894 | | if (osDirnameOri != osDirname) |
5895 | | { |
5896 | | if (GetCachedFileProp((GetURLFromFilename(osDirname) + "/").c_str(), |
5897 | | cachedFileProp) && |
5898 | | cachedFileProp.eExists == EXIST_YES && |
5899 | | !cachedFileProp.bIsDirectory) |
5900 | | { |
5901 | | if (pbGotFileList) |
5902 | | *pbGotFileList = true; |
5903 | | return nullptr; |
5904 | | } |
5905 | | } |
5906 | | else |
5907 | | { |
5908 | | if (pbGotFileList) |
5909 | | *pbGotFileList = true; |
5910 | | return nullptr; |
5911 | | } |
5912 | | } |
5913 | | |
5914 | | CachedDirList cachedDirList; |
5915 | | if (!GetCachedDirList(osDirname.c_str(), cachedDirList)) |
5916 | | { |
5917 | | cachedDirList.oFileList.Assign(GetFileList(osDirname.c_str(), nMaxFiles, |
5918 | | &cachedDirList.bGotFileList), |
5919 | | true); |
5920 | | if (cachedDirList.bGotFileList && cachedDirList.oFileList.empty()) |
5921 | | { |
5922 | | // To avoid an error to be reported |
5923 | | cachedDirList.oFileList.AddString("."); |
5924 | | } |
5925 | | if (nMaxFiles <= 0 || cachedDirList.oFileList.size() < nMaxFiles) |
5926 | | { |
5927 | | // Only cache content if we didn't hit the limitation |
5928 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5929 | | } |
5930 | | } |
5931 | | |
5932 | | if (pbGotFileList) |
5933 | | *pbGotFileList = cachedDirList.bGotFileList; |
5934 | | |
5935 | | return CSLDuplicate(cachedDirList.oFileList.List()); |
5936 | | } |
5937 | | |
5938 | | /************************************************************************/ |
5939 | | /* InvalidateDirContent() */ |
5940 | | /************************************************************************/ |
5941 | | |
5942 | | void VSICurlFilesystemHandlerBase::InvalidateDirContent( |
5943 | | const std::string &osDirname) |
5944 | | { |
5945 | | CPLMutexHolder oHolder(&hMutex); |
5946 | | |
5947 | | CachedDirList oCachedDirList; |
5948 | | if (oCacheDirList.tryGet(osDirname, oCachedDirList)) |
5949 | | { |
5950 | | nCachedFilesInDirList -= oCachedDirList.oFileList.size(); |
5951 | | oCacheDirList.remove(osDirname); |
5952 | | } |
5953 | | } |
5954 | | |
5955 | | /************************************************************************/ |
5956 | | /* ReadDirEx() */ |
5957 | | /************************************************************************/ |
5958 | | |
5959 | | char **VSICurlFilesystemHandlerBase::ReadDirEx(const char *pszDirname, |
5960 | | int nMaxFiles) |
5961 | | { |
5962 | | return ReadDirInternal(pszDirname, nMaxFiles, nullptr); |
5963 | | } |
5964 | | |
5965 | | /************************************************************************/ |
5966 | | /* SiblingFiles() */ |
5967 | | /************************************************************************/ |
5968 | | |
5969 | | char **VSICurlFilesystemHandlerBase::SiblingFiles(const char *pszFilename) |
5970 | | { |
5971 | | /* Small optimization to avoid unnecessary stat'ing from PAux or ENVI */ |
5972 | | /* drivers. The MBTiles driver needs no companion file. */ |
5973 | | if (EQUAL(CPLGetExtensionSafe(pszFilename).c_str(), "mbtiles")) |
5974 | | { |
5975 | | return static_cast<char **>(CPLCalloc(1, sizeof(char *))); |
5976 | | } |
5977 | | return nullptr; |
5978 | | } |
5979 | | |
5980 | | /************************************************************************/ |
5981 | | /* GetFileMetadata() */ |
5982 | | /************************************************************************/ |
5983 | | |
5984 | | char **VSICurlFilesystemHandlerBase::GetFileMetadata(const char *pszFilename, |
5985 | | const char *pszDomain, |
5986 | | CSLConstList) |
5987 | | { |
5988 | | if (pszDomain == nullptr || !EQUAL(pszDomain, "HEADERS")) |
5989 | | return nullptr; |
5990 | | std::unique_ptr<VSICurlHandle> poHandle(CreateFileHandle(pszFilename)); |
5991 | | if (poHandle == nullptr) |
5992 | | return nullptr; |
5993 | | |
5994 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5995 | | NetworkStatisticsAction oContextAction("GetFileMetadata"); |
5996 | | |
5997 | | poHandle->GetFileSizeOrHeaders(true, true); |
5998 | | return CSLDuplicate(poHandle->GetHeaders().List()); |
5999 | | } |
6000 | | |
6001 | | /************************************************************************/ |
6002 | | /* VSIAppendWriteHandle() */ |
6003 | | /************************************************************************/ |
6004 | | |
6005 | | VSIAppendWriteHandle::VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS, |
6006 | | const char *pszFSPrefix, |
6007 | | const char *pszFilename, |
6008 | | int nChunkSize) |
6009 | | : m_poFS(poFS), m_osFSPrefix(pszFSPrefix), m_osFilename(pszFilename), |
6010 | | m_oRetryParameters(CPLStringList(CPLHTTPGetOptionsFromEnv(pszFilename))), |
6011 | | m_nBufferSize(nChunkSize) |
6012 | | { |
6013 | | m_pabyBuffer = static_cast<GByte *>(VSIMalloc(m_nBufferSize)); |
6014 | | if (m_pabyBuffer == nullptr) |
6015 | | { |
6016 | | CPLError(CE_Failure, CPLE_AppDefined, |
6017 | | "Cannot allocate working buffer for %s writing", |
6018 | | m_osFSPrefix.c_str()); |
6019 | | } |
6020 | | } |
6021 | | |
6022 | | /************************************************************************/ |
6023 | | /* ~VSIAppendWriteHandle() */ |
6024 | | /************************************************************************/ |
6025 | | |
6026 | | VSIAppendWriteHandle::~VSIAppendWriteHandle() |
6027 | | { |
6028 | | /* WARNING: implementation should call Close() themselves */ |
6029 | | /* cannot be done safely from here, since Send() can be called. */ |
6030 | | CPLFree(m_pabyBuffer); |
6031 | | } |
6032 | | |
6033 | | /************************************************************************/ |
6034 | | /* Seek() */ |
6035 | | /************************************************************************/ |
6036 | | |
6037 | | int VSIAppendWriteHandle::Seek(vsi_l_offset nOffset, int nWhence) |
6038 | | { |
6039 | | if (!((nWhence == SEEK_SET && nOffset == m_nCurOffset) || |
6040 | | (nWhence == SEEK_CUR && nOffset == 0) || |
6041 | | (nWhence == SEEK_END && nOffset == 0))) |
6042 | | { |
6043 | | CPLError(CE_Failure, CPLE_NotSupported, |
6044 | | "Seek not supported on writable %s files", |
6045 | | m_osFSPrefix.c_str()); |
6046 | | m_bError = true; |
6047 | | return -1; |
6048 | | } |
6049 | | return 0; |
6050 | | } |
6051 | | |
6052 | | /************************************************************************/ |
6053 | | /* Tell() */ |
6054 | | /************************************************************************/ |
6055 | | |
6056 | | vsi_l_offset VSIAppendWriteHandle::Tell() |
6057 | | { |
6058 | | return m_nCurOffset; |
6059 | | } |
6060 | | |
6061 | | /************************************************************************/ |
6062 | | /* Read() */ |
6063 | | /************************************************************************/ |
6064 | | |
6065 | | size_t VSIAppendWriteHandle::Read(void * /* pBuffer */, size_t /* nBytes */) |
6066 | | { |
6067 | | CPLError(CE_Failure, CPLE_NotSupported, |
6068 | | "Read not supported on writable %s files", m_osFSPrefix.c_str()); |
6069 | | m_bError = true; |
6070 | | return 0; |
6071 | | } |
6072 | | |
6073 | | /************************************************************************/ |
6074 | | /* ReadCallBackBuffer() */ |
6075 | | /************************************************************************/ |
6076 | | |
6077 | | size_t VSIAppendWriteHandle::ReadCallBackBuffer(char *buffer, size_t size, |
6078 | | size_t nitems, void *instream) |
6079 | | { |
6080 | | VSIAppendWriteHandle *poThis = |
6081 | | static_cast<VSIAppendWriteHandle *>(instream); |
6082 | | const int nSizeMax = static_cast<int>(size * nitems); |
6083 | | const int nSizeToWrite = std::min( |
6084 | | nSizeMax, poThis->m_nBufferOff - poThis->m_nBufferOffReadCallback); |
6085 | | memcpy(buffer, poThis->m_pabyBuffer + poThis->m_nBufferOffReadCallback, |
6086 | | nSizeToWrite); |
6087 | | poThis->m_nBufferOffReadCallback += nSizeToWrite; |
6088 | | return nSizeToWrite; |
6089 | | } |
6090 | | |
6091 | | /************************************************************************/ |
6092 | | /* Write() */ |
6093 | | /************************************************************************/ |
6094 | | |
6095 | | size_t VSIAppendWriteHandle::Write(const void *pBuffer, size_t nBytes) |
6096 | | { |
6097 | | if (m_bError) |
6098 | | return 0; |
6099 | | |
6100 | | size_t nBytesToWrite = nBytes; |
6101 | | if (nBytesToWrite == 0) |
6102 | | return 0; |
6103 | | |
6104 | | const GByte *pabySrcBuffer = reinterpret_cast<const GByte *>(pBuffer); |
6105 | | while (nBytesToWrite > 0) |
6106 | | { |
6107 | | if (m_nBufferOff == m_nBufferSize) |
6108 | | { |
6109 | | if (!Send(false)) |
6110 | | { |
6111 | | m_bError = true; |
6112 | | return 0; |
6113 | | } |
6114 | | m_nBufferOff = 0; |
6115 | | } |
6116 | | |
6117 | | const int nToWriteInBuffer = static_cast<int>(std::min( |
6118 | | static_cast<size_t>(m_nBufferSize - m_nBufferOff), nBytesToWrite)); |
6119 | | memcpy(m_pabyBuffer + m_nBufferOff, pabySrcBuffer, nToWriteInBuffer); |
6120 | | pabySrcBuffer += nToWriteInBuffer; |
6121 | | m_nBufferOff += nToWriteInBuffer; |
6122 | | m_nCurOffset += nToWriteInBuffer; |
6123 | | nBytesToWrite -= nToWriteInBuffer; |
6124 | | } |
6125 | | return nBytes; |
6126 | | } |
6127 | | |
6128 | | /************************************************************************/ |
6129 | | /* Close() */ |
6130 | | /************************************************************************/ |
6131 | | |
6132 | | int VSIAppendWriteHandle::Close() |
6133 | | { |
6134 | | int nRet = 0; |
6135 | | if (!m_bClosed) |
6136 | | { |
6137 | | m_bClosed = true; |
6138 | | if (!m_bError && !Send(true)) |
6139 | | nRet = -1; |
6140 | | } |
6141 | | return nRet; |
6142 | | } |
6143 | | |
6144 | | /************************************************************************/ |
6145 | | /* CurlRequestHelper() */ |
6146 | | /************************************************************************/ |
6147 | | |
6148 | | CurlRequestHelper::CurlRequestHelper() |
6149 | | { |
6150 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
6151 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
6152 | | nullptr); |
6153 | | } |
6154 | | |
6155 | | /************************************************************************/ |
6156 | | /* ~CurlRequestHelper() */ |
6157 | | /************************************************************************/ |
6158 | | |
6159 | | CurlRequestHelper::~CurlRequestHelper() |
6160 | | { |
6161 | | CPLFree(sWriteFuncData.pBuffer); |
6162 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
6163 | | } |
6164 | | |
6165 | | /************************************************************************/ |
6166 | | /* perform() */ |
6167 | | /************************************************************************/ |
6168 | | |
6169 | | long CurlRequestHelper::perform(CURL *hCurlHandle, struct curl_slist *headers, |
6170 | | VSICurlFilesystemHandlerBase *poFS, |
6171 | | IVSIS3LikeHandleHelper *poS3HandleHelper) |
6172 | | { |
6173 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
6174 | | |
6175 | | poS3HandleHelper->ResetQueryParameters(); |
6176 | | |
6177 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
6178 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
6179 | | VSICurlHandleWriteFunc); |
6180 | | |
6181 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
6182 | | &sWriteFuncHeaderData); |
6183 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
6184 | | VSICurlHandleWriteFunc); |
6185 | | |
6186 | | szCurlErrBuf[0] = '\0'; |
6187 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
6188 | | |
6189 | | VSICURLMultiPerform(poFS->GetCurlMultiHandleFor(poS3HandleHelper->GetURL()), |
6190 | | hCurlHandle); |
6191 | | |
6192 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
6193 | | |
6194 | | curl_slist_free_all(headers); |
6195 | | |
6196 | | long response_code = 0; |
6197 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
6198 | | return response_code; |
6199 | | } |
6200 | | |
6201 | | /************************************************************************/ |
6202 | | /* NetworkStatisticsLogger */ |
6203 | | /************************************************************************/ |
6204 | | |
6205 | | // Global variable |
6206 | | NetworkStatisticsLogger NetworkStatisticsLogger::gInstance{}; |
6207 | | int NetworkStatisticsLogger::gnEnabled = -1; // unknown state |
6208 | | |
6209 | | static void ShowNetworkStats() |
6210 | | { |
6211 | | printf("Network statistics:\n%s\n", // ok |
6212 | | NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
6213 | | } |
6214 | | |
6215 | | void NetworkStatisticsLogger::ReadEnabled() |
6216 | | { |
6217 | | const bool bShowNetworkStats = |
6218 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_SHOW_NETWORK_STATS", "NO")); |
6219 | | gnEnabled = |
6220 | | (bShowNetworkStats || CPLTestBool(CPLGetConfigOption( |
6221 | | "CPL_VSIL_NETWORK_STATS_ENABLED", "NO"))) |
6222 | | ? TRUE |
6223 | | : FALSE; |
6224 | | if (bShowNetworkStats) |
6225 | | { |
6226 | | static bool bRegistered = false; |
6227 | | if (!bRegistered) |
6228 | | { |
6229 | | bRegistered = true; |
6230 | | atexit(ShowNetworkStats); |
6231 | | } |
6232 | | } |
6233 | | } |
6234 | | |
6235 | | void NetworkStatisticsLogger::EnterFileSystem(const char *pszName) |
6236 | | { |
6237 | | if (!IsEnabled()) |
6238 | | return; |
6239 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6240 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
6241 | | ContextPathItem(ContextPathType::FILESYSTEM, pszName)); |
6242 | | } |
6243 | | |
6244 | | void NetworkStatisticsLogger::LeaveFileSystem() |
6245 | | { |
6246 | | if (!IsEnabled()) |
6247 | | return; |
6248 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6249 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
6250 | | } |
6251 | | |
6252 | | void NetworkStatisticsLogger::EnterFile(const char *pszName) |
6253 | | { |
6254 | | if (!IsEnabled()) |
6255 | | return; |
6256 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6257 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
6258 | | ContextPathItem(ContextPathType::FILE, pszName)); |
6259 | | } |
6260 | | |
6261 | | void NetworkStatisticsLogger::LeaveFile() |
6262 | | { |
6263 | | if (!IsEnabled()) |
6264 | | return; |
6265 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6266 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
6267 | | } |
6268 | | |
6269 | | void NetworkStatisticsLogger::EnterAction(const char *pszName) |
6270 | | { |
6271 | | if (!IsEnabled()) |
6272 | | return; |
6273 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6274 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
6275 | | ContextPathItem(ContextPathType::ACTION, pszName)); |
6276 | | } |
6277 | | |
6278 | | void NetworkStatisticsLogger::LeaveAction() |
6279 | | { |
6280 | | if (!IsEnabled()) |
6281 | | return; |
6282 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6283 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
6284 | | } |
6285 | | |
6286 | | std::vector<NetworkStatisticsLogger::Counters *> |
6287 | | NetworkStatisticsLogger::GetCountersForContext() |
6288 | | { |
6289 | | std::vector<Counters *> v; |
6290 | | const auto &contextPath = gInstance.m_mapThreadIdToContextPath[CPLGetPID()]; |
6291 | | |
6292 | | Stats *curStats = &m_stats; |
6293 | | v.push_back(&(curStats->counters)); |
6294 | | |
6295 | | bool inFileSystem = false; |
6296 | | bool inFile = false; |
6297 | | bool inAction = false; |
6298 | | for (const auto &item : contextPath) |
6299 | | { |
6300 | | if (item.eType == ContextPathType::FILESYSTEM) |
6301 | | { |
6302 | | if (inFileSystem) |
6303 | | continue; |
6304 | | inFileSystem = true; |
6305 | | } |
6306 | | else if (item.eType == ContextPathType::FILE) |
6307 | | { |
6308 | | if (inFile) |
6309 | | continue; |
6310 | | inFile = true; |
6311 | | } |
6312 | | else if (item.eType == ContextPathType::ACTION) |
6313 | | { |
6314 | | if (inAction) |
6315 | | continue; |
6316 | | inAction = true; |
6317 | | } |
6318 | | |
6319 | | curStats = &(curStats->children[item]); |
6320 | | v.push_back(&(curStats->counters)); |
6321 | | } |
6322 | | |
6323 | | return v; |
6324 | | } |
6325 | | |
6326 | | void NetworkStatisticsLogger::LogGET(size_t nDownloadedBytes) |
6327 | | { |
6328 | | if (!IsEnabled()) |
6329 | | return; |
6330 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6331 | | for (auto counters : gInstance.GetCountersForContext()) |
6332 | | { |
6333 | | counters->nGET++; |
6334 | | counters->nGETDownloadedBytes += nDownloadedBytes; |
6335 | | } |
6336 | | } |
6337 | | |
6338 | | void NetworkStatisticsLogger::LogPUT(size_t nUploadedBytes) |
6339 | | { |
6340 | | if (!IsEnabled()) |
6341 | | return; |
6342 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6343 | | for (auto counters : gInstance.GetCountersForContext()) |
6344 | | { |
6345 | | counters->nPUT++; |
6346 | | counters->nPUTUploadedBytes += nUploadedBytes; |
6347 | | } |
6348 | | } |
6349 | | |
6350 | | void NetworkStatisticsLogger::LogHEAD() |
6351 | | { |
6352 | | if (!IsEnabled()) |
6353 | | return; |
6354 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6355 | | for (auto counters : gInstance.GetCountersForContext()) |
6356 | | { |
6357 | | counters->nHEAD++; |
6358 | | } |
6359 | | } |
6360 | | |
6361 | | void NetworkStatisticsLogger::LogPOST(size_t nUploadedBytes, |
6362 | | size_t nDownloadedBytes) |
6363 | | { |
6364 | | if (!IsEnabled()) |
6365 | | return; |
6366 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6367 | | for (auto counters : gInstance.GetCountersForContext()) |
6368 | | { |
6369 | | counters->nPOST++; |
6370 | | counters->nPOSTUploadedBytes += nUploadedBytes; |
6371 | | counters->nPOSTDownloadedBytes += nDownloadedBytes; |
6372 | | } |
6373 | | } |
6374 | | |
6375 | | void NetworkStatisticsLogger::LogDELETE() |
6376 | | { |
6377 | | if (!IsEnabled()) |
6378 | | return; |
6379 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6380 | | for (auto counters : gInstance.GetCountersForContext()) |
6381 | | { |
6382 | | counters->nDELETE++; |
6383 | | } |
6384 | | } |
6385 | | |
6386 | | void NetworkStatisticsLogger::Reset() |
6387 | | { |
6388 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6389 | | gInstance.m_stats = Stats(); |
6390 | | gnEnabled = -1; |
6391 | | } |
6392 | | |
6393 | | void NetworkStatisticsLogger::Stats::AsJSON(CPLJSONObject &oJSON) const |
6394 | | { |
6395 | | CPLJSONObject oMethods; |
6396 | | if (counters.nHEAD) |
6397 | | oMethods.Add("HEAD/count", counters.nHEAD); |
6398 | | if (counters.nGET) |
6399 | | oMethods.Add("GET/count", counters.nGET); |
6400 | | if (counters.nGETDownloadedBytes) |
6401 | | oMethods.Add("GET/downloaded_bytes", counters.nGETDownloadedBytes); |
6402 | | if (counters.nPUT) |
6403 | | oMethods.Add("PUT/count", counters.nPUT); |
6404 | | if (counters.nPUTUploadedBytes) |
6405 | | oMethods.Add("PUT/uploaded_bytes", counters.nPUTUploadedBytes); |
6406 | | if (counters.nPOST) |
6407 | | oMethods.Add("POST/count", counters.nPOST); |
6408 | | if (counters.nPOSTUploadedBytes) |
6409 | | oMethods.Add("POST/uploaded_bytes", counters.nPOSTUploadedBytes); |
6410 | | if (counters.nPOSTDownloadedBytes) |
6411 | | oMethods.Add("POST/downloaded_bytes", counters.nPOSTDownloadedBytes); |
6412 | | if (counters.nDELETE) |
6413 | | oMethods.Add("DELETE/count", counters.nDELETE); |
6414 | | oJSON.Add("methods", oMethods); |
6415 | | CPLJSONObject oFiles; |
6416 | | bool bFilesAdded = false; |
6417 | | for (const auto &kv : children) |
6418 | | { |
6419 | | CPLJSONObject childJSON; |
6420 | | kv.second.AsJSON(childJSON); |
6421 | | if (kv.first.eType == ContextPathType::FILESYSTEM) |
6422 | | { |
6423 | | std::string osName(kv.first.osName); |
6424 | | if (!osName.empty() && osName[0] == '/') |
6425 | | osName = osName.substr(1); |
6426 | | if (!osName.empty() && osName.back() == '/') |
6427 | | osName.pop_back(); |
6428 | | oJSON.Add(("handlers/" + osName).c_str(), childJSON); |
6429 | | } |
6430 | | else if (kv.first.eType == ContextPathType::FILE) |
6431 | | { |
6432 | | if (!bFilesAdded) |
6433 | | { |
6434 | | bFilesAdded = true; |
6435 | | oJSON.Add("files", oFiles); |
6436 | | } |
6437 | | oFiles.AddNoSplitName(kv.first.osName.c_str(), childJSON); |
6438 | | } |
6439 | | else if (kv.first.eType == ContextPathType::ACTION) |
6440 | | { |
6441 | | oJSON.Add(("actions/" + kv.first.osName).c_str(), childJSON); |
6442 | | } |
6443 | | } |
6444 | | } |
6445 | | |
6446 | | std::string NetworkStatisticsLogger::GetReportAsSerializedJSON() |
6447 | | { |
6448 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6449 | | |
6450 | | CPLJSONObject oJSON; |
6451 | | gInstance.m_stats.AsJSON(oJSON); |
6452 | | return oJSON.Format(CPLJSONObject::PrettyFormat::Pretty); |
6453 | | } |
6454 | | |
6455 | | } /* end of namespace cpl */ |
6456 | | |
6457 | | /************************************************************************/ |
6458 | | /* VSICurlParseUnixPermissions() */ |
6459 | | /************************************************************************/ |
6460 | | |
6461 | | int VSICurlParseUnixPermissions(const char *pszPermissions) |
6462 | | { |
6463 | | if (strlen(pszPermissions) != 9) |
6464 | | return 0; |
6465 | | int nMode = 0; |
6466 | | if (pszPermissions[0] == 'r') |
6467 | | nMode |= S_IRUSR; |
6468 | | if (pszPermissions[1] == 'w') |
6469 | | nMode |= S_IWUSR; |
6470 | | if (pszPermissions[2] == 'x') |
6471 | | nMode |= S_IXUSR; |
6472 | | if (pszPermissions[3] == 'r') |
6473 | | nMode |= S_IRGRP; |
6474 | | if (pszPermissions[4] == 'w') |
6475 | | nMode |= S_IWGRP; |
6476 | | if (pszPermissions[5] == 'x') |
6477 | | nMode |= S_IXGRP; |
6478 | | if (pszPermissions[6] == 'r') |
6479 | | nMode |= S_IROTH; |
6480 | | if (pszPermissions[7] == 'w') |
6481 | | nMode |= S_IWOTH; |
6482 | | if (pszPermissions[8] == 'x') |
6483 | | nMode |= S_IXOTH; |
6484 | | return nMode; |
6485 | | } |
6486 | | |
6487 | | /************************************************************************/ |
6488 | | /* Cache of file properties. */ |
6489 | | /************************************************************************/ |
6490 | | |
6491 | | static std::mutex oCacheFilePropMutex; |
6492 | | static lru11::Cache<std::string, cpl::FileProp> *poCacheFileProp = nullptr; |
6493 | | |
6494 | | /************************************************************************/ |
6495 | | /* VSICURLGetCachedFileProp() */ |
6496 | | /************************************************************************/ |
6497 | | |
6498 | | bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6499 | | { |
6500 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6501 | | return poCacheFileProp != nullptr && |
6502 | | poCacheFileProp->tryGet(std::string(pszURL), oFileProp) && |
6503 | | // Let a chance to use new auth parameters |
6504 | | !(oFileProp.eExists == cpl::EXIST_NO && |
6505 | | gnGenerationAuthParameters != oFileProp.nGenerationAuthParameters); |
6506 | | } |
6507 | | |
6508 | | /************************************************************************/ |
6509 | | /* VSICURLSetCachedFileProp() */ |
6510 | | /************************************************************************/ |
6511 | | |
6512 | | void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6513 | | { |
6514 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6515 | | if (poCacheFileProp == nullptr) |
6516 | | poCacheFileProp = |
6517 | | new lru11::Cache<std::string, cpl::FileProp>(100 * 1024); |
6518 | | oFileProp.nGenerationAuthParameters = gnGenerationAuthParameters; |
6519 | | poCacheFileProp->insert(std::string(pszURL), oFileProp); |
6520 | | } |
6521 | | |
6522 | | /************************************************************************/ |
6523 | | /* VSICURLInvalidateCachedFileProp() */ |
6524 | | /************************************************************************/ |
6525 | | |
6526 | | void VSICURLInvalidateCachedFileProp(const char *pszURL) |
6527 | | { |
6528 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6529 | | if (poCacheFileProp != nullptr) |
6530 | | poCacheFileProp->remove(std::string(pszURL)); |
6531 | | } |
6532 | | |
6533 | | /************************************************************************/ |
6534 | | /* VSICURLInvalidateCachedFilePropPrefix() */ |
6535 | | /************************************************************************/ |
6536 | | |
6537 | | void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL) |
6538 | | { |
6539 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6540 | | if (poCacheFileProp != nullptr) |
6541 | | { |
6542 | | std::list<std::string> keysToRemove; |
6543 | | const size_t nURLSize = strlen(pszURL); |
6544 | | auto lambda = |
6545 | | [&keysToRemove, &pszURL, nURLSize]( |
6546 | | const lru11::KeyValuePair<std::string, cpl::FileProp> &kv) |
6547 | | { |
6548 | | if (strncmp(kv.key.c_str(), pszURL, nURLSize) == 0) |
6549 | | keysToRemove.push_back(kv.key); |
6550 | | }; |
6551 | | poCacheFileProp->cwalk(lambda); |
6552 | | for (const auto &key : keysToRemove) |
6553 | | poCacheFileProp->remove(key); |
6554 | | } |
6555 | | } |
6556 | | |
6557 | | /************************************************************************/ |
6558 | | /* VSICURLDestroyCacheFileProp() */ |
6559 | | /************************************************************************/ |
6560 | | |
6561 | | void VSICURLDestroyCacheFileProp() |
6562 | | { |
6563 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6564 | | delete poCacheFileProp; |
6565 | | poCacheFileProp = nullptr; |
6566 | | } |
6567 | | |
6568 | | /************************************************************************/ |
6569 | | /* VSICURLMultiCleanup() */ |
6570 | | /************************************************************************/ |
6571 | | |
6572 | | void VSICURLMultiCleanup(CURLM *hCurlMultiHandle) |
6573 | | { |
6574 | | #if defined(CURL_AT_LEAST_VERSION) && defined(_WIN32) |
6575 | | // Since curl 8.20.0, auxiliary threads are used for DNS resolution |
6576 | | // Trying to join them when detaching the DLL results in a hang. |
6577 | | // See https://github.com/curl/curl/issues/21466#issuecomment-4372138595 |
6578 | | #if CURL_AT_LEAST_VERSION(8, 20, 0) |
6579 | | if (GDALIsInGlobalDestructorFromDLLMain()) |
6580 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_QUICK_EXIT, 1L); |
6581 | | #endif |
6582 | | #endif |
6583 | | |
6584 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
6585 | | curl_multi_cleanup(hCurlMultiHandle); |
6586 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
6587 | | } |
6588 | | |
6589 | | /************************************************************************/ |
6590 | | /* VSICurlInstallReadCbk() */ |
6591 | | /************************************************************************/ |
6592 | | |
6593 | | int VSICurlInstallReadCbk(VSILFILE *fp, VSICurlReadCbkFunc pfnReadCbk, |
6594 | | void *pfnUserData, int bStopOnInterruptUntilUninstall) |
6595 | | { |
6596 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->InstallReadCbk( |
6597 | | pfnReadCbk, pfnUserData, bStopOnInterruptUntilUninstall); |
6598 | | } |
6599 | | |
6600 | | /************************************************************************/ |
6601 | | /* VSICurlUninstallReadCbk() */ |
6602 | | /************************************************************************/ |
6603 | | |
6604 | | int VSICurlUninstallReadCbk(VSILFILE *fp) |
6605 | | { |
6606 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->UninstallReadCbk(); |
6607 | | } |
6608 | | |
6609 | | /************************************************************************/ |
6610 | | /* VSICurlSetOptions() */ |
6611 | | /************************************************************************/ |
6612 | | |
6613 | | struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL, |
6614 | | const char *const *papszOptions) |
6615 | | { |
6616 | | struct curl_slist *headers = static_cast<struct curl_slist *>( |
6617 | | CPLHTTPSetOptions(hCurlHandle, pszURL, papszOptions)); |
6618 | | |
6619 | | long option = CURLFTPMETHOD_SINGLECWD; |
6620 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_FILEMETHOD, option); |
6621 | | |
6622 | | // ftp://ftp2.cits.rncan.gc.ca/pub/cantopo/250k_tif/ |
6623 | | // doesn't like EPSV command, |
6624 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_USE_EPSV, 0); |
6625 | | |
6626 | | return headers; |
6627 | | } |
6628 | | |
6629 | | /************************************************************************/ |
6630 | | /* VSICurlSetContentTypeFromExt() */ |
6631 | | /************************************************************************/ |
6632 | | |
6633 | | struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *poList, |
6634 | | const char *pszPath) |
6635 | | { |
6636 | | struct curl_slist *iter = poList; |
6637 | | while (iter != nullptr) |
6638 | | { |
6639 | | if (STARTS_WITH_CI(iter->data, "Content-Type")) |
6640 | | { |
6641 | | return poList; |
6642 | | } |
6643 | | iter = iter->next; |
6644 | | } |
6645 | | |
6646 | | static const struct |
6647 | | { |
6648 | | const char *ext; |
6649 | | const char *mime; |
6650 | | } aosExtMimePairs[] = { |
6651 | | {"txt", "text/plain"}, {"json", "application/json"}, |
6652 | | {"tif", "image/tiff"}, {"tiff", "image/tiff"}, |
6653 | | {"jpg", "image/jpeg"}, {"jpeg", "image/jpeg"}, |
6654 | | {"jp2", "image/jp2"}, {"jpx", "image/jp2"}, |
6655 | | {"j2k", "image/jp2"}, {"jpc", "image/jp2"}, |
6656 | | {"png", "image/png"}, |
6657 | | }; |
6658 | | |
6659 | | const std::string osExt = CPLGetExtensionSafe(pszPath); |
6660 | | if (!osExt.empty()) |
6661 | | { |
6662 | | for (const auto &pair : aosExtMimePairs) |
6663 | | { |
6664 | | if (EQUAL(osExt.c_str(), pair.ext)) |
6665 | | { |
6666 | | |
6667 | | const std::string osContentType( |
6668 | | CPLSPrintf("Content-Type: %s", pair.mime)); |
6669 | | poList = curl_slist_append(poList, osContentType.c_str()); |
6670 | | #ifdef DEBUG_VERBOSE |
6671 | | CPLDebug("HTTP", "Setting %s, based on lookup table.", |
6672 | | osContentType.c_str()); |
6673 | | #endif |
6674 | | break; |
6675 | | } |
6676 | | } |
6677 | | } |
6678 | | |
6679 | | return poList; |
6680 | | } |
6681 | | |
6682 | | /************************************************************************/ |
6683 | | /* VSICurlSetCreationHeadersFromOptions() */ |
6684 | | /************************************************************************/ |
6685 | | |
6686 | | struct curl_slist *VSICurlSetCreationHeadersFromOptions( |
6687 | | struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath) |
6688 | | { |
6689 | | bool bContentTypeFound = false; |
6690 | | for (CSLConstList papszIter = papszOptions; papszIter && *papszIter; |
6691 | | ++papszIter) |
6692 | | { |
6693 | | char *pszKey = nullptr; |
6694 | | const char *pszValue = CPLParseNameValue(*papszIter, &pszKey); |
6695 | | if (pszKey && pszValue) |
6696 | | { |
6697 | | if (EQUAL(pszKey, "Content-Type")) |
6698 | | { |
6699 | | bContentTypeFound = true; |
6700 | | } |
6701 | | headers = curl_slist_append(headers, |
6702 | | CPLSPrintf("%s: %s", pszKey, pszValue)); |
6703 | | } |
6704 | | CPLFree(pszKey); |
6705 | | } |
6706 | | |
6707 | | // If Content-type not found in papszOptions, try to set it from the |
6708 | | // filename exstension. |
6709 | | if (!bContentTypeFound) |
6710 | | { |
6711 | | headers = VSICurlSetContentTypeFromExt(headers, pszPath); |
6712 | | } |
6713 | | |
6714 | | return headers; |
6715 | | } |
6716 | | |
6717 | | #endif // DOXYGEN_SKIP |
6718 | | //! @endcond |
6719 | | |
6720 | | /************************************************************************/ |
6721 | | /* VSIInstallCurlFileHandler() */ |
6722 | | /************************************************************************/ |
6723 | | |
6724 | | /*! |
6725 | | \brief Install /vsicurl/ HTTP/FTP file system handler (requires libcurl) |
6726 | | |
6727 | | \verbatim embed:rst |
6728 | | See :ref:`/vsicurl/ documentation <vsicurl>` |
6729 | | \endverbatim |
6730 | | |
6731 | | */ |
6732 | | void VSIInstallCurlFileHandler(void) |
6733 | | { |
6734 | | auto poHandler = std::make_shared<cpl::VSICurlFilesystemHandler>(); |
6735 | | for (const char *pszPrefix : VSICURL_PREFIXES) |
6736 | | { |
6737 | | VSIFileManager::InstallHandler(pszPrefix, poHandler); |
6738 | | } |
6739 | | } |
6740 | | |
6741 | | /************************************************************************/ |
6742 | | /* VSICurlClearCache() */ |
6743 | | /************************************************************************/ |
6744 | | |
6745 | | /** |
6746 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6747 | | * |
6748 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6749 | | * /vsiswift/) cache a number of |
6750 | | * metadata and data for faster execution in read-only scenarios. But when the |
6751 | | * content on the server-side may change during the same process, those |
6752 | | * mechanisms can prevent opening new files, or give an outdated version of |
6753 | | * them. |
6754 | | * |
6755 | | */ |
6756 | | |
6757 | | void VSICurlClearCache(void) |
6758 | | { |
6759 | | // FIXME ? Currently we have different filesystem instances for |
6760 | | // vsicurl/, /vsis3/, /vsigs/ . So each one has its own cache of regions. |
6761 | | // File properties cache are now shared |
6762 | | char **papszPrefix = VSIFileManager::GetPrefixes(); |
6763 | | for (size_t i = 0; papszPrefix && papszPrefix[i]; ++i) |
6764 | | { |
6765 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6766 | | VSIFileManager::GetHandler(papszPrefix[i])); |
6767 | | |
6768 | | if (poFSHandler) |
6769 | | poFSHandler->ClearCache(); |
6770 | | } |
6771 | | CSLDestroy(papszPrefix); |
6772 | | |
6773 | | VSICurlStreamingClearCache(); |
6774 | | } |
6775 | | |
6776 | | /************************************************************************/ |
6777 | | /* VSICurlPartialClearCache() */ |
6778 | | /************************************************************************/ |
6779 | | |
6780 | | /** |
6781 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6782 | | * for a given filename (and its subfiles and subdirectories if it is a |
6783 | | * directory) |
6784 | | * |
6785 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6786 | | * /vsiswift/) cache a number of |
6787 | | * metadata and data for faster execution in read-only scenarios. But when the |
6788 | | * content on the server-side may change during the same process, those |
6789 | | * mechanisms can prevent opening new files, or give an outdated version of |
6790 | | * them. |
6791 | | * |
6792 | | * The filename prefix must start with the name of a known virtual file system |
6793 | | * (such as "/vsicurl/", "/vsis3/") |
6794 | | * |
6795 | | * VSICurlPartialClearCache("/vsis3/b") will clear all cached state for any file |
6796 | | * or directory starting with that prefix, so potentially "/vsis3/bucket", |
6797 | | * "/vsis3/basket/" or "/vsis3/basket/object". |
6798 | | * |
6799 | | * @param pszFilenamePrefix Filename prefix |
6800 | | */ |
6801 | | |
6802 | | void VSICurlPartialClearCache(const char *pszFilenamePrefix) |
6803 | | { |
6804 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6805 | | VSIFileManager::GetHandler(pszFilenamePrefix)); |
6806 | | |
6807 | | if (poFSHandler) |
6808 | | poFSHandler->PartialClearCache(pszFilenamePrefix); |
6809 | | } |
6810 | | |
6811 | | /************************************************************************/ |
6812 | | /* VSINetworkStatsReset() */ |
6813 | | /************************************************************************/ |
6814 | | |
6815 | | /** |
6816 | | * \brief Clear network related statistics. |
6817 | | * |
6818 | | * The effect of the CPL_VSIL_NETWORK_STATS_ENABLED configuration option |
6819 | | * will also be reset. That is, that the next network access will check its |
6820 | | * value again. |
6821 | | * |
6822 | | * @since GDAL 3.2.0 |
6823 | | */ |
6824 | | |
6825 | | void VSINetworkStatsReset(void) |
6826 | | { |
6827 | | cpl::NetworkStatisticsLogger::Reset(); |
6828 | | } |
6829 | | |
6830 | | /************************************************************************/ |
6831 | | /* VSINetworkStatsGetAsSerializedJSON() */ |
6832 | | /************************************************************************/ |
6833 | | |
6834 | | /** |
6835 | | * \brief Return network related statistics, as a JSON serialized object. |
6836 | | * |
6837 | | * Statistics collecting should be enabled with the |
6838 | | CPL_VSIL_NETWORK_STATS_ENABLED |
6839 | | * configuration option set to YES before any network activity starts |
6840 | | * (for efficiency, reading it is cached on first access, until |
6841 | | VSINetworkStatsReset() is called) |
6842 | | * |
6843 | | * Statistics can also be emitted on standard output at process termination if |
6844 | | * the CPL_VSIL_SHOW_NETWORK_STATS configuration option is set to YES. |
6845 | | * |
6846 | | * Example of output: |
6847 | | * \code{.js} |
6848 | | * { |
6849 | | * "methods":{ |
6850 | | * "GET":{ |
6851 | | * "count":6, |
6852 | | * "downloaded_bytes":40825 |
6853 | | * }, |
6854 | | * "PUT":{ |
6855 | | * "count":1, |
6856 | | * "uploaded_bytes":35472 |
6857 | | * } |
6858 | | * }, |
6859 | | * "handlers":{ |
6860 | | * "vsigs":{ |
6861 | | * "methods":{ |
6862 | | * "GET":{ |
6863 | | * "count":2, |
6864 | | * "downloaded_bytes":446 |
6865 | | * }, |
6866 | | * "PUT":{ |
6867 | | * "count":1, |
6868 | | * "uploaded_bytes":35472 |
6869 | | * } |
6870 | | * }, |
6871 | | * "files":{ |
6872 | | * "\/vsigs\/spatialys\/byte.tif":{ |
6873 | | * "methods":{ |
6874 | | * "PUT":{ |
6875 | | * "count":1, |
6876 | | * "uploaded_bytes":35472 |
6877 | | * } |
6878 | | * }, |
6879 | | * "actions":{ |
6880 | | * "Write":{ |
6881 | | * "methods":{ |
6882 | | * "PUT":{ |
6883 | | * "count":1, |
6884 | | * "uploaded_bytes":35472 |
6885 | | * } |
6886 | | * } |
6887 | | * } |
6888 | | * } |
6889 | | * } |
6890 | | * }, |
6891 | | * "actions":{ |
6892 | | * "Stat":{ |
6893 | | * "methods":{ |
6894 | | * "GET":{ |
6895 | | * "count":2, |
6896 | | * "downloaded_bytes":446 |
6897 | | * } |
6898 | | * }, |
6899 | | * "files":{ |
6900 | | * "\/vsigs\/spatialys\/byte.tif\/":{ |
6901 | | * "methods":{ |
6902 | | * "GET":{ |
6903 | | * "count":1, |
6904 | | * "downloaded_bytes":181 |
6905 | | * } |
6906 | | * } |
6907 | | * } |
6908 | | * } |
6909 | | * } |
6910 | | * } |
6911 | | * }, |
6912 | | * "vsis3":{ |
6913 | | * [...] |
6914 | | * } |
6915 | | * } |
6916 | | * } |
6917 | | * \endcode |
6918 | | * |
6919 | | * @param papszOptions Unused. |
6920 | | * @return a JSON serialized string to free with VSIFree(), or nullptr |
6921 | | * @since GDAL 3.2.0 |
6922 | | */ |
6923 | | |
6924 | | char *VSINetworkStatsGetAsSerializedJSON(CPL_UNUSED char **papszOptions) |
6925 | | { |
6926 | | return CPLStrdup( |
6927 | | cpl::NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
6928 | | } |
6929 | | |
6930 | | #endif /* HAVE_CURL */ |
6931 | | |
6932 | | #undef ENABLE_DEBUG |