/src/gdal/port/cpl_vsil_curl.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: CPL - Common Portability Library |
4 | | * Purpose: Implement VSI large file api for HTTP/FTP files |
5 | | * Author: Even Rouault, even.rouault at spatialys.com |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_port.h" |
14 | | #include "cpl_vsil_curl_priv.h" |
15 | | #include "cpl_vsil_curl_class.h" |
16 | | |
17 | | #include <algorithm> |
18 | | #include <array> |
19 | | #include <limits> |
20 | | #include <map> |
21 | | #include <memory> |
22 | | #include <numeric> |
23 | | #include <set> |
24 | | #include <string_view> |
25 | | |
26 | | #include "cpl_aws.h" |
27 | | #include "cpl_json.h" |
28 | | #include "cpl_json_header.h" |
29 | | #include "cpl_minixml.h" |
30 | | #include "cpl_multiproc.h" |
31 | | #include "cpl_string.h" |
32 | | #include "cpl_time.h" |
33 | | #include "cpl_vsi.h" |
34 | | #include "cpl_vsi_virtual.h" |
35 | | #include "cpl_http.h" |
36 | | #include "cpl_mem_cache.h" |
37 | | |
38 | | #ifndef S_IRUSR |
39 | | #define S_IRUSR 00400 |
40 | | #define S_IWUSR 00200 |
41 | | #define S_IXUSR 00100 |
42 | | #define S_IRGRP 00040 |
43 | | #define S_IWGRP 00020 |
44 | | #define S_IXGRP 00010 |
45 | | #define S_IROTH 00004 |
46 | | #define S_IWOTH 00002 |
47 | | #define S_IXOTH 00001 |
48 | | #endif |
49 | | |
50 | | #ifndef HAVE_CURL |
51 | | |
52 | | void VSIInstallCurlFileHandler(void) |
53 | 0 | { |
54 | | // Not supported. |
55 | 0 | } |
56 | | |
57 | | void VSICurlClearCache(void) |
58 | 0 | { |
59 | | // Not supported. |
60 | 0 | } |
61 | | |
62 | | void VSICurlPartialClearCache(const char *) |
63 | 0 | { |
64 | | // Not supported. |
65 | 0 | } |
66 | | |
67 | | void VSICurlAuthParametersChanged() |
68 | 131 | { |
69 | | // Not supported. |
70 | 131 | } |
71 | | |
72 | | void VSINetworkStatsReset(void) |
73 | 0 | { |
74 | | // Not supported |
75 | 0 | } |
76 | | |
77 | | char *VSINetworkStatsGetAsSerializedJSON(char ** /* papszOptions */) |
78 | 0 | { |
79 | | // Not supported |
80 | 0 | return nullptr; |
81 | 0 | } |
82 | | |
83 | | /************************************************************************/ |
84 | | /* VSICurlInstallReadCbk() */ |
85 | | /************************************************************************/ |
86 | | |
87 | | int VSICurlInstallReadCbk(VSILFILE * /* fp */, |
88 | | VSICurlReadCbkFunc /* pfnReadCbk */, |
89 | | void * /* pfnUserData */, |
90 | | int /* bStopOnInterruptUntilUninstall */) |
91 | 0 | { |
92 | 0 | return FALSE; |
93 | 0 | } |
94 | | |
95 | | /************************************************************************/ |
96 | | /* VSICurlUninstallReadCbk() */ |
97 | | /************************************************************************/ |
98 | | |
99 | | int VSICurlUninstallReadCbk(VSILFILE * /* fp */) |
100 | 0 | { |
101 | 0 | return FALSE; |
102 | 0 | } |
103 | | |
104 | | #else |
105 | | |
106 | | //! @cond Doxygen_Suppress |
107 | | #ifndef DOXYGEN_SKIP |
108 | | |
109 | | #define ENABLE_DEBUG 1 |
110 | | #define ENABLE_DEBUG_VERBOSE 0 |
111 | | |
112 | | #define unchecked_curl_easy_setopt(handle, opt, param) \ |
113 | | CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param)) |
114 | | |
115 | | constexpr const char *const VSICURL_PREFIXES[] = {"/vsicurl/", "/vsicurl?"}; |
116 | | |
117 | | /***********************************************************รน************/ |
118 | | /* VSICurlAuthParametersChanged() */ |
119 | | /************************************************************************/ |
120 | | |
121 | | static unsigned int gnGenerationAuthParameters = 0; |
122 | | |
123 | | void VSICurlAuthParametersChanged() |
124 | | { |
125 | | gnGenerationAuthParameters++; |
126 | | } |
127 | | |
128 | | // Do not access those variables directly ! |
129 | | // Use VSICURLGetDownloadChunkSize() and GetMaxRegions() |
130 | | static int N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = 0; |
131 | | static int DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = 0; |
132 | | |
133 | | // RAII to potentially start/stop run thread. The run thread needs to running |
134 | | // for Perform() to succeed. |
135 | | cpl::RunThreadUser::RunThreadUser(VSICurlFilesystemHandlerBase &handler) |
136 | | : m_handler(handler) |
137 | | { |
138 | | m_handler.IncUseCount(); |
139 | | } |
140 | | |
141 | | cpl::RunThreadUser::~RunThreadUser() |
142 | | { |
143 | | m_handler.DecUseCount(); |
144 | | } |
145 | | |
146 | | /************************************************************************/ |
147 | | /* VSICURLMultiInit() */ |
148 | | /************************************************************************/ |
149 | | |
150 | | static CURLM *VSICURLMultiInit() |
151 | | { |
152 | | CURLM *hCurlMultiHandle = curl_multi_init(); |
153 | | |
154 | | if (const char *pszMAXCONNECTS = |
155 | | CPLGetConfigOption("GDAL_HTTP_MAX_CACHED_CONNECTIONS", nullptr)) |
156 | | { |
157 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAXCONNECTS, |
158 | | atoi(pszMAXCONNECTS)); |
159 | | } |
160 | | |
161 | | if (const char *pszMAX_TOTAL_CONNECTIONS = |
162 | | CPLGetConfigOption("GDAL_HTTP_MAX_TOTAL_CONNECTIONS", nullptr)) |
163 | | { |
164 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, |
165 | | atoi(pszMAX_TOTAL_CONNECTIONS)); |
166 | | } |
167 | | |
168 | | return hCurlMultiHandle; |
169 | | } |
170 | | |
171 | | /************************************************************************/ |
172 | | /* VSICURLReadGlobalEnvVariables() */ |
173 | | /************************************************************************/ |
174 | | |
175 | | static void VSICURLReadGlobalEnvVariables() |
176 | | { |
177 | | struct Initializer |
178 | | { |
179 | | Initializer() |
180 | | { |
181 | | constexpr int DOWNLOAD_CHUNK_SIZE_DEFAULT = 16384; |
182 | | const char *pszChunkSize = |
183 | | CPLGetConfigOption("CPL_VSIL_CURL_CHUNK_SIZE", nullptr); |
184 | | GIntBig nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
185 | | |
186 | | if (pszChunkSize) |
187 | | { |
188 | | if (CPLParseMemorySize(pszChunkSize, &nChunkSize, nullptr) != |
189 | | CE_None) |
190 | | { |
191 | | CPLError( |
192 | | CE_Warning, CPLE_AppDefined, |
193 | | "Could not parse value for CPL_VSIL_CURL_CHUNK_SIZE. " |
194 | | "Using default value of %d instead.", |
195 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
196 | | } |
197 | | } |
198 | | |
199 | | constexpr int MIN_CHUNK_SIZE = 1024; |
200 | | constexpr int MAX_CHUNK_SIZE = 10 * 1024 * 1024; |
201 | | if (nChunkSize < MIN_CHUNK_SIZE || nChunkSize > MAX_CHUNK_SIZE) |
202 | | { |
203 | | nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
204 | | CPLError(CE_Warning, CPLE_AppDefined, |
205 | | "Invalid value for CPL_VSIL_CURL_CHUNK_SIZE. " |
206 | | "Allowed range is [%d, %d]. " |
207 | | "Using CPL_VSIL_CURL_CHUNK_SIZE=%d instead", |
208 | | MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, |
209 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
210 | | } |
211 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = |
212 | | static_cast<int>(nChunkSize); |
213 | | |
214 | | constexpr int N_MAX_REGIONS_DEFAULT = 1000; |
215 | | constexpr int CACHE_SIZE_DEFAULT = |
216 | | N_MAX_REGIONS_DEFAULT * DOWNLOAD_CHUNK_SIZE_DEFAULT; |
217 | | |
218 | | const char *pszCacheSize = |
219 | | CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE", nullptr); |
220 | | GIntBig nCacheSize = CACHE_SIZE_DEFAULT; |
221 | | |
222 | | if (pszCacheSize) |
223 | | { |
224 | | if (CPLParseMemorySize(pszCacheSize, &nCacheSize, nullptr) != |
225 | | CE_None) |
226 | | { |
227 | | CPLError( |
228 | | CE_Warning, CPLE_AppDefined, |
229 | | "Could not parse value for CPL_VSIL_CURL_CACHE_SIZE. " |
230 | | "Using default value of " CPL_FRMT_GIB " instead.", |
231 | | nCacheSize); |
232 | | } |
233 | | } |
234 | | |
235 | | const auto nMaxRAM = CPLGetUsablePhysicalRAM(); |
236 | | const auto nMinVal = DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
237 | | auto nMaxVal = static_cast<GIntBig>(INT_MAX) * |
238 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
239 | | if (nMaxRAM > 0 && nMaxVal > nMaxRAM) |
240 | | nMaxVal = nMaxRAM; |
241 | | if (nCacheSize < nMinVal || nCacheSize > nMaxVal) |
242 | | { |
243 | | nCacheSize = nCacheSize < nMinVal ? nMinVal : nMaxVal; |
244 | | CPLError(CE_Warning, CPLE_AppDefined, |
245 | | "Invalid value for CPL_VSIL_CURL_CACHE_SIZE. " |
246 | | "Allowed range is [%d, " CPL_FRMT_GIB "]. " |
247 | | "Using CPL_VSIL_CURL_CACHE_SIZE=" CPL_FRMT_GIB |
248 | | " instead", |
249 | | nMinVal, nMaxVal, nCacheSize); |
250 | | } |
251 | | N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = std::max( |
252 | | 1, static_cast<int>(nCacheSize / |
253 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY)); |
254 | | } |
255 | | }; |
256 | | |
257 | | static Initializer initializer; |
258 | | } |
259 | | |
260 | | /************************************************************************/ |
261 | | /* VSICURLGetDownloadChunkSize() */ |
262 | | /************************************************************************/ |
263 | | |
264 | | int VSICURLGetDownloadChunkSize() |
265 | | { |
266 | | VSICURLReadGlobalEnvVariables(); |
267 | | return DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
268 | | } |
269 | | |
270 | | /************************************************************************/ |
271 | | /* GetMaxRegions() */ |
272 | | /************************************************************************/ |
273 | | |
274 | | static int GetMaxRegions() |
275 | | { |
276 | | VSICURLReadGlobalEnvVariables(); |
277 | | return N_MAX_REGIONS_DO_NOT_USE_DIRECTLY; |
278 | | } |
279 | | |
280 | | /************************************************************************/ |
281 | | /* VSICurlFindStringSensitiveExceptEscapeSequences() */ |
282 | | /************************************************************************/ |
283 | | |
284 | | static int |
285 | | VSICurlFindStringSensitiveExceptEscapeSequences(CSLConstList papszList, |
286 | | const char *pszTarget) |
287 | | |
288 | | { |
289 | | if (papszList == nullptr) |
290 | | return -1; |
291 | | |
292 | | for (int i = 0; papszList[i] != nullptr; i++) |
293 | | { |
294 | | const char *pszIter1 = papszList[i]; |
295 | | const char *pszIter2 = pszTarget; |
296 | | char ch1 = '\0'; |
297 | | char ch2 = '\0'; |
298 | | /* The comparison is case-sensitive, except for escaped */ |
299 | | /* sequences where letters of the hexadecimal sequence */ |
300 | | /* can be uppercase or lowercase depending on the quoting algorithm */ |
301 | | while (true) |
302 | | { |
303 | | ch1 = *pszIter1; |
304 | | ch2 = *pszIter2; |
305 | | if (ch1 == '\0' || ch2 == '\0') |
306 | | break; |
307 | | if (ch1 == '%' && ch2 == '%' && pszIter1[1] != '\0' && |
308 | | pszIter1[2] != '\0' && pszIter2[1] != '\0' && |
309 | | pszIter2[2] != '\0') |
310 | | { |
311 | | if (!EQUALN(pszIter1 + 1, pszIter2 + 1, 2)) |
312 | | break; |
313 | | pszIter1 += 2; |
314 | | pszIter2 += 2; |
315 | | } |
316 | | if (ch1 != ch2) |
317 | | break; |
318 | | pszIter1++; |
319 | | pszIter2++; |
320 | | } |
321 | | if (ch1 == ch2 && ch1 == '\0') |
322 | | return i; |
323 | | } |
324 | | |
325 | | return -1; |
326 | | } |
327 | | |
328 | | /************************************************************************/ |
329 | | /* VSICurlIsFileInList() */ |
330 | | /************************************************************************/ |
331 | | |
332 | | static int VSICurlIsFileInList(CSLConstList papszList, const char *pszTarget) |
333 | | { |
334 | | int nRet = |
335 | | VSICurlFindStringSensitiveExceptEscapeSequences(papszList, pszTarget); |
336 | | if (nRet >= 0) |
337 | | return nRet; |
338 | | |
339 | | // If we didn't find anything, try to URL-escape the target filename. |
340 | | char *pszEscaped = CPLEscapeString(pszTarget, -1, CPLES_URL); |
341 | | if (strcmp(pszTarget, pszEscaped) != 0) |
342 | | { |
343 | | nRet = VSICurlFindStringSensitiveExceptEscapeSequences(papszList, |
344 | | pszEscaped); |
345 | | } |
346 | | CPLFree(pszEscaped); |
347 | | return nRet; |
348 | | } |
349 | | |
350 | | /************************************************************************/ |
351 | | /* StartsWithVSICurlPrefix() */ |
352 | | /************************************************************************/ |
353 | | |
354 | | static bool StartsWithVSICurlPrefix(const char *pszFilename) |
355 | | { |
356 | | for (const char *pszPrefix : VSICURL_PREFIXES) |
357 | | { |
358 | | if (STARTS_WITH(pszFilename, pszPrefix)) |
359 | | { |
360 | | return true; |
361 | | } |
362 | | } |
363 | | return false; |
364 | | } |
365 | | |
366 | | /************************************************************************/ |
367 | | /* VSICurlGetURLFromFilename() */ |
368 | | /************************************************************************/ |
369 | | |
370 | | static std::string VSICurlGetURLFromFilename( |
371 | | const char *pszFilename, CPLHTTPRetryParameters *poRetryParameters, |
372 | | bool *pbUseHead, bool *pbUseRedirectURLIfNoQueryStringParams, |
373 | | bool *pbListDir, bool *pbEmptyDir, CPLStringList *paosHTTPOptions, |
374 | | bool *pbPlanetaryComputerURLSigning, char **ppszPlanetaryComputerCollection) |
375 | | { |
376 | | if (ppszPlanetaryComputerCollection) |
377 | | *ppszPlanetaryComputerCollection = nullptr; |
378 | | |
379 | | if (!StartsWithVSICurlPrefix(pszFilename)) |
380 | | return pszFilename; |
381 | | |
382 | | if (pbPlanetaryComputerURLSigning) |
383 | | { |
384 | | // It may be more convenient sometimes to store Planetary Computer URL |
385 | | // signing as a per-path specific option rather than capturing it in |
386 | | // the filename with the &pc_url_signing=yes option. |
387 | | if (CPLTestBool(VSIGetPathSpecificOption( |
388 | | pszFilename, "VSICURL_PC_URL_SIGNING", "FALSE"))) |
389 | | { |
390 | | *pbPlanetaryComputerURLSigning = true; |
391 | | } |
392 | | } |
393 | | |
394 | | pszFilename += strlen("/vsicurl/"); |
395 | | if (!STARTS_WITH(pszFilename, "http://") && |
396 | | !STARTS_WITH(pszFilename, "https://") && |
397 | | !STARTS_WITH(pszFilename, "ftp://") && |
398 | | !STARTS_WITH(pszFilename, "file://")) |
399 | | { |
400 | | if (*pszFilename == '?') |
401 | | pszFilename++; |
402 | | char **papszTokens = CSLTokenizeString2(pszFilename, "&", 0); |
403 | | for (int i = 0; papszTokens[i] != nullptr; i++) |
404 | | { |
405 | | char *pszUnescaped = |
406 | | CPLUnescapeString(papszTokens[i], nullptr, CPLES_URL); |
407 | | CPLFree(papszTokens[i]); |
408 | | papszTokens[i] = pszUnescaped; |
409 | | } |
410 | | |
411 | | std::string osURL; |
412 | | std::string osHeaders; |
413 | | for (int i = 0; papszTokens[i]; i++) |
414 | | { |
415 | | char *pszKey = nullptr; |
416 | | const char *pszValue = CPLParseNameValue(papszTokens[i], &pszKey); |
417 | | if (pszKey && pszValue) |
418 | | { |
419 | | if (EQUAL(pszKey, "max_retry")) |
420 | | { |
421 | | if (poRetryParameters) |
422 | | poRetryParameters->nMaxRetry = atoi(pszValue); |
423 | | } |
424 | | else if (EQUAL(pszKey, "retry_delay")) |
425 | | { |
426 | | if (poRetryParameters) |
427 | | poRetryParameters->dfInitialDelay = CPLAtof(pszValue); |
428 | | } |
429 | | else if (EQUAL(pszKey, "retry_codes")) |
430 | | { |
431 | | if (poRetryParameters) |
432 | | poRetryParameters->osRetryCodes = pszValue; |
433 | | } |
434 | | else if (EQUAL(pszKey, "use_head")) |
435 | | { |
436 | | if (pbUseHead) |
437 | | *pbUseHead = CPLTestBool(pszValue); |
438 | | } |
439 | | else if (EQUAL(pszKey, |
440 | | "use_redirect_url_if_no_query_string_params")) |
441 | | { |
442 | | /* Undocumented. Used by PLScenes driver */ |
443 | | if (pbUseRedirectURLIfNoQueryStringParams) |
444 | | *pbUseRedirectURLIfNoQueryStringParams = |
445 | | CPLTestBool(pszValue); |
446 | | } |
447 | | else if (EQUAL(pszKey, "list_dir")) |
448 | | { |
449 | | if (pbListDir) |
450 | | *pbListDir = CPLTestBool(pszValue); |
451 | | } |
452 | | else if (EQUAL(pszKey, "empty_dir")) |
453 | | { |
454 | | if (pbEmptyDir) |
455 | | *pbEmptyDir = CPLTestBool(pszValue); |
456 | | } |
457 | | else if (EQUAL(pszKey, "header_file")) |
458 | | { |
459 | | #if defined(CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED) |
460 | | constexpr bool CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED = |
461 | | true; |
462 | | #else |
463 | | constexpr bool CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED = |
464 | | false; |
465 | | #endif |
466 | | if (CPL_VSIL_CURL_HEADER_FILE_KVP_DISABLED) |
467 | | { |
468 | | CPLError(CE_Failure, CPLE_AppDefined, |
469 | | "Use of 'header_file' key-value pair in " |
470 | | "/vsicurl? is disabled in this build"); |
471 | | } |
472 | | else |
473 | | { |
474 | | bool bSetValue = false; |
475 | | const char *pszAllowHeaderFileKVP = CPLGetConfigOption( |
476 | | "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED", nullptr); |
477 | | if (!pszAllowHeaderFileKVP || |
478 | | pszAllowHeaderFileKVP[0] == 0 || |
479 | | EQUAL(pszAllowHeaderFileKVP, "ONLY_IN_TEMP")) |
480 | | { |
481 | | if (STARTS_WITH(pszValue, "/vsimem/")) |
482 | | { |
483 | | bSetValue = !CPLHasUnbalancedPathTraversal( |
484 | | pszValue + strlen("/vsimem/")); |
485 | | } |
486 | | else if (STARTS_WITH(pszValue, "/tmp/")) |
487 | | { |
488 | | bSetValue = !CPLHasUnbalancedPathTraversal( |
489 | | pszValue + strlen("/tmp/")); |
490 | | } |
491 | | else |
492 | | { |
493 | | for (const char *pszEnvVar : {"TEMP", "TMP"}) |
494 | | { |
495 | | if (const char *pszTemp = |
496 | | CPLGetConfigOption(pszEnvVar, |
497 | | nullptr)) |
498 | | { |
499 | | std::string osTemp = pszTemp; |
500 | | if (!osTemp.empty() && |
501 | | (osTemp.back() == '/' || |
502 | | osTemp.back() == '\\')) |
503 | | osTemp.pop_back(); |
504 | | if (!osTemp.empty() && |
505 | | cpl::starts_with( |
506 | | std::string_view(pszValue), |
507 | | osTemp) && |
508 | | (pszValue[osTemp.size()] == '/' || |
509 | | pszValue[osTemp.size()] == '\\')) |
510 | | { |
511 | | bSetValue = |
512 | | !CPLHasUnbalancedPathTraversal( |
513 | | pszValue + osTemp.size()); |
514 | | break; |
515 | | } |
516 | | } |
517 | | } |
518 | | } |
519 | | if (!bSetValue) |
520 | | { |
521 | | CPLError(CE_Failure, CPLE_AppDefined, |
522 | | "Use of 'header_file=%s' " |
523 | | "key-value pair in /vsicurl? is " |
524 | | "disabled because it refers to a " |
525 | | "file stored in a non-temporary " |
526 | | "location. You may set the " |
527 | | "CPL_VSIL_CURL_HEADER_FILE_KVP_" |
528 | | "ENABLED configuration option to " |
529 | | "YES to remove that restriction.", |
530 | | pszValue); |
531 | | } |
532 | | } |
533 | | else if (CPLTestBool(pszAllowHeaderFileKVP)) |
534 | | { |
535 | | bSetValue = true; |
536 | | } |
537 | | else |
538 | | { |
539 | | CPLError(CE_Failure, CPLE_AppDefined, |
540 | | "Use of 'header_file' key-value pair in " |
541 | | "/vsicurl? is disabled by the " |
542 | | "CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED " |
543 | | "configuration option"); |
544 | | } |
545 | | |
546 | | if (bSetValue && paosHTTPOptions) |
547 | | { |
548 | | paosHTTPOptions->SetNameValue(pszKey, pszValue); |
549 | | } |
550 | | } |
551 | | } |
552 | | else if (EQUAL(pszKey, "useragent") || |
553 | | EQUAL(pszKey, "referer") || EQUAL(pszKey, "cookie") || |
554 | | EQUAL(pszKey, "unsafessl") || |
555 | | #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION |
556 | | EQUAL(pszKey, "timeout") || |
557 | | EQUAL(pszKey, "connecttimeout") || |
558 | | #endif |
559 | | EQUAL(pszKey, "low_speed_time") || |
560 | | EQUAL(pszKey, "low_speed_limit") || |
561 | | EQUAL(pszKey, "proxy") || EQUAL(pszKey, "proxyauth") || |
562 | | EQUAL(pszKey, "proxyuserpwd")) |
563 | | { |
564 | | // Above names are the ones supported by |
565 | | // CPLHTTPSetOptions() |
566 | | if (paosHTTPOptions) |
567 | | { |
568 | | paosHTTPOptions->SetNameValue(pszKey, pszValue); |
569 | | } |
570 | | } |
571 | | else if (EQUAL(pszKey, "url")) |
572 | | { |
573 | | osURL = pszValue; |
574 | | } |
575 | | else if (EQUAL(pszKey, "pc_url_signing")) |
576 | | { |
577 | | if (pbPlanetaryComputerURLSigning) |
578 | | *pbPlanetaryComputerURLSigning = CPLTestBool(pszValue); |
579 | | } |
580 | | else if (EQUAL(pszKey, "pc_collection")) |
581 | | { |
582 | | if (ppszPlanetaryComputerCollection) |
583 | | { |
584 | | CPLFree(*ppszPlanetaryComputerCollection); |
585 | | *ppszPlanetaryComputerCollection = CPLStrdup(pszValue); |
586 | | } |
587 | | } |
588 | | else if (STARTS_WITH(pszKey, "header.")) |
589 | | { |
590 | | osHeaders += (pszKey + strlen("header.")); |
591 | | osHeaders += ':'; |
592 | | osHeaders += pszValue; |
593 | | osHeaders += "\r\n"; |
594 | | } |
595 | | else |
596 | | { |
597 | | CPLError(CE_Warning, CPLE_NotSupported, |
598 | | "Unsupported option: %s", pszKey); |
599 | | } |
600 | | } |
601 | | CPLFree(pszKey); |
602 | | } |
603 | | |
604 | | if (paosHTTPOptions && !osHeaders.empty()) |
605 | | paosHTTPOptions->SetNameValue("HEADERS", osHeaders.c_str()); |
606 | | |
607 | | CSLDestroy(papszTokens); |
608 | | if (osURL.empty()) |
609 | | { |
610 | | CPLError(CE_Failure, CPLE_IllegalArg, "Missing url parameter"); |
611 | | return pszFilename; |
612 | | } |
613 | | |
614 | | return osURL; |
615 | | } |
616 | | |
617 | | return pszFilename; |
618 | | } |
619 | | |
620 | | namespace cpl |
621 | | { |
622 | | |
623 | | /************************************************************************/ |
624 | | /* VSICurlHandle() */ |
625 | | /************************************************************************/ |
626 | | |
627 | | VSICurlHandle::VSICurlHandle(VSICurlFilesystemHandlerBase *poFSIn, |
628 | | const char *pszFilename, const char *pszURLIn) |
629 | | : poFS(poFSIn), m_osFilename(pszFilename), |
630 | | m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv(pszFilename)), |
631 | | m_oRetryParameters(m_aosHTTPOptions), |
632 | | m_bUseHead( |
633 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_USE_HEAD", "YES"))), |
634 | | m_runThreadUser(*poFSIn) |
635 | | { |
636 | | if (pszURLIn) |
637 | | { |
638 | | m_pszURL = CPLStrdup(pszURLIn); |
639 | | } |
640 | | else |
641 | | { |
642 | | char *pszPCCollection = nullptr; |
643 | | m_pszURL = |
644 | | CPLStrdup(VSICurlGetURLFromFilename( |
645 | | pszFilename, &m_oRetryParameters, &m_bUseHead, |
646 | | &m_bUseRedirectURLIfNoQueryStringParams, nullptr, |
647 | | nullptr, &m_aosHTTPOptions, |
648 | | &m_bPlanetaryComputerURLSigning, &pszPCCollection) |
649 | | .c_str()); |
650 | | if (pszPCCollection) |
651 | | m_osPlanetaryComputerCollection = pszPCCollection; |
652 | | CPLFree(pszPCCollection); |
653 | | } |
654 | | |
655 | | m_bCached = poFSIn->AllowCachedDataFor(pszFilename); |
656 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
657 | | } |
658 | | |
659 | | /************************************************************************/ |
660 | | /* ~VSICurlHandle() */ |
661 | | /************************************************************************/ |
662 | | |
663 | | VSICurlHandle::~VSICurlHandle() |
664 | | { |
665 | | if (m_oThreadAdviseRead.joinable()) |
666 | | { |
667 | | m_oThreadAdviseRead.join(); |
668 | | } |
669 | | if (m_hCurlMultiHandleForAdviseRead) |
670 | | { |
671 | | VSICURLMultiCleanup(m_hCurlMultiHandleForAdviseRead); |
672 | | } |
673 | | |
674 | | if (!m_bCached) |
675 | | { |
676 | | poFS->InvalidateCachedData(m_pszURL); |
677 | | poFS->InvalidateDirContent(CPLGetDirnameSafe(m_osFilename.c_str())); |
678 | | } |
679 | | CPLFree(m_pszURL); |
680 | | } |
681 | | |
682 | | /************************************************************************/ |
683 | | /* SetURL() */ |
684 | | /************************************************************************/ |
685 | | |
686 | | void VSICurlHandle::SetURL(const char *pszURLIn) |
687 | | { |
688 | | CPLFree(m_pszURL); |
689 | | m_pszURL = CPLStrdup(pszURLIn); |
690 | | } |
691 | | |
692 | | /************************************************************************/ |
693 | | /* InstallReadCbk() */ |
694 | | /************************************************************************/ |
695 | | |
696 | | int VSICurlHandle::InstallReadCbk(VSICurlReadCbkFunc pfnReadCbkIn, |
697 | | void *pfnUserDataIn, |
698 | | int bStopOnInterruptUntilUninstallIn) |
699 | | { |
700 | | if (pfnReadCbk != nullptr) |
701 | | return FALSE; |
702 | | |
703 | | pfnReadCbk = pfnReadCbkIn; |
704 | | pReadCbkUserData = pfnUserDataIn; |
705 | | bStopOnInterruptUntilUninstall = |
706 | | CPL_TO_BOOL(bStopOnInterruptUntilUninstallIn); |
707 | | bInterrupted = false; |
708 | | return TRUE; |
709 | | } |
710 | | |
711 | | /************************************************************************/ |
712 | | /* UninstallReadCbk() */ |
713 | | /************************************************************************/ |
714 | | |
715 | | int VSICurlHandle::UninstallReadCbk() |
716 | | { |
717 | | if (pfnReadCbk == nullptr) |
718 | | return FALSE; |
719 | | |
720 | | pfnReadCbk = nullptr; |
721 | | pReadCbkUserData = nullptr; |
722 | | bStopOnInterruptUntilUninstall = false; |
723 | | bInterrupted = false; |
724 | | return TRUE; |
725 | | } |
726 | | |
727 | | /************************************************************************/ |
728 | | /* Seek() */ |
729 | | /************************************************************************/ |
730 | | |
731 | | int VSICurlHandle::Seek(vsi_l_offset nOffset, int nWhence) |
732 | | { |
733 | | if (nWhence == SEEK_SET) |
734 | | { |
735 | | curOffset = nOffset; |
736 | | } |
737 | | else if (nWhence == SEEK_CUR) |
738 | | { |
739 | | curOffset = curOffset + nOffset; |
740 | | } |
741 | | else |
742 | | { |
743 | | curOffset = GetFileSize(false) + nOffset; |
744 | | } |
745 | | bEOF = false; |
746 | | return 0; |
747 | | } |
748 | | |
749 | | } // namespace cpl |
750 | | |
751 | | /************************************************************************/ |
752 | | /* VSICurlGetTimeStampFromRFC822DateTime() */ |
753 | | /************************************************************************/ |
754 | | |
755 | | static GIntBig VSICurlGetTimeStampFromRFC822DateTime(const char *pszDT) |
756 | | { |
757 | | // Sun, 03 Apr 2016 12:07:27 GMT |
758 | | if (strlen(pszDT) >= 5 && pszDT[3] == ',' && pszDT[4] == ' ') |
759 | | pszDT += 5; |
760 | | int nDay = 0; |
761 | | int nYear = 0; |
762 | | int nHour = 0; |
763 | | int nMinute = 0; |
764 | | int nSecond = 0; |
765 | | char szMonth[4] = {}; |
766 | | szMonth[3] = 0; |
767 | | if (sscanf(pszDT, "%02d %03s %04d %02d:%02d:%02d GMT", &nDay, szMonth, |
768 | | &nYear, &nHour, &nMinute, &nSecond) == 6) |
769 | | { |
770 | | static const char *const aszMonthStr[] = {"Jan", "Feb", "Mar", "Apr", |
771 | | "May", "Jun", "Jul", "Aug", |
772 | | "Sep", "Oct", "Nov", "Dec"}; |
773 | | |
774 | | int nMonthIdx0 = -1; |
775 | | for (int i = 0; i < 12; i++) |
776 | | { |
777 | | if (EQUAL(szMonth, aszMonthStr[i])) |
778 | | { |
779 | | nMonthIdx0 = i; |
780 | | break; |
781 | | } |
782 | | } |
783 | | if (nMonthIdx0 >= 0) |
784 | | { |
785 | | struct tm brokendowntime; |
786 | | brokendowntime.tm_year = nYear - 1900; |
787 | | brokendowntime.tm_mon = nMonthIdx0; |
788 | | brokendowntime.tm_mday = nDay; |
789 | | brokendowntime.tm_hour = nHour; |
790 | | brokendowntime.tm_min = nMinute; |
791 | | brokendowntime.tm_sec = nSecond; |
792 | | return CPLYMDHMSToUnixTime(&brokendowntime); |
793 | | } |
794 | | } |
795 | | return 0; |
796 | | } |
797 | | |
798 | | /************************************************************************/ |
799 | | /* VSICURLInitWriteFuncStruct() */ |
800 | | /************************************************************************/ |
801 | | |
802 | | void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp, |
803 | | VSICurlReadCbkFunc pfnReadCbk, |
804 | | void *pReadCbkUserData) |
805 | | { |
806 | | psStruct->pBuffer = nullptr; |
807 | | psStruct->nSize = 0; |
808 | | psStruct->bIsHTTP = false; |
809 | | psStruct->bMultiRange = false; |
810 | | psStruct->nStartOffset = 0; |
811 | | psStruct->nEndOffset = 0; |
812 | | psStruct->nHTTPCode = 0; |
813 | | psStruct->nFirstHTTPCode = 0; |
814 | | psStruct->nContentLength = 0; |
815 | | psStruct->bFoundContentRange = false; |
816 | | psStruct->bError = false; |
817 | | psStruct->bDetectRangeDownloadingError = true; |
818 | | psStruct->nTimestampDate = 0; |
819 | | |
820 | | psStruct->fp = fp; |
821 | | psStruct->pfnReadCbk = pfnReadCbk; |
822 | | psStruct->pReadCbkUserData = pReadCbkUserData; |
823 | | psStruct->bInterrupted = false; |
824 | | } |
825 | | |
826 | | /************************************************************************/ |
827 | | /* VSICurlHandleWriteFunc() */ |
828 | | /************************************************************************/ |
829 | | |
830 | | size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, |
831 | | void *req) |
832 | | { |
833 | | cpl::WriteFuncStruct *psStruct = static_cast<cpl::WriteFuncStruct *>(req); |
834 | | const size_t nSize = count * nmemb; |
835 | | |
836 | | if (psStruct->bInterrupted) |
837 | | { |
838 | | return 0; |
839 | | } |
840 | | |
841 | | char *pNewBuffer = static_cast<char *>( |
842 | | VSIRealloc(psStruct->pBuffer, psStruct->nSize + nSize + 1)); |
843 | | if (pNewBuffer) |
844 | | { |
845 | | psStruct->pBuffer = pNewBuffer; |
846 | | memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize); |
847 | | psStruct->pBuffer[psStruct->nSize + nSize] = '\0'; |
848 | | if (psStruct->bIsHTTP) |
849 | | { |
850 | | char *pszLine = psStruct->pBuffer + psStruct->nSize; |
851 | | if (STARTS_WITH_CI(pszLine, "HTTP/")) |
852 | | { |
853 | | char *pszSpace = strchr(pszLine, ' '); |
854 | | if (pszSpace) |
855 | | { |
856 | | const int nHTTPCode = atoi(pszSpace + 1); |
857 | | if (psStruct->nFirstHTTPCode == 0) |
858 | | psStruct->nFirstHTTPCode = nHTTPCode; |
859 | | psStruct->nHTTPCode = nHTTPCode; |
860 | | } |
861 | | } |
862 | | else if (STARTS_WITH_CI(pszLine, "Content-Length: ")) |
863 | | { |
864 | | psStruct->nContentLength = CPLScanUIntBig( |
865 | | pszLine + 16, static_cast<int>(strlen(pszLine + 16))); |
866 | | } |
867 | | else if (STARTS_WITH_CI(pszLine, "Content-Range: ")) |
868 | | { |
869 | | psStruct->bFoundContentRange = true; |
870 | | } |
871 | | else if (STARTS_WITH_CI(pszLine, "Date: ")) |
872 | | { |
873 | | CPLString osDate = pszLine + strlen("Date: "); |
874 | | size_t nSizeLine = osDate.size(); |
875 | | while (nSizeLine && (osDate[nSizeLine - 1] == '\r' || |
876 | | osDate[nSizeLine - 1] == '\n')) |
877 | | { |
878 | | osDate.resize(nSizeLine - 1); |
879 | | nSizeLine--; |
880 | | } |
881 | | osDate.Trim(); |
882 | | |
883 | | GIntBig nTimestampDate = |
884 | | VSICurlGetTimeStampFromRFC822DateTime(osDate.c_str()); |
885 | | #if DEBUG_VERBOSE |
886 | | CPLDebug("VSICURL", "Timestamp = " CPL_FRMT_GIB, |
887 | | nTimestampDate); |
888 | | #endif |
889 | | psStruct->nTimestampDate = nTimestampDate; |
890 | | } |
891 | | /*if( nSize > 2 && pszLine[nSize - 2] == '\r' && |
892 | | pszLine[nSize - 1] == '\n' ) |
893 | | { |
894 | | pszLine[nSize - 2] = 0; |
895 | | CPLDebug("VSICURL", "%s", pszLine); |
896 | | pszLine[nSize - 2] = '\r'; |
897 | | }*/ |
898 | | |
899 | | if (pszLine[0] == '\r' && pszLine[1] == '\n') |
900 | | { |
901 | | // Detect servers that don't support range downloading. |
902 | | if (psStruct->nHTTPCode == 200 && |
903 | | psStruct->bDetectRangeDownloadingError && |
904 | | !psStruct->bMultiRange && !psStruct->bFoundContentRange && |
905 | | (psStruct->nStartOffset != 0 || |
906 | | psStruct->nContentLength > |
907 | | 10 * (psStruct->nEndOffset - psStruct->nStartOffset + |
908 | | 1))) |
909 | | { |
910 | | CPLError(CE_Failure, CPLE_AppDefined, |
911 | | "Range downloading not supported by this " |
912 | | "server!"); |
913 | | psStruct->bError = true; |
914 | | return 0; |
915 | | } |
916 | | } |
917 | | } |
918 | | else |
919 | | { |
920 | | if (psStruct->pfnReadCbk) |
921 | | { |
922 | | if (!psStruct->pfnReadCbk(psStruct->fp, buffer, nSize, |
923 | | psStruct->pReadCbkUserData)) |
924 | | { |
925 | | psStruct->bInterrupted = true; |
926 | | return 0; |
927 | | } |
928 | | } |
929 | | } |
930 | | psStruct->nSize += nSize; |
931 | | return nmemb; |
932 | | } |
933 | | else |
934 | | { |
935 | | return 0; |
936 | | } |
937 | | } |
938 | | |
939 | | /************************************************************************/ |
940 | | /* VSICurlIsS3LikeSignedURL() */ |
941 | | /************************************************************************/ |
942 | | |
943 | | static bool VSICurlIsS3LikeSignedURL(const char *pszURL) |
944 | | { |
945 | | return ((strstr(pszURL, ".s3.amazonaws.com/") != nullptr || |
946 | | strstr(pszURL, ".s3.amazonaws.com:") != nullptr || |
947 | | strstr(pszURL, ".storage.googleapis.com/") != nullptr || |
948 | | strstr(pszURL, ".storage.googleapis.com:") != nullptr || |
949 | | strstr(pszURL, ".cloudfront.net/") != nullptr || |
950 | | strstr(pszURL, ".cloudfront.net:") != nullptr) && |
951 | | (strstr(pszURL, "&Signature=") != nullptr || |
952 | | strstr(pszURL, "?Signature=") != nullptr)) || |
953 | | strstr(pszURL, "&X-Amz-Signature=") != nullptr || |
954 | | strstr(pszURL, "?X-Amz-Signature=") != nullptr; |
955 | | } |
956 | | |
957 | | /************************************************************************/ |
958 | | /* VSICurlGetExpiresFromS3LikeSignedURL() */ |
959 | | /************************************************************************/ |
960 | | |
961 | | static GIntBig VSICurlGetExpiresFromS3LikeSignedURL(const char *pszURL) |
962 | | { |
963 | | const auto GetParamValue = [pszURL](const char *pszKey) -> const char * |
964 | | { |
965 | | for (const char *pszPrefix : {"&", "?"}) |
966 | | { |
967 | | std::string osNeedle(pszPrefix); |
968 | | osNeedle += pszKey; |
969 | | osNeedle += '='; |
970 | | const char *pszStr = strstr(pszURL, osNeedle.c_str()); |
971 | | if (pszStr) |
972 | | return pszStr + osNeedle.size(); |
973 | | } |
974 | | return nullptr; |
975 | | }; |
976 | | |
977 | | { |
978 | | // Expires= is a Unix timestamp |
979 | | const char *pszExpires = GetParamValue("Expires"); |
980 | | if (pszExpires != nullptr) |
981 | | return CPLAtoGIntBig(pszExpires); |
982 | | } |
983 | | |
984 | | // X-Amz-Expires= is a delay, to be combined with X-Amz-Date= |
985 | | const char *pszAmzExpires = GetParamValue("X-Amz-Expires"); |
986 | | if (pszAmzExpires == nullptr) |
987 | | return 0; |
988 | | const int nDelay = atoi(pszAmzExpires); |
989 | | |
990 | | const char *pszAmzDate = GetParamValue("X-Amz-Date"); |
991 | | if (pszAmzDate == nullptr) |
992 | | return 0; |
993 | | // pszAmzDate should be YYYYMMDDTHHMMSSZ |
994 | | if (strlen(pszAmzDate) < strlen("YYYYMMDDTHHMMSSZ")) |
995 | | return 0; |
996 | | if (pszAmzDate[strlen("YYYYMMDDTHHMMSSZ") - 1] != 'Z') |
997 | | return 0; |
998 | | struct tm brokendowntime; |
999 | | brokendowntime.tm_year = |
1000 | | atoi(std::string(pszAmzDate).substr(0, 4).c_str()) - 1900; |
1001 | | brokendowntime.tm_mon = |
1002 | | atoi(std::string(pszAmzDate).substr(4, 2).c_str()) - 1; |
1003 | | brokendowntime.tm_mday = atoi(std::string(pszAmzDate).substr(6, 2).c_str()); |
1004 | | brokendowntime.tm_hour = atoi(std::string(pszAmzDate).substr(9, 2).c_str()); |
1005 | | brokendowntime.tm_min = atoi(std::string(pszAmzDate).substr(11, 2).c_str()); |
1006 | | brokendowntime.tm_sec = atoi(std::string(pszAmzDate).substr(13, 2).c_str()); |
1007 | | return CPLYMDHMSToUnixTime(&brokendowntime) + nDelay; |
1008 | | } |
1009 | | |
1010 | | namespace cpl |
1011 | | { |
1012 | | |
1013 | | void VSICurlFilesystemHandlerBase::StartRunThread() |
1014 | | { |
1015 | | if (m_runThread) |
1016 | | return; |
1017 | | |
1018 | | m_stop = false; |
1019 | | m_runThread = |
1020 | | std::make_unique<std::thread>(&VSICurlFilesystemHandlerBase::Run, this); |
1021 | | } |
1022 | | |
1023 | | void VSICurlFilesystemHandlerBase::StopRunThread() |
1024 | | { |
1025 | | if (!m_runThread) |
1026 | | return; |
1027 | | |
1028 | | // Tell the thread to stop. |
1029 | | m_stop = true; |
1030 | | curl_multi_wakeup(m_multi); |
1031 | | |
1032 | | // Wait for run thread to stop. |
1033 | | m_runThread->join(); |
1034 | | m_runThread.reset(); |
1035 | | } |
1036 | | |
1037 | | // Increment the handle count. If this is the first handle, start the run loop. |
1038 | | void VSICurlFilesystemHandlerBase::IncUseCount() |
1039 | | { |
1040 | | std::lock_guard l(m_useMutex); |
1041 | | |
1042 | | if (m_useCount == 0) |
1043 | | StartRunThread(); |
1044 | | m_useCount++; |
1045 | | } |
1046 | | |
1047 | | // Decrement the handle count. If this is the last handle, stop and join the run loop. |
1048 | | void VSICurlFilesystemHandlerBase::DecUseCount() |
1049 | | { |
1050 | | std::lock_guard l(m_useMutex); |
1051 | | |
1052 | | m_useCount--; |
1053 | | if (m_useCount == 0) |
1054 | | StopRunThread(); |
1055 | | } |
1056 | | |
1057 | | // Must be called under lock. |
1058 | | void VSICurlFilesystemHandlerBase::HandleDebug(CURL *easyHandle) |
1059 | | { |
1060 | | // Remove 'easyHandle' from the handle list if it's done. |
1061 | | for (auto it = m_handles.begin(); it != m_handles.end(); ++it) |
1062 | | { |
1063 | | Handle &handle = *it; |
1064 | | if (handle.m_curl == easyHandle) |
1065 | | { |
1066 | | for (auto &debug : handle.m_debug) |
1067 | | CPLDebug(debug.first.c_str(), "%s", debug.second.c_str()); |
1068 | | handle.m_debug.clear(); |
1069 | | break; |
1070 | | } |
1071 | | } |
1072 | | } |
1073 | | |
1074 | | // Must be called under lock. |
1075 | | bool VSICurlFilesystemHandlerBase::RemoveDoneHandle(CURL *easyHandle) |
1076 | | { |
1077 | | // Remove 'easyHandle' from the handle list if it's done. |
1078 | | for (auto it = m_handles.begin(); it != m_handles.end(); ++it) |
1079 | | { |
1080 | | Handle &handle = *it; |
1081 | | if (handle.m_curl == easyHandle) |
1082 | | { |
1083 | | if (handle.m_state == HandleState::Done) |
1084 | | { |
1085 | | m_handles.erase(it); |
1086 | | return true; |
1087 | | } |
1088 | | break; |
1089 | | } |
1090 | | } |
1091 | | return false; |
1092 | | } |
1093 | | |
1094 | | // Perform work on handle and wait for completion. |
1095 | | void VSICurlFilesystemHandlerBase::Perform(CURL *easyHandle) |
1096 | | { |
1097 | | std::unique_lock l(m_runMutex); |
1098 | | |
1099 | | m_handles.push_back(Handle(easyHandle)); |
1100 | | curl_multi_wakeup(m_multi); |
1101 | | |
1102 | | // Wait until the handle is on the done list. |
1103 | | m_runCv.wait(l, |
1104 | | [easyHandle, this] |
1105 | | { |
1106 | | HandleDebug(easyHandle); |
1107 | | return RemoveDoneHandle(easyHandle); |
1108 | | }); |
1109 | | } |
1110 | | |
1111 | | // Perform work on some handles and wait for them to complete. |
1112 | | void VSICurlFilesystemHandlerBase::Perform( |
1113 | | const std::vector<CURL *> &easyHandles) |
1114 | | { |
1115 | | if (easyHandles.empty()) |
1116 | | return; |
1117 | | |
1118 | | std::unique_lock l(m_runMutex); |
1119 | | |
1120 | | for (CURL *easyHandle : easyHandles) |
1121 | | m_handles.push_back(Handle(easyHandle)); |
1122 | | curl_multi_wakeup(m_multi); |
1123 | | |
1124 | | // This seems kinda inefficient because we check easy handles for being done |
1125 | | // that we already know are done, but the numbers should be small so this is all |
1126 | | // pretty inconsequential and the penalty for removing the handles from the `easyHandles` |
1127 | | // list may be more than just re-checking. |
1128 | | size_t cnt = 0; |
1129 | | // Wait until all the requests have completed. |
1130 | | m_runCv.wait(l, |
1131 | | [&easyHandles, &cnt, this] |
1132 | | { |
1133 | | for (CURL *easyHandle : easyHandles) |
1134 | | HandleDebug(easyHandle); |
1135 | | for (CURL *easyHandle : easyHandles) |
1136 | | cnt += RemoveDoneHandle(easyHandle); |
1137 | | return cnt == easyHandles.size(); |
1138 | | }); |
1139 | | } |
1140 | | |
1141 | | // |
1142 | | void VSICurlFilesystemHandlerBase::Interrupt(CURL *easyHandle) |
1143 | | { |
1144 | | std::lock_guard l(m_runMutex); |
1145 | | |
1146 | | for (Handle &h : m_handles) |
1147 | | if (h.m_curl == easyHandle) |
1148 | | { |
1149 | | h.m_state = HandleState::Interrupted; |
1150 | | curl_multi_wakeup(m_multi); |
1151 | | return; |
1152 | | } |
1153 | | } |
1154 | | |
1155 | | // If we have any ready easy handles, add them to the multi handle and clear them from |
1156 | | // the ready list. |
1157 | | int VSICurlFilesystemHandlerBase::HandleReady() |
1158 | | { |
1159 | | std::lock_guard l(m_runMutex); |
1160 | | |
1161 | | for (Handle &handle : m_handles) |
1162 | | { |
1163 | | if (handle.m_state == HandleState::Ready) |
1164 | | { |
1165 | | curl_multi_add_handle(m_multi, handle.m_curl); |
1166 | | handle.m_state = HandleState::Running; |
1167 | | } |
1168 | | } |
1169 | | |
1170 | | int numRunning = 0; |
1171 | | for (Handle &handle : m_handles) |
1172 | | numRunning += (handle.m_state == HandleState::Running); |
1173 | | return numRunning; |
1174 | | } |
1175 | | |
1176 | | // Handle any interrupted handles. |
1177 | | bool VSICurlFilesystemHandlerBase::HandleInterrupted() |
1178 | | { |
1179 | | bool notify = false; |
1180 | | std::lock_guard l(m_runMutex); |
1181 | | |
1182 | | // The assumption here is that if you've interrupted a transfer, you don't really |
1183 | | // care about the result of the transfer, so just remove it and say we're done. |
1184 | | for (Handle &h : m_handles) |
1185 | | { |
1186 | | if (h.m_state == HandleState::Interrupted) |
1187 | | { |
1188 | | curl_multi_remove_handle(m_multi, h.m_curl); |
1189 | | h.m_state = HandleState::Done; |
1190 | | notify = true; |
1191 | | } |
1192 | | } |
1193 | | |
1194 | | return notify; |
1195 | | } |
1196 | | |
1197 | | // Handle any completed easy handles by removing them from the multi handle and |
1198 | | // setting their state to Done. |
1199 | | bool VSICurlFilesystemHandlerBase::HandleCompleted() |
1200 | | { |
1201 | | bool notify = false; |
1202 | | while (true) |
1203 | | { |
1204 | | int msgCnt; |
1205 | | CURLMsg *m = curl_multi_info_read(m_multi, &msgCnt); |
1206 | | if (!m) |
1207 | | break; |
1208 | | if (m->msg != CURLMSG_DONE) |
1209 | | continue; |
1210 | | |
1211 | | curl_multi_remove_handle(m_multi, m->easy_handle); |
1212 | | |
1213 | | std::lock_guard l(m_runMutex); |
1214 | | |
1215 | | for (Handle &h : m_handles) |
1216 | | if (h.m_curl == m->easy_handle) |
1217 | | { |
1218 | | h.m_state = HandleState::Done; |
1219 | | break; |
1220 | | } |
1221 | | |
1222 | | notify = true; |
1223 | | } |
1224 | | return notify; |
1225 | | } |
1226 | | |
1227 | | // Abort all the running transfers as curl has failed internally. Remove the handle. |
1228 | | bool VSICurlFilesystemHandlerBase::HandleFailure() |
1229 | | { |
1230 | | bool notify = false; |
1231 | | |
1232 | | std::lock_guard l(m_runMutex); |
1233 | | for (Handle &h : m_handles) |
1234 | | { |
1235 | | if (h.m_state == HandleState::Running) |
1236 | | { |
1237 | | h.m_state = HandleState::Done; |
1238 | | curl_multi_remove_handle(m_multi, h.m_curl); |
1239 | | notify = true; |
1240 | | } |
1241 | | } |
1242 | | return notify; |
1243 | | } |
1244 | | |
1245 | | // Loop to handle Curl requests. |
1246 | | void VSICurlFilesystemHandlerBase::Run() |
1247 | | { |
1248 | | m_multi = VSICURLMultiInit(); |
1249 | | while (true) |
1250 | | { |
1251 | | if (m_stop) |
1252 | | break; |
1253 | | |
1254 | | // Add ready transfers to the multi handle to run. If there is nothing to run |
1255 | | // (HandleReady() returns 0), wait for a second. However, the curl_multi_poll |
1256 | | // call will break before the 1 sec. timeout if a new handle is added. |
1257 | | if (HandleReady() == 0) |
1258 | | { |
1259 | | curl_multi_poll(m_multi, nullptr, 0, 1000, nullptr); |
1260 | | continue; |
1261 | | } |
1262 | | |
1263 | | int stillRunning; |
1264 | | CURLMcode result = curl_multi_perform(m_multi, &stillRunning); |
1265 | | if (result == CURLM_OK && stillRunning) |
1266 | | result = curl_multi_poll(m_multi, nullptr, 0, 200, nullptr); |
1267 | | |
1268 | | bool notify = HandleInterrupted(); |
1269 | | |
1270 | | if (result != CURLM_OK) |
1271 | | notify |= HandleFailure(); |
1272 | | else |
1273 | | notify |= HandleCompleted(); |
1274 | | if (notify) |
1275 | | m_runCv.notify_all(); |
1276 | | } |
1277 | | |
1278 | | VSICURLMultiCleanup(m_multi); |
1279 | | m_multi = nullptr; |
1280 | | } |
1281 | | |
1282 | | } // namespace cpl |
1283 | | |
1284 | | /************************************************************************/ |
1285 | | /* VSICURLMultiPerform() */ |
1286 | | /************************************************************************/ |
1287 | | |
1288 | | void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle, |
1289 | | std::atomic<bool> *pbInterrupt) |
1290 | | { |
1291 | | if (hEasyHandle) |
1292 | | curl_multi_add_handle(hCurlMultiHandle, hEasyHandle); |
1293 | | |
1294 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
1295 | | while (true) |
1296 | | { |
1297 | | int still_running; |
1298 | | while (curl_multi_perform(hCurlMultiHandle, &still_running) == |
1299 | | CURLM_CALL_MULTI_PERFORM) |
1300 | | { |
1301 | | // loop |
1302 | | } |
1303 | | if (!still_running) |
1304 | | { |
1305 | | break; |
1306 | | } |
1307 | | |
1308 | | #ifdef undef |
1309 | | CURLMsg *msg; |
1310 | | do |
1311 | | { |
1312 | | int msgq = 0; |
1313 | | msg = curl_multi_info_read(hCurlMultiHandle, &msgq); |
1314 | | if (msg && (msg->msg == CURLMSG_DONE)) |
1315 | | { |
1316 | | CURL *e = msg->easy_handle; |
1317 | | } |
1318 | | } while (msg); |
1319 | | #endif |
1320 | | |
1321 | | CPLMultiPerformWait(hCurlMultiHandle); |
1322 | | |
1323 | | if (pbInterrupt && *pbInterrupt) |
1324 | | break; |
1325 | | } |
1326 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
1327 | | |
1328 | | if (hEasyHandle) |
1329 | | curl_multi_remove_handle(hCurlMultiHandle, hEasyHandle); |
1330 | | } |
1331 | | |
1332 | | /************************************************************************/ |
1333 | | /* VSICurlDummyWriteFunc() */ |
1334 | | /************************************************************************/ |
1335 | | |
1336 | | static size_t VSICurlDummyWriteFunc(void *, size_t, size_t, void *) |
1337 | | { |
1338 | | return 0; |
1339 | | } |
1340 | | |
1341 | | /************************************************************************/ |
1342 | | /* VSICURLResetHeaderAndWriterFunctions() */ |
1343 | | /************************************************************************/ |
1344 | | |
1345 | | void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle) |
1346 | | { |
1347 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1348 | | VSICurlDummyWriteFunc); |
1349 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1350 | | VSICurlDummyWriteFunc); |
1351 | | } |
1352 | | |
1353 | | /************************************************************************/ |
1354 | | /* Iso8601ToUnixTime() */ |
1355 | | /************************************************************************/ |
1356 | | |
1357 | | static bool Iso8601ToUnixTime(const char *pszDT, GIntBig *pnUnixTime) |
1358 | | { |
1359 | | int nYear; |
1360 | | int nMonth; |
1361 | | int nDay; |
1362 | | int nHour; |
1363 | | int nMinute; |
1364 | | int nSecond; |
1365 | | if (sscanf(pszDT, "%04d-%02d-%02dT%02d:%02d:%02d", &nYear, &nMonth, &nDay, |
1366 | | &nHour, &nMinute, &nSecond) == 6) |
1367 | | { |
1368 | | struct tm brokendowntime; |
1369 | | brokendowntime.tm_year = nYear - 1900; |
1370 | | brokendowntime.tm_mon = nMonth - 1; |
1371 | | brokendowntime.tm_mday = nDay; |
1372 | | brokendowntime.tm_hour = nHour; |
1373 | | brokendowntime.tm_min = nMinute; |
1374 | | brokendowntime.tm_sec = nSecond; |
1375 | | *pnUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
1376 | | return true; |
1377 | | } |
1378 | | return false; |
1379 | | } |
1380 | | |
1381 | | namespace cpl |
1382 | | { |
1383 | | |
1384 | | /************************************************************************/ |
1385 | | /* ManagePlanetaryComputerSigning() */ |
1386 | | /************************************************************************/ |
1387 | | |
1388 | | void VSICurlHandle::ManagePlanetaryComputerSigning() const |
1389 | | { |
1390 | | // Take global lock |
1391 | | static std::mutex goMutex; |
1392 | | std::lock_guard<std::mutex> oLock(goMutex); |
1393 | | |
1394 | | struct PCSigningInfo |
1395 | | { |
1396 | | std::string osQueryString{}; |
1397 | | GIntBig nExpireTimestamp = 0; |
1398 | | }; |
1399 | | |
1400 | | PCSigningInfo sSigningInfo; |
1401 | | constexpr int knExpirationDelayMargin = 60; |
1402 | | |
1403 | | if (!m_osPlanetaryComputerCollection.empty()) |
1404 | | { |
1405 | | // key is the name of a collection |
1406 | | static lru11::Cache<std::string, PCSigningInfo> goCacheCollection{1024}; |
1407 | | |
1408 | | if (goCacheCollection.tryGet(m_osPlanetaryComputerCollection, |
1409 | | sSigningInfo) && |
1410 | | time(nullptr) + knExpirationDelayMargin <= |
1411 | | sSigningInfo.nExpireTimestamp) |
1412 | | { |
1413 | | m_osQueryString = sSigningInfo.osQueryString; |
1414 | | } |
1415 | | else |
1416 | | { |
1417 | | const auto psResult = |
1418 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
1419 | | "VSICURL_PC_SAS_TOKEN_URL", |
1420 | | "https://planetarycomputer.microsoft.com/api/" |
1421 | | "sas/v1/token/")) + |
1422 | | m_osPlanetaryComputerCollection) |
1423 | | .c_str(), |
1424 | | nullptr); |
1425 | | if (psResult) |
1426 | | { |
1427 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1428 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1429 | | const char *pszToken = aosKeyVals.FetchNameValue("token"); |
1430 | | if (pszToken) |
1431 | | { |
1432 | | m_osQueryString = '?'; |
1433 | | m_osQueryString += pszToken; |
1434 | | |
1435 | | sSigningInfo.osQueryString = m_osQueryString; |
1436 | | sSigningInfo.nExpireTimestamp = 0; |
1437 | | const char *pszExpiry = |
1438 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1439 | | if (pszExpiry) |
1440 | | { |
1441 | | Iso8601ToUnixTime(pszExpiry, |
1442 | | &sSigningInfo.nExpireTimestamp); |
1443 | | } |
1444 | | goCacheCollection.insert(m_osPlanetaryComputerCollection, |
1445 | | sSigningInfo); |
1446 | | |
1447 | | CPLDebug("VSICURL", "Got token from Planetary Computer: %s", |
1448 | | m_osQueryString.c_str()); |
1449 | | } |
1450 | | CPLHTTPDestroyResult(psResult); |
1451 | | } |
1452 | | } |
1453 | | } |
1454 | | else |
1455 | | { |
1456 | | // key is a URL |
1457 | | static lru11::Cache<std::string, PCSigningInfo> goCacheURL{1024}; |
1458 | | |
1459 | | if (goCacheURL.tryGet(m_pszURL, sSigningInfo) && |
1460 | | time(nullptr) + knExpirationDelayMargin <= |
1461 | | sSigningInfo.nExpireTimestamp) |
1462 | | { |
1463 | | m_osQueryString = sSigningInfo.osQueryString; |
1464 | | } |
1465 | | else |
1466 | | { |
1467 | | const auto psResult = |
1468 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
1469 | | "VSICURL_PC_SAS_SIGN_HREF_URL", |
1470 | | "https://planetarycomputer.microsoft.com/api/" |
1471 | | "sas/v1/sign?href=")) + |
1472 | | m_pszURL) |
1473 | | .c_str(), |
1474 | | nullptr); |
1475 | | if (psResult) |
1476 | | { |
1477 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1478 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1479 | | const char *pszHref = aosKeyVals.FetchNameValue("href"); |
1480 | | if (pszHref && STARTS_WITH(pszHref, m_pszURL)) |
1481 | | { |
1482 | | m_osQueryString = pszHref + strlen(m_pszURL); |
1483 | | |
1484 | | sSigningInfo.osQueryString = m_osQueryString; |
1485 | | sSigningInfo.nExpireTimestamp = 0; |
1486 | | const char *pszExpiry = |
1487 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1488 | | if (pszExpiry) |
1489 | | { |
1490 | | Iso8601ToUnixTime(pszExpiry, |
1491 | | &sSigningInfo.nExpireTimestamp); |
1492 | | } |
1493 | | goCacheURL.insert(m_pszURL, sSigningInfo); |
1494 | | |
1495 | | CPLDebug("VSICURL", |
1496 | | "Got signature from Planetary Computer: %s", |
1497 | | m_osQueryString.c_str()); |
1498 | | } |
1499 | | CPLHTTPDestroyResult(psResult); |
1500 | | } |
1501 | | } |
1502 | | } |
1503 | | } |
1504 | | |
1505 | | /************************************************************************/ |
1506 | | /* UpdateQueryString() */ |
1507 | | /************************************************************************/ |
1508 | | |
1509 | | void VSICurlHandle::UpdateQueryString() const |
1510 | | { |
1511 | | if (m_bPlanetaryComputerURLSigning) |
1512 | | { |
1513 | | ManagePlanetaryComputerSigning(); |
1514 | | } |
1515 | | else |
1516 | | { |
1517 | | const char *pszQueryString = VSIGetPathSpecificOption( |
1518 | | m_osFilename.c_str(), "VSICURL_QUERY_STRING", nullptr); |
1519 | | if (pszQueryString) |
1520 | | { |
1521 | | if (m_osFilename.back() == '?') |
1522 | | { |
1523 | | if (pszQueryString[0] == '?') |
1524 | | m_osQueryString = pszQueryString + 1; |
1525 | | else |
1526 | | m_osQueryString = pszQueryString; |
1527 | | } |
1528 | | else |
1529 | | { |
1530 | | if (pszQueryString[0] == '?') |
1531 | | m_osQueryString = pszQueryString; |
1532 | | else |
1533 | | { |
1534 | | m_osQueryString = "?"; |
1535 | | m_osQueryString.append(pszQueryString); |
1536 | | } |
1537 | | } |
1538 | | } |
1539 | | } |
1540 | | } |
1541 | | |
1542 | | /************************************************************************/ |
1543 | | /* GetFileSizeOrHeaders() */ |
1544 | | /************************************************************************/ |
1545 | | |
1546 | | vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, |
1547 | | bool bGetHeaders) |
1548 | | { |
1549 | | if (oFileProp.bHasComputedFileSize && !bGetHeaders) |
1550 | | return oFileProp.fileSize; |
1551 | | |
1552 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
1553 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
1554 | | NetworkStatisticsAction oContextAction("GetFileSize"); |
1555 | | |
1556 | | oFileProp.bHasComputedFileSize = true; |
1557 | | |
1558 | | UpdateQueryString(); |
1559 | | |
1560 | | std::string osURL(m_pszURL + m_osQueryString); |
1561 | | int nTryCount = 0; |
1562 | | bool bRetryWithGet = false; |
1563 | | bool bRetryWithLimitedRangeGet = false; |
1564 | | bool bS3LikeRedirect = false; |
1565 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
1566 | | |
1567 | | retry: |
1568 | | ++nTryCount; |
1569 | | CURL *hCurlHandle = curl_easy_init(); |
1570 | | |
1571 | | struct curl_slist *headers = nullptr; |
1572 | | if (bS3LikeRedirect) |
1573 | | { |
1574 | | // Do not propagate authentication sent to the original URL to a S3-like |
1575 | | // redirect. |
1576 | | CPLStringList aosHTTPOptions{}; |
1577 | | for (const auto &pszOption : m_aosHTTPOptions) |
1578 | | { |
1579 | | if (STARTS_WITH_CI(pszOption, "HTTPAUTH") || |
1580 | | STARTS_WITH_CI(pszOption, "HTTP_BEARER")) |
1581 | | continue; |
1582 | | aosHTTPOptions.AddString(pszOption); |
1583 | | } |
1584 | | headers = |
1585 | | poFS->SetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
1586 | | } |
1587 | | else |
1588 | | { |
1589 | | headers = poFS->SetOptions(hCurlHandle, osURL.c_str(), |
1590 | | m_aosHTTPOptions.List()); |
1591 | | } |
1592 | | |
1593 | | WriteFuncStruct sWriteFuncHeaderData; |
1594 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
1595 | | nullptr); |
1596 | | sWriteFuncHeaderData.bDetectRangeDownloadingError = false; |
1597 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(osURL.c_str(), "http"); |
1598 | | |
1599 | | WriteFuncStruct sWriteFuncData; |
1600 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
1601 | | |
1602 | | std::string osVerb; |
1603 | | std::string osRange; // leave in this scope ! |
1604 | | int nRoundedBufSize = 0; |
1605 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
1606 | | bool bHasUsedLimitedRangeGet = false; |
1607 | | if (bRetryWithLimitedRangeGet || UseLimitRangeGetInsteadOfHead()) |
1608 | | { |
1609 | | bHasUsedLimitedRangeGet = true; |
1610 | | osVerb = "GET"; |
1611 | | const int nBufSize = std::clamp( |
1612 | | atoi(CPLGetConfigOption("GDAL_INGESTED_BYTES_AT_OPEN", "1024")), |
1613 | | 1024, 10 * 1024 * 1024); |
1614 | | nRoundedBufSize = cpl::div_round_up(nBufSize, knDOWNLOAD_CHUNK_SIZE) * |
1615 | | knDOWNLOAD_CHUNK_SIZE; |
1616 | | |
1617 | | // so it gets included in Azure signature |
1618 | | osRange = CPLSPrintf("Range: bytes=0-%d", nRoundedBufSize - 1); |
1619 | | headers = curl_slist_append(headers, osRange.c_str()); |
1620 | | } |
1621 | | // HACK for mbtiles driver: http://a.tiles.mapbox.com/v3/ doesn't accept |
1622 | | // HEAD, as it is a redirect to AWS S3 signed URL, but those are only valid |
1623 | | // for a given type of HTTP request, and thus GET. This is valid for any |
1624 | | // signed URL for AWS S3. |
1625 | | else if (bRetryWithGet || |
1626 | | strstr(osURL.c_str(), ".tiles.mapbox.com/") != nullptr || |
1627 | | VSICurlIsS3LikeSignedURL(osURL.c_str()) || !m_bUseHead) |
1628 | | { |
1629 | | sWriteFuncData.bInterrupted = true; |
1630 | | osVerb = "GET"; |
1631 | | } |
1632 | | else |
1633 | | { |
1634 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1); |
1635 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 0); |
1636 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 1); |
1637 | | osVerb = "HEAD"; |
1638 | | } |
1639 | | |
1640 | | bRetryWithLimitedRangeGet = false; |
1641 | | |
1642 | | if (!AllowAutomaticRedirection()) |
1643 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
1644 | | |
1645 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
1646 | | &sWriteFuncHeaderData); |
1647 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1648 | | VSICurlHandleWriteFunc); |
1649 | | |
1650 | | // Bug with older curl versions (<=7.16.4) and FTP. |
1651 | | // See http://curl.haxx.se/mail/lib-2007-08/0312.html |
1652 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
1653 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1654 | | VSICurlHandleWriteFunc); |
1655 | | |
1656 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
1657 | | szCurlErrBuf[0] = '\0'; |
1658 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
1659 | | |
1660 | | headers = GetCurlHeaders(osVerb, headers); |
1661 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
1662 | | |
1663 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
1664 | | |
1665 | | poFS->Perform(hCurlHandle); |
1666 | | |
1667 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
1668 | | |
1669 | | curl_slist_free_all(headers); |
1670 | | |
1671 | | oFileProp.eExists = EXIST_UNKNOWN; |
1672 | | |
1673 | | curl_off_t filetime = -1; |
1674 | | GIntBig mtime = 0; |
1675 | | if (curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME_T, &filetime) == |
1676 | | CURLE_OK && |
1677 | | filetime != -1) |
1678 | | { |
1679 | | mtime = static_cast<GIntBig>(filetime); |
1680 | | } |
1681 | | |
1682 | | if (osVerb == "GET") |
1683 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
1684 | | else |
1685 | | NetworkStatisticsLogger::LogHEAD(); |
1686 | | |
1687 | | if (STARTS_WITH(osURL.c_str(), "ftp")) |
1688 | | { |
1689 | | if (sWriteFuncData.pBuffer != nullptr) |
1690 | | { |
1691 | | const char *pszContentLength = |
1692 | | strstr(const_cast<const char *>(sWriteFuncData.pBuffer), |
1693 | | "Content-Length: "); |
1694 | | if (pszContentLength) |
1695 | | { |
1696 | | pszContentLength += strlen("Content-Length: "); |
1697 | | oFileProp.eExists = EXIST_YES; |
1698 | | oFileProp.fileSize = |
1699 | | CPLScanUIntBig(pszContentLength, |
1700 | | static_cast<int>(strlen(pszContentLength))); |
1701 | | if constexpr (ENABLE_DEBUG) |
1702 | | { |
1703 | | CPLDebug(poFS->GetDebugKey(), |
1704 | | "GetFileSize(%s)=" CPL_FRMT_GUIB, osURL.c_str(), |
1705 | | oFileProp.fileSize); |
1706 | | } |
1707 | | } |
1708 | | } |
1709 | | } |
1710 | | |
1711 | | double dfSize = 0; |
1712 | | long response_code = -1; |
1713 | | if (oFileProp.eExists != EXIST_YES) |
1714 | | { |
1715 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
1716 | | |
1717 | | bool bAlreadyLogged = false; |
1718 | | if (response_code >= 400 && szCurlErrBuf[0] == '\0') |
1719 | | { |
1720 | | const bool bLogResponse = CPLTestConfigOption("CPL_CURL_VERBOSE"); |
1721 | | if (bLogResponse && sWriteFuncData.pBuffer) |
1722 | | { |
1723 | | const char *pszErrorMsg = |
1724 | | static_cast<const char *>(sWriteFuncData.pBuffer); |
1725 | | bAlreadyLogged = true; |
1726 | | CPLDebug( |
1727 | | poFS->GetDebugKey(), |
1728 | | "GetFileSize(%s): response_code=%d, server error msg=%s", |
1729 | | osURL.c_str(), static_cast<int>(response_code), |
1730 | | pszErrorMsg[0] ? pszErrorMsg : "(no message provided)"); |
1731 | | } |
1732 | | } |
1733 | | else if (szCurlErrBuf[0] != '\0') |
1734 | | { |
1735 | | bAlreadyLogged = true; |
1736 | | CPLDebug(poFS->GetDebugKey(), |
1737 | | "GetFileSize(%s): response_code=%d, curl error msg=%s", |
1738 | | osURL.c_str(), static_cast<int>(response_code), |
1739 | | szCurlErrBuf); |
1740 | | } |
1741 | | |
1742 | | std::string osEffectiveURL; |
1743 | | { |
1744 | | char *pszEffectiveURL = nullptr; |
1745 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
1746 | | &pszEffectiveURL); |
1747 | | if (pszEffectiveURL) |
1748 | | osEffectiveURL = pszEffectiveURL; |
1749 | | } |
1750 | | |
1751 | | if (!osEffectiveURL.empty() && |
1752 | | strstr(osEffectiveURL.c_str(), osURL.c_str()) == nullptr) |
1753 | | { |
1754 | | // Moved permanently ? |
1755 | | if (sWriteFuncHeaderData.nFirstHTTPCode == 301 || |
1756 | | (m_bUseRedirectURLIfNoQueryStringParams && |
1757 | | osEffectiveURL.find('?') == std::string::npos)) |
1758 | | { |
1759 | | CPLDebug(poFS->GetDebugKey(), |
1760 | | "Using effective URL %s permanently", |
1761 | | osEffectiveURL.c_str()); |
1762 | | oFileProp.osRedirectURL = osEffectiveURL; |
1763 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1764 | | } |
1765 | | else |
1766 | | { |
1767 | | CPLDebug(poFS->GetDebugKey(), |
1768 | | "Using effective URL %s temporarily", |
1769 | | osEffectiveURL.c_str()); |
1770 | | } |
1771 | | |
1772 | | // Is this is a redirect to a S3 URL? |
1773 | | if (VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
1774 | | !VSICurlIsS3LikeSignedURL(osURL.c_str())) |
1775 | | { |
1776 | | // Note that this is a redirect as we won't notice after the |
1777 | | // retry. |
1778 | | bS3LikeRedirect = true; |
1779 | | |
1780 | | if (!bRetryWithGet && osVerb == "HEAD" && response_code == 403) |
1781 | | { |
1782 | | CPLDebug(poFS->GetDebugKey(), |
1783 | | "Redirected to a AWS S3 signed URL. Retrying " |
1784 | | "with GET request instead of HEAD since the URL " |
1785 | | "might be valid only for GET"); |
1786 | | bRetryWithGet = true; |
1787 | | osURL = std::move(osEffectiveURL); |
1788 | | CPLFree(sWriteFuncData.pBuffer); |
1789 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1790 | | curl_easy_cleanup(hCurlHandle); |
1791 | | goto retry; |
1792 | | } |
1793 | | } |
1794 | | else if (oFileProp.osRedirectURL.empty() && nTryCount == 1 && |
1795 | | ((response_code >= 300 && response_code < 400) || |
1796 | | (osVerb == "HEAD" && response_code == 403))) |
1797 | | { |
1798 | | if (response_code == 403) |
1799 | | { |
1800 | | CPLDebug( |
1801 | | poFS->GetDebugKey(), |
1802 | | "Retrying redirected URL with GET instead of HEAD"); |
1803 | | bRetryWithGet = true; |
1804 | | } |
1805 | | osURL = std::move(osEffectiveURL); |
1806 | | CPLFree(sWriteFuncData.pBuffer); |
1807 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1808 | | curl_easy_cleanup(hCurlHandle); |
1809 | | goto retry; |
1810 | | } |
1811 | | } |
1812 | | |
1813 | | if (bS3LikeRedirect && response_code >= 200 && response_code < 300 && |
1814 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
1815 | | !osEffectiveURL.empty() && |
1816 | | CPLTestBool( |
1817 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
1818 | | { |
1819 | | const GIntBig nExpireTimestamp = |
1820 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
1821 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
1822 | | { |
1823 | | const int nValidity = static_cast<int>( |
1824 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
1825 | | CPLDebug(poFS->GetDebugKey(), |
1826 | | "Will use redirect URL for the next %d seconds", |
1827 | | nValidity); |
1828 | | // As our local clock might not be in sync with server clock, |
1829 | | // figure out the expiration timestamp in local time |
1830 | | oFileProp.bS3LikeRedirect = true; |
1831 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
1832 | | oFileProp.osRedirectURL = osEffectiveURL; |
1833 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1834 | | } |
1835 | | } |
1836 | | |
1837 | | // Split a string with the raw HTTP response headers as a key/value |
1838 | | // CPLStringList |
1839 | | const auto TokenizeHeaders = [](const char *pszHeaders) -> CPLStringList |
1840 | | { |
1841 | | CPLStringList aosHeaders; |
1842 | | while (pszHeaders) |
1843 | | { |
1844 | | const char *pszDelim = strchr(pszHeaders, ':'); |
1845 | | if (!pszDelim) |
1846 | | break; |
1847 | | const char *pszValue = pszDelim + 1; |
1848 | | |
1849 | | // Skip whitespace after colon |
1850 | | while (*pszValue == ' ' || *pszValue == '\t') |
1851 | | ++pszValue; |
1852 | | |
1853 | | // Find end of value |
1854 | | const char *pszEndOfValue = pszValue; |
1855 | | while (*pszEndOfValue && |
1856 | | !(*pszEndOfValue == '\r' && pszEndOfValue[1] == '\n')) |
1857 | | ++pszEndOfValue; |
1858 | | |
1859 | | aosHeaders.SetNameValue( |
1860 | | std::string(pszHeaders, pszDelim - pszHeaders).c_str(), |
1861 | | std::string(pszValue, pszEndOfValue - pszValue).c_str()); |
1862 | | |
1863 | | if (*pszEndOfValue == '\r' && pszEndOfValue[1] == '\n') |
1864 | | pszHeaders = pszEndOfValue + 2; |
1865 | | else |
1866 | | break; |
1867 | | } |
1868 | | return aosHeaders; |
1869 | | }; |
1870 | | |
1871 | | if (response_code < 300) |
1872 | | { |
1873 | | curl_off_t nSizeTmp = 0; |
1874 | | const CURLcode code = curl_easy_getinfo( |
1875 | | hCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &nSizeTmp); |
1876 | | CPL_IGNORE_RET_VAL(dfSize); |
1877 | | dfSize = static_cast<double>(nSizeTmp); |
1878 | | if (code == 0) |
1879 | | { |
1880 | | if (dfSize < 0) |
1881 | | { |
1882 | | if (osVerb == "HEAD" && !bRetryWithGet && |
1883 | | response_code == 200) |
1884 | | { |
1885 | | if (sWriteFuncHeaderData.pBuffer) |
1886 | | { |
1887 | | const CPLStringList aosHeaders( |
1888 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1889 | | if (strcmp(aosHeaders.FetchNameValueDef( |
1890 | | "accept-ranges", ""), |
1891 | | "bytes") == 0) |
1892 | | { |
1893 | | CPLDebug( |
1894 | | poFS->GetDebugKey(), |
1895 | | "HEAD did not provide file size. Retrying " |
1896 | | "with limited range GET"); |
1897 | | bRetryWithLimitedRangeGet = true; |
1898 | | CPLFree(sWriteFuncData.pBuffer); |
1899 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1900 | | curl_easy_cleanup(hCurlHandle); |
1901 | | goto retry; |
1902 | | } |
1903 | | } |
1904 | | |
1905 | | CPLDebug(poFS->GetDebugKey(), |
1906 | | "HEAD did not provide file size. Retrying " |
1907 | | "with GET"); |
1908 | | bRetryWithGet = true; |
1909 | | CPLFree(sWriteFuncData.pBuffer); |
1910 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1911 | | curl_easy_cleanup(hCurlHandle); |
1912 | | goto retry; |
1913 | | } |
1914 | | |
1915 | | if (poFS->GetFSPrefix() == "/vsicurl/" || |
1916 | | poFS->GetFSPrefix() == "/vsicurl?") |
1917 | | { |
1918 | | const CPLStringList aosHeaders( |
1919 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1920 | | if (strcmp(aosHeaders.FetchNameValueDef( |
1921 | | "transfer-encoding", ""), |
1922 | | "chunked") == 0) |
1923 | | { |
1924 | | CPLError( |
1925 | | CE_Failure, CPLE_AppDefined, |
1926 | | "Server does not seem to support range " |
1927 | | "requests. " |
1928 | | "Maybe retry with /vsicurl_streaming/ if the " |
1929 | | "read " |
1930 | | "access pattern is compatible with sequential " |
1931 | | "reading, or download the file entirely"); |
1932 | | } |
1933 | | } |
1934 | | } |
1935 | | else |
1936 | | { |
1937 | | oFileProp.eExists = EXIST_YES; |
1938 | | oFileProp.fileSize = static_cast<GUIntBig>(dfSize); |
1939 | | } |
1940 | | } |
1941 | | } |
1942 | | |
1943 | | if (sWriteFuncHeaderData.pBuffer != nullptr && |
1944 | | (response_code == 200 || response_code == 206)) |
1945 | | { |
1946 | | { |
1947 | | const CPLStringList aosHeaders( |
1948 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
1949 | | for (const auto &[pszKey, pszValue] : |
1950 | | cpl::IterateNameValue(aosHeaders)) |
1951 | | { |
1952 | | if (bGetHeaders) |
1953 | | { |
1954 | | m_aosHeaders.SetNameValue(pszKey, pszValue); |
1955 | | } |
1956 | | if (EQUAL(pszKey, "Cache-Control") && |
1957 | | EQUAL(pszValue, "no-cache") && |
1958 | | CPLTestBool(CPLGetConfigOption( |
1959 | | "CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES"))) |
1960 | | { |
1961 | | m_bCached = false; |
1962 | | } |
1963 | | |
1964 | | else if (EQUAL(pszKey, "ETag")) |
1965 | | { |
1966 | | std::string osValue(pszValue); |
1967 | | if (osValue.size() >= 2 && osValue.front() == '"' && |
1968 | | osValue.back() == '"') |
1969 | | osValue = osValue.substr(1, osValue.size() - 2); |
1970 | | oFileProp.ETag = std::move(osValue); |
1971 | | } |
1972 | | |
1973 | | // Azure Data Lake Storage |
1974 | | else if (EQUAL(pszKey, "x-ms-resource-type")) |
1975 | | { |
1976 | | if (EQUAL(pszValue, "file")) |
1977 | | { |
1978 | | oFileProp.nMode |= S_IFREG; |
1979 | | } |
1980 | | else if (EQUAL(pszValue, "directory")) |
1981 | | { |
1982 | | oFileProp.bIsDirectory = true; |
1983 | | oFileProp.nMode |= S_IFDIR; |
1984 | | } |
1985 | | } |
1986 | | else if (EQUAL(pszKey, "x-ms-permissions")) |
1987 | | { |
1988 | | oFileProp.nMode |= |
1989 | | VSICurlParseUnixPermissions(pszValue); |
1990 | | } |
1991 | | |
1992 | | // https://overturemapswestus2.blob.core.windows.net/release/2024-11-13.0/theme%3Ddivisions/type%3Ddivision_area |
1993 | | // returns a x-ms-meta-hdi_isfolder: true header |
1994 | | else if (EQUAL(pszKey, "x-ms-meta-hdi_isfolder") && |
1995 | | EQUAL(pszValue, "true")) |
1996 | | { |
1997 | | oFileProp.bIsAzureFolder = true; |
1998 | | oFileProp.bIsDirectory = true; |
1999 | | oFileProp.nMode |= S_IFDIR; |
2000 | | } |
2001 | | } |
2002 | | } |
2003 | | } |
2004 | | |
2005 | | if (bHasUsedLimitedRangeGet && response_code == 206) |
2006 | | { |
2007 | | oFileProp.eExists = EXIST_NO; |
2008 | | oFileProp.fileSize = 0; |
2009 | | if (sWriteFuncHeaderData.pBuffer != nullptr) |
2010 | | { |
2011 | | const CPLStringList aosHeaders( |
2012 | | TokenizeHeaders(sWriteFuncHeaderData.pBuffer)); |
2013 | | const char *pszContentRange = |
2014 | | aosHeaders.FetchNameValue("content-range"); |
2015 | | // Trailing space in string intended |
2016 | | if (pszContentRange && |
2017 | | STARTS_WITH_CI(pszContentRange, "bytes ")) |
2018 | | { |
2019 | | pszContentRange += strlen("bytes "); |
2020 | | pszContentRange = strchr(pszContentRange, '/'); |
2021 | | if (pszContentRange) |
2022 | | { |
2023 | | oFileProp.eExists = EXIST_YES; |
2024 | | oFileProp.fileSize = static_cast<GUIntBig>( |
2025 | | CPLAtoGIntBig(pszContentRange + 1)); |
2026 | | } |
2027 | | } |
2028 | | |
2029 | | // Add first bytes to cache |
2030 | | if (sWriteFuncData.pBuffer != nullptr) |
2031 | | { |
2032 | | size_t nOffset = 0; |
2033 | | while (nOffset < sWriteFuncData.nSize) |
2034 | | { |
2035 | | const size_t nToCache = |
2036 | | std::min<size_t>(sWriteFuncData.nSize - nOffset, |
2037 | | knDOWNLOAD_CHUNK_SIZE); |
2038 | | poFS->AddRegion(m_pszURL, nOffset, nToCache, |
2039 | | sWriteFuncData.pBuffer + nOffset); |
2040 | | nOffset += nToCache; |
2041 | | } |
2042 | | } |
2043 | | } |
2044 | | } |
2045 | | else if (IsDirectoryFromExists(osVerb.c_str(), |
2046 | | static_cast<int>(response_code))) |
2047 | | { |
2048 | | oFileProp.eExists = EXIST_YES; |
2049 | | oFileProp.fileSize = 0; |
2050 | | oFileProp.bIsDirectory = true; |
2051 | | } |
2052 | | // 405 = Method not allowed |
2053 | | else if (response_code == 405 && !bRetryWithGet && osVerb == "HEAD") |
2054 | | { |
2055 | | CPLDebug(poFS->GetDebugKey(), |
2056 | | "HEAD not allowed. Retrying with GET"); |
2057 | | bRetryWithGet = true; |
2058 | | CPLFree(sWriteFuncData.pBuffer); |
2059 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2060 | | curl_easy_cleanup(hCurlHandle); |
2061 | | goto retry; |
2062 | | } |
2063 | | else if (response_code == 416) |
2064 | | { |
2065 | | oFileProp.eExists = EXIST_YES; |
2066 | | oFileProp.fileSize = 0; |
2067 | | } |
2068 | | else if (response_code != 200) |
2069 | | { |
2070 | | // Look if we should attempt a retry |
2071 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
2072 | | sWriteFuncHeaderData.pBuffer, |
2073 | | szCurlErrBuf)) |
2074 | | { |
2075 | | CPLError(CE_Warning, CPLE_AppDefined, |
2076 | | "HTTP error code: %d - %s. " |
2077 | | "Retrying again in %.1f secs", |
2078 | | static_cast<int>(response_code), m_pszURL, |
2079 | | oRetryContext.GetCurrentDelay()); |
2080 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
2081 | | CPLFree(sWriteFuncData.pBuffer); |
2082 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2083 | | curl_easy_cleanup(hCurlHandle); |
2084 | | goto retry; |
2085 | | } |
2086 | | |
2087 | | if (sWriteFuncData.pBuffer != nullptr) |
2088 | | { |
2089 | | if (UseLimitRangeGetInsteadOfHead() && |
2090 | | CanRestartOnError(sWriteFuncData.pBuffer, |
2091 | | sWriteFuncHeaderData.pBuffer, bSetError)) |
2092 | | { |
2093 | | oFileProp.bHasComputedFileSize = false; |
2094 | | CPLFree(sWriteFuncData.pBuffer); |
2095 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2096 | | curl_easy_cleanup(hCurlHandle); |
2097 | | return GetFileSizeOrHeaders(bSetError, bGetHeaders); |
2098 | | } |
2099 | | else |
2100 | | { |
2101 | | CPL_IGNORE_RET_VAL(CanRestartOnError( |
2102 | | sWriteFuncData.pBuffer, sWriteFuncHeaderData.pBuffer, |
2103 | | bSetError)); |
2104 | | } |
2105 | | } |
2106 | | |
2107 | | // If there was no VSI error thrown in the process, |
2108 | | // fail by reporting the HTTP response code. |
2109 | | if (bSetError && VSIGetLastErrorNo() == 0) |
2110 | | { |
2111 | | if (strlen(szCurlErrBuf) > 0) |
2112 | | { |
2113 | | if (response_code == 0) |
2114 | | { |
2115 | | VSIError(VSIE_HttpError, "CURL error: %s", |
2116 | | szCurlErrBuf); |
2117 | | } |
2118 | | else |
2119 | | { |
2120 | | VSIError(VSIE_HttpError, "HTTP response code: %d - %s", |
2121 | | static_cast<int>(response_code), szCurlErrBuf); |
2122 | | } |
2123 | | } |
2124 | | else |
2125 | | { |
2126 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
2127 | | static_cast<int>(response_code)); |
2128 | | } |
2129 | | } |
2130 | | else |
2131 | | { |
2132 | | if (response_code != 400 && response_code != 404) |
2133 | | { |
2134 | | CPLError(CE_Warning, CPLE_AppDefined, |
2135 | | "HTTP response code on %s: %d", osURL.c_str(), |
2136 | | static_cast<int>(response_code)); |
2137 | | } |
2138 | | // else a CPLDebug() is emitted below |
2139 | | } |
2140 | | |
2141 | | oFileProp.eExists = EXIST_NO; |
2142 | | oFileProp.nHTTPCode = static_cast<int>(response_code); |
2143 | | oFileProp.fileSize = 0; |
2144 | | } |
2145 | | else if (sWriteFuncData.pBuffer != nullptr) |
2146 | | { |
2147 | | ProcessGetFileSizeResult( |
2148 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer)); |
2149 | | } |
2150 | | |
2151 | | // Try to guess if this is a directory. Generally if this is a |
2152 | | // directory, curl will retry with an URL with slash added. |
2153 | | if (!osEffectiveURL.empty() && |
2154 | | strncmp(osURL.c_str(), osEffectiveURL.c_str(), osURL.size()) == 0 && |
2155 | | osEffectiveURL[osURL.size()] == '/' && |
2156 | | oFileProp.eExists != EXIST_NO) |
2157 | | { |
2158 | | oFileProp.eExists = EXIST_YES; |
2159 | | oFileProp.fileSize = 0; |
2160 | | oFileProp.bIsDirectory = true; |
2161 | | } |
2162 | | else if (osURL.back() == '/') |
2163 | | { |
2164 | | oFileProp.bIsDirectory = true; |
2165 | | } |
2166 | | |
2167 | | if (!bAlreadyLogged) |
2168 | | { |
2169 | | CPLDebug(poFS->GetDebugKey(), |
2170 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
2171 | | osURL.c_str(), oFileProp.fileSize, |
2172 | | static_cast<int>(response_code)); |
2173 | | } |
2174 | | } |
2175 | | |
2176 | | CPLFree(sWriteFuncData.pBuffer); |
2177 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2178 | | curl_easy_cleanup(hCurlHandle); |
2179 | | |
2180 | | oFileProp.bHasComputedFileSize = true; |
2181 | | if (mtime > 0) |
2182 | | oFileProp.mTime = mtime; |
2183 | | // Do not update cached file properties if cURL returned a non-HTTP error |
2184 | | if (response_code != 0) |
2185 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2186 | | |
2187 | | return oFileProp.fileSize; |
2188 | | } |
2189 | | |
2190 | | /************************************************************************/ |
2191 | | /* Exists() */ |
2192 | | /************************************************************************/ |
2193 | | |
2194 | | bool VSICurlHandle::Exists(bool bSetError) |
2195 | | { |
2196 | | if (oFileProp.eExists == EXIST_UNKNOWN) |
2197 | | { |
2198 | | GetFileSize(bSetError); |
2199 | | } |
2200 | | else if (oFileProp.eExists == EXIST_NO) |
2201 | | { |
2202 | | // If there was no VSI error thrown in the process, |
2203 | | // and we know the HTTP error code of the first request where the |
2204 | | // file could not be retrieved, fail by reporting the HTTP code. |
2205 | | if (bSetError && VSIGetLastErrorNo() == 0 && oFileProp.nHTTPCode) |
2206 | | { |
2207 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
2208 | | oFileProp.nHTTPCode); |
2209 | | } |
2210 | | } |
2211 | | |
2212 | | return oFileProp.eExists == EXIST_YES; |
2213 | | } |
2214 | | |
2215 | | /************************************************************************/ |
2216 | | /* Tell() */ |
2217 | | /************************************************************************/ |
2218 | | |
2219 | | vsi_l_offset VSICurlHandle::Tell() |
2220 | | { |
2221 | | return curOffset; |
2222 | | } |
2223 | | |
2224 | | /************************************************************************/ |
2225 | | /* GetRedirectURLIfValid() */ |
2226 | | /************************************************************************/ |
2227 | | |
2228 | | std::string |
2229 | | VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired, |
2230 | | CPLStringList &aosHTTPOptions) const |
2231 | | { |
2232 | | bHasExpired = false; |
2233 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2234 | | |
2235 | | std::string osURL(m_pszURL + m_osQueryString); |
2236 | | if (oFileProp.bS3LikeRedirect) |
2237 | | { |
2238 | | if (time(nullptr) + 1 < oFileProp.nExpireTimestampLocal) |
2239 | | { |
2240 | | CPLDebug(poFS->GetDebugKey(), |
2241 | | "Using redirect URL as it looks to be still valid " |
2242 | | "(%d seconds left)", |
2243 | | static_cast<int>(oFileProp.nExpireTimestampLocal - |
2244 | | time(nullptr))); |
2245 | | osURL = oFileProp.osRedirectURL; |
2246 | | } |
2247 | | else |
2248 | | { |
2249 | | CPLDebug(poFS->GetDebugKey(), |
2250 | | "Redirect URL has expired. Using original URL"); |
2251 | | oFileProp.bS3LikeRedirect = false; |
2252 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2253 | | bHasExpired = true; |
2254 | | } |
2255 | | } |
2256 | | else if (!oFileProp.osRedirectURL.empty()) |
2257 | | { |
2258 | | osURL = oFileProp.osRedirectURL; |
2259 | | bHasExpired = false; |
2260 | | } |
2261 | | |
2262 | | if (m_pszURL != osURL) |
2263 | | { |
2264 | | const char *pszAuthorizationHeaderAllowed = VSIGetPathSpecificOption( |
2265 | | m_osFilename.c_str(), |
2266 | | "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT", |
2267 | | "IF_SAME_HOST"); |
2268 | | if (EQUAL(pszAuthorizationHeaderAllowed, "IF_SAME_HOST")) |
2269 | | { |
2270 | | const auto ExtractServer = [](const std::string &s) |
2271 | | { |
2272 | | size_t afterHTTPPos = 0; |
2273 | | if (STARTS_WITH(s.c_str(), "http://")) |
2274 | | afterHTTPPos = strlen("http://"); |
2275 | | else if (STARTS_WITH(s.c_str(), "https://")) |
2276 | | afterHTTPPos = strlen("https://"); |
2277 | | const auto posSlash = s.find('/', afterHTTPPos); |
2278 | | if (posSlash != std::string::npos) |
2279 | | return s.substr(afterHTTPPos, posSlash - afterHTTPPos); |
2280 | | else |
2281 | | return s.substr(afterHTTPPos); |
2282 | | }; |
2283 | | |
2284 | | if (ExtractServer(osURL) != ExtractServer(m_pszURL)) |
2285 | | { |
2286 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", |
2287 | | "NO"); |
2288 | | } |
2289 | | } |
2290 | | else if (!CPLTestBool(pszAuthorizationHeaderAllowed)) |
2291 | | { |
2292 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", "NO"); |
2293 | | } |
2294 | | } |
2295 | | |
2296 | | return osURL; |
2297 | | } |
2298 | | |
2299 | | /************************************************************************/ |
2300 | | /* CurrentDownload */ |
2301 | | /************************************************************************/ |
2302 | | |
2303 | | namespace |
2304 | | { |
2305 | | struct CurrentDownload |
2306 | | { |
2307 | | VSICurlFilesystemHandlerBase *m_poFS = nullptr; |
2308 | | std::string m_osURL{}; |
2309 | | vsi_l_offset m_nStartOffset = 0; |
2310 | | int m_nBlocks = 0; |
2311 | | std::string m_osAlreadyDownloadedData{}; |
2312 | | bool m_bHasAlreadyDownloadedData = false; |
2313 | | |
2314 | | CurrentDownload(VSICurlFilesystemHandlerBase *poFS, const char *pszURL, |
2315 | | vsi_l_offset startOffset, int nBlocks) |
2316 | | : m_poFS(poFS), m_osURL(pszURL), m_nStartOffset(startOffset), |
2317 | | m_nBlocks(nBlocks) |
2318 | | { |
2319 | | auto res = m_poFS->NotifyStartDownloadRegion(m_osURL, m_nStartOffset, |
2320 | | m_nBlocks); |
2321 | | m_bHasAlreadyDownloadedData = res.first; |
2322 | | m_osAlreadyDownloadedData = std::move(res.second); |
2323 | | } |
2324 | | |
2325 | | bool HasAlreadyDownloadedData() const |
2326 | | { |
2327 | | return m_bHasAlreadyDownloadedData; |
2328 | | } |
2329 | | |
2330 | | const std::string &GetAlreadyDownloadedData() const |
2331 | | { |
2332 | | return m_osAlreadyDownloadedData; |
2333 | | } |
2334 | | |
2335 | | void SetData(const std::string &osData) |
2336 | | { |
2337 | | CPLAssert(!m_bHasAlreadyDownloadedData); |
2338 | | m_bHasAlreadyDownloadedData = true; |
2339 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
2340 | | osData); |
2341 | | } |
2342 | | |
2343 | | ~CurrentDownload() |
2344 | | { |
2345 | | if (!m_bHasAlreadyDownloadedData) |
2346 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
2347 | | std::string()); |
2348 | | } |
2349 | | |
2350 | | CurrentDownload(const CurrentDownload &) = delete; |
2351 | | CurrentDownload &operator=(const CurrentDownload &) = delete; |
2352 | | }; |
2353 | | } // namespace |
2354 | | |
2355 | | /************************************************************************/ |
2356 | | /* NotifyStartDownloadRegion() */ |
2357 | | /************************************************************************/ |
2358 | | |
2359 | | /** Indicate intent at downloading a new region. |
2360 | | * |
2361 | | * If the region is already in download in another thread, then wait for its |
2362 | | * completion. |
2363 | | * |
2364 | | * Returns: |
2365 | | * - (false, empty string) if a new download is needed |
2366 | | * - (true, region_content) if we have been waiting for a download of the same |
2367 | | * region to be completed and got its result. Note that region_content will be |
2368 | | * empty if the download of that region failed. |
2369 | | */ |
2370 | | std::pair<bool, std::string> |
2371 | | VSICurlFilesystemHandlerBase::NotifyStartDownloadRegion( |
2372 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks) |
2373 | | { |
2374 | | std::string osId(osURL); |
2375 | | osId += '_'; |
2376 | | osId += std::to_string(startOffset); |
2377 | | osId += '_'; |
2378 | | osId += std::to_string(nBlocks); |
2379 | | |
2380 | | m_oMutex.lock(); |
2381 | | auto oIter = m_oMapRegionInDownload.find(osId); |
2382 | | if (oIter != m_oMapRegionInDownload.end()) |
2383 | | { |
2384 | | auto ®ion = *(oIter->second); |
2385 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
2386 | | m_oMutex.unlock(); |
2387 | | region.nWaiters++; |
2388 | | while (region.bDownloadInProgress) |
2389 | | { |
2390 | | region.oCond.wait(oRegionLock); |
2391 | | } |
2392 | | std::string osRet = region.osData; |
2393 | | region.nWaiters--; |
2394 | | region.oCond.notify_one(); |
2395 | | return std::pair<bool, std::string>(true, osRet); |
2396 | | } |
2397 | | else |
2398 | | { |
2399 | | auto poRegionInDownload = std::make_unique<RegionInDownload>(); |
2400 | | poRegionInDownload->bDownloadInProgress = true; |
2401 | | m_oMapRegionInDownload[osId] = std::move(poRegionInDownload); |
2402 | | m_oMutex.unlock(); |
2403 | | return std::pair<bool, std::string>(false, std::string()); |
2404 | | } |
2405 | | } |
2406 | | |
2407 | | /************************************************************************/ |
2408 | | /* NotifyStopDownloadRegion() */ |
2409 | | /************************************************************************/ |
2410 | | |
2411 | | void VSICurlFilesystemHandlerBase::NotifyStopDownloadRegion( |
2412 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks, |
2413 | | const std::string &osData) |
2414 | | { |
2415 | | std::string osId(osURL); |
2416 | | osId += '_'; |
2417 | | osId += std::to_string(startOffset); |
2418 | | osId += '_'; |
2419 | | osId += std::to_string(nBlocks); |
2420 | | |
2421 | | m_oMutex.lock(); |
2422 | | auto oIter = m_oMapRegionInDownload.find(osId); |
2423 | | CPLAssert(oIter != m_oMapRegionInDownload.end()); |
2424 | | auto ®ion = *(oIter->second); |
2425 | | { |
2426 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
2427 | | if (region.nWaiters) |
2428 | | { |
2429 | | region.osData = osData; |
2430 | | region.bDownloadInProgress = false; |
2431 | | region.oCond.notify_all(); |
2432 | | |
2433 | | while (region.nWaiters) |
2434 | | { |
2435 | | region.oCond.wait(oRegionLock); |
2436 | | } |
2437 | | } |
2438 | | } |
2439 | | m_oMapRegionInDownload.erase(oIter); |
2440 | | m_oMutex.unlock(); |
2441 | | } |
2442 | | |
2443 | | /************************************************************************/ |
2444 | | /* DownloadRegion() */ |
2445 | | /************************************************************************/ |
2446 | | |
2447 | | std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, |
2448 | | const int nBlocks) |
2449 | | { |
2450 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
2451 | | return std::string(); |
2452 | | |
2453 | | if (oFileProp.eExists == EXIST_NO) |
2454 | | return std::string(); |
2455 | | |
2456 | | // Check if there is not a download of the same region in progress in |
2457 | | // another thread, and if so wait for it to be completed |
2458 | | CurrentDownload currentDownload(poFS, m_pszURL, startOffset, nBlocks); |
2459 | | if (currentDownload.HasAlreadyDownloadedData()) |
2460 | | { |
2461 | | return currentDownload.GetAlreadyDownloadedData(); |
2462 | | } |
2463 | | |
2464 | | begin: |
2465 | | UpdateQueryString(); |
2466 | | |
2467 | | bool bHasExpired = false; |
2468 | | |
2469 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
2470 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
2471 | | bool bUsedRedirect = osURL != m_pszURL; |
2472 | | |
2473 | | WriteFuncStruct sWriteFuncData; |
2474 | | WriteFuncStruct sWriteFuncHeaderData; |
2475 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
2476 | | |
2477 | | retry: |
2478 | | CURL *hCurlHandle = curl_easy_init(); |
2479 | | struct curl_slist *headers = |
2480 | | poFS->SetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
2481 | | |
2482 | | if (!AllowAutomaticRedirection()) |
2483 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
2484 | | |
2485 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
2486 | | pReadCbkUserData); |
2487 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
2488 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2489 | | VSICurlHandleWriteFunc); |
2490 | | |
2491 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
2492 | | nullptr); |
2493 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2494 | | &sWriteFuncHeaderData); |
2495 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2496 | | VSICurlHandleWriteFunc); |
2497 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2498 | | sWriteFuncHeaderData.nStartOffset = startOffset; |
2499 | | sWriteFuncHeaderData.nEndOffset = |
2500 | | startOffset + |
2501 | | static_cast<vsi_l_offset>(nBlocks) * VSICURLGetDownloadChunkSize() - 1; |
2502 | | // Some servers don't like we try to read after end-of-file (#5786). |
2503 | | if (oFileProp.bHasComputedFileSize && |
2504 | | sWriteFuncHeaderData.nEndOffset >= oFileProp.fileSize) |
2505 | | { |
2506 | | sWriteFuncHeaderData.nEndOffset = oFileProp.fileSize - 1; |
2507 | | } |
2508 | | |
2509 | | char rangeStr[512] = {}; |
2510 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2511 | | startOffset, sWriteFuncHeaderData.nEndOffset); |
2512 | | |
2513 | | if constexpr (ENABLE_DEBUG) |
2514 | | { |
2515 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
2516 | | osURL.c_str()); |
2517 | | } |
2518 | | |
2519 | | std::string osHeaderRange; // leave in this scope |
2520 | | if (sWriteFuncHeaderData.bIsHTTP) |
2521 | | { |
2522 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
2523 | | // So it gets included in Azure signature |
2524 | | headers = curl_slist_append(headers, osHeaderRange.c_str()); |
2525 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
2526 | | } |
2527 | | else |
2528 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
2529 | | |
2530 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
2531 | | szCurlErrBuf[0] = '\0'; |
2532 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
2533 | | |
2534 | | headers = GetCurlHeaders("GET", headers); |
2535 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
2536 | | |
2537 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
2538 | | |
2539 | | poFS->Perform(hCurlHandle); |
2540 | | |
2541 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
2542 | | |
2543 | | curl_slist_free_all(headers); |
2544 | | |
2545 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
2546 | | |
2547 | | if (sWriteFuncData.bInterrupted || m_bInterrupt) |
2548 | | { |
2549 | | bInterrupted = true; |
2550 | | |
2551 | | // Notify that the download of the current region is finished |
2552 | | currentDownload.SetData(std::string()); |
2553 | | |
2554 | | CPLFree(sWriteFuncData.pBuffer); |
2555 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2556 | | curl_easy_cleanup(hCurlHandle); |
2557 | | |
2558 | | return std::string(); |
2559 | | } |
2560 | | |
2561 | | long response_code = 0; |
2562 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2563 | | |
2564 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
2565 | | { |
2566 | | CPLDebug(poFS->GetDebugKey(), |
2567 | | "DownloadRegion(%s): response_code=%d, msg=%s", osURL.c_str(), |
2568 | | static_cast<int>(response_code), szCurlErrBuf); |
2569 | | } |
2570 | | |
2571 | | long mtime = 0; |
2572 | | curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME, &mtime); |
2573 | | if (mtime > 0) |
2574 | | { |
2575 | | oFileProp.mTime = mtime; |
2576 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2577 | | } |
2578 | | |
2579 | | if constexpr (ENABLE_DEBUG) |
2580 | | { |
2581 | | CPLDebug(poFS->GetDebugKey(), "Got response_code=%ld", response_code); |
2582 | | } |
2583 | | |
2584 | | if (bUsedRedirect && |
2585 | | (response_code == 403 || |
2586 | | // Below case is in particular for |
2587 | | // gdalinfo |
2588 | | // /vsicurl/https://lpdaac.earthdata.nasa.gov/lp-prod-protected/HLSS30.015/HLS.S30.T10TEK.2020273T190109.v1.5.B8A.tif |
2589 | | // --config GDAL_DISABLE_READDIR_ON_OPEN EMPTY_DIR --config |
2590 | | // GDAL_HTTP_COOKIEFILE /tmp/cookie.txt --config GDAL_HTTP_COOKIEJAR |
2591 | | // /tmp/cookie.txt We got the redirect URL from a HEAD request, but it |
2592 | | // is not valid for a GET. So retry with GET on original URL to get a |
2593 | | // redirect URL valid for it. |
2594 | | (response_code == 400 && |
2595 | | osURL.find(".cloudfront.net") != std::string::npos))) |
2596 | | { |
2597 | | CPLDebug(poFS->GetDebugKey(), |
2598 | | "Got an error with redirect URL. Retrying with original one"); |
2599 | | oFileProp.bS3LikeRedirect = false; |
2600 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2601 | | bUsedRedirect = false; |
2602 | | osURL = m_pszURL; |
2603 | | CPLFree(sWriteFuncData.pBuffer); |
2604 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2605 | | curl_easy_cleanup(hCurlHandle); |
2606 | | goto retry; |
2607 | | } |
2608 | | |
2609 | | if (response_code == 401 && oRetryContext.CanRetry()) |
2610 | | { |
2611 | | CPLDebug(poFS->GetDebugKey(), "Unauthorized, trying to authenticate"); |
2612 | | CPLFree(sWriteFuncData.pBuffer); |
2613 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2614 | | curl_easy_cleanup(hCurlHandle); |
2615 | | if (Authenticate(m_osFilename.c_str())) |
2616 | | goto retry; |
2617 | | return std::string(); |
2618 | | } |
2619 | | |
2620 | | UpdateRedirectInfo(hCurlHandle, sWriteFuncHeaderData); |
2621 | | |
2622 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
2623 | | response_code != 226 && response_code != 426) || |
2624 | | sWriteFuncHeaderData.bError) |
2625 | | { |
2626 | | if (sWriteFuncData.pBuffer != nullptr && |
2627 | | CanRestartOnError( |
2628 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer), |
2629 | | reinterpret_cast<const char *>(sWriteFuncHeaderData.pBuffer), |
2630 | | true)) |
2631 | | { |
2632 | | CPLFree(sWriteFuncData.pBuffer); |
2633 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2634 | | curl_easy_cleanup(hCurlHandle); |
2635 | | goto begin; |
2636 | | } |
2637 | | |
2638 | | // Look if we should attempt a retry |
2639 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
2640 | | sWriteFuncHeaderData.pBuffer, szCurlErrBuf)) |
2641 | | { |
2642 | | CPLError(CE_Warning, CPLE_AppDefined, |
2643 | | "HTTP error code: %d - %s. " |
2644 | | "Retrying again in %.1f secs", |
2645 | | static_cast<int>(response_code), m_pszURL, |
2646 | | oRetryContext.GetCurrentDelay()); |
2647 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
2648 | | CPLFree(sWriteFuncData.pBuffer); |
2649 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2650 | | curl_easy_cleanup(hCurlHandle); |
2651 | | goto retry; |
2652 | | } |
2653 | | |
2654 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
2655 | | { |
2656 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
2657 | | CPLError( |
2658 | | CE_Failure, CPLE_AppDefined, |
2659 | | "%d: %s, Range downloading not supported by this server!", |
2660 | | static_cast<int>(response_code), szCurlErrBuf); |
2661 | | else |
2662 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
2663 | | static_cast<int>(response_code), szCurlErrBuf); |
2664 | | } |
2665 | | else if (response_code == 416) /* Range Not Satisfiable */ |
2666 | | { |
2667 | | if (sWriteFuncData.pBuffer) |
2668 | | { |
2669 | | CPLError( |
2670 | | CE_Failure, CPLE_AppDefined, |
2671 | | "%d: Range downloading not supported by this server: %s", |
2672 | | static_cast<int>(response_code), sWriteFuncData.pBuffer); |
2673 | | } |
2674 | | else |
2675 | | { |
2676 | | CPLError(CE_Failure, CPLE_AppDefined, |
2677 | | "%d: Range downloading not supported by this server", |
2678 | | static_cast<int>(response_code)); |
2679 | | } |
2680 | | } |
2681 | | if (!oFileProp.bHasComputedFileSize && startOffset == 0) |
2682 | | { |
2683 | | oFileProp.bHasComputedFileSize = true; |
2684 | | oFileProp.fileSize = 0; |
2685 | | oFileProp.eExists = EXIST_NO; |
2686 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2687 | | } |
2688 | | CPLFree(sWriteFuncData.pBuffer); |
2689 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2690 | | curl_easy_cleanup(hCurlHandle); |
2691 | | return std::string(); |
2692 | | } |
2693 | | |
2694 | | if (!oFileProp.bHasComputedFileSize && sWriteFuncHeaderData.pBuffer) |
2695 | | { |
2696 | | // Try to retrieve the filesize from the HTTP headers |
2697 | | // if in the form: "Content-Range: bytes x-y/filesize". |
2698 | | char *pszContentRange = |
2699 | | strstr(sWriteFuncHeaderData.pBuffer, "Content-Range: bytes "); |
2700 | | if (pszContentRange == nullptr) |
2701 | | pszContentRange = |
2702 | | strstr(sWriteFuncHeaderData.pBuffer, "content-range: bytes "); |
2703 | | if (pszContentRange) |
2704 | | { |
2705 | | char *pszEOL = strchr(pszContentRange, '\n'); |
2706 | | if (pszEOL) |
2707 | | { |
2708 | | *pszEOL = 0; |
2709 | | pszEOL = strchr(pszContentRange, '\r'); |
2710 | | if (pszEOL) |
2711 | | *pszEOL = 0; |
2712 | | char *pszSlash = strchr(pszContentRange, '/'); |
2713 | | if (pszSlash) |
2714 | | { |
2715 | | pszSlash++; |
2716 | | oFileProp.fileSize = CPLScanUIntBig( |
2717 | | pszSlash, static_cast<int>(strlen(pszSlash))); |
2718 | | } |
2719 | | } |
2720 | | } |
2721 | | else if (STARTS_WITH(m_pszURL, "ftp")) |
2722 | | { |
2723 | | // Parse 213 answer for FTP protocol. |
2724 | | char *pszSize = strstr(sWriteFuncHeaderData.pBuffer, "213 "); |
2725 | | if (pszSize) |
2726 | | { |
2727 | | pszSize += 4; |
2728 | | char *pszEOL = strchr(pszSize, '\n'); |
2729 | | if (pszEOL) |
2730 | | { |
2731 | | *pszEOL = 0; |
2732 | | pszEOL = strchr(pszSize, '\r'); |
2733 | | if (pszEOL) |
2734 | | *pszEOL = 0; |
2735 | | |
2736 | | oFileProp.fileSize = CPLScanUIntBig( |
2737 | | pszSize, static_cast<int>(strlen(pszSize))); |
2738 | | } |
2739 | | } |
2740 | | } |
2741 | | |
2742 | | if (oFileProp.fileSize != 0) |
2743 | | { |
2744 | | oFileProp.eExists = EXIST_YES; |
2745 | | |
2746 | | if constexpr (ENABLE_DEBUG) |
2747 | | { |
2748 | | CPLDebug(poFS->GetDebugKey(), |
2749 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
2750 | | m_pszURL, oFileProp.fileSize, |
2751 | | static_cast<int>(response_code)); |
2752 | | } |
2753 | | |
2754 | | oFileProp.bHasComputedFileSize = true; |
2755 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2756 | | } |
2757 | | } |
2758 | | |
2759 | | DownloadRegionPostProcess(startOffset, nBlocks, sWriteFuncData.pBuffer, |
2760 | | sWriteFuncData.nSize); |
2761 | | |
2762 | | std::string osRet; |
2763 | | osRet.assign(sWriteFuncData.pBuffer, sWriteFuncData.nSize); |
2764 | | |
2765 | | // Notify that the download of the current region is finished |
2766 | | currentDownload.SetData(osRet); |
2767 | | |
2768 | | CPLFree(sWriteFuncData.pBuffer); |
2769 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2770 | | curl_easy_cleanup(hCurlHandle); |
2771 | | |
2772 | | return osRet; |
2773 | | } |
2774 | | |
2775 | | /************************************************************************/ |
2776 | | /* UpdateRedirectInfo() */ |
2777 | | /************************************************************************/ |
2778 | | |
2779 | | void VSICurlHandle::UpdateRedirectInfo( |
2780 | | CURL *hCurlHandle, const WriteFuncStruct &sWriteFuncHeaderData) |
2781 | | { |
2782 | | std::string osEffectiveURL; |
2783 | | { |
2784 | | char *pszEffectiveURL = nullptr; |
2785 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
2786 | | &pszEffectiveURL); |
2787 | | if (pszEffectiveURL) |
2788 | | osEffectiveURL = pszEffectiveURL; |
2789 | | } |
2790 | | |
2791 | | if (!oFileProp.bS3LikeRedirect && !osEffectiveURL.empty() && |
2792 | | strstr(osEffectiveURL.c_str(), m_pszURL) == nullptr) |
2793 | | { |
2794 | | CPLDebug(poFS->GetDebugKey(), "Effective URL: %s", |
2795 | | osEffectiveURL.c_str()); |
2796 | | |
2797 | | long response_code = 0; |
2798 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2799 | | if (response_code >= 200 && response_code < 300 && |
2800 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
2801 | | VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
2802 | | !VSICurlIsS3LikeSignedURL(m_pszURL) && |
2803 | | CPLTestBool( |
2804 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
2805 | | { |
2806 | | GIntBig nExpireTimestamp = |
2807 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
2808 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
2809 | | { |
2810 | | const int nValidity = static_cast<int>( |
2811 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
2812 | | CPLDebug(poFS->GetDebugKey(), |
2813 | | "Will use redirect URL for the next %d seconds", |
2814 | | nValidity); |
2815 | | // As our local clock might not be in sync with server clock, |
2816 | | // figure out the expiration timestamp in local time. |
2817 | | oFileProp.bS3LikeRedirect = true; |
2818 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
2819 | | oFileProp.osRedirectURL = std::move(osEffectiveURL); |
2820 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2821 | | } |
2822 | | } |
2823 | | } |
2824 | | } |
2825 | | |
2826 | | /************************************************************************/ |
2827 | | /* DownloadRegionPostProcess() */ |
2828 | | /************************************************************************/ |
2829 | | |
2830 | | void VSICurlHandle::DownloadRegionPostProcess(const vsi_l_offset startOffset, |
2831 | | const int nBlocks, |
2832 | | const char *pBuffer, size_t nSize) |
2833 | | { |
2834 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2835 | | lastDownloadedOffset = startOffset + static_cast<vsi_l_offset>(nBlocks) * |
2836 | | knDOWNLOAD_CHUNK_SIZE; |
2837 | | |
2838 | | if (nSize > static_cast<size_t>(nBlocks) * knDOWNLOAD_CHUNK_SIZE) |
2839 | | { |
2840 | | if constexpr (ENABLE_DEBUG) |
2841 | | { |
2842 | | CPLDebug( |
2843 | | poFS->GetDebugKey(), |
2844 | | "Got more data than expected : %u instead of %u", |
2845 | | static_cast<unsigned int>(nSize), |
2846 | | static_cast<unsigned int>(nBlocks * knDOWNLOAD_CHUNK_SIZE)); |
2847 | | } |
2848 | | } |
2849 | | |
2850 | | vsi_l_offset l_startOffset = startOffset; |
2851 | | while (nSize > 0) |
2852 | | { |
2853 | | #if DEBUG_VERBOSE |
2854 | | if constexpr (ENABLE_DEBUG) |
2855 | | { |
2856 | | CPLDebug(poFS->GetDebugKey(), "Add region %u - %u", |
2857 | | static_cast<unsigned int>(startOffset), |
2858 | | static_cast<unsigned int>(std::min( |
2859 | | static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize))); |
2860 | | } |
2861 | | #endif |
2862 | | const size_t nChunkSize = |
2863 | | std::min(static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize); |
2864 | | poFS->AddRegion(m_pszURL, l_startOffset, nChunkSize, pBuffer); |
2865 | | l_startOffset += nChunkSize; |
2866 | | pBuffer += nChunkSize; |
2867 | | nSize -= nChunkSize; |
2868 | | } |
2869 | | } |
2870 | | |
2871 | | /************************************************************************/ |
2872 | | /* Read() */ |
2873 | | /************************************************************************/ |
2874 | | |
2875 | | size_t VSICurlHandle::Read(void *const pBufferIn, size_t const nBytes) |
2876 | | { |
2877 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2878 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2879 | | NetworkStatisticsAction oContextAction("Read"); |
2880 | | |
2881 | | size_t nBufferRequestSize = nBytes; |
2882 | | if (nBufferRequestSize == 0) |
2883 | | return 0; |
2884 | | |
2885 | | void *pBuffer = pBufferIn; |
2886 | | |
2887 | | #if DEBUG_VERBOSE |
2888 | | CPLDebug(poFS->GetDebugKey(), "offset=%d, size=%d", |
2889 | | static_cast<int>(curOffset), static_cast<int>(nBufferRequestSize)); |
2890 | | #endif |
2891 | | |
2892 | | vsi_l_offset iterOffset = curOffset; |
2893 | | const int knMAX_REGIONS = GetMaxRegions(); |
2894 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2895 | | while (nBufferRequestSize) |
2896 | | { |
2897 | | // Don't try to read after end of file. |
2898 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2899 | | if (oFileProp.bHasComputedFileSize && iterOffset >= oFileProp.fileSize) |
2900 | | { |
2901 | | if (iterOffset == curOffset) |
2902 | | { |
2903 | | CPLDebug(poFS->GetDebugKey(), |
2904 | | "Request at offset " CPL_FRMT_GUIB |
2905 | | ", after end of file", |
2906 | | iterOffset); |
2907 | | } |
2908 | | break; |
2909 | | } |
2910 | | |
2911 | | const vsi_l_offset nOffsetToDownload = |
2912 | | (iterOffset / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
2913 | | std::string osRegion; |
2914 | | std::shared_ptr<std::string> psRegion = |
2915 | | poFS->GetRegion(m_pszURL, nOffsetToDownload); |
2916 | | if (psRegion != nullptr) |
2917 | | { |
2918 | | osRegion = *psRegion; |
2919 | | } |
2920 | | else |
2921 | | { |
2922 | | if (nOffsetToDownload == lastDownloadedOffset) |
2923 | | { |
2924 | | // In case of consecutive reads (of small size), we use a |
2925 | | // heuristic that we will read the file sequentially, so |
2926 | | // we double the requested size to decrease the number of |
2927 | | // client/server roundtrips. |
2928 | | constexpr int MAX_CHUNK_SIZE_INCREASE_FACTOR = 128; |
2929 | | if (nBlocksToDownload < MAX_CHUNK_SIZE_INCREASE_FACTOR) |
2930 | | nBlocksToDownload *= 2; |
2931 | | } |
2932 | | else |
2933 | | { |
2934 | | // Random reads. Cancel the above heuristics. |
2935 | | nBlocksToDownload = 1; |
2936 | | } |
2937 | | |
2938 | | // Ensure that we will request at least the number of blocks |
2939 | | // to satisfy the remaining buffer size to read. |
2940 | | const vsi_l_offset nEndOffsetToDownload = |
2941 | | ((iterOffset + nBufferRequestSize + knDOWNLOAD_CHUNK_SIZE - 1) / |
2942 | | knDOWNLOAD_CHUNK_SIZE) * |
2943 | | knDOWNLOAD_CHUNK_SIZE; |
2944 | | const int nMinBlocksToDownload = |
2945 | | static_cast<int>((nEndOffsetToDownload - nOffsetToDownload) / |
2946 | | knDOWNLOAD_CHUNK_SIZE); |
2947 | | if (nBlocksToDownload < nMinBlocksToDownload) |
2948 | | nBlocksToDownload = nMinBlocksToDownload; |
2949 | | |
2950 | | // Avoid reading already cached data. |
2951 | | // Note: this might get evicted if concurrent reads are done, but |
2952 | | // this should not cause bugs. Just missed optimization. |
2953 | | for (int i = 1; i < nBlocksToDownload; i++) |
2954 | | { |
2955 | | if (poFS->GetRegion(m_pszURL, nOffsetToDownload + |
2956 | | static_cast<vsi_l_offset>(i) * |
2957 | | knDOWNLOAD_CHUNK_SIZE) != |
2958 | | nullptr) |
2959 | | { |
2960 | | nBlocksToDownload = i; |
2961 | | break; |
2962 | | } |
2963 | | } |
2964 | | |
2965 | | // We can't download more than knMAX_REGIONS chunks at a time, |
2966 | | // otherwise the cache will not be big enough to store them and |
2967 | | // copy their content to the target buffer. |
2968 | | if (nBlocksToDownload > knMAX_REGIONS) |
2969 | | nBlocksToDownload = knMAX_REGIONS; |
2970 | | |
2971 | | osRegion = DownloadRegion(nOffsetToDownload, nBlocksToDownload); |
2972 | | if (osRegion.empty()) |
2973 | | { |
2974 | | if (!bInterrupted) |
2975 | | bError = true; |
2976 | | return 0; |
2977 | | } |
2978 | | } |
2979 | | |
2980 | | const vsi_l_offset nRegionOffset = iterOffset - nOffsetToDownload; |
2981 | | if (osRegion.size() < nRegionOffset) |
2982 | | { |
2983 | | if (iterOffset == curOffset) |
2984 | | { |
2985 | | CPLDebug(poFS->GetDebugKey(), |
2986 | | "Request at offset " CPL_FRMT_GUIB |
2987 | | ", after end of file", |
2988 | | iterOffset); |
2989 | | } |
2990 | | break; |
2991 | | } |
2992 | | |
2993 | | const int nToCopy = static_cast<int>( |
2994 | | std::min(static_cast<vsi_l_offset>(nBufferRequestSize), |
2995 | | osRegion.size() - nRegionOffset)); |
2996 | | memcpy(pBuffer, osRegion.data() + nRegionOffset, nToCopy); |
2997 | | pBuffer = static_cast<char *>(pBuffer) + nToCopy; |
2998 | | iterOffset += nToCopy; |
2999 | | nBufferRequestSize -= nToCopy; |
3000 | | if (osRegion.size() < static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE) && |
3001 | | nBufferRequestSize != 0) |
3002 | | { |
3003 | | break; |
3004 | | } |
3005 | | } |
3006 | | |
3007 | | const size_t ret = static_cast<size_t>(iterOffset - curOffset); |
3008 | | if (ret != nBytes) |
3009 | | bEOF = true; |
3010 | | |
3011 | | curOffset = iterOffset; |
3012 | | |
3013 | | return ret; |
3014 | | } |
3015 | | |
3016 | | /************************************************************************/ |
3017 | | /* ReadMultiRange() */ |
3018 | | /************************************************************************/ |
3019 | | |
3020 | | int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, |
3021 | | const vsi_l_offset *const panOffsets, |
3022 | | const size_t *const panSizes) |
3023 | | { |
3024 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
3025 | | return FALSE; |
3026 | | |
3027 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
3028 | | if (oFileProp.eExists == EXIST_NO) |
3029 | | return -1; |
3030 | | |
3031 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3032 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3033 | | NetworkStatisticsAction oContextAction("ReadMultiRange"); |
3034 | | |
3035 | | const char *pszMultiRangeStrategy = |
3036 | | CPLGetConfigOption("GDAL_HTTP_MULTIRANGE", ""); |
3037 | | if (EQUAL(pszMultiRangeStrategy, "SINGLE_GET")) |
3038 | | { |
3039 | | // Just in case someone needs it, but the interest of this mode is |
3040 | | // rather dubious now. We could probably remove it |
3041 | | return ReadMultiRangeSingleGet(nRanges, ppData, panOffsets, panSizes); |
3042 | | } |
3043 | | else if (nRanges == 1 || EQUAL(pszMultiRangeStrategy, "SERIAL")) |
3044 | | { |
3045 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
3046 | | panSizes); |
3047 | | } |
3048 | | |
3049 | | UpdateQueryString(); |
3050 | | |
3051 | | bool bHasExpired = false; |
3052 | | |
3053 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3054 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
3055 | | if (bHasExpired) |
3056 | | { |
3057 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
3058 | | panSizes); |
3059 | | } |
3060 | | |
3061 | | struct CurlErrBuffer |
3062 | | { |
3063 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3064 | | }; |
3065 | | |
3066 | | // Sort ranges by file offset so the merge loop below can coalesce |
3067 | | // adjacent ranges regardless of the order the caller passed them. |
3068 | | // The ppData buffer pointers travel with their offsets, so the |
3069 | | // distribute logic fills the correct caller buffers after reading. |
3070 | | std::vector<int> anSortOrder(nRanges); |
3071 | | std::iota(anSortOrder.begin(), anSortOrder.end(), 0); |
3072 | | std::sort(anSortOrder.begin(), anSortOrder.end(), [panOffsets](int a, int b) |
3073 | | { return panOffsets[a] < panOffsets[b]; }); |
3074 | | |
3075 | | std::vector<void *> apSortedData(nRanges); |
3076 | | std::vector<vsi_l_offset> anSortedOffsets(nRanges); |
3077 | | std::vector<size_t> anSortedSizes(nRanges); |
3078 | | for (int i = 0; i < nRanges; ++i) |
3079 | | { |
3080 | | apSortedData[i] = ppData[anSortOrder[i]]; |
3081 | | anSortedOffsets[i] = panOffsets[anSortOrder[i]]; |
3082 | | anSortedSizes[i] = panSizes[anSortOrder[i]]; |
3083 | | } |
3084 | | |
3085 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
3086 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
3087 | | |
3088 | | // Build list of merged requests upfront, each with its own retry context |
3089 | | struct MergedRequest |
3090 | | { |
3091 | | int iFirstRange; |
3092 | | int iLastRange; |
3093 | | vsi_l_offset nStartOffset; |
3094 | | size_t nSize; |
3095 | | CPLHTTPRetryContext retryContext; |
3096 | | bool bToRetry = true; // true initially to trigger first attempt |
3097 | | |
3098 | | MergedRequest(int first, int last, vsi_l_offset start, size_t size, |
3099 | | const CPLHTTPRetryParameters ¶ms) |
3100 | | : iFirstRange(first), iLastRange(last), nStartOffset(start), |
3101 | | nSize(size), retryContext(params) |
3102 | | { |
3103 | | } |
3104 | | }; |
3105 | | |
3106 | | std::vector<MergedRequest> asMergedRequests; |
3107 | | for (int i = 0; i < nRanges;) |
3108 | | { |
3109 | | size_t nSize = 0; |
3110 | | int iNext = i; |
3111 | | // Identify consecutive ranges |
3112 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
3113 | | anSortedOffsets[iNext] + anSortedSizes[iNext] == |
3114 | | anSortedOffsets[iNext + 1]) |
3115 | | { |
3116 | | nSize += anSortedSizes[iNext]; |
3117 | | iNext++; |
3118 | | } |
3119 | | nSize += anSortedSizes[iNext]; |
3120 | | |
3121 | | if (nSize == 0) |
3122 | | { |
3123 | | i = iNext + 1; |
3124 | | continue; |
3125 | | } |
3126 | | |
3127 | | asMergedRequests.emplace_back(i, iNext, anSortedOffsets[i], nSize, |
3128 | | m_oRetryParameters); |
3129 | | i = iNext + 1; |
3130 | | } |
3131 | | |
3132 | | if (asMergedRequests.empty()) |
3133 | | return 0; |
3134 | | |
3135 | | int nRet = 0; |
3136 | | size_t nTotalDownloaded = 0; |
3137 | | |
3138 | | // Retry loop: re-issue only failed requests that are retryable |
3139 | | while (true) |
3140 | | { |
3141 | | const size_t nRequests = asMergedRequests.size(); |
3142 | | std::vector<CURL *> aHandles(nRequests, nullptr); |
3143 | | std::vector<CURL *> performHandles; |
3144 | | std::vector<WriteFuncStruct> asWriteFuncData(nRequests); |
3145 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData(nRequests); |
3146 | | std::vector<char *> apszRanges(nRequests, nullptr); |
3147 | | std::vector<struct curl_slist *> aHeaders(nRequests, nullptr); |
3148 | | std::vector<CurlErrBuffer> asCurlErrors(nRequests); |
3149 | | |
3150 | | for (size_t iReq = 0; iReq < nRequests; iReq++) |
3151 | | { |
3152 | | if (!asMergedRequests[iReq].bToRetry) |
3153 | | continue; |
3154 | | asMergedRequests[iReq].bToRetry = false; |
3155 | | |
3156 | | CURL *hCurlHandle = curl_easy_init(); |
3157 | | performHandles.push_back(hCurlHandle); |
3158 | | aHandles[iReq] = hCurlHandle; |
3159 | | |
3160 | | struct curl_slist *headers = poFS->SetOptions( |
3161 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3162 | | |
3163 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[iReq], this, pfnReadCbk, |
3164 | | pReadCbkUserData); |
3165 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
3166 | | &asWriteFuncData[iReq]); |
3167 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3168 | | VSICurlHandleWriteFunc); |
3169 | | |
3170 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[iReq], nullptr, |
3171 | | nullptr, nullptr); |
3172 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3173 | | &asWriteFuncHeaderData[iReq]); |
3174 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3175 | | VSICurlHandleWriteFunc); |
3176 | | asWriteFuncHeaderData[iReq].bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3177 | | asWriteFuncHeaderData[iReq].nStartOffset = |
3178 | | asMergedRequests[iReq].nStartOffset; |
3179 | | asWriteFuncHeaderData[iReq].nEndOffset = |
3180 | | asMergedRequests[iReq].nStartOffset + |
3181 | | asMergedRequests[iReq].nSize - 1; |
3182 | | |
3183 | | char rangeStr[512] = {}; |
3184 | | snprintf(rangeStr, sizeof(rangeStr), |
3185 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3186 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3187 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3188 | | |
3189 | | if constexpr (ENABLE_DEBUG) |
3190 | | { |
3191 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
3192 | | rangeStr, osURL.c_str()); |
3193 | | } |
3194 | | |
3195 | | if (asWriteFuncHeaderData[iReq].bIsHTTP) |
3196 | | { |
3197 | | // So it gets included in Azure signature |
3198 | | char *pszRange = |
3199 | | CPLStrdup(CPLSPrintf("Range: bytes=%s", rangeStr)); |
3200 | | apszRanges[iReq] = pszRange; |
3201 | | headers = curl_slist_append(headers, pszRange); |
3202 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
3203 | | } |
3204 | | else |
3205 | | { |
3206 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3207 | | rangeStr); |
3208 | | } |
3209 | | |
3210 | | asCurlErrors[iReq].szCurlErrBuf[0] = '\0'; |
3211 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3212 | | &asCurlErrors[iReq].szCurlErrBuf[0]); |
3213 | | |
3214 | | headers = GetCurlHeaders("GET", headers); |
3215 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
3216 | | headers); |
3217 | | aHeaders[iReq] = headers; |
3218 | | } |
3219 | | |
3220 | | poFS->Perform(performHandles); |
3221 | | |
3222 | | // Process results |
3223 | | bool bRetry = false; |
3224 | | double dfMaxDelay = 0.0; |
3225 | | for (size_t iReq = 0; iReq < nRequests; iReq++) |
3226 | | { |
3227 | | if (!aHandles[iReq]) |
3228 | | continue; |
3229 | | |
3230 | | long response_code = 0; |
3231 | | curl_easy_getinfo(aHandles[iReq], CURLINFO_HTTP_CODE, |
3232 | | &response_code); |
3233 | | |
3234 | | if (ENABLE_DEBUG && asCurlErrors[iReq].szCurlErrBuf[0] != '\0') |
3235 | | { |
3236 | | char rangeStr[512] = {}; |
3237 | | snprintf(rangeStr, sizeof(rangeStr), |
3238 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3239 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3240 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3241 | | |
3242 | | const char *pszErrorMsg = &asCurlErrors[iReq].szCurlErrBuf[0]; |
3243 | | CPLDebug(poFS->GetDebugKey(), |
3244 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
3245 | | osURL.c_str(), rangeStr, |
3246 | | static_cast<int>(response_code), pszErrorMsg); |
3247 | | } |
3248 | | |
3249 | | if ((response_code != 206 && response_code != 225) || |
3250 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
3251 | | asWriteFuncHeaderData[iReq].nStartOffset + |
3252 | | asWriteFuncData[iReq].nSize) |
3253 | | { |
3254 | | char rangeStr[512] = {}; |
3255 | | snprintf(rangeStr, sizeof(rangeStr), |
3256 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3257 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3258 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3259 | | |
3260 | | // Look if we should attempt a retry |
3261 | | if (asMergedRequests[iReq].retryContext.CanRetry( |
3262 | | static_cast<int>(response_code), |
3263 | | asWriteFuncData[iReq].pBuffer, |
3264 | | &asCurlErrors[iReq].szCurlErrBuf[0])) |
3265 | | { |
3266 | | CPLError( |
3267 | | CE_Warning, CPLE_AppDefined, |
3268 | | "HTTP error code for %s range %s: %d. " |
3269 | | "Retrying again in %.1f secs", |
3270 | | osURL.c_str(), rangeStr, |
3271 | | static_cast<int>(response_code), |
3272 | | asMergedRequests[iReq].retryContext.GetCurrentDelay()); |
3273 | | dfMaxDelay = std::max( |
3274 | | dfMaxDelay, |
3275 | | asMergedRequests[iReq].retryContext.GetCurrentDelay()); |
3276 | | asMergedRequests[iReq].bToRetry = true; |
3277 | | bRetry = true; |
3278 | | } |
3279 | | else |
3280 | | { |
3281 | | CPLError(CE_Failure, CPLE_AppDefined, |
3282 | | "Request for %s failed with response_code=%ld", |
3283 | | rangeStr, response_code); |
3284 | | nRet = -1; |
3285 | | } |
3286 | | } |
3287 | | else if (nRet == 0) |
3288 | | { |
3289 | | size_t nOffset = 0; |
3290 | | size_t nRemainingSize = asWriteFuncData[iReq].nSize; |
3291 | | nTotalDownloaded += nRemainingSize; |
3292 | | for (int iRange = asMergedRequests[iReq].iFirstRange; |
3293 | | iRange <= asMergedRequests[iReq].iLastRange; iRange++) |
3294 | | { |
3295 | | if (nRemainingSize < anSortedSizes[iRange]) |
3296 | | { |
3297 | | nRet = -1; |
3298 | | break; |
3299 | | } |
3300 | | |
3301 | | if (anSortedSizes[iRange] > 0) |
3302 | | { |
3303 | | memcpy(apSortedData[iRange], |
3304 | | asWriteFuncData[iReq].pBuffer + nOffset, |
3305 | | anSortedSizes[iRange]); |
3306 | | } |
3307 | | nOffset += anSortedSizes[iRange]; |
3308 | | nRemainingSize -= anSortedSizes[iRange]; |
3309 | | } |
3310 | | } |
3311 | | |
3312 | | VSICURLResetHeaderAndWriterFunctions(aHandles[iReq]); |
3313 | | curl_easy_cleanup(aHandles[iReq]); |
3314 | | CPLFree(apszRanges[iReq]); |
3315 | | CPLFree(asWriteFuncData[iReq].pBuffer); |
3316 | | CPLFree(asWriteFuncHeaderData[iReq].pBuffer); |
3317 | | if (aHeaders[iReq]) |
3318 | | curl_slist_free_all(aHeaders[iReq]); |
3319 | | } |
3320 | | |
3321 | | if (!bRetry || nRet != 0) |
3322 | | break; |
3323 | | CPLSleep(dfMaxDelay); |
3324 | | } |
3325 | | |
3326 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
3327 | | |
3328 | | if constexpr (ENABLE_DEBUG) |
3329 | | { |
3330 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
3331 | | } |
3332 | | |
3333 | | return nRet; |
3334 | | } |
3335 | | |
3336 | | /************************************************************************/ |
3337 | | /* ReadMultiRangeSingleGet() */ |
3338 | | /************************************************************************/ |
3339 | | |
3340 | | // TODO: the interest of this mode is rather dubious now. We could probably |
3341 | | // remove it |
3342 | | int VSICurlHandle::ReadMultiRangeSingleGet(int const nRanges, |
3343 | | void **const ppData, |
3344 | | const vsi_l_offset *const panOffsets, |
3345 | | const size_t *const panSizes) |
3346 | | { |
3347 | | std::string osRanges; |
3348 | | std::string osFirstRange; |
3349 | | std::string osLastRange; |
3350 | | int nMergedRanges = 0; |
3351 | | vsi_l_offset nTotalReqSize = 0; |
3352 | | for (int i = 0; i < nRanges; i++) |
3353 | | { |
3354 | | std::string osCurRange; |
3355 | | if (i != 0) |
3356 | | osRanges.append(","); |
3357 | | osCurRange = CPLSPrintf(CPL_FRMT_GUIB "-", panOffsets[i]); |
3358 | | while (i + 1 < nRanges && |
3359 | | panOffsets[i] + panSizes[i] == panOffsets[i + 1]) |
3360 | | { |
3361 | | nTotalReqSize += panSizes[i]; |
3362 | | i++; |
3363 | | } |
3364 | | nTotalReqSize += panSizes[i]; |
3365 | | osCurRange.append( |
3366 | | CPLSPrintf(CPL_FRMT_GUIB, panOffsets[i] + panSizes[i] - 1)); |
3367 | | nMergedRanges++; |
3368 | | |
3369 | | osRanges += osCurRange; |
3370 | | |
3371 | | if (nMergedRanges == 1) |
3372 | | osFirstRange = osCurRange; |
3373 | | osLastRange = std::move(osCurRange); |
3374 | | } |
3375 | | |
3376 | | const char *pszMaxRanges = |
3377 | | CPLGetConfigOption("CPL_VSIL_CURL_MAX_RANGES", "250"); |
3378 | | int nMaxRanges = atoi(pszMaxRanges); |
3379 | | if (nMaxRanges <= 0) |
3380 | | nMaxRanges = 250; |
3381 | | if (nMergedRanges > nMaxRanges) |
3382 | | { |
3383 | | const int nHalf = nRanges / 2; |
3384 | | const int nRet = ReadMultiRange(nHalf, ppData, panOffsets, panSizes); |
3385 | | if (nRet != 0) |
3386 | | return nRet; |
3387 | | return ReadMultiRange(nRanges - nHalf, ppData + nHalf, |
3388 | | panOffsets + nHalf, panSizes + nHalf); |
3389 | | } |
3390 | | |
3391 | | CURL *hCurlHandle = curl_easy_init(); |
3392 | | |
3393 | | struct curl_slist *headers = |
3394 | | poFS->SetOptions(hCurlHandle, m_pszURL, m_aosHTTPOptions.List()); |
3395 | | |
3396 | | WriteFuncStruct sWriteFuncData; |
3397 | | WriteFuncStruct sWriteFuncHeaderData; |
3398 | | |
3399 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
3400 | | pReadCbkUserData); |
3401 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
3402 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3403 | | VSICurlHandleWriteFunc); |
3404 | | |
3405 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
3406 | | nullptr); |
3407 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3408 | | &sWriteFuncHeaderData); |
3409 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3410 | | VSICurlHandleWriteFunc); |
3411 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3412 | | sWriteFuncHeaderData.bMultiRange = nMergedRanges > 1; |
3413 | | if (nMergedRanges == 1) |
3414 | | { |
3415 | | sWriteFuncHeaderData.nStartOffset = panOffsets[0]; |
3416 | | sWriteFuncHeaderData.nEndOffset = panOffsets[0] + nTotalReqSize - 1; |
3417 | | } |
3418 | | |
3419 | | if constexpr (ENABLE_DEBUG) |
3420 | | { |
3421 | | if (nMergedRanges == 1) |
3422 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
3423 | | osRanges.c_str(), m_pszURL); |
3424 | | else |
3425 | | CPLDebug(poFS->GetDebugKey(), |
3426 | | "Downloading %s, ..., %s (" CPL_FRMT_GUIB " bytes, %s)...", |
3427 | | osFirstRange.c_str(), osLastRange.c_str(), |
3428 | | static_cast<GUIntBig>(nTotalReqSize), m_pszURL); |
3429 | | } |
3430 | | |
3431 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, osRanges.c_str()); |
3432 | | |
3433 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
3434 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
3435 | | |
3436 | | headers = GetCurlHeaders("GET", headers); |
3437 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
3438 | | |
3439 | | poFS->Perform(hCurlHandle); |
3440 | | |
3441 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
3442 | | |
3443 | | curl_slist_free_all(headers); |
3444 | | |
3445 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
3446 | | |
3447 | | if (sWriteFuncData.bInterrupted) |
3448 | | { |
3449 | | bInterrupted = true; |
3450 | | |
3451 | | CPLFree(sWriteFuncData.pBuffer); |
3452 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3453 | | curl_easy_cleanup(hCurlHandle); |
3454 | | |
3455 | | return -1; |
3456 | | } |
3457 | | |
3458 | | long response_code = 0; |
3459 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
3460 | | |
3461 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
3462 | | response_code != 226 && response_code != 426) || |
3463 | | sWriteFuncHeaderData.bError) |
3464 | | { |
3465 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
3466 | | { |
3467 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
3468 | | CPLError( |
3469 | | CE_Failure, CPLE_AppDefined, |
3470 | | "%d: %s, Range downloading not supported by this server!", |
3471 | | static_cast<int>(response_code), szCurlErrBuf); |
3472 | | else |
3473 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
3474 | | static_cast<int>(response_code), szCurlErrBuf); |
3475 | | } |
3476 | | /* |
3477 | | if( !bHasComputedFileSize && startOffset == 0 ) |
3478 | | { |
3479 | | cachedFileProp->bHasComputedFileSize = bHasComputedFileSize = true; |
3480 | | cachedFileProp->fileSize = fileSize = 0; |
3481 | | cachedFileProp->eExists = eExists = EXIST_NO; |
3482 | | } |
3483 | | */ |
3484 | | CPLFree(sWriteFuncData.pBuffer); |
3485 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3486 | | curl_easy_cleanup(hCurlHandle); |
3487 | | return -1; |
3488 | | } |
3489 | | |
3490 | | char *pBuffer = sWriteFuncData.pBuffer; |
3491 | | size_t nSize = sWriteFuncData.nSize; |
3492 | | |
3493 | | // TODO(schwehr): Localize after removing gotos. |
3494 | | int nRet = -1; |
3495 | | char *pszBoundary; |
3496 | | std::string osBoundary; |
3497 | | char *pszNext = nullptr; |
3498 | | int iRange = 0; |
3499 | | int iPart = 0; |
3500 | | char *pszEOL = nullptr; |
3501 | | |
3502 | | /* -------------------------------------------------------------------- */ |
3503 | | /* No multipart if a single range has been requested */ |
3504 | | /* -------------------------------------------------------------------- */ |
3505 | | |
3506 | | if (nMergedRanges == 1) |
3507 | | { |
3508 | | size_t nAccSize = 0; |
3509 | | if (static_cast<vsi_l_offset>(nSize) < nTotalReqSize) |
3510 | | goto end; |
3511 | | |
3512 | | for (int i = 0; i < nRanges; i++) |
3513 | | { |
3514 | | memcpy(ppData[i], pBuffer + nAccSize, panSizes[i]); |
3515 | | nAccSize += panSizes[i]; |
3516 | | } |
3517 | | |
3518 | | nRet = 0; |
3519 | | goto end; |
3520 | | } |
3521 | | |
3522 | | /* -------------------------------------------------------------------- */ |
3523 | | /* Extract boundary name */ |
3524 | | /* -------------------------------------------------------------------- */ |
3525 | | |
3526 | | pszBoundary = strstr(sWriteFuncHeaderData.pBuffer, |
3527 | | "Content-Type: multipart/byteranges; boundary="); |
3528 | | if (pszBoundary == nullptr) |
3529 | | { |
3530 | | CPLError(CE_Failure, CPLE_AppDefined, "Could not find '%s'", |
3531 | | "Content-Type: multipart/byteranges; boundary="); |
3532 | | goto end; |
3533 | | } |
3534 | | |
3535 | | pszBoundary += strlen("Content-Type: multipart/byteranges; boundary="); |
3536 | | |
3537 | | pszEOL = strchr(pszBoundary, '\r'); |
3538 | | if (pszEOL) |
3539 | | *pszEOL = 0; |
3540 | | pszEOL = strchr(pszBoundary, '\n'); |
3541 | | if (pszEOL) |
3542 | | *pszEOL = 0; |
3543 | | |
3544 | | /* Remove optional double-quote character around boundary name */ |
3545 | | if (pszBoundary[0] == '"') |
3546 | | { |
3547 | | pszBoundary++; |
3548 | | char *pszLastDoubleQuote = strrchr(pszBoundary, '"'); |
3549 | | if (pszLastDoubleQuote) |
3550 | | *pszLastDoubleQuote = 0; |
3551 | | } |
3552 | | |
3553 | | osBoundary = "--"; |
3554 | | osBoundary += pszBoundary; |
3555 | | |
3556 | | /* -------------------------------------------------------------------- */ |
3557 | | /* Find the start of the first chunk. */ |
3558 | | /* -------------------------------------------------------------------- */ |
3559 | | pszNext = strstr(pBuffer, osBoundary.c_str()); |
3560 | | if (pszNext == nullptr) |
3561 | | { |
3562 | | CPLError(CE_Failure, CPLE_AppDefined, "No parts found."); |
3563 | | goto end; |
3564 | | } |
3565 | | |
3566 | | pszNext += osBoundary.size(); |
3567 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
3568 | | pszNext++; |
3569 | | if (*pszNext == '\r') |
3570 | | pszNext++; |
3571 | | if (*pszNext == '\n') |
3572 | | pszNext++; |
3573 | | |
3574 | | /* -------------------------------------------------------------------- */ |
3575 | | /* Loop over parts... */ |
3576 | | /* -------------------------------------------------------------------- */ |
3577 | | while (iPart < nRanges) |
3578 | | { |
3579 | | /* -------------------------------------------------------------------- |
3580 | | */ |
3581 | | /* Collect headers. */ |
3582 | | /* -------------------------------------------------------------------- |
3583 | | */ |
3584 | | bool bExpectedRange = false; |
3585 | | |
3586 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
3587 | | { |
3588 | | pszEOL = strstr(pszNext, "\n"); |
3589 | | |
3590 | | if (pszEOL == nullptr) |
3591 | | { |
3592 | | CPLError(CE_Failure, CPLE_AppDefined, |
3593 | | "Error while parsing multipart content (at line %d)", |
3594 | | __LINE__); |
3595 | | goto end; |
3596 | | } |
3597 | | |
3598 | | *pszEOL = '\0'; |
3599 | | bool bRestoreAntislashR = false; |
3600 | | if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r') |
3601 | | { |
3602 | | bRestoreAntislashR = true; |
3603 | | pszEOL[-1] = '\0'; |
3604 | | } |
3605 | | |
3606 | | if (STARTS_WITH_CI(pszNext, "Content-Range: bytes ")) |
3607 | | { |
3608 | | bExpectedRange = true; /* FIXME */ |
3609 | | } |
3610 | | |
3611 | | if (bRestoreAntislashR) |
3612 | | pszEOL[-1] = '\r'; |
3613 | | *pszEOL = '\n'; |
3614 | | |
3615 | | pszNext = pszEOL + 1; |
3616 | | } |
3617 | | |
3618 | | if (!bExpectedRange) |
3619 | | { |
3620 | | CPLError(CE_Failure, CPLE_AppDefined, |
3621 | | "Error while parsing multipart content (at line %d)", |
3622 | | __LINE__); |
3623 | | goto end; |
3624 | | } |
3625 | | |
3626 | | if (*pszNext == '\r') |
3627 | | pszNext++; |
3628 | | if (*pszNext == '\n') |
3629 | | pszNext++; |
3630 | | |
3631 | | /* -------------------------------------------------------------------- |
3632 | | */ |
3633 | | /* Work out the data block size. */ |
3634 | | /* -------------------------------------------------------------------- |
3635 | | */ |
3636 | | size_t nBytesAvail = nSize - (pszNext - pBuffer); |
3637 | | |
3638 | | while (true) |
3639 | | { |
3640 | | if (nBytesAvail < panSizes[iRange]) |
3641 | | { |
3642 | | CPLError(CE_Failure, CPLE_AppDefined, |
3643 | | "Error while parsing multipart content (at line %d)", |
3644 | | __LINE__); |
3645 | | goto end; |
3646 | | } |
3647 | | |
3648 | | memcpy(ppData[iRange], pszNext, panSizes[iRange]); |
3649 | | pszNext += panSizes[iRange]; |
3650 | | nBytesAvail -= panSizes[iRange]; |
3651 | | if (iRange + 1 < nRanges && |
3652 | | panOffsets[iRange] + panSizes[iRange] == panOffsets[iRange + 1]) |
3653 | | { |
3654 | | iRange++; |
3655 | | } |
3656 | | else |
3657 | | { |
3658 | | break; |
3659 | | } |
3660 | | } |
3661 | | |
3662 | | iPart++; |
3663 | | iRange++; |
3664 | | |
3665 | | while (nBytesAvail > 0 && |
3666 | | (*pszNext != '-' || |
3667 | | strncmp(pszNext, osBoundary.c_str(), osBoundary.size()) != 0)) |
3668 | | { |
3669 | | pszNext++; |
3670 | | nBytesAvail--; |
3671 | | } |
3672 | | |
3673 | | if (nBytesAvail == 0) |
3674 | | { |
3675 | | CPLError(CE_Failure, CPLE_AppDefined, |
3676 | | "Error while parsing multipart content (at line %d)", |
3677 | | __LINE__); |
3678 | | goto end; |
3679 | | } |
3680 | | |
3681 | | pszNext += osBoundary.size(); |
3682 | | if (STARTS_WITH(pszNext, "--")) |
3683 | | { |
3684 | | // End of multipart. |
3685 | | break; |
3686 | | } |
3687 | | |
3688 | | if (*pszNext == '\r') |
3689 | | pszNext++; |
3690 | | if (*pszNext == '\n') |
3691 | | pszNext++; |
3692 | | else |
3693 | | { |
3694 | | CPLError(CE_Failure, CPLE_AppDefined, |
3695 | | "Error while parsing multipart content (at line %d)", |
3696 | | __LINE__); |
3697 | | goto end; |
3698 | | } |
3699 | | } |
3700 | | |
3701 | | if (iPart == nMergedRanges) |
3702 | | nRet = 0; |
3703 | | else |
3704 | | CPLError(CE_Failure, CPLE_AppDefined, |
3705 | | "Got only %d parts, where %d were expected", iPart, |
3706 | | nMergedRanges); |
3707 | | |
3708 | | end: |
3709 | | CPLFree(sWriteFuncData.pBuffer); |
3710 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3711 | | curl_easy_cleanup(hCurlHandle); |
3712 | | |
3713 | | return nRet; |
3714 | | } |
3715 | | |
3716 | | /************************************************************************/ |
3717 | | /* PRead() */ |
3718 | | /************************************************************************/ |
3719 | | |
3720 | | size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, |
3721 | | vsi_l_offset nOffset) const |
3722 | | { |
3723 | | // Try to use AdviseRead ranges fetched asynchronously |
3724 | | if (!m_aoAdviseReadRanges.empty()) |
3725 | | { |
3726 | | for (auto &poRange : m_aoAdviseReadRanges) |
3727 | | { |
3728 | | if (nOffset >= poRange->nStartOffset && |
3729 | | nOffset + nSize <= poRange->nStartOffset + poRange->nSize) |
3730 | | { |
3731 | | { |
3732 | | std::unique_lock<std::mutex> oLock(poRange->oMutex); |
3733 | | // coverity[missing_lock:FALSE] |
3734 | | while (!poRange->bDone) |
3735 | | { |
3736 | | poRange->oCV.wait(oLock); |
3737 | | } |
3738 | | } |
3739 | | if (poRange->abyData.empty()) |
3740 | | return 0; |
3741 | | |
3742 | | auto nEndOffset = |
3743 | | poRange->nStartOffset + poRange->abyData.size(); |
3744 | | if (nOffset >= nEndOffset) |
3745 | | return 0; |
3746 | | const size_t nToCopy = static_cast<size_t>( |
3747 | | std::min<vsi_l_offset>(nSize, nEndOffset - nOffset)); |
3748 | | memcpy(pBuffer, |
3749 | | poRange->abyData.data() + |
3750 | | static_cast<size_t>(nOffset - poRange->nStartOffset), |
3751 | | nToCopy); |
3752 | | return nToCopy; |
3753 | | } |
3754 | | } |
3755 | | } |
3756 | | |
3757 | | // poFS has a global mutex |
3758 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
3759 | | if (oFileProp.eExists == EXIST_NO) |
3760 | | return static_cast<size_t>(-1); |
3761 | | |
3762 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3763 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3764 | | NetworkStatisticsAction oContextAction("PRead"); |
3765 | | |
3766 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3767 | | std::string osURL; |
3768 | | { |
3769 | | //PRead can be called by multiple threads for the same VSICurlHandle. |
3770 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3771 | | UpdateQueryString(); |
3772 | | bool bHasExpired; |
3773 | | osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions); |
3774 | | } |
3775 | | |
3776 | | CURL *hCurlHandle = curl_easy_init(); |
3777 | | |
3778 | | struct curl_slist *headers = |
3779 | | poFS->SetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3780 | | |
3781 | | WriteFuncStruct sWriteFuncData; |
3782 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
3783 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
3784 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3785 | | VSICurlHandleWriteFunc); |
3786 | | |
3787 | | WriteFuncStruct sWriteFuncHeaderData; |
3788 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
3789 | | nullptr); |
3790 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3791 | | &sWriteFuncHeaderData); |
3792 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3793 | | VSICurlHandleWriteFunc); |
3794 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3795 | | sWriteFuncHeaderData.nStartOffset = nOffset; |
3796 | | |
3797 | | sWriteFuncHeaderData.nEndOffset = nOffset + nSize - 1; |
3798 | | |
3799 | | char rangeStr[512] = {}; |
3800 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3801 | | sWriteFuncHeaderData.nStartOffset, |
3802 | | sWriteFuncHeaderData.nEndOffset); |
3803 | | |
3804 | | if constexpr (ENABLE_DEBUG) |
3805 | | { |
3806 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
3807 | | osURL.c_str()); |
3808 | | } |
3809 | | |
3810 | | std::string osHeaderRange; |
3811 | | if (sWriteFuncHeaderData.bIsHTTP) |
3812 | | { |
3813 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
3814 | | // So it gets included in Azure signature |
3815 | | headers = curl_slist_append(headers, osHeaderRange.data()); |
3816 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
3817 | | } |
3818 | | else |
3819 | | { |
3820 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
3821 | | } |
3822 | | |
3823 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3824 | | szCurlErrBuf[0] = '\0'; |
3825 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3826 | | &szCurlErrBuf[0]); |
3827 | | |
3828 | | { |
3829 | | //PRead can be called by multiple threads for the same VSICurlHandle. |
3830 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3831 | | headers = |
3832 | | const_cast<VSICurlHandle *>(this)->GetCurlHeaders("GET", headers); |
3833 | | } |
3834 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
3835 | | |
3836 | | poFS->Perform(hCurlHandle); |
3837 | | |
3838 | | { |
3839 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3840 | | const_cast<VSICurlHandle *>(this)->UpdateRedirectInfo( |
3841 | | hCurlHandle, sWriteFuncHeaderData); |
3842 | | } |
3843 | | |
3844 | | long response_code = 0; |
3845 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
3846 | | |
3847 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
3848 | | { |
3849 | | const char *pszErrorMsg = &szCurlErrBuf[0]; |
3850 | | CPLDebug(poFS->GetDebugKey(), "PRead(%s), %s: response_code=%d, msg=%s", |
3851 | | osURL.c_str(), rangeStr, static_cast<int>(response_code), |
3852 | | pszErrorMsg); |
3853 | | } |
3854 | | |
3855 | | size_t nRet; |
3856 | | if ((response_code != 206 && response_code != 225) || |
3857 | | sWriteFuncData.nSize == 0) |
3858 | | { |
3859 | | if (!m_bInterrupt) |
3860 | | { |
3861 | | CPLDebug(poFS->GetDebugKey(), |
3862 | | "Request for %s failed with response_code=%ld", rangeStr, |
3863 | | response_code); |
3864 | | } |
3865 | | nRet = static_cast<size_t>(-1); |
3866 | | } |
3867 | | else |
3868 | | { |
3869 | | nRet = std::min(sWriteFuncData.nSize, nSize); |
3870 | | if (nRet > 0) |
3871 | | memcpy(pBuffer, sWriteFuncData.pBuffer, nRet); |
3872 | | } |
3873 | | |
3874 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
3875 | | curl_easy_cleanup(hCurlHandle); |
3876 | | CPLFree(sWriteFuncData.pBuffer); |
3877 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3878 | | curl_slist_free_all(headers); |
3879 | | |
3880 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
3881 | | |
3882 | | #if 0 |
3883 | | if( ENABLE_DEBUG ) |
3884 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
3885 | | #endif |
3886 | | |
3887 | | return nRet; |
3888 | | } |
3889 | | |
3890 | | /************************************************************************/ |
3891 | | /* GetAdviseReadTotalBytesLimit() */ |
3892 | | /************************************************************************/ |
3893 | | |
3894 | | size_t VSICurlHandle::GetAdviseReadTotalBytesLimit() const |
3895 | | { |
3896 | | return static_cast<size_t>(std::min<unsigned long long>( |
3897 | | std::numeric_limits<size_t>::max(), |
3898 | | // 100 MB |
3899 | | std::strtoull( |
3900 | | CPLGetConfigOption("CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT", |
3901 | | "104857600"), |
3902 | | nullptr, 10))); |
3903 | | } |
3904 | | |
3905 | | /************************************************************************/ |
3906 | | /* AdviseRead() */ |
3907 | | /************************************************************************/ |
3908 | | |
3909 | | void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, |
3910 | | const size_t *panSizes) |
3911 | | { |
3912 | | if (!CPLTestBool( |
3913 | | CPLGetConfigOption("GDAL_HTTP_ENABLE_ADVISE_READ", "TRUE"))) |
3914 | | return; |
3915 | | |
3916 | | if (m_oThreadAdviseRead.joinable()) |
3917 | | { |
3918 | | m_oThreadAdviseRead.join(); |
3919 | | } |
3920 | | |
3921 | | // Give up if we need to allocate too much memory |
3922 | | vsi_l_offset nMaxSize = 0; |
3923 | | const size_t nLimit = GetAdviseReadTotalBytesLimit(); |
3924 | | for (int i = 0; i < nRanges; ++i) |
3925 | | { |
3926 | | if (panSizes[i] > nLimit - nMaxSize) |
3927 | | { |
3928 | | CPLDebug(poFS->GetDebugKey(), |
3929 | | "Trying to request too many bytes in AdviseRead()"); |
3930 | | return; |
3931 | | } |
3932 | | nMaxSize += panSizes[i]; |
3933 | | } |
3934 | | |
3935 | | UpdateQueryString(); |
3936 | | |
3937 | | bool bHasExpired = false; |
3938 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3939 | | const std::string l_osURL( |
3940 | | GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
3941 | | if (bHasExpired) |
3942 | | { |
3943 | | return; |
3944 | | } |
3945 | | |
3946 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
3947 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
3948 | | |
3949 | | try |
3950 | | { |
3951 | | m_aoAdviseReadRanges.clear(); |
3952 | | m_aoAdviseReadRanges.reserve(nRanges); |
3953 | | for (int i = 0; i < nRanges;) |
3954 | | { |
3955 | | int iNext = i; |
3956 | | // Identify consecutive ranges |
3957 | | constexpr size_t SIZE_COG_MARKERS = 2 * sizeof(uint32_t); |
3958 | | auto nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3959 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
3960 | | panOffsets[iNext + 1] > panOffsets[iNext] && |
3961 | | panOffsets[iNext] + panSizes[iNext] + SIZE_COG_MARKERS >= |
3962 | | panOffsets[iNext + 1] && |
3963 | | panOffsets[iNext + 1] + panSizes[iNext + 1] > nEndOffset) |
3964 | | { |
3965 | | iNext++; |
3966 | | nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3967 | | } |
3968 | | CPLAssert(panOffsets[i] <= nEndOffset); |
3969 | | const size_t nSize = |
3970 | | static_cast<size_t>(nEndOffset - panOffsets[i]); |
3971 | | |
3972 | | if (nSize == 0) |
3973 | | { |
3974 | | i = iNext + 1; |
3975 | | continue; |
3976 | | } |
3977 | | |
3978 | | auto newAdviseReadRange = |
3979 | | std::make_unique<AdviseReadRange>(m_oRetryParameters); |
3980 | | newAdviseReadRange->nStartOffset = panOffsets[i]; |
3981 | | newAdviseReadRange->nSize = nSize; |
3982 | | newAdviseReadRange->abyData.resize(nSize); |
3983 | | m_aoAdviseReadRanges.push_back(std::move(newAdviseReadRange)); |
3984 | | |
3985 | | i = iNext + 1; |
3986 | | } |
3987 | | } |
3988 | | catch (const std::exception &) |
3989 | | { |
3990 | | CPLError(CE_Failure, CPLE_OutOfMemory, |
3991 | | "Out of memory in VSICurlHandle::AdviseRead()"); |
3992 | | m_aoAdviseReadRanges.clear(); |
3993 | | } |
3994 | | |
3995 | | if (m_aoAdviseReadRanges.empty()) |
3996 | | return; |
3997 | | |
3998 | | #ifdef DEBUG |
3999 | | CPLDebug(poFS->GetDebugKey(), "AdviseRead(): fetching %u ranges", |
4000 | | static_cast<unsigned>(m_aoAdviseReadRanges.size())); |
4001 | | #endif |
4002 | | |
4003 | | const auto task = [this, aosHTTPOptions = std::move(aosHTTPOptions)]( |
4004 | | const std::string &osURL) |
4005 | | { |
4006 | | if (!m_hCurlMultiHandleForAdviseRead) |
4007 | | m_hCurlMultiHandleForAdviseRead = VSICURLMultiInit(); |
4008 | | |
4009 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
4010 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
4011 | | NetworkStatisticsAction oContextAction("AdviseRead"); |
4012 | | |
4013 | | size_t nTotalDownloaded = 0; |
4014 | | |
4015 | | while (true) |
4016 | | { |
4017 | | |
4018 | | std::vector<CURL *> aHandles; |
4019 | | std::vector<WriteFuncStruct> asWriteFuncData( |
4020 | | m_aoAdviseReadRanges.size()); |
4021 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData( |
4022 | | m_aoAdviseReadRanges.size()); |
4023 | | std::vector<char *> apszRanges; |
4024 | | std::vector<struct curl_slist *> aHeaders; |
4025 | | |
4026 | | struct CurlErrBuffer |
4027 | | { |
4028 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
4029 | | }; |
4030 | | std::vector<CurlErrBuffer> asCurlErrors( |
4031 | | m_aoAdviseReadRanges.size()); |
4032 | | |
4033 | | std::map<CURL *, size_t> oMapHandleToIdx; |
4034 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
4035 | | { |
4036 | | if (!m_aoAdviseReadRanges[i]->bToRetry) |
4037 | | { |
4038 | | aHandles.push_back(nullptr); |
4039 | | apszRanges.push_back(nullptr); |
4040 | | aHeaders.push_back(nullptr); |
4041 | | continue; |
4042 | | } |
4043 | | m_aoAdviseReadRanges[i]->bToRetry = false; |
4044 | | |
4045 | | CURL *hCurlHandle = curl_easy_init(); |
4046 | | oMapHandleToIdx[hCurlHandle] = i; |
4047 | | aHandles.push_back(hCurlHandle); |
4048 | | |
4049 | | // As the multi-range request is likely not the first one, we don't |
4050 | | // need to wait as we already know if pipelining is possible |
4051 | | // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); |
4052 | | |
4053 | | struct curl_slist *headers = poFS->SetOptions( |
4054 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
4055 | | |
4056 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[i], this, |
4057 | | pfnReadCbk, pReadCbkUserData); |
4058 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
4059 | | &asWriteFuncData[i]); |
4060 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
4061 | | VSICurlHandleWriteFunc); |
4062 | | |
4063 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[i], nullptr, |
4064 | | nullptr, nullptr); |
4065 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
4066 | | &asWriteFuncHeaderData[i]); |
4067 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
4068 | | VSICurlHandleWriteFunc); |
4069 | | asWriteFuncHeaderData[i].bIsHTTP = |
4070 | | STARTS_WITH(m_pszURL, "http"); |
4071 | | asWriteFuncHeaderData[i].nStartOffset = |
4072 | | m_aoAdviseReadRanges[i]->nStartOffset; |
4073 | | |
4074 | | asWriteFuncHeaderData[i].nEndOffset = |
4075 | | m_aoAdviseReadRanges[i]->nStartOffset + |
4076 | | m_aoAdviseReadRanges[i]->nSize - 1; |
4077 | | |
4078 | | char rangeStr[512] = {}; |
4079 | | snprintf(rangeStr, sizeof(rangeStr), |
4080 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
4081 | | asWriteFuncHeaderData[i].nStartOffset, |
4082 | | asWriteFuncHeaderData[i].nEndOffset); |
4083 | | |
4084 | | if constexpr (ENABLE_DEBUG) |
4085 | | { |
4086 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
4087 | | rangeStr, osURL.c_str()); |
4088 | | } |
4089 | | |
4090 | | if (asWriteFuncHeaderData[i].bIsHTTP) |
4091 | | { |
4092 | | std::string osHeaderRange( |
4093 | | CPLSPrintf("Range: bytes=%s", rangeStr)); |
4094 | | // So it gets included in Azure signature |
4095 | | char *pszRange = CPLStrdup(osHeaderRange.c_str()); |
4096 | | apszRanges.push_back(pszRange); |
4097 | | headers = curl_slist_append(headers, pszRange); |
4098 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
4099 | | nullptr); |
4100 | | } |
4101 | | else |
4102 | | { |
4103 | | apszRanges.push_back(nullptr); |
4104 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
4105 | | rangeStr); |
4106 | | } |
4107 | | |
4108 | | asCurlErrors[i].szCurlErrBuf[0] = '\0'; |
4109 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
4110 | | &asCurlErrors[i].szCurlErrBuf[0]); |
4111 | | |
4112 | | headers = GetCurlHeaders("GET", headers); |
4113 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
4114 | | headers); |
4115 | | aHeaders.push_back(headers); |
4116 | | curl_multi_add_handle(m_hCurlMultiHandleForAdviseRead, |
4117 | | hCurlHandle); |
4118 | | } |
4119 | | |
4120 | | const auto DealWithRequest = [this, &osURL, &nTotalDownloaded, |
4121 | | &oMapHandleToIdx, &asCurlErrors, |
4122 | | &asWriteFuncHeaderData, |
4123 | | &asWriteFuncData](CURL *hCurlHandle) |
4124 | | { |
4125 | | auto oIter = oMapHandleToIdx.find(hCurlHandle); |
4126 | | CPLAssert(oIter != oMapHandleToIdx.end()); |
4127 | | const auto iReq = oIter->second; |
4128 | | |
4129 | | long response_code = 0; |
4130 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, |
4131 | | &response_code); |
4132 | | |
4133 | | if (ENABLE_DEBUG && asCurlErrors[iReq].szCurlErrBuf[0] != '\0') |
4134 | | { |
4135 | | char rangeStr[512] = {}; |
4136 | | snprintf(rangeStr, sizeof(rangeStr), |
4137 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
4138 | | asWriteFuncHeaderData[iReq].nStartOffset, |
4139 | | asWriteFuncHeaderData[iReq].nEndOffset); |
4140 | | |
4141 | | const char *pszErrorMsg = |
4142 | | &asCurlErrors[iReq].szCurlErrBuf[0]; |
4143 | | CPLDebug(poFS->GetDebugKey(), |
4144 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
4145 | | osURL.c_str(), rangeStr, |
4146 | | static_cast<int>(response_code), pszErrorMsg); |
4147 | | } |
4148 | | |
4149 | | bool bToRetry = false; |
4150 | | if ((response_code != 206 && response_code != 225) || |
4151 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
4152 | | asWriteFuncHeaderData[iReq].nStartOffset + |
4153 | | asWriteFuncData[iReq].nSize) |
4154 | | { |
4155 | | char rangeStr[512] = {}; |
4156 | | snprintf(rangeStr, sizeof(rangeStr), |
4157 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
4158 | | asWriteFuncHeaderData[iReq].nStartOffset, |
4159 | | asWriteFuncHeaderData[iReq].nEndOffset); |
4160 | | |
4161 | | // Look if we should attempt a retry |
4162 | | if (m_aoAdviseReadRanges[iReq]->retryContext.CanRetry( |
4163 | | static_cast<int>(response_code), |
4164 | | asWriteFuncData[iReq].pBuffer, |
4165 | | &asCurlErrors[iReq].szCurlErrBuf[0])) |
4166 | | { |
4167 | | CPLError(CE_Warning, CPLE_AppDefined, |
4168 | | "HTTP error code for %s range %s: %d. " |
4169 | | "Retrying again in %.1f secs", |
4170 | | osURL.c_str(), rangeStr, |
4171 | | static_cast<int>(response_code), |
4172 | | m_aoAdviseReadRanges[iReq] |
4173 | | ->retryContext.GetCurrentDelay()); |
4174 | | m_aoAdviseReadRanges[iReq]->dfSleepDelay = |
4175 | | m_aoAdviseReadRanges[iReq] |
4176 | | ->retryContext.GetCurrentDelay(); |
4177 | | bToRetry = true; |
4178 | | } |
4179 | | else |
4180 | | { |
4181 | | CPLError(CE_Failure, CPLE_AppDefined, |
4182 | | "Request for %s range %s failed with " |
4183 | | "response_code=%ld", |
4184 | | osURL.c_str(), rangeStr, response_code); |
4185 | | } |
4186 | | } |
4187 | | else |
4188 | | { |
4189 | | const size_t nSize = asWriteFuncData[iReq].nSize; |
4190 | | memcpy(&m_aoAdviseReadRanges[iReq]->abyData[0], |
4191 | | asWriteFuncData[iReq].pBuffer, nSize); |
4192 | | m_aoAdviseReadRanges[iReq]->abyData.resize(nSize); |
4193 | | |
4194 | | nTotalDownloaded += nSize; |
4195 | | } |
4196 | | |
4197 | | m_aoAdviseReadRanges[iReq]->bToRetry = bToRetry; |
4198 | | |
4199 | | if (!bToRetry) |
4200 | | { |
4201 | | std::lock_guard<std::mutex> oLock( |
4202 | | m_aoAdviseReadRanges[iReq]->oMutex); |
4203 | | m_aoAdviseReadRanges[iReq]->bDone = true; |
4204 | | m_aoAdviseReadRanges[iReq]->oCV.notify_all(); |
4205 | | } |
4206 | | }; |
4207 | | |
4208 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
4209 | | while (true) |
4210 | | { |
4211 | | int still_running; |
4212 | | while (curl_multi_perform(m_hCurlMultiHandleForAdviseRead, |
4213 | | &still_running) == |
4214 | | CURLM_CALL_MULTI_PERFORM) |
4215 | | { |
4216 | | // loop |
4217 | | } |
4218 | | if (!still_running) |
4219 | | { |
4220 | | break; |
4221 | | } |
4222 | | |
4223 | | CURLMsg *msg; |
4224 | | do |
4225 | | { |
4226 | | int msgq = 0; |
4227 | | msg = curl_multi_info_read(m_hCurlMultiHandleForAdviseRead, |
4228 | | &msgq); |
4229 | | if (msg && (msg->msg == CURLMSG_DONE)) |
4230 | | { |
4231 | | DealWithRequest(msg->easy_handle); |
4232 | | } |
4233 | | } while (msg); |
4234 | | |
4235 | | CPLMultiPerformWait(m_hCurlMultiHandleForAdviseRead); |
4236 | | } |
4237 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
4238 | | |
4239 | | bool bRetry = false; |
4240 | | double dfDelay = 0.0; |
4241 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
4242 | | { |
4243 | | bool bReqDone; |
4244 | | { |
4245 | | // To please Coverity Scan |
4246 | | std::lock_guard<std::mutex> oLock( |
4247 | | m_aoAdviseReadRanges[i]->oMutex); |
4248 | | bReqDone = m_aoAdviseReadRanges[i]->bDone; |
4249 | | } |
4250 | | if (!bReqDone && !m_aoAdviseReadRanges[i]->bToRetry) |
4251 | | { |
4252 | | DealWithRequest(aHandles[i]); |
4253 | | } |
4254 | | if (m_aoAdviseReadRanges[i]->bToRetry) |
4255 | | dfDelay = std::max(dfDelay, |
4256 | | m_aoAdviseReadRanges[i]->dfSleepDelay); |
4257 | | bRetry = bRetry || m_aoAdviseReadRanges[i]->bToRetry; |
4258 | | if (aHandles[i]) |
4259 | | { |
4260 | | curl_multi_remove_handle(m_hCurlMultiHandleForAdviseRead, |
4261 | | aHandles[i]); |
4262 | | VSICURLResetHeaderAndWriterFunctions(aHandles[i]); |
4263 | | curl_easy_cleanup(aHandles[i]); |
4264 | | } |
4265 | | CPLFree(apszRanges[i]); |
4266 | | CPLFree(asWriteFuncData[i].pBuffer); |
4267 | | CPLFree(asWriteFuncHeaderData[i].pBuffer); |
4268 | | if (aHeaders[i]) |
4269 | | curl_slist_free_all(aHeaders[i]); |
4270 | | } |
4271 | | if (!bRetry) |
4272 | | break; |
4273 | | CPLSleep(dfDelay); |
4274 | | } |
4275 | | |
4276 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
4277 | | }; |
4278 | | |
4279 | | m_oThreadAdviseRead = std::thread(task, l_osURL); |
4280 | | } |
4281 | | |
4282 | | /************************************************************************/ |
4283 | | /* Write() */ |
4284 | | /************************************************************************/ |
4285 | | |
4286 | | size_t VSICurlHandle::Write(const void * /* pBuffer */, size_t /* nBytes */) |
4287 | | { |
4288 | | return 0; |
4289 | | } |
4290 | | |
4291 | | /************************************************************************/ |
4292 | | /* ClearErr() */ |
4293 | | /************************************************************************/ |
4294 | | |
4295 | | void VSICurlHandle::ClearErr() |
4296 | | |
4297 | | { |
4298 | | bEOF = false; |
4299 | | bError = false; |
4300 | | } |
4301 | | |
4302 | | /************************************************************************/ |
4303 | | /* Error() */ |
4304 | | /************************************************************************/ |
4305 | | |
4306 | | int VSICurlHandle::Error() |
4307 | | |
4308 | | { |
4309 | | return bError ? TRUE : FALSE; |
4310 | | } |
4311 | | |
4312 | | /************************************************************************/ |
4313 | | /* Eof() */ |
4314 | | /************************************************************************/ |
4315 | | |
4316 | | int VSICurlHandle::Eof() |
4317 | | |
4318 | | { |
4319 | | return bEOF ? TRUE : FALSE; |
4320 | | } |
4321 | | |
4322 | | /************************************************************************/ |
4323 | | /* Flush() */ |
4324 | | /************************************************************************/ |
4325 | | |
4326 | | int VSICurlHandle::Flush() |
4327 | | { |
4328 | | return 0; |
4329 | | } |
4330 | | |
4331 | | /************************************************************************/ |
4332 | | /* Close() */ |
4333 | | /************************************************************************/ |
4334 | | |
4335 | | int VSICurlHandle::Close() |
4336 | | { |
4337 | | return 0; |
4338 | | } |
4339 | | |
4340 | | /************************************************************************/ |
4341 | | /* VSICurlFilesystemHandlerBase() */ |
4342 | | /************************************************************************/ |
4343 | | |
4344 | | VSICurlFilesystemHandlerBase::VSICurlFilesystemHandlerBase() |
4345 | | : oCacheDirList{1024, 0} |
4346 | | { |
4347 | | } |
4348 | | |
4349 | | /************************************************************************/ |
4350 | | /* ~VSICurlFilesystemHandlerBase() */ |
4351 | | /************************************************************************/ |
4352 | | |
4353 | | VSICurlFilesystemHandlerBase::~VSICurlFilesystemHandlerBase() |
4354 | | { |
4355 | | StopRunThread(); |
4356 | | VSICurlFilesystemHandlerBase::ClearCache(); |
4357 | | |
4358 | | if (hMutex != nullptr) |
4359 | | CPLDestroyMutex(hMutex); |
4360 | | hMutex = nullptr; |
4361 | | } |
4362 | | |
4363 | | /************************************************************************/ |
4364 | | /* AllowCachedDataFor() */ |
4365 | | /************************************************************************/ |
4366 | | |
4367 | | bool VSICurlFilesystemHandlerBase::AllowCachedDataFor(const char *pszFilename) |
4368 | | { |
4369 | | bool bCachedAllowed = true; |
4370 | | char **papszTokens = CSLTokenizeString2( |
4371 | | CPLGetConfigOption("CPL_VSIL_CURL_NON_CACHED", ""), ":", 0); |
4372 | | for (int i = 0; papszTokens && papszTokens[i]; i++) |
4373 | | { |
4374 | | if (STARTS_WITH(pszFilename, papszTokens[i])) |
4375 | | { |
4376 | | bCachedAllowed = false; |
4377 | | break; |
4378 | | } |
4379 | | } |
4380 | | CSLDestroy(papszTokens); |
4381 | | return bCachedAllowed; |
4382 | | } |
4383 | | |
4384 | | /************************************************************************/ |
4385 | | /* GetRegionCache() */ |
4386 | | /************************************************************************/ |
4387 | | |
4388 | | VSICurlFilesystemHandlerBase::RegionCacheType * |
4389 | | VSICurlFilesystemHandlerBase::GetRegionCache() |
4390 | | { |
4391 | | // should be called under hMutex taken |
4392 | | if (m_poRegionCacheDoNotUseDirectly == nullptr) |
4393 | | { |
4394 | | m_poRegionCacheDoNotUseDirectly.reset( |
4395 | | new RegionCacheType(static_cast<size_t>(GetMaxRegions()))); |
4396 | | } |
4397 | | return m_poRegionCacheDoNotUseDirectly.get(); |
4398 | | } |
4399 | | |
4400 | | /************************************************************************/ |
4401 | | /* GetRegion() */ |
4402 | | /************************************************************************/ |
4403 | | |
4404 | | std::shared_ptr<std::string> |
4405 | | VSICurlFilesystemHandlerBase::GetRegion(const char *pszURL, |
4406 | | vsi_l_offset nFileOffsetStart) |
4407 | | { |
4408 | | CPLMutexHolder oHolder(&hMutex); |
4409 | | |
4410 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
4411 | | nFileOffsetStart = |
4412 | | (nFileOffsetStart / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
4413 | | |
4414 | | std::shared_ptr<std::string> out; |
4415 | | if (GetRegionCache()->tryGet( |
4416 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), out)) |
4417 | | { |
4418 | | return out; |
4419 | | } |
4420 | | |
4421 | | return nullptr; |
4422 | | } |
4423 | | |
4424 | | /************************************************************************/ |
4425 | | /* AddRegion() */ |
4426 | | /************************************************************************/ |
4427 | | |
4428 | | void VSICurlFilesystemHandlerBase::AddRegion(const char *pszURL, |
4429 | | vsi_l_offset nFileOffsetStart, |
4430 | | size_t nSize, const char *pData) |
4431 | | { |
4432 | | CPLMutexHolder oHolder(&hMutex); |
4433 | | |
4434 | | auto value = std::make_shared<std::string>(); |
4435 | | value->assign(pData, nSize); |
4436 | | GetRegionCache()->insert( |
4437 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), |
4438 | | std::move(value)); |
4439 | | } |
4440 | | |
4441 | | /************************************************************************/ |
4442 | | /* GetCachedFileProp() */ |
4443 | | /************************************************************************/ |
4444 | | |
4445 | | bool VSICurlFilesystemHandlerBase::GetCachedFileProp(const char *pszURL, |
4446 | | FileProp &oFileProp) |
4447 | | { |
4448 | | return VSICURLGetCachedFileProp(pszURL, oFileProp); |
4449 | | } |
4450 | | |
4451 | | /************************************************************************/ |
4452 | | /* SetCachedFileProp() */ |
4453 | | /************************************************************************/ |
4454 | | |
4455 | | void VSICurlFilesystemHandlerBase::SetCachedFileProp(const char *pszURL, |
4456 | | FileProp &oFileProp) |
4457 | | { |
4458 | | VSICURLSetCachedFileProp(pszURL, oFileProp); |
4459 | | } |
4460 | | |
4461 | | /************************************************************************/ |
4462 | | /* GetCachedDirList() */ |
4463 | | /************************************************************************/ |
4464 | | |
4465 | | bool VSICurlFilesystemHandlerBase::GetCachedDirList( |
4466 | | const char *pszURL, CachedDirList &oCachedDirList) |
4467 | | { |
4468 | | CPLMutexHolder oHolder(&hMutex); |
4469 | | |
4470 | | return oCacheDirList.tryGet(std::string(pszURL), oCachedDirList) && |
4471 | | // Let a chance to use new auth parameters |
4472 | | gnGenerationAuthParameters == |
4473 | | oCachedDirList.nGenerationAuthParameters; |
4474 | | } |
4475 | | |
4476 | | /************************************************************************/ |
4477 | | /* SetCachedDirList() */ |
4478 | | /************************************************************************/ |
4479 | | |
4480 | | void VSICurlFilesystemHandlerBase::SetCachedDirList( |
4481 | | const char *pszURL, CachedDirList &oCachedDirList) |
4482 | | { |
4483 | | CPLMutexHolder oHolder(&hMutex); |
4484 | | |
4485 | | std::string key(pszURL); |
4486 | | CachedDirList oldValue; |
4487 | | if (oCacheDirList.tryGet(key, oldValue)) |
4488 | | { |
4489 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4490 | | oCacheDirList.remove(key); |
4491 | | } |
4492 | | |
4493 | | while ((!oCacheDirList.empty() && |
4494 | | nCachedFilesInDirList + oCachedDirList.oFileList.size() > |
4495 | | 1024 * 1024) || |
4496 | | oCacheDirList.size() == oCacheDirList.getMaxAllowedSize()) |
4497 | | { |
4498 | | std::string oldestKey; |
4499 | | oCacheDirList.getOldestEntry(oldestKey, oldValue); |
4500 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4501 | | oCacheDirList.remove(oldestKey); |
4502 | | } |
4503 | | oCachedDirList.nGenerationAuthParameters = gnGenerationAuthParameters; |
4504 | | |
4505 | | nCachedFilesInDirList += oCachedDirList.oFileList.size(); |
4506 | | oCacheDirList.insert(key, oCachedDirList); |
4507 | | } |
4508 | | |
4509 | | /************************************************************************/ |
4510 | | /* ExistsInCacheDirList() */ |
4511 | | /************************************************************************/ |
4512 | | |
4513 | | bool VSICurlFilesystemHandlerBase::ExistsInCacheDirList( |
4514 | | const std::string &osDirname, bool *pbIsDir) |
4515 | | { |
4516 | | CachedDirList cachedDirList; |
4517 | | if (GetCachedDirList(osDirname.c_str(), cachedDirList)) |
4518 | | { |
4519 | | if (pbIsDir) |
4520 | | *pbIsDir = !cachedDirList.oFileList.empty(); |
4521 | | return false; |
4522 | | } |
4523 | | else |
4524 | | { |
4525 | | if (pbIsDir) |
4526 | | *pbIsDir = false; |
4527 | | return false; |
4528 | | } |
4529 | | } |
4530 | | |
4531 | | /************************************************************************/ |
4532 | | /* InvalidateCachedData() */ |
4533 | | /************************************************************************/ |
4534 | | |
4535 | | void VSICurlFilesystemHandlerBase::InvalidateCachedData(const char *pszURL) |
4536 | | { |
4537 | | CPLMutexHolder oHolder(&hMutex); |
4538 | | |
4539 | | VSICURLInvalidateCachedFileProp(pszURL); |
4540 | | |
4541 | | // Invalidate all cached regions for this URL |
4542 | | std::list<FilenameOffsetPair> keysToRemove; |
4543 | | std::string osURL(pszURL); |
4544 | | auto lambda = |
4545 | | [&keysToRemove, |
4546 | | &osURL](const lru11::KeyValuePair<FilenameOffsetPair, |
4547 | | std::shared_ptr<std::string>> &kv) |
4548 | | { |
4549 | | if (kv.key.filename_ == osURL) |
4550 | | keysToRemove.push_back(kv.key); |
4551 | | }; |
4552 | | auto *poRegionCache = GetRegionCache(); |
4553 | | poRegionCache->cwalk(lambda); |
4554 | | for (const auto &key : keysToRemove) |
4555 | | poRegionCache->remove(key); |
4556 | | } |
4557 | | |
4558 | | /************************************************************************/ |
4559 | | /* ClearCache() */ |
4560 | | /************************************************************************/ |
4561 | | |
4562 | | void VSICurlFilesystemHandlerBase::ClearCache() |
4563 | | { |
4564 | | CPLMutexHolder oHolder(&hMutex); |
4565 | | |
4566 | | GetRegionCache()->clear(); |
4567 | | |
4568 | | VSICURLDestroyCacheFileProp(); |
4569 | | |
4570 | | oCacheDirList.clear(); |
4571 | | nCachedFilesInDirList = 0; |
4572 | | } |
4573 | | |
4574 | | /************************************************************************/ |
4575 | | /* PartialClearCache() */ |
4576 | | /************************************************************************/ |
4577 | | |
4578 | | void VSICurlFilesystemHandlerBase::PartialClearCache( |
4579 | | const char *pszFilenamePrefix) |
4580 | | { |
4581 | | CPLMutexHolder oHolder(&hMutex); |
4582 | | |
4583 | | std::string osURL = GetURLFromFilename(pszFilenamePrefix); |
4584 | | { |
4585 | | std::list<FilenameOffsetPair> keysToRemove; |
4586 | | auto lambda = |
4587 | | [&keysToRemove, &osURL]( |
4588 | | const lru11::KeyValuePair<FilenameOffsetPair, |
4589 | | std::shared_ptr<std::string>> &kv) |
4590 | | { |
4591 | | if (strncmp(kv.key.filename_.c_str(), osURL.c_str(), |
4592 | | osURL.size()) == 0) |
4593 | | keysToRemove.push_back(kv.key); |
4594 | | }; |
4595 | | auto *poRegionCache = GetRegionCache(); |
4596 | | poRegionCache->cwalk(lambda); |
4597 | | for (const auto &key : keysToRemove) |
4598 | | poRegionCache->remove(key); |
4599 | | } |
4600 | | |
4601 | | VSICURLInvalidateCachedFilePropPrefix(osURL.c_str()); |
4602 | | |
4603 | | { |
4604 | | const size_t nLen = strlen(pszFilenamePrefix); |
4605 | | std::list<std::string> keysToRemove; |
4606 | | auto lambda = |
4607 | | [this, &keysToRemove, pszFilenamePrefix, |
4608 | | nLen](const lru11::KeyValuePair<std::string, CachedDirList> &kv) |
4609 | | { |
4610 | | if (strncmp(kv.key.c_str(), pszFilenamePrefix, nLen) == 0) |
4611 | | { |
4612 | | keysToRemove.push_back(kv.key); |
4613 | | nCachedFilesInDirList -= kv.value.oFileList.size(); |
4614 | | } |
4615 | | }; |
4616 | | oCacheDirList.cwalk(lambda); |
4617 | | for (const auto &key : keysToRemove) |
4618 | | oCacheDirList.remove(key); |
4619 | | } |
4620 | | } |
4621 | | |
4622 | | /************************************************************************/ |
4623 | | /* CreateFileHandle() */ |
4624 | | /************************************************************************/ |
4625 | | |
4626 | | VSICurlHandle * |
4627 | | VSICurlFilesystemHandlerBase::CreateFileHandle(const char *pszFilename) |
4628 | | { |
4629 | | return new VSICurlHandle(this, pszFilename); |
4630 | | } |
4631 | | |
4632 | | /************************************************************************/ |
4633 | | /* GetActualURL() */ |
4634 | | /************************************************************************/ |
4635 | | |
4636 | | const char *VSICurlFilesystemHandlerBase::GetActualURL(const char *pszFilename) |
4637 | | { |
4638 | | VSICurlHandle *poHandle = CreateFileHandle(pszFilename); |
4639 | | if (poHandle == nullptr) |
4640 | | return pszFilename; |
4641 | | std::string osURL(poHandle->GetURL()); |
4642 | | delete poHandle; |
4643 | | return CPLSPrintf("%s", osURL.c_str()); |
4644 | | } |
4645 | | |
4646 | | /************************************************************************/ |
4647 | | /* GetOptions() */ |
4648 | | /************************************************************************/ |
4649 | | |
4650 | | #define VSICURL_OPTIONS \ |
4651 | | " <Option name='GDAL_HTTP_MAX_RETRY' type='int' " \ |
4652 | | "description='Maximum number of retries' default='0'/>" \ |
4653 | | " <Option name='GDAL_HTTP_RETRY_DELAY' type='double' " \ |
4654 | | "description='Retry delay in seconds' default='30'/>" \ |
4655 | | " <Option name='GDAL_HTTP_HEADER_FILE' type='string' " \ |
4656 | | "description='Filename of a file that contains HTTP headers to " \ |
4657 | | "forward to the server'/>" \ |
4658 | | " <Option name='CPL_VSIL_CURL_USE_HEAD' type='boolean' " \ |
4659 | | "description='Whether to use HTTP HEAD verb to retrieve " \ |
4660 | | "file information' default='YES'/>" \ |
4661 | | " <Option name='GDAL_HTTP_MULTIRANGE' type='string-select' " \ |
4662 | | "description='Strategy to apply to run multi-range requests' " \ |
4663 | | "default='PARALLEL'>" \ |
4664 | | " <Value>PARALLEL</Value>" \ |
4665 | | " <Value>SERIAL</Value>" \ |
4666 | | " </Option>" \ |
4667 | | " <Option name='GDAL_HTTP_MULTIPLEX' type='boolean' " \ |
4668 | | "description='Whether to enable HTTP/2 multiplexing' default='YES'/>" \ |
4669 | | " <Option name='GDAL_HTTP_MERGE_CONSECUTIVE_RANGES' type='boolean' " \ |
4670 | | "description='Whether to merge consecutive ranges in multirange " \ |
4671 | | "requests' default='YES'/>" \ |
4672 | | " <Option name='CPL_VSIL_CURL_NON_CACHED' type='string' " \ |
4673 | | "description='Colon-separated list of filenames whose content" \ |
4674 | | "must not be cached across open attempts'/>" \ |
4675 | | " <Option name='CPL_VSIL_CURL_ALLOWED_FILENAME' type='string' " \ |
4676 | | "description='Single filename that is allowed to be opened'/>" \ |
4677 | | " <Option name='CPL_VSIL_CURL_ALLOWED_EXTENSIONS' type='string' " \ |
4678 | | "description='Comma or space separated list of allowed file " \ |
4679 | | "extensions'/>" \ |
4680 | | " <Option name='GDAL_DISABLE_READDIR_ON_OPEN' type='string-select' " \ |
4681 | | "description='Whether to disable establishing the list of files in " \ |
4682 | | "the directory of the current filename' default='NO'>" \ |
4683 | | " <Value>NO</Value>" \ |
4684 | | " <Value>YES</Value>" \ |
4685 | | " <Value>EMPTY_DIR</Value>" \ |
4686 | | " </Option>" \ |
4687 | | " <Option name='VSI_CACHE' type='boolean' " \ |
4688 | | "description='Whether to cache in memory the contents of the opened " \ |
4689 | | "file as soon as they are read' default='NO'/>" \ |
4690 | | " <Option name='CPL_VSIL_CURL_CHUNK_SIZE' type='integer' " \ |
4691 | | "description='Size in bytes of the minimum amount of data read in a " \ |
4692 | | "file' default='16384' min='1024' max='10485760'/>" \ |
4693 | | " <Option name='CPL_VSIL_CURL_CACHE_SIZE' type='integer' " \ |
4694 | | "description='Size in bytes of the global /vsicurl/ cache' " \ |
4695 | | "default='16384000'/>" \ |
4696 | | " <Option name='CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE' type='boolean' " \ |
4697 | | "description='Whether to skip files with Glacier storage class in " \ |
4698 | | "directory listing.' default='YES'/>" \ |
4699 | | " <Option name='CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT' " \ |
4700 | | "type='integer' description='Maximum number of bytes AdviseRead() is " \ |
4701 | | "allowed to fetch at once' default='104857600'/>" \ |
4702 | | " <Option name='CPL_VSIL_CURL_HEADER_FILE_KVP_ENABLED' " \ |
4703 | | "type='string-select' description='Whether the header-file key-value " \ |
4704 | | "pair can be used in /vsicurl? filenames' default='ONLY_IN_TEMP'>" \ |
4705 | | " <Value>ONLY_IN_TEMP</Value>" \ |
4706 | | " <Value>NO</Value>" \ |
4707 | | " <Value>YES</Value>" \ |
4708 | | " </Option>" \ |
4709 | | " <Option name='GDAL_HTTP_MAX_CACHED_CONNECTIONS' type='integer' " \ |
4710 | | "description='Maximum amount of connections that libcurl may keep alive " \ |
4711 | | "in its connection cache after use'/>" \ |
4712 | | " <Option name='GDAL_HTTP_MAX_TOTAL_CONNECTIONS' type='integer' " \ |
4713 | | "description='Maximum number of simultaneously open connections in " \ |
4714 | | "total'/>" |
4715 | | |
4716 | | const char *VSICurlFilesystemHandlerBase::GetOptionsStatic() |
4717 | | { |
4718 | | return VSICURL_OPTIONS; |
4719 | | } |
4720 | | |
4721 | | const char *VSICurlFilesystemHandlerBase::GetOptions() |
4722 | | { |
4723 | | static std::string osOptions(std::string("<Options>") + GetOptionsStatic() + |
4724 | | "</Options>"); |
4725 | | return osOptions.c_str(); |
4726 | | } |
4727 | | |
4728 | | /************************************************************************/ |
4729 | | /* SetOptions() */ |
4730 | | /************************************************************************/ |
4731 | | |
4732 | | struct curl_slist * |
4733 | | VSICurlFilesystemHandlerBase::SetOptions(CURL *hCurlHandle, const char *pszURL, |
4734 | | const char *const *papszOptions) |
4735 | | { |
4736 | | struct curl_slist *headers = static_cast<struct curl_slist *>( |
4737 | | CPLHTTPSetOptions(hCurlHandle, pszURL, papszOptions)); |
4738 | | // Override the debug output function. |
4739 | | if (CPLTestConfigOption("CPL_CURL_VERBOSE") && CPLIsDebugEnabled()) |
4740 | | { |
4741 | | unchecked_curl_easy_setopt( |
4742 | | hCurlHandle, CURLOPT_DEBUGFUNCTION, |
4743 | | &VSICurlFilesystemHandlerBase::CurlDebugStatic); |
4744 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_DEBUGDATA, |
4745 | | static_cast<void *>(this)); |
4746 | | } |
4747 | | |
4748 | | long option = CURLFTPMETHOD_SINGLECWD; |
4749 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_FILEMETHOD, option); |
4750 | | |
4751 | | // ftp://ftp2.cits.rncan.gc.ca/pub/cantopo/250k_tif/ |
4752 | | // doesn't like EPSV command, |
4753 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_USE_EPSV, 0); |
4754 | | |
4755 | | return headers; |
4756 | | } |
4757 | | |
4758 | | // Curl debug callback. Trim trailing newlines from the data and call the |
4759 | | // FilesystemHandler-specific debug handler. |
4760 | | int VSICurlFilesystemHandlerBase::CurlDebugStatic(CURL *handle, |
4761 | | curl_infotype type, |
4762 | | char *data, size_t size, |
4763 | | void *userp) |
4764 | | { |
4765 | | if (size && data[size - 1] == '\n') |
4766 | | size--; |
4767 | | |
4768 | | static_cast<VSICurlFilesystemHandlerBase *>(userp)->CurlDebug( |
4769 | | handle, type, std::string_view(data, size)); |
4770 | | return 0; |
4771 | | } |
4772 | | |
4773 | | // Handle curl debug. Stick the debug information on our handle object if it exists |
4774 | | // and notify so that a waiting thread will see the output. This makes the debug |
4775 | | // appear on the thread associated with the invoking CURL handle. |
4776 | | void VSICurlFilesystemHandlerBase::CurlDebug(CURL *handle, curl_infotype type, |
4777 | | std::string_view msg) |
4778 | | { |
4779 | | const char *pszDebugKey = nullptr; |
4780 | | if (type == CURLINFO_TEXT) |
4781 | | pszDebugKey = "CURL_INFO_TEXT"; |
4782 | | else if (type == CURLINFO_HEADER_OUT) |
4783 | | pszDebugKey = "CURL_INFO_HEADER_OUT"; |
4784 | | else if (type == CURLINFO_HEADER_IN) |
4785 | | pszDebugKey = "CURL_INFO_HEADER_IN"; |
4786 | | else if (type == CURLINFO_DATA_IN && |
4787 | | CPLTestConfigOption("CPL_CURL_VERBOSE_DATA_IN")) |
4788 | | pszDebugKey = "CURL_INFO_DATA_IN"; |
4789 | | if (!pszDebugKey) |
4790 | | return; |
4791 | | |
4792 | | std::lock_guard l(m_runMutex); |
4793 | | |
4794 | | // If we still have the handle for which there is debug information, stick it on |
4795 | | // the handle info and notify. |
4796 | | for (Handle &h : m_handles) |
4797 | | if (h.m_curl == handle) |
4798 | | { |
4799 | | h.m_debug.emplace_back(std::string(pszDebugKey), msg); |
4800 | | m_runCv.notify_all(); |
4801 | | break; |
4802 | | } |
4803 | | } |
4804 | | |
4805 | | /************************************************************************/ |
4806 | | /* IsAllowedFilename() */ |
4807 | | /************************************************************************/ |
4808 | | |
4809 | | bool VSICurlFilesystemHandlerBase::IsAllowedFilename(const char *pszFilename) |
4810 | | { |
4811 | | const char *pszAllowedFilename = |
4812 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_FILENAME", nullptr); |
4813 | | if (pszAllowedFilename != nullptr) |
4814 | | { |
4815 | | return strcmp(pszFilename, pszAllowedFilename) == 0; |
4816 | | } |
4817 | | |
4818 | | // Consider that only the files whose extension ends up with one that is |
4819 | | // listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server. This can |
4820 | | // speeds up dramatically open experience, in case the server cannot return |
4821 | | // a file list. {noext} can be used as a special token to mean file with no |
4822 | | // extension. |
4823 | | // For example: |
4824 | | // gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" |
4825 | | // /vsicurl/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif |
4826 | | const char *pszAllowedExtensions = |
4827 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", nullptr); |
4828 | | if (pszAllowedExtensions) |
4829 | | { |
4830 | | char **papszExtensions = |
4831 | | CSLTokenizeString2(pszAllowedExtensions, ", ", 0); |
4832 | | const char *queryStart = strchr(pszFilename, '?'); |
4833 | | char *pszFilenameWithoutQuery = nullptr; |
4834 | | if (queryStart != nullptr) |
4835 | | { |
4836 | | pszFilenameWithoutQuery = CPLStrdup(pszFilename); |
4837 | | pszFilenameWithoutQuery[queryStart - pszFilename] = '\0'; |
4838 | | pszFilename = pszFilenameWithoutQuery; |
4839 | | } |
4840 | | const size_t nURLLen = strlen(pszFilename); |
4841 | | bool bFound = false; |
4842 | | for (int i = 0; papszExtensions[i] != nullptr; i++) |
4843 | | { |
4844 | | const size_t nExtensionLen = strlen(papszExtensions[i]); |
4845 | | if (EQUAL(papszExtensions[i], "{noext}")) |
4846 | | { |
4847 | | const char *pszLastSlash = strrchr(pszFilename, '/'); |
4848 | | if (pszLastSlash != nullptr && |
4849 | | strchr(pszLastSlash, '.') == nullptr) |
4850 | | { |
4851 | | bFound = true; |
4852 | | break; |
4853 | | } |
4854 | | } |
4855 | | else if (nURLLen > nExtensionLen && |
4856 | | EQUAL(pszFilename + nURLLen - nExtensionLen, |
4857 | | papszExtensions[i])) |
4858 | | { |
4859 | | bFound = true; |
4860 | | break; |
4861 | | } |
4862 | | } |
4863 | | |
4864 | | CSLDestroy(papszExtensions); |
4865 | | if (pszFilenameWithoutQuery) |
4866 | | { |
4867 | | CPLFree(pszFilenameWithoutQuery); |
4868 | | } |
4869 | | |
4870 | | return bFound; |
4871 | | } |
4872 | | return TRUE; |
4873 | | } |
4874 | | |
4875 | | /************************************************************************/ |
4876 | | /* Open() */ |
4877 | | /************************************************************************/ |
4878 | | |
4879 | | VSIVirtualHandleUniquePtr |
4880 | | VSICurlFilesystemHandlerBase::Open(const char *pszFilename, |
4881 | | const char *pszAccess, bool bSetError, |
4882 | | CSLConstList papszOptions) |
4883 | | { |
4884 | | const bool bStartsWithVSICurlPrefix = StartsWithVSICurlPrefix(pszFilename); |
4885 | | if (!bStartsWithVSICurlPrefix && |
4886 | | !cpl::starts_with(std::string_view(pszFilename), GetFSPrefix())) |
4887 | | { |
4888 | | return nullptr; |
4889 | | } |
4890 | | |
4891 | | if (strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, '+') != nullptr) |
4892 | | { |
4893 | | if (bSetError) |
4894 | | { |
4895 | | VSIError(VSIE_FileError, |
4896 | | "Only read-only mode is supported for /vsicurl"); |
4897 | | } |
4898 | | return nullptr; |
4899 | | } |
4900 | | if (!papszOptions || |
4901 | | !CPLTestBool(CSLFetchNameValueDef( |
4902 | | papszOptions, "IGNORE_FILENAME_RESTRICTIONS", "NO"))) |
4903 | | { |
4904 | | if (!IsAllowedFilename(pszFilename)) |
4905 | | return nullptr; |
4906 | | } |
4907 | | |
4908 | | bool bListDir = true; |
4909 | | bool bEmptyDir = false; |
4910 | | std::string osURL = |
4911 | | bStartsWithVSICurlPrefix |
4912 | | ? VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, nullptr, |
4913 | | &bListDir, &bEmptyDir, nullptr, nullptr, |
4914 | | nullptr) |
4915 | | : GetURLFromFilename(pszFilename); |
4916 | | |
4917 | | const char *pszOptionVal = CSLFetchNameValueDef( |
4918 | | papszOptions, "DISABLE_READDIR_ON_OPEN", |
4919 | | VSIGetPathSpecificOption(pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", |
4920 | | "NO")); |
4921 | | const bool bCache = CPLTestBool(CSLFetchNameValueDef( |
4922 | | papszOptions, "CACHE", AllowCachedDataFor(pszFilename) ? "YES" : "NO")); |
4923 | | const bool bSkipReadDir = !bListDir || bEmptyDir || |
4924 | | EQUAL(pszOptionVal, "EMPTY_DIR") || |
4925 | | CPLTestBool(pszOptionVal) || !bCache; |
4926 | | |
4927 | | std::string osFilename(pszFilename); |
4928 | | bool bGotFileList = !bSkipReadDir; |
4929 | | bool bForceExistsCheck = false; |
4930 | | FileProp cachedFileProp; |
4931 | | if (!bSkipReadDir && |
4932 | | !(GetCachedFileProp(osURL.c_str(), cachedFileProp) && |
4933 | | cachedFileProp.eExists == EXIST_YES) && |
4934 | | strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
4935 | | !STARTS_WITH(CPLGetExtensionSafe(osFilename.c_str()).c_str(), "zip") && |
4936 | | // Likely a Kerchunk JSON reference file: no need to list siblings |
4937 | | !cpl::ends_with(osFilename, ".nc.zarr")) |
4938 | | { |
4939 | | // 1000 corresponds to the default page size of S3. |
4940 | | constexpr int FILE_COUNT_LIMIT = 1000; |
4941 | | const CPLStringList aosFileList(ReadDirInternal( |
4942 | | (CPLGetDirnameSafe(osFilename.c_str()) + '/').c_str(), |
4943 | | FILE_COUNT_LIMIT, &bGotFileList)); |
4944 | | const bool bFound = |
4945 | | VSICurlIsFileInList(aosFileList.List(), |
4946 | | CPLGetFilename(osFilename.c_str())) != -1; |
4947 | | if (bGotFileList && !bFound && aosFileList.size() < FILE_COUNT_LIMIT) |
4948 | | { |
4949 | | // Some file servers are case insensitive, so in case there is a |
4950 | | // match with case difference, do a full check just in case. |
4951 | | // e.g. |
4952 | | // http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/MEGA90N000CB.IMG |
4953 | | // that is queried by |
4954 | | // gdalinfo |
4955 | | // /vsicurl/http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/mega90n000cb.lbl |
4956 | | if (aosFileList.FindString(CPLGetFilename(osFilename.c_str())) != |
4957 | | -1) |
4958 | | { |
4959 | | bForceExistsCheck = true; |
4960 | | } |
4961 | | else |
4962 | | { |
4963 | | return nullptr; |
4964 | | } |
4965 | | } |
4966 | | } |
4967 | | if (!bStartsWithVSICurlPrefix) |
4968 | | osURL = GetURLFromFilename(pszFilename); |
4969 | | if (GetCachedFileProp(osURL.c_str(), cachedFileProp) && |
4970 | | cachedFileProp.eExists == EXIST_YES && cachedFileProp.bIsDirectory) |
4971 | | { |
4972 | | return nullptr; |
4973 | | } |
4974 | | |
4975 | | auto poHandle = |
4976 | | std::unique_ptr<VSICurlHandle>(CreateFileHandle(osFilename.c_str())); |
4977 | | if (poHandle == nullptr) |
4978 | | return nullptr; |
4979 | | poHandle->SetCache(bCache); |
4980 | | if (!bGotFileList || bForceExistsCheck) |
4981 | | { |
4982 | | // If we didn't get a filelist, check that the file really exists. |
4983 | | if (!poHandle->Exists(bSetError)) |
4984 | | { |
4985 | | return nullptr; |
4986 | | } |
4987 | | } |
4988 | | |
4989 | | if (CPLTestBool(CPLGetConfigOption("VSI_CACHE", "FALSE"))) |
4990 | | return VSIVirtualHandleUniquePtr( |
4991 | | VSICreateCachedFile(poHandle.release())); |
4992 | | else |
4993 | | return VSIVirtualHandleUniquePtr(poHandle.release()); |
4994 | | } |
4995 | | |
4996 | | /************************************************************************/ |
4997 | | /* VSICurlParserFindEOL() */ |
4998 | | /* */ |
4999 | | /* Small helper function for VSICurlPaseHTMLFileList() to find */ |
5000 | | /* the end of a line in the directory listing. Either a <br> */ |
5001 | | /* or newline. */ |
5002 | | /************************************************************************/ |
5003 | | |
5004 | | static char *VSICurlParserFindEOL(char *pszData) |
5005 | | |
5006 | | { |
5007 | | while (*pszData != '\0' && *pszData != '\n' && |
5008 | | !STARTS_WITH_CI(pszData, "<br>")) |
5009 | | pszData++; |
5010 | | |
5011 | | if (*pszData == '\0') |
5012 | | return nullptr; |
5013 | | |
5014 | | return pszData; |
5015 | | } |
5016 | | |
5017 | | /************************************************************************/ |
5018 | | /* ParseFileSize() */ |
5019 | | /************************************************************************/ |
5020 | | |
5021 | | static GUIntBig ParseFileSize(const char *pszStr) |
5022 | | { |
5023 | | GUIntBig nFileSize = 0; |
5024 | | while (*pszStr == ' ') |
5025 | | pszStr++; |
5026 | | if (*pszStr >= '1' && *pszStr <= '9') |
5027 | | { |
5028 | | const char *pszIter = pszStr + 1; |
5029 | | while (*pszIter >= '0' && *pszIter <= '9') |
5030 | | ++pszIter; |
5031 | | if (*pszIter == 0 || *pszIter == ' ' || *pszIter == '\t' || |
5032 | | *pszIter == '\r' || *pszIter == '\n') |
5033 | | { |
5034 | | nFileSize = |
5035 | | CPLScanUIntBig(pszStr, static_cast<int>(pszIter - pszStr)); |
5036 | | } |
5037 | | } |
5038 | | return nFileSize; |
5039 | | } |
5040 | | |
5041 | | /************************************************************************/ |
5042 | | /* VSICurlParseHTMLDateTimeFileSize() */ |
5043 | | /************************************************************************/ |
5044 | | |
5045 | | static const char *const apszMonths[] = { |
5046 | | "January", "February", "March", "April", "May", "June", |
5047 | | "July", "August", "September", "October", "November", "December"}; |
5048 | | |
5049 | | static bool VSICurlParseHTMLDateTimeFileSize(const char *pszStr, |
5050 | | struct tm &brokendowntime, |
5051 | | GUIntBig &nFileSize, |
5052 | | GIntBig &mTime) |
5053 | | { |
5054 | | for (int iMonth = 0; iMonth < 12; iMonth++) |
5055 | | { |
5056 | | nFileSize = 0; |
5057 | | |
5058 | | char szMonth[32] = {}; |
5059 | | szMonth[0] = '-'; |
5060 | | memcpy(szMonth + 1, apszMonths[iMonth], 3); |
5061 | | szMonth[4] = '-'; |
5062 | | szMonth[5] = '\0'; |
5063 | | const char *pszMonthFound = strstr(pszStr, szMonth); |
5064 | | if (pszMonthFound) |
5065 | | { |
5066 | | // Format of Apache, like in |
5067 | | // http://download.osgeo.org/gdal/data/gtiff/ |
5068 | | // "17-May-2010 12:26" |
5069 | | const auto nMonthFoundLen = strlen(pszMonthFound); |
5070 | | if (pszMonthFound - pszStr > 2 && nMonthFoundLen > 15 && |
5071 | | pszMonthFound[-2 + 11] == ' ' && pszMonthFound[-2 + 14] == ':') |
5072 | | { |
5073 | | pszMonthFound -= 2; |
5074 | | int nDay = atoi(pszMonthFound); |
5075 | | int nYear = atoi(pszMonthFound + 7); |
5076 | | int nHour = atoi(pszMonthFound + 12); |
5077 | | int nMin = atoi(pszMonthFound + 15); |
5078 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
5079 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
5080 | | { |
5081 | | brokendowntime.tm_year = nYear - 1900; |
5082 | | brokendowntime.tm_mon = iMonth; |
5083 | | brokendowntime.tm_mday = nDay; |
5084 | | brokendowntime.tm_hour = nHour; |
5085 | | brokendowntime.tm_min = nMin; |
5086 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
5087 | | |
5088 | | if (nMonthFoundLen > 15 + 2) |
5089 | | { |
5090 | | const char *pszFilesize = pszMonthFound + 15 + 2; |
5091 | | nFileSize = ParseFileSize(pszFilesize); |
5092 | | } |
5093 | | } |
5094 | | } |
5095 | | return nFileSize > 0; |
5096 | | } |
5097 | | |
5098 | | /* Microsoft IIS */ |
5099 | | snprintf(szMonth, sizeof(szMonth), " %s ", apszMonths[iMonth]); |
5100 | | pszMonthFound = strstr(pszStr, szMonth); |
5101 | | if (pszMonthFound) |
5102 | | { |
5103 | | int nLenMonth = static_cast<int>(strlen(apszMonths[iMonth])); |
5104 | | if (pszMonthFound - pszStr > 2 && pszMonthFound[-1] != ',' && |
5105 | | pszMonthFound[-2] != ' ' && |
5106 | | static_cast<int>(strlen(pszMonthFound - 2)) > |
5107 | | 2 + 1 + nLenMonth + 1 + 4 + 1 + 5 + 1 + 4) |
5108 | | { |
5109 | | /* Format of http://ortho.linz.govt.nz/tifs/1994_95/ */ |
5110 | | /* " Friday, 21 April 2006 12:05 p.m. 48062343 |
5111 | | * m35a_fy_94_95.tif" */ |
5112 | | pszMonthFound -= 2; |
5113 | | int nDay = atoi(pszMonthFound); |
5114 | | int nCurOffset = 2 + 1 + nLenMonth + 1; |
5115 | | int nYear = atoi(pszMonthFound + nCurOffset); |
5116 | | nCurOffset += 4 + 1; |
5117 | | int nHour = atoi(pszMonthFound + nCurOffset); |
5118 | | if (nHour < 10) |
5119 | | nCurOffset += 1 + 1; |
5120 | | else |
5121 | | nCurOffset += 2 + 1; |
5122 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
5123 | | nCurOffset += 2 + 1; |
5124 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "p.m.")) |
5125 | | nHour += 12; |
5126 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "a.m.")) |
5127 | | nHour = -1; |
5128 | | nCurOffset += 4; |
5129 | | |
5130 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
5131 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
5132 | | { |
5133 | | brokendowntime.tm_year = nYear - 1900; |
5134 | | brokendowntime.tm_mon = iMonth; |
5135 | | brokendowntime.tm_mday = nDay; |
5136 | | brokendowntime.tm_hour = nHour; |
5137 | | brokendowntime.tm_min = nMin; |
5138 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
5139 | | |
5140 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
5141 | | nFileSize = ParseFileSize(pszFilesize); |
5142 | | } |
5143 | | } |
5144 | | else if (pszMonthFound - pszStr > 1 && pszMonthFound[-1] == ',' && |
5145 | | static_cast<int>(strlen(pszMonthFound)) > |
5146 | | 1 + nLenMonth + 1 + 2 + 1 + 1 + 4 + 1 + 5 + 1 + 2) |
5147 | | { |
5148 | | // Format of |
5149 | | // http://publicfiles.dep.state.fl.us/dear/BWR_GIS/2007NWFLULC/ |
5150 | | // " Sunday, June 20, 2010 6:46 PM 233170905 |
5151 | | // NWF2007LULCForSDE.zip" |
5152 | | pszMonthFound += 1; |
5153 | | int nCurOffset = nLenMonth + 1; |
5154 | | int nDay = atoi(pszMonthFound + nCurOffset); |
5155 | | nCurOffset += 2 + 1 + 1; |
5156 | | int nYear = atoi(pszMonthFound + nCurOffset); |
5157 | | nCurOffset += 4 + 1; |
5158 | | int nHour = atoi(pszMonthFound + nCurOffset); |
5159 | | nCurOffset += 2 + 1; |
5160 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
5161 | | nCurOffset += 2 + 1; |
5162 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "PM")) |
5163 | | nHour += 12; |
5164 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "AM")) |
5165 | | nHour = -1; |
5166 | | nCurOffset += 2; |
5167 | | |
5168 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
5169 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
5170 | | { |
5171 | | brokendowntime.tm_year = nYear - 1900; |
5172 | | brokendowntime.tm_mon = iMonth; |
5173 | | brokendowntime.tm_mday = nDay; |
5174 | | brokendowntime.tm_hour = nHour; |
5175 | | brokendowntime.tm_min = nMin; |
5176 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
5177 | | |
5178 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
5179 | | nFileSize = ParseFileSize(pszFilesize); |
5180 | | } |
5181 | | } |
5182 | | |
5183 | | return nFileSize > 0; |
5184 | | } |
5185 | | } |
5186 | | |
5187 | | return false; |
5188 | | } |
5189 | | |
5190 | | /************************************************************************/ |
5191 | | /* ParseHTMLFileList() */ |
5192 | | /* */ |
5193 | | /* Parse a file list document and return all the components. */ |
5194 | | /************************************************************************/ |
5195 | | |
5196 | | char **VSICurlFilesystemHandlerBase::ParseHTMLFileList(const char *pszFilename, |
5197 | | int nMaxFiles, |
5198 | | char *pszData, |
5199 | | bool *pbGotFileList) |
5200 | | { |
5201 | | *pbGotFileList = false; |
5202 | | |
5203 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
5204 | | nullptr, nullptr, nullptr, |
5205 | | nullptr, nullptr, nullptr)); |
5206 | | const char *pszDir = nullptr; |
5207 | | if (STARTS_WITH_CI(osURL.c_str(), "http://")) |
5208 | | pszDir = strchr(osURL.c_str() + strlen("http://"), '/'); |
5209 | | else if (STARTS_WITH_CI(osURL.c_str(), "https://")) |
5210 | | pszDir = strchr(osURL.c_str() + strlen("https://"), '/'); |
5211 | | else if (STARTS_WITH_CI(osURL.c_str(), "ftp://")) |
5212 | | pszDir = strchr(osURL.c_str() + strlen("ftp://"), '/'); |
5213 | | if (pszDir == nullptr) |
5214 | | pszDir = ""; |
5215 | | |
5216 | | /* Apache / Nginx */ |
5217 | | /* Most of the time the format is <title>Index of {pszDir[/]}</title>, but |
5218 | | * there are special cases like https://cdn.star.nesdis.noaa.gov/GOES18/ABI/MESO/M1/GEOCOLOR/ |
5219 | | * where a CDN stuff makes that the title is <title>Index of /ma-cdn02/GOES/data/GOES18/ABI/MESO/M1/GEOCOLOR/</title> |
5220 | | */ |
5221 | | const std::string osTitleIndexOfPrefix = "<title>Index of "; |
5222 | | const std::string osExpectedSuffix = std::string(pszDir).append("</title>"); |
5223 | | const std::string osExpectedSuffixWithSlash = |
5224 | | std::string(pszDir).append("/</title>"); |
5225 | | /* FTP */ |
5226 | | const std::string osExpectedStringFTP = |
5227 | | std::string("FTP Listing of ").append(pszDir).append("/"); |
5228 | | /* Apache 1.3.33 */ |
5229 | | const std::string osExpectedStringOldApache = |
5230 | | std::string("<TITLE>Index of ").append(pszDir).append("</TITLE>"); |
5231 | | |
5232 | | // The listing of |
5233 | | // http://dds.cr.usgs.gov/srtm/SRTM_image_sample/picture%20examples/ |
5234 | | // has |
5235 | | // "<title>Index of /srtm/SRTM_image_sample/picture examples</title>" |
5236 | | // so we must try unescaped %20 also. |
5237 | | // Similar with |
5238 | | // http://datalib.usask.ca/gis/Data/Central_America_goodbutdoweown%3f/ |
5239 | | std::string osExpectedString_unescaped; |
5240 | | if (strchr(pszDir, '%')) |
5241 | | { |
5242 | | char *pszUnescapedDir = CPLUnescapeString(pszDir, nullptr, CPLES_URL); |
5243 | | osExpectedString_unescaped = osTitleIndexOfPrefix; |
5244 | | osExpectedString_unescaped += pszUnescapedDir; |
5245 | | osExpectedString_unescaped += "</title>"; |
5246 | | CPLFree(pszUnescapedDir); |
5247 | | } |
5248 | | |
5249 | | char *c = nullptr; |
5250 | | int nCount = 0; |
5251 | | int nCountTable = 0; |
5252 | | CPLStringList oFileList; |
5253 | | char *pszLine = pszData; |
5254 | | bool bIsHTMLDirList = false; |
5255 | | |
5256 | | while ((c = VSICurlParserFindEOL(pszLine)) != nullptr) |
5257 | | { |
5258 | | *c = '\0'; |
5259 | | |
5260 | | // To avoid false positive on pages such as |
5261 | | // http://www.ngs.noaa.gov/PC_PROD/USGG2009BETA |
5262 | | // This is a heuristics, but normal HTML listing of files have not more |
5263 | | // than one table. |
5264 | | if (strstr(pszLine, "<table")) |
5265 | | { |
5266 | | nCountTable++; |
5267 | | if (nCountTable == 2) |
5268 | | { |
5269 | | *pbGotFileList = false; |
5270 | | return nullptr; |
5271 | | } |
5272 | | } |
5273 | | |
5274 | | if (!bIsHTMLDirList && |
5275 | | ((strstr(pszLine, osTitleIndexOfPrefix.c_str()) && |
5276 | | (strstr(pszLine, osExpectedSuffix.c_str()) || |
5277 | | strstr(pszLine, osExpectedSuffixWithSlash.c_str()))) || |
5278 | | strstr(pszLine, osExpectedStringFTP.c_str()) || |
5279 | | strstr(pszLine, osExpectedStringOldApache.c_str()) || |
5280 | | (!osExpectedString_unescaped.empty() && |
5281 | | strstr(pszLine, osExpectedString_unescaped.c_str())))) |
5282 | | { |
5283 | | bIsHTMLDirList = true; |
5284 | | *pbGotFileList = true; |
5285 | | } |
5286 | | // Subversion HTTP listing |
5287 | | // or Microsoft-IIS/6.0 listing |
5288 | | // (e.g. http://ortho.linz.govt.nz/tifs/2005_06/) */ |
5289 | | else if (!bIsHTMLDirList && strstr(pszLine, "<title>")) |
5290 | | { |
5291 | | // Detect something like: |
5292 | | // <html><head><title>gdal - Revision 20739: |
5293 | | // /trunk/autotest/gcore/data</title></head> */ The annoying thing |
5294 | | // is that what is after ': ' is a subpart of what is after |
5295 | | // http://server/ |
5296 | | char *pszSubDir = strstr(pszLine, ": "); |
5297 | | if (pszSubDir == nullptr) |
5298 | | // or <title>ortho.linz.govt.nz - /tifs/2005_06/</title> |
5299 | | pszSubDir = strstr(pszLine, "- "); |
5300 | | if (pszSubDir) |
5301 | | { |
5302 | | pszSubDir += 2; |
5303 | | char *pszTmp = strstr(pszSubDir, "</title>"); |
5304 | | if (pszTmp) |
5305 | | { |
5306 | | if (pszTmp[-1] == '/') |
5307 | | pszTmp[-1] = 0; |
5308 | | else |
5309 | | *pszTmp = 0; |
5310 | | if (strstr(pszDir, pszSubDir)) |
5311 | | { |
5312 | | bIsHTMLDirList = true; |
5313 | | *pbGotFileList = true; |
5314 | | } |
5315 | | } |
5316 | | } |
5317 | | } |
5318 | | else if (bIsHTMLDirList && |
5319 | | (strstr(pszLine, "<a href=\"") != nullptr || |
5320 | | strstr(pszLine, "<A HREF=\"") != nullptr) && |
5321 | | // Exclude absolute links, like to subversion home. |
5322 | | strstr(pszLine, "<a href=\"http://") == nullptr && |
5323 | | // exclude parent directory. |
5324 | | strstr(pszLine, "Parent Directory") == nullptr) |
5325 | | { |
5326 | | char *beginFilename = strstr(pszLine, "<a href=\""); |
5327 | | if (beginFilename == nullptr) |
5328 | | beginFilename = strstr(pszLine, "<A HREF=\""); |
5329 | | beginFilename += strlen("<a href=\""); |
5330 | | char *endQuote = strchr(beginFilename, '"'); |
5331 | | if (endQuote && !STARTS_WITH(beginFilename, "?C=") && |
5332 | | !STARTS_WITH(beginFilename, "?N=")) |
5333 | | { |
5334 | | struct tm brokendowntime; |
5335 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
5336 | | GUIntBig nFileSize = 0; |
5337 | | GIntBig mTime = 0; |
5338 | | |
5339 | | VSICurlParseHTMLDateTimeFileSize(pszLine, brokendowntime, |
5340 | | nFileSize, mTime); |
5341 | | |
5342 | | *endQuote = '\0'; |
5343 | | |
5344 | | // Remove trailing slash, that are returned for directories by |
5345 | | // Apache. |
5346 | | bool bIsDirectory = false; |
5347 | | if (endQuote[-1] == '/') |
5348 | | { |
5349 | | bIsDirectory = true; |
5350 | | endQuote[-1] = 0; |
5351 | | } |
5352 | | |
5353 | | // shttpd links include slashes from the root directory. |
5354 | | // Skip them. |
5355 | | while (strchr(beginFilename, '/')) |
5356 | | beginFilename = strchr(beginFilename, '/') + 1; |
5357 | | |
5358 | | if (strcmp(beginFilename, ".") != 0 && |
5359 | | strcmp(beginFilename, "..") != 0) |
5360 | | { |
5361 | | std::string osCachedFilename = |
5362 | | CPLSPrintf("%s/%s", osURL.c_str(), beginFilename); |
5363 | | |
5364 | | FileProp cachedFileProp; |
5365 | | GetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
5366 | | cachedFileProp.eExists = EXIST_YES; |
5367 | | cachedFileProp.bIsDirectory = bIsDirectory; |
5368 | | if (mTime > 0) |
5369 | | { |
5370 | | cachedFileProp.mTime = static_cast<time_t>(mTime); |
5371 | | } |
5372 | | if (!cachedFileProp.bHasComputedFileSize) |
5373 | | { |
5374 | | cachedFileProp.bHasComputedFileSize = nFileSize > 0; |
5375 | | cachedFileProp.fileSize = nFileSize; |
5376 | | } |
5377 | | SetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
5378 | | |
5379 | | oFileList.AddString(beginFilename); |
5380 | | if constexpr (ENABLE_DEBUG_VERBOSE) |
5381 | | { |
5382 | | CPLDebug( |
5383 | | GetDebugKey(), |
5384 | | "File[%d] = %s, is_dir = %d, size = " CPL_FRMT_GUIB |
5385 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
5386 | | nCount, osCachedFilename.c_str(), |
5387 | | bIsDirectory ? 1 : 0, nFileSize, |
5388 | | brokendowntime.tm_year + 1900, |
5389 | | brokendowntime.tm_mon + 1, brokendowntime.tm_mday, |
5390 | | brokendowntime.tm_hour, brokendowntime.tm_min, |
5391 | | brokendowntime.tm_sec); |
5392 | | } |
5393 | | nCount++; |
5394 | | |
5395 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
5396 | | break; |
5397 | | } |
5398 | | } |
5399 | | } |
5400 | | pszLine = c + 1; |
5401 | | } |
5402 | | |
5403 | | return oFileList.StealList(); |
5404 | | } |
5405 | | |
5406 | | /************************************************************************/ |
5407 | | /* GetStreamingFilename() */ |
5408 | | /************************************************************************/ |
5409 | | |
5410 | | std::string VSICurlFilesystemHandler::GetStreamingFilename( |
5411 | | const std::string &osFilename) const |
5412 | | { |
5413 | | if (STARTS_WITH(osFilename.c_str(), GetFSPrefix().c_str())) |
5414 | | return "/vsicurl_streaming/" + osFilename.substr(GetFSPrefix().size()); |
5415 | | return osFilename; |
5416 | | } |
5417 | | |
5418 | | /************************************************************************/ |
5419 | | /* GetHintForPotentiallyRecognizedPath() */ |
5420 | | /************************************************************************/ |
5421 | | |
5422 | | std::string VSICurlFilesystemHandler::GetHintForPotentiallyRecognizedPath( |
5423 | | const std::string &osPath) |
5424 | | { |
5425 | | if (!StartsWithVSICurlPrefix(osPath.c_str()) && |
5426 | | !cpl::starts_with(osPath, GetStreamingFilename(GetFSPrefix()))) |
5427 | | { |
5428 | | for (const char *pszPrefix : {"http://", "https://"}) |
5429 | | { |
5430 | | if (cpl::starts_with(osPath, pszPrefix)) |
5431 | | { |
5432 | | return GetFSPrefix() + osPath; |
5433 | | } |
5434 | | } |
5435 | | } |
5436 | | return std::string(); |
5437 | | } |
5438 | | |
5439 | | /************************************************************************/ |
5440 | | /* VSICurlGetToken() */ |
5441 | | /************************************************************************/ |
5442 | | |
5443 | | static char *VSICurlGetToken(char *pszCurPtr, char **ppszNextToken) |
5444 | | { |
5445 | | if (pszCurPtr == nullptr) |
5446 | | return nullptr; |
5447 | | |
5448 | | while ((*pszCurPtr) == ' ') |
5449 | | pszCurPtr++; |
5450 | | if (*pszCurPtr == '\0') |
5451 | | return nullptr; |
5452 | | |
5453 | | char *pszToken = pszCurPtr; |
5454 | | while ((*pszCurPtr) != ' ' && (*pszCurPtr) != '\0') |
5455 | | pszCurPtr++; |
5456 | | if (*pszCurPtr == '\0') |
5457 | | { |
5458 | | *ppszNextToken = nullptr; |
5459 | | } |
5460 | | else |
5461 | | { |
5462 | | *pszCurPtr = '\0'; |
5463 | | pszCurPtr++; |
5464 | | while ((*pszCurPtr) == ' ') |
5465 | | pszCurPtr++; |
5466 | | *ppszNextToken = pszCurPtr; |
5467 | | } |
5468 | | |
5469 | | return pszToken; |
5470 | | } |
5471 | | |
5472 | | /************************************************************************/ |
5473 | | /* VSICurlParseFullFTPLine() */ |
5474 | | /************************************************************************/ |
5475 | | |
5476 | | /* Parse lines like the following ones : |
5477 | | -rw-r--r-- 1 10003 100 430 Jul 04 2008 COPYING |
5478 | | lrwxrwxrwx 1 ftp ftp 28 Jun 14 14:13 MPlayer -> |
5479 | | mirrors/mplayerhq.hu/MPlayer -rw-r--r-- 1 ftp ftp 725614592 May 13 |
5480 | | 20:13 Fedora-15-x86_64-Live-KDE.iso drwxr-xr-x 280 1003 1003 6656 Aug 26 |
5481 | | 04:17 gnu |
5482 | | */ |
5483 | | |
5484 | | static bool VSICurlParseFullFTPLine(char *pszLine, char *&pszFilename, |
5485 | | bool &bSizeValid, GUIntBig &nSize, |
5486 | | bool &bIsDirectory, GIntBig &nUnixTime) |
5487 | | { |
5488 | | char *pszNextToken = pszLine; |
5489 | | char *pszPermissions = VSICurlGetToken(pszNextToken, &pszNextToken); |
5490 | | if (pszPermissions == nullptr || strlen(pszPermissions) != 10) |
5491 | | return false; |
5492 | | bIsDirectory = pszPermissions[0] == 'd'; |
5493 | | |
5494 | | for (int i = 0; i < 3; i++) |
5495 | | { |
5496 | | if (VSICurlGetToken(pszNextToken, &pszNextToken) == nullptr) |
5497 | | return false; |
5498 | | } |
5499 | | |
5500 | | char *pszSize = VSICurlGetToken(pszNextToken, &pszNextToken); |
5501 | | if (pszSize == nullptr) |
5502 | | return false; |
5503 | | |
5504 | | if (pszPermissions[0] == '-') |
5505 | | { |
5506 | | // Regular file. |
5507 | | bSizeValid = true; |
5508 | | nSize = CPLScanUIntBig(pszSize, static_cast<int>(strlen(pszSize))); |
5509 | | } |
5510 | | |
5511 | | struct tm brokendowntime; |
5512 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
5513 | | bool bBrokenDownTimeValid = true; |
5514 | | |
5515 | | char *pszMonth = VSICurlGetToken(pszNextToken, &pszNextToken); |
5516 | | if (pszMonth == nullptr || strlen(pszMonth) != 3) |
5517 | | return false; |
5518 | | |
5519 | | int i = 0; // Used after for. |
5520 | | for (; i < 12; i++) |
5521 | | { |
5522 | | if (EQUALN(pszMonth, apszMonths[i], 3)) |
5523 | | break; |
5524 | | } |
5525 | | if (i < 12) |
5526 | | brokendowntime.tm_mon = i; |
5527 | | else |
5528 | | bBrokenDownTimeValid = false; |
5529 | | |
5530 | | char *pszDay = VSICurlGetToken(pszNextToken, &pszNextToken); |
5531 | | if (pszDay == nullptr || (strlen(pszDay) != 1 && strlen(pszDay) != 2)) |
5532 | | return false; |
5533 | | int nDay = atoi(pszDay); |
5534 | | if (nDay >= 1 && nDay <= 31) |
5535 | | brokendowntime.tm_mday = nDay; |
5536 | | else |
5537 | | bBrokenDownTimeValid = false; |
5538 | | |
5539 | | char *pszHourOrYear = VSICurlGetToken(pszNextToken, &pszNextToken); |
5540 | | if (pszHourOrYear == nullptr || |
5541 | | (strlen(pszHourOrYear) != 4 && strlen(pszHourOrYear) != 5)) |
5542 | | return false; |
5543 | | if (strlen(pszHourOrYear) == 4) |
5544 | | { |
5545 | | brokendowntime.tm_year = atoi(pszHourOrYear) - 1900; |
5546 | | } |
5547 | | else |
5548 | | { |
5549 | | time_t sTime; |
5550 | | time(&sTime); |
5551 | | struct tm currentBrokendowntime; |
5552 | | CPLUnixTimeToYMDHMS(static_cast<GIntBig>(sTime), |
5553 | | ¤tBrokendowntime); |
5554 | | brokendowntime.tm_year = currentBrokendowntime.tm_year; |
5555 | | brokendowntime.tm_hour = atoi(pszHourOrYear); |
5556 | | brokendowntime.tm_min = atoi(pszHourOrYear + 3); |
5557 | | } |
5558 | | |
5559 | | if (bBrokenDownTimeValid) |
5560 | | nUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
5561 | | else |
5562 | | nUnixTime = 0; |
5563 | | |
5564 | | if (pszNextToken == nullptr) |
5565 | | return false; |
5566 | | |
5567 | | pszFilename = pszNextToken; |
5568 | | |
5569 | | char *pszCurPtr = pszFilename; |
5570 | | while (*pszCurPtr != '\0') |
5571 | | { |
5572 | | // In case of a link, stop before the pointed part of the link. |
5573 | | if (pszPermissions[0] == 'l' && STARTS_WITH(pszCurPtr, " -> ")) |
5574 | | { |
5575 | | break; |
5576 | | } |
5577 | | pszCurPtr++; |
5578 | | } |
5579 | | *pszCurPtr = '\0'; |
5580 | | |
5581 | | return true; |
5582 | | } |
5583 | | |
5584 | | /************************************************************************/ |
5585 | | /* GetURLFromFilename() */ |
5586 | | /************************************************************************/ |
5587 | | |
5588 | | std::string VSICurlFilesystemHandlerBase::GetURLFromFilename( |
5589 | | const std::string &osFilename) const |
5590 | | { |
5591 | | return VSICurlGetURLFromFilename(osFilename.c_str(), nullptr, nullptr, |
5592 | | nullptr, nullptr, nullptr, nullptr, |
5593 | | nullptr, nullptr); |
5594 | | } |
5595 | | |
5596 | | /************************************************************************/ |
5597 | | /* RegisterEmptyDir() */ |
5598 | | /************************************************************************/ |
5599 | | |
5600 | | void VSICurlFilesystemHandlerBase::RegisterEmptyDir( |
5601 | | const std::string &osDirname) |
5602 | | { |
5603 | | CachedDirList cachedDirList; |
5604 | | cachedDirList.bGotFileList = true; |
5605 | | cachedDirList.oFileList.AddString("."); |
5606 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5607 | | } |
5608 | | |
5609 | | /************************************************************************/ |
5610 | | /* GetFileList() */ |
5611 | | /************************************************************************/ |
5612 | | |
5613 | | char **VSICurlFilesystemHandlerBase::GetFileList(const char *pszDirname, |
5614 | | int nMaxFiles, |
5615 | | bool *pbGotFileList) |
5616 | | { |
5617 | | if constexpr (ENABLE_DEBUG) |
5618 | | { |
5619 | | CPLDebug(GetDebugKey(), "GetFileList(%s)", pszDirname); |
5620 | | } |
5621 | | |
5622 | | *pbGotFileList = false; |
5623 | | |
5624 | | bool bListDir = true; |
5625 | | bool bEmptyDir = false; |
5626 | | std::string osURL(VSICurlGetURLFromFilename(pszDirname, nullptr, nullptr, |
5627 | | nullptr, &bListDir, &bEmptyDir, |
5628 | | nullptr, nullptr, nullptr)); |
5629 | | if (bEmptyDir) |
5630 | | { |
5631 | | *pbGotFileList = true; |
5632 | | return CSLAddString(nullptr, "."); |
5633 | | } |
5634 | | if (!bListDir) |
5635 | | return nullptr; |
5636 | | |
5637 | | // Deal with publicly visible Azure directories. |
5638 | | if (STARTS_WITH(osURL.c_str(), "https://")) |
5639 | | { |
5640 | | const char *pszBlobCore = |
5641 | | strstr(osURL.c_str(), ".blob.core.windows.net/"); |
5642 | | if (pszBlobCore) |
5643 | | { |
5644 | | FileProp cachedFileProp; |
5645 | | GetCachedFileProp(osURL.c_str(), cachedFileProp); |
5646 | | if (cachedFileProp.bIsAzureFolder) |
5647 | | { |
5648 | | const char *pszURLWithoutHTTPS = |
5649 | | osURL.c_str() + strlen("https://"); |
5650 | | const std::string osStorageAccount( |
5651 | | pszURLWithoutHTTPS, pszBlobCore - pszURLWithoutHTTPS); |
5652 | | CPLConfigOptionSetter oSetter1("AZURE_NO_SIGN_REQUEST", "YES", |
5653 | | false); |
5654 | | CPLConfigOptionSetter oSetter2("AZURE_STORAGE_ACCOUNT", |
5655 | | osStorageAccount.c_str(), false); |
5656 | | const std::string osVSIAZ(std::string("/vsiaz/").append( |
5657 | | pszBlobCore + strlen(".blob.core.windows.net/"))); |
5658 | | char **papszFileList = VSIReadDirEx(osVSIAZ.c_str(), nMaxFiles); |
5659 | | if (papszFileList) |
5660 | | { |
5661 | | *pbGotFileList = true; |
5662 | | return papszFileList; |
5663 | | } |
5664 | | } |
5665 | | } |
5666 | | } |
5667 | | |
5668 | | // HACK (optimization in fact) for MBTiles driver. |
5669 | | if (strstr(pszDirname, ".tiles.mapbox.com") != nullptr) |
5670 | | return nullptr; |
5671 | | |
5672 | | if (STARTS_WITH(osURL.c_str(), "ftp://")) |
5673 | | { |
5674 | | // Start Run thread if necessary. |
5675 | | RunThreadUser threadUser(*this); |
5676 | | |
5677 | | WriteFuncStruct sWriteFuncData; |
5678 | | sWriteFuncData.pBuffer = nullptr; |
5679 | | |
5680 | | std::string osDirname(osURL); |
5681 | | osDirname += '/'; |
5682 | | |
5683 | | char **papszFileList = nullptr; |
5684 | | CURL *hCurlHandle = curl_easy_init(); |
5685 | | |
5686 | | for (int iTry = 0; iTry < 2; iTry++) |
5687 | | { |
5688 | | struct curl_slist *headers = |
5689 | | SetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5690 | | |
5691 | | // On the first pass, we want to try fetching all the possible |
5692 | | // information (filename, file/directory, size). If that does not |
5693 | | // work, then try again with CURLOPT_DIRLISTONLY set. |
5694 | | if (iTry == 1) |
5695 | | { |
5696 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 1); |
5697 | | } |
5698 | | |
5699 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, |
5700 | | nullptr); |
5701 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5702 | | &sWriteFuncData); |
5703 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5704 | | VSICurlHandleWriteFunc); |
5705 | | |
5706 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5707 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5708 | | szCurlErrBuf); |
5709 | | |
5710 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
5711 | | headers); |
5712 | | |
5713 | | Perform(hCurlHandle); |
5714 | | |
5715 | | curl_slist_free_all(headers); |
5716 | | |
5717 | | if (sWriteFuncData.pBuffer == nullptr) |
5718 | | { |
5719 | | curl_easy_cleanup(hCurlHandle); |
5720 | | return nullptr; |
5721 | | } |
5722 | | |
5723 | | char *pszLine = sWriteFuncData.pBuffer; |
5724 | | char *c = nullptr; |
5725 | | int nCount = 0; |
5726 | | |
5727 | | if (STARTS_WITH_CI(pszLine, "<!DOCTYPE HTML") || |
5728 | | STARTS_WITH_CI(pszLine, "<HTML>")) |
5729 | | { |
5730 | | papszFileList = |
5731 | | ParseHTMLFileList(pszDirname, nMaxFiles, |
5732 | | sWriteFuncData.pBuffer, pbGotFileList); |
5733 | | break; |
5734 | | } |
5735 | | else if (iTry == 0) |
5736 | | { |
5737 | | CPLStringList oFileList; |
5738 | | *pbGotFileList = true; |
5739 | | |
5740 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5741 | | { |
5742 | | *c = 0; |
5743 | | if (c - pszLine > 0 && c[-1] == '\r') |
5744 | | c[-1] = 0; |
5745 | | |
5746 | | char *pszFilename = nullptr; |
5747 | | bool bSizeValid = false; |
5748 | | GUIntBig nFileSize = 0; |
5749 | | bool bIsDirectory = false; |
5750 | | GIntBig mUnixTime = 0; |
5751 | | if (!VSICurlParseFullFTPLine(pszLine, pszFilename, |
5752 | | bSizeValid, nFileSize, |
5753 | | bIsDirectory, mUnixTime)) |
5754 | | break; |
5755 | | |
5756 | | if (strcmp(pszFilename, ".") != 0 && |
5757 | | strcmp(pszFilename, "..") != 0) |
5758 | | { |
5759 | | if (CPLHasUnbalancedPathTraversal(pszFilename)) |
5760 | | { |
5761 | | CPLError(CE_Warning, CPLE_AppDefined, |
5762 | | "Ignoring '%s' that has a path traversal " |
5763 | | "pattern", |
5764 | | pszFilename); |
5765 | | } |
5766 | | else |
5767 | | { |
5768 | | std::string osCachedFilename = |
5769 | | CPLSPrintf("%s/%s", osURL.c_str(), pszFilename); |
5770 | | |
5771 | | FileProp cachedFileProp; |
5772 | | GetCachedFileProp(osCachedFilename.c_str(), |
5773 | | cachedFileProp); |
5774 | | cachedFileProp.eExists = EXIST_YES; |
5775 | | cachedFileProp.bIsDirectory = bIsDirectory; |
5776 | | cachedFileProp.mTime = |
5777 | | static_cast<time_t>(mUnixTime); |
5778 | | cachedFileProp.bHasComputedFileSize = bSizeValid; |
5779 | | cachedFileProp.fileSize = nFileSize; |
5780 | | SetCachedFileProp(osCachedFilename.c_str(), |
5781 | | cachedFileProp); |
5782 | | |
5783 | | oFileList.AddString(pszFilename); |
5784 | | if constexpr (ENABLE_DEBUG_VERBOSE) |
5785 | | { |
5786 | | struct tm brokendowntime; |
5787 | | CPLUnixTimeToYMDHMS(mUnixTime, &brokendowntime); |
5788 | | CPLDebug( |
5789 | | GetDebugKey(), |
5790 | | "File[%d] = %s, is_dir = %d, size " |
5791 | | "= " CPL_FRMT_GUIB |
5792 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
5793 | | nCount, pszFilename, bIsDirectory ? 1 : 0, |
5794 | | nFileSize, brokendowntime.tm_year + 1900, |
5795 | | brokendowntime.tm_mon + 1, |
5796 | | brokendowntime.tm_mday, |
5797 | | brokendowntime.tm_hour, |
5798 | | brokendowntime.tm_min, |
5799 | | brokendowntime.tm_sec); |
5800 | | } |
5801 | | |
5802 | | nCount++; |
5803 | | |
5804 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
5805 | | break; |
5806 | | } |
5807 | | } |
5808 | | |
5809 | | pszLine = c + 1; |
5810 | | } |
5811 | | |
5812 | | if (c == nullptr) |
5813 | | { |
5814 | | papszFileList = oFileList.StealList(); |
5815 | | break; |
5816 | | } |
5817 | | } |
5818 | | else |
5819 | | { |
5820 | | CPLStringList oFileList; |
5821 | | *pbGotFileList = true; |
5822 | | |
5823 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5824 | | { |
5825 | | *c = 0; |
5826 | | if (c - pszLine > 0 && c[-1] == '\r') |
5827 | | c[-1] = 0; |
5828 | | |
5829 | | if (strcmp(pszLine, ".") != 0 && strcmp(pszLine, "..") != 0) |
5830 | | { |
5831 | | oFileList.AddString(pszLine); |
5832 | | if constexpr (ENABLE_DEBUG_VERBOSE) |
5833 | | { |
5834 | | CPLDebug(GetDebugKey(), "File[%d] = %s", nCount, |
5835 | | pszLine); |
5836 | | } |
5837 | | nCount++; |
5838 | | } |
5839 | | |
5840 | | pszLine = c + 1; |
5841 | | } |
5842 | | |
5843 | | papszFileList = oFileList.StealList(); |
5844 | | } |
5845 | | |
5846 | | CPLFree(sWriteFuncData.pBuffer); |
5847 | | sWriteFuncData.pBuffer = nullptr; |
5848 | | } |
5849 | | |
5850 | | CPLFree(sWriteFuncData.pBuffer); |
5851 | | curl_easy_cleanup(hCurlHandle); |
5852 | | |
5853 | | return papszFileList; |
5854 | | } |
5855 | | |
5856 | | // Try to recognize HTML pages that list the content of a directory. |
5857 | | // Currently this supports what Apache and shttpd can return. |
5858 | | else if (STARTS_WITH(osURL.c_str(), "http://") || |
5859 | | STARTS_WITH(osURL.c_str(), "https://")) |
5860 | | { |
5861 | | RunThreadUser threadUser(*this); |
5862 | | |
5863 | | std::string osDirname(std::move(osURL)); |
5864 | | osDirname += '/'; |
5865 | | |
5866 | | CURL *hCurlHandle = curl_easy_init(); |
5867 | | |
5868 | | struct curl_slist *headers = |
5869 | | SetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5870 | | |
5871 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
5872 | | |
5873 | | WriteFuncStruct sWriteFuncData; |
5874 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
5875 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5876 | | &sWriteFuncData); |
5877 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5878 | | VSICurlHandleWriteFunc); |
5879 | | |
5880 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5881 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5882 | | szCurlErrBuf); |
5883 | | |
5884 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
5885 | | |
5886 | | Perform(hCurlHandle); |
5887 | | |
5888 | | curl_slist_free_all(headers); |
5889 | | |
5890 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
5891 | | |
5892 | | if (sWriteFuncData.pBuffer == nullptr) |
5893 | | { |
5894 | | curl_easy_cleanup(hCurlHandle); |
5895 | | return nullptr; |
5896 | | } |
5897 | | |
5898 | | char **papszFileList = nullptr; |
5899 | | if (STARTS_WITH_CI(sWriteFuncData.pBuffer, "<?xml") && |
5900 | | strstr(sWriteFuncData.pBuffer, "<ListBucketResult") != nullptr) |
5901 | | { |
5902 | | CPLStringList osFileList; |
5903 | | std::string osBaseURL(pszDirname); |
5904 | | osBaseURL += "/"; |
5905 | | bool bIsTruncated = true; |
5906 | | bool ret = AnalyseS3FileList( |
5907 | | osBaseURL, sWriteFuncData.pBuffer, osFileList, nMaxFiles, |
5908 | | GetS3IgnoredStorageClasses(), bIsTruncated); |
5909 | | // If the list is truncated, then don't report it. |
5910 | | if (ret && !bIsTruncated) |
5911 | | { |
5912 | | if (osFileList.empty()) |
5913 | | { |
5914 | | // To avoid an error to be reported |
5915 | | osFileList.AddString("."); |
5916 | | } |
5917 | | papszFileList = osFileList.StealList(); |
5918 | | *pbGotFileList = true; |
5919 | | } |
5920 | | } |
5921 | | else |
5922 | | { |
5923 | | papszFileList = ParseHTMLFileList( |
5924 | | pszDirname, nMaxFiles, sWriteFuncData.pBuffer, pbGotFileList); |
5925 | | } |
5926 | | |
5927 | | CPLFree(sWriteFuncData.pBuffer); |
5928 | | curl_easy_cleanup(hCurlHandle); |
5929 | | return papszFileList; |
5930 | | } |
5931 | | |
5932 | | return nullptr; |
5933 | | } |
5934 | | |
5935 | | /************************************************************************/ |
5936 | | /* GetS3IgnoredStorageClasses() */ |
5937 | | /************************************************************************/ |
5938 | | |
5939 | | std::set<std::string> VSICurlFilesystemHandlerBase::GetS3IgnoredStorageClasses() |
5940 | | { |
5941 | | std::set<std::string> oSetIgnoredStorageClasses; |
5942 | | const char *pszIgnoredStorageClasses = |
5943 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_STORAGE_CLASSES", nullptr); |
5944 | | const char *pszIgnoreGlacierStorage = |
5945 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE", nullptr); |
5946 | | CPLStringList aosIgnoredStorageClasses( |
5947 | | CSLTokenizeString2(pszIgnoredStorageClasses ? pszIgnoredStorageClasses |
5948 | | : "GLACIER,DEEP_ARCHIVE", |
5949 | | ",", 0)); |
5950 | | for (int i = 0; i < aosIgnoredStorageClasses.size(); ++i) |
5951 | | oSetIgnoredStorageClasses.insert(aosIgnoredStorageClasses[i]); |
5952 | | if (pszIgnoredStorageClasses == nullptr && |
5953 | | pszIgnoreGlacierStorage != nullptr && |
5954 | | !CPLTestBool(pszIgnoreGlacierStorage)) |
5955 | | { |
5956 | | oSetIgnoredStorageClasses.clear(); |
5957 | | } |
5958 | | return oSetIgnoredStorageClasses; |
5959 | | } |
5960 | | |
5961 | | /************************************************************************/ |
5962 | | /* Stat() */ |
5963 | | /************************************************************************/ |
5964 | | |
5965 | | int VSICurlFilesystemHandlerBase::Stat(const char *pszFilename, |
5966 | | VSIStatBufL *pStatBuf, int nFlags) |
5967 | | { |
5968 | | if (!cpl::starts_with(std::string_view(pszFilename), GetFSPrefix()) && |
5969 | | !StartsWithVSICurlPrefix(pszFilename)) |
5970 | | { |
5971 | | return -1; |
5972 | | } |
5973 | | |
5974 | | memset(pStatBuf, 0, sizeof(VSIStatBufL)); |
5975 | | |
5976 | | if ((nFlags & VSI_STAT_CACHE_ONLY) != 0) |
5977 | | { |
5978 | | cpl::FileProp oFileProp; |
5979 | | if (!GetCachedFileProp(GetURLFromFilename(pszFilename).c_str(), |
5980 | | oFileProp) || |
5981 | | oFileProp.eExists != EXIST_YES) |
5982 | | { |
5983 | | return -1; |
5984 | | } |
5985 | | pStatBuf->st_mode = static_cast<unsigned short>(oFileProp.nMode); |
5986 | | pStatBuf->st_mtime = oFileProp.mTime; |
5987 | | pStatBuf->st_size = oFileProp.fileSize; |
5988 | | return 0; |
5989 | | } |
5990 | | |
5991 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5992 | | NetworkStatisticsAction oContextAction("Stat"); |
5993 | | |
5994 | | const std::string osFilename(pszFilename); |
5995 | | |
5996 | | if (!IsAllowedFilename(pszFilename)) |
5997 | | return -1; |
5998 | | |
5999 | | bool bListDir = true; |
6000 | | bool bEmptyDir = false; |
6001 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
6002 | | nullptr, &bListDir, &bEmptyDir, |
6003 | | nullptr, nullptr, nullptr)); |
6004 | | |
6005 | | const char *pszOptionVal = VSIGetPathSpecificOption( |
6006 | | pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", "NO"); |
6007 | | const bool bSkipReadDir = |
6008 | | !bListDir || bEmptyDir || EQUAL(pszOptionVal, "EMPTY_DIR") || |
6009 | | CPLTestBool(pszOptionVal) || !AllowCachedDataFor(pszFilename); |
6010 | | |
6011 | | // Does it look like a FTP directory? |
6012 | | if (STARTS_WITH(osURL.c_str(), "ftp://") && osFilename.back() == '/' && |
6013 | | !bSkipReadDir) |
6014 | | { |
6015 | | char **papszFileList = ReadDirEx(osFilename.c_str(), 0); |
6016 | | if (papszFileList) |
6017 | | { |
6018 | | pStatBuf->st_mode = S_IFDIR; |
6019 | | pStatBuf->st_size = 0; |
6020 | | |
6021 | | CSLDestroy(papszFileList); |
6022 | | |
6023 | | return 0; |
6024 | | } |
6025 | | return -1; |
6026 | | } |
6027 | | else if (strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
6028 | | !STARTS_WITH_CI(CPLGetExtensionSafe(osFilename.c_str()).c_str(), |
6029 | | "zip") && |
6030 | | strstr(osFilename.c_str(), ".zip.") != nullptr && |
6031 | | strstr(osFilename.c_str(), ".ZIP.") != nullptr && !bSkipReadDir) |
6032 | | { |
6033 | | bool bGotFileList = false; |
6034 | | char **papszFileList = ReadDirInternal( |
6035 | | CPLGetDirnameSafe(osFilename.c_str()).c_str(), 0, &bGotFileList); |
6036 | | const bool bFound = |
6037 | | VSICurlIsFileInList(papszFileList, |
6038 | | CPLGetFilename(osFilename.c_str())) != -1; |
6039 | | CSLDestroy(papszFileList); |
6040 | | if (bGotFileList && !bFound) |
6041 | | { |
6042 | | return -1; |
6043 | | } |
6044 | | } |
6045 | | |
6046 | | VSICurlHandle *poHandle = CreateFileHandle(osFilename.c_str()); |
6047 | | if (poHandle == nullptr) |
6048 | | return -1; |
6049 | | |
6050 | | if (poHandle->IsKnownFileSize() || |
6051 | | ((nFlags & VSI_STAT_SIZE_FLAG) && !poHandle->IsDirectory() && |
6052 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES")))) |
6053 | | { |
6054 | | pStatBuf->st_size = poHandle->GetFileSize(true); |
6055 | | } |
6056 | | |
6057 | | const int nRet = |
6058 | | poHandle->Exists((nFlags & VSI_STAT_SET_ERROR_FLAG) > 0) ? 0 : -1; |
6059 | | pStatBuf->st_mtime = poHandle->GetMTime(); |
6060 | | pStatBuf->st_mode = static_cast<unsigned short>(poHandle->GetMode()); |
6061 | | if (pStatBuf->st_mode == 0) |
6062 | | pStatBuf->st_mode = poHandle->IsDirectory() ? S_IFDIR : S_IFREG; |
6063 | | delete poHandle; |
6064 | | return nRet; |
6065 | | } |
6066 | | |
6067 | | /************************************************************************/ |
6068 | | /* ReadDirInternal() */ |
6069 | | /************************************************************************/ |
6070 | | |
6071 | | char **VSICurlFilesystemHandlerBase::ReadDirInternal(const char *pszDirname, |
6072 | | int nMaxFiles, |
6073 | | bool *pbGotFileList) |
6074 | | { |
6075 | | std::string osDirname(pszDirname); |
6076 | | |
6077 | | // Replace a/b/../c by a/c |
6078 | | const auto posSlashDotDot = osDirname.find("/.."); |
6079 | | if (posSlashDotDot != std::string::npos && posSlashDotDot >= 1) |
6080 | | { |
6081 | | const auto posPrecedingSlash = |
6082 | | osDirname.find_last_of('/', posSlashDotDot - 1); |
6083 | | if (posPrecedingSlash != std::string::npos && posPrecedingSlash >= 1) |
6084 | | { |
6085 | | osDirname.erase(osDirname.begin() + posPrecedingSlash, |
6086 | | osDirname.begin() + posSlashDotDot + strlen("/..")); |
6087 | | } |
6088 | | } |
6089 | | |
6090 | | std::string osDirnameOri(osDirname); |
6091 | | if (osDirname + "/" == GetFSPrefix()) |
6092 | | { |
6093 | | osDirname += "/"; |
6094 | | } |
6095 | | else if (osDirname != GetFSPrefix()) |
6096 | | { |
6097 | | while (!osDirname.empty() && osDirname.back() == '/') |
6098 | | osDirname.erase(osDirname.size() - 1); |
6099 | | } |
6100 | | |
6101 | | if (osDirname.size() < GetFSPrefix().size()) |
6102 | | { |
6103 | | if (pbGotFileList) |
6104 | | *pbGotFileList = true; |
6105 | | return nullptr; |
6106 | | } |
6107 | | |
6108 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
6109 | | NetworkStatisticsAction oContextAction("ReadDir"); |
6110 | | |
6111 | | CPLMutexHolder oHolder(&hMutex); |
6112 | | |
6113 | | // If we know the file exists and is not a directory, |
6114 | | // then don't try to list its content. |
6115 | | FileProp cachedFileProp; |
6116 | | if (GetCachedFileProp(GetURLFromFilename(osDirname.c_str()).c_str(), |
6117 | | cachedFileProp) && |
6118 | | cachedFileProp.eExists == EXIST_YES && !cachedFileProp.bIsDirectory) |
6119 | | { |
6120 | | if (osDirnameOri != osDirname) |
6121 | | { |
6122 | | if (GetCachedFileProp((GetURLFromFilename(osDirname) + "/").c_str(), |
6123 | | cachedFileProp) && |
6124 | | cachedFileProp.eExists == EXIST_YES && |
6125 | | !cachedFileProp.bIsDirectory) |
6126 | | { |
6127 | | if (pbGotFileList) |
6128 | | *pbGotFileList = true; |
6129 | | return nullptr; |
6130 | | } |
6131 | | } |
6132 | | else |
6133 | | { |
6134 | | if (pbGotFileList) |
6135 | | *pbGotFileList = true; |
6136 | | return nullptr; |
6137 | | } |
6138 | | } |
6139 | | |
6140 | | CachedDirList cachedDirList; |
6141 | | if (!GetCachedDirList(osDirname.c_str(), cachedDirList)) |
6142 | | { |
6143 | | cachedDirList.oFileList.Assign(GetFileList(osDirname.c_str(), nMaxFiles, |
6144 | | &cachedDirList.bGotFileList), |
6145 | | true); |
6146 | | if (cachedDirList.bGotFileList && cachedDirList.oFileList.empty()) |
6147 | | { |
6148 | | // To avoid an error to be reported |
6149 | | cachedDirList.oFileList.AddString("."); |
6150 | | } |
6151 | | if (nMaxFiles <= 0 || cachedDirList.oFileList.size() < nMaxFiles) |
6152 | | { |
6153 | | // Only cache content if we didn't hit the limitation |
6154 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
6155 | | } |
6156 | | } |
6157 | | |
6158 | | if (pbGotFileList) |
6159 | | *pbGotFileList = cachedDirList.bGotFileList; |
6160 | | |
6161 | | return CSLDuplicate(cachedDirList.oFileList.List()); |
6162 | | } |
6163 | | |
6164 | | /************************************************************************/ |
6165 | | /* InvalidateDirContent() */ |
6166 | | /************************************************************************/ |
6167 | | |
6168 | | void VSICurlFilesystemHandlerBase::InvalidateDirContent( |
6169 | | const std::string &osDirname) |
6170 | | { |
6171 | | CPLMutexHolder oHolder(&hMutex); |
6172 | | |
6173 | | CachedDirList oCachedDirList; |
6174 | | if (oCacheDirList.tryGet(osDirname, oCachedDirList)) |
6175 | | { |
6176 | | nCachedFilesInDirList -= oCachedDirList.oFileList.size(); |
6177 | | oCacheDirList.remove(osDirname); |
6178 | | } |
6179 | | } |
6180 | | |
6181 | | /************************************************************************/ |
6182 | | /* ReadDirEx() */ |
6183 | | /************************************************************************/ |
6184 | | |
6185 | | char **VSICurlFilesystemHandlerBase::ReadDirEx(const char *pszDirname, |
6186 | | int nMaxFiles) |
6187 | | { |
6188 | | return ReadDirInternal(pszDirname, nMaxFiles, nullptr); |
6189 | | } |
6190 | | |
6191 | | /************************************************************************/ |
6192 | | /* SiblingFiles() */ |
6193 | | /************************************************************************/ |
6194 | | |
6195 | | char **VSICurlFilesystemHandlerBase::SiblingFiles(const char *pszFilename) |
6196 | | { |
6197 | | /* Small optimization to avoid unnecessary stat'ing from PAux or ENVI */ |
6198 | | /* drivers. The MBTiles driver needs no companion file. */ |
6199 | | if (EQUAL(CPLGetExtensionSafe(pszFilename).c_str(), "mbtiles")) |
6200 | | { |
6201 | | return static_cast<char **>(CPLCalloc(1, sizeof(char *))); |
6202 | | } |
6203 | | return nullptr; |
6204 | | } |
6205 | | |
6206 | | /************************************************************************/ |
6207 | | /* GetFileMetadata() */ |
6208 | | /************************************************************************/ |
6209 | | |
6210 | | char **VSICurlFilesystemHandlerBase::GetFileMetadata(const char *pszFilename, |
6211 | | const char *pszDomain, |
6212 | | CSLConstList) |
6213 | | { |
6214 | | if (pszDomain == nullptr || !EQUAL(pszDomain, "HEADERS")) |
6215 | | return nullptr; |
6216 | | std::unique_ptr<VSICurlHandle> poHandle(CreateFileHandle(pszFilename)); |
6217 | | if (poHandle == nullptr) |
6218 | | return nullptr; |
6219 | | |
6220 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
6221 | | NetworkStatisticsAction oContextAction("GetFileMetadata"); |
6222 | | |
6223 | | poHandle->GetFileSizeOrHeaders(true, true); |
6224 | | return CSLDuplicate(poHandle->GetHeaders().List()); |
6225 | | } |
6226 | | |
6227 | | /************************************************************************/ |
6228 | | /* VSIAppendWriteHandle() */ |
6229 | | /************************************************************************/ |
6230 | | |
6231 | | VSIAppendWriteHandle::VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS, |
6232 | | const char *pszFSPrefix, |
6233 | | const char *pszFilename, |
6234 | | int nChunkSize) |
6235 | | : m_poFS(poFS), m_osFSPrefix(pszFSPrefix), m_osFilename(pszFilename), |
6236 | | m_oRetryParameters(CPLStringList(CPLHTTPGetOptionsFromEnv(pszFilename))), |
6237 | | m_nBufferSize(nChunkSize) |
6238 | | { |
6239 | | m_pabyBuffer = static_cast<GByte *>(VSIMalloc(m_nBufferSize)); |
6240 | | if (m_pabyBuffer == nullptr) |
6241 | | { |
6242 | | CPLError(CE_Failure, CPLE_AppDefined, |
6243 | | "Cannot allocate working buffer for %s writing", |
6244 | | m_osFSPrefix.c_str()); |
6245 | | } |
6246 | | } |
6247 | | |
6248 | | /************************************************************************/ |
6249 | | /* ~VSIAppendWriteHandle() */ |
6250 | | /************************************************************************/ |
6251 | | |
6252 | | VSIAppendWriteHandle::~VSIAppendWriteHandle() |
6253 | | { |
6254 | | /* WARNING: implementation should call Close() themselves */ |
6255 | | /* cannot be done safely from here, since Send() can be called. */ |
6256 | | CPLFree(m_pabyBuffer); |
6257 | | } |
6258 | | |
6259 | | /************************************************************************/ |
6260 | | /* Seek() */ |
6261 | | /************************************************************************/ |
6262 | | |
6263 | | int VSIAppendWriteHandle::Seek(vsi_l_offset nOffset, int nWhence) |
6264 | | { |
6265 | | if (!((nWhence == SEEK_SET && nOffset == m_nCurOffset) || |
6266 | | (nWhence == SEEK_CUR && nOffset == 0) || |
6267 | | (nWhence == SEEK_END && nOffset == 0))) |
6268 | | { |
6269 | | CPLError(CE_Failure, CPLE_NotSupported, |
6270 | | "Seek not supported on writable %s files", |
6271 | | m_osFSPrefix.c_str()); |
6272 | | m_bError = true; |
6273 | | return -1; |
6274 | | } |
6275 | | return 0; |
6276 | | } |
6277 | | |
6278 | | /************************************************************************/ |
6279 | | /* Tell() */ |
6280 | | /************************************************************************/ |
6281 | | |
6282 | | vsi_l_offset VSIAppendWriteHandle::Tell() |
6283 | | { |
6284 | | return m_nCurOffset; |
6285 | | } |
6286 | | |
6287 | | /************************************************************************/ |
6288 | | /* Read() */ |
6289 | | /************************************************************************/ |
6290 | | |
6291 | | size_t VSIAppendWriteHandle::Read(void * /* pBuffer */, size_t /* nBytes */) |
6292 | | { |
6293 | | CPLError(CE_Failure, CPLE_NotSupported, |
6294 | | "Read not supported on writable %s files", m_osFSPrefix.c_str()); |
6295 | | m_bError = true; |
6296 | | return 0; |
6297 | | } |
6298 | | |
6299 | | /************************************************************************/ |
6300 | | /* ReadCallBackBuffer() */ |
6301 | | /************************************************************************/ |
6302 | | |
6303 | | size_t VSIAppendWriteHandle::ReadCallBackBuffer(char *buffer, size_t size, |
6304 | | size_t nitems, void *instream) |
6305 | | { |
6306 | | VSIAppendWriteHandle *poThis = |
6307 | | static_cast<VSIAppendWriteHandle *>(instream); |
6308 | | const int nSizeMax = static_cast<int>(size * nitems); |
6309 | | const int nSizeToWrite = std::min( |
6310 | | nSizeMax, poThis->m_nBufferOff - poThis->m_nBufferOffReadCallback); |
6311 | | memcpy(buffer, poThis->m_pabyBuffer + poThis->m_nBufferOffReadCallback, |
6312 | | nSizeToWrite); |
6313 | | poThis->m_nBufferOffReadCallback += nSizeToWrite; |
6314 | | return nSizeToWrite; |
6315 | | } |
6316 | | |
6317 | | /************************************************************************/ |
6318 | | /* Write() */ |
6319 | | /************************************************************************/ |
6320 | | |
6321 | | size_t VSIAppendWriteHandle::Write(const void *pBuffer, size_t nBytes) |
6322 | | { |
6323 | | if (m_bError) |
6324 | | return 0; |
6325 | | |
6326 | | size_t nBytesToWrite = nBytes; |
6327 | | if (nBytesToWrite == 0) |
6328 | | return 0; |
6329 | | |
6330 | | const GByte *pabySrcBuffer = reinterpret_cast<const GByte *>(pBuffer); |
6331 | | while (nBytesToWrite > 0) |
6332 | | { |
6333 | | if (m_nBufferOff == m_nBufferSize) |
6334 | | { |
6335 | | if (!Send(false)) |
6336 | | { |
6337 | | m_bError = true; |
6338 | | return 0; |
6339 | | } |
6340 | | m_nBufferOff = 0; |
6341 | | } |
6342 | | |
6343 | | const int nToWriteInBuffer = static_cast<int>(std::min( |
6344 | | static_cast<size_t>(m_nBufferSize - m_nBufferOff), nBytesToWrite)); |
6345 | | memcpy(m_pabyBuffer + m_nBufferOff, pabySrcBuffer, nToWriteInBuffer); |
6346 | | pabySrcBuffer += nToWriteInBuffer; |
6347 | | m_nBufferOff += nToWriteInBuffer; |
6348 | | m_nCurOffset += nToWriteInBuffer; |
6349 | | nBytesToWrite -= nToWriteInBuffer; |
6350 | | } |
6351 | | return nBytes; |
6352 | | } |
6353 | | |
6354 | | /************************************************************************/ |
6355 | | /* Close() */ |
6356 | | /************************************************************************/ |
6357 | | |
6358 | | int VSIAppendWriteHandle::Close() |
6359 | | { |
6360 | | int nRet = 0; |
6361 | | if (!m_bClosed) |
6362 | | { |
6363 | | m_bClosed = true; |
6364 | | if (!m_bError && !Send(true)) |
6365 | | nRet = -1; |
6366 | | } |
6367 | | return nRet; |
6368 | | } |
6369 | | |
6370 | | /************************************************************************/ |
6371 | | /* CurlRequestHelper() */ |
6372 | | /************************************************************************/ |
6373 | | |
6374 | | CurlRequestHelper::CurlRequestHelper() |
6375 | | { |
6376 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
6377 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
6378 | | nullptr); |
6379 | | } |
6380 | | |
6381 | | /************************************************************************/ |
6382 | | /* ~CurlRequestHelper() */ |
6383 | | /************************************************************************/ |
6384 | | |
6385 | | CurlRequestHelper::~CurlRequestHelper() |
6386 | | { |
6387 | | CPLFree(sWriteFuncData.pBuffer); |
6388 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
6389 | | } |
6390 | | |
6391 | | /************************************************************************/ |
6392 | | /* perform() */ |
6393 | | /************************************************************************/ |
6394 | | |
6395 | | long CurlRequestHelper::perform(CURL *hCurlHandle, struct curl_slist *headers, |
6396 | | VSICurlFilesystemHandlerBase *poFS, |
6397 | | IVSIS3LikeHandleHelper *poS3HandleHelper) |
6398 | | { |
6399 | | RunThreadUser threadUser(*poFS); |
6400 | | |
6401 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
6402 | | |
6403 | | poS3HandleHelper->ResetQueryParameters(); |
6404 | | |
6405 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
6406 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
6407 | | VSICurlHandleWriteFunc); |
6408 | | |
6409 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
6410 | | &sWriteFuncHeaderData); |
6411 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
6412 | | VSICurlHandleWriteFunc); |
6413 | | |
6414 | | szCurlErrBuf[0] = '\0'; |
6415 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
6416 | | |
6417 | | poFS->Perform(hCurlHandle); |
6418 | | |
6419 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
6420 | | |
6421 | | curl_slist_free_all(headers); |
6422 | | |
6423 | | long response_code = 0; |
6424 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
6425 | | return response_code; |
6426 | | } |
6427 | | |
6428 | | /************************************************************************/ |
6429 | | /* NetworkStatisticsLogger */ |
6430 | | /************************************************************************/ |
6431 | | |
6432 | | // Global variable |
6433 | | NetworkStatisticsLogger NetworkStatisticsLogger::gInstance{}; |
6434 | | int NetworkStatisticsLogger::gnEnabled = -1; // unknown state |
6435 | | |
6436 | | static void ShowNetworkStats() |
6437 | | { |
6438 | | printf("Network statistics:\n%s\n", // ok |
6439 | | NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
6440 | | } |
6441 | | |
6442 | | void NetworkStatisticsLogger::ReadEnabled() |
6443 | | { |
6444 | | const bool bShowNetworkStats = |
6445 | | CPLTestConfigOption("CPL_VSIL_SHOW_NETWORK_STATS"); |
6446 | | gnEnabled = bShowNetworkStats || |
6447 | | CPLTestConfigOption("CPL_VSIL_NETWORK_STATS_ENABLED"); |
6448 | | if (bShowNetworkStats) |
6449 | | { |
6450 | | static bool bRegistered = false; |
6451 | | if (!bRegistered) |
6452 | | { |
6453 | | bRegistered = true; |
6454 | | atexit(ShowNetworkStats); |
6455 | | } |
6456 | | } |
6457 | | } |
6458 | | |
6459 | | void NetworkStatisticsLogger::EnterFileSystem(const char *pszName) |
6460 | | { |
6461 | | if (!IsEnabled()) |
6462 | | return; |
6463 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6464 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
6465 | | ContextPathItem(ContextPathType::FILESYSTEM, pszName)); |
6466 | | } |
6467 | | |
6468 | | void NetworkStatisticsLogger::LeaveFileSystem() |
6469 | | { |
6470 | | if (!IsEnabled()) |
6471 | | return; |
6472 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6473 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
6474 | | } |
6475 | | |
6476 | | void NetworkStatisticsLogger::EnterFile(const char *pszName) |
6477 | | { |
6478 | | if (!IsEnabled()) |
6479 | | return; |
6480 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6481 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
6482 | | ContextPathItem(ContextPathType::FILE, pszName)); |
6483 | | } |
6484 | | |
6485 | | void NetworkStatisticsLogger::LeaveFile() |
6486 | | { |
6487 | | if (!IsEnabled()) |
6488 | | return; |
6489 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6490 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
6491 | | } |
6492 | | |
6493 | | void NetworkStatisticsLogger::EnterAction(const char *pszName) |
6494 | | { |
6495 | | if (!IsEnabled()) |
6496 | | return; |
6497 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6498 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
6499 | | ContextPathItem(ContextPathType::ACTION, pszName)); |
6500 | | } |
6501 | | |
6502 | | void NetworkStatisticsLogger::LeaveAction() |
6503 | | { |
6504 | | if (!IsEnabled()) |
6505 | | return; |
6506 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6507 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
6508 | | } |
6509 | | |
6510 | | std::vector<NetworkStatisticsLogger::Counters *> |
6511 | | NetworkStatisticsLogger::GetCountersForContext() |
6512 | | { |
6513 | | std::vector<Counters *> v; |
6514 | | const auto &contextPath = gInstance.m_mapThreadIdToContextPath[CPLGetPID()]; |
6515 | | |
6516 | | Stats *curStats = &m_stats; |
6517 | | v.push_back(&(curStats->counters)); |
6518 | | |
6519 | | bool inFileSystem = false; |
6520 | | bool inFile = false; |
6521 | | bool inAction = false; |
6522 | | for (const auto &item : contextPath) |
6523 | | { |
6524 | | if (item.eType == ContextPathType::FILESYSTEM) |
6525 | | { |
6526 | | if (inFileSystem) |
6527 | | continue; |
6528 | | inFileSystem = true; |
6529 | | } |
6530 | | else if (item.eType == ContextPathType::FILE) |
6531 | | { |
6532 | | if (inFile) |
6533 | | continue; |
6534 | | inFile = true; |
6535 | | } |
6536 | | else if (item.eType == ContextPathType::ACTION) |
6537 | | { |
6538 | | if (inAction) |
6539 | | continue; |
6540 | | inAction = true; |
6541 | | } |
6542 | | |
6543 | | curStats = &(curStats->children[item]); |
6544 | | v.push_back(&(curStats->counters)); |
6545 | | } |
6546 | | |
6547 | | return v; |
6548 | | } |
6549 | | |
6550 | | void NetworkStatisticsLogger::LogGET(size_t nDownloadedBytes) |
6551 | | { |
6552 | | if (!IsEnabled()) |
6553 | | return; |
6554 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6555 | | for (auto counters : gInstance.GetCountersForContext()) |
6556 | | { |
6557 | | counters->nGET++; |
6558 | | counters->nGETDownloadedBytes += nDownloadedBytes; |
6559 | | } |
6560 | | } |
6561 | | |
6562 | | void NetworkStatisticsLogger::LogPUT(size_t nUploadedBytes) |
6563 | | { |
6564 | | if (!IsEnabled()) |
6565 | | return; |
6566 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6567 | | for (auto counters : gInstance.GetCountersForContext()) |
6568 | | { |
6569 | | counters->nPUT++; |
6570 | | counters->nPUTUploadedBytes += nUploadedBytes; |
6571 | | } |
6572 | | } |
6573 | | |
6574 | | void NetworkStatisticsLogger::LogHEAD() |
6575 | | { |
6576 | | if (!IsEnabled()) |
6577 | | return; |
6578 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6579 | | for (auto counters : gInstance.GetCountersForContext()) |
6580 | | { |
6581 | | counters->nHEAD++; |
6582 | | } |
6583 | | } |
6584 | | |
6585 | | void NetworkStatisticsLogger::LogPOST(size_t nUploadedBytes, |
6586 | | size_t nDownloadedBytes) |
6587 | | { |
6588 | | if (!IsEnabled()) |
6589 | | return; |
6590 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6591 | | for (auto counters : gInstance.GetCountersForContext()) |
6592 | | { |
6593 | | counters->nPOST++; |
6594 | | counters->nPOSTUploadedBytes += nUploadedBytes; |
6595 | | counters->nPOSTDownloadedBytes += nDownloadedBytes; |
6596 | | } |
6597 | | } |
6598 | | |
6599 | | void NetworkStatisticsLogger::LogDELETE() |
6600 | | { |
6601 | | if (!IsEnabled()) |
6602 | | return; |
6603 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6604 | | for (auto counters : gInstance.GetCountersForContext()) |
6605 | | { |
6606 | | counters->nDELETE++; |
6607 | | } |
6608 | | } |
6609 | | |
6610 | | void NetworkStatisticsLogger::Reset() |
6611 | | { |
6612 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6613 | | gInstance.m_stats = Stats(); |
6614 | | gnEnabled = -1; |
6615 | | } |
6616 | | |
6617 | | void NetworkStatisticsLogger::Stats::AsJSON(CPLJSONObject &oJSON) const |
6618 | | { |
6619 | | CPLJSONObject oMethods; |
6620 | | if (counters.nHEAD) |
6621 | | oMethods.Add("HEAD/count", counters.nHEAD); |
6622 | | if (counters.nGET) |
6623 | | oMethods.Add("GET/count", counters.nGET); |
6624 | | if (counters.nGETDownloadedBytes) |
6625 | | oMethods.Add("GET/downloaded_bytes", counters.nGETDownloadedBytes); |
6626 | | if (counters.nPUT) |
6627 | | oMethods.Add("PUT/count", counters.nPUT); |
6628 | | if (counters.nPUTUploadedBytes) |
6629 | | oMethods.Add("PUT/uploaded_bytes", counters.nPUTUploadedBytes); |
6630 | | if (counters.nPOST) |
6631 | | oMethods.Add("POST/count", counters.nPOST); |
6632 | | if (counters.nPOSTUploadedBytes) |
6633 | | oMethods.Add("POST/uploaded_bytes", counters.nPOSTUploadedBytes); |
6634 | | if (counters.nPOSTDownloadedBytes) |
6635 | | oMethods.Add("POST/downloaded_bytes", counters.nPOSTDownloadedBytes); |
6636 | | if (counters.nDELETE) |
6637 | | oMethods.Add("DELETE/count", counters.nDELETE); |
6638 | | oJSON.Add("methods", oMethods); |
6639 | | CPLJSONObject oFiles; |
6640 | | bool bFilesAdded = false; |
6641 | | for (const auto &kv : children) |
6642 | | { |
6643 | | CPLJSONObject childJSON; |
6644 | | kv.second.AsJSON(childJSON); |
6645 | | if (kv.first.eType == ContextPathType::FILESYSTEM) |
6646 | | { |
6647 | | std::string osName(kv.first.osName); |
6648 | | if (!osName.empty() && osName[0] == '/') |
6649 | | osName = osName.substr(1); |
6650 | | if (!osName.empty() && osName.back() == '/') |
6651 | | osName.pop_back(); |
6652 | | oJSON.Add(("handlers/" + osName).c_str(), childJSON); |
6653 | | } |
6654 | | else if (kv.first.eType == ContextPathType::FILE) |
6655 | | { |
6656 | | if (!bFilesAdded) |
6657 | | { |
6658 | | bFilesAdded = true; |
6659 | | oJSON.Add("files", oFiles); |
6660 | | } |
6661 | | oFiles.AddNoSplitName(kv.first.osName.c_str(), childJSON); |
6662 | | } |
6663 | | else if (kv.first.eType == ContextPathType::ACTION) |
6664 | | { |
6665 | | oJSON.Add(("actions/" + kv.first.osName).c_str(), childJSON); |
6666 | | } |
6667 | | } |
6668 | | } |
6669 | | |
6670 | | std::string NetworkStatisticsLogger::GetReportAsSerializedJSON() |
6671 | | { |
6672 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6673 | | |
6674 | | CPLJSONObject oJSON; |
6675 | | gInstance.m_stats.AsJSON(oJSON); |
6676 | | return oJSON.Format(CPLJSONObject::PrettyFormat::Pretty); |
6677 | | } |
6678 | | |
6679 | | } /* end of namespace cpl */ |
6680 | | |
6681 | | /************************************************************************/ |
6682 | | /* VSICurlParseUnixPermissions() */ |
6683 | | /************************************************************************/ |
6684 | | |
6685 | | int VSICurlParseUnixPermissions(const char *pszPermissions) |
6686 | | { |
6687 | | if (strlen(pszPermissions) != 9) |
6688 | | return 0; |
6689 | | int nMode = 0; |
6690 | | if (pszPermissions[0] == 'r') |
6691 | | nMode |= S_IRUSR; |
6692 | | if (pszPermissions[1] == 'w') |
6693 | | nMode |= S_IWUSR; |
6694 | | if (pszPermissions[2] == 'x') |
6695 | | nMode |= S_IXUSR; |
6696 | | if (pszPermissions[3] == 'r') |
6697 | | nMode |= S_IRGRP; |
6698 | | if (pszPermissions[4] == 'w') |
6699 | | nMode |= S_IWGRP; |
6700 | | if (pszPermissions[5] == 'x') |
6701 | | nMode |= S_IXGRP; |
6702 | | if (pszPermissions[6] == 'r') |
6703 | | nMode |= S_IROTH; |
6704 | | if (pszPermissions[7] == 'w') |
6705 | | nMode |= S_IWOTH; |
6706 | | if (pszPermissions[8] == 'x') |
6707 | | nMode |= S_IXOTH; |
6708 | | return nMode; |
6709 | | } |
6710 | | |
6711 | | /************************************************************************/ |
6712 | | /* Cache of file properties. */ |
6713 | | /************************************************************************/ |
6714 | | |
6715 | | static std::mutex oCacheFilePropMutex; |
6716 | | static lru11::Cache<std::string, cpl::FileProp> *poCacheFileProp = nullptr; |
6717 | | |
6718 | | /************************************************************************/ |
6719 | | /* VSICURLGetCachedFileProp() */ |
6720 | | /************************************************************************/ |
6721 | | |
6722 | | bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6723 | | { |
6724 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6725 | | return poCacheFileProp != nullptr && |
6726 | | poCacheFileProp->tryGet(std::string(pszURL), oFileProp) && |
6727 | | // Let a chance to use new auth parameters |
6728 | | !(oFileProp.eExists == cpl::EXIST_NO && |
6729 | | gnGenerationAuthParameters != oFileProp.nGenerationAuthParameters); |
6730 | | } |
6731 | | |
6732 | | /************************************************************************/ |
6733 | | /* VSICURLSetCachedFileProp() */ |
6734 | | /************************************************************************/ |
6735 | | |
6736 | | void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6737 | | { |
6738 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6739 | | if (poCacheFileProp == nullptr) |
6740 | | poCacheFileProp = |
6741 | | new lru11::Cache<std::string, cpl::FileProp>(100 * 1024); |
6742 | | oFileProp.nGenerationAuthParameters = gnGenerationAuthParameters; |
6743 | | poCacheFileProp->insert(std::string(pszURL), oFileProp); |
6744 | | } |
6745 | | |
6746 | | /************************************************************************/ |
6747 | | /* VSICURLInvalidateCachedFileProp() */ |
6748 | | /************************************************************************/ |
6749 | | |
6750 | | void VSICURLInvalidateCachedFileProp(const char *pszURL) |
6751 | | { |
6752 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6753 | | if (poCacheFileProp != nullptr) |
6754 | | poCacheFileProp->remove(std::string(pszURL)); |
6755 | | } |
6756 | | |
6757 | | /************************************************************************/ |
6758 | | /* VSICURLInvalidateCachedFilePropPrefix() */ |
6759 | | /************************************************************************/ |
6760 | | |
6761 | | void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL) |
6762 | | { |
6763 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6764 | | if (poCacheFileProp != nullptr) |
6765 | | { |
6766 | | std::list<std::string> keysToRemove; |
6767 | | const size_t nURLSize = strlen(pszURL); |
6768 | | auto lambda = |
6769 | | [&keysToRemove, &pszURL, nURLSize]( |
6770 | | const lru11::KeyValuePair<std::string, cpl::FileProp> &kv) |
6771 | | { |
6772 | | if (strncmp(kv.key.c_str(), pszURL, nURLSize) == 0) |
6773 | | keysToRemove.push_back(kv.key); |
6774 | | }; |
6775 | | poCacheFileProp->cwalk(lambda); |
6776 | | for (const auto &key : keysToRemove) |
6777 | | poCacheFileProp->remove(key); |
6778 | | } |
6779 | | } |
6780 | | |
6781 | | /************************************************************************/ |
6782 | | /* VSICURLDestroyCacheFileProp() */ |
6783 | | /************************************************************************/ |
6784 | | |
6785 | | void VSICURLDestroyCacheFileProp() |
6786 | | { |
6787 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6788 | | delete poCacheFileProp; |
6789 | | poCacheFileProp = nullptr; |
6790 | | } |
6791 | | |
6792 | | /************************************************************************/ |
6793 | | /* VSICURLMultiCleanup() */ |
6794 | | /************************************************************************/ |
6795 | | |
6796 | | void VSICURLMultiCleanup(CURLM *hCurlMultiHandle) |
6797 | | { |
6798 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
6799 | | curl_multi_cleanup(hCurlMultiHandle); |
6800 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
6801 | | } |
6802 | | |
6803 | | /************************************************************************/ |
6804 | | /* VSICurlInstallReadCbk() */ |
6805 | | /************************************************************************/ |
6806 | | |
6807 | | int VSICurlInstallReadCbk(VSILFILE *fp, VSICurlReadCbkFunc pfnReadCbk, |
6808 | | void *pfnUserData, int bStopOnInterruptUntilUninstall) |
6809 | | { |
6810 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->InstallReadCbk( |
6811 | | pfnReadCbk, pfnUserData, bStopOnInterruptUntilUninstall); |
6812 | | } |
6813 | | |
6814 | | /************************************************************************/ |
6815 | | /* VSICurlUninstallReadCbk() */ |
6816 | | /************************************************************************/ |
6817 | | |
6818 | | int VSICurlUninstallReadCbk(VSILFILE *fp) |
6819 | | { |
6820 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->UninstallReadCbk(); |
6821 | | } |
6822 | | |
6823 | | /************************************************************************/ |
6824 | | /* VSICurlSetContentTypeFromExt() */ |
6825 | | /************************************************************************/ |
6826 | | |
6827 | | struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *poList, |
6828 | | const char *pszPath) |
6829 | | { |
6830 | | struct curl_slist *iter = poList; |
6831 | | while (iter != nullptr) |
6832 | | { |
6833 | | if (STARTS_WITH_CI(iter->data, "Content-Type")) |
6834 | | { |
6835 | | return poList; |
6836 | | } |
6837 | | iter = iter->next; |
6838 | | } |
6839 | | |
6840 | | static const struct |
6841 | | { |
6842 | | const char *ext; |
6843 | | const char *mime; |
6844 | | } aosExtMimePairs[] = { |
6845 | | {"txt", "text/plain"}, {"json", "application/json"}, |
6846 | | {"tif", "image/tiff"}, {"tiff", "image/tiff"}, |
6847 | | {"jpg", "image/jpeg"}, {"jpeg", "image/jpeg"}, |
6848 | | {"jp2", "image/jp2"}, {"jpx", "image/jp2"}, |
6849 | | {"j2k", "image/jp2"}, {"jpc", "image/jp2"}, |
6850 | | {"png", "image/png"}, |
6851 | | }; |
6852 | | |
6853 | | const std::string osExt = CPLGetExtensionSafe(pszPath); |
6854 | | if (!osExt.empty()) |
6855 | | { |
6856 | | for (const auto &pair : aosExtMimePairs) |
6857 | | { |
6858 | | if (EQUAL(osExt.c_str(), pair.ext)) |
6859 | | { |
6860 | | |
6861 | | const std::string osContentType( |
6862 | | CPLSPrintf("Content-Type: %s", pair.mime)); |
6863 | | poList = curl_slist_append(poList, osContentType.c_str()); |
6864 | | #ifdef DEBUG_VERBOSE |
6865 | | CPLDebug("HTTP", "Setting %s, based on lookup table.", |
6866 | | osContentType.c_str()); |
6867 | | #endif |
6868 | | break; |
6869 | | } |
6870 | | } |
6871 | | } |
6872 | | |
6873 | | return poList; |
6874 | | } |
6875 | | |
6876 | | /************************************************************************/ |
6877 | | /* VSICurlSetCreationHeadersFromOptions() */ |
6878 | | /************************************************************************/ |
6879 | | |
6880 | | struct curl_slist *VSICurlSetCreationHeadersFromOptions( |
6881 | | struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath) |
6882 | | { |
6883 | | bool bContentTypeFound = false; |
6884 | | for (CSLConstList papszIter = papszOptions; papszIter && *papszIter; |
6885 | | ++papszIter) |
6886 | | { |
6887 | | char *pszKey = nullptr; |
6888 | | const char *pszValue = CPLParseNameValue(*papszIter, &pszKey); |
6889 | | if (pszKey && pszValue) |
6890 | | { |
6891 | | if (EQUAL(pszKey, "Content-Type")) |
6892 | | { |
6893 | | bContentTypeFound = true; |
6894 | | } |
6895 | | headers = curl_slist_append(headers, |
6896 | | CPLSPrintf("%s: %s", pszKey, pszValue)); |
6897 | | } |
6898 | | CPLFree(pszKey); |
6899 | | } |
6900 | | |
6901 | | // If Content-type not found in papszOptions, try to set it from the |
6902 | | // filename exstension. |
6903 | | if (!bContentTypeFound) |
6904 | | { |
6905 | | headers = VSICurlSetContentTypeFromExt(headers, pszPath); |
6906 | | } |
6907 | | |
6908 | | return headers; |
6909 | | } |
6910 | | |
6911 | | #endif // DOXYGEN_SKIP |
6912 | | //! @endcond |
6913 | | |
6914 | | /************************************************************************/ |
6915 | | /* VSIInstallCurlFileHandler() */ |
6916 | | /************************************************************************/ |
6917 | | |
6918 | | /*! |
6919 | | \brief Install /vsicurl/ HTTP/FTP file system handler (requires libcurl) |
6920 | | |
6921 | | \verbatim embed:rst |
6922 | | See :ref:`/vsicurl/ documentation <vsicurl>` |
6923 | | \endverbatim |
6924 | | |
6925 | | */ |
6926 | | void VSIInstallCurlFileHandler(void) |
6927 | | { |
6928 | | auto poHandler = std::make_shared<cpl::VSICurlFilesystemHandler>(); |
6929 | | for (const char *pszPrefix : VSICURL_PREFIXES) |
6930 | | { |
6931 | | VSIFileManager::InstallHandler(pszPrefix, poHandler); |
6932 | | } |
6933 | | } |
6934 | | |
6935 | | /************************************************************************/ |
6936 | | /* VSICurlClearCache() */ |
6937 | | /************************************************************************/ |
6938 | | |
6939 | | /** |
6940 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6941 | | * |
6942 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6943 | | * /vsiswift/) cache a number of |
6944 | | * metadata and data for faster execution in read-only scenarios. But when the |
6945 | | * content on the server-side may change during the same process, those |
6946 | | * mechanisms can prevent opening new files, or give an outdated version of |
6947 | | * them. |
6948 | | * |
6949 | | */ |
6950 | | |
6951 | | void VSICurlClearCache(void) |
6952 | | { |
6953 | | // FIXME ? Currently we have different filesystem instances for |
6954 | | // vsicurl/, /vsis3/, /vsigs/ . So each one has its own cache of regions. |
6955 | | // File properties cache are now shared |
6956 | | char **papszPrefix = VSIFileManager::GetPrefixes(); |
6957 | | for (size_t i = 0; papszPrefix && papszPrefix[i]; ++i) |
6958 | | { |
6959 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6960 | | VSIFileManager::GetHandler(papszPrefix[i])); |
6961 | | |
6962 | | if (poFSHandler) |
6963 | | poFSHandler->ClearCache(); |
6964 | | } |
6965 | | CSLDestroy(papszPrefix); |
6966 | | |
6967 | | VSICurlStreamingClearCache(); |
6968 | | } |
6969 | | |
6970 | | /************************************************************************/ |
6971 | | /* VSICurlPartialClearCache() */ |
6972 | | /************************************************************************/ |
6973 | | |
6974 | | /** |
6975 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6976 | | * for a given filename (and its subfiles and subdirectories if it is a |
6977 | | * directory) |
6978 | | * |
6979 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6980 | | * /vsiswift/) cache a number of |
6981 | | * metadata and data for faster execution in read-only scenarios. But when the |
6982 | | * content on the server-side may change during the same process, those |
6983 | | * mechanisms can prevent opening new files, or give an outdated version of |
6984 | | * them. |
6985 | | * |
6986 | | * The filename prefix must start with the name of a known virtual file system |
6987 | | * (such as "/vsicurl/", "/vsis3/") |
6988 | | * |
6989 | | * VSICurlPartialClearCache("/vsis3/b") will clear all cached state for any file |
6990 | | * or directory starting with that prefix, so potentially "/vsis3/bucket", |
6991 | | * "/vsis3/basket/" or "/vsis3/basket/object". |
6992 | | * |
6993 | | * @param pszFilenamePrefix Filename prefix |
6994 | | */ |
6995 | | |
6996 | | void VSICurlPartialClearCache(const char *pszFilenamePrefix) |
6997 | | { |
6998 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6999 | | VSIFileManager::GetHandler(pszFilenamePrefix)); |
7000 | | |
7001 | | if (poFSHandler) |
7002 | | poFSHandler->PartialClearCache(pszFilenamePrefix); |
7003 | | } |
7004 | | |
7005 | | /************************************************************************/ |
7006 | | /* VSINetworkStatsReset() */ |
7007 | | /************************************************************************/ |
7008 | | |
7009 | | /** |
7010 | | * \brief Clear network related statistics. |
7011 | | * |
7012 | | * The effect of the CPL_VSIL_NETWORK_STATS_ENABLED configuration option |
7013 | | * will also be reset. That is, that the next network access will check its |
7014 | | * value again. |
7015 | | * |
7016 | | * @since GDAL 3.2.0 |
7017 | | */ |
7018 | | |
7019 | | void VSINetworkStatsReset(void) |
7020 | | { |
7021 | | cpl::NetworkStatisticsLogger::Reset(); |
7022 | | } |
7023 | | |
7024 | | /************************************************************************/ |
7025 | | /* VSINetworkStatsGetAsSerializedJSON() */ |
7026 | | /************************************************************************/ |
7027 | | |
7028 | | /** |
7029 | | * \brief Return network related statistics, as a JSON serialized object. |
7030 | | * |
7031 | | * Statistics collecting should be enabled with the |
7032 | | CPL_VSIL_NETWORK_STATS_ENABLED |
7033 | | * configuration option set to YES before any network activity starts |
7034 | | * (for efficiency, reading it is cached on first access, until |
7035 | | VSINetworkStatsReset() is called) |
7036 | | * |
7037 | | * Statistics can also be emitted on standard output at process termination if |
7038 | | * the CPL_VSIL_SHOW_NETWORK_STATS configuration option is set to YES. |
7039 | | * |
7040 | | * Example of output: |
7041 | | * \code{.js} |
7042 | | * { |
7043 | | * "methods":{ |
7044 | | * "GET":{ |
7045 | | * "count":6, |
7046 | | * "downloaded_bytes":40825 |
7047 | | * }, |
7048 | | * "PUT":{ |
7049 | | * "count":1, |
7050 | | * "uploaded_bytes":35472 |
7051 | | * } |
7052 | | * }, |
7053 | | * "handlers":{ |
7054 | | * "vsigs":{ |
7055 | | * "methods":{ |
7056 | | * "GET":{ |
7057 | | * "count":2, |
7058 | | * "downloaded_bytes":446 |
7059 | | * }, |
7060 | | * "PUT":{ |
7061 | | * "count":1, |
7062 | | * "uploaded_bytes":35472 |
7063 | | * } |
7064 | | * }, |
7065 | | * "files":{ |
7066 | | * "\/vsigs\/spatialys\/byte.tif":{ |
7067 | | * "methods":{ |
7068 | | * "PUT":{ |
7069 | | * "count":1, |
7070 | | * "uploaded_bytes":35472 |
7071 | | * } |
7072 | | * }, |
7073 | | * "actions":{ |
7074 | | * "Write":{ |
7075 | | * "methods":{ |
7076 | | * "PUT":{ |
7077 | | * "count":1, |
7078 | | * "uploaded_bytes":35472 |
7079 | | * } |
7080 | | * } |
7081 | | * } |
7082 | | * } |
7083 | | * } |
7084 | | * }, |
7085 | | * "actions":{ |
7086 | | * "Stat":{ |
7087 | | * "methods":{ |
7088 | | * "GET":{ |
7089 | | * "count":2, |
7090 | | * "downloaded_bytes":446 |
7091 | | * } |
7092 | | * }, |
7093 | | * "files":{ |
7094 | | * "\/vsigs\/spatialys\/byte.tif\/":{ |
7095 | | * "methods":{ |
7096 | | * "GET":{ |
7097 | | * "count":1, |
7098 | | * "downloaded_bytes":181 |
7099 | | * } |
7100 | | * } |
7101 | | * } |
7102 | | * } |
7103 | | * } |
7104 | | * } |
7105 | | * }, |
7106 | | * "vsis3":{ |
7107 | | * [...] |
7108 | | * } |
7109 | | * } |
7110 | | * } |
7111 | | * \endcode |
7112 | | * |
7113 | | * @param papszOptions Unused. |
7114 | | * @return a JSON serialized string to free with VSIFree(), or nullptr |
7115 | | * @since GDAL 3.2.0 |
7116 | | */ |
7117 | | |
7118 | | char *VSINetworkStatsGetAsSerializedJSON(CPL_UNUSED char **papszOptions) |
7119 | | { |
7120 | | return CPLStrdup( |
7121 | | cpl::NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
7122 | | } |
7123 | | |
7124 | | #endif /* HAVE_CURL */ |
7125 | | |
7126 | | #undef ENABLE_DEBUG |