/src/gdal/port/cpl_vsil_curl.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: CPL - Common Portability Library |
4 | | * Purpose: Implement VSI large file api for HTTP/FTP files |
5 | | * Author: Even Rouault, even.rouault at spatialys.com |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_port.h" |
14 | | #include "cpl_vsil_curl_priv.h" |
15 | | #include "cpl_vsil_curl_class.h" |
16 | | |
17 | | #include <algorithm> |
18 | | #include <array> |
19 | | #include <limits> |
20 | | #include <map> |
21 | | #include <memory> |
22 | | #include <set> |
23 | | |
24 | | #include "cpl_aws.h" |
25 | | #include "cpl_json.h" |
26 | | #include "cpl_json_header.h" |
27 | | #include "cpl_minixml.h" |
28 | | #include "cpl_multiproc.h" |
29 | | #include "cpl_string.h" |
30 | | #include "cpl_time.h" |
31 | | #include "cpl_vsi.h" |
32 | | #include "cpl_vsi_virtual.h" |
33 | | #include "cpl_http.h" |
34 | | #include "cpl_mem_cache.h" |
35 | | |
36 | | #ifndef S_IRUSR |
37 | | #define S_IRUSR 00400 |
38 | | #define S_IWUSR 00200 |
39 | | #define S_IXUSR 00100 |
40 | | #define S_IRGRP 00040 |
41 | | #define S_IWGRP 00020 |
42 | | #define S_IXGRP 00010 |
43 | | #define S_IROTH 00004 |
44 | | #define S_IWOTH 00002 |
45 | | #define S_IXOTH 00001 |
46 | | #endif |
47 | | |
48 | | #ifndef HAVE_CURL |
49 | | |
50 | | void VSIInstallCurlFileHandler(void) |
51 | 0 | { |
52 | | // Not supported. |
53 | 0 | } |
54 | | |
55 | | void VSICurlClearCache(void) |
56 | 0 | { |
57 | | // Not supported. |
58 | 0 | } |
59 | | |
60 | | void VSICurlPartialClearCache(const char *) |
61 | 0 | { |
62 | | // Not supported. |
63 | 0 | } |
64 | | |
65 | | void VSICurlAuthParametersChanged() |
66 | 812 | { |
67 | | // Not supported. |
68 | 812 | } |
69 | | |
70 | | void VSINetworkStatsReset(void) |
71 | 0 | { |
72 | | // Not supported |
73 | 0 | } |
74 | | |
75 | | char *VSINetworkStatsGetAsSerializedJSON(char ** /* papszOptions */) |
76 | 0 | { |
77 | | // Not supported |
78 | 0 | return nullptr; |
79 | 0 | } |
80 | | |
81 | | /************************************************************************/ |
82 | | /* VSICurlInstallReadCbk() */ |
83 | | /************************************************************************/ |
84 | | |
85 | | int VSICurlInstallReadCbk(VSILFILE * /* fp */, |
86 | | VSICurlReadCbkFunc /* pfnReadCbk */, |
87 | | void * /* pfnUserData */, |
88 | | int /* bStopOnInterruptUntilUninstall */) |
89 | 0 | { |
90 | 0 | return FALSE; |
91 | 0 | } |
92 | | |
93 | | /************************************************************************/ |
94 | | /* VSICurlUninstallReadCbk() */ |
95 | | /************************************************************************/ |
96 | | |
97 | | int VSICurlUninstallReadCbk(VSILFILE * /* fp */) |
98 | 0 | { |
99 | 0 | return FALSE; |
100 | 0 | } |
101 | | |
102 | | #else |
103 | | |
104 | | //! @cond Doxygen_Suppress |
105 | | #ifndef DOXYGEN_SKIP |
106 | | |
107 | | #define ENABLE_DEBUG 1 |
108 | | #define ENABLE_DEBUG_VERBOSE 0 |
109 | | |
110 | | #define unchecked_curl_easy_setopt(handle, opt, param) \ |
111 | | CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param)) |
112 | | |
113 | | /***********************************************************รน************/ |
114 | | /* VSICurlAuthParametersChanged() */ |
115 | | /************************************************************************/ |
116 | | |
117 | | static unsigned int gnGenerationAuthParameters = 0; |
118 | | |
119 | | void VSICurlAuthParametersChanged() |
120 | | { |
121 | | gnGenerationAuthParameters++; |
122 | | } |
123 | | |
124 | | // Do not access those variables directly ! |
125 | | // Use VSICURLGetDownloadChunkSize() and GetMaxRegions() |
126 | | static int N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = 0; |
127 | | static int DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = 0; |
128 | | |
129 | | /************************************************************************/ |
130 | | /* VSICURLReadGlobalEnvVariables() */ |
131 | | /************************************************************************/ |
132 | | |
133 | | static void VSICURLReadGlobalEnvVariables() |
134 | | { |
135 | | struct Initializer |
136 | | { |
137 | | Initializer() |
138 | | { |
139 | | constexpr int DOWNLOAD_CHUNK_SIZE_DEFAULT = 16384; |
140 | | const char *pszChunkSize = |
141 | | CPLGetConfigOption("CPL_VSIL_CURL_CHUNK_SIZE", nullptr); |
142 | | GIntBig nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
143 | | |
144 | | if (pszChunkSize) |
145 | | { |
146 | | if (CPLParseMemorySize(pszChunkSize, &nChunkSize, nullptr) != |
147 | | CE_None) |
148 | | { |
149 | | CPLError( |
150 | | CE_Warning, CPLE_AppDefined, |
151 | | "Could not parse value for CPL_VSIL_CURL_CHUNK_SIZE. " |
152 | | "Using default value of %d instead.", |
153 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
154 | | } |
155 | | } |
156 | | |
157 | | constexpr int MIN_CHUNK_SIZE = 1024; |
158 | | constexpr int MAX_CHUNK_SIZE = 10 * 1024 * 1024; |
159 | | if (nChunkSize < MIN_CHUNK_SIZE || nChunkSize > MAX_CHUNK_SIZE) |
160 | | { |
161 | | nChunkSize = DOWNLOAD_CHUNK_SIZE_DEFAULT; |
162 | | CPLError(CE_Warning, CPLE_AppDefined, |
163 | | "Invalid value for CPL_VSIL_CURL_CHUNK_SIZE. " |
164 | | "Allowed range is [%d, %d]. " |
165 | | "Using CPL_VSIL_CURL_CHUNK_SIZE=%d instead", |
166 | | MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, |
167 | | DOWNLOAD_CHUNK_SIZE_DEFAULT); |
168 | | } |
169 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY = |
170 | | static_cast<int>(nChunkSize); |
171 | | |
172 | | constexpr int N_MAX_REGIONS_DEFAULT = 1000; |
173 | | constexpr int CACHE_SIZE_DEFAULT = |
174 | | N_MAX_REGIONS_DEFAULT * DOWNLOAD_CHUNK_SIZE_DEFAULT; |
175 | | |
176 | | const char *pszCacheSize = |
177 | | CPLGetConfigOption("CPL_VSIL_CURL_CACHE_SIZE", nullptr); |
178 | | GIntBig nCacheSize = CACHE_SIZE_DEFAULT; |
179 | | |
180 | | if (pszCacheSize) |
181 | | { |
182 | | if (CPLParseMemorySize(pszCacheSize, &nCacheSize, nullptr) != |
183 | | CE_None) |
184 | | { |
185 | | CPLError( |
186 | | CE_Warning, CPLE_AppDefined, |
187 | | "Could not parse value for CPL_VSIL_CURL_CACHE_SIZE. " |
188 | | "Using default value of " CPL_FRMT_GIB " instead.", |
189 | | nCacheSize); |
190 | | } |
191 | | } |
192 | | |
193 | | const auto nMaxRAM = CPLGetUsablePhysicalRAM(); |
194 | | const auto nMinVal = DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
195 | | auto nMaxVal = static_cast<GIntBig>(INT_MAX) * |
196 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
197 | | if (nMaxRAM > 0 && nMaxVal > nMaxRAM) |
198 | | nMaxVal = nMaxRAM; |
199 | | if (nCacheSize < nMinVal || nCacheSize > nMaxVal) |
200 | | { |
201 | | nCacheSize = nCacheSize < nMinVal ? nMinVal : nMaxVal; |
202 | | CPLError(CE_Warning, CPLE_AppDefined, |
203 | | "Invalid value for CPL_VSIL_CURL_CACHE_SIZE. " |
204 | | "Allowed range is [%d, " CPL_FRMT_GIB "]. " |
205 | | "Using CPL_VSIL_CURL_CACHE_SIZE=" CPL_FRMT_GIB |
206 | | " instead", |
207 | | nMinVal, nMaxVal, nCacheSize); |
208 | | } |
209 | | N_MAX_REGIONS_DO_NOT_USE_DIRECTLY = std::max( |
210 | | 1, static_cast<int>(nCacheSize / |
211 | | DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY)); |
212 | | } |
213 | | }; |
214 | | |
215 | | static Initializer initializer; |
216 | | } |
217 | | |
218 | | /************************************************************************/ |
219 | | /* VSICURLGetDownloadChunkSize() */ |
220 | | /************************************************************************/ |
221 | | |
222 | | int VSICURLGetDownloadChunkSize() |
223 | | { |
224 | | VSICURLReadGlobalEnvVariables(); |
225 | | return DOWNLOAD_CHUNK_SIZE_DO_NOT_USE_DIRECTLY; |
226 | | } |
227 | | |
228 | | /************************************************************************/ |
229 | | /* GetMaxRegions() */ |
230 | | /************************************************************************/ |
231 | | |
232 | | static int GetMaxRegions() |
233 | | { |
234 | | VSICURLReadGlobalEnvVariables(); |
235 | | return N_MAX_REGIONS_DO_NOT_USE_DIRECTLY; |
236 | | } |
237 | | |
238 | | /************************************************************************/ |
239 | | /* VSICurlFindStringSensitiveExceptEscapeSequences() */ |
240 | | /************************************************************************/ |
241 | | |
242 | | static int |
243 | | VSICurlFindStringSensitiveExceptEscapeSequences(char **papszList, |
244 | | const char *pszTarget) |
245 | | |
246 | | { |
247 | | if (papszList == nullptr) |
248 | | return -1; |
249 | | |
250 | | for (int i = 0; papszList[i] != nullptr; i++) |
251 | | { |
252 | | const char *pszIter1 = papszList[i]; |
253 | | const char *pszIter2 = pszTarget; |
254 | | char ch1 = '\0'; |
255 | | char ch2 = '\0'; |
256 | | /* The comparison is case-sensitive, escape for escaped */ |
257 | | /* sequences where letters of the hexadecimal sequence */ |
258 | | /* can be uppercase or lowercase depending on the quoting algorithm */ |
259 | | while (true) |
260 | | { |
261 | | ch1 = *pszIter1; |
262 | | ch2 = *pszIter2; |
263 | | if (ch1 == '\0' || ch2 == '\0') |
264 | | break; |
265 | | if (ch1 == '%' && ch2 == '%' && pszIter1[1] != '\0' && |
266 | | pszIter1[2] != '\0' && pszIter2[1] != '\0' && |
267 | | pszIter2[2] != '\0') |
268 | | { |
269 | | if (!EQUALN(pszIter1 + 1, pszIter2 + 1, 2)) |
270 | | break; |
271 | | pszIter1 += 2; |
272 | | pszIter2 += 2; |
273 | | } |
274 | | if (ch1 != ch2) |
275 | | break; |
276 | | pszIter1++; |
277 | | pszIter2++; |
278 | | } |
279 | | if (ch1 == ch2 && ch1 == '\0') |
280 | | return i; |
281 | | } |
282 | | |
283 | | return -1; |
284 | | } |
285 | | |
286 | | /************************************************************************/ |
287 | | /* VSICurlIsFileInList() */ |
288 | | /************************************************************************/ |
289 | | |
290 | | static int VSICurlIsFileInList(char **papszList, const char *pszTarget) |
291 | | { |
292 | | int nRet = |
293 | | VSICurlFindStringSensitiveExceptEscapeSequences(papszList, pszTarget); |
294 | | if (nRet >= 0) |
295 | | return nRet; |
296 | | |
297 | | // If we didn't find anything, try to URL-escape the target filename. |
298 | | char *pszEscaped = CPLEscapeString(pszTarget, -1, CPLES_URL); |
299 | | if (strcmp(pszTarget, pszEscaped) != 0) |
300 | | { |
301 | | nRet = VSICurlFindStringSensitiveExceptEscapeSequences(papszList, |
302 | | pszEscaped); |
303 | | } |
304 | | CPLFree(pszEscaped); |
305 | | return nRet; |
306 | | } |
307 | | |
308 | | /************************************************************************/ |
309 | | /* VSICurlGetURLFromFilename() */ |
310 | | /************************************************************************/ |
311 | | |
312 | | static std::string VSICurlGetURLFromFilename( |
313 | | const char *pszFilename, CPLHTTPRetryParameters *poRetryParameters, |
314 | | bool *pbUseHead, bool *pbUseRedirectURLIfNoQueryStringParams, |
315 | | bool *pbListDir, bool *pbEmptyDir, CPLStringList *paosHTTPOptions, |
316 | | bool *pbPlanetaryComputerURLSigning, char **ppszPlanetaryComputerCollection) |
317 | | { |
318 | | if (ppszPlanetaryComputerCollection) |
319 | | *ppszPlanetaryComputerCollection = nullptr; |
320 | | |
321 | | if (!STARTS_WITH(pszFilename, "/vsicurl/") && |
322 | | !STARTS_WITH(pszFilename, "/vsicurl?")) |
323 | | return pszFilename; |
324 | | |
325 | | if (pbPlanetaryComputerURLSigning) |
326 | | { |
327 | | // It may be more convenient sometimes to store Planetary Computer URL |
328 | | // signing as a per-path specific option rather than capturing it in |
329 | | // the filename with the &pc_url_signing=yes option. |
330 | | if (CPLTestBool(VSIGetPathSpecificOption( |
331 | | pszFilename, "VSICURL_PC_URL_SIGNING", "FALSE"))) |
332 | | { |
333 | | *pbPlanetaryComputerURLSigning = true; |
334 | | } |
335 | | } |
336 | | |
337 | | pszFilename += strlen("/vsicurl/"); |
338 | | if (!STARTS_WITH(pszFilename, "http://") && |
339 | | !STARTS_WITH(pszFilename, "https://") && |
340 | | !STARTS_WITH(pszFilename, "ftp://") && |
341 | | !STARTS_WITH(pszFilename, "file://")) |
342 | | { |
343 | | if (*pszFilename == '?') |
344 | | pszFilename++; |
345 | | char **papszTokens = CSLTokenizeString2(pszFilename, "&", 0); |
346 | | for (int i = 0; papszTokens[i] != nullptr; i++) |
347 | | { |
348 | | char *pszUnescaped = |
349 | | CPLUnescapeString(papszTokens[i], nullptr, CPLES_URL); |
350 | | CPLFree(papszTokens[i]); |
351 | | papszTokens[i] = pszUnescaped; |
352 | | } |
353 | | |
354 | | std::string osURL; |
355 | | std::string osHeaders; |
356 | | for (int i = 0; papszTokens[i]; i++) |
357 | | { |
358 | | char *pszKey = nullptr; |
359 | | const char *pszValue = CPLParseNameValue(papszTokens[i], &pszKey); |
360 | | if (pszKey && pszValue) |
361 | | { |
362 | | if (EQUAL(pszKey, "max_retry")) |
363 | | { |
364 | | if (poRetryParameters) |
365 | | poRetryParameters->nMaxRetry = atoi(pszValue); |
366 | | } |
367 | | else if (EQUAL(pszKey, "retry_delay")) |
368 | | { |
369 | | if (poRetryParameters) |
370 | | poRetryParameters->dfInitialDelay = CPLAtof(pszValue); |
371 | | } |
372 | | else if (EQUAL(pszKey, "retry_codes")) |
373 | | { |
374 | | if (poRetryParameters) |
375 | | poRetryParameters->osRetryCodes = pszValue; |
376 | | } |
377 | | else if (EQUAL(pszKey, "use_head")) |
378 | | { |
379 | | if (pbUseHead) |
380 | | *pbUseHead = CPLTestBool(pszValue); |
381 | | } |
382 | | else if (EQUAL(pszKey, |
383 | | "use_redirect_url_if_no_query_string_params")) |
384 | | { |
385 | | /* Undocumented. Used by PLScenes driver */ |
386 | | if (pbUseRedirectURLIfNoQueryStringParams) |
387 | | *pbUseRedirectURLIfNoQueryStringParams = |
388 | | CPLTestBool(pszValue); |
389 | | } |
390 | | else if (EQUAL(pszKey, "list_dir")) |
391 | | { |
392 | | if (pbListDir) |
393 | | *pbListDir = CPLTestBool(pszValue); |
394 | | } |
395 | | else if (EQUAL(pszKey, "empty_dir")) |
396 | | { |
397 | | if (pbEmptyDir) |
398 | | *pbEmptyDir = CPLTestBool(pszValue); |
399 | | } |
400 | | else if (EQUAL(pszKey, "useragent") || |
401 | | EQUAL(pszKey, "referer") || EQUAL(pszKey, "cookie") || |
402 | | EQUAL(pszKey, "header_file") || |
403 | | EQUAL(pszKey, "unsafessl") || |
404 | | #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION |
405 | | EQUAL(pszKey, "timeout") || |
406 | | EQUAL(pszKey, "connecttimeout") || |
407 | | #endif |
408 | | EQUAL(pszKey, "low_speed_time") || |
409 | | EQUAL(pszKey, "low_speed_limit") || |
410 | | EQUAL(pszKey, "proxy") || EQUAL(pszKey, "proxyauth") || |
411 | | EQUAL(pszKey, "proxyuserpwd")) |
412 | | { |
413 | | // Above names are the ones supported by |
414 | | // CPLHTTPSetOptions() |
415 | | if (paosHTTPOptions) |
416 | | { |
417 | | paosHTTPOptions->SetNameValue(pszKey, pszValue); |
418 | | } |
419 | | } |
420 | | else if (EQUAL(pszKey, "url")) |
421 | | { |
422 | | osURL = pszValue; |
423 | | } |
424 | | else if (EQUAL(pszKey, "pc_url_signing")) |
425 | | { |
426 | | if (pbPlanetaryComputerURLSigning) |
427 | | *pbPlanetaryComputerURLSigning = CPLTestBool(pszValue); |
428 | | } |
429 | | else if (EQUAL(pszKey, "pc_collection")) |
430 | | { |
431 | | if (ppszPlanetaryComputerCollection) |
432 | | { |
433 | | CPLFree(*ppszPlanetaryComputerCollection); |
434 | | *ppszPlanetaryComputerCollection = CPLStrdup(pszValue); |
435 | | } |
436 | | } |
437 | | else if (STARTS_WITH(pszKey, "header.")) |
438 | | { |
439 | | osHeaders += (pszKey + strlen("header.")); |
440 | | osHeaders += ':'; |
441 | | osHeaders += pszValue; |
442 | | osHeaders += "\r\n"; |
443 | | } |
444 | | else |
445 | | { |
446 | | CPLError(CE_Warning, CPLE_NotSupported, |
447 | | "Unsupported option: %s", pszKey); |
448 | | } |
449 | | } |
450 | | CPLFree(pszKey); |
451 | | } |
452 | | |
453 | | if (paosHTTPOptions && !osHeaders.empty()) |
454 | | paosHTTPOptions->SetNameValue("HEADERS", osHeaders.c_str()); |
455 | | |
456 | | CSLDestroy(papszTokens); |
457 | | if (osURL.empty()) |
458 | | { |
459 | | CPLError(CE_Failure, CPLE_IllegalArg, "Missing url parameter"); |
460 | | return pszFilename; |
461 | | } |
462 | | |
463 | | return osURL; |
464 | | } |
465 | | |
466 | | return pszFilename; |
467 | | } |
468 | | |
469 | | namespace cpl |
470 | | { |
471 | | |
472 | | /************************************************************************/ |
473 | | /* VSICurlHandle() */ |
474 | | /************************************************************************/ |
475 | | |
476 | | VSICurlHandle::VSICurlHandle(VSICurlFilesystemHandlerBase *poFSIn, |
477 | | const char *pszFilename, const char *pszURLIn) |
478 | | : poFS(poFSIn), m_osFilename(pszFilename), |
479 | | m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv(pszFilename)), |
480 | | m_oRetryParameters(m_aosHTTPOptions), |
481 | | m_bUseHead( |
482 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_USE_HEAD", "YES"))) |
483 | | { |
484 | | if (pszURLIn) |
485 | | { |
486 | | m_pszURL = CPLStrdup(pszURLIn); |
487 | | } |
488 | | else |
489 | | { |
490 | | char *pszPCCollection = nullptr; |
491 | | m_pszURL = |
492 | | CPLStrdup(VSICurlGetURLFromFilename( |
493 | | pszFilename, &m_oRetryParameters, &m_bUseHead, |
494 | | &m_bUseRedirectURLIfNoQueryStringParams, nullptr, |
495 | | nullptr, &m_aosHTTPOptions, |
496 | | &m_bPlanetaryComputerURLSigning, &pszPCCollection) |
497 | | .c_str()); |
498 | | if (pszPCCollection) |
499 | | m_osPlanetaryComputerCollection = pszPCCollection; |
500 | | CPLFree(pszPCCollection); |
501 | | } |
502 | | |
503 | | m_bCached = poFSIn->AllowCachedDataFor(pszFilename); |
504 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
505 | | } |
506 | | |
507 | | /************************************************************************/ |
508 | | /* ~VSICurlHandle() */ |
509 | | /************************************************************************/ |
510 | | |
511 | | VSICurlHandle::~VSICurlHandle() |
512 | | { |
513 | | if (m_oThreadAdviseRead.joinable()) |
514 | | { |
515 | | m_oThreadAdviseRead.join(); |
516 | | } |
517 | | if (m_hCurlMultiHandleForAdviseRead) |
518 | | { |
519 | | curl_multi_cleanup(m_hCurlMultiHandleForAdviseRead); |
520 | | } |
521 | | |
522 | | if (!m_bCached) |
523 | | { |
524 | | poFS->InvalidateCachedData(m_pszURL); |
525 | | poFS->InvalidateDirContent(CPLGetDirnameSafe(m_osFilename.c_str())); |
526 | | } |
527 | | CPLFree(m_pszURL); |
528 | | } |
529 | | |
530 | | /************************************************************************/ |
531 | | /* SetURL() */ |
532 | | /************************************************************************/ |
533 | | |
534 | | void VSICurlHandle::SetURL(const char *pszURLIn) |
535 | | { |
536 | | CPLFree(m_pszURL); |
537 | | m_pszURL = CPLStrdup(pszURLIn); |
538 | | } |
539 | | |
540 | | /************************************************************************/ |
541 | | /* InstallReadCbk() */ |
542 | | /************************************************************************/ |
543 | | |
544 | | int VSICurlHandle::InstallReadCbk(VSICurlReadCbkFunc pfnReadCbkIn, |
545 | | void *pfnUserDataIn, |
546 | | int bStopOnInterruptUntilUninstallIn) |
547 | | { |
548 | | if (pfnReadCbk != nullptr) |
549 | | return FALSE; |
550 | | |
551 | | pfnReadCbk = pfnReadCbkIn; |
552 | | pReadCbkUserData = pfnUserDataIn; |
553 | | bStopOnInterruptUntilUninstall = |
554 | | CPL_TO_BOOL(bStopOnInterruptUntilUninstallIn); |
555 | | bInterrupted = false; |
556 | | return TRUE; |
557 | | } |
558 | | |
559 | | /************************************************************************/ |
560 | | /* UninstallReadCbk() */ |
561 | | /************************************************************************/ |
562 | | |
563 | | int VSICurlHandle::UninstallReadCbk() |
564 | | { |
565 | | if (pfnReadCbk == nullptr) |
566 | | return FALSE; |
567 | | |
568 | | pfnReadCbk = nullptr; |
569 | | pReadCbkUserData = nullptr; |
570 | | bStopOnInterruptUntilUninstall = false; |
571 | | bInterrupted = false; |
572 | | return TRUE; |
573 | | } |
574 | | |
575 | | /************************************************************************/ |
576 | | /* Seek() */ |
577 | | /************************************************************************/ |
578 | | |
579 | | int VSICurlHandle::Seek(vsi_l_offset nOffset, int nWhence) |
580 | | { |
581 | | if (nWhence == SEEK_SET) |
582 | | { |
583 | | curOffset = nOffset; |
584 | | } |
585 | | else if (nWhence == SEEK_CUR) |
586 | | { |
587 | | curOffset = curOffset + nOffset; |
588 | | } |
589 | | else |
590 | | { |
591 | | curOffset = GetFileSize(false) + nOffset; |
592 | | } |
593 | | bEOF = false; |
594 | | return 0; |
595 | | } |
596 | | |
597 | | } // namespace cpl |
598 | | |
599 | | /************************************************************************/ |
600 | | /* VSICurlGetTimeStampFromRFC822DateTime() */ |
601 | | /************************************************************************/ |
602 | | |
603 | | static GIntBig VSICurlGetTimeStampFromRFC822DateTime(const char *pszDT) |
604 | | { |
605 | | // Sun, 03 Apr 2016 12:07:27 GMT |
606 | | if (strlen(pszDT) >= 5 && pszDT[3] == ',' && pszDT[4] == ' ') |
607 | | pszDT += 5; |
608 | | int nDay = 0; |
609 | | int nYear = 0; |
610 | | int nHour = 0; |
611 | | int nMinute = 0; |
612 | | int nSecond = 0; |
613 | | char szMonth[4] = {}; |
614 | | szMonth[3] = 0; |
615 | | if (sscanf(pszDT, "%02d %03s %04d %02d:%02d:%02d GMT", &nDay, szMonth, |
616 | | &nYear, &nHour, &nMinute, &nSecond) == 6) |
617 | | { |
618 | | static const char *const aszMonthStr[] = {"Jan", "Feb", "Mar", "Apr", |
619 | | "May", "Jun", "Jul", "Aug", |
620 | | "Sep", "Oct", "Nov", "Dec"}; |
621 | | |
622 | | int nMonthIdx0 = -1; |
623 | | for (int i = 0; i < 12; i++) |
624 | | { |
625 | | if (EQUAL(szMonth, aszMonthStr[i])) |
626 | | { |
627 | | nMonthIdx0 = i; |
628 | | break; |
629 | | } |
630 | | } |
631 | | if (nMonthIdx0 >= 0) |
632 | | { |
633 | | struct tm brokendowntime; |
634 | | brokendowntime.tm_year = nYear - 1900; |
635 | | brokendowntime.tm_mon = nMonthIdx0; |
636 | | brokendowntime.tm_mday = nDay; |
637 | | brokendowntime.tm_hour = nHour; |
638 | | brokendowntime.tm_min = nMinute; |
639 | | brokendowntime.tm_sec = nSecond; |
640 | | return CPLYMDHMSToUnixTime(&brokendowntime); |
641 | | } |
642 | | } |
643 | | return 0; |
644 | | } |
645 | | |
646 | | /************************************************************************/ |
647 | | /* VSICURLInitWriteFuncStruct() */ |
648 | | /************************************************************************/ |
649 | | |
650 | | void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp, |
651 | | VSICurlReadCbkFunc pfnReadCbk, |
652 | | void *pReadCbkUserData) |
653 | | { |
654 | | psStruct->pBuffer = nullptr; |
655 | | psStruct->nSize = 0; |
656 | | psStruct->bIsHTTP = false; |
657 | | psStruct->bMultiRange = false; |
658 | | psStruct->nStartOffset = 0; |
659 | | psStruct->nEndOffset = 0; |
660 | | psStruct->nHTTPCode = 0; |
661 | | psStruct->nFirstHTTPCode = 0; |
662 | | psStruct->nContentLength = 0; |
663 | | psStruct->bFoundContentRange = false; |
664 | | psStruct->bError = false; |
665 | | psStruct->bDetectRangeDownloadingError = true; |
666 | | psStruct->nTimestampDate = 0; |
667 | | |
668 | | psStruct->fp = fp; |
669 | | psStruct->pfnReadCbk = pfnReadCbk; |
670 | | psStruct->pReadCbkUserData = pReadCbkUserData; |
671 | | psStruct->bInterrupted = false; |
672 | | } |
673 | | |
674 | | /************************************************************************/ |
675 | | /* VSICurlHandleWriteFunc() */ |
676 | | /************************************************************************/ |
677 | | |
678 | | size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, |
679 | | void *req) |
680 | | { |
681 | | cpl::WriteFuncStruct *psStruct = static_cast<cpl::WriteFuncStruct *>(req); |
682 | | const size_t nSize = count * nmemb; |
683 | | |
684 | | if (psStruct->bInterrupted) |
685 | | { |
686 | | return 0; |
687 | | } |
688 | | |
689 | | char *pNewBuffer = static_cast<char *>( |
690 | | VSIRealloc(psStruct->pBuffer, psStruct->nSize + nSize + 1)); |
691 | | if (pNewBuffer) |
692 | | { |
693 | | psStruct->pBuffer = pNewBuffer; |
694 | | memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize); |
695 | | psStruct->pBuffer[psStruct->nSize + nSize] = '\0'; |
696 | | if (psStruct->bIsHTTP) |
697 | | { |
698 | | char *pszLine = psStruct->pBuffer + psStruct->nSize; |
699 | | if (STARTS_WITH_CI(pszLine, "HTTP/")) |
700 | | { |
701 | | char *pszSpace = strchr(pszLine, ' '); |
702 | | if (pszSpace) |
703 | | { |
704 | | const int nHTTPCode = atoi(pszSpace + 1); |
705 | | if (psStruct->nFirstHTTPCode == 0) |
706 | | psStruct->nFirstHTTPCode = nHTTPCode; |
707 | | psStruct->nHTTPCode = nHTTPCode; |
708 | | } |
709 | | } |
710 | | else if (STARTS_WITH_CI(pszLine, "Content-Length: ")) |
711 | | { |
712 | | psStruct->nContentLength = CPLScanUIntBig( |
713 | | pszLine + 16, static_cast<int>(strlen(pszLine + 16))); |
714 | | } |
715 | | else if (STARTS_WITH_CI(pszLine, "Content-Range: ")) |
716 | | { |
717 | | psStruct->bFoundContentRange = true; |
718 | | } |
719 | | else if (STARTS_WITH_CI(pszLine, "Date: ")) |
720 | | { |
721 | | CPLString osDate = pszLine + strlen("Date: "); |
722 | | size_t nSizeLine = osDate.size(); |
723 | | while (nSizeLine && (osDate[nSizeLine - 1] == '\r' || |
724 | | osDate[nSizeLine - 1] == '\n')) |
725 | | { |
726 | | osDate.resize(nSizeLine - 1); |
727 | | nSizeLine--; |
728 | | } |
729 | | osDate.Trim(); |
730 | | |
731 | | GIntBig nTimestampDate = |
732 | | VSICurlGetTimeStampFromRFC822DateTime(osDate.c_str()); |
733 | | #if DEBUG_VERBOSE |
734 | | CPLDebug("VSICURL", "Timestamp = " CPL_FRMT_GIB, |
735 | | nTimestampDate); |
736 | | #endif |
737 | | psStruct->nTimestampDate = nTimestampDate; |
738 | | } |
739 | | /*if( nSize > 2 && pszLine[nSize - 2] == '\r' && |
740 | | pszLine[nSize - 1] == '\n' ) |
741 | | { |
742 | | pszLine[nSize - 2] = 0; |
743 | | CPLDebug("VSICURL", "%s", pszLine); |
744 | | pszLine[nSize - 2] = '\r'; |
745 | | }*/ |
746 | | |
747 | | if (pszLine[0] == '\r' && pszLine[1] == '\n') |
748 | | { |
749 | | // Detect servers that don't support range downloading. |
750 | | if (psStruct->nHTTPCode == 200 && |
751 | | psStruct->bDetectRangeDownloadingError && |
752 | | !psStruct->bMultiRange && !psStruct->bFoundContentRange && |
753 | | (psStruct->nStartOffset != 0 || |
754 | | psStruct->nContentLength > |
755 | | 10 * (psStruct->nEndOffset - psStruct->nStartOffset + |
756 | | 1))) |
757 | | { |
758 | | CPLError(CE_Failure, CPLE_AppDefined, |
759 | | "Range downloading not supported by this " |
760 | | "server!"); |
761 | | psStruct->bError = true; |
762 | | return 0; |
763 | | } |
764 | | } |
765 | | } |
766 | | else |
767 | | { |
768 | | if (psStruct->pfnReadCbk) |
769 | | { |
770 | | if (!psStruct->pfnReadCbk(psStruct->fp, buffer, nSize, |
771 | | psStruct->pReadCbkUserData)) |
772 | | { |
773 | | psStruct->bInterrupted = true; |
774 | | return 0; |
775 | | } |
776 | | } |
777 | | } |
778 | | psStruct->nSize += nSize; |
779 | | return nmemb; |
780 | | } |
781 | | else |
782 | | { |
783 | | return 0; |
784 | | } |
785 | | } |
786 | | |
787 | | /************************************************************************/ |
788 | | /* VSICurlIsS3LikeSignedURL() */ |
789 | | /************************************************************************/ |
790 | | |
791 | | static bool VSICurlIsS3LikeSignedURL(const char *pszURL) |
792 | | { |
793 | | return ((strstr(pszURL, ".s3.amazonaws.com/") != nullptr || |
794 | | strstr(pszURL, ".s3.amazonaws.com:") != nullptr || |
795 | | strstr(pszURL, ".storage.googleapis.com/") != nullptr || |
796 | | strstr(pszURL, ".storage.googleapis.com:") != nullptr || |
797 | | strstr(pszURL, ".cloudfront.net/") != nullptr || |
798 | | strstr(pszURL, ".cloudfront.net:") != nullptr) && |
799 | | (strstr(pszURL, "&Signature=") != nullptr || |
800 | | strstr(pszURL, "?Signature=") != nullptr)) || |
801 | | strstr(pszURL, "&X-Amz-Signature=") != nullptr || |
802 | | strstr(pszURL, "?X-Amz-Signature=") != nullptr; |
803 | | } |
804 | | |
805 | | /************************************************************************/ |
806 | | /* VSICurlGetExpiresFromS3LikeSignedURL() */ |
807 | | /************************************************************************/ |
808 | | |
809 | | static GIntBig VSICurlGetExpiresFromS3LikeSignedURL(const char *pszURL) |
810 | | { |
811 | | const auto GetParamValue = [pszURL](const char *pszKey) -> const char * |
812 | | { |
813 | | for (const char *pszPrefix : {"&", "?"}) |
814 | | { |
815 | | std::string osNeedle(pszPrefix); |
816 | | osNeedle += pszKey; |
817 | | osNeedle += '='; |
818 | | const char *pszStr = strstr(pszURL, osNeedle.c_str()); |
819 | | if (pszStr) |
820 | | return pszStr + osNeedle.size(); |
821 | | } |
822 | | return nullptr; |
823 | | }; |
824 | | |
825 | | { |
826 | | // Expires= is a Unix timestamp |
827 | | const char *pszExpires = GetParamValue("Expires"); |
828 | | if (pszExpires != nullptr) |
829 | | return CPLAtoGIntBig(pszExpires); |
830 | | } |
831 | | |
832 | | // X-Amz-Expires= is a delay, to be combined with X-Amz-Date= |
833 | | const char *pszAmzExpires = GetParamValue("X-Amz-Expires"); |
834 | | if (pszAmzExpires == nullptr) |
835 | | return 0; |
836 | | const int nDelay = atoi(pszAmzExpires); |
837 | | |
838 | | const char *pszAmzDate = GetParamValue("X-Amz-Date"); |
839 | | if (pszAmzDate == nullptr) |
840 | | return 0; |
841 | | // pszAmzDate should be YYYYMMDDTHHMMSSZ |
842 | | if (strlen(pszAmzDate) < strlen("YYYYMMDDTHHMMSSZ")) |
843 | | return 0; |
844 | | if (pszAmzDate[strlen("YYYYMMDDTHHMMSSZ") - 1] != 'Z') |
845 | | return 0; |
846 | | struct tm brokendowntime; |
847 | | brokendowntime.tm_year = |
848 | | atoi(std::string(pszAmzDate).substr(0, 4).c_str()) - 1900; |
849 | | brokendowntime.tm_mon = |
850 | | atoi(std::string(pszAmzDate).substr(4, 2).c_str()) - 1; |
851 | | brokendowntime.tm_mday = atoi(std::string(pszAmzDate).substr(6, 2).c_str()); |
852 | | brokendowntime.tm_hour = atoi(std::string(pszAmzDate).substr(9, 2).c_str()); |
853 | | brokendowntime.tm_min = atoi(std::string(pszAmzDate).substr(11, 2).c_str()); |
854 | | brokendowntime.tm_sec = atoi(std::string(pszAmzDate).substr(13, 2).c_str()); |
855 | | return CPLYMDHMSToUnixTime(&brokendowntime) + nDelay; |
856 | | } |
857 | | |
858 | | /************************************************************************/ |
859 | | /* VSICURLMultiPerform() */ |
860 | | /************************************************************************/ |
861 | | |
862 | | void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle, |
863 | | std::atomic<bool> *pbInterrupt) |
864 | | { |
865 | | int repeats = 0; |
866 | | |
867 | | if (hEasyHandle) |
868 | | curl_multi_add_handle(hCurlMultiHandle, hEasyHandle); |
869 | | |
870 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
871 | | while (true) |
872 | | { |
873 | | int still_running; |
874 | | while (curl_multi_perform(hCurlMultiHandle, &still_running) == |
875 | | CURLM_CALL_MULTI_PERFORM) |
876 | | { |
877 | | // loop |
878 | | } |
879 | | if (!still_running) |
880 | | { |
881 | | break; |
882 | | } |
883 | | |
884 | | #ifdef undef |
885 | | CURLMsg *msg; |
886 | | do |
887 | | { |
888 | | int msgq = 0; |
889 | | msg = curl_multi_info_read(hCurlMultiHandle, &msgq); |
890 | | if (msg && (msg->msg == CURLMSG_DONE)) |
891 | | { |
892 | | CURL *e = msg->easy_handle; |
893 | | } |
894 | | } while (msg); |
895 | | #endif |
896 | | |
897 | | CPLMultiPerformWait(hCurlMultiHandle, repeats); |
898 | | |
899 | | if (pbInterrupt && *pbInterrupt) |
900 | | break; |
901 | | } |
902 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
903 | | |
904 | | if (hEasyHandle) |
905 | | curl_multi_remove_handle(hCurlMultiHandle, hEasyHandle); |
906 | | } |
907 | | |
908 | | /************************************************************************/ |
909 | | /* VSICurlDummyWriteFunc() */ |
910 | | /************************************************************************/ |
911 | | |
912 | | static size_t VSICurlDummyWriteFunc(void *, size_t, size_t, void *) |
913 | | { |
914 | | return 0; |
915 | | } |
916 | | |
917 | | /************************************************************************/ |
918 | | /* VSICURLResetHeaderAndWriterFunctions() */ |
919 | | /************************************************************************/ |
920 | | |
921 | | void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle) |
922 | | { |
923 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
924 | | VSICurlDummyWriteFunc); |
925 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
926 | | VSICurlDummyWriteFunc); |
927 | | } |
928 | | |
929 | | /************************************************************************/ |
930 | | /* Iso8601ToUnixTime() */ |
931 | | /************************************************************************/ |
932 | | |
933 | | static bool Iso8601ToUnixTime(const char *pszDT, GIntBig *pnUnixTime) |
934 | | { |
935 | | int nYear; |
936 | | int nMonth; |
937 | | int nDay; |
938 | | int nHour; |
939 | | int nMinute; |
940 | | int nSecond; |
941 | | if (sscanf(pszDT, "%04d-%02d-%02dT%02d:%02d:%02d", &nYear, &nMonth, &nDay, |
942 | | &nHour, &nMinute, &nSecond) == 6) |
943 | | { |
944 | | struct tm brokendowntime; |
945 | | brokendowntime.tm_year = nYear - 1900; |
946 | | brokendowntime.tm_mon = nMonth - 1; |
947 | | brokendowntime.tm_mday = nDay; |
948 | | brokendowntime.tm_hour = nHour; |
949 | | brokendowntime.tm_min = nMinute; |
950 | | brokendowntime.tm_sec = nSecond; |
951 | | *pnUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
952 | | return true; |
953 | | } |
954 | | return false; |
955 | | } |
956 | | |
957 | | namespace cpl |
958 | | { |
959 | | |
960 | | /************************************************************************/ |
961 | | /* ManagePlanetaryComputerSigning() */ |
962 | | /************************************************************************/ |
963 | | |
964 | | void VSICurlHandle::ManagePlanetaryComputerSigning() const |
965 | | { |
966 | | // Take global lock |
967 | | static std::mutex goMutex; |
968 | | std::lock_guard<std::mutex> oLock(goMutex); |
969 | | |
970 | | struct PCSigningInfo |
971 | | { |
972 | | std::string osQueryString{}; |
973 | | GIntBig nExpireTimestamp = 0; |
974 | | }; |
975 | | |
976 | | PCSigningInfo sSigningInfo; |
977 | | constexpr int knExpirationDelayMargin = 60; |
978 | | |
979 | | if (!m_osPlanetaryComputerCollection.empty()) |
980 | | { |
981 | | // key is the name of a collection |
982 | | static lru11::Cache<std::string, PCSigningInfo> goCacheCollection{1024}; |
983 | | |
984 | | if (goCacheCollection.tryGet(m_osPlanetaryComputerCollection, |
985 | | sSigningInfo) && |
986 | | time(nullptr) + knExpirationDelayMargin <= |
987 | | sSigningInfo.nExpireTimestamp) |
988 | | { |
989 | | m_osQueryString = sSigningInfo.osQueryString; |
990 | | } |
991 | | else |
992 | | { |
993 | | const auto psResult = |
994 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
995 | | "VSICURL_PC_SAS_TOKEN_URL", |
996 | | "https://planetarycomputer.microsoft.com/api/" |
997 | | "sas/v1/token/")) + |
998 | | m_osPlanetaryComputerCollection) |
999 | | .c_str(), |
1000 | | nullptr); |
1001 | | if (psResult) |
1002 | | { |
1003 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1004 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1005 | | const char *pszToken = aosKeyVals.FetchNameValue("token"); |
1006 | | if (pszToken) |
1007 | | { |
1008 | | m_osQueryString = '?'; |
1009 | | m_osQueryString += pszToken; |
1010 | | |
1011 | | sSigningInfo.osQueryString = m_osQueryString; |
1012 | | sSigningInfo.nExpireTimestamp = 0; |
1013 | | const char *pszExpiry = |
1014 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1015 | | if (pszExpiry) |
1016 | | { |
1017 | | Iso8601ToUnixTime(pszExpiry, |
1018 | | &sSigningInfo.nExpireTimestamp); |
1019 | | } |
1020 | | goCacheCollection.insert(m_osPlanetaryComputerCollection, |
1021 | | sSigningInfo); |
1022 | | |
1023 | | CPLDebug("VSICURL", "Got token from Planetary Computer: %s", |
1024 | | m_osQueryString.c_str()); |
1025 | | } |
1026 | | CPLHTTPDestroyResult(psResult); |
1027 | | } |
1028 | | } |
1029 | | } |
1030 | | else |
1031 | | { |
1032 | | // key is a URL |
1033 | | static lru11::Cache<std::string, PCSigningInfo> goCacheURL{1024}; |
1034 | | |
1035 | | if (goCacheURL.tryGet(m_pszURL, sSigningInfo) && |
1036 | | time(nullptr) + knExpirationDelayMargin <= |
1037 | | sSigningInfo.nExpireTimestamp) |
1038 | | { |
1039 | | m_osQueryString = sSigningInfo.osQueryString; |
1040 | | } |
1041 | | else |
1042 | | { |
1043 | | const auto psResult = |
1044 | | CPLHTTPFetch((std::string(CPLGetConfigOption( |
1045 | | "VSICURL_PC_SAS_SIGN_HREF_URL", |
1046 | | "https://planetarycomputer.microsoft.com/api/" |
1047 | | "sas/v1/sign?href=")) + |
1048 | | m_pszURL) |
1049 | | .c_str(), |
1050 | | nullptr); |
1051 | | if (psResult) |
1052 | | { |
1053 | | const auto aosKeyVals = CPLParseKeyValueJson( |
1054 | | reinterpret_cast<const char *>(psResult->pabyData)); |
1055 | | const char *pszHref = aosKeyVals.FetchNameValue("href"); |
1056 | | if (pszHref && STARTS_WITH(pszHref, m_pszURL)) |
1057 | | { |
1058 | | m_osQueryString = pszHref + strlen(m_pszURL); |
1059 | | |
1060 | | sSigningInfo.osQueryString = m_osQueryString; |
1061 | | sSigningInfo.nExpireTimestamp = 0; |
1062 | | const char *pszExpiry = |
1063 | | aosKeyVals.FetchNameValue("msft:expiry"); |
1064 | | if (pszExpiry) |
1065 | | { |
1066 | | Iso8601ToUnixTime(pszExpiry, |
1067 | | &sSigningInfo.nExpireTimestamp); |
1068 | | } |
1069 | | goCacheURL.insert(m_pszURL, sSigningInfo); |
1070 | | |
1071 | | CPLDebug("VSICURL", |
1072 | | "Got signature from Planetary Computer: %s", |
1073 | | m_osQueryString.c_str()); |
1074 | | } |
1075 | | CPLHTTPDestroyResult(psResult); |
1076 | | } |
1077 | | } |
1078 | | } |
1079 | | } |
1080 | | |
1081 | | /************************************************************************/ |
1082 | | /* UpdateQueryString() */ |
1083 | | /************************************************************************/ |
1084 | | |
1085 | | void VSICurlHandle::UpdateQueryString() const |
1086 | | { |
1087 | | if (m_bPlanetaryComputerURLSigning) |
1088 | | { |
1089 | | ManagePlanetaryComputerSigning(); |
1090 | | } |
1091 | | else |
1092 | | { |
1093 | | const char *pszQueryString = VSIGetPathSpecificOption( |
1094 | | m_osFilename.c_str(), "VSICURL_QUERY_STRING", nullptr); |
1095 | | if (pszQueryString) |
1096 | | { |
1097 | | if (m_osFilename.back() == '?') |
1098 | | { |
1099 | | if (pszQueryString[0] == '?') |
1100 | | m_osQueryString = pszQueryString + 1; |
1101 | | else |
1102 | | m_osQueryString = pszQueryString; |
1103 | | } |
1104 | | else |
1105 | | { |
1106 | | if (pszQueryString[0] == '?') |
1107 | | m_osQueryString = pszQueryString; |
1108 | | else |
1109 | | { |
1110 | | m_osQueryString = "?"; |
1111 | | m_osQueryString.append(pszQueryString); |
1112 | | } |
1113 | | } |
1114 | | } |
1115 | | } |
1116 | | } |
1117 | | |
1118 | | /************************************************************************/ |
1119 | | /* GetFileSizeOrHeaders() */ |
1120 | | /************************************************************************/ |
1121 | | |
1122 | | vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, |
1123 | | bool bGetHeaders) |
1124 | | { |
1125 | | if (oFileProp.bHasComputedFileSize && !bGetHeaders) |
1126 | | return oFileProp.fileSize; |
1127 | | |
1128 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
1129 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
1130 | | NetworkStatisticsAction oContextAction("GetFileSize"); |
1131 | | |
1132 | | oFileProp.bHasComputedFileSize = true; |
1133 | | |
1134 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
1135 | | |
1136 | | UpdateQueryString(); |
1137 | | |
1138 | | std::string osURL(m_pszURL + m_osQueryString); |
1139 | | bool bRetryWithGet = false; |
1140 | | bool bS3LikeRedirect = false; |
1141 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
1142 | | |
1143 | | retry: |
1144 | | CURL *hCurlHandle = curl_easy_init(); |
1145 | | |
1146 | | struct curl_slist *headers = |
1147 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), m_aosHTTPOptions.List()); |
1148 | | |
1149 | | WriteFuncStruct sWriteFuncHeaderData; |
1150 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
1151 | | nullptr); |
1152 | | sWriteFuncHeaderData.bDetectRangeDownloadingError = false; |
1153 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(osURL.c_str(), "http"); |
1154 | | |
1155 | | WriteFuncStruct sWriteFuncData; |
1156 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
1157 | | |
1158 | | std::string osVerb; |
1159 | | std::string osRange; // leave in this scope ! |
1160 | | int nRoundedBufSize = 0; |
1161 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
1162 | | if (UseLimitRangeGetInsteadOfHead()) |
1163 | | { |
1164 | | osVerb = "GET"; |
1165 | | const int nBufSize = std::max( |
1166 | | 1024, std::min(10 * 1024 * 1024, |
1167 | | atoi(CPLGetConfigOption( |
1168 | | "GDAL_INGESTED_BYTES_AT_OPEN", "1024")))); |
1169 | | nRoundedBufSize = cpl::div_round_up(nBufSize, knDOWNLOAD_CHUNK_SIZE) * |
1170 | | knDOWNLOAD_CHUNK_SIZE; |
1171 | | |
1172 | | // so it gets included in Azure signature |
1173 | | osRange = CPLSPrintf("Range: bytes=0-%d", nRoundedBufSize - 1); |
1174 | | headers = curl_slist_append(headers, osRange.c_str()); |
1175 | | } |
1176 | | // HACK for mbtiles driver: http://a.tiles.mapbox.com/v3/ doesn't accept |
1177 | | // HEAD, as it is a redirect to AWS S3 signed URL, but those are only valid |
1178 | | // for a given type of HTTP request, and thus GET. This is valid for any |
1179 | | // signed URL for AWS S3. |
1180 | | else if (bRetryWithGet || |
1181 | | strstr(osURL.c_str(), ".tiles.mapbox.com/") != nullptr || |
1182 | | VSICurlIsS3LikeSignedURL(osURL.c_str()) || !m_bUseHead) |
1183 | | { |
1184 | | sWriteFuncData.bInterrupted = true; |
1185 | | osVerb = "GET"; |
1186 | | } |
1187 | | else |
1188 | | { |
1189 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1); |
1190 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 0); |
1191 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 1); |
1192 | | osVerb = "HEAD"; |
1193 | | } |
1194 | | |
1195 | | if (!AllowAutomaticRedirection()) |
1196 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
1197 | | |
1198 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
1199 | | &sWriteFuncHeaderData); |
1200 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1201 | | VSICurlHandleWriteFunc); |
1202 | | |
1203 | | // Bug with older curl versions (<=7.16.4) and FTP. |
1204 | | // See http://curl.haxx.se/mail/lib-2007-08/0312.html |
1205 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
1206 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1207 | | VSICurlHandleWriteFunc); |
1208 | | |
1209 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
1210 | | szCurlErrBuf[0] = '\0'; |
1211 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
1212 | | |
1213 | | headers = VSICurlMergeHeaders(headers, GetCurlHeaders(osVerb, headers)); |
1214 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
1215 | | |
1216 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
1217 | | |
1218 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); |
1219 | | |
1220 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
1221 | | |
1222 | | curl_slist_free_all(headers); |
1223 | | |
1224 | | oFileProp.eExists = EXIST_UNKNOWN; |
1225 | | |
1226 | | long mtime = 0; |
1227 | | curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME, &mtime); |
1228 | | |
1229 | | if (osVerb == "GET") |
1230 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
1231 | | else |
1232 | | NetworkStatisticsLogger::LogHEAD(); |
1233 | | |
1234 | | if (STARTS_WITH(osURL.c_str(), "ftp")) |
1235 | | { |
1236 | | if (sWriteFuncData.pBuffer != nullptr) |
1237 | | { |
1238 | | const char *pszContentLength = |
1239 | | strstr(const_cast<const char *>(sWriteFuncData.pBuffer), |
1240 | | "Content-Length: "); |
1241 | | if (pszContentLength) |
1242 | | { |
1243 | | pszContentLength += strlen("Content-Length: "); |
1244 | | oFileProp.eExists = EXIST_YES; |
1245 | | oFileProp.fileSize = |
1246 | | CPLScanUIntBig(pszContentLength, |
1247 | | static_cast<int>(strlen(pszContentLength))); |
1248 | | if (ENABLE_DEBUG) |
1249 | | CPLDebug(poFS->GetDebugKey(), |
1250 | | "GetFileSize(%s)=" CPL_FRMT_GUIB, osURL.c_str(), |
1251 | | oFileProp.fileSize); |
1252 | | } |
1253 | | } |
1254 | | } |
1255 | | |
1256 | | double dfSize = 0; |
1257 | | if (oFileProp.eExists != EXIST_YES) |
1258 | | { |
1259 | | long response_code = 0; |
1260 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
1261 | | |
1262 | | bool bAlreadyLogged = false; |
1263 | | if (response_code >= 400 && szCurlErrBuf[0] == '\0') |
1264 | | { |
1265 | | const bool bLogResponse = |
1266 | | CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")); |
1267 | | if (bLogResponse && sWriteFuncData.pBuffer) |
1268 | | { |
1269 | | const char *pszErrorMsg = |
1270 | | static_cast<const char *>(sWriteFuncData.pBuffer); |
1271 | | bAlreadyLogged = true; |
1272 | | CPLDebug( |
1273 | | poFS->GetDebugKey(), |
1274 | | "GetFileSize(%s): response_code=%d, server error msg=%s", |
1275 | | osURL.c_str(), static_cast<int>(response_code), |
1276 | | pszErrorMsg[0] ? pszErrorMsg : "(no message provided)"); |
1277 | | } |
1278 | | } |
1279 | | else if (szCurlErrBuf[0] != '\0') |
1280 | | { |
1281 | | bAlreadyLogged = true; |
1282 | | CPLDebug(poFS->GetDebugKey(), |
1283 | | "GetFileSize(%s): response_code=%d, curl error msg=%s", |
1284 | | osURL.c_str(), static_cast<int>(response_code), |
1285 | | szCurlErrBuf); |
1286 | | } |
1287 | | |
1288 | | std::string osEffectiveURL; |
1289 | | { |
1290 | | char *pszEffectiveURL = nullptr; |
1291 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
1292 | | &pszEffectiveURL); |
1293 | | if (pszEffectiveURL) |
1294 | | osEffectiveURL = pszEffectiveURL; |
1295 | | } |
1296 | | |
1297 | | if (!osEffectiveURL.empty() && |
1298 | | strstr(osEffectiveURL.c_str(), osURL.c_str()) == nullptr) |
1299 | | { |
1300 | | // Moved permanently ? |
1301 | | if (sWriteFuncHeaderData.nFirstHTTPCode == 301 || |
1302 | | (m_bUseRedirectURLIfNoQueryStringParams && |
1303 | | osEffectiveURL.find('?') == std::string::npos)) |
1304 | | { |
1305 | | CPLDebug(poFS->GetDebugKey(), |
1306 | | "Using effective URL %s permanently", |
1307 | | osEffectiveURL.c_str()); |
1308 | | oFileProp.osRedirectURL = osEffectiveURL; |
1309 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1310 | | } |
1311 | | else |
1312 | | { |
1313 | | CPLDebug(poFS->GetDebugKey(), |
1314 | | "Using effective URL %s temporarily", |
1315 | | osEffectiveURL.c_str()); |
1316 | | } |
1317 | | |
1318 | | // Is this is a redirect to a S3 URL? |
1319 | | if (VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
1320 | | !VSICurlIsS3LikeSignedURL(osURL.c_str())) |
1321 | | { |
1322 | | // Note that this is a redirect as we won't notice after the |
1323 | | // retry. |
1324 | | bS3LikeRedirect = true; |
1325 | | |
1326 | | if (!bRetryWithGet && osVerb == "HEAD" && response_code == 403) |
1327 | | { |
1328 | | CPLDebug(poFS->GetDebugKey(), |
1329 | | "Redirected to a AWS S3 signed URL. Retrying " |
1330 | | "with GET request instead of HEAD since the URL " |
1331 | | "might be valid only for GET"); |
1332 | | bRetryWithGet = true; |
1333 | | osURL = std::move(osEffectiveURL); |
1334 | | CPLFree(sWriteFuncData.pBuffer); |
1335 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1336 | | curl_easy_cleanup(hCurlHandle); |
1337 | | goto retry; |
1338 | | } |
1339 | | } |
1340 | | } |
1341 | | |
1342 | | if (bS3LikeRedirect && response_code >= 200 && response_code < 300 && |
1343 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
1344 | | !osEffectiveURL.empty() && |
1345 | | CPLTestBool( |
1346 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
1347 | | { |
1348 | | const GIntBig nExpireTimestamp = |
1349 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
1350 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
1351 | | { |
1352 | | const int nValidity = static_cast<int>( |
1353 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
1354 | | CPLDebug(poFS->GetDebugKey(), |
1355 | | "Will use redirect URL for the next %d seconds", |
1356 | | nValidity); |
1357 | | // As our local clock might not be in sync with server clock, |
1358 | | // figure out the expiration timestamp in local time |
1359 | | oFileProp.bS3LikeRedirect = true; |
1360 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
1361 | | oFileProp.osRedirectURL = osEffectiveURL; |
1362 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1363 | | } |
1364 | | } |
1365 | | |
1366 | | curl_off_t nSizeTmp = 0; |
1367 | | const CURLcode code = curl_easy_getinfo( |
1368 | | hCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &nSizeTmp); |
1369 | | CPL_IGNORE_RET_VAL(dfSize); |
1370 | | dfSize = static_cast<double>(nSizeTmp); |
1371 | | if (code == 0) |
1372 | | { |
1373 | | oFileProp.eExists = EXIST_YES; |
1374 | | if (dfSize < 0) |
1375 | | { |
1376 | | if (osVerb == "HEAD" && !bRetryWithGet && response_code == 200) |
1377 | | { |
1378 | | CPLDebug( |
1379 | | poFS->GetDebugKey(), |
1380 | | "HEAD did not provide file size. Retrying with GET"); |
1381 | | bRetryWithGet = true; |
1382 | | CPLFree(sWriteFuncData.pBuffer); |
1383 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1384 | | curl_easy_cleanup(hCurlHandle); |
1385 | | goto retry; |
1386 | | } |
1387 | | oFileProp.fileSize = 0; |
1388 | | } |
1389 | | else |
1390 | | oFileProp.fileSize = static_cast<GUIntBig>(dfSize); |
1391 | | } |
1392 | | |
1393 | | if (sWriteFuncHeaderData.pBuffer != nullptr && |
1394 | | (response_code == 200 || response_code == 206)) |
1395 | | { |
1396 | | { |
1397 | | char **papszHeaders = |
1398 | | CSLTokenizeString2(sWriteFuncHeaderData.pBuffer, "\r\n", 0); |
1399 | | for (int i = 0; papszHeaders[i]; ++i) |
1400 | | { |
1401 | | char *pszKey = nullptr; |
1402 | | const char *pszValue = |
1403 | | CPLParseNameValue(papszHeaders[i], &pszKey); |
1404 | | if (pszKey && pszValue) |
1405 | | { |
1406 | | if (bGetHeaders) |
1407 | | { |
1408 | | m_aosHeaders.SetNameValue(pszKey, pszValue); |
1409 | | } |
1410 | | if (EQUAL(pszKey, "Cache-Control") && |
1411 | | EQUAL(pszValue, "no-cache") && |
1412 | | CPLTestBool(CPLGetConfigOption( |
1413 | | "CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES"))) |
1414 | | { |
1415 | | m_bCached = false; |
1416 | | } |
1417 | | |
1418 | | else if (EQUAL(pszKey, "ETag")) |
1419 | | { |
1420 | | std::string osValue(pszValue); |
1421 | | if (osValue.size() >= 2 && osValue.front() == '"' && |
1422 | | osValue.back() == '"') |
1423 | | osValue = osValue.substr(1, osValue.size() - 2); |
1424 | | oFileProp.ETag = std::move(osValue); |
1425 | | } |
1426 | | |
1427 | | // Azure Data Lake Storage |
1428 | | else if (EQUAL(pszKey, "x-ms-resource-type")) |
1429 | | { |
1430 | | if (EQUAL(pszValue, "file")) |
1431 | | { |
1432 | | oFileProp.nMode |= S_IFREG; |
1433 | | } |
1434 | | else if (EQUAL(pszValue, "directory")) |
1435 | | { |
1436 | | oFileProp.bIsDirectory = true; |
1437 | | oFileProp.nMode |= S_IFDIR; |
1438 | | } |
1439 | | } |
1440 | | else if (EQUAL(pszKey, "x-ms-permissions")) |
1441 | | { |
1442 | | oFileProp.nMode |= |
1443 | | VSICurlParseUnixPermissions(pszValue); |
1444 | | } |
1445 | | |
1446 | | // https://overturemapswestus2.blob.core.windows.net/release/2024-11-13.0/theme%3Ddivisions/type%3Ddivision_area |
1447 | | // returns a x-ms-meta-hdi_isfolder: true header |
1448 | | else if (EQUAL(pszKey, "x-ms-meta-hdi_isfolder") && |
1449 | | EQUAL(pszValue, "true")) |
1450 | | { |
1451 | | oFileProp.bIsAzureFolder = true; |
1452 | | oFileProp.bIsDirectory = true; |
1453 | | oFileProp.nMode |= S_IFDIR; |
1454 | | } |
1455 | | } |
1456 | | CPLFree(pszKey); |
1457 | | } |
1458 | | CSLDestroy(papszHeaders); |
1459 | | } |
1460 | | } |
1461 | | |
1462 | | if (UseLimitRangeGetInsteadOfHead() && response_code == 206) |
1463 | | { |
1464 | | oFileProp.eExists = EXIST_NO; |
1465 | | oFileProp.fileSize = 0; |
1466 | | if (sWriteFuncHeaderData.pBuffer != nullptr) |
1467 | | { |
1468 | | const char *pszContentRange = strstr( |
1469 | | sWriteFuncHeaderData.pBuffer, "Content-Range: bytes "); |
1470 | | if (pszContentRange == nullptr) |
1471 | | pszContentRange = strstr(sWriteFuncHeaderData.pBuffer, |
1472 | | "content-range: bytes "); |
1473 | | if (pszContentRange) |
1474 | | pszContentRange = strchr(pszContentRange, '/'); |
1475 | | if (pszContentRange) |
1476 | | { |
1477 | | oFileProp.eExists = EXIST_YES; |
1478 | | oFileProp.fileSize = static_cast<GUIntBig>( |
1479 | | CPLAtoGIntBig(pszContentRange + 1)); |
1480 | | } |
1481 | | |
1482 | | // Add first bytes to cache |
1483 | | if (sWriteFuncData.pBuffer != nullptr) |
1484 | | { |
1485 | | size_t nOffset = 0; |
1486 | | while (nOffset < sWriteFuncData.nSize) |
1487 | | { |
1488 | | const size_t nToCache = |
1489 | | std::min<size_t>(sWriteFuncData.nSize - nOffset, |
1490 | | knDOWNLOAD_CHUNK_SIZE); |
1491 | | poFS->AddRegion(m_pszURL, nOffset, nToCache, |
1492 | | sWriteFuncData.pBuffer + nOffset); |
1493 | | nOffset += nToCache; |
1494 | | } |
1495 | | } |
1496 | | } |
1497 | | } |
1498 | | else if (IsDirectoryFromExists(osVerb.c_str(), |
1499 | | static_cast<int>(response_code))) |
1500 | | { |
1501 | | oFileProp.eExists = EXIST_YES; |
1502 | | oFileProp.fileSize = 0; |
1503 | | oFileProp.bIsDirectory = true; |
1504 | | } |
1505 | | // 405 = Method not allowed |
1506 | | else if (response_code == 405 && !bRetryWithGet && osVerb == "HEAD") |
1507 | | { |
1508 | | CPLDebug(poFS->GetDebugKey(), |
1509 | | "HEAD not allowed. Retrying with GET"); |
1510 | | bRetryWithGet = true; |
1511 | | CPLFree(sWriteFuncData.pBuffer); |
1512 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1513 | | curl_easy_cleanup(hCurlHandle); |
1514 | | goto retry; |
1515 | | } |
1516 | | else if (response_code == 416) |
1517 | | { |
1518 | | oFileProp.eExists = EXIST_YES; |
1519 | | oFileProp.fileSize = 0; |
1520 | | } |
1521 | | else if (response_code != 200) |
1522 | | { |
1523 | | // Look if we should attempt a retry |
1524 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
1525 | | sWriteFuncHeaderData.pBuffer, |
1526 | | szCurlErrBuf)) |
1527 | | { |
1528 | | CPLError(CE_Warning, CPLE_AppDefined, |
1529 | | "HTTP error code: %d - %s. " |
1530 | | "Retrying again in %.1f secs", |
1531 | | static_cast<int>(response_code), m_pszURL, |
1532 | | oRetryContext.GetCurrentDelay()); |
1533 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
1534 | | CPLFree(sWriteFuncData.pBuffer); |
1535 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1536 | | curl_easy_cleanup(hCurlHandle); |
1537 | | goto retry; |
1538 | | } |
1539 | | |
1540 | | if (sWriteFuncData.pBuffer != nullptr) |
1541 | | { |
1542 | | if (UseLimitRangeGetInsteadOfHead() && |
1543 | | CanRestartOnError(sWriteFuncData.pBuffer, |
1544 | | sWriteFuncHeaderData.pBuffer, bSetError)) |
1545 | | { |
1546 | | oFileProp.bHasComputedFileSize = false; |
1547 | | CPLFree(sWriteFuncData.pBuffer); |
1548 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1549 | | curl_easy_cleanup(hCurlHandle); |
1550 | | return GetFileSizeOrHeaders(bSetError, bGetHeaders); |
1551 | | } |
1552 | | else |
1553 | | { |
1554 | | CPL_IGNORE_RET_VAL(CanRestartOnError( |
1555 | | sWriteFuncData.pBuffer, sWriteFuncHeaderData.pBuffer, |
1556 | | bSetError)); |
1557 | | } |
1558 | | } |
1559 | | |
1560 | | // If there was no VSI error thrown in the process, |
1561 | | // fail by reporting the HTTP response code. |
1562 | | if (bSetError && VSIGetLastErrorNo() == 0) |
1563 | | { |
1564 | | if (strlen(szCurlErrBuf) > 0) |
1565 | | { |
1566 | | if (response_code == 0) |
1567 | | { |
1568 | | VSIError(VSIE_HttpError, "CURL error: %s", |
1569 | | szCurlErrBuf); |
1570 | | } |
1571 | | else |
1572 | | { |
1573 | | VSIError(VSIE_HttpError, "HTTP response code: %d - %s", |
1574 | | static_cast<int>(response_code), szCurlErrBuf); |
1575 | | } |
1576 | | } |
1577 | | else |
1578 | | { |
1579 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
1580 | | static_cast<int>(response_code)); |
1581 | | } |
1582 | | } |
1583 | | else |
1584 | | { |
1585 | | if (response_code != 400 && response_code != 404) |
1586 | | { |
1587 | | CPLError(CE_Warning, CPLE_AppDefined, |
1588 | | "HTTP response code on %s: %d", osURL.c_str(), |
1589 | | static_cast<int>(response_code)); |
1590 | | } |
1591 | | // else a CPLDebug() is emitted below |
1592 | | } |
1593 | | |
1594 | | oFileProp.eExists = EXIST_NO; |
1595 | | oFileProp.nHTTPCode = static_cast<int>(response_code); |
1596 | | oFileProp.fileSize = 0; |
1597 | | } |
1598 | | else if (sWriteFuncData.pBuffer != nullptr) |
1599 | | { |
1600 | | ProcessGetFileSizeResult( |
1601 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer)); |
1602 | | } |
1603 | | |
1604 | | // Try to guess if this is a directory. Generally if this is a |
1605 | | // directory, curl will retry with an URL with slash added. |
1606 | | if (!osEffectiveURL.empty() && |
1607 | | strncmp(osURL.c_str(), osEffectiveURL.c_str(), osURL.size()) == 0 && |
1608 | | osEffectiveURL[osURL.size()] == '/') |
1609 | | { |
1610 | | oFileProp.eExists = EXIST_YES; |
1611 | | oFileProp.fileSize = 0; |
1612 | | oFileProp.bIsDirectory = true; |
1613 | | } |
1614 | | else if (osURL.back() == '/') |
1615 | | { |
1616 | | oFileProp.bIsDirectory = true; |
1617 | | } |
1618 | | |
1619 | | if (!bAlreadyLogged) |
1620 | | { |
1621 | | CPLDebug(poFS->GetDebugKey(), |
1622 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
1623 | | osURL.c_str(), oFileProp.fileSize, |
1624 | | static_cast<int>(response_code)); |
1625 | | } |
1626 | | } |
1627 | | |
1628 | | CPLFree(sWriteFuncData.pBuffer); |
1629 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
1630 | | curl_easy_cleanup(hCurlHandle); |
1631 | | |
1632 | | oFileProp.bHasComputedFileSize = true; |
1633 | | if (mtime > 0) |
1634 | | oFileProp.mTime = mtime; |
1635 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1636 | | |
1637 | | return oFileProp.fileSize; |
1638 | | } |
1639 | | |
1640 | | /************************************************************************/ |
1641 | | /* Exists() */ |
1642 | | /************************************************************************/ |
1643 | | |
1644 | | bool VSICurlHandle::Exists(bool bSetError) |
1645 | | { |
1646 | | if (oFileProp.eExists == EXIST_UNKNOWN) |
1647 | | { |
1648 | | GetFileSize(bSetError); |
1649 | | } |
1650 | | else if (oFileProp.eExists == EXIST_NO) |
1651 | | { |
1652 | | // If there was no VSI error thrown in the process, |
1653 | | // and we know the HTTP error code of the first request where the |
1654 | | // file could not be retrieved, fail by reporting the HTTP code. |
1655 | | if (bSetError && VSIGetLastErrorNo() == 0 && oFileProp.nHTTPCode) |
1656 | | { |
1657 | | VSIError(VSIE_HttpError, "HTTP response code: %d", |
1658 | | oFileProp.nHTTPCode); |
1659 | | } |
1660 | | } |
1661 | | |
1662 | | return oFileProp.eExists == EXIST_YES; |
1663 | | } |
1664 | | |
1665 | | /************************************************************************/ |
1666 | | /* Tell() */ |
1667 | | /************************************************************************/ |
1668 | | |
1669 | | vsi_l_offset VSICurlHandle::Tell() |
1670 | | { |
1671 | | return curOffset; |
1672 | | } |
1673 | | |
1674 | | /************************************************************************/ |
1675 | | /* GetRedirectURLIfValid() */ |
1676 | | /************************************************************************/ |
1677 | | |
1678 | | std::string |
1679 | | VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired, |
1680 | | CPLStringList &aosHTTPOptions) const |
1681 | | { |
1682 | | bHasExpired = false; |
1683 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
1684 | | |
1685 | | std::string osURL(m_pszURL + m_osQueryString); |
1686 | | if (oFileProp.bS3LikeRedirect) |
1687 | | { |
1688 | | if (time(nullptr) + 1 < oFileProp.nExpireTimestampLocal) |
1689 | | { |
1690 | | CPLDebug(poFS->GetDebugKey(), |
1691 | | "Using redirect URL as it looks to be still valid " |
1692 | | "(%d seconds left)", |
1693 | | static_cast<int>(oFileProp.nExpireTimestampLocal - |
1694 | | time(nullptr))); |
1695 | | osURL = oFileProp.osRedirectURL; |
1696 | | } |
1697 | | else |
1698 | | { |
1699 | | CPLDebug(poFS->GetDebugKey(), |
1700 | | "Redirect URL has expired. Using original URL"); |
1701 | | oFileProp.bS3LikeRedirect = false; |
1702 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
1703 | | bHasExpired = true; |
1704 | | } |
1705 | | } |
1706 | | else if (!oFileProp.osRedirectURL.empty()) |
1707 | | { |
1708 | | osURL = oFileProp.osRedirectURL; |
1709 | | bHasExpired = false; |
1710 | | } |
1711 | | |
1712 | | if (m_pszURL != osURL) |
1713 | | { |
1714 | | const char *pszAuthorizationHeaderAllowed = CPLGetConfigOption( |
1715 | | "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT", |
1716 | | "IF_SAME_HOST"); |
1717 | | if (EQUAL(pszAuthorizationHeaderAllowed, "IF_SAME_HOST")) |
1718 | | { |
1719 | | const auto ExtractServer = [](const std::string &s) |
1720 | | { |
1721 | | size_t afterHTTPPos = 0; |
1722 | | if (STARTS_WITH(s.c_str(), "http://")) |
1723 | | afterHTTPPos = strlen("http://"); |
1724 | | else if (STARTS_WITH(s.c_str(), "https://")) |
1725 | | afterHTTPPos = strlen("https://"); |
1726 | | const auto posSlash = s.find('/', afterHTTPPos); |
1727 | | if (posSlash != std::string::npos) |
1728 | | return s.substr(afterHTTPPos, posSlash - afterHTTPPos); |
1729 | | else |
1730 | | return s.substr(afterHTTPPos); |
1731 | | }; |
1732 | | |
1733 | | if (ExtractServer(osURL) != ExtractServer(m_pszURL)) |
1734 | | { |
1735 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", |
1736 | | "NO"); |
1737 | | } |
1738 | | } |
1739 | | else if (!CPLTestBool(pszAuthorizationHeaderAllowed)) |
1740 | | { |
1741 | | aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", "NO"); |
1742 | | } |
1743 | | } |
1744 | | |
1745 | | return osURL; |
1746 | | } |
1747 | | |
1748 | | /************************************************************************/ |
1749 | | /* CurrentDownload */ |
1750 | | /************************************************************************/ |
1751 | | |
1752 | | namespace |
1753 | | { |
1754 | | struct CurrentDownload |
1755 | | { |
1756 | | VSICurlFilesystemHandlerBase *m_poFS = nullptr; |
1757 | | std::string m_osURL{}; |
1758 | | vsi_l_offset m_nStartOffset = 0; |
1759 | | int m_nBlocks = 0; |
1760 | | std::string m_osAlreadyDownloadedData{}; |
1761 | | bool m_bHasAlreadyDownloadedData = false; |
1762 | | |
1763 | | CurrentDownload(VSICurlFilesystemHandlerBase *poFS, const char *pszURL, |
1764 | | vsi_l_offset startOffset, int nBlocks) |
1765 | | : m_poFS(poFS), m_osURL(pszURL), m_nStartOffset(startOffset), |
1766 | | m_nBlocks(nBlocks) |
1767 | | { |
1768 | | auto res = m_poFS->NotifyStartDownloadRegion(m_osURL, m_nStartOffset, |
1769 | | m_nBlocks); |
1770 | | m_bHasAlreadyDownloadedData = res.first; |
1771 | | m_osAlreadyDownloadedData = std::move(res.second); |
1772 | | } |
1773 | | |
1774 | | bool HasAlreadyDownloadedData() const |
1775 | | { |
1776 | | return m_bHasAlreadyDownloadedData; |
1777 | | } |
1778 | | |
1779 | | const std::string &GetAlreadyDownloadedData() const |
1780 | | { |
1781 | | return m_osAlreadyDownloadedData; |
1782 | | } |
1783 | | |
1784 | | void SetData(const std::string &osData) |
1785 | | { |
1786 | | CPLAssert(!m_bHasAlreadyDownloadedData); |
1787 | | m_bHasAlreadyDownloadedData = true; |
1788 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
1789 | | osData); |
1790 | | } |
1791 | | |
1792 | | ~CurrentDownload() |
1793 | | { |
1794 | | if (!m_bHasAlreadyDownloadedData) |
1795 | | m_poFS->NotifyStopDownloadRegion(m_osURL, m_nStartOffset, m_nBlocks, |
1796 | | std::string()); |
1797 | | } |
1798 | | |
1799 | | CurrentDownload(const CurrentDownload &) = delete; |
1800 | | CurrentDownload &operator=(const CurrentDownload &) = delete; |
1801 | | }; |
1802 | | } // namespace |
1803 | | |
1804 | | /************************************************************************/ |
1805 | | /* NotifyStartDownloadRegion() */ |
1806 | | /************************************************************************/ |
1807 | | |
1808 | | /** Indicate intent at downloading a new region. |
1809 | | * |
1810 | | * If the region is already in download in another thread, then wait for its |
1811 | | * completion. |
1812 | | * |
1813 | | * Returns: |
1814 | | * - (false, empty string) if a new download is needed |
1815 | | * - (true, region_content) if we have been waiting for a download of the same |
1816 | | * region to be completed and got its result. Note that region_content will be |
1817 | | * empty if the download of that region failed. |
1818 | | */ |
1819 | | std::pair<bool, std::string> |
1820 | | VSICurlFilesystemHandlerBase::NotifyStartDownloadRegion( |
1821 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks) |
1822 | | { |
1823 | | std::string osId(osURL); |
1824 | | osId += '_'; |
1825 | | osId += std::to_string(startOffset); |
1826 | | osId += '_'; |
1827 | | osId += std::to_string(nBlocks); |
1828 | | |
1829 | | m_oMutex.lock(); |
1830 | | auto oIter = m_oMapRegionInDownload.find(osId); |
1831 | | if (oIter != m_oMapRegionInDownload.end()) |
1832 | | { |
1833 | | auto ®ion = *(oIter->second); |
1834 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
1835 | | m_oMutex.unlock(); |
1836 | | region.nWaiters++; |
1837 | | while (region.bDownloadInProgress) |
1838 | | { |
1839 | | region.oCond.wait(oRegionLock); |
1840 | | } |
1841 | | std::string osRet = region.osData; |
1842 | | region.nWaiters--; |
1843 | | region.oCond.notify_one(); |
1844 | | return std::pair<bool, std::string>(true, osRet); |
1845 | | } |
1846 | | else |
1847 | | { |
1848 | | auto poRegionInDownload = std::make_unique<RegionInDownload>(); |
1849 | | poRegionInDownload->bDownloadInProgress = true; |
1850 | | m_oMapRegionInDownload[osId] = std::move(poRegionInDownload); |
1851 | | m_oMutex.unlock(); |
1852 | | return std::pair<bool, std::string>(false, std::string()); |
1853 | | } |
1854 | | } |
1855 | | |
1856 | | /************************************************************************/ |
1857 | | /* NotifyStopDownloadRegion() */ |
1858 | | /************************************************************************/ |
1859 | | |
1860 | | void VSICurlFilesystemHandlerBase::NotifyStopDownloadRegion( |
1861 | | const std::string &osURL, vsi_l_offset startOffset, int nBlocks, |
1862 | | const std::string &osData) |
1863 | | { |
1864 | | std::string osId(osURL); |
1865 | | osId += '_'; |
1866 | | osId += std::to_string(startOffset); |
1867 | | osId += '_'; |
1868 | | osId += std::to_string(nBlocks); |
1869 | | |
1870 | | m_oMutex.lock(); |
1871 | | auto oIter = m_oMapRegionInDownload.find(osId); |
1872 | | CPLAssert(oIter != m_oMapRegionInDownload.end()); |
1873 | | auto ®ion = *(oIter->second); |
1874 | | { |
1875 | | std::unique_lock<std::mutex> oRegionLock(region.oMutex); |
1876 | | if (region.nWaiters) |
1877 | | { |
1878 | | region.osData = osData; |
1879 | | region.bDownloadInProgress = false; |
1880 | | region.oCond.notify_all(); |
1881 | | |
1882 | | while (region.nWaiters) |
1883 | | { |
1884 | | region.oCond.wait(oRegionLock); |
1885 | | } |
1886 | | } |
1887 | | } |
1888 | | m_oMapRegionInDownload.erase(oIter); |
1889 | | m_oMutex.unlock(); |
1890 | | } |
1891 | | |
1892 | | /************************************************************************/ |
1893 | | /* DownloadRegion() */ |
1894 | | /************************************************************************/ |
1895 | | |
1896 | | std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, |
1897 | | const int nBlocks) |
1898 | | { |
1899 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
1900 | | return std::string(); |
1901 | | |
1902 | | if (oFileProp.eExists == EXIST_NO) |
1903 | | return std::string(); |
1904 | | |
1905 | | // Check if there is not a download of the same region in progress in |
1906 | | // another thread, and if so wait for it to be completed |
1907 | | CurrentDownload currentDownload(poFS, m_pszURL, startOffset, nBlocks); |
1908 | | if (currentDownload.HasAlreadyDownloadedData()) |
1909 | | { |
1910 | | return currentDownload.GetAlreadyDownloadedData(); |
1911 | | } |
1912 | | |
1913 | | begin: |
1914 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
1915 | | |
1916 | | UpdateQueryString(); |
1917 | | |
1918 | | bool bHasExpired = false; |
1919 | | |
1920 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
1921 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
1922 | | bool bUsedRedirect = osURL != m_pszURL; |
1923 | | |
1924 | | WriteFuncStruct sWriteFuncData; |
1925 | | WriteFuncStruct sWriteFuncHeaderData; |
1926 | | CPLHTTPRetryContext oRetryContext(m_oRetryParameters); |
1927 | | |
1928 | | retry: |
1929 | | CURL *hCurlHandle = curl_easy_init(); |
1930 | | struct curl_slist *headers = |
1931 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
1932 | | |
1933 | | if (!AllowAutomaticRedirection()) |
1934 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); |
1935 | | |
1936 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
1937 | | pReadCbkUserData); |
1938 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
1939 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
1940 | | VSICurlHandleWriteFunc); |
1941 | | |
1942 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
1943 | | nullptr); |
1944 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
1945 | | &sWriteFuncHeaderData); |
1946 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
1947 | | VSICurlHandleWriteFunc); |
1948 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
1949 | | sWriteFuncHeaderData.nStartOffset = startOffset; |
1950 | | sWriteFuncHeaderData.nEndOffset = |
1951 | | startOffset + |
1952 | | static_cast<vsi_l_offset>(nBlocks) * VSICURLGetDownloadChunkSize() - 1; |
1953 | | // Some servers don't like we try to read after end-of-file (#5786). |
1954 | | if (oFileProp.bHasComputedFileSize && |
1955 | | sWriteFuncHeaderData.nEndOffset >= oFileProp.fileSize) |
1956 | | { |
1957 | | sWriteFuncHeaderData.nEndOffset = oFileProp.fileSize - 1; |
1958 | | } |
1959 | | |
1960 | | char rangeStr[512] = {}; |
1961 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
1962 | | startOffset, sWriteFuncHeaderData.nEndOffset); |
1963 | | |
1964 | | if (ENABLE_DEBUG) |
1965 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
1966 | | osURL.c_str()); |
1967 | | |
1968 | | std::string osHeaderRange; // leave in this scope |
1969 | | if (sWriteFuncHeaderData.bIsHTTP) |
1970 | | { |
1971 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
1972 | | // So it gets included in Azure signature |
1973 | | headers = curl_slist_append(headers, osHeaderRange.c_str()); |
1974 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
1975 | | } |
1976 | | else |
1977 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
1978 | | |
1979 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
1980 | | szCurlErrBuf[0] = '\0'; |
1981 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
1982 | | |
1983 | | headers = VSICurlMergeHeaders(headers, GetCurlHeaders("GET", headers)); |
1984 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
1985 | | |
1986 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); |
1987 | | |
1988 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); |
1989 | | |
1990 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
1991 | | |
1992 | | curl_slist_free_all(headers); |
1993 | | |
1994 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
1995 | | |
1996 | | if (sWriteFuncData.bInterrupted || m_bInterrupt) |
1997 | | { |
1998 | | bInterrupted = true; |
1999 | | |
2000 | | // Notify that the download of the current region is finished |
2001 | | currentDownload.SetData(std::string()); |
2002 | | |
2003 | | CPLFree(sWriteFuncData.pBuffer); |
2004 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2005 | | curl_easy_cleanup(hCurlHandle); |
2006 | | |
2007 | | return std::string(); |
2008 | | } |
2009 | | |
2010 | | long response_code = 0; |
2011 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2012 | | |
2013 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
2014 | | { |
2015 | | CPLDebug(poFS->GetDebugKey(), |
2016 | | "DownloadRegion(%s): response_code=%d, msg=%s", osURL.c_str(), |
2017 | | static_cast<int>(response_code), szCurlErrBuf); |
2018 | | } |
2019 | | |
2020 | | long mtime = 0; |
2021 | | curl_easy_getinfo(hCurlHandle, CURLINFO_FILETIME, &mtime); |
2022 | | if (mtime > 0) |
2023 | | { |
2024 | | oFileProp.mTime = mtime; |
2025 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2026 | | } |
2027 | | |
2028 | | if (ENABLE_DEBUG) |
2029 | | CPLDebug(poFS->GetDebugKey(), "Got response_code=%ld", response_code); |
2030 | | |
2031 | | if (bUsedRedirect && |
2032 | | (response_code == 403 || |
2033 | | // Below case is in particular for |
2034 | | // gdalinfo |
2035 | | // /vsicurl/https://lpdaac.earthdata.nasa.gov/lp-prod-protected/HLSS30.015/HLS.S30.T10TEK.2020273T190109.v1.5.B8A.tif |
2036 | | // --config GDAL_DISABLE_READDIR_ON_OPEN EMPTY_DIR --config |
2037 | | // GDAL_HTTP_COOKIEFILE /tmp/cookie.txt --config GDAL_HTTP_COOKIEJAR |
2038 | | // /tmp/cookie.txt We got the redirect URL from a HEAD request, but it |
2039 | | // is not valid for a GET. So retry with GET on original URL to get a |
2040 | | // redirect URL valid for it. |
2041 | | (response_code == 400 && |
2042 | | osURL.find(".cloudfront.net") != std::string::npos))) |
2043 | | { |
2044 | | CPLDebug(poFS->GetDebugKey(), |
2045 | | "Got an error with redirect URL. Retrying with original one"); |
2046 | | oFileProp.bS3LikeRedirect = false; |
2047 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2048 | | bUsedRedirect = false; |
2049 | | osURL = m_pszURL; |
2050 | | CPLFree(sWriteFuncData.pBuffer); |
2051 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2052 | | curl_easy_cleanup(hCurlHandle); |
2053 | | goto retry; |
2054 | | } |
2055 | | |
2056 | | if (response_code == 401 && oRetryContext.CanRetry()) |
2057 | | { |
2058 | | CPLDebug(poFS->GetDebugKey(), "Unauthorized, trying to authenticate"); |
2059 | | CPLFree(sWriteFuncData.pBuffer); |
2060 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2061 | | curl_easy_cleanup(hCurlHandle); |
2062 | | if (Authenticate(m_osFilename.c_str())) |
2063 | | goto retry; |
2064 | | return std::string(); |
2065 | | } |
2066 | | |
2067 | | UpdateRedirectInfo(hCurlHandle, sWriteFuncHeaderData); |
2068 | | |
2069 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
2070 | | response_code != 226 && response_code != 426) || |
2071 | | sWriteFuncHeaderData.bError) |
2072 | | { |
2073 | | if (sWriteFuncData.pBuffer != nullptr && |
2074 | | CanRestartOnError( |
2075 | | reinterpret_cast<const char *>(sWriteFuncData.pBuffer), |
2076 | | reinterpret_cast<const char *>(sWriteFuncHeaderData.pBuffer), |
2077 | | true)) |
2078 | | { |
2079 | | CPLFree(sWriteFuncData.pBuffer); |
2080 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2081 | | curl_easy_cleanup(hCurlHandle); |
2082 | | goto begin; |
2083 | | } |
2084 | | |
2085 | | // Look if we should attempt a retry |
2086 | | if (oRetryContext.CanRetry(static_cast<int>(response_code), |
2087 | | sWriteFuncHeaderData.pBuffer, szCurlErrBuf)) |
2088 | | { |
2089 | | CPLError(CE_Warning, CPLE_AppDefined, |
2090 | | "HTTP error code: %d - %s. " |
2091 | | "Retrying again in %.1f secs", |
2092 | | static_cast<int>(response_code), m_pszURL, |
2093 | | oRetryContext.GetCurrentDelay()); |
2094 | | CPLSleep(oRetryContext.GetCurrentDelay()); |
2095 | | CPLFree(sWriteFuncData.pBuffer); |
2096 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2097 | | curl_easy_cleanup(hCurlHandle); |
2098 | | goto retry; |
2099 | | } |
2100 | | |
2101 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
2102 | | { |
2103 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
2104 | | CPLError( |
2105 | | CE_Failure, CPLE_AppDefined, |
2106 | | "%d: %s, Range downloading not supported by this server!", |
2107 | | static_cast<int>(response_code), szCurlErrBuf); |
2108 | | else |
2109 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
2110 | | static_cast<int>(response_code), szCurlErrBuf); |
2111 | | } |
2112 | | else if (response_code == 416) /* Range Not Satisfiable */ |
2113 | | { |
2114 | | if (sWriteFuncData.pBuffer) |
2115 | | { |
2116 | | CPLError( |
2117 | | CE_Failure, CPLE_AppDefined, |
2118 | | "%d: Range downloading not supported by this server: %s", |
2119 | | static_cast<int>(response_code), sWriteFuncData.pBuffer); |
2120 | | } |
2121 | | else |
2122 | | { |
2123 | | CPLError(CE_Failure, CPLE_AppDefined, |
2124 | | "%d: Range downloading not supported by this server", |
2125 | | static_cast<int>(response_code)); |
2126 | | } |
2127 | | } |
2128 | | if (!oFileProp.bHasComputedFileSize && startOffset == 0) |
2129 | | { |
2130 | | oFileProp.bHasComputedFileSize = true; |
2131 | | oFileProp.fileSize = 0; |
2132 | | oFileProp.eExists = EXIST_NO; |
2133 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2134 | | } |
2135 | | CPLFree(sWriteFuncData.pBuffer); |
2136 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2137 | | curl_easy_cleanup(hCurlHandle); |
2138 | | return std::string(); |
2139 | | } |
2140 | | |
2141 | | if (!oFileProp.bHasComputedFileSize && sWriteFuncHeaderData.pBuffer) |
2142 | | { |
2143 | | // Try to retrieve the filesize from the HTTP headers |
2144 | | // if in the form: "Content-Range: bytes x-y/filesize". |
2145 | | char *pszContentRange = |
2146 | | strstr(sWriteFuncHeaderData.pBuffer, "Content-Range: bytes "); |
2147 | | if (pszContentRange == nullptr) |
2148 | | pszContentRange = |
2149 | | strstr(sWriteFuncHeaderData.pBuffer, "content-range: bytes "); |
2150 | | if (pszContentRange) |
2151 | | { |
2152 | | char *pszEOL = strchr(pszContentRange, '\n'); |
2153 | | if (pszEOL) |
2154 | | { |
2155 | | *pszEOL = 0; |
2156 | | pszEOL = strchr(pszContentRange, '\r'); |
2157 | | if (pszEOL) |
2158 | | *pszEOL = 0; |
2159 | | char *pszSlash = strchr(pszContentRange, '/'); |
2160 | | if (pszSlash) |
2161 | | { |
2162 | | pszSlash++; |
2163 | | oFileProp.fileSize = CPLScanUIntBig( |
2164 | | pszSlash, static_cast<int>(strlen(pszSlash))); |
2165 | | } |
2166 | | } |
2167 | | } |
2168 | | else if (STARTS_WITH(m_pszURL, "ftp")) |
2169 | | { |
2170 | | // Parse 213 answer for FTP protocol. |
2171 | | char *pszSize = strstr(sWriteFuncHeaderData.pBuffer, "213 "); |
2172 | | if (pszSize) |
2173 | | { |
2174 | | pszSize += 4; |
2175 | | char *pszEOL = strchr(pszSize, '\n'); |
2176 | | if (pszEOL) |
2177 | | { |
2178 | | *pszEOL = 0; |
2179 | | pszEOL = strchr(pszSize, '\r'); |
2180 | | if (pszEOL) |
2181 | | *pszEOL = 0; |
2182 | | |
2183 | | oFileProp.fileSize = CPLScanUIntBig( |
2184 | | pszSize, static_cast<int>(strlen(pszSize))); |
2185 | | } |
2186 | | } |
2187 | | } |
2188 | | |
2189 | | if (oFileProp.fileSize != 0) |
2190 | | { |
2191 | | oFileProp.eExists = EXIST_YES; |
2192 | | |
2193 | | if (ENABLE_DEBUG) |
2194 | | CPLDebug(poFS->GetDebugKey(), |
2195 | | "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d", |
2196 | | m_pszURL, oFileProp.fileSize, |
2197 | | static_cast<int>(response_code)); |
2198 | | |
2199 | | oFileProp.bHasComputedFileSize = true; |
2200 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2201 | | } |
2202 | | } |
2203 | | |
2204 | | DownloadRegionPostProcess(startOffset, nBlocks, sWriteFuncData.pBuffer, |
2205 | | sWriteFuncData.nSize); |
2206 | | |
2207 | | std::string osRet; |
2208 | | osRet.assign(sWriteFuncData.pBuffer, sWriteFuncData.nSize); |
2209 | | |
2210 | | // Notify that the download of the current region is finished |
2211 | | currentDownload.SetData(osRet); |
2212 | | |
2213 | | CPLFree(sWriteFuncData.pBuffer); |
2214 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2215 | | curl_easy_cleanup(hCurlHandle); |
2216 | | |
2217 | | return osRet; |
2218 | | } |
2219 | | |
2220 | | /************************************************************************/ |
2221 | | /* UpdateRedirectInfo() */ |
2222 | | /************************************************************************/ |
2223 | | |
2224 | | void VSICurlHandle::UpdateRedirectInfo( |
2225 | | CURL *hCurlHandle, const WriteFuncStruct &sWriteFuncHeaderData) |
2226 | | { |
2227 | | std::string osEffectiveURL; |
2228 | | { |
2229 | | char *pszEffectiveURL = nullptr; |
2230 | | curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, |
2231 | | &pszEffectiveURL); |
2232 | | if (pszEffectiveURL) |
2233 | | osEffectiveURL = pszEffectiveURL; |
2234 | | } |
2235 | | |
2236 | | if (!oFileProp.bS3LikeRedirect && !osEffectiveURL.empty() && |
2237 | | strstr(osEffectiveURL.c_str(), m_pszURL) == nullptr) |
2238 | | { |
2239 | | CPLDebug(poFS->GetDebugKey(), "Effective URL: %s", |
2240 | | osEffectiveURL.c_str()); |
2241 | | |
2242 | | long response_code = 0; |
2243 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2244 | | if (response_code >= 200 && response_code < 300 && |
2245 | | sWriteFuncHeaderData.nTimestampDate > 0 && |
2246 | | VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && |
2247 | | !VSICurlIsS3LikeSignedURL(m_pszURL) && |
2248 | | CPLTestBool( |
2249 | | CPLGetConfigOption("CPL_VSIL_CURL_USE_S3_REDIRECT", "TRUE"))) |
2250 | | { |
2251 | | GIntBig nExpireTimestamp = |
2252 | | VSICurlGetExpiresFromS3LikeSignedURL(osEffectiveURL.c_str()); |
2253 | | if (nExpireTimestamp > sWriteFuncHeaderData.nTimestampDate + 10) |
2254 | | { |
2255 | | const int nValidity = static_cast<int>( |
2256 | | nExpireTimestamp - sWriteFuncHeaderData.nTimestampDate); |
2257 | | CPLDebug(poFS->GetDebugKey(), |
2258 | | "Will use redirect URL for the next %d seconds", |
2259 | | nValidity); |
2260 | | // As our local clock might not be in sync with server clock, |
2261 | | // figure out the expiration timestamp in local time. |
2262 | | oFileProp.bS3LikeRedirect = true; |
2263 | | oFileProp.nExpireTimestampLocal = time(nullptr) + nValidity; |
2264 | | oFileProp.osRedirectURL = std::move(osEffectiveURL); |
2265 | | poFS->SetCachedFileProp(m_pszURL, oFileProp); |
2266 | | } |
2267 | | } |
2268 | | } |
2269 | | } |
2270 | | |
2271 | | /************************************************************************/ |
2272 | | /* DownloadRegionPostProcess() */ |
2273 | | /************************************************************************/ |
2274 | | |
2275 | | void VSICurlHandle::DownloadRegionPostProcess(const vsi_l_offset startOffset, |
2276 | | const int nBlocks, |
2277 | | const char *pBuffer, size_t nSize) |
2278 | | { |
2279 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2280 | | lastDownloadedOffset = startOffset + static_cast<vsi_l_offset>(nBlocks) * |
2281 | | knDOWNLOAD_CHUNK_SIZE; |
2282 | | |
2283 | | if (nSize > static_cast<size_t>(nBlocks) * knDOWNLOAD_CHUNK_SIZE) |
2284 | | { |
2285 | | if (ENABLE_DEBUG) |
2286 | | CPLDebug( |
2287 | | poFS->GetDebugKey(), |
2288 | | "Got more data than expected : %u instead of %u", |
2289 | | static_cast<unsigned int>(nSize), |
2290 | | static_cast<unsigned int>(nBlocks * knDOWNLOAD_CHUNK_SIZE)); |
2291 | | } |
2292 | | |
2293 | | vsi_l_offset l_startOffset = startOffset; |
2294 | | while (nSize > 0) |
2295 | | { |
2296 | | #if DEBUG_VERBOSE |
2297 | | if (ENABLE_DEBUG) |
2298 | | CPLDebug(poFS->GetDebugKey(), "Add region %u - %u", |
2299 | | static_cast<unsigned int>(startOffset), |
2300 | | static_cast<unsigned int>(std::min( |
2301 | | static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize))); |
2302 | | #endif |
2303 | | const size_t nChunkSize = |
2304 | | std::min(static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE), nSize); |
2305 | | poFS->AddRegion(m_pszURL, l_startOffset, nChunkSize, pBuffer); |
2306 | | l_startOffset += nChunkSize; |
2307 | | pBuffer += nChunkSize; |
2308 | | nSize -= nChunkSize; |
2309 | | } |
2310 | | } |
2311 | | |
2312 | | /************************************************************************/ |
2313 | | /* Read() */ |
2314 | | /************************************************************************/ |
2315 | | |
2316 | | size_t VSICurlHandle::Read(void *const pBufferIn, size_t const nSize, |
2317 | | size_t const nMemb) |
2318 | | { |
2319 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2320 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2321 | | NetworkStatisticsAction oContextAction("Read"); |
2322 | | |
2323 | | size_t nBufferRequestSize = nSize * nMemb; |
2324 | | if (nBufferRequestSize == 0) |
2325 | | return 0; |
2326 | | |
2327 | | void *pBuffer = pBufferIn; |
2328 | | |
2329 | | #if DEBUG_VERBOSE |
2330 | | CPLDebug(poFS->GetDebugKey(), "offset=%d, size=%d", |
2331 | | static_cast<int>(curOffset), static_cast<int>(nBufferRequestSize)); |
2332 | | #endif |
2333 | | |
2334 | | vsi_l_offset iterOffset = curOffset; |
2335 | | const int knMAX_REGIONS = GetMaxRegions(); |
2336 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
2337 | | while (nBufferRequestSize) |
2338 | | { |
2339 | | // Don't try to read after end of file. |
2340 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2341 | | if (oFileProp.bHasComputedFileSize && iterOffset >= oFileProp.fileSize) |
2342 | | { |
2343 | | if (iterOffset == curOffset) |
2344 | | { |
2345 | | CPLDebug(poFS->GetDebugKey(), |
2346 | | "Request at offset " CPL_FRMT_GUIB |
2347 | | ", after end of file", |
2348 | | iterOffset); |
2349 | | } |
2350 | | break; |
2351 | | } |
2352 | | |
2353 | | const vsi_l_offset nOffsetToDownload = |
2354 | | (iterOffset / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
2355 | | std::string osRegion; |
2356 | | std::shared_ptr<std::string> psRegion = |
2357 | | poFS->GetRegion(m_pszURL, nOffsetToDownload); |
2358 | | if (psRegion != nullptr) |
2359 | | { |
2360 | | osRegion = *psRegion; |
2361 | | } |
2362 | | else |
2363 | | { |
2364 | | if (nOffsetToDownload == lastDownloadedOffset) |
2365 | | { |
2366 | | // In case of consecutive reads (of small size), we use a |
2367 | | // heuristic that we will read the file sequentially, so |
2368 | | // we double the requested size to decrease the number of |
2369 | | // client/server roundtrips. |
2370 | | constexpr int MAX_CHUNK_SIZE_INCREASE_FACTOR = 128; |
2371 | | if (nBlocksToDownload < MAX_CHUNK_SIZE_INCREASE_FACTOR) |
2372 | | nBlocksToDownload *= 2; |
2373 | | } |
2374 | | else |
2375 | | { |
2376 | | // Random reads. Cancel the above heuristics. |
2377 | | nBlocksToDownload = 1; |
2378 | | } |
2379 | | |
2380 | | // Ensure that we will request at least the number of blocks |
2381 | | // to satisfy the remaining buffer size to read. |
2382 | | const vsi_l_offset nEndOffsetToDownload = |
2383 | | ((iterOffset + nBufferRequestSize + knDOWNLOAD_CHUNK_SIZE - 1) / |
2384 | | knDOWNLOAD_CHUNK_SIZE) * |
2385 | | knDOWNLOAD_CHUNK_SIZE; |
2386 | | const int nMinBlocksToDownload = |
2387 | | static_cast<int>((nEndOffsetToDownload - nOffsetToDownload) / |
2388 | | knDOWNLOAD_CHUNK_SIZE); |
2389 | | if (nBlocksToDownload < nMinBlocksToDownload) |
2390 | | nBlocksToDownload = nMinBlocksToDownload; |
2391 | | |
2392 | | // Avoid reading already cached data. |
2393 | | // Note: this might get evicted if concurrent reads are done, but |
2394 | | // this should not cause bugs. Just missed optimization. |
2395 | | for (int i = 1; i < nBlocksToDownload; i++) |
2396 | | { |
2397 | | if (poFS->GetRegion(m_pszURL, nOffsetToDownload + |
2398 | | static_cast<vsi_l_offset>(i) * |
2399 | | knDOWNLOAD_CHUNK_SIZE) != |
2400 | | nullptr) |
2401 | | { |
2402 | | nBlocksToDownload = i; |
2403 | | break; |
2404 | | } |
2405 | | } |
2406 | | |
2407 | | // We can't download more than knMAX_REGIONS chunks at a time, |
2408 | | // otherwise the cache will not be big enough to store them and |
2409 | | // copy their content to the target buffer. |
2410 | | if (nBlocksToDownload > knMAX_REGIONS) |
2411 | | nBlocksToDownload = knMAX_REGIONS; |
2412 | | |
2413 | | osRegion = DownloadRegion(nOffsetToDownload, nBlocksToDownload); |
2414 | | if (osRegion.empty()) |
2415 | | { |
2416 | | if (!bInterrupted) |
2417 | | bError = true; |
2418 | | return 0; |
2419 | | } |
2420 | | } |
2421 | | |
2422 | | const vsi_l_offset nRegionOffset = iterOffset - nOffsetToDownload; |
2423 | | if (osRegion.size() < nRegionOffset) |
2424 | | { |
2425 | | if (iterOffset == curOffset) |
2426 | | { |
2427 | | CPLDebug(poFS->GetDebugKey(), |
2428 | | "Request at offset " CPL_FRMT_GUIB |
2429 | | ", after end of file", |
2430 | | iterOffset); |
2431 | | } |
2432 | | break; |
2433 | | } |
2434 | | |
2435 | | const int nToCopy = static_cast<int>( |
2436 | | std::min(static_cast<vsi_l_offset>(nBufferRequestSize), |
2437 | | osRegion.size() - nRegionOffset)); |
2438 | | memcpy(pBuffer, osRegion.data() + nRegionOffset, nToCopy); |
2439 | | pBuffer = static_cast<char *>(pBuffer) + nToCopy; |
2440 | | iterOffset += nToCopy; |
2441 | | nBufferRequestSize -= nToCopy; |
2442 | | if (osRegion.size() < static_cast<size_t>(knDOWNLOAD_CHUNK_SIZE) && |
2443 | | nBufferRequestSize != 0) |
2444 | | { |
2445 | | break; |
2446 | | } |
2447 | | } |
2448 | | |
2449 | | const size_t ret = static_cast<size_t>((iterOffset - curOffset) / nSize); |
2450 | | if (ret != nMemb) |
2451 | | bEOF = true; |
2452 | | |
2453 | | curOffset = iterOffset; |
2454 | | |
2455 | | return ret; |
2456 | | } |
2457 | | |
2458 | | /************************************************************************/ |
2459 | | /* ReadMultiRange() */ |
2460 | | /************************************************************************/ |
2461 | | |
2462 | | int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, |
2463 | | const vsi_l_offset *const panOffsets, |
2464 | | const size_t *const panSizes) |
2465 | | { |
2466 | | if (bInterrupted && bStopOnInterruptUntilUninstall) |
2467 | | return FALSE; |
2468 | | |
2469 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
2470 | | if (oFileProp.eExists == EXIST_NO) |
2471 | | return -1; |
2472 | | |
2473 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
2474 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
2475 | | NetworkStatisticsAction oContextAction("ReadMultiRange"); |
2476 | | |
2477 | | const char *pszMultiRangeStrategy = |
2478 | | CPLGetConfigOption("GDAL_HTTP_MULTIRANGE", ""); |
2479 | | if (EQUAL(pszMultiRangeStrategy, "SINGLE_GET")) |
2480 | | { |
2481 | | // Just in case someone needs it, but the interest of this mode is |
2482 | | // rather dubious now. We could probably remove it |
2483 | | return ReadMultiRangeSingleGet(nRanges, ppData, panOffsets, panSizes); |
2484 | | } |
2485 | | else if (nRanges == 1 || EQUAL(pszMultiRangeStrategy, "SERIAL")) |
2486 | | { |
2487 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
2488 | | panSizes); |
2489 | | } |
2490 | | |
2491 | | UpdateQueryString(); |
2492 | | |
2493 | | bool bHasExpired = false; |
2494 | | |
2495 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
2496 | | std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
2497 | | if (bHasExpired) |
2498 | | { |
2499 | | return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, |
2500 | | panSizes); |
2501 | | } |
2502 | | |
2503 | | CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); |
2504 | | #ifdef CURLPIPE_MULTIPLEX |
2505 | | // Enable HTTP/2 multiplexing (ignored if an older version of HTTP is |
2506 | | // used) |
2507 | | // Not that this does not enable HTTP/1.1 pipeling, which is not |
2508 | | // recommended for example by Google Cloud Storage. |
2509 | | // For HTTP/1.1, parallel connections work better since you can get |
2510 | | // results out of order. |
2511 | | if (CPLTestBool(CPLGetConfigOption("GDAL_HTTP_MULTIPLEX", "YES"))) |
2512 | | { |
2513 | | curl_multi_setopt(hMultiHandle, CURLMOPT_PIPELINING, |
2514 | | CURLPIPE_MULTIPLEX); |
2515 | | } |
2516 | | #endif |
2517 | | |
2518 | | std::vector<CURL *> aHandles; |
2519 | | std::vector<WriteFuncStruct> asWriteFuncData(nRanges); |
2520 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData(nRanges); |
2521 | | std::vector<char *> apszRanges; |
2522 | | std::vector<struct curl_slist *> aHeaders; |
2523 | | |
2524 | | struct CurlErrBuffer |
2525 | | { |
2526 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
2527 | | }; |
2528 | | |
2529 | | std::vector<CurlErrBuffer> asCurlErrors(nRanges); |
2530 | | |
2531 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
2532 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
2533 | | |
2534 | | for (int i = 0, iRequest = 0; i < nRanges;) |
2535 | | { |
2536 | | size_t nSize = 0; |
2537 | | int iNext = i; |
2538 | | // Identify consecutive ranges |
2539 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
2540 | | panOffsets[iNext] + panSizes[iNext] == panOffsets[iNext + 1]) |
2541 | | { |
2542 | | nSize += panSizes[iNext]; |
2543 | | iNext++; |
2544 | | } |
2545 | | nSize += panSizes[iNext]; |
2546 | | |
2547 | | if (nSize == 0) |
2548 | | { |
2549 | | i = iNext + 1; |
2550 | | continue; |
2551 | | } |
2552 | | |
2553 | | CURL *hCurlHandle = curl_easy_init(); |
2554 | | aHandles.push_back(hCurlHandle); |
2555 | | |
2556 | | // As the multi-range request is likely not the first one, we don't |
2557 | | // need to wait as we already know if pipelining is possible |
2558 | | // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); |
2559 | | |
2560 | | struct curl_slist *headers = VSICurlSetOptions( |
2561 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
2562 | | |
2563 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[iRequest], this, pfnReadCbk, |
2564 | | pReadCbkUserData); |
2565 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
2566 | | &asWriteFuncData[iRequest]); |
2567 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2568 | | VSICurlHandleWriteFunc); |
2569 | | |
2570 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[iRequest], nullptr, |
2571 | | nullptr, nullptr); |
2572 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2573 | | &asWriteFuncHeaderData[iRequest]); |
2574 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2575 | | VSICurlHandleWriteFunc); |
2576 | | asWriteFuncHeaderData[iRequest].bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2577 | | asWriteFuncHeaderData[iRequest].nStartOffset = panOffsets[i]; |
2578 | | |
2579 | | asWriteFuncHeaderData[iRequest].nEndOffset = panOffsets[i] + nSize - 1; |
2580 | | |
2581 | | char rangeStr[512] = {}; |
2582 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2583 | | asWriteFuncHeaderData[iRequest].nStartOffset, |
2584 | | asWriteFuncHeaderData[iRequest].nEndOffset); |
2585 | | |
2586 | | if (ENABLE_DEBUG) |
2587 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", rangeStr, |
2588 | | osURL.c_str()); |
2589 | | |
2590 | | if (asWriteFuncHeaderData[iRequest].bIsHTTP) |
2591 | | { |
2592 | | // So it gets included in Azure signature |
2593 | | char *pszRange = CPLStrdup(CPLSPrintf("Range: bytes=%s", rangeStr)); |
2594 | | apszRanges.push_back(pszRange); |
2595 | | headers = curl_slist_append(headers, pszRange); |
2596 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
2597 | | } |
2598 | | else |
2599 | | { |
2600 | | apszRanges.push_back(nullptr); |
2601 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
2602 | | } |
2603 | | |
2604 | | asCurlErrors[iRequest].szCurlErrBuf[0] = '\0'; |
2605 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
2606 | | &asCurlErrors[iRequest].szCurlErrBuf[0]); |
2607 | | |
2608 | | headers = VSICurlMergeHeaders(headers, GetCurlHeaders("GET", headers)); |
2609 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
2610 | | aHeaders.push_back(headers); |
2611 | | curl_multi_add_handle(hMultiHandle, hCurlHandle); |
2612 | | |
2613 | | i = iNext + 1; |
2614 | | iRequest++; |
2615 | | } |
2616 | | |
2617 | | if (!aHandles.empty()) |
2618 | | { |
2619 | | VSICURLMultiPerform(hMultiHandle); |
2620 | | } |
2621 | | |
2622 | | int nRet = 0; |
2623 | | size_t iReq = 0; |
2624 | | int iRange = 0; |
2625 | | size_t nTotalDownloaded = 0; |
2626 | | for (; iReq < aHandles.size(); iReq++, iRange++) |
2627 | | { |
2628 | | while (iRange < nRanges && panSizes[iRange] == 0) |
2629 | | { |
2630 | | iRange++; |
2631 | | } |
2632 | | if (iRange == nRanges) |
2633 | | break; |
2634 | | |
2635 | | long response_code = 0; |
2636 | | curl_easy_getinfo(aHandles[iReq], CURLINFO_HTTP_CODE, &response_code); |
2637 | | |
2638 | | if (ENABLE_DEBUG && asCurlErrors[iRange].szCurlErrBuf[0] != '\0') |
2639 | | { |
2640 | | char rangeStr[512] = {}; |
2641 | | snprintf(rangeStr, sizeof(rangeStr), |
2642 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2643 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2644 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2645 | | |
2646 | | const char *pszErrorMsg = &asCurlErrors[iRange].szCurlErrBuf[0]; |
2647 | | CPLDebug(poFS->GetDebugKey(), |
2648 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
2649 | | osURL.c_str(), rangeStr, static_cast<int>(response_code), |
2650 | | pszErrorMsg); |
2651 | | } |
2652 | | |
2653 | | if ((response_code != 206 && response_code != 225) || |
2654 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
2655 | | asWriteFuncHeaderData[iReq].nStartOffset + |
2656 | | asWriteFuncData[iReq].nSize) |
2657 | | { |
2658 | | char rangeStr[512] = {}; |
2659 | | snprintf(rangeStr, sizeof(rangeStr), |
2660 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
2661 | | asWriteFuncHeaderData[iReq].nStartOffset, |
2662 | | asWriteFuncHeaderData[iReq].nEndOffset); |
2663 | | |
2664 | | CPLError(CE_Failure, CPLE_AppDefined, |
2665 | | "Request for %s failed with response_code=%ld", rangeStr, |
2666 | | response_code); |
2667 | | nRet = -1; |
2668 | | } |
2669 | | else if (nRet == 0) |
2670 | | { |
2671 | | size_t nOffset = 0; |
2672 | | size_t nRemainingSize = asWriteFuncData[iReq].nSize; |
2673 | | nTotalDownloaded += nRemainingSize; |
2674 | | CPLAssert(iRange < nRanges); |
2675 | | while (true) |
2676 | | { |
2677 | | if (nRemainingSize < panSizes[iRange]) |
2678 | | { |
2679 | | nRet = -1; |
2680 | | break; |
2681 | | } |
2682 | | |
2683 | | if (panSizes[iRange] > 0) |
2684 | | { |
2685 | | memcpy(ppData[iRange], |
2686 | | asWriteFuncData[iReq].pBuffer + nOffset, |
2687 | | panSizes[iRange]); |
2688 | | } |
2689 | | |
2690 | | if (bMergeConsecutiveRanges && iRange + 1 < nRanges && |
2691 | | panOffsets[iRange] + panSizes[iRange] == |
2692 | | panOffsets[iRange + 1]) |
2693 | | { |
2694 | | nOffset += panSizes[iRange]; |
2695 | | nRemainingSize -= panSizes[iRange]; |
2696 | | iRange++; |
2697 | | } |
2698 | | else |
2699 | | { |
2700 | | break; |
2701 | | } |
2702 | | } |
2703 | | } |
2704 | | |
2705 | | curl_multi_remove_handle(hMultiHandle, aHandles[iReq]); |
2706 | | VSICURLResetHeaderAndWriterFunctions(aHandles[iReq]); |
2707 | | curl_easy_cleanup(aHandles[iReq]); |
2708 | | CPLFree(apszRanges[iReq]); |
2709 | | CPLFree(asWriteFuncData[iReq].pBuffer); |
2710 | | CPLFree(asWriteFuncHeaderData[iReq].pBuffer); |
2711 | | curl_slist_free_all(aHeaders[iReq]); |
2712 | | } |
2713 | | |
2714 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
2715 | | |
2716 | | if (ENABLE_DEBUG) |
2717 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
2718 | | |
2719 | | return nRet; |
2720 | | } |
2721 | | |
2722 | | /************************************************************************/ |
2723 | | /* ReadMultiRangeSingleGet() */ |
2724 | | /************************************************************************/ |
2725 | | |
2726 | | // TODO: the interest of this mode is rather dubious now. We could probably |
2727 | | // remove it |
2728 | | int VSICurlHandle::ReadMultiRangeSingleGet(int const nRanges, |
2729 | | void **const ppData, |
2730 | | const vsi_l_offset *const panOffsets, |
2731 | | const size_t *const panSizes) |
2732 | | { |
2733 | | std::string osRanges; |
2734 | | std::string osFirstRange; |
2735 | | std::string osLastRange; |
2736 | | int nMergedRanges = 0; |
2737 | | vsi_l_offset nTotalReqSize = 0; |
2738 | | for (int i = 0; i < nRanges; i++) |
2739 | | { |
2740 | | std::string osCurRange; |
2741 | | if (i != 0) |
2742 | | osRanges.append(","); |
2743 | | osCurRange = CPLSPrintf(CPL_FRMT_GUIB "-", panOffsets[i]); |
2744 | | while (i + 1 < nRanges && |
2745 | | panOffsets[i] + panSizes[i] == panOffsets[i + 1]) |
2746 | | { |
2747 | | nTotalReqSize += panSizes[i]; |
2748 | | i++; |
2749 | | } |
2750 | | nTotalReqSize += panSizes[i]; |
2751 | | osCurRange.append( |
2752 | | CPLSPrintf(CPL_FRMT_GUIB, panOffsets[i] + panSizes[i] - 1)); |
2753 | | nMergedRanges++; |
2754 | | |
2755 | | osRanges += osCurRange; |
2756 | | |
2757 | | if (nMergedRanges == 1) |
2758 | | osFirstRange = osCurRange; |
2759 | | osLastRange = std::move(osCurRange); |
2760 | | } |
2761 | | |
2762 | | const char *pszMaxRanges = |
2763 | | CPLGetConfigOption("CPL_VSIL_CURL_MAX_RANGES", "250"); |
2764 | | int nMaxRanges = atoi(pszMaxRanges); |
2765 | | if (nMaxRanges <= 0) |
2766 | | nMaxRanges = 250; |
2767 | | if (nMergedRanges > nMaxRanges) |
2768 | | { |
2769 | | const int nHalf = nRanges / 2; |
2770 | | const int nRet = ReadMultiRange(nHalf, ppData, panOffsets, panSizes); |
2771 | | if (nRet != 0) |
2772 | | return nRet; |
2773 | | return ReadMultiRange(nRanges - nHalf, ppData + nHalf, |
2774 | | panOffsets + nHalf, panSizes + nHalf); |
2775 | | } |
2776 | | |
2777 | | CURLM *hCurlMultiHandle = poFS->GetCurlMultiHandleFor(m_pszURL); |
2778 | | CURL *hCurlHandle = curl_easy_init(); |
2779 | | |
2780 | | struct curl_slist *headers = |
2781 | | VSICurlSetOptions(hCurlHandle, m_pszURL, m_aosHTTPOptions.List()); |
2782 | | |
2783 | | WriteFuncStruct sWriteFuncData; |
2784 | | WriteFuncStruct sWriteFuncHeaderData; |
2785 | | |
2786 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, this, pfnReadCbk, |
2787 | | pReadCbkUserData); |
2788 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
2789 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
2790 | | VSICurlHandleWriteFunc); |
2791 | | |
2792 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
2793 | | nullptr); |
2794 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
2795 | | &sWriteFuncHeaderData); |
2796 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
2797 | | VSICurlHandleWriteFunc); |
2798 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
2799 | | sWriteFuncHeaderData.bMultiRange = nMergedRanges > 1; |
2800 | | if (nMergedRanges == 1) |
2801 | | { |
2802 | | sWriteFuncHeaderData.nStartOffset = panOffsets[0]; |
2803 | | sWriteFuncHeaderData.nEndOffset = panOffsets[0] + nTotalReqSize - 1; |
2804 | | } |
2805 | | |
2806 | | if (ENABLE_DEBUG) |
2807 | | { |
2808 | | if (nMergedRanges == 1) |
2809 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
2810 | | osRanges.c_str(), m_pszURL); |
2811 | | else |
2812 | | CPLDebug(poFS->GetDebugKey(), |
2813 | | "Downloading %s, ..., %s (" CPL_FRMT_GUIB " bytes, %s)...", |
2814 | | osFirstRange.c_str(), osLastRange.c_str(), |
2815 | | static_cast<GUIntBig>(nTotalReqSize), m_pszURL); |
2816 | | } |
2817 | | |
2818 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, osRanges.c_str()); |
2819 | | |
2820 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
2821 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
2822 | | |
2823 | | headers = VSICurlMergeHeaders(headers, GetCurlHeaders("GET", headers)); |
2824 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
2825 | | |
2826 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
2827 | | |
2828 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
2829 | | |
2830 | | curl_slist_free_all(headers); |
2831 | | |
2832 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
2833 | | |
2834 | | if (sWriteFuncData.bInterrupted) |
2835 | | { |
2836 | | bInterrupted = true; |
2837 | | |
2838 | | CPLFree(sWriteFuncData.pBuffer); |
2839 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2840 | | curl_easy_cleanup(hCurlHandle); |
2841 | | |
2842 | | return -1; |
2843 | | } |
2844 | | |
2845 | | long response_code = 0; |
2846 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
2847 | | |
2848 | | if ((response_code != 200 && response_code != 206 && response_code != 225 && |
2849 | | response_code != 226 && response_code != 426) || |
2850 | | sWriteFuncHeaderData.bError) |
2851 | | { |
2852 | | if (response_code >= 400 && szCurlErrBuf[0] != '\0') |
2853 | | { |
2854 | | if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0) |
2855 | | CPLError( |
2856 | | CE_Failure, CPLE_AppDefined, |
2857 | | "%d: %s, Range downloading not supported by this server!", |
2858 | | static_cast<int>(response_code), szCurlErrBuf); |
2859 | | else |
2860 | | CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", |
2861 | | static_cast<int>(response_code), szCurlErrBuf); |
2862 | | } |
2863 | | /* |
2864 | | if( !bHasComputedFileSize && startOffset == 0 ) |
2865 | | { |
2866 | | cachedFileProp->bHasComputedFileSize = bHasComputedFileSize = true; |
2867 | | cachedFileProp->fileSize = fileSize = 0; |
2868 | | cachedFileProp->eExists = eExists = EXIST_NO; |
2869 | | } |
2870 | | */ |
2871 | | CPLFree(sWriteFuncData.pBuffer); |
2872 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
2873 | | curl_easy_cleanup(hCurlHandle); |
2874 | | return -1; |
2875 | | } |
2876 | | |
2877 | | char *pBuffer = sWriteFuncData.pBuffer; |
2878 | | size_t nSize = sWriteFuncData.nSize; |
2879 | | |
2880 | | // TODO(schwehr): Localize after removing gotos. |
2881 | | int nRet = -1; |
2882 | | char *pszBoundary; |
2883 | | std::string osBoundary; |
2884 | | char *pszNext = nullptr; |
2885 | | int iRange = 0; |
2886 | | int iPart = 0; |
2887 | | char *pszEOL = nullptr; |
2888 | | |
2889 | | /* -------------------------------------------------------------------- */ |
2890 | | /* No multipart if a single range has been requested */ |
2891 | | /* -------------------------------------------------------------------- */ |
2892 | | |
2893 | | if (nMergedRanges == 1) |
2894 | | { |
2895 | | size_t nAccSize = 0; |
2896 | | if (static_cast<vsi_l_offset>(nSize) < nTotalReqSize) |
2897 | | goto end; |
2898 | | |
2899 | | for (int i = 0; i < nRanges; i++) |
2900 | | { |
2901 | | memcpy(ppData[i], pBuffer + nAccSize, panSizes[i]); |
2902 | | nAccSize += panSizes[i]; |
2903 | | } |
2904 | | |
2905 | | nRet = 0; |
2906 | | goto end; |
2907 | | } |
2908 | | |
2909 | | /* -------------------------------------------------------------------- */ |
2910 | | /* Extract boundary name */ |
2911 | | /* -------------------------------------------------------------------- */ |
2912 | | |
2913 | | pszBoundary = strstr(sWriteFuncHeaderData.pBuffer, |
2914 | | "Content-Type: multipart/byteranges; boundary="); |
2915 | | if (pszBoundary == nullptr) |
2916 | | { |
2917 | | CPLError(CE_Failure, CPLE_AppDefined, "Could not find '%s'", |
2918 | | "Content-Type: multipart/byteranges; boundary="); |
2919 | | goto end; |
2920 | | } |
2921 | | |
2922 | | pszBoundary += strlen("Content-Type: multipart/byteranges; boundary="); |
2923 | | |
2924 | | pszEOL = strchr(pszBoundary, '\r'); |
2925 | | if (pszEOL) |
2926 | | *pszEOL = 0; |
2927 | | pszEOL = strchr(pszBoundary, '\n'); |
2928 | | if (pszEOL) |
2929 | | *pszEOL = 0; |
2930 | | |
2931 | | /* Remove optional double-quote character around boundary name */ |
2932 | | if (pszBoundary[0] == '"') |
2933 | | { |
2934 | | pszBoundary++; |
2935 | | char *pszLastDoubleQuote = strrchr(pszBoundary, '"'); |
2936 | | if (pszLastDoubleQuote) |
2937 | | *pszLastDoubleQuote = 0; |
2938 | | } |
2939 | | |
2940 | | osBoundary = "--"; |
2941 | | osBoundary += pszBoundary; |
2942 | | |
2943 | | /* -------------------------------------------------------------------- */ |
2944 | | /* Find the start of the first chunk. */ |
2945 | | /* -------------------------------------------------------------------- */ |
2946 | | pszNext = strstr(pBuffer, osBoundary.c_str()); |
2947 | | if (pszNext == nullptr) |
2948 | | { |
2949 | | CPLError(CE_Failure, CPLE_AppDefined, "No parts found."); |
2950 | | goto end; |
2951 | | } |
2952 | | |
2953 | | pszNext += osBoundary.size(); |
2954 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
2955 | | pszNext++; |
2956 | | if (*pszNext == '\r') |
2957 | | pszNext++; |
2958 | | if (*pszNext == '\n') |
2959 | | pszNext++; |
2960 | | |
2961 | | /* -------------------------------------------------------------------- */ |
2962 | | /* Loop over parts... */ |
2963 | | /* -------------------------------------------------------------------- */ |
2964 | | while (iPart < nRanges) |
2965 | | { |
2966 | | /* -------------------------------------------------------------------- |
2967 | | */ |
2968 | | /* Collect headers. */ |
2969 | | /* -------------------------------------------------------------------- |
2970 | | */ |
2971 | | bool bExpectedRange = false; |
2972 | | |
2973 | | while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0') |
2974 | | { |
2975 | | pszEOL = strstr(pszNext, "\n"); |
2976 | | |
2977 | | if (pszEOL == nullptr) |
2978 | | { |
2979 | | CPLError(CE_Failure, CPLE_AppDefined, |
2980 | | "Error while parsing multipart content (at line %d)", |
2981 | | __LINE__); |
2982 | | goto end; |
2983 | | } |
2984 | | |
2985 | | *pszEOL = '\0'; |
2986 | | bool bRestoreAntislashR = false; |
2987 | | if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r') |
2988 | | { |
2989 | | bRestoreAntislashR = true; |
2990 | | pszEOL[-1] = '\0'; |
2991 | | } |
2992 | | |
2993 | | if (STARTS_WITH_CI(pszNext, "Content-Range: bytes ")) |
2994 | | { |
2995 | | bExpectedRange = true; /* FIXME */ |
2996 | | } |
2997 | | |
2998 | | if (bRestoreAntislashR) |
2999 | | pszEOL[-1] = '\r'; |
3000 | | *pszEOL = '\n'; |
3001 | | |
3002 | | pszNext = pszEOL + 1; |
3003 | | } |
3004 | | |
3005 | | if (!bExpectedRange) |
3006 | | { |
3007 | | CPLError(CE_Failure, CPLE_AppDefined, |
3008 | | "Error while parsing multipart content (at line %d)", |
3009 | | __LINE__); |
3010 | | goto end; |
3011 | | } |
3012 | | |
3013 | | if (*pszNext == '\r') |
3014 | | pszNext++; |
3015 | | if (*pszNext == '\n') |
3016 | | pszNext++; |
3017 | | |
3018 | | /* -------------------------------------------------------------------- |
3019 | | */ |
3020 | | /* Work out the data block size. */ |
3021 | | /* -------------------------------------------------------------------- |
3022 | | */ |
3023 | | size_t nBytesAvail = nSize - (pszNext - pBuffer); |
3024 | | |
3025 | | while (true) |
3026 | | { |
3027 | | if (nBytesAvail < panSizes[iRange]) |
3028 | | { |
3029 | | CPLError(CE_Failure, CPLE_AppDefined, |
3030 | | "Error while parsing multipart content (at line %d)", |
3031 | | __LINE__); |
3032 | | goto end; |
3033 | | } |
3034 | | |
3035 | | memcpy(ppData[iRange], pszNext, panSizes[iRange]); |
3036 | | pszNext += panSizes[iRange]; |
3037 | | nBytesAvail -= panSizes[iRange]; |
3038 | | if (iRange + 1 < nRanges && |
3039 | | panOffsets[iRange] + panSizes[iRange] == panOffsets[iRange + 1]) |
3040 | | { |
3041 | | iRange++; |
3042 | | } |
3043 | | else |
3044 | | { |
3045 | | break; |
3046 | | } |
3047 | | } |
3048 | | |
3049 | | iPart++; |
3050 | | iRange++; |
3051 | | |
3052 | | while (nBytesAvail > 0 && |
3053 | | (*pszNext != '-' || |
3054 | | strncmp(pszNext, osBoundary.c_str(), osBoundary.size()) != 0)) |
3055 | | { |
3056 | | pszNext++; |
3057 | | nBytesAvail--; |
3058 | | } |
3059 | | |
3060 | | if (nBytesAvail == 0) |
3061 | | { |
3062 | | CPLError(CE_Failure, CPLE_AppDefined, |
3063 | | "Error while parsing multipart content (at line %d)", |
3064 | | __LINE__); |
3065 | | goto end; |
3066 | | } |
3067 | | |
3068 | | pszNext += osBoundary.size(); |
3069 | | if (STARTS_WITH(pszNext, "--")) |
3070 | | { |
3071 | | // End of multipart. |
3072 | | break; |
3073 | | } |
3074 | | |
3075 | | if (*pszNext == '\r') |
3076 | | pszNext++; |
3077 | | if (*pszNext == '\n') |
3078 | | pszNext++; |
3079 | | else |
3080 | | { |
3081 | | CPLError(CE_Failure, CPLE_AppDefined, |
3082 | | "Error while parsing multipart content (at line %d)", |
3083 | | __LINE__); |
3084 | | goto end; |
3085 | | } |
3086 | | } |
3087 | | |
3088 | | if (iPart == nMergedRanges) |
3089 | | nRet = 0; |
3090 | | else |
3091 | | CPLError(CE_Failure, CPLE_AppDefined, |
3092 | | "Got only %d parts, where %d were expected", iPart, |
3093 | | nMergedRanges); |
3094 | | |
3095 | | end: |
3096 | | CPLFree(sWriteFuncData.pBuffer); |
3097 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3098 | | curl_easy_cleanup(hCurlHandle); |
3099 | | |
3100 | | return nRet; |
3101 | | } |
3102 | | |
3103 | | /************************************************************************/ |
3104 | | /* PRead() */ |
3105 | | /************************************************************************/ |
3106 | | |
3107 | | size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, |
3108 | | vsi_l_offset nOffset) const |
3109 | | { |
3110 | | // Try to use AdviseRead ranges fetched asynchronously |
3111 | | if (!m_aoAdviseReadRanges.empty()) |
3112 | | { |
3113 | | for (auto &poRange : m_aoAdviseReadRanges) |
3114 | | { |
3115 | | if (nOffset >= poRange->nStartOffset && |
3116 | | nOffset + nSize <= poRange->nStartOffset + poRange->nSize) |
3117 | | { |
3118 | | { |
3119 | | std::unique_lock<std::mutex> oLock(poRange->oMutex); |
3120 | | // coverity[missing_lock:FALSE] |
3121 | | while (!poRange->bDone) |
3122 | | { |
3123 | | poRange->oCV.wait(oLock); |
3124 | | } |
3125 | | } |
3126 | | if (poRange->abyData.empty()) |
3127 | | return 0; |
3128 | | |
3129 | | auto nEndOffset = |
3130 | | poRange->nStartOffset + poRange->abyData.size(); |
3131 | | if (nOffset >= nEndOffset) |
3132 | | return 0; |
3133 | | const size_t nToCopy = static_cast<size_t>( |
3134 | | std::min<vsi_l_offset>(nSize, nEndOffset - nOffset)); |
3135 | | memcpy(pBuffer, |
3136 | | poRange->abyData.data() + |
3137 | | static_cast<size_t>(nOffset - poRange->nStartOffset), |
3138 | | nToCopy); |
3139 | | return nToCopy; |
3140 | | } |
3141 | | } |
3142 | | } |
3143 | | |
3144 | | // poFS has a global mutex |
3145 | | poFS->GetCachedFileProp(m_pszURL, oFileProp); |
3146 | | if (oFileProp.eExists == EXIST_NO) |
3147 | | return static_cast<size_t>(-1); |
3148 | | |
3149 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3150 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3151 | | NetworkStatisticsAction oContextAction("PRead"); |
3152 | | |
3153 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3154 | | std::string osURL; |
3155 | | { |
3156 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3157 | | UpdateQueryString(); |
3158 | | bool bHasExpired; |
3159 | | osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions); |
3160 | | } |
3161 | | |
3162 | | CURL *hCurlHandle = curl_easy_init(); |
3163 | | |
3164 | | struct curl_slist *headers = |
3165 | | VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3166 | | |
3167 | | WriteFuncStruct sWriteFuncData; |
3168 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
3169 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
3170 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3171 | | VSICurlHandleWriteFunc); |
3172 | | |
3173 | | WriteFuncStruct sWriteFuncHeaderData; |
3174 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
3175 | | nullptr); |
3176 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3177 | | &sWriteFuncHeaderData); |
3178 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3179 | | VSICurlHandleWriteFunc); |
3180 | | sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http"); |
3181 | | sWriteFuncHeaderData.nStartOffset = nOffset; |
3182 | | |
3183 | | sWriteFuncHeaderData.nEndOffset = nOffset + nSize - 1; |
3184 | | |
3185 | | char rangeStr[512] = {}; |
3186 | | snprintf(rangeStr, sizeof(rangeStr), CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3187 | | sWriteFuncHeaderData.nStartOffset, |
3188 | | sWriteFuncHeaderData.nEndOffset); |
3189 | | |
3190 | | #if 0 |
3191 | | if( ENABLE_DEBUG ) |
3192 | | CPLDebug(poFS->GetDebugKey(), |
3193 | | "Downloading %s (%s)...", rangeStr, osURL.c_str()); |
3194 | | #endif |
3195 | | |
3196 | | std::string osHeaderRange; |
3197 | | if (sWriteFuncHeaderData.bIsHTTP) |
3198 | | { |
3199 | | osHeaderRange = CPLSPrintf("Range: bytes=%s", rangeStr); |
3200 | | // So it gets included in Azure signature |
3201 | | headers = curl_slist_append(headers, osHeaderRange.data()); |
3202 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
3203 | | } |
3204 | | else |
3205 | | { |
3206 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr); |
3207 | | } |
3208 | | |
3209 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3210 | | szCurlErrBuf[0] = '\0'; |
3211 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3212 | | &szCurlErrBuf[0]); |
3213 | | |
3214 | | { |
3215 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3216 | | auto newHeaders = |
3217 | | const_cast<VSICurlHandle *>(this)->GetCurlHeaders("GET", headers); |
3218 | | headers = VSICurlMergeHeaders(headers, newHeaders); |
3219 | | } |
3220 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
3221 | | |
3222 | | CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); |
3223 | | VSICURLMultiPerform(hMultiHandle, hCurlHandle, &m_bInterrupt); |
3224 | | |
3225 | | { |
3226 | | std::lock_guard<std::mutex> oLock(m_oMutex); |
3227 | | const_cast<VSICurlHandle *>(this)->UpdateRedirectInfo( |
3228 | | hCurlHandle, sWriteFuncHeaderData); |
3229 | | } |
3230 | | |
3231 | | long response_code = 0; |
3232 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
3233 | | |
3234 | | if (ENABLE_DEBUG && szCurlErrBuf[0] != '\0') |
3235 | | { |
3236 | | const char *pszErrorMsg = &szCurlErrBuf[0]; |
3237 | | CPLDebug(poFS->GetDebugKey(), "PRead(%s), %s: response_code=%d, msg=%s", |
3238 | | osURL.c_str(), rangeStr, static_cast<int>(response_code), |
3239 | | pszErrorMsg); |
3240 | | } |
3241 | | |
3242 | | size_t nRet; |
3243 | | if ((response_code != 206 && response_code != 225) || |
3244 | | sWriteFuncData.nSize == 0) |
3245 | | { |
3246 | | if (!m_bInterrupt) |
3247 | | { |
3248 | | CPLDebug(poFS->GetDebugKey(), |
3249 | | "Request for %s failed with response_code=%ld", rangeStr, |
3250 | | response_code); |
3251 | | } |
3252 | | nRet = static_cast<size_t>(-1); |
3253 | | } |
3254 | | else |
3255 | | { |
3256 | | nRet = std::min(sWriteFuncData.nSize, nSize); |
3257 | | if (nRet > 0) |
3258 | | memcpy(pBuffer, sWriteFuncData.pBuffer, nRet); |
3259 | | } |
3260 | | |
3261 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
3262 | | curl_easy_cleanup(hCurlHandle); |
3263 | | CPLFree(sWriteFuncData.pBuffer); |
3264 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
3265 | | curl_slist_free_all(headers); |
3266 | | |
3267 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
3268 | | |
3269 | | #if 0 |
3270 | | if( ENABLE_DEBUG ) |
3271 | | CPLDebug(poFS->GetDebugKey(), "Download completed"); |
3272 | | #endif |
3273 | | |
3274 | | return nRet; |
3275 | | } |
3276 | | |
3277 | | /************************************************************************/ |
3278 | | /* GetAdviseReadTotalBytesLimit() */ |
3279 | | /************************************************************************/ |
3280 | | |
3281 | | size_t VSICurlHandle::GetAdviseReadTotalBytesLimit() const |
3282 | | { |
3283 | | return static_cast<size_t>(std::min<unsigned long long>( |
3284 | | std::numeric_limits<size_t>::max(), |
3285 | | // 100 MB |
3286 | | std::strtoull( |
3287 | | CPLGetConfigOption("CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT", |
3288 | | "104857600"), |
3289 | | nullptr, 10))); |
3290 | | } |
3291 | | |
3292 | | /************************************************************************/ |
3293 | | /* VSICURLMultiInit() */ |
3294 | | /************************************************************************/ |
3295 | | |
3296 | | static CURLM *VSICURLMultiInit() |
3297 | | { |
3298 | | CURLM *hCurlMultiHandle = curl_multi_init(); |
3299 | | |
3300 | | if (const char *pszMAXCONNECTS = |
3301 | | CPLGetConfigOption("GDAL_HTTP_MAX_CACHED_CONNECTIONS", nullptr)) |
3302 | | { |
3303 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAXCONNECTS, |
3304 | | atoi(pszMAXCONNECTS)); |
3305 | | } |
3306 | | |
3307 | | if (const char *pszMAX_TOTAL_CONNECTIONS = |
3308 | | CPLGetConfigOption("GDAL_HTTP_MAX_TOTAL_CONNECTIONS", nullptr)) |
3309 | | { |
3310 | | curl_multi_setopt(hCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, |
3311 | | atoi(pszMAX_TOTAL_CONNECTIONS)); |
3312 | | } |
3313 | | |
3314 | | return hCurlMultiHandle; |
3315 | | } |
3316 | | |
3317 | | /************************************************************************/ |
3318 | | /* AdviseRead() */ |
3319 | | /************************************************************************/ |
3320 | | |
3321 | | void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, |
3322 | | const size_t *panSizes) |
3323 | | { |
3324 | | if (!CPLTestBool( |
3325 | | CPLGetConfigOption("GDAL_HTTP_ENABLE_ADVISE_READ", "TRUE"))) |
3326 | | return; |
3327 | | |
3328 | | if (m_oThreadAdviseRead.joinable()) |
3329 | | { |
3330 | | m_oThreadAdviseRead.join(); |
3331 | | } |
3332 | | |
3333 | | // Give up if we need to allocate too much memory |
3334 | | vsi_l_offset nMaxSize = 0; |
3335 | | const size_t nLimit = GetAdviseReadTotalBytesLimit(); |
3336 | | for (int i = 0; i < nRanges; ++i) |
3337 | | { |
3338 | | if (panSizes[i] > nLimit - nMaxSize) |
3339 | | { |
3340 | | CPLDebug(poFS->GetDebugKey(), |
3341 | | "Trying to request too many bytes in AdviseRead()"); |
3342 | | return; |
3343 | | } |
3344 | | nMaxSize += panSizes[i]; |
3345 | | } |
3346 | | |
3347 | | UpdateQueryString(); |
3348 | | |
3349 | | bool bHasExpired = false; |
3350 | | CPLStringList aosHTTPOptions(m_aosHTTPOptions); |
3351 | | const std::string l_osURL( |
3352 | | GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); |
3353 | | if (bHasExpired) |
3354 | | { |
3355 | | return; |
3356 | | } |
3357 | | |
3358 | | const bool bMergeConsecutiveRanges = CPLTestBool( |
3359 | | CPLGetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "TRUE")); |
3360 | | |
3361 | | try |
3362 | | { |
3363 | | m_aoAdviseReadRanges.clear(); |
3364 | | m_aoAdviseReadRanges.reserve(nRanges); |
3365 | | for (int i = 0; i < nRanges;) |
3366 | | { |
3367 | | int iNext = i; |
3368 | | // Identify consecutive ranges |
3369 | | constexpr size_t SIZE_COG_MARKERS = 2 * sizeof(uint32_t); |
3370 | | auto nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3371 | | while (bMergeConsecutiveRanges && iNext + 1 < nRanges && |
3372 | | panOffsets[iNext + 1] > panOffsets[iNext] && |
3373 | | panOffsets[iNext] + panSizes[iNext] + SIZE_COG_MARKERS >= |
3374 | | panOffsets[iNext + 1] && |
3375 | | panOffsets[iNext + 1] + panSizes[iNext + 1] > nEndOffset) |
3376 | | { |
3377 | | iNext++; |
3378 | | nEndOffset = panOffsets[iNext] + panSizes[iNext]; |
3379 | | } |
3380 | | CPLAssert(panOffsets[i] <= nEndOffset); |
3381 | | const size_t nSize = |
3382 | | static_cast<size_t>(nEndOffset - panOffsets[i]); |
3383 | | |
3384 | | if (nSize == 0) |
3385 | | { |
3386 | | i = iNext + 1; |
3387 | | continue; |
3388 | | } |
3389 | | |
3390 | | auto newAdviseReadRange = |
3391 | | std::make_unique<AdviseReadRange>(m_oRetryParameters); |
3392 | | newAdviseReadRange->nStartOffset = panOffsets[i]; |
3393 | | newAdviseReadRange->nSize = nSize; |
3394 | | newAdviseReadRange->abyData.resize(nSize); |
3395 | | m_aoAdviseReadRanges.push_back(std::move(newAdviseReadRange)); |
3396 | | |
3397 | | i = iNext + 1; |
3398 | | } |
3399 | | } |
3400 | | catch (const std::exception &) |
3401 | | { |
3402 | | CPLError(CE_Failure, CPLE_OutOfMemory, |
3403 | | "Out of memory in VSICurlHandle::AdviseRead()"); |
3404 | | m_aoAdviseReadRanges.clear(); |
3405 | | } |
3406 | | |
3407 | | if (m_aoAdviseReadRanges.empty()) |
3408 | | return; |
3409 | | |
3410 | | #ifdef DEBUG |
3411 | | CPLDebug(poFS->GetDebugKey(), "AdviseRead(): fetching %u ranges", |
3412 | | static_cast<unsigned>(m_aoAdviseReadRanges.size())); |
3413 | | #endif |
3414 | | |
3415 | | const auto task = [this, aosHTTPOptions = std::move(aosHTTPOptions)]( |
3416 | | const std::string &osURL) |
3417 | | { |
3418 | | if (!m_hCurlMultiHandleForAdviseRead) |
3419 | | m_hCurlMultiHandleForAdviseRead = VSICURLMultiInit(); |
3420 | | |
3421 | | NetworkStatisticsFileSystem oContextFS(poFS->GetFSPrefix().c_str()); |
3422 | | NetworkStatisticsFile oContextFile(m_osFilename.c_str()); |
3423 | | NetworkStatisticsAction oContextAction("AdviseRead"); |
3424 | | |
3425 | | #ifdef CURLPIPE_MULTIPLEX |
3426 | | // Enable HTTP/2 multiplexing (ignored if an older version of HTTP is |
3427 | | // used) |
3428 | | // Not that this does not enable HTTP/1.1 pipeling, which is not |
3429 | | // recommended for example by Google Cloud Storage. |
3430 | | // For HTTP/1.1, parallel connections work better since you can get |
3431 | | // results out of order. |
3432 | | if (CPLTestBool(CPLGetConfigOption("GDAL_HTTP_MULTIPLEX", "YES"))) |
3433 | | { |
3434 | | curl_multi_setopt(m_hCurlMultiHandleForAdviseRead, |
3435 | | CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); |
3436 | | } |
3437 | | #endif |
3438 | | |
3439 | | size_t nTotalDownloaded = 0; |
3440 | | |
3441 | | while (true) |
3442 | | { |
3443 | | |
3444 | | std::vector<CURL *> aHandles; |
3445 | | std::vector<WriteFuncStruct> asWriteFuncData( |
3446 | | m_aoAdviseReadRanges.size()); |
3447 | | std::vector<WriteFuncStruct> asWriteFuncHeaderData( |
3448 | | m_aoAdviseReadRanges.size()); |
3449 | | std::vector<char *> apszRanges; |
3450 | | std::vector<struct curl_slist *> aHeaders; |
3451 | | |
3452 | | struct CurlErrBuffer |
3453 | | { |
3454 | | std::array<char, CURL_ERROR_SIZE + 1> szCurlErrBuf; |
3455 | | }; |
3456 | | std::vector<CurlErrBuffer> asCurlErrors( |
3457 | | m_aoAdviseReadRanges.size()); |
3458 | | |
3459 | | std::map<CURL *, size_t> oMapHandleToIdx; |
3460 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
3461 | | { |
3462 | | if (!m_aoAdviseReadRanges[i]->bToRetry) |
3463 | | { |
3464 | | aHandles.push_back(nullptr); |
3465 | | apszRanges.push_back(nullptr); |
3466 | | aHeaders.push_back(nullptr); |
3467 | | continue; |
3468 | | } |
3469 | | m_aoAdviseReadRanges[i]->bToRetry = false; |
3470 | | |
3471 | | CURL *hCurlHandle = curl_easy_init(); |
3472 | | oMapHandleToIdx[hCurlHandle] = i; |
3473 | | aHandles.push_back(hCurlHandle); |
3474 | | |
3475 | | // As the multi-range request is likely not the first one, we don't |
3476 | | // need to wait as we already know if pipelining is possible |
3477 | | // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); |
3478 | | |
3479 | | struct curl_slist *headers = VSICurlSetOptions( |
3480 | | hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); |
3481 | | |
3482 | | VSICURLInitWriteFuncStruct(&asWriteFuncData[i], this, |
3483 | | pfnReadCbk, pReadCbkUserData); |
3484 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
3485 | | &asWriteFuncData[i]); |
3486 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
3487 | | VSICurlHandleWriteFunc); |
3488 | | |
3489 | | VSICURLInitWriteFuncStruct(&asWriteFuncHeaderData[i], nullptr, |
3490 | | nullptr, nullptr); |
3491 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
3492 | | &asWriteFuncHeaderData[i]); |
3493 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
3494 | | VSICurlHandleWriteFunc); |
3495 | | asWriteFuncHeaderData[i].bIsHTTP = |
3496 | | STARTS_WITH(m_pszURL, "http"); |
3497 | | asWriteFuncHeaderData[i].nStartOffset = |
3498 | | m_aoAdviseReadRanges[i]->nStartOffset; |
3499 | | |
3500 | | asWriteFuncHeaderData[i].nEndOffset = |
3501 | | m_aoAdviseReadRanges[i]->nStartOffset + |
3502 | | m_aoAdviseReadRanges[i]->nSize - 1; |
3503 | | |
3504 | | char rangeStr[512] = {}; |
3505 | | snprintf(rangeStr, sizeof(rangeStr), |
3506 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3507 | | asWriteFuncHeaderData[i].nStartOffset, |
3508 | | asWriteFuncHeaderData[i].nEndOffset); |
3509 | | |
3510 | | if (ENABLE_DEBUG) |
3511 | | CPLDebug(poFS->GetDebugKey(), "Downloading %s (%s)...", |
3512 | | rangeStr, osURL.c_str()); |
3513 | | |
3514 | | if (asWriteFuncHeaderData[i].bIsHTTP) |
3515 | | { |
3516 | | std::string osHeaderRange( |
3517 | | CPLSPrintf("Range: bytes=%s", rangeStr)); |
3518 | | // So it gets included in Azure signature |
3519 | | char *pszRange = CPLStrdup(osHeaderRange.c_str()); |
3520 | | apszRanges.push_back(pszRange); |
3521 | | headers = curl_slist_append(headers, pszRange); |
3522 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3523 | | nullptr); |
3524 | | } |
3525 | | else |
3526 | | { |
3527 | | apszRanges.push_back(nullptr); |
3528 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, |
3529 | | rangeStr); |
3530 | | } |
3531 | | |
3532 | | asCurlErrors[i].szCurlErrBuf[0] = '\0'; |
3533 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
3534 | | &asCurlErrors[i].szCurlErrBuf[0]); |
3535 | | |
3536 | | headers = VSICurlMergeHeaders(headers, |
3537 | | GetCurlHeaders("GET", headers)); |
3538 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
3539 | | headers); |
3540 | | aHeaders.push_back(headers); |
3541 | | curl_multi_add_handle(m_hCurlMultiHandleForAdviseRead, |
3542 | | hCurlHandle); |
3543 | | } |
3544 | | |
3545 | | const auto DealWithRequest = [this, &osURL, &nTotalDownloaded, |
3546 | | &oMapHandleToIdx, &asCurlErrors, |
3547 | | &asWriteFuncHeaderData, |
3548 | | &asWriteFuncData](CURL *hCurlHandle) |
3549 | | { |
3550 | | auto oIter = oMapHandleToIdx.find(hCurlHandle); |
3551 | | CPLAssert(oIter != oMapHandleToIdx.end()); |
3552 | | const auto iReq = oIter->second; |
3553 | | |
3554 | | long response_code = 0; |
3555 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, |
3556 | | &response_code); |
3557 | | |
3558 | | if (ENABLE_DEBUG && asCurlErrors[iReq].szCurlErrBuf[0] != '\0') |
3559 | | { |
3560 | | char rangeStr[512] = {}; |
3561 | | snprintf(rangeStr, sizeof(rangeStr), |
3562 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3563 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3564 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3565 | | |
3566 | | const char *pszErrorMsg = |
3567 | | &asCurlErrors[iReq].szCurlErrBuf[0]; |
3568 | | CPLDebug(poFS->GetDebugKey(), |
3569 | | "ReadMultiRange(%s), %s: response_code=%d, msg=%s", |
3570 | | osURL.c_str(), rangeStr, |
3571 | | static_cast<int>(response_code), pszErrorMsg); |
3572 | | } |
3573 | | |
3574 | | bool bToRetry = false; |
3575 | | if ((response_code != 206 && response_code != 225) || |
3576 | | asWriteFuncHeaderData[iReq].nEndOffset + 1 != |
3577 | | asWriteFuncHeaderData[iReq].nStartOffset + |
3578 | | asWriteFuncData[iReq].nSize) |
3579 | | { |
3580 | | char rangeStr[512] = {}; |
3581 | | snprintf(rangeStr, sizeof(rangeStr), |
3582 | | CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, |
3583 | | asWriteFuncHeaderData[iReq].nStartOffset, |
3584 | | asWriteFuncHeaderData[iReq].nEndOffset); |
3585 | | |
3586 | | // Look if we should attempt a retry |
3587 | | if (m_aoAdviseReadRanges[iReq]->retryContext.CanRetry( |
3588 | | static_cast<int>(response_code), |
3589 | | asWriteFuncData[iReq].pBuffer, |
3590 | | &asCurlErrors[iReq].szCurlErrBuf[0])) |
3591 | | { |
3592 | | CPLError(CE_Warning, CPLE_AppDefined, |
3593 | | "HTTP error code for %s range %s: %d. " |
3594 | | "Retrying again in %.1f secs", |
3595 | | osURL.c_str(), rangeStr, |
3596 | | static_cast<int>(response_code), |
3597 | | m_aoAdviseReadRanges[iReq] |
3598 | | ->retryContext.GetCurrentDelay()); |
3599 | | m_aoAdviseReadRanges[iReq]->dfSleepDelay = |
3600 | | m_aoAdviseReadRanges[iReq] |
3601 | | ->retryContext.GetCurrentDelay(); |
3602 | | bToRetry = true; |
3603 | | } |
3604 | | else |
3605 | | { |
3606 | | CPLError(CE_Failure, CPLE_AppDefined, |
3607 | | "Request for %s range %s failed with " |
3608 | | "response_code=%ld", |
3609 | | osURL.c_str(), rangeStr, response_code); |
3610 | | } |
3611 | | } |
3612 | | else |
3613 | | { |
3614 | | const size_t nSize = asWriteFuncData[iReq].nSize; |
3615 | | memcpy(&m_aoAdviseReadRanges[iReq]->abyData[0], |
3616 | | asWriteFuncData[iReq].pBuffer, nSize); |
3617 | | m_aoAdviseReadRanges[iReq]->abyData.resize(nSize); |
3618 | | |
3619 | | nTotalDownloaded += nSize; |
3620 | | } |
3621 | | |
3622 | | m_aoAdviseReadRanges[iReq]->bToRetry = bToRetry; |
3623 | | |
3624 | | if (!bToRetry) |
3625 | | { |
3626 | | std::lock_guard<std::mutex> oLock( |
3627 | | m_aoAdviseReadRanges[iReq]->oMutex); |
3628 | | m_aoAdviseReadRanges[iReq]->bDone = true; |
3629 | | m_aoAdviseReadRanges[iReq]->oCV.notify_all(); |
3630 | | } |
3631 | | }; |
3632 | | |
3633 | | int repeats = 0; |
3634 | | |
3635 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
3636 | | while (true) |
3637 | | { |
3638 | | int still_running; |
3639 | | while (curl_multi_perform(m_hCurlMultiHandleForAdviseRead, |
3640 | | &still_running) == |
3641 | | CURLM_CALL_MULTI_PERFORM) |
3642 | | { |
3643 | | // loop |
3644 | | } |
3645 | | if (!still_running) |
3646 | | { |
3647 | | break; |
3648 | | } |
3649 | | |
3650 | | CURLMsg *msg; |
3651 | | do |
3652 | | { |
3653 | | int msgq = 0; |
3654 | | msg = curl_multi_info_read(m_hCurlMultiHandleForAdviseRead, |
3655 | | &msgq); |
3656 | | if (msg && (msg->msg == CURLMSG_DONE)) |
3657 | | { |
3658 | | DealWithRequest(msg->easy_handle); |
3659 | | } |
3660 | | } while (msg); |
3661 | | |
3662 | | CPLMultiPerformWait(m_hCurlMultiHandleForAdviseRead, repeats); |
3663 | | } |
3664 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
3665 | | |
3666 | | bool bRetry = false; |
3667 | | double dfDelay = 0.0; |
3668 | | for (size_t i = 0; i < m_aoAdviseReadRanges.size(); ++i) |
3669 | | { |
3670 | | bool bReqDone; |
3671 | | { |
3672 | | // To please Coverity Scan |
3673 | | std::lock_guard<std::mutex> oLock( |
3674 | | m_aoAdviseReadRanges[i]->oMutex); |
3675 | | bReqDone = m_aoAdviseReadRanges[i]->bDone; |
3676 | | } |
3677 | | if (!bReqDone && !m_aoAdviseReadRanges[i]->bToRetry) |
3678 | | { |
3679 | | DealWithRequest(aHandles[i]); |
3680 | | } |
3681 | | if (m_aoAdviseReadRanges[i]->bToRetry) |
3682 | | dfDelay = std::max(dfDelay, |
3683 | | m_aoAdviseReadRanges[i]->dfSleepDelay); |
3684 | | bRetry = bRetry || m_aoAdviseReadRanges[i]->bToRetry; |
3685 | | if (aHandles[i]) |
3686 | | { |
3687 | | curl_multi_remove_handle(m_hCurlMultiHandleForAdviseRead, |
3688 | | aHandles[i]); |
3689 | | VSICURLResetHeaderAndWriterFunctions(aHandles[i]); |
3690 | | curl_easy_cleanup(aHandles[i]); |
3691 | | } |
3692 | | CPLFree(apszRanges[i]); |
3693 | | CPLFree(asWriteFuncData[i].pBuffer); |
3694 | | CPLFree(asWriteFuncHeaderData[i].pBuffer); |
3695 | | if (aHeaders[i]) |
3696 | | curl_slist_free_all(aHeaders[i]); |
3697 | | } |
3698 | | if (!bRetry) |
3699 | | break; |
3700 | | CPLSleep(dfDelay); |
3701 | | } |
3702 | | |
3703 | | NetworkStatisticsLogger::LogGET(nTotalDownloaded); |
3704 | | }; |
3705 | | |
3706 | | m_oThreadAdviseRead = std::thread(task, l_osURL); |
3707 | | } |
3708 | | |
3709 | | /************************************************************************/ |
3710 | | /* Write() */ |
3711 | | /************************************************************************/ |
3712 | | |
3713 | | size_t VSICurlHandle::Write(const void * /* pBuffer */, size_t /* nSize */, |
3714 | | size_t /* nMemb */) |
3715 | | { |
3716 | | return 0; |
3717 | | } |
3718 | | |
3719 | | /************************************************************************/ |
3720 | | /* ClearErr() */ |
3721 | | /************************************************************************/ |
3722 | | |
3723 | | void VSICurlHandle::ClearErr() |
3724 | | |
3725 | | { |
3726 | | bEOF = false; |
3727 | | bError = false; |
3728 | | } |
3729 | | |
3730 | | /************************************************************************/ |
3731 | | /* Error() */ |
3732 | | /************************************************************************/ |
3733 | | |
3734 | | int VSICurlHandle::Error() |
3735 | | |
3736 | | { |
3737 | | return bError ? TRUE : FALSE; |
3738 | | } |
3739 | | |
3740 | | /************************************************************************/ |
3741 | | /* Eof() */ |
3742 | | /************************************************************************/ |
3743 | | |
3744 | | int VSICurlHandle::Eof() |
3745 | | |
3746 | | { |
3747 | | return bEOF ? TRUE : FALSE; |
3748 | | } |
3749 | | |
3750 | | /************************************************************************/ |
3751 | | /* Flush() */ |
3752 | | /************************************************************************/ |
3753 | | |
3754 | | int VSICurlHandle::Flush() |
3755 | | { |
3756 | | return 0; |
3757 | | } |
3758 | | |
3759 | | /************************************************************************/ |
3760 | | /* Close() */ |
3761 | | /************************************************************************/ |
3762 | | |
3763 | | int VSICurlHandle::Close() |
3764 | | { |
3765 | | return 0; |
3766 | | } |
3767 | | |
3768 | | /************************************************************************/ |
3769 | | /* VSICurlFilesystemHandlerBase() */ |
3770 | | /************************************************************************/ |
3771 | | |
3772 | | VSICurlFilesystemHandlerBase::VSICurlFilesystemHandlerBase() |
3773 | | : oCacheFileProp{100 * 1024}, oCacheDirList{1024, 0} |
3774 | | { |
3775 | | } |
3776 | | |
3777 | | /************************************************************************/ |
3778 | | /* CachedConnection */ |
3779 | | /************************************************************************/ |
3780 | | |
3781 | | namespace |
3782 | | { |
3783 | | struct CachedConnection |
3784 | | { |
3785 | | CURLM *hCurlMultiHandle = nullptr; |
3786 | | void clear(); |
3787 | | |
3788 | | ~CachedConnection() |
3789 | | { |
3790 | | clear(); |
3791 | | } |
3792 | | }; |
3793 | | } // namespace |
3794 | | |
3795 | | #ifdef _WIN32 |
3796 | | // Currently thread_local and C++ objects don't work well with DLL on Windows |
3797 | | static void FreeCachedConnection(void *pData) |
3798 | | { |
3799 | | delete static_cast< |
3800 | | std::map<VSICurlFilesystemHandlerBase *, CachedConnection> *>(pData); |
3801 | | } |
3802 | | |
3803 | | // Per-thread and per-filesystem Curl connection cache. |
3804 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> & |
3805 | | GetConnectionCache() |
3806 | | { |
3807 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> |
3808 | | dummyCache; |
3809 | | int bMemoryErrorOccurred = false; |
3810 | | void *pData = |
3811 | | CPLGetTLSEx(CTLS_VSICURL_CACHEDCONNECTION, &bMemoryErrorOccurred); |
3812 | | if (bMemoryErrorOccurred) |
3813 | | { |
3814 | | return dummyCache; |
3815 | | } |
3816 | | if (pData == nullptr) |
3817 | | { |
3818 | | auto cachedConnection = |
3819 | | new std::map<VSICurlFilesystemHandlerBase *, CachedConnection>(); |
3820 | | CPLSetTLSWithFreeFuncEx(CTLS_VSICURL_CACHEDCONNECTION, cachedConnection, |
3821 | | FreeCachedConnection, &bMemoryErrorOccurred); |
3822 | | if (bMemoryErrorOccurred) |
3823 | | { |
3824 | | delete cachedConnection; |
3825 | | return dummyCache; |
3826 | | } |
3827 | | return *cachedConnection; |
3828 | | } |
3829 | | return *static_cast< |
3830 | | std::map<VSICurlFilesystemHandlerBase *, CachedConnection> *>(pData); |
3831 | | } |
3832 | | #else |
3833 | | static thread_local std::map<VSICurlFilesystemHandlerBase *, CachedConnection> |
3834 | | g_tls_connectionCache; |
3835 | | |
3836 | | static std::map<VSICurlFilesystemHandlerBase *, CachedConnection> & |
3837 | | GetConnectionCache() |
3838 | | { |
3839 | | return g_tls_connectionCache; |
3840 | | } |
3841 | | #endif |
3842 | | |
3843 | | /************************************************************************/ |
3844 | | /* clear() */ |
3845 | | /************************************************************************/ |
3846 | | |
3847 | | void CachedConnection::clear() |
3848 | | { |
3849 | | if (hCurlMultiHandle) |
3850 | | { |
3851 | | VSICURLMultiCleanup(hCurlMultiHandle); |
3852 | | hCurlMultiHandle = nullptr; |
3853 | | } |
3854 | | } |
3855 | | |
3856 | | /************************************************************************/ |
3857 | | /* ~VSICurlFilesystemHandlerBase() */ |
3858 | | /************************************************************************/ |
3859 | | |
3860 | | VSICurlFilesystemHandlerBase::~VSICurlFilesystemHandlerBase() |
3861 | | { |
3862 | | VSICurlFilesystemHandlerBase::ClearCache(); |
3863 | | GetConnectionCache().erase(this); |
3864 | | |
3865 | | if (hMutex != nullptr) |
3866 | | CPLDestroyMutex(hMutex); |
3867 | | hMutex = nullptr; |
3868 | | } |
3869 | | |
3870 | | /************************************************************************/ |
3871 | | /* AllowCachedDataFor() */ |
3872 | | /************************************************************************/ |
3873 | | |
3874 | | bool VSICurlFilesystemHandlerBase::AllowCachedDataFor(const char *pszFilename) |
3875 | | { |
3876 | | bool bCachedAllowed = true; |
3877 | | char **papszTokens = CSLTokenizeString2( |
3878 | | CPLGetConfigOption("CPL_VSIL_CURL_NON_CACHED", ""), ":", 0); |
3879 | | for (int i = 0; papszTokens && papszTokens[i]; i++) |
3880 | | { |
3881 | | if (STARTS_WITH(pszFilename, papszTokens[i])) |
3882 | | { |
3883 | | bCachedAllowed = false; |
3884 | | break; |
3885 | | } |
3886 | | } |
3887 | | CSLDestroy(papszTokens); |
3888 | | return bCachedAllowed; |
3889 | | } |
3890 | | |
3891 | | /************************************************************************/ |
3892 | | /* GetCurlMultiHandleFor() */ |
3893 | | /************************************************************************/ |
3894 | | |
3895 | | CURLM *VSICurlFilesystemHandlerBase::GetCurlMultiHandleFor( |
3896 | | const std::string & /*osURL*/) |
3897 | | { |
3898 | | auto &conn = GetConnectionCache()[this]; |
3899 | | if (conn.hCurlMultiHandle == nullptr) |
3900 | | { |
3901 | | conn.hCurlMultiHandle = VSICURLMultiInit(); |
3902 | | } |
3903 | | return conn.hCurlMultiHandle; |
3904 | | } |
3905 | | |
3906 | | /************************************************************************/ |
3907 | | /* GetRegionCache() */ |
3908 | | /************************************************************************/ |
3909 | | |
3910 | | VSICurlFilesystemHandlerBase::RegionCacheType * |
3911 | | VSICurlFilesystemHandlerBase::GetRegionCache() |
3912 | | { |
3913 | | // should be called under hMutex taken |
3914 | | if (m_poRegionCacheDoNotUseDirectly == nullptr) |
3915 | | { |
3916 | | m_poRegionCacheDoNotUseDirectly.reset( |
3917 | | new RegionCacheType(static_cast<size_t>(GetMaxRegions()))); |
3918 | | } |
3919 | | return m_poRegionCacheDoNotUseDirectly.get(); |
3920 | | } |
3921 | | |
3922 | | /************************************************************************/ |
3923 | | /* GetRegion() */ |
3924 | | /************************************************************************/ |
3925 | | |
3926 | | std::shared_ptr<std::string> |
3927 | | VSICurlFilesystemHandlerBase::GetRegion(const char *pszURL, |
3928 | | vsi_l_offset nFileOffsetStart) |
3929 | | { |
3930 | | CPLMutexHolder oHolder(&hMutex); |
3931 | | |
3932 | | const int knDOWNLOAD_CHUNK_SIZE = VSICURLGetDownloadChunkSize(); |
3933 | | nFileOffsetStart = |
3934 | | (nFileOffsetStart / knDOWNLOAD_CHUNK_SIZE) * knDOWNLOAD_CHUNK_SIZE; |
3935 | | |
3936 | | std::shared_ptr<std::string> out; |
3937 | | if (GetRegionCache()->tryGet( |
3938 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), out)) |
3939 | | { |
3940 | | return out; |
3941 | | } |
3942 | | |
3943 | | return nullptr; |
3944 | | } |
3945 | | |
3946 | | /************************************************************************/ |
3947 | | /* AddRegion() */ |
3948 | | /************************************************************************/ |
3949 | | |
3950 | | void VSICurlFilesystemHandlerBase::AddRegion(const char *pszURL, |
3951 | | vsi_l_offset nFileOffsetStart, |
3952 | | size_t nSize, const char *pData) |
3953 | | { |
3954 | | CPLMutexHolder oHolder(&hMutex); |
3955 | | |
3956 | | std::shared_ptr<std::string> value(new std::string()); |
3957 | | value->assign(pData, nSize); |
3958 | | GetRegionCache()->insert( |
3959 | | FilenameOffsetPair(std::string(pszURL), nFileOffsetStart), value); |
3960 | | } |
3961 | | |
3962 | | /************************************************************************/ |
3963 | | /* GetCachedFileProp() */ |
3964 | | /************************************************************************/ |
3965 | | |
3966 | | bool VSICurlFilesystemHandlerBase::GetCachedFileProp(const char *pszURL, |
3967 | | FileProp &oFileProp) |
3968 | | { |
3969 | | CPLMutexHolder oHolder(&hMutex); |
3970 | | bool inCache; |
3971 | | if (oCacheFileProp.tryGet(std::string(pszURL), inCache)) |
3972 | | { |
3973 | | if (VSICURLGetCachedFileProp(pszURL, oFileProp)) |
3974 | | { |
3975 | | return true; |
3976 | | } |
3977 | | oCacheFileProp.remove(std::string(pszURL)); |
3978 | | } |
3979 | | return false; |
3980 | | } |
3981 | | |
3982 | | /************************************************************************/ |
3983 | | /* SetCachedFileProp() */ |
3984 | | /************************************************************************/ |
3985 | | |
3986 | | void VSICurlFilesystemHandlerBase::SetCachedFileProp(const char *pszURL, |
3987 | | FileProp &oFileProp) |
3988 | | { |
3989 | | CPLMutexHolder oHolder(&hMutex); |
3990 | | oCacheFileProp.insert(std::string(pszURL), true); |
3991 | | VSICURLSetCachedFileProp(pszURL, oFileProp); |
3992 | | } |
3993 | | |
3994 | | /************************************************************************/ |
3995 | | /* GetCachedDirList() */ |
3996 | | /************************************************************************/ |
3997 | | |
3998 | | bool VSICurlFilesystemHandlerBase::GetCachedDirList( |
3999 | | const char *pszURL, CachedDirList &oCachedDirList) |
4000 | | { |
4001 | | CPLMutexHolder oHolder(&hMutex); |
4002 | | |
4003 | | return oCacheDirList.tryGet(std::string(pszURL), oCachedDirList) && |
4004 | | // Let a chance to use new auth parameters |
4005 | | gnGenerationAuthParameters == |
4006 | | oCachedDirList.nGenerationAuthParameters; |
4007 | | } |
4008 | | |
4009 | | /************************************************************************/ |
4010 | | /* SetCachedDirList() */ |
4011 | | /************************************************************************/ |
4012 | | |
4013 | | void VSICurlFilesystemHandlerBase::SetCachedDirList( |
4014 | | const char *pszURL, CachedDirList &oCachedDirList) |
4015 | | { |
4016 | | CPLMutexHolder oHolder(&hMutex); |
4017 | | |
4018 | | std::string key(pszURL); |
4019 | | CachedDirList oldValue; |
4020 | | if (oCacheDirList.tryGet(key, oldValue)) |
4021 | | { |
4022 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4023 | | oCacheDirList.remove(key); |
4024 | | } |
4025 | | |
4026 | | while ((!oCacheDirList.empty() && |
4027 | | nCachedFilesInDirList + oCachedDirList.oFileList.size() > |
4028 | | 1024 * 1024) || |
4029 | | oCacheDirList.size() == oCacheDirList.getMaxAllowedSize()) |
4030 | | { |
4031 | | std::string oldestKey; |
4032 | | oCacheDirList.getOldestEntry(oldestKey, oldValue); |
4033 | | nCachedFilesInDirList -= oldValue.oFileList.size(); |
4034 | | oCacheDirList.remove(oldestKey); |
4035 | | } |
4036 | | oCachedDirList.nGenerationAuthParameters = gnGenerationAuthParameters; |
4037 | | |
4038 | | nCachedFilesInDirList += oCachedDirList.oFileList.size(); |
4039 | | oCacheDirList.insert(key, oCachedDirList); |
4040 | | } |
4041 | | |
4042 | | /************************************************************************/ |
4043 | | /* ExistsInCacheDirList() */ |
4044 | | /************************************************************************/ |
4045 | | |
4046 | | bool VSICurlFilesystemHandlerBase::ExistsInCacheDirList( |
4047 | | const std::string &osDirname, bool *pbIsDir) |
4048 | | { |
4049 | | CachedDirList cachedDirList; |
4050 | | if (GetCachedDirList(osDirname.c_str(), cachedDirList)) |
4051 | | { |
4052 | | if (pbIsDir) |
4053 | | *pbIsDir = !cachedDirList.oFileList.empty(); |
4054 | | return false; |
4055 | | } |
4056 | | else |
4057 | | { |
4058 | | if (pbIsDir) |
4059 | | *pbIsDir = false; |
4060 | | return false; |
4061 | | } |
4062 | | } |
4063 | | |
4064 | | /************************************************************************/ |
4065 | | /* InvalidateCachedData() */ |
4066 | | /************************************************************************/ |
4067 | | |
4068 | | void VSICurlFilesystemHandlerBase::InvalidateCachedData(const char *pszURL) |
4069 | | { |
4070 | | CPLMutexHolder oHolder(&hMutex); |
4071 | | |
4072 | | oCacheFileProp.remove(std::string(pszURL)); |
4073 | | |
4074 | | // Invalidate all cached regions for this URL |
4075 | | std::list<FilenameOffsetPair> keysToRemove; |
4076 | | std::string osURL(pszURL); |
4077 | | auto lambda = |
4078 | | [&keysToRemove, |
4079 | | &osURL](const lru11::KeyValuePair<FilenameOffsetPair, |
4080 | | std::shared_ptr<std::string>> &kv) |
4081 | | { |
4082 | | if (kv.key.filename_ == osURL) |
4083 | | keysToRemove.push_back(kv.key); |
4084 | | }; |
4085 | | auto *poRegionCache = GetRegionCache(); |
4086 | | poRegionCache->cwalk(lambda); |
4087 | | for (const auto &key : keysToRemove) |
4088 | | poRegionCache->remove(key); |
4089 | | } |
4090 | | |
4091 | | /************************************************************************/ |
4092 | | /* ClearCache() */ |
4093 | | /************************************************************************/ |
4094 | | |
4095 | | void VSICurlFilesystemHandlerBase::ClearCache() |
4096 | | { |
4097 | | CPLMutexHolder oHolder(&hMutex); |
4098 | | |
4099 | | GetRegionCache()->clear(); |
4100 | | |
4101 | | { |
4102 | | const auto lambda = [](const lru11::KeyValuePair<std::string, bool> &kv) |
4103 | | { VSICURLInvalidateCachedFileProp(kv.key.c_str()); }; |
4104 | | oCacheFileProp.cwalk(lambda); |
4105 | | oCacheFileProp.clear(); |
4106 | | } |
4107 | | |
4108 | | oCacheDirList.clear(); |
4109 | | nCachedFilesInDirList = 0; |
4110 | | |
4111 | | GetConnectionCache()[this].clear(); |
4112 | | } |
4113 | | |
4114 | | /************************************************************************/ |
4115 | | /* PartialClearCache() */ |
4116 | | /************************************************************************/ |
4117 | | |
4118 | | void VSICurlFilesystemHandlerBase::PartialClearCache( |
4119 | | const char *pszFilenamePrefix) |
4120 | | { |
4121 | | CPLMutexHolder oHolder(&hMutex); |
4122 | | |
4123 | | std::string osURL = GetURLFromFilename(pszFilenamePrefix); |
4124 | | { |
4125 | | std::list<FilenameOffsetPair> keysToRemove; |
4126 | | auto lambda = |
4127 | | [&keysToRemove, &osURL]( |
4128 | | const lru11::KeyValuePair<FilenameOffsetPair, |
4129 | | std::shared_ptr<std::string>> &kv) |
4130 | | { |
4131 | | if (strncmp(kv.key.filename_.c_str(), osURL.c_str(), |
4132 | | osURL.size()) == 0) |
4133 | | keysToRemove.push_back(kv.key); |
4134 | | }; |
4135 | | auto *poRegionCache = GetRegionCache(); |
4136 | | poRegionCache->cwalk(lambda); |
4137 | | for (const auto &key : keysToRemove) |
4138 | | poRegionCache->remove(key); |
4139 | | } |
4140 | | |
4141 | | { |
4142 | | std::list<std::string> keysToRemove; |
4143 | | auto lambda = [&keysToRemove, |
4144 | | &osURL](const lru11::KeyValuePair<std::string, bool> &kv) |
4145 | | { |
4146 | | if (strncmp(kv.key.c_str(), osURL.c_str(), osURL.size()) == 0) |
4147 | | keysToRemove.push_back(kv.key); |
4148 | | }; |
4149 | | oCacheFileProp.cwalk(lambda); |
4150 | | for (const auto &key : keysToRemove) |
4151 | | oCacheFileProp.remove(key); |
4152 | | } |
4153 | | VSICURLInvalidateCachedFilePropPrefix(osURL.c_str()); |
4154 | | |
4155 | | { |
4156 | | const size_t nLen = strlen(pszFilenamePrefix); |
4157 | | std::list<std::string> keysToRemove; |
4158 | | auto lambda = |
4159 | | [this, &keysToRemove, pszFilenamePrefix, |
4160 | | nLen](const lru11::KeyValuePair<std::string, CachedDirList> &kv) |
4161 | | { |
4162 | | if (strncmp(kv.key.c_str(), pszFilenamePrefix, nLen) == 0) |
4163 | | { |
4164 | | keysToRemove.push_back(kv.key); |
4165 | | nCachedFilesInDirList -= kv.value.oFileList.size(); |
4166 | | } |
4167 | | }; |
4168 | | oCacheDirList.cwalk(lambda); |
4169 | | for (const auto &key : keysToRemove) |
4170 | | oCacheDirList.remove(key); |
4171 | | } |
4172 | | } |
4173 | | |
4174 | | /************************************************************************/ |
4175 | | /* CreateFileHandle() */ |
4176 | | /************************************************************************/ |
4177 | | |
4178 | | VSICurlHandle * |
4179 | | VSICurlFilesystemHandlerBase::CreateFileHandle(const char *pszFilename) |
4180 | | { |
4181 | | return new VSICurlHandle(this, pszFilename); |
4182 | | } |
4183 | | |
4184 | | /************************************************************************/ |
4185 | | /* GetActualURL() */ |
4186 | | /************************************************************************/ |
4187 | | |
4188 | | const char *VSICurlFilesystemHandlerBase::GetActualURL(const char *pszFilename) |
4189 | | { |
4190 | | VSICurlHandle *poHandle = CreateFileHandle(pszFilename); |
4191 | | if (poHandle == nullptr) |
4192 | | return pszFilename; |
4193 | | std::string osURL(poHandle->GetURL()); |
4194 | | delete poHandle; |
4195 | | return CPLSPrintf("%s", osURL.c_str()); |
4196 | | } |
4197 | | |
4198 | | /************************************************************************/ |
4199 | | /* GetOptions() */ |
4200 | | /************************************************************************/ |
4201 | | |
4202 | | #define VSICURL_OPTIONS \ |
4203 | | " <Option name='GDAL_HTTP_MAX_RETRY' type='int' " \ |
4204 | | "description='Maximum number of retries' default='0'/>" \ |
4205 | | " <Option name='GDAL_HTTP_RETRY_DELAY' type='double' " \ |
4206 | | "description='Retry delay in seconds' default='30'/>" \ |
4207 | | " <Option name='GDAL_HTTP_HEADER_FILE' type='string' " \ |
4208 | | "description='Filename of a file that contains HTTP headers to " \ |
4209 | | "forward to the server'/>" \ |
4210 | | " <Option name='CPL_VSIL_CURL_USE_HEAD' type='boolean' " \ |
4211 | | "description='Whether to use HTTP HEAD verb to retrieve " \ |
4212 | | "file information' default='YES'/>" \ |
4213 | | " <Option name='GDAL_HTTP_MULTIRANGE' type='string-select' " \ |
4214 | | "description='Strategy to apply to run multi-range requests' " \ |
4215 | | "default='PARALLEL'>" \ |
4216 | | " <Value>PARALLEL</Value>" \ |
4217 | | " <Value>SERIAL</Value>" \ |
4218 | | " </Option>" \ |
4219 | | " <Option name='GDAL_HTTP_MULTIPLEX' type='boolean' " \ |
4220 | | "description='Whether to enable HTTP/2 multiplexing' default='YES'/>" \ |
4221 | | " <Option name='GDAL_HTTP_MERGE_CONSECUTIVE_RANGES' type='boolean' " \ |
4222 | | "description='Whether to merge consecutive ranges in multirange " \ |
4223 | | "requests' default='YES'/>" \ |
4224 | | " <Option name='CPL_VSIL_CURL_NON_CACHED' type='string' " \ |
4225 | | "description='Colon-separated list of filenames whose content" \ |
4226 | | "must not be cached across open attempts'/>" \ |
4227 | | " <Option name='CPL_VSIL_CURL_ALLOWED_FILENAME' type='string' " \ |
4228 | | "description='Single filename that is allowed to be opened'/>" \ |
4229 | | " <Option name='CPL_VSIL_CURL_ALLOWED_EXTENSIONS' type='string' " \ |
4230 | | "description='Comma or space separated list of allowed file " \ |
4231 | | "extensions'/>" \ |
4232 | | " <Option name='GDAL_DISABLE_READDIR_ON_OPEN' type='string-select' " \ |
4233 | | "description='Whether to disable establishing the list of files in " \ |
4234 | | "the directory of the current filename' default='NO'>" \ |
4235 | | " <Value>NO</Value>" \ |
4236 | | " <Value>YES</Value>" \ |
4237 | | " <Value>EMPTY_DIR</Value>" \ |
4238 | | " </Option>" \ |
4239 | | " <Option name='VSI_CACHE' type='boolean' " \ |
4240 | | "description='Whether to cache in memory the contents of the opened " \ |
4241 | | "file as soon as they are read' default='NO'/>" \ |
4242 | | " <Option name='CPL_VSIL_CURL_CHUNK_SIZE' type='integer' " \ |
4243 | | "description='Size in bytes of the minimum amount of data read in a " \ |
4244 | | "file' default='16384' min='1024' max='10485760'/>" \ |
4245 | | " <Option name='CPL_VSIL_CURL_CACHE_SIZE' type='integer' " \ |
4246 | | "description='Size in bytes of the global /vsicurl/ cache' " \ |
4247 | | "default='16384000'/>" \ |
4248 | | " <Option name='CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE' type='boolean' " \ |
4249 | | "description='Whether to skip files with Glacier storage class in " \ |
4250 | | "directory listing.' default='YES'/>" \ |
4251 | | " <Option name='CPL_VSIL_CURL_ADVISE_READ_TOTAL_BYTES_LIMIT' " \ |
4252 | | "type='integer' description='Maximum number of bytes AdviseRead() is " \ |
4253 | | "allowed to fetch at once' default='104857600'/>" \ |
4254 | | " <Option name='GDAL_HTTP_MAX_CACHED_CONNECTIONS' type='integer' " \ |
4255 | | "description='Maximum amount of connections that libcurl may keep alive " \ |
4256 | | "in its connection cache after use'/>" \ |
4257 | | " <Option name='GDAL_HTTP_MAX_TOTAL_CONNECTIONS' type='integer' " \ |
4258 | | "description='Maximum number of simultaneously open connections in " \ |
4259 | | "total'/>" |
4260 | | |
4261 | | const char *VSICurlFilesystemHandlerBase::GetOptionsStatic() |
4262 | | { |
4263 | | return VSICURL_OPTIONS; |
4264 | | } |
4265 | | |
4266 | | const char *VSICurlFilesystemHandlerBase::GetOptions() |
4267 | | { |
4268 | | static std::string osOptions(std::string("<Options>") + GetOptionsStatic() + |
4269 | | "</Options>"); |
4270 | | return osOptions.c_str(); |
4271 | | } |
4272 | | |
4273 | | /************************************************************************/ |
4274 | | /* IsAllowedFilename() */ |
4275 | | /************************************************************************/ |
4276 | | |
4277 | | bool VSICurlFilesystemHandlerBase::IsAllowedFilename(const char *pszFilename) |
4278 | | { |
4279 | | const char *pszAllowedFilename = |
4280 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_FILENAME", nullptr); |
4281 | | if (pszAllowedFilename != nullptr) |
4282 | | { |
4283 | | return strcmp(pszFilename, pszAllowedFilename) == 0; |
4284 | | } |
4285 | | |
4286 | | // Consider that only the files whose extension ends up with one that is |
4287 | | // listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server. This can |
4288 | | // speeds up dramatically open experience, in case the server cannot return |
4289 | | // a file list. {noext} can be used as a special token to mean file with no |
4290 | | // extension. |
4291 | | // For example: |
4292 | | // gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" |
4293 | | // /vsicurl/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif |
4294 | | const char *pszAllowedExtensions = |
4295 | | CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", nullptr); |
4296 | | if (pszAllowedExtensions) |
4297 | | { |
4298 | | char **papszExtensions = |
4299 | | CSLTokenizeString2(pszAllowedExtensions, ", ", 0); |
4300 | | const char *queryStart = strchr(pszFilename, '?'); |
4301 | | char *pszFilenameWithoutQuery = nullptr; |
4302 | | if (queryStart != nullptr) |
4303 | | { |
4304 | | pszFilenameWithoutQuery = CPLStrdup(pszFilename); |
4305 | | pszFilenameWithoutQuery[queryStart - pszFilename] = '\0'; |
4306 | | pszFilename = pszFilenameWithoutQuery; |
4307 | | } |
4308 | | const size_t nURLLen = strlen(pszFilename); |
4309 | | bool bFound = false; |
4310 | | for (int i = 0; papszExtensions[i] != nullptr; i++) |
4311 | | { |
4312 | | const size_t nExtensionLen = strlen(papszExtensions[i]); |
4313 | | if (EQUAL(papszExtensions[i], "{noext}")) |
4314 | | { |
4315 | | const char *pszLastSlash = strrchr(pszFilename, '/'); |
4316 | | if (pszLastSlash != nullptr && |
4317 | | strchr(pszLastSlash, '.') == nullptr) |
4318 | | { |
4319 | | bFound = true; |
4320 | | break; |
4321 | | } |
4322 | | } |
4323 | | else if (nURLLen > nExtensionLen && |
4324 | | EQUAL(pszFilename + nURLLen - nExtensionLen, |
4325 | | papszExtensions[i])) |
4326 | | { |
4327 | | bFound = true; |
4328 | | break; |
4329 | | } |
4330 | | } |
4331 | | |
4332 | | CSLDestroy(papszExtensions); |
4333 | | if (pszFilenameWithoutQuery) |
4334 | | { |
4335 | | CPLFree(pszFilenameWithoutQuery); |
4336 | | } |
4337 | | |
4338 | | return bFound; |
4339 | | } |
4340 | | return TRUE; |
4341 | | } |
4342 | | |
4343 | | /************************************************************************/ |
4344 | | /* Open() */ |
4345 | | /************************************************************************/ |
4346 | | |
4347 | | VSIVirtualHandle *VSICurlFilesystemHandlerBase::Open(const char *pszFilename, |
4348 | | const char *pszAccess, |
4349 | | bool bSetError, |
4350 | | CSLConstList papszOptions) |
4351 | | { |
4352 | | if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str()) && |
4353 | | !STARTS_WITH_CI(pszFilename, "/vsicurl?")) |
4354 | | return nullptr; |
4355 | | |
4356 | | if (strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, '+') != nullptr) |
4357 | | { |
4358 | | if (bSetError) |
4359 | | { |
4360 | | VSIError(VSIE_FileError, |
4361 | | "Only read-only mode is supported for /vsicurl"); |
4362 | | } |
4363 | | return nullptr; |
4364 | | } |
4365 | | if (!papszOptions || |
4366 | | !CPLTestBool(CSLFetchNameValueDef( |
4367 | | papszOptions, "IGNORE_FILENAME_RESTRICTIONS", "NO"))) |
4368 | | { |
4369 | | if (!IsAllowedFilename(pszFilename)) |
4370 | | return nullptr; |
4371 | | } |
4372 | | |
4373 | | bool bListDir = true; |
4374 | | bool bEmptyDir = false; |
4375 | | CPL_IGNORE_RET_VAL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
4376 | | nullptr, &bListDir, &bEmptyDir, |
4377 | | nullptr, nullptr, nullptr)); |
4378 | | |
4379 | | const char *pszOptionVal = CSLFetchNameValueDef( |
4380 | | papszOptions, "DISABLE_READDIR_ON_OPEN", |
4381 | | VSIGetPathSpecificOption(pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", |
4382 | | "NO")); |
4383 | | const bool bSkipReadDir = |
4384 | | !bListDir || bEmptyDir || EQUAL(pszOptionVal, "EMPTY_DIR") || |
4385 | | CPLTestBool(pszOptionVal) || !AllowCachedDataFor(pszFilename); |
4386 | | |
4387 | | std::string osFilename(pszFilename); |
4388 | | bool bGotFileList = !bSkipReadDir; |
4389 | | bool bForceExistsCheck = false; |
4390 | | FileProp cachedFileProp; |
4391 | | if (!(GetCachedFileProp(osFilename.c_str() + strlen(GetFSPrefix().c_str()), |
4392 | | cachedFileProp) && |
4393 | | cachedFileProp.eExists == EXIST_YES) && |
4394 | | strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
4395 | | !STARTS_WITH(CPLGetExtensionSafe(osFilename.c_str()).c_str(), "zip") && |
4396 | | !bSkipReadDir) |
4397 | | { |
4398 | | char **papszFileList = ReadDirInternal( |
4399 | | (CPLGetDirnameSafe(osFilename.c_str()) + '/').c_str(), 0, |
4400 | | &bGotFileList); |
4401 | | const bool bFound = |
4402 | | VSICurlIsFileInList(papszFileList, |
4403 | | CPLGetFilename(osFilename.c_str())) != -1; |
4404 | | if (bGotFileList && !bFound) |
4405 | | { |
4406 | | // Some file servers are case insensitive, so in case there is a |
4407 | | // match with case difference, do a full check just in case. |
4408 | | // e.g. |
4409 | | // http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/MEGA90N000CB.IMG |
4410 | | // that is queried by |
4411 | | // gdalinfo |
4412 | | // /vsicurl/http://pds-geosciences.wustl.edu/mgs/mgs-m-mola-5-megdr-l3-v1/mgsl_300x/meg004/mega90n000cb.lbl |
4413 | | if (CSLFindString(papszFileList, |
4414 | | CPLGetFilename(osFilename.c_str())) != -1) |
4415 | | { |
4416 | | bForceExistsCheck = true; |
4417 | | } |
4418 | | else |
4419 | | { |
4420 | | CSLDestroy(papszFileList); |
4421 | | return nullptr; |
4422 | | } |
4423 | | } |
4424 | | CSLDestroy(papszFileList); |
4425 | | } |
4426 | | |
4427 | | VSICurlHandle *poHandle = CreateFileHandle(osFilename.c_str()); |
4428 | | if (poHandle == nullptr) |
4429 | | return nullptr; |
4430 | | if (!bGotFileList || bForceExistsCheck) |
4431 | | { |
4432 | | // If we didn't get a filelist, check that the file really exists. |
4433 | | if (!poHandle->Exists(bSetError)) |
4434 | | { |
4435 | | delete poHandle; |
4436 | | return nullptr; |
4437 | | } |
4438 | | } |
4439 | | |
4440 | | if (CPLTestBool(CPLGetConfigOption("VSI_CACHE", "FALSE"))) |
4441 | | return VSICreateCachedFile(poHandle); |
4442 | | else |
4443 | | return poHandle; |
4444 | | } |
4445 | | |
4446 | | /************************************************************************/ |
4447 | | /* VSICurlParserFindEOL() */ |
4448 | | /* */ |
4449 | | /* Small helper function for VSICurlPaseHTMLFileList() to find */ |
4450 | | /* the end of a line in the directory listing. Either a <br> */ |
4451 | | /* or newline. */ |
4452 | | /************************************************************************/ |
4453 | | |
4454 | | static char *VSICurlParserFindEOL(char *pszData) |
4455 | | |
4456 | | { |
4457 | | while (*pszData != '\0' && *pszData != '\n' && |
4458 | | !STARTS_WITH_CI(pszData, "<br>")) |
4459 | | pszData++; |
4460 | | |
4461 | | if (*pszData == '\0') |
4462 | | return nullptr; |
4463 | | |
4464 | | return pszData; |
4465 | | } |
4466 | | |
4467 | | /************************************************************************/ |
4468 | | /* VSICurlParseHTMLDateTimeFileSize() */ |
4469 | | /************************************************************************/ |
4470 | | |
4471 | | static const char *const apszMonths[] = { |
4472 | | "January", "February", "March", "April", "May", "June", |
4473 | | "July", "August", "September", "October", "November", "December"}; |
4474 | | |
4475 | | static bool VSICurlParseHTMLDateTimeFileSize(const char *pszStr, |
4476 | | struct tm &brokendowntime, |
4477 | | GUIntBig &nFileSize, |
4478 | | GIntBig &mTime) |
4479 | | { |
4480 | | for (int iMonth = 0; iMonth < 12; iMonth++) |
4481 | | { |
4482 | | char szMonth[32] = {}; |
4483 | | szMonth[0] = '-'; |
4484 | | memcpy(szMonth + 1, apszMonths[iMonth], 3); |
4485 | | szMonth[4] = '-'; |
4486 | | szMonth[5] = '\0'; |
4487 | | const char *pszMonthFound = strstr(pszStr, szMonth); |
4488 | | if (pszMonthFound) |
4489 | | { |
4490 | | // Format of Apache, like in |
4491 | | // http://download.osgeo.org/gdal/data/gtiff/ |
4492 | | // "17-May-2010 12:26" |
4493 | | if (pszMonthFound - pszStr > 2 && strlen(pszMonthFound) > 15 && |
4494 | | pszMonthFound[-2 + 11] == ' ' && pszMonthFound[-2 + 14] == ':') |
4495 | | { |
4496 | | pszMonthFound -= 2; |
4497 | | int nDay = atoi(pszMonthFound); |
4498 | | int nYear = atoi(pszMonthFound + 7); |
4499 | | int nHour = atoi(pszMonthFound + 12); |
4500 | | int nMin = atoi(pszMonthFound + 15); |
4501 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4502 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4503 | | { |
4504 | | brokendowntime.tm_year = nYear - 1900; |
4505 | | brokendowntime.tm_mon = iMonth; |
4506 | | brokendowntime.tm_mday = nDay; |
4507 | | brokendowntime.tm_hour = nHour; |
4508 | | brokendowntime.tm_min = nMin; |
4509 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4510 | | |
4511 | | return true; |
4512 | | } |
4513 | | } |
4514 | | return false; |
4515 | | } |
4516 | | |
4517 | | /* Microsoft IIS */ |
4518 | | snprintf(szMonth, sizeof(szMonth), " %s ", apszMonths[iMonth]); |
4519 | | pszMonthFound = strstr(pszStr, szMonth); |
4520 | | if (pszMonthFound) |
4521 | | { |
4522 | | int nLenMonth = static_cast<int>(strlen(apszMonths[iMonth])); |
4523 | | if (pszMonthFound - pszStr > 2 && pszMonthFound[-1] != ',' && |
4524 | | pszMonthFound[-2] != ' ' && |
4525 | | static_cast<int>(strlen(pszMonthFound - 2)) > |
4526 | | 2 + 1 + nLenMonth + 1 + 4 + 1 + 5 + 1 + 4) |
4527 | | { |
4528 | | /* Format of http://ortho.linz.govt.nz/tifs/1994_95/ */ |
4529 | | /* " Friday, 21 April 2006 12:05 p.m. 48062343 |
4530 | | * m35a_fy_94_95.tif" */ |
4531 | | pszMonthFound -= 2; |
4532 | | int nDay = atoi(pszMonthFound); |
4533 | | int nCurOffset = 2 + 1 + nLenMonth + 1; |
4534 | | int nYear = atoi(pszMonthFound + nCurOffset); |
4535 | | nCurOffset += 4 + 1; |
4536 | | int nHour = atoi(pszMonthFound + nCurOffset); |
4537 | | if (nHour < 10) |
4538 | | nCurOffset += 1 + 1; |
4539 | | else |
4540 | | nCurOffset += 2 + 1; |
4541 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
4542 | | nCurOffset += 2 + 1; |
4543 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "p.m.")) |
4544 | | nHour += 12; |
4545 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "a.m.")) |
4546 | | nHour = -1; |
4547 | | nCurOffset += 4; |
4548 | | |
4549 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
4550 | | while (*pszFilesize == ' ') |
4551 | | pszFilesize++; |
4552 | | if (*pszFilesize >= '1' && *pszFilesize <= '9') |
4553 | | nFileSize = CPLScanUIntBig( |
4554 | | pszFilesize, static_cast<int>(strlen(pszFilesize))); |
4555 | | |
4556 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4557 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4558 | | { |
4559 | | brokendowntime.tm_year = nYear - 1900; |
4560 | | brokendowntime.tm_mon = iMonth; |
4561 | | brokendowntime.tm_mday = nDay; |
4562 | | brokendowntime.tm_hour = nHour; |
4563 | | brokendowntime.tm_min = nMin; |
4564 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4565 | | |
4566 | | return true; |
4567 | | } |
4568 | | nFileSize = 0; |
4569 | | } |
4570 | | else if (pszMonthFound - pszStr > 1 && pszMonthFound[-1] == ',' && |
4571 | | static_cast<int>(strlen(pszMonthFound)) > |
4572 | | 1 + nLenMonth + 1 + 2 + 1 + 1 + 4 + 1 + 5 + 1 + 2) |
4573 | | { |
4574 | | // Format of |
4575 | | // http://publicfiles.dep.state.fl.us/dear/BWR_GIS/2007NWFLULC/ |
4576 | | // " Sunday, June 20, 2010 6:46 PM 233170905 |
4577 | | // NWF2007LULCForSDE.zip" |
4578 | | pszMonthFound += 1; |
4579 | | int nCurOffset = nLenMonth + 1; |
4580 | | int nDay = atoi(pszMonthFound + nCurOffset); |
4581 | | nCurOffset += 2 + 1 + 1; |
4582 | | int nYear = atoi(pszMonthFound + nCurOffset); |
4583 | | nCurOffset += 4 + 1; |
4584 | | int nHour = atoi(pszMonthFound + nCurOffset); |
4585 | | nCurOffset += 2 + 1; |
4586 | | const int nMin = atoi(pszMonthFound + nCurOffset); |
4587 | | nCurOffset += 2 + 1; |
4588 | | if (STARTS_WITH(pszMonthFound + nCurOffset, "PM")) |
4589 | | nHour += 12; |
4590 | | else if (!STARTS_WITH(pszMonthFound + nCurOffset, "AM")) |
4591 | | nHour = -1; |
4592 | | nCurOffset += 2; |
4593 | | |
4594 | | const char *pszFilesize = pszMonthFound + nCurOffset; |
4595 | | while (*pszFilesize == ' ') |
4596 | | pszFilesize++; |
4597 | | if (*pszFilesize >= '1' && *pszFilesize <= '9') |
4598 | | nFileSize = CPLScanUIntBig( |
4599 | | pszFilesize, static_cast<int>(strlen(pszFilesize))); |
4600 | | |
4601 | | if (nDay >= 1 && nDay <= 31 && nYear >= 1900 && nHour >= 0 && |
4602 | | nHour <= 24 && nMin >= 0 && nMin < 60) |
4603 | | { |
4604 | | brokendowntime.tm_year = nYear - 1900; |
4605 | | brokendowntime.tm_mon = iMonth; |
4606 | | brokendowntime.tm_mday = nDay; |
4607 | | brokendowntime.tm_hour = nHour; |
4608 | | brokendowntime.tm_min = nMin; |
4609 | | mTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4610 | | |
4611 | | return true; |
4612 | | } |
4613 | | nFileSize = 0; |
4614 | | } |
4615 | | return false; |
4616 | | } |
4617 | | } |
4618 | | |
4619 | | return false; |
4620 | | } |
4621 | | |
4622 | | /************************************************************************/ |
4623 | | /* ParseHTMLFileList() */ |
4624 | | /* */ |
4625 | | /* Parse a file list document and return all the components. */ |
4626 | | /************************************************************************/ |
4627 | | |
4628 | | char **VSICurlFilesystemHandlerBase::ParseHTMLFileList(const char *pszFilename, |
4629 | | int nMaxFiles, |
4630 | | char *pszData, |
4631 | | bool *pbGotFileList) |
4632 | | { |
4633 | | *pbGotFileList = false; |
4634 | | |
4635 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
4636 | | nullptr, nullptr, nullptr, |
4637 | | nullptr, nullptr, nullptr)); |
4638 | | const char *pszDir = nullptr; |
4639 | | if (STARTS_WITH_CI(osURL.c_str(), "http://")) |
4640 | | pszDir = strchr(osURL.c_str() + strlen("http://"), '/'); |
4641 | | else if (STARTS_WITH_CI(osURL.c_str(), "https://")) |
4642 | | pszDir = strchr(osURL.c_str() + strlen("https://"), '/'); |
4643 | | else if (STARTS_WITH_CI(osURL.c_str(), "ftp://")) |
4644 | | pszDir = strchr(osURL.c_str() + strlen("ftp://"), '/'); |
4645 | | if (pszDir == nullptr) |
4646 | | pszDir = ""; |
4647 | | |
4648 | | /* Apache */ |
4649 | | std::string osExpectedString = "<title>Index of "; |
4650 | | osExpectedString += pszDir; |
4651 | | osExpectedString += "</title>"; |
4652 | | /* shttpd */ |
4653 | | std::string osExpectedString2 = "<title>Index of "; |
4654 | | osExpectedString2 += pszDir; |
4655 | | osExpectedString2 += "/</title>"; |
4656 | | /* FTP */ |
4657 | | std::string osExpectedString3 = "FTP Listing of "; |
4658 | | osExpectedString3 += pszDir; |
4659 | | osExpectedString3 += "/"; |
4660 | | /* Apache 1.3.33 */ |
4661 | | std::string osExpectedString4 = "<TITLE>Index of "; |
4662 | | osExpectedString4 += pszDir; |
4663 | | osExpectedString4 += "</TITLE>"; |
4664 | | |
4665 | | // The listing of |
4666 | | // http://dds.cr.usgs.gov/srtm/SRTM_image_sample/picture%20examples/ |
4667 | | // has |
4668 | | // "<title>Index of /srtm/SRTM_image_sample/picture examples</title>" |
4669 | | // so we must try unescaped %20 also. |
4670 | | // Similar with |
4671 | | // http://datalib.usask.ca/gis/Data/Central_America_goodbutdoweown%3f/ |
4672 | | std::string osExpectedString_unescaped; |
4673 | | if (strchr(pszDir, '%')) |
4674 | | { |
4675 | | char *pszUnescapedDir = CPLUnescapeString(pszDir, nullptr, CPLES_URL); |
4676 | | osExpectedString_unescaped = "<title>Index of "; |
4677 | | osExpectedString_unescaped += pszUnescapedDir; |
4678 | | osExpectedString_unescaped += "</title>"; |
4679 | | CPLFree(pszUnescapedDir); |
4680 | | } |
4681 | | |
4682 | | char *c = nullptr; |
4683 | | int nCount = 0; |
4684 | | int nCountTable = 0; |
4685 | | CPLStringList oFileList; |
4686 | | char *pszLine = pszData; |
4687 | | bool bIsHTMLDirList = false; |
4688 | | |
4689 | | while ((c = VSICurlParserFindEOL(pszLine)) != nullptr) |
4690 | | { |
4691 | | *c = '\0'; |
4692 | | |
4693 | | // To avoid false positive on pages such as |
4694 | | // http://www.ngs.noaa.gov/PC_PROD/USGG2009BETA |
4695 | | // This is a heuristics, but normal HTML listing of files have not more |
4696 | | // than one table. |
4697 | | if (strstr(pszLine, "<table")) |
4698 | | { |
4699 | | nCountTable++; |
4700 | | if (nCountTable == 2) |
4701 | | { |
4702 | | *pbGotFileList = false; |
4703 | | return nullptr; |
4704 | | } |
4705 | | } |
4706 | | |
4707 | | if (!bIsHTMLDirList && |
4708 | | (strstr(pszLine, osExpectedString.c_str()) || |
4709 | | strstr(pszLine, osExpectedString2.c_str()) || |
4710 | | strstr(pszLine, osExpectedString3.c_str()) || |
4711 | | strstr(pszLine, osExpectedString4.c_str()) || |
4712 | | (!osExpectedString_unescaped.empty() && |
4713 | | strstr(pszLine, osExpectedString_unescaped.c_str())))) |
4714 | | { |
4715 | | bIsHTMLDirList = true; |
4716 | | *pbGotFileList = true; |
4717 | | } |
4718 | | // Subversion HTTP listing |
4719 | | // or Microsoft-IIS/6.0 listing |
4720 | | // (e.g. http://ortho.linz.govt.nz/tifs/2005_06/) */ |
4721 | | else if (!bIsHTMLDirList && strstr(pszLine, "<title>")) |
4722 | | { |
4723 | | // Detect something like: |
4724 | | // <html><head><title>gdal - Revision 20739: |
4725 | | // /trunk/autotest/gcore/data</title></head> */ The annoying thing |
4726 | | // is that what is after ': ' is a subpart of what is after |
4727 | | // http://server/ |
4728 | | char *pszSubDir = strstr(pszLine, ": "); |
4729 | | if (pszSubDir == nullptr) |
4730 | | // or <title>ortho.linz.govt.nz - /tifs/2005_06/</title> |
4731 | | pszSubDir = strstr(pszLine, "- "); |
4732 | | if (pszSubDir) |
4733 | | { |
4734 | | pszSubDir += 2; |
4735 | | char *pszTmp = strstr(pszSubDir, "</title>"); |
4736 | | if (pszTmp) |
4737 | | { |
4738 | | if (pszTmp[-1] == '/') |
4739 | | pszTmp[-1] = 0; |
4740 | | else |
4741 | | *pszTmp = 0; |
4742 | | if (strstr(pszDir, pszSubDir)) |
4743 | | { |
4744 | | bIsHTMLDirList = true; |
4745 | | *pbGotFileList = true; |
4746 | | } |
4747 | | } |
4748 | | } |
4749 | | } |
4750 | | else if (bIsHTMLDirList && |
4751 | | (strstr(pszLine, "<a href=\"") != nullptr || |
4752 | | strstr(pszLine, "<A HREF=\"") != nullptr) && |
4753 | | // Exclude absolute links, like to subversion home. |
4754 | | strstr(pszLine, "<a href=\"http://") == nullptr && |
4755 | | // exclude parent directory. |
4756 | | strstr(pszLine, "Parent Directory") == nullptr) |
4757 | | { |
4758 | | char *beginFilename = strstr(pszLine, "<a href=\""); |
4759 | | if (beginFilename == nullptr) |
4760 | | beginFilename = strstr(pszLine, "<A HREF=\""); |
4761 | | beginFilename += strlen("<a href=\""); |
4762 | | char *endQuote = strchr(beginFilename, '"'); |
4763 | | if (endQuote && !STARTS_WITH(beginFilename, "?C=") && |
4764 | | !STARTS_WITH(beginFilename, "?N=")) |
4765 | | { |
4766 | | struct tm brokendowntime; |
4767 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
4768 | | GUIntBig nFileSize = 0; |
4769 | | GIntBig mTime = 0; |
4770 | | |
4771 | | VSICurlParseHTMLDateTimeFileSize(pszLine, brokendowntime, |
4772 | | nFileSize, mTime); |
4773 | | |
4774 | | *endQuote = '\0'; |
4775 | | |
4776 | | // Remove trailing slash, that are returned for directories by |
4777 | | // Apache. |
4778 | | bool bIsDirectory = false; |
4779 | | if (endQuote[-1] == '/') |
4780 | | { |
4781 | | bIsDirectory = true; |
4782 | | endQuote[-1] = 0; |
4783 | | } |
4784 | | |
4785 | | // shttpd links include slashes from the root directory. |
4786 | | // Skip them. |
4787 | | while (strchr(beginFilename, '/')) |
4788 | | beginFilename = strchr(beginFilename, '/') + 1; |
4789 | | |
4790 | | if (strcmp(beginFilename, ".") != 0 && |
4791 | | strcmp(beginFilename, "..") != 0) |
4792 | | { |
4793 | | std::string osCachedFilename = |
4794 | | CPLSPrintf("%s/%s", osURL.c_str(), beginFilename); |
4795 | | |
4796 | | FileProp cachedFileProp; |
4797 | | GetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
4798 | | cachedFileProp.eExists = EXIST_YES; |
4799 | | cachedFileProp.bIsDirectory = bIsDirectory; |
4800 | | cachedFileProp.mTime = static_cast<time_t>(mTime); |
4801 | | cachedFileProp.bHasComputedFileSize = nFileSize > 0; |
4802 | | cachedFileProp.fileSize = nFileSize; |
4803 | | SetCachedFileProp(osCachedFilename.c_str(), cachedFileProp); |
4804 | | |
4805 | | oFileList.AddString(beginFilename); |
4806 | | if (ENABLE_DEBUG_VERBOSE) |
4807 | | { |
4808 | | CPLDebug( |
4809 | | GetDebugKey(), |
4810 | | "File[%d] = %s, is_dir = %d, size = " CPL_FRMT_GUIB |
4811 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
4812 | | nCount, osCachedFilename.c_str(), |
4813 | | bIsDirectory ? 1 : 0, nFileSize, |
4814 | | brokendowntime.tm_year + 1900, |
4815 | | brokendowntime.tm_mon + 1, brokendowntime.tm_mday, |
4816 | | brokendowntime.tm_hour, brokendowntime.tm_min, |
4817 | | brokendowntime.tm_sec); |
4818 | | } |
4819 | | nCount++; |
4820 | | |
4821 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
4822 | | break; |
4823 | | } |
4824 | | } |
4825 | | } |
4826 | | pszLine = c + 1; |
4827 | | } |
4828 | | |
4829 | | return oFileList.StealList(); |
4830 | | } |
4831 | | |
4832 | | /************************************************************************/ |
4833 | | /* GetStreamingFilename() */ |
4834 | | /************************************************************************/ |
4835 | | |
4836 | | std::string VSICurlFilesystemHandler::GetStreamingFilename( |
4837 | | const std::string &osFilename) const |
4838 | | { |
4839 | | if (STARTS_WITH(osFilename.c_str(), GetFSPrefix().c_str())) |
4840 | | return "/vsicurl_streaming/" + osFilename.substr(GetFSPrefix().size()); |
4841 | | return osFilename; |
4842 | | } |
4843 | | |
4844 | | /************************************************************************/ |
4845 | | /* VSICurlGetToken() */ |
4846 | | /************************************************************************/ |
4847 | | |
4848 | | static char *VSICurlGetToken(char *pszCurPtr, char **ppszNextToken) |
4849 | | { |
4850 | | if (pszCurPtr == nullptr) |
4851 | | return nullptr; |
4852 | | |
4853 | | while ((*pszCurPtr) == ' ') |
4854 | | pszCurPtr++; |
4855 | | if (*pszCurPtr == '\0') |
4856 | | return nullptr; |
4857 | | |
4858 | | char *pszToken = pszCurPtr; |
4859 | | while ((*pszCurPtr) != ' ' && (*pszCurPtr) != '\0') |
4860 | | pszCurPtr++; |
4861 | | if (*pszCurPtr == '\0') |
4862 | | { |
4863 | | *ppszNextToken = nullptr; |
4864 | | } |
4865 | | else |
4866 | | { |
4867 | | *pszCurPtr = '\0'; |
4868 | | pszCurPtr++; |
4869 | | while ((*pszCurPtr) == ' ') |
4870 | | pszCurPtr++; |
4871 | | *ppszNextToken = pszCurPtr; |
4872 | | } |
4873 | | |
4874 | | return pszToken; |
4875 | | } |
4876 | | |
4877 | | /************************************************************************/ |
4878 | | /* VSICurlParseFullFTPLine() */ |
4879 | | /************************************************************************/ |
4880 | | |
4881 | | /* Parse lines like the following ones : |
4882 | | -rw-r--r-- 1 10003 100 430 Jul 04 2008 COPYING |
4883 | | lrwxrwxrwx 1 ftp ftp 28 Jun 14 14:13 MPlayer -> |
4884 | | mirrors/mplayerhq.hu/MPlayer -rw-r--r-- 1 ftp ftp 725614592 May 13 |
4885 | | 20:13 Fedora-15-x86_64-Live-KDE.iso drwxr-xr-x 280 1003 1003 6656 Aug 26 |
4886 | | 04:17 gnu |
4887 | | */ |
4888 | | |
4889 | | static bool VSICurlParseFullFTPLine(char *pszLine, char *&pszFilename, |
4890 | | bool &bSizeValid, GUIntBig &nSize, |
4891 | | bool &bIsDirectory, GIntBig &nUnixTime) |
4892 | | { |
4893 | | char *pszNextToken = pszLine; |
4894 | | char *pszPermissions = VSICurlGetToken(pszNextToken, &pszNextToken); |
4895 | | if (pszPermissions == nullptr || strlen(pszPermissions) != 10) |
4896 | | return false; |
4897 | | bIsDirectory = pszPermissions[0] == 'd'; |
4898 | | |
4899 | | for (int i = 0; i < 3; i++) |
4900 | | { |
4901 | | if (VSICurlGetToken(pszNextToken, &pszNextToken) == nullptr) |
4902 | | return false; |
4903 | | } |
4904 | | |
4905 | | char *pszSize = VSICurlGetToken(pszNextToken, &pszNextToken); |
4906 | | if (pszSize == nullptr) |
4907 | | return false; |
4908 | | |
4909 | | if (pszPermissions[0] == '-') |
4910 | | { |
4911 | | // Regular file. |
4912 | | bSizeValid = true; |
4913 | | nSize = CPLScanUIntBig(pszSize, static_cast<int>(strlen(pszSize))); |
4914 | | } |
4915 | | |
4916 | | struct tm brokendowntime; |
4917 | | memset(&brokendowntime, 0, sizeof(brokendowntime)); |
4918 | | bool bBrokenDownTimeValid = true; |
4919 | | |
4920 | | char *pszMonth = VSICurlGetToken(pszNextToken, &pszNextToken); |
4921 | | if (pszMonth == nullptr || strlen(pszMonth) != 3) |
4922 | | return false; |
4923 | | |
4924 | | int i = 0; // Used after for. |
4925 | | for (; i < 12; i++) |
4926 | | { |
4927 | | if (EQUALN(pszMonth, apszMonths[i], 3)) |
4928 | | break; |
4929 | | } |
4930 | | if (i < 12) |
4931 | | brokendowntime.tm_mon = i; |
4932 | | else |
4933 | | bBrokenDownTimeValid = false; |
4934 | | |
4935 | | char *pszDay = VSICurlGetToken(pszNextToken, &pszNextToken); |
4936 | | if (pszDay == nullptr || (strlen(pszDay) != 1 && strlen(pszDay) != 2)) |
4937 | | return false; |
4938 | | int nDay = atoi(pszDay); |
4939 | | if (nDay >= 1 && nDay <= 31) |
4940 | | brokendowntime.tm_mday = nDay; |
4941 | | else |
4942 | | bBrokenDownTimeValid = false; |
4943 | | |
4944 | | char *pszHourOrYear = VSICurlGetToken(pszNextToken, &pszNextToken); |
4945 | | if (pszHourOrYear == nullptr || |
4946 | | (strlen(pszHourOrYear) != 4 && strlen(pszHourOrYear) != 5)) |
4947 | | return false; |
4948 | | if (strlen(pszHourOrYear) == 4) |
4949 | | { |
4950 | | brokendowntime.tm_year = atoi(pszHourOrYear) - 1900; |
4951 | | } |
4952 | | else |
4953 | | { |
4954 | | time_t sTime; |
4955 | | time(&sTime); |
4956 | | struct tm currentBrokendowntime; |
4957 | | CPLUnixTimeToYMDHMS(static_cast<GIntBig>(sTime), |
4958 | | ¤tBrokendowntime); |
4959 | | brokendowntime.tm_year = currentBrokendowntime.tm_year; |
4960 | | brokendowntime.tm_hour = atoi(pszHourOrYear); |
4961 | | brokendowntime.tm_min = atoi(pszHourOrYear + 3); |
4962 | | } |
4963 | | |
4964 | | if (bBrokenDownTimeValid) |
4965 | | nUnixTime = CPLYMDHMSToUnixTime(&brokendowntime); |
4966 | | else |
4967 | | nUnixTime = 0; |
4968 | | |
4969 | | if (pszNextToken == nullptr) |
4970 | | return false; |
4971 | | |
4972 | | pszFilename = pszNextToken; |
4973 | | |
4974 | | char *pszCurPtr = pszFilename; |
4975 | | while (*pszCurPtr != '\0') |
4976 | | { |
4977 | | // In case of a link, stop before the pointed part of the link. |
4978 | | if (pszPermissions[0] == 'l' && STARTS_WITH(pszCurPtr, " -> ")) |
4979 | | { |
4980 | | break; |
4981 | | } |
4982 | | pszCurPtr++; |
4983 | | } |
4984 | | *pszCurPtr = '\0'; |
4985 | | |
4986 | | return true; |
4987 | | } |
4988 | | |
4989 | | /************************************************************************/ |
4990 | | /* GetURLFromFilename() */ |
4991 | | /************************************************************************/ |
4992 | | |
4993 | | std::string VSICurlFilesystemHandlerBase::GetURLFromFilename( |
4994 | | const std::string &osFilename) const |
4995 | | { |
4996 | | return VSICurlGetURLFromFilename(osFilename.c_str(), nullptr, nullptr, |
4997 | | nullptr, nullptr, nullptr, nullptr, |
4998 | | nullptr, nullptr); |
4999 | | } |
5000 | | |
5001 | | /************************************************************************/ |
5002 | | /* RegisterEmptyDir() */ |
5003 | | /************************************************************************/ |
5004 | | |
5005 | | void VSICurlFilesystemHandlerBase::RegisterEmptyDir( |
5006 | | const std::string &osDirname) |
5007 | | { |
5008 | | CachedDirList cachedDirList; |
5009 | | cachedDirList.bGotFileList = true; |
5010 | | cachedDirList.oFileList.AddString("."); |
5011 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5012 | | } |
5013 | | |
5014 | | /************************************************************************/ |
5015 | | /* GetFileList() */ |
5016 | | /************************************************************************/ |
5017 | | |
5018 | | char **VSICurlFilesystemHandlerBase::GetFileList(const char *pszDirname, |
5019 | | int nMaxFiles, |
5020 | | bool *pbGotFileList) |
5021 | | { |
5022 | | if (ENABLE_DEBUG) |
5023 | | CPLDebug(GetDebugKey(), "GetFileList(%s)", pszDirname); |
5024 | | |
5025 | | *pbGotFileList = false; |
5026 | | |
5027 | | bool bListDir = true; |
5028 | | bool bEmptyDir = false; |
5029 | | std::string osURL(VSICurlGetURLFromFilename(pszDirname, nullptr, nullptr, |
5030 | | nullptr, &bListDir, &bEmptyDir, |
5031 | | nullptr, nullptr, nullptr)); |
5032 | | if (bEmptyDir) |
5033 | | { |
5034 | | *pbGotFileList = true; |
5035 | | return CSLAddString(nullptr, "."); |
5036 | | } |
5037 | | if (!bListDir) |
5038 | | return nullptr; |
5039 | | |
5040 | | // Deal with publicly visible Azure directories. |
5041 | | if (STARTS_WITH(osURL.c_str(), "https://")) |
5042 | | { |
5043 | | const char *pszBlobCore = |
5044 | | strstr(osURL.c_str(), ".blob.core.windows.net/"); |
5045 | | if (pszBlobCore) |
5046 | | { |
5047 | | FileProp cachedFileProp; |
5048 | | GetCachedFileProp(osURL.c_str(), cachedFileProp); |
5049 | | if (cachedFileProp.bIsAzureFolder) |
5050 | | { |
5051 | | const char *pszURLWithoutHTTPS = |
5052 | | osURL.c_str() + strlen("https://"); |
5053 | | const std::string osStorageAccount( |
5054 | | pszURLWithoutHTTPS, pszBlobCore - pszURLWithoutHTTPS); |
5055 | | CPLConfigOptionSetter oSetter1("AZURE_NO_SIGN_REQUEST", "YES", |
5056 | | false); |
5057 | | CPLConfigOptionSetter oSetter2("AZURE_STORAGE_ACCOUNT", |
5058 | | osStorageAccount.c_str(), false); |
5059 | | const std::string osVSIAZ(std::string("/vsiaz/").append( |
5060 | | pszBlobCore + strlen(".blob.core.windows.net/"))); |
5061 | | char **papszFileList = VSIReadDirEx(osVSIAZ.c_str(), nMaxFiles); |
5062 | | if (papszFileList) |
5063 | | { |
5064 | | *pbGotFileList = true; |
5065 | | return papszFileList; |
5066 | | } |
5067 | | } |
5068 | | } |
5069 | | } |
5070 | | |
5071 | | // HACK (optimization in fact) for MBTiles driver. |
5072 | | if (strstr(pszDirname, ".tiles.mapbox.com") != nullptr) |
5073 | | return nullptr; |
5074 | | |
5075 | | if (STARTS_WITH(osURL.c_str(), "ftp://")) |
5076 | | { |
5077 | | WriteFuncStruct sWriteFuncData; |
5078 | | sWriteFuncData.pBuffer = nullptr; |
5079 | | |
5080 | | std::string osDirname(osURL); |
5081 | | osDirname += '/'; |
5082 | | |
5083 | | char **papszFileList = nullptr; |
5084 | | |
5085 | | CURLM *hCurlMultiHandle = GetCurlMultiHandleFor(osDirname); |
5086 | | CURL *hCurlHandle = curl_easy_init(); |
5087 | | |
5088 | | for (int iTry = 0; iTry < 2; iTry++) |
5089 | | { |
5090 | | struct curl_slist *headers = |
5091 | | VSICurlSetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5092 | | |
5093 | | // On the first pass, we want to try fetching all the possible |
5094 | | // information (filename, file/directory, size). If that does not |
5095 | | // work, then try again with CURLOPT_DIRLISTONLY set. |
5096 | | if (iTry == 1) |
5097 | | { |
5098 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 1); |
5099 | | } |
5100 | | |
5101 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, |
5102 | | nullptr); |
5103 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5104 | | &sWriteFuncData); |
5105 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5106 | | VSICurlHandleWriteFunc); |
5107 | | |
5108 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5109 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5110 | | szCurlErrBuf); |
5111 | | |
5112 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, |
5113 | | headers); |
5114 | | |
5115 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
5116 | | |
5117 | | curl_slist_free_all(headers); |
5118 | | |
5119 | | if (sWriteFuncData.pBuffer == nullptr) |
5120 | | { |
5121 | | curl_easy_cleanup(hCurlHandle); |
5122 | | return nullptr; |
5123 | | } |
5124 | | |
5125 | | char *pszLine = sWriteFuncData.pBuffer; |
5126 | | char *c = nullptr; |
5127 | | int nCount = 0; |
5128 | | |
5129 | | if (STARTS_WITH_CI(pszLine, "<!DOCTYPE HTML") || |
5130 | | STARTS_WITH_CI(pszLine, "<HTML>")) |
5131 | | { |
5132 | | papszFileList = |
5133 | | ParseHTMLFileList(pszDirname, nMaxFiles, |
5134 | | sWriteFuncData.pBuffer, pbGotFileList); |
5135 | | break; |
5136 | | } |
5137 | | else if (iTry == 0) |
5138 | | { |
5139 | | CPLStringList oFileList; |
5140 | | *pbGotFileList = true; |
5141 | | |
5142 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5143 | | { |
5144 | | *c = 0; |
5145 | | if (c - pszLine > 0 && c[-1] == '\r') |
5146 | | c[-1] = 0; |
5147 | | |
5148 | | char *pszFilename = nullptr; |
5149 | | bool bSizeValid = false; |
5150 | | GUIntBig nFileSize = 0; |
5151 | | bool bIsDirectory = false; |
5152 | | GIntBig mUnixTime = 0; |
5153 | | if (!VSICurlParseFullFTPLine(pszLine, pszFilename, |
5154 | | bSizeValid, nFileSize, |
5155 | | bIsDirectory, mUnixTime)) |
5156 | | break; |
5157 | | |
5158 | | if (strcmp(pszFilename, ".") != 0 && |
5159 | | strcmp(pszFilename, "..") != 0) |
5160 | | { |
5161 | | std::string osCachedFilename = |
5162 | | CPLSPrintf("%s/%s", osURL.c_str(), pszFilename); |
5163 | | |
5164 | | FileProp cachedFileProp; |
5165 | | GetCachedFileProp(osCachedFilename.c_str(), |
5166 | | cachedFileProp); |
5167 | | cachedFileProp.eExists = EXIST_YES; |
5168 | | cachedFileProp.bIsDirectory = bIsDirectory; |
5169 | | cachedFileProp.mTime = static_cast<time_t>(mUnixTime); |
5170 | | cachedFileProp.bHasComputedFileSize = bSizeValid; |
5171 | | cachedFileProp.fileSize = nFileSize; |
5172 | | SetCachedFileProp(osCachedFilename.c_str(), |
5173 | | cachedFileProp); |
5174 | | |
5175 | | oFileList.AddString(pszFilename); |
5176 | | if (ENABLE_DEBUG_VERBOSE) |
5177 | | { |
5178 | | struct tm brokendowntime; |
5179 | | CPLUnixTimeToYMDHMS(mUnixTime, &brokendowntime); |
5180 | | CPLDebug( |
5181 | | GetDebugKey(), |
5182 | | "File[%d] = %s, is_dir = %d, size " |
5183 | | "= " CPL_FRMT_GUIB |
5184 | | ", time = %04d/%02d/%02d %02d:%02d:%02d", |
5185 | | nCount, pszFilename, bIsDirectory ? 1 : 0, |
5186 | | nFileSize, brokendowntime.tm_year + 1900, |
5187 | | brokendowntime.tm_mon + 1, |
5188 | | brokendowntime.tm_mday, brokendowntime.tm_hour, |
5189 | | brokendowntime.tm_min, brokendowntime.tm_sec); |
5190 | | } |
5191 | | |
5192 | | nCount++; |
5193 | | |
5194 | | if (nMaxFiles > 0 && oFileList.Count() > nMaxFiles) |
5195 | | break; |
5196 | | } |
5197 | | |
5198 | | pszLine = c + 1; |
5199 | | } |
5200 | | |
5201 | | if (c == nullptr) |
5202 | | { |
5203 | | papszFileList = oFileList.StealList(); |
5204 | | break; |
5205 | | } |
5206 | | } |
5207 | | else |
5208 | | { |
5209 | | CPLStringList oFileList; |
5210 | | *pbGotFileList = true; |
5211 | | |
5212 | | while ((c = strchr(pszLine, '\n')) != nullptr) |
5213 | | { |
5214 | | *c = 0; |
5215 | | if (c - pszLine > 0 && c[-1] == '\r') |
5216 | | c[-1] = 0; |
5217 | | |
5218 | | if (strcmp(pszLine, ".") != 0 && strcmp(pszLine, "..") != 0) |
5219 | | { |
5220 | | oFileList.AddString(pszLine); |
5221 | | if (ENABLE_DEBUG_VERBOSE) |
5222 | | { |
5223 | | CPLDebug(GetDebugKey(), "File[%d] = %s", nCount, |
5224 | | pszLine); |
5225 | | } |
5226 | | nCount++; |
5227 | | } |
5228 | | |
5229 | | pszLine = c + 1; |
5230 | | } |
5231 | | |
5232 | | papszFileList = oFileList.StealList(); |
5233 | | } |
5234 | | |
5235 | | CPLFree(sWriteFuncData.pBuffer); |
5236 | | sWriteFuncData.pBuffer = nullptr; |
5237 | | } |
5238 | | |
5239 | | CPLFree(sWriteFuncData.pBuffer); |
5240 | | curl_easy_cleanup(hCurlHandle); |
5241 | | |
5242 | | return papszFileList; |
5243 | | } |
5244 | | |
5245 | | // Try to recognize HTML pages that list the content of a directory. |
5246 | | // Currently this supports what Apache and shttpd can return. |
5247 | | else if (STARTS_WITH(osURL.c_str(), "http://") || |
5248 | | STARTS_WITH(osURL.c_str(), "https://")) |
5249 | | { |
5250 | | std::string osDirname(std::move(osURL)); |
5251 | | osDirname += '/'; |
5252 | | |
5253 | | CURLM *hCurlMultiHandle = GetCurlMultiHandleFor(osDirname); |
5254 | | CURL *hCurlHandle = curl_easy_init(); |
5255 | | |
5256 | | struct curl_slist *headers = |
5257 | | VSICurlSetOptions(hCurlHandle, osDirname.c_str(), nullptr); |
5258 | | |
5259 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr); |
5260 | | |
5261 | | WriteFuncStruct sWriteFuncData; |
5262 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
5263 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, |
5264 | | &sWriteFuncData); |
5265 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5266 | | VSICurlHandleWriteFunc); |
5267 | | |
5268 | | char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; |
5269 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, |
5270 | | szCurlErrBuf); |
5271 | | |
5272 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
5273 | | |
5274 | | VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); |
5275 | | |
5276 | | curl_slist_free_all(headers); |
5277 | | |
5278 | | NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); |
5279 | | |
5280 | | if (sWriteFuncData.pBuffer == nullptr) |
5281 | | { |
5282 | | curl_easy_cleanup(hCurlHandle); |
5283 | | return nullptr; |
5284 | | } |
5285 | | |
5286 | | char **papszFileList = nullptr; |
5287 | | if (STARTS_WITH_CI(sWriteFuncData.pBuffer, "<?xml") && |
5288 | | strstr(sWriteFuncData.pBuffer, "<ListBucketResult") != nullptr) |
5289 | | { |
5290 | | CPLStringList osFileList; |
5291 | | std::string osBaseURL(pszDirname); |
5292 | | osBaseURL += "/"; |
5293 | | bool bIsTruncated = true; |
5294 | | bool ret = AnalyseS3FileList( |
5295 | | osBaseURL, sWriteFuncData.pBuffer, osFileList, nMaxFiles, |
5296 | | GetS3IgnoredStorageClasses(), bIsTruncated); |
5297 | | // If the list is truncated, then don't report it. |
5298 | | if (ret && !bIsTruncated) |
5299 | | { |
5300 | | if (osFileList.empty()) |
5301 | | { |
5302 | | // To avoid an error to be reported |
5303 | | osFileList.AddString("."); |
5304 | | } |
5305 | | papszFileList = osFileList.StealList(); |
5306 | | *pbGotFileList = true; |
5307 | | } |
5308 | | } |
5309 | | else |
5310 | | { |
5311 | | papszFileList = ParseHTMLFileList( |
5312 | | pszDirname, nMaxFiles, sWriteFuncData.pBuffer, pbGotFileList); |
5313 | | } |
5314 | | |
5315 | | CPLFree(sWriteFuncData.pBuffer); |
5316 | | curl_easy_cleanup(hCurlHandle); |
5317 | | return papszFileList; |
5318 | | } |
5319 | | |
5320 | | return nullptr; |
5321 | | } |
5322 | | |
5323 | | /************************************************************************/ |
5324 | | /* GetS3IgnoredStorageClasses() */ |
5325 | | /************************************************************************/ |
5326 | | |
5327 | | std::set<std::string> VSICurlFilesystemHandlerBase::GetS3IgnoredStorageClasses() |
5328 | | { |
5329 | | std::set<std::string> oSetIgnoredStorageClasses; |
5330 | | const char *pszIgnoredStorageClasses = |
5331 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_STORAGE_CLASSES", nullptr); |
5332 | | const char *pszIgnoreGlacierStorage = |
5333 | | CPLGetConfigOption("CPL_VSIL_CURL_IGNORE_GLACIER_STORAGE", nullptr); |
5334 | | CPLStringList aosIgnoredStorageClasses( |
5335 | | CSLTokenizeString2(pszIgnoredStorageClasses ? pszIgnoredStorageClasses |
5336 | | : "GLACIER,DEEP_ARCHIVE", |
5337 | | ",", 0)); |
5338 | | for (int i = 0; i < aosIgnoredStorageClasses.size(); ++i) |
5339 | | oSetIgnoredStorageClasses.insert(aosIgnoredStorageClasses[i]); |
5340 | | if (pszIgnoredStorageClasses == nullptr && |
5341 | | pszIgnoreGlacierStorage != nullptr && |
5342 | | !CPLTestBool(pszIgnoreGlacierStorage)) |
5343 | | { |
5344 | | oSetIgnoredStorageClasses.clear(); |
5345 | | } |
5346 | | return oSetIgnoredStorageClasses; |
5347 | | } |
5348 | | |
5349 | | /************************************************************************/ |
5350 | | /* Stat() */ |
5351 | | /************************************************************************/ |
5352 | | |
5353 | | int VSICurlFilesystemHandlerBase::Stat(const char *pszFilename, |
5354 | | VSIStatBufL *pStatBuf, int nFlags) |
5355 | | { |
5356 | | if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str()) && |
5357 | | !STARTS_WITH_CI(pszFilename, "/vsicurl?")) |
5358 | | return -1; |
5359 | | |
5360 | | memset(pStatBuf, 0, sizeof(VSIStatBufL)); |
5361 | | |
5362 | | if ((nFlags & VSI_STAT_CACHE_ONLY) != 0) |
5363 | | { |
5364 | | cpl::FileProp oFileProp; |
5365 | | if (!GetCachedFileProp(GetURLFromFilename(pszFilename).c_str(), |
5366 | | oFileProp) || |
5367 | | oFileProp.eExists != EXIST_YES) |
5368 | | { |
5369 | | return -1; |
5370 | | } |
5371 | | pStatBuf->st_mode = static_cast<unsigned short>(oFileProp.nMode); |
5372 | | pStatBuf->st_mtime = oFileProp.mTime; |
5373 | | pStatBuf->st_size = oFileProp.fileSize; |
5374 | | return 0; |
5375 | | } |
5376 | | |
5377 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5378 | | NetworkStatisticsAction oContextAction("Stat"); |
5379 | | |
5380 | | const std::string osFilename(pszFilename); |
5381 | | |
5382 | | if (!IsAllowedFilename(pszFilename)) |
5383 | | return -1; |
5384 | | |
5385 | | bool bListDir = true; |
5386 | | bool bEmptyDir = false; |
5387 | | std::string osURL(VSICurlGetURLFromFilename(pszFilename, nullptr, nullptr, |
5388 | | nullptr, &bListDir, &bEmptyDir, |
5389 | | nullptr, nullptr, nullptr)); |
5390 | | |
5391 | | const char *pszOptionVal = VSIGetPathSpecificOption( |
5392 | | pszFilename, "GDAL_DISABLE_READDIR_ON_OPEN", "NO"); |
5393 | | const bool bSkipReadDir = |
5394 | | !bListDir || bEmptyDir || EQUAL(pszOptionVal, "EMPTY_DIR") || |
5395 | | CPLTestBool(pszOptionVal) || !AllowCachedDataFor(pszFilename); |
5396 | | |
5397 | | // Does it look like a FTP directory? |
5398 | | if (STARTS_WITH(osURL.c_str(), "ftp://") && osFilename.back() == '/' && |
5399 | | !bSkipReadDir) |
5400 | | { |
5401 | | char **papszFileList = ReadDirEx(osFilename.c_str(), 0); |
5402 | | if (papszFileList) |
5403 | | { |
5404 | | pStatBuf->st_mode = S_IFDIR; |
5405 | | pStatBuf->st_size = 0; |
5406 | | |
5407 | | CSLDestroy(papszFileList); |
5408 | | |
5409 | | return 0; |
5410 | | } |
5411 | | return -1; |
5412 | | } |
5413 | | else if (strchr(CPLGetFilename(osFilename.c_str()), '.') != nullptr && |
5414 | | !STARTS_WITH_CI(CPLGetExtensionSafe(osFilename.c_str()).c_str(), |
5415 | | "zip") && |
5416 | | strstr(osFilename.c_str(), ".zip.") != nullptr && |
5417 | | strstr(osFilename.c_str(), ".ZIP.") != nullptr && !bSkipReadDir) |
5418 | | { |
5419 | | bool bGotFileList = false; |
5420 | | char **papszFileList = ReadDirInternal( |
5421 | | CPLGetDirnameSafe(osFilename.c_str()).c_str(), 0, &bGotFileList); |
5422 | | const bool bFound = |
5423 | | VSICurlIsFileInList(papszFileList, |
5424 | | CPLGetFilename(osFilename.c_str())) != -1; |
5425 | | CSLDestroy(papszFileList); |
5426 | | if (bGotFileList && !bFound) |
5427 | | { |
5428 | | return -1; |
5429 | | } |
5430 | | } |
5431 | | |
5432 | | VSICurlHandle *poHandle = CreateFileHandle(osFilename.c_str()); |
5433 | | if (poHandle == nullptr) |
5434 | | return -1; |
5435 | | |
5436 | | if (poHandle->IsKnownFileSize() || |
5437 | | ((nFlags & VSI_STAT_SIZE_FLAG) && !poHandle->IsDirectory() && |
5438 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES")))) |
5439 | | { |
5440 | | pStatBuf->st_size = poHandle->GetFileSize(true); |
5441 | | } |
5442 | | |
5443 | | const int nRet = |
5444 | | poHandle->Exists((nFlags & VSI_STAT_SET_ERROR_FLAG) > 0) ? 0 : -1; |
5445 | | pStatBuf->st_mtime = poHandle->GetMTime(); |
5446 | | pStatBuf->st_mode = static_cast<unsigned short>(poHandle->GetMode()); |
5447 | | if (pStatBuf->st_mode == 0) |
5448 | | pStatBuf->st_mode = poHandle->IsDirectory() ? S_IFDIR : S_IFREG; |
5449 | | delete poHandle; |
5450 | | return nRet; |
5451 | | } |
5452 | | |
5453 | | /************************************************************************/ |
5454 | | /* ReadDirInternal() */ |
5455 | | /************************************************************************/ |
5456 | | |
5457 | | char **VSICurlFilesystemHandlerBase::ReadDirInternal(const char *pszDirname, |
5458 | | int nMaxFiles, |
5459 | | bool *pbGotFileList) |
5460 | | { |
5461 | | std::string osDirname(pszDirname); |
5462 | | |
5463 | | // Replace a/b/../c by a/c |
5464 | | const auto posSlashDotDot = osDirname.find("/.."); |
5465 | | if (posSlashDotDot != std::string::npos && posSlashDotDot >= 1) |
5466 | | { |
5467 | | const auto posPrecedingSlash = |
5468 | | osDirname.find_last_of('/', posSlashDotDot - 1); |
5469 | | if (posPrecedingSlash != std::string::npos && posPrecedingSlash >= 1) |
5470 | | { |
5471 | | osDirname.erase(osDirname.begin() + posPrecedingSlash, |
5472 | | osDirname.begin() + posSlashDotDot + strlen("/..")); |
5473 | | } |
5474 | | } |
5475 | | |
5476 | | std::string osDirnameOri(osDirname); |
5477 | | if (osDirname + "/" == GetFSPrefix()) |
5478 | | { |
5479 | | osDirname += "/"; |
5480 | | } |
5481 | | else if (osDirname != GetFSPrefix()) |
5482 | | { |
5483 | | while (!osDirname.empty() && osDirname.back() == '/') |
5484 | | osDirname.erase(osDirname.size() - 1); |
5485 | | } |
5486 | | |
5487 | | if (osDirname.size() < GetFSPrefix().size()) |
5488 | | { |
5489 | | if (pbGotFileList) |
5490 | | *pbGotFileList = true; |
5491 | | return nullptr; |
5492 | | } |
5493 | | |
5494 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5495 | | NetworkStatisticsAction oContextAction("ReadDir"); |
5496 | | |
5497 | | CPLMutexHolder oHolder(&hMutex); |
5498 | | |
5499 | | // If we know the file exists and is not a directory, |
5500 | | // then don't try to list its content. |
5501 | | FileProp cachedFileProp; |
5502 | | if (GetCachedFileProp(GetURLFromFilename(osDirname.c_str()).c_str(), |
5503 | | cachedFileProp) && |
5504 | | cachedFileProp.eExists == EXIST_YES && !cachedFileProp.bIsDirectory) |
5505 | | { |
5506 | | if (osDirnameOri != osDirname) |
5507 | | { |
5508 | | if (GetCachedFileProp((GetURLFromFilename(osDirname) + "/").c_str(), |
5509 | | cachedFileProp) && |
5510 | | cachedFileProp.eExists == EXIST_YES && |
5511 | | !cachedFileProp.bIsDirectory) |
5512 | | { |
5513 | | if (pbGotFileList) |
5514 | | *pbGotFileList = true; |
5515 | | return nullptr; |
5516 | | } |
5517 | | } |
5518 | | else |
5519 | | { |
5520 | | if (pbGotFileList) |
5521 | | *pbGotFileList = true; |
5522 | | return nullptr; |
5523 | | } |
5524 | | } |
5525 | | |
5526 | | CachedDirList cachedDirList; |
5527 | | if (!GetCachedDirList(osDirname.c_str(), cachedDirList)) |
5528 | | { |
5529 | | cachedDirList.oFileList.Assign(GetFileList(osDirname.c_str(), nMaxFiles, |
5530 | | &cachedDirList.bGotFileList), |
5531 | | true); |
5532 | | if (cachedDirList.bGotFileList && cachedDirList.oFileList.empty()) |
5533 | | { |
5534 | | // To avoid an error to be reported |
5535 | | cachedDirList.oFileList.AddString("."); |
5536 | | } |
5537 | | if (nMaxFiles <= 0 || cachedDirList.oFileList.size() < nMaxFiles) |
5538 | | { |
5539 | | // Only cache content if we didn't hit the limitation |
5540 | | SetCachedDirList(osDirname.c_str(), cachedDirList); |
5541 | | } |
5542 | | } |
5543 | | |
5544 | | if (pbGotFileList) |
5545 | | *pbGotFileList = cachedDirList.bGotFileList; |
5546 | | |
5547 | | return CSLDuplicate(cachedDirList.oFileList.List()); |
5548 | | } |
5549 | | |
5550 | | /************************************************************************/ |
5551 | | /* InvalidateDirContent() */ |
5552 | | /************************************************************************/ |
5553 | | |
5554 | | void VSICurlFilesystemHandlerBase::InvalidateDirContent( |
5555 | | const std::string &osDirname) |
5556 | | { |
5557 | | CPLMutexHolder oHolder(&hMutex); |
5558 | | |
5559 | | CachedDirList oCachedDirList; |
5560 | | if (oCacheDirList.tryGet(osDirname, oCachedDirList)) |
5561 | | { |
5562 | | nCachedFilesInDirList -= oCachedDirList.oFileList.size(); |
5563 | | oCacheDirList.remove(osDirname); |
5564 | | } |
5565 | | } |
5566 | | |
5567 | | /************************************************************************/ |
5568 | | /* ReadDirEx() */ |
5569 | | /************************************************************************/ |
5570 | | |
5571 | | char **VSICurlFilesystemHandlerBase::ReadDirEx(const char *pszDirname, |
5572 | | int nMaxFiles) |
5573 | | { |
5574 | | return ReadDirInternal(pszDirname, nMaxFiles, nullptr); |
5575 | | } |
5576 | | |
5577 | | /************************************************************************/ |
5578 | | /* SiblingFiles() */ |
5579 | | /************************************************************************/ |
5580 | | |
5581 | | char **VSICurlFilesystemHandlerBase::SiblingFiles(const char *pszFilename) |
5582 | | { |
5583 | | /* Small optimization to avoid unnecessary stat'ing from PAux or ENVI */ |
5584 | | /* drivers. The MBTiles driver needs no companion file. */ |
5585 | | if (EQUAL(CPLGetExtensionSafe(pszFilename).c_str(), "mbtiles")) |
5586 | | { |
5587 | | return static_cast<char **>(CPLCalloc(1, sizeof(char *))); |
5588 | | } |
5589 | | return nullptr; |
5590 | | } |
5591 | | |
5592 | | /************************************************************************/ |
5593 | | /* GetFileMetadata() */ |
5594 | | /************************************************************************/ |
5595 | | |
5596 | | char **VSICurlFilesystemHandlerBase::GetFileMetadata(const char *pszFilename, |
5597 | | const char *pszDomain, |
5598 | | CSLConstList) |
5599 | | { |
5600 | | if (pszDomain == nullptr || !EQUAL(pszDomain, "HEADERS")) |
5601 | | return nullptr; |
5602 | | std::unique_ptr<VSICurlHandle> poHandle(CreateFileHandle(pszFilename)); |
5603 | | if (poHandle == nullptr) |
5604 | | return nullptr; |
5605 | | |
5606 | | NetworkStatisticsFileSystem oContextFS(GetFSPrefix().c_str()); |
5607 | | NetworkStatisticsAction oContextAction("GetFileMetadata"); |
5608 | | |
5609 | | poHandle->GetFileSizeOrHeaders(true, true); |
5610 | | return CSLDuplicate(poHandle->GetHeaders().List()); |
5611 | | } |
5612 | | |
5613 | | /************************************************************************/ |
5614 | | /* VSIAppendWriteHandle() */ |
5615 | | /************************************************************************/ |
5616 | | |
5617 | | VSIAppendWriteHandle::VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS, |
5618 | | const char *pszFSPrefix, |
5619 | | const char *pszFilename, |
5620 | | int nChunkSize) |
5621 | | : m_poFS(poFS), m_osFSPrefix(pszFSPrefix), m_osFilename(pszFilename), |
5622 | | m_oRetryParameters(CPLStringList(CPLHTTPGetOptionsFromEnv(pszFilename))), |
5623 | | m_nBufferSize(nChunkSize) |
5624 | | { |
5625 | | m_pabyBuffer = static_cast<GByte *>(VSIMalloc(m_nBufferSize)); |
5626 | | if (m_pabyBuffer == nullptr) |
5627 | | { |
5628 | | CPLError(CE_Failure, CPLE_AppDefined, |
5629 | | "Cannot allocate working buffer for %s writing", |
5630 | | m_osFSPrefix.c_str()); |
5631 | | } |
5632 | | } |
5633 | | |
5634 | | /************************************************************************/ |
5635 | | /* ~VSIAppendWriteHandle() */ |
5636 | | /************************************************************************/ |
5637 | | |
5638 | | VSIAppendWriteHandle::~VSIAppendWriteHandle() |
5639 | | { |
5640 | | /* WARNING: implementation should call Close() themselves */ |
5641 | | /* cannot be done safely from here, since Send() can be called. */ |
5642 | | CPLFree(m_pabyBuffer); |
5643 | | } |
5644 | | |
5645 | | /************************************************************************/ |
5646 | | /* Seek() */ |
5647 | | /************************************************************************/ |
5648 | | |
5649 | | int VSIAppendWriteHandle::Seek(vsi_l_offset nOffset, int nWhence) |
5650 | | { |
5651 | | if (!((nWhence == SEEK_SET && nOffset == m_nCurOffset) || |
5652 | | (nWhence == SEEK_CUR && nOffset == 0) || |
5653 | | (nWhence == SEEK_END && nOffset == 0))) |
5654 | | { |
5655 | | CPLError(CE_Failure, CPLE_NotSupported, |
5656 | | "Seek not supported on writable %s files", |
5657 | | m_osFSPrefix.c_str()); |
5658 | | m_bError = true; |
5659 | | return -1; |
5660 | | } |
5661 | | return 0; |
5662 | | } |
5663 | | |
5664 | | /************************************************************************/ |
5665 | | /* Tell() */ |
5666 | | /************************************************************************/ |
5667 | | |
5668 | | vsi_l_offset VSIAppendWriteHandle::Tell() |
5669 | | { |
5670 | | return m_nCurOffset; |
5671 | | } |
5672 | | |
5673 | | /************************************************************************/ |
5674 | | /* Read() */ |
5675 | | /************************************************************************/ |
5676 | | |
5677 | | size_t VSIAppendWriteHandle::Read(void * /* pBuffer */, size_t /* nSize */, |
5678 | | size_t /* nMemb */) |
5679 | | { |
5680 | | CPLError(CE_Failure, CPLE_NotSupported, |
5681 | | "Read not supported on writable %s files", m_osFSPrefix.c_str()); |
5682 | | m_bError = true; |
5683 | | return 0; |
5684 | | } |
5685 | | |
5686 | | /************************************************************************/ |
5687 | | /* ReadCallBackBuffer() */ |
5688 | | /************************************************************************/ |
5689 | | |
5690 | | size_t VSIAppendWriteHandle::ReadCallBackBuffer(char *buffer, size_t size, |
5691 | | size_t nitems, void *instream) |
5692 | | { |
5693 | | VSIAppendWriteHandle *poThis = |
5694 | | static_cast<VSIAppendWriteHandle *>(instream); |
5695 | | const int nSizeMax = static_cast<int>(size * nitems); |
5696 | | const int nSizeToWrite = std::min( |
5697 | | nSizeMax, poThis->m_nBufferOff - poThis->m_nBufferOffReadCallback); |
5698 | | memcpy(buffer, poThis->m_pabyBuffer + poThis->m_nBufferOffReadCallback, |
5699 | | nSizeToWrite); |
5700 | | poThis->m_nBufferOffReadCallback += nSizeToWrite; |
5701 | | return nSizeToWrite; |
5702 | | } |
5703 | | |
5704 | | /************************************************************************/ |
5705 | | /* Write() */ |
5706 | | /************************************************************************/ |
5707 | | |
5708 | | size_t VSIAppendWriteHandle::Write(const void *pBuffer, size_t nSize, |
5709 | | size_t nMemb) |
5710 | | { |
5711 | | if (m_bError) |
5712 | | return 0; |
5713 | | |
5714 | | size_t nBytesToWrite = nSize * nMemb; |
5715 | | if (nBytesToWrite == 0) |
5716 | | return 0; |
5717 | | |
5718 | | const GByte *pabySrcBuffer = reinterpret_cast<const GByte *>(pBuffer); |
5719 | | while (nBytesToWrite > 0) |
5720 | | { |
5721 | | if (m_nBufferOff == m_nBufferSize) |
5722 | | { |
5723 | | if (!Send(false)) |
5724 | | { |
5725 | | m_bError = true; |
5726 | | return 0; |
5727 | | } |
5728 | | m_nBufferOff = 0; |
5729 | | } |
5730 | | |
5731 | | const int nToWriteInBuffer = static_cast<int>(std::min( |
5732 | | static_cast<size_t>(m_nBufferSize - m_nBufferOff), nBytesToWrite)); |
5733 | | memcpy(m_pabyBuffer + m_nBufferOff, pabySrcBuffer, nToWriteInBuffer); |
5734 | | pabySrcBuffer += nToWriteInBuffer; |
5735 | | m_nBufferOff += nToWriteInBuffer; |
5736 | | m_nCurOffset += nToWriteInBuffer; |
5737 | | nBytesToWrite -= nToWriteInBuffer; |
5738 | | } |
5739 | | return nMemb; |
5740 | | } |
5741 | | |
5742 | | /************************************************************************/ |
5743 | | /* Close() */ |
5744 | | /************************************************************************/ |
5745 | | |
5746 | | int VSIAppendWriteHandle::Close() |
5747 | | { |
5748 | | int nRet = 0; |
5749 | | if (!m_bClosed) |
5750 | | { |
5751 | | m_bClosed = true; |
5752 | | if (!m_bError && !Send(true)) |
5753 | | nRet = -1; |
5754 | | } |
5755 | | return nRet; |
5756 | | } |
5757 | | |
5758 | | /************************************************************************/ |
5759 | | /* CurlRequestHelper() */ |
5760 | | /************************************************************************/ |
5761 | | |
5762 | | CurlRequestHelper::CurlRequestHelper() |
5763 | | { |
5764 | | VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); |
5765 | | VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, |
5766 | | nullptr); |
5767 | | } |
5768 | | |
5769 | | /************************************************************************/ |
5770 | | /* ~CurlRequestHelper() */ |
5771 | | /************************************************************************/ |
5772 | | |
5773 | | CurlRequestHelper::~CurlRequestHelper() |
5774 | | { |
5775 | | CPLFree(sWriteFuncData.pBuffer); |
5776 | | CPLFree(sWriteFuncHeaderData.pBuffer); |
5777 | | } |
5778 | | |
5779 | | /************************************************************************/ |
5780 | | /* perform() */ |
5781 | | /************************************************************************/ |
5782 | | |
5783 | | long CurlRequestHelper::perform(CURL *hCurlHandle, struct curl_slist *headers, |
5784 | | VSICurlFilesystemHandlerBase *poFS, |
5785 | | IVSIS3LikeHandleHelper *poS3HandleHelper) |
5786 | | { |
5787 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); |
5788 | | |
5789 | | poS3HandleHelper->ResetQueryParameters(); |
5790 | | |
5791 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData); |
5792 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, |
5793 | | VSICurlHandleWriteFunc); |
5794 | | |
5795 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, |
5796 | | &sWriteFuncHeaderData); |
5797 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, |
5798 | | VSICurlHandleWriteFunc); |
5799 | | |
5800 | | szCurlErrBuf[0] = '\0'; |
5801 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); |
5802 | | |
5803 | | VSICURLMultiPerform(poFS->GetCurlMultiHandleFor(poS3HandleHelper->GetURL()), |
5804 | | hCurlHandle); |
5805 | | |
5806 | | VSICURLResetHeaderAndWriterFunctions(hCurlHandle); |
5807 | | |
5808 | | curl_slist_free_all(headers); |
5809 | | |
5810 | | long response_code = 0; |
5811 | | curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); |
5812 | | return response_code; |
5813 | | } |
5814 | | |
5815 | | /************************************************************************/ |
5816 | | /* NetworkStatisticsLogger */ |
5817 | | /************************************************************************/ |
5818 | | |
5819 | | // Global variable |
5820 | | NetworkStatisticsLogger NetworkStatisticsLogger::gInstance{}; |
5821 | | int NetworkStatisticsLogger::gnEnabled = -1; // unknown state |
5822 | | |
5823 | | static void ShowNetworkStats() |
5824 | | { |
5825 | | printf("Network statistics:\n%s\n", // ok |
5826 | | NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
5827 | | } |
5828 | | |
5829 | | void NetworkStatisticsLogger::ReadEnabled() |
5830 | | { |
5831 | | const bool bShowNetworkStats = |
5832 | | CPLTestBool(CPLGetConfigOption("CPL_VSIL_SHOW_NETWORK_STATS", "NO")); |
5833 | | gnEnabled = |
5834 | | (bShowNetworkStats || CPLTestBool(CPLGetConfigOption( |
5835 | | "CPL_VSIL_NETWORK_STATS_ENABLED", "NO"))) |
5836 | | ? TRUE |
5837 | | : FALSE; |
5838 | | if (bShowNetworkStats) |
5839 | | { |
5840 | | static bool bRegistered = false; |
5841 | | if (!bRegistered) |
5842 | | { |
5843 | | bRegistered = true; |
5844 | | atexit(ShowNetworkStats); |
5845 | | } |
5846 | | } |
5847 | | } |
5848 | | |
5849 | | void NetworkStatisticsLogger::EnterFileSystem(const char *pszName) |
5850 | | { |
5851 | | if (!IsEnabled()) |
5852 | | return; |
5853 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5854 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
5855 | | ContextPathItem(ContextPathType::FILESYSTEM, pszName)); |
5856 | | } |
5857 | | |
5858 | | void NetworkStatisticsLogger::LeaveFileSystem() |
5859 | | { |
5860 | | if (!IsEnabled()) |
5861 | | return; |
5862 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5863 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
5864 | | } |
5865 | | |
5866 | | void NetworkStatisticsLogger::EnterFile(const char *pszName) |
5867 | | { |
5868 | | if (!IsEnabled()) |
5869 | | return; |
5870 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5871 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
5872 | | ContextPathItem(ContextPathType::FILE, pszName)); |
5873 | | } |
5874 | | |
5875 | | void NetworkStatisticsLogger::LeaveFile() |
5876 | | { |
5877 | | if (!IsEnabled()) |
5878 | | return; |
5879 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5880 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
5881 | | } |
5882 | | |
5883 | | void NetworkStatisticsLogger::EnterAction(const char *pszName) |
5884 | | { |
5885 | | if (!IsEnabled()) |
5886 | | return; |
5887 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5888 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].push_back( |
5889 | | ContextPathItem(ContextPathType::ACTION, pszName)); |
5890 | | } |
5891 | | |
5892 | | void NetworkStatisticsLogger::LeaveAction() |
5893 | | { |
5894 | | if (!IsEnabled()) |
5895 | | return; |
5896 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5897 | | gInstance.m_mapThreadIdToContextPath[CPLGetPID()].pop_back(); |
5898 | | } |
5899 | | |
5900 | | std::vector<NetworkStatisticsLogger::Counters *> |
5901 | | NetworkStatisticsLogger::GetCountersForContext() |
5902 | | { |
5903 | | std::vector<Counters *> v; |
5904 | | const auto &contextPath = gInstance.m_mapThreadIdToContextPath[CPLGetPID()]; |
5905 | | |
5906 | | Stats *curStats = &m_stats; |
5907 | | v.push_back(&(curStats->counters)); |
5908 | | |
5909 | | bool inFileSystem = false; |
5910 | | bool inFile = false; |
5911 | | bool inAction = false; |
5912 | | for (const auto &item : contextPath) |
5913 | | { |
5914 | | if (item.eType == ContextPathType::FILESYSTEM) |
5915 | | { |
5916 | | if (inFileSystem) |
5917 | | continue; |
5918 | | inFileSystem = true; |
5919 | | } |
5920 | | else if (item.eType == ContextPathType::FILE) |
5921 | | { |
5922 | | if (inFile) |
5923 | | continue; |
5924 | | inFile = true; |
5925 | | } |
5926 | | else if (item.eType == ContextPathType::ACTION) |
5927 | | { |
5928 | | if (inAction) |
5929 | | continue; |
5930 | | inAction = true; |
5931 | | } |
5932 | | |
5933 | | curStats = &(curStats->children[item]); |
5934 | | v.push_back(&(curStats->counters)); |
5935 | | } |
5936 | | |
5937 | | return v; |
5938 | | } |
5939 | | |
5940 | | void NetworkStatisticsLogger::LogGET(size_t nDownloadedBytes) |
5941 | | { |
5942 | | if (!IsEnabled()) |
5943 | | return; |
5944 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5945 | | for (auto counters : gInstance.GetCountersForContext()) |
5946 | | { |
5947 | | counters->nGET++; |
5948 | | counters->nGETDownloadedBytes += nDownloadedBytes; |
5949 | | } |
5950 | | } |
5951 | | |
5952 | | void NetworkStatisticsLogger::LogPUT(size_t nUploadedBytes) |
5953 | | { |
5954 | | if (!IsEnabled()) |
5955 | | return; |
5956 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5957 | | for (auto counters : gInstance.GetCountersForContext()) |
5958 | | { |
5959 | | counters->nPUT++; |
5960 | | counters->nPUTUploadedBytes += nUploadedBytes; |
5961 | | } |
5962 | | } |
5963 | | |
5964 | | void NetworkStatisticsLogger::LogHEAD() |
5965 | | { |
5966 | | if (!IsEnabled()) |
5967 | | return; |
5968 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5969 | | for (auto counters : gInstance.GetCountersForContext()) |
5970 | | { |
5971 | | counters->nHEAD++; |
5972 | | } |
5973 | | } |
5974 | | |
5975 | | void NetworkStatisticsLogger::LogPOST(size_t nUploadedBytes, |
5976 | | size_t nDownloadedBytes) |
5977 | | { |
5978 | | if (!IsEnabled()) |
5979 | | return; |
5980 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5981 | | for (auto counters : gInstance.GetCountersForContext()) |
5982 | | { |
5983 | | counters->nPOST++; |
5984 | | counters->nPOSTUploadedBytes += nUploadedBytes; |
5985 | | counters->nPOSTDownloadedBytes += nDownloadedBytes; |
5986 | | } |
5987 | | } |
5988 | | |
5989 | | void NetworkStatisticsLogger::LogDELETE() |
5990 | | { |
5991 | | if (!IsEnabled()) |
5992 | | return; |
5993 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
5994 | | for (auto counters : gInstance.GetCountersForContext()) |
5995 | | { |
5996 | | counters->nDELETE++; |
5997 | | } |
5998 | | } |
5999 | | |
6000 | | void NetworkStatisticsLogger::Reset() |
6001 | | { |
6002 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6003 | | gInstance.m_stats = Stats(); |
6004 | | gnEnabled = -1; |
6005 | | } |
6006 | | |
6007 | | void NetworkStatisticsLogger::Stats::AsJSON(CPLJSONObject &oJSON) const |
6008 | | { |
6009 | | CPLJSONObject oMethods; |
6010 | | if (counters.nHEAD) |
6011 | | oMethods.Add("HEAD/count", counters.nHEAD); |
6012 | | if (counters.nGET) |
6013 | | oMethods.Add("GET/count", counters.nGET); |
6014 | | if (counters.nGETDownloadedBytes) |
6015 | | oMethods.Add("GET/downloaded_bytes", counters.nGETDownloadedBytes); |
6016 | | if (counters.nPUT) |
6017 | | oMethods.Add("PUT/count", counters.nPUT); |
6018 | | if (counters.nPUTUploadedBytes) |
6019 | | oMethods.Add("PUT/uploaded_bytes", counters.nPUTUploadedBytes); |
6020 | | if (counters.nPOST) |
6021 | | oMethods.Add("POST/count", counters.nPOST); |
6022 | | if (counters.nPOSTUploadedBytes) |
6023 | | oMethods.Add("POST/uploaded_bytes", counters.nPOSTUploadedBytes); |
6024 | | if (counters.nPOSTDownloadedBytes) |
6025 | | oMethods.Add("POST/downloaded_bytes", counters.nPOSTDownloadedBytes); |
6026 | | if (counters.nDELETE) |
6027 | | oMethods.Add("DELETE/count", counters.nDELETE); |
6028 | | oJSON.Add("methods", oMethods); |
6029 | | CPLJSONObject oFiles; |
6030 | | bool bFilesAdded = false; |
6031 | | for (const auto &kv : children) |
6032 | | { |
6033 | | CPLJSONObject childJSON; |
6034 | | kv.second.AsJSON(childJSON); |
6035 | | if (kv.first.eType == ContextPathType::FILESYSTEM) |
6036 | | { |
6037 | | std::string osName(kv.first.osName); |
6038 | | if (!osName.empty() && osName[0] == '/') |
6039 | | osName = osName.substr(1); |
6040 | | if (!osName.empty() && osName.back() == '/') |
6041 | | osName.pop_back(); |
6042 | | oJSON.Add(("handlers/" + osName).c_str(), childJSON); |
6043 | | } |
6044 | | else if (kv.first.eType == ContextPathType::FILE) |
6045 | | { |
6046 | | if (!bFilesAdded) |
6047 | | { |
6048 | | bFilesAdded = true; |
6049 | | oJSON.Add("files", oFiles); |
6050 | | } |
6051 | | oFiles.AddNoSplitName(kv.first.osName.c_str(), childJSON); |
6052 | | } |
6053 | | else if (kv.first.eType == ContextPathType::ACTION) |
6054 | | { |
6055 | | oJSON.Add(("actions/" + kv.first.osName).c_str(), childJSON); |
6056 | | } |
6057 | | } |
6058 | | } |
6059 | | |
6060 | | std::string NetworkStatisticsLogger::GetReportAsSerializedJSON() |
6061 | | { |
6062 | | std::lock_guard<std::mutex> oLock(gInstance.m_mutex); |
6063 | | |
6064 | | CPLJSONObject oJSON; |
6065 | | gInstance.m_stats.AsJSON(oJSON); |
6066 | | return oJSON.Format(CPLJSONObject::PrettyFormat::Pretty); |
6067 | | } |
6068 | | |
6069 | | } /* end of namespace cpl */ |
6070 | | |
6071 | | /************************************************************************/ |
6072 | | /* VSICurlParseUnixPermissions() */ |
6073 | | /************************************************************************/ |
6074 | | |
6075 | | int VSICurlParseUnixPermissions(const char *pszPermissions) |
6076 | | { |
6077 | | if (strlen(pszPermissions) != 9) |
6078 | | return 0; |
6079 | | int nMode = 0; |
6080 | | if (pszPermissions[0] == 'r') |
6081 | | nMode |= S_IRUSR; |
6082 | | if (pszPermissions[1] == 'w') |
6083 | | nMode |= S_IWUSR; |
6084 | | if (pszPermissions[2] == 'x') |
6085 | | nMode |= S_IXUSR; |
6086 | | if (pszPermissions[3] == 'r') |
6087 | | nMode |= S_IRGRP; |
6088 | | if (pszPermissions[4] == 'w') |
6089 | | nMode |= S_IWGRP; |
6090 | | if (pszPermissions[5] == 'x') |
6091 | | nMode |= S_IXGRP; |
6092 | | if (pszPermissions[6] == 'r') |
6093 | | nMode |= S_IROTH; |
6094 | | if (pszPermissions[7] == 'w') |
6095 | | nMode |= S_IWOTH; |
6096 | | if (pszPermissions[8] == 'x') |
6097 | | nMode |= S_IXOTH; |
6098 | | return nMode; |
6099 | | } |
6100 | | |
6101 | | /************************************************************************/ |
6102 | | /* Cache of file properties. */ |
6103 | | /************************************************************************/ |
6104 | | |
6105 | | static std::mutex oCacheFilePropMutex; |
6106 | | static lru11::Cache<std::string, cpl::FileProp> *poCacheFileProp = nullptr; |
6107 | | |
6108 | | /************************************************************************/ |
6109 | | /* VSICURLGetCachedFileProp() */ |
6110 | | /************************************************************************/ |
6111 | | |
6112 | | bool VSICURLGetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6113 | | { |
6114 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6115 | | return poCacheFileProp != nullptr && |
6116 | | poCacheFileProp->tryGet(std::string(pszURL), oFileProp) && |
6117 | | // Let a chance to use new auth parameters |
6118 | | !(oFileProp.eExists == cpl::EXIST_NO && |
6119 | | gnGenerationAuthParameters != oFileProp.nGenerationAuthParameters); |
6120 | | } |
6121 | | |
6122 | | /************************************************************************/ |
6123 | | /* VSICURLSetCachedFileProp() */ |
6124 | | /************************************************************************/ |
6125 | | |
6126 | | void VSICURLSetCachedFileProp(const char *pszURL, cpl::FileProp &oFileProp) |
6127 | | { |
6128 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6129 | | if (poCacheFileProp == nullptr) |
6130 | | poCacheFileProp = |
6131 | | new lru11::Cache<std::string, cpl::FileProp>(100 * 1024); |
6132 | | oFileProp.nGenerationAuthParameters = gnGenerationAuthParameters; |
6133 | | poCacheFileProp->insert(std::string(pszURL), oFileProp); |
6134 | | } |
6135 | | |
6136 | | /************************************************************************/ |
6137 | | /* VSICURLInvalidateCachedFileProp() */ |
6138 | | /************************************************************************/ |
6139 | | |
6140 | | void VSICURLInvalidateCachedFileProp(const char *pszURL) |
6141 | | { |
6142 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6143 | | if (poCacheFileProp != nullptr) |
6144 | | poCacheFileProp->remove(std::string(pszURL)); |
6145 | | } |
6146 | | |
6147 | | /************************************************************************/ |
6148 | | /* VSICURLInvalidateCachedFilePropPrefix() */ |
6149 | | /************************************************************************/ |
6150 | | |
6151 | | void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL) |
6152 | | { |
6153 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6154 | | if (poCacheFileProp != nullptr) |
6155 | | { |
6156 | | std::list<std::string> keysToRemove; |
6157 | | const size_t nURLSize = strlen(pszURL); |
6158 | | auto lambda = |
6159 | | [&keysToRemove, &pszURL, nURLSize]( |
6160 | | const lru11::KeyValuePair<std::string, cpl::FileProp> &kv) |
6161 | | { |
6162 | | if (strncmp(kv.key.c_str(), pszURL, nURLSize) == 0) |
6163 | | keysToRemove.push_back(kv.key); |
6164 | | }; |
6165 | | poCacheFileProp->cwalk(lambda); |
6166 | | for (const auto &key : keysToRemove) |
6167 | | poCacheFileProp->remove(key); |
6168 | | } |
6169 | | } |
6170 | | |
6171 | | /************************************************************************/ |
6172 | | /* VSICURLDestroyCacheFileProp() */ |
6173 | | /************************************************************************/ |
6174 | | |
6175 | | void VSICURLDestroyCacheFileProp() |
6176 | | { |
6177 | | std::lock_guard<std::mutex> oLock(oCacheFilePropMutex); |
6178 | | delete poCacheFileProp; |
6179 | | poCacheFileProp = nullptr; |
6180 | | } |
6181 | | |
6182 | | /************************************************************************/ |
6183 | | /* VSICURLMultiCleanup() */ |
6184 | | /************************************************************************/ |
6185 | | |
6186 | | void VSICURLMultiCleanup(CURLM *hCurlMultiHandle) |
6187 | | { |
6188 | | void *old_handler = CPLHTTPIgnoreSigPipe(); |
6189 | | curl_multi_cleanup(hCurlMultiHandle); |
6190 | | CPLHTTPRestoreSigPipeHandler(old_handler); |
6191 | | } |
6192 | | |
6193 | | /************************************************************************/ |
6194 | | /* VSICurlInstallReadCbk() */ |
6195 | | /************************************************************************/ |
6196 | | |
6197 | | int VSICurlInstallReadCbk(VSILFILE *fp, VSICurlReadCbkFunc pfnReadCbk, |
6198 | | void *pfnUserData, int bStopOnInterruptUntilUninstall) |
6199 | | { |
6200 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->InstallReadCbk( |
6201 | | pfnReadCbk, pfnUserData, bStopOnInterruptUntilUninstall); |
6202 | | } |
6203 | | |
6204 | | /************************************************************************/ |
6205 | | /* VSICurlUninstallReadCbk() */ |
6206 | | /************************************************************************/ |
6207 | | |
6208 | | int VSICurlUninstallReadCbk(VSILFILE *fp) |
6209 | | { |
6210 | | return reinterpret_cast<cpl::VSICurlHandle *>(fp)->UninstallReadCbk(); |
6211 | | } |
6212 | | |
6213 | | /************************************************************************/ |
6214 | | /* VSICurlSetOptions() */ |
6215 | | /************************************************************************/ |
6216 | | |
6217 | | struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL, |
6218 | | const char *const *papszOptions) |
6219 | | { |
6220 | | struct curl_slist *headers = static_cast<struct curl_slist *>( |
6221 | | CPLHTTPSetOptions(hCurlHandle, pszURL, papszOptions)); |
6222 | | |
6223 | | long option = CURLFTPMETHOD_SINGLECWD; |
6224 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_FILEMETHOD, option); |
6225 | | |
6226 | | // ftp://ftp2.cits.rncan.gc.ca/pub/cantopo/250k_tif/ |
6227 | | // doesn't like EPSV command, |
6228 | | unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FTP_USE_EPSV, 0); |
6229 | | |
6230 | | return headers; |
6231 | | } |
6232 | | |
6233 | | /************************************************************************/ |
6234 | | /* VSICurlMergeHeaders() */ |
6235 | | /************************************************************************/ |
6236 | | |
6237 | | struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest, |
6238 | | struct curl_slist *poSrcToDestroy) |
6239 | | { |
6240 | | struct curl_slist *iter = poSrcToDestroy; |
6241 | | while (iter != nullptr) |
6242 | | { |
6243 | | poDest = curl_slist_append(poDest, iter->data); |
6244 | | iter = iter->next; |
6245 | | } |
6246 | | if (poSrcToDestroy) |
6247 | | curl_slist_free_all(poSrcToDestroy); |
6248 | | return poDest; |
6249 | | } |
6250 | | |
6251 | | /************************************************************************/ |
6252 | | /* VSICurlSetContentTypeFromExt() */ |
6253 | | /************************************************************************/ |
6254 | | |
6255 | | struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *poList, |
6256 | | const char *pszPath) |
6257 | | { |
6258 | | struct curl_slist *iter = poList; |
6259 | | while (iter != nullptr) |
6260 | | { |
6261 | | if (STARTS_WITH_CI(iter->data, "Content-Type")) |
6262 | | { |
6263 | | return poList; |
6264 | | } |
6265 | | iter = iter->next; |
6266 | | } |
6267 | | |
6268 | | static const struct |
6269 | | { |
6270 | | const char *ext; |
6271 | | const char *mime; |
6272 | | } aosExtMimePairs[] = { |
6273 | | {"txt", "text/plain"}, {"json", "application/json"}, |
6274 | | {"tif", "image/tiff"}, {"tiff", "image/tiff"}, |
6275 | | {"jpg", "image/jpeg"}, {"jpeg", "image/jpeg"}, |
6276 | | {"jp2", "image/jp2"}, {"jpx", "image/jp2"}, |
6277 | | {"j2k", "image/jp2"}, {"jpc", "image/jp2"}, |
6278 | | {"png", "image/png"}, |
6279 | | }; |
6280 | | |
6281 | | const std::string osExt = CPLGetExtensionSafe(pszPath); |
6282 | | if (!osExt.empty()) |
6283 | | { |
6284 | | for (const auto &pair : aosExtMimePairs) |
6285 | | { |
6286 | | if (EQUAL(osExt.c_str(), pair.ext)) |
6287 | | { |
6288 | | |
6289 | | const std::string osContentType( |
6290 | | CPLSPrintf("Content-Type: %s", pair.mime)); |
6291 | | poList = curl_slist_append(poList, osContentType.c_str()); |
6292 | | #ifdef DEBUG_VERBOSE |
6293 | | CPLDebug("HTTP", "Setting %s, based on lookup table.", |
6294 | | osContentType.c_str()); |
6295 | | #endif |
6296 | | break; |
6297 | | } |
6298 | | } |
6299 | | } |
6300 | | |
6301 | | return poList; |
6302 | | } |
6303 | | |
6304 | | /************************************************************************/ |
6305 | | /* VSICurlSetCreationHeadersFromOptions() */ |
6306 | | /************************************************************************/ |
6307 | | |
6308 | | struct curl_slist *VSICurlSetCreationHeadersFromOptions( |
6309 | | struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath) |
6310 | | { |
6311 | | bool bContentTypeFound = false; |
6312 | | for (CSLConstList papszIter = papszOptions; papszIter && *papszIter; |
6313 | | ++papszIter) |
6314 | | { |
6315 | | char *pszKey = nullptr; |
6316 | | const char *pszValue = CPLParseNameValue(*papszIter, &pszKey); |
6317 | | if (pszKey && pszValue) |
6318 | | { |
6319 | | if (EQUAL(pszKey, "Content-Type")) |
6320 | | { |
6321 | | bContentTypeFound = true; |
6322 | | } |
6323 | | headers = curl_slist_append(headers, |
6324 | | CPLSPrintf("%s: %s", pszKey, pszValue)); |
6325 | | } |
6326 | | CPLFree(pszKey); |
6327 | | } |
6328 | | |
6329 | | // If Content-type not found in papszOptions, try to set it from the |
6330 | | // filename exstension. |
6331 | | if (!bContentTypeFound) |
6332 | | { |
6333 | | headers = VSICurlSetContentTypeFromExt(headers, pszPath); |
6334 | | } |
6335 | | |
6336 | | return headers; |
6337 | | } |
6338 | | |
6339 | | #endif // DOXYGEN_SKIP |
6340 | | //! @endcond |
6341 | | |
6342 | | /************************************************************************/ |
6343 | | /* VSIInstallCurlFileHandler() */ |
6344 | | /************************************************************************/ |
6345 | | |
6346 | | /*! |
6347 | | \brief Install /vsicurl/ HTTP/FTP file system handler (requires libcurl) |
6348 | | |
6349 | | \verbatim embed:rst |
6350 | | See :ref:`/vsicurl/ documentation <vsicurl>` |
6351 | | \endverbatim |
6352 | | |
6353 | | @since GDAL 1.8.0 |
6354 | | */ |
6355 | | void VSIInstallCurlFileHandler(void) |
6356 | | { |
6357 | | VSIFilesystemHandler *poHandler = new cpl::VSICurlFilesystemHandler; |
6358 | | VSIFileManager::InstallHandler("/vsicurl/", poHandler); |
6359 | | VSIFileManager::InstallHandler("/vsicurl?", poHandler); |
6360 | | } |
6361 | | |
6362 | | /************************************************************************/ |
6363 | | /* VSICurlClearCache() */ |
6364 | | /************************************************************************/ |
6365 | | |
6366 | | /** |
6367 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6368 | | * |
6369 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6370 | | * /vsiswift/) cache a number of |
6371 | | * metadata and data for faster execution in read-only scenarios. But when the |
6372 | | * content on the server-side may change during the same process, those |
6373 | | * mechanisms can prevent opening new files, or give an outdated version of |
6374 | | * them. |
6375 | | * |
6376 | | * @since GDAL 2.2.1 |
6377 | | */ |
6378 | | |
6379 | | void VSICurlClearCache(void) |
6380 | | { |
6381 | | // FIXME ? Currently we have different filesystem instances for |
6382 | | // vsicurl/, /vsis3/, /vsigs/ . So each one has its own cache of regions. |
6383 | | // File properties cache are now shared |
6384 | | char **papszPrefix = VSIFileManager::GetPrefixes(); |
6385 | | for (size_t i = 0; papszPrefix && papszPrefix[i]; ++i) |
6386 | | { |
6387 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6388 | | VSIFileManager::GetHandler(papszPrefix[i])); |
6389 | | |
6390 | | if (poFSHandler) |
6391 | | poFSHandler->ClearCache(); |
6392 | | } |
6393 | | CSLDestroy(papszPrefix); |
6394 | | |
6395 | | VSICurlStreamingClearCache(); |
6396 | | } |
6397 | | |
6398 | | /************************************************************************/ |
6399 | | /* VSICurlPartialClearCache() */ |
6400 | | /************************************************************************/ |
6401 | | |
6402 | | /** |
6403 | | * \brief Clean local cache associated with /vsicurl/ (and related file systems) |
6404 | | * for a given filename (and its subfiles and subdirectories if it is a |
6405 | | * directory) |
6406 | | * |
6407 | | * /vsicurl (and related file systems like /vsis3/, /vsigs/, /vsiaz/, /vsioss/, |
6408 | | * /vsiswift/) cache a number of |
6409 | | * metadata and data for faster execution in read-only scenarios. But when the |
6410 | | * content on the server-side may change during the same process, those |
6411 | | * mechanisms can prevent opening new files, or give an outdated version of |
6412 | | * them. |
6413 | | * |
6414 | | * The filename prefix must start with the name of a known virtual file system |
6415 | | * (such as "/vsicurl/", "/vsis3/") |
6416 | | * |
6417 | | * VSICurlPartialClearCache("/vsis3/b") will clear all cached state for any file |
6418 | | * or directory starting with that prefix, so potentially "/vsis3/bucket", |
6419 | | * "/vsis3/basket/" or "/vsis3/basket/object". |
6420 | | * |
6421 | | * @param pszFilenamePrefix Filename prefix |
6422 | | * @since GDAL 2.4.0 |
6423 | | */ |
6424 | | |
6425 | | void VSICurlPartialClearCache(const char *pszFilenamePrefix) |
6426 | | { |
6427 | | auto poFSHandler = dynamic_cast<cpl::VSICurlFilesystemHandlerBase *>( |
6428 | | VSIFileManager::GetHandler(pszFilenamePrefix)); |
6429 | | |
6430 | | if (poFSHandler) |
6431 | | poFSHandler->PartialClearCache(pszFilenamePrefix); |
6432 | | } |
6433 | | |
6434 | | /************************************************************************/ |
6435 | | /* VSINetworkStatsReset() */ |
6436 | | /************************************************************************/ |
6437 | | |
6438 | | /** |
6439 | | * \brief Clear network related statistics. |
6440 | | * |
6441 | | * The effect of the CPL_VSIL_NETWORK_STATS_ENABLED configuration option |
6442 | | * will also be reset. That is, that the next network access will check its |
6443 | | * value again. |
6444 | | * |
6445 | | * @since GDAL 3.2.0 |
6446 | | */ |
6447 | | |
6448 | | void VSINetworkStatsReset(void) |
6449 | | { |
6450 | | cpl::NetworkStatisticsLogger::Reset(); |
6451 | | } |
6452 | | |
6453 | | /************************************************************************/ |
6454 | | /* VSINetworkStatsGetAsSerializedJSON() */ |
6455 | | /************************************************************************/ |
6456 | | |
6457 | | /** |
6458 | | * \brief Return network related statistics, as a JSON serialized object. |
6459 | | * |
6460 | | * Statistics collecting should be enabled with the |
6461 | | CPL_VSIL_NETWORK_STATS_ENABLED |
6462 | | * configuration option set to YES before any network activity starts |
6463 | | * (for efficiency, reading it is cached on first access, until |
6464 | | VSINetworkStatsReset() is called) |
6465 | | * |
6466 | | * Statistics can also be emitted on standard output at process termination if |
6467 | | * the CPL_VSIL_SHOW_NETWORK_STATS configuration option is set to YES. |
6468 | | * |
6469 | | * Example of output: |
6470 | | * \code{.js} |
6471 | | * { |
6472 | | * "methods":{ |
6473 | | * "GET":{ |
6474 | | * "count":6, |
6475 | | * "downloaded_bytes":40825 |
6476 | | * }, |
6477 | | * "PUT":{ |
6478 | | * "count":1, |
6479 | | * "uploaded_bytes":35472 |
6480 | | * } |
6481 | | * }, |
6482 | | * "handlers":{ |
6483 | | * "vsigs":{ |
6484 | | * "methods":{ |
6485 | | * "GET":{ |
6486 | | * "count":2, |
6487 | | * "downloaded_bytes":446 |
6488 | | * }, |
6489 | | * "PUT":{ |
6490 | | * "count":1, |
6491 | | * "uploaded_bytes":35472 |
6492 | | * } |
6493 | | * }, |
6494 | | * "files":{ |
6495 | | * "\/vsigs\/spatialys\/byte.tif":{ |
6496 | | * "methods":{ |
6497 | | * "PUT":{ |
6498 | | * "count":1, |
6499 | | * "uploaded_bytes":35472 |
6500 | | * } |
6501 | | * }, |
6502 | | * "actions":{ |
6503 | | * "Write":{ |
6504 | | * "methods":{ |
6505 | | * "PUT":{ |
6506 | | * "count":1, |
6507 | | * "uploaded_bytes":35472 |
6508 | | * } |
6509 | | * } |
6510 | | * } |
6511 | | * } |
6512 | | * } |
6513 | | * }, |
6514 | | * "actions":{ |
6515 | | * "Stat":{ |
6516 | | * "methods":{ |
6517 | | * "GET":{ |
6518 | | * "count":2, |
6519 | | * "downloaded_bytes":446 |
6520 | | * } |
6521 | | * }, |
6522 | | * "files":{ |
6523 | | * "\/vsigs\/spatialys\/byte.tif\/":{ |
6524 | | * "methods":{ |
6525 | | * "GET":{ |
6526 | | * "count":1, |
6527 | | * "downloaded_bytes":181 |
6528 | | * } |
6529 | | * } |
6530 | | * } |
6531 | | * } |
6532 | | * } |
6533 | | * } |
6534 | | * }, |
6535 | | * "vsis3":{ |
6536 | | * [...] |
6537 | | * } |
6538 | | * } |
6539 | | * } |
6540 | | * \endcode |
6541 | | * |
6542 | | * @param papszOptions Unused. |
6543 | | * @return a JSON serialized string to free with VSIFree(), or nullptr |
6544 | | * @since GDAL 3.2.0 |
6545 | | */ |
6546 | | |
6547 | | char *VSINetworkStatsGetAsSerializedJSON(CPL_UNUSED char **papszOptions) |
6548 | | { |
6549 | | return CPLStrdup( |
6550 | | cpl::NetworkStatisticsLogger::GetReportAsSerializedJSON().c_str()); |
6551 | | } |
6552 | | |
6553 | | #endif /* HAVE_CURL */ |
6554 | | |
6555 | | #undef ENABLE_DEBUG |