/src/gdal/frmts/wms/gdalhttp.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: WMS Client Driver |
4 | | * Purpose: Implementation of Dataset and RasterBand classes for WMS |
5 | | * and other similar services. |
6 | | * Author: Adam Nowacki, nowak@xpam.de |
7 | | * |
8 | | ****************************************************************************** |
9 | | * Copyright (c) 2007, Adam Nowacki |
10 | | * Copyright (c) 2007-2013, Even Rouault <even dot rouault at spatialys.com> |
11 | | * Copyright (c) 2016, Lucian Plesea |
12 | | * |
13 | | * SPDX-License-Identifier: MIT |
14 | | ****************************************************************************/ |
15 | | |
16 | | #include "wmsdriver.h" |
17 | | #include <algorithm> |
18 | | |
19 | | static size_t WriteFunc(void *buffer, size_t count, size_t nmemb, void *req) |
20 | 0 | { |
21 | 0 | WMSHTTPRequest *psRequest = reinterpret_cast<WMSHTTPRequest *>(req); |
22 | 0 | size_t size = count * nmemb; |
23 | |
|
24 | 0 | if (size == 0) |
25 | 0 | return 0; |
26 | | |
27 | 0 | const size_t required_size = psRequest->nDataLen + size + 1; |
28 | 0 | if (required_size > psRequest->nDataAlloc) |
29 | 0 | { |
30 | 0 | size_t new_size = required_size * 2; |
31 | 0 | if (new_size < 512) |
32 | 0 | new_size = 512; |
33 | 0 | psRequest->nDataAlloc = new_size; |
34 | 0 | GByte *pabyNewData = reinterpret_cast<GByte *>( |
35 | 0 | VSIRealloc(psRequest->pabyData, new_size)); |
36 | 0 | if (pabyNewData == nullptr) |
37 | 0 | { |
38 | 0 | VSIFree(psRequest->pabyData); |
39 | 0 | psRequest->pabyData = nullptr; |
40 | 0 | psRequest->Error.Printf( |
41 | 0 | "Out of memory allocating %u bytes for HTTP data buffer.", |
42 | 0 | static_cast<unsigned int>(new_size)); |
43 | 0 | psRequest->nDataAlloc = 0; |
44 | 0 | psRequest->nDataLen = 0; |
45 | 0 | return 0; |
46 | 0 | } |
47 | 0 | psRequest->pabyData = pabyNewData; |
48 | 0 | } |
49 | 0 | memcpy(psRequest->pabyData + psRequest->nDataLen, buffer, size); |
50 | 0 | psRequest->nDataLen += size; |
51 | 0 | psRequest->pabyData[psRequest->nDataLen] = 0; |
52 | 0 | return nmemb; |
53 | 0 | } |
54 | | |
55 | | // Process curl errors |
56 | | static void ProcessCurlErrors(CURLMsg *msg, WMSHTTPRequest *pasRequest, |
57 | | int nRequestCount) |
58 | 0 | { |
59 | 0 | CPLAssert(msg != nullptr); |
60 | 0 | CPLAssert(msg->msg == CURLMSG_DONE); |
61 | | |
62 | | // in case of local file error: update status code |
63 | 0 | if (msg->data.result == CURLE_FILE_COULDNT_READ_FILE) |
64 | 0 | { |
65 | | // identify current request |
66 | 0 | for (int current_req_i = 0; current_req_i < nRequestCount; |
67 | 0 | ++current_req_i) |
68 | 0 | { |
69 | 0 | WMSHTTPRequest *const psRequest = &pasRequest[current_req_i]; |
70 | 0 | if (psRequest->m_curl_handle != msg->easy_handle) |
71 | 0 | continue; |
72 | | |
73 | | // sanity check for local files |
74 | 0 | if (STARTS_WITH(psRequest->URL.c_str(), "file://")) |
75 | 0 | { |
76 | 0 | psRequest->nStatus = 404; |
77 | 0 | break; |
78 | 0 | } |
79 | 0 | } |
80 | 0 | } |
81 | 0 | } |
82 | | |
83 | | // Builds a curl request |
84 | | void WMSHTTPInitializeRequest(WMSHTTPRequest *psRequest) |
85 | 0 | { |
86 | 0 | psRequest->nStatus = 0; |
87 | 0 | psRequest->pabyData = nullptr; |
88 | 0 | psRequest->nDataLen = 0; |
89 | 0 | psRequest->nDataAlloc = 0; |
90 | |
|
91 | 0 | psRequest->m_curl_handle = curl_easy_init(); |
92 | 0 | if (psRequest->m_curl_handle == nullptr) |
93 | 0 | { |
94 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
95 | 0 | "CPLHTTPInitializeRequest(): Unable to create CURL handle."); |
96 | 0 | return; |
97 | 0 | } |
98 | | |
99 | 0 | if (!psRequest->Range.empty()) |
100 | 0 | { |
101 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt( |
102 | 0 | psRequest->m_curl_handle, CURLOPT_RANGE, psRequest->Range.c_str())); |
103 | 0 | } |
104 | |
|
105 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
106 | 0 | CURLOPT_WRITEDATA, psRequest)); |
107 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
108 | 0 | CURLOPT_WRITEFUNCTION, WriteFunc)); |
109 | |
|
110 | 0 | psRequest->m_curl_error.resize(CURL_ERROR_SIZE + 1); |
111 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
112 | 0 | CURLOPT_ERRORBUFFER, |
113 | 0 | &psRequest->m_curl_error[0])); |
114 | |
|
115 | 0 | psRequest->m_headers = static_cast<struct curl_slist *>(CPLHTTPSetOptions( |
116 | 0 | psRequest->m_curl_handle, psRequest->URL.c_str(), psRequest->options)); |
117 | 0 | if (psRequest->m_headers != nullptr) |
118 | 0 | { |
119 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
120 | 0 | CURLOPT_HTTPHEADER, |
121 | 0 | psRequest->m_headers)); |
122 | 0 | } |
123 | 0 | } |
124 | | |
125 | | WMSHTTPRequest::~WMSHTTPRequest() |
126 | 0 | { |
127 | 0 | if (m_curl_handle != nullptr) |
128 | 0 | curl_easy_cleanup(m_curl_handle); |
129 | 0 | if (m_headers != nullptr) |
130 | 0 | curl_slist_free_all(m_headers); |
131 | 0 | if (pabyData != nullptr) |
132 | 0 | CPLFree(pabyData); |
133 | 0 | } |
134 | | |
135 | | // |
136 | | // Like CPLHTTPFetch, but multiple requests in parallel |
137 | | // By default it uses 5 connections |
138 | | // |
139 | | CPLErr WMSHTTPFetchMulti(WMSHTTPRequest *pasRequest, int nRequestCount) |
140 | 0 | { |
141 | 0 | CPLErr ret = CE_None; |
142 | 0 | CURLM *curl_multi = nullptr; |
143 | 0 | int max_conn; |
144 | 0 | int i, conn_i; |
145 | |
|
146 | 0 | CPLAssert(nRequestCount >= 0); |
147 | 0 | if (nRequestCount == 0) |
148 | 0 | return CE_None; |
149 | | |
150 | 0 | const char *max_conn_opt = |
151 | 0 | CSLFetchNameValue(const_cast<char **>(pasRequest->options), "MAXCONN"); |
152 | 0 | max_conn = |
153 | 0 | (max_conn_opt == nullptr) ? 5 : MAX(1, MIN(atoi(max_conn_opt), 1000)); |
154 | | |
155 | | // If the first url starts with vsimem, assume all do and defer to |
156 | | // CPLHTTPFetch |
157 | 0 | if (STARTS_WITH(pasRequest[0].URL.c_str(), "/vsimem/") && |
158 | | /* Disabled by default for potential security issues */ |
159 | 0 | CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE"))) |
160 | 0 | { |
161 | 0 | for (i = 0; i < nRequestCount; i++) |
162 | 0 | { |
163 | 0 | CPLHTTPResult *psResult = |
164 | 0 | CPLHTTPFetch(pasRequest[i].URL.c_str(), |
165 | 0 | const_cast<char **>(pasRequest[i].options)); |
166 | 0 | pasRequest[i].pabyData = psResult->pabyData; |
167 | 0 | pasRequest[i].nDataLen = psResult->nDataLen; |
168 | 0 | pasRequest[i].Error = |
169 | 0 | psResult->pszErrBuf ? psResult->pszErrBuf : ""; |
170 | | // Conventions are different between this module and cpl_http... |
171 | 0 | if (psResult->pszErrBuf != nullptr && |
172 | 0 | strcmp(psResult->pszErrBuf, "HTTP error code : 404") == 0) |
173 | 0 | pasRequest[i].nStatus = 404; |
174 | 0 | else |
175 | 0 | pasRequest[i].nStatus = 200; |
176 | 0 | pasRequest[i].ContentType = |
177 | 0 | psResult->pszContentType ? psResult->pszContentType : ""; |
178 | | // took ownership of content, we're done with the rest |
179 | 0 | psResult->pabyData = nullptr; |
180 | 0 | psResult->nDataLen = 0; |
181 | 0 | CPLHTTPDestroyResult(psResult); |
182 | 0 | } |
183 | 0 | return CE_None; |
184 | 0 | } |
185 | | |
186 | 0 | curl_multi = curl_multi_init(); |
187 | 0 | if (curl_multi == nullptr) |
188 | 0 | { |
189 | 0 | CPLError(CE_Fatal, CPLE_AppDefined, |
190 | 0 | "CPLHTTPFetchMulti(): Unable to create CURL multi-handle."); |
191 | 0 | } |
192 | | |
193 | | // add at most max_conn requests |
194 | 0 | int torun = std::min(nRequestCount, max_conn); |
195 | 0 | for (conn_i = 0; conn_i < torun; ++conn_i) |
196 | 0 | { |
197 | 0 | WMSHTTPRequest *const psRequest = &pasRequest[conn_i]; |
198 | 0 | CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount, |
199 | 0 | pasRequest[conn_i].URL.c_str()); |
200 | 0 | curl_multi_add_handle(curl_multi, psRequest->m_curl_handle); |
201 | 0 | } |
202 | |
|
203 | 0 | void *old_handler = CPLHTTPIgnoreSigPipe(); |
204 | 0 | int still_running; |
205 | 0 | do |
206 | 0 | { |
207 | 0 | CURLMcode mc; |
208 | 0 | do |
209 | 0 | { |
210 | 0 | mc = curl_multi_perform(curl_multi, &still_running); |
211 | 0 | } while (CURLM_CALL_MULTI_PERFORM == mc); |
212 | | |
213 | | // Pick up messages, clean up the completed ones, add more |
214 | 0 | int msgs_in_queue = 0; |
215 | 0 | do |
216 | 0 | { |
217 | 0 | CURLMsg *m = curl_multi_info_read(curl_multi, &msgs_in_queue); |
218 | 0 | if (m && (m->msg == CURLMSG_DONE)) |
219 | 0 | { |
220 | 0 | ProcessCurlErrors(m, pasRequest, nRequestCount); |
221 | |
|
222 | 0 | curl_multi_remove_handle(curl_multi, m->easy_handle); |
223 | 0 | if (conn_i < nRequestCount) |
224 | 0 | { |
225 | 0 | auto psRequest = &pasRequest[conn_i]; |
226 | 0 | CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, |
227 | 0 | nRequestCount, pasRequest[conn_i].URL.c_str()); |
228 | 0 | curl_multi_add_handle(curl_multi, psRequest->m_curl_handle); |
229 | 0 | ++conn_i; |
230 | 0 | still_running = 1; // Still have request pending |
231 | 0 | } |
232 | 0 | } |
233 | 0 | } while (msgs_in_queue); |
234 | |
|
235 | 0 | if (CURLM_OK == mc) |
236 | 0 | { |
237 | 0 | int numfds; |
238 | 0 | curl_multi_wait(curl_multi, nullptr, 0, 100, &numfds); |
239 | 0 | } |
240 | 0 | } while (still_running || conn_i != nRequestCount); |
241 | | |
242 | | // process any message still in queue |
243 | 0 | CURLMsg *msg; |
244 | 0 | int msgs_in_queue; |
245 | 0 | do |
246 | 0 | { |
247 | 0 | msg = curl_multi_info_read(curl_multi, &msgs_in_queue); |
248 | 0 | if (msg != nullptr) |
249 | 0 | { |
250 | 0 | if (msg->msg == CURLMSG_DONE) |
251 | 0 | { |
252 | 0 | ProcessCurlErrors(msg, pasRequest, nRequestCount); |
253 | 0 | } |
254 | 0 | } |
255 | 0 | } while (msg != nullptr); |
256 | |
|
257 | 0 | CPLHTTPRestoreSigPipeHandler(old_handler); |
258 | |
|
259 | 0 | if (conn_i != nRequestCount) |
260 | 0 | { // something gone really really wrong |
261 | | // oddly built libcurl or perhaps absence of network interface |
262 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
263 | 0 | "WMSHTTPFetchMulti(): conn_i != nRequestCount, this should " |
264 | 0 | "never happen ..."); |
265 | 0 | nRequestCount = conn_i; |
266 | 0 | ret = CE_Failure; |
267 | 0 | } |
268 | |
|
269 | 0 | for (i = 0; i < nRequestCount; ++i) |
270 | 0 | { |
271 | 0 | WMSHTTPRequest *const psRequest = &pasRequest[i]; |
272 | |
|
273 | 0 | long response_code; |
274 | 0 | curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_RESPONSE_CODE, |
275 | 0 | &response_code); |
276 | | // for local files, don't update the status code if one is already set |
277 | 0 | if (!(psRequest->nStatus != 0 && |
278 | 0 | STARTS_WITH(psRequest->URL.c_str(), "file://"))) |
279 | 0 | psRequest->nStatus = static_cast<int>(response_code); |
280 | |
|
281 | 0 | char *content_type = nullptr; |
282 | 0 | curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_CONTENT_TYPE, |
283 | 0 | &content_type); |
284 | 0 | psRequest->ContentType = content_type ? content_type : ""; |
285 | |
|
286 | 0 | if (psRequest->Error.empty()) |
287 | 0 | psRequest->Error = &psRequest->m_curl_error[0]; |
288 | | |
289 | | /* In the case of a file:// URL, curl will return a status == 0, so if |
290 | | * there's no */ |
291 | | /* error returned, patch the status code to be 200, as it would be for |
292 | | * http:// */ |
293 | 0 | if (psRequest->nStatus == 0 && psRequest->Error.empty() && |
294 | 0 | STARTS_WITH(psRequest->URL.c_str(), "file://")) |
295 | 0 | psRequest->nStatus = 200; |
296 | | |
297 | | // If there is an error with no error message, use the content if it is |
298 | | // text |
299 | 0 | if (psRequest->Error.empty() && psRequest->nStatus != 0 && |
300 | 0 | psRequest->nStatus != 200 && |
301 | 0 | strstr(psRequest->ContentType, "text") && |
302 | 0 | psRequest->pabyData != nullptr) |
303 | 0 | psRequest->Error = |
304 | 0 | reinterpret_cast<const char *>(psRequest->pabyData); |
305 | |
|
306 | 0 | CPLDebug( |
307 | 0 | "HTTP", "Request [%d] %s : status = %d, type = %s, error = %s", i, |
308 | 0 | psRequest->URL.c_str(), psRequest->nStatus, |
309 | 0 | !psRequest->ContentType.empty() ? psRequest->ContentType.c_str() |
310 | 0 | : "(null)", |
311 | 0 | !psRequest->Error.empty() ? psRequest->Error.c_str() : "(null)"); |
312 | |
|
313 | 0 | curl_multi_remove_handle(curl_multi, pasRequest->m_curl_handle); |
314 | 0 | } |
315 | |
|
316 | 0 | curl_multi_cleanup(curl_multi); |
317 | |
|
318 | 0 | return ret; |
319 | 0 | } |