/src/gdal/frmts/wms/gdalhttp.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: WMS Client Driver |
4 | | * Purpose: Implementation of Dataset and RasterBand classes for WMS |
5 | | * and other similar services. |
6 | | * Author: Adam Nowacki, nowak@xpam.de |
7 | | * |
8 | | ****************************************************************************** |
9 | | * Copyright (c) 2007, Adam Nowacki |
10 | | * Copyright (c) 2007-2013, Even Rouault <even dot rouault at spatialys.com> |
11 | | * Copyright (c) 2016, Lucian Plesea |
12 | | * |
13 | | * SPDX-License-Identifier: MIT |
14 | | ****************************************************************************/ |
15 | | |
16 | | #include "wmsdriver.h" |
17 | | #include <algorithm> |
18 | | |
19 | | static size_t WriteFunc(void *buffer, size_t count, size_t nmemb, void *req) |
20 | 485 | { |
21 | 485 | WMSHTTPRequest *psRequest = reinterpret_cast<WMSHTTPRequest *>(req); |
22 | 485 | size_t size = count * nmemb; |
23 | | |
24 | 485 | if (size == 0) |
25 | 0 | return 0; |
26 | | |
27 | 485 | const size_t required_size = psRequest->nDataLen + size + 1; |
28 | 485 | if (required_size > psRequest->nDataAlloc) |
29 | 357 | { |
30 | 357 | size_t new_size = required_size * 2; |
31 | 357 | if (new_size < 512) |
32 | 0 | new_size = 512; |
33 | 357 | psRequest->nDataAlloc = new_size; |
34 | 357 | GByte *pabyNewData = reinterpret_cast<GByte *>( |
35 | 357 | VSIRealloc(psRequest->pabyData, new_size)); |
36 | 357 | if (pabyNewData == nullptr) |
37 | 0 | { |
38 | 0 | VSIFree(psRequest->pabyData); |
39 | 0 | psRequest->pabyData = nullptr; |
40 | 0 | psRequest->Error.Printf( |
41 | 0 | "Out of memory allocating %u bytes for HTTP data buffer.", |
42 | 0 | static_cast<unsigned int>(new_size)); |
43 | 0 | psRequest->nDataAlloc = 0; |
44 | 0 | psRequest->nDataLen = 0; |
45 | 0 | return 0; |
46 | 0 | } |
47 | 357 | psRequest->pabyData = pabyNewData; |
48 | 357 | } |
49 | 485 | memcpy(psRequest->pabyData + psRequest->nDataLen, buffer, size); |
50 | 485 | psRequest->nDataLen += size; |
51 | 485 | psRequest->pabyData[psRequest->nDataLen] = 0; |
52 | 485 | return nmemb; |
53 | 485 | } |
54 | | |
55 | | // Process curl errors |
56 | | static void ProcessCurlErrors(CURLMsg *msg, WMSHTTPRequest *pasRequest, |
57 | | int nRequestCount) |
58 | 2.75k | { |
59 | 2.75k | CPLAssert(msg != nullptr); |
60 | 2.75k | CPLAssert(msg->msg == CURLMSG_DONE); |
61 | | |
62 | | // in case of local file error: update status code |
63 | 2.75k | if (msg->data.result == CURLE_FILE_COULDNT_READ_FILE) |
64 | 0 | { |
65 | | // identify current request |
66 | 0 | for (int current_req_i = 0; current_req_i < nRequestCount; |
67 | 0 | ++current_req_i) |
68 | 0 | { |
69 | 0 | WMSHTTPRequest *const psRequest = &pasRequest[current_req_i]; |
70 | 0 | if (psRequest->m_curl_handle != msg->easy_handle) |
71 | 0 | continue; |
72 | | |
73 | | // sanity check for local files |
74 | 0 | if (STARTS_WITH(psRequest->URL.c_str(), "file://")) |
75 | 0 | { |
76 | 0 | psRequest->nStatus = 404; |
77 | 0 | break; |
78 | 0 | } |
79 | 0 | } |
80 | 0 | } |
81 | 2.75k | } |
82 | | |
83 | | // Builds a curl request |
84 | | void WMSHTTPInitializeRequest(WMSHTTPRequest *psRequest) |
85 | 2.75k | { |
86 | 2.75k | psRequest->nStatus = 0; |
87 | 2.75k | psRequest->pabyData = nullptr; |
88 | 2.75k | psRequest->nDataLen = 0; |
89 | 2.75k | psRequest->nDataAlloc = 0; |
90 | | |
91 | 2.75k | psRequest->m_curl_handle = curl_easy_init(); |
92 | 2.75k | if (psRequest->m_curl_handle == nullptr) |
93 | 0 | { |
94 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
95 | 0 | "CPLHTTPInitializeRequest(): Unable to create CURL handle."); |
96 | 0 | return; |
97 | 0 | } |
98 | | |
99 | 2.75k | if (!psRequest->Range.empty()) |
100 | 0 | { |
101 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt( |
102 | 0 | psRequest->m_curl_handle, CURLOPT_RANGE, psRequest->Range.c_str())); |
103 | 0 | } |
104 | | |
105 | 2.75k | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
106 | 2.75k | CURLOPT_WRITEDATA, psRequest)); |
107 | 2.75k | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
108 | 2.75k | CURLOPT_WRITEFUNCTION, WriteFunc)); |
109 | | |
110 | 2.75k | psRequest->m_curl_error.resize(CURL_ERROR_SIZE + 1); |
111 | 2.75k | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
112 | 2.75k | CURLOPT_ERRORBUFFER, |
113 | 2.75k | &psRequest->m_curl_error[0])); |
114 | | |
115 | 2.75k | psRequest->m_headers = static_cast<struct curl_slist *>(CPLHTTPSetOptions( |
116 | 2.75k | psRequest->m_curl_handle, psRequest->URL.URLEncode().c_str(), |
117 | 2.75k | psRequest->options)); |
118 | 2.75k | if (psRequest->m_headers != nullptr) |
119 | 0 | { |
120 | 0 | CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle, |
121 | 0 | CURLOPT_HTTPHEADER, |
122 | 0 | psRequest->m_headers)); |
123 | 0 | } |
124 | 2.75k | } |
125 | | |
126 | | WMSHTTPRequest::~WMSHTTPRequest() |
127 | 2.94k | { |
128 | 2.94k | if (m_curl_handle != nullptr) |
129 | 2.75k | curl_easy_cleanup(m_curl_handle); |
130 | 2.94k | if (m_headers != nullptr) |
131 | 0 | curl_slist_free_all(m_headers); |
132 | 2.94k | if (pabyData != nullptr) |
133 | 324 | CPLFree(pabyData); |
134 | 2.94k | } |
135 | | |
136 | | // |
137 | | // Like CPLHTTPFetch, but multiple requests in parallel |
138 | | // By default it uses 5 connections |
139 | | // |
140 | | CPLErr WMSHTTPFetchMulti(WMSHTTPRequest *pasRequest, int nRequestCount) |
141 | 1.18k | { |
142 | 1.18k | CPLErr ret = CE_None; |
143 | 1.18k | CURLM *curl_multi = nullptr; |
144 | 1.18k | int max_conn; |
145 | 1.18k | int i, conn_i; |
146 | | |
147 | 1.18k | CPLAssert(nRequestCount >= 0); |
148 | 1.18k | if (nRequestCount == 0) |
149 | 34 | return CE_None; |
150 | | |
151 | 1.14k | const char *max_conn_opt = |
152 | 1.14k | CSLFetchNameValue(const_cast<char **>(pasRequest->options), "MAXCONN"); |
153 | 1.14k | max_conn = |
154 | 1.14k | (max_conn_opt == nullptr) ? 5 : MAX(1, MIN(atoi(max_conn_opt), 1000)); |
155 | | |
156 | | // If the first url starts with vsimem, assume all do and defer to |
157 | | // CPLHTTPFetch |
158 | 1.14k | if (STARTS_WITH(pasRequest[0].URL.c_str(), "/vsimem/") && |
159 | | /* Disabled by default for potential security issues */ |
160 | 0 | CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE"))) |
161 | 0 | { |
162 | 0 | for (i = 0; i < nRequestCount; i++) |
163 | 0 | { |
164 | 0 | CPLHTTPResult *psResult = |
165 | 0 | CPLHTTPFetch(pasRequest[i].URL.c_str(), |
166 | 0 | const_cast<char **>(pasRequest[i].options)); |
167 | 0 | pasRequest[i].pabyData = psResult->pabyData; |
168 | 0 | pasRequest[i].nDataLen = psResult->nDataLen; |
169 | 0 | pasRequest[i].Error = |
170 | 0 | psResult->pszErrBuf ? psResult->pszErrBuf : ""; |
171 | | // Conventions are different between this module and cpl_http... |
172 | 0 | if (psResult->pszErrBuf != nullptr && |
173 | 0 | strcmp(psResult->pszErrBuf, "HTTP error code : 404") == 0) |
174 | 0 | pasRequest[i].nStatus = 404; |
175 | 0 | else |
176 | 0 | pasRequest[i].nStatus = 200; |
177 | 0 | pasRequest[i].ContentType = |
178 | 0 | psResult->pszContentType ? psResult->pszContentType : ""; |
179 | | // took ownership of content, we're done with the rest |
180 | 0 | psResult->pabyData = nullptr; |
181 | 0 | psResult->nDataLen = 0; |
182 | 0 | CPLHTTPDestroyResult(psResult); |
183 | 0 | } |
184 | 0 | return CE_None; |
185 | 0 | } |
186 | | |
187 | 1.14k | curl_multi = curl_multi_init(); |
188 | 1.14k | if (curl_multi == nullptr) |
189 | 0 | { |
190 | 0 | CPLError(CE_Fatal, CPLE_AppDefined, |
191 | 0 | "CPLHTTPFetchMulti(): Unable to create CURL multi-handle."); |
192 | 0 | } |
193 | | |
194 | | // add at most max_conn requests |
195 | 1.14k | int torun = std::min(nRequestCount, max_conn); |
196 | 2.36k | for (conn_i = 0; conn_i < torun; ++conn_i) |
197 | 1.21k | { |
198 | 1.21k | WMSHTTPRequest *const psRequest = &pasRequest[conn_i]; |
199 | 1.21k | CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount, |
200 | 1.21k | pasRequest[conn_i].URL.c_str()); |
201 | 1.21k | curl_multi_add_handle(curl_multi, psRequest->m_curl_handle); |
202 | 1.21k | } |
203 | | |
204 | 1.14k | void *old_handler = CPLHTTPIgnoreSigPipe(); |
205 | 1.14k | int still_running; |
206 | 1.14k | do |
207 | 10.4k | { |
208 | 10.4k | CURLMcode mc; |
209 | 10.4k | do |
210 | 10.4k | { |
211 | 10.4k | mc = curl_multi_perform(curl_multi, &still_running); |
212 | 10.4k | } while (CURLM_CALL_MULTI_PERFORM == mc); |
213 | | |
214 | | // Pick up messages, clean up the completed ones, add more |
215 | 10.4k | int msgs_in_queue = 0; |
216 | 10.4k | do |
217 | 11.0k | { |
218 | 11.0k | CURLMsg *m = curl_multi_info_read(curl_multi, &msgs_in_queue); |
219 | 11.0k | if (m && (m->msg == CURLMSG_DONE)) |
220 | 2.75k | { |
221 | 2.75k | ProcessCurlErrors(m, pasRequest, nRequestCount); |
222 | | |
223 | 2.75k | curl_multi_remove_handle(curl_multi, m->easy_handle); |
224 | 2.75k | if (conn_i < nRequestCount) |
225 | 1.53k | { |
226 | 1.53k | auto psRequest = &pasRequest[conn_i]; |
227 | 1.53k | CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, |
228 | 1.53k | nRequestCount, pasRequest[conn_i].URL.c_str()); |
229 | 1.53k | curl_multi_add_handle(curl_multi, psRequest->m_curl_handle); |
230 | 1.53k | ++conn_i; |
231 | 1.53k | still_running = 1; // Still have request pending |
232 | 1.53k | } |
233 | 2.75k | } |
234 | 11.0k | } while (msgs_in_queue); |
235 | | |
236 | 10.4k | if (CURLM_OK == mc) |
237 | 10.4k | { |
238 | 10.4k | int numfds; |
239 | 10.4k | curl_multi_wait(curl_multi, nullptr, 0, 100, &numfds); |
240 | 10.4k | } |
241 | 10.4k | } while (still_running || conn_i != nRequestCount); |
242 | | |
243 | | // process any message still in queue |
244 | 1.14k | CURLMsg *msg; |
245 | 1.14k | int msgs_in_queue; |
246 | 1.14k | do |
247 | 1.14k | { |
248 | 1.14k | msg = curl_multi_info_read(curl_multi, &msgs_in_queue); |
249 | 1.14k | if (msg != nullptr) |
250 | 0 | { |
251 | 0 | if (msg->msg == CURLMSG_DONE) |
252 | 0 | { |
253 | 0 | ProcessCurlErrors(msg, pasRequest, nRequestCount); |
254 | 0 | } |
255 | 0 | } |
256 | 1.14k | } while (msg != nullptr); |
257 | | |
258 | 1.14k | CPLHTTPRestoreSigPipeHandler(old_handler); |
259 | | |
260 | 1.14k | if (conn_i != nRequestCount) |
261 | 0 | { // something gone really really wrong |
262 | | // oddly built libcurl or perhaps absence of network interface |
263 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
264 | 0 | "WMSHTTPFetchMulti(): conn_i != nRequestCount, this should " |
265 | 0 | "never happen ..."); |
266 | 0 | nRequestCount = conn_i; |
267 | 0 | ret = CE_Failure; |
268 | 0 | } |
269 | | |
270 | 3.90k | for (i = 0; i < nRequestCount; ++i) |
271 | 2.75k | { |
272 | 2.75k | WMSHTTPRequest *const psRequest = &pasRequest[i]; |
273 | | |
274 | 2.75k | long response_code; |
275 | 2.75k | curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_RESPONSE_CODE, |
276 | 2.75k | &response_code); |
277 | | // for local files, don't update the status code if one is already set |
278 | 2.75k | if (!(psRequest->nStatus != 0 && |
279 | 0 | STARTS_WITH(psRequest->URL.c_str(), "file://"))) |
280 | 2.75k | psRequest->nStatus = static_cast<int>(response_code); |
281 | | |
282 | 2.75k | char *content_type = nullptr; |
283 | 2.75k | curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_CONTENT_TYPE, |
284 | 2.75k | &content_type); |
285 | 2.75k | psRequest->ContentType = content_type ? content_type : ""; |
286 | | |
287 | 2.75k | if (psRequest->Error.empty()) |
288 | 2.75k | psRequest->Error = &psRequest->m_curl_error[0]; |
289 | | |
290 | | /* In the case of a file:// URL, curl will return a status == 0, so if |
291 | | * there's no */ |
292 | | /* error returned, patch the status code to be 200, as it would be for |
293 | | * http:// */ |
294 | 2.75k | if (psRequest->nStatus == 0 && psRequest->Error.empty() && |
295 | 0 | STARTS_WITH(psRequest->URL.c_str(), "file://")) |
296 | 0 | psRequest->nStatus = 200; |
297 | | |
298 | | // If there is an error with no error message, use the content if it is |
299 | | // text |
300 | 2.75k | if (psRequest->Error.empty() && psRequest->nStatus != 0 && |
301 | 274 | psRequest->nStatus != 200 && |
302 | 0 | strstr(psRequest->ContentType, "text") && |
303 | 0 | psRequest->pabyData != nullptr) |
304 | 0 | psRequest->Error = |
305 | 0 | reinterpret_cast<const char *>(psRequest->pabyData); |
306 | | |
307 | 2.75k | CPLDebug( |
308 | 2.75k | "HTTP", "Request [%d] %s : status = %d, type = %s, error = %s", i, |
309 | 2.75k | psRequest->URL.c_str(), psRequest->nStatus, |
310 | 2.75k | !psRequest->ContentType.empty() ? psRequest->ContentType.c_str() |
311 | 2.75k | : "(null)", |
312 | 2.75k | !psRequest->Error.empty() ? psRequest->Error.c_str() : "(null)"); |
313 | | |
314 | 2.75k | curl_multi_remove_handle(curl_multi, pasRequest->m_curl_handle); |
315 | 2.75k | } |
316 | | |
317 | 1.14k | curl_multi_cleanup(curl_multi); |
318 | | |
319 | 1.14k | return ret; |
320 | 1.14k | } |