/src/gdal/port/cpl_vsil_oss.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: CPL - Common Portability Library |
4 | | * Purpose: Implement VSI large file api for Alibaba Object Storage Service |
5 | | * Author: Even Rouault, even.rouault at spatialys.com |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2017-2018, Even Rouault <even.rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_port.h" |
14 | | #include "cpl_http.h" |
15 | | #include "cpl_minixml.h" |
16 | | #include "cpl_vsil_curl_priv.h" |
17 | | #include "cpl_vsil_curl_class.h" |
18 | | |
19 | | #include <errno.h> |
20 | | |
21 | | #include <algorithm> |
22 | | #include <set> |
23 | | #include <map> |
24 | | #include <memory> |
25 | | |
26 | | #include "cpl_alibaba_oss.h" |
27 | | |
28 | | #ifndef HAVE_CURL |
29 | | |
30 | | void VSIInstallOSSFileHandler(void) |
31 | | { |
32 | | // Not supported |
33 | | } |
34 | | |
35 | | #else |
36 | | |
37 | | //! @cond Doxygen_Suppress |
38 | | #ifndef DOXYGEN_SKIP |
39 | | |
40 | | #define ENABLE_DEBUG 0 |
41 | | |
42 | | namespace cpl |
43 | | { |
44 | | |
45 | | /************************************************************************/ |
46 | | /* VSIOSSFSHandler */ |
47 | | /************************************************************************/ |
48 | | |
49 | | class VSIOSSFSHandler final : public IVSIS3LikeFSHandlerWithMultipartUpload |
50 | | { |
51 | | CPL_DISALLOW_COPY_ASSIGN(VSIOSSFSHandler) |
52 | | |
53 | | protected: |
54 | | VSICurlHandle *CreateFileHandle(const char *pszFilename) override; |
55 | | std::string |
56 | | GetURLFromFilename(const std::string &osFilename) const override; |
57 | | |
58 | | const char *GetDebugKey() const override |
59 | 0 | { |
60 | 0 | return "OSS"; |
61 | 0 | } |
62 | | |
63 | | IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI, |
64 | | bool bAllowNoObject) override; |
65 | | |
66 | | std::string GetFSPrefix() const override |
67 | 144k | { |
68 | 144k | return "/vsioss/"; |
69 | 144k | } |
70 | | |
71 | | void ClearCache() override; |
72 | | |
73 | | VSIVirtualHandleUniquePtr |
74 | | CreateWriteHandle(const char *pszFilename, |
75 | | CSLConstList papszOptions) override; |
76 | | |
77 | | public: |
78 | 88 | VSIOSSFSHandler() = default; |
79 | | ~VSIOSSFSHandler() override; |
80 | | |
81 | | const char *GetOptions() override; |
82 | | |
83 | | char *GetSignedURL(const char *pszFilename, |
84 | | CSLConstList papszOptions) override; |
85 | | |
86 | | std::string |
87 | | GetStreamingFilename(const std::string &osFilename) const override |
88 | 0 | { |
89 | 0 | return osFilename; |
90 | 0 | } |
91 | | |
92 | | bool SupportsMultipartAbort() const override |
93 | 0 | { |
94 | 0 | return true; |
95 | 0 | } |
96 | | }; |
97 | | |
98 | | /************************************************************************/ |
99 | | /* VSIOSSHandle */ |
100 | | /************************************************************************/ |
101 | | |
102 | | class VSIOSSHandle final : public IVSIS3LikeHandle |
103 | | { |
104 | | CPL_DISALLOW_COPY_ASSIGN(VSIOSSHandle) |
105 | | |
106 | | VSIOSSHandleHelper *m_poHandleHelper = nullptr; |
107 | | |
108 | | protected: |
109 | | struct curl_slist * |
110 | | GetCurlHeaders(const std::string &osVerb, |
111 | | const struct curl_slist *psExistingHeaders) override; |
112 | | bool CanRestartOnError(const char *, const char *, bool) override; |
113 | | |
114 | | public: |
115 | | VSIOSSHandle(VSIOSSFSHandler *poFS, const char *pszFilename, |
116 | | VSIOSSHandleHelper *poHandleHelper); |
117 | | ~VSIOSSHandle() override; |
118 | | }; |
119 | | |
120 | | /************************************************************************/ |
121 | | /* ~VSIOSSFSHandler() */ |
122 | | /************************************************************************/ |
123 | | |
124 | | VSIOSSFSHandler::~VSIOSSFSHandler() |
125 | 0 | { |
126 | 0 | VSIOSSFSHandler::ClearCache(); |
127 | 0 | } |
128 | | |
129 | | /************************************************************************/ |
130 | | /* CreateWriteHandle() */ |
131 | | /************************************************************************/ |
132 | | |
133 | | VSIVirtualHandleUniquePtr |
134 | | VSIOSSFSHandler::CreateWriteHandle(const char *pszFilename, |
135 | | CSLConstList papszOptions) |
136 | 0 | { |
137 | 0 | auto poHandleHelper = |
138 | 0 | CreateHandleHelper(pszFilename + GetFSPrefix().size(), false); |
139 | 0 | if (poHandleHelper == nullptr) |
140 | 0 | return nullptr; |
141 | 0 | auto poHandle = std::make_unique<VSIMultipartWriteHandle>( |
142 | 0 | this, pszFilename, poHandleHelper, papszOptions); |
143 | 0 | if (!poHandle->IsOK()) |
144 | 0 | { |
145 | 0 | return nullptr; |
146 | 0 | } |
147 | 0 | return VSIVirtualHandleUniquePtr(poHandle.release()); |
148 | 0 | } |
149 | | |
150 | | /************************************************************************/ |
151 | | /* ClearCache() */ |
152 | | /************************************************************************/ |
153 | | |
154 | | void VSIOSSFSHandler::ClearCache() |
155 | 0 | { |
156 | 0 | VSICurlFilesystemHandlerBase::ClearCache(); |
157 | |
|
158 | 0 | VSIOSSUpdateParams::ClearCache(); |
159 | 0 | } |
160 | | |
161 | | /************************************************************************/ |
162 | | /* GetOptions() */ |
163 | | /************************************************************************/ |
164 | | |
165 | | const char *VSIOSSFSHandler::GetOptions() |
166 | 0 | { |
167 | 0 | static std::string osOptions( |
168 | 0 | std::string("<Options>") + |
169 | 0 | " <Option name='OSS_SECRET_ACCESS_KEY' type='string' " |
170 | 0 | "description='Secret access key. To use with OSS_ACCESS_KEY_ID'/>" |
171 | 0 | " <Option name='OSS_ACCESS_KEY_ID' type='string' " |
172 | 0 | "description='Access key id'/>" |
173 | 0 | " <Option name='OSS_ENDPOINT' type='string' " |
174 | 0 | "description='Default endpoint' default='oss-us-east-1.aliyuncs.com'/>" |
175 | 0 | " <Option name='VSIOSS_CHUNK_SIZE' type='int' " |
176 | 0 | "description='Size in MB for chunks of files that are uploaded. The" |
177 | 0 | "default value of 50 MB allows for files up to 500 GB each' " |
178 | 0 | "default='50' min='1' max='1000'/>" + |
179 | 0 | VSICurlFilesystemHandlerBase::GetOptionsStatic() + "</Options>"); |
180 | 0 | return osOptions.c_str(); |
181 | 0 | } |
182 | | |
183 | | /************************************************************************/ |
184 | | /* GetSignedURL() */ |
185 | | /************************************************************************/ |
186 | | |
187 | | char *VSIOSSFSHandler::GetSignedURL(const char *pszFilename, |
188 | | CSLConstList papszOptions) |
189 | 0 | { |
190 | 0 | if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str())) |
191 | 0 | return nullptr; |
192 | | |
193 | 0 | VSIOSSHandleHelper *poHandleHelper = VSIOSSHandleHelper::BuildFromURI( |
194 | 0 | pszFilename + GetFSPrefix().size(), GetFSPrefix().c_str(), false, |
195 | 0 | papszOptions); |
196 | 0 | if (poHandleHelper == nullptr) |
197 | 0 | { |
198 | 0 | return nullptr; |
199 | 0 | } |
200 | | |
201 | 0 | std::string osRet(poHandleHelper->GetSignedURL(papszOptions)); |
202 | |
|
203 | 0 | delete poHandleHelper; |
204 | 0 | return CPLStrdup(osRet.c_str()); |
205 | 0 | } |
206 | | |
207 | | /************************************************************************/ |
208 | | /* CreateFileHandle() */ |
209 | | /************************************************************************/ |
210 | | |
211 | | VSICurlHandle *VSIOSSFSHandler::CreateFileHandle(const char *pszFilename) |
212 | 8.49k | { |
213 | 8.49k | VSIOSSHandleHelper *poHandleHelper = VSIOSSHandleHelper::BuildFromURI( |
214 | 8.49k | pszFilename + GetFSPrefix().size(), GetFSPrefix().c_str(), false); |
215 | 8.49k | if (poHandleHelper) |
216 | 0 | { |
217 | 0 | return new VSIOSSHandle(this, pszFilename, poHandleHelper); |
218 | 0 | } |
219 | 8.49k | return nullptr; |
220 | 8.49k | } |
221 | | |
222 | | /************************************************************************/ |
223 | | /* GetURLFromFilename() */ |
224 | | /************************************************************************/ |
225 | | |
226 | | std::string |
227 | | VSIOSSFSHandler::GetURLFromFilename(const std::string &osFilename) const |
228 | 9.02k | { |
229 | 9.02k | const std::string osFilenameWithoutPrefix = |
230 | 9.02k | osFilename.substr(GetFSPrefix().size()); |
231 | | |
232 | 9.02k | auto poHandleHelper = |
233 | 9.02k | std::unique_ptr<VSIOSSHandleHelper>(VSIOSSHandleHelper::BuildFromURI( |
234 | 9.02k | osFilenameWithoutPrefix.c_str(), GetFSPrefix().c_str(), true)); |
235 | 9.02k | if (!poHandleHelper) |
236 | 9.02k | { |
237 | 9.02k | return std::string(); |
238 | 9.02k | } |
239 | | |
240 | 0 | std::string osBaseURL(poHandleHelper->GetURL()); |
241 | 0 | if (!osBaseURL.empty() && osBaseURL.back() == '/') |
242 | 0 | osBaseURL.pop_back(); |
243 | 0 | return osBaseURL; |
244 | 9.02k | } |
245 | | |
246 | | /************************************************************************/ |
247 | | /* CreateHandleHelper() */ |
248 | | /************************************************************************/ |
249 | | |
250 | | IVSIS3LikeHandleHelper *VSIOSSFSHandler::CreateHandleHelper(const char *pszURI, |
251 | | bool bAllowNoObject) |
252 | 525 | { |
253 | 525 | return VSIOSSHandleHelper::BuildFromURI(pszURI, GetFSPrefix().c_str(), |
254 | 525 | bAllowNoObject); |
255 | 525 | } |
256 | | |
257 | | /************************************************************************/ |
258 | | /* VSIOSSHandle() */ |
259 | | /************************************************************************/ |
260 | | |
261 | | VSIOSSHandle::VSIOSSHandle(VSIOSSFSHandler *poFSIn, const char *pszFilename, |
262 | | VSIOSSHandleHelper *poHandleHelper) |
263 | 0 | : IVSIS3LikeHandle(poFSIn, pszFilename, poHandleHelper->GetURL().c_str()), |
264 | 0 | m_poHandleHelper(poHandleHelper) |
265 | 0 | { |
266 | 0 | } |
267 | | |
268 | | /************************************************************************/ |
269 | | /* ~VSIOSSHandle() */ |
270 | | /************************************************************************/ |
271 | | |
272 | | VSIOSSHandle::~VSIOSSHandle() |
273 | 0 | { |
274 | 0 | delete m_poHandleHelper; |
275 | 0 | } |
276 | | |
277 | | /************************************************************************/ |
278 | | /* GetCurlHeaders() */ |
279 | | /************************************************************************/ |
280 | | |
281 | | struct curl_slist * |
282 | | VSIOSSHandle::GetCurlHeaders(const std::string &osVerb, |
283 | | const struct curl_slist *psExistingHeaders) |
284 | 0 | { |
285 | 0 | return m_poHandleHelper->GetCurlHeaders(osVerb, psExistingHeaders); |
286 | 0 | } |
287 | | |
288 | | /************************************************************************/ |
289 | | /* CanRestartOnError() */ |
290 | | /************************************************************************/ |
291 | | |
292 | | bool VSIOSSHandle::CanRestartOnError(const char *pszErrorMsg, |
293 | | const char *pszHeaders, bool bSetError) |
294 | 0 | { |
295 | 0 | if (m_poHandleHelper->CanRestartOnError(pszErrorMsg, pszHeaders, bSetError)) |
296 | 0 | { |
297 | 0 | SetURL(m_poHandleHelper->GetURL().c_str()); |
298 | 0 | return true; |
299 | 0 | } |
300 | 0 | return false; |
301 | 0 | } |
302 | | |
303 | | } /* end of namespace cpl */ |
304 | | |
305 | | #endif // DOXYGEN_SKIP |
306 | | //! @endcond |
307 | | |
308 | | /************************************************************************/ |
309 | | /* VSIInstallOSSFileHandler() */ |
310 | | /************************************************************************/ |
311 | | |
312 | | /*! |
313 | | \brief Install /vsioss/ Alibaba Cloud Object Storage Service (OSS) file |
314 | | system handler (requires libcurl) |
315 | | |
316 | | \verbatim embed:rst |
317 | | See :ref:`/vsioss/ documentation <vsioss>` |
318 | | \endverbatim |
319 | | |
320 | | @since GDAL 2.3 |
321 | | */ |
322 | | void VSIInstallOSSFileHandler(void) |
323 | 88 | { |
324 | 88 | VSIFileManager::InstallHandler("/vsioss/", new cpl::VSIOSSFSHandler); |
325 | 88 | } |
326 | | |
327 | | #endif /* HAVE_CURL */ |