/src/gdal/port/cpl_vsil_oss.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: CPL - Common Portability Library |
4 | | * Purpose: Implement VSI large file api for Alibaba Object Storage Service |
5 | | * Author: Even Rouault, even.rouault at spatialys.com |
6 | | * |
7 | | ****************************************************************************** |
8 | | * Copyright (c) 2017-2018, Even Rouault <even.rouault at spatialys.com> |
9 | | * |
10 | | * SPDX-License-Identifier: MIT |
11 | | ****************************************************************************/ |
12 | | |
13 | | #include "cpl_port.h" |
14 | | #include "cpl_http.h" |
15 | | #include "cpl_minixml.h" |
16 | | #include "cpl_vsil_curl_priv.h" |
17 | | #include "cpl_vsil_curl_class.h" |
18 | | |
19 | | #include <errno.h> |
20 | | |
21 | | #include <algorithm> |
22 | | #include <set> |
23 | | #include <map> |
24 | | #include <memory> |
25 | | |
26 | | #include "cpl_alibaba_oss.h" |
27 | | |
28 | | #ifndef HAVE_CURL |
29 | | |
30 | | void VSIInstallOSSFileHandler(void) |
31 | | { |
32 | | // Not supported |
33 | | } |
34 | | |
35 | | #else |
36 | | |
37 | | //! @cond Doxygen_Suppress |
38 | | #ifndef DOXYGEN_SKIP |
39 | | |
40 | | #define ENABLE_DEBUG 0 |
41 | | |
42 | | namespace cpl |
43 | | { |
44 | | |
45 | | /************************************************************************/ |
46 | | /* VSIOSSFSHandler */ |
47 | | /************************************************************************/ |
48 | | |
49 | | class VSIOSSFSHandler final : public IVSIS3LikeFSHandlerWithMultipartUpload |
50 | | { |
51 | | CPL_DISALLOW_COPY_ASSIGN(VSIOSSFSHandler) |
52 | | |
53 | | protected: |
54 | | VSICurlHandle *CreateFileHandle(const char *pszFilename) override; |
55 | | std::string |
56 | | GetURLFromFilename(const std::string &osFilename) const override; |
57 | | |
58 | | const char *GetDebugKey() const override |
59 | 0 | { |
60 | 0 | return "OSS"; |
61 | 0 | } |
62 | | |
63 | | IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI, |
64 | | bool bAllowNoObject) override; |
65 | | |
66 | | std::string GetFSPrefix() const override |
67 | 19.0k | { |
68 | 19.0k | return "/vsioss/"; |
69 | 19.0k | } |
70 | | |
71 | | void ClearCache() override; |
72 | | |
73 | | VSIVirtualHandleUniquePtr |
74 | | CreateWriteHandle(const char *pszFilename, |
75 | | CSLConstList papszOptions) override; |
76 | | |
77 | | public: |
78 | 83 | VSIOSSFSHandler() = default; |
79 | | ~VSIOSSFSHandler() override; |
80 | | |
81 | | const char *GetOptions() override; |
82 | | |
83 | | char *GetSignedURL(const char *pszFilename, |
84 | | CSLConstList papszOptions) override; |
85 | | |
86 | | std::string |
87 | | GetStreamingFilename(const std::string &osFilename) const override |
88 | 0 | { |
89 | 0 | return osFilename; |
90 | 0 | } |
91 | | |
92 | | bool SupportsMultipartAbort() const override |
93 | 0 | { |
94 | 0 | return true; |
95 | 0 | } |
96 | | }; |
97 | | |
98 | | /************************************************************************/ |
99 | | /* VSIOSSHandle */ |
100 | | /************************************************************************/ |
101 | | |
102 | | class VSIOSSHandle final : public IVSIS3LikeHandle |
103 | | { |
104 | | CPL_DISALLOW_COPY_ASSIGN(VSIOSSHandle) |
105 | | |
106 | | VSIOSSHandleHelper *m_poHandleHelper = nullptr; |
107 | | |
108 | | protected: |
109 | | struct curl_slist *GetCurlHeaders(const std::string &osVerb, |
110 | | struct curl_slist *psHeaders) override; |
111 | | bool CanRestartOnError(const char *, const char *, bool) override; |
112 | | |
113 | | public: |
114 | | VSIOSSHandle(VSIOSSFSHandler *poFS, const char *pszFilename, |
115 | | VSIOSSHandleHelper *poHandleHelper); |
116 | | ~VSIOSSHandle() override; |
117 | | }; |
118 | | |
119 | | /************************************************************************/ |
120 | | /* ~VSIOSSFSHandler() */ |
121 | | /************************************************************************/ |
122 | | |
123 | | VSIOSSFSHandler::~VSIOSSFSHandler() |
124 | 0 | { |
125 | 0 | VSIOSSFSHandler::ClearCache(); |
126 | 0 | } |
127 | | |
128 | | /************************************************************************/ |
129 | | /* CreateWriteHandle() */ |
130 | | /************************************************************************/ |
131 | | |
132 | | VSIVirtualHandleUniquePtr |
133 | | VSIOSSFSHandler::CreateWriteHandle(const char *pszFilename, |
134 | | CSLConstList papszOptions) |
135 | 0 | { |
136 | 0 | auto poHandleHelper = |
137 | 0 | CreateHandleHelper(pszFilename + GetFSPrefix().size(), false); |
138 | 0 | if (poHandleHelper == nullptr) |
139 | 0 | return nullptr; |
140 | 0 | auto poHandle = std::make_unique<VSIMultipartWriteHandle>( |
141 | 0 | this, pszFilename, poHandleHelper, papszOptions); |
142 | 0 | if (!poHandle->IsOK()) |
143 | 0 | { |
144 | 0 | return nullptr; |
145 | 0 | } |
146 | 0 | return VSIVirtualHandleUniquePtr(poHandle.release()); |
147 | 0 | } |
148 | | |
149 | | /************************************************************************/ |
150 | | /* ClearCache() */ |
151 | | /************************************************************************/ |
152 | | |
153 | | void VSIOSSFSHandler::ClearCache() |
154 | 0 | { |
155 | 0 | VSICurlFilesystemHandlerBase::ClearCache(); |
156 | |
|
157 | 0 | VSIOSSUpdateParams::ClearCache(); |
158 | 0 | } |
159 | | |
160 | | /************************************************************************/ |
161 | | /* GetOptions() */ |
162 | | /************************************************************************/ |
163 | | |
164 | | const char *VSIOSSFSHandler::GetOptions() |
165 | 0 | { |
166 | 0 | static std::string osOptions( |
167 | 0 | std::string("<Options>") + |
168 | 0 | " <Option name='OSS_SECRET_ACCESS_KEY' type='string' " |
169 | 0 | "description='Secret access key. To use with OSS_ACCESS_KEY_ID'/>" |
170 | 0 | " <Option name='OSS_ACCESS_KEY_ID' type='string' " |
171 | 0 | "description='Access key id'/>" |
172 | 0 | " <Option name='OSS_ENDPOINT' type='string' " |
173 | 0 | "description='Default endpoint' default='oss-us-east-1.aliyuncs.com'/>" |
174 | 0 | " <Option name='VSIOSS_CHUNK_SIZE' type='int' " |
175 | 0 | "description='Size in MB for chunks of files that are uploaded. The" |
176 | 0 | "default value of 50 MB allows for files up to 500 GB each' " |
177 | 0 | "default='50' min='1' max='1000'/>" + |
178 | 0 | VSICurlFilesystemHandlerBase::GetOptionsStatic() + "</Options>"); |
179 | 0 | return osOptions.c_str(); |
180 | 0 | } |
181 | | |
182 | | /************************************************************************/ |
183 | | /* GetSignedURL() */ |
184 | | /************************************************************************/ |
185 | | |
186 | | char *VSIOSSFSHandler::GetSignedURL(const char *pszFilename, |
187 | | CSLConstList papszOptions) |
188 | 0 | { |
189 | 0 | if (!STARTS_WITH_CI(pszFilename, GetFSPrefix().c_str())) |
190 | 0 | return nullptr; |
191 | | |
192 | 0 | VSIOSSHandleHelper *poHandleHelper = VSIOSSHandleHelper::BuildFromURI( |
193 | 0 | pszFilename + GetFSPrefix().size(), GetFSPrefix().c_str(), false, |
194 | 0 | papszOptions); |
195 | 0 | if (poHandleHelper == nullptr) |
196 | 0 | { |
197 | 0 | return nullptr; |
198 | 0 | } |
199 | | |
200 | 0 | std::string osRet(poHandleHelper->GetSignedURL(papszOptions)); |
201 | |
|
202 | 0 | delete poHandleHelper; |
203 | 0 | return CPLStrdup(osRet.c_str()); |
204 | 0 | } |
205 | | |
206 | | /************************************************************************/ |
207 | | /* CreateFileHandle() */ |
208 | | /************************************************************************/ |
209 | | |
210 | | VSICurlHandle *VSIOSSFSHandler::CreateFileHandle(const char *pszFilename) |
211 | 1.07k | { |
212 | 1.07k | VSIOSSHandleHelper *poHandleHelper = VSIOSSHandleHelper::BuildFromURI( |
213 | 1.07k | pszFilename + GetFSPrefix().size(), GetFSPrefix().c_str(), false); |
214 | 1.07k | if (poHandleHelper) |
215 | 0 | { |
216 | 0 | return new VSIOSSHandle(this, pszFilename, poHandleHelper); |
217 | 0 | } |
218 | 1.07k | return nullptr; |
219 | 1.07k | } |
220 | | |
221 | | /************************************************************************/ |
222 | | /* GetURLFromFilename() */ |
223 | | /************************************************************************/ |
224 | | |
225 | | std::string |
226 | | VSIOSSFSHandler::GetURLFromFilename(const std::string &osFilename) const |
227 | 1.07k | { |
228 | 1.07k | const std::string osFilenameWithoutPrefix = |
229 | 1.07k | osFilename.substr(GetFSPrefix().size()); |
230 | | |
231 | 1.07k | auto poHandleHelper = |
232 | 1.07k | std::unique_ptr<VSIOSSHandleHelper>(VSIOSSHandleHelper::BuildFromURI( |
233 | 1.07k | osFilenameWithoutPrefix.c_str(), GetFSPrefix().c_str(), true)); |
234 | 1.07k | if (!poHandleHelper) |
235 | 1.07k | { |
236 | 1.07k | return std::string(); |
237 | 1.07k | } |
238 | | |
239 | 0 | std::string osBaseURL(poHandleHelper->GetURL()); |
240 | 0 | if (!osBaseURL.empty() && osBaseURL.back() == '/') |
241 | 0 | osBaseURL.pop_back(); |
242 | 0 | return osBaseURL; |
243 | 1.07k | } |
244 | | |
245 | | /************************************************************************/ |
246 | | /* CreateHandleHelper() */ |
247 | | /************************************************************************/ |
248 | | |
249 | | IVSIS3LikeHandleHelper *VSIOSSFSHandler::CreateHandleHelper(const char *pszURI, |
250 | | bool bAllowNoObject) |
251 | 141 | { |
252 | 141 | return VSIOSSHandleHelper::BuildFromURI(pszURI, GetFSPrefix().c_str(), |
253 | 141 | bAllowNoObject); |
254 | 141 | } |
255 | | |
256 | | /************************************************************************/ |
257 | | /* VSIOSSHandle() */ |
258 | | /************************************************************************/ |
259 | | |
260 | | VSIOSSHandle::VSIOSSHandle(VSIOSSFSHandler *poFSIn, const char *pszFilename, |
261 | | VSIOSSHandleHelper *poHandleHelper) |
262 | 0 | : IVSIS3LikeHandle(poFSIn, pszFilename, poHandleHelper->GetURL().c_str()), |
263 | 0 | m_poHandleHelper(poHandleHelper) |
264 | 0 | { |
265 | 0 | } |
266 | | |
267 | | /************************************************************************/ |
268 | | /* ~VSIOSSHandle() */ |
269 | | /************************************************************************/ |
270 | | |
271 | | VSIOSSHandle::~VSIOSSHandle() |
272 | 0 | { |
273 | 0 | delete m_poHandleHelper; |
274 | 0 | } |
275 | | |
276 | | /************************************************************************/ |
277 | | /* GetCurlHeaders() */ |
278 | | /************************************************************************/ |
279 | | |
280 | | struct curl_slist *VSIOSSHandle::GetCurlHeaders(const std::string &osVerb, |
281 | | struct curl_slist *psHeaders) |
282 | 0 | { |
283 | 0 | return m_poHandleHelper->GetCurlHeaders(osVerb, psHeaders); |
284 | 0 | } |
285 | | |
286 | | /************************************************************************/ |
287 | | /* CanRestartOnError() */ |
288 | | /************************************************************************/ |
289 | | |
290 | | bool VSIOSSHandle::CanRestartOnError(const char *pszErrorMsg, |
291 | | const char *pszHeaders, bool bSetError) |
292 | 0 | { |
293 | 0 | if (m_poHandleHelper->CanRestartOnError(pszErrorMsg, pszHeaders, bSetError)) |
294 | 0 | { |
295 | 0 | SetURL(m_poHandleHelper->GetURL().c_str()); |
296 | 0 | return true; |
297 | 0 | } |
298 | 0 | return false; |
299 | 0 | } |
300 | | |
301 | | } /* end of namespace cpl */ |
302 | | |
303 | | #endif // DOXYGEN_SKIP |
304 | | //! @endcond |
305 | | |
306 | | /************************************************************************/ |
307 | | /* VSIInstallOSSFileHandler() */ |
308 | | /************************************************************************/ |
309 | | |
310 | | /*! |
311 | | \brief Install /vsioss/ Alibaba Cloud Object Storage Service (OSS) file |
312 | | system handler (requires libcurl) |
313 | | |
314 | | \verbatim embed:rst |
315 | | See :ref:`/vsioss/ documentation <vsioss>` |
316 | | \endverbatim |
317 | | |
318 | | */ |
319 | | void VSIInstallOSSFileHandler(void) |
320 | 83 | { |
321 | 83 | VSIFileManager::InstallHandler("/vsioss/", |
322 | 83 | std::make_shared<cpl::VSIOSSFSHandler>()); |
323 | 83 | } |
324 | | |
325 | | #endif /* HAVE_CURL */ |