1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Implement VSI large file api for Alibaba Object Storage Service
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2017-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #include "cpl_port.h"
30 #include "cpl_http.h"
31 #include "cpl_minixml.h"
32 #include "cpl_vsil_curl_priv.h"
33 #include "cpl_vsil_curl_class.h"
34
35 #include <errno.h>
36
37 #include <algorithm>
38 #include <set>
39 #include <map>
40 #include <memory>
41
42 #include "cpl_alibaba_oss.h"
43
44 CPL_CVSID("$Id: cpl_vsil_oss.cpp 86933038c3926cd4dc3ff37c431b317abb69e602 2021-03-27 23:20:49 +0100 Even Rouault $")
45
46 #ifndef HAVE_CURL
47
VSIInstallOSSFileHandler(void)48 void VSIInstallOSSFileHandler( void )
49 {
50 // Not supported
51 }
52
53 #else
54
55 //! @cond Doxygen_Suppress
56 #ifndef DOXYGEN_SKIP
57
58 #define ENABLE_DEBUG 0
59
60 namespace cpl {
61
62 /************************************************************************/
63 /* VSIOSSFSHandler */
64 /************************************************************************/
65
66 class VSIOSSFSHandler final : public IVSIS3LikeFSHandler
67 {
68 CPL_DISALLOW_COPY_ASSIGN(VSIOSSFSHandler)
69
70 std::map< CPLString, VSIOSSUpdateParams > oMapBucketsToOSSParams{};
71
72 protected:
73 VSICurlHandle* CreateFileHandle( const char* pszFilename ) override;
74 CPLString GetURLFromFilename( const CPLString& osFilename ) override;
75
76 const char* GetDebugKey() const override { return "OSS"; }
77
78 IVSIS3LikeHandleHelper* CreateHandleHelper(
79 const char* pszURI, bool bAllowNoObject) override;
80
81 CPLString GetFSPrefix() const override { return "/vsioss/"; }
82
83 void ClearCache() override;
84
85 public:
86 VSIOSSFSHandler() = default;
87 ~VSIOSSFSHandler() override;
88
89 VSIVirtualHandle *Open( const char *pszFilename,
90 const char *pszAccess,
91 bool bSetError,
92 CSLConstList papszOptions ) override;
93
94 const char* GetOptions() override;
95
96 void UpdateMapFromHandle(
97 IVSIS3LikeHandleHelper * poHandleHelper ) override;
98 void UpdateHandleFromMap(
99 IVSIS3LikeHandleHelper * poHandleHelper ) override;
100
101 char* GetSignedURL( const char* pszFilename, CSLConstList papszOptions ) override;
102 };
103
104 /************************************************************************/
105 /* VSIOSSHandle */
106 /************************************************************************/
107
108 class VSIOSSHandle final : public IVSIS3LikeHandle
109 {
110 CPL_DISALLOW_COPY_ASSIGN(VSIOSSHandle)
111
112 VSIOSSHandleHelper* m_poHandleHelper = nullptr;
113
114 protected:
115 struct curl_slist* GetCurlHeaders(
116 const CPLString& osVerb,
117 const struct curl_slist* psExistingHeaders ) override;
118 bool CanRestartOnError( const char*, const char*, bool ) override;
119
120 public:
121 VSIOSSHandle( VSIOSSFSHandler* poFS,
122 const char* pszFilename,
123 VSIOSSHandleHelper* poHandleHelper );
124 ~VSIOSSHandle() override;
125 };
126
127 /************************************************************************/
128 /* Open() */
129 /************************************************************************/
130
131 VSIVirtualHandle* VSIOSSFSHandler::Open( const char *pszFilename,
132 const char *pszAccess,
133 bool bSetError,
134 CSLConstList papszOptions )
135 {
136 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
137 return nullptr;
138
139 if( strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, 'a') != nullptr )
140 {
141 if( strchr(pszAccess, '+') != nullptr &&
142 !CPLTestBool(CPLGetConfigOption("CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE", "NO")) )
143 {
144 CPLError(CE_Failure, CPLE_AppDefined,
145 "w+ not supported for /vsioss, unless "
146 "CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE is set to YES");
147 errno = EACCES;
148 return nullptr;
149 }
150
151 VSIOSSHandleHelper* poHandleHelper =
152 VSIOSSHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
153 GetFSPrefix().c_str(), false);
154 if( poHandleHelper == nullptr )
155 return nullptr;
156 UpdateHandleFromMap(poHandleHelper);
157 VSIS3WriteHandle* poHandle =
158 new VSIS3WriteHandle(this, pszFilename, poHandleHelper, false, papszOptions);
159 if( !poHandle->IsOK() )
160 {
161 delete poHandle;
162 return nullptr;
163 }
164 if( strchr(pszAccess, '+') != nullptr)
165 {
166 return VSICreateUploadOnCloseFile(poHandle);
167 }
168 return poHandle;
169 }
170
171 return
172 VSICurlFilesystemHandler::Open(pszFilename, pszAccess, bSetError, papszOptions);
173 }
174
175 /************************************************************************/
176 /* ~VSIOSSFSHandler() */
177 /************************************************************************/
178
179 VSIOSSFSHandler::~VSIOSSFSHandler()
180 {
181 VSIOSSFSHandler::ClearCache();
182 }
183
184 /************************************************************************/
185 /* ClearCache() */
186 /************************************************************************/
187
188 void VSIOSSFSHandler::ClearCache()
189 {
190 VSICurlFilesystemHandler::ClearCache();
191
192 oMapBucketsToOSSParams.clear();
193 }
194
195 /************************************************************************/
196 /* GetOptions() */
197 /************************************************************************/
198
199 const char* VSIOSSFSHandler::GetOptions()
200 {
201 static CPLString osOptions(
202 CPLString("<Options>") +
203 " <Option name='OSS_SECRET_ACCESS_KEY' type='string' "
204 "description='Secret access key. To use with OSS_ACCESS_KEY_ID'/>"
205 " <Option name='OSS_ACCESS_KEY_ID' type='string' "
206 "description='Access key id'/>"
207 " <Option name='OSS_ENDPOINT' type='string' "
208 "description='Default endpoint' default='oss-us-east-1.aliyuncs.com'/>"
209 " <Option name='VSIOSS_CHUNK_SIZE' type='int' "
210 "description='Size in MB for chunks of files that are uploaded. The"
211 "default value of 50 MB allows for files up to 500 GB each' "
212 "default='50' min='1' max='1000'/>" +
213 VSICurlFilesystemHandler::GetOptionsStatic() +
214 "</Options>");
215 return osOptions.c_str();
216 }
217
218 /************************************************************************/
219 /* GetSignedURL() */
220 /************************************************************************/
221
222 char* VSIOSSFSHandler::GetSignedURL(const char* pszFilename, CSLConstList papszOptions )
223 {
224 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
225 return nullptr;
226
227 VSIOSSHandleHelper* poHandleHelper =
228 VSIOSSHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
229 GetFSPrefix().c_str(), false,
230 papszOptions);
231 if( poHandleHelper == nullptr )
232 {
233 return nullptr;
234 }
235
236 CPLString osRet(poHandleHelper->GetSignedURL(papszOptions));
237
238 delete poHandleHelper;
239 return CPLStrdup(osRet);
240 }
241
242 /************************************************************************/
243 /* CreateFileHandle() */
244 /************************************************************************/
245
246 VSICurlHandle* VSIOSSFSHandler::CreateFileHandle(const char* pszFilename)
247 {
248 VSIOSSHandleHelper* poHandleHelper =
249 VSIOSSHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
250 GetFSPrefix().c_str(), false);
251 if( poHandleHelper )
252 {
253 UpdateHandleFromMap(poHandleHelper);
254 return new VSIOSSHandle(this, pszFilename, poHandleHelper);
255 }
256 return nullptr;
257 }
258
259 /************************************************************************/
260 /* GetURLFromFilename() */
261 /************************************************************************/
262
263 CPLString VSIOSSFSHandler::GetURLFromFilename( const CPLString& osFilename )
264 {
265 CPLString osFilenameWithoutPrefix = osFilename.substr(GetFSPrefix().size());
266
267 VSIOSSHandleHelper* poHandleHelper =
268 VSIOSSHandleHelper::BuildFromURI(osFilenameWithoutPrefix,
269 GetFSPrefix().c_str(), true);
270 if( poHandleHelper == nullptr )
271 {
272 return "";
273 }
274 UpdateHandleFromMap(poHandleHelper);
275 CPLString osBaseURL(poHandleHelper->GetURL());
276 if( !osBaseURL.empty() && osBaseURL.back() == '/' )
277 osBaseURL.resize(osBaseURL.size()-1);
278 delete poHandleHelper;
279
280 return osBaseURL;
281 }
282
283 /************************************************************************/
284 /* CreateHandleHelper() */
285 /************************************************************************/
286
287 IVSIS3LikeHandleHelper* VSIOSSFSHandler::CreateHandleHelper(const char* pszURI,
288 bool bAllowNoObject)
289 {
290 return VSIOSSHandleHelper::BuildFromURI(
291 pszURI, GetFSPrefix().c_str(), bAllowNoObject);
292 }
293
294 /************************************************************************/
295 /* UpdateMapFromHandle() */
296 /************************************************************************/
297
298 void VSIOSSFSHandler::UpdateMapFromHandle( IVSIS3LikeHandleHelper * poHandleHelper )
299 {
300 CPLMutexHolder oHolder( &hMutex );
301
302 VSIOSSHandleHelper * poOSSHandleHelper =
303 cpl::down_cast<VSIOSSHandleHelper *>(poHandleHelper);
304 oMapBucketsToOSSParams[ poOSSHandleHelper->GetBucket() ] =
305 VSIOSSUpdateParams ( poOSSHandleHelper );
306 }
307
308 /************************************************************************/
309 /* UpdateHandleFromMap() */
310 /************************************************************************/
311
312 void VSIOSSFSHandler::UpdateHandleFromMap( IVSIS3LikeHandleHelper * poHandleHelper )
313 {
314 CPLMutexHolder oHolder( &hMutex );
315
316 VSIOSSHandleHelper * poOSSHandleHelper =
317 cpl::down_cast<VSIOSSHandleHelper *>(poHandleHelper);
318 std::map< CPLString, VSIOSSUpdateParams>::iterator oIter =
319 oMapBucketsToOSSParams.find(poOSSHandleHelper->GetBucket());
320 if( oIter != oMapBucketsToOSSParams.end() )
321 {
322 oIter->second.UpdateHandlerHelper(poOSSHandleHelper);
323 }
324 }
325
326 /************************************************************************/
327 /* VSIOSSHandle() */
328 /************************************************************************/
329
330 VSIOSSHandle::VSIOSSHandle( VSIOSSFSHandler* poFSIn,
331 const char* pszFilename,
332 VSIOSSHandleHelper* poHandleHelper ) :
333 IVSIS3LikeHandle(poFSIn, pszFilename, poHandleHelper->GetURL()),
334 m_poHandleHelper(poHandleHelper)
335 {
336 }
337
338 /************************************************************************/
339 /* ~VSIOSSHandle() */
340 /************************************************************************/
341
342 VSIOSSHandle::~VSIOSSHandle()
343 {
344 delete m_poHandleHelper;
345 }
346
347 /************************************************************************/
348 /* GetCurlHeaders() */
349 /************************************************************************/
350
351 struct curl_slist* VSIOSSHandle::GetCurlHeaders( const CPLString& osVerb,
352 const struct curl_slist* psExistingHeaders )
353 {
354 return m_poHandleHelper->GetCurlHeaders(osVerb, psExistingHeaders);
355 }
356
357 /************************************************************************/
358 /* CanRestartOnError() */
359 /************************************************************************/
360
361 bool VSIOSSHandle::CanRestartOnError(const char* pszErrorMsg,
362 const char* pszHeaders, bool bSetError)
363 {
364 if( m_poHandleHelper->CanRestartOnError(pszErrorMsg, pszHeaders,
365 bSetError, nullptr) )
366 {
367 static_cast<VSIOSSFSHandler *>(poFS)->
368 UpdateMapFromHandle(m_poHandleHelper);
369
370 SetURL(m_poHandleHelper->GetURL());
371 return true;
372 }
373 return false;
374 }
375
376 } /* end of namespace cpl */
377
378
379 #endif // DOXYGEN_SKIP
380 //! @endcond
381
382 /************************************************************************/
383 /* VSIInstallOSSFileHandler() */
384 /************************************************************************/
385
386 /**
387 * \brief Install /vsioss/ Alibaba Cloud Object Storage Service (OSS) file
388 * system handler (requires libcurl)
389 *
390 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsioss">/vsioss/ documentation</a>
391 *
392 * @since GDAL 2.3
393 */
394 void VSIInstallOSSFileHandler( void )
395 {
396 VSIFileManager::InstallHandler( "/vsioss/", new cpl::VSIOSSFSHandler );
397 }
398
399 #endif /* HAVE_CURL */
400