1#coding: utf-8 2 3import http.client 4import urllib.request, urllib.error, urllib.parse 5import json 6from seafobj.backends.base import AbstractObjStore 7from seafobj.exceptions import GetObjectError, SwiftAuthenticateError 8 9class SwiftConf(object): 10 def __init__(self, user_name, password, container, auth_host, auth_ver, tenant, use_https, region, domain): 11 self.user_name = user_name 12 self.password = password 13 self.container = container 14 self.auth_host = auth_host 15 self.auth_ver = auth_ver 16 self.tenant = tenant 17 self.use_https = use_https 18 self.region = region 19 self.domain = domain 20 21class SeafSwiftClient(object): 22 MAX_RETRY = 2 23 24 def __init__(self, swift_conf): 25 self.swift_conf = swift_conf 26 self.token = None 27 self.storage_url = None 28 if swift_conf.use_https: 29 self.base_url = 'https://%s' % swift_conf.auth_host 30 else: 31 self.base_url = 'http://%s' % swift_conf.auth_host 32 33 def authenticated(self): 34 if self.token is not None and self.storage_url is not None: 35 return True 36 return False 37 38 def authenticate(self): 39 if self.swift_conf.auth_ver == 'v1.0': 40 self.authenticate_v1() 41 elif self.swift_conf.auth_ver == "v2.0": 42 self.authenticate_v2() 43 else: 44 self.authenticate_v3() 45 46 def authenticate_v1(self): 47 url = '%s/auth/%s' % (self.base_url, self.swift_conf.auth_ver) 48 49 hdr = {'X-Storage-User': self.swift_conf.user_name, 50 'X-Storage-Pass': self.swift_conf.password} 51 req = urllib.request.Request(url, None, hdr) 52 try: 53 resp = urllib.request.urlopen(req) 54 except urllib.error.HTTPError as e: 55 raise SwiftAuthenticateError('[swift] Failed to authenticate: %d.' % 56 (SeafSwiftClient.MAX_RETRY, e.getcode())) 57 except urllib.error.URLError as e: 58 raise SwiftAuthenticateError('[swift] Failed to authenticate: %s.' % 59 (SeafSwiftClient.MAX_RETRY, e.reason)) 60 61 ret_code = resp.getcode() 62 if ret_code == http.client.OK or ret_code == http.client.NON_AUTHORITATIVE_INFORMATION: 63 self.storage_url = resp.headers['x-storage-url'] 64 self.token = resp.headers['x-auth-token'] 65 else: 66 raise SwiftAuthenticateError('[swift] Unexpected code when authenticate: %d' % 67 ret_code) 68 if self.storage_url == None: 69 raise SwiftAuthenticateError('[swift] Failed to authenticate.') 70 71 def authenticate_v2(self): 72 url = '%s/%s/tokens' % (self.base_url, self.swift_conf.auth_ver) 73 hdr = {'Content-Type': 'application/json'} 74 auth_data = {'auth': {'passwordCredentials': {'username': self.swift_conf.user_name, 75 'password': self.swift_conf.password}, 76 'tenantName': self.swift_conf.tenant}} 77 78 req = urllib.request.Request(url, json.dumps(auth_data).encode('utf8'), hdr) 79 try: 80 resp = urllib.request.urlopen(req) 81 except urllib.error.HTTPError as e: 82 raise SwiftAuthenticateError('[swift] Failed to authenticate: %d.' % 83 (SeafSwiftClient.MAX_RETRY, e.getcode())) 84 except urllib.error.URLError as e: 85 raise SwiftAuthenticateError('[swift] Failed to authenticate: %s.' % 86 (SeafSwiftClient.MAX_RETRY, e.reason)) 87 88 ret_code = resp.getcode() 89 ret_data = resp.read() 90 91 if ret_code == http.client.OK or ret_code == http.client.NON_AUTHORITATIVE_INFORMATION: 92 data_json = json.loads(ret_data) 93 self.token = data_json['access']['token']['id'] 94 catalogs = data_json['access']['serviceCatalog'] 95 for catalog in catalogs: 96 if catalog['type'] == 'object-store': 97 if self.swift_conf.region: 98 for endpoint in catalog['endpoints']: 99 if endpoint['region'] == self.swift_conf.region: 100 self.storage_url = endpoint['publicURL'] 101 return 102 else: 103 self.storage_url = catalog['endpoints'][0]['publicURL'] 104 return 105 else: 106 raise SwiftAuthenticateError('[swift] Unexpected code when authenticate: %d' % 107 ret_code) 108 if self.swift_conf.region and self.storage_url == None: 109 raise SwiftAuthenticateError('[swift] Region \'%s\' not found.' % self.swift_conf.region) 110 111 def authenticate_v3(self): 112 url = '%s/v3/auth/tokens' % self.base_url 113 hdr = {'Content-Type': 'application/json'} 114 115 if self.swift_conf.domain: 116 domain_value = self.swift_conf.domain 117 else: 118 domain_value = 'default' 119 auth_data = {'auth': {'identity': {'methods': ['password'], 120 'password': {'user': {'domain': {'id': domain_value}, 121 'name': self.swift_conf.user_name, 122 'password': self.swift_conf.password}}}, 123 'scope': {'project': {'domain': {'id': domain_value}, 124 'name': self.swift_conf.tenant}}}} 125 126 req = urllib.request.Request(url, json.dumps(auth_data).encode('utf8'), hdr) 127 try: 128 resp = urllib.request.urlopen(req) 129 except urllib.error.HTTPError as e: 130 raise SwiftAuthenticateError('[swift] Failed to authenticate: %d.' % 131 (SeafSwiftClient.MAX_RETRY, e.getcode())) 132 except urllib.error.URLError as e: 133 raise SwiftAuthenticateError('[swift] Failed to authenticate: %s.' % 134 (SeafSwiftClient.MAX_RETRY, e.reason)) 135 136 ret_code = resp.getcode() 137 ret_data = resp.read() 138 139 if ret_code == http.client.OK or ret_code == http.client.NON_AUTHORITATIVE_INFORMATION or ret_code == http.client.CREATED: 140 self.token = resp.headers['X-Subject-Token'] 141 data_json = json.loads(ret_data) 142 catalogs = data_json['token']['catalog'] 143 for catalog in catalogs: 144 if catalog['type'] == 'object-store': 145 if self.swift_conf.region: 146 for endpoint in catalog['endpoints']: 147 if endpoint['region'] == self.swift_conf.region and endpoint['interface'] == 'public': 148 self.storage_url = endpoint['url'] 149 return 150 else: 151 for endpoint in catalog['endpoints']: 152 if endpoint ['interface'] == 'public': 153 self.storage_url = endpoint['url'] 154 return 155 else: 156 raise SwiftAuthenticateError('[swift] Unexpected code when authenticate: %d' % 157 ret_code) 158 if self.swift_conf.region and self.storage_url == None: 159 raise SwiftAuthenticateError('[swift] Region \'%s\' not found.' % self.swift_conf.region) 160 161 def read_object_content(self, obj_id): 162 i = 0 163 while i <= SeafSwiftClient.MAX_RETRY: 164 if not self.authenticated(): 165 self.authenticate() 166 167 url = '%s/%s/%s' % (self.storage_url, self.swift_conf.container, obj_id) 168 hdr = {'X-Auth-Token': self.token} 169 req = urllib.request.Request(url, headers=hdr) 170 try: 171 resp = urllib.request.urlopen(req) 172 except urllib.error.HTTPError as e: 173 err_code = e.getcode() 174 if err_code == http.client.UNAUTHORIZED: 175 # Reset token and storage_url 176 self.token = None 177 self.storage_url = None 178 i += 1 179 continue 180 else: 181 raise GetObjectError('[swift] Failed to read %s: %d' % (obj_id, err_code)) 182 except urllib.error.URLError as e: 183 raise GetObjectError('[swift] Failed to read %s: %s' % (obj_id, e.reason)) 184 185 ret_code = resp.getcode() 186 ret_data = resp.read() 187 188 if ret_code == http.client.OK: 189 return ret_data 190 else: 191 raise GetObjectError('[swift] Unexpected code when read %s: %d' % 192 (obj_id, ret_code)) 193 raise GetObjectError('[swift] Failed to read %s: quit after %d unauthorized retries.' % 194 (obj_id, SeafSwiftClient.MAX_RETRY)) 195 196 def list_objs(self): 197 i = 0 198 while i <= SeafSwiftClient.MAX_RETRY: 199 if not self.authenticated(): 200 self.authenticate() 201 202 url = '%s/%s' % (self.storage_url, self.swift_conf.container) 203 hdr = {'X-Auth-Token': self.token} 204 req = urllib.request.Request(url, headers=hdr) 205 try: 206 resp = urllib.request.urlopen(req) 207 except urllib.error.HTTPError as e: 208 err_code = e.getcode() 209 if err_code == http.client.UNAUTHORIZED: 210 # Reset token and storage_url 211 self.token = None 212 self.storage_url = None 213 i += 1 214 continue 215 else: 216 raise GetObjectError('[swift] Failed to list objs %s' % err_code) 217 except urllib.error.URLError as e: 218 raise GetObjectError('[swift] Failed to list objs' % e.reason) 219 220 ret_code = resp.getcode() 221 ret_data = resp.read() 222 223 if ret_code == http.client.OK: 224 return ret_data 225 else: 226 raise GetObjectError('[swift] Unexpected code when list objs %s' % ret_code) 227 raise GetObjectError('[swift] Failed to list objs: quit after %d unauthorized retries.' % 228 SeafSwiftClient.MAX_RETRY) 229 230 def remove_obj(self, obj_id): 231 i = 0 232 while i <= SeafSwiftClient.MAX_RETRY: 233 if not self.authenticated(): 234 self.authenticate() 235 236 url = '%s/%s/%s' % (self.storage_url, self.swift_conf.container, obj_id) 237 hdr = {'X-Auth-Token': self.token} 238 req = urllib.request.Request(url, headers=hdr, method='DELETE') 239 try: 240 resp = urllib.request.urlopen(req) 241 except urllib.error.HTTPError as e: 242 err_code = e.getcode() 243 if err_code == http.client.UNAUTHORIZED: 244 # Reset token and storage_url 245 self.token = None 246 self.storage_url = None 247 i += 1 248 continue 249 else: 250 raise GetObjectError('[swift] Failed to remove %s: %d' % (obj_id, err_code)) 251 except urllib.error.URLError as e: 252 raise GetObjectError('[swift] Failed to remove %s: %s' % (obj_id, e.reason)) 253 254 ret_code = resp.getcode() 255 return ret_code 256 raise GetObjectError('[swift] Failed to remove obj %s: quit after %d unauthorized retries.' % 257 (obj_id, SeafSwiftClient.MAX_RETRY)) 258 259 def stat_obj(self, obj_id): 260 i = 0 261 while i <= SeafSwiftClient.MAX_RETRY: 262 if not self.authenticated(): 263 self.authenticate() 264 265 url = '%s/%s/%s' % (self.storage_url, self.swift_conf.container, obj_id) 266 hdr = {'X-Auth-Token': self.token} 267 req = urllib.request.Request(url, headers=hdr, method='HEAD') 268 try: 269 resp = urllib.request.urlopen(req) 270 except urllib.error.HTTPError as e: 271 err_code = e.getcode() 272 if err_code == http.client.UNAUTHORIZED: 273 # Reset token and storage_url 274 self.token = None 275 self.storage_url = None 276 i += 1 277 continue 278 else: 279 raise GetObjectError('[swift] Failed to remove %s: %d' % (obj_id, err_code)) 280 except urllib.error.URLError as e: 281 raise GetObjectError('[swift] Failed to remove %s: %s' % (obj_id, e.reason)) 282 283 for k, v in resp.headers.items(): 284 if k == 'Content-Length': 285 return int(v) 286 raise GetObjectError('[swift] Failed to remove obj %s: quit after %d unauthorized retries.' % 287 (obj_id, SeafSwiftClient.MAX_RETRY)) 288 289class SeafObjStoreSwift(AbstractObjStore): 290 '''Swift backend for seafile objecs''' 291 def __init__(self, compressed, swift_conf, crypto=None): 292 AbstractObjStore.__init__(self, compressed, crypto) 293 self.swift_client = SeafSwiftClient(swift_conf) 294 295 def read_obj_raw(self, repo_id, version, obj_id): 296 real_obj_id = '%s/%s' % (repo_id, obj_id) 297 data = self.swift_client.read_object_content(real_obj_id) 298 return data 299 300 def get_name(self): 301 return 'Swift storage backend' 302 303 def list_objs(self, repo_id=None): 304 objs = self.swift_client.list_objs().decode('utf8').split('\n') 305 if repo_id: 306 for obj in objs: 307 tokens = obj.split('/') 308 if tokens[0] == repo_id and len(tokens) == 2: 309 yield [tokens[0], tokens[1], 0] 310 else: 311 for obj in objs: 312 tokens = obj.split('/') 313 if len(tokens) == 2: 314 yield [tokens[0], tokens[1], 0] 315 316 def remove_obj(self, repo_id, obj_id): 317 key = '%s/%s' % (repo_id, obj_id) 318 319 self.swift_client.remove_obj(key) 320 321 def stat_raw(self, repo_id, obj_id): 322 key = '%s/%s' % (repo_id, obj_id) 323 324 return self.swift_client.stat_obj(key) 325