1# test_swift.py -- Unittests for the Swift backend. 2# Copyright (C) 2013 eNovance SAS <licensing@enovance.com> 3# 4# Author: Fabien Boucher <fabien.boucher@enovance.com> 5# 6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU 7# General Public License as public by the Free Software Foundation; version 2.0 8# or (at your option) any later version. You can redistribute it and/or 9# modify it under the terms of either of these two licenses. 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17# You should have received a copy of the licenses; if not, see 18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License 19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache 20# License, Version 2.0. 21# 22 23"""Tests for dulwich.contrib.swift.""" 24 25import posixpath 26 27from time import time 28from io import BytesIO 29try: 30 from StringIO import StringIO 31except ImportError: 32 from io import StringIO 33 34import sys 35from unittest import skipIf 36 37from dulwich.tests import ( 38 TestCase, 39 ) 40from dulwich.tests.test_object_store import ( 41 ObjectStoreTests, 42 ) 43from dulwich.tests.utils import ( 44 build_pack, 45 ) 46from dulwich.objects import ( 47 Blob, 48 Commit, 49 Tree, 50 Tag, 51 parse_timezone, 52 ) 53from dulwich.pack import ( 54 REF_DELTA, 55 write_pack_index_v2, 56 PackData, 57 load_pack_index_file, 58 ) 59 60try: 61 from simplejson import dumps as json_dumps 62except ImportError: 63 from json import dumps as json_dumps 64 65missing_libs = [] 66 67try: 68 import gevent # noqa:F401 69except ImportError: 70 missing_libs.append("gevent") 71 72try: 73 import geventhttpclient # noqa:F401 74except ImportError: 75 missing_libs.append("geventhttpclient") 76 77try: 78 from mock import patch 79except ImportError: 80 missing_libs.append("mock") 81 82skipmsg = "Required libraries are not installed (%r)" % missing_libs 83 84skipIfPY3 = skipIf(sys.version_info[0] == 3, 85 "SWIFT module not yet ported to python3.") 86 87if not missing_libs: 88 from dulwich.contrib import swift 89 90config_file = """[swift] 91auth_url = http://127.0.0.1:8080/auth/%(version_str)s 92auth_ver = %(version_int)s 93username = test;tester 94password = testing 95region_name = %(region_name)s 96endpoint_type = %(endpoint_type)s 97concurrency = %(concurrency)s 98chunk_length = %(chunk_length)s 99cache_length = %(cache_length)s 100http_pool_length = %(http_pool_length)s 101http_timeout = %(http_timeout)s 102""" 103 104def_config_file = {'version_str': 'v1.0', 105 'version_int': 1, 106 'concurrency': 1, 107 'chunk_length': 12228, 108 'cache_length': 1, 109 'region_name': 'test', 110 'endpoint_type': 'internalURL', 111 'http_pool_length': 1, 112 'http_timeout': 1} 113 114 115def create_swift_connector(store={}): 116 return lambda root, conf: FakeSwiftConnector(root, 117 conf=conf, 118 store=store) 119 120 121class Response(object): 122 123 def __init__(self, headers={}, status=200, content=None): 124 self.headers = headers 125 self.status_code = status 126 self.content = content 127 128 def __getitem__(self, key): 129 return self.headers[key] 130 131 def items(self): 132 return self.headers.items() 133 134 def read(self): 135 return self.content 136 137 138def fake_auth_request_v1(*args, **kwargs): 139 ret = Response({'X-Storage-Url': 140 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser', 141 'X-Auth-Token': '12' * 10}, 142 200) 143 return ret 144 145 146def fake_auth_request_v1_error(*args, **kwargs): 147 ret = Response({}, 148 401) 149 return ret 150 151 152def fake_auth_request_v2(*args, **kwargs): 153 s_url = 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser' 154 resp = {'access': {'token': {'id': '12' * 10}, 155 'serviceCatalog': 156 [ 157 {'type': 'object-store', 158 'endpoints': [{'region': 'test', 159 'internalURL': s_url, 160 }, 161 ] 162 }, 163 ] 164 } 165 } 166 ret = Response(status=200, content=json_dumps(resp)) 167 return ret 168 169 170def create_commit(data, marker=b'Default', blob=None): 171 if not blob: 172 blob = Blob.from_string(b'The blob content ' + marker) 173 tree = Tree() 174 tree.add(b"thefile_" + marker, 0o100644, blob.id) 175 cmt = Commit() 176 if data: 177 assert isinstance(data[-1], Commit) 178 cmt.parents = [data[-1].id] 179 cmt.tree = tree.id 180 author = b"John Doe " + marker + b" <john@doe.net>" 181 cmt.author = cmt.committer = author 182 tz = parse_timezone(b'-0200')[0] 183 cmt.commit_time = cmt.author_time = int(time()) 184 cmt.commit_timezone = cmt.author_timezone = tz 185 cmt.encoding = b"UTF-8" 186 cmt.message = b"The commit message " + marker 187 tag = Tag() 188 tag.tagger = b"john@doe.net" 189 tag.message = b"Annotated tag" 190 tag.tag_timezone = parse_timezone(b'-0200')[0] 191 tag.tag_time = cmt.author_time 192 tag.object = (Commit, cmt.id) 193 tag.name = b"v_" + marker + b"_0.1" 194 return blob, tree, tag, cmt 195 196 197def create_commits(length=1, marker=b'Default'): 198 data = [] 199 for i in range(0, length): 200 _marker = ("%s_%s" % (marker, i)).encode() 201 blob, tree, tag, cmt = create_commit(data, _marker) 202 data.extend([blob, tree, tag, cmt]) 203 return data 204 205 206@skipIf(missing_libs, skipmsg) 207class FakeSwiftConnector(object): 208 209 def __init__(self, root, conf, store=None): 210 if store: 211 self.store = store 212 else: 213 self.store = {} 214 self.conf = conf 215 self.root = root 216 self.concurrency = 1 217 self.chunk_length = 12228 218 self.cache_length = 1 219 220 def put_object(self, name, content): 221 name = posixpath.join(self.root, name) 222 if hasattr(content, 'seek'): 223 content.seek(0) 224 content = content.read() 225 self.store[name] = content 226 227 def get_object(self, name, range=None): 228 name = posixpath.join(self.root, name) 229 if not range: 230 try: 231 return BytesIO(self.store[name]) 232 except KeyError: 233 return None 234 else: 235 l, r = range.split('-') 236 try: 237 if not l: 238 r = -int(r) 239 return self.store[name][r:] 240 else: 241 return self.store[name][int(l):int(r)] 242 except KeyError: 243 return None 244 245 def get_container_objects(self): 246 return [{'name': k.replace(self.root + '/', '')} 247 for k in self.store] 248 249 def create_root(self): 250 if self.root in self.store.keys(): 251 pass 252 else: 253 self.store[self.root] = '' 254 255 def get_object_stat(self, name): 256 name = posixpath.join(self.root, name) 257 if name not in self.store: 258 return None 259 return {'content-length': len(self.store[name])} 260 261 262@skipIf(missing_libs, skipmsg) 263@skipIfPY3 264class TestSwiftObjectStore(TestCase): 265 266 def setUp(self): 267 super(TestSwiftObjectStore, self).setUp() 268 self.conf = swift.load_conf(file=StringIO(config_file % 269 def_config_file)) 270 self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf) 271 272 def _put_pack(self, sos, commit_amount=1, marker='Default'): 273 odata = create_commits(length=commit_amount, marker=marker) 274 data = [(d.type_num, d.as_raw_string()) for d in odata] 275 f = BytesIO() 276 build_pack(f, data, store=sos) 277 sos.add_thin_pack(f.read, None) 278 return odata 279 280 def test_load_packs(self): 281 store = {'fakerepo/objects/pack/pack-'+'1'*40+'.idx': '', 282 'fakerepo/objects/pack/pack-'+'1'*40+'.pack': '', 283 'fakerepo/objects/pack/pack-'+'1'*40+'.info': '', 284 'fakerepo/objects/pack/pack-'+'2'*40+'.idx': '', 285 'fakerepo/objects/pack/pack-'+'2'*40+'.pack': '', 286 'fakerepo/objects/pack/pack-'+'2'*40+'.info': ''} 287 fsc = FakeSwiftConnector('fakerepo', conf=self.conf, store=store) 288 sos = swift.SwiftObjectStore(fsc) 289 packs = sos.packs 290 self.assertEqual(len(packs), 2) 291 for pack in packs: 292 self.assertTrue(isinstance(pack, swift.SwiftPack)) 293 294 def test_add_thin_pack(self): 295 sos = swift.SwiftObjectStore(self.fsc) 296 self._put_pack(sos, 1, 'Default') 297 self.assertEqual(len(self.fsc.store), 3) 298 299 def test_find_missing_objects(self): 300 commit_amount = 3 301 sos = swift.SwiftObjectStore(self.fsc) 302 odata = self._put_pack(sos, commit_amount, 'Default') 303 head = odata[-1].id 304 i = sos.iter_shas(sos.find_missing_objects([], 305 [head, ], 306 progress=None, 307 get_tagged=None)) 308 self.assertEqual(len(i), commit_amount * 3) 309 shas = [d.id for d in odata] 310 for sha, path in i: 311 self.assertIn(sha.id, shas) 312 313 def test_find_missing_objects_with_tag(self): 314 commit_amount = 3 315 sos = swift.SwiftObjectStore(self.fsc) 316 odata = self._put_pack(sos, commit_amount, 'Default') 317 head = odata[-1].id 318 peeled_sha = dict([(sha.object[1], sha.id) 319 for sha in odata if isinstance(sha, Tag)]) 320 321 def get_tagged(): 322 return peeled_sha 323 i = sos.iter_shas(sos.find_missing_objects([], 324 [head, ], 325 progress=None, 326 get_tagged=get_tagged)) 327 self.assertEqual(len(i), commit_amount * 4) 328 shas = [d.id for d in odata] 329 for sha, path in i: 330 self.assertIn(sha.id, shas) 331 332 def test_find_missing_objects_with_common(self): 333 commit_amount = 3 334 sos = swift.SwiftObjectStore(self.fsc) 335 odata = self._put_pack(sos, commit_amount, 'Default') 336 head = odata[-1].id 337 have = odata[7].id 338 i = sos.iter_shas(sos.find_missing_objects([have, ], 339 [head, ], 340 progress=None, 341 get_tagged=None)) 342 self.assertEqual(len(i), 3) 343 344 def test_find_missing_objects_multiple_packs(self): 345 sos = swift.SwiftObjectStore(self.fsc) 346 commit_amount_a = 3 347 odataa = self._put_pack(sos, commit_amount_a, 'Default1') 348 heada = odataa[-1].id 349 commit_amount_b = 2 350 odatab = self._put_pack(sos, commit_amount_b, 'Default2') 351 headb = odatab[-1].id 352 i = sos.iter_shas(sos.find_missing_objects([], 353 [heada, headb], 354 progress=None, 355 get_tagged=None)) 356 self.assertEqual(len(self.fsc.store), 6) 357 self.assertEqual(len(i), 358 commit_amount_a * 3 + 359 commit_amount_b * 3) 360 shas = [d.id for d in odataa] 361 shas.extend([d.id for d in odatab]) 362 for sha, path in i: 363 self.assertIn(sha.id, shas) 364 365 def test_add_thin_pack_ext_ref(self): 366 sos = swift.SwiftObjectStore(self.fsc) 367 odata = self._put_pack(sos, 1, 'Default1') 368 ref_blob_content = odata[0].as_raw_string() 369 ref_blob_id = odata[0].id 370 new_blob = Blob.from_string(ref_blob_content.replace('blob', 371 'yummy blob')) 372 blob, tree, tag, cmt = \ 373 create_commit([], marker='Default2', blob=new_blob) 374 data = [(REF_DELTA, (ref_blob_id, blob.as_raw_string())), 375 (tree.type_num, tree.as_raw_string()), 376 (cmt.type_num, cmt.as_raw_string()), 377 (tag.type_num, tag.as_raw_string())] 378 f = BytesIO() 379 build_pack(f, data, store=sos) 380 sos.add_thin_pack(f.read, None) 381 self.assertEqual(len(self.fsc.store), 6) 382 383 384@skipIf(missing_libs, skipmsg) 385class TestSwiftRepo(TestCase): 386 387 def setUp(self): 388 super(TestSwiftRepo, self).setUp() 389 self.conf = swift.load_conf(file=StringIO(config_file % 390 def_config_file)) 391 392 def test_init(self): 393 store = {'fakerepo/objects/pack': ''} 394 with patch('dulwich.contrib.swift.SwiftConnector', 395 new_callable=create_swift_connector, 396 store=store): 397 swift.SwiftRepo('fakerepo', conf=self.conf) 398 399 def test_init_no_data(self): 400 with patch('dulwich.contrib.swift.SwiftConnector', 401 new_callable=create_swift_connector): 402 self.assertRaises(Exception, swift.SwiftRepo, 403 'fakerepo', self.conf) 404 405 def test_init_bad_data(self): 406 store = {'fakerepo/.git/objects/pack': ''} 407 with patch('dulwich.contrib.swift.SwiftConnector', 408 new_callable=create_swift_connector, 409 store=store): 410 self.assertRaises(Exception, swift.SwiftRepo, 411 'fakerepo', self.conf) 412 413 def test_put_named_file(self): 414 store = {'fakerepo/objects/pack': ''} 415 with patch('dulwich.contrib.swift.SwiftConnector', 416 new_callable=create_swift_connector, 417 store=store): 418 repo = swift.SwiftRepo('fakerepo', conf=self.conf) 419 desc = b'Fake repo' 420 repo._put_named_file('description', desc) 421 self.assertEqual(repo.scon.store['fakerepo/description'], 422 desc) 423 424 def test_init_bare(self): 425 fsc = FakeSwiftConnector('fakeroot', conf=self.conf) 426 with patch('dulwich.contrib.swift.SwiftConnector', 427 new_callable=create_swift_connector, 428 store=fsc.store): 429 swift.SwiftRepo.init_bare(fsc, conf=self.conf) 430 self.assertIn('fakeroot/objects/pack', fsc.store) 431 self.assertIn('fakeroot/info/refs', fsc.store) 432 self.assertIn('fakeroot/description', fsc.store) 433 434 435@skipIf(missing_libs, skipmsg) 436@skipIfPY3 437class TestPackInfoLoadDump(TestCase): 438 439 def setUp(self): 440 super(TestPackInfoLoadDump, self).setUp() 441 conf = swift.load_conf(file=StringIO(config_file % 442 def_config_file)) 443 sos = swift.SwiftObjectStore( 444 FakeSwiftConnector('fakerepo', conf=conf)) 445 commit_amount = 10 446 self.commits = create_commits(length=commit_amount, marker="m") 447 data = [(d.type_num, d.as_raw_string()) for d in self.commits] 448 f = BytesIO() 449 fi = BytesIO() 450 expected = build_pack(f, data, store=sos) 451 entries = [(sha, ofs, checksum) for 452 ofs, _, _, sha, checksum in expected] 453 self.pack_data = PackData.from_file(file=f, size=None) 454 write_pack_index_v2( 455 fi, entries, self.pack_data.calculate_checksum()) 456 fi.seek(0) 457 self.pack_index = load_pack_index_file('', fi) 458 459# def test_pack_info_perf(self): 460# dump_time = [] 461# load_time = [] 462# for i in range(0, 100): 463# start = time() 464# dumps = swift.pack_info_create(self.pack_data, self.pack_index) 465# dump_time.append(time() - start) 466# for i in range(0, 100): 467# start = time() 468# pack_infos = swift.load_pack_info('', file=BytesIO(dumps)) 469# load_time.append(time() - start) 470# print sum(dump_time) / float(len(dump_time)) 471# print sum(load_time) / float(len(load_time)) 472 473 def test_pack_info(self): 474 dumps = swift.pack_info_create(self.pack_data, self.pack_index) 475 pack_infos = swift.load_pack_info('', file=BytesIO(dumps)) 476 for obj in self.commits: 477 self.assertIn(obj.id, pack_infos) 478 479 480@skipIf(missing_libs, skipmsg) 481class TestSwiftInfoRefsContainer(TestCase): 482 483 def setUp(self): 484 super(TestSwiftInfoRefsContainer, self).setUp() 485 content = ( 486 b"22effb216e3a82f97da599b8885a6cadb488b4c5\trefs/heads/master\n" 487 b"cca703b0e1399008b53a1a236d6b4584737649e4\trefs/heads/dev") 488 self.store = {'fakerepo/info/refs': content} 489 self.conf = swift.load_conf(file=StringIO(config_file % 490 def_config_file)) 491 self.fsc = FakeSwiftConnector('fakerepo', conf=self.conf) 492 self.object_store = {} 493 494 def test_init(self): 495 """info/refs does not exists""" 496 irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) 497 self.assertEqual(len(irc._refs), 0) 498 self.fsc.store = self.store 499 irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) 500 self.assertIn(b'refs/heads/dev', irc.allkeys()) 501 self.assertIn(b'refs/heads/master', irc.allkeys()) 502 503 def test_set_if_equals(self): 504 self.fsc.store = self.store 505 irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) 506 irc.set_if_equals(b'refs/heads/dev', 507 b"cca703b0e1399008b53a1a236d6b4584737649e4", b'1'*40) 508 self.assertEqual(irc[b'refs/heads/dev'], b'1'*40) 509 510 def test_remove_if_equals(self): 511 self.fsc.store = self.store 512 irc = swift.SwiftInfoRefsContainer(self.fsc, self.object_store) 513 irc.remove_if_equals(b'refs/heads/dev', 514 b"cca703b0e1399008b53a1a236d6b4584737649e4") 515 self.assertNotIn(b'refs/heads/dev', irc.allkeys()) 516 517 518@skipIf(missing_libs, skipmsg) 519class TestSwiftConnector(TestCase): 520 521 def setUp(self): 522 super(TestSwiftConnector, self).setUp() 523 self.conf = swift.load_conf(file=StringIO(config_file % 524 def_config_file)) 525 with patch('geventhttpclient.HTTPClient.request', 526 fake_auth_request_v1): 527 self.conn = swift.SwiftConnector('fakerepo', conf=self.conf) 528 529 def test_init_connector(self): 530 self.assertEqual(self.conn.auth_ver, '1') 531 self.assertEqual(self.conn.auth_url, 532 'http://127.0.0.1:8080/auth/v1.0') 533 self.assertEqual(self.conn.user, 'test:tester') 534 self.assertEqual(self.conn.password, 'testing') 535 self.assertEqual(self.conn.root, 'fakerepo') 536 self.assertEqual(self.conn.storage_url, 537 'http://127.0.0.1:8080/v1.0/AUTH_fakeuser') 538 self.assertEqual(self.conn.token, '12' * 10) 539 self.assertEqual(self.conn.http_timeout, 1) 540 self.assertEqual(self.conn.http_pool_length, 1) 541 self.assertEqual(self.conn.concurrency, 1) 542 self.conf.set('swift', 'auth_ver', '2') 543 self.conf.set('swift', 'auth_url', 'http://127.0.0.1:8080/auth/v2.0') 544 with patch('geventhttpclient.HTTPClient.request', 545 fake_auth_request_v2): 546 conn = swift.SwiftConnector('fakerepo', conf=self.conf) 547 self.assertEqual(conn.user, 'tester') 548 self.assertEqual(conn.tenant, 'test') 549 self.conf.set('swift', 'auth_ver', '1') 550 self.conf.set('swift', 'auth_url', 'http://127.0.0.1:8080/auth/v1.0') 551 with patch('geventhttpclient.HTTPClient.request', 552 fake_auth_request_v1_error): 553 self.assertRaises(swift.SwiftException, 554 lambda: swift.SwiftConnector('fakerepo', 555 conf=self.conf)) 556 557 def test_root_exists(self): 558 with patch('geventhttpclient.HTTPClient.request', 559 lambda *args: Response()): 560 self.assertEqual(self.conn.test_root_exists(), True) 561 562 def test_root_not_exists(self): 563 with patch('geventhttpclient.HTTPClient.request', 564 lambda *args: Response(status=404)): 565 self.assertEqual(self.conn.test_root_exists(), None) 566 567 def test_create_root(self): 568 with patch('dulwich.contrib.swift.SwiftConnector.test_root_exists', 569 lambda *args: None): 570 with patch('geventhttpclient.HTTPClient.request', 571 lambda *args: Response()): 572 self.assertEqual(self.conn.create_root(), None) 573 574 def test_create_root_fails(self): 575 with patch('dulwich.contrib.swift.SwiftConnector.test_root_exists', 576 lambda *args: None): 577 with patch('geventhttpclient.HTTPClient.request', 578 lambda *args: Response(status=404)): 579 self.assertRaises(swift.SwiftException, 580 lambda: self.conn.create_root()) 581 582 def test_get_container_objects(self): 583 with patch('geventhttpclient.HTTPClient.request', 584 lambda *args: Response(content=json_dumps( 585 (({'name': 'a'}, {'name': 'b'}))))): 586 self.assertEqual(len(self.conn.get_container_objects()), 2) 587 588 def test_get_container_objects_fails(self): 589 with patch('geventhttpclient.HTTPClient.request', 590 lambda *args: Response(status=404)): 591 self.assertEqual(self.conn.get_container_objects(), None) 592 593 def test_get_object_stat(self): 594 with patch('geventhttpclient.HTTPClient.request', 595 lambda *args: Response(headers={'content-length': '10'})): 596 self.assertEqual(self.conn.get_object_stat('a')['content-length'], 597 '10') 598 599 def test_get_object_stat_fails(self): 600 with patch('geventhttpclient.HTTPClient.request', 601 lambda *args: Response(status=404)): 602 self.assertEqual(self.conn.get_object_stat('a'), None) 603 604 def test_put_object(self): 605 with patch('geventhttpclient.HTTPClient.request', 606 lambda *args, **kwargs: Response()): 607 self.assertEqual(self.conn.put_object('a', BytesIO(b'content')), 608 None) 609 610 def test_put_object_fails(self): 611 with patch('geventhttpclient.HTTPClient.request', 612 lambda *args, **kwargs: Response(status=400)): 613 self.assertRaises(swift.SwiftException, 614 lambda: self.conn.put_object( 615 'a', BytesIO(b'content'))) 616 617 def test_get_object(self): 618 with patch('geventhttpclient.HTTPClient.request', 619 lambda *args, **kwargs: Response(content=b'content')): 620 self.assertEqual(self.conn.get_object('a').read(), b'content') 621 with patch('geventhttpclient.HTTPClient.request', 622 lambda *args, **kwargs: Response(content=b'content')): 623 self.assertEqual( 624 self.conn.get_object('a', range='0-6'), 625 b'content') 626 627 def test_get_object_fails(self): 628 with patch('geventhttpclient.HTTPClient.request', 629 lambda *args, **kwargs: Response(status=404)): 630 self.assertEqual(self.conn.get_object('a'), None) 631 632 def test_del_object(self): 633 with patch('geventhttpclient.HTTPClient.request', 634 lambda *args: Response()): 635 self.assertEqual(self.conn.del_object('a'), None) 636 637 def test_del_root(self): 638 with patch('dulwich.contrib.swift.SwiftConnector.del_object', 639 lambda *args: None): 640 with patch('dulwich.contrib.swift.SwiftConnector.' 641 'get_container_objects', 642 lambda *args: ({'name': 'a'}, {'name': 'b'})): 643 with patch('geventhttpclient.HTTPClient.request', 644 lambda *args: Response()): 645 self.assertEqual(self.conn.del_root(), None) 646 647 648@skipIf(missing_libs, skipmsg) 649class SwiftObjectStoreTests(ObjectStoreTests, TestCase): 650 651 def setUp(self): 652 TestCase.setUp(self) 653 conf = swift.load_conf(file=StringIO(config_file % 654 def_config_file)) 655 fsc = FakeSwiftConnector('fakerepo', conf=conf) 656 self.store = swift.SwiftObjectStore(fsc) 657