1# swift.py -- Repo implementation atop OpenStack SWIFT 2# Copyright (C) 2013 eNovance SAS <licensing@enovance.com> 3# 4# Author: Fabien Boucher <fabien.boucher@enovance.com> 5# 6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU 7# General Public License as public by the Free Software Foundation; version 2.0 8# or (at your option) any later version. You can redistribute it and/or 9# modify it under the terms of either of these two licenses. 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17# You should have received a copy of the licenses; if not, see 18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License 19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache 20# License, Version 2.0. 21# 22 23"""Repo implementation atop OpenStack SWIFT.""" 24 25# TODO: Refactor to share more code with dulwich/repo.py. 26# TODO(fbo): Second attempt to _send() must be notified via real log 27# TODO(fbo): More logs for operations 28 29import os 30import stat 31import zlib 32import tempfile 33import posixpath 34 35try: 36 import urlparse 37except ImportError: 38 import urllib.parse as urlparse 39 40from io import BytesIO 41try: 42 from ConfigParser import ConfigParser 43except ImportError: 44 from configparser import ConfigParser 45from geventhttpclient import HTTPClient 46 47from dulwich.greenthreads import ( 48 GreenThreadsMissingObjectFinder, 49 GreenThreadsObjectStoreIterator, 50 ) 51 52from dulwich.lru_cache import LRUSizeCache 53from dulwich.objects import ( 54 Blob, 55 Commit, 56 Tree, 57 Tag, 58 S_ISGITLINK, 59 ) 60from dulwich.object_store import ( 61 PackBasedObjectStore, 62 PACKDIR, 63 INFODIR, 64 ) 65from dulwich.pack import ( 66 PackData, 67 Pack, 68 PackIndexer, 69 PackStreamCopier, 70 write_pack_header, 71 compute_file_sha, 72 iter_sha1, 73 write_pack_index_v2, 74 load_pack_index_file, 75 read_pack_header, 76 _compute_object_size, 77 unpack_object, 78 write_pack_object, 79 ) 80from dulwich.protocol import TCP_GIT_PORT 81from dulwich.refs import ( 82 InfoRefsContainer, 83 read_info_refs, 84 write_info_refs, 85 ) 86from dulwich.repo import ( 87 BaseRepo, 88 OBJECTDIR, 89 ) 90from dulwich.server import ( 91 Backend, 92 TCPGitServer, 93 ) 94 95try: 96 from simplejson import loads as json_loads 97 from simplejson import dumps as json_dumps 98except ImportError: 99 from json import loads as json_loads 100 from json import dumps as json_dumps 101 102import sys 103 104 105""" 106# Configuration file sample 107[swift] 108# Authentication URL (Keystone or Swift) 109auth_url = http://127.0.0.1:5000/v2.0 110# Authentication version to use 111auth_ver = 2 112# The tenant and username separated by a semicolon 113username = admin;admin 114# The user password 115password = pass 116# The Object storage region to use (auth v2) (Default RegionOne) 117region_name = RegionOne 118# The Object storage endpoint URL to use (auth v2) (Default internalURL) 119endpoint_type = internalURL 120# Concurrency to use for parallel tasks (Default 10) 121concurrency = 10 122# Size of the HTTP pool (Default 10) 123http_pool_length = 10 124# Timeout delay for HTTP connections (Default 20) 125http_timeout = 20 126# Chunk size to read from pack (Bytes) (Default 12228) 127chunk_length = 12228 128# Cache size (MBytes) (Default 20) 129cache_length = 20 130""" 131 132 133class PackInfoObjectStoreIterator(GreenThreadsObjectStoreIterator): 134 135 def __len__(self): 136 while len(self.finder.objects_to_send): 137 for _ in range(0, len(self.finder.objects_to_send)): 138 sha = self.finder.next() 139 self._shas.append(sha) 140 return len(self._shas) 141 142 143class PackInfoMissingObjectFinder(GreenThreadsMissingObjectFinder): 144 145 def next(self): 146 while True: 147 if not self.objects_to_send: 148 return None 149 (sha, name, leaf) = self.objects_to_send.pop() 150 if sha not in self.sha_done: 151 break 152 if not leaf: 153 info = self.object_store.pack_info_get(sha) 154 if info[0] == Commit.type_num: 155 self.add_todo([(info[2], "", False)]) 156 elif info[0] == Tree.type_num: 157 self.add_todo([tuple(i) for i in info[1]]) 158 elif info[0] == Tag.type_num: 159 self.add_todo([(info[1], None, False)]) 160 if sha in self._tagged: 161 self.add_todo([(self._tagged[sha], None, True)]) 162 self.sha_done.add(sha) 163 self.progress("counting objects: %d\r" % len(self.sha_done)) 164 return (sha, name) 165 166 167def load_conf(path=None, file=None): 168 """Load configuration in global var CONF 169 170 Args: 171 path: The path to the configuration file 172 file: If provided read instead the file like object 173 """ 174 conf = ConfigParser() 175 if file: 176 try: 177 conf.read_file(file, path) 178 except AttributeError: 179 # read_file only exists in Python3 180 conf.readfp(file) 181 return conf 182 confpath = None 183 if not path: 184 try: 185 confpath = os.environ['DULWICH_SWIFT_CFG'] 186 except KeyError: 187 raise Exception("You need to specify a configuration file") 188 else: 189 confpath = path 190 if not os.path.isfile(confpath): 191 raise Exception("Unable to read configuration file %s" % confpath) 192 conf.read(confpath) 193 return conf 194 195 196def swift_load_pack_index(scon, filename): 197 """Read a pack index file from Swift 198 199 Args: 200 scon: a `SwiftConnector` instance 201 filename: Path to the index file objectise 202 Returns: a `PackIndexer` instance 203 """ 204 with scon.get_object(filename) as f: 205 return load_pack_index_file(filename, f) 206 207 208def pack_info_create(pack_data, pack_index): 209 pack = Pack.from_objects(pack_data, pack_index) 210 info = {} 211 for obj in pack.iterobjects(): 212 # Commit 213 if obj.type_num == Commit.type_num: 214 info[obj.id] = (obj.type_num, obj.parents, obj.tree) 215 # Tree 216 elif obj.type_num == Tree.type_num: 217 shas = [(s, n, not stat.S_ISDIR(m)) for 218 n, m, s in obj.items() if not S_ISGITLINK(m)] 219 info[obj.id] = (obj.type_num, shas) 220 # Blob 221 elif obj.type_num == Blob.type_num: 222 info[obj.id] = None 223 # Tag 224 elif obj.type_num == Tag.type_num: 225 info[obj.id] = (obj.type_num, obj.object[1]) 226 return zlib.compress(json_dumps(info)) 227 228 229def load_pack_info(filename, scon=None, file=None): 230 if not file: 231 f = scon.get_object(filename) 232 else: 233 f = file 234 if not f: 235 return None 236 try: 237 return json_loads(zlib.decompress(f.read())) 238 finally: 239 f.close() 240 241 242class SwiftException(Exception): 243 pass 244 245 246class SwiftConnector(object): 247 """A Connector to swift that manage authentication and errors catching 248 """ 249 250 def __init__(self, root, conf): 251 """ Initialize a SwiftConnector 252 253 Args: 254 root: The swift container that will act as Git bare repository 255 conf: A ConfigParser Object 256 """ 257 self.conf = conf 258 self.auth_ver = self.conf.get("swift", "auth_ver") 259 if self.auth_ver not in ["1", "2"]: 260 raise NotImplementedError( 261 "Wrong authentication version use either 1 or 2") 262 self.auth_url = self.conf.get("swift", "auth_url") 263 self.user = self.conf.get("swift", "username") 264 self.password = self.conf.get("swift", "password") 265 self.concurrency = self.conf.getint('swift', 'concurrency') or 10 266 self.http_timeout = self.conf.getint('swift', 'http_timeout') or 20 267 self.http_pool_length = \ 268 self.conf.getint('swift', 'http_pool_length') or 10 269 self.region_name = self.conf.get("swift", "region_name") or "RegionOne" 270 self.endpoint_type = \ 271 self.conf.get("swift", "endpoint_type") or "internalURL" 272 self.cache_length = self.conf.getint("swift", "cache_length") or 20 273 self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228 274 self.root = root 275 block_size = 1024 * 12 # 12KB 276 if self.auth_ver == "1": 277 self.storage_url, self.token = self.swift_auth_v1() 278 else: 279 self.storage_url, self.token = self.swift_auth_v2() 280 281 token_header = {'X-Auth-Token': str(self.token)} 282 self.httpclient = \ 283 HTTPClient.from_url(str(self.storage_url), 284 concurrency=self.http_pool_length, 285 block_size=block_size, 286 connection_timeout=self.http_timeout, 287 network_timeout=self.http_timeout, 288 headers=token_header) 289 self.base_path = str(posixpath.join( 290 urlparse.urlparse(self.storage_url).path, self.root)) 291 292 def swift_auth_v1(self): 293 self.user = self.user.replace(";", ":") 294 auth_httpclient = HTTPClient.from_url( 295 self.auth_url, 296 connection_timeout=self.http_timeout, 297 network_timeout=self.http_timeout, 298 ) 299 headers = {'X-Auth-User': self.user, 300 'X-Auth-Key': self.password} 301 path = urlparse.urlparse(self.auth_url).path 302 303 ret = auth_httpclient.request('GET', path, headers=headers) 304 305 # Should do something with redirections (301 in my case) 306 307 if ret.status_code < 200 or ret.status_code >= 300: 308 raise SwiftException('AUTH v1.0 request failed on ' + 309 '%s with error code %s (%s)' 310 % (str(auth_httpclient.get_base_url()) + 311 path, ret.status_code, 312 str(ret.items()))) 313 storage_url = ret['X-Storage-Url'] 314 token = ret['X-Auth-Token'] 315 return storage_url, token 316 317 def swift_auth_v2(self): 318 self.tenant, self.user = self.user.split(';') 319 auth_dict = {} 320 auth_dict['auth'] = {'passwordCredentials': 321 { 322 'username': self.user, 323 'password': self.password, 324 }, 325 'tenantName': self.tenant} 326 auth_json = json_dumps(auth_dict) 327 headers = {'Content-Type': 'application/json'} 328 auth_httpclient = HTTPClient.from_url( 329 self.auth_url, 330 connection_timeout=self.http_timeout, 331 network_timeout=self.http_timeout, 332 ) 333 path = urlparse.urlparse(self.auth_url).path 334 if not path.endswith('tokens'): 335 path = posixpath.join(path, 'tokens') 336 ret = auth_httpclient.request('POST', path, 337 body=auth_json, 338 headers=headers) 339 340 if ret.status_code < 200 or ret.status_code >= 300: 341 raise SwiftException('AUTH v2.0 request failed on ' + 342 '%s with error code %s (%s)' 343 % (str(auth_httpclient.get_base_url()) + 344 path, ret.status_code, 345 str(ret.items()))) 346 auth_ret_json = json_loads(ret.read()) 347 token = auth_ret_json['access']['token']['id'] 348 catalogs = auth_ret_json['access']['serviceCatalog'] 349 object_store = [o_store for o_store in catalogs if 350 o_store['type'] == 'object-store'][0] 351 endpoints = object_store['endpoints'] 352 endpoint = [endp for endp in endpoints if 353 endp["region"] == self.region_name][0] 354 return endpoint[self.endpoint_type], token 355 356 def test_root_exists(self): 357 """Check that Swift container exist 358 359 Returns: True if exist or None it not 360 """ 361 ret = self.httpclient.request('HEAD', self.base_path) 362 if ret.status_code == 404: 363 return None 364 if ret.status_code < 200 or ret.status_code > 300: 365 raise SwiftException('HEAD request failed with error code %s' 366 % ret.status_code) 367 return True 368 369 def create_root(self): 370 """Create the Swift container 371 372 Raises: 373 SwiftException: if unable to create 374 """ 375 if not self.test_root_exists(): 376 ret = self.httpclient.request('PUT', self.base_path) 377 if ret.status_code < 200 or ret.status_code > 300: 378 raise SwiftException('PUT request failed with error code %s' 379 % ret.status_code) 380 381 def get_container_objects(self): 382 """Retrieve objects list in a container 383 384 Returns: A list of dict that describe objects 385 or None if container does not exist 386 """ 387 qs = '?format=json' 388 path = self.base_path + qs 389 ret = self.httpclient.request('GET', path) 390 if ret.status_code == 404: 391 return None 392 if ret.status_code < 200 or ret.status_code > 300: 393 raise SwiftException('GET request failed with error code %s' 394 % ret.status_code) 395 content = ret.read() 396 return json_loads(content) 397 398 def get_object_stat(self, name): 399 """Retrieve object stat 400 401 Args: 402 name: The object name 403 Returns: 404 A dict that describe the object or None if object does not exist 405 """ 406 path = self.base_path + '/' + name 407 ret = self.httpclient.request('HEAD', path) 408 if ret.status_code == 404: 409 return None 410 if ret.status_code < 200 or ret.status_code > 300: 411 raise SwiftException('HEAD request failed with error code %s' 412 % ret.status_code) 413 resp_headers = {} 414 for header, value in ret.items(): 415 resp_headers[header.lower()] = value 416 return resp_headers 417 418 def put_object(self, name, content): 419 """Put an object 420 421 Args: 422 name: The object name 423 content: A file object 424 Raises: 425 SwiftException: if unable to create 426 """ 427 content.seek(0) 428 data = content.read() 429 path = self.base_path + '/' + name 430 headers = {'Content-Length': str(len(data))} 431 432 def _send(): 433 ret = self.httpclient.request('PUT', path, 434 body=data, 435 headers=headers) 436 return ret 437 438 try: 439 # Sometime got Broken Pipe - Dirty workaround 440 ret = _send() 441 except Exception: 442 # Second attempt work 443 ret = _send() 444 445 if ret.status_code < 200 or ret.status_code > 300: 446 raise SwiftException('PUT request failed with error code %s' 447 % ret.status_code) 448 449 def get_object(self, name, range=None): 450 """Retrieve an object 451 452 Args: 453 name: The object name 454 range: A string range like "0-10" to 455 retrieve specified bytes in object content 456 Returns: 457 A file like instance or bytestring if range is specified 458 """ 459 headers = {} 460 if range: 461 headers['Range'] = 'bytes=%s' % range 462 path = self.base_path + '/' + name 463 ret = self.httpclient.request('GET', path, headers=headers) 464 if ret.status_code == 404: 465 return None 466 if ret.status_code < 200 or ret.status_code > 300: 467 raise SwiftException('GET request failed with error code %s' 468 % ret.status_code) 469 content = ret.read() 470 471 if range: 472 return content 473 return BytesIO(content) 474 475 def del_object(self, name): 476 """Delete an object 477 478 Args: 479 name: The object name 480 Raises: 481 SwiftException: if unable to delete 482 """ 483 path = self.base_path + '/' + name 484 ret = self.httpclient.request('DELETE', path) 485 if ret.status_code < 200 or ret.status_code > 300: 486 raise SwiftException('DELETE request failed with error code %s' 487 % ret.status_code) 488 489 def del_root(self): 490 """Delete the root container by removing container content 491 492 Raises: 493 SwiftException: if unable to delete 494 """ 495 for obj in self.get_container_objects(): 496 self.del_object(obj['name']) 497 ret = self.httpclient.request('DELETE', self.base_path) 498 if ret.status_code < 200 or ret.status_code > 300: 499 raise SwiftException('DELETE request failed with error code %s' 500 % ret.status_code) 501 502 503class SwiftPackReader(object): 504 """A SwiftPackReader that mimic read and sync method 505 506 The reader allows to read a specified amount of bytes from 507 a given offset of a Swift object. A read offset is kept internaly. 508 The reader will read from Swift a specified amount of data to complete 509 its internal buffer. chunk_length specifiy the amount of data 510 to read from Swift. 511 """ 512 513 def __init__(self, scon, filename, pack_length): 514 """Initialize a SwiftPackReader 515 516 Args: 517 scon: a `SwiftConnector` instance 518 filename: the pack filename 519 pack_length: The size of the pack object 520 """ 521 self.scon = scon 522 self.filename = filename 523 self.pack_length = pack_length 524 self.offset = 0 525 self.base_offset = 0 526 self.buff = b'' 527 self.buff_length = self.scon.chunk_length 528 529 def _read(self, more=False): 530 if more: 531 self.buff_length = self.buff_length * 2 532 offset = self.base_offset 533 r = min(self.base_offset + self.buff_length, self.pack_length) 534 ret = self.scon.get_object(self.filename, range="%s-%s" % (offset, r)) 535 self.buff = ret 536 537 def read(self, length): 538 """Read a specified amount of Bytes form the pack object 539 540 Args: 541 length: amount of bytes to read 542 Returns: 543 a bytestring 544 """ 545 end = self.offset+length 546 if self.base_offset + end > self.pack_length: 547 data = self.buff[self.offset:] 548 self.offset = end 549 return data 550 if end > len(self.buff): 551 # Need to read more from swift 552 self._read(more=True) 553 return self.read(length) 554 data = self.buff[self.offset:end] 555 self.offset = end 556 return data 557 558 def seek(self, offset): 559 """Seek to a specified offset 560 561 Args: 562 offset: the offset to seek to 563 """ 564 self.base_offset = offset 565 self._read() 566 self.offset = 0 567 568 def read_checksum(self): 569 """Read the checksum from the pack 570 571 Returns: the checksum bytestring 572 """ 573 return self.scon.get_object(self.filename, range="-20") 574 575 576class SwiftPackData(PackData): 577 """The data contained in a packfile. 578 579 We use the SwiftPackReader to read bytes from packs stored in Swift 580 using the Range header feature of Swift. 581 """ 582 583 def __init__(self, scon, filename): 584 """ Initialize a SwiftPackReader 585 586 Args: 587 scon: a `SwiftConnector` instance 588 filename: the pack filename 589 """ 590 self.scon = scon 591 self._filename = filename 592 self._header_size = 12 593 headers = self.scon.get_object_stat(self._filename) 594 self.pack_length = int(headers['content-length']) 595 pack_reader = SwiftPackReader(self.scon, self._filename, 596 self.pack_length) 597 (version, self._num_objects) = read_pack_header(pack_reader.read) 598 self._offset_cache = LRUSizeCache(1024*1024*self.scon.cache_length, 599 compute_size=_compute_object_size) 600 self.pack = None 601 602 def get_object_at(self, offset): 603 if offset in self._offset_cache: 604 return self._offset_cache[offset] 605 assert offset >= self._header_size 606 pack_reader = SwiftPackReader(self.scon, self._filename, 607 self.pack_length) 608 pack_reader.seek(offset) 609 unpacked, _ = unpack_object(pack_reader.read) 610 return (unpacked.pack_type_num, unpacked._obj()) 611 612 def get_stored_checksum(self): 613 pack_reader = SwiftPackReader(self.scon, self._filename, 614 self.pack_length) 615 return pack_reader.read_checksum() 616 617 def close(self): 618 pass 619 620 621class SwiftPack(Pack): 622 """A Git pack object. 623 624 Same implementation as pack.Pack except that _idx_load and 625 _data_load are bounded to Swift version of load_pack_index and 626 PackData. 627 """ 628 629 def __init__(self, *args, **kwargs): 630 self.scon = kwargs['scon'] 631 del kwargs['scon'] 632 super(SwiftPack, self).__init__(*args, **kwargs) 633 self._pack_info_path = self._basename + '.info' 634 self._pack_info = None 635 self._pack_info_load = lambda: load_pack_info(self._pack_info_path, 636 self.scon) 637 self._idx_load = lambda: swift_load_pack_index(self.scon, 638 self._idx_path) 639 self._data_load = lambda: SwiftPackData(self.scon, self._data_path) 640 641 @property 642 def pack_info(self): 643 """The pack data object being used.""" 644 if self._pack_info is None: 645 self._pack_info = self._pack_info_load() 646 return self._pack_info 647 648 649class SwiftObjectStore(PackBasedObjectStore): 650 """A Swift Object Store 651 652 Allow to manage a bare Git repository from Openstack Swift. 653 This object store only supports pack files and not loose objects. 654 """ 655 def __init__(self, scon): 656 """Open a Swift object store. 657 658 Args: 659 scon: A `SwiftConnector` instance 660 """ 661 super(SwiftObjectStore, self).__init__() 662 self.scon = scon 663 self.root = self.scon.root 664 self.pack_dir = posixpath.join(OBJECTDIR, PACKDIR) 665 self._alternates = None 666 667 def _update_pack_cache(self): 668 objects = self.scon.get_container_objects() 669 pack_files = [o['name'].replace(".pack", "") 670 for o in objects if o['name'].endswith(".pack")] 671 ret = [] 672 for basename in pack_files: 673 pack = SwiftPack(basename, scon=self.scon) 674 self._pack_cache[basename] = pack 675 ret.append(pack) 676 return ret 677 678 def _iter_loose_objects(self): 679 """Loose objects are not supported by this repository 680 """ 681 return [] 682 683 def iter_shas(self, finder): 684 """An iterator over pack's ObjectStore. 685 686 Returns: a `ObjectStoreIterator` or `GreenThreadsObjectStoreIterator` 687 instance if gevent is enabled 688 """ 689 shas = iter(finder.next, None) 690 return PackInfoObjectStoreIterator( 691 self, shas, finder, self.scon.concurrency) 692 693 def find_missing_objects(self, *args, **kwargs): 694 kwargs['concurrency'] = self.scon.concurrency 695 return PackInfoMissingObjectFinder(self, *args, **kwargs) 696 697 def pack_info_get(self, sha): 698 for pack in self.packs: 699 if sha in pack: 700 return pack.pack_info[sha] 701 702 def _collect_ancestors(self, heads, common=set()): 703 def _find_parents(commit): 704 for pack in self.packs: 705 if commit in pack: 706 try: 707 parents = pack.pack_info[commit][1] 708 except KeyError: 709 # Seems to have no parents 710 return [] 711 return parents 712 713 bases = set() 714 commits = set() 715 queue = [] 716 queue.extend(heads) 717 while queue: 718 e = queue.pop(0) 719 if e in common: 720 bases.add(e) 721 elif e not in commits: 722 commits.add(e) 723 parents = _find_parents(e) 724 queue.extend(parents) 725 return (commits, bases) 726 727 def add_pack(self): 728 """Add a new pack to this object store. 729 730 Returns: Fileobject to write to and a commit function to 731 call when the pack is finished. 732 """ 733 f = BytesIO() 734 735 def commit(): 736 f.seek(0) 737 pack = PackData(file=f, filename="") 738 entries = pack.sorted_entries() 739 if len(entries): 740 basename = posixpath.join(self.pack_dir, 741 "pack-%s" % 742 iter_sha1(entry[0] for 743 entry in entries)) 744 index = BytesIO() 745 write_pack_index_v2(index, entries, pack.get_stored_checksum()) 746 self.scon.put_object(basename + ".pack", f) 747 f.close() 748 self.scon.put_object(basename + ".idx", index) 749 index.close() 750 final_pack = SwiftPack(basename, scon=self.scon) 751 final_pack.check_length_and_checksum() 752 self._add_cached_pack(basename, final_pack) 753 return final_pack 754 else: 755 return None 756 757 def abort(): 758 pass 759 return f, commit, abort 760 761 def add_object(self, obj): 762 self.add_objects([(obj, None), ]) 763 764 def _pack_cache_stale(self): 765 return False 766 767 def _get_loose_object(self, sha): 768 return None 769 770 def add_thin_pack(self, read_all, read_some): 771 """Read a thin pack 772 773 Read it from a stream and complete it in a temporary file. 774 Then the pack and the corresponding index file are uploaded to Swift. 775 """ 776 fd, path = tempfile.mkstemp(prefix='tmp_pack_') 777 f = os.fdopen(fd, 'w+b') 778 try: 779 indexer = PackIndexer(f, resolve_ext_ref=self.get_raw) 780 copier = PackStreamCopier(read_all, read_some, f, 781 delta_iter=indexer) 782 copier.verify() 783 return self._complete_thin_pack(f, path, copier, indexer) 784 finally: 785 f.close() 786 os.unlink(path) 787 788 def _complete_thin_pack(self, f, path, copier, indexer): 789 entries = list(indexer) 790 791 # Update the header with the new number of objects. 792 f.seek(0) 793 write_pack_header(f, len(entries) + len(indexer.ext_refs())) 794 795 # Must flush before reading (http://bugs.python.org/issue3207) 796 f.flush() 797 798 # Rescan the rest of the pack, computing the SHA with the new header. 799 new_sha = compute_file_sha(f, end_ofs=-20) 800 801 # Must reposition before writing (http://bugs.python.org/issue3207) 802 f.seek(0, os.SEEK_CUR) 803 804 # Complete the pack. 805 for ext_sha in indexer.ext_refs(): 806 assert len(ext_sha) == 20 807 type_num, data = self.get_raw(ext_sha) 808 offset = f.tell() 809 crc32 = write_pack_object(f, type_num, data, sha=new_sha) 810 entries.append((ext_sha, offset, crc32)) 811 pack_sha = new_sha.digest() 812 f.write(pack_sha) 813 f.flush() 814 815 # Move the pack in. 816 entries.sort() 817 pack_base_name = posixpath.join( 818 self.pack_dir, 819 'pack-' + iter_sha1(e[0] for e in entries).decode( 820 sys.getfilesystemencoding())) 821 self.scon.put_object(pack_base_name + '.pack', f) 822 823 # Write the index. 824 filename = pack_base_name + '.idx' 825 index_file = BytesIO() 826 write_pack_index_v2(index_file, entries, pack_sha) 827 self.scon.put_object(filename, index_file) 828 829 # Write pack info. 830 f.seek(0) 831 pack_data = PackData(filename="", file=f) 832 index_file.seek(0) 833 pack_index = load_pack_index_file('', index_file) 834 serialized_pack_info = pack_info_create(pack_data, pack_index) 835 f.close() 836 index_file.close() 837 pack_info_file = BytesIO(serialized_pack_info) 838 filename = pack_base_name + '.info' 839 self.scon.put_object(filename, pack_info_file) 840 pack_info_file.close() 841 842 # Add the pack to the store and return it. 843 final_pack = SwiftPack(pack_base_name, scon=self.scon) 844 final_pack.check_length_and_checksum() 845 self._add_cached_pack(pack_base_name, final_pack) 846 return final_pack 847 848 849class SwiftInfoRefsContainer(InfoRefsContainer): 850 """Manage references in info/refs object. 851 """ 852 853 def __init__(self, scon, store): 854 self.scon = scon 855 self.filename = 'info/refs' 856 self.store = store 857 f = self.scon.get_object(self.filename) 858 if not f: 859 f = BytesIO(b'') 860 super(SwiftInfoRefsContainer, self).__init__(f) 861 862 def _load_check_ref(self, name, old_ref): 863 self._check_refname(name) 864 f = self.scon.get_object(self.filename) 865 if not f: 866 return {} 867 refs = read_info_refs(f) 868 if old_ref is not None: 869 if refs[name] != old_ref: 870 return False 871 return refs 872 873 def _write_refs(self, refs): 874 f = BytesIO() 875 f.writelines(write_info_refs(refs, self.store)) 876 self.scon.put_object(self.filename, f) 877 878 def set_if_equals(self, name, old_ref, new_ref): 879 """Set a refname to new_ref only if it currently equals old_ref. 880 """ 881 if name == 'HEAD': 882 return True 883 refs = self._load_check_ref(name, old_ref) 884 if not isinstance(refs, dict): 885 return False 886 refs[name] = new_ref 887 self._write_refs(refs) 888 self._refs[name] = new_ref 889 return True 890 891 def remove_if_equals(self, name, old_ref): 892 """Remove a refname only if it currently equals old_ref. 893 """ 894 if name == 'HEAD': 895 return True 896 refs = self._load_check_ref(name, old_ref) 897 if not isinstance(refs, dict): 898 return False 899 del refs[name] 900 self._write_refs(refs) 901 del self._refs[name] 902 return True 903 904 def allkeys(self): 905 try: 906 self._refs['HEAD'] = self._refs['refs/heads/master'] 907 except KeyError: 908 pass 909 return self._refs.keys() 910 911 912class SwiftRepo(BaseRepo): 913 914 def __init__(self, root, conf): 915 """Init a Git bare Repository on top of a Swift container. 916 917 References are managed in info/refs objects by 918 `SwiftInfoRefsContainer`. The root attribute is the Swift 919 container that contain the Git bare repository. 920 921 Args: 922 root: The container which contains the bare repo 923 conf: A ConfigParser object 924 """ 925 self.root = root.lstrip('/') 926 self.conf = conf 927 self.scon = SwiftConnector(self.root, self.conf) 928 objects = self.scon.get_container_objects() 929 if not objects: 930 raise Exception('There is not any GIT repo here : %s' % self.root) 931 objects = [o['name'].split('/')[0] for o in objects] 932 if OBJECTDIR not in objects: 933 raise Exception('This repository (%s) is not bare.' % self.root) 934 self.bare = True 935 self._controldir = self.root 936 object_store = SwiftObjectStore(self.scon) 937 refs = SwiftInfoRefsContainer(self.scon, object_store) 938 BaseRepo.__init__(self, object_store, refs) 939 940 def _determine_file_mode(self): 941 """Probe the file-system to determine whether permissions can be trusted. 942 943 Returns: True if permissions can be trusted, False otherwise. 944 """ 945 return False 946 947 def _put_named_file(self, filename, contents): 948 """Put an object in a Swift container 949 950 Args: 951 filename: the path to the object to put on Swift 952 contents: the content as bytestring 953 """ 954 with BytesIO() as f: 955 f.write(contents) 956 self.scon.put_object(filename, f) 957 958 @classmethod 959 def init_bare(cls, scon, conf): 960 """Create a new bare repository. 961 962 Args: 963 scon: a `SwiftConnector` instance 964 conf: a ConfigParser object 965 Returns: 966 a `SwiftRepo` instance 967 """ 968 scon.create_root() 969 for obj in [posixpath.join(OBJECTDIR, PACKDIR), 970 posixpath.join(INFODIR, 'refs')]: 971 scon.put_object(obj, BytesIO(b'')) 972 ret = cls(scon.root, conf) 973 ret._init_files(True) 974 return ret 975 976 977class SwiftSystemBackend(Backend): 978 979 def __init__(self, logger, conf): 980 self.conf = conf 981 self.logger = logger 982 983 def open_repository(self, path): 984 self.logger.info('opening repository at %s', path) 985 return SwiftRepo(path, self.conf) 986 987 988def cmd_daemon(args): 989 """Entry point for starting a TCP git server.""" 990 import optparse 991 parser = optparse.OptionParser() 992 parser.add_option("-l", "--listen_address", dest="listen_address", 993 default="127.0.0.1", 994 help="Binding IP address.") 995 parser.add_option("-p", "--port", dest="port", type=int, 996 default=TCP_GIT_PORT, 997 help="Binding TCP port.") 998 parser.add_option("-c", "--swift_config", dest="swift_config", 999 default="", 1000 help="Path to the configuration file for Swift backend.") 1001 options, args = parser.parse_args(args) 1002 1003 try: 1004 import gevent 1005 import geventhttpclient # noqa: F401 1006 except ImportError: 1007 print("gevent and geventhttpclient libraries are mandatory " 1008 " for use the Swift backend.") 1009 sys.exit(1) 1010 import gevent.monkey 1011 gevent.monkey.patch_socket() 1012 from dulwich import log_utils 1013 logger = log_utils.getLogger(__name__) 1014 conf = load_conf(options.swift_config) 1015 backend = SwiftSystemBackend(logger, conf) 1016 1017 log_utils.default_logging_config() 1018 server = TCPGitServer(backend, options.listen_address, 1019 port=options.port) 1020 server.serve_forever() 1021 1022 1023def cmd_init(args): 1024 import optparse 1025 parser = optparse.OptionParser() 1026 parser.add_option("-c", "--swift_config", dest="swift_config", 1027 default="", 1028 help="Path to the configuration file for Swift backend.") 1029 options, args = parser.parse_args(args) 1030 1031 conf = load_conf(options.swift_config) 1032 if args == []: 1033 parser.error("missing repository name") 1034 repo = args[0] 1035 scon = SwiftConnector(repo, conf) 1036 SwiftRepo.init_bare(scon, conf) 1037 1038 1039def main(argv=sys.argv): 1040 commands = { 1041 "init": cmd_init, 1042 "daemon": cmd_daemon, 1043 } 1044 1045 if len(sys.argv) < 2: 1046 print("Usage: %s <%s> [OPTIONS...]" % ( 1047 sys.argv[0], "|".join(commands.keys()))) 1048 sys.exit(1) 1049 1050 cmd = sys.argv[1] 1051 if cmd not in commands: 1052 print("No such subcommand: %s" % cmd) 1053 sys.exit(1) 1054 commands[cmd](sys.argv[2:]) 1055 1056 1057if __name__ == '__main__': 1058 main() 1059