1import datetime 2import os 3import shutil 4import time 5 6from .crypto.key import KeyfileKey, KeyfileNotFoundError 7from .constants import REPOSITORY_README 8from .helpers import ProgressIndicatorPercent 9from .helpers import get_base_dir, get_keys_dir, get_cache_dir 10from .locking import Lock 11from .logger import create_logger 12from .repository import Repository, MAGIC 13 14logger = create_logger(__name__) 15 16ATTIC_MAGIC = b'ATTICSEG' 17 18 19class AtticRepositoryUpgrader(Repository): 20 def __init__(self, *args, **kw): 21 kw['lock'] = False # do not create borg lock files (now) in attic repo 22 kw['check_segment_magic'] = False # skip the Attic check when upgrading 23 super().__init__(*args, **kw) 24 25 def upgrade(self, dryrun=True, inplace=False, progress=False): 26 """convert an attic repository to a borg repository 27 28 those are the files that need to be upgraded here, from most 29 important to least important: segments, key files, and various 30 caches, the latter being optional, as they will be rebuilt if 31 missing. 32 33 we nevertheless do the order in reverse, as we prefer to do 34 the fast stuff first, to improve interactivity. 35 """ 36 with self: 37 backup = None 38 if not inplace: 39 backup = '{}.before-upgrade-{:%Y-%m-%d-%H:%M:%S}'.format(self.path, datetime.datetime.now()) 40 logger.info('making a hardlink copy in %s', backup) 41 if not dryrun: 42 shutil.copytree(self.path, backup, copy_function=os.link) 43 logger.info("opening attic repository with borg and converting") 44 # now lock the repo, after we have made the copy 45 self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() 46 segments = [filename for i, filename in self.io.segment_iterator()] 47 try: 48 keyfile = self.find_attic_keyfile() 49 except KeyfileNotFoundError: 50 logger.warning("no key file found for repository") 51 else: 52 self.convert_keyfiles(keyfile, dryrun) 53 # partial open: just hold on to the lock 54 self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True).acquire() 55 try: 56 self.convert_cache(dryrun) 57 self.convert_repo_index(dryrun=dryrun, inplace=inplace) 58 self.convert_segments(segments, dryrun=dryrun, inplace=inplace, progress=progress) 59 self.borg_readme() 60 finally: 61 self.lock.release() 62 self.lock = None 63 return backup 64 65 def borg_readme(self): 66 readme = os.path.join(self.path, 'README') 67 os.remove(readme) 68 with open(readme, 'w') as fd: 69 fd.write(REPOSITORY_README) 70 71 @staticmethod 72 def convert_segments(segments, dryrun=True, inplace=False, progress=False): 73 """convert repository segments from attic to borg 74 75 replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in 76 `$ATTIC_REPO/data/**`. 77 78 luckily the magic string length didn't change so we can just 79 replace the 8 first bytes of all regular files in there.""" 80 logger.info("converting %d segments..." % len(segments)) 81 segment_count = len(segments) 82 pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%", msgid='upgrade.convert_segments') 83 for i, filename in enumerate(segments): 84 if progress: 85 pi.show(i) 86 if dryrun: 87 time.sleep(0.001) 88 else: 89 AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace) 90 if progress: 91 pi.finish() 92 93 @staticmethod 94 def header_replace(filename, old_magic, new_magic, inplace=True): 95 with open(filename, 'r+b') as segment: 96 segment.seek(0) 97 # only write if necessary 98 if segment.read(len(old_magic)) == old_magic: 99 if inplace: 100 segment.seek(0) 101 segment.write(new_magic) 102 else: 103 # rename the hardlink and rewrite the file. this works 104 # because the file is still open. so even though the file 105 # is renamed, we can still read it until it is closed. 106 os.rename(filename, filename + '.tmp') 107 with open(filename, 'wb') as new_segment: 108 new_segment.write(new_magic) 109 new_segment.write(segment.read()) 110 # the little dance with the .tmp file is necessary 111 # because Windows won't allow overwriting an open file. 112 os.unlink(filename + '.tmp') 113 114 def find_attic_keyfile(self): 115 """find the attic keyfiles 116 117 the keyfiles are loaded by `KeyfileKey.find_key_file()`. that 118 finds the keys with the right identifier for the repo. 119 120 this is expected to look into $HOME/.attic/keys or 121 $ATTIC_KEYS_DIR for key files matching the given Borg 122 repository. 123 124 it is expected to raise an exception (KeyfileNotFoundError) if 125 no key is found. whether that exception is from Borg or Attic 126 is unclear. 127 128 this is split in a separate function in case we want to use 129 the attic code here directly, instead of our local 130 implementation.""" 131 return AtticKeyfileKey.find_key_file(self) 132 133 @staticmethod 134 def convert_keyfiles(keyfile, dryrun): 135 """convert key files from attic to borg 136 137 replacement pattern is `s/ATTIC KEY/BORG_KEY/` in 138 `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or 139 `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or 140 `$HOME/.config/borg/keys`. 141 142 no need to decrypt to convert. we need to rewrite the whole 143 key file because magic string length changed, but that's not a 144 problem because the keyfiles are small (compared to, say, 145 all the segments).""" 146 logger.info("converting keyfile %s" % keyfile) 147 with open(keyfile, 'r') as f: 148 data = f.read() 149 data = data.replace(AtticKeyfileKey.FILE_ID, KeyfileKey.FILE_ID, 1) 150 keyfile = os.path.join(get_keys_dir(), os.path.basename(keyfile)) 151 logger.info("writing borg keyfile to %s" % keyfile) 152 if not dryrun: 153 with open(keyfile, 'w') as f: 154 f.write(data) 155 156 def convert_repo_index(self, dryrun, inplace): 157 """convert some repo files 158 159 those are all hash indexes, so we need to 160 `s/ATTICIDX/BORG_IDX/` in a few locations: 161 162 * the repository index (in `$ATTIC_REPO/index.%d`, where `%d` 163 is the `Repository.get_index_transaction_id()`), which we 164 should probably update, with a lock, see 165 `Repository.open()`, which i'm not sure we should use 166 because it may write data on `Repository.close()`... 167 """ 168 transaction_id = self.get_index_transaction_id() 169 if transaction_id is None: 170 logger.warning('no index file found for repository %s' % self.path) 171 else: 172 index = os.path.join(self.path, 'index.%d' % transaction_id) 173 logger.info("converting repo index %s" % index) 174 if not dryrun: 175 AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX', inplace=inplace) 176 177 def convert_cache(self, dryrun): 178 """convert caches from attic to borg 179 180 those are all hash indexes, so we need to 181 `s/ATTICIDX/BORG_IDX/` in a few locations: 182 183 * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or 184 `$HOME/.cache/attic/<repoid>/`), which we could just drop, 185 but if we'd want to convert, we could open it with the 186 `Cache.open()`, edit in place and then `Cache.close()` to 187 make sure we have locking right 188 """ 189 # copy of attic's get_cache_dir() 190 attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR', 191 os.path.join(get_base_dir(), 192 '.cache', 'attic')) 193 attic_cache_dir = os.path.join(attic_cache_dir, self.id_str) 194 borg_cache_dir = os.path.join(get_cache_dir(), self.id_str) 195 196 def copy_cache_file(path): 197 """copy the given attic cache path into the borg directory 198 199 does nothing if dryrun is True. also expects 200 attic_cache_dir and borg_cache_dir to be set in the parent 201 scope, to the directories path including the repository 202 identifier. 203 204 :params path: the basename of the cache file to copy 205 (example: "files" or "chunks") as a string 206 207 :returns: the borg file that was created or None if no 208 Attic cache file was found. 209 210 """ 211 attic_file = os.path.join(attic_cache_dir, path) 212 if os.path.exists(attic_file): 213 borg_file = os.path.join(borg_cache_dir, path) 214 if os.path.exists(borg_file): 215 logger.warning("borg cache file already exists in %s, not copying from Attic", borg_file) 216 else: 217 logger.info("copying attic cache file from %s to %s" % (attic_file, borg_file)) 218 if not dryrun: 219 shutil.copyfile(attic_file, borg_file) 220 return borg_file 221 else: 222 logger.warning("no %s cache file found in %s" % (path, attic_file)) 223 return None 224 225 # XXX: untested, because generating cache files is a PITA, see 226 # Archiver.do_create() for proof 227 if os.path.exists(attic_cache_dir): 228 if not os.path.exists(borg_cache_dir): 229 os.makedirs(borg_cache_dir) 230 231 # file that we don't have a header to convert, just copy 232 for cache in ['config', 'files']: 233 copy_cache_file(cache) 234 235 # we need to convert the headers of those files, copy first 236 for cache in ['chunks']: 237 cache = copy_cache_file(cache) 238 logger.info("converting cache %s" % cache) 239 if not dryrun: 240 AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX') 241 242 243class AtticKeyfileKey(KeyfileKey): 244 """backwards compatible Attic key file parser""" 245 FILE_ID = 'ATTIC KEY' 246 247 # verbatim copy from attic 248 @staticmethod 249 def get_keys_dir(): 250 """Determine where to repository keys and cache""" 251 return os.environ.get('ATTIC_KEYS_DIR', 252 os.path.join(get_base_dir(), '.attic', 'keys')) 253 254 @classmethod 255 def find_key_file(cls, repository): 256 """copy of attic's `find_key_file`_ 257 258 this has two small modifications: 259 260 1. it uses the above `get_keys_dir`_ instead of the global one, 261 assumed to be borg's 262 263 2. it uses `repository.path`_ instead of 264 `repository._location.canonical_path`_ because we can't 265 assume the repository has been opened by the archiver yet 266 """ 267 get_keys_dir = cls.get_keys_dir 268 keys_dir = get_keys_dir() 269 if not os.path.exists(keys_dir): 270 raise KeyfileNotFoundError(repository.path, keys_dir) 271 for name in os.listdir(keys_dir): 272 filename = os.path.join(keys_dir, name) 273 with open(filename, 'r') as fd: 274 line = fd.readline().strip() 275 if line and line.startswith(cls.FILE_ID) and line[10:] == repository.id_str: 276 return filename 277 raise KeyfileNotFoundError(repository.path, keys_dir) 278 279 280class BorgRepositoryUpgrader(Repository): 281 def upgrade(self, dryrun=True, inplace=False, progress=False): 282 """convert an old borg repository to a current borg repository 283 """ 284 logger.info("converting borg 0.xx to borg current") 285 with self: 286 try: 287 keyfile = self.find_borg0xx_keyfile() 288 except KeyfileNotFoundError: 289 logger.warning("no key file found for repository") 290 else: 291 self.move_keyfiles(keyfile, dryrun) 292 293 def find_borg0xx_keyfile(self): 294 return Borg0xxKeyfileKey.find_key_file(self) 295 296 def move_keyfiles(self, keyfile, dryrun): 297 filename = os.path.basename(keyfile) 298 new_keyfile = os.path.join(get_keys_dir(), filename) 299 try: 300 os.rename(keyfile, new_keyfile) 301 except FileExistsError: 302 # likely the attic -> borg upgrader already put it in the final location 303 pass 304 305 306class Borg0xxKeyfileKey(KeyfileKey): 307 """backwards compatible borg 0.xx key file parser""" 308 309 @staticmethod 310 def get_keys_dir(): 311 return os.environ.get('BORG_KEYS_DIR', 312 os.path.join(get_base_dir(), '.borg', 'keys')) 313 314 @classmethod 315 def find_key_file(cls, repository): 316 get_keys_dir = cls.get_keys_dir 317 keys_dir = get_keys_dir() 318 if not os.path.exists(keys_dir): 319 raise KeyfileNotFoundError(repository.path, keys_dir) 320 for name in os.listdir(keys_dir): 321 filename = os.path.join(keys_dir, name) 322 with open(filename, 'r') as fd: 323 line = fd.readline().strip() 324 if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID) + 1:] == repository.id_str: 325 return filename 326 raise KeyfileNotFoundError(repository.path, keys_dir) 327