1import datetime
2import os
3import shutil
4import time
5
6from .crypto.key import KeyfileKey, KeyfileNotFoundError
7from .constants import REPOSITORY_README
8from .helpers import ProgressIndicatorPercent
9from .helpers import get_base_dir, get_keys_dir, get_cache_dir
10from .locking import Lock
11from .logger import create_logger
12from .repository import Repository, MAGIC
13
14logger = create_logger(__name__)
15
16ATTIC_MAGIC = b'ATTICSEG'
17
18
19class AtticRepositoryUpgrader(Repository):
20    def __init__(self, *args, **kw):
21        kw['lock'] = False  # do not create borg lock files (now) in attic repo
22        kw['check_segment_magic'] = False  # skip the Attic check when upgrading
23        super().__init__(*args, **kw)
24
25    def upgrade(self, dryrun=True, inplace=False, progress=False):
26        """convert an attic repository to a borg repository
27
28        those are the files that need to be upgraded here, from most
29        important to least important: segments, key files, and various
30        caches, the latter being optional, as they will be rebuilt if
31        missing.
32
33        we nevertheless do the order in reverse, as we prefer to do
34        the fast stuff first, to improve interactivity.
35        """
36        with self:
37            backup = None
38            if not inplace:
39                backup = '{}.before-upgrade-{:%Y-%m-%d-%H:%M:%S}'.format(self.path, datetime.datetime.now())
40                logger.info('making a hardlink copy in %s', backup)
41                if not dryrun:
42                    shutil.copytree(self.path, backup, copy_function=os.link)
43            logger.info("opening attic repository with borg and converting")
44            # now lock the repo, after we have made the copy
45            self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire()
46            segments = [filename for i, filename in self.io.segment_iterator()]
47            try:
48                keyfile = self.find_attic_keyfile()
49            except KeyfileNotFoundError:
50                logger.warning("no key file found for repository")
51            else:
52                self.convert_keyfiles(keyfile, dryrun)
53        # partial open: just hold on to the lock
54        self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True).acquire()
55        try:
56            self.convert_cache(dryrun)
57            self.convert_repo_index(dryrun=dryrun, inplace=inplace)
58            self.convert_segments(segments, dryrun=dryrun, inplace=inplace, progress=progress)
59            self.borg_readme()
60        finally:
61            self.lock.release()
62            self.lock = None
63        return backup
64
65    def borg_readme(self):
66        readme = os.path.join(self.path, 'README')
67        os.remove(readme)
68        with open(readme, 'w') as fd:
69            fd.write(REPOSITORY_README)
70
71    @staticmethod
72    def convert_segments(segments, dryrun=True, inplace=False, progress=False):
73        """convert repository segments from attic to borg
74
75        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
76        `$ATTIC_REPO/data/**`.
77
78        luckily the magic string length didn't change so we can just
79        replace the 8 first bytes of all regular files in there."""
80        logger.info("converting %d segments..." % len(segments))
81        segment_count = len(segments)
82        pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%", msgid='upgrade.convert_segments')
83        for i, filename in enumerate(segments):
84            if progress:
85                pi.show(i)
86            if dryrun:
87                time.sleep(0.001)
88            else:
89                AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace)
90        if progress:
91            pi.finish()
92
93    @staticmethod
94    def header_replace(filename, old_magic, new_magic, inplace=True):
95        with open(filename, 'r+b') as segment:
96            segment.seek(0)
97            # only write if necessary
98            if segment.read(len(old_magic)) == old_magic:
99                if inplace:
100                    segment.seek(0)
101                    segment.write(new_magic)
102                else:
103                    # rename the hardlink and rewrite the file. this works
104                    # because the file is still open. so even though the file
105                    # is renamed, we can still read it until it is closed.
106                    os.rename(filename, filename + '.tmp')
107                    with open(filename, 'wb') as new_segment:
108                        new_segment.write(new_magic)
109                        new_segment.write(segment.read())
110                    # the little dance with the .tmp file is necessary
111                    # because Windows won't allow overwriting an open file.
112                    os.unlink(filename + '.tmp')
113
114    def find_attic_keyfile(self):
115        """find the attic keyfiles
116
117        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
118        finds the keys with the right identifier for the repo.
119
120        this is expected to look into $HOME/.attic/keys or
121        $ATTIC_KEYS_DIR for key files matching the given Borg
122        repository.
123
124        it is expected to raise an exception (KeyfileNotFoundError) if
125        no key is found. whether that exception is from Borg or Attic
126        is unclear.
127
128        this is split in a separate function in case we want to use
129        the attic code here directly, instead of our local
130        implementation."""
131        return AtticKeyfileKey.find_key_file(self)
132
133    @staticmethod
134    def convert_keyfiles(keyfile, dryrun):
135        """convert key files from attic to borg
136
137        replacement pattern is `s/ATTIC KEY/BORG_KEY/` in
138        `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or
139        `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or
140        `$HOME/.config/borg/keys`.
141
142        no need to decrypt to convert. we need to rewrite the whole
143        key file because magic string length changed, but that's not a
144        problem because the keyfiles are small (compared to, say,
145        all the segments)."""
146        logger.info("converting keyfile %s" % keyfile)
147        with open(keyfile, 'r') as f:
148            data = f.read()
149        data = data.replace(AtticKeyfileKey.FILE_ID, KeyfileKey.FILE_ID, 1)
150        keyfile = os.path.join(get_keys_dir(), os.path.basename(keyfile))
151        logger.info("writing borg keyfile to %s" % keyfile)
152        if not dryrun:
153            with open(keyfile, 'w') as f:
154                f.write(data)
155
156    def convert_repo_index(self, dryrun, inplace):
157        """convert some repo files
158
159        those are all hash indexes, so we need to
160        `s/ATTICIDX/BORG_IDX/` in a few locations:
161
162        * the repository index (in `$ATTIC_REPO/index.%d`, where `%d`
163          is the `Repository.get_index_transaction_id()`), which we
164          should probably update, with a lock, see
165          `Repository.open()`, which i'm not sure we should use
166          because it may write data on `Repository.close()`...
167        """
168        transaction_id = self.get_index_transaction_id()
169        if transaction_id is None:
170            logger.warning('no index file found for repository %s' % self.path)
171        else:
172            index = os.path.join(self.path, 'index.%d' % transaction_id)
173            logger.info("converting repo index %s" % index)
174            if not dryrun:
175                AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX', inplace=inplace)
176
177    def convert_cache(self, dryrun):
178        """convert caches from attic to borg
179
180        those are all hash indexes, so we need to
181        `s/ATTICIDX/BORG_IDX/` in a few locations:
182
183        * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or
184          `$HOME/.cache/attic/<repoid>/`), which we could just drop,
185          but if we'd want to convert, we could open it with the
186          `Cache.open()`, edit in place and then `Cache.close()` to
187          make sure we have locking right
188        """
189        # copy of attic's get_cache_dir()
190        attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR',
191                                         os.path.join(get_base_dir(),
192                                                      '.cache', 'attic'))
193        attic_cache_dir = os.path.join(attic_cache_dir, self.id_str)
194        borg_cache_dir = os.path.join(get_cache_dir(), self.id_str)
195
196        def copy_cache_file(path):
197            """copy the given attic cache path into the borg directory
198
199            does nothing if dryrun is True. also expects
200            attic_cache_dir and borg_cache_dir to be set in the parent
201            scope, to the directories path including the repository
202            identifier.
203
204            :params path: the basename of the cache file to copy
205            (example: "files" or "chunks") as a string
206
207            :returns: the borg file that was created or None if no
208            Attic cache file was found.
209
210            """
211            attic_file = os.path.join(attic_cache_dir, path)
212            if os.path.exists(attic_file):
213                borg_file = os.path.join(borg_cache_dir, path)
214                if os.path.exists(borg_file):
215                    logger.warning("borg cache file already exists in %s, not copying from Attic", borg_file)
216                else:
217                    logger.info("copying attic cache file from %s to %s" % (attic_file, borg_file))
218                    if not dryrun:
219                        shutil.copyfile(attic_file, borg_file)
220                return borg_file
221            else:
222                logger.warning("no %s cache file found in %s" % (path, attic_file))
223                return None
224
225        # XXX: untested, because generating cache files is a PITA, see
226        # Archiver.do_create() for proof
227        if os.path.exists(attic_cache_dir):
228            if not os.path.exists(borg_cache_dir):
229                os.makedirs(borg_cache_dir)
230
231            # file that we don't have a header to convert, just copy
232            for cache in ['config', 'files']:
233                copy_cache_file(cache)
234
235            # we need to convert the headers of those files, copy first
236            for cache in ['chunks']:
237                cache = copy_cache_file(cache)
238                logger.info("converting cache %s" % cache)
239                if not dryrun:
240                    AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX')
241
242
243class AtticKeyfileKey(KeyfileKey):
244    """backwards compatible Attic key file parser"""
245    FILE_ID = 'ATTIC KEY'
246
247    # verbatim copy from attic
248    @staticmethod
249    def get_keys_dir():
250        """Determine where to repository keys and cache"""
251        return os.environ.get('ATTIC_KEYS_DIR',
252                              os.path.join(get_base_dir(), '.attic', 'keys'))
253
254    @classmethod
255    def find_key_file(cls, repository):
256        """copy of attic's `find_key_file`_
257
258        this has two small modifications:
259
260        1. it uses the above `get_keys_dir`_ instead of the global one,
261           assumed to be borg's
262
263        2. it uses `repository.path`_ instead of
264           `repository._location.canonical_path`_ because we can't
265           assume the repository has been opened by the archiver yet
266        """
267        get_keys_dir = cls.get_keys_dir
268        keys_dir = get_keys_dir()
269        if not os.path.exists(keys_dir):
270            raise KeyfileNotFoundError(repository.path, keys_dir)
271        for name in os.listdir(keys_dir):
272            filename = os.path.join(keys_dir, name)
273            with open(filename, 'r') as fd:
274                line = fd.readline().strip()
275                if line and line.startswith(cls.FILE_ID) and line[10:] == repository.id_str:
276                    return filename
277        raise KeyfileNotFoundError(repository.path, keys_dir)
278
279
280class BorgRepositoryUpgrader(Repository):
281    def upgrade(self, dryrun=True, inplace=False, progress=False):
282        """convert an old borg repository to a current borg repository
283        """
284        logger.info("converting borg 0.xx to borg current")
285        with self:
286            try:
287                keyfile = self.find_borg0xx_keyfile()
288            except KeyfileNotFoundError:
289                logger.warning("no key file found for repository")
290            else:
291                self.move_keyfiles(keyfile, dryrun)
292
293    def find_borg0xx_keyfile(self):
294        return Borg0xxKeyfileKey.find_key_file(self)
295
296    def move_keyfiles(self, keyfile, dryrun):
297        filename = os.path.basename(keyfile)
298        new_keyfile = os.path.join(get_keys_dir(), filename)
299        try:
300            os.rename(keyfile, new_keyfile)
301        except FileExistsError:
302            # likely the attic -> borg upgrader already put it in the final location
303            pass
304
305
306class Borg0xxKeyfileKey(KeyfileKey):
307    """backwards compatible borg 0.xx key file parser"""
308
309    @staticmethod
310    def get_keys_dir():
311        return os.environ.get('BORG_KEYS_DIR',
312                              os.path.join(get_base_dir(), '.borg', 'keys'))
313
314    @classmethod
315    def find_key_file(cls, repository):
316        get_keys_dir = cls.get_keys_dir
317        keys_dir = get_keys_dir()
318        if not os.path.exists(keys_dir):
319            raise KeyfileNotFoundError(repository.path, keys_dir)
320        for name in os.listdir(keys_dir):
321            filename = os.path.join(keys_dir, name)
322            with open(filename, 'r') as fd:
323                line = fd.readline().strip()
324                if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID) + 1:] == repository.id_str:
325                    return filename
326        raise KeyfileNotFoundError(repository.path, keys_dir)
327