1import argparse 2import errno 3import io 4import json 5import logging 6import os 7import pstats 8import random 9import shutil 10import socket 11import stat 12import subprocess 13import sys 14import tempfile 15import time 16import re 17import unittest 18from binascii import unhexlify, b2a_base64 19from configparser import ConfigParser 20from datetime import datetime 21from datetime import timezone 22from datetime import timedelta 23from hashlib import sha256 24from io import BytesIO, StringIO 25from unittest.mock import patch 26 27import pytest 28 29try: 30 import llfuse 31except ImportError: 32 pass 33 34import borg 35from .. import xattr, helpers, platform 36from ..archive import Archive, ChunkBuffer, flags_noatime, flags_normal 37from ..archiver import Archiver, parse_storage_quota, PURE_PYTHON_MSGPACK_WARNING 38from ..cache import Cache, LocalCache 39from ..constants import * # NOQA 40from ..crypto.low_level import bytes_to_long, num_aes_blocks 41from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError 42from ..crypto.keymanager import RepoIdMismatch, NotABorgKeyFile 43from ..crypto.file_integrity import FileIntegrityError 44from ..helpers import Location, get_security_dir 45from ..helpers import Manifest, MandatoryFeatureUnsupported 46from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR 47from ..helpers import bin_to_hex 48from ..helpers import MAX_S 49from ..helpers import msgpack 50from ..nanorst import RstToTextLazy, rst_to_terminal 51from ..patterns import IECommand, PatternMatcher, parse_pattern 52from ..item import Item 53from ..locking import LockFailed 54from ..logger import setup_logging 55from ..remote import RemoteRepository, PathNotAllowed 56from ..repository import Repository 57from . import has_lchflags, has_llfuse 58from . import BaseTestCase, changedir, environment_variable, no_selinux 59from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported 60from .platform import fakeroot_detected 61from .upgrader import make_attic_repo 62from . import key 63 64 65src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 66 67 68def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): 69 if fork: 70 try: 71 if exe is None: 72 borg = (sys.executable, '-m', 'borg.archiver') 73 elif isinstance(exe, str): 74 borg = (exe, ) 75 elif not isinstance(exe, tuple): 76 raise ValueError('exe must be None, a tuple or a str') 77 output = subprocess.check_output(borg + args, stderr=subprocess.STDOUT, input=input) 78 ret = 0 79 except subprocess.CalledProcessError as e: 80 output = e.output 81 ret = e.returncode 82 except SystemExit as e: # possibly raised by argparse 83 output = '' 84 ret = e.code 85 if binary_output: 86 return ret, output 87 else: 88 return ret, os.fsdecode(output) 89 else: 90 stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr 91 try: 92 sys.stdin = StringIO(input.decode()) 93 sys.stdin.buffer = BytesIO(input) 94 output = BytesIO() 95 # Always use utf-8 here, to simply .decode() below 96 output_text = sys.stdout = sys.stderr = io.TextIOWrapper(output, encoding='utf-8') 97 if archiver is None: 98 archiver = Archiver() 99 archiver.prerun_checks = lambda *args: None 100 archiver.exit_code = EXIT_SUCCESS 101 helpers.exit_code = EXIT_SUCCESS 102 try: 103 args = archiver.parse_args(list(args)) 104 # argparse parsing may raise SystemExit when the command line is bad or 105 # actions that abort early (eg. --help) where given. Catch this and return 106 # the error code as-if we invoked a Borg binary. 107 except SystemExit as e: 108 output_text.flush() 109 return e.code, output.getvalue() if binary_output else output.getvalue().decode() 110 ret = archiver.run(args) 111 output_text.flush() 112 return ret, output.getvalue() if binary_output else output.getvalue().decode() 113 finally: 114 sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr 115 116 117def have_gnutar(): 118 if not shutil.which('tar'): 119 return False 120 popen = subprocess.Popen(['tar', '--version'], stdout=subprocess.PIPE) 121 stdout, stderr = popen.communicate() 122 return b'GNU tar' in stdout 123 124 125# check if the binary "borg.exe" is available (for local testing a symlink to virtualenv/bin/borg should do) 126try: 127 exec_cmd('help', exe='borg.exe', fork=True) 128 BORG_EXES = ['python', 'binary', ] 129except FileNotFoundError: 130 BORG_EXES = ['python', ] 131 132 133@pytest.fixture(params=BORG_EXES) 134def cmd(request): 135 if request.param == 'python': 136 exe = None 137 elif request.param == 'binary': 138 exe = 'borg.exe' 139 else: 140 raise ValueError("param must be 'python' or 'binary'") 141 142 def exec_fn(*args, **kw): 143 return exec_cmd(*args, exe=exe, fork=True, **kw) 144 return exec_fn 145 146 147def test_return_codes(cmd, tmpdir): 148 repo = tmpdir.mkdir('repo') 149 input = tmpdir.mkdir('input') 150 output = tmpdir.mkdir('output') 151 input.join('test_file').write('content') 152 rc, out = cmd('init', '--encryption=none', '%s' % str(repo)) 153 assert rc == EXIT_SUCCESS 154 rc, out = cmd('create', '%s::archive' % repo, str(input)) 155 assert rc == EXIT_SUCCESS 156 with changedir(str(output)): 157 rc, out = cmd('extract', '%s::archive' % repo) 158 assert rc == EXIT_SUCCESS 159 rc, out = cmd('extract', '%s::archive' % repo, 'does/not/match') 160 assert rc == EXIT_WARNING # pattern did not match 161 rc, out = cmd('create', '%s::archive' % repo, str(input)) 162 assert rc == EXIT_ERROR # duplicate archive name 163 164 165""" 166test_disk_full is very slow and not recommended to be included in daily testing. 167for this test, an empty, writable 16MB filesystem mounted on DF_MOUNT is required. 168for speed and other reasons, it is recommended that the underlying block device is 169in RAM, not a magnetic or flash disk. 170 171assuming /tmp is a tmpfs (in memory filesystem), one can use this: 172dd if=/dev/zero of=/tmp/borg-disk bs=16M count=1 173mkfs.ext4 /tmp/borg-disk 174mkdir /tmp/borg-mount 175sudo mount /tmp/borg-disk /tmp/borg-mount 176 177if the directory does not exist, the test will be skipped. 178""" 179DF_MOUNT = '/tmp/borg-mount' 180 181 182@pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT) 183def test_disk_full(cmd): 184 def make_files(dir, count, size, rnd=True): 185 shutil.rmtree(dir, ignore_errors=True) 186 os.mkdir(dir) 187 if rnd: 188 count = random.randint(1, count) 189 if size > 1: 190 size = random.randint(1, size) 191 for i in range(count): 192 fn = os.path.join(dir, "file%03d" % i) 193 with open(fn, 'wb') as f: 194 data = os.urandom(size) 195 f.write(data) 196 197 with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='YES'): 198 mount = DF_MOUNT 199 assert os.path.exists(mount) 200 repo = os.path.join(mount, 'repo') 201 input = os.path.join(mount, 'input') 202 reserve = os.path.join(mount, 'reserve') 203 for j in range(100): 204 shutil.rmtree(repo, ignore_errors=True) 205 shutil.rmtree(input, ignore_errors=True) 206 # keep some space and some inodes in reserve that we can free up later: 207 make_files(reserve, 80, 100000, rnd=False) 208 rc, out = cmd('init', repo) 209 if rc != EXIT_SUCCESS: 210 print('init', rc, out) 211 assert rc == EXIT_SUCCESS 212 try: 213 success, i = True, 0 214 while success: 215 i += 1 216 try: 217 make_files(input, 20, 200000) 218 except OSError as err: 219 if err.errno == errno.ENOSPC: 220 # already out of space 221 break 222 raise 223 try: 224 rc, out = cmd('create', '%s::test%03d' % (repo, i), input) 225 success = rc == EXIT_SUCCESS 226 if not success: 227 print('create', rc, out) 228 finally: 229 # make sure repo is not locked 230 shutil.rmtree(os.path.join(repo, 'lock.exclusive'), ignore_errors=True) 231 os.remove(os.path.join(repo, 'lock.roster')) 232 finally: 233 # now some error happened, likely we are out of disk space. 234 # free some space so we can expect borg to be able to work normally: 235 shutil.rmtree(reserve, ignore_errors=True) 236 rc, out = cmd('list', repo) 237 if rc != EXIT_SUCCESS: 238 print('list', rc, out) 239 rc, out = cmd('check', '--repair', repo) 240 if rc != EXIT_SUCCESS: 241 print('check', rc, out) 242 assert rc == EXIT_SUCCESS 243 244 245class ArchiverTestCaseBase(BaseTestCase): 246 EXE = None # python source based 247 FORK_DEFAULT = False 248 prefix = '' 249 250 def setUp(self): 251 os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = 'YES' 252 os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' 253 os.environ['BORG_PASSPHRASE'] = 'waytooeasyonlyfortests' 254 os.environ['BORG_SELFTEST'] = 'disabled' 255 self.archiver = not self.FORK_DEFAULT and Archiver() or None 256 self.tmpdir = tempfile.mkdtemp() 257 self.repository_path = os.path.join(self.tmpdir, 'repository') 258 self.repository_location = self.prefix + self.repository_path 259 self.input_path = os.path.join(self.tmpdir, 'input') 260 self.output_path = os.path.join(self.tmpdir, 'output') 261 self.keys_path = os.path.join(self.tmpdir, 'keys') 262 self.cache_path = os.path.join(self.tmpdir, 'cache') 263 self.exclude_file_path = os.path.join(self.tmpdir, 'excludes') 264 self.patterns_file_path = os.path.join(self.tmpdir, 'patterns') 265 os.environ['BORG_KEYS_DIR'] = self.keys_path 266 os.environ['BORG_CACHE_DIR'] = self.cache_path 267 os.mkdir(self.input_path) 268 os.chmod(self.input_path, 0o777) # avoid troubles with fakeroot / FUSE 269 os.mkdir(self.output_path) 270 os.mkdir(self.keys_path) 271 os.mkdir(self.cache_path) 272 with open(self.exclude_file_path, 'wb') as fd: 273 fd.write(b'input/file2\n# A comment line, then a blank line\n\n') 274 with open(self.patterns_file_path, 'wb') as fd: 275 fd.write(b'+input/file_important\n- input/file*\n# A comment line, then a blank line\n\n') 276 self._old_wd = os.getcwd() 277 os.chdir(self.tmpdir) 278 279 def tearDown(self): 280 os.chdir(self._old_wd) 281 # note: ignore_errors=True as workaround for issue #862 282 shutil.rmtree(self.tmpdir, ignore_errors=True) 283 setup_logging() 284 285 def cmd(self, *args, **kw): 286 exit_code = kw.pop('exit_code', 0) 287 fork = kw.pop('fork', None) 288 binary_output = kw.get('binary_output', False) 289 if fork is None: 290 fork = self.FORK_DEFAULT 291 ret, output = exec_cmd(*args, fork=fork, exe=self.EXE, archiver=self.archiver, **kw) 292 if ret != exit_code: 293 print(output) 294 self.assert_equal(ret, exit_code) 295 # if tests are run with the pure-python msgpack, there will be warnings about 296 # this in the output, which would make a lot of tests fail. 297 pp_msg = PURE_PYTHON_MSGPACK_WARNING.encode() if binary_output else PURE_PYTHON_MSGPACK_WARNING 298 empty = b'' if binary_output else '' 299 output = empty.join(line for line in output.splitlines(keepends=True) 300 if pp_msg not in line) 301 return output 302 303 def create_src_archive(self, name): 304 self.cmd('create', '--compression=lz4', self.repository_location + '::' + name, src_dir) 305 306 def open_archive(self, name): 307 repository = Repository(self.repository_path, exclusive=True) 308 with repository: 309 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 310 archive = Archive(repository, key, manifest, name) 311 return archive, repository 312 313 def open_repository(self): 314 return Repository(self.repository_path, exclusive=True) 315 316 def create_regular_file(self, name, size=0, contents=None): 317 assert not (size != 0 and contents and len(contents) != size), 'size and contents do not match' 318 filename = os.path.join(self.input_path, name) 319 if not os.path.exists(os.path.dirname(filename)): 320 os.makedirs(os.path.dirname(filename)) 321 with open(filename, 'wb') as fd: 322 if contents is None: 323 contents = b'X' * size 324 fd.write(contents) 325 326 def create_test_files(self): 327 """Create a minimal test case including all supported file types 328 """ 329 # File 330 self.create_regular_file('file1', size=1024 * 80) 331 self.create_regular_file('flagfile', size=1024) 332 # Directory 333 self.create_regular_file('dir2/file2', size=1024 * 80) 334 # File mode 335 os.chmod('input/file1', 0o4755) 336 # Hard link 337 if are_hardlinks_supported(): 338 os.link(os.path.join(self.input_path, 'file1'), 339 os.path.join(self.input_path, 'hardlink')) 340 # Symlink 341 if are_symlinks_supported(): 342 os.symlink('somewhere', os.path.join(self.input_path, 'link1')) 343 self.create_regular_file('fusexattr', size=1) 344 if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): 345 # ironically, due to the way how fakeroot works, comparing FUSE file xattrs to orig file xattrs 346 # will FAIL if fakeroot supports xattrs, thus we only set the xattr if XATTR_FAKEROOT is False. 347 # This is because fakeroot with xattr-support does not propagate xattrs of the underlying file 348 # into "fakeroot space". Because the xattrs exposed by borgfs are these of an underlying file 349 # (from fakeroots point of view) they are invisible to the test process inside the fakeroot. 350 xattr.setxattr(os.path.join(self.input_path, 'fusexattr'), 'user.foo', b'bar') 351 xattr.setxattr(os.path.join(self.input_path, 'fusexattr'), 'user.empty', b'') 352 # XXX this always fails for me 353 # ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot 354 # same for newer ubuntu and centos. 355 # if this is supported just on specific platform, platform should be checked first, 356 # so that the test setup for all tests using it does not fail here always for others. 357 # xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False) 358 # FIFO node 359 if are_fifos_supported(): 360 os.mkfifo(os.path.join(self.input_path, 'fifo1')) 361 if has_lchflags: 362 platform.set_flags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP) 363 try: 364 # Block device 365 os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20)) 366 # Char device 367 os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40)) 368 # File mode 369 os.chmod('input/dir2', 0o555) # if we take away write perms, we need root to remove contents 370 # File owner 371 os.chown('input/file1', 100, 200) # raises OSError invalid argument on cygwin 372 have_root = True # we have (fake)root 373 except PermissionError: 374 have_root = False 375 except OSError as e: 376 # Note: ENOSYS "Function not implemented" happens as non-root on Win 10 Linux Subsystem. 377 if e.errno not in (errno.EINVAL, errno.ENOSYS): 378 raise 379 have_root = False 380 time.sleep(1) # "empty" must have newer timestamp than other files 381 self.create_regular_file('empty', size=0) 382 return have_root 383 384 385class ArchiverTestCase(ArchiverTestCaseBase): 386 requires_hardlinks = pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') 387 388 def test_basic_functionality(self): 389 have_root = self.create_test_files() 390 # fork required to test show-rc output 391 output = self.cmd('init', '--encryption=repokey', '--show-version', '--show-rc', self.repository_location, fork=True) 392 self.assert_in('borgbackup version', output) 393 self.assert_in('terminating with success status, rc 0', output) 394 self.cmd('create', '--exclude-nodump', self.repository_location + '::test', 'input') 395 output = self.cmd('create', '--exclude-nodump', '--stats', self.repository_location + '::test.2', 'input') 396 self.assert_in('Archive name: test.2', output) 397 self.assert_in('This archive: ', output) 398 with changedir('output'): 399 self.cmd('extract', self.repository_location + '::test') 400 list_output = self.cmd('list', '--short', self.repository_location) 401 self.assert_in('test', list_output) 402 self.assert_in('test.2', list_output) 403 expected = [ 404 'input', 405 'input/bdev', 406 'input/cdev', 407 'input/dir2', 408 'input/dir2/file2', 409 'input/empty', 410 'input/file1', 411 'input/flagfile', 412 ] 413 if are_fifos_supported(): 414 expected.append('input/fifo1') 415 if are_symlinks_supported(): 416 expected.append('input/link1') 417 if are_hardlinks_supported(): 418 expected.append('input/hardlink') 419 if not have_root: 420 # we could not create these device files without (fake)root 421 expected.remove('input/bdev') 422 expected.remove('input/cdev') 423 if has_lchflags: 424 # remove the file we did not backup, so input and output become equal 425 expected.remove('input/flagfile') # this file is UF_NODUMP 426 os.remove(os.path.join('input', 'flagfile')) 427 list_output = self.cmd('list', '--short', self.repository_location + '::test') 428 for name in expected: 429 self.assert_in(name, list_output) 430 self.assert_dirs_equal('input', 'output/input') 431 info_output = self.cmd('info', self.repository_location + '::test') 432 item_count = 4 if has_lchflags else 5 # one file is UF_NODUMP 433 self.assert_in('Number of files: %d' % item_count, info_output) 434 shutil.rmtree(self.cache_path) 435 info_output2 = self.cmd('info', self.repository_location + '::test') 436 437 def filter(output): 438 # filter for interesting "info" output, ignore cache rebuilding related stuff 439 prefixes = ['Name:', 'Fingerprint:', 'Number of files:', 'This archive:', 440 'All archives:', 'Chunk index:', ] 441 result = [] 442 for line in output.splitlines(): 443 for prefix in prefixes: 444 if line.startswith(prefix): 445 result.append(line) 446 return '\n'.join(result) 447 448 # the interesting parts of info_output2 and info_output should be same 449 self.assert_equal(filter(info_output), filter(info_output2)) 450 451 @requires_hardlinks 452 def test_create_duplicate_root(self): 453 # setup for #5603 454 path_a = os.path.join(self.input_path, 'a') 455 path_b = os.path.join(self.input_path, 'b') 456 os.mkdir(path_a) 457 os.mkdir(path_b) 458 hl_a = os.path.join(path_a, 'hardlink') 459 hl_b = os.path.join(path_b, 'hardlink') 460 self.create_regular_file(hl_a, contents=b'123456') 461 os.link(hl_a, hl_b) 462 self.cmd('init', '--encryption=none', self.repository_location) 463 self.cmd('create', self.repository_location + '::test', 'input', 'input') # give input twice! 464 # test if created archive has 'input' contents twice: 465 archive_list = self.cmd('list', '--json-lines', self.repository_location + '::test') 466 paths = [json.loads(line)['path'] for line in archive_list.split('\n') if line] 467 # we have all fs items exactly once! 468 assert sorted(paths) == ['input', 'input/a', 'input/a/hardlink', 'input/b', 'input/b/hardlink'] 469 470 def test_init_parent_dirs(self): 471 parent_path = os.path.join(self.tmpdir, 'parent1', 'parent2') 472 repository_path = os.path.join(parent_path, 'repository') 473 repository_location = self.prefix + repository_path 474 with pytest.raises(Repository.ParentPathDoesNotExist): 475 # normal borg init does NOT create missing parent dirs 476 self.cmd('init', '--encryption=none', repository_location) 477 # but if told so, it does: 478 self.cmd('init', '--encryption=none', '--make-parent-dirs', repository_location) 479 assert os.path.exists(parent_path) 480 481 def test_unix_socket(self): 482 self.cmd('init', '--encryption=repokey', self.repository_location) 483 try: 484 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 485 sock.bind(os.path.join(self.input_path, 'unix-socket')) 486 except PermissionError as err: 487 if err.errno == errno.EPERM: 488 pytest.skip('unix sockets disabled or not supported') 489 elif err.errno == errno.EACCES: 490 pytest.skip('permission denied to create unix sockets') 491 self.cmd('create', self.repository_location + '::test', 'input') 492 sock.close() 493 with changedir('output'): 494 self.cmd('extract', self.repository_location + '::test') 495 assert not os.path.exists('input/unix-socket') 496 497 @pytest.mark.skipif(not are_symlinks_supported(), reason='symlinks not supported') 498 def test_symlink_extract(self): 499 self.create_test_files() 500 self.cmd('init', '--encryption=repokey', self.repository_location) 501 self.cmd('create', self.repository_location + '::test', 'input') 502 with changedir('output'): 503 self.cmd('extract', self.repository_location + '::test') 504 assert os.readlink('input/link1') == 'somewhere' 505 506 @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') 507 def test_atime(self): 508 def has_noatime(some_file): 509 atime_before = os.stat(some_file).st_atime_ns 510 try: 511 with open(os.open(some_file, flags_noatime)) as file: 512 file.read() 513 except PermissionError: 514 return False 515 else: 516 atime_after = os.stat(some_file).st_atime_ns 517 noatime_used = flags_noatime != flags_normal 518 return noatime_used and atime_before == atime_after 519 520 self.create_test_files() 521 atime, mtime = 123456780, 234567890 522 have_noatime = has_noatime('input/file1') 523 os.utime('input/file1', (atime, mtime)) 524 self.cmd('init', '--encryption=repokey', self.repository_location) 525 self.cmd('create', self.repository_location + '::test', 'input') 526 with changedir('output'): 527 self.cmd('extract', self.repository_location + '::test') 528 sti = os.stat('input/file1') 529 sto = os.stat('output/input/file1') 530 assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 531 if have_noatime: 532 assert sti.st_atime_ns == sto.st_atime_ns == atime * 1e9 533 else: 534 # it touched the input file's atime while backing it up 535 assert sto.st_atime_ns == atime * 1e9 536 537 @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') 538 @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime') 539 def test_birthtime(self): 540 self.create_test_files() 541 birthtime, mtime, atime = 946598400, 946684800, 946771200 542 os.utime('input/file1', (atime, birthtime)) 543 os.utime('input/file1', (atime, mtime)) 544 self.cmd('init', '--encryption=repokey', self.repository_location) 545 self.cmd('create', self.repository_location + '::test', 'input') 546 with changedir('output'): 547 self.cmd('extract', self.repository_location + '::test') 548 sti = os.stat('input/file1') 549 sto = os.stat('output/input/file1') 550 assert int(sti.st_birthtime * 1e9) == int(sto.st_birthtime * 1e9) == birthtime * 1e9 551 assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 552 553 @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') 554 @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime') 555 def test_nobirthtime(self): 556 self.create_test_files() 557 birthtime, mtime, atime = 946598400, 946684800, 946771200 558 os.utime('input/file1', (atime, birthtime)) 559 os.utime('input/file1', (atime, mtime)) 560 self.cmd('init', '--encryption=repokey', self.repository_location) 561 self.cmd('create', '--nobirthtime', self.repository_location + '::test', 'input') 562 with changedir('output'): 563 self.cmd('extract', self.repository_location + '::test') 564 sti = os.stat('input/file1') 565 sto = os.stat('output/input/file1') 566 assert int(sti.st_birthtime * 1e9) == birthtime * 1e9 567 assert int(sto.st_birthtime * 1e9) == mtime * 1e9 568 assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 569 570 def _extract_repository_id(self, path): 571 with Repository(self.repository_path) as repository: 572 return repository.id 573 574 def _set_repository_id(self, path, id): 575 config = ConfigParser(interpolation=None) 576 config.read(os.path.join(path, 'config')) 577 config.set('repository', 'id', bin_to_hex(id)) 578 with open(os.path.join(path, 'config'), 'w') as fd: 579 config.write(fd) 580 with Repository(self.repository_path) as repository: 581 return repository.id 582 583 def test_sparse_file(self): 584 def is_sparse(fn, total_size, hole_size): 585 st = os.stat(fn) 586 assert st.st_size == total_size 587 sparse = True 588 if sparse and hasattr(st, 'st_blocks') and st.st_blocks * 512 >= st.st_size: 589 sparse = False 590 if sparse and hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): 591 with open(fn, 'rb') as fd: 592 # only check if the first hole is as expected, because the 2nd hole check 593 # is problematic on xfs due to its "dynamic speculative EOF preallocation 594 try: 595 if fd.seek(0, os.SEEK_HOLE) != 0: 596 sparse = False 597 if fd.seek(0, os.SEEK_DATA) != hole_size: 598 sparse = False 599 except OSError: 600 # OS/FS does not really support SEEK_HOLE/SEEK_DATA 601 sparse = False 602 return sparse 603 604 filename = os.path.join(self.input_path, 'sparse') 605 content = b'foobar' 606 hole_size = 5 * (1 << CHUNK_MAX_EXP) # 5 full chunker buffers 607 total_size = hole_size + len(content) + hole_size 608 with open(filename, 'wb') as fd: 609 # create a file that has a hole at the beginning and end (if the 610 # OS and filesystem supports sparse files) 611 fd.seek(hole_size, 1) 612 fd.write(content) 613 fd.seek(hole_size, 1) 614 pos = fd.tell() 615 fd.truncate(pos) 616 # we first check if we could create a sparse input file: 617 sparse_support = is_sparse(filename, total_size, hole_size) 618 if sparse_support: 619 # we could create a sparse input file, so creating a backup of it and 620 # extracting it again (as sparse) should also work: 621 self.cmd('init', '--encryption=repokey', self.repository_location) 622 self.cmd('create', self.repository_location + '::test', 'input') 623 with changedir(self.output_path): 624 self.cmd('extract', '--sparse', self.repository_location + '::test') 625 self.assert_dirs_equal('input', 'output/input') 626 filename = os.path.join(self.output_path, 'input', 'sparse') 627 with open(filename, 'rb') as fd: 628 # check if file contents are as expected 629 self.assert_equal(fd.read(hole_size), b'\0' * hole_size) 630 self.assert_equal(fd.read(len(content)), content) 631 self.assert_equal(fd.read(hole_size), b'\0' * hole_size) 632 self.assert_true(is_sparse(filename, total_size, hole_size)) 633 634 def test_unusual_filenames(self): 635 filenames = ['normal', 'with some blanks', '(with_parens)', ] 636 for filename in filenames: 637 filename = os.path.join(self.input_path, filename) 638 with open(filename, 'wb'): 639 pass 640 self.cmd('init', '--encryption=repokey', self.repository_location) 641 self.cmd('create', self.repository_location + '::test', 'input') 642 for filename in filenames: 643 with changedir('output'): 644 self.cmd('extract', self.repository_location + '::test', os.path.join('input', filename)) 645 assert os.path.exists(os.path.join('output', 'input', filename)) 646 647 def test_repository_swap_detection(self): 648 self.create_test_files() 649 os.environ['BORG_PASSPHRASE'] = 'passphrase' 650 self.cmd('init', '--encryption=repokey', self.repository_location) 651 repository_id = self._extract_repository_id(self.repository_path) 652 self.cmd('create', self.repository_location + '::test', 'input') 653 shutil.rmtree(self.repository_path) 654 self.cmd('init', '--encryption=none', self.repository_location) 655 self._set_repository_id(self.repository_path, repository_id) 656 self.assert_equal(repository_id, self._extract_repository_id(self.repository_path)) 657 if self.FORK_DEFAULT: 658 self.cmd('create', self.repository_location + '::test.2', 'input', exit_code=EXIT_ERROR) 659 else: 660 with pytest.raises(Cache.EncryptionMethodMismatch): 661 self.cmd('create', self.repository_location + '::test.2', 'input') 662 663 def test_repository_swap_detection2(self): 664 self.create_test_files() 665 self.cmd('init', '--encryption=none', self.repository_location + '_unencrypted') 666 os.environ['BORG_PASSPHRASE'] = 'passphrase' 667 self.cmd('init', '--encryption=repokey', self.repository_location + '_encrypted') 668 self.cmd('create', self.repository_location + '_encrypted::test', 'input') 669 shutil.rmtree(self.repository_path + '_encrypted') 670 os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') 671 if self.FORK_DEFAULT: 672 self.cmd('create', self.repository_location + '_encrypted::test.2', 'input', exit_code=EXIT_ERROR) 673 else: 674 with pytest.raises(Cache.RepositoryAccessAborted): 675 self.cmd('create', self.repository_location + '_encrypted::test.2', 'input') 676 677 def test_repository_swap_detection_no_cache(self): 678 self.create_test_files() 679 os.environ['BORG_PASSPHRASE'] = 'passphrase' 680 self.cmd('init', '--encryption=repokey', self.repository_location) 681 repository_id = self._extract_repository_id(self.repository_path) 682 self.cmd('create', self.repository_location + '::test', 'input') 683 shutil.rmtree(self.repository_path) 684 self.cmd('init', '--encryption=none', self.repository_location) 685 self._set_repository_id(self.repository_path, repository_id) 686 self.assert_equal(repository_id, self._extract_repository_id(self.repository_path)) 687 self.cmd('delete', '--cache-only', self.repository_location) 688 if self.FORK_DEFAULT: 689 self.cmd('create', self.repository_location + '::test.2', 'input', exit_code=EXIT_ERROR) 690 else: 691 with pytest.raises(Cache.EncryptionMethodMismatch): 692 self.cmd('create', self.repository_location + '::test.2', 'input') 693 694 def test_repository_swap_detection2_no_cache(self): 695 self.create_test_files() 696 self.cmd('init', '--encryption=none', self.repository_location + '_unencrypted') 697 os.environ['BORG_PASSPHRASE'] = 'passphrase' 698 self.cmd('init', '--encryption=repokey', self.repository_location + '_encrypted') 699 self.cmd('create', self.repository_location + '_encrypted::test', 'input') 700 self.cmd('delete', '--cache-only', self.repository_location + '_unencrypted') 701 self.cmd('delete', '--cache-only', self.repository_location + '_encrypted') 702 shutil.rmtree(self.repository_path + '_encrypted') 703 os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') 704 if self.FORK_DEFAULT: 705 self.cmd('create', self.repository_location + '_encrypted::test.2', 'input', exit_code=EXIT_ERROR) 706 else: 707 with pytest.raises(Cache.RepositoryAccessAborted): 708 self.cmd('create', self.repository_location + '_encrypted::test.2', 'input') 709 710 def test_repository_swap_detection_repokey_blank_passphrase(self): 711 # Check that a repokey repo with a blank passphrase is considered like a plaintext repo. 712 self.create_test_files() 713 # User initializes her repository with her passphrase 714 self.cmd('init', '--encryption=repokey', self.repository_location) 715 self.cmd('create', self.repository_location + '::test', 'input') 716 # Attacker replaces it with her own repository, which is encrypted but has no passphrase set 717 shutil.rmtree(self.repository_path) 718 with environment_variable(BORG_PASSPHRASE=''): 719 self.cmd('init', '--encryption=repokey', self.repository_location) 720 # Delete cache & security database, AKA switch to user perspective 721 self.cmd('delete', '--cache-only', self.repository_location) 722 repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) 723 shutil.rmtree(get_security_dir(repository_id)) 724 with environment_variable(BORG_PASSPHRASE=None): 725 # This is the part were the user would be tricked, e.g. she assumes that BORG_PASSPHRASE 726 # is set, while it isn't. Previously this raised no warning, 727 # since the repository is, technically, encrypted. 728 if self.FORK_DEFAULT: 729 self.cmd('create', self.repository_location + '::test.2', 'input', exit_code=EXIT_ERROR) 730 else: 731 with pytest.raises(Cache.CacheInitAbortedError): 732 self.cmd('create', self.repository_location + '::test.2', 'input') 733 734 def test_repository_move(self): 735 self.cmd('init', '--encryption=repokey', self.repository_location) 736 repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) 737 os.rename(self.repository_path, self.repository_path + '_new') 738 with environment_variable(BORG_RELOCATED_REPO_ACCESS_IS_OK='yes'): 739 self.cmd('info', self.repository_location + '_new') 740 security_dir = get_security_dir(repository_id) 741 with open(os.path.join(security_dir, 'location')) as fd: 742 location = fd.read() 743 assert location == Location(self.repository_location + '_new').canonical_path() 744 # Needs no confirmation anymore 745 self.cmd('info', self.repository_location + '_new') 746 shutil.rmtree(self.cache_path) 747 self.cmd('info', self.repository_location + '_new') 748 shutil.rmtree(security_dir) 749 self.cmd('info', self.repository_location + '_new') 750 for file in ('location', 'key-type', 'manifest-timestamp'): 751 assert os.path.exists(os.path.join(security_dir, file)) 752 753 def test_security_dir_compat(self): 754 self.cmd('init', '--encryption=repokey', self.repository_location) 755 repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) 756 security_dir = get_security_dir(repository_id) 757 with open(os.path.join(security_dir, 'location'), 'w') as fd: 758 fd.write('something outdated') 759 # This is fine, because the cache still has the correct information. security_dir and cache can disagree 760 # if older versions are used to confirm a renamed repository. 761 self.cmd('info', self.repository_location) 762 763 def test_unknown_unencrypted(self): 764 self.cmd('init', '--encryption=none', self.repository_location) 765 repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) 766 security_dir = get_security_dir(repository_id) 767 # Ok: repository is known 768 self.cmd('info', self.repository_location) 769 770 # Ok: repository is still known (through security_dir) 771 shutil.rmtree(self.cache_path) 772 self.cmd('info', self.repository_location) 773 774 # Needs confirmation: cache and security dir both gone (eg. another host or rm -rf ~) 775 shutil.rmtree(self.cache_path) 776 shutil.rmtree(security_dir) 777 if self.FORK_DEFAULT: 778 self.cmd('info', self.repository_location, exit_code=EXIT_ERROR) 779 else: 780 with pytest.raises(Cache.CacheInitAbortedError): 781 self.cmd('info', self.repository_location) 782 with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'): 783 self.cmd('info', self.repository_location) 784 785 def test_strip_components(self): 786 self.cmd('init', '--encryption=repokey', self.repository_location) 787 self.create_regular_file('dir/file') 788 self.cmd('create', self.repository_location + '::test', 'input') 789 with changedir('output'): 790 self.cmd('extract', self.repository_location + '::test', '--strip-components', '3') 791 self.assert_true(not os.path.exists('file')) 792 with self.assert_creates_file('file'): 793 self.cmd('extract', self.repository_location + '::test', '--strip-components', '2') 794 with self.assert_creates_file('dir/file'): 795 self.cmd('extract', self.repository_location + '::test', '--strip-components', '1') 796 with self.assert_creates_file('input/dir/file'): 797 self.cmd('extract', self.repository_location + '::test', '--strip-components', '0') 798 799 def _extract_hardlinks_setup(self): 800 os.mkdir(os.path.join(self.input_path, 'dir1')) 801 os.mkdir(os.path.join(self.input_path, 'dir1/subdir')) 802 803 self.create_regular_file('source', contents=b'123456') 804 os.link(os.path.join(self.input_path, 'source'), 805 os.path.join(self.input_path, 'abba')) 806 os.link(os.path.join(self.input_path, 'source'), 807 os.path.join(self.input_path, 'dir1/hardlink')) 808 os.link(os.path.join(self.input_path, 'source'), 809 os.path.join(self.input_path, 'dir1/subdir/hardlink')) 810 811 self.create_regular_file('dir1/source2') 812 os.link(os.path.join(self.input_path, 'dir1/source2'), 813 os.path.join(self.input_path, 'dir1/aaaa')) 814 815 self.cmd('init', '--encryption=repokey', self.repository_location) 816 self.cmd('create', self.repository_location + '::test', 'input') 817 818 @requires_hardlinks 819 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 820 def test_fuse_mount_hardlinks(self): 821 self._extract_hardlinks_setup() 822 mountpoint = os.path.join(self.tmpdir, 'mountpoint') 823 # we need to get rid of permissions checking because fakeroot causes issues with it. 824 # On all platforms, borg defaults to "default_permissions" and we need to get rid of it via "ignore_permissions". 825 # On macOS (darwin), we additionally need "defer_permissions" to switch off the checks in osxfuse. 826 if sys.platform == 'darwin': 827 ignore_perms = ['-o', 'ignore_permissions,defer_permissions'] 828 else: 829 ignore_perms = ['-o', 'ignore_permissions'] 830 with self.fuse_mount(self.repository_location + '::test', mountpoint, '--strip-components=2', *ignore_perms), \ 831 changedir(mountpoint): 832 assert os.stat('hardlink').st_nlink == 2 833 assert os.stat('subdir/hardlink').st_nlink == 2 834 assert open('subdir/hardlink', 'rb').read() == b'123456' 835 assert os.stat('aaaa').st_nlink == 2 836 assert os.stat('source2').st_nlink == 2 837 with self.fuse_mount(self.repository_location + '::test', mountpoint, 'input/dir1', *ignore_perms), \ 838 changedir(mountpoint): 839 assert os.stat('input/dir1/hardlink').st_nlink == 2 840 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 841 assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' 842 assert os.stat('input/dir1/aaaa').st_nlink == 2 843 assert os.stat('input/dir1/source2').st_nlink == 2 844 with self.fuse_mount(self.repository_location + '::test', mountpoint, *ignore_perms), \ 845 changedir(mountpoint): 846 assert os.stat('input/source').st_nlink == 4 847 assert os.stat('input/abba').st_nlink == 4 848 assert os.stat('input/dir1/hardlink').st_nlink == 4 849 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4 850 assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' 851 852 @requires_hardlinks 853 def test_extract_hardlinks1(self): 854 self._extract_hardlinks_setup() 855 with changedir('output'): 856 self.cmd('extract', self.repository_location + '::test') 857 assert os.stat('input/source').st_nlink == 4 858 assert os.stat('input/abba').st_nlink == 4 859 assert os.stat('input/dir1/hardlink').st_nlink == 4 860 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4 861 assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' 862 863 @requires_hardlinks 864 def test_extract_hardlinks2(self): 865 self._extract_hardlinks_setup() 866 with changedir('output'): 867 self.cmd('extract', self.repository_location + '::test', '--strip-components', '2') 868 assert os.stat('hardlink').st_nlink == 2 869 assert os.stat('subdir/hardlink').st_nlink == 2 870 assert open('subdir/hardlink', 'rb').read() == b'123456' 871 assert os.stat('aaaa').st_nlink == 2 872 assert os.stat('source2').st_nlink == 2 873 with changedir('output'): 874 self.cmd('extract', self.repository_location + '::test', 'input/dir1') 875 assert os.stat('input/dir1/hardlink').st_nlink == 2 876 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 877 assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' 878 assert os.stat('input/dir1/aaaa').st_nlink == 2 879 assert os.stat('input/dir1/source2').st_nlink == 2 880 881 @requires_hardlinks 882 def test_extract_hardlinks_twice(self): 883 # setup for #5603 884 path_a = os.path.join(self.input_path, 'a') 885 path_b = os.path.join(self.input_path, 'b') 886 os.mkdir(path_a) 887 os.mkdir(path_b) 888 hl_a = os.path.join(path_a, 'hardlink') 889 hl_b = os.path.join(path_b, 'hardlink') 890 self.create_regular_file(hl_a, contents=b'123456') 891 os.link(hl_a, hl_b) 892 self.cmd('init', '--encryption=none', self.repository_location) 893 self.cmd('create', self.repository_location + '::test', 'input', 'input') # give input twice! 894 # now test extraction 895 with changedir('output'): 896 self.cmd('extract', self.repository_location + '::test') 897 # if issue #5603 happens, extraction gives rc == 1 (triggering AssertionError) and warnings like: 898 # input/a/hardlink: link: [Errno 2] No such file or directory: 'input/a/hardlink' -> 'input/a/hardlink' 899 # input/b/hardlink: link: [Errno 2] No such file or directory: 'input/a/hardlink' -> 'input/b/hardlink' 900 # otherwise, when fixed, the hardlinks should be there and have a link count of 2 901 assert os.stat('input/a/hardlink').st_nlink == 2 902 assert os.stat('input/b/hardlink').st_nlink == 2 903 904 def test_extract_include_exclude(self): 905 self.cmd('init', '--encryption=repokey', self.repository_location) 906 self.create_regular_file('file1', size=1024 * 80) 907 self.create_regular_file('file2', size=1024 * 80) 908 self.create_regular_file('file3', size=1024 * 80) 909 self.create_regular_file('file4', size=1024 * 80) 910 self.cmd('create', '--exclude=input/file4', self.repository_location + '::test', 'input') 911 with changedir('output'): 912 self.cmd('extract', self.repository_location + '::test', 'input/file1', ) 913 self.assert_equal(sorted(os.listdir('output/input')), ['file1']) 914 with changedir('output'): 915 self.cmd('extract', '--exclude=input/file2', self.repository_location + '::test') 916 self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3']) 917 with changedir('output'): 918 self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') 919 self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3']) 920 921 def test_extract_include_exclude_regex(self): 922 self.cmd('init', '--encryption=repokey', self.repository_location) 923 self.create_regular_file('file1', size=1024 * 80) 924 self.create_regular_file('file2', size=1024 * 80) 925 self.create_regular_file('file3', size=1024 * 80) 926 self.create_regular_file('file4', size=1024 * 80) 927 self.create_regular_file('file333', size=1024 * 80) 928 929 # Create with regular expression exclusion for file4 930 self.cmd('create', '--exclude=re:input/file4$', self.repository_location + '::test', 'input') 931 with changedir('output'): 932 self.cmd('extract', self.repository_location + '::test') 933 self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2', 'file3', 'file333']) 934 shutil.rmtree('output/input') 935 936 # Extract with regular expression exclusion 937 with changedir('output'): 938 self.cmd('extract', '--exclude=re:file3+', self.repository_location + '::test') 939 self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2']) 940 shutil.rmtree('output/input') 941 942 # Combine --exclude with fnmatch and regular expression 943 with changedir('output'): 944 self.cmd('extract', '--exclude=input/file2', '--exclude=re:file[01]', self.repository_location + '::test') 945 self.assert_equal(sorted(os.listdir('output/input')), ['file3', 'file333']) 946 shutil.rmtree('output/input') 947 948 # Combine --exclude-from and regular expression exclusion 949 with changedir('output'): 950 self.cmd('extract', '--exclude-from=' + self.exclude_file_path, '--exclude=re:file1', 951 '--exclude=re:file(\\d)\\1\\1$', self.repository_location + '::test') 952 self.assert_equal(sorted(os.listdir('output/input')), ['file3']) 953 954 def test_extract_include_exclude_regex_from_file(self): 955 self.cmd('init', '--encryption=repokey', self.repository_location) 956 self.create_regular_file('file1', size=1024 * 80) 957 self.create_regular_file('file2', size=1024 * 80) 958 self.create_regular_file('file3', size=1024 * 80) 959 self.create_regular_file('file4', size=1024 * 80) 960 self.create_regular_file('file333', size=1024 * 80) 961 self.create_regular_file('aa:something', size=1024 * 80) 962 963 # Create while excluding using mixed pattern styles 964 with open(self.exclude_file_path, 'wb') as fd: 965 fd.write(b're:input/file4$\n') 966 fd.write(b'fm:*aa:*thing\n') 967 968 self.cmd('create', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test', 'input') 969 with changedir('output'): 970 self.cmd('extract', self.repository_location + '::test') 971 self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2', 'file3', 'file333']) 972 shutil.rmtree('output/input') 973 974 # Exclude using regular expression 975 with open(self.exclude_file_path, 'wb') as fd: 976 fd.write(b're:file3+\n') 977 978 with changedir('output'): 979 self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') 980 self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2']) 981 shutil.rmtree('output/input') 982 983 # Mixed exclude pattern styles 984 with open(self.exclude_file_path, 'wb') as fd: 985 fd.write(b're:file(\\d)\\1\\1$\n') 986 fd.write(b'fm:nothingwillmatchthis\n') 987 fd.write(b'*/file1\n') 988 fd.write(b're:file2$\n') 989 990 with changedir('output'): 991 self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') 992 self.assert_equal(sorted(os.listdir('output/input')), ['file3']) 993 994 def test_extract_with_pattern(self): 995 self.cmd("init", '--encryption=repokey', self.repository_location) 996 self.create_regular_file("file1", size=1024 * 80) 997 self.create_regular_file("file2", size=1024 * 80) 998 self.create_regular_file("file3", size=1024 * 80) 999 self.create_regular_file("file4", size=1024 * 80) 1000 self.create_regular_file("file333", size=1024 * 80) 1001 1002 self.cmd("create", self.repository_location + "::test", "input") 1003 1004 # Extract everything with regular expression 1005 with changedir("output"): 1006 self.cmd("extract", self.repository_location + "::test", "re:.*") 1007 self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file3", "file333", "file4"]) 1008 shutil.rmtree("output/input") 1009 1010 # Extract with pattern while also excluding files 1011 with changedir("output"): 1012 self.cmd("extract", "--exclude=re:file[34]$", self.repository_location + "::test", r"re:file\d$") 1013 self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2"]) 1014 shutil.rmtree("output/input") 1015 1016 # Combine --exclude with pattern for extraction 1017 with changedir("output"): 1018 self.cmd("extract", "--exclude=input/file1", self.repository_location + "::test", "re:file[12]$") 1019 self.assert_equal(sorted(os.listdir("output/input")), ["file2"]) 1020 shutil.rmtree("output/input") 1021 1022 # Multiple pattern 1023 with changedir("output"): 1024 self.cmd("extract", self.repository_location + "::test", "fm:input/file1", "fm:*file33*", "input/file2") 1025 self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file333"]) 1026 1027 def test_extract_list_output(self): 1028 self.cmd('init', '--encryption=repokey', self.repository_location) 1029 self.create_regular_file('file', size=1024 * 80) 1030 1031 self.cmd('create', self.repository_location + '::test', 'input') 1032 1033 with changedir('output'): 1034 output = self.cmd('extract', self.repository_location + '::test') 1035 self.assert_not_in("input/file", output) 1036 shutil.rmtree('output/input') 1037 1038 with changedir('output'): 1039 output = self.cmd('extract', '--info', self.repository_location + '::test') 1040 self.assert_not_in("input/file", output) 1041 shutil.rmtree('output/input') 1042 1043 with changedir('output'): 1044 output = self.cmd('extract', '--list', self.repository_location + '::test') 1045 self.assert_in("input/file", output) 1046 shutil.rmtree('output/input') 1047 1048 with changedir('output'): 1049 output = self.cmd('extract', '--list', '--info', self.repository_location + '::test') 1050 self.assert_in("input/file", output) 1051 1052 def test_extract_progress(self): 1053 self.cmd('init', '--encryption=repokey', self.repository_location) 1054 self.create_regular_file('file', size=1024 * 80) 1055 self.cmd('create', self.repository_location + '::test', 'input') 1056 1057 with changedir('output'): 1058 output = self.cmd('extract', self.repository_location + '::test', '--progress') 1059 assert 'Extracting:' in output 1060 1061 def _create_test_caches(self): 1062 self.cmd('init', '--encryption=repokey', self.repository_location) 1063 self.create_regular_file('file1', size=1024 * 80) 1064 self.create_regular_file('cache1/%s' % CACHE_TAG_NAME, 1065 contents=CACHE_TAG_CONTENTS + b' extra stuff') 1066 self.create_regular_file('cache2/%s' % CACHE_TAG_NAME, 1067 contents=b'invalid signature') 1068 os.mkdir('input/cache3') 1069 if are_hardlinks_supported(): 1070 os.link('input/cache1/%s' % CACHE_TAG_NAME, 'input/cache3/%s' % CACHE_TAG_NAME) 1071 else: 1072 self.create_regular_file('cache3/%s' % CACHE_TAG_NAME, 1073 contents=CACHE_TAG_CONTENTS + b' extra stuff') 1074 1075 def test_create_stdin(self): 1076 self.cmd('init', '--encryption=repokey', self.repository_location) 1077 input_data = b'\x00foo\n\nbar\n \n' 1078 self.cmd('create', self.repository_location + '::test', '-', input=input_data) 1079 item = json.loads(self.cmd('list', '--json-lines', self.repository_location + '::test')) 1080 assert item['uid'] == 0 1081 assert item['gid'] == 0 1082 assert item['size'] == len(input_data) 1083 assert item['path'] == 'stdin' 1084 extracted_data = self.cmd('extract', '--stdout', self.repository_location + '::test', binary_output=True) 1085 assert extracted_data == input_data 1086 1087 def test_create_without_root(self): 1088 """test create without a root""" 1089 self.cmd('init', '--encryption=repokey', self.repository_location) 1090 self.cmd('create', self.repository_location + '::test', exit_code=2) 1091 1092 def test_create_pattern_root(self): 1093 """test create with only a root pattern""" 1094 self.cmd('init', '--encryption=repokey', self.repository_location) 1095 self.create_regular_file('file1', size=1024 * 80) 1096 self.create_regular_file('file2', size=1024 * 80) 1097 output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test') 1098 self.assert_in("A input/file1", output) 1099 self.assert_in("A input/file2", output) 1100 1101 def test_create_pattern(self): 1102 """test file patterns during create""" 1103 self.cmd('init', '--encryption=repokey', self.repository_location) 1104 self.create_regular_file('file1', size=1024 * 80) 1105 self.create_regular_file('file2', size=1024 * 80) 1106 self.create_regular_file('file_important', size=1024 * 80) 1107 output = self.cmd('create', '-v', '--list', 1108 '--pattern=+input/file_important', '--pattern=-input/file*', 1109 self.repository_location + '::test', 'input') 1110 self.assert_in("A input/file_important", output) 1111 self.assert_in('x input/file1', output) 1112 self.assert_in('x input/file2', output) 1113 1114 def test_create_pattern_file(self): 1115 """test file patterns during create""" 1116 self.cmd('init', '--encryption=repokey', self.repository_location) 1117 self.create_regular_file('file1', size=1024 * 80) 1118 self.create_regular_file('file2', size=1024 * 80) 1119 self.create_regular_file('otherfile', size=1024 * 80) 1120 self.create_regular_file('file_important', size=1024 * 80) 1121 output = self.cmd('create', '-v', '--list', 1122 '--pattern=-input/otherfile', '--patterns-from=' + self.patterns_file_path, 1123 self.repository_location + '::test', 'input') 1124 self.assert_in("A input/file_important", output) 1125 self.assert_in('x input/file1', output) 1126 self.assert_in('x input/file2', output) 1127 self.assert_in('x input/otherfile', output) 1128 1129 def test_create_pattern_exclude_folder_but_recurse(self): 1130 """test when patterns exclude a parent folder, but include a child""" 1131 self.patterns_file_path2 = os.path.join(self.tmpdir, 'patterns2') 1132 with open(self.patterns_file_path2, 'wb') as fd: 1133 fd.write(b'+ input/x/b\n- input/x*\n') 1134 1135 self.cmd('init', '--encryption=repokey', self.repository_location) 1136 self.create_regular_file('x/a/foo_a', size=1024 * 80) 1137 self.create_regular_file('x/b/foo_b', size=1024 * 80) 1138 self.create_regular_file('y/foo_y', size=1024 * 80) 1139 output = self.cmd('create', '-v', '--list', 1140 '--patterns-from=' + self.patterns_file_path2, 1141 self.repository_location + '::test', 'input') 1142 self.assert_in('x input/x/a/foo_a', output) 1143 self.assert_in("A input/x/b/foo_b", output) 1144 self.assert_in('A input/y/foo_y', output) 1145 1146 def test_create_pattern_exclude_folder_no_recurse(self): 1147 """test when patterns exclude a parent folder and, but include a child""" 1148 self.patterns_file_path2 = os.path.join(self.tmpdir, 'patterns2') 1149 with open(self.patterns_file_path2, 'wb') as fd: 1150 fd.write(b'+ input/x/b\n! input/x*\n') 1151 1152 self.cmd('init', '--encryption=repokey', self.repository_location) 1153 self.create_regular_file('x/a/foo_a', size=1024 * 80) 1154 self.create_regular_file('x/b/foo_b', size=1024 * 80) 1155 self.create_regular_file('y/foo_y', size=1024 * 80) 1156 output = self.cmd('create', '-v', '--list', 1157 '--patterns-from=' + self.patterns_file_path2, 1158 self.repository_location + '::test', 'input') 1159 self.assert_not_in('input/x/a/foo_a', output) 1160 self.assert_not_in('input/x/a', output) 1161 self.assert_in('A input/y/foo_y', output) 1162 1163 def test_create_pattern_intermediate_folders_first(self): 1164 """test that intermediate folders appear first when patterns exclude a parent folder but include a child""" 1165 self.patterns_file_path2 = os.path.join(self.tmpdir, 'patterns2') 1166 with open(self.patterns_file_path2, 'wb') as fd: 1167 fd.write(b'+ input/x/a\n+ input/x/b\n- input/x*\n') 1168 1169 self.cmd('init', '--encryption=repokey', self.repository_location) 1170 1171 self.create_regular_file('x/a/foo_a', size=1024 * 80) 1172 self.create_regular_file('x/b/foo_b', size=1024 * 80) 1173 with changedir('input'): 1174 self.cmd('create', '--patterns-from=' + self.patterns_file_path2, 1175 self.repository_location + '::test', '.') 1176 1177 # list the archive and verify that the "intermediate" folders appear before 1178 # their contents 1179 out = self.cmd('list', '--format', '{type} {path}{NL}', self.repository_location + '::test') 1180 out_list = out.splitlines() 1181 1182 self.assert_in('d x/a', out_list) 1183 self.assert_in('d x/b', out_list) 1184 1185 assert out_list.index('d x/a') < out_list.index('- x/a/foo_a') 1186 assert out_list.index('d x/b') < out_list.index('- x/b/foo_b') 1187 1188 def test_create_no_cache_sync(self): 1189 self.create_test_files() 1190 self.cmd('init', '--encryption=repokey', self.repository_location) 1191 self.cmd('delete', '--cache-only', self.repository_location) 1192 create_json = json.loads(self.cmd('create', '--no-cache-sync', self.repository_location + '::test', 'input', 1193 '--json', '--error')) # ignore experimental warning 1194 info_json = json.loads(self.cmd('info', self.repository_location + '::test', '--json')) 1195 create_stats = create_json['cache']['stats'] 1196 info_stats = info_json['cache']['stats'] 1197 assert create_stats == info_stats 1198 self.cmd('delete', '--cache-only', self.repository_location) 1199 self.cmd('create', '--no-cache-sync', self.repository_location + '::test2', 'input') 1200 self.cmd('info', self.repository_location) 1201 self.cmd('check', self.repository_location) 1202 1203 def test_extract_pattern_opt(self): 1204 self.cmd('init', '--encryption=repokey', self.repository_location) 1205 self.create_regular_file('file1', size=1024 * 80) 1206 self.create_regular_file('file2', size=1024 * 80) 1207 self.create_regular_file('file_important', size=1024 * 80) 1208 self.cmd('create', self.repository_location + '::test', 'input') 1209 with changedir('output'): 1210 self.cmd('extract', 1211 '--pattern=+input/file_important', '--pattern=-input/file*', 1212 self.repository_location + '::test') 1213 self.assert_equal(sorted(os.listdir('output/input')), ['file_important']) 1214 1215 def _assert_test_caches(self): 1216 with changedir('output'): 1217 self.cmd('extract', self.repository_location + '::test') 1218 self.assert_equal(sorted(os.listdir('output/input')), ['cache2', 'file1']) 1219 self.assert_equal(sorted(os.listdir('output/input/cache2')), [CACHE_TAG_NAME]) 1220 1221 def test_exclude_caches(self): 1222 self._create_test_caches() 1223 self.cmd('create', '--exclude-caches', self.repository_location + '::test', 'input') 1224 self._assert_test_caches() 1225 1226 def test_recreate_exclude_caches(self): 1227 self._create_test_caches() 1228 self.cmd('create', self.repository_location + '::test', 'input') 1229 self.cmd('recreate', '--exclude-caches', self.repository_location + '::test') 1230 self._assert_test_caches() 1231 1232 def _create_test_tagged(self): 1233 self.cmd('init', '--encryption=repokey', self.repository_location) 1234 self.create_regular_file('file1', size=1024 * 80) 1235 self.create_regular_file('tagged1/.NOBACKUP') 1236 self.create_regular_file('tagged2/00-NOBACKUP') 1237 self.create_regular_file('tagged3/.NOBACKUP/file2', size=1024) 1238 1239 def _assert_test_tagged(self): 1240 with changedir('output'): 1241 self.cmd('extract', self.repository_location + '::test') 1242 self.assert_equal(sorted(os.listdir('output/input')), ['file1']) 1243 1244 def test_exclude_tagged(self): 1245 self._create_test_tagged() 1246 self.cmd('create', '--exclude-if-present', '.NOBACKUP', '--exclude-if-present', '00-NOBACKUP', self.repository_location + '::test', 'input') 1247 self._assert_test_tagged() 1248 1249 def test_recreate_exclude_tagged(self): 1250 self._create_test_tagged() 1251 self.cmd('create', self.repository_location + '::test', 'input') 1252 self.cmd('recreate', '--exclude-if-present', '.NOBACKUP', '--exclude-if-present', '00-NOBACKUP', 1253 self.repository_location + '::test') 1254 self._assert_test_tagged() 1255 1256 def _create_test_keep_tagged(self): 1257 self.cmd('init', '--encryption=repokey', self.repository_location) 1258 self.create_regular_file('file0', size=1024) 1259 self.create_regular_file('tagged1/.NOBACKUP1') 1260 self.create_regular_file('tagged1/file1', size=1024) 1261 self.create_regular_file('tagged2/.NOBACKUP2/subfile1', size=1024) 1262 self.create_regular_file('tagged2/file2', size=1024) 1263 self.create_regular_file('tagged3/%s' % CACHE_TAG_NAME, 1264 contents=CACHE_TAG_CONTENTS + b' extra stuff') 1265 self.create_regular_file('tagged3/file3', size=1024) 1266 self.create_regular_file('taggedall/.NOBACKUP1') 1267 self.create_regular_file('taggedall/.NOBACKUP2/subfile1', size=1024) 1268 self.create_regular_file('taggedall/%s' % CACHE_TAG_NAME, 1269 contents=CACHE_TAG_CONTENTS + b' extra stuff') 1270 self.create_regular_file('taggedall/file4', size=1024) 1271 1272 def _assert_test_keep_tagged(self): 1273 with changedir('output'): 1274 self.cmd('extract', self.repository_location + '::test') 1275 self.assert_equal(sorted(os.listdir('output/input')), ['file0', 'tagged1', 'tagged2', 'tagged3', 'taggedall']) 1276 self.assert_equal(os.listdir('output/input/tagged1'), ['.NOBACKUP1']) 1277 self.assert_equal(os.listdir('output/input/tagged2'), ['.NOBACKUP2']) 1278 self.assert_equal(os.listdir('output/input/tagged3'), [CACHE_TAG_NAME]) 1279 self.assert_equal(sorted(os.listdir('output/input/taggedall')), 1280 ['.NOBACKUP1', '.NOBACKUP2', CACHE_TAG_NAME, ]) 1281 1282 def test_exclude_keep_tagged_deprecation(self): 1283 self.cmd('init', '--encryption=repokey', self.repository_location) 1284 output_warn = self.cmd('create', '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', src_dir) 1285 self.assert_in('--keep-tag-files" has been deprecated.', output_warn) 1286 1287 def test_exclude_keep_tagged(self): 1288 self._create_test_keep_tagged() 1289 self.cmd('create', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2', 1290 '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test', 'input') 1291 self._assert_test_keep_tagged() 1292 1293 def test_recreate_exclude_keep_tagged(self): 1294 self._create_test_keep_tagged() 1295 self.cmd('create', self.repository_location + '::test', 'input') 1296 self.cmd('recreate', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2', 1297 '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test') 1298 self._assert_test_keep_tagged() 1299 1300 @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') 1301 def test_recreate_hardlinked_tags(self): # test for issue #4911 1302 self.cmd('init', '--encryption=none', self.repository_location) 1303 self.create_regular_file('file1', contents=CACHE_TAG_CONTENTS) # "wrong" filename, but correct tag contents 1304 os.mkdir(os.path.join(self.input_path, 'subdir')) # to make sure the tag is encountered *after* file1 1305 os.link(os.path.join(self.input_path, 'file1'), 1306 os.path.join(self.input_path, 'subdir', CACHE_TAG_NAME)) # correct tag name, hardlink to file1 1307 self.cmd('create', self.repository_location + '::test', 'input') 1308 # in the "test" archive, we now have, in this order: 1309 # - a regular file item for "file1" 1310 # - a hardlink item for "CACHEDIR.TAG" referring back to file1 for its contents 1311 self.cmd('recreate', '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test') 1312 # if issue #4911 is present, the recreate will crash with a KeyError for "input/file1" 1313 1314 @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='Linux capabilities test, requires fakeroot >= 1.20.2') 1315 def test_extract_capabilities(self): 1316 fchown = os.fchown 1317 1318 # We need to manually patch chown to get the behaviour Linux has, since fakeroot does not 1319 # accurately model the interaction of chown(2) and Linux capabilities, i.e. it does not remove them. 1320 def patched_fchown(fd, uid, gid): 1321 xattr.setxattr(fd, 'security.capability', None, follow_symlinks=False) 1322 fchown(fd, uid, gid) 1323 1324 # The capability descriptor used here is valid and taken from a /usr/bin/ping 1325 capabilities = b'\x01\x00\x00\x02\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' 1326 self.create_regular_file('file') 1327 xattr.setxattr('input/file', 'security.capability', capabilities) 1328 self.cmd('init', '--encryption=repokey', self.repository_location) 1329 self.cmd('create', self.repository_location + '::test', 'input') 1330 with changedir('output'): 1331 with patch.object(os, 'fchown', patched_fchown): 1332 self.cmd('extract', self.repository_location + '::test') 1333 assert xattr.getxattr('input/file', 'security.capability') == capabilities 1334 1335 @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='xattr not supported on this system or on this version of' 1336 'fakeroot') 1337 def test_extract_xattrs_errors(self): 1338 def patched_setxattr_E2BIG(*args, **kwargs): 1339 raise OSError(errno.E2BIG, 'E2BIG') 1340 1341 def patched_setxattr_ENOTSUP(*args, **kwargs): 1342 raise OSError(errno.ENOTSUP, 'ENOTSUP') 1343 1344 def patched_setxattr_EACCES(*args, **kwargs): 1345 raise OSError(errno.EACCES, 'EACCES') 1346 1347 self.create_regular_file('file') 1348 xattr.setxattr('input/file', 'user.attribute', 'value') 1349 self.cmd('init', self.repository_location, '-e' 'none') 1350 self.cmd('create', self.repository_location + '::test', 'input') 1351 with changedir('output'): 1352 input_abspath = os.path.abspath('input/file') 1353 with patch.object(xattr, 'setxattr', patched_setxattr_E2BIG): 1354 out = self.cmd('extract', self.repository_location + '::test', exit_code=EXIT_WARNING) 1355 assert out == (input_abspath + ': when setting extended attribute user.attribute: too big for this filesystem\n') 1356 os.remove(input_abspath) 1357 with patch.object(xattr, 'setxattr', patched_setxattr_ENOTSUP): 1358 out = self.cmd('extract', self.repository_location + '::test', exit_code=EXIT_WARNING) 1359 assert out == (input_abspath + ': when setting extended attribute user.attribute: xattrs not supported on this filesystem\n') 1360 os.remove(input_abspath) 1361 with patch.object(xattr, 'setxattr', patched_setxattr_EACCES): 1362 out = self.cmd('extract', self.repository_location + '::test', exit_code=EXIT_WARNING) 1363 assert out == (input_abspath + ': when setting extended attribute user.attribute: Permission denied\n') 1364 assert os.path.isfile(input_abspath) 1365 1366 def test_path_normalization(self): 1367 self.cmd('init', '--encryption=repokey', self.repository_location) 1368 self.create_regular_file('dir1/dir2/file', size=1024 * 80) 1369 with changedir('input/dir1/dir2'): 1370 self.cmd('create', self.repository_location + '::test', '../../../input/dir1/../dir1/dir2/..') 1371 output = self.cmd('list', self.repository_location + '::test') 1372 self.assert_not_in('..', output) 1373 self.assert_in(' input/dir1/dir2/file', output) 1374 1375 def test_exclude_normalization(self): 1376 self.cmd('init', '--encryption=repokey', self.repository_location) 1377 self.create_regular_file('file1', size=1024 * 80) 1378 self.create_regular_file('file2', size=1024 * 80) 1379 with changedir('input'): 1380 self.cmd('create', '--exclude=file1', self.repository_location + '::test1', '.') 1381 with changedir('output'): 1382 self.cmd('extract', self.repository_location + '::test1') 1383 self.assert_equal(sorted(os.listdir('output')), ['file2']) 1384 with changedir('input'): 1385 self.cmd('create', '--exclude=./file1', self.repository_location + '::test2', '.') 1386 with changedir('output'): 1387 self.cmd('extract', self.repository_location + '::test2') 1388 self.assert_equal(sorted(os.listdir('output')), ['file2']) 1389 self.cmd('create', '--exclude=input/./file1', self.repository_location + '::test3', 'input') 1390 with changedir('output'): 1391 self.cmd('extract', self.repository_location + '::test3') 1392 self.assert_equal(sorted(os.listdir('output/input')), ['file2']) 1393 1394 def test_repeated_files(self): 1395 self.create_regular_file('file1', size=1024 * 80) 1396 self.cmd('init', '--encryption=repokey', self.repository_location) 1397 self.cmd('create', self.repository_location + '::test', 'input', 'input') 1398 1399 def test_overwrite(self): 1400 self.create_regular_file('file1', size=1024 * 80) 1401 self.create_regular_file('dir2/file2', size=1024 * 80) 1402 self.cmd('init', '--encryption=repokey', self.repository_location) 1403 self.cmd('create', self.repository_location + '::test', 'input') 1404 # Overwriting regular files and directories should be supported 1405 os.mkdir('output/input') 1406 os.mkdir('output/input/file1') 1407 os.mkdir('output/input/dir2') 1408 with changedir('output'): 1409 self.cmd('extract', self.repository_location + '::test') 1410 self.assert_dirs_equal('input', 'output/input') 1411 # But non-empty dirs should fail 1412 os.unlink('output/input/file1') 1413 os.mkdir('output/input/file1') 1414 os.mkdir('output/input/file1/dir') 1415 with changedir('output'): 1416 self.cmd('extract', self.repository_location + '::test', exit_code=1) 1417 1418 def test_rename(self): 1419 self.create_regular_file('file1', size=1024 * 80) 1420 self.create_regular_file('dir2/file2', size=1024 * 80) 1421 self.cmd('init', '--encryption=repokey', self.repository_location) 1422 self.cmd('create', self.repository_location + '::test', 'input') 1423 self.cmd('create', self.repository_location + '::test.2', 'input') 1424 self.cmd('extract', '--dry-run', self.repository_location + '::test') 1425 self.cmd('extract', '--dry-run', self.repository_location + '::test.2') 1426 self.cmd('rename', self.repository_location + '::test', 'test.3') 1427 self.cmd('extract', '--dry-run', self.repository_location + '::test.2') 1428 self.cmd('rename', self.repository_location + '::test.2', 'test.4') 1429 self.cmd('extract', '--dry-run', self.repository_location + '::test.3') 1430 self.cmd('extract', '--dry-run', self.repository_location + '::test.4') 1431 # Make sure both archives have been renamed 1432 with Repository(self.repository_path) as repository: 1433 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1434 self.assert_equal(len(manifest.archives), 2) 1435 self.assert_in('test.3', manifest.archives) 1436 self.assert_in('test.4', manifest.archives) 1437 1438 def test_info(self): 1439 self.create_regular_file('file1', size=1024 * 80) 1440 self.cmd('init', '--encryption=repokey', self.repository_location) 1441 self.cmd('create', self.repository_location + '::test', 'input') 1442 info_repo = self.cmd('info', self.repository_location) 1443 assert 'All archives:' in info_repo 1444 info_archive = self.cmd('info', self.repository_location + '::test') 1445 assert 'Archive name: test\n' in info_archive 1446 info_archive = self.cmd('info', '--first', '1', self.repository_location) 1447 assert 'Archive name: test\n' in info_archive 1448 1449 def test_info_json(self): 1450 self.create_regular_file('file1', size=1024 * 80) 1451 self.cmd('init', '--encryption=repokey', self.repository_location) 1452 self.cmd('create', self.repository_location + '::test', 'input') 1453 info_repo = json.loads(self.cmd('info', '--json', self.repository_location)) 1454 repository = info_repo['repository'] 1455 assert len(repository['id']) == 64 1456 assert 'last_modified' in repository 1457 assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise 1458 assert info_repo['encryption']['mode'] == 'repokey' 1459 assert 'keyfile' not in info_repo['encryption'] 1460 cache = info_repo['cache'] 1461 stats = cache['stats'] 1462 assert all(isinstance(o, int) for o in stats.values()) 1463 assert all(key in stats for key in ('total_chunks', 'total_csize', 'total_size', 'total_unique_chunks', 'unique_csize', 'unique_size')) 1464 1465 info_archive = json.loads(self.cmd('info', '--json', self.repository_location + '::test')) 1466 assert info_repo['repository'] == info_archive['repository'] 1467 assert info_repo['cache'] == info_archive['cache'] 1468 archives = info_archive['archives'] 1469 assert len(archives) == 1 1470 archive = archives[0] 1471 assert archive['name'] == 'test' 1472 assert isinstance(archive['command_line'], list) 1473 assert isinstance(archive['duration'], float) 1474 assert len(archive['id']) == 64 1475 assert 'stats' in archive 1476 assert datetime.strptime(archive['start'], ISO_FORMAT) 1477 assert datetime.strptime(archive['end'], ISO_FORMAT) 1478 1479 def test_comment(self): 1480 self.create_regular_file('file1', size=1024 * 80) 1481 self.cmd('init', '--encryption=repokey', self.repository_location) 1482 self.cmd('create', self.repository_location + '::test1', 'input') 1483 self.cmd('create', '--comment', 'this is the comment', self.repository_location + '::test2', 'input') 1484 self.cmd('create', '--comment', '"deleted" comment', self.repository_location + '::test3', 'input') 1485 self.cmd('create', '--comment', 'preserved comment', self.repository_location + '::test4', 'input') 1486 assert 'Comment: \n' in self.cmd('info', self.repository_location + '::test1') 1487 assert 'Comment: this is the comment' in self.cmd('info', self.repository_location + '::test2') 1488 1489 self.cmd('recreate', self.repository_location + '::test1', '--comment', 'added comment') 1490 self.cmd('recreate', self.repository_location + '::test2', '--comment', 'modified comment') 1491 self.cmd('recreate', self.repository_location + '::test3', '--comment', '') 1492 self.cmd('recreate', self.repository_location + '::test4', '12345') 1493 assert 'Comment: added comment' in self.cmd('info', self.repository_location + '::test1') 1494 assert 'Comment: modified comment' in self.cmd('info', self.repository_location + '::test2') 1495 assert 'Comment: \n' in self.cmd('info', self.repository_location + '::test3') 1496 assert 'Comment: preserved comment' in self.cmd('info', self.repository_location + '::test4') 1497 1498 def test_delete(self): 1499 self.create_regular_file('file1', size=1024 * 80) 1500 self.create_regular_file('dir2/file2', size=1024 * 80) 1501 self.cmd('init', '--encryption=repokey', self.repository_location) 1502 self.cmd('create', self.repository_location + '::test', 'input') 1503 self.cmd('create', self.repository_location + '::test.2', 'input') 1504 self.cmd('create', self.repository_location + '::test.3', 'input') 1505 self.cmd('create', self.repository_location + '::another_test.1', 'input') 1506 self.cmd('create', self.repository_location + '::another_test.2', 'input') 1507 self.cmd('extract', '--dry-run', self.repository_location + '::test') 1508 self.cmd('extract', '--dry-run', self.repository_location + '::test.2') 1509 self.cmd('delete', '--prefix', 'another_', self.repository_location) 1510 self.cmd('delete', '--last', '1', self.repository_location) 1511 self.cmd('delete', self.repository_location + '::test') 1512 self.cmd('extract', '--dry-run', self.repository_location + '::test.2') 1513 output = self.cmd('delete', '--stats', self.repository_location + '::test.2') 1514 self.assert_in('Deleted data:', output) 1515 # Make sure all data except the manifest has been deleted 1516 with Repository(self.repository_path) as repository: 1517 self.assert_equal(len(repository), 1) 1518 1519 def test_delete_multiple(self): 1520 self.create_regular_file('file1', size=1024 * 80) 1521 self.cmd('init', '--encryption=repokey', self.repository_location) 1522 self.cmd('create', self.repository_location + '::test1', 'input') 1523 self.cmd('create', self.repository_location + '::test2', 'input') 1524 self.cmd('create', self.repository_location + '::test3', 'input') 1525 self.cmd('delete', self.repository_location + '::test1', 'test2') 1526 self.cmd('extract', '--dry-run', self.repository_location + '::test3') 1527 self.cmd('delete', self.repository_location, 'test3') 1528 assert not self.cmd('list', self.repository_location) 1529 1530 def test_delete_repo(self): 1531 self.create_regular_file('file1', size=1024 * 80) 1532 self.create_regular_file('dir2/file2', size=1024 * 80) 1533 self.cmd('init', '--encryption=repokey', self.repository_location) 1534 self.cmd('create', self.repository_location + '::test', 'input') 1535 self.cmd('create', self.repository_location + '::test.2', 'input') 1536 os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'no' 1537 self.cmd('delete', self.repository_location, exit_code=2) 1538 assert os.path.exists(self.repository_path) 1539 os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' 1540 self.cmd('delete', self.repository_location) 1541 # Make sure the repo is gone 1542 self.assertFalse(os.path.exists(self.repository_path)) 1543 1544 def test_delete_force(self): 1545 self.cmd('init', '--encryption=none', self.repository_location) 1546 self.create_src_archive('test') 1547 with Repository(self.repository_path, exclusive=True) as repository: 1548 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1549 archive = Archive(repository, key, manifest, 'test') 1550 for item in archive.iter_items(): 1551 if item.path.endswith('testsuite/archiver.py'): 1552 repository.delete(item.chunks[-1].id) 1553 break 1554 else: 1555 assert False # missed the file 1556 repository.commit() 1557 output = self.cmd('delete', '--force', self.repository_location + '::test') 1558 self.assert_in('deleted archive was corrupted', output) 1559 self.cmd('check', '--repair', self.repository_location) 1560 output = self.cmd('list', self.repository_location) 1561 self.assert_not_in('test', output) 1562 1563 def test_delete_double_force(self): 1564 self.cmd('init', '--encryption=none', self.repository_location) 1565 self.create_src_archive('test') 1566 with Repository(self.repository_path, exclusive=True) as repository: 1567 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1568 archive = Archive(repository, key, manifest, 'test') 1569 id = archive.metadata.items[0] 1570 repository.put(id, b'corrupted items metadata stream chunk') 1571 repository.commit() 1572 self.cmd('delete', '--force', '--force', self.repository_location + '::test') 1573 self.cmd('check', '--repair', self.repository_location) 1574 output = self.cmd('list', self.repository_location) 1575 self.assert_not_in('test', output) 1576 1577 def test_corrupted_repository(self): 1578 self.cmd('init', '--encryption=repokey', self.repository_location) 1579 self.create_src_archive('test') 1580 self.cmd('extract', '--dry-run', self.repository_location + '::test') 1581 output = self.cmd('check', '--show-version', self.repository_location) 1582 self.assert_in('borgbackup version', output) # implied output even without --info given 1583 self.assert_not_in('Starting repository check', output) # --info not given for root logger 1584 1585 name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[1] 1586 with open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+b') as fd: 1587 fd.seek(100) 1588 fd.write(b'XXXX') 1589 output = self.cmd('check', '--info', self.repository_location, exit_code=1) 1590 self.assert_in('Starting repository check', output) # --info given for root logger 1591 1592 def test_readonly_check(self): 1593 self.cmd('init', '--encryption=repokey', self.repository_location) 1594 self.create_src_archive('test') 1595 with self.read_only(self.repository_path): 1596 # verify that command normally doesn't work with read-only repo 1597 if self.FORK_DEFAULT: 1598 self.cmd('check', '--verify-data', self.repository_location, exit_code=EXIT_ERROR) 1599 else: 1600 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1601 self.cmd('check', '--verify-data', self.repository_location) 1602 if isinstance(excinfo.value, RemoteRepository.RPCError): 1603 assert excinfo.value.exception_class == 'LockFailed' 1604 # verify that command works with read-only repo when using --bypass-lock 1605 self.cmd('check', '--verify-data', self.repository_location, '--bypass-lock') 1606 1607 def test_readonly_diff(self): 1608 self.cmd('init', '--encryption=repokey', self.repository_location) 1609 self.create_src_archive('a') 1610 self.create_src_archive('b') 1611 with self.read_only(self.repository_path): 1612 # verify that command normally doesn't work with read-only repo 1613 if self.FORK_DEFAULT: 1614 self.cmd('diff', '%s::a' % self.repository_location, 'b', exit_code=EXIT_ERROR) 1615 else: 1616 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1617 self.cmd('diff', '%s::a' % self.repository_location, 'b') 1618 if isinstance(excinfo.value, RemoteRepository.RPCError): 1619 assert excinfo.value.exception_class == 'LockFailed' 1620 # verify that command works with read-only repo when using --bypass-lock 1621 self.cmd('diff', '%s::a' % self.repository_location, 'b', '--bypass-lock') 1622 1623 def test_readonly_export_tar(self): 1624 self.cmd('init', '--encryption=repokey', self.repository_location) 1625 self.create_src_archive('test') 1626 with self.read_only(self.repository_path): 1627 # verify that command normally doesn't work with read-only repo 1628 if self.FORK_DEFAULT: 1629 self.cmd('export-tar', '%s::test' % self.repository_location, 'test.tar', exit_code=EXIT_ERROR) 1630 else: 1631 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1632 self.cmd('export-tar', '%s::test' % self.repository_location, 'test.tar') 1633 if isinstance(excinfo.value, RemoteRepository.RPCError): 1634 assert excinfo.value.exception_class == 'LockFailed' 1635 # verify that command works with read-only repo when using --bypass-lock 1636 self.cmd('export-tar', '%s::test' % self.repository_location, 'test.tar', '--bypass-lock') 1637 1638 def test_readonly_extract(self): 1639 self.cmd('init', '--encryption=repokey', self.repository_location) 1640 self.create_src_archive('test') 1641 with self.read_only(self.repository_path): 1642 # verify that command normally doesn't work with read-only repo 1643 if self.FORK_DEFAULT: 1644 self.cmd('extract', '%s::test' % self.repository_location, exit_code=EXIT_ERROR) 1645 else: 1646 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1647 self.cmd('extract', '%s::test' % self.repository_location) 1648 if isinstance(excinfo.value, RemoteRepository.RPCError): 1649 assert excinfo.value.exception_class == 'LockFailed' 1650 # verify that command works with read-only repo when using --bypass-lock 1651 self.cmd('extract', '%s::test' % self.repository_location, '--bypass-lock') 1652 1653 def test_readonly_info(self): 1654 self.cmd('init', '--encryption=repokey', self.repository_location) 1655 self.create_src_archive('test') 1656 with self.read_only(self.repository_path): 1657 # verify that command normally doesn't work with read-only repo 1658 if self.FORK_DEFAULT: 1659 self.cmd('info', self.repository_location, exit_code=EXIT_ERROR) 1660 else: 1661 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1662 self.cmd('info', self.repository_location) 1663 if isinstance(excinfo.value, RemoteRepository.RPCError): 1664 assert excinfo.value.exception_class == 'LockFailed' 1665 # verify that command works with read-only repo when using --bypass-lock 1666 self.cmd('info', self.repository_location, '--bypass-lock') 1667 1668 def test_readonly_list(self): 1669 self.cmd('init', '--encryption=repokey', self.repository_location) 1670 self.create_src_archive('test') 1671 with self.read_only(self.repository_path): 1672 # verify that command normally doesn't work with read-only repo 1673 if self.FORK_DEFAULT: 1674 self.cmd('list', self.repository_location, exit_code=EXIT_ERROR) 1675 else: 1676 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1677 self.cmd('list', self.repository_location) 1678 if isinstance(excinfo.value, RemoteRepository.RPCError): 1679 assert excinfo.value.exception_class == 'LockFailed' 1680 # verify that command works with read-only repo when using --bypass-lock 1681 self.cmd('list', self.repository_location, '--bypass-lock') 1682 1683 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 1684 def test_readonly_mount(self): 1685 self.cmd('init', '--encryption=repokey', self.repository_location) 1686 self.create_src_archive('test') 1687 with self.read_only(self.repository_path): 1688 # verify that command normally doesn't work with read-only repo 1689 if self.FORK_DEFAULT: 1690 with self.fuse_mount(self.repository_location, exit_code=EXIT_ERROR): 1691 pass 1692 else: 1693 with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: 1694 # self.fuse_mount always assumes fork=True, so for this test we have to manually set fork=False 1695 with self.fuse_mount(self.repository_location, fork=False): 1696 pass 1697 if isinstance(excinfo.value, RemoteRepository.RPCError): 1698 assert excinfo.value.exception_class == 'LockFailed' 1699 # verify that command works with read-only repo when using --bypass-lock 1700 with self.fuse_mount(self.repository_location, None, '--bypass-lock'): 1701 pass 1702 1703 @pytest.mark.skipif('BORG_TESTS_IGNORE_MODES' in os.environ, reason='modes unreliable') 1704 def test_umask(self): 1705 self.create_regular_file('file1', size=1024 * 80) 1706 self.cmd('init', '--encryption=repokey', self.repository_location) 1707 self.cmd('create', self.repository_location + '::test', 'input') 1708 mode = os.stat(self.repository_path).st_mode 1709 self.assertEqual(stat.S_IMODE(mode), 0o700) 1710 1711 def test_create_dry_run(self): 1712 self.cmd('init', '--encryption=repokey', self.repository_location) 1713 self.cmd('create', '--dry-run', self.repository_location + '::test', 'input') 1714 # Make sure no archive has been created 1715 with Repository(self.repository_path) as repository: 1716 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1717 self.assert_equal(len(manifest.archives), 0) 1718 1719 def add_unknown_feature(self, operation): 1720 with Repository(self.repository_path, exclusive=True) as repository: 1721 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1722 manifest.config[b'feature_flags'] = {operation.value.encode(): {b'mandatory': [b'unknown-feature']}} 1723 manifest.write() 1724 repository.commit() 1725 1726 def cmd_raises_unknown_feature(self, args): 1727 if self.FORK_DEFAULT: 1728 self.cmd(*args, exit_code=EXIT_ERROR) 1729 else: 1730 with pytest.raises(MandatoryFeatureUnsupported) as excinfo: 1731 self.cmd(*args) 1732 assert excinfo.value.args == (['unknown-feature'],) 1733 1734 def test_unknown_feature_on_create(self): 1735 print(self.cmd('init', '--encryption=repokey', self.repository_location)) 1736 self.add_unknown_feature(Manifest.Operation.WRITE) 1737 self.cmd_raises_unknown_feature(['create', self.repository_location + '::test', 'input']) 1738 1739 def test_unknown_feature_on_cache_sync(self): 1740 self.cmd('init', '--encryption=repokey', self.repository_location) 1741 self.cmd('delete', '--cache-only', self.repository_location) 1742 self.add_unknown_feature(Manifest.Operation.READ) 1743 self.cmd_raises_unknown_feature(['create', self.repository_location + '::test', 'input']) 1744 1745 def test_unknown_feature_on_change_passphrase(self): 1746 print(self.cmd('init', '--encryption=repokey', self.repository_location)) 1747 self.add_unknown_feature(Manifest.Operation.CHECK) 1748 self.cmd_raises_unknown_feature(['change-passphrase', self.repository_location]) 1749 1750 def test_unknown_feature_on_read(self): 1751 print(self.cmd('init', '--encryption=repokey', self.repository_location)) 1752 self.cmd('create', self.repository_location + '::test', 'input') 1753 self.add_unknown_feature(Manifest.Operation.READ) 1754 with changedir('output'): 1755 self.cmd_raises_unknown_feature(['extract', self.repository_location + '::test']) 1756 1757 self.cmd_raises_unknown_feature(['list', self.repository_location]) 1758 self.cmd_raises_unknown_feature(['info', self.repository_location + '::test']) 1759 1760 def test_unknown_feature_on_rename(self): 1761 print(self.cmd('init', '--encryption=repokey', self.repository_location)) 1762 self.cmd('create', self.repository_location + '::test', 'input') 1763 self.add_unknown_feature(Manifest.Operation.CHECK) 1764 self.cmd_raises_unknown_feature(['rename', self.repository_location + '::test', 'other']) 1765 1766 def test_unknown_feature_on_delete(self): 1767 print(self.cmd('init', '--encryption=repokey', self.repository_location)) 1768 self.cmd('create', self.repository_location + '::test', 'input') 1769 self.add_unknown_feature(Manifest.Operation.DELETE) 1770 # delete of an archive raises 1771 self.cmd_raises_unknown_feature(['delete', self.repository_location + '::test']) 1772 self.cmd_raises_unknown_feature(['prune', '--keep-daily=3', self.repository_location]) 1773 # delete of the whole repository ignores features 1774 self.cmd('delete', self.repository_location) 1775 1776 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 1777 def test_unknown_feature_on_mount(self): 1778 self.cmd('init', '--encryption=repokey', self.repository_location) 1779 self.cmd('create', self.repository_location + '::test', 'input') 1780 self.add_unknown_feature(Manifest.Operation.READ) 1781 mountpoint = os.path.join(self.tmpdir, 'mountpoint') 1782 os.mkdir(mountpoint) 1783 # XXX this might hang if it doesn't raise an error 1784 self.cmd_raises_unknown_feature(['mount', self.repository_location + '::test', mountpoint]) 1785 1786 @pytest.mark.allow_cache_wipe 1787 def test_unknown_mandatory_feature_in_cache(self): 1788 if self.prefix: 1789 path_prefix = 'ssh://__testsuite__' 1790 else: 1791 path_prefix = '' 1792 1793 print(self.cmd('init', '--encryption=repokey', self.repository_location)) 1794 1795 with Repository(self.repository_path, exclusive=True) as repository: 1796 if path_prefix: 1797 repository._location = Location(self.repository_location) 1798 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1799 with Cache(repository, key, manifest) as cache: 1800 cache.begin_txn() 1801 cache.cache_config.mandatory_features = set(['unknown-feature']) 1802 cache.commit() 1803 1804 if self.FORK_DEFAULT: 1805 self.cmd('create', self.repository_location + '::test', 'input') 1806 else: 1807 called = False 1808 wipe_cache_safe = LocalCache.wipe_cache 1809 1810 def wipe_wrapper(*args): 1811 nonlocal called 1812 called = True 1813 wipe_cache_safe(*args) 1814 1815 with patch.object(LocalCache, 'wipe_cache', wipe_wrapper): 1816 self.cmd('create', self.repository_location + '::test', 'input') 1817 1818 assert called 1819 1820 with Repository(self.repository_path, exclusive=True) as repository: 1821 if path_prefix: 1822 repository._location = Location(self.repository_location) 1823 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 1824 with Cache(repository, key, manifest) as cache: 1825 assert cache.cache_config.mandatory_features == set([]) 1826 1827 def test_progress_on(self): 1828 self.create_regular_file('file1', size=1024 * 80) 1829 self.cmd('init', '--encryption=repokey', self.repository_location) 1830 output = self.cmd('create', '--progress', self.repository_location + '::test4', 'input') 1831 self.assert_in("\r", output) 1832 1833 def test_progress_off(self): 1834 self.create_regular_file('file1', size=1024 * 80) 1835 self.cmd('init', '--encryption=repokey', self.repository_location) 1836 output = self.cmd('create', self.repository_location + '::test5', 'input') 1837 self.assert_not_in("\r", output) 1838 1839 def test_file_status(self): 1840 """test that various file status show expected results 1841 1842 clearly incomplete: only tests for the weird "unchanged" status for now""" 1843 self.create_regular_file('file1', size=1024 * 80) 1844 time.sleep(1) # file2 must have newer timestamps than file1 1845 self.create_regular_file('file2', size=1024 * 80) 1846 self.cmd('init', '--encryption=repokey', self.repository_location) 1847 output = self.cmd('create', '--list', self.repository_location + '::test', 'input') 1848 self.assert_in("A input/file1", output) 1849 self.assert_in("A input/file2", output) 1850 # should find first file as unmodified 1851 output = self.cmd('create', '--list', self.repository_location + '::test1', 'input') 1852 self.assert_in("U input/file1", output) 1853 # this is expected, although surprising, for why, see: 1854 # https://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file 1855 self.assert_in("A input/file2", output) 1856 1857 def test_file_status_cs_cache_mode(self): 1858 """test that a changed file with faked "previous" mtime still gets backed up in ctime,size cache_mode""" 1859 self.create_regular_file('file1', contents=b'123') 1860 time.sleep(1) # file2 must have newer timestamps than file1 1861 self.create_regular_file('file2', size=10) 1862 self.cmd('init', '--encryption=repokey', self.repository_location) 1863 output = self.cmd('create', '--list', '--files-cache=ctime,size', self.repository_location + '::test1', 'input') 1864 # modify file1, but cheat with the mtime (and atime) and also keep same size: 1865 st = os.stat('input/file1') 1866 self.create_regular_file('file1', contents=b'321') 1867 os.utime('input/file1', ns=(st.st_atime_ns, st.st_mtime_ns)) 1868 # this mode uses ctime for change detection, so it should find file1 as modified 1869 output = self.cmd('create', '--list', '--files-cache=ctime,size', self.repository_location + '::test2', 'input') 1870 self.assert_in("M input/file1", output) 1871 1872 def test_file_status_ms_cache_mode(self): 1873 """test that a chmod'ed file with no content changes does not get chunked again in mtime,size cache_mode""" 1874 self.create_regular_file('file1', size=10) 1875 time.sleep(1) # file2 must have newer timestamps than file1 1876 self.create_regular_file('file2', size=10) 1877 self.cmd('init', '--encryption=repokey', self.repository_location) 1878 output = self.cmd('create', '--list', '--files-cache=mtime,size', self.repository_location + '::test1', 'input') 1879 # change mode of file1, no content change: 1880 st = os.stat('input/file1') 1881 os.chmod('input/file1', st.st_mode ^ stat.S_IRWXO) # this triggers a ctime change, but mtime is unchanged 1882 # this mode uses mtime for change detection, so it should find file1 as unmodified 1883 output = self.cmd('create', '--list', '--files-cache=mtime,size', self.repository_location + '::test2', 'input') 1884 self.assert_in("U input/file1", output) 1885 1886 def test_file_status_rc_cache_mode(self): 1887 """test that files get rechunked unconditionally in rechunk,ctime cache mode""" 1888 self.create_regular_file('file1', size=10) 1889 time.sleep(1) # file2 must have newer timestamps than file1 1890 self.create_regular_file('file2', size=10) 1891 self.cmd('init', '--encryption=repokey', self.repository_location) 1892 output = self.cmd('create', '--list', '--files-cache=rechunk,ctime', self.repository_location + '::test1', 'input') 1893 # no changes here, but this mode rechunks unconditionally 1894 output = self.cmd('create', '--list', '--files-cache=rechunk,ctime', self.repository_location + '::test2', 'input') 1895 self.assert_in("A input/file1", output) 1896 1897 def test_file_status_excluded(self): 1898 """test that excluded paths are listed""" 1899 1900 self.create_regular_file('file1', size=1024 * 80) 1901 time.sleep(1) # file2 must have newer timestamps than file1 1902 self.create_regular_file('file2', size=1024 * 80) 1903 if has_lchflags: 1904 self.create_regular_file('file3', size=1024 * 80) 1905 platform.set_flags(os.path.join(self.input_path, 'file3'), stat.UF_NODUMP) 1906 self.cmd('init', '--encryption=repokey', self.repository_location) 1907 output = self.cmd('create', '--list', '--exclude-nodump', self.repository_location + '::test', 'input') 1908 self.assert_in("A input/file1", output) 1909 self.assert_in("A input/file2", output) 1910 if has_lchflags: 1911 self.assert_in("x input/file3", output) 1912 # should find second file as excluded 1913 output = self.cmd('create', '--list', '--exclude-nodump', self.repository_location + '::test1', 'input', '--exclude', '*/file2') 1914 self.assert_in("U input/file1", output) 1915 self.assert_in("x input/file2", output) 1916 if has_lchflags: 1917 self.assert_in("x input/file3", output) 1918 1919 def test_create_json(self): 1920 self.create_regular_file('file1', size=1024 * 80) 1921 self.cmd('init', '--encryption=repokey', self.repository_location) 1922 create_info = json.loads(self.cmd('create', '--json', self.repository_location + '::test', 'input')) 1923 # The usual keys 1924 assert 'encryption' in create_info 1925 assert 'repository' in create_info 1926 assert 'cache' in create_info 1927 assert 'last_modified' in create_info['repository'] 1928 1929 archive = create_info['archive'] 1930 assert archive['name'] == 'test' 1931 assert isinstance(archive['command_line'], list) 1932 assert isinstance(archive['duration'], float) 1933 assert len(archive['id']) == 64 1934 assert 'stats' in archive 1935 1936 def test_create_topical(self): 1937 self.create_regular_file('file1', size=1024 * 80) 1938 time.sleep(1) # file2 must have newer timestamps than file1 1939 self.create_regular_file('file2', size=1024 * 80) 1940 self.cmd('init', '--encryption=repokey', self.repository_location) 1941 # no listing by default 1942 output = self.cmd('create', self.repository_location + '::test', 'input') 1943 self.assert_not_in('file1', output) 1944 # shouldn't be listed even if unchanged 1945 output = self.cmd('create', self.repository_location + '::test0', 'input') 1946 self.assert_not_in('file1', output) 1947 # should list the file as unchanged 1948 output = self.cmd('create', '--list', '--filter=U', self.repository_location + '::test1', 'input') 1949 self.assert_in('file1', output) 1950 # should *not* list the file as changed 1951 output = self.cmd('create', '--list', '--filter=AM', self.repository_location + '::test2', 'input') 1952 self.assert_not_in('file1', output) 1953 # change the file 1954 self.create_regular_file('file1', size=1024 * 100) 1955 # should list the file as changed 1956 output = self.cmd('create', '--list', '--filter=AM', self.repository_location + '::test3', 'input') 1957 self.assert_in('file1', output) 1958 1959 def test_create_read_special_broken_symlink(self): 1960 os.symlink('somewhere does not exist', os.path.join(self.input_path, 'link')) 1961 self.cmd('init', '--encryption=repokey', self.repository_location) 1962 archive = self.repository_location + '::test' 1963 self.cmd('create', '--read-special', archive, 'input') 1964 output = self.cmd('list', archive) 1965 assert 'input/link -> somewhere does not exist' in output 1966 1967 # def test_cmdline_compatibility(self): 1968 # self.create_regular_file('file1', size=1024 * 80) 1969 # self.cmd('init', '--encryption=repokey', self.repository_location) 1970 # self.cmd('create', self.repository_location + '::test', 'input') 1971 # output = self.cmd('foo', self.repository_location, '--old') 1972 # self.assert_in('"--old" has been deprecated. Use "--new" instead', output) 1973 1974 def test_prune_repository(self): 1975 self.cmd('init', '--encryption=repokey', self.repository_location) 1976 self.cmd('create', self.repository_location + '::test1', src_dir) 1977 self.cmd('create', self.repository_location + '::test2', src_dir) 1978 # these are not really a checkpoints, but they look like some: 1979 self.cmd('create', self.repository_location + '::test3.checkpoint', src_dir) 1980 self.cmd('create', self.repository_location + '::test3.checkpoint.1', src_dir) 1981 self.cmd('create', self.repository_location + '::test4.checkpoint', src_dir) 1982 output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2') 1983 self.assert_in('Keeping archive: test2', output) 1984 self.assert_in('Would prune: test1', output) 1985 # must keep the latest non-checkpoint archive: 1986 self.assert_in('Keeping archive: test2', output) 1987 # must keep the latest checkpoint archive: 1988 self.assert_in('Keeping archive: test4.checkpoint', output) 1989 output = self.cmd('list', self.repository_location) 1990 self.assert_in('test1', output) 1991 self.assert_in('test2', output) 1992 self.assert_in('test3.checkpoint', output) 1993 self.assert_in('test3.checkpoint.1', output) 1994 self.assert_in('test4.checkpoint', output) 1995 self.cmd('prune', self.repository_location, '--keep-daily=2') 1996 output = self.cmd('list', self.repository_location) 1997 self.assert_not_in('test1', output) 1998 # the latest non-checkpoint archive must be still there: 1999 self.assert_in('test2', output) 2000 # only the latest checkpoint archive must still be there: 2001 self.assert_not_in('test3.checkpoint', output) 2002 self.assert_not_in('test3.checkpoint.1', output) 2003 self.assert_in('test4.checkpoint', output) 2004 # now we supercede the latest checkpoint by a successful backup: 2005 self.cmd('create', self.repository_location + '::test5', src_dir) 2006 self.cmd('prune', self.repository_location, '--keep-daily=2') 2007 output = self.cmd('list', self.repository_location) 2008 # all checkpoints should be gone now: 2009 self.assert_not_in('checkpoint', output) 2010 # the latest archive must be still there 2011 self.assert_in('test5', output) 2012 2013 def test_prune_repository_save_space(self): 2014 self.cmd('init', '--encryption=repokey', self.repository_location) 2015 self.cmd('create', self.repository_location + '::test1', src_dir) 2016 self.cmd('create', self.repository_location + '::test2', src_dir) 2017 output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2') 2018 self.assert_in('Keeping archive: test2', output) 2019 self.assert_in('Would prune: test1', output) 2020 output = self.cmd('list', self.repository_location) 2021 self.assert_in('test1', output) 2022 self.assert_in('test2', output) 2023 self.cmd('prune', '--save-space', self.repository_location, '--keep-daily=2') 2024 output = self.cmd('list', self.repository_location) 2025 self.assert_not_in('test1', output) 2026 self.assert_in('test2', output) 2027 2028 def test_prune_repository_prefix(self): 2029 self.cmd('init', '--encryption=repokey', self.repository_location) 2030 self.cmd('create', self.repository_location + '::foo-2015-08-12-10:00', src_dir) 2031 self.cmd('create', self.repository_location + '::foo-2015-08-12-20:00', src_dir) 2032 self.cmd('create', self.repository_location + '::bar-2015-08-12-10:00', src_dir) 2033 self.cmd('create', self.repository_location + '::bar-2015-08-12-20:00', src_dir) 2034 output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2', '--prefix=foo-') 2035 self.assert_in('Keeping archive: foo-2015-08-12-20:00', output) 2036 self.assert_in('Would prune: foo-2015-08-12-10:00', output) 2037 output = self.cmd('list', self.repository_location) 2038 self.assert_in('foo-2015-08-12-10:00', output) 2039 self.assert_in('foo-2015-08-12-20:00', output) 2040 self.assert_in('bar-2015-08-12-10:00', output) 2041 self.assert_in('bar-2015-08-12-20:00', output) 2042 self.cmd('prune', self.repository_location, '--keep-daily=2', '--prefix=foo-') 2043 output = self.cmd('list', self.repository_location) 2044 self.assert_not_in('foo-2015-08-12-10:00', output) 2045 self.assert_in('foo-2015-08-12-20:00', output) 2046 self.assert_in('bar-2015-08-12-10:00', output) 2047 self.assert_in('bar-2015-08-12-20:00', output) 2048 2049 def test_prune_repository_glob(self): 2050 self.cmd('init', '--encryption=repokey', self.repository_location) 2051 self.cmd('create', self.repository_location + '::2015-08-12-10:00-foo', src_dir) 2052 self.cmd('create', self.repository_location + '::2015-08-12-20:00-foo', src_dir) 2053 self.cmd('create', self.repository_location + '::2015-08-12-10:00-bar', src_dir) 2054 self.cmd('create', self.repository_location + '::2015-08-12-20:00-bar', src_dir) 2055 output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo') 2056 self.assert_in('Keeping archive: 2015-08-12-20:00-foo', output) 2057 self.assert_in('Would prune: 2015-08-12-10:00-foo', output) 2058 output = self.cmd('list', self.repository_location) 2059 self.assert_in('2015-08-12-10:00-foo', output) 2060 self.assert_in('2015-08-12-20:00-foo', output) 2061 self.assert_in('2015-08-12-10:00-bar', output) 2062 self.assert_in('2015-08-12-20:00-bar', output) 2063 self.cmd('prune', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo') 2064 output = self.cmd('list', self.repository_location) 2065 self.assert_not_in('2015-08-12-10:00-foo', output) 2066 self.assert_in('2015-08-12-20:00-foo', output) 2067 self.assert_in('2015-08-12-10:00-bar', output) 2068 self.assert_in('2015-08-12-20:00-bar', output) 2069 2070 def test_list_prefix(self): 2071 self.cmd('init', '--encryption=repokey', self.repository_location) 2072 self.cmd('create', self.repository_location + '::test-1', src_dir) 2073 self.cmd('create', self.repository_location + '::something-else-than-test-1', src_dir) 2074 self.cmd('create', self.repository_location + '::test-2', src_dir) 2075 output = self.cmd('list', '--prefix=test-', self.repository_location) 2076 self.assert_in('test-1', output) 2077 self.assert_in('test-2', output) 2078 self.assert_not_in('something-else', output) 2079 2080 def test_list_format(self): 2081 self.cmd('init', '--encryption=repokey', self.repository_location) 2082 test_archive = self.repository_location + '::test' 2083 self.cmd('create', test_archive, src_dir) 2084 output_warn = self.cmd('list', '--list-format', '-', test_archive) 2085 self.assert_in('--list-format" has been deprecated.', output_warn) 2086 output_1 = self.cmd('list', test_archive) 2087 output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}', test_archive) 2088 output_3 = self.cmd('list', '--format', '{mtime:%s} {path}{NL}', test_archive) 2089 self.assertEqual(output_1, output_2) 2090 self.assertNotEqual(output_1, output_3) 2091 2092 def test_list_repository_format(self): 2093 self.cmd('init', '--encryption=repokey', self.repository_location) 2094 self.cmd('create', '--comment', 'comment 1', self.repository_location + '::test-1', src_dir) 2095 self.cmd('create', '--comment', 'comment 2', self.repository_location + '::test-2', src_dir) 2096 output_1 = self.cmd('list', self.repository_location) 2097 output_2 = self.cmd('list', '--format', '{archive:<36} {time} [{id}]{NL}', self.repository_location) 2098 self.assertEqual(output_1, output_2) 2099 output_1 = self.cmd('list', '--short', self.repository_location) 2100 self.assertEqual(output_1, 'test-1\ntest-2\n') 2101 output_1 = self.cmd('list', '--format', '{barchive}/', self.repository_location) 2102 self.assertEqual(output_1, 'test-1/test-2/') 2103 output_3 = self.cmd('list', '--format', '{name} {comment}{NL}', self.repository_location) 2104 self.assert_in('test-1 comment 1\n', output_3) 2105 self.assert_in('test-2 comment 2\n', output_3) 2106 2107 def test_list_hash(self): 2108 self.create_regular_file('empty_file', size=0) 2109 self.create_regular_file('amb', contents=b'a' * 1000000) 2110 self.cmd('init', '--encryption=repokey', self.repository_location) 2111 test_archive = self.repository_location + '::test' 2112 self.cmd('create', test_archive, 'input') 2113 output = self.cmd('list', '--format', '{sha256} {path}{NL}', test_archive) 2114 assert "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0 input/amb" in output 2115 assert "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 input/empty_file" in output 2116 2117 def test_list_chunk_counts(self): 2118 self.create_regular_file('empty_file', size=0) 2119 self.create_regular_file('two_chunks') 2120 with open(os.path.join(self.input_path, 'two_chunks'), 'wb') as fd: 2121 fd.write(b'abba' * 2000000) 2122 fd.write(b'baab' * 2000000) 2123 self.cmd('init', '--encryption=repokey', self.repository_location) 2124 test_archive = self.repository_location + '::test' 2125 self.cmd('create', test_archive, 'input') 2126 output = self.cmd('list', '--format', '{num_chunks} {unique_chunks} {path}{NL}', test_archive) 2127 assert "0 0 input/empty_file" in output 2128 assert "2 2 input/two_chunks" in output 2129 2130 def test_list_size(self): 2131 self.create_regular_file('compressible_file', size=10000) 2132 self.cmd('init', '--encryption=repokey', self.repository_location) 2133 test_archive = self.repository_location + '::test' 2134 self.cmd('create', '-C', 'lz4', test_archive, 'input') 2135 output = self.cmd('list', '--format', '{size} {csize} {dsize} {dcsize} {path}{NL}', test_archive) 2136 size, csize, dsize, dcsize, path = output.split("\n")[1].split(" ") 2137 assert int(csize) < int(size) 2138 assert int(dcsize) < int(dsize) 2139 assert int(dsize) <= int(size) 2140 assert int(dcsize) <= int(csize) 2141 2142 def test_list_json(self): 2143 self.create_regular_file('file1', size=1024 * 80) 2144 self.cmd('init', '--encryption=repokey', self.repository_location) 2145 self.cmd('create', self.repository_location + '::test', 'input') 2146 list_repo = json.loads(self.cmd('list', '--json', self.repository_location)) 2147 repository = list_repo['repository'] 2148 assert len(repository['id']) == 64 2149 assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise 2150 assert list_repo['encryption']['mode'] == 'repokey' 2151 assert 'keyfile' not in list_repo['encryption'] 2152 archive0 = list_repo['archives'][0] 2153 assert datetime.strptime(archive0['time'], ISO_FORMAT) # must not raise 2154 2155 list_archive = self.cmd('list', '--json-lines', self.repository_location + '::test') 2156 items = [json.loads(s) for s in list_archive.splitlines()] 2157 assert len(items) == 2 2158 file1 = items[1] 2159 assert file1['path'] == 'input/file1' 2160 assert file1['size'] == 81920 2161 assert datetime.strptime(file1['mtime'], ISO_FORMAT) # must not raise 2162 2163 list_archive = self.cmd('list', '--json-lines', '--format={sha256}', self.repository_location + '::test') 2164 items = [json.loads(s) for s in list_archive.splitlines()] 2165 assert len(items) == 2 2166 file1 = items[1] 2167 assert file1['path'] == 'input/file1' 2168 assert file1['sha256'] == 'b2915eb69f260d8d3c25249195f2c8f4f716ea82ec760ae929732c0262442b2b' 2169 2170 def test_list_json_args(self): 2171 self.cmd('init', '--encryption=repokey', self.repository_location) 2172 self.cmd('list', '--json-lines', self.repository_location, exit_code=2) 2173 self.cmd('list', '--json', self.repository_location + '::archive', exit_code=2) 2174 2175 def test_log_json(self): 2176 self.create_test_files() 2177 self.cmd('init', '--encryption=repokey', self.repository_location) 2178 log = self.cmd('create', '--log-json', self.repository_location + '::test', 'input', '--list', '--debug') 2179 messages = {} # type -> message, one of each kind 2180 for line in log.splitlines(): 2181 msg = json.loads(line) 2182 messages[msg['type']] = msg 2183 2184 file_status = messages['file_status'] 2185 assert 'status' in file_status 2186 assert file_status['path'].startswith('input') 2187 2188 log_message = messages['log_message'] 2189 assert isinstance(log_message['time'], float) 2190 assert log_message['levelname'] == 'DEBUG' # there should only be DEBUG messages 2191 assert isinstance(log_message['message'], str) 2192 2193 def test_debug_profile(self): 2194 self.create_test_files() 2195 self.cmd('init', '--encryption=repokey', self.repository_location) 2196 self.cmd('create', self.repository_location + '::test', 'input', '--debug-profile=create.prof') 2197 self.cmd('debug', 'convert-profile', 'create.prof', 'create.pyprof') 2198 stats = pstats.Stats('create.pyprof') 2199 stats.strip_dirs() 2200 stats.sort_stats('cumtime') 2201 2202 self.cmd('create', self.repository_location + '::test2', 'input', '--debug-profile=create.pyprof') 2203 stats = pstats.Stats('create.pyprof') # Only do this on trusted data! 2204 stats.strip_dirs() 2205 stats.sort_stats('cumtime') 2206 2207 def test_common_options(self): 2208 self.create_test_files() 2209 self.cmd('init', '--encryption=repokey', self.repository_location) 2210 log = self.cmd('--debug', 'create', self.repository_location + '::test', 'input') 2211 assert 'security: read previous location' in log 2212 2213 def _get_sizes(self, compression, compressible, size=10000): 2214 if compressible: 2215 contents = b'X' * size 2216 else: 2217 contents = os.urandom(size) 2218 self.create_regular_file('file', contents=contents) 2219 self.cmd('init', '--encryption=none', self.repository_location) 2220 archive = self.repository_location + '::test' 2221 self.cmd('create', '-C', compression, archive, 'input') 2222 output = self.cmd('list', '--format', '{size} {csize} {path}{NL}', archive) 2223 size, csize, path = output.split("\n")[1].split(" ") 2224 return int(size), int(csize) 2225 2226 def test_compression_none_compressible(self): 2227 size, csize = self._get_sizes('none', compressible=True) 2228 assert csize >= size 2229 assert csize == size + 3 2230 2231 def test_compression_none_uncompressible(self): 2232 size, csize = self._get_sizes('none', compressible=False) 2233 assert csize >= size 2234 assert csize == size + 3 2235 2236 def test_compression_zlib_compressible(self): 2237 size, csize = self._get_sizes('zlib', compressible=True) 2238 assert csize < size * 0.1 2239 assert csize == 35 2240 2241 def test_compression_zlib_uncompressible(self): 2242 size, csize = self._get_sizes('zlib', compressible=False) 2243 assert csize >= size 2244 2245 def test_compression_auto_compressible(self): 2246 size, csize = self._get_sizes('auto,zlib', compressible=True) 2247 assert csize < size * 0.1 2248 assert csize == 35 # same as compression 'zlib' 2249 2250 def test_compression_auto_uncompressible(self): 2251 size, csize = self._get_sizes('auto,zlib', compressible=False) 2252 assert csize >= size 2253 assert csize == size + 3 # same as compression 'none' 2254 2255 def test_compression_lz4_compressible(self): 2256 size, csize = self._get_sizes('lz4', compressible=True) 2257 assert csize < size * 0.1 2258 2259 def test_compression_lz4_uncompressible(self): 2260 size, csize = self._get_sizes('lz4', compressible=False) 2261 assert csize >= size 2262 2263 def test_compression_lzma_compressible(self): 2264 size, csize = self._get_sizes('lzma', compressible=True) 2265 assert csize < size * 0.1 2266 2267 def test_compression_lzma_uncompressible(self): 2268 size, csize = self._get_sizes('lzma', compressible=False) 2269 assert csize >= size 2270 2271 def test_change_passphrase(self): 2272 self.cmd('init', '--encryption=repokey', self.repository_location) 2273 os.environ['BORG_NEW_PASSPHRASE'] = 'newpassphrase' 2274 # here we have both BORG_PASSPHRASE and BORG_NEW_PASSPHRASE set: 2275 self.cmd('change-passphrase', self.repository_location) 2276 os.environ['BORG_PASSPHRASE'] = 'newpassphrase' 2277 self.cmd('list', self.repository_location) 2278 2279 def test_break_lock(self): 2280 self.cmd('init', '--encryption=repokey', self.repository_location) 2281 self.cmd('break-lock', self.repository_location) 2282 2283 def test_usage(self): 2284 self.cmd() 2285 self.cmd('-h') 2286 2287 def test_help(self): 2288 assert 'Borg' in self.cmd('help') 2289 assert 'patterns' in self.cmd('help', 'patterns') 2290 assert 'Initialize' in self.cmd('help', 'init') 2291 assert 'positional arguments' not in self.cmd('help', 'init', '--epilog-only') 2292 assert 'This command initializes' not in self.cmd('help', 'init', '--usage-only') 2293 2294 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 2295 def test_fuse(self): 2296 def has_noatime(some_file): 2297 atime_before = os.stat(some_file).st_atime_ns 2298 try: 2299 os.close(os.open(some_file, flags_noatime)) 2300 except PermissionError: 2301 return False 2302 else: 2303 atime_after = os.stat(some_file).st_atime_ns 2304 noatime_used = flags_noatime != flags_normal 2305 return noatime_used and atime_before == atime_after 2306 2307 self.cmd('init', '--encryption=repokey', self.repository_location) 2308 self.create_test_files() 2309 have_noatime = has_noatime('input/file1') 2310 self.cmd('create', '--exclude-nodump', self.repository_location + '::archive', 'input') 2311 self.cmd('create', '--exclude-nodump', self.repository_location + '::archive2', 'input') 2312 if has_lchflags: 2313 # remove the file we did not backup, so input and output become equal 2314 os.remove(os.path.join('input', 'flagfile')) 2315 mountpoint = os.path.join(self.tmpdir, 'mountpoint') 2316 # mount the whole repository, archive contents shall show up in archivename subdirs of mountpoint: 2317 with self.fuse_mount(self.repository_location, mountpoint): 2318 # bsdflags are not supported by the FUSE mount 2319 # we also ignore xattrs here, they are tested separately 2320 self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'), 2321 ignore_bsdflags=True, ignore_xattrs=True) 2322 self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'), 2323 ignore_bsdflags=True, ignore_xattrs=True) 2324 # mount only 1 archive, its contents shall show up directly in mountpoint: 2325 with self.fuse_mount(self.repository_location + '::archive', mountpoint): 2326 self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'), 2327 ignore_bsdflags=True, ignore_xattrs=True) 2328 # regular file 2329 in_fn = 'input/file1' 2330 out_fn = os.path.join(mountpoint, 'input', 'file1') 2331 # stat 2332 sti1 = os.stat(in_fn) 2333 sto1 = os.stat(out_fn) 2334 assert sti1.st_mode == sto1.st_mode 2335 assert sti1.st_uid == sto1.st_uid 2336 assert sti1.st_gid == sto1.st_gid 2337 assert sti1.st_size == sto1.st_size 2338 if have_noatime: 2339 assert sti1.st_atime == sto1.st_atime 2340 assert sti1.st_ctime == sto1.st_ctime 2341 assert sti1.st_mtime == sto1.st_mtime 2342 if are_hardlinks_supported(): 2343 # note: there is another hardlink to this, see below 2344 assert sti1.st_nlink == sto1.st_nlink == 2 2345 # read 2346 with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f: 2347 assert in_f.read() == out_f.read() 2348 # hardlink (to 'input/file1') 2349 if are_hardlinks_supported(): 2350 in_fn = 'input/hardlink' 2351 out_fn = os.path.join(mountpoint, 'input', 'hardlink') 2352 sti2 = os.stat(in_fn) 2353 sto2 = os.stat(out_fn) 2354 assert sti2.st_nlink == sto2.st_nlink == 2 2355 assert sto1.st_ino == sto2.st_ino 2356 # symlink 2357 if are_symlinks_supported(): 2358 in_fn = 'input/link1' 2359 out_fn = os.path.join(mountpoint, 'input', 'link1') 2360 sti = os.stat(in_fn, follow_symlinks=False) 2361 sto = os.stat(out_fn, follow_symlinks=False) 2362 assert sti.st_size == len('somewhere') 2363 assert sto.st_size == len('somewhere') 2364 assert stat.S_ISLNK(sti.st_mode) 2365 assert stat.S_ISLNK(sto.st_mode) 2366 assert os.readlink(in_fn) == os.readlink(out_fn) 2367 # FIFO 2368 if are_fifos_supported(): 2369 out_fn = os.path.join(mountpoint, 'input', 'fifo1') 2370 sto = os.stat(out_fn) 2371 assert stat.S_ISFIFO(sto.st_mode) 2372 # list/read xattrs 2373 try: 2374 in_fn = 'input/fusexattr' 2375 out_fn = os.path.join(mountpoint, 'input', 'fusexattr') 2376 if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): 2377 assert sorted(no_selinux(xattr.listxattr(out_fn))) == ['user.empty', 'user.foo', ] 2378 assert xattr.getxattr(out_fn, 'user.foo') == b'bar' 2379 # Special case: getxattr returns None (not b'') when reading an empty xattr. 2380 assert xattr.getxattr(out_fn, 'user.empty') is None 2381 else: 2382 assert no_selinux(xattr.listxattr(out_fn)) == [] 2383 try: 2384 xattr.getxattr(out_fn, 'user.foo') 2385 except OSError as e: 2386 assert e.errno == llfuse.ENOATTR 2387 else: 2388 assert False, "expected OSError(ENOATTR), but no error was raised" 2389 except OSError as err: 2390 if sys.platform.startswith(('freebsd', )) and err.errno == errno.ENOTSUP: 2391 # some systems have no xattr support on FUSE 2392 pass 2393 else: 2394 raise 2395 2396 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 2397 def test_fuse_versions_view(self): 2398 self.cmd('init', '--encryption=repokey', self.repository_location) 2399 self.create_regular_file('test', contents=b'first') 2400 if are_hardlinks_supported(): 2401 self.create_regular_file('hardlink1', contents=b'123456') 2402 os.link('input/hardlink1', 'input/hardlink2') 2403 os.link('input/hardlink1', 'input/hardlink3') 2404 self.cmd('create', self.repository_location + '::archive1', 'input') 2405 self.create_regular_file('test', contents=b'second') 2406 self.cmd('create', self.repository_location + '::archive2', 'input') 2407 mountpoint = os.path.join(self.tmpdir, 'mountpoint') 2408 # mount the whole repository, archive contents shall show up in versioned view: 2409 with self.fuse_mount(self.repository_location, mountpoint, '-o', 'versions'): 2410 path = os.path.join(mountpoint, 'input', 'test') # filename shows up as directory ... 2411 files = os.listdir(path) 2412 assert all(f.startswith('test.') for f in files) # ... with files test.xxxxx in there 2413 assert {b'first', b'second'} == {open(os.path.join(path, f), 'rb').read() for f in files} 2414 if are_hardlinks_supported(): 2415 hl1 = os.path.join(mountpoint, 'input', 'hardlink1', 'hardlink1.00001') 2416 hl2 = os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001') 2417 hl3 = os.path.join(mountpoint, 'input', 'hardlink3', 'hardlink3.00001') 2418 assert os.stat(hl1).st_ino == os.stat(hl2).st_ino == os.stat(hl3).st_ino 2419 assert open(hl3, 'rb').read() == b'123456' 2420 # similar again, but exclude the hardlink master: 2421 with self.fuse_mount(self.repository_location, mountpoint, '-o', 'versions', '-e', 'input/hardlink1'): 2422 if are_hardlinks_supported(): 2423 hl2 = os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001') 2424 hl3 = os.path.join(mountpoint, 'input', 'hardlink3', 'hardlink3.00001') 2425 assert os.stat(hl2).st_ino == os.stat(hl3).st_ino 2426 assert open(hl3, 'rb').read() == b'123456' 2427 2428 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 2429 def test_fuse_allow_damaged_files(self): 2430 self.cmd('init', '--encryption=repokey', self.repository_location) 2431 self.create_src_archive('archive') 2432 # Get rid of a chunk and repair it 2433 archive, repository = self.open_archive('archive') 2434 with repository: 2435 for item in archive.iter_items(): 2436 if item.path.endswith('testsuite/archiver.py'): 2437 repository.delete(item.chunks[-1].id) 2438 path = item.path # store full path for later 2439 break 2440 else: 2441 assert False # missed the file 2442 repository.commit() 2443 self.cmd('check', '--repair', self.repository_location, exit_code=0) 2444 2445 mountpoint = os.path.join(self.tmpdir, 'mountpoint') 2446 with self.fuse_mount(self.repository_location + '::archive', mountpoint): 2447 with pytest.raises(OSError) as excinfo: 2448 open(os.path.join(mountpoint, path)) 2449 assert excinfo.value.errno == errno.EIO 2450 with self.fuse_mount(self.repository_location + '::archive', mountpoint, '-o', 'allow_damaged_files'): 2451 open(os.path.join(mountpoint, path)).close() 2452 2453 @unittest.skipUnless(has_llfuse, 'llfuse not installed') 2454 def test_fuse_mount_options(self): 2455 self.cmd('init', '--encryption=repokey', self.repository_location) 2456 self.create_src_archive('arch11') 2457 self.create_src_archive('arch12') 2458 self.create_src_archive('arch21') 2459 self.create_src_archive('arch22') 2460 2461 mountpoint = os.path.join(self.tmpdir, 'mountpoint') 2462 with self.fuse_mount(self.repository_location, mountpoint, '--first=2', '--sort=name'): 2463 assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch11', 'arch12'] 2464 with self.fuse_mount(self.repository_location, mountpoint, '--last=2', '--sort=name'): 2465 assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch21', 'arch22'] 2466 with self.fuse_mount(self.repository_location, mountpoint, '--prefix=arch1'): 2467 assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch11', 'arch12'] 2468 with self.fuse_mount(self.repository_location, mountpoint, '--prefix=arch2'): 2469 assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch21', 'arch22'] 2470 with self.fuse_mount(self.repository_location, mountpoint, '--prefix=arch'): 2471 assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch11', 'arch12', 'arch21', 'arch22'] 2472 with self.fuse_mount(self.repository_location, mountpoint, '--prefix=nope'): 2473 assert sorted(os.listdir(os.path.join(mountpoint))) == [] 2474 2475 def verify_aes_counter_uniqueness(self, method): 2476 seen = set() # Chunks already seen 2477 used = set() # counter values already used 2478 2479 def verify_uniqueness(): 2480 with Repository(self.repository_path) as repository: 2481 for id, _ in repository.open_index(repository.get_transaction_id()).iteritems(): 2482 data = repository.get(id) 2483 hash = sha256(data).digest() 2484 if hash not in seen: 2485 seen.add(hash) 2486 num_blocks = num_aes_blocks(len(data) - 41) 2487 nonce = bytes_to_long(data[33:41]) 2488 for counter in range(nonce, nonce + num_blocks): 2489 self.assert_not_in(counter, used) 2490 used.add(counter) 2491 2492 self.create_test_files() 2493 os.environ['BORG_PASSPHRASE'] = 'passphrase' 2494 self.cmd('init', '--encryption=' + method, self.repository_location) 2495 verify_uniqueness() 2496 self.cmd('create', self.repository_location + '::test', 'input') 2497 verify_uniqueness() 2498 self.cmd('create', self.repository_location + '::test.2', 'input') 2499 verify_uniqueness() 2500 self.cmd('delete', self.repository_location + '::test.2') 2501 verify_uniqueness() 2502 2503 def test_aes_counter_uniqueness_keyfile(self): 2504 self.verify_aes_counter_uniqueness('keyfile') 2505 2506 def test_aes_counter_uniqueness_passphrase(self): 2507 self.verify_aes_counter_uniqueness('repokey') 2508 2509 def test_debug_dump_archive_items(self): 2510 self.create_test_files() 2511 self.cmd('init', '--encryption=repokey', self.repository_location) 2512 self.cmd('create', self.repository_location + '::test', 'input') 2513 with changedir('output'): 2514 output = self.cmd('debug', 'dump-archive-items', self.repository_location + '::test') 2515 output_dir = sorted(os.listdir('output')) 2516 assert len(output_dir) > 0 and output_dir[0].startswith('000000_') 2517 assert 'Done.' in output 2518 2519 def test_debug_dump_repo_objs(self): 2520 self.create_test_files() 2521 self.cmd('init', '--encryption=repokey', self.repository_location) 2522 self.cmd('create', self.repository_location + '::test', 'input') 2523 with changedir('output'): 2524 output = self.cmd('debug', 'dump-repo-objs', self.repository_location) 2525 output_dir = sorted(os.listdir('output')) 2526 assert len(output_dir) > 0 and output_dir[0].startswith('00000000_') 2527 assert 'Done.' in output 2528 2529 def test_debug_put_get_delete_obj(self): 2530 self.cmd('init', '--encryption=repokey', self.repository_location) 2531 data = b'some data' 2532 hexkey = sha256(data).hexdigest() 2533 self.create_regular_file('file', contents=data) 2534 output = self.cmd('debug', 'put-obj', self.repository_location, 'input/file') 2535 assert hexkey in output 2536 output = self.cmd('debug', 'get-obj', self.repository_location, hexkey, 'output/file') 2537 assert hexkey in output 2538 with open('output/file', 'rb') as f: 2539 data_read = f.read() 2540 assert data == data_read 2541 output = self.cmd('debug', 'delete-obj', self.repository_location, hexkey) 2542 assert "deleted" in output 2543 output = self.cmd('debug', 'delete-obj', self.repository_location, hexkey) 2544 assert "not found" in output 2545 output = self.cmd('debug', 'delete-obj', self.repository_location, 'invalid') 2546 assert "is invalid" in output 2547 2548 def test_init_interrupt(self): 2549 def raise_eof(*args): 2550 raise EOFError 2551 2552 with patch.object(KeyfileKeyBase, 'create', raise_eof): 2553 self.cmd('init', '--encryption=repokey', self.repository_location, exit_code=1) 2554 assert not os.path.exists(self.repository_location) 2555 2556 def test_init_requires_encryption_option(self): 2557 self.cmd('init', self.repository_location, exit_code=2) 2558 2559 def test_init_nested_repositories(self): 2560 self.cmd('init', '--encryption=repokey', self.repository_location) 2561 if self.FORK_DEFAULT: 2562 self.cmd('init', '--encryption=repokey', self.repository_location + '/nested', exit_code=2) 2563 else: 2564 with pytest.raises(Repository.AlreadyExists): 2565 self.cmd('init', '--encryption=repokey', self.repository_location + '/nested') 2566 2567 def check_cache(self): 2568 # First run a regular borg check 2569 self.cmd('check', self.repository_location) 2570 # Then check that the cache on disk matches exactly what's in the repo. 2571 with self.open_repository() as repository: 2572 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 2573 with Cache(repository, key, manifest, sync=False) as cache: 2574 original_chunks = cache.chunks 2575 Cache.destroy(repository) 2576 with Cache(repository, key, manifest) as cache: 2577 correct_chunks = cache.chunks 2578 assert original_chunks is not correct_chunks 2579 seen = set() 2580 for id, (refcount, size, csize) in correct_chunks.iteritems(): 2581 o_refcount, o_size, o_csize = original_chunks[id] 2582 assert refcount == o_refcount 2583 assert size == o_size 2584 assert csize == o_csize 2585 seen.add(id) 2586 for id, (refcount, size, csize) in original_chunks.iteritems(): 2587 assert id in seen 2588 2589 def test_check_cache(self): 2590 self.cmd('init', '--encryption=repokey', self.repository_location) 2591 self.cmd('create', self.repository_location + '::test', 'input') 2592 with self.open_repository() as repository: 2593 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 2594 with Cache(repository, key, manifest, sync=False) as cache: 2595 cache.begin_txn() 2596 cache.chunks.incref(list(cache.chunks.iteritems())[0][0]) 2597 cache.commit() 2598 with pytest.raises(AssertionError): 2599 self.check_cache() 2600 2601 def test_recreate_target_rc(self): 2602 self.cmd('init', '--encryption=repokey', self.repository_location) 2603 output = self.cmd('recreate', self.repository_location, '--target=asdf', exit_code=2) 2604 assert 'Need to specify single archive' in output 2605 2606 def test_recreate_target(self): 2607 self.create_test_files() 2608 self.cmd('init', '--encryption=repokey', self.repository_location) 2609 self.check_cache() 2610 archive = self.repository_location + '::test0' 2611 self.cmd('create', archive, 'input') 2612 self.check_cache() 2613 original_archive = self.cmd('list', self.repository_location) 2614 self.cmd('recreate', archive, 'input/dir2', '-e', 'input/dir2/file3', '--target=new-archive') 2615 self.check_cache() 2616 archives = self.cmd('list', self.repository_location) 2617 assert original_archive in archives 2618 assert 'new-archive' in archives 2619 2620 archive = self.repository_location + '::new-archive' 2621 listing = self.cmd('list', '--short', archive) 2622 assert 'file1' not in listing 2623 assert 'dir2/file2' in listing 2624 assert 'dir2/file3' not in listing 2625 2626 def test_recreate_basic(self): 2627 self.create_test_files() 2628 self.create_regular_file('dir2/file3', size=1024 * 80) 2629 self.cmd('init', '--encryption=repokey', self.repository_location) 2630 archive = self.repository_location + '::test0' 2631 self.cmd('create', archive, 'input') 2632 self.cmd('recreate', archive, 'input/dir2', '-e', 'input/dir2/file3') 2633 self.check_cache() 2634 listing = self.cmd('list', '--short', archive) 2635 assert 'file1' not in listing 2636 assert 'dir2/file2' in listing 2637 assert 'dir2/file3' not in listing 2638 2639 @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') 2640 def test_recreate_subtree_hardlinks(self): 2641 # This is essentially the same problem set as in test_extract_hardlinks 2642 self._extract_hardlinks_setup() 2643 self.cmd('create', self.repository_location + '::test2', 'input') 2644 self.cmd('recreate', self.repository_location + '::test', 'input/dir1') 2645 self.check_cache() 2646 with changedir('output'): 2647 self.cmd('extract', self.repository_location + '::test') 2648 assert os.stat('input/dir1/hardlink').st_nlink == 2 2649 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 2650 assert os.stat('input/dir1/aaaa').st_nlink == 2 2651 assert os.stat('input/dir1/source2').st_nlink == 2 2652 with changedir('output'): 2653 self.cmd('extract', self.repository_location + '::test2') 2654 assert os.stat('input/dir1/hardlink').st_nlink == 4 2655 2656 def test_recreate_rechunkify(self): 2657 with open(os.path.join(self.input_path, 'large_file'), 'wb') as fd: 2658 fd.write(b'a' * 280) 2659 fd.write(b'b' * 280) 2660 self.cmd('init', '--encryption=repokey', self.repository_location) 2661 self.cmd('create', '--chunker-params', '7,9,8,128', self.repository_location + '::test1', 'input') 2662 self.cmd('create', self.repository_location + '::test2', 'input', '--files-cache=disabled') 2663 list = self.cmd('list', self.repository_location + '::test1', 'input/large_file', 2664 '--format', '{num_chunks} {unique_chunks}') 2665 num_chunks, unique_chunks = map(int, list.split(' ')) 2666 # test1 and test2 do not deduplicate 2667 assert num_chunks == unique_chunks 2668 self.cmd('recreate', self.repository_location, '--chunker-params', 'default') 2669 self.check_cache() 2670 # test1 and test2 do deduplicate after recreate 2671 assert int(self.cmd('list', self.repository_location + '::test1', 'input/large_file', '--format={size}')) 2672 assert not int(self.cmd('list', self.repository_location + '::test1', 'input/large_file', 2673 '--format', '{unique_chunks}')) 2674 2675 def test_recreate_recompress(self): 2676 self.create_regular_file('compressible', size=10000) 2677 self.cmd('init', '--encryption=repokey', self.repository_location) 2678 self.cmd('create', self.repository_location + '::test', 'input', '-C', 'none') 2679 file_list = self.cmd('list', self.repository_location + '::test', 'input/compressible', 2680 '--format', '{size} {csize} {sha256}') 2681 size, csize, sha256_before = file_list.split(' ') 2682 assert int(csize) >= int(size) # >= due to metadata overhead 2683 self.cmd('recreate', self.repository_location, '-C', 'lz4', '--recompress') 2684 self.check_cache() 2685 file_list = self.cmd('list', self.repository_location + '::test', 'input/compressible', 2686 '--format', '{size} {csize} {sha256}') 2687 size, csize, sha256_after = file_list.split(' ') 2688 assert int(csize) < int(size) 2689 assert sha256_before == sha256_after 2690 2691 def test_recreate_timestamp(self): 2692 local_timezone = datetime.now(timezone(timedelta(0))).astimezone().tzinfo 2693 self.create_test_files() 2694 self.cmd('init', '--encryption=repokey', self.repository_location) 2695 archive = self.repository_location + '::test0' 2696 self.cmd('create', archive, 'input') 2697 self.cmd('recreate', '--timestamp', "1970-01-02T00:00:00", '--comment', 2698 'test', archive) 2699 info = self.cmd('info', archive).splitlines() 2700 dtime = datetime(1970, 1, 2) + local_timezone.utcoffset(None) 2701 s_time = dtime.strftime("%Y-%m-%d") 2702 assert any([re.search(r'Time \(start\).+ %s' % s_time, item) for item in info]) 2703 assert any([re.search(r'Time \(end\).+ %s' % s_time, item) for item in info]) 2704 2705 def test_recreate_dry_run(self): 2706 self.create_regular_file('compressible', size=10000) 2707 self.cmd('init', '--encryption=repokey', self.repository_location) 2708 self.cmd('create', self.repository_location + '::test', 'input') 2709 archives_before = self.cmd('list', self.repository_location + '::test') 2710 self.cmd('recreate', self.repository_location, '-n', '-e', 'input/compressible') 2711 self.check_cache() 2712 archives_after = self.cmd('list', self.repository_location + '::test') 2713 assert archives_after == archives_before 2714 2715 def test_recreate_skips_nothing_to_do(self): 2716 self.create_regular_file('file1', size=1024 * 80) 2717 self.cmd('init', '--encryption=repokey', self.repository_location) 2718 self.cmd('create', self.repository_location + '::test', 'input') 2719 info_before = self.cmd('info', self.repository_location + '::test') 2720 self.cmd('recreate', self.repository_location, '--chunker-params', 'default') 2721 self.check_cache() 2722 info_after = self.cmd('info', self.repository_location + '::test') 2723 assert info_before == info_after # includes archive ID 2724 2725 def test_with_lock(self): 2726 self.cmd('init', '--encryption=repokey', self.repository_location) 2727 lock_path = os.path.join(self.repository_path, 'lock.exclusive') 2728 cmd = 'python3', '-c', 'import os, sys; sys.exit(42 if os.path.exists("%s") else 23)' % lock_path 2729 self.cmd('with-lock', self.repository_location, *cmd, fork=True, exit_code=42) 2730 2731 def test_recreate_list_output(self): 2732 self.cmd('init', '--encryption=repokey', self.repository_location) 2733 self.create_regular_file('file1', size=0) 2734 self.create_regular_file('file2', size=0) 2735 self.create_regular_file('file3', size=0) 2736 self.create_regular_file('file4', size=0) 2737 self.create_regular_file('file5', size=0) 2738 2739 self.cmd('create', self.repository_location + '::test', 'input') 2740 2741 output = self.cmd('recreate', '--list', '--info', self.repository_location + '::test', '-e', 'input/file2') 2742 self.check_cache() 2743 self.assert_in("input/file1", output) 2744 self.assert_in("x input/file2", output) 2745 2746 output = self.cmd('recreate', '--list', self.repository_location + '::test', '-e', 'input/file3') 2747 self.check_cache() 2748 self.assert_in("input/file1", output) 2749 self.assert_in("x input/file3", output) 2750 2751 output = self.cmd('recreate', self.repository_location + '::test', '-e', 'input/file4') 2752 self.check_cache() 2753 self.assert_not_in("input/file1", output) 2754 self.assert_not_in("x input/file4", output) 2755 2756 output = self.cmd('recreate', '--info', self.repository_location + '::test', '-e', 'input/file5') 2757 self.check_cache() 2758 self.assert_not_in("input/file1", output) 2759 self.assert_not_in("x input/file5", output) 2760 2761 def test_bad_filters(self): 2762 self.cmd('init', '--encryption=repokey', self.repository_location) 2763 self.cmd('create', self.repository_location + '::test', 'input') 2764 self.cmd('delete', '--first', '1', '--last', '1', self.repository_location, fork=True, exit_code=2) 2765 2766 def test_key_export_keyfile(self): 2767 export_file = self.output_path + '/exported' 2768 self.cmd('init', self.repository_location, '--encryption', 'keyfile') 2769 repo_id = self._extract_repository_id(self.repository_path) 2770 self.cmd('key', 'export', self.repository_location, export_file) 2771 2772 with open(export_file, 'r') as fd: 2773 export_contents = fd.read() 2774 2775 assert export_contents.startswith('BORG_KEY ' + bin_to_hex(repo_id) + '\n') 2776 2777 key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0] 2778 2779 with open(key_file, 'r') as fd: 2780 key_contents = fd.read() 2781 2782 assert key_contents == export_contents 2783 2784 os.unlink(key_file) 2785 2786 self.cmd('key', 'import', self.repository_location, export_file) 2787 2788 with open(key_file, 'r') as fd: 2789 key_contents2 = fd.read() 2790 2791 assert key_contents2 == key_contents 2792 2793 def test_key_export_repokey(self): 2794 export_file = self.output_path + '/exported' 2795 self.cmd('init', self.repository_location, '--encryption', 'repokey') 2796 repo_id = self._extract_repository_id(self.repository_path) 2797 self.cmd('key', 'export', self.repository_location, export_file) 2798 2799 with open(export_file, 'r') as fd: 2800 export_contents = fd.read() 2801 2802 assert export_contents.startswith('BORG_KEY ' + bin_to_hex(repo_id) + '\n') 2803 2804 with Repository(self.repository_path) as repository: 2805 repo_key = RepoKey(repository) 2806 repo_key.load(None, Passphrase.env_passphrase()) 2807 2808 backup_key = KeyfileKey(key.TestKey.MockRepository()) 2809 backup_key.load(export_file, Passphrase.env_passphrase()) 2810 2811 assert repo_key.enc_key == backup_key.enc_key 2812 2813 with Repository(self.repository_path) as repository: 2814 repository.save_key(b'') 2815 2816 self.cmd('key', 'import', self.repository_location, export_file) 2817 2818 with Repository(self.repository_path) as repository: 2819 repo_key2 = RepoKey(repository) 2820 repo_key2.load(None, Passphrase.env_passphrase()) 2821 2822 assert repo_key2.enc_key == repo_key2.enc_key 2823 2824 def test_key_export_qr(self): 2825 export_file = self.output_path + '/exported.html' 2826 self.cmd('init', self.repository_location, '--encryption', 'repokey') 2827 repo_id = self._extract_repository_id(self.repository_path) 2828 self.cmd('key', 'export', '--qr-html', self.repository_location, export_file) 2829 2830 with open(export_file, 'r', encoding='utf-8') as fd: 2831 export_contents = fd.read() 2832 2833 assert bin_to_hex(repo_id) in export_contents 2834 assert export_contents.startswith('<!doctype html>') 2835 assert export_contents.endswith('</html>\n') 2836 2837 def test_key_export_directory(self): 2838 export_directory = self.output_path + '/exported' 2839 os.mkdir(export_directory) 2840 2841 self.cmd('init', self.repository_location, '--encryption', 'repokey') 2842 2843 self.cmd('key', 'export', self.repository_location, export_directory, exit_code=EXIT_ERROR) 2844 2845 def test_key_import_errors(self): 2846 export_file = self.output_path + '/exported' 2847 self.cmd('init', self.repository_location, '--encryption', 'keyfile') 2848 2849 self.cmd('key', 'import', self.repository_location, export_file, exit_code=EXIT_ERROR) 2850 2851 with open(export_file, 'w') as fd: 2852 fd.write('something not a key\n') 2853 2854 if self.FORK_DEFAULT: 2855 self.cmd('key', 'import', self.repository_location, export_file, exit_code=2) 2856 else: 2857 with pytest.raises(NotABorgKeyFile): 2858 self.cmd('key', 'import', self.repository_location, export_file) 2859 2860 with open(export_file, 'w') as fd: 2861 fd.write('BORG_KEY a0a0a0\n') 2862 2863 if self.FORK_DEFAULT: 2864 self.cmd('key', 'import', self.repository_location, export_file, exit_code=2) 2865 else: 2866 with pytest.raises(RepoIdMismatch): 2867 self.cmd('key', 'import', self.repository_location, export_file) 2868 2869 def test_key_export_paperkey(self): 2870 repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239' 2871 2872 export_file = self.output_path + '/exported' 2873 self.cmd('init', self.repository_location, '--encryption', 'keyfile') 2874 self._set_repository_id(self.repository_path, unhexlify(repo_id)) 2875 2876 key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0] 2877 2878 with open(key_file, 'w') as fd: 2879 fd.write(KeyfileKey.FILE_ID + ' ' + repo_id + '\n') 2880 fd.write(b2a_base64(b'abcdefghijklmnopqrstu').decode()) 2881 2882 self.cmd('key', 'export', '--paper', self.repository_location, export_file) 2883 2884 with open(export_file, 'r') as fd: 2885 export_contents = fd.read() 2886 2887 assert export_contents == """To restore key use borg key import --paper /path/to/repo 2888 2889BORG PAPER KEY v1 2890id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02 2891 1: 616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d 2892 2: 737475 - 88 2893""" 2894 2895 def test_key_import_paperkey(self): 2896 repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239' 2897 self.cmd('init', self.repository_location, '--encryption', 'keyfile') 2898 self._set_repository_id(self.repository_path, unhexlify(repo_id)) 2899 2900 key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0] 2901 with open(key_file, 'w') as fd: 2902 fd.write(KeyfileKey.FILE_ID + ' ' + repo_id + '\n') 2903 fd.write(b2a_base64(b'abcdefghijklmnopqrstu').decode()) 2904 2905 typed_input = ( 2906 b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 02\n' # Forgot to type "-" 2907 b'2 / e29442 3506da 4e1ea7 25f62a 5a3d41 - 02\n' # Forgot to type second "/" 2908 b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d42 - 02\n' # Typo (..42 not ..41) 2909 b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n' # Correct! Congratulations 2910 b'616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d\n' 2911 b'\n\n' # Abort [yN] => N 2912 b'737475 88\n' # missing "-" 2913 b'73747i - 88\n' # typo 2914 b'73747 - 88\n' # missing nibble 2915 b'73 74 75 - 89\n' # line checksum mismatch 2916 b'00a1 - 88\n' # line hash collision - overall hash mismatch, have to start over 2917 2918 b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n' 2919 b'616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d\n' 2920 b'73 74 75 - 88\n' 2921 ) 2922 2923 # In case that this has to change, here is a quick way to find a colliding line hash: 2924 # 2925 # from hashlib import sha256 2926 # hash_fn = lambda x: sha256(b'\x00\x02' + x).hexdigest()[:2] 2927 # for i in range(1000): 2928 # if hash_fn(i.to_bytes(2, byteorder='big')) == '88': # 88 = line hash 2929 # print(i.to_bytes(2, 'big')) 2930 # break 2931 2932 self.cmd('key', 'import', '--paper', self.repository_location, input=typed_input) 2933 2934 # Test abort paths 2935 typed_input = b'\ny\n' 2936 self.cmd('key', 'import', '--paper', self.repository_location, input=typed_input) 2937 typed_input = b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n\ny\n' 2938 self.cmd('key', 'import', '--paper', self.repository_location, input=typed_input) 2939 2940 def test_debug_dump_manifest(self): 2941 self.create_regular_file('file1', size=1024 * 80) 2942 self.cmd('init', '--encryption=repokey', self.repository_location) 2943 self.cmd('create', self.repository_location + '::test', 'input') 2944 dump_file = self.output_path + '/dump' 2945 output = self.cmd('debug', 'dump-manifest', self.repository_location, dump_file) 2946 assert output == "" 2947 with open(dump_file, "r") as f: 2948 result = json.load(f) 2949 assert 'archives' in result 2950 assert 'config' in result 2951 assert 'item_keys' in result 2952 assert 'timestamp' in result 2953 assert 'version' in result 2954 2955 def test_debug_dump_archive(self): 2956 self.create_regular_file('file1', size=1024 * 80) 2957 self.cmd('init', '--encryption=repokey', self.repository_location) 2958 self.cmd('create', self.repository_location + '::test', 'input') 2959 dump_file = self.output_path + '/dump' 2960 output = self.cmd('debug', 'dump-archive', self.repository_location + "::test", dump_file) 2961 assert output == "" 2962 with open(dump_file, "r") as f: 2963 result = json.load(f) 2964 assert '_name' in result 2965 assert '_manifest_entry' in result 2966 assert '_meta' in result 2967 assert '_items' in result 2968 2969 def test_debug_refcount_obj(self): 2970 self.cmd('init', '--encryption=repokey', self.repository_location) 2971 output = self.cmd('debug', 'refcount-obj', self.repository_location, '0' * 64).strip() 2972 assert output == 'object 0000000000000000000000000000000000000000000000000000000000000000 not found [info from chunks cache].' 2973 2974 create_json = json.loads(self.cmd('create', '--json', self.repository_location + '::test', 'input')) 2975 archive_id = create_json['archive']['id'] 2976 output = self.cmd('debug', 'refcount-obj', self.repository_location, archive_id).strip() 2977 assert output == 'object ' + archive_id + ' has 1 referrers [info from chunks cache].' 2978 2979 # Invalid IDs do not abort or return an error 2980 output = self.cmd('debug', 'refcount-obj', self.repository_location, '124', 'xyza').strip() 2981 assert output == 'object id 124 is invalid.\nobject id xyza is invalid.' 2982 2983 def test_debug_info(self): 2984 output = self.cmd('debug', 'info') 2985 assert 'CRC implementation' in output 2986 assert 'Python' in output 2987 2988 def test_benchmark_crud(self): 2989 self.cmd('init', '--encryption=repokey', self.repository_location) 2990 with environment_variable(_BORG_BENCHMARK_CRUD_TEST='YES'): 2991 self.cmd('benchmark', 'crud', self.repository_location, self.input_path) 2992 2993 def test_config(self): 2994 self.create_test_files() 2995 os.unlink('input/flagfile') 2996 self.cmd('init', '--encryption=repokey', self.repository_location) 2997 output = self.cmd('config', '--list', self.repository_location) 2998 self.assert_in('[repository]', output) 2999 self.assert_in('version', output) 3000 self.assert_in('segments_per_dir', output) 3001 self.assert_in('storage_quota', output) 3002 self.assert_in('append_only', output) 3003 self.assert_in('additional_free_space', output) 3004 self.assert_in('id', output) 3005 for cfg_key, cfg_value in [ 3006 ('additional_free_space', '2G'), 3007 ('repository.append_only', '1'), 3008 ]: 3009 output = self.cmd('config', self.repository_location, cfg_key) 3010 assert output == '0' + '\n' 3011 self.cmd('config', self.repository_location, cfg_key, cfg_value) 3012 output = self.cmd('config', self.repository_location, cfg_key) 3013 assert output == cfg_value + '\n' 3014 self.cmd('config', '--delete', self.repository_location, cfg_key) 3015 self.cmd('config', self.repository_location, cfg_key, exit_code=1) 3016 self.cmd('config', '--list', '--delete', self.repository_location, exit_code=2) 3017 self.cmd('config', self.repository_location, exit_code=2) 3018 self.cmd('config', self.repository_location, 'invalid-option', exit_code=1) 3019 3020 requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') 3021 requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') 3022 3023 @requires_gnutar 3024 def test_export_tar(self): 3025 self.create_test_files() 3026 os.unlink('input/flagfile') 3027 self.cmd('init', '--encryption=repokey', self.repository_location) 3028 self.cmd('create', self.repository_location + '::test', 'input') 3029 self.cmd('export-tar', self.repository_location + '::test', 'simple.tar', '--progress') 3030 with changedir('output'): 3031 # This probably assumes GNU tar. Note -p switch to extract permissions regardless of umask. 3032 subprocess.check_call(['tar', 'xpf', '../simple.tar', '--warning=no-timestamp']) 3033 self.assert_dirs_equal('input', 'output/input', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True) 3034 3035 @requires_gnutar 3036 @requires_gzip 3037 def test_export_tar_gz(self): 3038 if not shutil.which('gzip'): 3039 pytest.skip('gzip is not installed') 3040 self.create_test_files() 3041 os.unlink('input/flagfile') 3042 self.cmd('init', '--encryption=repokey', self.repository_location) 3043 self.cmd('create', self.repository_location + '::test', 'input') 3044 list = self.cmd('export-tar', self.repository_location + '::test', 'simple.tar.gz', '--list') 3045 assert 'input/file1\n' in list 3046 assert 'input/dir2\n' in list 3047 with changedir('output'): 3048 subprocess.check_call(['tar', 'xpf', '../simple.tar.gz', '--warning=no-timestamp']) 3049 self.assert_dirs_equal('input', 'output/input', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True) 3050 3051 @requires_gnutar 3052 def test_export_tar_strip_components(self): 3053 if not shutil.which('gzip'): 3054 pytest.skip('gzip is not installed') 3055 self.create_test_files() 3056 os.unlink('input/flagfile') 3057 self.cmd('init', '--encryption=repokey', self.repository_location) 3058 self.cmd('create', self.repository_location + '::test', 'input') 3059 list = self.cmd('export-tar', self.repository_location + '::test', 'simple.tar', '--strip-components=1', '--list') 3060 # --list's path are those before processing with --strip-components 3061 assert 'input/file1\n' in list 3062 assert 'input/dir2\n' in list 3063 with changedir('output'): 3064 subprocess.check_call(['tar', 'xpf', '../simple.tar', '--warning=no-timestamp']) 3065 self.assert_dirs_equal('input', 'output/', ignore_bsdflags=True, ignore_xattrs=True, ignore_ns=True) 3066 3067 @requires_hardlinks 3068 @requires_gnutar 3069 def test_export_tar_strip_components_links(self): 3070 self._extract_hardlinks_setup() 3071 self.cmd('export-tar', self.repository_location + '::test', 'output.tar', '--strip-components=2') 3072 with changedir('output'): 3073 subprocess.check_call(['tar', 'xpf', '../output.tar', '--warning=no-timestamp']) 3074 assert os.stat('hardlink').st_nlink == 2 3075 assert os.stat('subdir/hardlink').st_nlink == 2 3076 assert os.stat('aaaa').st_nlink == 2 3077 assert os.stat('source2').st_nlink == 2 3078 3079 @requires_hardlinks 3080 @requires_gnutar 3081 def test_extract_hardlinks_tar(self): 3082 self._extract_hardlinks_setup() 3083 self.cmd('export-tar', self.repository_location + '::test', 'output.tar', 'input/dir1') 3084 with changedir('output'): 3085 subprocess.check_call(['tar', 'xpf', '../output.tar', '--warning=no-timestamp']) 3086 assert os.stat('input/dir1/hardlink').st_nlink == 2 3087 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 3088 assert os.stat('input/dir1/aaaa').st_nlink == 2 3089 assert os.stat('input/dir1/source2').st_nlink == 2 3090 3091 def test_detect_attic_repo(self): 3092 path = make_attic_repo(self.repository_path) 3093 cmds = [ 3094 ['create', path + '::test', self.tmpdir], 3095 ['extract', path + '::test'], 3096 ['check', path], 3097 ['rename', path + '::test', 'newname'], 3098 ['list', path], 3099 ['delete', path], 3100 ['prune', path], 3101 ['info', path + '::test'], 3102 ['key', 'export', path, 'exported'], 3103 ['key', 'import', path, 'import'], 3104 ['change-passphrase', path], 3105 ['break-lock', path], 3106 ] 3107 for args in cmds: 3108 output = self.cmd(*args, fork=True, exit_code=2) 3109 assert 'Attic repository detected.' in output 3110 3111 3112@unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') 3113class ArchiverTestCaseBinary(ArchiverTestCase): 3114 EXE = 'borg.exe' 3115 FORK_DEFAULT = True 3116 3117 @unittest.skip('does not raise Exception, but sets rc==2') 3118 def test_init_parent_dirs(self): 3119 pass 3120 3121 @unittest.skip('patches objects') 3122 def test_init_interrupt(self): 3123 pass 3124 3125 @unittest.skip('patches objects') 3126 def test_extract_capabilities(self): 3127 pass 3128 3129 @unittest.skip('patches objects') 3130 def test_extract_xattrs_errors(self): 3131 pass 3132 3133 @unittest.skip('test_basic_functionality seems incompatible with fakeroot and/or the binary.') 3134 def test_basic_functionality(self): 3135 pass 3136 3137 @unittest.skip('test_overwrite seems incompatible with fakeroot and/or the binary.') 3138 def test_overwrite(self): 3139 pass 3140 3141 def test_fuse(self): 3142 if fakeroot_detected(): 3143 unittest.skip('test_fuse with the binary is not compatible with fakeroot') 3144 else: 3145 super().test_fuse() 3146 3147 3148class ArchiverCheckTestCase(ArchiverTestCaseBase): 3149 3150 def setUp(self): 3151 super().setUp() 3152 with patch.object(ChunkBuffer, 'BUFFER_SIZE', 10): 3153 self.cmd('init', '--encryption=repokey', self.repository_location) 3154 self.create_src_archive('archive1') 3155 self.create_src_archive('archive2') 3156 3157 def test_check_usage(self): 3158 output = self.cmd('check', '-v', '--progress', self.repository_location, exit_code=0) 3159 self.assert_in('Starting repository check', output) 3160 self.assert_in('Starting archive consistency check', output) 3161 self.assert_in('Checking segments', output) 3162 # reset logging to new process default to avoid need for fork=True on next check 3163 logging.getLogger('borg.output.progress').setLevel(logging.NOTSET) 3164 output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0) 3165 self.assert_in('Starting repository check', output) 3166 self.assert_not_in('Starting archive consistency check', output) 3167 self.assert_not_in('Checking segments', output) 3168 output = self.cmd('check', '-v', '--archives-only', self.repository_location, exit_code=0) 3169 self.assert_not_in('Starting repository check', output) 3170 self.assert_in('Starting archive consistency check', output) 3171 output = self.cmd('check', '-v', '--archives-only', '--prefix=archive2', self.repository_location, exit_code=0) 3172 self.assert_not_in('archive1', output) 3173 output = self.cmd('check', '-v', '--archives-only', '--first=1', self.repository_location, exit_code=0) 3174 self.assert_in('archive1', output) 3175 self.assert_not_in('archive2', output) 3176 output = self.cmd('check', '-v', '--archives-only', '--last=1', self.repository_location, exit_code=0) 3177 self.assert_not_in('archive1', output) 3178 self.assert_in('archive2', output) 3179 3180 def test_missing_file_chunk(self): 3181 archive, repository = self.open_archive('archive1') 3182 with repository: 3183 for item in archive.iter_items(): 3184 if item.path.endswith('testsuite/archiver.py'): 3185 valid_chunks = item.chunks 3186 killed_chunk = valid_chunks[-1] 3187 repository.delete(killed_chunk.id) 3188 break 3189 else: 3190 self.fail('should not happen') 3191 repository.commit() 3192 self.cmd('check', self.repository_location, exit_code=1) 3193 output = self.cmd('check', '--repair', self.repository_location, exit_code=0) 3194 self.assert_in('New missing file chunk detected', output) 3195 self.cmd('check', self.repository_location, exit_code=0) 3196 output = self.cmd('list', '--format={health}#{path}{LF}', self.repository_location + '::archive1', exit_code=0) 3197 self.assert_in('broken#', output) 3198 # check that the file in the old archives has now a different chunk list without the killed chunk 3199 for archive_name in ('archive1', 'archive2'): 3200 archive, repository = self.open_archive(archive_name) 3201 with repository: 3202 for item in archive.iter_items(): 3203 if item.path.endswith('testsuite/archiver.py'): 3204 self.assert_not_equal(valid_chunks, item.chunks) 3205 self.assert_not_in(killed_chunk, item.chunks) 3206 break 3207 else: 3208 self.fail('should not happen') 3209 # do a fresh backup (that will include the killed chunk) 3210 with patch.object(ChunkBuffer, 'BUFFER_SIZE', 10): 3211 self.create_src_archive('archive3') 3212 # check should be able to heal the file now: 3213 output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) 3214 self.assert_in('Healed previously missing file chunk', output) 3215 self.assert_in('testsuite/archiver.py: Completely healed previously damaged file!', output) 3216 # check that the file in the old archives has the correct chunks again 3217 for archive_name in ('archive1', 'archive2'): 3218 archive, repository = self.open_archive(archive_name) 3219 with repository: 3220 for item in archive.iter_items(): 3221 if item.path.endswith('testsuite/archiver.py'): 3222 self.assert_equal(valid_chunks, item.chunks) 3223 break 3224 else: 3225 self.fail('should not happen') 3226 # list is also all-healthy again 3227 output = self.cmd('list', '--format={health}#{path}{LF}', self.repository_location + '::archive1', exit_code=0) 3228 self.assert_not_in('broken#', output) 3229 3230 def test_missing_archive_item_chunk(self): 3231 archive, repository = self.open_archive('archive1') 3232 with repository: 3233 repository.delete(archive.metadata.items[0]) 3234 repository.commit() 3235 self.cmd('check', self.repository_location, exit_code=1) 3236 self.cmd('check', '--repair', self.repository_location, exit_code=0) 3237 self.cmd('check', self.repository_location, exit_code=0) 3238 3239 def test_missing_archive_metadata(self): 3240 archive, repository = self.open_archive('archive1') 3241 with repository: 3242 repository.delete(archive.id) 3243 repository.commit() 3244 self.cmd('check', self.repository_location, exit_code=1) 3245 self.cmd('check', '--repair', self.repository_location, exit_code=0) 3246 self.cmd('check', self.repository_location, exit_code=0) 3247 3248 def test_missing_manifest(self): 3249 archive, repository = self.open_archive('archive1') 3250 with repository: 3251 repository.delete(Manifest.MANIFEST_ID) 3252 repository.commit() 3253 self.cmd('check', self.repository_location, exit_code=1) 3254 output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) 3255 self.assert_in('archive1', output) 3256 self.assert_in('archive2', output) 3257 self.cmd('check', self.repository_location, exit_code=0) 3258 3259 def test_corrupted_manifest(self): 3260 archive, repository = self.open_archive('archive1') 3261 with repository: 3262 manifest = repository.get(Manifest.MANIFEST_ID) 3263 corrupted_manifest = manifest + b'corrupted!' 3264 repository.put(Manifest.MANIFEST_ID, corrupted_manifest) 3265 repository.commit() 3266 self.cmd('check', self.repository_location, exit_code=1) 3267 output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) 3268 self.assert_in('archive1', output) 3269 self.assert_in('archive2', output) 3270 self.cmd('check', self.repository_location, exit_code=0) 3271 3272 def test_manifest_rebuild_corrupted_chunk(self): 3273 archive, repository = self.open_archive('archive1') 3274 with repository: 3275 manifest = repository.get(Manifest.MANIFEST_ID) 3276 corrupted_manifest = manifest + b'corrupted!' 3277 repository.put(Manifest.MANIFEST_ID, corrupted_manifest) 3278 3279 chunk = repository.get(archive.id) 3280 corrupted_chunk = chunk + b'corrupted!' 3281 repository.put(archive.id, corrupted_chunk) 3282 repository.commit() 3283 self.cmd('check', self.repository_location, exit_code=1) 3284 output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) 3285 self.assert_in('archive2', output) 3286 self.cmd('check', self.repository_location, exit_code=0) 3287 3288 def test_manifest_rebuild_duplicate_archive(self): 3289 archive, repository = self.open_archive('archive1') 3290 key = archive.key 3291 with repository: 3292 manifest = repository.get(Manifest.MANIFEST_ID) 3293 corrupted_manifest = manifest + b'corrupted!' 3294 repository.put(Manifest.MANIFEST_ID, corrupted_manifest) 3295 3296 archive = msgpack.packb({ 3297 'cmdline': [], 3298 'items': [], 3299 'hostname': 'foo', 3300 'username': 'bar', 3301 'name': 'archive1', 3302 'time': '2016-12-15T18:49:51.849711', 3303 'version': 1, 3304 }) 3305 archive_id = key.id_hash(archive) 3306 repository.put(archive_id, key.encrypt(archive)) 3307 repository.commit() 3308 self.cmd('check', self.repository_location, exit_code=1) 3309 self.cmd('check', '--repair', self.repository_location, exit_code=0) 3310 output = self.cmd('list', self.repository_location) 3311 self.assert_in('archive1', output) 3312 self.assert_in('archive1.1', output) 3313 self.assert_in('archive2', output) 3314 3315 def test_extra_chunks(self): 3316 self.cmd('check', self.repository_location, exit_code=0) 3317 with Repository(self.repository_location, exclusive=True) as repository: 3318 repository.put(b'01234567890123456789012345678901', b'xxxx') 3319 repository.commit() 3320 self.cmd('check', self.repository_location, exit_code=1) 3321 self.cmd('check', self.repository_location, exit_code=1) 3322 self.cmd('check', '--repair', self.repository_location, exit_code=0) 3323 self.cmd('check', self.repository_location, exit_code=0) 3324 self.cmd('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0) 3325 3326 def _test_verify_data(self, *init_args): 3327 shutil.rmtree(self.repository_path) 3328 self.cmd('init', self.repository_location, *init_args) 3329 self.create_src_archive('archive1') 3330 archive, repository = self.open_archive('archive1') 3331 with repository: 3332 for item in archive.iter_items(): 3333 if item.path.endswith('testsuite/archiver.py'): 3334 chunk = item.chunks[-1] 3335 data = repository.get(chunk.id) + b'1234' 3336 repository.put(chunk.id, data) 3337 break 3338 repository.commit() 3339 self.cmd('check', self.repository_location, exit_code=0) 3340 output = self.cmd('check', '--verify-data', self.repository_location, exit_code=1) 3341 assert bin_to_hex(chunk.id) + ', integrity error' in output 3342 # repair (heal is tested in another test) 3343 output = self.cmd('check', '--repair', '--verify-data', self.repository_location, exit_code=0) 3344 assert bin_to_hex(chunk.id) + ', integrity error' in output 3345 assert 'testsuite/archiver.py: New missing file chunk detected' in output 3346 3347 def test_verify_data(self): 3348 self._test_verify_data('--encryption', 'repokey') 3349 3350 def test_verify_data_unencrypted(self): 3351 self._test_verify_data('--encryption', 'none') 3352 3353 def test_empty_repository(self): 3354 with Repository(self.repository_location, exclusive=True) as repository: 3355 for id_ in repository.list(): 3356 repository.delete(id_) 3357 repository.commit() 3358 self.cmd('check', self.repository_location, exit_code=1) 3359 3360 def test_attic013_acl_bug(self): 3361 # Attic up to release 0.13 contained a bug where every item unintentionally received 3362 # a b'acl'=None key-value pair. 3363 # This bug can still live on in Borg repositories (through borg upgrade). 3364 class Attic013Item: 3365 def as_dict(self): 3366 return { 3367 # These are required 3368 b'path': '1234', 3369 b'mtime': 0, 3370 b'mode': 0, 3371 b'user': b'0', 3372 b'group': b'0', 3373 b'uid': 0, 3374 b'gid': 0, 3375 # acl is the offending key. 3376 b'acl': None, 3377 } 3378 3379 archive, repository = self.open_archive('archive1') 3380 with repository: 3381 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 3382 with Cache(repository, key, manifest) as cache: 3383 archive = Archive(repository, key, manifest, '0.13', cache=cache, create=True) 3384 archive.items_buffer.add(Attic013Item()) 3385 archive.save() 3386 self.cmd('check', self.repository_location, exit_code=0) 3387 self.cmd('list', self.repository_location + '::0.13', exit_code=0) 3388 3389 3390class ManifestAuthenticationTest(ArchiverTestCaseBase): 3391 def spoof_manifest(self, repository): 3392 with repository: 3393 _, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 3394 repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ 3395 'version': 1, 3396 'archives': {}, 3397 'config': {}, 3398 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), 3399 }))) 3400 repository.commit() 3401 3402 def test_fresh_init_tam_required(self): 3403 self.cmd('init', '--encryption=repokey', self.repository_location) 3404 repository = Repository(self.repository_path, exclusive=True) 3405 with repository: 3406 manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 3407 repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ 3408 'version': 1, 3409 'archives': {}, 3410 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), 3411 }))) 3412 repository.commit() 3413 3414 with pytest.raises(TAMRequiredError): 3415 self.cmd('list', self.repository_location) 3416 3417 def test_not_required(self): 3418 self.cmd('init', '--encryption=repokey', self.repository_location) 3419 self.create_src_archive('archive1234') 3420 repository = Repository(self.repository_path, exclusive=True) 3421 with repository: 3422 shutil.rmtree(get_security_dir(bin_to_hex(repository.id))) 3423 _, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) 3424 key.tam_required = False 3425 key.change_passphrase(key._passphrase) 3426 3427 manifest = msgpack.unpackb(key.decrypt(None, repository.get(Manifest.MANIFEST_ID))) 3428 del manifest[b'tam'] 3429 repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb(manifest))) 3430 repository.commit() 3431 output = self.cmd('list', '--debug', self.repository_location) 3432 assert 'archive1234' in output 3433 assert 'TAM not found and not required' in output 3434 # Run upgrade 3435 self.cmd('upgrade', '--tam', self.repository_location) 3436 # Manifest must be authenticated now 3437 output = self.cmd('list', '--debug', self.repository_location) 3438 assert 'archive1234' in output 3439 assert 'TAM-verified manifest' in output 3440 # Try to spoof / modify pre-1.0.9 3441 self.spoof_manifest(repository) 3442 # Fails 3443 with pytest.raises(TAMRequiredError): 3444 self.cmd('list', self.repository_location) 3445 # Force upgrade 3446 self.cmd('upgrade', '--tam', '--force', self.repository_location) 3447 self.cmd('list', self.repository_location) 3448 3449 def test_disable(self): 3450 self.cmd('init', '--encryption=repokey', self.repository_location) 3451 self.create_src_archive('archive1234') 3452 self.cmd('upgrade', '--disable-tam', self.repository_location) 3453 repository = Repository(self.repository_path, exclusive=True) 3454 self.spoof_manifest(repository) 3455 assert not self.cmd('list', self.repository_location) 3456 3457 def test_disable2(self): 3458 self.cmd('init', '--encryption=repokey', self.repository_location) 3459 self.create_src_archive('archive1234') 3460 repository = Repository(self.repository_path, exclusive=True) 3461 self.spoof_manifest(repository) 3462 self.cmd('upgrade', '--disable-tam', self.repository_location) 3463 assert not self.cmd('list', self.repository_location) 3464 3465 3466class RemoteArchiverTestCase(ArchiverTestCase): 3467 prefix = '__testsuite__:' 3468 3469 def open_repository(self): 3470 return RemoteRepository(Location(self.repository_location)) 3471 3472 def test_remote_repo_restrict_to_path(self): 3473 # restricted to repo directory itself: 3474 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', self.repository_path]): 3475 self.cmd('init', '--encryption=repokey', self.repository_location) 3476 # restricted to repo directory itself, fail for other directories with same prefix: 3477 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', self.repository_path]): 3478 with pytest.raises(PathNotAllowed): 3479 self.cmd('init', '--encryption=repokey', self.repository_location + '_0') 3480 3481 # restricted to a completely different path: 3482 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo']): 3483 with pytest.raises(PathNotAllowed): 3484 self.cmd('init', '--encryption=repokey', self.repository_location + '_1') 3485 path_prefix = os.path.dirname(self.repository_path) 3486 # restrict to repo directory's parent directory: 3487 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', path_prefix]): 3488 self.cmd('init', '--encryption=repokey', self.repository_location + '_2') 3489 # restrict to repo directory's parent directory and another directory: 3490 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo', '--restrict-to-path', path_prefix]): 3491 self.cmd('init', '--encryption=repokey', self.repository_location + '_3') 3492 3493 def test_remote_repo_restrict_to_repository(self): 3494 # restricted to repo directory itself: 3495 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-repository', self.repository_path]): 3496 self.cmd('init', '--encryption=repokey', self.repository_location) 3497 parent_path = os.path.join(self.repository_path, '..') 3498 with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-repository', parent_path]): 3499 with pytest.raises(PathNotAllowed): 3500 self.cmd('init', '--encryption=repokey', self.repository_location) 3501 3502 @unittest.skip('only works locally') 3503 def test_debug_put_get_delete_obj(self): 3504 pass 3505 3506 @unittest.skip('only works locally') 3507 def test_config(self): 3508 pass 3509 3510 def test_strip_components_doesnt_leak(self): 3511 self.cmd('init', '--encryption=repokey', self.repository_location) 3512 self.create_regular_file('dir/file', contents=b"test file contents 1") 3513 self.create_regular_file('dir/file2', contents=b"test file contents 2") 3514 self.create_regular_file('skipped-file1', contents=b"test file contents 3") 3515 self.create_regular_file('skipped-file2', contents=b"test file contents 4") 3516 self.create_regular_file('skipped-file3', contents=b"test file contents 5") 3517 self.cmd('create', self.repository_location + '::test', 'input') 3518 marker = 'cached responses left in RemoteRepository' 3519 with changedir('output'): 3520 res = self.cmd('extract', "--debug", self.repository_location + '::test', '--strip-components', '3') 3521 self.assert_true(marker not in res) 3522 with self.assert_creates_file('file'): 3523 res = self.cmd('extract', "--debug", self.repository_location + '::test', '--strip-components', '2') 3524 self.assert_true(marker not in res) 3525 with self.assert_creates_file('dir/file'): 3526 res = self.cmd('extract', "--debug", self.repository_location + '::test', '--strip-components', '1') 3527 self.assert_true(marker not in res) 3528 with self.assert_creates_file('input/dir/file'): 3529 res = self.cmd('extract', "--debug", self.repository_location + '::test', '--strip-components', '0') 3530 self.assert_true(marker not in res) 3531 3532 3533class ArchiverCorruptionTestCase(ArchiverTestCaseBase): 3534 def setUp(self): 3535 super().setUp() 3536 self.create_test_files() 3537 self.cmd('init', '--encryption=repokey', self.repository_location) 3538 self.cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path'] 3539 3540 def corrupt(self, file, amount=1): 3541 with open(file, 'r+b') as fd: 3542 fd.seek(-amount, io.SEEK_END) 3543 corrupted = bytes(255-c for c in fd.read(amount)) 3544 fd.seek(-amount, io.SEEK_END) 3545 fd.write(corrupted) 3546 3547 def test_cache_chunks(self): 3548 self.corrupt(os.path.join(self.cache_path, 'chunks')) 3549 3550 if self.FORK_DEFAULT: 3551 out = self.cmd('info', self.repository_location, exit_code=2) 3552 assert 'failed integrity check' in out 3553 else: 3554 with pytest.raises(FileIntegrityError): 3555 self.cmd('info', self.repository_location) 3556 3557 def test_cache_files(self): 3558 self.cmd('create', self.repository_location + '::test', 'input') 3559 self.corrupt(os.path.join(self.cache_path, 'files')) 3560 out = self.cmd('create', self.repository_location + '::test1', 'input') 3561 # borg warns about the corrupt files cache, but then continues without files cache. 3562 assert 'files cache is corrupted' in out 3563 3564 def test_chunks_archive(self): 3565 self.cmd('create', self.repository_location + '::test1', 'input') 3566 # Find ID of test1 so we can corrupt it later :) 3567 target_id = self.cmd('list', self.repository_location, '--format={id}{LF}').strip() 3568 self.cmd('create', self.repository_location + '::test2', 'input') 3569 3570 # Force cache sync, creating archive chunks of test1 and test2 in chunks.archive.d 3571 self.cmd('delete', '--cache-only', self.repository_location) 3572 self.cmd('info', self.repository_location, '--json') 3573 3574 chunks_archive = os.path.join(self.cache_path, 'chunks.archive.d') 3575 assert len(os.listdir(chunks_archive)) == 4 # two archives, one chunks cache and one .integrity file each 3576 3577 self.corrupt(os.path.join(chunks_archive, target_id + '.compact')) 3578 3579 # Trigger cache sync by changing the manifest ID in the cache config 3580 config_path = os.path.join(self.cache_path, 'config') 3581 config = ConfigParser(interpolation=None) 3582 config.read(config_path) 3583 config.set('cache', 'manifest', bin_to_hex(bytes(32))) 3584 with open(config_path, 'w') as fd: 3585 config.write(fd) 3586 3587 # Cache sync notices corrupted archive chunks, but automatically recovers. 3588 out = self.cmd('create', '-v', self.repository_location + '::test3', 'input', exit_code=1) 3589 assert 'Reading cached archive chunk index for test1' in out 3590 assert 'Cached archive chunk index of test1 is corrupted' in out 3591 assert 'Fetching and building archive index for test1' in out 3592 3593 def test_old_version_interfered(self): 3594 # Modify the main manifest ID without touching the manifest ID in the integrity section. 3595 # This happens if a version without integrity checking modifies the cache. 3596 config_path = os.path.join(self.cache_path, 'config') 3597 config = ConfigParser(interpolation=None) 3598 config.read(config_path) 3599 config.set('cache', 'manifest', bin_to_hex(bytes(32))) 3600 with open(config_path, 'w') as fd: 3601 config.write(fd) 3602 3603 out = self.cmd('info', self.repository_location) 3604 assert 'Cache integrity data not available: old Borg version modified the cache.' in out 3605 3606 3607class DiffArchiverTestCase(ArchiverTestCaseBase): 3608 def test_basic_functionality(self): 3609 # Initialize test folder 3610 self.create_test_files() 3611 self.cmd('init', '--encryption=repokey', self.repository_location) 3612 3613 # Setup files for the first snapshot 3614 self.create_regular_file('file_unchanged', size=128) 3615 self.create_regular_file('file_removed', size=256) 3616 self.create_regular_file('file_removed2', size=512) 3617 self.create_regular_file('file_replaced', size=1024) 3618 os.mkdir('input/dir_replaced_with_file') 3619 os.chmod('input/dir_replaced_with_file', stat.S_IFDIR | 0o755) 3620 os.mkdir('input/dir_removed') 3621 if are_symlinks_supported(): 3622 os.mkdir('input/dir_replaced_with_link') 3623 os.symlink('input/dir_replaced_with_file', 'input/link_changed') 3624 os.symlink('input/file_unchanged', 'input/link_removed') 3625 os.symlink('input/file_removed2', 'input/link_target_removed') 3626 os.symlink('input/empty', 'input/link_target_contents_changed') 3627 os.symlink('input/empty', 'input/link_replaced_by_file') 3628 if are_hardlinks_supported(): 3629 os.link('input/file_replaced', 'input/hardlink_target_replaced') 3630 os.link('input/empty', 'input/hardlink_contents_changed') 3631 os.link('input/file_removed', 'input/hardlink_removed') 3632 os.link('input/file_removed2', 'input/hardlink_target_removed') 3633 3634 # Create the first snapshot 3635 self.cmd('create', self.repository_location + '::test0', 'input') 3636 3637 # Setup files for the second snapshot 3638 self.create_regular_file('file_added', size=2048) 3639 self.create_regular_file('file_empty_added', size=0) 3640 os.unlink('input/file_replaced') 3641 self.create_regular_file('file_replaced', contents=b'0' * 4096) 3642 os.unlink('input/file_removed') 3643 os.unlink('input/file_removed2') 3644 os.rmdir('input/dir_replaced_with_file') 3645 self.create_regular_file('dir_replaced_with_file', size=8192) 3646 os.chmod('input/dir_replaced_with_file', stat.S_IFREG | 0o755) 3647 os.mkdir('input/dir_added') 3648 os.rmdir('input/dir_removed') 3649 if are_symlinks_supported(): 3650 os.rmdir('input/dir_replaced_with_link') 3651 os.symlink('input/dir_added', 'input/dir_replaced_with_link') 3652 os.unlink('input/link_changed') 3653 os.symlink('input/dir_added', 'input/link_changed') 3654 os.symlink('input/dir_added', 'input/link_added') 3655 os.unlink('input/link_replaced_by_file') 3656 self.create_regular_file('link_replaced_by_file', size=16384) 3657 os.unlink('input/link_removed') 3658 if are_hardlinks_supported(): 3659 os.unlink('input/hardlink_removed') 3660 os.link('input/file_added', 'input/hardlink_added') 3661 3662 with open('input/empty', 'ab') as fd: 3663 fd.write(b'appended_data') 3664 3665 # Create the second snapshot 3666 self.cmd('create', self.repository_location + '::test1a', 'input') 3667 self.cmd('create', '--chunker-params', '16,18,17,4095', self.repository_location + '::test1b', 'input') 3668 3669 def do_asserts(output, can_compare_ids): 3670 # File contents changed (deleted and replaced with a new file) 3671 change = 'B' if can_compare_ids else '{:<19}'.format('modified') 3672 assert 'file_replaced' in output # added to debug #3494 3673 assert '{} input/file_replaced'.format(change) in output 3674 3675 # File unchanged 3676 assert 'input/file_unchanged' not in output 3677 3678 # Directory replaced with a regular file 3679 if 'BORG_TESTS_IGNORE_MODES' not in os.environ: 3680 assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output 3681 3682 # Basic directory cases 3683 assert 'added directory input/dir_added' in output 3684 assert 'removed directory input/dir_removed' in output 3685 3686 if are_symlinks_supported(): 3687 # Basic symlink cases 3688 assert 'changed link input/link_changed' in output 3689 assert 'added link input/link_added' in output 3690 assert 'removed link input/link_removed' in output 3691 3692 # Symlink replacing or being replaced 3693 assert '] input/dir_replaced_with_link' in output 3694 assert '] input/link_replaced_by_file' in output 3695 3696 # Symlink target removed. Should not affect the symlink at all. 3697 assert 'input/link_target_removed' not in output 3698 3699 # The inode has two links and the file contents changed. Borg 3700 # should notice the changes in both links. However, the symlink 3701 # pointing to the file is not changed. 3702 change = '0 B' if can_compare_ids else '{:<19}'.format('modified') 3703 assert '{} input/empty'.format(change) in output 3704 if are_hardlinks_supported(): 3705 assert '{} input/hardlink_contents_changed'.format(change) in output 3706 if are_symlinks_supported(): 3707 assert 'input/link_target_contents_changed' not in output 3708 3709 # Added a new file and a hard link to it. Both links to the same 3710 # inode should appear as separate files. 3711 assert 'added 2.05 kB input/file_added' in output 3712 if are_hardlinks_supported(): 3713 assert 'added 2.05 kB input/hardlink_added' in output 3714 3715 # check if a diff between non-existent and empty new file is found 3716 assert 'added 0 B input/file_empty_added' in output 3717 3718 # The inode has two links and both of them are deleted. They should 3719 # appear as two deleted files. 3720 assert 'removed 256 B input/file_removed' in output 3721 if are_hardlinks_supported(): 3722 assert 'removed 256 B input/hardlink_removed' in output 3723 3724 # Another link (marked previously as the source in borg) to the 3725 # same inode was removed. This should not change this link at all. 3726 if are_hardlinks_supported(): 3727 assert 'input/hardlink_target_removed' not in output 3728 3729 # Another link (marked previously as the source in borg) to the 3730 # same inode was replaced with a new regular file. This should not 3731 # change this link at all. 3732 if are_hardlinks_supported(): 3733 assert 'input/hardlink_target_replaced' not in output 3734 3735 def do_json_asserts(output, can_compare_ids): 3736 def get_changes(filename, data): 3737 chgsets = [j['changes'] for j in data if j['path'] == filename] 3738 assert len(chgsets) < 2 3739 # return a flattened list of changes for given filename 3740 return [chg for chgset in chgsets for chg in chgset] 3741 3742 # convert output to list of dicts 3743 joutput = [json.loads(line) for line in output.split('\n') if line] 3744 3745 # File contents changed (deleted and replaced with a new file) 3746 expected = {'type': 'modified', 'added': 4096, 'removed': 1024} if can_compare_ids else {'type': 'modified'} 3747 assert expected in get_changes('input/file_replaced', joutput) 3748 3749 # File unchanged 3750 assert not any(get_changes('input/file_unchanged', joutput)) 3751 3752 # Directory replaced with a regular file 3753 if 'BORG_TESTS_IGNORE_MODES' not in os.environ: 3754 assert {'type': 'mode', 'old_mode': 'drwxr-xr-x', 'new_mode': '-rwxr-xr-x'} in \ 3755 get_changes('input/dir_replaced_with_file', joutput) 3756 3757 # Basic directory cases 3758 assert {'type': 'added directory'} in get_changes('input/dir_added', joutput) 3759 assert {'type': 'removed directory'} in get_changes('input/dir_removed', joutput) 3760 3761 if are_symlinks_supported(): 3762 # Basic symlink cases 3763 assert {'type': 'changed link'} in get_changes('input/link_changed', joutput) 3764 assert {'type': 'added link'} in get_changes('input/link_added', joutput) 3765 assert {'type': 'removed link'} in get_changes('input/link_removed', joutput) 3766 3767 # Symlink replacing or being replaced 3768 assert any(chg['type'] == 'mode' and chg['new_mode'].startswith('l') for chg in 3769 get_changes('input/dir_replaced_with_link', joutput)) 3770 assert any(chg['type'] == 'mode' and chg['old_mode'].startswith('l') for chg in 3771 get_changes('input/link_replaced_by_file', joutput)) 3772 3773 # Symlink target removed. Should not affect the symlink at all. 3774 assert not any(get_changes('input/link_target_removed', joutput)) 3775 3776 # The inode has two links and the file contents changed. Borg 3777 # should notice the changes in both links. However, the symlink 3778 # pointing to the file is not changed. 3779 expected = {'type': 'modified', 'added': 13, 'removed': 0} if can_compare_ids else {'type': 'modified'} 3780 assert expected in get_changes('input/empty', joutput) 3781 if are_hardlinks_supported(): 3782 assert expected in get_changes('input/hardlink_contents_changed', joutput) 3783 if are_symlinks_supported(): 3784 assert not any(get_changes('input/link_target_contents_changed', joutput)) 3785 3786 # Added a new file and a hard link to it. Both links to the same 3787 # inode should appear as separate files. 3788 assert {'type': 'added', 'size': 2048} in get_changes('input/file_added', joutput) 3789 if are_hardlinks_supported(): 3790 assert {'type': 'added', 'size': 2048} in get_changes('input/hardlink_added', joutput) 3791 3792 # check if a diff between non-existent and empty new file is found 3793 assert {'type': 'added', 'size': 0} in get_changes('input/file_empty_added', joutput) 3794 3795 # The inode has two links and both of them are deleted. They should 3796 # appear as two deleted files. 3797 assert {'type': 'removed', 'size': 256} in get_changes('input/file_removed', joutput) 3798 if are_hardlinks_supported(): 3799 assert {'type': 'removed', 'size': 256} in get_changes('input/hardlink_removed', joutput) 3800 3801 # Another link (marked previously as the source in borg) to the 3802 # same inode was removed. This should not change this link at all. 3803 if are_hardlinks_supported(): 3804 assert not any(get_changes('input/hardlink_target_removed', joutput)) 3805 3806 # Another link (marked previously as the source in borg) to the 3807 # same inode was replaced with a new regular file. This should not 3808 # change this link at all. 3809 if are_hardlinks_supported(): 3810 assert not any(get_changes('input/hardlink_target_replaced', joutput)) 3811 3812 do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), True) 3813 # We expect exit_code=1 due to the chunker params warning 3814 do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1b', exit_code=1), False) 3815 do_json_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a', '--json-lines'), True) 3816 3817 def test_sort_option(self): 3818 self.cmd('init', '--encryption=repokey', self.repository_location) 3819 3820 self.create_regular_file('a_file_removed', size=8) 3821 self.create_regular_file('f_file_removed', size=16) 3822 self.create_regular_file('c_file_changed', size=32) 3823 self.create_regular_file('e_file_changed', size=64) 3824 self.cmd('create', self.repository_location + '::test0', 'input') 3825 3826 os.unlink('input/a_file_removed') 3827 os.unlink('input/f_file_removed') 3828 os.unlink('input/c_file_changed') 3829 os.unlink('input/e_file_changed') 3830 self.create_regular_file('c_file_changed', size=512) 3831 self.create_regular_file('e_file_changed', size=1024) 3832 self.create_regular_file('b_file_added', size=128) 3833 self.create_regular_file('d_file_added', size=256) 3834 self.cmd('create', self.repository_location + '::test1', 'input') 3835 3836 output = self.cmd('diff', '--sort', self.repository_location + '::test0', 'test1') 3837 expected = [ 3838 'a_file_removed', 3839 'b_file_added', 3840 'c_file_changed', 3841 'd_file_added', 3842 'e_file_changed', 3843 'f_file_removed', 3844 ] 3845 3846 assert all(x in line for x, line in zip(expected, output.splitlines())) 3847 3848 3849def test_get_args(): 3850 archiver = Archiver() 3851 # everything normal: 3852 # first param is argv as produced by ssh forced command, 3853 # second param is like from SSH_ORIGINAL_COMMAND env variable 3854 args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], 3855 'borg serve --info --umask=0027') 3856 assert args.func == archiver.do_serve 3857 assert args.restrict_to_paths == ['/p1', '/p2'] 3858 assert args.umask == 0o027 3859 assert args.log_level == 'info' 3860 # similar, but with --restrict-to-repository 3861 args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], 3862 'borg serve --info --umask=0027') 3863 assert args.restrict_to_repositories == ['/r1', '/r2'] 3864 # trying to cheat - break out of path restriction 3865 args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], 3866 'borg serve --restrict-to-path=/') 3867 assert args.restrict_to_paths == ['/p1', '/p2'] 3868 # trying to cheat - break out of repository restriction 3869 args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], 3870 'borg serve --restrict-to-repository=/') 3871 assert args.restrict_to_repositories == ['/r1', '/r2'] 3872 # trying to cheat - break below repository restriction 3873 args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], 3874 'borg serve --restrict-to-repository=/r1/below') 3875 assert args.restrict_to_repositories == ['/r1', '/r2'] 3876 # trying to cheat - try to execute different subcommand 3877 args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], 3878 'borg init --encryption=repokey /') 3879 assert args.func == archiver.do_serve 3880 3881 # Check that environment variables in the forced command don't cause issues. If the command 3882 # were not forced, environment variables would be interpreted by the shell, but this does not 3883 # happen for forced commands - we get the verbatim command line and need to deal with env vars. 3884 args = archiver.get_args(['borg', 'serve', ], 3885 'BORG_HOSTNAME_IS_UNIQUE=yes borg serve --info') 3886 assert args.func == archiver.do_serve 3887 3888 3889def test_compare_chunk_contents(): 3890 def ccc(a, b): 3891 chunks_a = [data for data in a] 3892 chunks_b = [data for data in b] 3893 compare1 = Archiver.compare_chunk_contents(iter(chunks_a), iter(chunks_b)) 3894 compare2 = Archiver.compare_chunk_contents(iter(chunks_b), iter(chunks_a)) 3895 assert compare1 == compare2 3896 return compare1 3897 assert ccc([ 3898 b'1234', b'567A', b'bC' 3899 ], [ 3900 b'1', b'23', b'4567A', b'b', b'C' 3901 ]) 3902 # one iterator exhausted before the other 3903 assert not ccc([ 3904 b'12345', 3905 ], [ 3906 b'1234', b'56' 3907 ]) 3908 # content mismatch 3909 assert not ccc([ 3910 b'1234', b'65' 3911 ], [ 3912 b'1234', b'56' 3913 ]) 3914 # first is the prefix of second 3915 assert not ccc([ 3916 b'1234', b'56' 3917 ], [ 3918 b'1234', b'565' 3919 ]) 3920 3921 3922class TestBuildFilter: 3923 @staticmethod 3924 def peek_and_store_hardlink_masters(item, matched): 3925 pass 3926 3927 def test_basic(self): 3928 matcher = PatternMatcher() 3929 matcher.add([parse_pattern('included')], IECommand.Include) 3930 filter = Archiver.build_filter(matcher, self.peek_and_store_hardlink_masters, 0) 3931 assert filter(Item(path='included')) 3932 assert filter(Item(path='included/file')) 3933 assert not filter(Item(path='something else')) 3934 3935 def test_empty(self): 3936 matcher = PatternMatcher(fallback=True) 3937 filter = Archiver.build_filter(matcher, self.peek_and_store_hardlink_masters, 0) 3938 assert filter(Item(path='anything')) 3939 3940 def test_strip_components(self): 3941 matcher = PatternMatcher(fallback=True) 3942 filter = Archiver.build_filter(matcher, self.peek_and_store_hardlink_masters, strip_components=1) 3943 assert not filter(Item(path='shallow')) 3944 assert not filter(Item(path='shallow/')) # can this even happen? paths are normalized... 3945 assert filter(Item(path='deep enough/file')) 3946 assert filter(Item(path='something/dir/file')) 3947 3948 3949class TestCommonOptions: 3950 @staticmethod 3951 def define_common_options(add_common_option): 3952 add_common_option('-h', '--help', action='help', help='show this help message and exit') 3953 add_common_option('--critical', dest='log_level', help='foo', 3954 action='store_const', const='critical', default='warning') 3955 add_common_option('--error', dest='log_level', help='foo', 3956 action='store_const', const='error', default='warning') 3957 add_common_option('--append', dest='append', help='foo', 3958 action='append', metavar='TOPIC', default=[]) 3959 add_common_option('-p', '--progress', dest='progress', action='store_true', help='foo') 3960 add_common_option('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, 3961 help='(default: %(default)d).') 3962 3963 @pytest.fixture 3964 def basic_parser(self): 3965 parser = argparse.ArgumentParser(prog='test', description='test parser', add_help=False) 3966 parser.common_options = Archiver.CommonOptions(self.define_common_options, 3967 suffix_precedence=('_level0', '_level1')) 3968 return parser 3969 3970 @pytest.fixture 3971 def subparsers(self, basic_parser): 3972 return basic_parser.add_subparsers(title='required arguments', metavar='<command>') 3973 3974 @pytest.fixture 3975 def parser(self, basic_parser): 3976 basic_parser.common_options.add_common_group(basic_parser, '_level0', provide_defaults=True) 3977 return basic_parser 3978 3979 @pytest.fixture 3980 def common_parser(self, parser): 3981 common_parser = argparse.ArgumentParser(add_help=False, prog='test') 3982 parser.common_options.add_common_group(common_parser, '_level1') 3983 return common_parser 3984 3985 @pytest.fixture 3986 def parse_vars_from_line(self, parser, subparsers, common_parser): 3987 subparser = subparsers.add_parser('subcommand', parents=[common_parser], add_help=False, 3988 description='foo', epilog='bar', help='baz', 3989 formatter_class=argparse.RawDescriptionHelpFormatter) 3990 subparser.set_defaults(func=1234) 3991 subparser.add_argument('--append-only', dest='append_only', action='store_true') 3992 3993 def parse_vars_from_line(*line): 3994 print(line) 3995 args = parser.parse_args(line) 3996 parser.common_options.resolve(args) 3997 return vars(args) 3998 3999 return parse_vars_from_line 4000 4001 def test_simple(self, parse_vars_from_line): 4002 assert parse_vars_from_line('--error') == { 4003 'append': [], 4004 'lock_wait': 1, 4005 'log_level': 'error', 4006 'progress': False 4007 } 4008 4009 assert parse_vars_from_line('--error', 'subcommand', '--critical') == { 4010 'append': [], 4011 'lock_wait': 1, 4012 'log_level': 'critical', 4013 'progress': False, 4014 'append_only': False, 4015 'func': 1234, 4016 } 4017 4018 with pytest.raises(SystemExit): 4019 parse_vars_from_line('--append-only', 'subcommand') 4020 4021 assert parse_vars_from_line('--append=foo', '--append', 'bar', 'subcommand', '--append', 'baz') == { 4022 'append': ['foo', 'bar', 'baz'], 4023 'lock_wait': 1, 4024 'log_level': 'warning', 4025 'progress': False, 4026 'append_only': False, 4027 'func': 1234, 4028 } 4029 4030 @pytest.mark.parametrize('position', ('before', 'after', 'both')) 4031 @pytest.mark.parametrize('flag,args_key,args_value', ( 4032 ('-p', 'progress', True), 4033 ('--lock-wait=3', 'lock_wait', 3), 4034 )) 4035 def test_flag_position_independence(self, parse_vars_from_line, position, flag, args_key, args_value): 4036 line = [] 4037 if position in ('before', 'both'): 4038 line.append(flag) 4039 line.append('subcommand') 4040 if position in ('after', 'both'): 4041 line.append(flag) 4042 4043 result = { 4044 'append': [], 4045 'lock_wait': 1, 4046 'log_level': 'warning', 4047 'progress': False, 4048 'append_only': False, 4049 'func': 1234, 4050 } 4051 result[args_key] = args_value 4052 4053 assert parse_vars_from_line(*line) == result 4054 4055 4056def test_parse_storage_quota(): 4057 assert parse_storage_quota('50M') == 50 * 1000**2 4058 with pytest.raises(argparse.ArgumentTypeError): 4059 parse_storage_quota('5M') 4060 4061 4062def get_all_parsers(): 4063 """ 4064 Return dict mapping command to parser. 4065 """ 4066 parser = Archiver(prog='borg').build_parser() 4067 borgfs_parser = Archiver(prog='borgfs').build_parser() 4068 parsers = {} 4069 4070 def discover_level(prefix, parser, Archiver, extra_choices=None): 4071 choices = {} 4072 for action in parser._actions: 4073 if action.choices is not None and 'SubParsersAction' in str(action.__class__): 4074 for cmd, parser in action.choices.items(): 4075 choices[prefix + cmd] = parser 4076 if extra_choices is not None: 4077 choices.update(extra_choices) 4078 if prefix and not choices: 4079 return 4080 4081 for command, parser in sorted(choices.items()): 4082 discover_level(command + " ", parser, Archiver) 4083 parsers[command] = parser 4084 4085 discover_level("", parser, Archiver, {'borgfs': borgfs_parser}) 4086 return parsers 4087 4088 4089@pytest.mark.parametrize('command, parser', list(get_all_parsers().items())) 4090def test_help_formatting(command, parser): 4091 if isinstance(parser.epilog, RstToTextLazy): 4092 assert parser.epilog.rst 4093 4094 4095@pytest.mark.parametrize('topic, helptext', list(Archiver.helptext.items())) 4096def test_help_formatting_helptexts(topic, helptext): 4097 assert str(rst_to_terminal(helptext)) 4098