1"""commontest - Some functions and constants common to several test cases. 2Can be called also directly to setup the test environment""" 3import os 4import sys 5import code 6import shutil 7import subprocess 8# Avoid circularities 9from rdiff_backup.log import Log 10from rdiff_backup import Globals, Hardlink, SetConnections, Main, \ 11 selection, rpath, eas_acls, rorpiter, Security, hash 12 13RBBin = os.fsencode(shutil.which("rdiff-backup")) 14 15# Working directory is defined by Tox, venv or the current build directory 16abs_work_dir = os.getenvb( 17 b'TOX_ENV_DIR', 18 os.getenvb(b'VIRTUAL_ENV', os.path.join(os.getcwdb(), b'build'))) 19abs_test_dir = os.path.join(abs_work_dir, b'testfiles') 20abs_output_dir = os.path.join(abs_test_dir, b'output') 21abs_restore_dir = os.path.join(abs_test_dir, b'restore') 22 23# the directory with the testfiles used as input is in the parent directory of the Git clone 24old_test_dir = os.path.join(os.path.dirname(os.getcwdb()), 25 b'rdiff-backup_testfiles') 26old_inc1_dir = os.path.join(old_test_dir, b'increment1') 27old_inc2_dir = os.path.join(old_test_dir, b'increment2') 28old_inc3_dir = os.path.join(old_test_dir, b'increment3') 29old_inc4_dir = os.path.join(old_test_dir, b'increment4') 30 31# the directory in which all testing scripts are placed is the one 32abs_testing_dir = os.path.dirname(os.path.abspath(os.fsencode(sys.argv[0]))) 33 34__no_execute__ = 1 # Keeps the actual rdiff-backup program from running 35 36 37def Myrm(dirstring): 38 """Run myrm on given directory string""" 39 root_rp = rpath.RPath(Globals.local_connection, dirstring) 40 for rp in selection.Select(root_rp).set_iter(): 41 if rp.isdir(): 42 rp.chmod(0o700) # otherwise may not be able to remove 43 assert not os.system(b"rm -rf %s" % (root_rp.path, )) 44 45 46def re_init_rpath_dir(rp, uid=-1, gid=-1): 47 """Delete directory if present, then recreate""" 48 if rp.lstat(): 49 Myrm(rp.path) 50 rp.setdata() 51 rp.mkdir() 52 rp.chown(uid, gid) 53 54 55def re_init_subdir(maindir, subdir): 56 """Remove a sub-directory and return its name joined 57 to the main directory as an empty directory""" 58 dir = os.path.join(maindir, subdir) 59 Myrm(dir) 60 os.makedirs(dir) 61 return dir 62 63 64# two temporary directories to simulate remote actions 65abs_remote1_dir = re_init_subdir(abs_test_dir, b'remote1') 66abs_remote2_dir = re_init_subdir(abs_test_dir, b'remote2') 67 68 69def MakeOutputDir(): 70 """Initialize the output directory""" 71 Myrm(abs_output_dir) 72 rp = rpath.RPath(Globals.local_connection, abs_output_dir) 73 rp.mkdir() 74 return rp 75 76 77def rdiff_backup(source_local, 78 dest_local, 79 src_dir, 80 dest_dir, 81 current_time=None, 82 extra_options=b"", 83 input=None, 84 check_return_val=1, 85 expected_ret_val=0): 86 """Run rdiff-backup with the given options 87 88 source_local and dest_local are boolean values. If either is 89 false, then rdiff-backup will be run pretending that src_dir and 90 dest_dir, respectively, are remote. The server process will be 91 run in directories remote1 and remote2 respectively. 92 93 src_dir and dest_dir are the source and destination 94 (mirror) directories, relative to the testing directory. 95 96 If current time is true, add the --current-time option with the 97 given number of seconds. 98 99 extra_options are just added to the command line. 100 101 """ 102 if not source_local: 103 src_dir = (b"'cd %s; %s --server'::%s" % 104 (abs_remote1_dir, RBBin, src_dir)) 105 if dest_dir and not dest_local: 106 dest_dir = (b"'cd %s; %s --server'::%s" % 107 (abs_remote2_dir, RBBin, dest_dir)) 108 109 cmdargs = [RBBin, extra_options] 110 if not (source_local and dest_local): 111 cmdargs.append(b"--remote-schema %s") 112 113 if current_time: 114 cmdargs.append(b"--current-time %i" % current_time) 115 cmdargs.append(src_dir) 116 if dest_dir: 117 cmdargs.append(dest_dir) 118 cmdline = b" ".join(cmdargs) 119 print("Executing: ", cmdline) 120 ret_val = subprocess.run(cmdline, 121 shell=True, 122 input=input, 123 universal_newlines=False).returncode 124 if check_return_val: 125 # the construct is needed because os.system seemingly doesn't 126 # respect expected return values (FIXME) 127 assert ((expected_ret_val == 0 and ret_val == 0) or (expected_ret_val > 0 and ret_val > 0)), \ 128 "Return code %d of command `%a` isn't expected %d." % \ 129 (ret_val, cmdline, expected_ret_val) 130 return ret_val 131 132 133def InternalBackup(source_local, 134 dest_local, 135 src_dir, 136 dest_dir, 137 current_time=None, 138 eas=None, 139 acls=None): 140 """Backup src to dest internally 141 142 This is like rdiff_backup but instead of running a separate 143 rdiff-backup script, use the separate *.py files. This way the 144 script doesn't have to be rebuild constantly, and stacktraces have 145 correct line/file references. 146 147 """ 148 Globals.current_time = current_time 149 Globals.security_level = "override" 150 remote_schema = b'%s' 151 152 if not source_local: 153 src_dir = b"cd %s; %s/server.py::%s" % (abs_remote1_dir, abs_testing_dir, src_dir) 154 if not dest_local: 155 dest_dir = b"cd %s; %s/server.py::%s" % (abs_remote2_dir, abs_testing_dir, dest_dir) 156 157 cmdpairs = SetConnections.get_cmd_pairs([src_dir, dest_dir], remote_schema) 158 Security.initialize("backup", cmdpairs) 159 rpin, rpout = list(map(SetConnections.cmdpair2rp, cmdpairs)) 160 for attr in ('eas_active', 'eas_write', 'eas_conn'): 161 SetConnections.UpdateGlobal(attr, eas) 162 for attr in ('acls_active', 'acls_write', 'acls_conn'): 163 SetConnections.UpdateGlobal(attr, acls) 164 Main.misc_setup([rpin, rpout]) 165 Main.Backup(rpin, rpout) 166 Main.cleanup() 167 168 169def InternalMirror(source_local, dest_local, src_dir, dest_dir): 170 """Mirror src to dest internally 171 172 like InternalBackup, but only mirror. Do this through 173 InternalBackup, but then delete rdiff-backup-data directory. 174 175 """ 176 # Save attributes of root to restore later 177 src_root = rpath.RPath(Globals.local_connection, src_dir) 178 dest_root = rpath.RPath(Globals.local_connection, dest_dir) 179 dest_rbdir = dest_root.append("rdiff-backup-data") 180 181 InternalBackup(source_local, dest_local, src_dir, dest_dir) 182 dest_root.setdata() 183 Myrm(dest_rbdir.path) 184 # Restore old attributes 185 rpath.copy_attribs(src_root, dest_root) 186 187 188def InternalRestore(mirror_local, 189 dest_local, 190 mirror_dir, 191 dest_dir, 192 time, 193 eas=None, 194 acls=None): 195 """Restore mirror_dir to dest_dir at given time 196 197 This will automatically find the increments.XXX.dir representing 198 the time specified. The mirror_dir and dest_dir are relative to 199 the testing directory and will be modified for remote trials. 200 201 """ 202 Main.force = 1 203 Main.restore_root_set = 0 204 remote_schema = b'%s' 205 Globals.security_level = "override" 206 if not mirror_local: 207 mirror_dir = b"cd %s; %s/server.py::%s" % (abs_remote1_dir, abs_testing_dir, mirror_dir) 208 if not dest_local: 209 dest_dir = b"cd %s; %s/server.py::%s" % (abs_remote2_dir, abs_testing_dir, dest_dir) 210 211 cmdpairs = SetConnections.get_cmd_pairs([mirror_dir, dest_dir], 212 remote_schema) 213 Security.initialize("restore", cmdpairs) 214 mirror_rp, dest_rp = list(map(SetConnections.cmdpair2rp, cmdpairs)) 215 for attr in ('eas_active', 'eas_write', 'eas_conn'): 216 SetConnections.UpdateGlobal(attr, eas) 217 for attr in ('acls_active', 'acls_write', 'acls_conn'): 218 SetConnections.UpdateGlobal(attr, acls) 219 Main.misc_setup([mirror_rp, dest_rp]) 220 inc = get_increment_rp(mirror_rp, time) 221 if inc: 222 Main.Restore(get_increment_rp(mirror_rp, time), dest_rp) 223 else: # use alternate syntax 224 Main.restore_timestr = str(time) 225 Main.Restore(mirror_rp, dest_rp, restore_as_of=1) 226 Main.cleanup() 227 228 229def get_increment_rp(mirror_rp, time): 230 """Return increment rp matching time in seconds""" 231 data_rp = mirror_rp.append("rdiff-backup-data") 232 if not data_rp.isdir(): 233 return None 234 for filename in data_rp.listdir(): 235 rp = data_rp.append(filename) 236 if rp.isincfile() and rp.getincbase_bname() == b"increments": 237 if rp.getinctime() == time: 238 return rp 239 return None # Couldn't find appropriate increment 240 241 242def _reset_connections(src_rp, dest_rp): 243 """Reset some global connection information""" 244 Globals.security_level = "override" 245 Globals.isbackup_reader = Globals.isbackup_writer = None 246 SetConnections.UpdateGlobal('rbdir', None) 247 Main.misc_setup([src_rp, dest_rp]) 248 249 250def CompareRecursive(src_rp, 251 dest_rp, 252 compare_hardlinks=1, 253 equality_func=None, 254 exclude_rbdir=1, 255 ignore_tmp_files=None, 256 compare_ownership=0, 257 compare_eas=0, 258 compare_acls=0): 259 """Compare src_rp and dest_rp, which can be directories 260 261 This only compares file attributes, not the actual data. This 262 will overwrite the hardlink dictionaries if compare_hardlinks is 263 specified. 264 265 """ 266 267 def get_selection_functions(): 268 """Return generators of files in source, dest""" 269 src_rp.setdata() 270 dest_rp.setdata() 271 src_select = selection.Select(src_rp) 272 dest_select = selection.Select(dest_rp) 273 274 if ignore_tmp_files: 275 # Ignoring temp files can be useful when we want to check the 276 # correctness of a backup which aborted in the middle. In 277 # these cases it is OK to have tmp files lying around. 278 src_select.add_selection_func( 279 src_select.regexp_get_sf(".*rdiff-backup.tmp.[^/]+$", 0)) 280 dest_select.add_selection_func( 281 dest_select.regexp_get_sf(".*rdiff-backup.tmp.[^/]+$", 0)) 282 283 if exclude_rbdir: # Exclude rdiff-backup-data directory 284 src_select.parse_rbdir_exclude() 285 dest_select.parse_rbdir_exclude() 286 287 return src_select.set_iter(), dest_select.set_iter() 288 289 def hardlink_rorp_eq(src_rorp, dest_rorp): 290 Hardlink.add_rorp(dest_rorp) 291 Hardlink.add_rorp(src_rorp, dest_rorp) 292 rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp) 293 if not src_rorp.isreg() or not dest_rorp.isreg() or src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1: 294 if not rorp_eq: 295 Log("Hardlink compare error with when no links exist exist", 3) 296 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 297 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 298 return 0 299 elif src_rorp.getnumlinks() > 1 and not Hardlink.islinked(src_rorp): 300 if rorp_eq: 301 Log("Hardlink compare error with first linked src_rorp and no dest_rorp sha1", 3) 302 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 303 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 304 return 0 305 hash.compute_sha1(dest_rorp) 306 rorp_eq = Hardlink.rorp_eq(src_rorp, dest_rorp) 307 if src_rorp.getnumlinks() != dest_rorp.getnumlinks(): 308 if rorp_eq: 309 Log("Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with differing link counts", 3) 310 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 311 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 312 return 0 313 elif not rorp_eq: 314 Log("Hardlink compare error with first linked src_rorp, with dest_rorp sha1, and with equal link counts", 3) 315 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 316 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 317 return 0 318 elif src_rorp.getnumlinks() != dest_rorp.getnumlinks(): 319 if rorp_eq: 320 Log("Hardlink compare error with non-first linked src_rorp and with differing link counts", 3) 321 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 322 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 323 return 0 324 elif not rorp_eq: 325 Log("Hardlink compare error with non-first linked src_rorp and with equal link counts", 3) 326 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 327 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 328 return 0 329 Hardlink.del_rorp(src_rorp) 330 Hardlink.del_rorp(dest_rorp) 331 return 1 332 333 def equality_func(src_rorp, dest_rorp): 334 """Combined eq func returns true if two files compare same""" 335 if not src_rorp: 336 Log("Source rorp missing: %s" % str(dest_rorp), 3) 337 return 0 338 if not dest_rorp: 339 Log("Dest rorp missing: %s" % str(src_rorp), 3) 340 return 0 341 if not src_rorp.equal_verbose(dest_rorp, 342 compare_ownership=compare_ownership): 343 return 0 344 if compare_hardlinks and not hardlink_rorp_eq(src_rorp, dest_rorp): 345 return 0 346 if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp): 347 Log( 348 "Different EAs in files %s and %s" % 349 (src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3) 350 return 0 351 if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp): 352 Log( 353 "Different ACLs in files %s and %s" % 354 (src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3) 355 return 0 356 return 1 357 358 Log( 359 "Comparing %s and %s, hardlinks %s, eas %s, acls %s" % 360 (src_rp.get_safepath(), dest_rp.get_safepath(), compare_hardlinks, 361 compare_eas, compare_acls), 3) 362 if compare_hardlinks: 363 reset_hardlink_dicts() 364 src_iter, dest_iter = get_selection_functions() 365 for src_rorp, dest_rorp in rorpiter.Collate2Iters(src_iter, dest_iter): 366 if not equality_func(src_rorp, dest_rorp): 367 return 0 368 return 1 369 370 def rbdir_equal(src_rorp, dest_rorp): 371 """Like hardlink_equal, but make allowances for data directories""" 372 if not src_rorp.index and not dest_rorp.index: 373 return 1 374 if (src_rorp.index and src_rorp.index[0] == 'rdiff-backup-data' and src_rorp.index == dest_rorp.index): 375 # Don't compare dirs - they don't carry significant info 376 if dest_rorp.isdir() and src_rorp.isdir(): 377 return 1 378 if dest_rorp.isreg() and src_rorp.isreg(): 379 # Don't compare gzipped files because it is apparently 380 # non-deterministic. 381 if dest_rorp.index[-1].endswith('gz'): 382 return 1 383 # Don't compare .missing increments because they don't matter 384 if dest_rorp.index[-1].endswith('.missing'): 385 return 1 386 if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp): 387 Log("Different EAs in files %s and %s" % 388 (src_rorp.get_indexpath(), dest_rorp.get_indexpath())) 389 return None 390 if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp): 391 Log( 392 "Different ACLs in files %s and %s" % 393 (src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3) 394 return None 395 if compare_hardlinks: 396 if Hardlink.rorp_eq(src_rorp, dest_rorp): 397 return 1 398 elif src_rorp.equal_verbose(dest_rorp, 399 compare_ownership=compare_ownership): 400 return 1 401 Log("%s: %s" % (src_rorp.index, Hardlink.get_inode_key(src_rorp)), 3) 402 Log("%s: %s" % (dest_rorp.index, Hardlink.get_inode_key(dest_rorp)), 3) 403 return None 404 405 406def reset_hardlink_dicts(): 407 """Clear the hardlink dictionaries""" 408 Hardlink._inode_index = {} 409 410 411def BackupRestoreSeries(source_local, 412 dest_local, 413 list_of_dirnames, 414 compare_hardlinks=1, 415 dest_dirname=abs_output_dir, 416 restore_dirname=abs_restore_dir, 417 compare_backups=1, 418 compare_eas=0, 419 compare_acls=0, 420 compare_ownership=0): 421 """Test backing up/restoring of a series of directories 422 423 The dirnames correspond to a single directory at different times. 424 After each backup, the dest dir will be compared. After the whole 425 set, each of the earlier directories will be recovered to the 426 restore_dirname and compared. 427 428 """ 429 Globals.set('preserve_hardlinks', compare_hardlinks) 430 time = 10000 431 dest_rp = rpath.RPath(Globals.local_connection, dest_dirname) 432 restore_rp = rpath.RPath(Globals.local_connection, restore_dirname) 433 434 Myrm(dest_dirname) 435 for dirname in list_of_dirnames: 436 src_rp = rpath.RPath(Globals.local_connection, dirname) 437 reset_hardlink_dicts() 438 _reset_connections(src_rp, dest_rp) 439 440 InternalBackup(source_local, 441 dest_local, 442 dirname, 443 dest_dirname, 444 time, 445 eas=compare_eas, 446 acls=compare_acls) 447 time += 10000 448 _reset_connections(src_rp, dest_rp) 449 if compare_backups: 450 assert CompareRecursive(src_rp, 451 dest_rp, 452 compare_hardlinks, 453 compare_eas=compare_eas, 454 compare_acls=compare_acls, 455 compare_ownership=compare_ownership) 456 457 time = 10000 458 for dirname in list_of_dirnames[:-1]: 459 reset_hardlink_dicts() 460 Myrm(restore_dirname) 461 InternalRestore(dest_local, 462 source_local, 463 dest_dirname, 464 restore_dirname, 465 time, 466 eas=compare_eas, 467 acls=compare_acls) 468 src_rp = rpath.RPath(Globals.local_connection, dirname) 469 assert CompareRecursive(src_rp, 470 restore_rp, 471 compare_eas=compare_eas, 472 compare_acls=compare_acls, 473 compare_ownership=compare_ownership) 474 475 # Restore should default back to newest time older than it 476 # with a backup then. 477 if time == 20000: 478 time = 21000 479 480 time += 10000 481 482 483def MirrorTest(source_local, 484 dest_local, 485 list_of_dirnames, 486 compare_hardlinks=1, 487 dest_dirname=abs_output_dir): 488 """Mirror each of list_of_dirnames, and compare after each""" 489 Globals.set('preserve_hardlinks', compare_hardlinks) 490 dest_rp = rpath.RPath(Globals.local_connection, dest_dirname) 491 old_force_val = Main.force 492 Main.force = 1 493 494 Myrm(dest_dirname) 495 for dirname in list_of_dirnames: 496 src_rp = rpath.RPath(Globals.local_connection, dirname) 497 reset_hardlink_dicts() 498 _reset_connections(src_rp, dest_rp) 499 500 InternalMirror(source_local, dest_local, dirname, dest_dirname) 501 _reset_connections(src_rp, dest_rp) 502 assert CompareRecursive(src_rp, dest_rp, compare_hardlinks) 503 Main.force = old_force_val 504 505 506def raise_interpreter(use_locals=None): 507 """Start Python interpreter, with local variables if locals is true""" 508 if use_locals: 509 local_dict = locals() 510 else: 511 local_dict = globals() 512 code.InteractiveConsole(local_dict).interact() 513 514 515def getrefs(i, depth): 516 """Get the i'th object in memory, return objects that reference it""" 517 import sys 518 import gc 519 import types 520 o = sys.getobjects(i)[-1] 521 for d in range(depth): 522 for ref in gc.get_referrers(o): 523 if type(ref) in (list, dict, types.InstanceType): 524 if type(ref) is dict and 'copyright' in ref: 525 continue 526 o = ref 527 break 528 else: 529 print("Max depth ", d) 530 return o 531 return o 532 533 534def iter_equal(iter1, iter2, verbose=None, operator=lambda x, y: x == y): 535 """True if iterator 1 has same elements as iterator 2 536 537 Use equality operator, or == if it is unspecified. 538 539 """ 540 for i1 in iter1: 541 try: 542 i2 = next(iter2) 543 except StopIteration: 544 if verbose: 545 print("End when i1 = %s" % (i1, )) 546 return False 547 if not operator(i1, i2): 548 if verbose: 549 print("%s not equal to %s" % (i1, i2)) 550 return False 551 try: 552 i2 = next(iter2) 553 except StopIteration: 554 return True 555 if verbose: 556 print("End when i2 = %s" % (i2, )) 557 return False 558 559 560def iter_map(function, iterator): 561 """Like map in a lazy functional programming language""" 562 for i in iterator: 563 yield function(i) 564 565 566if __name__ == '__main__': 567 os.makedirs(abs_test_dir, exist_ok=True) 568