1# Copyright 2002, 2003, 2004 Ben Escoto 2# 3# This file is part of rdiff-backup. 4# 5# rdiff-backup is free software; you can redistribute it and/or modify 6# under the terms of the GNU General Public License as published by the 7# Free Software Foundation; either version 2 of the License, or (at your 8# option) any later version. 9# 10# rdiff-backup is distributed in the hope that it will be useful, but 11# WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13# General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with rdiff-backup; if not, write to the Free Software 17# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 18# 02110-1301, USA 19"""Wrapper class around a real path like "/usr/bin/env" 20 21The RPath (short for Remote Path) and associated classes make some 22function calls more convenient and also make working with files on 23remote systems transparent. 24 25For instance, suppose 26 27rp = RPath(connection_object, "/usr/bin/env") 28 29Then rp.getperms() returns the permissions of that file, and 30rp.delete() deletes that file. Both of these will work the same even 31if "usr/bin/env" is on a different computer. So many rdiff-backup 32functions use rpaths so they don't have to know whether the files they 33are dealing with are local or remote. 34 35""" 36 37import os 38import stat 39import re 40import gzip 41import time 42import errno 43from . import Globals, Time, log, user_group, C 44 45try: 46 import win32api 47 import win32con 48 import pywintypes 49except ImportError: 50 pass 51 52 53class SkipFileException(Exception): 54 """Signal that the current file should be skipped but then continue 55 56 This exception will often be raised when there is problem reading 57 an individual file, but it makes sense for the rest of the backup 58 to keep going. 59 60 """ 61 pass 62 63 64class RPathException(Exception): 65 pass 66 67 68def copyfileobj(inputfp, outputfp): 69 """Copies file inputfp to outputfp in blocksize intervals""" 70 blocksize = Globals.blocksize 71 72 sparse = False 73 """Negative seeks are not supported by GzipFile""" 74 compressed = False 75 if isinstance(outputfp, gzip.GzipFile): 76 compressed = True 77 78 while 1: 79 inbuf = inputfp.read(blocksize) 80 if not inbuf: 81 break 82 83 buflen = len(inbuf) 84 if not compressed and inbuf == b"\x00" * buflen: 85 outputfp.seek(buflen, os.SEEK_CUR) 86 # flag sparse=True, that we seek()ed, but have not written yet 87 # The filesize is wrong until we write 88 sparse = True 89 else: 90 outputfp.write(inbuf) 91 # We wrote, so clear sparse. 92 sparse = False 93 94 if sparse: 95 outputfp.seek(-1, os.SEEK_CUR) 96 outputfp.write(b"\x00") 97 98 99def cmpfileobj(fp1, fp2): 100 """True if file objects fp1 and fp2 contain same data""" 101 blocksize = Globals.blocksize 102 while 1: 103 buf1 = fp1.read(blocksize) 104 buf2 = fp2.read(blocksize) 105 if buf1 != buf2: 106 return None 107 elif not buf1: 108 return 1 109 110 111def check_for_files(*rps): 112 """Make sure that all the rps exist, raise error if not""" 113 for rp in rps: 114 if not rp.lstat(): 115 raise RPathException( 116 "File %s does not exist" % rp.get_safeindexpath()) 117 118 119def move(rpin, rpout): 120 """Move rpin to rpout, renaming if possible""" 121 try: 122 rename(rpin, rpout) 123 except os.error: 124 copy(rpin, rpout) 125 rpin.delete() 126 127 128def copy(rpin, rpout, compress=0): 129 """Copy RPath rpin to rpout. Works for symlinks, dirs, etc. 130 131 Returns close value of input for regular file, which can be used 132 to pass hashes on. 133 134 """ 135 log.Log("Regular copying %s to %s" % (rpin.index, rpout.get_safepath()), 6) 136 if not rpin.lstat(): 137 if rpout.lstat(): 138 rpout.delete() 139 return 140 141 if rpout.lstat(): 142 if rpin.isreg() or not cmp(rpin, rpout): 143 rpout.delete() # easier to write than compare 144 else: 145 return 146 147 if rpin.isreg(): 148 return copy_reg_file(rpin, rpout, compress) 149 elif rpin.isdir(): 150 rpout.mkdir() 151 elif rpin.issym(): 152 # some systems support permissions for symlinks, but 153 # only by setting at creation via the umask 154 if Globals.symlink_perms: 155 orig_umask = os.umask(0o777 & ~rpin.getperms()) 156 rpout.symlink(rpin.readlink()) 157 if Globals.symlink_perms: 158 os.umask(orig_umask) # restore previous umask 159 elif rpin.isdev(): 160 dev_type, major, minor = rpin.getdevnums() 161 rpout.makedev(dev_type, major, minor) 162 elif rpin.isfifo(): 163 rpout.mkfifo() 164 elif rpin.issock(): 165 rpout.mksock() 166 else: 167 raise RPathException("File '%s' has unknown type." % rpin.get_safepath()) 168 169 170def copy_reg_file(rpin, rpout, compress=0): 171 """Copy regular file rpin to rpout, possibly avoiding connection""" 172 try: 173 if (rpout.conn is rpin.conn 174 and rpout.conn is not Globals.local_connection): 175 v = rpout.conn.rpath.copy_reg_file(rpin.path, rpout.path, compress) 176 rpout.setdata() 177 return v 178 except AttributeError: 179 pass 180 try: 181 return rpout.write_from_fileobj(rpin.open("rb"), compress=compress) 182 except IOError as e: 183 if (e.errno == errno.ERANGE): 184 log.Log.FatalError( 185 "'IOError - Result too large' while reading %s. " 186 "If you are using a Mac, this is probably " 187 "the result of HFS+ filesystem corruption. " 188 "Please exclude this file from your backup " 189 "before proceeding." % rpin.get_safepath()) 190 else: 191 raise 192 193 194def cmp(rpin, rpout): 195 """True if rpin has the same data as rpout 196 197 cmp does not compare file ownership, permissions, or times, or 198 examine the contents of a directory. 199 200 """ 201 check_for_files(rpin, rpout) 202 if rpin.isreg(): 203 if not rpout.isreg(): 204 return None 205 fp1, fp2 = rpin.open("rb"), rpout.open("rb") 206 result = cmpfileobj(fp1, fp2) 207 if fp1.close() or fp2.close(): 208 raise RPathException("Error closing file") 209 return result 210 elif rpin.isdir(): 211 return rpout.isdir() 212 elif rpin.issym(): 213 return rpout.issym() and (rpin.readlink() == rpout.readlink()) 214 elif rpin.isdev(): 215 return rpout.isdev() and (rpin.getdevnums() == rpout.getdevnums()) 216 elif rpin.isfifo(): 217 return rpout.isfifo() 218 elif rpin.issock(): 219 return rpout.issock() 220 else: 221 raise RPathException("File %s has unknown type" % rpin.get_safepath()) 222 223 224def copy_attribs(rpin, rpout): 225 """Change file attributes of rpout to match rpin 226 227 Only changes the chmoddable bits, uid/gid ownership, and 228 timestamps, so both must already exist. 229 230 """ 231 log.Log( 232 "Copying attributes from %s to %s" % (rpin.index, 233 rpout.get_safepath()), 7) 234 assert rpin.lstat() == rpout.lstat() or rpin.isspecial() 235 if Globals.change_ownership: 236 rpout.chown(*rpout.conn.user_group.map_rpath(rpin)) 237 if Globals.eas_write: 238 rpout.write_ea(rpin.get_ea()) 239 if rpin.issym(): 240 return # symlinks don't have times or perms 241 if (Globals.resource_forks_write and rpin.isreg() 242 and rpin.has_resource_fork()): 243 rpout.write_resource_fork(rpin.get_resource_fork()) 244 if (Globals.carbonfile_write and rpin.isreg() and rpin.has_carbonfile()): 245 rpout.write_carbonfile(rpin.get_carbonfile()) 246 rpout.chmod(rpin.getperms()) 247 if Globals.acls_write: 248 rpout.write_acl(rpin.get_acl()) 249 if not rpin.isdev(): 250 rpout.setmtime(rpin.getmtime()) 251 if Globals.win_acls_write: 252 rpout.write_win_acl(rpin.get_win_acl()) 253 254 255def copy_attribs_inc(rpin, rpout): 256 """Change file attributes of rpout to match rpin 257 258 Like above, but used to give increments the same attributes as the 259 originals. Therefore, don't copy all directory acl and 260 permissions. 261 262 """ 263 log.Log( 264 "Copying inc attrs from %s to %s" % (rpin.index, rpout.get_safepath()), 265 7) 266 check_for_files(rpin, rpout) 267 if Globals.change_ownership: 268 rpout.chown(*rpin.getuidgid()) 269 if Globals.eas_write: 270 rpout.write_ea(rpin.get_ea()) 271 if rpin.issym(): 272 return # symlinks don't have times or perms 273 if (Globals.resource_forks_write and rpin.isreg() 274 and rpin.has_resource_fork() and rpout.isreg()): 275 rpout.write_resource_fork(rpin.get_resource_fork()) 276 if (Globals.carbonfile_write and rpin.isreg() and rpin.has_carbonfile() 277 and rpout.isreg()): 278 rpout.write_carbonfile(rpin.get_carbonfile()) 279 if rpin.isdir() and not rpout.isdir(): 280 rpout.chmod(rpin.getperms() & 0o777) 281 else: 282 rpout.chmod(rpin.getperms()) 283 if Globals.acls_write: 284 rpout.write_acl(rpin.get_acl(), map_names=0) 285 if not rpin.isdev(): 286 rpout.setmtime(rpin.getmtime()) 287 288 289def cmp_attribs(rp1, rp2): 290 """True if rp1 has the same file attributes as rp2 291 292 Does not compare file access times. If not changing 293 ownership, do not check user/group id. 294 295 """ 296 check_for_files(rp1, rp2) 297 if Globals.change_ownership and rp1.getuidgid() != rp2.getuidgid(): 298 result = None 299 elif rp1.getperms() != rp2.getperms(): 300 result = None 301 elif rp1.issym() and rp2.issym(): # Don't check times for some types 302 result = 1 303 elif rp1.isblkdev() and rp2.isblkdev(): 304 result = 1 305 elif rp1.ischardev() and rp2.ischardev(): 306 result = 1 307 else: 308 result = ((rp1.getctime() == rp2.getctime()) 309 and (rp1.getmtime() == rp2.getmtime())) 310 log.Log( 311 "Compare attribs of %s and %s: %s" % (rp1.get_safeindexpath(), 312 rp2.get_safeindexpath(), result), 313 7) 314 return result 315 316 317def copy_with_attribs(rpin, rpout, compress=0): 318 """Copy file and then copy over attributes""" 319 copy(rpin, rpout, compress) 320 if rpin.lstat(): 321 copy_attribs(rpin, rpout) 322 323 324def rename(rp_source, rp_dest): 325 """Rename rp_source to rp_dest""" 326 assert rp_source.conn is rp_dest.conn 327 log.Log(lambda: "Renaming %s to %s" % (rp_source.get_safepath(), rp_dest.get_safepath()), 7) 328 if not rp_source.lstat(): 329 rp_dest.delete() 330 else: 331 if rp_dest.lstat() and rp_source.getinode() == rp_dest.getinode() and \ 332 rp_source.getinode() != 0: 333 log.Log( 334 "Warning: Attempt to rename over same inode: %s to %s" % 335 (rp_source.get_safepath(), rp_dest.get_safepath()), 2) 336 # You can't rename one hard linked file over another 337 rp_source.delete() 338 else: 339 try: 340 rp_source.conn.os.rename(rp_source.path, rp_dest.path) 341 except OSError as error: 342 # XXX errno.EINVAL and len(rp_dest.path) >= 260 indicates 343 # pathname too long on Windows 344 if error.errno != errno.EEXIST: 345 log.Log( 346 "OSError while renaming %s to %s" % 347 (rp_source.get_safepath(), rp_dest.get_safepath()), 1) 348 raise 349 350 # On Windows, files can't be renamed on top of an existing file 351 rp_source.conn.os.chmod(rp_dest.path, 0o700) 352 rp_source.conn.os.unlink(rp_dest.path) 353 rp_source.conn.os.rename(rp_source.path, rp_dest.path) 354 355 rp_dest.data = rp_source.data 356 rp_source.data = {'type': None} 357 358 359def make_file_dict(filename): 360 """Generate the data dictionary for the given RPath 361 362 This is a global function so that os.name can be called locally, 363 thus avoiding network lag and so that we only need to send the 364 filename over the network, thus avoiding the need to pickle an 365 (incomplete) rpath object. 366 """ 367 368 def _readlink(filename): 369 """FIXME wrapper function to workaround a bug in os.readlink on Windows 370 not accepting bytes path. This function can be removed once pyinstaller 371 supports Python 3.8 and a new release can be made. 372 See https://github.com/pyinstaller/pyinstaller/issues/4311 373 """ 374 375 if os.name == 'nt' and not isinstance(filename, str): 376 # we assume a bytes representation 377 return os.fsencode(os.readlink(os.fsdecode(filename))) 378 else: 379 return os.readlink(filename) 380 381 try: 382 statblock = os.lstat(filename) 383 except (FileNotFoundError, NotADirectoryError): 384 # FIXME not sure if this shouldn't trigger a warning but doing it 385 # generates (too) many messages during the tests 386 # log.Log("Warning: missing file '%s' couldn't be assessed." % filename, 2) 387 return {'type': None} 388 data = {} 389 mode = statblock[stat.ST_MODE] 390 391 if stat.S_ISREG(mode): 392 type_ = 'reg' 393 elif stat.S_ISDIR(mode): 394 type_ = 'dir' 395 elif stat.S_ISCHR(mode): 396 type_ = 'dev' 397 s = statblock.st_rdev 398 data['devnums'] = ('c', os.major(s), os.minor(s)) 399 elif stat.S_ISBLK(mode): 400 type_ = 'dev' 401 s = statblock.st_rdev 402 data['devnums'] = ('b', os.major(s), os.minor(s)) 403 elif stat.S_ISFIFO(mode): 404 type_ = 'fifo' 405 elif stat.S_ISLNK(mode): 406 type_ = 'sym' 407 # FIXME reverse once Python 3.8 can be used under Windows 408 # data['linkname'] = os.readlink(filename) 409 data['linkname'] = _readlink(filename) 410 elif stat.S_ISSOCK(mode): 411 type_ = 'sock' 412 else: 413 raise C.UnknownFileError(filename) 414 data['type'] = type_ 415 data['size'] = statblock[stat.ST_SIZE] 416 data['perms'] = stat.S_IMODE(mode) 417 data['uid'] = statblock[stat.ST_UID] 418 data['gid'] = statblock[stat.ST_GID] 419 data['inode'] = statblock[stat.ST_INO] 420 data['devloc'] = statblock[stat.ST_DEV] 421 data['nlink'] = statblock[stat.ST_NLINK] 422 423 if os.name == 'nt': 424 try: 425 attribs = win32api.GetFileAttributes(os.fsdecode(filename)) 426 except pywintypes.error as exc: 427 if (exc.args[0] == 32): # file in use 428 # we could also ignore with: return {'type': None} 429 # but this approach seems to be better handled 430 attribs = 0 431 else: 432 # we replace the specific Windows exception by a generic 433 # one also understood by a potential Linux client/server 434 raise OSError(None, exc.args[1] + " - " + exc.args[2], 435 filename, exc.args[0]) from None 436 if attribs & win32con.FILE_ATTRIBUTE_REPARSE_POINT: 437 data['type'] = 'sym' 438 data['linkname'] = None 439 440 if not (type_ == 'sym' or type_ == 'dev'): 441 # mtimes on symlinks and dev files don't work consistently 442 data['mtime'] = int(statblock[stat.ST_MTIME]) 443 data['atime'] = int(statblock[stat.ST_ATIME]) 444 data['ctime'] = int(statblock[stat.ST_CTIME]) 445 return data 446 447 448def make_socket_local(rpath): 449 """Make a local socket at the given path 450 451 This takes an rpath so that it will be checked by Security. 452 (Miscellaneous strings will not be.) 453 """ 454 assert rpath.conn is Globals.local_connection 455 rpath.conn.os.mknod(rpath.path, stat.S_IFSOCK) 456 457 458def gzip_open_local_read(rpath): 459 """Return open GzipFile. See security note directly above""" 460 assert rpath.conn is Globals.local_connection 461 return gzip.GzipFile(rpath.path, "rb") 462 463 464def open_local_read(rpath): 465 """Return open file (provided for security reasons)""" 466 assert rpath.conn is Globals.local_connection 467 return open(rpath.path, "rb") 468 469 470def get_incfile_info(basename): 471 """Returns None or tuple of 472 (is_compressed, timestr, type, and basename)""" 473 dotsplit = basename.split(b'.') 474 if dotsplit[-1] == b'gz': 475 compressed = 1 476 if len(dotsplit) < 4: 477 return None 478 timestring, ext = dotsplit[-3:-1] 479 else: 480 compressed = None 481 if len(dotsplit) < 3: 482 return None 483 timestring, ext = dotsplit[-2:] 484 if Time.bytestotime(timestring) is None: 485 return None 486 if not (ext == b"snapshot" or ext == b"dir" or ext == b"missing" 487 or ext == b"diff" or ext == b"data"): 488 return None 489 if compressed: 490 basestr = b'.'.join(dotsplit[:-3]) 491 else: 492 basestr = b'.'.join(dotsplit[:-2]) 493 return (compressed, timestring, ext, basestr) 494 495 496def delete_dir_no_files(rp): 497 """Deletes the directory at rp.path if empty. Raises if the 498 directory contains files.""" 499 assert rp.isdir() 500 if rp.contains_files(): 501 raise RPathException("Directory contains files.") 502 rp.delete() 503 504 505class RORPath: 506 """Read Only RPath - carry information about a path 507 508 These contain information about a file, and possible the file's 509 data, but do not have a connection and cannot be written to or 510 changed. The advantage of these objects is that they can be 511 communicated by encoding their index and data dictionary. 512 513 """ 514 515 def __init__(self, index, data=None): 516 self.index = tuple(map(os.fsencode, index)) 517 if data: 518 self.data = data 519 else: 520 self.data = {'type': None} # signify empty file 521 self.file = None 522 523 def zero(self): 524 """Set inside of self to type None""" 525 self.data = {'type': None} 526 self.file = None 527 528 def make_zero_dir(self, dir_rp): 529 """Set self.data the same as dir_rp.data but with safe permissions""" 530 self.data = dir_rp.data.copy() 531 self.data['perms'] = 0o700 532 533 def __eq__(self, other): 534 """True iff the two rorpaths are equivalent""" 535 if self.index != other.index: 536 return None 537 538 for key in list(self.data.keys()): # compare dicts key by key 539 if self.issym() and key in ('uid', 'gid', 'uname', 'gname'): 540 pass # Don't compare gid/uid for symlinks 541 elif key == 'atime' and not Globals.preserve_atime: 542 pass 543 elif key == 'ctime': 544 pass 545 elif key == 'nlink': 546 pass 547 elif key == 'size' and not self.isreg(): 548 pass 549 elif key == 'ea' and not Globals.eas_active: 550 pass 551 elif key == 'acl' and not Globals.acls_active: 552 pass 553 elif key == 'win_acl' and not Globals.win_acls_active: 554 pass 555 elif key == 'carbonfile' and not Globals.carbonfile_active: 556 pass 557 elif key == 'resourcefork' and not Globals.resource_forks_active: 558 pass 559 elif key == 'uname' or key == 'gname': 560 # here for legacy reasons - 0.12.x didn't store u/gnames 561 other_name = other.data.get(key, None) 562 if (other_name and other_name != "None" 563 and other_name != self.data[key]): 564 return None 565 elif ((key == 'inode' or key == 'devloc') 566 and (not self.isreg() or self.getnumlinks() == 1 567 or not Globals.compare_inode 568 or not Globals.preserve_hardlinks)): 569 pass 570 else: 571 try: 572 other_val = other.data[key] 573 except KeyError: 574 return None 575 if self.data[key] != other_val: 576 return None 577 return 1 578 579 def equal_loose(self, other): 580 """True iff the two rorpaths are kinda equivalent 581 582 Sometimes because permissions cannot be set, a file cannot be 583 replicated exactly on the remote side. This function tells 584 you whether the two files are close enough. self must be the 585 original rpath. 586 587 """ 588 for key in list(self.data.keys()): # compare dicts key by key 589 if key in ('uid', 'gid', 'uname', 'gname'): 590 pass 591 elif (key == 'type' and self.isspecial() and other.isreg() 592 and other.getsize() == 0): 593 pass # Special files may be replaced with empty regular files 594 elif key == 'atime' and not Globals.preserve_atime: 595 pass 596 elif key == 'ctime': 597 pass 598 elif key == 'devloc' or key == 'nlink': 599 pass 600 elif key == 'size' and not self.isreg(): 601 pass 602 elif key == 'inode': 603 pass 604 elif key == 'ea' and not Globals.eas_write: 605 pass 606 elif key == 'acl' and not Globals.acls_write: 607 pass 608 elif key == 'win_acl' and not Globals.win_acls_write: 609 pass 610 elif key == 'carbonfile' and not Globals.carbonfile_write: 611 pass 612 elif key == 'resourcefork' and not Globals.resource_forks_write: 613 pass 614 elif key == 'sha1': 615 pass # one or other may not have set 616 elif key == 'mirrorname' or key == 'incname': 617 pass 618 elif (key not in other.data or self.data[key] != other.data[key]): 619 return 0 620 621 if self.lstat() and not self.issym() and Globals.change_ownership: 622 # Now compare ownership. Symlinks don't have ownership 623 try: 624 if user_group.map_rpath(self) != other.getuidgid(): 625 return 0 626 except KeyError: 627 return 0 # uid/gid might be missing if metadata file is corrupt 628 629 return 1 630 631 def equal_verbose(self, 632 other, 633 check_index=1, 634 compare_inodes=0, 635 compare_ownership=0, 636 compare_acls=0, 637 compare_eas=0, 638 compare_win_acls=0, 639 compare_size=1, 640 compare_type=1, 641 verbosity=2): 642 """Like __eq__, but log more information. Useful when testing""" 643 if check_index and self.index != other.index: 644 log.Log("Index %s != index %s" % (self.index, other.index), 645 verbosity) 646 return None 647 648 for key in list(self.data.keys()): # compare dicts key by key 649 if (key in ('uid', 'gid', 'uname', 'gname') 650 and (self.issym() or not compare_ownership)): 651 # Don't compare gid/uid for symlinks, or if told not to 652 pass 653 elif key == 'type' and not compare_type: 654 pass 655 elif key == 'atime' and not Globals.preserve_atime: 656 pass 657 elif key == 'ctime': 658 pass 659 elif key == 'devloc' or key == 'nlink': 660 pass 661 elif key == 'size' and (not self.isreg() or not compare_size): 662 pass 663 elif key == 'inode' and (not self.isreg() or not compare_inodes): 664 pass 665 elif key == 'ea' and not compare_eas: 666 pass 667 elif key == 'acl' and not compare_acls: 668 pass 669 elif key == 'win_acl' and not compare_win_acls: 670 pass 671 elif (key not in other.data or self.data[key] != other.data[key]): 672 if key not in other.data: 673 log.Log("Second is missing key %s" % (key, ), verbosity) 674 else: 675 log.Log( 676 "Value of %s differs between %s and %s: %s vs %s" % 677 (key, self.get_indexpath(), other.get_indexpath(), 678 self.data[key], other.data[key]), verbosity) 679 return None 680 return 1 681 682 def equal_verbose_auto(self, other, verbosity=2): 683 """Like equal_verbose, but set parameters like __eq__ does""" 684 compare_inodes = ((self.getnumlinks() != 1) and Globals.compare_inode 685 and Globals.preserve_hardlinks) 686 return self.equal_verbose( 687 other, 688 compare_inodes=compare_inodes, 689 compare_eas=Globals.eas_active, 690 compare_acls=Globals.acls_active, 691 compare_win_acls=Globals.win_acls_active) 692 693 def __ne__(self, other): 694 return not self.__eq__(other) 695 696 def __str__(self): 697 """Pretty print file statistics""" 698 return "Index: %s\nData: %s" % (self.get_safeindex(), self.data) 699 700 def summary_string(self): 701 """Return summary string""" 702 return "%s %s" % (self.get_safeindexpath(), self.lstat()) 703 704 def __getstate__(self): 705 """Return picklable state 706 707 This is necessary in case the RORPath is carrying around a 708 file object, which can't/shouldn't be pickled. 709 710 """ 711 return (self.index, self.data) 712 713 def __setstate__(self, rorp_state): 714 """Reproduce RORPath from __getstate__ output""" 715 self.index, self.data = rorp_state 716 717 def getRORPath(self): 718 """Return new rorpath based on self""" 719 return RORPath(self.index, self.data.copy()) 720 721 @classmethod 722 def path_join(self, *filenames): 723 """Simulate the os.path.join function to have the same separator '/' on all platforms""" 724 725 def abs_drive(filename): 726 """Because os.path.join does make out of a tuple with a drive letter under Windows 727 a path _relative_ to the current directory on the drive, we need to make it 728 absolute ourselves by adding a slash at the end.""" 729 if re.match(b"^[A-Za-z]:$", filename): # if filename is a drive 730 return filename + b'/' # os.path.join won't make a drive absolute for us 731 else: 732 return filename 733 734 if os.path.altsep: # only Windows has an alternative separator for paths 735 filenames = tuple(map(abs_drive, filenames)) 736 return os.path.join(*filenames).replace( 737 os.fsencode(os.path.sep), b'/') 738 else: 739 return os.path.join(*filenames) 740 741 @classmethod 742 def getcwdb(self): 743 """A getcwdb function that makes sure that also under Windows '/' are used""" 744 if os.path.altsep: # only Windows has an alternative separator for paths 745 return os.getcwdb().replace(os.fsencode(os.path.sep), b'/') 746 else: 747 return os.getcwdb() 748 749 def lstat(self): 750 """Returns type of file 751 752 The allowable types are None if the file doesn't exist, 'reg' 753 for a regular file, 'dir' for a directory, 'dev' for a device 754 file, 'fifo' for a fifo, 'sock' for a socket, and 'sym' for a 755 symlink. 756 """ 757 return self.data['type'] 758 759 gettype = lstat 760 761 def isdir(self): 762 """True if self is a dir""" 763 return self.data['type'] == 'dir' 764 765 def isreg(self): 766 """True if self is a regular file""" 767 return self.data['type'] == 'reg' 768 769 def issym(self): 770 """True if path is of a symlink""" 771 return self.data['type'] == 'sym' 772 773 def isfifo(self): 774 """True if path is a fifo""" 775 return self.data['type'] == 'fifo' 776 777 def ischardev(self): 778 """True if path is a character device file""" 779 return self.data['type'] == 'dev' and self.data['devnums'][0] == 'c' 780 781 def isblkdev(self): 782 """True if path is a block device file""" 783 return self.data['type'] == 'dev' and self.data['devnums'][0] == 'b' 784 785 def isdev(self): 786 """True if path is a device file""" 787 return self.data['type'] == 'dev' 788 789 def issock(self): 790 """True if path is a socket""" 791 return self.data['type'] == 'sock' 792 793 def isspecial(self): 794 """True if the file is a sock, symlink, device, or fifo""" 795 type = self.data['type'] 796 return (type == 'dev' or type == 'sock' or type == 'fifo' 797 or type == 'sym') 798 799 def getperms(self): 800 """Return permission block of file""" 801 if 'perms' in self.data: 802 return self.data['perms'] 803 else: 804 return 0 805 806 def getuname(self): 807 """Return username that owns the file""" 808 try: 809 return self.data['uname'] 810 except KeyError: 811 return None 812 813 def getgname(self): 814 """Return groupname that owns the file""" 815 try: 816 return self.data['gname'] 817 except KeyError: 818 return None 819 820 def hassize(self): 821 """True if rpath has a size parameter""" 822 return 'size' in self.data 823 824 def getsize(self): 825 """Return length of file in bytes""" 826 return self.data['size'] 827 828 def getuidgid(self): 829 """Return userid/groupid of file""" 830 return self.data['uid'], self.data['gid'] 831 832 def getatime(self): 833 """Return access time in seconds""" 834 return self.data['atime'] 835 836 def getmtime(self): 837 """Return modification time in seconds""" 838 return self.data['mtime'] 839 840 def getctime(self): 841 """Return change time in seconds""" 842 return self.data['ctime'] 843 844 def getinode(self): 845 """Return inode number of file""" 846 return self.data['inode'] 847 848 def getdevloc(self): 849 """Device number file resides on""" 850 return self.data['devloc'] 851 852 def getnumlinks(self): 853 """Number of places inode is linked to""" 854 if 'nlink' in self.data: 855 return self.data['nlink'] 856 else: 857 return 1 858 859 def readlink(self): 860 """Wrapper around os.readlink()""" 861 return self.data['linkname'] 862 863 def getdevnums(self): 864 """Return a device's type and major/minor numbers from dictionary""" 865 return self.data['devnums'] 866 867 def setfile(self, file): 868 """Right now just set self.file to be the already opened file""" 869 assert file and not self.file 870 871 def closing_hook(): 872 self.file_already_open = None 873 874 self.file = RPathFileHook(file, closing_hook) 875 self.file_already_open = None 876 877 def get_safeindex(self): 878 """Return index as a tuple of strings with safe decoding 879 880 For instance, if the index is (b"a", b"b"), return ("a", "b") 881 882 """ 883 return tuple(map(lambda f: f.decode(errors='replace'), self.index)) 884 885 def get_indexpath(self): 886 """Return path of index portion 887 888 For instance, if the index is ("a", "b"), return "a/b". 889 890 """ 891 if not self.index: 892 return b'.' 893 return self.path_join(*self.index) 894 895 def get_safeindexpath(self): 896 """Return safe path of index even with names throwing UnicodeEncodeError 897 898 For instance, if the index is ("a", "b"), return "'a/b'". 899 900 """ 901 return self.get_indexpath().decode(errors='replace') 902 903 def get_attached_filetype(self): 904 """If there is a file attached, say what it is 905 906 Currently the choices are 'snapshot' meaning an exact copy of 907 something, and 'diff' for an rdiff style diff. 908 909 """ 910 return self.data['filetype'] 911 912 def set_attached_filetype(self, type): 913 """Set the type of the attached file""" 914 self.data['filetype'] = type 915 916 def isflaglinked(self): 917 """True if rorp is a signature/diff for a hardlink file 918 919 This indicates that a file's data need not be transferred 920 because it is hardlinked on the remote side. 921 922 """ 923 return 'linked' in self.data 924 925 def get_link_flag(self): 926 """Return previous index that a file is hard linked to""" 927 return self.data['linked'] 928 929 def flaglinked(self, index): 930 """Signal that rorp is a signature/diff for a hardlink file""" 931 self.data['linked'] = index 932 933 def open(self, mode): 934 """Return file type object if any was given using self.setfile""" 935 if mode != "rb": 936 raise RPathException("Bad mode %s" % mode) 937 if self.file_already_open: 938 raise RPathException("Attempt to open same file twice") 939 self.file_already_open = 1 940 return self.file 941 942 def close_if_necessary(self): 943 """If file is present, discard data and close""" 944 if self.file: 945 while self.file.read(Globals.blocksize): 946 pass 947 assert not self.file.close(), \ 948 "Error closing file\ndata = %s\nindex = %s\n" % (self.data, 949 self.get_safeindex()) 950 self.file_already_open = None 951 952 def set_acl(self, acl): 953 """Record access control list in dictionary. Does not write""" 954 self.data['acl'] = acl 955 956 def get_acl(self): 957 """Return access control list object from dictionary""" 958 try: 959 return self.data['acl'] 960 except KeyError: 961 acl = self.data['acl'] = get_blank_acl(self.index) 962 return acl 963 964 def set_ea(self, ea): 965 """Record extended attributes in dictionary. Does not write""" 966 self.data['ea'] = ea 967 968 def get_ea(self): 969 """Return extended attributes object""" 970 try: 971 return self.data['ea'] 972 except KeyError: 973 ea = self.data['ea'] = get_blank_ea(self.index) 974 return ea 975 976 def has_carbonfile(self): 977 """True if rpath has a carbonfile parameter""" 978 return 'carbonfile' in self.data 979 980 def get_carbonfile(self): 981 """Returns the carbonfile data""" 982 return self.data['carbonfile'] 983 984 def set_carbonfile(self, cfile): 985 """Record carbonfile data in dictionary. Does not write.""" 986 self.data['carbonfile'] = cfile 987 988 def has_resource_fork(self): 989 """True if rpath has a resourcefork parameter""" 990 return 'resourcefork' in self.data 991 992 def get_resource_fork(self): 993 """Return the resource fork in binary data""" 994 return self.data['resourcefork'] 995 996 def set_resource_fork(self, rfork): 997 """Record resource fork in dictionary. Does not write""" 998 self.data['resourcefork'] = rfork 999 1000 def set_win_acl(self, acl): 1001 """Record Windows access control list in dictionary. Does not write""" 1002 self.data['win_acl'] = acl 1003 1004 def get_win_acl(self): 1005 """Return access control list object from dictionary""" 1006 try: 1007 return self.data['win_acl'] 1008 except KeyError: 1009 acl = self.data['win_acl'] = get_blank_win_acl(self.index) 1010 return acl 1011 1012 def has_alt_mirror_name(self): 1013 """True if rorp has an alternate mirror name specified""" 1014 return 'mirrorname' in self.data 1015 1016 def get_alt_mirror_name(self): 1017 """Return alternate mirror name (for long filenames)""" 1018 return self.data['mirrorname'] 1019 1020 def set_alt_mirror_name(self, filename): 1021 """Set alternate mirror name to filename 1022 1023 Instead of writing to the traditional mirror file, store 1024 mirror information in filename in the long filename 1025 directory. 1026 1027 """ 1028 self.data['mirrorname'] = filename 1029 1030 def has_alt_inc_name(self): 1031 """True if rorp has an alternate increment base specified""" 1032 return 'incname' in self.data 1033 1034 def get_alt_inc_name(self): 1035 """Return alternate increment base (used for long name support)""" 1036 return self.data['incname'] 1037 1038 def set_alt_inc_name(self, name): 1039 """Set alternate increment name to name 1040 1041 If set, increments will be in the long name directory with 1042 name as their base. If the alt mirror name is set, this 1043 should be set to the same. 1044 1045 """ 1046 self.data['incname'] = name 1047 1048 def has_sha1(self): 1049 """True iff self has its sha1 digest set""" 1050 return 'sha1' in self.data 1051 1052 def get_sha1(self): 1053 """Return sha1 digest. Causes exception unless set_sha1 first""" 1054 return self.data['sha1'] 1055 1056 def set_sha1(self, digest): 1057 """Set sha1 hash (should be in hexdecimal)""" 1058 self.data['sha1'] = digest 1059 1060 1061class RPath(RORPath): 1062 """Remote Path class - wrapper around a possibly non-local pathname 1063 1064 This class contains a dictionary called "data" which should 1065 contain all the information about the file sufficient for 1066 identification (i.e. if two files have the the same (==) data 1067 dictionary, they are the same file). 1068 1069 """ 1070 regex_chars_to_quote = re.compile(b"[\\\\\\\"\\$`]") 1071 1072 def __init__(self, connection, base, index=(), data=None): 1073 """RPath constructor 1074 1075 connection = self.conn is the Connection the RPath will use to 1076 make system calls, and index is the name of the rpath used for 1077 comparison, and should be a tuple consisting of the parts of 1078 the rpath after the base split up. For instance ("foo", 1079 "bar") for "foo/bar" (no base), and ("local", "bin") for 1080 "/usr/local/bin" if the base is "/usr". 1081 1082 For the root directory "/", the index is empty and the base is 1083 "/". 1084 1085 """ 1086 super().__init__(index, data) 1087 self.conn = connection 1088 if base is not None: 1089 self.base = os.fsencode(base) # path is always bytes 1090 self.path = self.path_join(self.base, *self.index) 1091 if data is None: 1092 self.setdata() 1093 else: 1094 self.base = None 1095 1096 def __str__(self): 1097 return "%s: Path: %s\nIndex: %s\nData: %s" \ 1098 % (self.__class__.__name__, self.get_safepath(), self.get_safeindex(), self.data) 1099 1100 def __getstate__(self): 1101 """Return picklable state 1102 1103 The rpath's connection will be encoded as its conn_number. It 1104 and the other information is put in a tuple. Data and any attached 1105 file won't be saved. 1106 1107 """ 1108 return (self.conn.conn_number, self.base, self.index, self.data) 1109 1110 def __setstate__(self, rpath_state): 1111 """Reproduce RPath from __getstate__ output""" 1112 conn_number, self.base, self.index, self.data = rpath_state 1113 self.conn = Globals.connection_dict[conn_number] 1114 self.path = self.path_join(self.base, *self.index) 1115 1116 def setdata(self): 1117 """Set data dictionary using the wrapper""" 1118 self.data = self.conn.rpath.make_file_dict(self.path) 1119 if self.lstat(): 1120 self.conn.rpath.setdata_local(self) 1121 1122 def check_consistency(self): 1123 """Raise an error if consistency of rp broken 1124 1125 This is useful for debugging when the cache and disk get out 1126 of sync and you need to find out where it happened. 1127 1128 """ 1129 temptype = self.data['type'] 1130 self.setdata() 1131 assert temptype == self.data['type'], \ 1132 "\nName: %s\nOld: %s --> New: %s\n" % \ 1133 (self.path, temptype, self.data['type']) 1134 1135 def chmod(self, permissions, loglevel=2): 1136 """Wrapper around os.chmod""" 1137 try: 1138 self.conn.os.chmod(self.path, 1139 permissions & Globals.permission_mask) 1140 except OSError as e: 1141 if e.strerror == "Inappropriate file type or format" \ 1142 and not self.isdir(): 1143 # Some systems throw this error if try to set sticky bit 1144 # on a non-directory. Remove sticky bit and try again. 1145 log.Log( 1146 "Warning: Unable to set permissions of %s to %o - " 1147 "trying again without sticky bit (%o)" % 1148 (self.path, permissions, permissions & 0o6777), loglevel) 1149 self.conn.os.chmod( 1150 self.path, permissions 1151 & 0o6777 & Globals.permission_mask) 1152 else: 1153 raise 1154 self.data['perms'] = permissions 1155 1156 def settime(self, accesstime, modtime): 1157 """Change file modification times""" 1158 log.Log("Setting time of %s to %d" % (self.get_safepath(), modtime), 7) 1159 try: 1160 self.conn.os.utime(self.path, (accesstime, modtime)) 1161 except OverflowError: 1162 log.Log( 1163 "Cannot change times of %s to %s - problem is probably" 1164 "64->32bit conversion" % (self.path, (accesstime, modtime)), 2) 1165 else: 1166 self.data['atime'] = accesstime 1167 self.data['mtime'] = modtime 1168 1169 def setmtime(self, modtime): 1170 """Set only modtime (access time to present)""" 1171 log.Log( 1172 lambda: "Setting time of %s to %d" % (self.get_safepath(), modtime), 1173 7) 1174 if modtime < 0: 1175 log.Log( 1176 "Warning: modification time of %s is" 1177 "before 1970" % self.path, 2) 1178 try: 1179 self.conn.os.utime(self.path, (int(time.time()), modtime)) 1180 except OverflowError: 1181 log.Log( 1182 "Cannot change mtime of %s to %s - problem is probably" 1183 "64->32bit conversion" % (self.path, modtime), 2) 1184 except OSError: 1185 # It's not possible to set a modification time for 1186 # directories on Windows. 1187 if self.conn.os.name != 'nt' or not self.isdir(): 1188 raise 1189 else: 1190 self.data['mtime'] = modtime 1191 1192 def chown(self, uid, gid): 1193 """Set file's uid and gid""" 1194 if self.issym(): 1195 try: 1196 self.conn.os.lchown(self.path, uid, gid) 1197 except AttributeError: 1198 log.Log( 1199 "Warning: lchown missing, cannot change ownership " 1200 "of symlink %s" % self.get_safepath(), 2) 1201 else: 1202 self.conn.os.chown(self.path, uid, gid) 1203 # uid/gid equal to -1 is ignored by chown/lchown 1204 if uid >= 0: 1205 self.data['uid'] = uid 1206 if gid >= 0: 1207 self.data['gid'] = gid 1208 1209 def mkdir(self): 1210 log.Log("Making directory %s" % self.get_safepath(), 6) 1211 self.conn.os.mkdir(self.path) 1212 self.setdata() 1213 1214 def makedirs(self): 1215 log.Log("Making directory path %s" % self.get_safepath(), 6) 1216 self.conn.os.makedirs(self.path) 1217 self.setdata() 1218 1219 def rmdir(self): 1220 log.Log("Removing directory %s" % self.get_safepath(), 6) 1221 self.conn.os.chmod(self.path, 0o700) 1222 self.conn.os.rmdir(self.path) 1223 self.data = {'type': None} 1224 1225 def listdir(self): 1226 """Return list of string paths returned by os.listdir""" 1227 path = self.path 1228 return self.conn.os.listdir(path) 1229 1230 def symlink(self, linktext): 1231 """Make symlink at self.path pointing to linktext""" 1232 self.conn.os.symlink(linktext, self.path) 1233 self.setdata() 1234 assert self.issym() 1235 1236 def hardlink(self, linkpath): 1237 """Make self into a hardlink joined to linkpath""" 1238 log.Log( 1239 "Hard linking %s to %s" % (self.get_safepath(), 1240 self.get_safepath(linkpath)), 6) 1241 self.conn.os.link(linkpath, self.path) 1242 self.setdata() 1243 1244 def mkfifo(self): 1245 """Make a fifo at self.path""" 1246 self.conn.os.mkfifo(self.path) 1247 self.setdata() 1248 assert self.isfifo() 1249 1250 def mksock(self): 1251 """Make a socket at self.path""" 1252 self.conn.rpath.make_socket_local(self) 1253 self.setdata() 1254 assert self.issock() 1255 1256 def touch(self): 1257 """Make sure file at self.path exists""" 1258 log.Log("Touching %s" % self.get_safepath(), 7) 1259 self.conn.open(self.path, "wb").close() 1260 self.setdata() 1261 assert self.isreg(), self.path 1262 1263 def hasfullperms(self): 1264 """Return true if current process has full permissions on the file""" 1265 if self.isowner(): 1266 return self.getperms() % 0o1000 >= 0o700 1267 elif self.isgroup(): 1268 return self.getperms() % 0o100 >= 0o70 1269 else: 1270 return self.getperms() % 0o10 >= 0o7 1271 1272 def readable(self): 1273 """Return true if current process has read permissions on the file""" 1274 if self.isowner(): 1275 return self.getperms() % 0o1000 >= 0o400 1276 elif self.isgroup(): 1277 return self.getperms() % 0o100 >= 0o40 1278 else: 1279 return self.getperms() % 0o10 >= 0o4 1280 1281 def executable(self): 1282 """Return true if current process has execute permissions""" 1283 if self.isowner(): 1284 return self.getperms() % 0o200 >= 0o100 1285 elif self.isgroup(): 1286 return self.getperms() % 0o20 >= 0o10 1287 else: 1288 return self.getperms() % 0o2 >= 0o1 1289 1290 def isowner(self): 1291 """Return true if current process is owner of rp or root""" 1292 try: 1293 uid = self.conn.os.getuid() 1294 except AttributeError: 1295 return True # Windows doesn't have getuid(), so hope for the best 1296 return uid == 0 or \ 1297 ('uid' in self.data and uid == self.data['uid']) 1298 1299 def isgroup(self): 1300 """Return true if process has group of rp""" 1301 return ('gid' in self.data 1302 and self.data['gid'] in self.conn.Globals.get('process_groups')) 1303 1304 def delete(self): 1305 """Delete file at self.path. Recursively deletes directories.""" 1306 log.Log("Deleting %s" % self.get_safepath(), 7) 1307 if self.isdir(): 1308 try: 1309 self.rmdir() 1310 except os.error: 1311 if Globals.fsync_directories: 1312 self.fsync() 1313 self.conn.shutil.rmtree(self.path) 1314 else: 1315 try: 1316 self.conn.os.unlink(self.path) 1317 except OSError as error: 1318 if error.errno in (errno.EPERM, errno.EACCES): 1319 # On Windows, read-only files cannot be deleted. 1320 # Remove the read-only attribute and try again. 1321 self.chmod(0o700) 1322 self.conn.os.unlink(self.path) 1323 else: 1324 raise 1325 1326 self.setdata() 1327 1328 def contains_files(self): 1329 """Returns true if self (or subdir) contains any regular files.""" 1330 log.Log( 1331 "Determining if directory contains files: %s" % 1332 self.get_safepath(), 7) 1333 if not self.isdir(): 1334 return False 1335 dir_entries = self.listdir() 1336 for entry in dir_entries: 1337 child_rp = self.append(entry) 1338 if not child_rp.isdir(): 1339 return True 1340 else: 1341 if child_rp.contains_files(): 1342 return True 1343 return False 1344 1345 def quote(self): 1346 """Return quoted self.path for use with os.system()""" 1347 return b'"%s"' % self.regex_chars_to_quote.sub( 1348 lambda m: b"\\" + m.group(0), self.path) 1349 1350 def normalize(self): 1351 """Return RPath canonical version of self.path 1352 1353 This just means that redundant /'s will be removed, including 1354 the trailing one, even for directories. ".." components will 1355 be retained. 1356 1357 """ 1358 newpath = self.path_join( 1359 b'', *[x for x in self.path.split(b"/") if x and x != b"."]) 1360 if self.path[0:1] == b"/": 1361 newpath = b"/" + newpath 1362 if self.path[1:2] == b"/" and self.path[2:3].isalnum(): # we assume a Windows share 1363 newpath = b"/" + newpath 1364 elif not newpath: 1365 newpath = b"." 1366 return self.newpath(newpath) 1367 1368 def dirsplit(self): 1369 """Returns a tuple of strings (dirname, basename) 1370 1371 Basename is never '' unless self is root, so it is unlike 1372 os.path.basename. If path is just above root (so dirname is 1373 root), then dirname is ''. In all other cases dirname is not 1374 the empty string. Also, dirsplit depends on the format of 1375 self, so basename could be ".." and dirname could be a 1376 subdirectory. For an atomic relative path, dirname will be 1377 '.'. 1378 1379 """ 1380 normed = self.normalize() 1381 if normed.path.find(b"/") == -1: 1382 return (b".", normed.path) 1383 comps = normed.path.split(b"/") 1384 return b"/".join(comps[:-1]), comps[-1] 1385 1386 def get_path(self): 1387 """Just a getter for the path variable that can be overwritten by QuotedRPath""" 1388 return self.path 1389 1390 def get_safepath(self, somepath=None): 1391 """Return safely decoded version of path into the current encoding 1392 1393 it's meant only for logging and outputting to user 1394 1395 """ 1396 if somepath is not None: 1397 # somepath should never be a string but just to be sure 1398 # we check before we decode it 1399 if isinstance(somepath, str): 1400 return somepath 1401 else: 1402 return somepath.decode(errors='replace') 1403 else: 1404 return self.path.decode(errors='replace') 1405 1406 def get_parent_rp(self): 1407 """Return new RPath of directory self is in""" 1408 if self.index: 1409 return self.__class__(self.conn, self.base, self.index[:-1]) 1410 dirname = self.dirsplit()[0] 1411 if dirname: 1412 return self.__class__(self.conn, dirname) 1413 else: 1414 return self.__class__(self.conn, b"/") 1415 1416 def newpath(self, newpath, index=()): 1417 """Return new RPath with the same connection but different path""" 1418 return self.__class__(self.conn, newpath, index) 1419 1420 def append(self, *ext): 1421 """Return new RPath with same connection by adjoining ext""" 1422 return self.__class__(self.conn, self.base, self.index + ext) 1423 1424 def append_path(self, ext, new_index=()): 1425 """Like append, but add ext to path instead of to index""" 1426 # ext can be a string but shouldn't hence we transform it into bytes 1427 return self.__class__(self.conn, 1428 self.path_join(self.base, os.fsencode(ext)), 1429 new_index) 1430 1431 def new_index(self, index): 1432 """Return similar RPath but with new index""" 1433 return self.__class__(self.conn, self.base, index) 1434 1435 def new_index_empty(self, index): 1436 """Return similar RPath with given index, but initialize to empty""" 1437 return self.__class__(self.conn, self.base, index, {'type': None}) 1438 1439 def open(self, mode, compress=None): 1440 """Return open file. Supports modes "w" and "r". 1441 1442 If compress is true, data written/read will be gzip 1443 compressed/decompressed on the fly. The extra complications 1444 below are for security reasons - try to make the extent of the 1445 risk apparent from the remote call. 1446 1447 """ 1448 if self.conn is Globals.local_connection: 1449 if compress: 1450 return gzip.GzipFile(self.path, mode) 1451 else: 1452 return open(self.path, mode) 1453 1454 if compress: 1455 if mode == "r" or mode == "rb": 1456 return self.conn.rpath.gzip_open_local_read(self) 1457 else: 1458 return self.conn.gzip.GzipFile(self.path, mode) 1459 else: 1460 if mode == "r" or mode == "rb": 1461 return self.conn.rpath.open_local_read(self) 1462 else: 1463 return self.conn.open(self.path, mode) 1464 1465 def write_from_fileobj(self, fp, compress=None): 1466 """Reads fp and writes to self.path. Closes both when done 1467 1468 If compress is true, fp will be gzip compressed before being 1469 written to self. Returns closing value of fp. 1470 1471 """ 1472 log.Log("Writing file object to %s" % self.get_safepath(), 7) 1473 assert not self.lstat(), "File %s already exists" % self.path 1474 outfp = self.open("wb", compress=compress) 1475 copyfileobj(fp, outfp) 1476 if outfp.close(): 1477 raise RPathException("Error closing file") 1478 self.setdata() 1479 return fp.close() 1480 1481 def write_string(self, s, compress=None): 1482 """Write string s into rpath""" 1483 assert not self.lstat(), "File %s already exists" % (self.path, ) 1484 with self.open("w", compress=compress) as outfp: 1485 outfp.write(s) 1486 self.setdata() 1487 1488 def write_bytes(self, s, compress=None): 1489 """Write data s into rpath""" 1490 assert not self.lstat(), "File %s already exists" % (self.path, ) 1491 with self.open("wb", compress=compress) as outfp: 1492 outfp.write(s) 1493 self.setdata() 1494 1495 def isincfile(self): 1496 """Return true if path looks like an increment file 1497 1498 Also sets various inc information used by the *inc* functions. 1499 1500 """ 1501 if self.index: 1502 basename = self.index[-1] 1503 else: 1504 basename = self.base 1505 1506 inc_info = get_incfile_info(basename) 1507 1508 if inc_info: 1509 self.inc_compressed, self.inc_timestr, \ 1510 self.inc_type, self.inc_basestr = inc_info 1511 return 1 1512 else: 1513 return None 1514 1515 def isinccompressed(self): 1516 """Return true if inc file is compressed""" 1517 return self.inc_compressed 1518 1519 def getinctype(self): 1520 """Return type of an increment file""" 1521 return self.inc_type 1522 1523 def getinctime(self): 1524 """Return time in seconds of an increment file""" 1525 return Time.bytestotime(self.inc_timestr) 1526 1527 def getincbase(self): 1528 """Return the base filename of an increment file in rp form""" 1529 if self.index: 1530 return self.__class__(self.conn, self.base, 1531 self.index[:-1] + (self.inc_basestr, )) 1532 else: 1533 return self.__class__(self.conn, self.inc_basestr) 1534 1535 def getincbase_bname(self): 1536 """Return the base filename as bytes of an increment file""" 1537 rp = self.getincbase() 1538 if rp.index: 1539 return rp.index[-1] 1540 else: 1541 return rp.dirsplit()[1] 1542 1543 def makedev(self, type, major, minor): 1544 """Make a special file with specified type, and major/minor nums""" 1545 if type == 'c': 1546 mode = stat.S_IFCHR | 0o600 1547 elif type == 'b': 1548 mode = stat.S_IFBLK | 0o600 1549 else: 1550 raise RPathException 1551 try: 1552 self.conn.os.mknod(self.path, mode, 1553 self.conn.os.makedev(major, minor)) 1554 except (OSError, AttributeError) as e: 1555 if isinstance(e, AttributeError) or e.errno == errno.EPERM: 1556 # AttributeError will be raised by Python 2.2, which 1557 # doesn't have os.mknod 1558 log.Log( 1559 "unable to mknod %s -- using touch instead" % 1560 self.get_safepath(), 4) 1561 self.touch() 1562 self.setdata() 1563 1564 def fsync(self, fp=None): 1565 """fsync the current file or directory 1566 1567 If fp is none, get the file description by opening the file. 1568 This can be useful for directories. 1569 1570 """ 1571 if Globals.do_fsync: 1572 if not fp: 1573 self.conn.rpath.RPath.fsync_local(self) 1574 else: 1575 os.fsync(fp.fileno()) 1576 1577 def fsync_local(self, thunk=None): 1578 """fsync current file, run locally 1579 1580 If thunk is given, run it before syncing but after gathering 1581 the file's file descriptor. 1582 1583 """ 1584 assert Globals.do_fsync 1585 assert self.conn is Globals.local_connection 1586 try: 1587 fd = os.open(self.path, os.O_RDONLY) 1588 os.fsync(fd) 1589 os.close(fd) 1590 except OSError as e: 1591 if 'fd' in locals(): 1592 os.close(fd) 1593 if (e.errno not in (errno.EPERM, errno.EACCES, errno.EBADF)) or self.isdir(): 1594 raise 1595 1596 # Maybe the system doesn't like read-only fsyncing. 1597 # However, to open RDWR, we may need to alter permissions 1598 # temporarily. 1599 if self.hasfullperms(): 1600 oldperms = None 1601 else: 1602 oldperms = self.getperms() 1603 if not oldperms: # self.data['perms'] is probably out of sync 1604 self.setdata() 1605 oldperms = self.getperms() 1606 self.chmod(0o700) 1607 fd = os.open(self.path, os.O_RDWR) 1608 if oldperms is not None: 1609 self.chmod(oldperms) 1610 if thunk: 1611 thunk() 1612 os.fsync(fd) # Sync after we switch back permissions! 1613 os.close(fd) 1614 1615 def fsync_with_dir(self, fp=None): 1616 """fsync self and directory self is under""" 1617 self.fsync(fp) 1618 if Globals.fsync_directories: 1619 self.get_parent_rp().fsync() 1620 1621 def get_bytes(self, compressed=None): 1622 """Open file as a regular file, read data, close, return data""" 1623 fp = self.open("rb", compressed) 1624 d = fp.read() 1625 assert not fp.close() 1626 return d 1627 1628 def get_string(self, compressed=None): 1629 """Open file as a regular file, read string, close, return string""" 1630 fp = self.open("r", compressed) 1631 s = fp.read() 1632 assert not fp.close() 1633 if isinstance(s, bytes) or isinstance(s, bytearray): 1634 s = s.decode() 1635 return s 1636 1637 def get_acl(self): 1638 """Return access control list object, setting if necessary""" 1639 try: 1640 acl = self.data['acl'] 1641 except KeyError: 1642 acl = self.data['acl'] = acl_get(self) 1643 return acl 1644 1645 def write_acl(self, acl, map_names=1): 1646 """Change access control list of rp 1647 1648 If map_names is true, map the ids in acl by user/group names. 1649 1650 """ 1651 acl.write_to_rp(self, map_names) 1652 self.data['acl'] = acl 1653 1654 def get_ea(self): 1655 """Return extended attributes object, setting if necessary""" 1656 try: 1657 ea = self.data['ea'] 1658 except KeyError: 1659 ea = self.data['ea'] = ea_get(self) 1660 return ea 1661 1662 def write_ea(self, ea): 1663 """Change extended attributes of rp""" 1664 ea.write_to_rp(self) 1665 self.data['ea'] = ea 1666 1667 def write_carbonfile(self, cfile): 1668 """Write new carbon data to self.""" 1669 if not cfile: 1670 return 1671 log.Log("Writing carbon data to %s" % (self.index, ), 7) 1672 from Carbon.File import FSSpec 1673 from Carbon.File import FSRef 1674 import Carbon.Files 1675 fsobj = FSSpec(self.path) 1676 finderinfo = fsobj.FSpGetFInfo() 1677 finderinfo.Creator = cfile['creator'] 1678 finderinfo.Type = cfile['type'] 1679 finderinfo.Location = cfile['location'] 1680 finderinfo.Flags = cfile['flags'] 1681 fsobj.FSpSetFInfo(finderinfo) 1682 """Write Creation Date to self (if stored in metadata).""" 1683 try: 1684 cdate = cfile['createDate'] 1685 fsref = FSRef(fsobj) 1686 cataloginfo, d1, d2, d3 = fsref.FSGetCatalogInfo( 1687 Carbon.Files.kFSCatInfoCreateDate) 1688 cataloginfo.createDate = (0, cdate, 0) 1689 fsref.FSSetCatalogInfo(Carbon.Files.kFSCatInfoCreateDate, 1690 cataloginfo) 1691 self.set_carbonfile(cfile) 1692 except KeyError: 1693 self.set_carbonfile(cfile) 1694 1695 def get_resource_fork(self): 1696 """Return resource fork data, setting if necessary""" 1697 assert self.isreg() 1698 try: 1699 rfork = self.data['resourcefork'] 1700 except KeyError: 1701 try: 1702 rfork_fp = self.conn.open( 1703 os.path.join(self.path, b'..namedfork', b'rsrc'), 'rb') 1704 rfork = rfork_fp.read() 1705 assert not rfork_fp.close() 1706 except (IOError, OSError): 1707 rfork = b'' 1708 self.data['resourcefork'] = rfork 1709 return rfork 1710 1711 def write_resource_fork(self, rfork_data): 1712 """Write new resource fork to self""" 1713 log.Log("Writing resource fork to %s" % (self.index, ), 7) 1714 fp = self.conn.open( 1715 os.path.join(self.path, b'..namedfork', b'rsrc'), 'wb') 1716 fp.write(rfork_data) 1717 assert not fp.close() 1718 self.set_resource_fork(rfork_data) 1719 1720 def get_win_acl(self): 1721 """Return Windows access control list, setting if necessary""" 1722 try: 1723 acl = self.data['win_acl'] 1724 except KeyError: 1725 acl = self.data['win_acl'] = win_acl_get(self) 1726 return acl 1727 1728 def write_win_acl(self, acl): 1729 """Change access control list of rp""" 1730 write_win_acl(self, acl) 1731 self.data['win_acl'] = acl 1732 1733 1734class RPathFileHook: 1735 """Look like a file, but add closing hook""" 1736 1737 def __init__(self, file, closing_thunk): 1738 self.file = file 1739 self.closing_thunk = closing_thunk 1740 1741 def read(self, length=-1): 1742 return self.file.read(length) 1743 1744 def write(self, buf): 1745 return self.file.write(buf) 1746 1747 def close(self): 1748 """Close file and then run closing thunk""" 1749 result = self.file.close() 1750 self.closing_thunk() 1751 return result 1752 1753 1754class MaybeGzip: 1755 """Represent a file object that may or may not be compressed 1756 1757 We don't want to compress 0 length files. This class lets us 1758 delay the opening of the file until either the first write (so we 1759 know it has data and should be compressed), or close (when there's 1760 no data). 1761 1762 """ 1763 1764 def __init__(self, base_rp, callback=None): 1765 """Return file-like object with filename based on base_rp""" 1766 assert not base_rp.lstat(), base_rp 1767 self.base_rp = base_rp 1768 # callback will be called with final write rp as only argument 1769 self.callback = callback 1770 self.fileobj = None # Will be None unless data gets written 1771 self.closed = 0 1772 1773 def __getattr__(self, name): 1774 if name == 'fileno': 1775 return self.fileobj.fileno 1776 else: 1777 raise AttributeError(name) 1778 1779 def get_gzipped_rp(self): 1780 """Return gzipped rp by adding .gz to base_rp""" 1781 if self.base_rp.index: 1782 newind = self.base_rp.index[:-1] + ( 1783 self.base_rp.index[-1] + b'.gz', ) 1784 return self.base_rp.new_index(newind) 1785 else: 1786 return self.base_rp.append_path(b'.gz') 1787 1788 def write(self, buf): 1789 """Write buf to fileobj""" 1790 if isinstance(buf, str): 1791 buf = buf.encode() 1792 if self.fileobj: 1793 return self.fileobj.write(buf) 1794 if not buf: 1795 return 1796 1797 new_rp = self.get_gzipped_rp() 1798 if self.callback: 1799 self.callback(new_rp) 1800 self.fileobj = new_rp.open("wb", compress=1) 1801 return self.fileobj.write(buf) 1802 1803 def close(self): 1804 """Close related fileobj, pass return value""" 1805 if self.closed: 1806 return None 1807 self.closed = 1 1808 if self.fileobj: 1809 return self.fileobj.close() 1810 if self.callback: 1811 self.callback(self.base_rp) 1812 self.base_rp.touch() 1813 1814 1815def setdata_local(rpath): 1816 """Set eas/acls, uid/gid, resource fork in data dictionary 1817 1818 This is a global function because it must be called locally, since 1819 these features may exist or not depending on the connection. 1820 1821 """ 1822 assert rpath.conn is Globals.local_connection 1823 reset_perms = False 1824 if (Globals.process_uid != 0 and not rpath.readable() and rpath.isowner()): 1825 reset_perms = True 1826 rpath.chmod(0o400 | rpath.getperms()) 1827 1828 rpath.data['uname'] = user_group.uid2uname(rpath.data['uid']) 1829 rpath.data['gname'] = user_group.gid2gname(rpath.data['gid']) 1830 if Globals.eas_conn: 1831 rpath.data['ea'] = ea_get(rpath) 1832 if Globals.acls_conn: 1833 rpath.data['acl'] = acl_get(rpath) 1834 if Globals.win_acls_conn: 1835 rpath.data['win_acl'] = win_acl_get(rpath) 1836 if Globals.resource_forks_conn and rpath.isreg(): 1837 rpath.get_resource_fork() 1838 if Globals.carbonfile_conn and rpath.isreg(): 1839 rpath.data['carbonfile'] = carbonfile_get(rpath) 1840 1841 if reset_perms: 1842 rpath.chmod(rpath.getperms() & ~0o400) 1843 1844 1845def carbonfile_get(rpath): 1846 """Return carbonfile value for local rpath""" 1847 # Note, after we drop support for Mac OS X 10.0 - 10.3, it will no longer 1848 # be necessary to read the finderinfo struct since it is a strict subset 1849 # of the data stored in the com.apple.FinderInfo extended attribute 1850 # introduced in 10.4. Indeed, FSpGetFInfo() is deprecated on 10.4. 1851 from Carbon.File import FSSpec 1852 from Carbon.File import FSRef 1853 import Carbon.Files 1854 import MacOS 1855 try: 1856 fsobj = FSSpec(rpath.path) 1857 finderinfo = fsobj.FSpGetFInfo() 1858 cataloginfo, d1, d2, d3 = FSRef(fsobj).FSGetCatalogInfo( 1859 Carbon.Files.kFSCatInfoCreateDate) 1860 cfile = { 1861 'creator': finderinfo.Creator, 1862 'type': finderinfo.Type, 1863 'location': finderinfo.Location, 1864 'flags': finderinfo.Flags, 1865 'createDate': cataloginfo.createDate[1] 1866 } 1867 return cfile 1868 except MacOS.Error: 1869 log.Log("Cannot read carbonfile information from %s" % (rpath.path, ), 1870 2) 1871 return None 1872 1873 1874# These functions are overwritten by the eas_acls.py module. We can't 1875# import that module directly because of circular dependency problems. 1876def acl_get(rp): 1877 assert 0 1878 1879 1880def get_blank_acl(index): 1881 assert 0 1882 1883 1884def ea_get(rp): 1885 assert 0 1886 1887 1888def get_blank_ea(index): 1889 assert 0 1890 1891 1892def win_acl_get(rp): 1893 assert 0 1894 1895 1896def write_win_acl(rp): 1897 assert 0 1898 1899 1900def get_blank_win_acl(): 1901 assert 0 1902