1""" 2Functions for working with files 3""" 4 5 6import codecs 7import contextlib 8import errno 9import logging 10import os 11import re 12import shutil 13import stat 14import subprocess 15import tempfile 16import time 17import urllib.parse 18 19import salt.modules.selinux 20import salt.utils.path 21import salt.utils.platform 22import salt.utils.stringutils 23from salt.exceptions import CommandExecutionError, FileLockError, MinionError 24from salt.utils.decorators.jinja import jinja_filter 25 26try: 27 import fcntl 28 29 HAS_FCNTL = True 30except ImportError: 31 # fcntl is not available on windows 32 HAS_FCNTL = False 33 34log = logging.getLogger(__name__) 35 36LOCAL_PROTOS = ("", "file") 37REMOTE_PROTOS = ("http", "https", "ftp", "swift", "s3") 38VALID_PROTOS = ("salt", "file") + REMOTE_PROTOS 39TEMPFILE_PREFIX = "__salt.tmp." 40 41HASHES = { 42 "sha512": 128, 43 "sha384": 96, 44 "sha256": 64, 45 "sha224": 56, 46 "sha1": 40, 47 "md5": 32, 48} 49HASHES_REVMAP = {y: x for x, y in HASHES.items()} 50 51 52def __clean_tmp(tmp): 53 """ 54 Remove temporary files 55 """ 56 try: 57 rm_rf(tmp) 58 except Exception as exc: # pylint: disable=broad-except 59 log.error( 60 "Exception while removing temp directory: %s", 61 exc, 62 exc_info_on_loglevel=logging.DEBUG, 63 ) 64 65 66def guess_archive_type(name): 67 """ 68 Guess an archive type (tar, zip, or rar) by its file extension 69 """ 70 name = name.lower() 71 for ending in ( 72 "tar", 73 "tar.gz", 74 "tgz", 75 "tar.bz2", 76 "tbz2", 77 "tbz", 78 "tar.xz", 79 "txz", 80 "tar.lzma", 81 "tlz", 82 ): 83 if name.endswith("." + ending): 84 return "tar" 85 for ending in ("zip", "rar"): 86 if name.endswith("." + ending): 87 return ending 88 return None 89 90 91def mkstemp(*args, **kwargs): 92 """ 93 Helper function which does exactly what ``tempfile.mkstemp()`` does but 94 accepts another argument, ``close_fd``, which, by default, is true and closes 95 the fd before returning the file path. Something commonly done throughout 96 Salt's code. 97 """ 98 if "prefix" not in kwargs: 99 kwargs["prefix"] = "__salt.tmp." 100 close_fd = kwargs.pop("close_fd", True) 101 fd_, f_path = tempfile.mkstemp(*args, **kwargs) 102 if close_fd is False: 103 return fd_, f_path 104 os.close(fd_) 105 del fd_ 106 return f_path 107 108 109def recursive_copy(source, dest): 110 """ 111 Recursively copy the source directory to the destination, 112 leaving files with the source does not explicitly overwrite. 113 114 (identical to cp -r on a unix machine) 115 """ 116 for root, _, files in salt.utils.path.os_walk(source): 117 path_from_source = root.replace(source, "").lstrip(os.sep) 118 target_directory = os.path.join(dest, path_from_source) 119 if not os.path.exists(target_directory): 120 os.makedirs(target_directory) 121 for name in files: 122 file_path_from_source = os.path.join(source, path_from_source, name) 123 target_path = os.path.join(target_directory, name) 124 shutil.copyfile(file_path_from_source, target_path) 125 126 127def copyfile(source, dest, backup_mode="", cachedir=""): 128 """ 129 Copy files from a source to a destination in an atomic way, and if 130 specified cache the file. 131 """ 132 if not os.path.isfile(source): 133 raise OSError("[Errno 2] No such file or directory: {}".format(source)) 134 if not os.path.isdir(os.path.dirname(dest)): 135 raise OSError("[Errno 2] No such file or directory: {}".format(dest)) 136 bname = os.path.basename(dest) 137 dname = os.path.dirname(os.path.abspath(dest)) 138 tgt = mkstemp(prefix=bname, dir=dname) 139 shutil.copyfile(source, tgt) 140 bkroot = "" 141 if cachedir: 142 bkroot = os.path.join(cachedir, "file_backup") 143 if backup_mode == "minion" or backup_mode == "both" and bkroot: 144 if os.path.exists(dest): 145 backup_minion(dest, bkroot) 146 if backup_mode == "master" or backup_mode == "both" and bkroot: 147 # TODO, backup to master 148 pass 149 # Get current file stats to they can be replicated after the new file is 150 # moved to the destination path. 151 fstat = None 152 if not salt.utils.platform.is_windows(): 153 try: 154 fstat = os.stat(dest) 155 except OSError: 156 pass 157 158 # The move could fail if the dest has xattr protections, so delete the 159 # temp file in this case 160 try: 161 shutil.move(tgt, dest) 162 except Exception: # pylint: disable=broad-except 163 __clean_tmp(tgt) 164 raise 165 166 if fstat is not None: 167 os.chown(dest, fstat.st_uid, fstat.st_gid) 168 os.chmod(dest, fstat.st_mode) 169 # If SELINUX is available run a restorecon on the file 170 rcon = salt.utils.path.which("restorecon") 171 if rcon: 172 policy = False 173 try: 174 policy = salt.modules.selinux.getenforce() 175 except (ImportError, CommandExecutionError): 176 pass 177 if policy == "Enforcing": 178 with fopen(os.devnull, "w") as dev_null: 179 cmd = [rcon, dest] 180 subprocess.call(cmd, stdout=dev_null, stderr=dev_null) 181 if os.path.isfile(tgt): 182 # The temp file failed to move 183 __clean_tmp(tgt) 184 185 186def rename(src, dst): 187 """ 188 On Windows, os.rename() will fail with a WindowsError exception if a file 189 exists at the destination path. This function checks for this error and if 190 found, it deletes the destination path first. 191 """ 192 try: 193 os.rename(src, dst) 194 except OSError as exc: 195 if exc.errno != errno.EEXIST: 196 raise 197 try: 198 os.remove(dst) 199 except OSError as exc: 200 if exc.errno != errno.ENOENT: 201 raise MinionError( 202 "Error: Unable to remove {}: {}".format(dst, exc.strerror) 203 ) 204 os.rename(src, dst) 205 206 207def process_read_exception(exc, path, ignore=None): 208 """ 209 Common code for raising exceptions when reading a file fails 210 211 The ignore argument can be an iterable of integer error codes (or a single 212 integer error code) that should be ignored. 213 """ 214 if ignore is not None: 215 if isinstance(ignore, int): 216 ignore = (ignore,) 217 else: 218 ignore = () 219 220 if exc.errno in ignore: 221 return 222 223 if exc.errno == errno.ENOENT: 224 raise CommandExecutionError("{} does not exist".format(path)) 225 elif exc.errno == errno.EACCES: 226 raise CommandExecutionError("Permission denied reading from {}".format(path)) 227 else: 228 raise CommandExecutionError( 229 "Error {} encountered reading from {}: {}".format( 230 exc.errno, path, exc.strerror 231 ) 232 ) 233 234 235@contextlib.contextmanager 236def wait_lock(path, lock_fn=None, timeout=5, sleep=0.1, time_start=None): 237 """ 238 Obtain a write lock. If one exists, wait for it to release first 239 """ 240 if not isinstance(path, str): 241 raise FileLockError("path must be a string") 242 if lock_fn is None: 243 lock_fn = path + ".w" 244 if time_start is None: 245 time_start = time.time() 246 obtained_lock = False 247 248 def _raise_error(msg, race=False): 249 """ 250 Raise a FileLockError 251 """ 252 raise FileLockError(msg, time_start=time_start) 253 254 try: 255 if os.path.exists(lock_fn) and not os.path.isfile(lock_fn): 256 _raise_error("lock_fn {} exists and is not a file".format(lock_fn)) 257 258 open_flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY 259 while time.time() - time_start < timeout: 260 try: 261 # Use os.open() to obtain filehandle so that we can force an 262 # exception if the file already exists. Concept found here: 263 # http://stackoverflow.com/a/10979569 264 fh_ = os.open(lock_fn, open_flags) 265 except OSError as exc: 266 if exc.errno != errno.EEXIST: 267 _raise_error( 268 "Error {} encountered obtaining file lock {}: {}".format( 269 exc.errno, lock_fn, exc.strerror 270 ) 271 ) 272 log.trace("Lock file %s exists, sleeping %f seconds", lock_fn, sleep) 273 time.sleep(sleep) 274 else: 275 # Write the lock file 276 with os.fdopen(fh_, "w"): 277 pass 278 # Lock successfully acquired 279 log.trace("Write lock %s obtained", lock_fn) 280 obtained_lock = True 281 # Transfer control back to the code inside the with block 282 yield 283 # Exit the loop 284 break 285 286 else: 287 _raise_error( 288 "Timeout of {} seconds exceeded waiting for lock_fn {} " 289 "to be released".format(timeout, lock_fn) 290 ) 291 292 except FileLockError: 293 raise 294 295 except Exception as exc: # pylint: disable=broad-except 296 _raise_error( 297 "Error encountered obtaining file lock {}: {}".format(lock_fn, exc) 298 ) 299 300 finally: 301 if obtained_lock: 302 os.remove(lock_fn) 303 log.trace("Write lock for %s (%s) released", path, lock_fn) 304 305 306def get_umask(): 307 """ 308 Returns the current umask 309 """ 310 ret = os.umask(0) # pylint: disable=blacklisted-function 311 os.umask(ret) # pylint: disable=blacklisted-function 312 return ret 313 314 315@contextlib.contextmanager 316def set_umask(mask): 317 """ 318 Temporarily set the umask and restore once the contextmanager exits 319 """ 320 if mask is None or salt.utils.platform.is_windows(): 321 # Don't attempt on Windows, or if no mask was passed 322 yield 323 else: 324 try: 325 orig_mask = os.umask(mask) # pylint: disable=blacklisted-function 326 yield 327 finally: 328 os.umask(orig_mask) # pylint: disable=blacklisted-function 329 330 331def fopen(*args, **kwargs): 332 """ 333 Wrapper around open() built-in to set CLOEXEC on the fd. 334 335 This flag specifies that the file descriptor should be closed when an exec 336 function is invoked; 337 338 When a file descriptor is allocated (as with open or dup), this bit is 339 initially cleared on the new file descriptor, meaning that descriptor will 340 survive into the new program after exec. 341 342 NB! We still have small race condition between open and fcntl. 343 """ 344 try: 345 # Don't permit stdin/stdout/stderr to be opened. The boolean False 346 # and True are treated by Python 3's open() as file descriptors 0 347 # and 1, respectively. 348 if args[0] in (0, 1, 2): 349 raise TypeError("{} is not a permitted file descriptor".format(args[0])) 350 except IndexError: 351 pass 352 binary = None 353 if kwargs.pop("binary", None): 354 if len(args) > 1: 355 args = list(args) 356 if "b" not in args[1]: 357 args[1] = args[1].replace("t", "b") 358 if "b" not in args[1]: 359 args[1] += "b" 360 elif kwargs.get("mode"): 361 if "b" not in kwargs["mode"]: 362 kwargs["mode"] = kwargs["mode"].replace("t", "b") 363 if "b" not in kwargs["mode"]: 364 kwargs["mode"] += "b" 365 else: 366 # the default is to read 367 kwargs["mode"] = "rb" 368 if "encoding" not in kwargs: 369 # In Python 3, if text mode is used and the encoding 370 # is not specified, set the encoding to 'utf-8'. 371 binary = False 372 if len(args) > 1: 373 args = list(args) 374 if "b" in args[1]: 375 binary = True 376 if kwargs.get("mode", None): 377 if "b" in kwargs["mode"]: 378 binary = True 379 if not binary: 380 kwargs["encoding"] = __salt_system_encoding__ 381 382 if not binary and not kwargs.get("newline", None): 383 kwargs["newline"] = "" 384 385 f_handle = open(*args, **kwargs) # pylint: disable=resource-leakage 386 387 if is_fcntl_available(): 388 # modify the file descriptor on systems with fcntl 389 # unix and unix-like systems only 390 try: 391 FD_CLOEXEC = fcntl.FD_CLOEXEC # pylint: disable=C0103 392 except AttributeError: 393 FD_CLOEXEC = 1 # pylint: disable=C0103 394 old_flags = fcntl.fcntl(f_handle.fileno(), fcntl.F_GETFD) 395 fcntl.fcntl(f_handle.fileno(), fcntl.F_SETFD, old_flags | FD_CLOEXEC) 396 397 return f_handle 398 399 400@contextlib.contextmanager 401def flopen(*args, **kwargs): 402 """ 403 Shortcut for fopen with lock and context manager. 404 """ 405 filename, args = args[0], args[1:] 406 writing = "wa" 407 with fopen(filename, *args, **kwargs) as f_handle: 408 try: 409 if is_fcntl_available(check_sunos=True): 410 lock_type = fcntl.LOCK_SH 411 if args and any([write in args[0] for write in writing]): 412 lock_type = fcntl.LOCK_EX 413 fcntl.flock(f_handle.fileno(), lock_type) 414 yield f_handle 415 finally: 416 if is_fcntl_available(check_sunos=True): 417 fcntl.flock(f_handle.fileno(), fcntl.LOCK_UN) 418 419 420@contextlib.contextmanager 421def fpopen(*args, **kwargs): 422 """ 423 Shortcut for fopen with extra uid, gid, and mode options. 424 425 Supported optional Keyword Arguments: 426 427 mode 428 Explicit mode to set. Mode is anything os.chmod would accept 429 as input for mode. Works only on unix/unix-like systems. 430 431 uid 432 The uid to set, if not set, or it is None or -1 no changes are 433 made. Same applies if the path is already owned by this uid. 434 Must be int. Works only on unix/unix-like systems. 435 436 gid 437 The gid to set, if not set, or it is None or -1 no changes are 438 made. Same applies if the path is already owned by this gid. 439 Must be int. Works only on unix/unix-like systems. 440 441 """ 442 # Remove uid, gid and mode from kwargs if present 443 uid = kwargs.pop("uid", -1) # -1 means no change to current uid 444 gid = kwargs.pop("gid", -1) # -1 means no change to current gid 445 mode = kwargs.pop("mode", None) 446 with fopen(*args, **kwargs) as f_handle: 447 path = args[0] 448 d_stat = os.stat(path) 449 450 if hasattr(os, "chown"): 451 # if uid and gid are both -1 then go ahead with 452 # no changes at all 453 if (d_stat.st_uid != uid or d_stat.st_gid != gid) and [ 454 i for i in (uid, gid) if i != -1 455 ]: 456 os.chown(path, uid, gid) 457 458 if mode is not None: 459 mode_part = stat.S_IMODE(d_stat.st_mode) 460 if mode_part != mode: 461 os.chmod(path, (d_stat.st_mode ^ mode_part) | mode) 462 463 yield f_handle 464 465 466def safe_walk(top, topdown=True, onerror=None, followlinks=True, _seen=None): 467 """ 468 A clone of the python os.walk function with some checks for recursive 469 symlinks. Unlike os.walk this follows symlinks by default. 470 """ 471 if _seen is None: 472 _seen = set() 473 474 # We may not have read permission for top, in which case we can't 475 # get a list of the files the directory contains. os.path.walk 476 # always suppressed the exception then, rather than blow up for a 477 # minor reason when (say) a thousand readable directories are still 478 # left to visit. That logic is copied here. 479 try: 480 # Note that listdir and error are globals in this module due 481 # to earlier import-*. 482 names = os.listdir(top) 483 except os.error as err: 484 if onerror is not None: 485 onerror(err) 486 return 487 488 if followlinks: 489 status = os.stat(top) 490 # st_ino is always 0 on some filesystems (FAT, NTFS); ignore them 491 if status.st_ino != 0: 492 node = (status.st_dev, status.st_ino) 493 if node in _seen: 494 return 495 _seen.add(node) 496 497 dirs, nondirs = [], [] 498 for name in names: 499 full_path = os.path.join(top, name) 500 if os.path.isdir(full_path): 501 dirs.append(name) 502 else: 503 nondirs.append(name) 504 505 if topdown: 506 yield top, dirs, nondirs 507 for name in dirs: 508 new_path = os.path.join(top, name) 509 if followlinks or not os.path.islink(new_path): 510 yield from safe_walk(new_path, topdown, onerror, followlinks, _seen) 511 if not topdown: 512 yield top, dirs, nondirs 513 514 515def safe_rm(tgt): 516 """ 517 Safely remove a file 518 """ 519 try: 520 os.remove(tgt) 521 except OSError: 522 pass 523 524 525def rm_rf(path): 526 """ 527 Platform-independent recursive delete. Includes code from 528 http://stackoverflow.com/a/2656405 529 """ 530 531 def _onerror(func, path, exc_info): 532 """ 533 Error handler for `shutil.rmtree`. 534 535 If the error is due to an access error (read only file) 536 it attempts to add write permission and then retries. 537 538 If the error is for another reason it re-raises the error. 539 540 Usage : `shutil.rmtree(path, onerror=onerror)` 541 """ 542 if salt.utils.platform.is_windows() and not os.access(path, os.W_OK): 543 # Is the error an access error ? 544 os.chmod(path, stat.S_IWUSR) 545 func(path) 546 else: 547 raise # pylint: disable=E0704 548 549 if os.path.islink(path) or not os.path.isdir(path): 550 os.remove(path) 551 else: 552 if salt.utils.platform.is_windows(): 553 try: 554 path = salt.utils.stringutils.to_unicode(path) 555 except TypeError: 556 pass 557 shutil.rmtree(path, onerror=_onerror) 558 559 560@jinja_filter("is_empty") 561def is_empty(filename): 562 """ 563 Is a file empty? 564 """ 565 try: 566 return os.stat(filename).st_size == 0 567 except OSError: 568 # Non-existent file or permission denied to the parent dir 569 return False 570 571 572def is_fcntl_available(check_sunos=False): 573 """ 574 Simple function to check if the ``fcntl`` module is available or not. 575 576 If ``check_sunos`` is passed as ``True`` an additional check to see if host is 577 SunOS is also made. For additional information see: http://goo.gl/159FF8 578 """ 579 if check_sunos and salt.utils.platform.is_sunos(): 580 return False 581 return HAS_FCNTL 582 583 584def safe_filename_leaf(file_basename): 585 """ 586 Input the basename of a file, without the directory tree, and returns a safe name to use 587 i.e. only the required characters are converted by urllib.parse.quote 588 If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. 589 For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible 590 windows is \\ / : * ? " < > | posix is / 591 592 .. versionadded:: 2017.7.2 593 594 :codeauthor: Damon Atkins <https://github.com/damon-atkins> 595 """ 596 597 def _replace(re_obj): 598 return urllib.parse.quote(re_obj.group(0), safe="") 599 600 if not isinstance(file_basename, str): 601 # the following string is not prefixed with u 602 return re.sub( 603 '[\\\\:/*?"<>|]', 604 _replace, 605 str(file_basename, "utf8").encode("ascii", "backslashreplace"), 606 ) 607 # the following string is prefixed with u 608 return re.sub('[\\\\:/*?"<>|]', _replace, file_basename, flags=re.UNICODE) 609 610 611def safe_filepath(file_path_name, dir_sep=None): 612 """ 613 Input the full path and filename, splits on directory separator and calls safe_filename_leaf for 614 each part of the path. dir_sep allows coder to force a directory separate to a particular character 615 616 .. versionadded:: 2017.7.2 617 618 :codeauthor: Damon Atkins <https://github.com/damon-atkins> 619 """ 620 if not dir_sep: 621 dir_sep = os.sep 622 # Normally if file_path_name or dir_sep is Unicode then the output will be Unicode 623 # This code ensure the output type is the same as file_path_name 624 if not isinstance(file_path_name, str) and isinstance(dir_sep, str): 625 dir_sep = dir_sep.encode("ascii") # This should not be executed under PY3 626 # splitdrive only set drive on windows platform 627 (drive, path) = os.path.splitdrive(file_path_name) 628 path = dir_sep.join( 629 [safe_filename_leaf(file_section) for file_section in path.rsplit(dir_sep)] 630 ) 631 if drive: 632 path = dir_sep.join([drive, path]) 633 return path 634 635 636@jinja_filter("is_text_file") 637def is_text(fp_, blocksize=512): 638 """ 639 Uses heuristics to guess whether the given file is text or binary, 640 by reading a single block of bytes from the file. 641 If more than 30% of the chars in the block are non-text, or there 642 are NUL ('\x00') bytes in the block, assume this is a binary file. 643 """ 644 int2byte = lambda x: bytes((x,)) 645 text_characters = b"".join(int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b" 646 try: 647 block = fp_.read(blocksize) 648 except AttributeError: 649 # This wasn't an open filehandle, so treat it as a file path and try to 650 # open the file 651 try: 652 with fopen(fp_, "rb") as fp2_: 653 block = fp2_.read(blocksize) 654 except OSError: 655 # Unable to open file, bail out and return false 656 return False 657 if b"\x00" in block: 658 # Files with null bytes are binary 659 return False 660 elif not block: 661 # An empty file is considered a valid text file 662 return True 663 try: 664 block.decode("utf-8") 665 return True 666 except UnicodeDecodeError: 667 pass 668 669 nontext = block.translate(None, text_characters) 670 return float(len(nontext)) / len(block) <= 0.30 671 672 673@jinja_filter("is_bin_file") 674def is_binary(path): 675 """ 676 Detects if the file is a binary, returns bool. Returns True if the file is 677 a bin, False if the file is not and None if the file is not available. 678 """ 679 if not os.path.isfile(path): 680 return False 681 try: 682 with fopen(path, "rb") as fp_: 683 try: 684 data = fp_.read(2048) 685 data = data.decode(__salt_system_encoding__) 686 return salt.utils.stringutils.is_binary(data) 687 except UnicodeDecodeError: 688 return True 689 except os.error: 690 return False 691 692 693def remove(path): 694 """ 695 Runs os.remove(path) and suppresses the OSError if the file doesn't exist 696 """ 697 try: 698 os.remove(path) 699 except OSError as exc: 700 if exc.errno != errno.ENOENT: 701 raise 702 703 704@jinja_filter("list_files") 705def list_files(directory): 706 """ 707 Return a list of all files found under directory (and its subdirectories) 708 """ 709 ret = set() 710 ret.add(directory) 711 for root, dirs, files in safe_walk(directory): 712 for name in files: 713 ret.add(os.path.join(root, name)) 714 for name in dirs: 715 ret.add(os.path.join(root, name)) 716 717 return list(ret) 718 719 720def st_mode_to_octal(mode): 721 """ 722 Convert the st_mode value from a stat(2) call (as returned from os.stat()) 723 to an octal mode. 724 """ 725 try: 726 return oct(mode)[-4:] 727 except (TypeError, IndexError): 728 return "" 729 730 731def normalize_mode(mode): 732 """ 733 Return a mode value, normalized to a string and containing a leading zero 734 if it does not have one. 735 736 Allow "keep" as a valid mode (used by file state/module to preserve mode 737 from the Salt fileserver in file states). 738 """ 739 if mode is None: 740 return None 741 if not isinstance(mode, str): 742 mode = str(mode) 743 mode = mode.replace("0o", "0") 744 # Strip any quotes any initial zeroes, then though zero-pad it up to 4. 745 # This ensures that somethign like '00644' is normalized to '0644' 746 return mode.strip('"').strip("'").lstrip("0").zfill(4) 747 748 749def human_size_to_bytes(human_size): 750 """ 751 Convert human-readable units to bytes 752 """ 753 size_exp_map = {"K": 1, "M": 2, "G": 3, "T": 4, "P": 5} 754 human_size_str = str(human_size) 755 match = re.match(r"^(\d+)([KMGTP])?$", human_size_str) 756 if not match: 757 raise ValueError( 758 "Size must be all digits, with an optional unit type (K, M, G, T, or P)" 759 ) 760 size_num = int(match.group(1)) 761 unit_multiplier = 1024 ** size_exp_map.get(match.group(2), 0) 762 return size_num * unit_multiplier 763 764 765def backup_minion(path, bkroot): 766 """ 767 Backup a file on the minion 768 """ 769 dname, bname = os.path.split(path) 770 if salt.utils.platform.is_windows(): 771 src_dir = dname.replace(":", "_") 772 else: 773 src_dir = dname[1:] 774 if not salt.utils.platform.is_windows(): 775 fstat = os.stat(path) 776 msecs = str(int(time.time() * 1000000))[-6:] 777 if salt.utils.platform.is_windows(): 778 # ':' is an illegal filesystem path character on Windows 779 stamp = time.strftime("%a_%b_%d_%H-%M-%S_%Y") 780 else: 781 stamp = time.strftime("%a_%b_%d_%H:%M:%S_%Y") 782 stamp = "{}{}_{}".format(stamp[:-4], msecs, stamp[-4:]) 783 bkpath = os.path.join(bkroot, src_dir, "{}_{}".format(bname, stamp)) 784 if not os.path.isdir(os.path.dirname(bkpath)): 785 os.makedirs(os.path.dirname(bkpath)) 786 shutil.copyfile(path, bkpath) 787 if not salt.utils.platform.is_windows(): 788 os.chown(bkpath, fstat.st_uid, fstat.st_gid) 789 os.chmod(bkpath, fstat.st_mode) 790 791 792def case_insensitive_filesystem(path=None): 793 """ 794 Detect case insensitivity on a system. 795 796 Returns: 797 bool: Flag to indicate case insensitivity 798 799 .. versionadded:: 3004 800 801 """ 802 with tempfile.NamedTemporaryFile(prefix="TmP", dir=path, delete=True) as tmp_file: 803 return os.path.exists(tmp_file.name.lower()) 804 805 806def get_encoding(path): 807 """ 808 Detect a file's encoding using the following: 809 - Check for Byte Order Marks (BOM) 810 - Check for UTF-8 Markers 811 - Check System Encoding 812 - Check for ascii 813 814 Args: 815 816 path (str): The path to the file to check 817 818 Returns: 819 str: The encoding of the file 820 821 Raises: 822 CommandExecutionError: If the encoding cannot be detected 823 """ 824 825 def check_ascii(_data): 826 # If all characters can be decoded to ASCII, then it's ASCII 827 try: 828 _data.decode("ASCII") 829 log.debug("Found ASCII") 830 except UnicodeDecodeError: 831 return False 832 else: 833 return True 834 835 def check_bom(_data): 836 # Supported Python Codecs 837 # https://docs.python.org/2/library/codecs.html 838 # https://docs.python.org/3/library/codecs.html 839 boms = [ 840 ("UTF-32-BE", salt.utils.stringutils.to_bytes(codecs.BOM_UTF32_BE)), 841 ("UTF-32-LE", salt.utils.stringutils.to_bytes(codecs.BOM_UTF32_LE)), 842 ("UTF-16-BE", salt.utils.stringutils.to_bytes(codecs.BOM_UTF16_BE)), 843 ("UTF-16-LE", salt.utils.stringutils.to_bytes(codecs.BOM_UTF16_LE)), 844 ("UTF-8", salt.utils.stringutils.to_bytes(codecs.BOM_UTF8)), 845 ("UTF-7", salt.utils.stringutils.to_bytes("\x2b\x2f\x76\x38\x2D")), 846 ("UTF-7", salt.utils.stringutils.to_bytes("\x2b\x2f\x76\x38")), 847 ("UTF-7", salt.utils.stringutils.to_bytes("\x2b\x2f\x76\x39")), 848 ("UTF-7", salt.utils.stringutils.to_bytes("\x2b\x2f\x76\x2b")), 849 ("UTF-7", salt.utils.stringutils.to_bytes("\x2b\x2f\x76\x2f")), 850 ] 851 for _encoding, bom in boms: 852 if _data.startswith(bom): 853 log.debug("Found BOM for %s", _encoding) 854 return _encoding 855 return False 856 857 def check_utf8_markers(_data): 858 try: 859 decoded = _data.decode("UTF-8") 860 except UnicodeDecodeError: 861 return False 862 else: 863 return True 864 865 def check_system_encoding(_data): 866 try: 867 _data.decode(__salt_system_encoding__) 868 except UnicodeDecodeError: 869 return False 870 else: 871 return True 872 873 if not os.path.isfile(path): 874 raise CommandExecutionError("Not a file") 875 try: 876 with fopen(path, "rb") as fp_: 877 data = fp_.read(2048) 878 except os.error: 879 raise CommandExecutionError("Failed to open file") 880 881 # Check for Unicode BOM 882 encoding = check_bom(data) 883 if encoding: 884 return encoding 885 886 # Check for UTF-8 markers 887 if check_utf8_markers(data): 888 return "UTF-8" 889 890 # Check system encoding 891 if check_system_encoding(data): 892 return __salt_system_encoding__ 893 894 # Check for ASCII first 895 if check_ascii(data): 896 return "ASCII" 897 898 raise CommandExecutionError("Could not detect file encoding") 899