1""" 2Classes that manage file clients 3""" 4import contextlib 5import errno 6import ftplib # nosec 7import http.server 8import logging 9import os 10import shutil 11import string 12import urllib.error 13import urllib.parse 14 15import salt.client 16import salt.crypt 17import salt.fileserver 18import salt.loader 19import salt.payload 20import salt.transport.client 21import salt.utils.atomicfile 22import salt.utils.data 23import salt.utils.files 24import salt.utils.gzip_util 25import salt.utils.hashutils 26import salt.utils.http 27import salt.utils.path 28import salt.utils.platform 29import salt.utils.stringutils 30import salt.utils.templates 31import salt.utils.url 32import salt.utils.verify 33import salt.utils.versions 34from salt.exceptions import CommandExecutionError, MinionError 35from salt.ext.tornado.httputil import ( 36 HTTPHeaders, 37 HTTPInputError, 38 parse_response_start_line, 39) 40from salt.utils.openstack.swift import SaltSwift 41 42log = logging.getLogger(__name__) 43MAX_FILENAME_LENGTH = 255 44 45 46def get_file_client(opts, pillar=False): 47 """ 48 Read in the ``file_client`` option and return the correct type of file 49 server 50 """ 51 client = opts.get("file_client", "remote") 52 if pillar and client == "local": 53 client = "pillar" 54 return {"remote": RemoteClient, "local": FSClient, "pillar": PillarClient}.get( 55 client, RemoteClient 56 )(opts) 57 58 59def decode_dict_keys_to_str(src): 60 """ 61 Convert top level keys from bytes to strings if possible. 62 This is necessary because Python 3 makes a distinction 63 between these types. 64 """ 65 if not isinstance(src, dict): 66 return src 67 68 output = {} 69 for key, val in src.items(): 70 if isinstance(key, bytes): 71 try: 72 key = key.decode() 73 except UnicodeError: 74 pass 75 output[key] = val 76 return output 77 78 79class Client: 80 """ 81 Base class for Salt file interactions 82 """ 83 84 def __init__(self, opts): 85 self.opts = opts 86 self.utils = salt.loader.utils(self.opts) 87 88 # Add __setstate__ and __getstate__ so that the object may be 89 # deep copied. It normally can't be deep copied because its 90 # constructor requires an 'opts' parameter. 91 # The TCP transport needs to be able to deep copy this class 92 # due to 'salt.utils.context.ContextDict.clone'. 93 def __setstate__(self, state): 94 # This will polymorphically call __init__ 95 # in the derived class. 96 self.__init__(state["opts"]) 97 98 def __getstate__(self): 99 return {"opts": self.opts} 100 101 def _check_proto(self, path): 102 """ 103 Make sure that this path is intended for the salt master and trim it 104 """ 105 if not path.startswith("salt://"): 106 raise MinionError("Unsupported path: {}".format(path)) 107 file_path, saltenv = salt.utils.url.parse(path) 108 return file_path 109 110 def _file_local_list(self, dest): 111 """ 112 Helper util to return a list of files in a directory 113 """ 114 if os.path.isdir(dest): 115 destdir = dest 116 else: 117 destdir = os.path.dirname(dest) 118 119 filelist = set() 120 121 for root, dirs, files in salt.utils.path.os_walk(destdir, followlinks=True): 122 for name in files: 123 path = os.path.join(root, name) 124 filelist.add(path) 125 126 return filelist 127 128 @contextlib.contextmanager 129 def _cache_loc(self, path, saltenv="base", cachedir=None): 130 """ 131 Return the local location to cache the file, cache dirs will be made 132 """ 133 cachedir = self.get_cachedir(cachedir) 134 dest = salt.utils.path.join(cachedir, "files", saltenv, path) 135 destdir = os.path.dirname(dest) 136 with salt.utils.files.set_umask(0o077): 137 # remove destdir if it is a regular file to avoid an OSError when 138 # running os.makedirs below 139 if os.path.isfile(destdir): 140 os.remove(destdir) 141 142 # ensure destdir exists 143 try: 144 os.makedirs(destdir) 145 except OSError as exc: 146 if exc.errno != errno.EEXIST: # ignore if it was there already 147 raise 148 149 yield dest 150 151 def get_cachedir(self, cachedir=None): 152 if cachedir is None: 153 cachedir = self.opts["cachedir"] 154 elif not os.path.isabs(cachedir): 155 cachedir = os.path.join(self.opts["cachedir"], cachedir) 156 return cachedir 157 158 def get_file( 159 self, path, dest="", makedirs=False, saltenv="base", gzip=None, cachedir=None 160 ): 161 """ 162 Copies a file from the local files or master depending on 163 implementation 164 """ 165 raise NotImplementedError 166 167 def file_list_emptydirs(self, saltenv="base", prefix=""): 168 """ 169 List the empty dirs 170 """ 171 raise NotImplementedError 172 173 def cache_file( 174 self, path, saltenv="base", cachedir=None, source_hash=None, verify_ssl=True 175 ): 176 """ 177 Pull a file down from the file server and store it in the minion 178 file cache 179 """ 180 return self.get_url( 181 path, 182 "", 183 True, 184 saltenv, 185 cachedir=cachedir, 186 source_hash=source_hash, 187 verify_ssl=verify_ssl, 188 ) 189 190 def cache_files(self, paths, saltenv="base", cachedir=None): 191 """ 192 Download a list of files stored on the master and put them in the 193 minion file cache 194 """ 195 ret = [] 196 if isinstance(paths, str): 197 paths = paths.split(",") 198 for path in paths: 199 ret.append(self.cache_file(path, saltenv, cachedir=cachedir)) 200 return ret 201 202 def cache_master(self, saltenv="base", cachedir=None): 203 """ 204 Download and cache all files on a master in a specified environment 205 """ 206 ret = [] 207 for path in self.file_list(saltenv): 208 ret.append( 209 self.cache_file(salt.utils.url.create(path), saltenv, cachedir=cachedir) 210 ) 211 return ret 212 213 def cache_dir( 214 self, 215 path, 216 saltenv="base", 217 include_empty=False, 218 include_pat=None, 219 exclude_pat=None, 220 cachedir=None, 221 ): 222 """ 223 Download all of the files in a subdir of the master 224 """ 225 ret = [] 226 227 path = self._check_proto(salt.utils.data.decode(path)) 228 # We want to make sure files start with this *directory*, use 229 # '/' explicitly because the master (that's generating the 230 # list of files) only runs on POSIX 231 if not path.endswith("/"): 232 path = path + "/" 233 234 log.info("Caching directory '%s' for environment '%s'", path, saltenv) 235 # go through the list of all files finding ones that are in 236 # the target directory and caching them 237 for fn_ in self.file_list(saltenv): 238 fn_ = salt.utils.data.decode(fn_) 239 if fn_.strip() and fn_.startswith(path): 240 if salt.utils.stringutils.check_include_exclude( 241 fn_, include_pat, exclude_pat 242 ): 243 fn_ = self.cache_file( 244 salt.utils.url.create(fn_), saltenv, cachedir=cachedir 245 ) 246 if fn_: 247 ret.append(fn_) 248 249 if include_empty: 250 # Break up the path into a list containing the bottom-level 251 # directory (the one being recursively copied) and the directories 252 # preceding it 253 # separated = string.rsplit(path, '/', 1) 254 # if len(separated) != 2: 255 # # No slashes in path. (So all files in saltenv will be copied) 256 # prefix = '' 257 # else: 258 # prefix = separated[0] 259 cachedir = self.get_cachedir(cachedir) 260 261 dest = salt.utils.path.join(cachedir, "files", saltenv) 262 for fn_ in self.file_list_emptydirs(saltenv): 263 fn_ = salt.utils.data.decode(fn_) 264 if fn_.startswith(path): 265 minion_dir = "{}/{}".format(dest, fn_) 266 if not os.path.isdir(minion_dir): 267 os.makedirs(minion_dir) 268 ret.append(minion_dir) 269 return ret 270 271 def cache_local_file(self, path, **kwargs): 272 """ 273 Cache a local file on the minion in the localfiles cache 274 """ 275 dest = os.path.join(self.opts["cachedir"], "localfiles", path.lstrip("/")) 276 destdir = os.path.dirname(dest) 277 278 if not os.path.isdir(destdir): 279 os.makedirs(destdir) 280 281 shutil.copyfile(path, dest) 282 return dest 283 284 def file_local_list(self, saltenv="base"): 285 """ 286 List files in the local minion files and localfiles caches 287 """ 288 filesdest = os.path.join(self.opts["cachedir"], "files", saltenv) 289 localfilesdest = os.path.join(self.opts["cachedir"], "localfiles") 290 291 fdest = self._file_local_list(filesdest) 292 ldest = self._file_local_list(localfilesdest) 293 return sorted(fdest.union(ldest)) 294 295 def file_list(self, saltenv="base", prefix=""): 296 """ 297 This function must be overwritten 298 """ 299 return [] 300 301 def dir_list(self, saltenv="base", prefix=""): 302 """ 303 This function must be overwritten 304 """ 305 return [] 306 307 def symlink_list(self, saltenv="base", prefix=""): 308 """ 309 This function must be overwritten 310 """ 311 return {} 312 313 def is_cached(self, path, saltenv="base", cachedir=None): 314 """ 315 Returns the full path to a file if it is cached locally on the minion 316 otherwise returns a blank string 317 """ 318 if path.startswith("salt://"): 319 path, senv = salt.utils.url.parse(path) 320 if senv: 321 saltenv = senv 322 323 escaped = True if salt.utils.url.is_escaped(path) else False 324 325 # also strip escape character '|' 326 localsfilesdest = os.path.join( 327 self.opts["cachedir"], "localfiles", path.lstrip("|/") 328 ) 329 filesdest = os.path.join( 330 self.opts["cachedir"], "files", saltenv, path.lstrip("|/") 331 ) 332 extrndest = self._extrn_path(path, saltenv, cachedir=cachedir) 333 334 if os.path.exists(filesdest): 335 return salt.utils.url.escape(filesdest) if escaped else filesdest 336 elif os.path.exists(localsfilesdest): 337 return ( 338 salt.utils.url.escape(localsfilesdest) if escaped else localsfilesdest 339 ) 340 elif os.path.exists(extrndest): 341 return extrndest 342 343 return "" 344 345 def cache_dest(self, url, saltenv="base", cachedir=None): 346 """ 347 Return the expected cache location for the specified URL and 348 environment. 349 """ 350 proto = urllib.parse.urlparse(url).scheme 351 352 if proto == "": 353 # Local file path 354 return url 355 356 if proto == "salt": 357 url, senv = salt.utils.url.parse(url) 358 if senv: 359 saltenv = senv 360 return salt.utils.path.join( 361 self.opts["cachedir"], "files", saltenv, url.lstrip("|/") 362 ) 363 364 return self._extrn_path(url, saltenv, cachedir=cachedir) 365 366 def list_states(self, saltenv): 367 """ 368 Return a list of all available sls modules on the master for a given 369 environment 370 """ 371 states = set() 372 for path in self.file_list(saltenv): 373 if salt.utils.platform.is_windows(): 374 path = path.replace("\\", "/") 375 if path.endswith(".sls"): 376 # is an sls module! 377 if path.endswith("/init.sls"): 378 states.add(path.replace("/", ".")[:-9]) 379 else: 380 states.add(path.replace("/", ".")[:-4]) 381 return sorted(states) 382 383 def get_state(self, sls, saltenv, cachedir=None): 384 """ 385 Get a state file from the master and store it in the local minion 386 cache; return the location of the file 387 """ 388 if "." in sls: 389 sls = sls.replace(".", "/") 390 sls_url = salt.utils.url.create(sls + ".sls") 391 init_url = salt.utils.url.create(sls + "/init.sls") 392 for path in [sls_url, init_url]: 393 dest = self.cache_file(path, saltenv, cachedir=cachedir) 394 if dest: 395 return {"source": path, "dest": dest} 396 return {} 397 398 def get_dir(self, path, dest="", saltenv="base", gzip=None, cachedir=None): 399 """ 400 Get a directory recursively from the salt-master 401 """ 402 ret = [] 403 # Strip trailing slash 404 path = self._check_proto(path).rstrip("/") 405 # Break up the path into a list containing the bottom-level directory 406 # (the one being recursively copied) and the directories preceding it 407 separated = path.rsplit("/", 1) 408 if len(separated) != 2: 409 # No slashes in path. (This means all files in saltenv will be 410 # copied) 411 prefix = "" 412 else: 413 prefix = separated[0] 414 415 # Copy files from master 416 for fn_ in self.file_list(saltenv, prefix=path): 417 # Prevent files in "salt://foobar/" (or salt://foo.sh) from 418 # matching a path of "salt://foo" 419 try: 420 if fn_[len(path)] != "/": 421 continue 422 except IndexError: 423 continue 424 # Remove the leading directories from path to derive 425 # the relative path on the minion. 426 minion_relpath = fn_[len(prefix) :].lstrip("/") 427 ret.append( 428 self.get_file( 429 salt.utils.url.create(fn_), 430 "{}/{}".format(dest, minion_relpath), 431 True, 432 saltenv, 433 gzip, 434 ) 435 ) 436 # Replicate empty dirs from master 437 try: 438 for fn_ in self.file_list_emptydirs(saltenv, prefix=path): 439 # Prevent an empty dir "salt://foobar/" from matching a path of 440 # "salt://foo" 441 try: 442 if fn_[len(path)] != "/": 443 continue 444 except IndexError: 445 continue 446 # Remove the leading directories from path to derive 447 # the relative path on the minion. 448 minion_relpath = fn_[len(prefix) :].lstrip("/") 449 minion_mkdir = "{}/{}".format(dest, minion_relpath) 450 if not os.path.isdir(minion_mkdir): 451 os.makedirs(minion_mkdir) 452 ret.append(minion_mkdir) 453 except TypeError: 454 pass 455 ret.sort() 456 return ret 457 458 def get_url( 459 self, 460 url, 461 dest, 462 makedirs=False, 463 saltenv="base", 464 no_cache=False, 465 cachedir=None, 466 source_hash=None, 467 verify_ssl=True, 468 ): 469 """ 470 Get a single file from a URL. 471 """ 472 url_data = urllib.parse.urlparse(url) 473 url_scheme = url_data.scheme 474 url_path = os.path.join(url_data.netloc, url_data.path).rstrip(os.sep) 475 476 # If dest is a directory, rewrite dest with filename 477 if dest is not None and (os.path.isdir(dest) or dest.endswith(("/", "\\"))): 478 if ( 479 url_data.query 480 or len(url_data.path) > 1 481 and not url_data.path.endswith("/") 482 ): 483 strpath = url.split("/")[-1] 484 else: 485 strpath = "index.html" 486 487 if salt.utils.platform.is_windows(): 488 strpath = salt.utils.path.sanitize_win_path(strpath) 489 490 dest = os.path.join(dest, strpath) 491 492 if url_scheme and url_scheme.lower() in string.ascii_lowercase: 493 url_path = ":".join((url_scheme, url_path)) 494 url_scheme = "file" 495 496 if url_scheme in ("file", ""): 497 # Local filesystem 498 if not os.path.isabs(url_path): 499 raise CommandExecutionError( 500 "Path '{}' is not absolute".format(url_path) 501 ) 502 if dest is None: 503 with salt.utils.files.fopen(url_path, "rb") as fp_: 504 data = fp_.read() 505 return data 506 return url_path 507 508 if url_scheme == "salt": 509 result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir) 510 if result and dest is None: 511 with salt.utils.files.fopen(result, "rb") as fp_: 512 data = fp_.read() 513 return data 514 return result 515 516 if dest: 517 destdir = os.path.dirname(dest) 518 if not os.path.isdir(destdir): 519 if makedirs: 520 os.makedirs(destdir) 521 else: 522 return "" 523 elif not no_cache: 524 dest = self._extrn_path(url, saltenv, cachedir=cachedir) 525 if source_hash is not None: 526 try: 527 source_hash = source_hash.split("=")[-1] 528 form = salt.utils.files.HASHES_REVMAP[len(source_hash)] 529 if salt.utils.hashutils.get_hash(dest, form) == source_hash: 530 log.debug( 531 "Cached copy of %s (%s) matches source_hash %s, " 532 "skipping download", 533 url, 534 dest, 535 source_hash, 536 ) 537 return dest 538 except (AttributeError, KeyError, OSError): 539 pass 540 destdir = os.path.dirname(dest) 541 if not os.path.isdir(destdir): 542 os.makedirs(destdir) 543 544 if url_data.scheme == "s3": 545 try: 546 547 def s3_opt(key, default=None): 548 """ 549 Get value of s3.<key> from Minion config or from Pillar 550 """ 551 if "s3." + key in self.opts: 552 return self.opts["s3." + key] 553 try: 554 return self.opts["pillar"]["s3"][key] 555 except (KeyError, TypeError): 556 return default 557 558 self.utils["s3.query"]( 559 method="GET", 560 bucket=url_data.netloc, 561 path=url_data.path[1:], 562 return_bin=False, 563 local_file=dest, 564 action=None, 565 key=s3_opt("key"), 566 keyid=s3_opt("keyid"), 567 service_url=s3_opt("service_url"), 568 verify_ssl=s3_opt("verify_ssl", True), 569 location=s3_opt("location"), 570 path_style=s3_opt("path_style", False), 571 https_enable=s3_opt("https_enable", True), 572 ) 573 return dest 574 except Exception as exc: # pylint: disable=broad-except 575 raise MinionError( 576 "Could not fetch from {}. Exception: {}".format(url, exc) 577 ) 578 if url_data.scheme == "ftp": 579 try: 580 ftp = ftplib.FTP() # nosec 581 ftp_port = url_data.port 582 if not ftp_port: 583 ftp_port = 21 584 ftp.connect(url_data.hostname, ftp_port) 585 ftp.login(url_data.username, url_data.password) 586 remote_file_path = url_data.path.lstrip("/") 587 with salt.utils.files.fopen(dest, "wb") as fp_: 588 ftp.retrbinary("RETR {}".format(remote_file_path), fp_.write) 589 ftp.quit() 590 return dest 591 except Exception as exc: # pylint: disable=broad-except 592 raise MinionError( 593 "Could not retrieve {} from FTP server. Exception: {}".format( 594 url, exc 595 ) 596 ) 597 598 if url_data.scheme == "swift": 599 try: 600 601 def swift_opt(key, default): 602 """ 603 Get value of <key> from Minion config or from Pillar 604 """ 605 if key in self.opts: 606 return self.opts[key] 607 try: 608 return self.opts["pillar"][key] 609 except (KeyError, TypeError): 610 return default 611 612 swift_conn = SaltSwift( 613 swift_opt("keystone.user", None), 614 swift_opt("keystone.tenant", None), 615 swift_opt("keystone.auth_url", None), 616 swift_opt("keystone.password", None), 617 ) 618 619 swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) 620 return dest 621 except Exception: # pylint: disable=broad-except 622 raise MinionError("Could not fetch from {}".format(url)) 623 624 get_kwargs = {} 625 if url_data.username is not None and url_data.scheme in ("http", "https"): 626 netloc = url_data.netloc 627 at_sign_pos = netloc.rfind("@") 628 if at_sign_pos != -1: 629 netloc = netloc[at_sign_pos + 1 :] 630 fixed_url = urllib.parse.urlunparse( 631 ( 632 url_data.scheme, 633 netloc, 634 url_data.path, 635 url_data.params, 636 url_data.query, 637 url_data.fragment, 638 ) 639 ) 640 get_kwargs["auth"] = (url_data.username, url_data.password) 641 else: 642 fixed_url = url 643 644 destfp = None 645 try: 646 # Tornado calls streaming_callback on redirect response bodies. 647 # But we need streaming to support fetching large files (> RAM 648 # avail). Here we are working around this by disabling recording 649 # the body for redirections. The issue is fixed in Tornado 4.3.0 650 # so on_header callback could be removed when we'll deprecate 651 # Tornado<4.3.0. See #27093 and #30431 for details. 652 653 # Use list here to make it writable inside the on_header callback. 654 # Simple bool doesn't work here: on_header creates a new local 655 # variable instead. This could be avoided in Py3 with 'nonlocal' 656 # statement. There is no Py2 alternative for this. 657 # 658 # write_body[0] is used by the on_chunk callback to tell it whether 659 # or not we need to write the body of the request to disk. For 660 # 30x redirects we set this to False because we don't want to 661 # write the contents to disk, as we will need to wait until we 662 # get to the redirected URL. 663 # 664 # write_body[1] will contain a tornado.httputil.HTTPHeaders 665 # instance that we will use to parse each header line. We 666 # initialize this to False, and after we parse the status line we 667 # will replace it with the HTTPHeaders instance. If/when we have 668 # found the encoding used in the request, we set this value to 669 # False to signify that we are done parsing. 670 # 671 # write_body[2] is where the encoding will be stored 672 write_body = [None, False, None] 673 674 def on_header(hdr): 675 if write_body[1] is not False and write_body[2] is None: 676 if not hdr.strip() and "Content-Type" not in write_body[1]: 677 # If write_body[0] is True, then we are not following a 678 # redirect (initial response was a 200 OK). So there is 679 # no need to reset write_body[0]. 680 if write_body[0] is not True: 681 # We are following a redirect, so we need to reset 682 # write_body[0] so that we properly follow it. 683 write_body[0] = None 684 # We don't need the HTTPHeaders object anymore 685 write_body[1] = False 686 return 687 # Try to find out what content type encoding is used if 688 # this is a text file 689 write_body[1].parse_line(hdr) # pylint: disable=no-member 690 if "Content-Type" in write_body[1]: 691 content_type = write_body[1].get( 692 "Content-Type" 693 ) # pylint: disable=no-member 694 if not content_type.startswith("text"): 695 write_body[1] = write_body[2] = False 696 else: 697 encoding = "utf-8" 698 fields = content_type.split(";") 699 for field in fields: 700 if "encoding" in field: 701 encoding = field.split("encoding=")[-1] 702 write_body[2] = encoding 703 # We have found our encoding. Stop processing headers. 704 write_body[1] = False 705 706 # If write_body[0] is False, this means that this 707 # header is a 30x redirect, so we need to reset 708 # write_body[0] to None so that we parse the HTTP 709 # status code from the redirect target. Additionally, 710 # we need to reset write_body[2] so that we inspect the 711 # headers for the Content-Type of the URL we're 712 # following. 713 if write_body[0] is write_body[1] is False: 714 write_body[0] = write_body[2] = None 715 716 # Check the status line of the HTTP request 717 if write_body[0] is None: 718 try: 719 hdr = parse_response_start_line(hdr) 720 except HTTPInputError: 721 # Not the first line, do nothing 722 return 723 write_body[0] = hdr.code not in [301, 302, 303, 307] 724 write_body[1] = HTTPHeaders() 725 726 if no_cache: 727 result = [] 728 729 def on_chunk(chunk): 730 if write_body[0]: 731 if write_body[2]: 732 chunk = chunk.decode(write_body[2]) 733 result.append(chunk) 734 735 else: 736 dest_tmp = "{}.part".format(dest) 737 # We need an open filehandle to use in the on_chunk callback, 738 # that's why we're not using a with clause here. 739 # pylint: disable=resource-leakage 740 destfp = salt.utils.files.fopen(dest_tmp, "wb") 741 # pylint: enable=resource-leakage 742 743 def on_chunk(chunk): 744 if write_body[0]: 745 destfp.write(chunk) 746 747 query = salt.utils.http.query( 748 fixed_url, 749 stream=True, 750 streaming_callback=on_chunk, 751 header_callback=on_header, 752 username=url_data.username, 753 password=url_data.password, 754 opts=self.opts, 755 verify_ssl=verify_ssl, 756 **get_kwargs 757 ) 758 if "handle" not in query: 759 raise MinionError( 760 "Error: {} reading {}".format(query["error"], url_data.path) 761 ) 762 if no_cache: 763 if write_body[2]: 764 return "".join(result) 765 return b"".join(result) 766 else: 767 destfp.close() 768 destfp = None 769 salt.utils.files.rename(dest_tmp, dest) 770 return dest 771 except urllib.error.HTTPError as exc: 772 raise MinionError( 773 "HTTP error {0} reading {1}: {3}".format( 774 exc.code, 775 url, 776 *http.server.BaseHTTPRequestHandler.responses[exc.code] 777 ) 778 ) 779 except urllib.error.URLError as exc: 780 raise MinionError("Error reading {}: {}".format(url, exc.reason)) 781 finally: 782 if destfp is not None: 783 destfp.close() 784 785 def get_template( 786 self, 787 url, 788 dest, 789 template="jinja", 790 makedirs=False, 791 saltenv="base", 792 cachedir=None, 793 **kwargs 794 ): 795 """ 796 Cache a file then process it as a template 797 """ 798 if "env" in kwargs: 799 # "env" is not supported; Use "saltenv". 800 kwargs.pop("env") 801 802 kwargs["saltenv"] = saltenv 803 url_data = urllib.parse.urlparse(url) 804 sfn = self.cache_file(url, saltenv, cachedir=cachedir) 805 if not sfn or not os.path.exists(sfn): 806 return "" 807 if template in salt.utils.templates.TEMPLATE_REGISTRY: 808 data = salt.utils.templates.TEMPLATE_REGISTRY[template](sfn, **kwargs) 809 else: 810 log.error( 811 "Attempted to render template with unavailable engine %s", template 812 ) 813 return "" 814 if not data["result"]: 815 # Failed to render the template 816 log.error("Failed to render template with error: %s", data["data"]) 817 return "" 818 if not dest: 819 # No destination passed, set the dest as an extrn_files cache 820 dest = self._extrn_path(url, saltenv, cachedir=cachedir) 821 # If Salt generated the dest name, create any required dirs 822 makedirs = True 823 824 destdir = os.path.dirname(dest) 825 if not os.path.isdir(destdir): 826 if makedirs: 827 os.makedirs(destdir) 828 else: 829 salt.utils.files.safe_rm(data["data"]) 830 return "" 831 shutil.move(data["data"], dest) 832 return dest 833 834 def _extrn_path(self, url, saltenv, cachedir=None): 835 """ 836 Return the extrn_filepath for a given url 837 """ 838 url_data = urllib.parse.urlparse(url) 839 if salt.utils.platform.is_windows(): 840 netloc = salt.utils.path.sanitize_win_path(url_data.netloc) 841 else: 842 netloc = url_data.netloc 843 844 # Strip user:pass from URLs 845 netloc = netloc.split("@")[-1] 846 847 if cachedir is None: 848 cachedir = self.opts["cachedir"] 849 elif not os.path.isabs(cachedir): 850 cachedir = os.path.join(self.opts["cachedir"], cachedir) 851 852 if url_data.query: 853 file_name = "-".join([url_data.path, url_data.query]) 854 else: 855 file_name = url_data.path 856 857 # clean_path returns an empty string if the check fails 858 root_path = salt.utils.path.join(cachedir, "extrn_files", saltenv, netloc) 859 new_path = os.path.sep.join([root_path, file_name]) 860 if not salt.utils.verify.clean_path(root_path, new_path, subdir=True): 861 return "Invalid path" 862 863 if len(file_name) > MAX_FILENAME_LENGTH: 864 file_name = salt.utils.hashutils.sha256_digest(file_name) 865 866 return salt.utils.path.join(cachedir, "extrn_files", saltenv, netloc, file_name) 867 868 869class PillarClient(Client): 870 """ 871 Used by pillar to handle fileclient requests 872 """ 873 874 def _find_file(self, path, saltenv="base"): 875 """ 876 Locate the file path 877 """ 878 fnd = {"path": "", "rel": ""} 879 880 if salt.utils.url.is_escaped(path): 881 # The path arguments are escaped 882 path = salt.utils.url.unescape(path) 883 for root in self.opts["pillar_roots"].get(saltenv, []): 884 full = os.path.join(root, path) 885 if os.path.isfile(full): 886 fnd["path"] = full 887 fnd["rel"] = path 888 return fnd 889 return fnd 890 891 def get_file( 892 self, path, dest="", makedirs=False, saltenv="base", gzip=None, cachedir=None 893 ): 894 """ 895 Copies a file from the local files directory into :param:`dest` 896 gzip compression settings are ignored for local files 897 """ 898 path = self._check_proto(path) 899 fnd = self._find_file(path, saltenv) 900 fnd_path = fnd.get("path") 901 if not fnd_path: 902 return "" 903 904 return fnd_path 905 906 def file_list(self, saltenv="base", prefix=""): 907 """ 908 Return a list of files in the given environment 909 with optional relative prefix path to limit directory traversal 910 """ 911 ret = [] 912 prefix = prefix.strip("/") 913 for path in self.opts["pillar_roots"].get(saltenv, []): 914 for root, dirs, files in salt.utils.path.os_walk( 915 os.path.join(path, prefix), followlinks=True 916 ): 917 # Don't walk any directories that match file_ignore_regex or glob 918 dirs[:] = [ 919 d for d in dirs if not salt.fileserver.is_file_ignored(self.opts, d) 920 ] 921 for fname in files: 922 relpath = os.path.relpath(os.path.join(root, fname), path) 923 ret.append(salt.utils.data.decode(relpath)) 924 return ret 925 926 def file_list_emptydirs(self, saltenv="base", prefix=""): 927 """ 928 List the empty dirs in the pillar_roots 929 with optional relative prefix path to limit directory traversal 930 """ 931 ret = [] 932 prefix = prefix.strip("/") 933 for path in self.opts["pillar_roots"].get(saltenv, []): 934 for root, dirs, files in salt.utils.path.os_walk( 935 os.path.join(path, prefix), followlinks=True 936 ): 937 # Don't walk any directories that match file_ignore_regex or glob 938 dirs[:] = [ 939 d for d in dirs if not salt.fileserver.is_file_ignored(self.opts, d) 940 ] 941 if not dirs and not files: 942 ret.append(salt.utils.data.decode(os.path.relpath(root, path))) 943 return ret 944 945 def dir_list(self, saltenv="base", prefix=""): 946 """ 947 List the dirs in the pillar_roots 948 with optional relative prefix path to limit directory traversal 949 """ 950 ret = [] 951 prefix = prefix.strip("/") 952 for path in self.opts["pillar_roots"].get(saltenv, []): 953 for root, dirs, files in salt.utils.path.os_walk( 954 os.path.join(path, prefix), followlinks=True 955 ): 956 ret.append(salt.utils.data.decode(os.path.relpath(root, path))) 957 return ret 958 959 def __get_file_path(self, path, saltenv="base"): 960 """ 961 Return either a file path or the result of a remote find_file call. 962 """ 963 try: 964 path = self._check_proto(path) 965 except MinionError as err: 966 # Local file path 967 if not os.path.isfile(path): 968 log.warning( 969 "specified file %s is not present to generate hash: %s", path, err 970 ) 971 return None 972 else: 973 return path 974 return self._find_file(path, saltenv) 975 976 def hash_file(self, path, saltenv="base"): 977 """ 978 Return the hash of a file, to get the hash of a file in the pillar_roots 979 prepend the path with salt://<file on server> otherwise, prepend the 980 file with / for a local file. 981 """ 982 ret = {} 983 fnd = self.__get_file_path(path, saltenv) 984 if fnd is None: 985 return ret 986 987 try: 988 # Remote file path (self._find_file() invoked) 989 fnd_path = fnd["path"] 990 except TypeError: 991 # Local file path 992 fnd_path = fnd 993 994 hash_type = self.opts.get("hash_type", "md5") 995 ret["hsum"] = salt.utils.hashutils.get_hash(fnd_path, form=hash_type) 996 ret["hash_type"] = hash_type 997 return ret 998 999 def hash_and_stat_file(self, path, saltenv="base"): 1000 """ 1001 Return the hash of a file, to get the hash of a file in the pillar_roots 1002 prepend the path with salt://<file on server> otherwise, prepend the 1003 file with / for a local file. 1004 1005 Additionally, return the stat result of the file, or None if no stat 1006 results were found. 1007 """ 1008 ret = {} 1009 fnd = self.__get_file_path(path, saltenv) 1010 if fnd is None: 1011 return ret, None 1012 1013 try: 1014 # Remote file path (self._find_file() invoked) 1015 fnd_path = fnd["path"] 1016 fnd_stat = fnd.get("stat") 1017 except TypeError: 1018 # Local file path 1019 fnd_path = fnd 1020 try: 1021 fnd_stat = list(os.stat(fnd_path)) 1022 except Exception: # pylint: disable=broad-except 1023 fnd_stat = None 1024 1025 hash_type = self.opts.get("hash_type", "md5") 1026 ret["hsum"] = salt.utils.hashutils.get_hash(fnd_path, form=hash_type) 1027 ret["hash_type"] = hash_type 1028 return ret, fnd_stat 1029 1030 def list_env(self, saltenv="base"): 1031 """ 1032 Return a list of the files in the file server's specified environment 1033 """ 1034 return self.file_list(saltenv) 1035 1036 def master_opts(self): 1037 """ 1038 Return the master opts data 1039 """ 1040 return self.opts 1041 1042 def envs(self): 1043 """ 1044 Return the available environments 1045 """ 1046 ret = [] 1047 for saltenv in self.opts["pillar_roots"]: 1048 ret.append(saltenv) 1049 return ret 1050 1051 def master_tops(self): 1052 """ 1053 Originally returned information via the external_nodes subsystem. 1054 External_nodes was deprecated and removed in 1055 2014.1.6 in favor of master_tops (which had been around since pre-0.17). 1056 salt-call --local state.show_top 1057 ends up here, but master_tops has not been extended to support 1058 show_top in a completely local environment yet. It's worth noting 1059 that originally this fn started with 1060 if 'external_nodes' not in opts: return {} 1061 So since external_nodes is gone now, we are just returning the 1062 empty dict. 1063 """ 1064 return {} 1065 1066 1067class RemoteClient(Client): 1068 """ 1069 Interact with the salt master file server. 1070 """ 1071 1072 def __init__(self, opts): 1073 Client.__init__(self, opts) 1074 self._closing = False 1075 self.channel = salt.transport.client.ReqChannel.factory(self.opts) 1076 if hasattr(self.channel, "auth"): 1077 self.auth = self.channel.auth 1078 else: 1079 self.auth = "" 1080 1081 def _refresh_channel(self): 1082 """ 1083 Reset the channel, in the event of an interruption 1084 """ 1085 # Close the previous channel 1086 self.channel.close() 1087 # Instantiate a new one 1088 self.channel = salt.transport.client.ReqChannel.factory(self.opts) 1089 return self.channel 1090 1091 # pylint: disable=no-dunder-del 1092 def __del__(self): 1093 self.destroy() 1094 1095 # pylint: enable=no-dunder-del 1096 1097 def destroy(self): 1098 if self._closing: 1099 return 1100 1101 self._closing = True 1102 channel = None 1103 try: 1104 channel = self.channel 1105 except AttributeError: 1106 pass 1107 if channel is not None: 1108 channel.close() 1109 1110 def get_file( 1111 self, path, dest="", makedirs=False, saltenv="base", gzip=None, cachedir=None 1112 ): 1113 """ 1114 Get a single file from the salt-master 1115 path must be a salt server location, aka, salt://path/to/file, if 1116 dest is omitted, then the downloaded file will be placed in the minion 1117 cache 1118 """ 1119 path, senv = salt.utils.url.split_env(path) 1120 if senv: 1121 saltenv = senv 1122 1123 if not salt.utils.platform.is_windows(): 1124 hash_server, stat_server = self.hash_and_stat_file(path, saltenv) 1125 try: 1126 mode_server = stat_server[0] 1127 except (IndexError, TypeError): 1128 mode_server = None 1129 else: 1130 hash_server = self.hash_file(path, saltenv) 1131 mode_server = None 1132 1133 # Check if file exists on server, before creating files and 1134 # directories 1135 if hash_server == "": 1136 log.debug("Could not find file '%s' in saltenv '%s'", path, saltenv) 1137 return False 1138 1139 # If dest is a directory, rewrite dest with filename 1140 if dest is not None and (os.path.isdir(dest) or dest.endswith(("/", "\\"))): 1141 dest = os.path.join(dest, os.path.basename(path)) 1142 log.debug( 1143 "In saltenv '%s', '%s' is a directory. Changing dest to '%s'", 1144 saltenv, 1145 os.path.dirname(dest), 1146 dest, 1147 ) 1148 1149 # Hash compare local copy with master and skip download 1150 # if no difference found. 1151 dest2check = dest 1152 if not dest2check: 1153 rel_path = self._check_proto(path) 1154 1155 log.debug( 1156 "In saltenv '%s', looking at rel_path '%s' to resolve '%s'", 1157 saltenv, 1158 rel_path, 1159 path, 1160 ) 1161 with self._cache_loc(rel_path, saltenv, cachedir=cachedir) as cache_dest: 1162 dest2check = cache_dest 1163 1164 log.debug( 1165 "In saltenv '%s', ** considering ** path '%s' to resolve '%s'", 1166 saltenv, 1167 dest2check, 1168 path, 1169 ) 1170 1171 if dest2check and os.path.isfile(dest2check): 1172 if not salt.utils.platform.is_windows(): 1173 hash_local, stat_local = self.hash_and_stat_file(dest2check, saltenv) 1174 try: 1175 mode_local = stat_local[0] 1176 except (IndexError, TypeError): 1177 mode_local = None 1178 else: 1179 hash_local = self.hash_file(dest2check, saltenv) 1180 mode_local = None 1181 1182 if hash_local == hash_server: 1183 return dest2check 1184 1185 log.debug( 1186 "Fetching file from saltenv '%s', ** attempting ** '%s'", saltenv, path 1187 ) 1188 d_tries = 0 1189 transport_tries = 0 1190 path = self._check_proto(path) 1191 load = {"path": path, "saltenv": saltenv, "cmd": "_serve_file"} 1192 if gzip: 1193 gzip = int(gzip) 1194 load["gzip"] = gzip 1195 1196 fn_ = None 1197 if dest: 1198 destdir = os.path.dirname(dest) 1199 if not os.path.isdir(destdir): 1200 if makedirs: 1201 try: 1202 os.makedirs(destdir) 1203 except OSError as exc: 1204 if exc.errno != errno.EEXIST: # ignore if it was there already 1205 raise 1206 else: 1207 return False 1208 # We need an open filehandle here, that's why we're not using a 1209 # with clause: 1210 # pylint: disable=resource-leakage 1211 fn_ = salt.utils.files.fopen(dest, "wb+") 1212 # pylint: enable=resource-leakage 1213 else: 1214 log.debug("No dest file found") 1215 1216 while True: 1217 if not fn_: 1218 load["loc"] = 0 1219 else: 1220 load["loc"] = fn_.tell() 1221 data = self.channel.send(load, raw=True) 1222 # Sometimes the source is local (eg when using 1223 # 'salt.fileserver.FSChan'), in which case the keys are 1224 # already strings. Sometimes the source is remote, in which 1225 # case the keys are bytes due to raw mode. Standardize on 1226 # strings for the top-level keys to simplify things. 1227 data = decode_dict_keys_to_str(data) 1228 try: 1229 if not data["data"]: 1230 if not fn_ and data["dest"]: 1231 # This is a 0 byte file on the master 1232 with self._cache_loc( 1233 data["dest"], saltenv, cachedir=cachedir 1234 ) as cache_dest: 1235 dest = cache_dest 1236 with salt.utils.files.fopen(cache_dest, "wb+") as ofile: 1237 ofile.write(data["data"]) 1238 if "hsum" in data and d_tries < 3: 1239 # Master has prompted a file verification, if the 1240 # verification fails, re-download the file. Try 3 times 1241 d_tries += 1 1242 hsum = salt.utils.hashutils.get_hash( 1243 dest, 1244 salt.utils.stringutils.to_str( 1245 data.get("hash_type", b"md5") 1246 ), 1247 ) 1248 if hsum != data["hsum"]: 1249 log.warning( 1250 "Bad download of file %s, attempt %d of 3", 1251 path, 1252 d_tries, 1253 ) 1254 continue 1255 break 1256 if not fn_: 1257 with self._cache_loc( 1258 data["dest"], saltenv, cachedir=cachedir 1259 ) as cache_dest: 1260 dest = cache_dest 1261 # If a directory was formerly cached at this path, then 1262 # remove it to avoid a traceback trying to write the file 1263 if os.path.isdir(dest): 1264 salt.utils.files.rm_rf(dest) 1265 fn_ = salt.utils.atomicfile.atomic_open(dest, "wb+") 1266 if data.get("gzip", None): 1267 data = salt.utils.gzip_util.uncompress(data["data"]) 1268 else: 1269 data = data["data"] 1270 if isinstance(data, str): 1271 data = data.encode() 1272 fn_.write(data) 1273 except (TypeError, KeyError) as exc: 1274 try: 1275 data_type = type(data).__name__ 1276 except AttributeError: 1277 # Shouldn't happen, but don't let this cause a traceback. 1278 data_type = str(type(data)) 1279 transport_tries += 1 1280 log.warning( 1281 "Data transport is broken, got: %s, type: %s, " 1282 "exception: %s, attempt %d of 3", 1283 data, 1284 data_type, 1285 exc, 1286 transport_tries, 1287 ) 1288 self._refresh_channel() 1289 if transport_tries > 3: 1290 log.error( 1291 "Data transport is broken, got: %s, type: %s, " 1292 "exception: %s, retry attempts exhausted", 1293 data, 1294 data_type, 1295 exc, 1296 ) 1297 break 1298 1299 if fn_: 1300 fn_.close() 1301 log.info("Fetching file from saltenv '%s', ** done ** '%s'", saltenv, path) 1302 else: 1303 log.debug( 1304 "In saltenv '%s', we are ** missing ** the file '%s'", saltenv, path 1305 ) 1306 1307 return dest 1308 1309 def file_list(self, saltenv="base", prefix=""): 1310 """ 1311 List the files on the master 1312 """ 1313 load = {"saltenv": saltenv, "prefix": prefix, "cmd": "_file_list"} 1314 return self.channel.send(load) 1315 1316 def file_list_emptydirs(self, saltenv="base", prefix=""): 1317 """ 1318 List the empty dirs on the master 1319 """ 1320 load = {"saltenv": saltenv, "prefix": prefix, "cmd": "_file_list_emptydirs"} 1321 return self.channel.send(load) 1322 1323 def dir_list(self, saltenv="base", prefix=""): 1324 """ 1325 List the dirs on the master 1326 """ 1327 load = {"saltenv": saltenv, "prefix": prefix, "cmd": "_dir_list"} 1328 return self.channel.send(load) 1329 1330 def symlink_list(self, saltenv="base", prefix=""): 1331 """ 1332 List symlinked files and dirs on the master 1333 """ 1334 load = {"saltenv": saltenv, "prefix": prefix, "cmd": "_symlink_list"} 1335 return self.channel.send(load) 1336 1337 def __hash_and_stat_file(self, path, saltenv="base"): 1338 """ 1339 Common code for hashing and stating files 1340 """ 1341 try: 1342 path = self._check_proto(path) 1343 except MinionError as err: 1344 if not os.path.isfile(path): 1345 log.warning( 1346 "specified file %s is not present to generate hash: %s", path, err 1347 ) 1348 return {}, None 1349 else: 1350 ret = {} 1351 hash_type = self.opts.get("hash_type", "md5") 1352 ret["hsum"] = salt.utils.hashutils.get_hash(path, form=hash_type) 1353 ret["hash_type"] = hash_type 1354 return ret 1355 load = {"path": path, "saltenv": saltenv, "cmd": "_file_hash"} 1356 return self.channel.send(load) 1357 1358 def hash_file(self, path, saltenv="base"): 1359 """ 1360 Return the hash of a file, to get the hash of a file on the salt 1361 master file server prepend the path with salt://<file on server> 1362 otherwise, prepend the file with / for a local file. 1363 """ 1364 return self.__hash_and_stat_file(path, saltenv) 1365 1366 def hash_and_stat_file(self, path, saltenv="base"): 1367 """ 1368 The same as hash_file, but also return the file's mode, or None if no 1369 mode data is present. 1370 """ 1371 hash_result = self.hash_file(path, saltenv) 1372 try: 1373 path = self._check_proto(path) 1374 except MinionError as err: 1375 if not os.path.isfile(path): 1376 return hash_result, None 1377 else: 1378 try: 1379 return hash_result, list(os.stat(path)) 1380 except Exception: # pylint: disable=broad-except 1381 return hash_result, None 1382 load = {"path": path, "saltenv": saltenv, "cmd": "_file_find"} 1383 fnd = self.channel.send(load) 1384 try: 1385 stat_result = fnd.get("stat") 1386 except AttributeError: 1387 stat_result = None 1388 return hash_result, stat_result 1389 1390 def list_env(self, saltenv="base"): 1391 """ 1392 Return a list of the files in the file server's specified environment 1393 """ 1394 load = {"saltenv": saltenv, "cmd": "_file_list"} 1395 return self.channel.send(load) 1396 1397 def envs(self): 1398 """ 1399 Return a list of available environments 1400 """ 1401 load = {"cmd": "_file_envs"} 1402 return self.channel.send(load) 1403 1404 def master_opts(self): 1405 """ 1406 Return the master opts data 1407 """ 1408 load = {"cmd": "_master_opts"} 1409 return self.channel.send(load) 1410 1411 def master_tops(self): 1412 """ 1413 Return the metadata derived from the master_tops system 1414 """ 1415 load = {"cmd": "_master_tops", "id": self.opts["id"], "opts": self.opts} 1416 if self.auth: 1417 load["tok"] = self.auth.gen_token(b"salt") 1418 return self.channel.send(load) 1419 1420 1421class FSClient(RemoteClient): 1422 """ 1423 A local client that uses the RemoteClient but substitutes the channel for 1424 the FSChan object 1425 """ 1426 1427 def __init__(self, opts): # pylint: disable=W0231 1428 Client.__init__(self, opts) # pylint: disable=W0233 1429 self._closing = False 1430 self.channel = salt.fileserver.FSChan(opts) 1431 self.auth = DumbAuth() 1432 1433 1434# Provide backward compatibility for anyone directly using LocalClient (but no 1435# one should be doing this). 1436LocalClient = FSClient 1437 1438 1439class DumbAuth: 1440 """ 1441 The dumbauth class is used to stub out auth calls fired from the FSClient 1442 subsystem 1443 """ 1444 1445 def gen_token(self, clear_tok): 1446 return clear_tok 1447