1""" 2Generate the salt thin tarball from the installed python files 3""" 4 5import contextvars as py_contextvars 6import copy 7import importlib.util 8import logging 9import os 10import shutil 11import site 12import subprocess 13import sys 14import tarfile 15import tempfile 16import zipfile 17 18import distro 19import jinja2 20import msgpack 21import salt 22import salt.exceptions 23import salt.ext.tornado as tornado 24import salt.utils.files 25import salt.utils.hashutils 26import salt.utils.json 27import salt.utils.path 28import salt.utils.stringutils 29import salt.version 30import yaml 31 32# This is needed until we drop support for python 3.6 33has_immutables = False 34try: 35 import immutables 36 37 has_immutables = True 38except ImportError: 39 pass 40 41 42try: 43 import zlib 44except ImportError: 45 zlib = None 46 47# pylint: disable=import-error,no-name-in-module 48try: 49 import certifi 50except ImportError: 51 certifi = None 52 53try: 54 import singledispatch 55except ImportError: 56 singledispatch = None 57 58try: 59 import singledispatch_helpers 60except ImportError: 61 singledispatch_helpers = None 62 63try: 64 import backports_abc 65except ImportError: 66 import salt.ext.backports_abc as backports_abc 67 68try: 69 # New Jinja only 70 import markupsafe 71except ImportError: 72 markupsafe = None 73 74 75try: 76 # Older python where the backport from pypi is installed 77 from backports import ssl_match_hostname 78except ImportError: 79 # Other older python we use our bundled copy 80 try: 81 from salt.ext import ssl_match_hostname 82 except ImportError: 83 ssl_match_hostname = None 84 85concurrent = None 86 87 88log = logging.getLogger(__name__) 89 90 91def import_module(name, path): 92 """ 93 Import a module from a specific path. Path can be a full or relative path 94 to a .py file. 95 96 :name: The name of the module to import 97 :path: The path of the module to import 98 """ 99 try: 100 spec = importlib.util.spec_from_file_location(name, path) 101 except ValueError: 102 spec = None 103 if spec is not None: 104 lib = importlib.util.module_from_spec(spec) 105 try: 106 spec.loader.exec_module(lib) 107 except OSError: 108 pass 109 else: 110 return lib 111 112 113def getsitepackages(): 114 """ 115 Some versions of Virtualenv ship a site.py without getsitepackages. This 116 method will first try and return sitepackages from the default site module 117 if no method exists we will try importing the site module from every other 118 path in sys.paths until we find a getsitepackages method to return the 119 results from. If for some reason no gesitepackages method can be found a 120 RuntimeError will be raised 121 122 :return: A list containing all global site-packages directories. 123 """ 124 if hasattr(site, "getsitepackages"): 125 return site.getsitepackages() 126 for path in sys.path: 127 lib = import_module("site", os.path.join(path, "site.py")) 128 if hasattr(lib, "getsitepackages"): 129 return lib.getsitepackages() 130 raise RuntimeError("Unable to locate a getsitepackages method") 131 132 133def find_site_modules(name): 134 """ 135 Finds and imports a module from site packages directories. 136 137 :name: The name of the module to import 138 :return: A list of imported modules, if no modules are imported an empty 139 list is returned. 140 """ 141 libs = [] 142 site_paths = [] 143 try: 144 site_paths = getsitepackages() 145 except RuntimeError: 146 log.debug("No site package directories found") 147 for site_path in site_paths: 148 path = os.path.join(site_path, "{}.py".format(name)) 149 lib = import_module(name, path) 150 if lib: 151 libs.append(lib) 152 path = os.path.join(site_path, name, "__init__.py") 153 lib = import_module(name, path) 154 if lib: 155 libs.append(lib) 156 return libs 157 158 159def _get_salt_call(*dirs, **namespaces): 160 """ 161 Return salt-call source, based on configuration. 162 This will include additional namespaces for another versions of Salt, 163 if needed (e.g. older interpreters etc). 164 165 :dirs: List of directories to include in the system path 166 :namespaces: Dictionary of namespace 167 :return: 168 """ 169 template = """# -*- coding: utf-8 -*- 170import os 171import sys 172 173# Namespaces is a map: {namespace: major/minor version}, like {'2016.11.4': [2, 6]} 174# Appears only when configured in Master configuration. 175namespaces = %namespaces% 176 177# Default system paths alongside the namespaces 178syspaths = %dirs% 179syspaths.append('py{0}'.format(sys.version_info[0])) 180 181curr_ver = (sys.version_info[0], sys.version_info[1],) 182 183namespace = '' 184for ns in namespaces: 185 if curr_ver == tuple(namespaces[ns]): 186 namespace = ns 187 break 188 189for base in syspaths: 190 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 191 namespace and os.path.join(namespace, base) or base)) 192 193if __name__ == '__main__': 194 from salt.scripts import salt_call 195 salt_call() 196""" 197 198 for tgt, cnt in [("%dirs%", dirs), ("%namespaces%", namespaces)]: 199 template = template.replace(tgt, salt.utils.json.dumps(cnt)) 200 201 return salt.utils.stringutils.to_bytes(template) 202 203 204def thin_path(cachedir): 205 """ 206 Return the path to the thin tarball 207 """ 208 return os.path.join(cachedir, "thin", "thin.tgz") 209 210 211def _is_shareable(mod): 212 """ 213 Return True if module is share-able between major Python versions. 214 215 :param mod: 216 :return: 217 """ 218 # This list is subject to change 219 shareable = ["salt", "jinja2", "msgpack", "certifi"] 220 221 return os.path.basename(mod) in shareable 222 223 224def _add_dependency(container, obj): 225 """ 226 Add a dependency to the top list. 227 228 :param obj: 229 :param is_file: 230 :return: 231 """ 232 if os.path.basename(obj.__file__).split(".")[0] == "__init__": 233 container.append(os.path.dirname(obj.__file__)) 234 else: 235 container.append(obj.__file__.replace(".pyc", ".py")) 236 237 238def gte(): 239 """ 240 This function is called externally from the alternative 241 Python interpreter from within _get_tops function. 242 243 :param extra_mods: 244 :param so_mods: 245 :return: 246 """ 247 extra = salt.utils.json.loads(sys.argv[1]) 248 tops = get_tops(**extra) 249 250 return salt.utils.json.dumps(tops, ensure_ascii=False) 251 252 253def get_tops_python(py_ver, exclude=None, ext_py_ver=None): 254 """ 255 Get top directories for the ssh_ext_alternatives dependencies 256 automatically for the given python version. This allows 257 the user to add the dependency paths automatically. 258 259 :param py_ver: 260 python binary to use to detect binaries 261 262 :param exclude: 263 list of modules not to auto detect 264 265 :param ext_py_ver: 266 the py-version from the ssh_ext_alternatives config 267 """ 268 files = {} 269 mods = [ 270 "jinja2", 271 "yaml", 272 "tornado", 273 "msgpack", 274 "certifi", 275 "singledispatch", 276 "concurrent", 277 "singledispatch_helpers", 278 "ssl_match_hostname", 279 "markupsafe", 280 "backports_abc", 281 ] 282 if ext_py_ver and tuple(ext_py_ver) >= (3, 0): 283 mods.append("distro") 284 285 for mod in mods: 286 if exclude and mod in exclude: 287 continue 288 289 if not salt.utils.path.which(py_ver): 290 log.error("%s does not exist. Could not auto detect dependencies", py_ver) 291 return {} 292 py_shell_cmd = [py_ver, "-c", "import {0}; print({0}.__file__)".format(mod)] 293 cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE) 294 stdout, _ = cmd.communicate() 295 mod_file = os.path.abspath(salt.utils.data.decode(stdout).rstrip("\n")) 296 297 if not stdout or not os.path.exists(mod_file): 298 log.error( 299 "Could not auto detect file location for module %s for python version %s", 300 mod, 301 py_ver, 302 ) 303 continue 304 305 if os.path.basename(mod_file).split(".")[0] == "__init__": 306 mod_file = os.path.dirname(mod_file) 307 else: 308 mod_file = mod_file.replace("pyc", "py") 309 310 files[mod] = mod_file 311 return files 312 313 314def get_ext_tops(config): 315 """ 316 Get top directories for the dependencies, based on external configuration. 317 318 :return: 319 """ 320 config = copy.deepcopy(config) or {} 321 alternatives = {} 322 required = ["jinja2", "yaml", "tornado", "msgpack"] 323 tops = [] 324 for ns, cfg in config.items(): 325 alternatives[ns] = cfg 326 locked_py_version = cfg.get("py-version") 327 err_msg = None 328 if not locked_py_version: 329 err_msg = "Alternative Salt library: missing specific locked Python version" 330 elif not isinstance(locked_py_version, (tuple, list)): 331 err_msg = ( 332 "Alternative Salt library: specific locked Python version " 333 "should be a list of major/minor version" 334 ) 335 if err_msg: 336 raise salt.exceptions.SaltSystemExit(err_msg) 337 338 if tuple(locked_py_version) >= (3, 0) and "distro" not in required: 339 required.append("distro") 340 341 if cfg.get("dependencies") == "inherit": 342 # TODO: implement inheritance of the modules from _here_ 343 raise NotImplementedError("This feature is not yet implemented") 344 else: 345 for dep in cfg.get("dependencies"): 346 mod = cfg["dependencies"][dep] or "" 347 if not mod: 348 log.warning("Module %s has missing configuration", dep) 349 continue 350 elif mod.endswith(".py") and not os.path.isfile(mod): 351 log.warning( 352 "Module %s configured with not a file or does not exist: %s", 353 dep, 354 mod, 355 ) 356 continue 357 elif not mod.endswith(".py") and not os.path.isfile( 358 os.path.join(mod, "__init__.py") 359 ): 360 log.warning( 361 "Module %s is not a Python importable module with %s", dep, mod 362 ) 363 continue 364 tops.append(mod) 365 366 if dep in required: 367 required.pop(required.index(dep)) 368 369 required = ", ".join(required) 370 if required: 371 msg = ( 372 "Missing dependencies for the alternative version" 373 " in the external configuration: {}".format(required) 374 ) 375 log.error(msg) 376 raise salt.exceptions.SaltSystemExit(msg=msg) 377 alternatives[ns]["dependencies"] = tops 378 return alternatives 379 380 381def _get_ext_namespaces(config): 382 """ 383 Get namespaces from the existing configuration. 384 385 :param config: 386 :return: 387 """ 388 namespaces = {} 389 if not config: 390 return namespaces 391 392 for ns in config: 393 constraint_version = tuple(config[ns].get("py-version", [])) 394 if not constraint_version: 395 raise salt.exceptions.SaltSystemExit( 396 "An alternative version is configured, but not defined " 397 "to what Python's major/minor version it should be constrained." 398 ) 399 else: 400 namespaces[ns] = constraint_version 401 402 return namespaces 403 404 405def get_tops(extra_mods="", so_mods=""): 406 """ 407 Get top directories for the dependencies, based on Python interpreter. 408 409 :param extra_mods: 410 :param so_mods: 411 :return: 412 """ 413 tops = [] 414 mods = [ 415 salt, 416 distro, 417 jinja2, 418 yaml, 419 tornado, 420 msgpack, 421 certifi, 422 singledispatch, 423 concurrent, 424 singledispatch_helpers, 425 ssl_match_hostname, 426 markupsafe, 427 backports_abc, 428 ] 429 modules = find_site_modules("contextvars") 430 if modules: 431 contextvars = modules[0] 432 else: 433 contextvars = py_contextvars 434 log.debug("Using contextvars %r", contextvars) 435 mods.append(contextvars) 436 if has_immutables: 437 mods.append(immutables) 438 for mod in mods: 439 if mod: 440 log.debug('Adding module to the tops: "%s"', mod.__name__) 441 _add_dependency(tops, mod) 442 443 for mod in [m for m in extra_mods.split(",") if m]: 444 if mod not in locals() and mod not in globals(): 445 try: 446 locals()[mod] = __import__(mod) 447 moddir, modname = os.path.split(locals()[mod].__file__) 448 base, _ = os.path.splitext(modname) 449 if base == "__init__": 450 tops.append(moddir) 451 else: 452 tops.append(os.path.join(moddir, base + ".py")) 453 except ImportError as err: 454 log.error( 455 'Unable to import extra-module "%s": %s', mod, err, exc_info=True 456 ) 457 458 for mod in [m for m in so_mods.split(",") if m]: 459 try: 460 locals()[mod] = __import__(mod) 461 tops.append(locals()[mod].__file__) 462 except ImportError as err: 463 log.error('Unable to import so-module "%s"', mod, exc_info=True) 464 465 return tops 466 467 468def _get_supported_py_config(tops, extended_cfg): 469 """ 470 Based on the Salt SSH configuration, create a YAML configuration 471 for the supported Python interpreter versions. This is then written into the thin.tgz 472 archive and then verified by salt.client.ssh.ssh_py_shim.get_executable() 473 474 Note: Current versions of Salt only Support Python 3, but the versions of Python 475 (2.7,3.0) remain to include support for ssh_ext_alternatives if user is targeting an 476 older version of Salt. 477 :return: 478 """ 479 pymap = [] 480 for py_ver, tops in copy.deepcopy(tops).items(): 481 py_ver = int(py_ver) 482 if py_ver == 2: 483 pymap.append("py2:2:7") 484 elif py_ver == 3: 485 pymap.append("py3:3:0") 486 cfg_copy = copy.deepcopy(extended_cfg) or {} 487 for ns, cfg in cfg_copy.items(): 488 pymap.append("{}:{}:{}".format(ns, *cfg.get("py-version"))) 489 pymap.append("") 490 491 return salt.utils.stringutils.to_bytes(os.linesep.join(pymap)) 492 493 494def _get_thintar_prefix(tarname): 495 """ 496 Make sure thintar temporary name is concurrent and secure. 497 498 :param tarname: name of the chosen tarball 499 :return: prefixed tarname 500 """ 501 tfd, tmp_tarname = tempfile.mkstemp( 502 dir=os.path.dirname(tarname), 503 prefix=".thin-", 504 suffix=os.path.splitext(tarname)[1], 505 ) 506 os.close(tfd) 507 508 return tmp_tarname 509 510 511def _pack_alternative(extended_cfg, digest_collector, tfp): 512 # Pack alternative data 513 config = copy.deepcopy(extended_cfg) 514 # Check if auto_detect is enabled and update dependencies 515 for ns, cfg in config.items(): 516 if cfg.get("auto_detect"): 517 py_ver = "python" + str(cfg.get("py-version", [""])[0]) 518 if cfg.get("py_bin"): 519 py_ver = cfg["py_bin"] 520 521 exclude = [] 522 # get any manually set deps 523 deps = config[ns].get("dependencies") 524 if deps: 525 for dep in deps.keys(): 526 exclude.append(dep) 527 else: 528 config[ns]["dependencies"] = {} 529 530 # get auto deps 531 auto_deps = get_tops_python( 532 py_ver, exclude=exclude, ext_py_ver=cfg["py-version"] 533 ) 534 for dep in auto_deps: 535 config[ns]["dependencies"][dep] = auto_deps[dep] 536 537 for ns, cfg in get_ext_tops(config).items(): 538 tops = [cfg.get("path")] + cfg.get("dependencies") 539 py_ver_major, py_ver_minor = cfg.get("py-version") 540 541 for top in tops: 542 top = os.path.normpath(top) 543 base, top_dirname = os.path.basename(top), os.path.dirname(top) 544 os.chdir(top_dirname) 545 site_pkg_dir = ( 546 _is_shareable(base) and "pyall" or "py{}".format(py_ver_major) 547 ) 548 log.debug( 549 'Packing alternative "%s" to "%s/%s" destination', 550 base, 551 ns, 552 site_pkg_dir, 553 ) 554 if not os.path.exists(top): 555 log.error( 556 "File path %s does not exist. Unable to add to salt-ssh thin", top 557 ) 558 continue 559 if not os.path.isdir(top): 560 # top is a single file module 561 if os.path.exists(os.path.join(top_dirname, base)): 562 tfp.add(base, arcname=os.path.join(ns, site_pkg_dir, base)) 563 continue 564 for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True): 565 for name in files: 566 if not name.endswith((".pyc", ".pyo")): 567 digest_collector.add(os.path.join(root, name)) 568 arcname = os.path.join(ns, site_pkg_dir, root, name) 569 if hasattr(tfp, "getinfo"): 570 try: 571 tfp.getinfo(os.path.join(site_pkg_dir, root, name)) 572 arcname = None 573 except KeyError: 574 log.debug( 575 'ZIP: Unable to add "%s" with "getinfo"', arcname 576 ) 577 if arcname: 578 tfp.add(os.path.join(root, name), arcname=arcname) 579 580 581def gen_thin( 582 cachedir, 583 extra_mods="", 584 overwrite=False, 585 so_mods="", 586 absonly=True, 587 compress="gzip", 588 extended_cfg=None, 589): 590 """ 591 Generate the salt-thin tarball and print the location of the tarball 592 Optional additional mods to include (e.g. mako) can be supplied as a comma 593 delimited string. Permits forcing an overwrite of the output file as well. 594 595 CLI Example: 596 597 .. code-block:: bash 598 599 salt-run thin.generate 600 salt-run thin.generate mako 601 salt-run thin.generate mako,wempy 1 602 salt-run thin.generate overwrite=1 603 """ 604 if sys.version_info < (3,): 605 raise salt.exceptions.SaltSystemExit( 606 'The minimum required python version to run salt-ssh is "3".' 607 ) 608 if compress not in ["gzip", "zip"]: 609 log.warning( 610 'Unknown compression type: "%s". Falling back to "gzip" compression.', 611 compress, 612 ) 613 compress = "gzip" 614 615 thindir = os.path.join(cachedir, "thin") 616 if not os.path.isdir(thindir): 617 os.makedirs(thindir) 618 thintar = os.path.join(thindir, "thin." + (compress == "gzip" and "tgz" or "zip")) 619 thinver = os.path.join(thindir, "version") 620 pythinver = os.path.join(thindir, ".thin-gen-py-version") 621 salt_call = os.path.join(thindir, "salt-call") 622 pymap_cfg = os.path.join(thindir, "supported-versions") 623 code_checksum = os.path.join(thindir, "code-checksum") 624 digest_collector = salt.utils.hashutils.DigestCollector() 625 626 with salt.utils.files.fopen(salt_call, "wb") as fp_: 627 fp_.write(_get_salt_call("pyall", **_get_ext_namespaces(extended_cfg))) 628 629 if os.path.isfile(thintar): 630 if not overwrite: 631 if os.path.isfile(thinver): 632 with salt.utils.files.fopen(thinver) as fh_: 633 overwrite = fh_.read() != salt.version.__version__ 634 if overwrite is False and os.path.isfile(pythinver): 635 with salt.utils.files.fopen(pythinver) as fh_: 636 overwrite = fh_.read() != str(sys.version_info[0]) 637 else: 638 overwrite = True 639 640 if overwrite: 641 try: 642 log.debug("Removing %s archive file", thintar) 643 os.remove(thintar) 644 except OSError as exc: 645 log.error("Error while removing %s file: %s", thintar, exc) 646 if os.path.exists(thintar): 647 raise salt.exceptions.SaltSystemExit( 648 "Unable to remove {} file. See logs for details.".format( 649 thintar 650 ) 651 ) 652 else: 653 return thintar 654 655 tops_failure_msg = "Failed %s tops for Python binary %s." 656 tops_py_version_mapping = {} 657 tops = get_tops(extra_mods=extra_mods, so_mods=so_mods) 658 tops_py_version_mapping[sys.version_info.major] = tops 659 660 with salt.utils.files.fopen(pymap_cfg, "wb") as fp_: 661 fp_.write( 662 _get_supported_py_config( 663 tops=tops_py_version_mapping, extended_cfg=extended_cfg 664 ) 665 ) 666 667 tmp_thintar = _get_thintar_prefix(thintar) 668 if compress == "gzip": 669 tfp = tarfile.open(tmp_thintar, "w:gz", dereference=True) 670 elif compress == "zip": 671 tfp = zipfile.ZipFile( 672 tmp_thintar, 673 "w", 674 compression=zlib and zipfile.ZIP_DEFLATED or zipfile.ZIP_STORED, 675 ) 676 tfp.add = tfp.write 677 try: # cwd may not exist if it was removed but salt was run from it 678 start_dir = os.getcwd() 679 except OSError: 680 start_dir = None 681 tempdir = None 682 683 # Pack default data 684 log.debug("Packing default libraries based on current Salt version") 685 for py_ver, tops in tops_py_version_mapping.items(): 686 for top in tops: 687 if absonly and not os.path.isabs(top): 688 continue 689 base = os.path.basename(top) 690 top_dirname = os.path.dirname(top) 691 if os.path.isdir(top_dirname): 692 os.chdir(top_dirname) 693 else: 694 # This is likely a compressed python .egg 695 tempdir = tempfile.mkdtemp() 696 egg = zipfile.ZipFile(top_dirname) 697 egg.extractall(tempdir) 698 top = os.path.join(tempdir, base) 699 os.chdir(tempdir) 700 701 site_pkg_dir = _is_shareable(base) and "pyall" or "py{}".format(py_ver) 702 703 log.debug('Packing "%s" to "%s" destination', base, site_pkg_dir) 704 if not os.path.isdir(top): 705 # top is a single file module 706 if os.path.exists(os.path.join(top_dirname, base)): 707 tfp.add(base, arcname=os.path.join(site_pkg_dir, base)) 708 continue 709 for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True): 710 for name in files: 711 if not name.endswith((".pyc", ".pyo")): 712 digest_collector.add(os.path.join(root, name)) 713 arcname = os.path.join(site_pkg_dir, root, name) 714 if hasattr(tfp, "getinfo"): 715 try: 716 # This is a little slow but there's no clear way to detect duplicates 717 tfp.getinfo(os.path.join(site_pkg_dir, root, name)) 718 arcname = None 719 except KeyError: 720 log.debug( 721 'ZIP: Unable to add "%s" with "getinfo"', arcname 722 ) 723 if arcname: 724 tfp.add(os.path.join(root, name), arcname=arcname) 725 726 if tempdir is not None: 727 shutil.rmtree(tempdir) 728 tempdir = None 729 730 if extended_cfg: 731 log.debug("Packing libraries based on alternative Salt versions") 732 _pack_alternative(extended_cfg, digest_collector, tfp) 733 734 os.chdir(thindir) 735 with salt.utils.files.fopen(thinver, "w+") as fp_: 736 fp_.write(salt.version.__version__) 737 with salt.utils.files.fopen(pythinver, "w+") as fp_: 738 fp_.write(str(sys.version_info.major)) 739 with salt.utils.files.fopen(code_checksum, "w+") as fp_: 740 fp_.write(digest_collector.digest()) 741 os.chdir(os.path.dirname(thinver)) 742 743 for fname in [ 744 "version", 745 ".thin-gen-py-version", 746 "salt-call", 747 "supported-versions", 748 "code-checksum", 749 ]: 750 tfp.add(fname) 751 752 if start_dir and os.access(start_dir, os.R_OK) and os.access(start_dir, os.X_OK): 753 os.chdir(start_dir) 754 tfp.close() 755 756 shutil.move(tmp_thintar, thintar) 757 758 return thintar 759 760 761def thin_sum(cachedir, form="sha1"): 762 """ 763 Return the checksum of the current thin tarball 764 """ 765 thintar = gen_thin(cachedir) 766 code_checksum_path = os.path.join(cachedir, "thin", "code-checksum") 767 if os.path.isfile(code_checksum_path): 768 with salt.utils.files.fopen(code_checksum_path, "r") as fh: 769 code_checksum = "'{}'".format(fh.read().strip()) 770 else: 771 code_checksum = "'0'" 772 773 return code_checksum, salt.utils.hashutils.get_hash(thintar, form) 774 775 776def gen_min( 777 cachedir, 778 extra_mods="", 779 overwrite=False, 780 so_mods="", 781): 782 """ 783 Generate the salt-min tarball and print the location of the tarball 784 Optional additional mods to include (e.g. mako) can be supplied as a comma 785 delimited string. Permits forcing an overwrite of the output file as well. 786 787 CLI Example: 788 789 .. code-block:: bash 790 791 salt-run min.generate 792 salt-run min.generate mako 793 salt-run min.generate mako,wempy 1 794 salt-run min.generate overwrite=1 795 """ 796 mindir = os.path.join(cachedir, "min") 797 if not os.path.isdir(mindir): 798 os.makedirs(mindir) 799 mintar = os.path.join(mindir, "min.tgz") 800 minver = os.path.join(mindir, "version") 801 pyminver = os.path.join(mindir, ".min-gen-py-version") 802 salt_call = os.path.join(mindir, "salt-call") 803 with salt.utils.files.fopen(salt_call, "wb") as fp_: 804 fp_.write(_get_salt_call()) 805 if os.path.isfile(mintar): 806 if not overwrite: 807 if os.path.isfile(minver): 808 with salt.utils.files.fopen(minver) as fh_: 809 overwrite = fh_.read() != salt.version.__version__ 810 if overwrite is False and os.path.isfile(pyminver): 811 with salt.utils.files.fopen(pyminver) as fh_: 812 overwrite = fh_.read() != str(sys.version_info[0]) 813 else: 814 overwrite = True 815 816 if overwrite: 817 try: 818 os.remove(mintar) 819 except OSError: 820 pass 821 else: 822 return mintar 823 824 tops_py_version_mapping = {} 825 tops = get_tops(extra_mods=extra_mods, so_mods=so_mods) 826 tops_py_version_mapping["3"] = tops 827 828 tfp = tarfile.open(mintar, "w:gz", dereference=True) 829 try: # cwd may not exist if it was removed but salt was run from it 830 start_dir = os.getcwd() 831 except OSError: 832 start_dir = None 833 tempdir = None 834 835 # This is the absolute minimum set of files required to run salt-call 836 min_files = ( 837 "salt/__init__.py", 838 "salt/utils", 839 "salt/utils/__init__.py", 840 "salt/utils/atomicfile.py", 841 "salt/utils/validate", 842 "salt/utils/validate/__init__.py", 843 "salt/utils/validate/path.py", 844 "salt/utils/decorators", 845 "salt/utils/decorators/__init__.py", 846 "salt/utils/cache.py", 847 "salt/utils/xdg.py", 848 "salt/utils/odict.py", 849 "salt/utils/minions.py", 850 "salt/utils/dicttrim.py", 851 "salt/utils/sdb.py", 852 "salt/utils/migrations.py", 853 "salt/utils/files.py", 854 "salt/utils/parsers.py", 855 "salt/utils/locales.py", 856 "salt/utils/lazy.py", 857 "salt/utils/s3.py", 858 "salt/utils/dictupdate.py", 859 "salt/utils/verify.py", 860 "salt/utils/args.py", 861 "salt/utils/kinds.py", 862 "salt/utils/xmlutil.py", 863 "salt/utils/debug.py", 864 "salt/utils/jid.py", 865 "salt/utils/openstack", 866 "salt/utils/openstack/__init__.py", 867 "salt/utils/openstack/swift.py", 868 "salt/utils/asynchronous.py", 869 "salt/utils/process.py", 870 "salt/utils/jinja.py", 871 "salt/utils/rsax931.py", 872 "salt/utils/context.py", 873 "salt/utils/minion.py", 874 "salt/utils/error.py", 875 "salt/utils/aws.py", 876 "salt/utils/timed_subprocess.py", 877 "salt/utils/zeromq.py", 878 "salt/utils/schedule.py", 879 "salt/utils/url.py", 880 "salt/utils/yamlencoding.py", 881 "salt/utils/network.py", 882 "salt/utils/http.py", 883 "salt/utils/gzip_util.py", 884 "salt/utils/vt.py", 885 "salt/utils/templates.py", 886 "salt/utils/aggregation.py", 887 "salt/utils/yaml.py", 888 "salt/utils/yamldumper.py", 889 "salt/utils/yamlloader.py", 890 "salt/utils/event.py", 891 "salt/utils/state.py", 892 "salt/serializers", 893 "salt/serializers/__init__.py", 894 "salt/serializers/yamlex.py", 895 "salt/template.py", 896 "salt/_compat.py", 897 "salt/loader.py", 898 "salt/client", 899 "salt/client/__init__.py", 900 "salt/ext", 901 "salt/ext/__init__.py", 902 "salt/ext/six.py", 903 "salt/ext/ipaddress.py", 904 "salt/version.py", 905 "salt/syspaths.py", 906 "salt/defaults", 907 "salt/defaults/__init__.py", 908 "salt/defaults/exitcodes.py", 909 "salt/renderers", 910 "salt/renderers/__init__.py", 911 "salt/renderers/jinja.py", 912 "salt/renderers/yaml.py", 913 "salt/modules", 914 "salt/modules/__init__.py", 915 "salt/modules/test.py", 916 "salt/modules/selinux.py", 917 "salt/modules/cmdmod.py", 918 "salt/modules/saltutil.py", 919 "salt/minion.py", 920 "salt/pillar", 921 "salt/pillar/__init__.py", 922 "salt/utils/textformat.py", 923 "salt/log", 924 "salt/log/__init__.py", 925 "salt/log/handlers", 926 "salt/log/handlers/__init__.py", 927 "salt/log/mixins.py", 928 "salt/log/setup.py", 929 "salt/cli", 930 "salt/cli/__init__.py", 931 "salt/cli/caller.py", 932 "salt/cli/daemons.py", 933 "salt/cli/salt.py", 934 "salt/cli/call.py", 935 "salt/fileserver", 936 "salt/fileserver/__init__.py", 937 "salt/transport", 938 "salt/transport/__init__.py", 939 "salt/transport/client.py", 940 "salt/exceptions.py", 941 "salt/grains", 942 "salt/grains/__init__.py", 943 "salt/grains/extra.py", 944 "salt/scripts.py", 945 "salt/state.py", 946 "salt/fileclient.py", 947 "salt/crypt.py", 948 "salt/config.py", 949 "salt/beacons", 950 "salt/beacons/__init__.py", 951 "salt/payload.py", 952 "salt/output", 953 "salt/output/__init__.py", 954 "salt/output/nested.py", 955 ) 956 957 for py_ver, tops in tops_py_version_mapping.items(): 958 for top in tops: 959 base = os.path.basename(top) 960 top_dirname = os.path.dirname(top) 961 if os.path.isdir(top_dirname): 962 os.chdir(top_dirname) 963 else: 964 # This is likely a compressed python .egg 965 tempdir = tempfile.mkdtemp() 966 egg = zipfile.ZipFile(top_dirname) 967 egg.extractall(tempdir) 968 top = os.path.join(tempdir, base) 969 os.chdir(tempdir) 970 if not os.path.isdir(top): 971 # top is a single file module 972 tfp.add(base, arcname=os.path.join("py{}".format(py_ver), base)) 973 continue 974 for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True): 975 for name in files: 976 if name.endswith((".pyc", ".pyo")): 977 continue 978 if ( 979 root.startswith("salt") 980 and os.path.join(root, name) not in min_files 981 ): 982 continue 983 tfp.add( 984 os.path.join(root, name), 985 arcname=os.path.join("py{}".format(py_ver), root, name), 986 ) 987 if tempdir is not None: 988 shutil.rmtree(tempdir) 989 tempdir = None 990 991 os.chdir(mindir) 992 tfp.add("salt-call") 993 with salt.utils.files.fopen(minver, "w+") as fp_: 994 fp_.write(salt.version.__version__) 995 with salt.utils.files.fopen(pyminver, "w+") as fp_: 996 fp_.write(str(sys.version_info[0])) 997 os.chdir(os.path.dirname(minver)) 998 tfp.add("version") 999 tfp.add(".min-gen-py-version") 1000 if start_dir: 1001 os.chdir(start_dir) 1002 tfp.close() 1003 return mintar 1004 1005 1006def min_sum(cachedir, form="sha1"): 1007 """ 1008 Return the checksum of the current thin tarball 1009 """ 1010 mintar = gen_min(cachedir) 1011 return salt.utils.hashutils.get_hash(mintar, form) 1012